@michaelthielemann/kestrel 4.1.3 → 5.0.0

package/dist/testing/runPipeline.js

@@ -0,0 +1,22 @@
+import { silentLogger } from "../logger.js";
+import { StepRegistry } from "../registry.js";
+import { runPipeline as execute } from "../runner.js";
+export function runPipeline(pipeline, input, options) {
+    const registry = new StepRegistry();
+    const byPrefix = new Map();
+    for (const [name, fn] of Object.entries(options.steps)) {
+        const dot = name.indexOf(".");
+        if (dot === -1)
+            throw new Error(`fake step "${name}" must be named "<module>.<step>"`);
+        const prefix = name.slice(0, dot);
+        const group = byPrefix.get(prefix) ?? {};
+        group[name.slice(dot + 1)] = fn;
+        byPrefix.set(prefix, group);
+    }
+    for (const [prefix, group] of byPrefix) {
+        const descriptions = Object.fromEntries(Object.keys(group).map((key) => [key, { summary: `${prefix}.${key}`, reads: [], writes: ["result"] }]));
+        registry.register("test", prefix, group, descriptions);
+    }
+    const resolved = { name: pipeline.name, steps: pipeline.steps.map((s) => registry.resolve(s, `pipelines/${pipeline.name}`)) };
+    return execute(resolved, { trigger: { kind: "http", name: "test" }, ...input }, options.logger ?? silentLogger);
+}

package/dist/triggers/cron.d.ts

@@ -0,0 +1,10 @@
+import { type Logger } from "../logger.ts";
+import type { Runner } from "../runner.ts";
+import { type CronSpec } from "./cronMatch.ts";
+export interface CronEntry {
+    expression: string;
+    spec: CronSpec;
+    pipeline: string;
+}
+export declare function createCronEntry(expression: string, pipeline: string): CronEntry;
+export declare function startCron(entries: readonly CronEntry[], run: Runner, logger: Logger, now?: () => Date): () => void;

package/dist/triggers/cron.js

@@ -0,0 +1,44 @@
+import { logWarn } from "../logger.js";
+import { cronMatches, parseCron } from "./cronMatch.js";
+export function createCronEntry(expression, pipeline) {
+    return { expression, spec: parseCron(expression), pipeline };
+}
+export function startCron(entries, run, logger, now = () => new Date()) {
+    let timer;
+    let stopped = false;
+    const running = new Set();
+    const tick = () => {
+        const date = now();
+        for (const entry of entries) {
+            if (!cronMatches(entry.spec, date))
+                continue;
+            if (running.has(entry.pipeline)) {
+                logWarn(logger, `cron "${entry.expression}" skipped: pipeline "${entry.pipeline}" is still running`, { cron: entry.expression, pipeline: entry.pipeline });
+                continue;
+            }
+            running.add(entry.pipeline);
+            void run(entry.pipeline, { trigger: { kind: "cron", name: entry.expression } })
+                .then((res) => {
+                if (res.status >= 400)
+                    logger.error(`cron "${entry.expression}" pipeline "${entry.pipeline}" ended with ${res.status}`, { runId: res.runId, error: res.error });
+            }, (err) => {
+                logger.error(`cron "${entry.expression}" pipeline "${entry.pipeline}" threw`, { error: err instanceof Error ? (err.stack ?? err.message) : String(err) });
+            })
+                .finally(() => running.delete(entry.pipeline));
+        }
+        schedule();
+    };
+    const schedule = () => {
+        if (stopped)
+            return;
+        const ms = 60_000 - (now().getTime() % 60_000);
+        timer = setTimeout(tick, ms);
+        timer.unref();
+    };
+    schedule();
+    return () => {
+        stopped = true;
+        if (timer)
+            clearTimeout(timer);
+    };
+}

package/dist/triggers/cronMatch.d.ts

@@ -0,0 +1,10 @@
+export interface CronSpec {
+    minute: Set<number>;
+    hour: Set<number>;
+    dayOfMonth: Set<number>;
+    month: Set<number>;
+    dayOfWeek: Set<number>;
+    anyDay: boolean;
+}
+export declare function parseCron(expression: string): CronSpec;
+export declare function cronMatches(spec: CronSpec, date: Date): boolean;

package/dist/triggers/cronMatch.js

@@ -0,0 +1,68 @@
+const RANGES = [
+    ["minute", 0, 59],
+    ["hour", 0, 23],
+    ["day of month", 1, 31],
+    ["month", 1, 12],
+    ["day of week", 0, 7],
+];
+function parseField(field, label, min, max) {
+    const values = new Set();
+    let restricted = false;
+    for (const part of field.split(",")) {
+        const [rangePart, stepPart] = part.split("/");
+        if (rangePart === undefined || rangePart === "")
+            throw new Error(`cron ${label}: empty entry in "${field}"`);
+        const step = stepPart === undefined ? 1 : Number(stepPart);
+        if (!Number.isInteger(step) || step < 1)
+            throw new Error(`cron ${label}: invalid step in "${part}"`);
+        let lo;
+        let hi;
+        if (rangePart === "*") {
+            lo = min;
+            hi = max;
+            if (stepPart !== undefined)
+                restricted = true;
+        }
+        else {
+            restricted = true;
+            const [a, b] = rangePart.split("-");
+            lo = Number(a);
+            hi = b === undefined ? (stepPart === undefined ? lo : max) : Number(b);
+            if (!Number.isInteger(lo) || !Number.isInteger(hi) || lo < min || hi > max || lo > hi) {
+                throw new Error(`cron ${label}: "${part}" is outside ${min}-${max}`);
+            }
+        }
+        for (let v = lo; v <= hi; v += step)
+            values.add(v);
+    }
+    return { values, restricted };
+}
+export function parseCron(expression) {
+    const fields = expression.trim().split(/\s+/);
+    if (fields.length !== 5)
+        throw new Error(`cron expression "${expression}" must have 5 fields`);
+    const parsed = fields.map((f, i) => {
+        const [label, min, max] = RANGES[i];
+        return parseField(f, label, min, max);
+    });
+    const dow = parsed[4].values;
+    if (dow.has(7)) {
+        dow.delete(7);
+        dow.add(0);
+    }
+    return {
+        minute: parsed[0].values,
+        hour: parsed[1].values,
+        dayOfMonth: parsed[2].values,
+        month: parsed[3].values,
+        dayOfWeek: dow,
+        anyDay: parsed[2].restricted && parsed[4].restricted,
+    };
+}
+export function cronMatches(spec, date) {
+    if (!spec.minute.has(date.getMinutes()) || !spec.hour.has(date.getHours()) || !spec.month.has(date.getMonth() + 1))
+        return false;
+    const domOk = spec.dayOfMonth.has(date.getDate());
+    const dowOk = spec.dayOfWeek.has(date.getDay());
+    return spec.anyDay ? domOk || dowOk : domOk && dowOk;
+}

package/dist/triggers/http.d.ts

@@ -0,0 +1,58 @@
+import { type IncomingMessage, type Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import { type UploadedFile } from "../context.ts";
+import type { Logger } from "../logger.ts";
+import type { RunResult, Runner } from "../runner.ts";
+export interface Route {
+    method: string;
+    segments: readonly string[];
+    pipeline: string;
+}
+export declare function parseRoute(http: string, pipeline: string): Route;
+/** Method plus segment shape; two routes differing only in a param name are the same pattern. */
+export declare function routePattern(route: Route): string;
+export declare function matchRoute(routes: readonly Route[], method: string, pathname: string): {
+    route: Route;
+    params: Record<string, string>;
+} | undefined;
+export declare class BodyTooLarge extends Error {
+}
+export declare function parseQuery(searchParams: URLSearchParams): Record<string, unknown>;
+export interface ParsedBody {
+    payload: Record<string, unknown>;
+    files: UploadedFile[];
+}
+export declare function parseRequestBody(contentType: string, body: Uint8Array): Promise<ParsedBody>;
+export interface HttpResponse {
+    status: number;
+    headers: Record<string, string>;
+    body: Uint8Array | string;
+}
+export interface ResponseMeta {
+    runId?: string;
+    requestId?: string;
+    retryable?: boolean;
+    retryAfterSeconds?: number;
+}
+export declare function buildResponse(status: number, body: unknown, meta?: ResponseMeta, extraInline?: readonly string[]): HttpResponse;
+export declare function responseForRun(result: RunResult, extraInline?: readonly string[]): HttpResponse;
+/** The one shape for errors raised before a pipeline runs (unknown route, unreadable body). */
+export declare function errorResponse(status: number, error: string, requestId?: string): HttpResponse;
+/** RFC 6266: a quoted ASCII fallback, plus filename* only when the name is not printable ASCII. */
+export declare function disposition(contentType: string, filename: string | undefined, extraInline?: readonly string[]): string;
+export interface HttpTimeouts {
+    requestMs?: number;
+    headersMs?: number;
+    keepAliveMs?: number;
+}
+export interface HttpOptions {
+    corsOrigin?: string;
+    maxBodyBytes?: number;
+    trustProxy?: boolean;
+    healthPath?: string | null;
+    inlineTypes?: readonly string[];
+    timeouts?: HttpTimeouts;
+}
+export declare function clientIp(req: IncomingMessage, trustProxy: boolean): string | undefined;
+export declare function createHttpServer(routes: readonly Route[], run: Runner, logger: Logger, options?: HttpOptions): Server;
+export declare function listen(server: Server, port: number, host: string): Promise<AddressInfo>;

package/dist/triggers/http.js

@@ -0,0 +1,292 @@
+import { randomUUID } from "node:crypto";
+import { createServer } from "node:http";
+import { isBinaryResult, requestIdOf } from "../context.js";
+export function parseRoute(http, pipeline) {
+    const [method, path] = http.split(" ");
+    if (method === undefined || path === undefined || !path.startsWith("/"))
+        throw new Error(`invalid http trigger "${http}"`);
+    const segments = path === "/" ? [] : path.slice(1).split("/");
+    if (segments.some((s) => s === ""))
+        throw new Error(`invalid http trigger "${http}": empty path segment`);
+    const wildcard = segments.findIndex((s) => s.startsWith("*"));
+    if (wildcard !== -1 && wildcard !== segments.length - 1)
+        throw new Error(`invalid http trigger "${http}": wildcard must be the last segment`);
+    return { method, segments, pipeline };
+}
+/** literal > :param > end of route > *rest, so a literal route wins no matter when it was registered. */
+function specificity(segment) {
+    if (segment === undefined)
+        return 1;
+    if (segment.startsWith("*"))
+        return 0;
+    if (segment.startsWith(":"))
+        return 2;
+    return 3;
+}
+function bySpecificity(a, b) {
+    const len = Math.max(a.segments.length, b.segments.length);
+    for (let i = 0; i < len; i++) {
+        const diff = specificity(b.segments[i]) - specificity(a.segments[i]);
+        if (diff !== 0)
+            return diff;
+    }
+    return 0;
+}
+function decodeSegment(part) {
+    try {
+        return decodeURIComponent(part);
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Method plus segment shape; two routes differing only in a param name are the same pattern. */
+export function routePattern(route) {
+    return `${route.method} /${route.segments.map((s) => (s.startsWith(":") ? ":" : s.startsWith("*") ? "*" : s)).join("/")}`;
+}
+export function matchRoute(routes, method, pathname) {
+    const parts = pathname === "/" ? [] : pathname.replace(/\/+$/, "").slice(1).split("/");
+    for (const route of [...routes].sort(bySpecificity)) {
+        if (route.method !== method)
+            continue;
+        const last = route.segments[route.segments.length - 1];
+        const wildcard = last?.startsWith("*") ? route.segments.length - 1 : -1;
+        if (wildcard === -1 ? route.segments.length !== parts.length : parts.length < wildcard)
+            continue;
+        const params = {};
+        let ok = true;
+        for (let i = 0; i < route.segments.length; i++) {
+            const seg = route.segments[i];
+            if (i === wildcard) {
+                const rest = parts.slice(i).map(decodeSegment);
+                // An encoded "/" would make the rest indistinguishable from a deeper path, so it never matches.
+                if (rest.some((p) => p === undefined || p.includes("/")))
+                    ok = false;
+                else
+                    params[seg.slice(1)] = rest.join("/");
+                break;
+            }
+            const part = decodeSegment(parts[i]);
+            if (part === undefined)
+                ok = false;
+            else if (seg.startsWith(":"))
+                params[seg.slice(1)] = part;
+            else if (seg !== part)
+                ok = false;
+            if (!ok)
+                break;
+        }
+        if (ok)
+            return { route, params };
+    }
+    return undefined;
+}
+export class BodyTooLarge extends Error {
+}
+async function readBody(req, maxBytes) {
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of req) {
+        size += chunk.length;
+        if (size > maxBytes)
+            throw new BodyTooLarge(`body exceeds ${maxBytes} bytes`);
+        chunks.push(chunk);
+    }
+    return Buffer.concat(chunks);
+}
+export function parseQuery(searchParams) {
+    const payload = {};
+    for (const key of new Set(searchParams.keys())) {
+        const values = searchParams.getAll(key);
+        payload[key] = values.length > 1 ? values : values[0];
+    }
+    return payload;
+}
+export async function parseRequestBody(contentType, body) {
+    if (contentType.startsWith("multipart/form-data")) {
+        const copy = new Uint8Array(body.byteLength);
+        copy.set(body);
+        const form = await new Response(copy.buffer, { headers: { "content-type": contentType } }).formData();
+        const payload = {};
+        const files = [];
+        for (const [field, value] of form) {
+            if (typeof value === "string")
+                payload[field] = value;
+            else
+                files.push({ field, filename: value.name, contentType: value.type || "application/octet-stream", data: new Uint8Array(await value.arrayBuffer()) });
+        }
+        return { payload, files };
+    }
+    const text = Buffer.from(body).toString("utf8");
+    if (text.trim() === "")
+        return { payload: {}, files: [] };
+    const parsed = JSON.parse(text);
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
+        throw new Error("body must be a JSON object");
+    return { payload: parsed, files: [] };
+}
+const DEFAULT_RETRY_AFTER_SECONDS = 1;
+function retryAfterSecondsOf(meta) {
+    const seconds = meta.retryAfterSeconds;
+    return typeof seconds === "number" && Number.isInteger(seconds) && seconds > 0 ? seconds : DEFAULT_RETRY_AFTER_SECONDS;
+}
+export function buildResponse(status, body, meta = {}, extraInline = []) {
+    const headers = { "x-content-type-options": "nosniff", "cache-control": "no-store" };
+    if (meta.runId !== undefined)
+        headers["x-kestrel-run-id"] = meta.runId;
+    if (meta.requestId !== undefined)
+        headers["x-request-id"] = meta.requestId;
+    if (meta.retryable === true && (status === 429 || status === 503))
+        headers["retry-after"] = String(retryAfterSecondsOf(meta));
+    if (isBinaryResult(body)) {
+        return {
+            status,
+            headers: {
+                // Caller headers may add, never weaken: the mandatory ones are spread last.
+                ...Object.fromEntries(Object.entries(body.headers ?? {}).map(([k, v]) => [k.toLowerCase(), v])),
+                ...headers,
+                "content-type": body.contentType,
+                "content-length": String(body.data.byteLength),
+                "content-disposition": disposition(body.contentType, body.filename, extraInline),
+                "content-security-policy": "default-src 'none'; sandbox",
+            },
+            body: body.data,
+        };
+    }
+    return { status, headers: { ...headers, "content-type": "application/json; charset=utf-8" }, body: JSON.stringify(body ?? null) };
+}
+function errorBody(result) {
+    const body = { error: result.error, code: result.code, retryable: result.retryable, runId: result.runId };
+    if (result.step !== undefined)
+        body.step = result.step;
+    if (result.details !== undefined)
+        body.details = result.details;
+    return body;
+}
+function metaOf(result) {
+    const meta = { runId: result.runId };
+    if (result.requestId !== undefined)
+        meta.requestId = result.requestId;
+    if (result.retryable !== undefined)
+        meta.retryable = result.retryable;
+    const retryAfterSeconds = result.details?.retryAfterSeconds;
+    if (typeof retryAfterSeconds === "number")
+        meta.retryAfterSeconds = retryAfterSeconds;
+    return meta;
+}
+export function responseForRun(result, extraInline = []) {
+    if (result.status >= 400)
+        return buildResponse(result.status, errorBody(result), metaOf(result), extraInline);
+    return buildResponse(result.status, result.result, metaOf(result), extraInline);
+}
+/** Fixed code for errors raised before a pipeline runs, where there is no KestrelError to read one from. */
+const EDGE_CODE_BY_STATUS = {
+    400: "VALIDATION",
+    404: "NOT_FOUND",
+    413: "PAYLOAD_TOO_LARGE",
+    500: "INTERNAL",
+};
+/** The one shape for errors raised before a pipeline runs (unknown route, unreadable body). */
+export function errorResponse(status, error, requestId) {
+    const runId = randomUUID();
+    const code = EDGE_CODE_BY_STATUS[status] ?? "INTERNAL";
+    const body = { error, code, retryable: false, runId };
+    return buildResponse(status, body, requestId === undefined ? { runId } : { runId, requestId });
+}
+const INLINE_SAFE = new Set(["image/png", "image/jpeg", "image/gif", "image/webp", "image/avif", "video/mp4", "video/webm", "audio/mpeg", "audio/ogg", "audio/wav"]);
+/** RFC 6266: a quoted ASCII fallback, plus filename* only when the name is not printable ASCII. */
+export function disposition(contentType, filename, extraInline = []) {
+    const mediaType = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
+    const mode = INLINE_SAFE.has(mediaType) || extraInline.includes(mediaType) ? "inline" : "attachment";
+    if (filename === undefined)
+        return mode;
+    const ascii = `${mode}; filename="${filename.replace(/[^\x20-\x7e]/g, "_").replace(/["\\]/g, "\\$&")}"`;
+    if (/^[\x20-\x7e]*$/.test(filename))
+        return ascii;
+    const encoded = encodeURIComponent(filename).replace(/['()!*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
+    return `${ascii}; filename*=UTF-8''${encoded}`;
+}
+function sendError(res, status, error, requestId) {
+    write(res, errorResponse(status, error, requestId));
+}
+function write(res, response) {
+    res.writeHead(response.status, response.headers);
+    res.end(typeof response.body === "string" ? response.body : Buffer.from(response.body));
+}
+export function clientIp(req, trustProxy) {
+    if (trustProxy) {
+        const forwarded = req.headers["x-forwarded-for"];
+        const first = (Array.isArray(forwarded) ? forwarded[0] : forwarded)?.split(",")[0]?.trim();
+        if (first)
+            return first;
+    }
+    return req.socket.remoteAddress ?? undefined;
+}
+export function createHttpServer(routes, run, logger, options = {}) {
+    const started = Date.now();
+    const healthPath = options.healthPath === undefined ? "/health" : options.healthPath;
+    const server = createServer((req, res) => {
+        if (options.corsOrigin !== undefined) {
+            res.setHeader("access-control-allow-origin", options.corsOrigin);
+            res.setHeader("access-control-allow-headers", "content-type, authorization");
+            res.setHeader("access-control-allow-methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS");
+            res.setHeader("vary", "origin");
+            if (req.method === "OPTIONS") {
+                res.writeHead(204, { "x-content-type-options": "nosniff", "cache-control": "no-store", "access-control-max-age": "600" });
+                res.end();
+                return;
+            }
+        }
+        void handle(req, res).catch((err) => {
+            logger.error("http handler failed", { error: err instanceof Error ? err.stack : String(err) });
+            if (!res.headersSent)
+                sendError(res, 500, "internal error", requestIdOf(req.headers["x-request-id"]));
+        });
+    });
+    server.requestTimeout = options.timeouts?.requestMs ?? 30_000;
+    server.headersTimeout = options.timeouts?.headersMs ?? 10_000;
+    server.keepAliveTimeout = options.timeouts?.keepAliveMs ?? 5_000;
+    return server;
+    async function handle(req, res) {
+        const url = new URL(req.url ?? "/", "http://localhost");
+        const requestId = requestIdOf(req.headers["x-request-id"]);
+        if (healthPath && url.pathname === healthPath && req.method === "GET") {
+            const meta = requestId === undefined ? { runId: randomUUID() } : { runId: randomUUID(), requestId };
+            return write(res, buildResponse(200, { ok: true, uptimeSeconds: Math.round((Date.now() - started) / 1000) }, meta));
+        }
+        const begin = performance.now();
+        const match = matchRoute(routes, req.method ?? "GET", url.pathname);
+        if (!match)
+            return sendError(res, 404, `no route for ${req.method} ${url.pathname}`, requestId);
+        let body;
+        try {
+            body = await parseRequestBody(req.headers["content-type"] ?? "", await readBody(req, options.maxBodyBytes ?? 10 * 1024 * 1024));
+        }
+        catch (err) {
+            if (err instanceof BodyTooLarge)
+                return sendError(res, 413, err.message, requestId);
+            return sendError(res, 400, `invalid body: ${err instanceof Error ? err.message : String(err)}`, requestId);
+        }
+        const payload = { ...parseQuery(url.searchParams), ...body.payload };
+        const headers = {};
+        for (const [k, v] of Object.entries(req.headers))
+            if (typeof v === "string")
+                headers[k] = v;
+        const input = { trigger: { kind: "http", name: `${req.method} ${url.pathname}` }, payload, params: match.params, headers, files: body.files };
+        const ip = clientIp(req, options.trustProxy ?? false);
+        if (ip !== undefined)
+            input.ip = ip;
+        const result = await run(match.route.pipeline, input);
+        logger.info("http", { runId: result.runId, ...(requestId === undefined ? {} : { requestId }), method: req.method, path: url.pathname, pipeline: match.route.pipeline, status: result.status, ms: Math.round((performance.now() - begin) * 100) / 100, ip });
+        write(res, responseForRun(result, options.inlineTypes));
+    }
+}
+export function listen(server, port, host) {
+    return new Promise((resolve, reject) => {
+        server.once("error", reject);
+        server.listen(port, host, () => {
+            server.off("error", reject);
+            resolve(server.address());
+        });
+    });
+}