loopengine 0.3.0 → 0.3.2

package/dist/core/workflow-triggers.js

@@ -0,0 +1,378 @@
+// Webhook triggers: a secret URL per workflow that an outside service
+// (Stripe, GitHub, Shopify, a form or monitoring tool) can post its own
+// events to, to start a run, with no API key and no body format of ours:
+//
+//   POST /hooks/workflows/<token>
+//
+// The token in the URL is the credential (it's kept here only as a
+// SHA-256 hash, so the full URL is shown once, when it's made or
+// rotated). A trigger can also check the sender's own signature, keep
+// only some events (filter), and fill the workflow's parameters from the
+// payload (mapping, or the whole payload as one parameter). A delivery a
+// sender retries is recognised by its id and starts no second run.
+//
+// Stored like runs and schedules: one JSON file per trigger under
+// `<LOOPENGINE_SCHEDULER_DIR or .scheduler>/triggers/`.
+import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { KeyedMutex } from './session-store.js';
+import { checkCallbackUrl } from './run-callbacks.js';
+export class TriggerError extends Error {
+}
+const MAX_DELIVERIES = 20;
+const MAX_SEEN = 200;
+const MAX_PAYLOAD_PARAM = 20_000;
+const SENDERS = ['stripe', 'github', 'shopify', 'standard', 'custom'];
+const ALGORITHMS = ['sha256', 'sha1', 'sha512'];
+/** How old a signed timestamp (Stripe, Standard Webhooks) may be. */
+const SIGNATURE_TOLERANCE_S = 300;
+const fileMutex = new KeyedMutex();
+/** One JSON file per trigger under `<dir>/triggers/` — one server process. */
+export class FileTriggerStore {
+    dir;
+    constructor(dir) {
+        this.dir = path.join(dir, 'triggers');
+    }
+    file(id) {
+        if (!/^[A-Za-z0-9_-]+$/.test(id))
+            throw new TriggerError(`invalid trigger id "${id}"`);
+        return path.join(this.dir, `${id}.json`);
+    }
+    async list() {
+        if (!existsSync(this.dir))
+            return [];
+        const triggers = [];
+        for (const name of readdirSync(this.dir)) {
+            if (!name.endsWith('.json'))
+                continue;
+            try {
+                triggers.push(JSON.parse(readFileSync(path.join(this.dir, name), 'utf8')));
+            }
+            catch {
+                // A file being written, or not a trigger: skip it.
+            }
+        }
+        return triggers.sort((a, b) => a.created_at.localeCompare(b.created_at));
+    }
+    async get(id) {
+        const file = this.file(id);
+        return existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')) : undefined;
+    }
+    async put(trigger) {
+        mkdirSync(this.dir, { recursive: true });
+        const file = this.file(trigger.trigger_id);
+        writeFileSync(file + '.tmp', JSON.stringify(trigger, null, 2), { mode: 0o600 });
+        renameSync(file + '.tmp', file);
+    }
+    async remove(id) {
+        rmSync(this.file(id), { force: true });
+    }
+    async update(id, fn) {
+        return fileMutex.run(id, async () => {
+            const next = fn(await this.get(id));
+            if (next)
+                await this.put(next);
+            return next;
+        });
+    }
+}
+let store;
+export function getTriggerStore() {
+    store ??= new FileTriggerStore(process.env.LOOPENGINE_SCHEDULER_DIR || path.join(process.cwd(), '.scheduler'));
+    return store;
+}
+export function setTriggerStore(next) {
+    store = next;
+}
+// ---------- making and finding ----------
+function hashToken(token) {
+    return createHash('sha256').update(token).digest('hex');
+}
+function newToken() {
+    return `whk_${randomBytes(24).toString('base64url')}`;
+}
+export function publicTrigger(t) {
+    const { token_hash: _hash, verify, seen_ids: _seen, ...rest } = t;
+    if (!verify)
+        return rest;
+    const { secret, ...shown } = verify;
+    return { ...rest, verify: { ...shown, secret_set: Boolean(secret) } };
+}
+/** Checks settings for a trigger of `workflow`; throws TriggerError saying
+ * what's wrong. With `existing` (a change), a setting left out keeps its
+ * value and null removes it, and a sender's secret is kept when only the
+ * sender is given again (the console never sees the secret).
+ *
+ * A trigger that checks a signature can be made turned off without its
+ * secret yet: the workflow planner makes one that way, for the person to
+ * add the secret the sender shows them. It can't be turned on until the
+ * secret is there. */
+export function checkTriggerSettings(input, workflow, existing) {
+    if (existing) {
+        const keys = ['verify', 'filter', 'mapping', 'payload_param', 'notify', 'callback_url'];
+        input = { ...input };
+        for (const key of keys)
+            if (input[key] === undefined && existing[key] !== undefined)
+                input[key] = existing[key];
+    }
+    const name = typeof input.name === 'string' && input.name.trim() ? input.name.trim().slice(0, 80) : existing?.name ?? 'Webhook';
+    if (input.enabled !== undefined && typeof input.enabled !== 'boolean')
+        throw new TriggerError('enabled must be true or false');
+    const enabled = input.enabled === undefined ? existing?.enabled ?? true : input.enabled;
+    let verify;
+    if (input.verify !== undefined && input.verify !== null) {
+        const v = input.verify;
+        if (typeof v !== 'object' || !SENDERS.includes(v.sender))
+            throw new TriggerError(`verify.sender must be one of ${SENDERS.join(', ')}`);
+        const kept = existing?.verify && existing.verify.sender === v.sender ? existing.verify.secret : '';
+        const secret = typeof v.secret === 'string' && v.secret.trim() ? v.secret.trim() : kept;
+        if (!secret && enabled)
+            throw new TriggerError(`Give the signing secret ${v.sender === 'custom' ? 'the sender' : v.sender} shows for this webhook, so its signature can be checked.`);
+        verify = { sender: v.sender, secret };
+        if (v.sender === 'standard' && secret && !/^(whsec_)?[A-Za-z0-9+/]+=*$/.test(secret))
+            throw new TriggerError('A Standard Webhooks secret looks like "whsec_" followed by base64 text.');
+        if (v.sender === 'custom') {
+            const headerName = typeof v.header === 'string' ? v.header.trim() : '';
+            if (!/^[A-Za-z0-9-]{1,80}$/.test(headerName))
+                throw new TriggerError('Give the header the signature arrives in, e.g. "Linear-Signature" (letters, digits and dashes).');
+            const algorithm = (v.algorithm ?? 'sha256');
+            if (!ALGORITHMS.includes(algorithm))
+                throw new TriggerError(`algorithm must be one of ${ALGORITHMS.join(', ')}`);
+            const encoding = v.encoding ?? 'hex';
+            if (encoding !== 'hex' && encoding !== 'base64')
+                throw new TriggerError('encoding must be hex or base64');
+            const prefix = typeof v.prefix === 'string' ? v.prefix.trim() : '';
+            if (prefix.length > 20)
+                throw new TriggerError('prefix is too long: keep it under 20 characters, e.g. "sha256="');
+            verify = { ...verify, header: headerName.toLowerCase(), algorithm, encoding, ...(prefix ? { prefix } : {}) };
+        }
+    }
+    let filter;
+    if (input.filter !== undefined && input.filter !== null) {
+        const f = input.filter;
+        const values = Array.isArray(f.values) ? f.values.filter((x) => typeof x === 'string' && x.trim()).map((x) => x.trim()) : [];
+        if (typeof f.path !== 'string' || !f.path.trim() || !values.length)
+            throw new TriggerError('filter needs a path, e.g. "type", and at least one value, e.g. ["charge.dispute.created"]');
+        filter = { path: f.path.trim(), values };
+    }
+    const declared = Object.keys(workflow.parameters);
+    let mapping;
+    if (input.mapping !== undefined && input.mapping !== null) {
+        if (typeof input.mapping !== 'object' || Array.isArray(input.mapping))
+            throw new TriggerError('mapping must be an object of parameter names to payload paths');
+        mapping = {};
+        for (const [param, p] of Object.entries(input.mapping)) {
+            if (typeof p !== 'string' || !p.trim())
+                continue;
+            if (!declared.includes(param))
+                throw new TriggerError(`mapping: ${workflow.title} has no parameter "${param}"${declared.length ? ` (its parameters: ${declared.join(', ')})` : ''}`);
+            mapping[param] = p.trim();
+        }
+        if (!Object.keys(mapping).length)
+            mapping = undefined;
+    }
+    let payloadParam;
+    if (typeof input.payload_param === 'string' && input.payload_param) {
+        if (!declared.includes(input.payload_param))
+            throw new TriggerError(`payload_param: ${workflow.title} has no parameter "${input.payload_param}"`);
+        payloadParam = input.payload_param;
+    }
+    if (input.notify !== undefined && typeof input.notify !== 'boolean')
+        throw new TriggerError('notify must be true or false');
+    const callbackUrl = input.callback_url ? checkCallbackUrl(input.callback_url) : undefined;
+    return {
+        name,
+        enabled,
+        ...(verify ? { verify } : {}),
+        ...(filter ? { filter } : {}),
+        ...(mapping ? { mapping } : {}),
+        ...(payloadParam ? { payload_param: payloadParam } : {}),
+        ...(input.notify === true ? { notify: true } : {}),
+        ...(callbackUrl ? { callback_url: callbackUrl } : {}),
+    };
+}
+/** Makes a trigger for a workflow; returns it with its token (shown once). */
+export async function createTrigger(agent, workflow, input) {
+    const settings = checkTriggerSettings(input, workflow);
+    const token = newToken();
+    const trigger = {
+        trigger_id: randomUUID(),
+        agent,
+        workflow_id: workflow.id,
+        ...settings,
+        token_hash: hashToken(token),
+        token_hint: `${token.slice(0, 8)}…`,
+        created_at: new Date().toISOString(),
+        deliveries: [],
+        seen_ids: [],
+    };
+    await getTriggerStore().put(trigger);
+    return { trigger, token };
+}
+export async function updateTrigger(id, workflow, input) {
+    const existing = await getTriggerStore().get(id);
+    if (!existing)
+        throw new TriggerError(`no trigger ${id}`);
+    const settings = checkTriggerSettings(input, workflow, existing);
+    const { verify: _v, filter: _f, mapping: _m, payload_param: _p, notify: _n, callback_url: _c, ...kept } = existing;
+    const next = { ...kept, ...settings };
+    await getTriggerStore().put(next);
+    return next;
+}
+/** A new URL for a trigger; the old one stops working. */
+export async function rotateTrigger(id) {
+    const token = newToken();
+    const trigger = await getTriggerStore().update(id, (t) => t && { ...t, token_hash: hashToken(token), token_hint: `${token.slice(0, 8)}…` });
+    if (!trigger)
+        throw new TriggerError(`no trigger ${id}`);
+    return { trigger, token };
+}
+export async function listTriggers(agent, workflowId) {
+    return (await getTriggerStore().list()).filter((t) => t.agent === agent && (!workflowId || t.workflow_id === workflowId));
+}
+/** The trigger a URL's token belongs to. */
+export async function findTriggerByToken(token) {
+    if (!/^whk_[A-Za-z0-9_-]{16,}$/.test(token))
+        return undefined;
+    const hash = Buffer.from(hashToken(token), 'hex');
+    return (await getTriggerStore().list()).find((t) => {
+        const other = Buffer.from(t.token_hash, 'hex');
+        return other.length === hash.length && timingSafeEqual(other, hash);
+    });
+}
+// ---------- a delivery ----------
+const hmacHex = (secret, data) => createHmac('sha256', secret).update(data).digest('hex');
+const sameText = (a, b) => {
+    const x = Buffer.from(a);
+    const y = Buffer.from(b);
+    return x.length === y.length && timingSafeEqual(x, y);
+};
+const header = (headers, name) => {
+    const v = headers[name];
+    return Array.isArray(v) ? v[0] : v;
+};
+/** Whether the sender's own signature on the raw body is right. Never
+ * with no secret: anyone can sign with an empty key. */
+export function verifySenderSignature(verify, headers, raw, nowS = Math.floor(Date.now() / 1000)) {
+    if (!verify.secret)
+        return false;
+    if (verify.sender === 'stripe') {
+        const parts = (header(headers, 'stripe-signature') ?? '').split(',').map((p) => p.split('='));
+        const t = Number(parts.find(([k]) => k === 't')?.[1]);
+        if (!Number.isFinite(t) || Math.abs(nowS - t) > SIGNATURE_TOLERANCE_S)
+            return false;
+        const expected = hmacHex(verify.secret, Buffer.concat([Buffer.from(`${t}.`), raw]));
+        return parts.some(([k, v]) => k === 'v1' && typeof v === 'string' && sameText(v, expected));
+    }
+    if (verify.sender === 'github') {
+        const sig = header(headers, 'x-hub-signature-256') ?? '';
+        return sig.startsWith('sha256=') && sameText(sig.slice(7), hmacHex(verify.secret, raw));
+    }
+    if (verify.sender === 'shopify') {
+        const sig = header(headers, 'x-shopify-hmac-sha256') ?? '';
+        return Boolean(sig) && sameText(sig, createHmac('sha256', verify.secret).update(raw).digest('base64'));
+    }
+    if (verify.sender === 'standard') {
+        // webhook-* headers, or Svix's own svix-* names for the same thing.
+        const id = header(headers, 'webhook-id') ?? header(headers, 'svix-id');
+        const t = Number(header(headers, 'webhook-timestamp') ?? header(headers, 'svix-timestamp'));
+        const sigs = header(headers, 'webhook-signature') ?? header(headers, 'svix-signature') ?? '';
+        if (!id || !Number.isFinite(t) || Math.abs(nowS - t) > SIGNATURE_TOLERANCE_S)
+            return false;
+        const key = Buffer.from(verify.secret.replace(/^whsec_/, ''), 'base64');
+        const expected = createHmac('sha256', key).update(Buffer.concat([Buffer.from(`${id}.${t}.`), raw])).digest('base64');
+        return sigs.split(' ').some((part) => {
+            const [version, sig] = part.split(',');
+            return version === 'v1' && typeof sig === 'string' && sameText(sig, expected);
+        });
+    }
+    // custom: an HMAC of the raw body, in the header the sender names.
+    const got = (header(headers, verify.header ?? '') ?? '').trim();
+    const prefix = verify.prefix ?? '';
+    if (!got || (prefix && !got.startsWith(prefix)))
+        return false;
+    const expected = createHmac(verify.algorithm ?? 'sha256', verify.secret).update(raw).digest(verify.encoding ?? 'hex');
+    const value = got.slice(prefix.length);
+    return verify.encoding === 'base64' ? sameText(value, expected) : sameText(value.toLowerCase(), expected);
+}
+/** The payload: JSON, a form, or plain text (as { body }). */
+export function parsePayload(raw, contentType) {
+    const text = raw.toString('utf8');
+    if (/application\/x-www-form-urlencoded/i.test(contentType ?? '')) {
+        const form = Object.fromEntries(new URLSearchParams(text));
+        // Slack-style and some tools put JSON in a "payload" field.
+        if (typeof form.payload === 'string') {
+            try {
+                return JSON.parse(form.payload);
+            }
+            catch {
+                // Leave it as a form.
+            }
+        }
+        return form;
+    }
+    try {
+        return text.trim() ? JSON.parse(text) : {};
+    }
+    catch {
+        return { body: text };
+    }
+}
+/** The value at a dot path: "data.object.id", "items.0.sku". */
+export function valueAt(payload, dotPath) {
+    let value = payload;
+    for (const key of dotPath.split('.')) {
+        if (value === null || value === undefined || typeof value !== 'object')
+            return undefined;
+        value = value[key];
+    }
+    return value;
+}
+function asText(value) {
+    if (value === undefined || value === null)
+        return undefined;
+    if (typeof value === 'string')
+        return value;
+    if (typeof value === 'number' || typeof value === 'boolean')
+        return String(value);
+    return JSON.stringify(value);
+}
+/** The sender's id for this delivery, if it gives one: what a retry
+ * repeats. */
+export function deliveryId(headers, payload) {
+    for (const name of ['x-github-delivery', 'x-shopify-webhook-id', 'idempotency-key', 'x-delivery-id', 'x-request-id']) {
+        const v = header(headers, name);
+        if (v)
+            return v.slice(0, 200);
+    }
+    const id = valueAt(payload, 'id');
+    return typeof id === 'string' && id ? id.slice(0, 200) : undefined;
+}
+/** Whether the filter keeps this payload. */
+export function passesFilter(filter, payload) {
+    if (!filter)
+        return true;
+    const value = asText(valueAt(payload, filter.path));
+    return value !== undefined && filter.values.includes(value);
+}
+/** The workflow's parameters from the payload. */
+export function paramsFromPayload(trigger, payload) {
+    const params = {};
+    for (const [param, dotPath] of Object.entries(trigger.mapping ?? {})) {
+        const value = asText(valueAt(payload, dotPath));
+        if (value !== undefined && value !== '')
+            params[param] = value.slice(0, MAX_PAYLOAD_PARAM);
+    }
+    if (trigger.payload_param)
+        params[trigger.payload_param] = JSON.stringify(payload).slice(0, MAX_PAYLOAD_PARAM);
+    return params;
+}
+/** Notes a delivery on its trigger, and its id as handled. */
+export async function recordDelivery(id, delivery, seenId) {
+    await getTriggerStore().update(id, (t) => t && {
+        ...t,
+        deliveries: [delivery, ...t.deliveries].slice(0, MAX_DELIVERIES),
+        seen_ids: seenId ? [...t.seen_ids.filter((s) => s !== seenId), seenId].slice(-MAX_SEEN) : t.seen_ids,
+    });
+}