loopengine 0.3.0 → 0.3.2

package/dist/core/workflow-runs.js

@@ -0,0 +1,229 @@
+// Workflow runs: started by hand or by an app (Run now in the console, or
+// POST /agents/:name/workflows/:id/run), or by a schedule that runs the
+// workflow. Each is a record of its own here, so a workflow has one
+// history of every run whatever started it, the Schedules tab only lists
+// what is set to run, and each run can be looked up by its id afterwards
+// (GET /agents/:name/runs/:id).
+//
+// A run executes the same way a scheduled task fires (supplied by the
+// server, adapters/http.ts): an unattended agent turn whose approvals and
+// questions go to the agent's messaging app and wait durably. A run that
+// pauses there is settled by settleFromResume once someone decides.
+//
+// Stored like scheduled tasks: one JSON file per run under
+// `<LOOPENGINE_SCHEDULER_DIR or .scheduler>/runs/`. Each agent keeps its
+// latest MAX_FINISHED_PER_AGENT finished runs; older ones are removed as
+// new runs start.
+import { randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { KeyedMutex } from './session-store.js';
+const MAX_FINISHED_PER_AGENT = 500;
+const SESSION_PREFIX = 'run-';
+/** A scheduled task's sessions (core/scheduler.ts). */
+const SCHEDULE_SESSION_PREFIX = 'sched-';
+export class RunError extends Error {
+}
+const fileMutex = new KeyedMutex();
+/** One JSON file per run under `<dir>/runs/` — one server process. */
+export class FileRunStore {
+    dir;
+    constructor(dir) {
+        this.dir = path.join(dir, 'runs');
+    }
+    file(runId) {
+        if (!/^[A-Za-z0-9_-]+$/.test(runId))
+            throw new RunError(`invalid run id "${runId}"`);
+        return path.join(this.dir, `${runId}.json`);
+    }
+    async list() {
+        if (!existsSync(this.dir))
+            return [];
+        const runs = [];
+        for (const name of readdirSync(this.dir)) {
+            if (!name.endsWith('.json'))
+                continue;
+            try {
+                runs.push(JSON.parse(readFileSync(path.join(this.dir, name), 'utf8')));
+            }
+            catch {
+                // A file being written, or not a run: skip it.
+            }
+        }
+        return runs.sort((a, b) => b.started_at.localeCompare(a.started_at));
+    }
+    async get(runId) {
+        const file = this.file(runId);
+        return existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')) : undefined;
+    }
+    async put(run) {
+        mkdirSync(this.dir, { recursive: true });
+        const file = this.file(run.run_id);
+        writeFileSync(file + '.tmp', JSON.stringify(run, null, 2));
+        renameSync(file + '.tmp', file);
+    }
+    async remove(runId) {
+        rmSync(this.file(runId), { force: true });
+    }
+    async update(runId, fn) {
+        return fileMutex.run(runId, async () => {
+            const next = fn(await this.get(runId));
+            if (next)
+                await this.put(next);
+            return next;
+        });
+    }
+}
+let store;
+/** The process-wide store: `runs/` in the scheduler's folder. */
+export function getRunStore() {
+    store ??= new FileRunStore(process.env.LOOPENGINE_SCHEDULER_DIR || path.join(process.cwd(), '.scheduler'));
+    return store;
+}
+export function setRunStore(next) {
+    store = next;
+}
+// ---------- reading ----------
+/** An agent's runs, newest first; of one workflow when `workflowId` is given. */
+export async function listRuns(agent, filter = {}) {
+    const runs = (await getRunStore().list()).filter((r) => r.agent === agent && (!filter.workflowId || r.workflow_id === filter.workflowId));
+    return filter.limit ? runs.slice(0, filter.limit) : runs;
+}
+/** One of an agent's runs; undefined for another agent's. */
+export async function getRun(agent, runId) {
+    if (!/^[A-Za-z0-9_-]+$/.test(runId))
+        return undefined;
+    const run = await getRunStore().get(runId);
+    return run && run.agent === agent ? run : undefined;
+}
+export function isUnderWay(run) {
+    return run.status === 'running' || run.status === 'awaiting_approval';
+}
+export class WorkflowRunner {
+    execute;
+    maxFinishedPerAgent;
+    onChange;
+    inFlight = new Set();
+    /** `onChange`: called with the run each time it finishes, fails or
+     * stops for a decision (the server sends its notify and callback). */
+    constructor(options) {
+        this.execute = options.execute;
+        this.maxFinishedPerAgent = options.maxFinishedPerAgent ?? MAX_FINISHED_PER_AGENT;
+        this.onChange = options.onChange;
+    }
+    changed(run) {
+        if (!run || !this.onChange)
+            return;
+        void Promise.resolve()
+            .then(() => this.onChange(run))
+            .catch((err) => console.error(`[loopengine] reporting run ${run.run_id} failed:`, err));
+    }
+    /** Records a run and starts it; returns at once with the record. */
+    async start(input) {
+        const runId = randomUUID();
+        const run = {
+            run_id: runId,
+            agent: input.agent,
+            workflow_id: input.workflow_id,
+            ...(input.params && Object.keys(input.params).length ? { params: input.params } : {}),
+            source: input.source,
+            ...(input.notify ? { notify: true } : {}),
+            ...(input.callback_url ? { callback_url: input.callback_url } : {}),
+            ...(input.trigger_id ? { trigger_id: input.trigger_id } : {}),
+            status: 'running',
+            started_at: new Date().toISOString(),
+            message: input.message,
+            session_id: `${SESSION_PREFIX}${runId.slice(0, 8)}-${Date.now().toString(36)}`,
+        };
+        await getRunStore().put(run);
+        await this.prune(input.agent);
+        const job = this.run(run).finally(() => this.inFlight.delete(job));
+        this.inFlight.add(job);
+        return run;
+    }
+    /** A schedule firing one of its agent's workflows (the scheduler's fire,
+     * for a workflow task): recorded as a run of the workflow, then run the
+     * usual way. Returns the outcome for the scheduler, and throws what the
+     * run threw, so the schedule's own history records it too. */
+    async runForSchedule(input) {
+        const run = {
+            run_id: randomUUID(),
+            agent: input.agent,
+            workflow_id: input.workflow_id,
+            ...(input.params && Object.keys(input.params).length ? { params: input.params } : {}),
+            source: 'schedule',
+            task_id: input.task_id,
+            ...(input.task_name ? { task_name: input.task_name } : {}),
+            status: 'running',
+            started_at: new Date().toISOString(),
+            message: input.message,
+            session_id: input.session_id,
+        };
+        await getRunStore().put(run);
+        await this.prune(input.agent);
+        try {
+            const outcome = await this.execute(run.agent, run.message, run.session_id);
+            if (outcome.kind === 'pending')
+                this.changed(await getRunStore().update(run.run_id, (r) => r && { ...r, status: 'awaiting_approval', pending_ids: outcome.pendingIds }));
+            else
+                await this.finish(run.run_id, { status: 'done', result: outcome.text });
+            return outcome;
+        }
+        catch (err) {
+            await this.finish(run.run_id, { status: 'error', error: err instanceof Error ? err.message : String(err) });
+            throw err;
+        }
+    }
+    /** Waits for the runs this process has started (tests, shutdown). */
+    async idle() {
+        await Promise.all([...this.inFlight]);
+    }
+    /** How many runs are executing right now (for idle-shutdown checks). */
+    get running() {
+        return this.inFlight.size;
+    }
+    async run(run) {
+        try {
+            const outcome = await this.execute(run.agent, run.message, run.session_id);
+            if (outcome.kind === 'pending') {
+                this.changed(await getRunStore().update(run.run_id, (r) => r && { ...r, status: 'awaiting_approval', pending_ids: outcome.pendingIds }));
+                return;
+            }
+            await this.finish(run.run_id, { status: 'done', result: outcome.text });
+        }
+        catch (err) {
+            await this.finish(run.run_id, { status: 'error', error: err instanceof Error ? err.message : String(err) });
+        }
+    }
+    async finish(runId, end) {
+        this.changed(await getRunStore().update(runId, (r) => r && { ...r, ...end, pending_ids: undefined, finished_at: new Date().toISOString() }));
+    }
+    /** A paused run resumed after a decision (the server calls this from
+     * its resolve path, for every resumed session): settle it, or record
+     * what it's waiting on now. */
+    async settleFromResume(sessionId, outcome) {
+        if (!sessionId.startsWith(SESSION_PREFIX) && !sessionId.startsWith(SCHEDULE_SESSION_PREFIX))
+            return;
+        const run = (await getRunStore().list()).find((r) => r.session_id === sessionId);
+        if (!run)
+            return;
+        if (outcome.pendingIds?.length) {
+            this.changed(await getRunStore().update(run.run_id, (r) => r && { ...r, status: 'awaiting_approval', pending_ids: outcome.pendingIds }));
+            return;
+        }
+        await this.finish(run.run_id, { status: 'done', result: outcome.text });
+    }
+    /** On startup: a run a restart cut off can't continue, so it's an
+     * error. One waiting on a decision is kept: its checkpoint survives. */
+    async recover() {
+        for (const run of await getRunStore().list()) {
+            if (run.status === 'running')
+                await this.finish(run.run_id, { status: 'error', error: 'The server restarted while this run was in progress.' });
+        }
+    }
+    async prune(agent) {
+        const finished = (await listRuns(agent)).filter((r) => !isUnderWay(r));
+        for (const old of finished.slice(this.maxFinishedPerAgent))
+            await getRunStore().remove(old.run_id);
+    }
+}

package/dist/core/workflow-triggers.d.ts

@@ -0,0 +1,140 @@
+import type { Workflow } from './workflows.js';
+/** Whose signature scheme a trigger checks. "standard" is Standard
+ * Webhooks (Svix: used by Clerk, Resend, OpenAI and others); "custom" is
+ * an HMAC of the raw body in a header the sender names (Typeform, Linear,
+ * Sentry, Intercom and most others that sign the body). */
+export type TriggerSender = 'stripe' | 'github' | 'shopify' | 'standard' | 'custom';
+export type SignatureAlgorithm = 'sha256' | 'sha1' | 'sha512';
+export interface TriggerVerify {
+    sender: TriggerSender;
+    /** The signing secret the sender gives (Stripe's whsec_…, GitHub's,
+     * Shopify's, a Standard Webhooks whsec_…, or a custom one). */
+    secret: string;
+    /** custom only: the header the signature is in, e.g. "Linear-Signature". */
+    header?: string;
+    /** custom only: the HMAC's hash (default sha256). */
+    algorithm?: SignatureAlgorithm;
+    /** custom only: how the signature is written (default hex). */
+    encoding?: 'hex' | 'base64';
+    /** custom only: text before the signature, e.g. "sha256=". */
+    prefix?: string;
+}
+export interface TriggerFilter {
+    /** Where in the payload, as a dot path: "type", "action", "data.object.status". */
+    path: string;
+    /** Run only when the value there is one of these. */
+    values: string[];
+}
+export type TriggerOutcome = 'started' | 'filtered' | 'duplicate' | 'busy' | 'rejected' | 'invalid' | 'error';
+export interface TriggerDelivery {
+    at: string;
+    outcome: TriggerOutcome;
+    delivery_id?: string;
+    run_id?: string;
+    detail?: string;
+}
+export interface WorkflowTrigger {
+    trigger_id: string;
+    agent: string;
+    workflow_id: string;
+    name: string;
+    /** SHA-256 of the URL's token. */
+    token_hash: string;
+    /** The token's start, to tell triggers apart: "whk_8f3k…". */
+    token_hint: string;
+    enabled: boolean;
+    created_at: string;
+    verify?: TriggerVerify;
+    filter?: TriggerFilter;
+    /** Workflow parameter → dot path in the payload. */
+    mapping?: Record<string, string>;
+    /** A parameter that gets the whole payload, as JSON. */
+    payload_param?: string;
+    notify?: boolean;
+    callback_url?: string;
+    /** The latest deliveries, newest first (20). */
+    deliveries: TriggerDelivery[];
+    /** Delivery ids already handled (the latest 200), for retries. */
+    seen_ids: string[];
+}
+/** A trigger as the console and API show it: no token hash, no secret,
+ * only whether one is set. */
+export type PublicTrigger = Omit<WorkflowTrigger, 'token_hash' | 'verify' | 'seen_ids'> & {
+    verify?: Omit<TriggerVerify, 'secret'> & {
+        secret_set: boolean;
+    };
+};
+export declare class TriggerError extends Error {
+}
+export interface TriggerStore {
+    list(): Promise<WorkflowTrigger[]>;
+    get(id: string): Promise<WorkflowTrigger | undefined>;
+    put(trigger: WorkflowTrigger): Promise<void>;
+    remove(id: string): Promise<void>;
+    update(id: string, fn: (t: WorkflowTrigger | undefined) => WorkflowTrigger | undefined): Promise<WorkflowTrigger | undefined>;
+}
+/** One JSON file per trigger under `<dir>/triggers/` — one server process. */
+export declare class FileTriggerStore implements TriggerStore {
+    private readonly dir;
+    constructor(dir: string);
+    private file;
+    list(): Promise<WorkflowTrigger[]>;
+    get(id: string): Promise<WorkflowTrigger | undefined>;
+    put(trigger: WorkflowTrigger): Promise<void>;
+    remove(id: string): Promise<void>;
+    update(id: string, fn: (t: WorkflowTrigger | undefined) => WorkflowTrigger | undefined): Promise<WorkflowTrigger | undefined>;
+}
+export declare function getTriggerStore(): TriggerStore;
+export declare function setTriggerStore(next: TriggerStore): void;
+export declare function publicTrigger(t: WorkflowTrigger): PublicTrigger;
+/** The settings a trigger can have, checked against its workflow. */
+export interface TriggerSettings {
+    name?: unknown;
+    enabled?: unknown;
+    verify?: unknown;
+    filter?: unknown;
+    mapping?: unknown;
+    payload_param?: unknown;
+    notify?: unknown;
+    callback_url?: unknown;
+}
+/** Checks settings for a trigger of `workflow`; throws TriggerError saying
+ * what's wrong. With `existing` (a change), a setting left out keeps its
+ * value and null removes it, and a sender's secret is kept when only the
+ * sender is given again (the console never sees the secret).
+ *
+ * A trigger that checks a signature can be made turned off without its
+ * secret yet: the workflow planner makes one that way, for the person to
+ * add the secret the sender shows them. It can't be turned on until the
+ * secret is there. */
+export declare function checkTriggerSettings(input: TriggerSettings, workflow: Workflow, existing?: WorkflowTrigger): Pick<WorkflowTrigger, 'name' | 'enabled' | 'verify' | 'filter' | 'mapping' | 'payload_param' | 'notify' | 'callback_url'>;
+/** Makes a trigger for a workflow; returns it with its token (shown once). */
+export declare function createTrigger(agent: string, workflow: Workflow, input: TriggerSettings): Promise<{
+    trigger: WorkflowTrigger;
+    token: string;
+}>;
+export declare function updateTrigger(id: string, workflow: Workflow, input: TriggerSettings): Promise<WorkflowTrigger>;
+/** A new URL for a trigger; the old one stops working. */
+export declare function rotateTrigger(id: string): Promise<{
+    trigger: WorkflowTrigger;
+    token: string;
+}>;
+export declare function listTriggers(agent: string, workflowId?: string): Promise<WorkflowTrigger[]>;
+/** The trigger a URL's token belongs to. */
+export declare function findTriggerByToken(token: string): Promise<WorkflowTrigger | undefined>;
+/** Whether the sender's own signature on the raw body is right. Never
+ * with no secret: anyone can sign with an empty key. */
+export declare function verifySenderSignature(verify: TriggerVerify, headers: Record<string, string | string[] | undefined>, raw: Buffer, nowS?: number): boolean;
+/** The payload: JSON, a form, or plain text (as { body }). */
+export declare function parsePayload(raw: Buffer, contentType: string | undefined): unknown;
+/** The value at a dot path: "data.object.id", "items.0.sku". */
+export declare function valueAt(payload: unknown, dotPath: string): unknown;
+/** The sender's id for this delivery, if it gives one: what a retry
+ * repeats. */
+export declare function deliveryId(headers: Record<string, string | string[] | undefined>, payload: unknown): string | undefined;
+/** Whether the filter keeps this payload. */
+export declare function passesFilter(filter: TriggerFilter | undefined, payload: unknown): boolean;
+/** The workflow's parameters from the payload. */
+export declare function paramsFromPayload(trigger: Pick<WorkflowTrigger, 'mapping' | 'payload_param'>, payload: unknown): Record<string, string>;
+/** Notes a delivery on its trigger, and its id as handled. */
+export declare function recordDelivery(id: string, delivery: TriggerDelivery, seenId?: string): Promise<void>;