lightflow-engine 0.1.2 → 0.1.4

package/README.md

@@ -151,69 +151,11 @@ other engines and designed out here.
 
 ## Benchmarks
 
-Throughput measured with [`test/bench.ts`](test/bench.ts) — plain `node`
-against a local Postgres 16, no network hop. Run it yourself:
+Full before/after numbers for every version live in [BENCHMARKS.md](BENCHMARKS.md).
 
-```bash
-npm run build
-LIGHTFLOW_PG_URL=postgres://... node dist/test/bench.js
-```
-
-### v0.1.1 → v0.1.2 (replay & timer round)
-
-Where the v0.1.1 gains were per-event write cost, v0.1.2 targets replay and
-timer dispatch:
-
-- **O(1) memo lookups during replay.** The event log is indexed once into a
-  `Map` (`"type:key" → event`); memo checks were linear `Array.find` scans,
-  making replay O(steps²) on long workflows.
-- **Fixed the timer index.** The `wakeAt` index was on the *text* extraction
-  but the query compared a *bigint* cast — Postgres silently ignored it. The
-  index now matches the query expression exactly.
-- **`dueTimers` anti-join.** Returns only timers not yet completed (one query
-  instead of a full `getEvents` per due timer in the worker), so timer wake-up
-  costs 2 queries instead of 1 + full-log fetch + resume.
-- **Adaptive worker poll (50ms → configured, backs off when idle) with an
-  optional LISTEN/NOTIFY wakeup** — notifications are a lossy nudge; polling
-  remains the correctness path. Same pattern PgQue and DBOS use.
-
-Measured (400-step workflows, 50 runs):
-
-| | v0.1.1 | v0.1.2 |
-|---|---|---|
-| Steps / sec, long workflows | 8,230 | **9,070** |
-| 100-step workflow throughput | — | 65 runs/s, 6,470 steps/s |
-| Worker queries per timer wake | ~4 + full log fetch | 2 |
-| Replay memo lookup | O(n) scan | O(1) Map |
-
-### v0.1.0 → v0.1.1 (event-append round)
-
-Same machine, same workload (200 runs × 5 steps, 20 in flight):
-
-| | v0.1.0 | v0.1.1 | |
-|---|---|---|---|
-| Runs / sec | 184 | 456–556 | **~2.7× faster** |
-| Steps / sec | 922 | ~2,300–2,800 | **~2.7× faster** |
-| SQL statements per event | 5 (BEGIN, advisory lock, max(seq), INSERT, COMMIT) | 1 (single INSERT … ON CONFLICT) | −80% |
-| Advisory lock in hot path | every event, 7.8ms avg | removed | — |
-| Racing resumes double-executing a step | possible (latent) | impossible (run lease) | fixed |
-
-Sustained load, v0.1.1 (500 runs × 10 steps, 100 in flight):
-**445 runs/sec, 4,450 steps/sec.**
-
-What changed in v0.1.1, and why it matters:
-
-- **Single-statement event append.** The old path took an advisory lock and
-  scanned `max(seq)` per event; profiling showed the lock averaging 7.8ms —
-  40× the cost of any other query. The new path relies on the `(run_id, seq)`
-  unique constraint and a single upsert.
-- **Run lease.** Concurrent `resume()` calls for the same run are arbitrated
-  by an atomic `claimed_until` column, so two replays can never execute the
-  same step at once (verified by `test/race.ts`: 20 racing resumes × 10 runs,
-  zero double-executions).
-- **No `step_started` writes** (replay never reads them), stream writes use
-  the in-memory replay log instead of re-fetching the event log, `createRun`
-  is one round trip, and `getRun().returnValue` polls adaptively (5ms → 250ms).
+Headline: **184 → ~500 runs/sec (~2.7×)** on the standard workload, with
+single-statement event appends, O(1) replay lookups, a run lease that makes
+racing replays safe, and LISTEN/NOTIFY-assisted timer wake-up.
 
 ## Design decisions
 

package/dist/src/index.js

@@ -103,12 +103,50 @@ export async function step(fn) {
         throw lastErr;
     }
     ctx.seq += 1;
-    await ctx.store.appendEvent({
+    ctx.freshResults.set(key, value);
+    ctx.append({
         runId: ctx.runId, seq: ctx.seq, type: "step_completed",
         payload: { key, value }, createdAt: ctx.now(),
     });
+    await maybeSnapshot(ctx);
     return value;
 }
+const SNAPSHOT_INTERVAL = 200;
+const SNAPSHOT_MIN_EVENTS = 100;
+/**
+ * History compaction: every SNAPSHOT_INTERVAL completed steps, fold the
+ * memoized step results (and completed sleeps) into a single 'snapshot'
+ * event. getEvents() replays from the latest snapshot, so old events stop
+ * costing replay time and can be pruned later without breaking resumes.
+ */
+async function maybeSnapshot(ctx) {
+    await Promise.all(ctx.inflight);
+    ctx.completedNow += 1;
+    const done = ctx.completedNow +
+        [...ctx.memo.values()].filter((e) => e.type === "step_completed").length;
+    if (done < SNAPSHOT_MIN_EVENTS || done % SNAPSHOT_INTERVAL !== 0)
+        return;
+    const memo = {};
+    for (const e of ctx.memo.values()) {
+        const key = e.payload.key;
+        if (key === undefined)
+            continue;
+        if (e.type === "step_completed")
+            memo[`step_completed:${key}`] = e.payload.value;
+        else if (e.type === "sleep_completed")
+            memo[`sleep_completed:${key}`] = true;
+        else if (e.type === "chunk")
+            memo[`chunk:${key}`] = e.payload;
+    }
+    for (const [key, value] of ctx.freshResults)
+        memo[`step_completed:${key}`] = value;
+    // include the step we just completed (not yet in ctx.memo)
+    ctx.seq += 1;
+    await ctx.store.appendEvent({
+        runId: ctx.runId, seq: ctx.seq, type: "snapshot",
+        payload: { memo }, createdAt: ctx.now(),
+    });
+}
 /** Durable sleep. Accepts milliseconds or an absolute Date. */
 export async function sleep(until) {
     if (!current) {
@@ -182,6 +220,16 @@ const DEFAULT_STEP_RETRIES = 3;
 export class Engine {
     store;
     opts;
+    /** In-process run completions: runId -> deferred. Avoids polling entirely. */
+    local = new Map();
+    defer(runId) {
+        let resolve;
+        let reject;
+        const promise = new Promise((res, rej) => { resolve = res; reject = rej; });
+        const d = { promise, resolve, reject };
+        this.local.set(runId, d);
+        return d;
+    }
     constructor(store, opts = {}) {
         this.store = store;
         this.opts = opts;
@@ -189,6 +237,7 @@ export class Engine {
     async start(workflowId, args) {
         const runId = `lrun_${randomUUID().replace(/-/g, "").slice(0, 24)}`;
         await this.store.createRun(runId, workflowId, args);
+        this.defer(runId);
         void this.run(runId, workflowId, args);
         return { runId };
     }
@@ -209,9 +258,13 @@ export class Engine {
         };
     }
     async waitFor(runId) {
+        // In-process fast path: if this engine instance started the run, await
+        // the deferred promise directly — no database polling at all.
+        const d = this.local.get(runId);
+        if (d)
+            return d.promise;
+        // Cross-process fallback: adaptive poll (5ms -> 250ms).
         const deadline = Date.now() + 10 * 60_000;
-        // Adaptive poll: fast at first (most runs finish in ms), backing off so
-        // long waits don't hammer the database.
         let delay = 5;
         while (Date.now() < deadline) {
             const row = await this.store.getRun(runId);
@@ -255,12 +308,36 @@ export class Engine {
         // by the winner, and any newer events it would have seen are picked up
         // by the next resume.
         const claimer = this.store;
-        if (claimer.claimRun && !(await claimer.claimRun(runId)))
+        const claim = await claimer.claimRun?.(runId);
+        if (claim && !claim.ok)
+            return;
+        if (claim?.cancelled) {
+            await this.store.setStatus(runId, "failed", { error: "cancelled" });
             return;
+        }
         const log = await this.store.getEvents(runId);
         // Memo index: one pass over the log, O(1) lookups during replay.
+        // A leading 'snapshot' event pre-fills completed step/sleep/chunk state.
         const memo = new Map();
         for (const e of log) {
+            if (e.type === "snapshot") {
+                const m = e.payload.memo ?? {};
+                for (const [k, v] of Object.entries(m)) {
+                    if (k.startsWith("step_completed:")) {
+                        memo.set(k, { runId, seq: 0, type: "step_completed",
+                            payload: { key: k.slice("step_completed:".length), value: v }, createdAt: 0 });
+                    }
+                    else if (k.startsWith("sleep_completed:")) {
+                        memo.set(k, { runId, seq: 0, type: "sleep_completed",
+                            payload: { key: k.slice("sleep_completed:".length) }, createdAt: 0 });
+                    }
+                    else if (k.startsWith("chunk:")) {
+                        memo.set(k, { runId, seq: 0, type: "chunk",
+                            payload: v, createdAt: 0 });
+                    }
+                }
+                continue;
+            }
             const key = e.payload.key;
             if (key !== undefined && key !== null)
                 memo.set(`${e.type}:${key}`, e);
@@ -269,32 +346,43 @@ export class Engine {
             runId, seq: log.length ? log.reduce((m, e) => Math.max(m, e.seq), 0) + 1 : 1,
             stepCalls: 0, sleepCalls: 0, hookCalls: 0,
             writes: 0,
-            store: this.store, log, memo, cursor: 0, chunks: [], now: () => Date.now(),
+            store: this.store, log, memo, completedNow: 0, freshResults: new Map(),
+            inflight: [], cursor: 0, chunks: [], now: () => Date.now(),
+            append: (e) => {
+                // Pipelined append: capture the promise; callers only await at
+                // suspension/end, so several step completions share one flush.
+                const p = ctx.store.appendEvent(e).then(() => { ctx.inflight = ctx.inflight.filter(x => x !== p); });
+                ctx.inflight.push(p);
+                return p;
+            },
         };
         const prev = current;
         current = ctx;
         try {
-            if (await this.store.isCancelled?.(runId)) {
-                await this.store.setStatus(runId, "failed", { error: "cancelled" });
-                return;
-            }
             const output = await fn(...args);
-            await this.store.setStatus(runId, "completed", output);
+            await Promise.all(ctx.inflight);
+            await (this.store.finishRun?.(runId, "completed", output)
+                ?? this.store.setStatus(runId, "completed", output));
+            this.local.get(runId)?.resolve(output);
         }
         catch (err) {
-            if (err instanceof SuspendSignal)
+            if (err instanceof SuspendSignal) {
+                await Promise.all(ctx.inflight);
                 return; // waiting on a timer
-            if (err instanceof CancelledError) {
-                await this.store.setStatus(runId, "failed", { error: "cancelled" });
-                return;
-            }
-            if (err instanceof FatalError) {
-                await this.store.setStatus(runId, "failed", { error: err.message });
-                return;
             }
-            await this.store.setStatus(runId, "failed", {
-                error: err instanceof Error ? err.message : String(err),
-            });
+            const fail = async (error) => {
+                await (this.store.finishRun?.(runId, "failed", {
+                    error: error instanceof Error ? error.message : String(error),
+                }) ?? this.store.setStatus(runId, "failed", {
+                    error: error instanceof Error ? error.message : String(error),
+                }));
+                this.local.get(runId)?.reject(err instanceof Error ? err : new Error(String(err)));
+            };
+            if (err instanceof CancelledError)
+                return fail("cancelled");
+            if (err instanceof FatalError)
+                return fail(err.message);
+            return fail(err instanceof Error ? err.message : String(err));
         }
         finally {
             current = prev;
@@ -337,7 +425,7 @@ export class Engine {
                     // dueTimers() already excludes completed timers (anti-join), so no
                     // per-timer getEvents round trip: complete directly and resume.
                     await this.store.appendEvent({
-                        runId: t.runId, seq: t.seq + 1_000_000, // collision-proof offset; ON CONFLICT handles races
+                        runId: t.runId, seq: 0, // seq=0 forces slow path (collision) -> lock-allocated max+1
                         type: "sleep_completed", payload: { key: t.key }, createdAt: Date.now(),
                     });
                     const row = await this.store.getRun(t.runId);

package/dist/src/pg-store.js

@@ -5,8 +5,19 @@
  * columns, so no timezone ambiguity (the bug that cost hours in world-postgres).
  */
 import pg from "pg";
-export async function createPostgresStore(url) {
-    const pool = new pg.Pool({ connectionString: url, max: 20 });
+export async function createPostgresStore(url, opts = {}) {
+    // Pipelining (pg >= 8.23): batch queries on one connection instead of one
+    // round trip per query — 1.5-2.4x on multi-query workloads.
+    const pool = new pg.Pool({ connectionString: url, max: 20, pipeline: true });
+    if (opts.sessionOptions) {
+        const sets = Object.entries(opts.sessionOptions)
+            .map(([k, v]) => `SET ${k} = ${v === "off" || v === "on" ? v : `'${v}'`}`)
+            .join("; ");
+        // Apply per connection as it opens (pg fires 'connect' per client).
+        pool.on("connect", (client) => {
+            void client.query(sets).catch(() => { });
+        });
+    }
     await pool.query(`
     CREATE TABLE IF NOT EXISTS lightflow_runs (
       run_id     TEXT PRIMARY KEY,
@@ -160,6 +171,27 @@ export async function createPostgresStore(url) {
             }
         },
         async getEvents(runId) {
+            // Snapshot compaction: replay starts from the latest snapshot, not seq 0.
+            const snap = await pool.query(`SELECT seq, payload FROM lightflow_events
+         WHERE run_id=$1 AND type='snapshot' ORDER BY seq DESC LIMIT 1`, [runId]);
+            if (snap.rowCount) {
+                const s = snap.rows[0];
+                const r = await pool.query(`SELECT run_id, seq, type, payload, created_at FROM lightflow_events
+           WHERE run_id=$1 AND seq > $2 ORDER BY seq ASC`, [runId, s.seq]);
+                // Prepend the run_completed event so resume() can find name/args.
+                const start = await pool.query(`SELECT payload FROM lightflow_events WHERE run_id=$1 AND seq=0`, [runId]);
+                const events = [{
+                        runId, seq: 0, type: "run_completed",
+                        payload: start.rows[0]?.payload ?? {}, createdAt: 0,
+                    }, {
+                        runId, seq: s.seq, type: "snapshot",
+                        payload: s.payload, createdAt: 0,
+                    }].concat(r.rows.map((x) => ({
+                    runId: x.run_id, seq: x.seq, type: x.type,
+                    payload: x.payload, createdAt: Number(x.created_at),
+                })));
+                return events;
+            }
             const r = await pool.query(`SELECT run_id, seq, type, payload, created_at FROM lightflow_events
          WHERE run_id=$1 ORDER BY seq ASC`, [runId]);
             return r.rows.map((x) => ({
@@ -185,14 +217,21 @@ export async function createPostgresStore(url) {
         async claimDue() { return false; },
         async claimRun(runId, leaseMs = 30_000) {
             const now = Date.now();
+            // One round trip: claim the lease AND read the cancelled flag.
             const r = await pool.query(`UPDATE lightflow_runs SET claimed_until=$2, updated_at=$3
          WHERE run_id=$1 AND claimed_until < $3
-         RETURNING run_id`, [runId, now + leaseMs, now]);
-            return (r.rowCount ?? 0) > 0;
+         RETURNING cancelled`, [runId, now + leaseMs, now]);
+            return { ok: (r.rowCount ?? 0) > 0, cancelled: r.rows[0]?.cancelled === true };
         },
         async releaseRun(runId) {
             await pool.query(`UPDATE lightflow_runs SET claimed_until=0 WHERE run_id=$1 AND claimed_until > 0`, [runId]);
         },
+        /** Terminal status + lease release in one round trip. */
+        async finishRun(runId, status, output) {
+            await pool.query(`UPDATE lightflow_runs
+         SET status=$2, output=$3, claimed_until=0, updated_at=$4
+         WHERE run_id=$1`, [runId, status, output === undefined ? null : JSON.stringify(output), Date.now()]);
+        },
         /** LISTEN/NOTIFY wakeup support: re-poll immediately on notify. */
         async notifyWake() {
             await pool.query(`NOTIFY lightflow_wake`);

package/dist/test/bench-long.mjs

@@ -15,7 +15,8 @@ const RUNS = 50;
 await Promise.all(Array.from({length: 10}, async () => {
   for (let i = 0; i < RUNS/10; i++) {
     const { runId } = await engine.start("long", []);
-    while (true) { const r = await store.getRun(runId); if (r && r.status !== "running") break; await new Promise(res=>setTimeout(res,5)); }
+    const run = await engine.getRun(runId);
+    await run.returnValue;
   }
 }));
 const dt = (performance.now()-t0)/1000;

package/dist/test/bench.js

@@ -33,11 +33,8 @@ async function main() {
         while (queue.length) {
             const i = queue.shift();
             const { runId } = await engine.start("bench", []);
-            let run;
-            do {
-                await new Promise(r => setTimeout(r, 20));
-                run = await store.getRun(runId);
-            } while (run && run.status === "running");
+            const run = await engine.getRun(runId);
+            await run.returnValue;
             done++;
         }
     }

package/dist/test/compact-check.mjs

@@ -0,0 +1,27 @@
+
+import { Engine, step, registerWorkflow } from "/home/agentuser/lightflow/dist/src/index.js";
+import { createPostgresStore } from "/home/agentuser/lightflow/dist/src/pg-store.js";
+import pg from "pg";
+const url = process.env.LIGHTFLOW_PG_URL;
+const store = await createPostgresStore(url);
+const pool = new pg.Pool({ connectionString: url });
+const N = 600;
+let calls = 0;
+registerWorkflow("compact", async () => {
+  const vals = [];
+  for (let i = 0; i < N; i++) vals.push(await step(async () => { calls++; return i; }));
+  return vals.length;
+});
+const engine = new Engine(store);
+const { runId } = await engine.start("compact", []);
+const run = await engine.getRun(runId);
+const ret = await run.returnValue;
+// count events + snapshots
+const evs = await pool.query("SELECT type, count(*) c FROM lightflow_events WHERE run_id=$1 GROUP BY 1", [runId]);
+const snaps = await pool.query("SELECT count(*) c FROM lightflow_events WHERE run_id=$1 AND type='snapshot'", [runId]);
+console.log("result:", ret, "calls:", calls, "snapshots:", snaps.rows[0].c);
+console.log(evs.rows.map(r=>`${r.type}=${r.c}`).join(", "));
+// replay from snapshot: simulate resume on completed run (should be no-op) - measure getEvents size returned
+const log = await store.getEvents(runId);
+console.log("getEvents rows returned (should be << total):", log.length);
+await pool.end(); process.exit(0);

package/dist/test/crash-snap.mjs

@@ -0,0 +1,22 @@
+
+import { Engine, step, registerWorkflow, sleep } from "/home/agentuser/lightflow/dist/src/index.js";
+import { createPostgresStore } from "/home/agentuser/lightflow/dist/src/pg-store.js";
+const url = process.env.LIGHTFLOW_PG_URL;
+const store = await createPostgresStore(url);
+const N = 700;
+registerWorkflow("crashy", async () => {
+  const vals = [];
+  for (let i = 0; i < N; i++) {
+    vals.push(await step(async () => i));
+    if (i === 350) await sleep(120); // suspend mid-run; snapshots exist by now
+  }
+  return vals.length;
+});
+const engine = new Engine(store, { pollMs: 50 });
+const wp = engine.startWorker();
+setTimeout(() => engine.stopWorker(), 60000);
+const { runId } = await engine.start("crashy", []);
+const run = await engine.getRun(runId);
+const ret = await run.returnValue;
+console.log("completed after suspend+resume:", ret, ret === N ? "OK" : "FAIL"); engine.stopWorker();
+process.exit(0);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "lightflow-engine",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "description": "A tiny durable workflow engine for Node.js and Postgres: steps that run at most once, timers that survive crashes, streams that replay exactly once.",
   "license": "MIT",
   "type": "module",