@mneme-ai/core 0.24.0 → 0.26.0

package/dist/pipeline/super-pipeline.test.js

@@ -0,0 +1,130 @@
+import { describe, it, expect } from "vitest";
+import { runPipeline } from "./super-pipeline.js";
+import { reorderBySeq } from "./superscalar.js";
+import { defineStage } from "./index.js";
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+async function* fromArray(items) {
+    for (const v of items)
+        yield v;
+}
+async function collect(it) {
+    const out = [];
+    for await (const v of it)
+        out.push(v);
+    return out;
+}
+describe("runPipeline — fan-through correctness", () => {
+    it("two-stage chain produces all outputs", async () => {
+        const stages = [
+            defineStage("plus1", "plus1", (n) => n + 1),
+            defineStage("times2", "times2", (n) => n * 2),
+        ];
+        const out = await collect(reorderBySeq(runPipeline({ stages }, [1, 2, 3])));
+        expect(out.map((x) => x.value)).toEqual([4, 6, 8]);
+    });
+    it("preserves input order via reorderBySeq", async () => {
+        // Stage 0 sleeps inversely so later items finish first.
+        const stages = [
+            defineStage("sleep", "sleep", async (n) => {
+                await sleep((5 - n) * 5);
+                return n;
+            }),
+            defineStage("id", "id", (n) => n),
+        ];
+        const out = await collect(reorderBySeq(runPipeline({ stages, width: 4 }, [0, 1, 2, 3, 4])));
+        expect(out.map((x) => x.seq)).toEqual([0, 1, 2, 3, 4]);
+        expect(out.map((x) => x.value)).toEqual([0, 1, 2, 3, 4]);
+    });
+});
+describe("runPipeline — superscalar speedup", () => {
+    it("width=4 finishes a slow stage faster than width=1", async () => {
+        const inputs = Array.from({ length: 8 }, (_, i) => i);
+        const slow = defineStage("slow", "slow", async (n) => {
+            await sleep(20);
+            return n;
+        });
+        const t0 = Date.now();
+        await collect(runPipeline({ stages: [slow], width: 1 }, inputs));
+        const seq = Date.now() - t0;
+        const t1 = Date.now();
+        await collect(runPipeline({ stages: [slow], width: 4 }, inputs));
+        const par = Date.now() - t1;
+        // 4 workers should finish noticeably faster — give a generous bound to
+        // keep the test stable on slow CI.
+        expect(par).toBeLessThan(seq);
+    });
+});
+describe("runPipeline — backpressure", () => {
+    it("a slow downstream stage caps in-flight queue depth", async () => {
+        // When stage 2 is slow + bufferSize=2, stage 1 cannot run more than
+        // ~2 items ahead (queue capacity).
+        let stage1InFlight = 0;
+        let maxInFlight = 0;
+        const stages = [
+            defineStage("fast", "fast", async (n) => {
+                stage1InFlight += 1;
+                maxInFlight = Math.max(maxInFlight, stage1InFlight);
+                await sleep(2);
+                stage1InFlight -= 1;
+                return n;
+            }),
+            defineStage("slow", "slow", async (n) => {
+                await sleep(40);
+                return n;
+            }),
+        ];
+        await collect(runPipeline({ stages, width: 1, bufferSize: 2 }, Array.from({ length: 10 }, (_, i) => i)));
+        // Without backpressure, stage 1 would race ahead and finish all 10 ~immediately.
+        // With bufferSize=2, the running window is bounded.
+        expect(maxInFlight).toBeLessThanOrEqual(2);
+    });
+});
+describe("runPipeline — failure isolation", () => {
+    it("a single stage failure does not stop the rest of the items", async () => {
+        const events = [];
+        const stages = [
+            defineStage("guard", "guard", (n) => {
+                if (n === 2)
+                    throw new Error("nope");
+                return n;
+            }),
+            defineStage("ok", "ok", (n) => n + 100),
+        ];
+        const out = await collect(runPipeline({ stages, onEvent: (e) => events.push(e) }, [0, 1, 2, 3]));
+        // 3 succeed, 1 dropped — but pipeline finishes.
+        expect(out.map((x) => x.value).sort((a, b) => a - b)).toEqual([100, 101, 103]);
+        expect(events.some((e) => e.kind === "stage-fail")).toBe(true);
+    });
+});
+describe("runPipeline — events + MPE telemetry", () => {
+    it("emits stage-start and stage-done for every successful run", async () => {
+        const events = [];
+        const stages = [defineStage("e", "e", (n) => n)];
+        await collect(runPipeline({ stages, onEvent: (e) => events.push(e) }, [0, 1, 2]));
+        const starts = events.filter((e) => e.kind === "stage-start").length;
+        const dones = events.filter((e) => e.kind === "stage-done").length;
+        expect(starts).toBe(3);
+        expect(dones).toBe(3);
+    });
+    it("requires at least one stage", async () => {
+        await expect(async () => {
+            for await (const _ of runPipeline({ stages: [] }, [1])) {
+                // unreachable
+            }
+        }).rejects.toThrow(/at least one stage/);
+    });
+    it("stage receives StageContext.trust ∈ [0,1]", async () => {
+        const trusts = [];
+        const stage = {
+            id: "trust",
+            description: "t",
+            async process(n, ctx) {
+                trusts.push(ctx.trust);
+                return n;
+            },
+        };
+        await collect(runPipeline({ stages: [stage] }, [0]));
+        expect(trusts.every((t) => t >= 0 && t <= 1)).toBe(true);
+    });
+});
+//# sourceMappingURL=super-pipeline.test.js.map

package/dist/pipeline/super-pipeline.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"super-pipeline.test.js","sourceRoot":"","sources":["../../src/pipeline/super-pipeline.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAM,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAGzC,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAE1E,KAAK,SAAS,CAAC,CAAC,SAAS,CAAI,KAAU;IACrC,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,MAAM,CAAC,CAAC;AACjC,CAAC;AAED,KAAK,UAAU,OAAO,CAAI,EAAoB;IAC5C,MAAM,GAAG,GAAQ,EAAE,CAAC;IACpB,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE;QAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,QAAQ,CAAC,uCAAuC,EAAE,GAAG,EAAE;IACrD,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACpD,MAAM,MAAM,GAAG;YACb,WAAW,CAAiB,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;YAC3D,WAAW,CAAiB,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;SACrD,CAAC;QACX,MAAM,GAAG,GAAG,MAAM,OAAO,CACvB,YAAY,CAAC,WAAW,CAAiB,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CACjE,CAAC;QACF,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,wDAAwD;QACxD,MAAM,MAAM,GAAG;YACb,WAAW,CAAiB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;gBACxD,MAAM,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACzB,OAAO,CAAC,CAAC;YACX,CAAC,CAAC;YACF,WAAW,CAAiB,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;SACzC,CAAC;QACX,MAAM,GAAG,GAAG,MAAM,OAAO,CACvB,YAAY,CAAC,WAAW,CAAiB,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CACjF,CAAC;QACF,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,mCAAmC,EAAE,GAAG,EAAE;IACjD,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,WAAW,CAAiB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;YACnE,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;YAChB,OAAO,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;QAEH,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,OAAO,CAAC,WAAW,CAAiB,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACjF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;QAE5B,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,OAAO,CAAC,WAAW,CAAiB,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACjF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;QAE5B,uEAAuE;QACvE,mCAAmC;QACnC,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;IAC1C,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,oEAAoE;QACpE,mCAAmC;QACnC,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,MAAM,MAAM,GAAG;YACb,WAAW,CAAiB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;gBACtD,cAAc,IAAI,CAAC,CAAC;gBACpB,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;gBACpD,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC;gBACf,cAAc,IAAI,CAAC,CAAC;gBACpB,OAAO,CAAC,CAAC;YACX,CAAC,CAAC;YACF,WAAW,CAAiB,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;gBACtD,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAChB,OAAO,CAAC,CAAC;YACX,CAAC,CAAC;SACM,CAAC;QAEX,MAAM,OAAO,CACX,WAAW,CACT,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,EACnC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CACxC,CACF,CAAC;QACF,iFAAiF;QACjF,oDAAoD;QACpD,MAAM,CAAC,WAAW,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,iCAAiC,EAAE,GAAG,EAAE;IAC/C,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG;YACb,WAAW,CAAiB,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;gBAClD,IAAI,CAAC,KAAK,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrC,OAAO,CAAC,CAAC;YACX,CAAC,CAAC;YACF,WAAW,CAAiB,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC;SAC/C,CAAC;QACX,MAAM,GAAG,GAAG,MAAM,OAAO,CACvB,WAAW,CAAiB,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CACtF,CAAC;QACF,gDAAgD;QAChD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/E,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,sCAAsC,EAAE,GAAG,EAAE;IACpD,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,CAAC,WAAW,CAAiB,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAU,CAAC;QAC1E,MAAM,OAAO,CACX,WAAW,CAAiB,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CACnF,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,CAAC,MAAM,CAAC;QACrE,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC,MAAM,CAAC;QACnE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvB,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;YACtB,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,EAAa,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClE,cAAc;YAChB,CAAC;QACH,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAkC;YAC3C,EAAE,EAAE,OAAO;YACX,WAAW,EAAE,GAAG;YAChB,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG;gBAClB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBACvB,OAAO,CAAC,CAAC;YACX,CAAC;SACF,CAAC;QACF,MAAM,OAAO,CAAC,WAAW,CAAiB,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/pipeline/superscalar.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Superscalar layer — N parallel workers per stage with sequence-id
+ * preservation, plus speculative pre-fetch helpers.
+ *
+ * "Superscalar" here means: multiple identical workers can pull items from
+ * the same input queue and process them concurrently. Outputs may complete
+ * out of order (one item is slow while a later one is fast). Each item
+ * carries a seq number so downstream code can re-sort if needed.
+ */
+import type { PipelineStage, StageContext, SeqItem } from "./types.js";
+/**
+ * Spawn `width` parallel workers for a single stage. Reads SeqItem<I> from
+ * `inputs`, processes through `stage.process`, and yields SeqItem<O> in the
+ * order they complete (NOT the order they entered — see reorderBySeq).
+ */
+export declare function superscalar<I, O>(stage: PipelineStage<I, O>, inputs: AsyncIterable<SeqItem<I>>, width: number, baseCtx: Omit<StageContext, "workerId">): AsyncIterable<SeqItem<O>>;
+/**
+ * Re-emit SeqItem<T> in monotonically-increasing seq order. Buffers
+ * out-of-order items in a small heap-like structure (insertion sort —
+ * pipelines rarely have huge reorder windows so O(n) per insert is fine).
+ *
+ * `startAt` is the first seq number to emit (default 0).
+ */
+export declare function reorderBySeq<T>(inputs: AsyncIterable<SeqItem<T>>, startAt?: number): AsyncIterable<SeqItem<T>>;
+/**
+ * Speculative pre-fetch: start nextStage on a predicted input before the
+ * upstream finalizes. Returns a promise + a cancel() function. cancel()
+ * does NOT abort the underlying process call (we can't cancel arbitrary
+ * promises) but flips an internal flag so callers can ignore the result.
+ */
+export declare function speculatePrefetch<I, O>(nextStage: PipelineStage<I, O>, predicted: I, ctx?: StageContext): {
+    promise: Promise<O>;
+    cancel: () => void;
+    cancelled: () => boolean;
+};
+//# sourceMappingURL=superscalar.d.ts.map

package/dist/pipeline/superscalar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"superscalar.d.ts","sourceRoot":"","sources":["../../src/pipeline/superscalar.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAEvE;;;;GAIG;AACH,wBAAuB,WAAW,CAAC,CAAC,EAAE,CAAC,EACrC,KAAK,EAAE,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,EAC1B,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EACjC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,GACtC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAoE3B;AAED;;;;;;GAMG;AACH,wBAAuB,YAAY,CAAC,CAAC,EACnC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EACjC,OAAO,SAAI,GACV,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CA6B3B;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,CAAC,EACpC,SAAS,EAAE,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,EAC9B,SAAS,EAAE,CAAC,EACZ,GAAG,GAAE,YAAyC,GAC7C;IAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAAC,MAAM,EAAE,MAAM,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,OAAO,CAAA;CAAE,CAgBvE"}

package/dist/pipeline/superscalar.js

@@ -0,0 +1,130 @@
+/**
+ * Spawn `width` parallel workers for a single stage. Reads SeqItem<I> from
+ * `inputs`, processes through `stage.process`, and yields SeqItem<O> in the
+ * order they complete (NOT the order they entered — see reorderBySeq).
+ */
+export async function* superscalar(stage, inputs, width, baseCtx) {
+    // Bridge an AsyncIterator into a single shared "next()" channel that
+    // multiple workers can pull from sequentially. JS iterators aren't
+    // thread-safe, but they ARE single-threaded — calls to next() are atomic
+    // with respect to each other on the event loop, so workers naturally
+    // round-robin through the queue.
+    const it = inputs[Symbol.asyncIterator]();
+    // We use a small in-flight buffer + manual completion signalling so
+    // outputs surface as soon as ANY worker finishes, not in worker-id order.
+    const ready = [];
+    let pendingResolve = null;
+    const wakeMain = () => {
+        if (pendingResolve) {
+            const r = pendingResolve;
+            pendingResolve = null;
+            r();
+        }
+    };
+    let activeWorkers = width;
+    const workerLoop = async (workerId) => {
+        const ctx = { ...baseCtx, workerId };
+        while (true) {
+            const next = await it.next();
+            if (next.done)
+                break;
+            const { seq, value } = next.value;
+            try {
+                const out = await stage.process(value, ctx);
+                ready.push({ seq, value: out });
+            }
+            catch (err) {
+                // Re-throw on the main stream by stashing as a pending error.
+                // We push a sentinel that downstream filters out via failure path.
+                ready.push({ seq, value: err });
+                wakeMain();
+                throw err;
+            }
+            wakeMain();
+        }
+    };
+    const workers = Array.from({ length: Math.max(1, width) }, (_, i) => workerLoop(i).finally(() => {
+        activeWorkers -= 1;
+        wakeMain();
+    }));
+    // Drain loop: yield whatever's ready; wait when nothing is.
+    try {
+        while (true) {
+            while (ready.length > 0) {
+                yield ready.shift();
+            }
+            if (activeWorkers === 0 && ready.length === 0)
+                break;
+            await new Promise((resolve) => {
+                pendingResolve = resolve;
+            });
+        }
+        // Surface any worker-side errors.
+        await Promise.all(workers);
+    }
+    catch (err) {
+        // Make sure workers settle even if one threw.
+        await Promise.allSettled(workers);
+        throw err;
+    }
+}
+/**
+ * Re-emit SeqItem<T> in monotonically-increasing seq order. Buffers
+ * out-of-order items in a small heap-like structure (insertion sort —
+ * pipelines rarely have huge reorder windows so O(n) per insert is fine).
+ *
+ * `startAt` is the first seq number to emit (default 0).
+ */
+export async function* reorderBySeq(inputs, startAt = 0) {
+    let next = startAt;
+    const buffer = [];
+    for await (const item of inputs) {
+        // Insert sorted by seq so the smallest is at index 0.
+        let i = 0;
+        while (i < buffer.length && buffer[i].seq < item.seq)
+            i++;
+        buffer.splice(i, 0, item);
+        while (buffer.length > 0 && buffer[0].seq === next) {
+            yield buffer.shift();
+            next += 1;
+        }
+    }
+    // Drain remaining (gaps surface naturally since we only advance on match).
+    while (buffer.length > 0) {
+        const head = buffer.shift();
+        if (head.seq === next) {
+            yield head;
+            next += 1;
+        }
+        else {
+            // Skipped seq — yield in order anyway (this should never happen if
+            // the upstream is well-formed, but it's a safe fallback).
+            yield head;
+            next = head.seq + 1;
+        }
+    }
+}
+/**
+ * Speculative pre-fetch: start nextStage on a predicted input before the
+ * upstream finalizes. Returns a promise + a cancel() function. cancel()
+ * does NOT abort the underlying process call (we can't cancel arbitrary
+ * promises) but flips an internal flag so callers can ignore the result.
+ */
+export function speculatePrefetch(nextStage, predicted, ctx = { trust: 1, workerId: -1 }) {
+    let cancelled = false;
+    const promise = nextStage.process(predicted, ctx).then((v) => {
+        if (cancelled) {
+            ctx.emit?.({ kind: "drop", stage: nextStage.id, reason: "speculation cancelled" });
+        }
+        return v;
+    });
+    ctx.emit?.({ kind: "speculate", stage: nextStage.id, reason: "prefetch" });
+    return {
+        promise,
+        cancel: () => {
+            cancelled = true;
+        },
+        cancelled: () => cancelled,
+    };
+}
+//# sourceMappingURL=superscalar.js.map

package/dist/pipeline/superscalar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"superscalar.js","sourceRoot":"","sources":["../../src/pipeline/superscalar.ts"],"names":[],"mappings":"AAWA;;;;GAIG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,WAAW,CAChC,KAA0B,EAC1B,MAAiC,EACjC,KAAa,EACb,OAAuC;IAEvC,qEAAqE;IACrE,mEAAmE;IACnE,yEAAyE;IACzE,qEAAqE;IACrE,iCAAiC;IACjC,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IAE1C,oEAAoE;IACpE,0EAA0E;IAC1E,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,IAAI,cAAc,GAAwB,IAAI,CAAC;IAE/C,MAAM,QAAQ,GAAG,GAAG,EAAE;QACpB,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,CAAC,GAAG,cAAc,CAAC;YACzB,cAAc,GAAG,IAAI,CAAC;YACtB,CAAC,EAAE,CAAC;QACN,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,aAAa,GAAG,KAAK,CAAC;IAE1B,MAAM,UAAU,GAAG,KAAK,EAAE,QAAgB,EAAiB,EAAE;QAC3D,MAAM,GAAG,GAAiB,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,CAAC;QACnD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,IAAI;gBAAE,MAAM;YACrB,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC;YAClC,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;gBAC5C,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YAClC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,8DAA8D;gBAC9D,mEAAmE;gBACnE,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAY,EAAE,CAAC,CAAC;gBACzC,QAAQ,EAAE,CAAC;gBACX,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,QAAQ,EAAE,CAAC;QACb,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAClE,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;QACzB,aAAa,IAAI,CAAC,CAAC;QACnB,QAAQ,EAAE,CAAC;IACb,CAAC,CAAC,CACH,CAAC;IAEF,4DAA4D;IAC5D,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,KAAK,CAAC,KAAK,EAAG,CAAC;YACvB,CAAC;YACD,IAAI,aAAa,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YACrD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAClC,cAAc,GAAG,OAAO,CAAC;YAC3B,CAAC,CAAC,CAAC;QACL,CAAC;QACD,kCAAkC;QAClC,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,8CAA8C;QAC9C,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAClC,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,YAAY,CACjC,MAAiC,EACjC,OAAO,GAAG,CAAC;IAEX,IAAI,IAAI,GAAG,OAAO,CAAC;IACnB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAChC,sDAAsD;QACtD,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG;YAAE,CAAC,EAAE,CAAC;QAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;QAE1B,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;YACnD,MAAM,MAAM,CAAC,KAAK,EAAG,CAAC;YACtB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,EAAG,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC;YACX,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,mEAAmE;YACnE,0DAA0D;YAC1D,MAAM,IAAI,CAAC;YACX,IAAI,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;QACtB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAC/B,SAA8B,EAC9B,SAAY,EACZ,MAAoB,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE;IAE9C,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3D,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,uBAAuB,EAAE,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;IACH,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IAC3E,OAAO;QACL,OAAO;QACP,MAAM,EAAE,GAAG,EAAE;YACX,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC;QACD,SAAS,EAAE,GAAG,EAAE,CAAC,SAAS;KAC3B,CAAC;AACJ,CAAC"}

package/dist/pipeline/superscalar.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=superscalar.test.d.ts.map

package/dist/pipeline/superscalar.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"superscalar.test.d.ts","sourceRoot":"","sources":["../../src/pipeline/superscalar.test.ts"],"names":[],"mappings":""}

package/dist/pipeline/superscalar.test.js

@@ -0,0 +1,130 @@
+import { describe, it, expect, vi } from "vitest";
+import { superscalar, reorderBySeq, speculatePrefetch } from "./superscalar.js";
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+function makeStage(id, fn) {
+    return { id, description: id, async process(input) { return fn(input); } };
+}
+async function* fromArray(items) {
+    for (const v of items)
+        yield v;
+}
+async function collect(it) {
+    const out = [];
+    for await (const v of it)
+        out.push(v);
+    return out;
+}
+describe("superscalar — N parallel workers", () => {
+    it("processes every input exactly once with width=4", async () => {
+        const seen = [];
+        const stage = makeStage("double", async (n) => {
+            seen.push(n);
+            return n * 2;
+        });
+        const inputs = Array.from({ length: 20 }, (_, i) => ({
+            seq: i,
+            value: i,
+        }));
+        const out = await collect(superscalar(stage, fromArray(inputs), 4, { trust: 1 }));
+        expect(out).toHaveLength(20);
+        expect(seen.sort((a, b) => a - b)).toEqual(Array.from({ length: 20 }, (_, i) => i));
+    });
+    it("with width>1, slow items don't block fast ones (out-of-order)", async () => {
+        // Item 0 sleeps 60ms, items 1..3 sleep 5ms. With width=2, item 1 should
+        // finish before item 0.
+        const stage = makeStage("vary", async (n) => {
+            await sleep(n === 0 ? 60 : 5);
+            return n;
+        });
+        const inputs = [0, 1, 2, 3].map((v, i) => ({ seq: i, value: v }));
+        const out = await collect(superscalar(stage, fromArray(inputs), 2, { trust: 1 }));
+        // First item to complete should NOT be seq 0.
+        expect(out[0].seq).not.toBe(0);
+        expect(out).toHaveLength(4);
+    });
+    it("workerId is forwarded into ctx (0..width-1)", async () => {
+        const ids = new Set();
+        const stage = {
+            id: "id-stage",
+            description: "id",
+            async process(n, ctx) {
+                ids.add(ctx.workerId);
+                await sleep(2);
+                return n;
+            },
+        };
+        const inputs = Array.from({ length: 8 }, (_, i) => ({ seq: i, value: i }));
+        await collect(superscalar(stage, fromArray(inputs), 3, { trust: 1 }));
+        expect([...ids].every((i) => i >= 0 && i < 3)).toBe(true);
+    });
+    it("a thrown stage error propagates to the consumer", async () => {
+        const stage = makeStage("err", async (n) => {
+            if (n === 1)
+                throw new Error("boom");
+            return n;
+        });
+        const inputs = [0, 1, 2].map((v, i) => ({ seq: i, value: v }));
+        await expect(collect(superscalar(stage, fromArray(inputs), 1, { trust: 1 }))).rejects.toThrow(/boom/);
+    });
+});
+describe("reorderBySeq — input-order recovery", () => {
+    it("yields items in monotonic seq order", async () => {
+        // Out-of-order source: 2, 0, 1, 3
+        const src = [
+            { seq: 2, value: "c" },
+            { seq: 0, value: "a" },
+            { seq: 1, value: "b" },
+            { seq: 3, value: "d" },
+        ];
+        const out = await collect(reorderBySeq(fromArray(src)));
+        expect(out.map((x) => x.value)).toEqual(["a", "b", "c", "d"]);
+    });
+    it("respects a non-zero startAt", async () => {
+        const src = [
+            { seq: 11, value: 11 },
+            { seq: 10, value: 10 },
+        ];
+        const out = await collect(reorderBySeq(fromArray(src), 10));
+        expect(out.map((x) => x.seq)).toEqual([10, 11]);
+    });
+    it("drains buffered items once the source closes (gap fallback)", async () => {
+        // Seq 1 is missing — reorder still emits the remaining items.
+        const src = [
+            { seq: 0, value: 0 },
+            { seq: 2, value: 2 },
+        ];
+        const out = await collect(reorderBySeq(fromArray(src)));
+        expect(out.map((x) => x.value)).toEqual([0, 2]);
+    });
+});
+describe("speculatePrefetch", () => {
+    it("starts the next stage immediately on the predicted input", async () => {
+        const fn = vi.fn(async (n) => n * 10);
+        const stage = makeStage("predict", fn);
+        const { promise } = speculatePrefetch(stage, 7);
+        expect(fn).toHaveBeenCalledTimes(1);
+        await expect(promise).resolves.toBe(70);
+    });
+    it("cancel() flips the cancelled flag (the underlying promise still resolves)", async () => {
+        const stage = makeStage("predict", async (n) => {
+            await sleep(10);
+            return n + 1;
+        });
+        const { promise, cancel, cancelled } = speculatePrefetch(stage, 1);
+        cancel();
+        expect(cancelled()).toBe(true);
+        // The promise still resolves — cancel is advisory.
+        await expect(promise).resolves.toBe(2);
+    });
+    it("emits a 'speculate' event when an emit sink is provided", async () => {
+        const events = [];
+        const stage = makeStage("predict", async (n) => n);
+        speculatePrefetch(stage, 0, {
+            trust: 1,
+            workerId: -1,
+            emit: (e) => events.push(e.kind),
+        });
+        expect(events).toContain("speculate");
+    });
+});
+//# sourceMappingURL=superscalar.test.js.map

package/dist/pipeline/superscalar.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"superscalar.test.js","sourceRoot":"","sources":["../../src/pipeline/superscalar.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAGhF,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAE1E,SAAS,SAAS,CAChB,EAAU,EACV,EAA4B;IAE5B,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC7E,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,SAAS,CAAI,KAAU;IACrC,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,MAAM,CAAC,CAAC;AACjC,CAAC;AAED,KAAK,UAAU,OAAO,CAAI,EAAoB;IAC5C,MAAM,GAAG,GAAQ,EAAE,CAAC;IACpB,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE;QAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,QAAQ,CAAC,kCAAkC,EAAE,GAAG,EAAE;IAChD,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,SAAS,CAAiB,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;YAC5D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAsB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACtE,GAAG,EAAE,CAAC;YACN,KAAK,EAAE,CAAC;SACT,CAAC,CAAC,CAAC;QACJ,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAClF,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,wEAAwE;QACxE,wBAAwB;QACxB,MAAM,KAAK,GAAG,SAAS,CAAiB,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;YAC1D,MAAM,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9B,OAAO,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAsB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACrF,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAClF,8CAA8C;QAC9C,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;QAC3D,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;QAC9B,MAAM,KAAK,GAAkC;YAC3C,EAAE,EAAE,UAAU;YACd,WAAW,EAAE,IAAI;YACjB,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG;gBAClB,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtB,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC;gBACf,OAAO,CAAC,CAAC;YACX,CAAC;SACF,CAAC;QACF,MAAM,MAAM,GAAsB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9F,MAAM,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,KAAK,GAAG,SAAS,CAAiB,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;YACzD,IAAI,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC;YACrC,OAAO,CAAC,CAAC;QACX,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAsB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC3F,MAAM,CACP,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,qCAAqC,EAAE,GAAG,EAAE;IACnD,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACnD,kCAAkC;QAClC,MAAM,GAAG,GAAsB;YAC7B,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE;YACtB,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE;YACtB,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE;YACtB,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE;SACvB,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;QAC3C,MAAM,GAAG,GAAsB;YAC7B,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;YACtB,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;SACvB,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;QAC3E,8DAA8D;QAC9D,MAAM,GAAG,GAAsB;YAC7B,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;YACpB,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;SACrB,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACvC,MAAM,EAAE,OAAO,EAAE,GAAG,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,EAAE,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2EAA2E,EAAE,KAAK,IAAI,EAAE;QACzF,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,CAAS,EAAE,EAAE;YACrD,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;YAChB,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QACH,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,EAAE,CAAC;QACT,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,mDAAmD;QACnD,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACvE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,KAAK,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3D,iBAAiB,CAAC,KAAK,EAAE,CAAC,EAAE;YAC1B,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE,CAAC,CAAC;YACZ,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;SACjC,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/pipeline/types.d.ts

@@ -0,0 +1,104 @@
+/**
+ * Super Pipeline Engine — types.
+ *
+ * Borrows two ideas from CPU architecture:
+ *   1. Deep pipelining — many small stages, each running concurrently on
+ *      different items (high throughput, item-level parallelism).
+ *   2. Superscalar — each stage may have multiple workers (width N) so
+ *      independent items at the same stage run in parallel.
+ *
+ * The novel ingredient is MPE (Multi-stage Pipelined Eigentrust): an
+ * eigentrust-style update that auto-tunes per-stage trust + width based
+ * on success rate and latency. See ./mpe.ts.
+ */
+/**
+ * A single stage in the deep pipeline. Tasks flow through stages; each stage
+ * runs in its own worker pool (superscalar width N) and can speculatively
+ * pre-fetch from the next stage's input queue.
+ */
+export interface PipelineStage<I, O> {
+    /** Stage identifier — used by MPE for trust tracking. */
+    readonly id: string;
+    /** Plain-English name for events / logs. */
+    readonly description: string;
+    /** Pure function: input → output. Called with the stage's allocated worker. */
+    process(input: I, ctx: StageContext): Promise<O>;
+    /** Soft target latency (ms). MPE uses this for backpressure. */
+    readonly targetMs?: number;
+}
+/**
+ * Per-call context passed into PipelineStage.process(). Workers receive a
+ * read-only view of their stage's running trust score plus an optional event
+ * sink for telemetry.
+ */
+export interface StageContext {
+    /** Stage's running success rate (0..1) per MPE. */
+    trust: number;
+    /** Worker index (0..width-1) — for tracing. */
+    workerId: number;
+    /** Optional event sink — emits stage-start, stage-done, etc. */
+    emit?: (e: PipelineEvent) => void;
+}
+/**
+ * Events emitted by the runtime for tracing + MPE training. Consumers may
+ * subscribe via PipelineConfig.onEvent (added by super-pipeline.ts).
+ */
+export type PipelineEvent = {
+    kind: "stage-start";
+    stage: string;
+    workerId: number;
+    inputHash?: string;
+} | {
+    kind: "stage-done";
+    stage: string;
+    workerId: number;
+    latencyMs: number;
+} | {
+    kind: "stage-fail";
+    stage: string;
+    workerId: number;
+    error: string;
+} | {
+    kind: "speculate";
+    stage: string;
+    reason: string;
+} | {
+    kind: "drop";
+    stage: string;
+    reason: string;
+};
+/** MPE configuration block. See ./mpe.ts for the math. */
+export interface MpeConfig {
+    /** PageRank-style decay (0..1). Higher = more recency bias. Default 0.85. */
+    decay?: number;
+    /** Trust threshold below which speculative pre-fetch is suppressed. Default 0.3. */
+    speculateThreshold?: number;
+    /** Whether to read/write .mneme/mpe.json — false in tests. Default false. */
+    persist?: boolean;
+    /** Repo root for persistence (only used when persist=true). */
+    repoRoot?: string;
+}
+/**
+ * Configuration for runPipeline. `Stages` is an ordered tuple — the runtime
+ * does not check type compatibility between adjacent stages, but the helper
+ * runDeepPipeline in pipeline/index.ts does at the type level.
+ */
+export interface PipelineConfig<Stages extends ReadonlyArray<PipelineStage<unknown, unknown>> = ReadonlyArray<PipelineStage<unknown, unknown>>> {
+    stages: Stages;
+    /** Superscalar width per stage (default 1). Pass an array to vary per-stage. */
+    width?: number | number[];
+    /** Backpressure queue size between stages (default 16). */
+    bufferSize?: number;
+    /** MPE config — auto-tune trust eigenvector per stage. */
+    mpe?: MpeConfig;
+    /** Optional event subscriber. Receives every PipelineEvent. */
+    onEvent?: (e: PipelineEvent) => void;
+}
+/** Internal envelope: items flow through the pipeline tagged with a sequence id
+ *  so the runtime can yield outputs in original input order even when stages
+ *  complete out of order. */
+export interface SeqItem<T> {
+    readonly seq: number;
+    readonly value: T;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/pipeline/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/pipeline/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,CAAC,EAAE,CAAC;IACjC,yDAAyD;IACzD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,4CAA4C;IAC5C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+EAA+E;IAC/E,OAAO,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACjD,gEAAgE;IAChE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,IAAI,CAAC;CACnC;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAC5E;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC1E;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACtE;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpD,0DAA0D;AAC1D,MAAM,WAAW,SAAS;IACxB,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oFAAoF;IACpF,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,6EAA6E;IAC7E,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;GAIG;AACH,MAAM,WAAW,cAAc,CAC7B,MAAM,SAAS,aAAa,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,GAAG,aAAa,CAC3E,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAChC;IAED,MAAM,EAAE,MAAM,CAAC;IACf,gFAAgF;IAChF,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC1B,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,GAAG,CAAC,EAAE,SAAS,CAAC;IAChB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,IAAI,CAAC;CACtC;AAED;;6BAE6B;AAC7B,MAAM,WAAW,OAAO,CAAC,CAAC;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;CACnB"}

package/dist/pipeline/types.js

@@ -0,0 +1,15 @@
+/**
+ * Super Pipeline Engine — types.
+ *
+ * Borrows two ideas from CPU architecture:
+ *   1. Deep pipelining — many small stages, each running concurrently on
+ *      different items (high throughput, item-level parallelism).
+ *   2. Superscalar — each stage may have multiple workers (width N) so
+ *      independent items at the same stage run in parallel.
+ *
+ * The novel ingredient is MPE (Multi-stage Pipelined Eigentrust): an
+ * eigentrust-style update that auto-tunes per-stage trust + width based
+ * on success rate and latency. See ./mpe.ts.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/pipeline/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/pipeline/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mneme-ai/core",
-  "version": "0.24.0",
+  "version": "0.26.0",
   "description": "Core indexing, retrieval, and graph engine for Mneme",
   "type": "module",
   "main": "./dist/index.js",