@wenathlan/devthink 2.0.15 → 2.0.19

package/tests/torture-data.test.ts

@@ -1,923 +0,0 @@
-import { describe, expect, it } from "vitest";
-import {
-  advancecursor,
-  advancestream,
-  artifactrecordof,
-  backpressure,
-  builddataset,
-  checksum,
-  chunkextractprogress,
-  chunkextracttaskof,
-  chunkextractwindow,
-  chunkplan,
-  deduperows,
-  exportartifact,
-  exportcontent,
-  extractbatchof,
-  gridpreview,
-  interpolate,
-  loopstep,
-  loopvariables,
-  mergechunkwindows,
-  mergetaskrules,
-  newextractsession,
-  newstream,
-  openchunkextractcursor,
-  pipelinestate,
-  provenancefor,
-  pipelinegridpreview,
-  provlog,
-  provlogquery,
-  remainingpages,
-  resumechunkextract,
-  resumeextract,
-  retainedexports,
-  samplerows,
-  sheetpayload,
-  sortrows,
-  sourcestamp,
-  streamdisk,
-  streamfrom,
-  streamparsechunkof,
-  streamparsememorybound,
-  streamparseobservations,
-  streamparsewindows,
-  streamparsetaskof,
-  transformvalues,
-  validatesetreamparsechunk,
-} from "../data.js";
-import type {
-  columnspec,
-  dataset,
-  extractpipeline,
-  extractrow,
-  streamcursor,
-  streamstate,
-  transformrule,
-} from "../types.js";
-
-const now = 1_800_000_000_000;
-
-/** Builds one reviewed pipeline fixture of the pricing extraction. */
-function pipeline(over: Partial<extractpipeline> = {}): extractpipeline {
-  return {
-    id: "p1",
-    runid: "run1",
-    name: "Pricing rows",
-    source: "table.pricing",
-    steps: ["s1", "s2"],
-    transforms: [],
-    sink: "csv",
-    state: "running",
-    origin: "https://example.com",
-    createdat: now,
-    updatedat: now,
-    ...over,
-  };
-}
-
-/** Builds one extracted row fixture with its key and values. */
-function row(key: string, values: Record<string, string>, over: Partial<extractrow> = {}): extractrow {
-  return { key, values, stamps: [], provenance: [], ...over };
-}
-
-/** Builds one dataset fixture with the given columns and rows. */
-function datasetof(columns: columnspec[], rows: Array<Record<string, string>>): dataset {
-  return builddataset("d1", "Pricing", { columns, rows }, now);
-}
-
-const textcolumn = (key: string, label = key): columnspec => ({ key, label, kind: "text", normalized: "none" });
-const numbercolumn = (key: string, label = key): columnspec => ({ key, label, kind: "number", normalized: "none" });
-
-describe("torture: dataset records, checksums and artifact records", () => {
-  it("checksums deterministically over empty, ascii, unicode and null byte payloads", () => {
-    expect(checksum("")).toBe(checksum(""));
-    expect(checksum("devthink")).toMatch(/^fnv1a-[0-9a-f]+$/);
-    expect(checksum("a")).not.toBe(checksum("b"));
-    expect(checksum("éé")).not.toBe(checksum("ée"));
-    expect(checksum("x\0y")).toMatch(/^fnv1a-/);
-    expect(checksum("𝕏")).toMatch(/^fnv1a-/);
-  });
-
-  it("builds datasets with an empty name falling back to the id and passing the grid through untouched", () => {
-    const bare = builddataset("d9", "", { columns: [textcolumn("a")], rows: [{ a: "1" }] }, now);
-    expect(bare.name).toBe("d9");
-    expect(bare.sources).toEqual([]);
-    expect(bare.at).toBe(now);
-    expect(bare.rows).toEqual([{ a: "1" }]);
-  });
-
-  it("exports artifacts with format extensions, row counts and checksums that track the content", () => {
-    const ds = datasetof([textcolumn("a")], [{ a: "1" }, { a: "2" }]);
-    const csv = exportartifact("art1", ds, "csv", "s1", "a,b\n1,2", now);
-    expect(csv.name).toBe("Pricing.csv");
-    expect(csv.rowcount).toBe(2);
-    expect(csv.checksum).toBe(checksum("a,b\n1,2"));
-    expect(exportartifact("art2", ds, "excel", "s1", "<workbook/>", now).name).toBe("Pricing.xml");
-    expect(exportartifact("art3", ds, "json", "s1", "{}", now).name).toBe("Pricing.json");
-    expect(artifactrecordof(csv)).toEqual({ id: "art1", kind: "csv", name: "Pricing.csv", stepid: "s1", at: now });
-  });
-
-  it("renders the csv, json and excel export formats with escaping and a custom delimiter", () => {
-    const columns = [textcolumn("note", "Note"), numbercolumn("qty", "Qty")];
-    const rows = [{ note: 'has "quotes", commas\nand newlines', qty: "2" }];
-    const csv = exportcontent(datasetof(columns, rows), "csv");
-    expect(csv).toContain('has ""quotes"", commas\nand newlines');
-    const semicolon = exportcontent(datasetof(columns, rows), "csv", ";");
-    expect(semicolon.split("\n")[0]).toBe("Note;Qty");
-    const json = exportcontent(datasetof(columns, rows), "json");
-    expect(JSON.parse(json).rows[0].qty).toBe("2");
-    const excel = exportcontent(datasetof(columns, rows), "excel");
-    expect(excel).toContain("&quot;quotes&quot;");
-    expect(excel).not.toContain("<script");
-  });
-
-  it("escapes xml hostile cell payloads inside the excel export", () => {
-    const excel = exportcontent(datasetof([textcolumn("cell")], [{ cell: "<script>alert(1)</script>&amp;" }]), "excel");
-    expect(excel).toContain("&lt;script&gt;");
-    expect(excel).not.toContain("<script>alert");
-  });
-
-  // REPORTED: shared-module bug — see worklog 2-c: page.ts tocsv ships formula-leading cells (= + @ tab/CR) raw through exportcontent, so the datacommand csv export carries spreadsheet formula payloads; page.ts is outside this agent's owned files.
-  it.skip("neutralizes spreadsheet formula payloads in the csv export", () => {
-    const csv = exportcontent(
-      datasetof([textcolumn("cell")], [{ cell: "=1+1" }, { cell: "+cmd|' /C calc'!A0" }, { cell: "@SUM(A1:A9)" }]),
-      "csv",
-    );
-    expect(csv).not.toMatch(/^[=+@]/m);
-  });
-
-  it("keeps formula cells as inert json values", () => {
-    const json = exportcontent(datasetof([textcolumn("cell")], [{ cell: "=1+1" }]), "json");
-    expect(JSON.parse(json).rows[0].cell).toBe("=1+1");
-  });
-
-  it("plans chunk boundaries for zero, exact, fractional, negative and huge chunk sizes", () => {
-    expect(chunkplan(0, 5)).toEqual([{ index: 0, from: 0, to: 0 }]);
-    expect(chunkplan(10, 3)).toHaveLength(4);
-    expect(chunkplan(10, 3).at(-1)).toEqual({ index: 3, from: 9, to: 10 });
-    expect(chunkplan(10, 0.9)).toHaveLength(10);
-    expect(chunkplan(10, -5)).toHaveLength(10);
-    expect(chunkplan(10, 10)).toEqual([{ index: 0, from: 0, to: 10 }]);
-    expect(chunkplan(10, 1_000_000)).toEqual([{ index: 0, from: 0, to: 10 }]);
-    expect(chunkplan(7, 2).every((chunk) => chunk.to - chunk.from <= 2)).toBe(true);
-  });
-
-  it("gates backpressure at the in flight budget of one", () => {
-    expect(backpressure(0, 0)).toBe(false);
-    expect(backpressure(1, 0)).toBe(true);
-    expect(backpressure(5, 5)).toBe(false);
-    expect(backpressure(5, 4)).toBe(true);
-  });
-
-  it("advances stream state and reads the first unwritten row from fresh, done and past the end states", () => {
-    const stream: streamstate = newstream(datasetof([textcolumn("a")], [{ a: "1" }]), 3, now);
-    expect(stream).toEqual({ datasetid: "d1", name: "Pricing", chunk: 0, chunks: 3, written: 0, at: now });
-    const advanced = advancestream(stream, { index: 1, to: 4 }, now + 10, false);
-    expect(advanced).toEqual({ datasetid: "d1", name: "Pricing", chunk: 2, chunks: 3, written: 4, at: now + 10 });
-    expect(advanced.done).toBeUndefined();
-    const finished = advancestream(stream, { index: 2, to: 10 }, now + 20, true);
-    expect(finished.done).toBe(true);
-    expect(streamfrom(undefined, 5)).toBe(0);
-    expect(streamfrom(finished, 5)).toBe(0);
-    expect(streamfrom(advanced, 2)).toBe(2);
-    expect(streamfrom({ ...advanced, written: 7 }, 5)).toBe(5);
-  });
-});
-
-describe("torture: extraction sessions, provenance and retention", () => {
-  it("walks extraction sessions page by page until the planned cursor completes", () => {
-    const session = newextractsession("e1", "d1", "Pricing", "https://example.com", "?page=2", 2, now);
-    expect(session).toMatchObject({ id: "e1", datasetid: "d1", cursor: 0, rows: 0, pages: [] });
-    const one = advancecursor(session, "?page=2", 20, now + 10, false);
-    expect(one.cursor).toBe(1);
-    expect(one.rows).toBe(20);
-    expect(one.done).toBeUndefined();
-    const two = advancecursor(one, "?page=3", 10, now + 20, false);
-    expect(two.done).toBe(true);
-    expect(two.pages).toEqual(["?page=2", "?page=3"]);
-    expect(remainingpages(two, 5)).toBe(0);
-    expect(remainingpages(one, 5)).toBe(4);
-    expect(remainingpages(one, 2)).toBe(1);
-    expect(remainingpages(newextractsession("e2", "d1", "Pricing", "https://example.com", "", 5, now), 2)).toBe(5);
-  });
-
-  it("stamps provenance with one based row ranges and applies retention as a head slice", () => {
-    const provenance = provenancefor(
-      { id: "art1", name: "Pricing.csv", rowcount: 42, checksum: "fnv1a-1" },
-      "https://example.com",
-      "s1",
-      now,
-    );
-    expect(provenance).toEqual({
-      artifact: "art1",
-      name: "Pricing.csv",
-      url: "https://example.com",
-      stepid: "s1",
-      rowstart: 1,
-      rowend: 42,
-      checksum: "fnv1a-1",
-      at: now,
-    });
-    expect(provenancefor({ id: "a", name: "n", rowcount: 0, checksum: "c" }, "u", "s", now).rowstart).toBe(0);
-    expect(retainedexports([1, 2, 3], undefined)).toEqual([1, 2, 3]);
-    expect(retainedexports([1, 2, 3], 0)).toEqual([]);
-    expect(retainedexports([1, 2, 3], 2)).toEqual([1, 2]);
-    expect(retainedexports([1, 2, 3], 99)).toEqual([1, 2, 3]);
-    expect(retainedexports([1, 2, 3], -5)).toEqual([]);
-  });
-});
-
-describe("torture: row interpolation and loop variables", () => {
-  it("interpolates {{column}} tokens with trim, missing keys and repeated tokens", () => {
-    expect(interpolate("{{name}} costs {{ price }}", { name: "Devthink", price: "1299.50" })).toBe(
-      "Devthink costs 1299.50",
-    );
-    expect(interpolate("{{missing}}", { name: "x" })).toBe("");
-    expect(interpolate("{{a}}{{a}}{{a}}", { a: "x" })).toBe("xxx");
-    expect(interpolate("no tokens", { a: "x" })).toBe("no tokens");
-    expect(interpolate("{{ name", { name: "x" })).toBe("{{ name");
-    expect(interpolate("{{}}", { "": "empty key" })).toBe("{{}}");
-  });
-
-  it("refuses to interpolate inherited properties so a crafted token never reads object internals", () => {
-    expect(interpolate("{{constructor}}", {} as Record<string, string>)).toBe("");
-    expect(interpolate("{{toString}}", {} as Record<string, string>)).toBe("");
-    expect(interpolate("{{__proto__}}", {} as Record<string, string>)).toBe("");
-    expect(interpolate("{{ hasOwnProperty }}", {} as Record<string, string>)).toBe("");
-  });
-
-  it("substitutes row variables into the loop step target, value and options", () => {
-    const step = {
-      id: "s1",
-      kind: "click" as const,
-      target: "#{{name}}",
-      value: "{{price}}",
-      options: '{"row":"{{name}}"}',
-      summary: "",
-      risk: "interaction" as const,
-    };
-    const substituted = loopstep(step, { name: "buy", price: "12" });
-    expect(substituted.target).toBe("#buy");
-    expect(substituted.value).toBe("12");
-    expect(JSON.parse(substituted.options ?? "{}")).toEqual({ row: "buy" });
-    const untouched = loopstep({ id: "s2", kind: "wait" as const, summary: "", risk: "read" as const }, { name: "x" });
-    expect(untouched.target).toBeUndefined();
-    expect(untouched.options).toBeUndefined();
-    const variables = loopvariables({ name: "x", nested: "y" });
-    expect(variables).toEqual({ name: "x", nested: "y" });
-    variables.name = "changed";
-    expect(loopvariables({ name: "x" }).name).toBe("x");
-  });
-});
-
-describe("torture: grid preview, sorting and task rules", () => {
-  it("previews the grid with zero, negative, fractional and huge sample bounds", () => {
-    const ds = datasetof([textcolumn("a")], [{ a: "1" }, { a: "2" }, { a: "3" }]);
-    expect(gridpreview(ds, 0).sample).toEqual([]);
-    expect(gridpreview(ds, -3).sample).toEqual([]);
-    expect(gridpreview(ds, 2.9).sample).toHaveLength(2);
-    expect(gridpreview(ds, 1_000_000).sample).toHaveLength(3);
-    expect(gridpreview(ds, 1).columns).toEqual(["a"]);
-    expect(gridpreview(ds, 1).rows).toBe(3);
-  });
-
-  it("sorts numerically when both cells parse and falls back to collation otherwise", () => {
-    const rows = [{ n: "10" }, { n: "9" }, { n: "2.5" }, { n: "-1" }];
-    expect(sortrows(rows, "n", "asc").map((entry) => entry.n)).toEqual(["-1", "2.5", "9", "10"]);
-    expect(sortrows(rows, "n", "desc").map((entry) => entry.n)).toEqual(["10", "9", "2.5", "-1"]);
-    const textrows = [{ n: "beta" }, { n: "Alpha" }, { n: "" }];
-    expect(sortrows(textrows, "n", "asc").map((entry) => entry.n)).toEqual(["", "Alpha", "beta"]);
-    const missing = sortrows([{ n: "1" }, { m: "2" }], "ghost", "asc");
-    expect(missing.map((entry) => entry.n ?? "")).toEqual(["1", ""]);
-    expect(sortrows([], "n", "asc")).toEqual([]);
-  });
-
-  it("merges task rules keeping the stored list when the new list is empty", () => {
-    const stored = {
-      taskid: "t1",
-      transforms: [{ expression: "", sources: [], target: "a", field: "a", operation: "trim" as const }],
-      dedupekeys: ["a"],
-    };
-    const merged = mergetaskrules(stored, "t1", [], ["b"], now);
-    expect(merged.transforms).toEqual(stored.transforms);
-    expect(merged.dedupekeys).toEqual(["b"]);
-    const fresh = mergetaskrules(undefined, "t2", [], [], now);
-    expect(fresh).toEqual({ taskid: "t2", transforms: [], dedupekeys: [], at: now });
-    const sheet = sheetpayload(datasetof([textcolumn("a")], [{ a: "1" }]), "Inventory");
-    expect(sheet).toEqual({ sheet: "Inventory", columns: ["a"], rows: [{ a: "1" }] });
-  });
-});
-
-describe("torture: streamdisk, resume and transform passes", () => {
-  it("streams empty extracts without one write and survives a cursor pointing past the end", () => {
-    const writes: number[] = [];
-    const empty = streamdisk({
-      pipeline: pipeline(),
-      rows: [],
-      chunk: 5,
-      write: (chunk) => {
-        writes.push(chunk.length);
-        return 1;
-      },
-      now,
-    });
-    expect(writes).toEqual([]);
-    expect(empty.rowswritten).toBe(0);
-    expect(empty.chunks).toBe(0);
-    expect(empty.cursor).toEqual({ pipelineid: "p1", offset: 0, chunk: 0, updatedat: now });
-    const past = streamdisk({
-      pipeline: pipeline(),
-      rows: [row("r1", { a: "1" })],
-      cursor: { pipelineid: "p1", offset: 100, chunk: 9, updatedat: now },
-      chunk: 5,
-      write: () => 1,
-      now,
-    });
-    expect(past.rowswritten).toBe(0);
-    expect(past.cursor.offset).toBe(100);
-  });
-
-  it("refuses invalid chunk sizes and propagates a sink error mid stream", () => {
-    expect(() =>
-      streamdisk({ pipeline: pipeline(), rows: [row("r1", { a: "1" })], chunk: 0, write: () => 1, now }),
-    ).toThrow(/positive whole number/);
-    expect(() =>
-      streamdisk({ pipeline: pipeline(), rows: [row("r1", { a: "1" })], chunk: -2, write: () => 1, now }),
-    ).toThrow(/positive whole number/);
-    expect(() =>
-      streamdisk({ pipeline: pipeline(), rows: [row("r1", { a: "1" })], chunk: 1.5, write: () => 1, now }),
-    ).toThrow(/positive whole number/);
-    expect(() => streamdisk({ pipeline: pipeline(), rows: [], write: () => 1, now })).not.toThrow();
-    expect(() => streamdisk({ pipeline: pipeline({ id: " " }), rows: [], write: () => 1, now })).toThrow(
-      /needs its pipeline/,
-    );
-    let wrote = 0;
-    expect(() =>
-      streamdisk({
-        pipeline: pipeline(),
-        rows: [row("r1", { a: "1" }), row("r2", { a: "2" })],
-        chunk: 1,
-        write: () => {
-          wrote += 1;
-          if (wrote === 2) throw new Error("sink exploded");
-          return 1;
-        },
-        now,
-      }),
-    ).toThrow(/sink exploded/);
-    expect(wrote).toBe(2);
-  });
-
-  it("resumes with a cursor at the end, all keys persisted and refuses finished pipelines and foreign cursors", () => {
-    const rows = [row("r1", { a: "1" }), row("r2", { a: "2" })];
-    const atend = resumeextract({
-      pipeline: pipeline({ state: "paused" }),
-      rows,
-      cursor: { pipelineid: "p1", offset: 2, chunk: 2, updatedat: now },
-      now,
-    });
-    expect(atend.rows).toEqual([]);
-    expect(atend.skipped).toBe(2);
-    const allpersisted = resumeextract({
-      pipeline: pipeline({ state: "paused" }),
-      rows,
-      persistedkeys: ["r1", "r2"],
-      now,
-    });
-    expect(allpersisted.rows).toEqual([]);
-    expect(allpersisted.skipped).toBe(2);
-    expect(() => resumeextract({ pipeline: pipeline({ state: "finished" }), rows, now })).toThrow(/already finished/);
-    expect(() =>
-      resumeextract({
-        pipeline: pipeline(),
-        rows,
-        cursor: { pipelineid: "p9", offset: 0, chunk: 0, updatedat: now },
-        now,
-      }),
-    ).toThrow(/another pipeline/);
-    expect(() => resumeextract({ pipeline: pipeline({ id: " " }), rows, now })).toThrow(/names its pipeline/);
-    const partial = resumeextract({
-      pipeline: pipeline({ state: "paused" }),
-      rows,
-      cursor: { pipelineid: "p1", offset: 1, chunk: 1, updatedat: now },
-      persistedkeys: ["r2"],
-      now,
-    });
-    expect(partial.rows).toEqual([]);
-    expect(partial.skipped).toBe(2);
-  });
-
-  it("keeps every row through number, case, trim and date transforms with refusals reported", () => {
-    const rules: transformrule[] = [
-      { expression: "", sources: [], target: "price", field: "price", operation: "number" },
-      { expression: "", sources: [], target: "name", field: "name", operation: "case" },
-      { expression: "", sources: [], target: "seen", field: "seen", operation: "date" },
-      {
-        expression: "",
-        sources: [],
-        target: "mystery",
-        field: "mystery",
-        operation: "hex",
-      } as unknown as transformrule,
-    ];
-    const rows = [
-      row("r1", { price: "$ 1,299.50", name: "  dev think  ", seen: "March 3, 2026", mystery: "x" }),
-      row("r2", { price: "free", name: "plain", seen: "no date", mystery: "y" }),
-    ];
-    const result = transformvalues({ rows, rules });
-    expect(result.rows).toHaveLength(2);
-    expect(result.rows[0]?.values.price).toBe("1299.50");
-    expect(result.rows[0]?.rawvalues?.price).toBe("$ 1,299.50");
-    expect(result.rows[0]?.values.name).toBe("  DEV THINK  ");
-    expect(result.rows[1]?.values.price).toBe("");
-    expect(result.rows[1]?.rawvalues?.price).toBe("free");
-    expect(result.rows[1]?.values.seen).toBe("no date");
-    expect(result.rows[1]?.rawvalues?.seen).toBeUndefined();
-    expect(result.refusals.join(" ")).toContain("not a reviewed transform");
-    expect(result.refusals.join(" ")).toContain("parses as no date");
-    expect(result.refusals).toHaveLength(3);
-  });
-
-  it("falls back to the rule target when the field name is absent", () => {
-    const result = transformvalues({
-      rows: [row("r1", { sku: "  a  " })],
-      rules: [{ expression: "", sources: [], target: "sku", operation: "trim" } as unknown as transformrule],
-    });
-    expect(result.rows[0]?.values.sku).toBe("a");
-    expect(result.rows[0]?.rawvalues?.sku).toBe("  a  ");
-    expect(result.applied).toBe(1);
-  });
-});
-
-describe("torture: dedupe normalization collisions and sampler bounds", () => {
-  it("keeps distinct rows apart when a cell value embeds the join separator", () => {
-    const rows = [row("r1", { a: "x|y", b: "1" }), row("r2", { a: "x", b: "y|1" }), row("r3", { a: "x", b: "1" })];
-    const deduped = deduperows({ rows, key: { columns: ["a", "b"], normalization: "none" } });
-    expect(deduped.kept.map((entry) => entry.key)).toEqual(["r1", "r2", "r3"]);
-    expect(deduped.dropped).toBe(0);
-  });
-
-  it("folds whitespace and case exactly as the normalization names", () => {
-    const rows = [
-      row("r1", { a: "Devthink" }),
-      row("r2", { a: " devthink " }),
-      row("r3", { a: "DEVTHINK" }),
-      row("r4", { a: "Other" }),
-    ];
-    const both = deduperows({ rows, key: { columns: ["a"], normalization: "whitespace and case" } });
-    expect(both.kept.map((entry) => entry.key)).toEqual(["r1", "r4"]);
-    const whitespace = deduperows({ rows, key: { columns: ["a"], normalization: "whitespace" } });
-    expect(whitespace.kept.map((entry) => entry.key)).toEqual(["r1", "r2", "r3", "r4"]);
-  });
-
-  it("refuses empty and whitespace only key columns while trimming the configured ones", () => {
-    const rows = [row("r1", { a: "1" })];
-    expect(() => deduperows({ rows, key: { columns: [], normalization: "none" } })).toThrow(/names its columns/);
-    expect(() => deduperows({ rows, key: { columns: ["  "], normalization: "none" } })).toThrow(/names its columns/);
-    expect(deduperows({ rows, key: { columns: ["  a  "], normalization: "none" } }).kept).toHaveLength(1);
-    expect(deduperows({ rows: [], key: { columns: ["a"], normalization: "none" } })).toMatchObject({
-      kept: [],
-      dropped: 0,
-    });
-  });
-
-  it("samples with one row, huge bounds, deterministic seeds and empty extracts", () => {
-    const rows = Array.from({ length: 12 }, (_, index) => row(`r${index + 1}`, { a: String(index) }));
-    expect(samplerows({ rows, policy: { rows: 1, strategy: "first" } }).rows.map((entry) => entry.key)).toEqual(["r1"]);
-    expect(samplerows({ rows, policy: { rows: 999, strategy: "first" } }).rows).toHaveLength(12);
-    expect(samplerows({ rows: [], policy: { rows: 5, strategy: "stratified" } }).rows).toEqual([]);
-    expect(samplerows({ rows: [], policy: { rows: 5, strategy: "random" } }).rows).toEqual([]);
-    const one = samplerows({ rows, policy: { rows: 5, strategy: "random" }, seed: 7 });
-    const two = samplerows({ rows, policy: { rows: 5, strategy: "random" }, seed: 7 });
-    expect(one.rows.map((entry) => entry.key)).toEqual(two.rows.map((entry) => entry.key));
-    expect(new Set(one.rows.map((entry) => entry.key)).size).toBe(5);
-    const stratified = samplerows({ rows, policy: { rows: 12, strategy: "stratified" } });
-    expect(stratified.rows.map((entry) => entry.key)).toEqual(rows.map((entry) => entry.key));
-    expect(() => samplerows({ rows, policy: { rows: 0, strategy: "first" } })).toThrow(/positive whole number/);
-    expect(() => samplerows({ rows, policy: { rows: -1, strategy: "first" } })).toThrow(/positive whole number/);
-    expect(() => samplerows({ rows, policy: { rows: 2.5, strategy: "first" } })).toThrow(/positive whole number/);
-    const copied = samplerows({ rows, policy: { rows: 2, strategy: "first" } });
-    copied.rows[0]!.values.a = "mutated";
-    expect(rows[0]!.values.a).toBe("0");
-  });
-});
-
-describe("torture: source stamps and grid projection", () => {
-  it("stamps rows and cells while a repeated stamp id never duplicates and a second stamp overwrites cell links", () => {
-    const rows = [row("r1", { a: "1", b: "2" })];
-    const stamped = sourcestamp({ rows, stampid: "st1", url: "https://example.com", stepid: "s1", now });
-    expect(stamped.rows[0]?.cellstamps).toEqual({ a: "st1", b: "st1" });
-    const restamped = sourcestamp({
-      rows: stamped.rows,
-      stampid: "st1",
-      url: "https://example.com",
-      stepid: "s1",
-      now: now + 5,
-    });
-    expect(restamped.rows[0]?.stamps).toHaveLength(1);
-    expect(restamped.rows[0]?.stamps[0]?.capturedat).toBe(now);
-    const second = sourcestamp({
-      rows: stamped.rows,
-      stampid: "st2",
-      url: "https://example.com/2",
-      stepid: "s2",
-      now: now + 9,
-    });
-    expect(second.rows[0]?.stamps.map((stamp) => stamp.id)).toEqual(["st1", "st2"]);
-    expect(second.rows[0]?.cellstamps?.a).toBe("st2");
-    expect(() => sourcestamp({ rows, stampid: " ", url: "u", stepid: "s", now })).toThrow(/needs its id/);
-    expect(() => sourcestamp({ rows, stampid: "s", url: " ", stepid: "s", now })).toThrow(/names its url/);
-    expect(() => sourcestamp({ rows, stampid: "s", url: "u", stepid: " ", now })).toThrow(/names its step/);
-  });
-
-  it("projects the grid with column kinds, sorting, filtering, marks and refusals for unknown sort columns", () => {
-    const rows = [
-      row("r1", { name: "Devthink", price: "1299.50", seen: "2026-03-03", live: "true" }),
-      row("r2", { name: "Other", price: "42", seen: "2026-03-04", live: "false" }),
-      row("r3", { name: "Mixed", price: "n/a", seen: "2026-03-05", live: "true" }),
-    ];
-    const grid = pipelinegridpreview({ rows });
-    const kinds = Object.fromEntries(grid.columns.map((column) => [column.name, column.kind]));
-    expect(kinds.name).toBe("text");
-    expect(kinds.price).toBe("text");
-    expect(kinds.seen).toBe("date");
-    expect(kinds.live).toBe("boolean");
-    expect(grid.total).toBe(3);
-    expect(pipelinegridpreview({ rows: [] }).columns).toEqual([]);
-    expect(pipelinegridpreview({ rows: [], filter: "anything" }).rows).toEqual([]);
-    expect(pipelinegridpreview({ rows, filter: "DEVTHINK" }).rows.map((entry) => entry.key)).toEqual(["r1"]);
-    expect(pipelinegridpreview({ rows, filter: "  " }).rows).toHaveLength(3);
-    expect(pipelinegridpreview({ rows, dedupekeys: ["r1", "r2"] }).rows.map((entry) => entry.deduplicated)).toEqual([
-      true,
-      true,
-      false,
-    ]);
-    expect(() => pipelinegridpreview({ rows, sort: { column: "ghost", direction: "ascending" } })).toThrow(
-      /knows no ghost/,
-    );
-    expect(
-      pipelinegridpreview({ rows, sort: { column: "name", direction: "descending" } }).rows.map((entry) => entry.key),
-    ).toEqual(["r2", "r3", "r1"]);
-  });
-});
-
-describe("torture: provlog redaction, append only and queries", () => {
-  it("redacts every secret shape from summaries before an entry lands", () => {
-    const entry = provlog({
-      entries: [],
-      id: "l1",
-      pipelineid: "p1",
-      runid: "run1",
-      operation: "stream",
-      rowkeys: ["r1"],
-      summary:
-        "streamed with token=abc123 and api-key: xyz and sk-abcdefghijklmnopqrstuv and ghp_AAAA0123456789aaaaaa and password:hunter2",
-      now,
-    }).entry;
-    expect(entry.summary).toContain("token=[redacted]");
-    expect(entry.summary).toContain("api-key: [redacted]");
-    expect(entry.summary).toContain("[redacted key]");
-    expect(entry.summary).toContain("[redacted token]");
-    expect(entry.summary).toContain("password:[redacted]");
-    expect(entry.summary).not.toContain("hunter2");
-    expect(entry.summary).not.toContain("abc123");
-    expect(entry.summary).not.toContain("ghp_");
-  });
-
-  it("redacts pem private key blocks inside a summary", () => {
-    const pem = `-----BEGIN RSA PRIVATE KEY-----${"A".repeat(64)}-----END RSA PRIVATE KEY-----`;
-    const entry = provlog({
-      entries: [],
-      id: "l2",
-      pipelineid: "p1",
-      runid: "run1",
-      operation: "transform",
-      rowkeys: [],
-      summary: `the pass carried ${pem} through`,
-      now,
-    }).entry;
-    expect(entry.summary).toContain("[redacted private key]");
-    expect(entry.summary).not.toContain("BEGIN RSA");
-  });
-
-  it("refuses duplicate ids, empty ids and empty summaries while the query orders by time then id", () => {
-    const one = provlog({
-      entries: [],
-      id: "l1",
-      pipelineid: "p1",
-      runid: "run1",
-      operation: "stream",
-      rowkeys: ["r1"],
-      summary: "one",
-      now,
-    });
-    const two = provlog({
-      entries: one.entries,
-      id: "l2",
-      pipelineid: "p1",
-      runid: "run1",
-      operation: "dedupe",
-      rowkeys: ["r1"],
-      summary: "two",
-      now,
-    });
-    expect(() =>
-      provlog({
-        entries: two.entries,
-        id: "l1",
-        pipelineid: "p1",
-        runid: "run1",
-        operation: "stream",
-        rowkeys: [],
-        summary: "again",
-        now,
-      }),
-    ).toThrow(/append only/);
-    expect(() =>
-      provlog({
-        entries: [],
-        id: " ",
-        pipelineid: "p1",
-        runid: "run1",
-        operation: "stream",
-        rowkeys: [],
-        summary: "s",
-        now,
-      }),
-    ).toThrow(/needs its id/);
-    expect(() =>
-      provlog({
-        entries: [],
-        id: "l3",
-        pipelineid: " ",
-        runid: "run1",
-        operation: "stream",
-        rowkeys: [],
-        summary: "s",
-        now,
-      }),
-    ).toThrow(/names its pipeline/);
-    expect(() =>
-      provlog({
-        entries: [],
-        id: "l4",
-        pipelineid: "p1",
-        runid: "run1",
-        operation: "stream",
-        rowkeys: [],
-        summary: "  ",
-        now,
-      }),
-    ).toThrow(/needs its summary/);
-    const tie = provlog({
-      entries: two.entries,
-      id: "l0",
-      pipelineid: "p1",
-      runid: "run1",
-      operation: "sample",
-      rowkeys: ["r1"],
-      summary: "tie",
-      now,
-    });
-    expect(provlogquery(tie.entries, "r1").map((entry) => entry.id)).toEqual(["l0", "l1", "l2"]);
-    expect(provlogquery(tie.entries, "ghost")).toEqual([]);
-  });
-
-  it("counts the pipeline states per run with empty inputs", () => {
-    expect(pipelinestate({ pipelines: [] })).toEqual({ running: 0, paused: 0, finished: 0, perrun: [] });
-    const state = pipelinestate({
-      pipelines: [
-        pipeline(),
-        pipeline({ id: "p2", state: "paused", runid: "run2" }),
-        pipeline({ id: "p3", state: "finished" }),
-      ],
-    });
-    expect(state).toEqual({
-      running: 1,
-      paused: 1,
-      finished: 1,
-      perrun: [
-        { runid: "run1", running: 1, paused: 0, finished: 1 },
-        { runid: "run2", running: 0, paused: 1, finished: 0 },
-      ],
-    });
-    const batch = extractbatchof({
-      id: "b1",
-      pipeline: pipeline(),
-      rows: [row("r1", { a: "1" })],
-      cursor: { pipelineid: "p1", offset: 1, chunk: 1, updatedat: now },
-      now,
-    });
-    expect(batch).toMatchObject({ id: "b1", pipelineid: "p1", at: now });
-    expect(() =>
-      extractbatchof({
-        id: " ",
-        pipeline: pipeline(),
-        rows: [],
-        cursor: { pipelineid: "p1", offset: 0, chunk: 0, updatedat: now },
-        now,
-      }),
-    ).toThrow(/needs its id/);
-  });
-});
-
-describe("torture: streamparse windows, chunks and validation", () => {
-  it("splits byte ranges with zero, fractional and huge bounds while refusing non positive chunk bytes", () => {
-    expect(streamparsewindows({ bytes: 0, chunkbytes: 5 })).toEqual({ chunks: 0, windows: [] });
-    expect(streamparsewindows({ bytes: 10, chunkbytes: 4 }).windows).toEqual([
-      { index: 0, start: 0, end: 4 },
-      { index: 1, start: 4, end: 8 },
-      { index: 2, start: 8, end: 10 },
-    ]);
-    expect(streamparsewindows({ bytes: 10, chunkbytes: 2.5 }).chunks).toBe(4);
-    expect(streamparsewindows({ bytes: 10, chunkbytes: 1_000_000 })).toEqual({
-      chunks: 1,
-      windows: [{ index: 0, start: 0, end: 10 }],
-    });
-    expect(() => streamparsewindows({ bytes: 10, chunkbytes: 0 })).toThrow(/positive number of bytes/);
-    expect(() => streamparsewindows({ bytes: 10, chunkbytes: -1 })).toThrow(/positive number of bytes/);
-  });
-
-  it("tokenizes chunks with element, attribute and text tokens while refusing over bound chunks", () => {
-    const chunk = streamparsechunkof({
-      index: 0,
-      text: 'hello <div class="a">world</div> tail',
-      chunkbytes: 100,
-      last: true,
-    });
-    expect(chunk.tokens.map((token) => token.kind)).toEqual([
-      "text",
-      "element",
-      "attribute",
-      "text",
-      "element",
-      "text",
-    ]);
-    expect(chunk.tokens.find((token) => token.kind === "attribute")?.text).toBe('class="a"');
-    expect(chunk.observations).toEqual(["hello", "world", "tail"]);
-    expect(chunk.bytes).toBe(37);
-    expect(chunk.complete).toBe(true);
-    expect(streamparsechunkof({ index: 3, text: "", chunkbytes: 10, last: false }).tokens).toEqual([]);
-    expect(() => streamparsechunkof({ index: 0, text: "a".repeat(11), chunkbytes: 10, last: true })).toThrow(
-      /over the 10 byte bound/,
-    );
-  });
-
-  it("validates chunk carriers through schemastrict refusals", () => {
-    const chunk = streamparsechunkof({ index: 2, text: "x", chunkbytes: 10, last: false });
-    expect(validatesetreamparsechunk(chunk, 2).valid).toBe(true);
-    expect(validatesetreamparsechunk(null, 0).reason).toMatch(/plain object/);
-    expect(validatesetreamparsechunk([chunk], 2).reason).toMatch(/plain object/);
-    expect(validatesetreamparsechunk({ ...chunk, index: 3 }, 2).reason).toMatch(/index 2/);
-    expect(validatesetreamparsechunk({ ...chunk, tokens: "no" }, 2).reason).toMatch(/tokens/);
-    expect(validatesetreamparsechunk({ ...chunk, bytes: "no" }, 2).reason).toMatch(/byte count/);
-    expect(validatesetreamparsechunk({ ...chunk, complete: "no" }, 2).reason).toMatch(/completion flag/);
-  });
-
-  it("yields progressive observations and memory bounds", () => {
-    expect(streamparseobservations([])).toEqual({ progressive: [], total: 0 });
-    const chunks = [
-      streamparsechunkof({ index: 0, text: "one", chunkbytes: 10, last: false }),
-      streamparsechunkof({ index: 1, text: "two <b>three</b>", chunkbytes: 20, last: true }),
-    ];
-    const yielded = streamparseobservations(chunks);
-    expect(yielded.progressive).toEqual([["one"], ["two", "three"]]);
-    expect(yielded.total).toBe(3);
-    expect(streamparsememorybound({ chunkbytes: 64, chunks: 0 })).toEqual({ peak: 64, fullpage: 0, saved: 0 });
-    expect(streamparsememorybound({ chunkbytes: 64, chunks: 100 })).toEqual({ peak: 64, fullpage: 6400, saved: 6336 });
-  });
-
-  it("shapes worker tasks with positive bounds", () => {
-    expect(streamparsetaskof({ runid: "r1", stepid: "s1", chunkbytes: 64, chunks: 3 })).toEqual({
-      task: "streamparse",
-      runid: "r1",
-      stepid: "s1",
-      chunkbytes: 64,
-      chunks: 3,
-    });
-    expect(() => streamparsetaskof({ runid: "r1", stepid: "s1", chunkbytes: 0, chunks: 3 })).toThrow(
-      /positive chunk bound/,
-    );
-    expect(() => streamparsetaskof({ runid: "r1", stepid: "s1", chunkbytes: -1, chunks: 3 })).toThrow(
-      /positive chunk bound/,
-    );
-    expect(
-      chunkextracttaskof({ runid: "r1", stepid: "s1", tableid: "t1", fingerprint: "f1", rowindex: 4, window: 2 }),
-    ).toMatchObject({ task: "chunkextract", tableid: "t1" });
-    expect(() =>
-      chunkextracttaskof({ runid: "r1", stepid: "s1", tableid: "t1", fingerprint: "f1", rowindex: 4, window: 0 }),
-    ).toThrow(/positive row window/);
-    expect(() =>
-      chunkextracttaskof({ runid: "r1", stepid: "s1", tableid: "t1", fingerprint: "f1", rowindex: 4, window: -3 }),
-    ).toThrow(/positive row window/);
-  });
-});
-
-describe("torture: chunkextract cursors, windows and merges", () => {
-  it("opens cursors with zero row tables, defaults and refusals", () => {
-    expect(openchunkextractcursor({ tableid: "t1", fingerprint: "f1", rows: 0 })).toMatchObject({
-      rowindex: 0,
-      complete: true,
-    });
-    expect(openchunkextractcursor({ tableid: "t1", fingerprint: "f1", rows: 10 })).toMatchObject({
-      window: 10,
-      complete: false,
-    });
-    expect(openchunkextractcursor({ tableid: "t1", fingerprint: "f1", rows: 10, window: 3 })).toMatchObject({
-      window: 3,
-      complete: false,
-    });
-    expect(() => openchunkextractcursor({ tableid: " ", fingerprint: "f", rows: 1 })).toThrow(/table id/);
-    expect(() => openchunkextractcursor({ tableid: "t", fingerprint: " ", rows: 1 })).toThrow(/fingerprint/);
-    expect(() => openchunkextractcursor({ tableid: "t", fingerprint: "f", rows: 1, window: 0 })).toThrow(
-      /positive row count/,
-    );
-    expect(() => openchunkextractcursor({ tableid: "t", fingerprint: "f", rows: 1, window: -2 })).toThrow(
-      /positive row count/,
-    );
-  });
-
-  it("slices windows at the end, past the end and larger than the table", () => {
-    const rows = [{ a: "1" }, { a: "2" }, { a: "3" }, { a: "4" }, { a: "5" }];
-    const first = chunkextractwindow({
-      cursor: openchunkextractcursor({ tableid: "t1", fingerprint: "f1", rows: 5, window: 2 }),
-      rows,
-    });
-    expect(first.rows).toEqual([{ a: "1" }, { a: "2" }]);
-    expect(first.complete).toBe(false);
-    const oversized = chunkextractwindow({
-      cursor: { tableid: "t1", fingerprint: "f1", rowindex: 0, window: 100, complete: false },
-      rows,
-    });
-    expect(oversized.rows).toHaveLength(5);
-    expect(oversized.complete).toBe(true);
-    const atend = chunkextractwindow({
-      cursor: { tableid: "t1", fingerprint: "f1", rowindex: 5, window: 2, complete: true },
-      rows,
-    });
-    expect(atend.rows).toEqual([]);
-    expect(atend.complete).toBe(true);
-    expect(() =>
-      chunkextractwindow({
-        cursor: { tableid: "t1", fingerprint: "f1", rowindex: 6, window: 2, complete: false },
-        rows,
-      }),
-    ).toThrow(/past the end/);
-    expect(
-      chunkextractwindow({ cursor: openchunkextractcursor({ tableid: "t", fingerprint: "f", rows: 0 }), rows: [] }),
-    ).toMatchObject({ rows: [], complete: true });
-  });
-
-  it("resumes only the same table with the same fingerprint while a completed cursor never resumes", () => {
-    const cursor = openchunkextractcursor({ tableid: "t1", fingerprint: "f1", rows: 10, window: 2 });
-    expect(resumechunkextract({ cursor, tableid: "t1", fingerprint: "f1" })).toMatchObject({ resumed: true });
-    expect(resumechunkextract({ cursor, tableid: "t2", fingerprint: "f1" }).reason).toMatch(/belongs to the table t1/);
-    expect(resumechunkextract({ cursor, tableid: "t1", fingerprint: "f2" }).reason).toMatch(/fingerprint changed/);
-    expect(
-      resumechunkextract({ cursor: { ...cursor, complete: true }, tableid: "t1", fingerprint: "f1" }).reason,
-    ).toMatch(/already completed/);
-  });
-
-  it("merges windows without duplicating rows and reports the duplicates", () => {
-    const rows = [{ a: "1" }, { a: "2" }];
-    const windows = [
-      { cursor: { tableid: "t", fingerprint: "f", rowindex: 2, window: 2, complete: true }, rows, complete: true },
-      { cursor: { tableid: "t", fingerprint: "f", rowindex: 2, window: 2, complete: true }, rows, complete: true },
-      {
-        cursor: { tableid: "t", fingerprint: "f", rowindex: 4, window: 2, complete: true },
-        rows: [{ a: "3" }],
-        complete: true,
-      },
-    ];
-    const merged = mergechunkwindows(windows);
-    expect(merged.rows).toEqual([{ a: "1" }, { a: "2" }, { a: "3" }]);
-    expect(merged.duplicates).toBe(2);
-    expect(mergechunkwindows([])).toEqual({ rows: [], duplicates: 0 });
-    const reordered = mergechunkwindows([
-      {
-        cursor: { tableid: "t", fingerprint: "f", rowindex: 1, window: 1, complete: true },
-        rows: [{ b: "2", a: "1" }],
-        complete: true,
-      },
-      {
-        cursor: { tableid: "t", fingerprint: "f", rowindex: 0, window: 1, complete: true },
-        rows: [{ a: "1", b: "2" }],
-        complete: true,
-      },
-    ]);
-    expect(reordered.duplicates).toBe(1);
-  });
-
-  it("reads progress with zero totals and cursors past the end", () => {
-    expect(
-      chunkextractprogress({ tableid: "t", fingerprint: "f", rowindex: 3, window: 2, complete: false }, 5),
-    ).toEqual({ extracted: 3, total: 5, share: 0.6, complete: false });
-    expect(
-      chunkextractprogress({ tableid: "t", fingerprint: "f", rowindex: 0, window: 2, complete: false }, 0).share,
-    ).toBe(1);
-    expect(
-      chunkextractprogress({ tableid: "t", fingerprint: "f", rowindex: 9, window: 2, complete: false }, 5).share,
-    ).toBe(1.8);
-  });
-});