@inflexa-ai/harness 0.20.0 → 0.21.0

package/dist/report-model/pin-snapshot.js

@@ -7,17 +7,209 @@
  *
  * The artifact ledger holds one row for each path, and it keeps no history. Thus the set at a past
  * moment is not recoverable later, and the pin must run at the anchor itself.
+ *
+ * The snapshot carries the citation evidence beside the artifact map. The pinned set is the literature
+ * that the synthesis engaged, and two fields of a run synthesis carry it: the key references and the
+ * references of each finding. The records live in the run tree on disk. Thus the pin reads that tree
+ * through the workspace-root seam of the caller.
+ *
+ * A reference that carries a short citation also pins a bibliography record beside its key. The key list
+ * keeps the membership role, thus the record map is optional and it changes no resolution.
  */
 import { err, ok } from "neverthrow";
+import { open } from "node:fs/promises";
+import { join } from "node:path";
+import { tryFs } from "../lib/fs-result.js";
 import { queryAnalysisArtifacts } from "../state/artifacts.js";
+import { queryRunsByAnalysis } from "../state/runs.js";
+import { isSafeId, runDir } from "../workspace/paths.js";
+/** The file name of the synthesis record inside the run directory. */
+const SYNTHESIS_FILE = "synthesis.json";
+/**
+ * The cap of one synthesis read. A synthesis record is small by construction, and the cap bounds the
+ * cost of a file that is not. A file over the cap gives no key, because a cut JSON text cannot parse.
+ */
+const SYNTHESIS_CAP_BYTES = 1024 * 1024;
+/** The page size of the run listing. The listing walks the pages until a short page ends the walk. */
+const RUN_PAGE_SIZE = 200;
+/**
+ * Read a synthesis record under the cap, or give `undefined` when the bytes do not come back.
+ *
+ * The read takes a bounded byte window, thus a file of any size costs the cap. An absent file, a genuine
+ * read fault, and a file over the cap each give `undefined`, because the collection reports no error for
+ * one record.
+ */
+async function readSynthesisText(absolute) {
+    return tryFs("pinReportSnapshot.readSynthesis", async () => {
+        const handle = await open(absolute, "r");
+        try {
+            // The window takes one byte more than the cap. A read that fills it states that the file
+            // holds more bytes than the cap admits, thus the two conditions stay apart.
+            const buffer = Buffer.alloc(SYNTHESIS_CAP_BYTES + 1);
+            // One read of a network-backed file can give fewer bytes than the window holds. Such a
+            // short read cuts the text in the middle of the JSON, and the record then parses to no
+            // key at all. Thus the loop reads again from the offset that the last read reached, and
+            // it stops when the window fills or when a read gives no byte.
+            let filled = 0;
+            while (filled < buffer.length) {
+                const { bytesRead } = await handle.read(buffer, filled, buffer.length - filled, filled);
+                if (bytesRead === 0) {
+                    break;
+                }
+                filled += bytesRead;
+            }
+            return filled > SYNTHESIS_CAP_BYTES ? undefined : buffer.subarray(0, filled).toString("utf8");
+        }
+        finally {
+            await handle.close();
+        }
+    }, { path: absolute, onAbsent: () => undefined }).unwrapOr(undefined);
+}
+/** The trimmed text of a field, or `undefined` when the field is not a non-empty string. */
+function trimmedText(value) {
+    if (typeof value !== "string") {
+        return undefined;
+    }
+    const text = value.trim();
+    return text.length > 0 ? text : undefined;
+}
+/**
+ * The bibliography record of one reference entry, or `undefined` when the entry carries no citation text.
+ *
+ * A key reference carries `citation` and `description`. A reference of a finding carries `citation` and a
+ * narrower `relevance`, thus it gives the short citation alone. A record without a citation would state
+ * nothing, thus such an entry gives a bare key.
+ */
+function recordOf(reference) {
+    const citation = trimmedText(reference.citation);
+    if (citation === undefined) {
+        return undefined;
+    }
+    const description = trimmedText(reference.description);
+    return description === undefined ? { citation } : { citation, description };
+}
+/**
+ * Take one `pmid:` citation from each entry of one reference list, and add it to `entries`.
+ *
+ * The walk is lenient: a value that is not a list, an entry that is not an object, and a PMID that is
+ * not a non-empty string each give no citation.
+ */
+function pushPmidCitations(list, entries) {
+    if (!Array.isArray(list)) {
+        return;
+    }
+    for (const reference of list) {
+        if (typeof reference !== "object" || reference === null) {
+            continue;
+        }
+        const pmid = reference.pmid;
+        if (typeof pmid !== "string") {
+            continue;
+        }
+        const id = pmid.trim();
+        if (id.length > 0) {
+            const record = recordOf(reference);
+            entries.push(record === undefined ? { key: `pmid:${id}` } : { key: `pmid:${id}`, record });
+        }
+    }
+}
 /**
- * Pin the snapshot of one analysis from the artifact ledger.
+ * The citations of one synthesis text, in the order that the record carries them.
+ *
+ * The extraction is lenient: it parses the JSON, and it takes each PMID of `keyReferences` and of the
+ * references of each finding. The key references are the papers that the synthesis names as primary, and
+ * the references of the findings are the superset. Thus a citation over either one resolves. A
+ * whole-schema parse would empty the citation list of the whole analysis for one record that a different
+ * schema version wrote. Malformed JSON gives no citation.
+ *
+ * The key references come first, because the caller keeps the first record of a key and the curated
+ * description sits there.
+ */
+function citationsOf(text) {
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        return [];
+    }
+    if (typeof parsed !== "object" || parsed === null) {
+        return [];
+    }
+    const entries = [];
+    pushPmidCitations(parsed.keyReferences, entries);
+    const findings = parsed.findings;
+    if (Array.isArray(findings)) {
+        for (const finding of findings) {
+            if (typeof finding !== "object" || finding === null) {
+                continue;
+            }
+            pushPmidCitations(finding.references, entries);
+        }
+    }
+    return entries;
+}
+/**
+ * Collect the citation evidence of one analysis from the synthesis record of each of its runs.
+ *
+ * The pinned set is the literature that the synthesis engaged, and both fields of a record carry it: the
+ * key references and the references of each finding. The keys dedupe, and they sort in code-unit order.
+ * Thus one disk state gives one list, and a second pin over that state gives the same list.
+ *
+ * The first record of a key wins. The key references of a record walk before its findings, thus the
+ * curated description survives a duplicate that names the same paper with a narrower text.
+ *
+ * A run listing that fails fails the collection, because a store fault is not absence. Each other fault
+ * is a normal condition: an unresolvable workspace root, an absent record, an unreadable record, and a
+ * malformed record each give no key and no error.
+ */
+async function collectCitations(pool, resolveWorkspaceRoot, analysisId) {
+    let root;
+    try {
+        root = resolveWorkspaceRoot(analysisId);
+    }
+    catch {
+        // The seam signals an unresolvable resource by a throw. The artifact map still states the
+        // membership of the session, thus the fault costs the citation list alone.
+        return ok({ keys: [], records: new Map() });
+    }
+    const keys = new Set();
+    const records = new Map();
+    for (let offset = 0;; offset += RUN_PAGE_SIZE) {
+        const page = await queryRunsByAnalysis(pool, analysisId, { limit: RUN_PAGE_SIZE, offset });
+        if (page.isErr()) {
+            return err({ kind: "run-listing-failed", cause: page.error.cause });
+        }
+        for (const run of page.value) {
+            // The path builder refuses an id that can escape the run tree. A refusal is a throw, thus
+            // the guard keeps the collection on the value channel.
+            if (!isSafeId(run.runId)) {
+                continue;
+            }
+            const text = await readSynthesisText(join(root, runDir(run.runId), SYNTHESIS_FILE));
+            if (text === undefined) {
+                continue;
+            }
+            for (const entry of citationsOf(text)) {
+                keys.add(entry.key);
+                if (entry.record !== undefined && !records.has(entry.key)) {
+                    records.set(entry.key, entry.record);
+                }
+            }
+        }
+        if (page.value.length < RUN_PAGE_SIZE) {
+            return ok({ keys: [...keys].sort(), records });
+        }
+    }
+}
+/**
+ * Pin the snapshot of one analysis from the artifact ledger and from the synthesis records of its runs.
  *
  * The pin copies no cell and no byte. An entry pins identity alone, thus the snapshot grows with the
  * count of artifacts and never with the size of the data. A read of an artifact belongs to the value
  * tier, which runs one time for each report version.
  */
-export async function pinReportSnapshot(pool, analysisId) {
+export async function pinReportSnapshot(pool, analysisId, options = {}) {
     let ledgerRows;
     try {
         ledgerRows = await queryAnalysisArtifacts(pool, analysisId);
@@ -33,8 +225,14 @@ export async function pinReportSnapshot(pool, analysisId) {
     for (const row of ledgerRows) {
         artifacts[row.path] = { hash: row.hash, fileType: row.fileType };
     }
-    // The pin fills no citation. No store holds the citation ids of an analysis, and validation
-    // matches a citation against the external authorities and never against a pinned list. Thus such
-    // a list would state nothing that a later read consumes.
-    return ok({ artifacts });
+    // A citation reference resolves against this list: the resolver refuses a key that the list does not
+    // hold. Thus the pin must state which external ids the analysis engaged, and a session with an empty
+    // list can bind no citation block.
+    if (options.resolveWorkspaceRoot === undefined) {
+        return ok({ artifacts, citations: [] });
+    }
+    const citations = await collectCitations(pool, options.resolveWorkspaceRoot, analysisId);
+    // A collection that recorded nothing stores no map. Thus the snapshot of an analysis whose synthesis
+    // carries no citation text reads the same as a pin that predates the map.
+    return citations.map(({ keys, records }) => records.size === 0 ? { artifacts, citations: keys } : { artifacts, citations: keys, citationRecords: Object.fromEntries(records) });
 }

package/dist/report-model/production-resolver.js

@@ -25,7 +25,7 @@ import { allWithConcurrency } from "../lib/async-utils.js";
 import { computeSha256File } from "../lib/fs-helpers.js";
 import { resolveWorkspacePath } from "../workspace/paths.js";
 import { asFiniteNumber, cellMatchesFilterValue, checkCitationAssertion, checkValueAssertion } from "./assert-rules.js";
-import { columnsHeldByNoRow, fileTypeHoldsNoCell, snapshotEntry, } from "./reference-resolver.js";
+import { applyRowBound, columnsHeldByNoRow, fileTypeHoldsNoCell, snapshotEntry, } from "./reference-resolver.js";
 /**
  * The default cap on the file size that the host fast path reads. A larger file takes the fall-through.
  * The cap gates the cell extraction only. The identity compare streams the hash at every size.
@@ -623,10 +623,21 @@ export function createProductionResolver(deps) {
         if (gated.isErr()) {
             return err(gated.error);
         }
-        const rows = gated.value;
+        // The bound ranks the rows as the artifact holds them, thus a bound over a column that the subset
+        // below leaves out still ranks them. An unknown bound column addresses nothing, exactly as an
+        // unknown subset column does.
+        const bound = reference.rowBound;
+        const allRows = gated.value;
+        if (bound !== undefined && columnsHeldByNoRow(allRows, [bound.column]).length > 0) {
+            return fail(reference, "locator-out-of-range", `the row bound names column ${bound.column}, which the table at ${reference.path} does not hold`);
+        }
+        const rows = bound === undefined ? allRows : applyRowBound(allRows, bound);
+        // The page states the shown count against the count of the artifact. This read is the one place that
+        // holds both, thus a bounded table carries the pre-bound total out with its rows.
+        const total = bound === undefined ? {} : { total: allRows.length };
         const columns = reference.columns;
         if (columns === undefined) {
-            return ok({ type: "table", rows });
+            return ok({ type: "table", rows, ...total });
         }
         // A name that no row holds addresses nothing. Without this check a projection onto an invented
         // column gives an empty cell for each row and still resolves, which grounding exists to reject.
@@ -645,7 +656,7 @@ export function createProductionResolver(deps) {
             }
             return projected;
         });
-        return ok({ type: "table", rows: projectedRows, columns });
+        return ok({ type: "table", rows: projectedRows, columns, ...total });
     }
     async function resolveArtifactFile(reference, snapshot) {
         const identity = await getIdentity(reference.path, snapshot);

package/dist/report-model/reference-resolver.d.ts

@@ -6,7 +6,7 @@
  * and a fixture realization can read an in-memory snapshot. The two obey the same contract.
  */
 import type { Result } from "neverthrow";
-import type { Reference, UnresolvedReference } from "../contracts/report-reference.js";
+import type { Reference, RowBound, UnresolvedReference } from "../contracts/report-reference.js";
 /**
  * One pinned artifact: its content hash, and its rows as plain cells.
  *
@@ -31,20 +31,39 @@ export interface ArtifactSnapshot {
     fileType?: string | null;
     rows?: Array<Record<string, string | number>>;
 }
+/**
+ * One pinned citation record: the short citation of the paper, and the description that the synthesis
+ * gave. The record is the bibliography of a citation key, and the key alone states the membership.
+ */
+export interface CitationRecord {
+    citation: string;
+    description?: string;
+}
+/** The pinned citation records, keyed by the citation key, for example `pmid:12345`. */
+export type CitationRecords = Record<string, CitationRecord>;
 /**
  * The pinned evidence that a resolver reads. The `artifacts` map is keyed by the analysis-relative path,
  * the same path that an artifact reference names and the same key as `cortex_artifacts.path`. That path
  * is unique across the analysis, thus one map holds the artifacts of every run without a collision.
  * `citations` holds each known external id as an `idKind:id` string, for example `pmid:12345`.
+ *
+ * `citationRecords` carries the bibliography of a key that the synthesis described. The key list keeps the
+ * membership role, thus a key with no record is a pinned citation and a stored pin with no map reads the
+ * same as one that predates the map.
  */
 export interface ReportSnapshot {
     artifacts: Record<string, ArtifactSnapshot>;
     citations?: string[];
+    citationRecords?: CitationRecords;
 }
 /**
  * The concrete value that a reference resolves to. A scalar comes from one artifact cell or one
  * derivation. A table comes from a whole-table artifact. A file echoes the pin of a whole-file artifact,
  * for example an image. A citation echo confirms that an external id is in the pinned evidence.
+ *
+ * A table carries `total` when a row bound cut the rows. The resolution is the one step that reads the
+ * whole artifact, thus it is the one step that knows the count before the cut. A table with no bound
+ * carries none, and the row count answers for it.
  */
 export type ResolvedValue = {
     type: "scalar";
@@ -53,6 +72,7 @@ export type ResolvedValue = {
     type: "table";
     rows: Array<Record<string, string | number>>;
     columns?: string[];
+    total?: number;
 } | {
     type: "file";
     path: string;
@@ -92,6 +112,23 @@ export interface ReferenceResolver {
  * result table is a real scientific outcome that must not fail for the shape of its emptiness.
  */
 export declare function columnsHeldByNoRow(rows: Array<Record<string, string | number>>, columns: readonly string[]): string[];
+/**
+ * Apply a row bound to a table: rank the rows by the bound column, and keep the first `count` of them.
+ *
+ * The kind of the column decides one time, over the whole column, and never for one pair. A column is
+ * numeric when one cell of it reads as a finite number. A text-backed artifact holds every cell as a
+ * string, thus the parse is what tells a measurement from a label, and a code-unit order would put `"10"`
+ * between `"1"` and `"2"`. A column where nothing parses ranks by the code-unit order of its text.
+ *
+ * The sort is stable by construction. The walk carries the source index of each row and it breaks a tie on
+ * that index, thus two rows of equal rank keep the order of the file and two runs over one table give one
+ * result. `Array.prototype.sort` is specified as stable in the current language, and the explicit tiebreak
+ * states the property at the site that depends on it.
+ *
+ * The bound reads the rows as they arrive. A caller that projects onto a column subset projects after the
+ * bound, thus a bound over a column that the subset leaves out still ranks the rows.
+ */
+export declare function applyRowBound(rows: Array<Record<string, string | number>>, bound: RowBound): Array<Record<string, string | number>>;
 /**
  * Give back `true` for a file type that holds no cell for a reference to address.
  *
@@ -117,3 +154,15 @@ export declare function fileTypeHoldsNoCell(fileType: string | null | undefined)
  * disagree about membership.
  */
 export declare function snapshotEntry(snapshot: ReportSnapshot, path: string): ArtifactSnapshot | undefined;
+/**
+ * Give back the citation record at `key`, or `undefined` when the map holds no such own key.
+ *
+ * An agent authors the citation key of a block, thus the key is untrusted text. The map can be a plain
+ * object from a `JSON.parse` of a stored snapshot, and a bracket lookup with a key such as `constructor`
+ * finds an inherited member. The `Object.hasOwn` guard admits an own key only, thus an inherited member
+ * reads as absent.
+ *
+ * The card and the appendix both read this one lookup. Thus the two surfaces can never disagree about the
+ * bibliography of one key.
+ */
+export declare function citationRecordOf(records: CitationRecords | undefined, key: string): CitationRecord | undefined;

package/dist/report-model/reference-resolver.js

@@ -5,6 +5,7 @@
  * belief points at. The harness declares the seam here, thus a production realization can read storage,
  * and a fixture realization can read an in-memory snapshot. The two obey the same contract.
  */
+import { asFiniteNumber } from "./assert-rules.js";
 /**
  * Give back each name in `columns` that no row holds, in the order of `columns`.
  *
@@ -19,7 +20,85 @@ export function columnsHeldByNoRow(rows, columns) {
     if (rows.length === 0) {
         return [];
     }
-    return columns.filter((column) => !rows.some((row) => column in row));
+    // An agent authors a column name, thus the name is untrusted text. A row can be a plain object from a
+    // `JSON.parse` of a stored snapshot, and `in` finds an inherited member such as `constructor`. The name
+    // would then read as a column, and the read of the cell would give a function. The own-key test admits
+    // a real column alone.
+    return columns.filter((column) => !rows.some((row) => Object.hasOwn(row, column)));
+}
+/** The cell of one row under an own key, or `undefined` when the row holds no such column. */
+function cellOf(row, column) {
+    return Object.hasOwn(row, column) ? row[column] : undefined;
+}
+/**
+ * The code-unit rank of the string form of two cells.
+ *
+ * The comparison never calls `localeCompare`, because the ICU of a host decides that order and the bounded
+ * rows reach the bytes of the data asset. Thus one table gives one payload and one asset name on every
+ * host.
+ */
+function rankText(left, right) {
+    const leftText = String(left);
+    const rightText = String(right);
+    if (leftText < rightText)
+        return -1;
+    if (leftText > rightText)
+        return 1;
+    return 0;
+}
+/**
+ * A row that holds no rank in the column, which the bound keeps last in either direction.
+ *
+ * A row that the column does not reach carries no value to rank. In a numeric column a cell that reads as
+ * no number carries none either: a sentinel such as `NA` is the absence of a measurement, and a rank for it
+ * against a real value would be invented. Thus a descending bound of the top 20 rows gives 20 measurements,
+ * and never 20 sentinels.
+ */
+function holdsNoRank(entry, numeric) {
+    return entry.cell === undefined || (numeric && entry.value === undefined);
+}
+/**
+ * The rank of two rows that both hold one. A numeric column compares the parsed numbers, and a column
+ * where nothing parses compares the code-unit order of the text.
+ */
+function rankRankable(left, right, numeric) {
+    if (numeric && left.value !== undefined && right.value !== undefined) {
+        return left.value - right.value;
+    }
+    return rankText(left.cell, right.cell);
+}
+/**
+ * Apply a row bound to a table: rank the rows by the bound column, and keep the first `count` of them.
+ *
+ * The kind of the column decides one time, over the whole column, and never for one pair. A column is
+ * numeric when one cell of it reads as a finite number. A text-backed artifact holds every cell as a
+ * string, thus the parse is what tells a measurement from a label, and a code-unit order would put `"10"`
+ * between `"1"` and `"2"`. A column where nothing parses ranks by the code-unit order of its text.
+ *
+ * The sort is stable by construction. The walk carries the source index of each row and it breaks a tie on
+ * that index, thus two rows of equal rank keep the order of the file and two runs over one table give one
+ * result. `Array.prototype.sort` is specified as stable in the current language, and the explicit tiebreak
+ * states the property at the site that depends on it.
+ *
+ * The bound reads the rows as they arrive. A caller that projects onto a column subset projects after the
+ * bound, thus a bound over a column that the subset leaves out still ranks the rows.
+ */
+export function applyRowBound(rows, bound) {
+    const descending = (bound.order ?? "desc") === "desc";
+    const ranked = rows.map((row, index) => {
+        const cell = cellOf(row, bound.column);
+        return { row, index, cell, value: cell === undefined ? undefined : asFiniteNumber(cell) };
+    });
+    const numeric = ranked.some((entry) => entry.value !== undefined);
+    // A row with no rank stays behind every ranked row, and the direction never lifts it. The two lists
+    // keep their source order, thus the tail is as stable as the head.
+    const rankable = ranked.filter((entry) => !holdsNoRank(entry, numeric));
+    const unrankable = ranked.filter((entry) => holdsNoRank(entry, numeric));
+    const sorted = [...rankable].sort((left, right) => {
+        const rank = rankRankable(left, right, numeric);
+        return rank === 0 ? left.index - right.index : descending ? -rank : rank;
+    });
+    return [...sorted, ...unrankable].slice(0, bound.count).map((entry) => entry.row);
 }
 /**
  * The file type of an entry that holds no cell for a reference to address.
@@ -58,3 +137,17 @@ export function fileTypeHoldsNoCell(fileType) {
 export function snapshotEntry(snapshot, path) {
     return Object.hasOwn(snapshot.artifacts, path) ? snapshot.artifacts[path] : undefined;
 }
+/**
+ * Give back the citation record at `key`, or `undefined` when the map holds no such own key.
+ *
+ * An agent authors the citation key of a block, thus the key is untrusted text. The map can be a plain
+ * object from a `JSON.parse` of a stored snapshot, and a bracket lookup with a key such as `constructor`
+ * finds an inherited member. The `Object.hasOwn` guard admits an own key only, thus an inherited member
+ * reads as absent.
+ *
+ * The card and the appendix both read this one lookup. Thus the two surfaces can never disagree about the
+ * bibliography of one key.
+ */
+export function citationRecordOf(records, key) {
+    return records !== undefined && Object.hasOwn(records, key) ? records[key] : undefined;
+}

package/dist/report-model/reference-stamp.d.ts

@@ -0,0 +1,45 @@
+/**
+ * The hash stamp of the land path, and the hash elision of the read.
+ *
+ * An artifact reference pins a path and a content hash. The pinned snapshot already maps each path onto
+ * its hash, thus the author of a block gives the path alone. The stamp fills the hash from the snapshot,
+ * and a reference that names the path lands.
+ *
+ * The stamp also drops a hash that a payload carries. The snapshot owns the value. Thus an echoed binding
+ * and a mistyped one both land with the hash of the snapshot entry, and the structural tier reads no stale
+ * pin on the land path.
+ *
+ * The walk is structural, because it runs before the grammar parse. It reads the artifact-reference shape
+ * alone: a pinned kind and a string path. Every other value passes through untouched, thus a nested block,
+ * a derivation input, and a plain field are all safe.
+ *
+ * A reference whose path the snapshot does not hold refuses here. Without this arm the parse refuses the
+ * absent hash, and that message sends the author to the wrong repair.
+ *
+ * The elision walks the same shape in the other direction: a read of a landed block gives each binding
+ * with no hash. Thus one module holds both directions, and the two agree on which object carries a pin.
+ */
+import { type Result } from "neverthrow";
+import type { DraftRefusal } from "./draft-operations.js";
+import { type ReportSnapshot } from "./reference-resolver.js";
+/**
+ * Stamp the hash of each artifact reference in a block payload, from the snapshot entry at its path.
+ *
+ * The membership read is `snapshotEntry`, because the path is author text. An own key alone counts, thus
+ * an inherited member of a plain object reads as absent and the stamp agrees with the structural tier.
+ *
+ * The whole call refuses when one reference names a path that the snapshot does not hold. A partial stamp
+ * beside a refusal would land nothing, thus the refusal carries every unknown path at one time and the
+ * author repairs them together.
+ */
+export declare function stampReferenceHashes(payload: unknown, snapshot: ReportSnapshot): Result<unknown, DraftRefusal>;
+/**
+ * Give one value with the stamped hash of each artifact reference removed.
+ *
+ * The snapshot owns the hash, thus a read that echoed one would give an agent a value to mistype. A binding
+ * still gives its path and its locator, which are what an agent authored and what it changes.
+ *
+ * The walk builds a copy on the changed path only. A branch that carries no pin comes back as the same
+ * reference. Thus the read costs the size of the pins, and never the size of the block.
+ */
+export declare function stripReferenceHashes(value: unknown): unknown;