@inflexa-ai/harness 0.47.0 → 0.48.1

package/dist/state/step-executions.js

@@ -172,6 +172,28 @@ export function queryStepsByRun(pool, runId) {
         return result.rows.map(mapStepExecutionRow);
     });
 }
+/**
+ * The step counts of each run of a list, in one statement. `totalSteps` counts each row, the `synthesis` row
+ * included, and `doneSteps` counts the `completed` rows: the rule of the analysis summary reads. A step that
+ * waits counts in `totalSteps` only. A run with no row has no entry, and a caller reads it as 0.
+ */
+export function countStepsByRuns(pool, analysisId, runIds) {
+    return tryQuery("stepExecutions.countStepsByRuns", async () => {
+        if (runIds.length === 0)
+            return [];
+        const result = await pool.query({
+            text: `SELECT run_id,
+                 COUNT(*) FILTER (WHERE status = 'completed')::int AS done,
+                 COUNT(*) FILTER (WHERE status = 'running')::int AS running,
+                 COUNT(*)::int AS total
+          FROM cortex_step_executions
+          WHERE analysis_id = $1 AND run_id = ANY($2::text[])
+          GROUP BY run_id`,
+            values: [analysisId, [...runIds]],
+        });
+        return result.rows.map((r) => ({ runId: r.run_id, doneSteps: r.done, runningSteps: r.running, totalSteps: r.total }));
+    });
+}
 function mapStepExecutionRow(row) {
     return StepExecutionRowSchema.parse({
         runId: row.run_id,

package/dist/tools/report-authoring/authoring-tools.d.ts

@@ -16,8 +16,8 @@
  * The primitive admits a flat object input only. Thus a destination is four optional fields, and not a
  * nested union. A conflict between the flat fields refuses before the core runs.
  *
- * Inside a durable step, a mutation stores its result for the thread and the step, in the transaction of
- * the document and under the lock of the row. A rerun of the step after the landing, before its
+ * Inside a durable step, a mutation stores its result for the thread, the step, and the call, in the
+ * transaction of the document and under the lock of the row. A rerun of the step after the landing, before its
  * checkpoint, gives the stored result. Thus a rerun of `add_block` does not refuse its own block id, and a
  * rerun of `remove_block` does not refuse its own removed target.
  */

package/dist/tools/report-authoring/authoring-tools.js

@@ -16,8 +16,8 @@
  * The primitive admits a flat object input only. Thus a destination is four optional fields, and not a
  * nested union. A conflict between the flat fields refuses before the core runs.
  *
- * Inside a durable step, a mutation stores its result for the thread and the step, in the transaction of
- * the document and under the lock of the row. A rerun of the step after the landing, before its
+ * Inside a durable step, a mutation stores its result for the thread, the step, and the call, in the
+ * transaction of the document and under the lock of the row. A rerun of the step after the landing, before its
  * checkpoint, gives the stored result. Thus a rerun of `add_block` does not refuse its own block id, and a
  * rerun of `remove_block` does not refuse its own removed target.
  */
@@ -133,6 +133,23 @@ const LandedResultSchema = z.object({
         })),
     })),
 });
+/** A refusal as a script reads it: the closed reason and the prose detail. */
+const RefusalOutputSchema = z.object({ reason: z.string(), detail: z.string() });
+const OutlineEntryOutputSchema = z.object({ id: z.string(), kind: z.string(), depth: z.number(), label: z.string() });
+/** The result of an edit, as `run_script` shows it to a script. A refused edit lands nothing. */
+const MutationOutputSchema = z.union([
+    z.object({
+        applied: z.literal(true),
+        changed: z.array(z.object({ parentId: z.string().optional(), children: z.array(OutlineEntryOutputSchema) })),
+    }),
+    z.object({ applied: z.literal(false), refusal: RefusalOutputSchema }),
+]);
+const OutlineOutputSchema = z.union([z.object({ outline: z.array(OutlineEntryOutputSchema) }), z.object({ refused: RefusalOutputSchema })]);
+const ReadBlockOutputSchema = z.union([
+    z.object({ found: z.literal(true), block: z.record(z.string(), z.unknown()) }),
+    z.object({ found: z.literal(false) }),
+    z.object({ refused: RefusalOutputSchema }),
+]);
 /**
  * Resolve the report thread of a call, and load its state through the gateway.
  *
@@ -439,7 +456,9 @@ export function createReportAuthoringTools(gateway, deps = {}) {
      * the landing of its own earlier run.
      */
     const openMutation = async (ctx) => {
-        const callKey = stepKey();
+        // The calls of one script share the step of `run_script`, thus the key names the call inside its step.
+        const step = stepKey();
+        const callKey = step === undefined ? undefined : `${step}:${ctx.invocationId}`;
         const opened = await openReportThread(gateway, ctx.session.scope, callKey);
         if (opened.isErr())
             return { done: refuse(opened.error) };
@@ -504,6 +523,7 @@ export function createReportAuthoringTools(gateway, deps = {}) {
             "The root admits a section only, and an atom needs a section as its parent. " +
             "A reference names the path of a pinned artifact, and the session stamps the hash from the pinned evidence.",
         inputSchema: addBlockInput,
+        outputSchema: MutationOutputSchema,
         describeCall: (input) => {
             const block = decodeBlockPayload(input.block);
             const kind = readKind(block);
@@ -534,10 +554,11 @@ export function createReportAuthoringTools(gateway, deps = {}) {
     });
     const change_block = defineTool({
         id: "change_block",
-        description: "Change one block by its id. Give a `title` to retitle a section, or a `block` to replace an atom, and never both. " +
+        description: "Change one block by its id: a `title` retitles a section, or a `block` replaces an atom, and never both. " +
             "The change keeps the id of the target, thus the `id` of the payload does not matter. A kind change is permitted. " +
             "A reference names the path of a pinned artifact, and the session stamps the hash from the pinned evidence.",
         inputSchema: changeBlockInput,
+        outputSchema: MutationOutputSchema,
         describeCall: (input) => `change ${input.targetId}`,
         execute: async (input, ctx) => {
             const opening = await openMutation(ctx);
@@ -560,6 +581,7 @@ export function createReportAuthoringTools(gateway, deps = {}) {
         id: "remove_block",
         description: "Remove one block by its id. A removed section takes its whole subtree with it, and the id is free for a later add.",
         inputSchema: removeBlockInput,
+        outputSchema: MutationOutputSchema,
         describeCall: (input) => `remove ${input.targetId}`,
         execute: async (input, ctx) => {
             const opening = await openMutation(ctx);
@@ -579,6 +601,7 @@ export function createReportAuthoringTools(gateway, deps = {}) {
         description: "Move one block by its id to a new destination. Name the destination with `parentId` and one of `place`, `before`, or `after`. " +
             "A section cannot move into its own subtree.",
         inputSchema: moveBlockInput,
+        outputSchema: MutationOutputSchema,
         describeCall: (input) => `move ${input.targetId}`,
         execute: async (input, ctx) => {
             const opening = await openMutation(ctx);
@@ -596,9 +619,9 @@ export function createReportAuthoringTools(gateway, deps = {}) {
     });
     const set_title = defineTool({
         id: "set_title",
-        description: "Set the title of the report document. A draft starts with no title, and a document needs one. " +
-            "The title names the whole report, and it is not the title of a section.",
+        description: "Set the title of the whole report, not the title of a section. A draft starts with no title, and a document needs one.",
         inputSchema: setTitleInput,
+        outputSchema: MutationOutputSchema,
         describeCall: (input) => `title the report ${input.title}`,
         // The title sits on the document, thus no container changes its child order.
         execute: async (input, ctx) => {
@@ -624,6 +647,7 @@ export function createReportAuthoringTools(gateway, deps = {}) {
             "A label that ends with the character … is clipped. Use the outline as the primary view of the draft, " +
             "and read one block when the label is not enough.",
         inputSchema: readOutlineInput,
+        outputSchema: OutlineOutputSchema,
         // The input carries no field, thus a hook can only restate the name of the tool.
         describeCall: "none",
         execute: async (_input, ctx) => {
@@ -640,6 +664,7 @@ export function createReportAuthoringTools(gateway, deps = {}) {
             "because the session owns the hash. A section gives its title and the id of each child, " +
             "because the outline already names every block under it. Use it when the outline label is not enough.",
         inputSchema: readBlockInput,
+        outputSchema: ReadBlockOutputSchema,
         describeCall: (input) => `read ${input.targetId}`,
         execute: async (input, ctx) => {
             const opened = await openThread(ctx);

package/dist/tools/report-session/script-tools.d.ts

@@ -18,12 +18,15 @@ import type { WorkspaceFilesystem } from "../../workspace/filesystem.js";
 import { type ResolveWorkspaceRoot } from "../../workspace/paths.js";
 import { type Tool } from "../define-tool.js";
 import { type ReportSessionStateGateway } from "../report-authoring/authoring-tools.js";
+import { type SyncSessionFile } from "../../workspace/session-files.js";
 export type ReportScriptToolsDeps = {
     readonly gateway: ReportSessionStateGateway;
     readonly workspaceFs: WorkspaceFilesystem;
     readonly resolveWorkspaceRoot: ResolveWorkspaceRoot;
     readonly derivations: Pick<ReportSessionStateStore, "appendDerivation">;
     readonly provenance?: ProvenanceSeam;
+    /** The host copy of each derived table, owned by the thread of the session. */
+    readonly syncSessionFile?: SyncSessionFile;
     readonly logger?: Logger;
 };
 /** The script tools of a report session. */

package/dist/tools/report-session/script-tools.js

@@ -24,6 +24,7 @@ import { openReportThread } from "../report-authoring/authoring-tools.js";
 import { ReadTableOutputSchema, readTableRows, rowsText } from "../script/file-tools.js";
 import { DERIVED_FILE_CAP_BYTES } from "../script/scratch.js";
 import { confinedWrite } from "../workspace/mutator.js";
+import { syncWrittenFile } from "../../workspace/session-files.js";
 function refused(error) {
     return err({ error, retryable: false });
 }
@@ -57,7 +58,7 @@ function createReadTableTool(deps) {
             });
             if (table.isErr())
                 return refused(table.error);
-            const hash = computeSha256(table.value.content);
+            const hash = computeSha256(table.value.source);
             if (hash !== entry.hash)
                 return refused(`${path} changed after the session pinned it, thus its bytes are not the evidence of this session`);
             const { rows } = table.value;
@@ -167,6 +168,14 @@ function createWriteDerivedTool(deps) {
             if (appended.value === "absent")
                 return refused("no report session state row exists to hold the derivation");
             observeDerived(analysisId, threadId, record);
+            await syncWrittenFile(deps.syncSessionFile, logger, {
+                session: ctx.session,
+                owner: { kind: "thread", threadId },
+                path: outputPath,
+                hash: record.outputHash,
+                size: landed.bytes.length,
+                content: landed.bytes,
+            });
             return ok(derivedOutput(record, landed.bytes.length, columns));
         },
     });

package/dist/tools/script/archive.d.ts

@@ -0,0 +1,34 @@
+/**
+ * The entries of a compressed file, for the read tools of a script: a zip, a tar, a gzip-compressed tar, or one
+ * gzip-compressed file.
+ *
+ * Each read runs on the host and holds the bytes in memory, thus each one has a cap. A zip names the size of each
+ * entry in its index, thus a list reads no entry and a read decompresses one entry. A tar has no index, thus a list
+ * and a read of a `.tar.gz` decompress the whole stream, up to {@link ARCHIVE_SCAN_CAP_BYTES}.
+ */
+import { type Result } from "neverthrow";
+export type ArchiveKind = "zip" | "tar" | "tar-gzip" | "gzip";
+/** The cap of the decompressed stream of a gzip-compressed tar or file that a list or a read decompresses. */
+export declare const ARCHIVE_SCAN_CAP_BYTES: number;
+export type ArchiveEntry = {
+    readonly name: string;
+    readonly bytes: number;
+};
+/** Why an entry did not read. `too_large` carries the size of the entry, when the archive names it. */
+export type EntryRefusal = {
+    readonly kind: "not_found";
+} | {
+    readonly kind: "too_large";
+    readonly bytes: number | undefined;
+} | {
+    readonly kind: "unparsable";
+    readonly detail: string;
+};
+/** The kind of a compressed file by its name, or `undefined` for a file that is not one. */
+export declare function archiveKindOf(path: string): ArchiveKind | undefined;
+/** The one entry of a gzip-compressed file: its name with no `.gz`. */
+export declare function gzipEntryName(path: string): string;
+/** Each file of a compressed file, with its decompressed size. */
+export declare function listArchive(kind: ArchiveKind, path: string, content: Buffer): Result<ArchiveEntry[], EntryRefusal>;
+/** The decompressed bytes of one entry, up to `cap`. */
+export declare function readArchiveEntry(kind: ArchiveKind, path: string, content: Buffer, entry: string, cap: number): Result<Buffer, EntryRefusal>;

package/dist/tools/script/archive.js

@@ -0,0 +1,182 @@
+/**
+ * The entries of a compressed file, for the read tools of a script: a zip, a tar, a gzip-compressed tar, or one
+ * gzip-compressed file.
+ *
+ * Each read runs on the host and holds the bytes in memory, thus each one has a cap. A zip names the size of each
+ * entry in its index, thus a list reads no entry and a read decompresses one entry. A tar has no index, thus a list
+ * and a read of a `.tar.gz` decompress the whole stream, up to {@link ARCHIVE_SCAN_CAP_BYTES}.
+ */
+import { basename } from "node:path";
+import { gunzipSync } from "node:zlib";
+import { unzipSync } from "fflate";
+import { err, ok } from "neverthrow";
+/** The cap of the decompressed stream of a gzip-compressed tar or file that a list or a read decompresses. */
+export const ARCHIVE_SCAN_CAP_BYTES = 64 * 1024 * 1024;
+/** The kind of a compressed file by its name, or `undefined` for a file that is not one. */
+export function archiveKindOf(path) {
+    const lower = path.toLowerCase();
+    if (lower.endsWith(".zip"))
+        return "zip";
+    if (lower.endsWith(".tar"))
+        return "tar";
+    if (lower.endsWith(".tar.gz") || lower.endsWith(".tgz"))
+        return "tar-gzip";
+    if (lower.endsWith(".gz"))
+        return "gzip";
+    return undefined;
+}
+/** The one entry of a gzip-compressed file: its name with no `.gz`. */
+export function gzipEntryName(path) {
+    return basename(path).slice(0, -".gz".length);
+}
+/** The decompressed bytes of a gzip stream, or `too_large` past `cap`. */
+function gunzip(content, cap) {
+    try {
+        return ok(gunzipSync(content, { maxOutputLength: cap }));
+    }
+    catch (cause) {
+        if (cause instanceof RangeError)
+            return err({ kind: "too_large", bytes: undefined });
+        return err({ kind: "unparsable", detail: "the gzip stream is damaged" });
+    }
+}
+const TAR_BLOCK = 512;
+/** One text field of a tar header: the bytes up to the first NUL. */
+function tarText(block, start, length) {
+    const field = block.subarray(start, start + length);
+    const end = field.indexOf(0);
+    return field.subarray(0, end === -1 ? field.length : end).toString("utf8");
+}
+/** The `path` record of a pax extended header, which names an entry longer than the header field holds. */
+function paxPath(data) {
+    for (const line of data.toString("utf8").split("\n")) {
+        const record = line.slice(line.indexOf(" ") + 1);
+        if (record.startsWith("path="))
+            return record.slice("path=".length);
+    }
+    return undefined;
+}
+/**
+ * Each regular file of a tar, with its data. A GNU long name and a pax `path` record rename the entry that follows
+ * them. A directory, a link, and each other special entry is no file, thus it is skipped.
+ */
+function tarFiles(tar) {
+    const files = [];
+    let offset = 0;
+    let longName;
+    while (offset + TAR_BLOCK <= tar.length) {
+        const header = tar.subarray(offset, offset + TAR_BLOCK);
+        if (header.every((byte) => byte === 0))
+            break;
+        const size = Number.parseInt(tarText(header, 124, 12).trim() || "0", 8);
+        if (!Number.isSafeInteger(size) || size < 0)
+            return err({ kind: "unparsable", detail: "a tar header has no valid size" });
+        const dataStart = offset + TAR_BLOCK;
+        if (dataStart + size > tar.length)
+            return err({ kind: "unparsable", detail: "the tar ends inside an entry" });
+        const data = tar.subarray(dataStart, dataStart + size);
+        const type = String.fromCharCode(header[156] ?? 0);
+        const prefix = tarText(header, 257, 6) === "ustar" ? tarText(header, 345, 155) : "";
+        const name = longName ?? (prefix === "" ? tarText(header, 0, 100) : `${prefix}/${tarText(header, 0, 100)}`);
+        if (type === "L")
+            longName = tarText(data, 0, data.length);
+        else if (type === "x")
+            longName = paxPath(data) ?? longName;
+        else {
+            longName = undefined;
+            if (type === "0" || type === "\0")
+                files.push({ name, data });
+        }
+        offset = dataStart + Math.ceil(size / TAR_BLOCK) * TAR_BLOCK;
+    }
+    return ok(files);
+}
+/** The tar inside a tar archive: the bytes themselves, or the decompressed stream of a `.tar.gz`. */
+function tarOf(kind, content) {
+    return kind === "tar" ? ok(content) : gunzip(content, ARCHIVE_SCAN_CAP_BYTES);
+}
+/** The files of a zip, as its index names them. A directory entry is no file. */
+function zipIndex(content) {
+    const entries = [];
+    try {
+        unzipSync(content, {
+            filter: (file) => {
+                if (!file.name.endsWith("/"))
+                    entries.push({ name: file.name, bytes: file.originalSize });
+                return false;
+            },
+        });
+    }
+    catch {
+        return err({ kind: "unparsable", detail: "the zip is damaged" });
+    }
+    return ok(entries);
+}
+/** Each file of a compressed file, with its decompressed size. */
+export function listArchive(kind, path, content) {
+    switch (kind) {
+        case "zip":
+            return zipIndex(content);
+        case "gzip": {
+            // The last 4 bytes of a gzip stream hold the decompressed size, modulo 2^32.
+            const bytes = content.length >= 4 ? content.readUInt32LE(content.length - 4) : 0;
+            return ok([{ name: gzipEntryName(path), bytes }]);
+        }
+        case "tar":
+        case "tar-gzip":
+            return tarOf(kind, content)
+                .andThen(tarFiles)
+                .map((files) => files.map((file) => ({ name: file.name, bytes: file.data.length })));
+        default: {
+            const unhandled = kind;
+            throw new Error(`unhandled archive kind: ${String(unhandled)}`);
+        }
+    }
+}
+/** The decompressed bytes of one entry, up to `cap`. */
+export function readArchiveEntry(kind, path, content, entry, cap) {
+    switch (kind) {
+        case "zip": {
+            const index = zipIndex(content);
+            if (index.isErr())
+                return err(index.error);
+            const found = index.value.find((file) => file.name === entry);
+            if (found === undefined)
+                return err({ kind: "not_found" });
+            if (found.bytes > cap)
+                return err({ kind: "too_large", bytes: found.bytes });
+            try {
+                const data = unzipSync(content, { filter: (file) => file.name === entry })[entry];
+                if (data === undefined)
+                    return err({ kind: "not_found" });
+                // The index can lie about the size, thus the cap holds on the bytes too.
+                if (data.length > cap)
+                    return err({ kind: "too_large", bytes: data.length });
+                return ok(Buffer.from(data.buffer, data.byteOffset, data.length));
+            }
+            catch {
+                return err({ kind: "unparsable", detail: `the entry ${entry} did not decompress` });
+            }
+        }
+        case "gzip":
+            if (entry !== gzipEntryName(path))
+                return err({ kind: "not_found" });
+            return gunzip(content, cap);
+        case "tar":
+        case "tar-gzip": {
+            const files = tarOf(kind, content).andThen(tarFiles);
+            if (files.isErr())
+                return err(files.error);
+            const found = files.value.find((file) => file.name === entry);
+            if (found === undefined)
+                return err({ kind: "not_found" });
+            if (found.data.length > cap)
+                return err({ kind: "too_large", bytes: found.data.length });
+            return ok(found.data);
+        }
+        default: {
+            const unhandled = kind;
+            throw new Error(`unhandled archive kind: ${String(unhandled)}`);
+        }
+    }
+}

package/dist/tools/script/file-tools.d.ts

@@ -7,7 +7,8 @@
  * directory of the scratch of the session.
  *
  * A table read holds at most 16 MiB, and the parse runs on the host. A larger table, or a format with no parser
- * here, is work for an analysis run, and the refusal says so.
+ * here, is work for an analysis run, and the refusal says so. `list_archive` lists the files of a compressed file, and
+ * a read names one of them as its `entry`. A gzip-compressed table reads with no entry.
  *
  * Each tool gives the files that it read or wrote as its call record. `run_script` collects the records of the
  * calls of one script, thus the provenance of the script chains each derived file to the tables that it read.
@@ -44,9 +45,12 @@ export type ScriptFileToolsDeps = {
     readonly resolveWorkspaceRoot: ResolveWorkspaceRoot;
     readonly scratch: ScriptScratch;
 };
-/** The rows of one table, the bytes that they came from, and the columns that they keep. */
+/**
+ * The rows of one table, the columns that they keep, and the bytes of the file on disk that held them. A read of an
+ * entry names the compressed file, thus its hash is the hash of that file.
+ */
 export type TableRows = {
-    readonly content: Buffer;
+    readonly source: Buffer;
     readonly columns: string[];
     readonly rows: Row[];
 };
@@ -58,6 +62,7 @@ export declare function readTableRows(args: {
     readonly workspaceFs: WorkspaceFilesystem;
     readonly session: AgentSession;
     readonly path: string;
+    readonly entry?: string;
     readonly columns: readonly string[] | undefined;
     readonly tooLarge: string;
 }): Promise<Result<TableRows, string>>;
@@ -69,6 +74,7 @@ export declare const ReadTableOutputSchema: z.ZodObject<{
 }, z.core.$strip>;
 declare const readPdfInput: z.ZodObject<{
     path: z.ZodString;
+    entry: z.ZodOptional<z.ZodString>;
     firstPage: z.ZodOptional<z.ZodNumber>;
     lastPage: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
@@ -93,7 +99,7 @@ export type DocumentToolsDeps = ScriptFileToolsDeps & {
     readonly imageScope?: (session: AgentSession) => string | undefined;
 };
 export declare function createScriptFileTools(deps: ScriptFileToolsDeps): Tool[];
-/** The script tools that read a table and a PDF of the workspace, and write nothing. */
+/** The script tools that read a table, a PDF, and the files of a compressed file of the workspace, and write nothing. */
 export declare function createReadScriptTools(deps: Pick<ScriptFileToolsDeps, "workspaceFs" | "resolveWorkspaceRoot">): Tool[];
 /** The tools that make a document in a script: a chart as an SVG file, and a PDF from markdown. */
 export declare function createDocumentTools(deps: DocumentToolsDeps): Tool[];