@inflexa-ai/harness 0.24.0 → 0.25.0

package/dist/tools/research/inspect-data-profile.d.ts

@@ -46,8 +46,34 @@ interface OverviewOutput extends ProfileEnvelope {
     readonly experimentalDesign?: string;
     readonly qualityAssessment?: DataProfileResult["qualityAssessment"];
     readonly accessions?: string[];
-    /** How many per-file records `scope:"files"` would page through. */
-    readonly fileCount: number;
+    /** How many per-file records `scope:"files"` would page through — NOT the dataset's size. */
+    readonly describedFileCount: number;
+    /**
+     * How many files the dataset holds, or `null` when the snapshot cannot say (one
+     * written before kinds and coverage existed). Reported separately from
+     * `describedFileCount` because a profile describing eight files of several thousand
+     * is correct under this capability, and a single count would read as a dataset of
+     * eight files.
+     */
+    readonly datasetFileCount: number | null;
+    /** How many kinds `scope:"kinds"` would return; `null` on a pre-kinds snapshot. */
+    readonly kindCount: number | null;
+    /** Present when the two counts differ — says where the rest of the dataset is described. */
+    readonly structureNote?: string;
+    readonly coverage?: DataProfileResult["coverage"];
+}
+/**
+ * The dataset's structure. A snapshot predating kinds reports `available: false`
+ * rather than an empty list: "this profile predates the structure" and "this dataset
+ * has no structure" are different facts, and only one of them is true.
+ */
+interface KindsOutput extends ProfileEnvelope {
+    readonly scope: "kinds";
+    readonly available: boolean;
+    readonly kinds?: DataProfileResult["kinds"];
+    readonly axes?: DataProfileResult["axes"];
+    readonly coverage?: DataProfileResult["coverage"];
+    readonly message?: string;
 }
 interface FilesOutput extends ProfileEnvelope {
     readonly scope: "files";
@@ -101,9 +127,9 @@ interface FailedOutput {
     readonly failedAt: string | null;
     readonly message: string;
 }
-export type InspectDataProfileOutput = OverviewOutput | FilesOutput | AbsentOutput | PendingOutput | FailedOutput;
+export type InspectDataProfileOutput = OverviewOutput | KindsOutput | FilesOutput | AbsentOutput | PendingOutput | FailedOutput;
 export declare function createInspectDataProfileTool(pool: Pool): import("../define-tool.js").Tool<{
-    scope?: "files" | "overview" | undefined;
+    scope?: "overview" | "files" | "kinds" | undefined;
     page?: number | undefined;
     pageSize?: number | undefined;
 }, InspectDataProfileOutput>;

package/dist/tools/research/inspect-data-profile.js

@@ -31,6 +31,20 @@ import { defineTool } from "../define-tool.js";
 const DEFAULT_PAGE_SIZE = 20;
 /** Ceiling on `pageSize` — a page is a context-window budget, not a dump. */
 const MAX_PAGE_SIZE = 100;
+/**
+ * How many files the dataset holds, as far as the snapshot can establish it.
+ *
+ * Coverage counts the scanned tree, so it is the authority where present. Summed kind
+ * counts are the fallback. A snapshot with neither returns `null` rather than the
+ * described-file count, which would be a dataset size the row cannot support.
+ */
+function datasetFileCount(result) {
+    if (result.coverage)
+        return result.coverage.total;
+    if (result.kinds && result.kinds.length > 0)
+        return result.kinds.reduce((sum, kind) => sum + kind.count, 0);
+    return null;
+}
 /**
  * Name every reason the served profile may not describe the current inputs.
  *
@@ -44,7 +58,7 @@ const MAX_PAGE_SIZE = 100;
  */
 function stalenessReasons(status, result) {
     const reasons = [];
-    if (isDataProfileStale(status.seedInputFileIds ?? [], result.inputFileIds)) {
+    if (isDataProfileStale({ fileIds: status.seedInputFileIds ?? [] }, result)) {
         reasons.push("the analysis's input file set changed after this profile was taken");
     }
     if (status.status === "running" || status.status === "pending") {
@@ -69,14 +83,20 @@ export function createInspectDataProfileTool(pool) {
         id: "inspect_data_profile",
         description: "Read this analysis's data profile — the AUTHORITATIVE record of what the input dataset is: " +
             "organism and taxon id, scientific domain and subtype, tissue, cell type, condition, experimental design, " +
-            "dataset-wide quality concerns and strengths, public accessions, and per-file data type, format, " +
-            "row/column dimensions, tags, warnings, and profiling metrics. " +
+            "dataset-wide quality concerns and strengths, public accessions, the KINDS of file the dataset is made of " +
+            "and what varies across them, and per-file data type, format, row/column dimensions, tags and warnings for " +
+            "the files described individually. " +
             "There is NO data-profile file in the workspace — this tool is the only way to read it. Do not search for one, " +
             "and do not rediscover these facts by listing or reading the raw input files. " +
             "Call it before you reason about the data (planning, writing analysis code, interpreting results). " +
-            "scope:'overview' (the default) returns the dataset-level orientation plus the number of profiled files. " +
-            "scope:'files' returns the per-file records, paged: page (1-based, default 1) and pageSize (default 20, max 100), " +
+            "scope:'overview' (the default) returns the dataset-level orientation, how many files are described individually, " +
+            "and how many files the dataset holds — those two differ on purpose, because a large dataset is described by its " +
+            "kinds rather than file by file. " +
+            "scope:'kinds' returns those kinds with their counts, path patterns, and axes — this is where a dataset of thousands " +
+            "of files is described. A profile taken before kinds existed reports the scope unavailable. " +
+            "scope:'files' returns the individually described files, paged: page (1-based, default 1) and pageSize (default 20, max 100), " +
             "always with the true total and hasMore, so you can see exactly what you have not read yet. " +
+            "For the paths of files no record describes, list the workspace tree. " +
             "The state field says what you got: 'ready'; 'stale' (a profile is returned but may not describe the current " +
             "inputs — staleReason says why); 'pending' (profiling is still running); 'failed'; or 'absent' (never profiled, " +
             "or the analysis has no input files). " +
@@ -86,9 +106,11 @@ export function createInspectDataProfileTool(pool) {
             "needs re-running rather than diagnosing the data itself.",
         inputSchema: z.object({
             scope: z
-                .enum(["overview", "files"])
+                .enum(["overview", "kinds", "files"])
                 .optional()
-                .describe("'overview' (default): dataset-level facts + file count. 'files': paged per-file records."),
+                .describe("'overview' (default): dataset-level facts, described-file count, and dataset file count. " +
+                "'kinds': the sets the dataset is made of and what varies across them. " +
+                "'files': paged records for the individually described files."),
             page: z.number().int().min(1).optional().describe("1-based page of per-file records. Only used when scope is 'files'. Default 1."),
             pageSize: z
                 .number()
@@ -101,7 +123,7 @@ export function createInspectDataProfileTool(pool) {
         // `execute` defaults both fields internally, and the hook applies the same
         // defaults. Without them the ordinary call, which names no field at all,
         // gives no detail. `page` acts only on the `files` scope, as in `execute`.
-        describeCall: ({ scope, page }) => ((scope ?? "overview") === "files" ? `files (page ${page ?? 1})` : "overview"),
+        describeCall: ({ scope, page }) => ((scope ?? "overview") === "files" ? `files (page ${page ?? 1})` : (scope ?? "overview")),
         execute: async (input, ctx) => {
             const resourceId = scopeResource(ctx.session.scope).resourceId;
             const status = unwrapOrThrow(await loadDataProfileStatus(pool, resourceId));
@@ -145,7 +167,17 @@ export function createInspectDataProfileTool(pool) {
             const envelope = reasons.length > 0
                 ? { state: "stale", staleReason: reasons.join("; "), profiledAt: result.profiledAt }
                 : { state: "ready", profiledAt: result.profiledAt };
-            if ((input.scope ?? "overview") === "overview") {
+            const scope = input.scope ?? "overview";
+            if (scope === "overview") {
+                const dataset = datasetFileCount(result);
+                const described = result.files.length;
+                const structureNote = dataset === null
+                    ? "This profile predates the dataset-structure record, so only the individually described files are known; " +
+                        "list the workspace tree for the rest."
+                    : dataset !== described
+                        ? `${described} of ${dataset} files are described individually. Call scope:'kinds' for what the rest are, ` +
+                            "and list the workspace tree for their paths."
+                        : undefined;
                 return ok({
                     ...envelope,
                     scope: "overview",
@@ -159,7 +191,31 @@ export function createInspectDataProfileTool(pool) {
                     experimentalDesign: result.experimentalDesign,
                     qualityAssessment: result.qualityAssessment,
                     accessions: result.accessions,
-                    fileCount: result.files.length,
+                    describedFileCount: described,
+                    datasetFileCount: dataset,
+                    kindCount: result.kinds ? result.kinds.length : null,
+                    ...(structureNote ? { structureNote } : {}),
+                    ...(result.coverage ? { coverage: result.coverage } : {}),
+                });
+            }
+            if (scope === "kinds") {
+                if (!result.kinds) {
+                    return ok({
+                        ...envelope,
+                        scope: "kinds",
+                        available: false,
+                        message: "This profile was taken before the dataset-structure record existed, so it carries no kinds. " +
+                            "That is a fact about the profile, not about the dataset: use scope:'files' and the workspace " +
+                            "listing tools, or re-profile the analysis.",
+                    });
+                }
+                return ok({
+                    ...envelope,
+                    scope: "kinds",
+                    available: true,
+                    kinds: result.kinds,
+                    ...(result.axes ? { axes: result.axes } : {}),
+                    ...(result.coverage ? { coverage: result.coverage } : {}),
                 });
             }
             const pageSize = input.pageSize ?? DEFAULT_PAGE_SIZE;

package/dist/tools/workspace/scan-inputs.d.ts

@@ -0,0 +1,38 @@
+/**
+ * `scan_inputs` tool — re-run the deterministic input scan over a subtree.
+ *
+ * The data-profile body runs the scan once before the agent loop and injects the
+ * manifest into the briefing, because the scan is always needed and its result does
+ * not depend on agent judgement. This tool is the other half: an agent that disagrees
+ * with the observed shapes, or wants a directory the injected manifest summarised, can
+ * get the evidence to group differently.
+ *
+ * Dependency-bearing factory. The walk and the shape observation run in-process over
+ * the workspace read seam; only the per-shape header readout reaches into the sandbox
+ * (see `input-scan/enrich.ts`).
+ */
+import type { SandboxClient } from "../../sandbox/client.js";
+import type { SandboxRef } from "../../sandbox/types.js";
+import type { WorkspaceFilesystem } from "../../workspace/filesystem.js";
+export interface ScanInputsDeps {
+    readonly workspaceFs: WorkspaceFilesystem;
+    readonly analysisId: string;
+    /** Omit to run the host-side scan with no header readout (no sandbox wired). */
+    readonly sandboxClient?: SandboxClient;
+    readonly sandbox?: SandboxRef;
+    readonly workflowId: string;
+    readonly stepId: string;
+    readonly nextFunctionId: () => string;
+    readonly deadlineMs: () => number;
+}
+type ScanInputsOutput = {
+    readonly path: string;
+    readonly fileCount: number;
+    readonly shapeCount: number;
+    /** The rendered manifest — bounded whatever the subtree's size. */
+    readonly manifest: string;
+};
+export declare function createScanInputsTool(deps: ScanInputsDeps): import("../define-tool.js").Tool<{
+    path: string;
+}, ScanInputsOutput>;
+export {};

package/dist/tools/workspace/scan-inputs.js

@@ -0,0 +1,61 @@
+/**
+ * `scan_inputs` tool — re-run the deterministic input scan over a subtree.
+ *
+ * The data-profile body runs the scan once before the agent loop and injects the
+ * manifest into the briefing, because the scan is always needed and its result does
+ * not depend on agent judgement. This tool is the other half: an agent that disagrees
+ * with the observed shapes, or wants a directory the injected manifest summarised, can
+ * get the evidence to group differently.
+ *
+ * Dependency-bearing factory. The walk and the shape observation run in-process over
+ * the workspace read seam; only the per-shape header readout reaches into the sandbox
+ * (see `input-scan/enrich.ts`).
+ */
+import { ok } from "neverthrow";
+import { z } from "zod";
+import { enrichShapes } from "../../input-scan/enrich.js";
+import { renderInputScanManifest, scanInputTree } from "../../input-scan/scan.js";
+import { defineTool } from "../define-tool.js";
+export function createScanInputsTool(deps) {
+    return defineTool({
+        id: "scan_inputs",
+        description: "Scan a directory of input files and report what is OBSERVABLE about it: per-format counts, " +
+            "the shapes its filenames form (sets of files whose names differ only at marked positions), " +
+            "the distinct values each varying position takes, how those positions co-occur, value overlap " +
+            "between shapes, and the files that share structure with nothing else. " +
+            "Deterministic — no model, no per-file decoding — so it costs the same on three files as on three thousand. " +
+            "Your briefing already carries a scan of the whole input tree; call this to look at a subtree more closely, " +
+            "or to check a grouping you are unsure about. " +
+            "It reports observations only: which files are one KIND of data, and what a varying position MEANS, are yours to decide.",
+        inputSchema: z.object({
+            path: z.string().min(1).describe("Directory to scan, relative to the analysis root (e.g. 'data/inputs' or 'data/inputs/vcf')."),
+        }),
+        describeCall: ({ path }) => path,
+        execute: async ({ path }, ctx) => {
+            const scan = await scanInputTree({
+                session: ctx.session,
+                fs: deps.workspaceFs,
+                root: path.replace(/^\/+/, "").replace(/\/+$/, ""),
+                signal: ctx.signal,
+            });
+            const shapes = deps.sandboxClient && deps.sandbox
+                ? await enrichShapes({
+                    shapes: scan.manifest.shapes,
+                    sandboxClient: deps.sandboxClient,
+                    sandbox: deps.sandbox,
+                    mountRoot: `/${deps.analysisId}`,
+                    execId: `${deps.workflowId}:${deps.stepId}:${deps.nextFunctionId()}`,
+                    deadlineMs: deps.deadlineMs(),
+                    emit: ctx.emit,
+                })
+                : scan.manifest.shapes;
+            const manifest = { ...scan.manifest, shapes };
+            return ok({
+                path: manifest.root,
+                fileCount: manifest.fileCount,
+                shapeCount: manifest.shapes.length,
+                manifest: renderInputScanManifest(manifest),
+            });
+        },
+    });
+}

package/dist/tools/workspace/workspace-search.d.ts

@@ -15,7 +15,7 @@ import type { EmbeddingProvider } from "../../providers/types.js";
 export declare function createWorkspaceSearchTool(pool: Pool, embedding: EmbeddingProvider): import("../define-tool.js").Tool<{
     query: string;
     limit: number;
-    type?: "output" | "input" | "synthesis" | "summary" | undefined;
+    type?: "output" | "input" | "synthesis" | "summary" | "input-kind" | undefined;
 }, {
     results: {
         id: string;

package/dist/tools/workspace/workspace-search.js

@@ -32,19 +32,22 @@ function indexName(analysisId) {
 export function createWorkspaceSearchTool(pool, embedding) {
     return defineTool({
         id: "workspace_search",
-        description: "Semantic search over the analysis workspace. Returns ranked file paths " +
-            "with descriptions and metadata; read a file separately to see its " +
-            "contents. Indexed entries are exactly four types, and `type` restricts " +
-            'results to one of them: "input" (staged input data files — their ' +
-            'descriptions come from the data profiler), "output" (files a step ' +
-            'produced), "summary" (a step\'s summary), "synthesis" (a run\'s ' +
-            "literature-grounded synthesis).",
+        description: "Semantic search over the analysis workspace. Returns ranked entries with descriptions " +
+            "and metadata; read a file separately to see its contents. Indexed entries are exactly " +
+            'five types, and `type` restricts results to one of them: "input-kind" (a KIND of input ' +
+            "data — a repeating set the data profiler named, carrying its count and a pathPattern " +
+            'matching its members), "input" (one entity the dataset is about — a subject, sample, or ' +
+            'other axis value — with the kinds of file it carries), "output" (files a step produced), ' +
+            '"summary" (a step\'s summary), "synthesis" (a run\'s literature-grounded synthesis). ' +
+            "For output/summary/synthesis the entry id IS the workspace path; for the two input types " +
+            "it is not, because a kind and an entity each span many files — search input-kind for a " +
+            "set's pathPattern, then list the tree for its members.",
         inputSchema: z.object({
             query: z.string().min(1).describe("What to search for, in natural language"),
             type: z
-                .enum(["input", "output", "summary", "synthesis"])
+                .enum(["input-kind", "input", "output", "summary", "synthesis"])
                 .optional()
-                .describe('Restrict results to one entry type: "input", "output", "summary", or "synthesis"'),
+                .describe('Restrict results to one entry type: "input-kind", "input", "output", "summary", or "synthesis"'),
             limit: z.number().int().min(1).max(50).default(8).describe("Maximum number of results to return"),
         }),
         describeCall: ({ query, type }) => (type === undefined ? query : `${query} (${type})`),

package/dist/workspace/filesystem.d.ts

@@ -67,6 +67,15 @@ export type StatResult = {
 } | {
     readonly kind: "out_of_scope";
 };
+/** A bounded prefix of a file's raw bytes. `bytes` may be shorter than requested. */
+export type ReadBytesResult = {
+    readonly kind: "ok";
+    readonly bytes: Buffer;
+} | {
+    readonly kind: "not_found";
+} | {
+    readonly kind: "out_of_scope";
+};
 export interface WorkspaceFilesystem {
     /**
      * Read a workspace file. `headLines` / `tailLines` (mutually exclusive,
@@ -86,6 +95,24 @@ export interface WorkspaceFilesystem {
         /** Absolute host base for relative paths. Defaults to the analysis root. */
         readonly workingDir?: string;
     }): ResultAsync<ReadFileResult, FsError>;
+    /**
+     * Read the first `length` bytes of a file, raw.
+     *
+     * Distinct from `readFile` because that surface is line- and text-oriented — it
+     * decodes to UTF-8 for its line windows and its byte cap is an outer bound on a
+     * whole-file read, not a prefix request. Magic-byte format detection needs an
+     * exact, small prefix of the raw bytes and nothing else.
+     *
+     * Local files only: a prefix of a presigned object would mean fetching the whole
+     * object to read its first 512 bytes, which defeats the bound. An absent local
+     * file is `not_found`.
+     */
+    readBytes(args: {
+        readonly session: AgentSession;
+        readonly path: string;
+        readonly length: number;
+        readonly workingDir?: string;
+    }): ResultAsync<ReadBytesResult, FsError>;
     list(args: {
         readonly session: AgentSession;
         readonly path: string;

package/dist/workspace/filesystem.js

@@ -117,6 +117,33 @@ export function createWorkspaceFilesystem(deps) {
                 });
             });
         },
+        readBytes({ session, path, length, workingDir }) {
+            return confinedResolve(session, path, workingDir).andThen((resolved) => {
+                if (resolved.kind === "out_of_scope")
+                    return okAsync({ kind: "out_of_scope" });
+                const absolute = resolved.absolute;
+                return safeStat(absolute).andThen((localStat) => {
+                    if (!localStat?.isFile())
+                        return okAsync({ kind: "not_found" });
+                    return openReadNoFollow(absolute).andThen((opened) => {
+                        if (opened.kind === "out_of_scope")
+                            return okAsync({ kind: "out_of_scope" });
+                        const fh = opened.fh;
+                        return tryFs("workspace.readBytes", async () => {
+                            try {
+                                const want = Math.max(0, Math.min(length, localStat.size));
+                                const buf = Buffer.alloc(want);
+                                const { bytesRead } = want > 0 ? await fh.read(buf, 0, want, 0) : { bytesRead: 0 };
+                                return { kind: "ok", bytes: buf.subarray(0, bytesRead) };
+                            }
+                            finally {
+                                await fh.close();
+                            }
+                        }, { path: absolute });
+                    });
+                });
+            });
+        },
         list({ session, path, workingDir }) {
             return confinedResolve(session, path, workingDir).andThen((resolved) => {
                 if (resolved.kind === "out_of_scope") {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@inflexa-ai/harness",
-    "version": "0.24.0",
+    "version": "0.25.0",
     "description": "Host-agnostic bioinformatics agent harness — hand-rolled agent loop, DBOS-durable workflows, sandboxed compute.",
     "license": "Apache-2.0",
     "type": "module",