@inflexa-ai/harness 0.28.1 → 0.30.0

package/dist/tools/research/generate-plan.js

@@ -35,6 +35,12 @@ import { defineTool } from "../define-tool.js";
 import { createListAvailablePackagesTool } from "../sandbox/list-available-packages.js";
 import { createListAvailableRefsTool } from "../sandbox/list-available-refs.js";
 import { createReportBlockerToolFor } from "../sandbox/report-blocker.js";
+import { searchGeoDatasetsTool } from "../bio/search-geo-datasets.js";
+import { createNcbiTools } from "../bio/keys.js";
+import { queryDocsTool, resolveLibraryIdTool } from "./context7-docs.js";
+import { searchArxivTool } from "./search-arxiv.js";
+import { createSearchGithubReposTool } from "./search-github-repos.js";
+import { createSearchSemanticScholarTool } from "./search-semantic-scholar.js";
 import { DATA_PROFILE_ORIENTATION_MAX_CHARS, buildDataProfileOrientation } from "../../app/data-profile-orientation.js";
 import { DEFAULT_SANDBOX_MAX_STEPS } from "../../config/resource-limits.js";
 import { plannerPrompt } from "../../prompts/planner.js";
@@ -49,10 +55,17 @@ import { insertPlan, loadDataProfileStatus, loadPlan } from "../../state/index.j
 /** Sub-agent identity for the planner — provenance only. */
 const PLANNER_AGENT_ID = "planner";
 /**
- * Budget for the planner's internal loop: one draft/submit attempt plus
- * correction retries and headroom.
+ * Budget for the planner's internal loop: the research phase, one draft/submit
+ * attempt, correction retries, and headroom.
+ *
+ * The number is deliberately far above what a plan costs. A well-grounded plan
+ * still costs two or three calls, because the prompt gates the research phase on
+ * what the seed does not already answer. What the ceiling must never do is stop
+ * a planner that is searching correctly. A capped run takes the forced wrap-up
+ * path and submits the plan that it had, not the plan that it was building.
+ * Thus the wall-clock guard, not this number, is what bounds the worst case.
  */
-const PLANNER_MAX_ITERATIONS = 13;
+const PLANNER_MAX_ITERATIONS = 200;
 /** Wall-clock guard for a single plan-generation invocation. */
 const PLAN_TIMEOUT_MS = 600_000;
 // ── Diagnostic bounds ───────────────────────────────────────────────
@@ -71,7 +84,7 @@ const MAX_LOGGED_ISSUE_CHARS = 240;
 const MAX_LOGGED_REJECTIONS = 6;
 /** Excerpt of the planner's last words — the one artifact that explains a run ending on prose. */
 const MAX_LOGGED_PROSE_CHARS = 800;
-/** Tool-call trace length. The planner's whole budget is ~16 turns, so this rarely truncates. */
+/** Tool-call trace length. A long research phase passes it, and `toolCallsTruncated` marks the record when it does. */
 const MAX_LOGGED_TOOL_CALLS = 48;
 /** Trim to `max`, marking the cut so a truncated value never reads as a complete one. */
 function bounded(value, max) {
@@ -629,6 +642,44 @@ const OUTCOME_LEVEL = {
     invalid_parent_plan: "warn",
     parent_plan_load_failed: "error",
 };
+/**
+ * The search tools of the planner — built one time, not per invocation.
+ *
+ * The set answers the two questions that a seed cannot. "What did a study of
+ * this kind do before?" reaches the public dataset records and the literature.
+ * "What does this package actually give me?" reaches the developer docs. The
+ * planner reads its reference and package inventories from the seed, thus the
+ * docs pair is what turns a package name into a real function.
+ *
+ * A tool here never writes and never computes. Thus the worst outcome of a
+ * needless call is latency, and the prompt is what bounds that.
+ */
+function buildPlannerSearchTools(deps) {
+    const bioKeys = deps.bioKeys;
+    const ncbi = createNcbiTools(bioKeys);
+    return [
+        // Comparable public studies — the design, the platform, and the sample
+        // count that a plan for the same assay must answer to.
+        searchGeoDatasetsTool,
+        // Prior methods, in three corpora that do not overlap: the biomedical
+        // literature, the statistics and machine-learning preprints, and the
+        // cross-field index that ranks a method paper best.
+        ncbi.pubmed,
+        searchArxivTool,
+        createSearchSemanticScholarTool({ ...(bioKeys.semanticScholar === undefined ? {} : { apiKey: bioKeys.semanticScholar }) }),
+        // The same approach as running code — a pipeline that someone published.
+        createSearchGithubReposTool({ githubToken: bioKeys.github }),
+        // The API of a staged package, so a step names a function that exists.
+        resolveLibraryIdTool,
+        queryDocsTool,
+        // The environment itself. The seed already carries a rendered census of
+        // both stores. These two tools are for the narrow second look: one
+        // collection of the reference store, or one package name that a step is
+        // about to import.
+        createListAvailableRefsTool(deps.refStorePath === undefined ? {} : { refStorePath: deps.refStorePath }),
+        createListAvailablePackagesTool(deps.packagesFile === undefined ? {} : { packagesFile: deps.packagesFile }),
+    ];
+}
 /** Build the `generate_plan` tool bound to its provider and pool. */
 export function createGeneratePlanTool(deps) {
     const baseLogger = (deps.logger ?? createNoopLogger()).named("generate-plan");
@@ -806,11 +857,19 @@ export function createGeneratePlanTool(deps) {
                 parentPlanId: input.parentPlanId ?? null,
             };
             const innerTools = buildInnerTools(holder, trace, persistCtx, deps.pool, deps.resourcePolicy, logger);
+            // Built here rather than at construction: a `describeCall` hook reads no
+            // dep, thus the tool must stay constructible from an empty bag. The tool
+            // definitions are identical across invocations, thus the request prefix
+            // that the cache keys on does not move.
+            const searchTools = buildPlannerSearchTools(deps);
             const planner = {
                 id: PLANNER_AGENT_ID,
                 systemPrompt: composeSystemPrompt(plannerInstructions(deps.resourcePolicy)),
                 model: deps.conversation.model,
-                tools: innerTools.terminal,
+                // The search tools come first, and the terminal tools come last.
+                // The order is the order of the prompt, and it is stable across
+                // every invocation, thus the cached request prefix holds.
+                tools: [...searchTools, ...innerTools.terminal],
                 maxIterations: PLANNER_MAX_ITERATIONS,
             };
             // Merge the outer abort signal with a wall-clock timeout — either

package/dist/tools/research/inspect-data-profile.d.ts

@@ -8,13 +8,16 @@
  * **There is no data-profile file.** The profiler's `runs/data-profile/` scratch tree
  * is deleted when profiling completes; the profile's only durable home is the
  * `cortex_analysis_state` row this tool reads. That makes this the single authoritative
- * source for dataset facts (organism, domain, design, per-file type/format/dimensions),
- * and it is why the record is served in full rather than summarized: an agent that
- * cannot pull a fact here has no fallback but to re-derive it from the raw bytes.
+ * source for dataset facts — the identity fields, the resolved groups with their slots,
+ * the dimensions with their observations, and the partition census — and it is why the
+ * record is served in full rather than summarized: an agent that cannot pull a fact here
+ * has no fallback but to re-derive it from the raw bytes.
  *
- * Bounded by construction. Like `inspect_run`, the per-file scope is explicitly
- * paged and always reports `total` and `hasMore`: an elided tail is a fact the
- * model can see and act on, never a silent truncation.
+ * Bounded by construction. Like `inspect_run`, the member scope is explicitly paged and
+ * always reports `total` and `hasMore`: an elided tail is a fact the model can see and
+ * act on, never a silent truncation. The annotated-member count and the dataset's file
+ * count are reported as two distinct figures, because a profile annotating eight members
+ * of several thousand files is correct and a single number would read as a tiny dataset.
  *
  * Lifecycle states are data variants, not errors (the `defineTool` contract): a missing,
  * in-flight, failed, or stale profile is an ordinary, expected outcome the model must be
@@ -50,45 +53,68 @@ interface OverviewOutput extends ProfileEnvelope {
     readonly cellType?: string | null;
     readonly condition?: string | null;
     readonly experimentalDesign?: string;
-    readonly qualityAssessment?: DataProfileResult["qualityAssessment"];
+    /** Dataset-wide findings the agent wrote, from whichever field the snapshot carries them in. */
+    readonly caveats?: string[];
     readonly accessions?: string[];
-    /** How many per-file records `scope:"files"` would page through — NOT the dataset's size. */
+    /** How many member records `scope:"files"` would page through — NOT the dataset's size. */
     readonly describedFileCount: number;
     /**
      * How many files the dataset holds, or `null` when the snapshot cannot say (one
-     * written before kinds and coverage existed). Reported separately from
-     * `describedFileCount` because a profile describing eight files of several thousand
-     * is correct under this capability, and a single count would read as a dataset of
-     * eight files.
+     * written before any structure record existed). Reported separately from
+     * `describedFileCount` because a profile describing eight members of several thousand
+     * files is correct under this capability, and a single count would read as a dataset
+     * of eight files.
      */
     readonly datasetFileCount: number | null;
-    /** How many kinds `scope:"kinds"` would return; `null` on a pre-kinds snapshot. */
-    readonly kindCount: number | null;
+    /** How many groups `scope:"groups"` would return; `null` on a snapshot that carries no structure. */
+    readonly groupCount: number | null;
     /** Present when the two counts differ — says where the rest of the dataset is described. */
     readonly structureNote?: string;
+    /**
+     * The census, on a snapshot resolved under the partition: kept files, per-group
+     * counts, the unclassified sweep, and the quarantine with its reasons.
+     */
+    readonly partition?: DataProfileResult["partition"];
     readonly coverage?: DataProfileResult["coverage"];
 }
 /**
- * The dataset's structure. A snapshot predating kinds reports `available: false`
- * rather than an empty list: "this profile predates the structure" and "this dataset
- * has no structure" are different facts, and only one of them is true.
+ * The dataset's structure. A snapshot carrying none reports `available: false` rather
+ * than an empty list: "this profile predates the structure" and "this dataset has no
+ * structure" are different facts, and only one of them is true.
+ *
+ * A snapshot written under the previous model is served through `kinds`/`axes`, labelled
+ * — the structure exists, and an agent must not be told the dataset has none.
  */
-interface KindsOutput extends ProfileEnvelope {
-    readonly scope: "kinds";
+interface GroupsOutput extends ProfileEnvelope {
+    readonly scope: "groups";
     readonly available: boolean;
+    /** Authored under the previous model — `kinds`/`axes` rather than resolved groups. */
+    readonly legacy?: boolean;
+    readonly groups?: DataProfileResult["groups"];
+    readonly dimensions?: DataProfileResult["dimensions"];
+    readonly probes?: DataProfileResult["probes"];
+    readonly partition?: DataProfileResult["partition"];
+    readonly recipe?: DataProfileResult["recipe"];
     readonly kinds?: DataProfileResult["kinds"];
     readonly axes?: DataProfileResult["axes"];
     readonly coverage?: DataProfileResult["coverage"];
     readonly message?: string;
 }
+/**
+ * The members the profile describes individually — annotated members on a resolved
+ * snapshot, the notable-file records on an older one. Never the dataset: `datasetFileCount`
+ * on the overview is what says how big that is.
+ */
 interface FilesOutput extends ProfileEnvelope {
     readonly scope: "files";
     readonly page: number;
     readonly pageSize: number;
-    /** Every per-file record the profile holds — not just this page. */
+    /** Every member record the profile holds — not just this page. */
     readonly total: number;
     /** True when records remain past this page. Truncation is always stated, never silent. */
     readonly hasMore: boolean;
+    /** How many files the dataset holds, so a page of 20 never reads as the whole of it. */
+    readonly datasetFileCount: number | null;
     readonly files: DataProfileFile[];
 }
 /**
@@ -133,9 +159,9 @@ interface FailedOutput {
     readonly failedAt: string | null;
     readonly message: string;
 }
-export type InspectDataProfileOutput = OverviewOutput | KindsOutput | FilesOutput | AbsentOutput | PendingOutput | FailedOutput;
+export type InspectDataProfileOutput = OverviewOutput | GroupsOutput | FilesOutput | AbsentOutput | PendingOutput | FailedOutput;
 export declare function createInspectDataProfileTool(pool: Pool): import("../define-tool.js").Tool<{
-    scope?: "overview" | "files" | "kinds" | undefined;
+    scope?: "overview" | "groups" | "files" | undefined;
     page?: number | undefined;
     pageSize?: number | undefined;
 }, InspectDataProfileOutput>;

package/dist/tools/research/inspect-data-profile.js

@@ -8,13 +8,16 @@
  * **There is no data-profile file.** The profiler's `runs/data-profile/` scratch tree
  * is deleted when profiling completes; the profile's only durable home is the
  * `cortex_analysis_state` row this tool reads. That makes this the single authoritative
- * source for dataset facts (organism, domain, design, per-file type/format/dimensions),
- * and it is why the record is served in full rather than summarized: an agent that
- * cannot pull a fact here has no fallback but to re-derive it from the raw bytes.
+ * source for dataset facts — the identity fields, the resolved groups with their slots,
+ * the dimensions with their observations, and the partition census — and it is why the
+ * record is served in full rather than summarized: an agent that cannot pull a fact here
+ * has no fallback but to re-derive it from the raw bytes.
  *
- * Bounded by construction. Like `inspect_run`, the per-file scope is explicitly
- * paged and always reports `total` and `hasMore`: an elided tail is a fact the
- * model can see and act on, never a silent truncation.
+ * Bounded by construction. Like `inspect_run`, the member scope is explicitly paged and
+ * always reports `total` and `hasMore`: an elided tail is a fact the model can see and
+ * act on, never a silent truncation. The annotated-member count and the dataset's file
+ * count are reported as two distinct figures, because a profile annotating eight members
+ * of several thousand files is correct and a single number would read as a tiny dataset.
  *
  * Lifecycle states are data variants, not errors (the `defineTool` contract): a missing,
  * in-flight, failed, or stale profile is an ordinary, expected outcome the model must be
@@ -28,6 +31,7 @@
  */
 import { ok } from "neverthrow";
 import { z } from "zod";
+import { profileCaveats, profileDatasetFileCount, profileFileRecords, profileGroups } from "../../app/data-profile-view.js";
 import { scopeResource } from "../../auth/types.js";
 import { unwrapOrThrow } from "../../lib/result.js";
 import { loadDataProfileStatus } from "../../state/index.js";
@@ -37,31 +41,19 @@ import { defineTool } from "../define-tool.js";
 const DEFAULT_PAGE_SIZE = 20;
 /** Ceiling on `pageSize` — a page is a context-window budget, not a dump. */
 const MAX_PAGE_SIZE = 100;
-/**
- * How many files the dataset holds, as far as the snapshot can establish it.
- *
- * Coverage counts the scanned tree, so it is the authority where present. Summed kind
- * counts are the fallback. A snapshot with neither returns `null` rather than the
- * described-file count, which would be a dataset size the row cannot support.
- */
-function datasetFileCount(result) {
-    if (result.coverage)
-        return result.coverage.total;
-    if (result.kinds && result.kinds.length > 0)
-        return result.kinds.reduce((sum, kind) => sum + kind.count, 0);
-    return null;
-}
 /** Serve each per-file record with a path that resolves in the frame of the caller. */
 function rootFilePaths(files, analysisId) {
     return files.map((file) => ({ ...file, path: toAnalysisRootPath(analysisId, file.path) }));
 }
 /**
- * Serve each kind with a glob that resolves in the frame of the caller.
+ * Serve each group with a display pattern that resolves in the frame of the caller.
  *
- * A `pathPattern` is a path with a wildcard segment, thus it takes the same
- * projection as a file path. The wildcard characters survive it: the projection
- * only re-roots the pattern.
+ * A pattern is a path with varying segments, thus it takes the same projection as a file
+ * path. Its placeholders survive it: the projection only re-roots the pattern.
  */
+function rootGroupPatterns(groups, analysisId) {
+    return groups.map((group) => ({ ...group, displayPattern: toAnalysisRootPath(analysisId, group.displayPattern) }));
+}
 function rootKindPatterns(kinds, analysisId) {
     return kinds.map((kind) => ({ ...kind, pathPattern: toAnalysisRootPath(analysisId, kind.pathPattern) }));
 }
@@ -103,20 +95,21 @@ export function createInspectDataProfileTool(pool) {
         id: "inspect_data_profile",
         description: "Read this analysis's data profile — the AUTHORITATIVE record of what the input dataset is: " +
             "organism and taxon id, scientific domain and subtype, tissue, cell type, condition, experimental design, " +
-            "dataset-wide quality concerns and strengths, public accessions, the KINDS of file the dataset is made of " +
-            "and what varies across them, and per-file data type, format, row/column dimensions, tags and warnings for " +
-            "the files described individually. " +
+            "dataset-wide caveats, public accessions, the GROUPS of file the dataset is made of, the DIMENSIONS that vary " +
+            "across it with the evidence behind each, and the file census (kept, unclassified, quarantined). " +
             "There is NO data-profile file in the workspace — this tool is the only way to read it. Do not search for one, " +
             "and do not rediscover these facts by listing or reading the raw input files. " +
             "Call it before you reason about the data (planning, writing analysis code, interpreting results). " +
-            "scope:'overview' (the default) returns the dataset-level orientation, how many files are described individually, " +
-            "and how many files the dataset holds — those two differ on purpose, because a large dataset is described by its " +
-            "kinds rather than file by file. " +
-            "scope:'kinds' returns those kinds with their counts, path patterns, and axes — this is where a dataset of thousands " +
-            "of files is described. A profile taken before kinds existed reports the scope unavailable. " +
-            "scope:'files' returns the individually described files, paged: page (1-based, default 1) and pageSize (default 20, max 100), " +
+            "scope:'overview' (the default) returns the dataset-level orientation plus the partition accounting — how many " +
+            "files were kept, how many landed in no group, how many were quarantined — and how many members are described " +
+            "individually. That last figure and the dataset's file count differ on purpose: a large dataset is described by " +
+            "its groups, not file by file. " +
+            "scope:'groups' returns the groups with their derived counts, display patterns, and slots, plus the dimensions " +
+            "with their observations, reconciliations, and probe outcomes — this is where a dataset of thousands of files is " +
+            "described. A profile written under the previous model returns its kinds and axes, labelled as such. " +
+            "scope:'files' returns the individually annotated members, paged: page (1-based, default 1) and pageSize (default 20, max 100), " +
             "always with the true total and hasMore, so you can see exactly what you have not read yet. " +
-            "For the paths of files no record describes, list the workspace tree. " +
+            "For the paths of files no record describes, use scope:'groups' for the structure and list the workspace tree for the paths. " +
             "The state field says what you got: 'ready'; 'stale' (a profile is returned but may not describe the current " +
             "inputs — staleReason says why); 'pending' (profiling is still running); 'failed'; or 'absent' (never profiled, " +
             "or the analysis has no input files). " +
@@ -126,19 +119,19 @@ export function createInspectDataProfileTool(pool) {
             "needs re-running rather than diagnosing the data itself.",
         inputSchema: z.object({
             scope: z
-                .enum(["overview", "kinds", "files"])
+                .enum(["overview", "groups", "files"])
                 .optional()
-                .describe("'overview' (default): dataset-level facts, described-file count, and dataset file count. " +
-                "'kinds': the sets the dataset is made of and what varies across them. " +
-                "'files': paged records for the individually described files."),
-            page: z.number().int().min(1).optional().describe("1-based page of per-file records. Only used when scope is 'files'. Default 1."),
+                .describe("'overview' (default): dataset-level facts, the partition accounting, the annotated-member count, and the dataset file count. " +
+                "'groups': the groups the dataset is made of, with their slots, and the dimensions that vary across it. " +
+                "'files': paged records for the individually annotated members."),
+            page: z.number().int().min(1).optional().describe("1-based page of member records. Only used when scope is 'files'. Default 1."),
             pageSize: z
                 .number()
                 .int()
                 .min(1)
                 .max(MAX_PAGE_SIZE)
                 .optional()
-                .describe(`Per-file records per page. Only used when scope is 'files'. Default ${DEFAULT_PAGE_SIZE}, max ${MAX_PAGE_SIZE}.`),
+                .describe(`Member records per page. Only used when scope is 'files'. Default ${DEFAULT_PAGE_SIZE}, max ${MAX_PAGE_SIZE}.`),
         }),
         // `execute` defaults both fields internally, and the hook applies the same
         // defaults. Without them the ordinary call, which names no field at all,
@@ -160,8 +153,8 @@ export function createInspectDataProfileTool(pool) {
                 if (status.status === "failed") {
                     // A failure that DID leave a prior profile on the row never reaches
                     // here — it is served as `stale`, where `stalenessReasons` names the
-                    // failed re-profile and compares the input sets through the single
-                    // shared predicate. This branch is the case with nothing to compare.
+                    // failed re-profile. This branch is the case with no prior profile to
+                    // serve at all.
                     return ok({
                         state: "failed",
                         error: status.error,
@@ -188,14 +181,16 @@ export function createInspectDataProfileTool(pool) {
                 ? { state: "stale", staleReason: reasons.join("; "), profiledAt: result.profiledAt }
                 : { state: "ready", profiledAt: result.profiledAt };
             const scope = input.scope ?? "overview";
+            const describedFiles = profileFileRecords(result);
+            const groups = profileGroups(result);
             if (scope === "overview") {
-                const dataset = datasetFileCount(result);
-                const described = result.files.length;
+                const dataset = profileDatasetFileCount(result);
+                const described = describedFiles.length;
                 const structureNote = dataset === null
                     ? "This profile predates the dataset-structure record, so only the individually described files are known; " +
                         "list the workspace tree for the rest."
                     : dataset !== described
-                        ? `${described} of ${dataset} files are described individually. Call scope:'kinds' for what the rest are, ` +
+                        ? `${described} of ${dataset} files are described individually. Call scope:'groups' for what the rest are, ` +
                             "and list the workspace tree for their paths."
                         : undefined;
                 return ok({
@@ -209,40 +204,63 @@ export function createInspectDataProfileTool(pool) {
                     cellType: result.cellType,
                     condition: result.condition,
                     experimentalDesign: result.experimentalDesign,
-                    qualityAssessment: result.qualityAssessment,
+                    caveats: profileCaveats(result),
                     accessions: result.accessions,
                     describedFileCount: described,
                     datasetFileCount: dataset,
-                    kindCount: result.kinds ? result.kinds.length : null,
+                    // Presence, not truthiness: a completed profile over a tree the scan kept
+                    // nothing of carries `groups: []`, and "zero groups" is a finding — reading
+                    // it as "no structure record" would report the analysis unprofiled.
+                    groupCount: result.groups !== undefined || result.kinds !== undefined ? groups.length : null,
                     ...(structureNote ? { structureNote } : {}),
+                    ...(result.partition ? { partition: result.partition } : {}),
                     ...(result.coverage ? { coverage: result.coverage } : {}),
                 });
             }
-            if (scope === "kinds") {
-                if (!result.kinds) {
+            if (scope === "groups") {
+                if (result.groups !== undefined) {
+                    return ok({
+                        ...envelope,
+                        scope: "groups",
+                        available: true,
+                        groups: rootGroupPatterns(result.groups, resourceId),
+                        ...(result.dimensions ? { dimensions: result.dimensions } : {}),
+                        ...(result.probes ? { probes: result.probes } : {}),
+                        ...(result.partition ? { partition: result.partition } : {}),
+                        ...(result.recipe ? { recipe: result.recipe } : {}),
+                    });
+                }
+                if (result.kinds === undefined) {
                     return ok({
                         ...envelope,
-                        scope: "kinds",
+                        scope: "groups",
                         available: false,
-                        message: "This profile was taken before the dataset-structure record existed, so it carries no kinds. " +
+                        message: "This profile was taken before the dataset-structure record existed, so it carries no groups. " +
                             "That is a fact about the profile, not about the dataset: use scope:'files' and the workspace " +
                             "listing tools, or re-profile the analysis.",
                     });
                 }
+                // The structure exists; only its vocabulary is older. Serving it labelled beats
+                // reporting the scope unavailable, which would tell an agent the dataset has no
+                // structure when the row plainly holds some.
                 return ok({
                     ...envelope,
-                    scope: "kinds",
+                    scope: "groups",
                     available: true,
+                    legacy: true,
                     kinds: rootKindPatterns(result.kinds, resourceId),
                     ...(result.axes ? { axes: result.axes } : {}),
                     ...(result.coverage ? { coverage: result.coverage } : {}),
+                    message: "Authored under the previous model: these are kinds and axes, not resolved groups and dimensions. " +
+                        "Counts and patterns were the profiler's own claims rather than a computed partition, and there is no " +
+                        "unclassified sweep or quarantine accounting on a profile of that era.",
                 });
             }
             const pageSize = input.pageSize ?? DEFAULT_PAGE_SIZE;
             const page = input.page ?? 1;
-            const total = result.files.length;
+            const total = describedFiles.length;
             const start = (page - 1) * pageSize;
-            const files = result.files.slice(start, start + pageSize);
+            const files = describedFiles.slice(start, start + pageSize);
             return ok({
                 ...envelope,
                 scope: "files",
@@ -250,6 +268,7 @@ export function createInspectDataProfileTool(pool) {
                 pageSize,
                 total,
                 hasMore: start + files.length < total,
+                datasetFileCount: profileDatasetFileCount(result),
                 files: rootFilePaths(files, resourceId),
             });
         },

package/dist/tools/research/resolve-citation.js

@@ -4,10 +4,15 @@ import { defineTool } from "../define-tool.js";
 export function createResolveCitationTool(resolver) {
     return defineTool({
         id: "resolve_citation",
-        description: "Verify one supplied citation against applicable bibliographic authorities. " +
-            "Use this for 'does this reference exist and does its metadata agree?', not for topical literature discovery. " +
+        description: "Verify one supplied citation against the applicable bibliographic authorities — the DOI registry, Crossref, PubMed, arXiv and Semantic " +
+            "Scholar — and report where they agree. " +
+            "Use this for 'does this reference exist and does its metadata agree?', not for topical literature discovery: pubmed, " +
+            "search_semantic_scholar and search_arxiv are the discovery tools.\n" +
+            "ACCEPTED IDENTIFIERS: a DOI ('10.1038/nature12373'), a PMID ('23945592'), an arXiv ID ('2301.00001'), or raw bibliographic text. Each " +
+            "authority that the input cannot address is marked not_applicable rather than queried.\n" +
             "Returns source-labelled records, field comparisons, conflicts, coverage, and one of verified, metadata_mismatch, " +
-            "not_found, unverifiable, or inconclusive. Source failure never becomes not_found.",
+            "not_found, unverifiable, or inconclusive. Source failure never becomes not_found — an unreachable authority reports unavailable, and " +
+            "not_found means the authorities that answered hold no such record.",
         inputSchema: CitationInputSchema,
         describeCall: ({ citation }) => `verify ${citation}`,
         execute: async (input, ctx) => ok(await resolver.resolveOne(input, { signal: ctx.signal })),

package/dist/tools/research/search-arxiv.js

@@ -7,9 +7,14 @@ export { parseArxivAtom } from "../../literature/sources/arxiv.js";
 const source = createArxivSource({ maxRetries: 3, retryDelayMs: 1_000, timeoutMs: 90_000 });
 export const searchArxivTool = defineTool({
     id: "search_arxiv",
-    description: "Search arXiv for preprints in ML, physics, math, control theory, " +
-        "economics, and related quantitative fields. Returns id, title, " +
-        "abstract, authors, publication date, arXiv categories, and URLs.",
+    description: "Search arXiv — the open preprint server of Cornell University — for work in machine learning, physics, math, quantitative biology, control " +
+        "theory, economics and the related quantitative fields. Returns id, title, abstract, authors, publication date, arXiv categories, and URLs.\n" +
+        "It carries PREPRINTS, thus a paper here may not be peer reviewed, and it is where a method appears months before a journal holds it. For the " +
+        "clinical and biomedical literature use pubmed, which indexes MEDLINE; for cross-domain coverage of any field use search_semantic_scholar.\n" +
+        "ACCEPTED IDENTIFIERS: free text in `query`, and arXiv category codes in `categories` ('cs.LG', 'q-bio.GN', 'math.OC'). An arXiv ID " +
+        "('2301.00001') also matches as free text.\n" +
+        "ABSENCE IS NORMAL: an empty papers array means arXiv holds nothing for the query, and `success: false` with an `error` string means the service " +
+        "could not be reached. Report either one and continue, do not retry unchanged.",
     inputSchema: z.object({
         query: z.string().describe('Free-text query. Example: "adaptive control feedback stabilization".'),
         categories: z

package/dist/tools/research/search-semantic-scholar.js

@@ -13,10 +13,16 @@ export function createSearchSemanticScholarTool(options = {}) {
     });
     return defineTool({
         id: "search_semantic_scholar",
-        description: "Search Semantic Scholar for academic papers across all sciences " +
-            "(biology, ML, physics, math, economics, engineering, etc.). Returns " +
-            "paper id, title, abstract, year, venue, citation count, authors, and " +
-            "external IDs (DOI, ArXiv). Best for cross-domain analogical search.",
+        description: "Search Semantic Scholar — the cross-disciplinary literature corpus of the Allen Institute for AI, which indexes over 200M papers across " +
+            "biology, medicine, machine learning, physics, math, economics and engineering. Returns paper id, title, abstract, year, venue, citation " +
+            "count, authors, and external IDs (DOI, arXiv). Its citation counts are what rank a field by influence, and its breadth is what makes it the " +
+            "tool for a cross-domain analogical search.\n" +
+            "For the clinical and biomedical literature alone, pubmed indexes MEDLINE more precisely; for preprints in the quantitative fields use " +
+            "search_arxiv.\n" +
+            "ACCEPTED IDENTIFIERS: free text in `query`. A DOI or an arXiv ID also matches as free text, and each result carries its DOI and arXiv ID " +
+            "back for a citation.\n" +
+            "ABSENCE IS NORMAL: an empty papers array means the corpus holds nothing for the query, and `success: false` with an `error` string means the " +
+            "service could not be reached (its unkeyed rate limit lands here). Report either one and continue, do not retry unchanged.",
         inputSchema: z.object({
             query: z
                 .string()

package/dist/tools/workspace/scan-inputs.d.ts

@@ -7,9 +7,9 @@
  * with the observed shapes, or wants a directory the injected manifest summarised, can
  * get the evidence to group differently.
  *
- * Dependency-bearing factory. The walk and the shape observation run in-process over
- * the workspace read seam; only the per-shape header readout reaches into the sandbox
- * (see `input-scan/enrich.ts`).
+ * Dependency-bearing factory. The walk, the shape observation, and every
+ * prefix-sufficient header readout run in-process over the workspace read seam; only a
+ * footer-indexed container reaches into the sandbox (see `input-scan/enrich.ts`).
  */
 import type { SandboxClient } from "../../sandbox/client.js";
 import type { SandboxRef } from "../../sandbox/types.js";
@@ -17,7 +17,7 @@ import type { WorkspaceFilesystem } from "../../workspace/filesystem.js";
 export interface ScanInputsDeps {
     readonly workspaceFs: WorkspaceFilesystem;
     readonly analysisId: string;
-    /** Omit to run the host-side scan with no header readout (no sandbox wired). */
+    /** Omit to skip the container readouts; the prefix readouts run regardless. */
     readonly sandboxClient?: SandboxClient;
     readonly sandbox?: SandboxRef;
     readonly workflowId: string;

package/dist/tools/workspace/scan-inputs.js

@@ -7,9 +7,9 @@
  * with the observed shapes, or wants a directory the injected manifest summarised, can
  * get the evidence to group differently.
  *
- * Dependency-bearing factory. The walk and the shape observation run in-process over
- * the workspace read seam; only the per-shape header readout reaches into the sandbox
- * (see `input-scan/enrich.ts`).
+ * Dependency-bearing factory. The walk, the shape observation, and every
+ * prefix-sufficient header readout run in-process over the workspace read seam; only a
+ * footer-indexed container reaches into the sandbox (see `input-scan/enrich.ts`).
  */
 import { ok } from "neverthrow";
 import { z } from "zod";
@@ -38,17 +38,16 @@ export function createScanInputsTool(deps) {
                 root: path.replace(/^\/+/, "").replace(/\/+$/, ""),
                 signal: ctx.signal,
             });
-            const shapes = deps.sandboxClient && deps.sandbox
-                ? await enrichShapes({
-                    shapes: scan.manifest.shapes,
-                    sandboxClient: deps.sandboxClient,
-                    sandbox: deps.sandbox,
-                    mountRoot: `/${deps.analysisId}`,
-                    execId: `${deps.workflowId}:${deps.stepId}:${deps.nextFunctionId()}`,
-                    deadlineMs: deps.deadlineMs(),
-                    emit: ctx.emit,
-                })
-                : scan.manifest.shapes;
+            const shapes = await enrichShapes({
+                shapes: scan.manifest.shapes,
+                session: ctx.session,
+                fs: deps.workspaceFs,
+                ...(deps.sandboxClient && deps.sandbox ? { sandboxClient: deps.sandboxClient, sandbox: deps.sandbox } : {}),
+                mountRoot: `/${deps.analysisId}`,
+                execId: `${deps.workflowId}:${deps.stepId}:${deps.nextFunctionId()}`,
+                deadlineMs: deps.deadlineMs(),
+                emit: ctx.emit,
+            });
             const manifest = { ...scan.manifest, shapes };
             return ok({
                 path: manifest.root,

package/dist/tools/workspace/workspace-search.d.ts

@@ -12,14 +12,23 @@
  */
 import type { Pool } from "pg";
 import type { EmbeddingProvider } from "../../providers/types.js";
+/**
+ * The workspace path a hit addresses, or `undefined` for a pattern entry. Resolved here,
+ * once, across every index era — group/dimension/kind entries and previous-era entity
+ * entries (`entity` in metadata) are patterns; everything else is a file whose path is
+ * stamped in metadata or, on rows older than the stamp, is the id minus the resource
+ * segment. The consumer branches on presence, nothing else.
+ */
+export declare function fileTargetOf(id: string, metadata: unknown): string | undefined;
 export declare function createWorkspaceSearchTool(pool: Pool, embedding: EmbeddingProvider): import("../define-tool.js").Tool<{
     query: string;
     limit: number;
-    type?: "output" | "input" | "synthesis" | "summary" | "input-kind" | undefined;
+    type?: "output" | "input" | "synthesis" | "summary" | "input-group" | "input-dimension" | "input-kind" | undefined;
 }, {
     results: {
+        metadata: unknown;
+        path?: string | undefined;
         id: string;
         score: number;
-        metadata: unknown;
     }[];
 }>;