@inflexa-ai/harness 0.24.0 → 0.25.0

package/dist/tasks/data-profile-activity.d.ts

@@ -50,6 +50,14 @@ export type ActivityWrite = (part: unknown) => Promise<void>;
 export interface ProfileActivityEmitter {
     /** Reported before the sandbox is created — the longest wait in a profile, and the one that precedes the agent entirely. */
     sandboxInit(): Promise<void>;
+    /**
+     * Reported before the deterministic input scan, which runs between a ready sandbox
+     * and the agent's first turn. On a large tree it is the second-longest operation in
+     * a profile, and left unreported it would read as `Running data-profiler` for
+     * minutes before the agent had begun — the misreport `sandbox-init` exists to
+     * prevent.
+     */
+    scanning(): Promise<void>;
     /** Reported once the sandbox is ready, covering the gap before the agent's first tool call. */
     agentStarting(): Promise<void>;
     /**

package/dist/tasks/data-profile-activity.js

@@ -47,6 +47,7 @@ export function createProfileActivityEmitter(write, frame, logger) {
     };
     return {
         sandboxInit: () => emit("sandbox-init", "Starting sandbox"),
+        scanning: () => emit("executing", "Scanning input files"),
         agentStarting: () => emit("executing", "Running data-profiler"),
         forTool: (name, input, resolveDetail) => emit("executing", activityForTool(name, input, resolveDetail)),
         indexing: () => emit("indexing", "Indexing input descriptions for search"),

package/dist/tasks/data-profile-index.d.ts

@@ -0,0 +1,45 @@
+/**
+ * The profile's vector index, as a pure projection.
+ *
+ * The index is a function of the scan crossed with the submitted kinds: for each kind,
+ * one entry; for each entity, one entry templated from the kinds it participates in and
+ * its own axis value. It has no persisted representation of its own and nothing derives
+ * from it, so it can be rebuilt at any time by re-running the scan and reading `kinds`
+ * off the existing profile — no model call, no re-profile. Improving the template,
+ * adding a tier, or folding in entity attributes later are all re-projections.
+ *
+ * Two tiers, because they answer different queries. A query naming a KIND of data
+ * ("variant calls") wants one result carrying the set's count and path pattern; a
+ * per-entity tier would return five arbitrary subjects out of thousands of
+ * indistinguishable ones. A query naming an ENTITY ("PT0421") wants that entity. A
+ * per-file tier adds nothing: a file is identified by its kind and its entity, and its
+ * path is on the filesystem.
+ *
+ * `type: "input"` keeps its existing meaning for the entity tier, so searches written
+ * against it keep matching; `type: "input-kind"` is additive. Renaming `"input"` would
+ * have silently broken existing search on every new analysis.
+ */
+import type { InputScan } from "../input-scan/types.js";
+import type { DataProfileAxis, DataProfileFile, DataProfileKind } from "../state/data-profile.js";
+/** Entity entries built per analysis. Past this the entity tier is a liability, not a search aid. */
+export declare const MAX_ENTITY_ENTRIES = 20000;
+export interface ProfileIndexEntry {
+    readonly id: string;
+    readonly text: string;
+    readonly metadata: Record<string, unknown>;
+}
+export interface BuildProfileIndexArgs {
+    readonly analysisId: string;
+    readonly kinds: readonly DataProfileKind[];
+    readonly axes?: readonly DataProfileAxis[];
+    /** The scan the profile was taken against — the source of the entity value set. */
+    readonly scan?: InputScan;
+    /** Individually described files, indexed only when there are no kinds to index. */
+    readonly files?: readonly DataProfileFile[];
+}
+/**
+ * Build every index entry for a completed profile. Deterministic and model-free: the
+ * text is templated from the kind's own description and the entity's axis value, so
+ * indexing costs no LLM tokens whatever the dataset's size.
+ */
+export declare function buildProfileIndexEntries(args: BuildProfileIndexArgs): ProfileIndexEntry[];

package/dist/tasks/data-profile-index.js

@@ -0,0 +1,134 @@
+/**
+ * The profile's vector index, as a pure projection.
+ *
+ * The index is a function of the scan crossed with the submitted kinds: for each kind,
+ * one entry; for each entity, one entry templated from the kinds it participates in and
+ * its own axis value. It has no persisted representation of its own and nothing derives
+ * from it, so it can be rebuilt at any time by re-running the scan and reading `kinds`
+ * off the existing profile — no model call, no re-profile. Improving the template,
+ * adding a tier, or folding in entity attributes later are all re-projections.
+ *
+ * Two tiers, because they answer different queries. A query naming a KIND of data
+ * ("variant calls") wants one result carrying the set's count and path pattern; a
+ * per-entity tier would return five arbitrary subjects out of thousands of
+ * indistinguishable ones. A query naming an ENTITY ("PT0421") wants that entity. A
+ * per-file tier adds nothing: a file is identified by its kind and its entity, and its
+ * path is on the filesystem.
+ *
+ * `type: "input"` keeps its existing meaning for the entity tier, so searches written
+ * against it keep matching; `type: "input-kind"` is additive. Renaming `"input"` would
+ * have silently broken existing search on every new analysis.
+ */
+/** Entity entries built per analysis. Past this the entity tier is a liability, not a search aid. */
+export const MAX_ENTITY_ENTRIES = 20_000;
+/** Kind names named in one entity's entry text. */
+const MAX_KINDS_IN_ENTITY_TEXT = 6;
+function slug(value) {
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-|-$/g, "")
+        .slice(0, 60);
+}
+/**
+ * The entity value set: the largest distinct-value set any single variable position
+ * carries.
+ *
+ * Entities are what a dataset is ABOUT, and the position that varies most widely across
+ * one shape is what identifies them — 1171 subjects, against three timepoints and two
+ * replicates. Taking the widest position rather than the cross product is deliberate:
+ * the cross product is the file set, which is the tier this projection exists to avoid.
+ */
+function entityValues(scan) {
+    if (!scan)
+        return { values: [], distinct: 0 };
+    let best = [];
+    for (const shape of scan.manifest.shapes) {
+        const positions = scan.positionValues.get(shape.id);
+        if (!positions)
+            continue;
+        for (const position of shape.variablePositions) {
+            const values = positions[position.index];
+            if (values && values.length > best.length)
+                best = values;
+        }
+    }
+    return { values: [...best].sort((a, b) => a.localeCompare(b, "en")), distinct: best.length };
+}
+/**
+ * The axis label for an entity set. The agent labels axes; the scan counts values, so
+ * the axis whose cardinality matches the observed set is the one the agent was naming.
+ * With no match the entries still index under a neutral label rather than not at all.
+ */
+function entityAxisLabel(axes, distinct) {
+    if (!axes || axes.length === 0)
+        return "entity";
+    const exact = axes.find((axis) => axis.cardinality === distinct);
+    if (exact)
+        return exact.label;
+    const widest = [...axes].sort((a, b) => b.cardinality - a.cardinality)[0];
+    return widest?.label ?? "entity";
+}
+function kindEntry(analysisId, kind) {
+    const facts = [`${kind.count} files`, kind.format, kind.pathPattern].filter((f) => Boolean(f)).join(", ");
+    return {
+        id: `/${analysisId}/kind/${slug(kind.name)}`,
+        text: `${kind.name} — ${kind.description} One member is ${kind.memberRepresents}. (${facts})`,
+        metadata: {
+            type: "input-kind",
+            kind: kind.name,
+            pathPattern: kind.pathPattern,
+            count: kind.count,
+            ...(kind.format ? { format: kind.format } : {}),
+            ...(kind.axisLabels ? { axisLabels: kind.axisLabels } : {}),
+        },
+    };
+}
+function entityEntry(analysisId, label, value, kinds) {
+    const named = kinds.slice(0, MAX_KINDS_IN_ENTITY_TEXT);
+    const carried = named.map((kind) => `${kind.name} (${kind.memberRepresents})`).join("; ");
+    return {
+        id: `/${analysisId}/entity/${encodeURIComponent(value)}`,
+        text: `${label} ${value} — input data for this ${label}: ${carried}.`,
+        metadata: {
+            type: "input",
+            entity: value,
+            axis: label,
+            kinds: named.map((kind) => kind.name),
+        },
+    };
+}
+/** A file entry, for the degenerate case where a profile submitted no kinds at all. */
+function fileEntry(analysisId, file) {
+    return {
+        id: `/${analysisId}/${file.path}`,
+        text: `${file.path} — ${file.description}`,
+        metadata: {
+            type: "input",
+            ...(file.dataType ? { dataType: file.dataType } : {}),
+            ...(file.format ? { format: file.format } : {}),
+            ...(file.tags ? { tags: file.tags } : {}),
+        },
+    };
+}
+/**
+ * Build every index entry for a completed profile. Deterministic and model-free: the
+ * text is templated from the kind's own description and the entity's axis value, so
+ * indexing costs no LLM tokens whatever the dataset's size.
+ */
+export function buildProfileIndexEntries(args) {
+    const { analysisId, kinds, axes, scan, files } = args;
+    if (kinds.length === 0) {
+        // No kinds is a degenerate profile, but the notable files it did describe stay
+        // discoverable — the same losslessness the per-file fallback description had.
+        return (files ?? []).map((file) => fileEntry(analysisId, file));
+    }
+    const entries = kinds.map((kind) => kindEntry(analysisId, kind));
+    const { values, distinct } = entityValues(scan);
+    if (values.length > 1 && values.length <= MAX_ENTITY_ENTRIES) {
+        const label = entityAxisLabel(axes, distinct);
+        for (const value of values)
+            entries.push(entityEntry(analysisId, label, value, kinds));
+    }
+    return entries;
+}

package/dist/tasks/data-profile.d.ts

@@ -1,9 +1,10 @@
 /**
- * Data-profile DBOS workflow — profiles the already-staged input files, runs
- * the data-profiler sandbox agent, writes per-file metadata to
- * cortex_artifacts, and indexes descriptions into the pgvector store. Inputs
- * are staged under `data/inputs/` by the embedder before the run (see the data-profile-init spec);
- * the body assumes a populated tree and never downloads.
+ * Data-profile DBOS workflow — scans the already-staged input tree, runs the
+ * data-profiler sandbox agent over the resulting manifest, registers each staged
+ * file in cortex_artifacts, and indexes the profile's kinds and entities into the
+ * pgvector store. Inputs are staged under `data/inputs/` by the embedder before the
+ * run (see the data-profile-init spec); the body assumes a populated tree and never
+ * downloads.
  *
  * Recoverable: a crashed Cortex pod resumes the workflow from the DBOS step
  * cache. The run authorization is minted at the async edge (`triggerDataProfile`)
@@ -29,7 +30,7 @@ import type { SandboxClient } from "../sandbox/client.js";
 import type { WorkspaceFilesystem } from "../workspace/filesystem.js";
 import { type ResolveWorkspaceRoot } from "../workspace/paths.js";
 import { type ProfilerOutput } from "../schemas/data-profile-schemas.js";
-import { type DataProfileInputFile, type DataProfileResult } from "../state/index.js";
+import { type DataProfileCoverage, type DataProfileResult } from "../state/index.js";
 /** The body's construction-time deps — closed over at registration. */
 export interface DataProfileDeps extends EnvironmentStorePaths {
     /** Operational logging seam; omitted falls back to no-op. */
@@ -92,22 +93,6 @@ export interface DataProfileWorkflowInput {
      */
     readonly ownsMandate?: boolean;
 }
-/**
- * Build the drift-signature comparand a completed profile persists, or `undefined`
- * to omit it. `inputFiles` records whether the profiled files still hold the same
- * bytes; it sits beside the `inputFileIds` audit record (WHICH files) in the stored
- * `DataProfileResult`.
- *
- * A workflow input persisted before `StagedInput.mtimeMs` existed (recovered across
- * the deploy that added it) carries entries with NO `mtimeMs` despite the compile-time
- * type — the same recovered-legacy-input case `ownsMandate` handles. A per-entry
- * `{ …, mtimeMs: undefined }` would JSON.stringify to `{ fileId, size }` and violate
- * `DataProfileInputFile`, so the WHOLE signature is omitted when ANY entry lacks
- * `mtimeMs`: an absent comparand reads back as drift and buys one clean re-profile,
- * exactly as a wholly-absent `result` already does (see `DataProfileResult.inputFiles`).
- * The cast defeats the required-`number` type precisely to observe that legacy gap.
- */
-export declare function buildDriftSignature(stagedInputs: readonly StagedInput[]): DataProfileInputFile[] | undefined;
 /**
  * Project the profiler's structured output plus the staged manifest into the record
  * persisted on `cortex_analysis_state`.
@@ -124,7 +109,7 @@ export declare function buildDriftSignature(stagedInputs: readonly StagedInput[]
  * indistinguishable from a legacy snapshot that predates the field, which is exactly
  * the reading a consumer must already tolerate.
  */
-export declare function buildDataProfileResult(profile: ProfilerOutput, stagedInputs: readonly StagedInput[], profiledAt: string): DataProfileResult;
+export declare function buildDataProfileResult(profile: ProfilerOutput, stagedInputs: readonly StagedInput[], profiledAt: string, coverage?: DataProfileCoverage): DataProfileResult;
 /**
  * Register the data-profile workflow with DBOS. Returns the registered
  * callable so `triggerDataProfile` can dispatch via `DBOS.startWorkflow`.

package/dist/tasks/data-profile.js

@@ -1,9 +1,10 @@
 /**
- * Data-profile DBOS workflow — profiles the already-staged input files, runs
- * the data-profiler sandbox agent, writes per-file metadata to
- * cortex_artifacts, and indexes descriptions into the pgvector store. Inputs
- * are staged under `data/inputs/` by the embedder before the run (see the data-profile-init spec);
- * the body assumes a populated tree and never downloads.
+ * Data-profile DBOS workflow — scans the already-staged input tree, runs the
+ * data-profiler sandbox agent over the resulting manifest, registers each staged
+ * file in cortex_artifacts, and indexes the profile's kinds and entities into the
+ * pgvector store. Inputs are staged under `data/inputs/` by the embedder before the
+ * run (see the data-profile-init spec); the body assumes a populated tree and never
+ * downloads.
  *
  * Recoverable: a crashed Cortex pod resumes the workflow from the DBOS step
  * cache. The run authorization is minted at the async edge (`triggerDataProfile`)
@@ -34,6 +35,11 @@ import { generateExecutionId } from "../sandbox/execution-id.js";
 import { mintSandboxIdentity } from "../sandbox/identity.js";
 import { createVectorStore } from "../state/vector-store.js";
 import { ProfilerOutputSchema } from "../schemas/data-profile-schemas.js";
+import { computeInputSignature } from "../execution/input-signature.js";
+import { computeCoverage } from "../input-scan/coverage.js";
+import { enrichShapes } from "../input-scan/enrich.js";
+import { renderInputScanManifest, scanInputTree } from "../input-scan/scan.js";
+import { buildProfileIndexEntries } from "./data-profile-index.js";
 import { completeDataProfile, failDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, tryRerunDataProfile, tryStartDataProfile, upsertArtifacts, } from "../state/index.js";
 import { createProfileActivityEmitter } from "./data-profile-activity.js";
 import { ensureSearchIndex, searchIndexName } from "../workspace/search-config.js";
@@ -42,39 +48,20 @@ import { ensureSearchIndex, searchIndexName } from "../workspace/search-config.j
 import { DATA_PROFILE_RUN_LITERAL } from "../contracts/data-profile.js";
 const DATA_PROFILE_STEP_LITERAL = "profile";
 const DATA_PROFILE_AGENT_ID = "data-profiler";
-/** Sandbox-server exec budget for the profile run. */
-const DEFAULT_DEADLINE_MS = 300_000;
+/**
+ * Sandbox-server exec budget for the profile run. Twenty minutes: the deterministic
+ * scan bounds the discovery half, and what remains is the agent's own reading of a
+ * bounded number of example files.
+ */
+const DEFAULT_DEADLINE_MS = 1_200_000;
+/** Index entries embedded and upserted per round trip. */
+const INDEX_BATCH_SIZE = 256;
+/** Where the embedder stages this analysis's inputs (see the data-profile-init spec). */
+const STAGED_INPUT_ROOT = "data/inputs";
 /** Build the canonical artifact path for a staged input file. */
 function inputArtifactPath(f) {
     return `data/${f.relativePath}`;
 }
-/** Lowercased extension of a path, or `"unknown"` when none. */
-function fileExtension(p) {
-    const base = p.slice(p.lastIndexOf("/") + 1);
-    const dot = base.lastIndexOf(".");
-    return dot > 0 ? base.slice(dot + 1).toLowerCase() : "unknown";
-}
-/**
- * Build the drift-signature comparand a completed profile persists, or `undefined`
- * to omit it. `inputFiles` records whether the profiled files still hold the same
- * bytes; it sits beside the `inputFileIds` audit record (WHICH files) in the stored
- * `DataProfileResult`.
- *
- * A workflow input persisted before `StagedInput.mtimeMs` existed (recovered across
- * the deploy that added it) carries entries with NO `mtimeMs` despite the compile-time
- * type — the same recovered-legacy-input case `ownsMandate` handles. A per-entry
- * `{ …, mtimeMs: undefined }` would JSON.stringify to `{ fileId, size }` and violate
- * `DataProfileInputFile`, so the WHOLE signature is omitted when ANY entry lacks
- * `mtimeMs`: an absent comparand reads back as drift and buys one clean re-profile,
- * exactly as a wholly-absent `result` already does (see `DataProfileResult.inputFiles`).
- * The cast defeats the required-`number` type precisely to observe that legacy gap.
- */
-export function buildDriftSignature(stagedInputs) {
-    const mtimeOf = (f) => f.mtimeMs;
-    if (!stagedInputs.every((f) => mtimeOf(f) !== undefined))
-        return undefined;
-    return stagedInputs.map((f) => ({ fileId: f.fileId, size: f.size, mtimeMs: mtimeOf(f) }));
-}
 /**
  * Project the profiler's structured output plus the staged manifest into the record
  * persisted on `cortex_analysis_state`.
@@ -91,7 +78,7 @@ export function buildDriftSignature(stagedInputs) {
  * indistinguishable from a legacy snapshot that predates the field, which is exactly
  * the reading a consumer must already tolerate.
  */
-export function buildDataProfileResult(profile, stagedInputs, profiledAt) {
+export function buildDataProfileResult(profile, stagedInputs, profiledAt, coverage) {
     return {
         summary: profile.analysisSummary,
         files: profile.files.map((f) => ({
@@ -105,8 +92,23 @@ export function buildDataProfileResult(profile, stagedInputs, profiledAt) {
             warnings: f.warnings,
             metrics: f.metrics,
         })),
-        inputFileIds: stagedInputs.map((f) => f.fileId),
-        inputFiles: buildDriftSignature(stagedInputs),
+        kinds: profile.kinds.map((k) => ({
+            name: k.name,
+            memberRepresents: k.memberRepresents,
+            description: k.description,
+            count: k.count,
+            pathPattern: k.pathPattern,
+            format: k.format,
+            axisLabels: k.axisLabels,
+        })),
+        axes: profile.axes?.map((a) => ({
+            label: a.label,
+            cardinality: a.cardinality,
+            exampleValues: a.exampleValues,
+            description: a.description,
+        })),
+        inputSignature: computeInputSignature(stagedInputs),
+        coverage,
         profiledAt,
         domain: profile.domain,
         subtype: profile.subtype,
@@ -119,6 +121,37 @@ export function buildDataProfileResult(profile, stagedInputs, profiledAt) {
         qualityAssessment: profile.qualityAssessment,
     };
 }
+/**
+ * Walk the staged tree and enrich its shapes with a header readout.
+ *
+ * The walk and the shape observation run in this process over the workspace read seam;
+ * only the readout reaches into the sandbox, and only once per shape. A readout that
+ * fails is logged and dropped: it is enrichment, and a manifest without it still
+ * carries every structural observation the agent's grouping rests on, so failing the
+ * profile over it would trade the whole capability for a nicety.
+ */
+async function runInputScan(args) {
+    const { session, deps, analysisId, sandbox, execId, deadlineMs, logger } = args;
+    const scan = await scanInputTree({ session, fs: deps.workspaceFs, root: STAGED_INPUT_ROOT });
+    if (scan.manifest.shapes.length === 0)
+        return scan;
+    try {
+        const shapes = await enrichShapes({
+            shapes: scan.manifest.shapes,
+            sandboxClient: deps.sandboxClient,
+            sandbox,
+            mountRoot: `/${analysisId}`,
+            execId,
+            deadlineMs,
+            emit: async () => { },
+        });
+        return { ...scan, manifest: { ...scan.manifest, shapes } };
+    }
+    catch (err) {
+        logger.warn("input-scan header readout failed (non-fatal)", logger.errorFields(err));
+        return scan;
+    }
+}
 /**
  * Register the data-profile workflow with DBOS. Returns the registered
  * callable so `triggerDataProfile` can dispatch via `DBOS.startWorkflow`.
@@ -192,34 +225,6 @@ export async function runDataProfileBody(input, deps) {
         // meaningless, so no softer handling is warranted.
         const workspaceRoot = deps.resolveWorkspaceRoot(analysisId);
         const profileWritePrefix = `${workspaceRoot}/runs/${DATA_PROFILE_RUN_LITERAL}/${DATA_PROFILE_STEP_LITERAL}`;
-        const fileList = stagedInputs.map((f) => inputArtifactPath(f));
-        // The profiler runs outside `executeAnalysis`, so no scheduler composes a
-        // briefing for it — this prompt IS its briefing, and the sandbox system
-        // prompt names no paths (it is static per agent type, for the prompt cache).
-        // Its workspace frame therefore has to be stated here, in the same words the
-        // step briefing uses.
-        const prompt = [
-            `Profile all input data files for this analysis.`,
-            ``,
-            renderWorkspace({
-                analysisRoot: `/${analysisId}`,
-                workingDir: toSandboxPath(workspaceRoot, analysisId, profileWritePrefix),
-            }),
-            ``,
-            `IMPORTANT: You MUST use execute_command to profile files before submitting results. Do NOT submit empty files or placeholder text.`,
-            ``,
-            `CRITICAL: File and directory paths may contain spaces. You MUST always double-quote paths in shell commands (e.g. head "/${analysisId}/data/inputs/My Folder/file.csv"). Unquoted paths with spaces will silently break commands.`,
-            ``,
-            `The following input files are available (paths relative to the analysis root):`,
-            ...fileList.map((f) => `- ${f}`),
-            ``,
-            `Steps:`,
-            `1. For each file, use head and wc to inspect structure (ALWAYS double-quote file paths)`,
-            `2. Write a Python script for detailed profiling — use pathlib or os.path for path handling (handles spaces natively)`,
-            `3. Call the \`submit_profile\` tool with structured metadata for EVERY file listed above`,
-            ``,
-            `All file paths in your response must be relative to the analysis root and must EXACTLY match the paths listed above.`,
-        ].join("\n");
         logger.info("starting sandbox", { executionId });
         // Reported BEFORE the container exists, deliberately: provisioning is the longest single
         // operation in a profile and it precedes the agent entirely, so a body that reported only
@@ -255,6 +260,48 @@ export async function runDataProfileBody(input, deps) {
             // which captures its step deadline the same way).
             const deadlineAbs = (await DBOS.now()) + DEFAULT_DEADLINE_MS;
             const nextFunctionId = makeNextFunctionId();
+            // The deterministic scan, before the agent's first turn. It is always needed
+            // and its result does not depend on agent judgement, so spending an agent turn
+            // to request it would be waste — and a briefing carrying one line per input
+            // file consumes context that carries no structure (~270 KB of bare paths for
+            // the tree that motivated this, against ~2 KB of shapes).
+            await activity.scanning();
+            const scan = await runInputScan({
+                session: childSession,
+                deps,
+                analysisId,
+                sandbox,
+                execId: `${workflowId}:${DATA_PROFILE_STEP_LITERAL}:${nextFunctionId()}`,
+                deadlineMs: deadlineAbs,
+                logger,
+            });
+            logger.info("input scan complete", {
+                files: scan.manifest.fileCount,
+                shapes: scan.manifest.shapes.length,
+                unstructured: scan.manifest.unstructured.count,
+                truncated: scan.manifest.truncated,
+            });
+            // The profiler runs outside `executeAnalysis`, so no scheduler composes a
+            // briefing for it — this prompt IS its briefing, and the sandbox system
+            // prompt names no paths (it is static per agent type, for the prompt cache).
+            // Its workspace frame therefore has to be stated here, in the same words the
+            // step briefing uses.
+            const prompt = [
+                `Profile the input data for this analysis: say what the dataset IS, so planning can proceed.`,
+                ``,
+                renderWorkspace({
+                    analysisRoot: `/${analysisId}`,
+                    workingDir: toSandboxPath(workspaceRoot, analysisId, profileWritePrefix),
+                }),
+                ``,
+                `CRITICAL: File and directory paths may contain spaces. You MUST always double-quote paths in shell commands (e.g. head "/${analysisId}/data/inputs/My Folder/file.csv"). Unquoted paths with spaces will silently break commands.`,
+                ``,
+                renderInputScanManifest(scan.manifest),
+                ``,
+                `Decide the dataset's kinds from these observations — you may split a shape, span several shapes, or`,
+                `label an axis the scan did not observe. Inspect ONE example file per kind where a description needs`,
+                `content the scan did not capture, then call \`submit_profile\` once.`,
+            ].join("\n");
             const sandboxAgentDeps = {
                 provider: deps.provider,
                 pool: deps.pool,
@@ -322,78 +369,64 @@ export async function runDataProfileBody(input, deps) {
             }, {
                 tools: [submitProfileTool],
                 nudge: "You stopped without calling submit_profile, so no profile was " +
-                    "recorded. Call submit_profile now with structured metadata for " +
-                    "every input file — base it on the profiling you already did.",
+                    "recorded. Call submit_profile now with the kinds this dataset is " +
+                    "made of, the axes that vary across them, and any individually " +
+                    "notable files — base it on the scan and the work you already did.",
             });
             if (!capturedProfile) {
                 throw new Error("Data profiling failed: agent did not call submit_profile");
             }
             const profilerData = capturedProfile;
-            // 4. Index into vector store — real files only, consistent type metadata
+            // 4. Index into vector store — one entry per kind, one per entity, none per file.
             //
             // Reported as `indexing`, not `persisting`: the contract defines `persisting` as step
             // bytes uploading to an artifact store, and a profile uploads nothing — its durable
             // products are this vector index and the ledger row.
             await activity.indexing();
-            const profilerByPath = new Map(profilerData.files.map((f) => [f.path, f]));
-            const normPath = (p) => p
-                .replace(/\/+/g, "/")
-                .replace(/^\/|\/$/g, "")
-                .trim();
-            const profilerByNorm = new Map(profilerData.files.map((f) => [normPath(f.path), f]));
+            const profileRecord = buildDataProfileResult(profilerData, stagedInputs, new Date().toISOString(), 
+            // Coverage measures the PROFILE, not the scan: the agent's own patterns are
+            // matched against the scanned tree, so a profile whose kinds leave most of
+            // the tree unmatched is visibly incomplete instead of reading as fresh.
+            computeCoverage(scan.files.map((f) => f.path), profilerData.kinds.map((kind) => kind.pathPattern)));
             await ensureSearchIndex(deps.pool, analysisId, deps.embedding.dimensions);
             const vectorStore = createVectorStore(deps.pool);
-            const embedOne = async (text, session) => {
-                const [vec] = unwrapOrThrow(await deps.embedding.embed([text], session));
-                if (!vec)
-                    throw new Error("data-profile: empty embedding response");
-                return vec;
-            };
             const indexName = searchIndexName(analysisId);
-            let indexed = 0;
-            let fallbackCount = 0;
-            for (const matFile of stagedInputs) {
-                const dbPath = inputArtifactPath(matFile);
-                const desc = profilerByPath.get(dbPath) ?? profilerByNorm.get(normPath(dbPath));
-                // Lossless: a materialized input file the profiler agent omitted
-                // still gets a deterministic, no-LLM description so it stays
-                // discoverable via search. The agent's profile is enrichment, not a
-                // gate on indexing — mirrors the step-output metadata fallback.
-                const searchMeta = desc
-                    ? {
-                        text: desc.description,
-                        type: "input",
-                        dataType: desc.dataType,
-                        format: desc.format,
-                        ...(desc.tags ? { tags: desc.tags } : {}),
-                    }
-                    : {
-                        text: `${dbPath} — input file (${fileExtension(dbPath)}, ${matFile.size} bytes); automated profile unavailable.`,
-                        type: "input",
-                        dataType: "unknown",
-                        format: fileExtension(dbPath),
-                    };
-                if (!desc)
-                    fallbackCount++;
-                const embedding = await embedOne(searchMeta.text, runSession);
+            const entries = buildProfileIndexEntries({
+                analysisId,
+                kinds: profileRecord.kinds ?? [],
+                ...(profileRecord.axes ? { axes: profileRecord.axes } : {}),
+                scan,
+                files: profileRecord.files,
+            });
+            // Batched: both interfaces already take arrays, and one round trip per entry is
+            // what made indexing 12 of the 39 minutes the motivating profile took.
+            for (let start = 0; start < entries.length; start += INDEX_BATCH_SIZE) {
+                const batch = entries.slice(start, start + INDEX_BATCH_SIZE);
+                const vectors = unwrapOrThrow(await deps.embedding.embed(batch.map((entry) => entry.text), runSession));
+                if (vectors.length !== batch.length)
+                    throw new Error(`data-profile: embedding returned ${vectors.length} vectors for ${batch.length} entries`);
                 unwrapOrThrow(await vectorStore.upsert({
                     indexName,
-                    vectors: [embedding],
-                    metadata: [searchMeta],
-                    ids: [`/${analysisId}/${dbPath}`],
+                    vectors,
+                    metadata: batch.map((entry) => ({ ...entry.metadata, text: entry.text })),
+                    ids: batch.map((entry) => entry.id),
                 }));
-                indexed++;
             }
-            if (fallbackCount > 0) {
-                logger.warn("input file(s) used a deterministic fallback description — profiler omitted them", {
-                    fallbackCount,
-                    stagedCount: stagedInputs.length,
+            logger.info("indexed profile", {
+                entries: entries.length,
+                kinds: profileRecord.kinds?.length ?? 0,
+                stagedCount: stagedInputs.length,
+            });
+            if (profileRecord.coverage && profileRecord.coverage.unmatched > 0) {
+                logger.warn("profile kinds do not cover the scanned tree", {
+                    matched: profileRecord.coverage.matched,
+                    unmatched: profileRecord.coverage.unmatched,
+                    total: profileRecord.coverage.total,
                 });
             }
-            logger.info("indexed file(s)", { indexed });
-            // 5. Complete — store the FULL profiler finding plus the input snapshot for
+            // 5. Complete — store the FULL profiler finding plus the input signature for
             // staleness detection. The scratch tree is gone; this row is all that survives.
-            if (!unwrapOrThrow(await completeDataProfile(deps.pool, analysisId, buildDataProfileResult(profilerData, stagedInputs, new Date().toISOString())))) {
+            if (!unwrapOrThrow(await completeDataProfile(deps.pool, analysisId, profileRecord))) {
                 logTerminalNoop(logger, analysisId, "completion");
             }
             await deps.runAuthorizer.revoke(authorization, "data-profile-completed");

package/dist/tools/research/generate-plan.js

@@ -182,7 +182,7 @@ function classifyGrounding(status) {
     // preserve `data_profile_result` on purpose, so a non-`completed` status
     // carrying a result means what is on the row is the PREVIOUS profile.
     const reasons = [];
-    if (isDataProfileStale(status.seedInputFileIds ?? [], result.inputFileIds)) {
+    if (isDataProfileStale({ fileIds: status.seedInputFileIds ?? [] }, result)) {
         reasons.push("the analysis's input file set changed after this profile was taken");
     }
     if (status.status === "pending" || status.status === "running") {