@inflexa-ai/harness 0.28.1 → 0.29.0

package/dist/tasks/data-profile.js

@@ -1,8 +1,8 @@
 /**
  * Data-profile DBOS workflow — scans the already-staged input tree, runs the
  * data-profiler sandbox agent over the resulting manifest, registers each staged
- * file in cortex_artifacts, and indexes the profile's kinds and entities into the
- * pgvector store. Inputs are staged under `data/inputs/` by the embedder before the
+ * file in cortex_artifacts, and indexes the profile's groups, dimensions, and annotated
+ * members into the pgvector store. Inputs are staged under `data/inputs/` by the embedder before the
  * run (see the data-profile-init spec); the body assumes a populated tree and never
  * downloads.
  *
@@ -34,12 +34,16 @@ import { estimateDataProfileResources } from "../sandbox/estimate-data-profile-r
 import { generateExecutionId } from "../sandbox/execution-id.js";
 import { mintSandboxIdentity } from "../sandbox/identity.js";
 import { createVectorStore } from "../state/vector-store.js";
-import { ProfilerOutputSchema } from "../schemas/data-profile-schemas.js";
+import { ProfileSubmissionSchema } from "../schemas/data-profile-schemas.js";
 import { computeInputSignature } from "../execution/input-signature.js";
-import { computeCoverage } from "../input-scan/coverage.js";
-import { enrichShapes } from "../input-scan/enrich.js";
-import { renderInputScanManifest, scanInputTree } from "../input-scan/scan.js";
-import { buildProfileIndexEntries } from "./data-profile-index.js";
+import { readHeaders } from "../input-scan/enrich.js";
+import { detectSets } from "../input-scan/detect-sets.js";
+import { buildSetMenu, renderSetMenu } from "../input-scan/menu.js";
+import { readoutTargets } from "../input-scan/readout-budget.js";
+import { scanInputTree } from "../input-scan/scan.js";
+import { PROFILE_INDEX_TYPES, buildProfileIndexEntries } from "./data-profile-index.js";
+import { absorbRecipe, renderAbsorbDelta } from "./data-profile-absorb.js";
+import { formatResolutionErrors, resolveProfileSubmission } from "./data-profile-resolve.js";
 import { completeDataProfile, failDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, tryRerunDataProfile, tryStartDataProfile, upsertArtifacts, } from "../state/index.js";
 import { createProfileActivityEmitter } from "./data-profile-activity.js";
 import { ensureSearchIndex, searchIndexName } from "../workspace/search-config.js";
@@ -56,6 +60,15 @@ const DATA_PROFILE_AGENT_ID = "data-profiler";
 const DEFAULT_DEADLINE_MS = 1_200_000;
 /** Index entries embedded and upserted per round trip. */
 const INDEX_BATCH_SIZE = 256;
+/**
+ * Submissions the tool accepts: the first, plus one repair.
+ *
+ * The repair is a FULL resubmit — the agent replaces the whole operation list and nothing
+ * is merged into the prior attempt. Past it the resolution stands as it resolved: an
+ * operation that still does not resolve produces no group, and its members land in the
+ * visible `unclassified` group rather than blocking a profile that gates planning.
+ */
+const MAX_SUBMIT_ROUNDS = 2;
 /** Where the embedder stages this analysis's inputs (see the data-profile-init spec). */
 const STAGED_INPUT_ROOT = "data/inputs";
 /** Build the canonical artifact path for a staged input file. */
@@ -78,66 +91,44 @@ function inputArtifactPath(f) {
  * indistinguishable from a legacy snapshot that predates the field, which is exactly
  * the reading a consumer must already tolerate.
  */
-export function buildDataProfileResult(profile, stagedInputs, profiledAt, coverage) {
+export function buildDataProfileResult(submission, resolution, stagedInputs, profiledAt) {
     return {
-        summary: profile.analysisSummary,
-        files: profile.files.map((f) => ({
-            path: f.path,
-            description: f.description,
-            dataType: f.dataType,
-            format: f.format,
-            rows: f.rows,
-            cols: f.cols,
-            tags: f.tags,
-            warnings: f.warnings,
-            metrics: f.metrics,
-        })),
-        kinds: profile.kinds.map((k) => ({
-            name: k.name,
-            memberRepresents: k.memberRepresents,
-            description: k.description,
-            count: k.count,
-            pathPattern: k.pathPattern,
-            format: k.format,
-            axisLabels: k.axisLabels,
-        })),
-        axes: profile.axes?.map((a) => ({
-            label: a.label,
-            cardinality: a.cardinality,
-            exampleValues: a.exampleValues,
-            description: a.description,
-        })),
+        summary: submission.analysisSummary,
+        groups: withoutMembers(resolution.groups),
+        dimensions: [...resolution.dimensions],
+        probes: [...resolution.probes],
+        partition: resolution.partition,
+        recipe: [...resolution.recipe],
+        caveats: submission.caveats,
         inputSignature: computeInputSignature(stagedInputs),
-        coverage,
         profiledAt,
-        domain: profile.domain,
-        subtype: profile.subtype,
-        organism: profile.organism,
-        tissue: profile.tissue,
-        cellType: profile.cellType,
-        condition: profile.condition,
-        accessions: profile.accessions,
-        experimentalDesign: profile.experimentalDesign,
-        qualityAssessment: profile.qualityAssessment,
+        domain: submission.domain,
+        subtype: submission.subtype,
+        organism: submission.organism,
+        tissue: submission.tissue,
+        cellType: submission.cellType,
+        condition: submission.condition,
+        accessions: submission.accessions,
+        experimentalDesign: submission.experimentalDesign,
     };
 }
 /**
- * Walk the staged tree and enrich its shapes with a header readout.
+ * Read one header per set, once a container exists.
  *
- * The walk and the shape observation run in this process over the workspace read seam;
- * only the readout reaches into the sandbox, and only once per shape. A readout that
- * fails is logged and dropped: it is enrichment, and a manifest without it still
- * carries every structural observation the agent's grouping rests on, so failing the
- * profile over it would trade the whole capability for a nicety.
+ * The walk and the set detection run before any sandbox — a profile that absorbs a prior
+ * recipe never provisions one. Every prefix-sufficient readout also runs in this process
+ * over the workspace read seam; only a footer-indexed container reaches into the sandbox,
+ * and only once per set. A readout that fails is logged and dropped: it is enrichment,
+ * and a menu without it still carries every structural observation the agent's grouping
+ * rests on, so failing the profile over it would trade the whole capability for a nicety.
  */
-async function runInputScan(args) {
-    const { session, deps, analysisId, sandbox, execId, deadlineMs, logger } = args;
-    const scan = await scanInputTree({ session, fs: deps.workspaceFs, root: STAGED_INPUT_ROOT });
-    if (scan.manifest.shapes.length === 0)
-        return scan;
+async function readSetHeaders(args) {
+    const { session, deps, analysisId, detected, sandbox, execId, deadlineMs, logger } = args;
     try {
-        const shapes = await enrichShapes({
-            shapes: scan.manifest.shapes,
+        return await readHeaders({
+            targets: readoutTargets(detected),
+            session,
+            fs: deps.workspaceFs,
             sandboxClient: deps.sandboxClient,
             sandbox,
             mountRoot: `/${analysisId}`,
@@ -145,12 +136,112 @@ async function runInputScan(args) {
             deadlineMs,
             emit: async () => { },
         });
-        return { ...scan, manifest: { ...scan.manifest, shapes } };
     }
     catch (err) {
         logger.warn("input-scan header readout failed (non-fatal)", logger.errorFields(err));
-        return scan;
+        return new Map();
+    }
+}
+/**
+ * The submission a tree with nothing to group would have produced.
+ *
+ * Authored here rather than by a model: with no kept file there is no judgement to make,
+ * and the fields below say exactly that. Resolution turns it into the census — zero
+ * groups against a full quarantine accounting — which is the honest record of such a tree
+ * and is what keeps `inspect_data_profile` from reporting the analysis unprofiled.
+ */
+function emptySubmission() {
+    return {
+        operations: [],
+        analysisSummary: "The input scan kept no files: every staged file was quarantined as OS junk, an editor temp, or a partial download. " +
+            "There is no dataset structure to describe. See the quarantine accounting for what was set aside and why.",
+        domain: "",
+        organism: null,
+    };
+}
+/** The membership the persisted record deliberately does not carry. */
+function withoutMembers(groups) {
+    return groups.map(({ memberPaths, ...group }) => {
+        void memberPaths;
+        return group;
+    });
+}
+/**
+ * The prior profile re-stamped over a deterministic absorb.
+ *
+ * Membership, counts, the partition, the recipe, and the signature come from the replay;
+ * everything the agent authored — the summary, the identity fields, the dimensions, the
+ * probes, the caveats — is the profile's existing finding and is carried verbatim. An
+ * absorb re-derives arithmetic, and re-deciding judgement without a model is not something
+ * this path is entitled to do.
+ */
+export function buildAbsorbedResult(prior, resolution, stagedInputs, profiledAt) {
+    return {
+        ...prior,
+        groups: withoutMembers(resolution.groups),
+        // Re-resolved, not carried: a slot observation's cardinality and values are computed
+        // from the scan, and the scan just changed. The labels and the reasoning are the
+        // profile's existing finding; the numbers under them are this replay's.
+        dimensions: [...resolution.dimensions],
+        partition: resolution.partition,
+        recipe: [...resolution.recipe],
+        inputSignature: computeInputSignature(stagedInputs),
+        profiledAt,
+    };
+}
+/**
+ * Build and upsert the profile's vector index — one entry per group, one per dimension,
+ * one per annotated member, none per unannotated file.
+ *
+ * A pure projection of the persisted profile crossed with the scan, so an absorb rebuilds
+ * it from the same two things the agent path does. Embedding is not a model judgement:
+ * leaving the index behind after an absorb would make the newly absorbed members
+ * unsearchable while the record said they were profiled.
+ */
+async function indexProfile(args) {
+    const { deps, analysisId, session, record, scan, logger } = args;
+    await ensureSearchIndex(deps.pool, analysisId, deps.embedding.dimensions);
+    const vectorStore = createVectorStore(deps.pool);
+    const indexName = searchIndexName(analysisId);
+    const entries = buildProfileIndexEntries({ analysisId, result: record, scan });
+    // Replace, don't merge. The tiers are a projection of THIS profile, and an upsert keyed
+    // by entry id leaves a renamed group, a dropped dimension, and a de-annotated member
+    // behind — searchable, and describing a profile that no longer exists.
+    unwrapOrThrow(await vectorStore.deleteByType({ indexName, types: [...PROFILE_INDEX_TYPES] }));
+    // Batched: both interfaces already take arrays, and one round trip per entry is
+    // what made indexing 12 of the 39 minutes the motivating profile took.
+    for (let start = 0; start < entries.length; start += INDEX_BATCH_SIZE) {
+        const batch = entries.slice(start, start + INDEX_BATCH_SIZE);
+        const vectors = unwrapOrThrow(await deps.embedding.embed(batch.map((entry) => entry.text), session));
+        if (vectors.length !== batch.length)
+            throw new Error(`data-profile: embedding returned ${vectors.length} vectors for ${batch.length} entries`);
+        unwrapOrThrow(await vectorStore.upsert({
+            indexName,
+            vectors,
+            metadata: batch.map((entry) => ({ ...entry.metadata, text: entry.text })),
+            ids: batch.map((entry) => entry.id),
+        }));
     }
+    logger.info("indexed profile", { entries: entries.length, groups: record.groups?.length ?? 0 });
+}
+/**
+ * The counters the catalogue's fit against real use is measured from — one structured
+ * event per completed profile, emitted whether or not anything went wrong.
+ */
+function logProfileMonitoring(logger, args) {
+    const { resolution, dimensions, probes, repairRounds, absorb } = args;
+    logger.info("profile monitoring", {
+        absorb,
+        repairRounds,
+        unresolvedOperations: resolution?.errors.length ?? 0,
+        contestedFiles: resolution?.contested.length ?? 0,
+        unclassifiedFiles: resolution?.partition.unclassifiedFiles ?? 0,
+        keptFiles: resolution?.partition.keptFiles ?? 0,
+        scanTruncated: resolution?.partition.scanTruncated ?? false,
+        otherGroupCategories: (resolution?.groups ?? []).filter((group) => group.category === "other" && !group.unclassified).length,
+        otherDimensionCategories: dimensions.filter((dimension) => dimension.category === "other").length,
+        probeNotFound: probes.filter((probe) => probe.outcome === "not-found").length,
+    });
 }
 /**
  * Register the data-profile workflow with DBOS. Returns the registered
@@ -199,6 +290,10 @@ export async function runDataProfileBody(input, deps) {
         // input. The body never downloads.
         if (stagedInputs.length === 0) {
             logger.warn("no input files staged");
+            // One event per completed profile, this one included: the counters are how the
+            // catalogue's fit is measured, and a completion that emitted none would make the
+            // empty-manifest path invisible to the same monitoring that watches every other.
+            logProfileMonitoring(logger, { dimensions: [], probes: [], repairRounds: 0, absorb: "none" });
             if (!unwrapOrThrow(await completeDataProfile(deps.pool, analysisId)))
                 logTerminalNoop(logger, analysisId, "completion");
             await deps.runAuthorizer.revoke(authorization, "data-profile-completed");
@@ -216,7 +311,6 @@ export async function runDataProfileBody(input, deps) {
             role: "input",
             fileId: f.fileId,
         })));
-        // 3. Run data-profiler sandbox agent
         const executionId = generateExecutionId(DATA_PROFILE_AGENT_ID);
         const workflowId = DBOS.workflowID ?? `${DATA_PROFILE_RUN_LITERAL}:${executionId}`;
         const childSession = forSubAgent(runSession, DATA_PROFILE_AGENT_ID);
@@ -225,6 +319,69 @@ export async function runDataProfileBody(input, deps) {
         // meaningless, so no softer handling is warranted.
         const workspaceRoot = deps.resolveWorkspaceRoot(analysisId);
         const profileWritePrefix = `${workspaceRoot}/runs/${DATA_PROFILE_RUN_LITERAL}/${DATA_PROFILE_STEP_LITERAL}`;
+        // 3. The deterministic scan, before any container and before the agent's first turn.
+        // It is always needed and its result does not depend on agent judgement, so spending
+        // an agent turn to request it would be waste — and a briefing carrying one line per
+        // input file consumes context that carries no structure. It runs BEFORE the sandbox
+        // because the absorb below may complete the profile without ever needing one.
+        await activity.scanning();
+        const scan = await scanInputTree({ session: childSession, fs: deps.workspaceFs, root: STAGED_INPUT_ROOT });
+        const detected = detectSets(scan.files);
+        const menu = buildSetMenu(detected, { truncated: scan.manifest.truncated });
+        logger.info("input scan complete", {
+            files: detected.fileCount,
+            kept: detected.keptFileCount,
+            sets: detected.sets.length,
+            listedSets: menu.sets.length,
+            leftovers: detected.leftovers.memberCount,
+            quarantined: detected.quarantine.count,
+            truncated: scan.manifest.truncated,
+        });
+        // 3a. A tree the scan kept nothing of. There is no menu to author against and no file
+        // to describe, so a sandbox and a model pass would produce an empty submission at the
+        // cost of a container. The quarantine accounting is the whole finding, and it is a
+        // real one: it names every file and why each was set aside.
+        if (detected.keptFileCount === 0) {
+            logger.warn("every scanned file was quarantined", { scanned: detected.fileCount, quarantined: detected.quarantine.count });
+            const resolution = resolveProfileSubmission(emptySubmission(), detected, menu, { finalRound: true });
+            const record = buildDataProfileResult(emptySubmission(), resolution, stagedInputs, new Date().toISOString());
+            await activity.indexing();
+            await indexProfile({ deps, analysisId, session: runSession, record, scan, logger });
+            logProfileMonitoring(logger, { resolution, dimensions: [], probes: [], repairRounds: 0, absorb: "none" });
+            if (!unwrapOrThrow(await completeDataProfile(deps.pool, analysisId, record)))
+                logTerminalNoop(logger, analysisId, "completion");
+            await deps.runAuthorizer.revoke(authorization, "data-profile-completed");
+            await activity.complete();
+            return;
+        }
+        // 3b. Absorption — replay a prior profile's recipe against the fresh scan. A recipe
+        // covering every kept file completes the profile here: an added directory of files
+        // that instantiate templates the profile already describes is arithmetic, and paying
+        // a container and a model pass for it is what this pre-step exists to remove.
+        const prior = unwrapOrThrow(await loadDataProfileStatus(deps.pool, analysisId))?.result ?? null;
+        const absorb = absorbRecipe(prior, detected, menu);
+        if (absorb.kind === "stranded") {
+            logger.warn("prior recipe stranded — re-profiling from scratch", { reason: absorb.reason });
+        }
+        if (absorb.kind === "full" && prior) {
+            logger.info("absorbed the prior recipe", { groups: absorb.resolution.groups.length, keptFiles: absorb.resolution.partition.keptFiles });
+            await activity.indexing();
+            const absorbed = buildAbsorbedResult(prior, absorb.resolution, stagedInputs, new Date().toISOString());
+            await indexProfile({ deps, analysisId, session: runSession, record: absorbed, scan, logger });
+            logProfileMonitoring(logger, {
+                resolution: absorb.resolution,
+                dimensions: absorbed.dimensions ?? [],
+                probes: absorbed.probes ?? [],
+                repairRounds: 0,
+                absorb: absorb.kind,
+            });
+            if (!unwrapOrThrow(await completeDataProfile(deps.pool, analysisId, absorbed)))
+                logTerminalNoop(logger, analysisId, "completion");
+            await deps.runAuthorizer.revoke(authorization, "data-profile-completed");
+            await activity.complete();
+            return;
+        }
+        // 4. Run the data-profiler sandbox agent.
         logger.info("starting sandbox", { executionId });
         // Reported BEFORE the container exists, deliberately: provisioning is the longest single
         // operation in a profile and it precedes the agent entirely, so a body that reported only
@@ -260,27 +417,17 @@ export async function runDataProfileBody(input, deps) {
             // which captures its step deadline the same way).
             const deadlineAbs = (await DBOS.now()) + DEFAULT_DEADLINE_MS;
             const nextFunctionId = makeNextFunctionId();
-            // The deterministic scan, before the agent's first turn. It is always needed
-            // and its result does not depend on agent judgement, so spending an agent turn
-            // to request it would be waste — and a briefing carrying one line per input
-            // file consumes context that carries no structure (~270 KB of bare paths for
-            // the tree that motivated this, against ~2 KB of shapes).
-            await activity.scanning();
-            const scan = await runInputScan({
+            const headers = await readSetHeaders({
                 session: childSession,
                 deps,
                 analysisId,
+                detected,
                 sandbox,
                 execId: `${workflowId}:${DATA_PROFILE_STEP_LITERAL}:${nextFunctionId()}`,
                 deadlineMs: deadlineAbs,
                 logger,
             });
-            logger.info("input scan complete", {
-                files: scan.manifest.fileCount,
-                shapes: scan.manifest.shapes.length,
-                unstructured: scan.manifest.unstructured.count,
-                truncated: scan.manifest.truncated,
-            });
+            const briefingMenu = { ...menu, headers };
             // The profiler runs outside `executeAnalysis`, so no scheduler composes a
             // briefing for it — this prompt IS its briefing, and the sandbox system
             // prompt names no paths (it is static per agent type, for the prompt cache).
@@ -296,11 +443,16 @@ export async function runDataProfileBody(input, deps) {
                 ``,
                 `CRITICAL: File and directory paths may contain spaces. You MUST always double-quote paths in shell commands (e.g. head "/${analysisId}/data/inputs/My Folder/file.csv"). Unquoted paths with spaces will silently break commands.`,
                 ``,
-                renderInputScanManifest(scan.manifest),
+                renderSetMenu(briefingMenu, STAGED_INPUT_ROOT),
                 ``,
-                `Decide the dataset's kinds from these observations — you may split a shape, span several shapes, or`,
-                `label an axis the scan did not observe. Inspect ONE example file per kind where a description needs`,
-                `content the scan did not capture, then call \`submit_profile\` once.`,
+                ...(absorb.kind === "partial"
+                    ? [renderAbsorbDelta(absorb), ``]
+                    : [
+                        `Author the dataset's groups as operations on this menu: \`use\` a set, \`split\` one by a slot or by a`,
+                        `value mapping, \`merge\` several, or \`group\` explicit paths the scan left over. Membership and counts`,
+                        `are computed from your operations — you do not state them. Inspect ONE example file per group where a`,
+                        `description needs content the scan did not capture, then call \`submit_profile\` once.`,
+                    ]),
             ].join("\n");
             const sandboxAgentDeps = {
                 provider: deps.provider,
@@ -328,15 +480,34 @@ export async function runDataProfileBody(input, deps) {
                     deadlineMs: () => deadlineAbs,
                 },
             };
-            let capturedProfile = null;
+            let accepted = null;
+            let rounds = 0;
             const submitProfileTool = defineTool({
                 id: "submit_profile",
-                description: "Submit the profiling results. Call this tool once after completing " + "all profiling work — it validates and records your findings.",
-                inputSchema: ProfilerOutputSchema,
+                description: "Submit the profiling results. Call this tool once after completing all profiling work — it resolves your " +
+                    "operations against the scan, computes every group's membership and counts, and records the profile. " +
+                    "A submission whose operations do not resolve, or that leaves kept files unclaimed, comes back once with " +
+                    "the errors; the repair is a FULL resubmit of the whole operation list.",
+                inputSchema: ProfileSubmissionSchema,
                 describeCall: "none",
                 execute: async (input) => {
-                    capturedProfile = input;
-                    return ok({ status: "accepted" });
+                    rounds++;
+                    // The last round resolves as final: nothing that follows can repair an
+                    // overlap, so a contested file sweeps rather than being awarded to whichever
+                    // operation the agent happened to write first.
+                    const finalRound = rounds >= MAX_SUBMIT_ROUNDS;
+                    const resolution = resolveProfileSubmission(input, detected, briefingMenu, { finalRound });
+                    if (!finalRound && (resolution.errors.length > 0 || resolution.unclaimed.length > 0)) {
+                        return ok({ status: "needs_repair", detail: formatResolutionErrors(resolution) });
+                    }
+                    accepted = { submission: input, resolution };
+                    return ok({
+                        status: "accepted",
+                        groups: resolution.groups.length,
+                        keptFiles: resolution.partition.keptFiles,
+                        unclassifiedFiles: resolution.partition.unclassifiedFiles,
+                        unresolvedOperations: resolution.errors.length,
+                    });
                 },
             });
             const baseAgent = createDataProfilerAgent(sandboxAgentDeps);
@@ -364,67 +535,35 @@ export async function runDataProfileBody(input, deps) {
                         await activity.forTool(event.name, event.input, resolveDetail);
                 },
                 runStep: durableStep,
-                resolved: () => capturedProfile !== null,
+                resolved: () => accepted !== null,
                 usageRecorder: deps.usageRecorder,
             }, {
                 tools: [submitProfileTool],
                 nudge: "You stopped without calling submit_profile, so no profile was " +
-                    "recorded. Call submit_profile now with the kinds this dataset is " +
-                    "made of, the axes that vary across them, and any individually " +
-                    "notable files — base it on the scan and the work you already did.",
+                    "recorded. Call submit_profile now with your operations on the menu, " +
+                    "what each resulting group means, and the dimensions you saw with " +
+                    "their observations — base it on the scan and the work you already did.",
             });
-            if (!capturedProfile) {
+            if (!accepted) {
                 throw new Error("Data profiling failed: agent did not call submit_profile");
             }
-            const profilerData = capturedProfile;
-            // 4. Index into vector store — one entry per kind, one per entity, none per file.
+            const { submission, resolution } = accepted;
+            // 5. Index into vector store — one entry per group, one per dimension, one per annotated member.
             //
             // Reported as `indexing`, not `persisting`: the contract defines `persisting` as step
             // bytes uploading to an artifact store, and a profile uploads nothing — its durable
             // products are this vector index and the ledger row.
             await activity.indexing();
-            const profileRecord = buildDataProfileResult(profilerData, stagedInputs, new Date().toISOString(), 
-            // Coverage measures the PROFILE, not the scan: the agent's own patterns are
-            // matched against the scanned tree, so a profile whose kinds leave most of
-            // the tree unmatched is visibly incomplete instead of reading as fresh.
-            computeCoverage(scan.files.map((f) => f.path), profilerData.kinds.map((kind) => kind.pathPattern)));
-            await ensureSearchIndex(deps.pool, analysisId, deps.embedding.dimensions);
-            const vectorStore = createVectorStore(deps.pool);
-            const indexName = searchIndexName(analysisId);
-            const entries = buildProfileIndexEntries({
-                analysisId,
-                kinds: profileRecord.kinds ?? [],
-                ...(profileRecord.axes ? { axes: profileRecord.axes } : {}),
-                scan,
-                files: profileRecord.files,
+            const profileRecord = buildDataProfileResult(submission, resolution, stagedInputs, new Date().toISOString());
+            await indexProfile({ deps, analysisId, session: runSession, record: profileRecord, scan, logger });
+            logProfileMonitoring(logger, {
+                resolution,
+                dimensions: resolution.dimensions,
+                probes: resolution.probes,
+                repairRounds: rounds - 1,
+                absorb: absorb.kind,
             });
-            // Batched: both interfaces already take arrays, and one round trip per entry is
-            // what made indexing 12 of the 39 minutes the motivating profile took.
-            for (let start = 0; start < entries.length; start += INDEX_BATCH_SIZE) {
-                const batch = entries.slice(start, start + INDEX_BATCH_SIZE);
-                const vectors = unwrapOrThrow(await deps.embedding.embed(batch.map((entry) => entry.text), runSession));
-                if (vectors.length !== batch.length)
-                    throw new Error(`data-profile: embedding returned ${vectors.length} vectors for ${batch.length} entries`);
-                unwrapOrThrow(await vectorStore.upsert({
-                    indexName,
-                    vectors,
-                    metadata: batch.map((entry) => ({ ...entry.metadata, text: entry.text })),
-                    ids: batch.map((entry) => entry.id),
-                }));
-            }
-            logger.info("indexed profile", {
-                entries: entries.length,
-                kinds: profileRecord.kinds?.length ?? 0,
-                stagedCount: stagedInputs.length,
-            });
-            if (profileRecord.coverage && profileRecord.coverage.unmatched > 0) {
-                logger.warn("profile kinds do not cover the scanned tree", {
-                    matched: profileRecord.coverage.matched,
-                    unmatched: profileRecord.coverage.unmatched,
-                    total: profileRecord.coverage.total,
-                });
-            }
-            // 5. Complete — store the FULL profiler finding plus the input signature for
+            // 6. Complete — store the FULL profiler finding plus the input signature for
             // staleness detection. The scratch tree is gone; this row is all that survives.
             if (!unwrapOrThrow(await completeDataProfile(deps.pool, analysisId, profileRecord))) {
                 logTerminalNoop(logger, analysisId, "completion");

package/dist/tools/research/inspect-data-profile.d.ts

@@ -8,13 +8,16 @@
  * **There is no data-profile file.** The profiler's `runs/data-profile/` scratch tree
  * is deleted when profiling completes; the profile's only durable home is the
  * `cortex_analysis_state` row this tool reads. That makes this the single authoritative
- * source for dataset facts (organism, domain, design, per-file type/format/dimensions),
- * and it is why the record is served in full rather than summarized: an agent that
- * cannot pull a fact here has no fallback but to re-derive it from the raw bytes.
+ * source for dataset facts — the identity fields, the resolved groups with their slots,
+ * the dimensions with their observations, and the partition census — and it is why the
+ * record is served in full rather than summarized: an agent that cannot pull a fact here
+ * has no fallback but to re-derive it from the raw bytes.
  *
- * Bounded by construction. Like `inspect_run`, the per-file scope is explicitly
- * paged and always reports `total` and `hasMore`: an elided tail is a fact the
- * model can see and act on, never a silent truncation.
+ * Bounded by construction. Like `inspect_run`, the member scope is explicitly paged and
+ * always reports `total` and `hasMore`: an elided tail is a fact the model can see and
+ * act on, never a silent truncation. The annotated-member count and the dataset's file
+ * count are reported as two distinct figures, because a profile annotating eight members
+ * of several thousand files is correct and a single number would read as a tiny dataset.
  *
  * Lifecycle states are data variants, not errors (the `defineTool` contract): a missing,
  * in-flight, failed, or stale profile is an ordinary, expected outcome the model must be
@@ -50,45 +53,68 @@ interface OverviewOutput extends ProfileEnvelope {
     readonly cellType?: string | null;
     readonly condition?: string | null;
     readonly experimentalDesign?: string;
-    readonly qualityAssessment?: DataProfileResult["qualityAssessment"];
+    /** Dataset-wide findings the agent wrote, from whichever field the snapshot carries them in. */
+    readonly caveats?: string[];
     readonly accessions?: string[];
-    /** How many per-file records `scope:"files"` would page through — NOT the dataset's size. */
+    /** How many member records `scope:"files"` would page through — NOT the dataset's size. */
     readonly describedFileCount: number;
     /**
      * How many files the dataset holds, or `null` when the snapshot cannot say (one
-     * written before kinds and coverage existed). Reported separately from
-     * `describedFileCount` because a profile describing eight files of several thousand
-     * is correct under this capability, and a single count would read as a dataset of
-     * eight files.
+     * written before any structure record existed). Reported separately from
+     * `describedFileCount` because a profile describing eight members of several thousand
+     * files is correct under this capability, and a single count would read as a dataset
+     * of eight files.
      */
     readonly datasetFileCount: number | null;
-    /** How many kinds `scope:"kinds"` would return; `null` on a pre-kinds snapshot. */
-    readonly kindCount: number | null;
+    /** How many groups `scope:"groups"` would return; `null` on a snapshot that carries no structure. */
+    readonly groupCount: number | null;
     /** Present when the two counts differ — says where the rest of the dataset is described. */
     readonly structureNote?: string;
+    /**
+     * The census, on a snapshot resolved under the partition: kept files, per-group
+     * counts, the unclassified sweep, and the quarantine with its reasons.
+     */
+    readonly partition?: DataProfileResult["partition"];
     readonly coverage?: DataProfileResult["coverage"];
 }
 /**
- * The dataset's structure. A snapshot predating kinds reports `available: false`
- * rather than an empty list: "this profile predates the structure" and "this dataset
- * has no structure" are different facts, and only one of them is true.
+ * The dataset's structure. A snapshot carrying none reports `available: false` rather
+ * than an empty list: "this profile predates the structure" and "this dataset has no
+ * structure" are different facts, and only one of them is true.
+ *
+ * A snapshot written under the previous model is served through `kinds`/`axes`, labelled
+ * — the structure exists, and an agent must not be told the dataset has none.
  */
-interface KindsOutput extends ProfileEnvelope {
-    readonly scope: "kinds";
+interface GroupsOutput extends ProfileEnvelope {
+    readonly scope: "groups";
     readonly available: boolean;
+    /** Authored under the previous model — `kinds`/`axes` rather than resolved groups. */
+    readonly legacy?: boolean;
+    readonly groups?: DataProfileResult["groups"];
+    readonly dimensions?: DataProfileResult["dimensions"];
+    readonly probes?: DataProfileResult["probes"];
+    readonly partition?: DataProfileResult["partition"];
+    readonly recipe?: DataProfileResult["recipe"];
     readonly kinds?: DataProfileResult["kinds"];
     readonly axes?: DataProfileResult["axes"];
     readonly coverage?: DataProfileResult["coverage"];
     readonly message?: string;
 }
+/**
+ * The members the profile describes individually — annotated members on a resolved
+ * snapshot, the notable-file records on an older one. Never the dataset: `datasetFileCount`
+ * on the overview is what says how big that is.
+ */
 interface FilesOutput extends ProfileEnvelope {
     readonly scope: "files";
     readonly page: number;
     readonly pageSize: number;
-    /** Every per-file record the profile holds — not just this page. */
+    /** Every member record the profile holds — not just this page. */
     readonly total: number;
     /** True when records remain past this page. Truncation is always stated, never silent. */
     readonly hasMore: boolean;
+    /** How many files the dataset holds, so a page of 20 never reads as the whole of it. */
+    readonly datasetFileCount: number | null;
     readonly files: DataProfileFile[];
 }
 /**
@@ -133,9 +159,9 @@ interface FailedOutput {
     readonly failedAt: string | null;
     readonly message: string;
 }
-export type InspectDataProfileOutput = OverviewOutput | KindsOutput | FilesOutput | AbsentOutput | PendingOutput | FailedOutput;
+export type InspectDataProfileOutput = OverviewOutput | GroupsOutput | FilesOutput | AbsentOutput | PendingOutput | FailedOutput;
 export declare function createInspectDataProfileTool(pool: Pool): import("../define-tool.js").Tool<{
-    scope?: "overview" | "files" | "kinds" | undefined;
+    scope?: "overview" | "groups" | "files" | undefined;
     page?: number | undefined;
     pageSize?: number | undefined;
 }, InspectDataProfileOutput>;