@inflexa-ai/harness 0.28.1 → 0.30.0

package/dist/tasks/data-profile.js

@@ -1,8 +1,8 @@
 /**
  * Data-profile DBOS workflow — scans the already-staged input tree, runs the
  * data-profiler sandbox agent over the resulting manifest, registers each staged
- * file in cortex_artifacts, and indexes the profile's kinds and entities into the
- * pgvector store. Inputs are staged under `data/inputs/` by the embedder before the
+ * file in cortex_artifacts, and indexes the profile's groups, dimensions, and annotated
+ * members into the pgvector store. Inputs are staged under `data/inputs/` by the embedder before the
  * run (see the data-profile-init spec); the body assumes a populated tree and never
  * downloads.
  *
@@ -34,12 +34,16 @@ import { estimateDataProfileResources } from "../sandbox/estimate-data-profile-r
 import { generateExecutionId } from "../sandbox/execution-id.js";
 import { mintSandboxIdentity } from "../sandbox/identity.js";
 import { createVectorStore } from "../state/vector-store.js";
-import { ProfilerOutputSchema } from "../schemas/data-profile-schemas.js";
+import { ProfileSubmissionSchema } from "../schemas/data-profile-schemas.js";
 import { computeInputSignature } from "../execution/input-signature.js";
-import { computeCoverage } from "../input-scan/coverage.js";
-import { enrichShapes } from "../input-scan/enrich.js";
-import { renderInputScanManifest, scanInputTree } from "../input-scan/scan.js";
-import { buildProfileIndexEntries } from "./data-profile-index.js";
+import { readHeaders } from "../input-scan/enrich.js";
+import { detectSets } from "../input-scan/detect-sets.js";
+import { buildSetMenu, renderSetMenu } from "../input-scan/menu.js";
+import { readoutTargets } from "../input-scan/readout-budget.js";
+import { scanInputTree } from "../input-scan/scan.js";
+import { PROFILE_INDEX_TYPES, buildProfileIndexEntries } from "./data-profile-index.js";
+import { absorbRecipe, renderAbsorbDelta } from "./data-profile-absorb.js";
+import { formatResolutionErrors, resolveProfileSubmission } from "./data-profile-resolve.js";
 import { completeDataProfile, failDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, tryRerunDataProfile, tryStartDataProfile, upsertArtifacts, } from "../state/index.js";
 import { createProfileActivityEmitter } from "./data-profile-activity.js";
 import { ensureSearchIndex, searchIndexName } from "../workspace/search-config.js";
@@ -56,6 +60,15 @@ const DATA_PROFILE_AGENT_ID = "data-profiler";
 const DEFAULT_DEADLINE_MS = 1_200_000;
 /** Index entries embedded and upserted per round trip. */
 const INDEX_BATCH_SIZE = 256;
+/**
+ * Submissions the tool accepts: the first, plus one repair.
+ *
+ * The repair is a FULL resubmit — the agent replaces the whole operation list and nothing
+ * is merged into the prior attempt. Past it the resolution stands as it resolved: an
+ * operation that still does not resolve produces no group, and its members land in the
+ * visible `unclassified` group rather than blocking a profile that gates planning.
+ */
+const MAX_SUBMIT_ROUNDS = 2;
 /** Where the embedder stages this analysis's inputs (see the data-profile-init spec). */
 const STAGED_INPUT_ROOT = "data/inputs";
 /** Build the canonical artifact path for a staged input file. */
@@ -78,66 +91,44 @@ function inputArtifactPath(f) {
  * indistinguishable from a legacy snapshot that predates the field, which is exactly
  * the reading a consumer must already tolerate.
  */
-export function buildDataProfileResult(profile, stagedInputs, profiledAt, coverage) {
+export function buildDataProfileResult(submission, resolution, stagedInputs, profiledAt) {
     return {
-        summary: profile.analysisSummary,
-        files: profile.files.map((f) => ({
-            path: f.path,
-            description: f.description,
-            dataType: f.dataType,
-            format: f.format,
-            rows: f.rows,
-            cols: f.cols,
-            tags: f.tags,
-            warnings: f.warnings,
-            metrics: f.metrics,
-        })),
-        kinds: profile.kinds.map((k) => ({
-            name: k.name,
-            memberRepresents: k.memberRepresents,
-            description: k.description,
-            count: k.count,
-            pathPattern: k.pathPattern,
-            format: k.format,
-            axisLabels: k.axisLabels,
-        })),
-        axes: profile.axes?.map((a) => ({
-            label: a.label,
-            cardinality: a.cardinality,
-            exampleValues: a.exampleValues,
-            description: a.description,
-        })),
+        summary: submission.analysisSummary,
+        groups: withoutMembers(resolution.groups),
+        dimensions: [...resolution.dimensions],
+        probes: [...resolution.probes],
+        partition: resolution.partition,
+        recipe: [...resolution.recipe],
+        caveats: submission.caveats,
         inputSignature: computeInputSignature(stagedInputs),
-        coverage,
         profiledAt,
-        domain: profile.domain,
-        subtype: profile.subtype,
-        organism: profile.organism,
-        tissue: profile.tissue,
-        cellType: profile.cellType,
-        condition: profile.condition,
-        accessions: profile.accessions,
-        experimentalDesign: profile.experimentalDesign,
-        qualityAssessment: profile.qualityAssessment,
+        domain: submission.domain,
+        subtype: submission.subtype,
+        organism: submission.organism,
+        tissue: submission.tissue,
+        cellType: submission.cellType,
+        condition: submission.condition,
+        accessions: submission.accessions,
+        experimentalDesign: submission.experimentalDesign,
     };
 }
 /**
- * Walk the staged tree and enrich its shapes with a header readout.
+ * Read one header per set, once a container exists.
  *
- * The walk and the shape observation run in this process over the workspace read seam;
- * only the readout reaches into the sandbox, and only once per shape. A readout that
- * fails is logged and dropped: it is enrichment, and a manifest without it still
- * carries every structural observation the agent's grouping rests on, so failing the
- * profile over it would trade the whole capability for a nicety.
+ * The walk and the set detection run before any sandbox — a profile that absorbs a prior
+ * recipe never provisions one. Every prefix-sufficient readout also runs in this process
+ * over the workspace read seam; only a footer-indexed container reaches into the sandbox,
+ * and only once per set. A readout that fails is logged and dropped: it is enrichment,
+ * and a menu without it still carries every structural observation the agent's grouping
+ * rests on, so failing the profile over it would trade the whole capability for a nicety.
  */
-async function runInputScan(args) {
-    const { session, deps, analysisId, sandbox, execId, deadlineMs, logger } = args;
-    const scan = await scanInputTree({ session, fs: deps.workspaceFs, root: STAGED_INPUT_ROOT });
-    if (scan.manifest.shapes.length === 0)
-        return scan;
+async function readSetHeaders(args) {
+    const { session, deps, analysisId, detected, sandbox, execId, deadlineMs, logger } = args;
     try {
-        const shapes = await enrichShapes({
-            shapes: scan.manifest.shapes,
+        return await readHeaders({
+            targets: readoutTargets(detected),
+            session,
+            fs: deps.workspaceFs,
             sandboxClient: deps.sandboxClient,
             sandbox,
             mountRoot: `/${analysisId}`,
@@ -145,12 +136,112 @@ async function runInputScan(args) {
             deadlineMs,
             emit: async () => { },
         });
-        return { ...scan, manifest: { ...scan.manifest, shapes } };
     }
     catch (err) {
         logger.warn("input-scan header readout failed (non-fatal)", logger.errorFields(err));
-        return scan;
+        return new Map();
+    }
+}
+/**
+ * The submission a tree with nothing to group would have produced.
+ *
+ * Authored here rather than by a model: with no kept file there is no judgement to make,
+ * and the fields below say exactly that. Resolution turns it into the census — zero
+ * groups against a full quarantine accounting — which is the honest record of such a tree
+ * and is what keeps `inspect_data_profile` from reporting the analysis unprofiled.
+ */
+function emptySubmission() {
+    return {
+        operations: [],
+        analysisSummary: "The input scan kept no files: every staged file was quarantined as OS junk, an editor temp, or a partial download. " +
+            "There is no dataset structure to describe. See the quarantine accounting for what was set aside and why.",
+        domain: "",
+        organism: null,
+    };
+}
+/** The membership the persisted record deliberately does not carry. */
+function withoutMembers(groups) {
+    return groups.map(({ memberPaths, ...group }) => {
+        void memberPaths;
+        return group;
+    });
+}
+/**
+ * The prior profile re-stamped over a deterministic absorb.
+ *
+ * Membership, counts, the partition, the recipe, and the signature come from the replay;
+ * everything the agent authored — the summary, the identity fields, the dimensions, the
+ * probes, the caveats — is the profile's existing finding and is carried verbatim. An
+ * absorb re-derives arithmetic, and re-deciding judgement without a model is not something
+ * this path is entitled to do.
+ */
+export function buildAbsorbedResult(prior, resolution, stagedInputs, profiledAt) {
+    return {
+        ...prior,
+        groups: withoutMembers(resolution.groups),
+        // Re-resolved, not carried: a slot observation's cardinality and values are computed
+        // from the scan, and the scan just changed. The labels and the reasoning are the
+        // profile's existing finding; the numbers under them are this replay's.
+        dimensions: [...resolution.dimensions],
+        partition: resolution.partition,
+        recipe: [...resolution.recipe],
+        inputSignature: computeInputSignature(stagedInputs),
+        profiledAt,
+    };
+}
+/**
+ * Build and upsert the profile's vector index — one entry per group, one per dimension,
+ * one per annotated member, none per unannotated file.
+ *
+ * A pure projection of the persisted profile crossed with the scan, so an absorb rebuilds
+ * it from the same two things the agent path does. Embedding is not a model judgement:
+ * leaving the index behind after an absorb would make the newly absorbed members
+ * unsearchable while the record said they were profiled.
+ */
+async function indexProfile(args) {
+    const { deps, analysisId, session, record, scan, logger } = args;
+    await ensureSearchIndex(deps.pool, analysisId, deps.embedding.dimensions);
+    const vectorStore = createVectorStore(deps.pool);
+    const indexName = searchIndexName(analysisId);
+    const entries = buildProfileIndexEntries({ analysisId, result: record, scan });
+    // Replace, don't merge. The tiers are a projection of THIS profile, and an upsert keyed
+    // by entry id leaves a renamed group, a dropped dimension, and a de-annotated member
+    // behind — searchable, and describing a profile that no longer exists.
+    unwrapOrThrow(await vectorStore.deleteByType({ indexName, types: [...PROFILE_INDEX_TYPES] }));
+    // Batched: both interfaces already take arrays, and one round trip per entry is
+    // what made indexing 12 of the 39 minutes the motivating profile took.
+    for (let start = 0; start < entries.length; start += INDEX_BATCH_SIZE) {
+        const batch = entries.slice(start, start + INDEX_BATCH_SIZE);
+        const vectors = unwrapOrThrow(await deps.embedding.embed(batch.map((entry) => entry.text), session));
+        if (vectors.length !== batch.length)
+            throw new Error(`data-profile: embedding returned ${vectors.length} vectors for ${batch.length} entries`);
+        unwrapOrThrow(await vectorStore.upsert({
+            indexName,
+            vectors,
+            metadata: batch.map((entry) => ({ ...entry.metadata, text: entry.text })),
+            ids: batch.map((entry) => entry.id),
+        }));
     }
+    logger.info("indexed profile", { entries: entries.length, groups: record.groups?.length ?? 0 });
+}
+/**
+ * The counters the catalogue's fit against real use is measured from — one structured
+ * event per completed profile, emitted whether or not anything went wrong.
+ */
+function logProfileMonitoring(logger, args) {
+    const { resolution, dimensions, probes, repairRounds, absorb } = args;
+    logger.info("profile monitoring", {
+        absorb,
+        repairRounds,
+        unresolvedOperations: resolution?.errors.length ?? 0,
+        contestedFiles: resolution?.contested.length ?? 0,
+        unclassifiedFiles: resolution?.partition.unclassifiedFiles ?? 0,
+        keptFiles: resolution?.partition.keptFiles ?? 0,
+        scanTruncated: resolution?.partition.scanTruncated ?? false,
+        otherGroupCategories: (resolution?.groups ?? []).filter((group) => group.category === "other" && !group.unclassified).length,
+        otherDimensionCategories: dimensions.filter((dimension) => dimension.category === "other").length,
+        probeNotFound: probes.filter((probe) => probe.outcome === "not-found").length,
+    });
 }
 /**
  * Register the data-profile workflow with DBOS. Returns the registered
@@ -199,6 +290,10 @@ export async function runDataProfileBody(input, deps) {
         // input. The body never downloads.
         if (stagedInputs.length === 0) {
             logger.warn("no input files staged");
+            // One event per completed profile, this one included: the counters are how the
+            // catalogue's fit is measured, and a completion that emitted none would make the
+            // empty-manifest path invisible to the same monitoring that watches every other.
+            logProfileMonitoring(logger, { dimensions: [], probes: [], repairRounds: 0, absorb: "none" });
             if (!unwrapOrThrow(await completeDataProfile(deps.pool, analysisId)))
                 logTerminalNoop(logger, analysisId, "completion");
             await deps.runAuthorizer.revoke(authorization, "data-profile-completed");
@@ -216,7 +311,6 @@ export async function runDataProfileBody(input, deps) {
             role: "input",
             fileId: f.fileId,
         })));
-        // 3. Run data-profiler sandbox agent
         const executionId = generateExecutionId(DATA_PROFILE_AGENT_ID);
         const workflowId = DBOS.workflowID ?? `${DATA_PROFILE_RUN_LITERAL}:${executionId}`;
         const childSession = forSubAgent(runSession, DATA_PROFILE_AGENT_ID);
@@ -225,6 +319,69 @@ export async function runDataProfileBody(input, deps) {
         // meaningless, so no softer handling is warranted.
         const workspaceRoot = deps.resolveWorkspaceRoot(analysisId);
         const profileWritePrefix = `${workspaceRoot}/runs/${DATA_PROFILE_RUN_LITERAL}/${DATA_PROFILE_STEP_LITERAL}`;
+        // 3. The deterministic scan, before any container and before the agent's first turn.
+        // It is always needed and its result does not depend on agent judgement, so spending
+        // an agent turn to request it would be waste — and a briefing carrying one line per
+        // input file consumes context that carries no structure. It runs BEFORE the sandbox
+        // because the absorb below may complete the profile without ever needing one.
+        await activity.scanning();
+        const scan = await scanInputTree({ session: childSession, fs: deps.workspaceFs, root: STAGED_INPUT_ROOT });
+        const detected = detectSets(scan.files);
+        const menu = buildSetMenu(detected, { truncated: scan.manifest.truncated });
+        logger.info("input scan complete", {
+            files: detected.fileCount,
+            kept: detected.keptFileCount,
+            sets: detected.sets.length,
+            listedSets: menu.sets.length,
+            leftovers: detected.leftovers.memberCount,
+            quarantined: detected.quarantine.count,
+            truncated: scan.manifest.truncated,
+        });
+        // 3a. A tree the scan kept nothing of. There is no menu to author against and no file
+        // to describe, so a sandbox and a model pass would produce an empty submission at the
+        // cost of a container. The quarantine accounting is the whole finding, and it is a
+        // real one: it names every file and why each was set aside.
+        if (detected.keptFileCount === 0) {
+            logger.warn("every scanned file was quarantined", { scanned: detected.fileCount, quarantined: detected.quarantine.count });
+            const resolution = resolveProfileSubmission(emptySubmission(), detected, menu, { finalRound: true });
+            const record = buildDataProfileResult(emptySubmission(), resolution, stagedInputs, new Date().toISOString());
+            await activity.indexing();
+            await indexProfile({ deps, analysisId, session: runSession, record, scan, logger });
+            logProfileMonitoring(logger, { resolution, dimensions: [], probes: [], repairRounds: 0, absorb: "none" });
+            if (!unwrapOrThrow(await completeDataProfile(deps.pool, analysisId, record)))
+                logTerminalNoop(logger, analysisId, "completion");
+            await deps.runAuthorizer.revoke(authorization, "data-profile-completed");
+            await activity.complete();
+            return;
+        }
+        // 3b. Absorption — replay a prior profile's recipe against the fresh scan. A recipe
+        // covering every kept file completes the profile here: an added directory of files
+        // that instantiate templates the profile already describes is arithmetic, and paying
+        // a container and a model pass for it is what this pre-step exists to remove.
+        const prior = unwrapOrThrow(await loadDataProfileStatus(deps.pool, analysisId))?.result ?? null;
+        const absorb = absorbRecipe(prior, detected, menu);
+        if (absorb.kind === "stranded") {
+            logger.warn("prior recipe stranded — re-profiling from scratch", { reason: absorb.reason });
+        }
+        if (absorb.kind === "full" && prior) {
+            logger.info("absorbed the prior recipe", { groups: absorb.resolution.groups.length, keptFiles: absorb.resolution.partition.keptFiles });
+            await activity.indexing();
+            const absorbed = buildAbsorbedResult(prior, absorb.resolution, stagedInputs, new Date().toISOString());
+            await indexProfile({ deps, analysisId, session: runSession, record: absorbed, scan, logger });
+            logProfileMonitoring(logger, {
+                resolution: absorb.resolution,
+                dimensions: absorbed.dimensions ?? [],
+                probes: absorbed.probes ?? [],
+                repairRounds: 0,
+                absorb: absorb.kind,
+            });
+            if (!unwrapOrThrow(await completeDataProfile(deps.pool, analysisId, absorbed)))
+                logTerminalNoop(logger, analysisId, "completion");
+            await deps.runAuthorizer.revoke(authorization, "data-profile-completed");
+            await activity.complete();
+            return;
+        }
+        // 4. Run the data-profiler sandbox agent.
         logger.info("starting sandbox", { executionId });
         // Reported BEFORE the container exists, deliberately: provisioning is the longest single
         // operation in a profile and it precedes the agent entirely, so a body that reported only
@@ -260,27 +417,17 @@ export async function runDataProfileBody(input, deps) {
             // which captures its step deadline the same way).
             const deadlineAbs = (await DBOS.now()) + DEFAULT_DEADLINE_MS;
             const nextFunctionId = makeNextFunctionId();
-            // The deterministic scan, before the agent's first turn. It is always needed
-            // and its result does not depend on agent judgement, so spending an agent turn
-            // to request it would be waste — and a briefing carrying one line per input
-            // file consumes context that carries no structure (~270 KB of bare paths for
-            // the tree that motivated this, against ~2 KB of shapes).
-            await activity.scanning();
-            const scan = await runInputScan({
+            const headers = await readSetHeaders({
                 session: childSession,
                 deps,
                 analysisId,
+                detected,
                 sandbox,
                 execId: `${workflowId}:${DATA_PROFILE_STEP_LITERAL}:${nextFunctionId()}`,
                 deadlineMs: deadlineAbs,
                 logger,
             });
-            logger.info("input scan complete", {
-                files: scan.manifest.fileCount,
-                shapes: scan.manifest.shapes.length,
-                unstructured: scan.manifest.unstructured.count,
-                truncated: scan.manifest.truncated,
-            });
+            const briefingMenu = { ...menu, headers };
             // The profiler runs outside `executeAnalysis`, so no scheduler composes a
             // briefing for it — this prompt IS its briefing, and the sandbox system
             // prompt names no paths (it is static per agent type, for the prompt cache).
@@ -296,11 +443,16 @@ export async function runDataProfileBody(input, deps) {
                 ``,
                 `CRITICAL: File and directory paths may contain spaces. You MUST always double-quote paths in shell commands (e.g. head "/${analysisId}/data/inputs/My Folder/file.csv"). Unquoted paths with spaces will silently break commands.`,
                 ``,
-                renderInputScanManifest(scan.manifest),
+                renderSetMenu(briefingMenu, STAGED_INPUT_ROOT),
                 ``,
-                `Decide the dataset's kinds from these observations — you may split a shape, span several shapes, or`,
-                `label an axis the scan did not observe. Inspect ONE example file per kind where a description needs`,
-                `content the scan did not capture, then call \`submit_profile\` once.`,
+                ...(absorb.kind === "partial"
+                    ? [renderAbsorbDelta(absorb), ``]
+                    : [
+                        `Author the dataset's groups as operations on this menu: \`use\` a set, \`split\` one by a slot or by a`,
+                        `value mapping, \`merge\` several, or \`group\` explicit paths the scan left over. Membership and counts`,
+                        `are computed from your operations — you do not state them. Inspect ONE example file per group where a`,
+                        `description needs content the scan did not capture, then call \`submit_profile\` once.`,
+                    ]),
             ].join("\n");
             const sandboxAgentDeps = {
                 provider: deps.provider,
@@ -328,15 +480,34 @@ export async function runDataProfileBody(input, deps) {
                     deadlineMs: () => deadlineAbs,
                 },
             };
-            let capturedProfile = null;
+            let accepted = null;
+            let rounds = 0;
             const submitProfileTool = defineTool({
                 id: "submit_profile",
-                description: "Submit the profiling results. Call this tool once after completing " + "all profiling work — it validates and records your findings.",
-                inputSchema: ProfilerOutputSchema,
+                description: "Submit the profiling results. Call this tool once after completing all profiling work — it resolves your " +
+                    "operations against the scan, computes every group's membership and counts, and records the profile. " +
+                    "A submission whose operations do not resolve, or that leaves kept files unclaimed, comes back once with " +
+                    "the errors; the repair is a FULL resubmit of the whole operation list.",
+                inputSchema: ProfileSubmissionSchema,
                 describeCall: "none",
                 execute: async (input) => {
-                    capturedProfile = input;
-                    return ok({ status: "accepted" });
+                    rounds++;
+                    // The last round resolves as final: nothing that follows can repair an
+                    // overlap, so a contested file sweeps rather than being awarded to whichever
+                    // operation the agent happened to write first.
+                    const finalRound = rounds >= MAX_SUBMIT_ROUNDS;
+                    const resolution = resolveProfileSubmission(input, detected, briefingMenu, { finalRound });
+                    if (!finalRound && (resolution.errors.length > 0 || resolution.unclaimed.length > 0)) {
+                        return ok({ status: "needs_repair", detail: formatResolutionErrors(resolution) });
+                    }
+                    accepted = { submission: input, resolution };
+                    return ok({
+                        status: "accepted",
+                        groups: resolution.groups.length,
+                        keptFiles: resolution.partition.keptFiles,
+                        unclassifiedFiles: resolution.partition.unclassifiedFiles,
+                        unresolvedOperations: resolution.errors.length,
+                    });
                 },
             });
             const baseAgent = createDataProfilerAgent(sandboxAgentDeps);
@@ -364,67 +535,35 @@ export async function runDataProfileBody(input, deps) {
                         await activity.forTool(event.name, event.input, resolveDetail);
                 },
                 runStep: durableStep,
-                resolved: () => capturedProfile !== null,
+                resolved: () => accepted !== null,
                 usageRecorder: deps.usageRecorder,
             }, {
                 tools: [submitProfileTool],
                 nudge: "You stopped without calling submit_profile, so no profile was " +
-                    "recorded. Call submit_profile now with the kinds this dataset is " +
-                    "made of, the axes that vary across them, and any individually " +
-                    "notable files — base it on the scan and the work you already did.",
+                    "recorded. Call submit_profile now with your operations on the menu, " +
+                    "what each resulting group means, and the dimensions you saw with " +
+                    "their observations — base it on the scan and the work you already did.",
             });
-            if (!capturedProfile) {
+            if (!accepted) {
                 throw new Error("Data profiling failed: agent did not call submit_profile");
             }
-            const profilerData = capturedProfile;
-            // 4. Index into vector store — one entry per kind, one per entity, none per file.
+            const { submission, resolution } = accepted;
+            // 5. Index into vector store — one entry per group, one per dimension, one per annotated member.
             //
             // Reported as `indexing`, not `persisting`: the contract defines `persisting` as step
             // bytes uploading to an artifact store, and a profile uploads nothing — its durable
             // products are this vector index and the ledger row.
             await activity.indexing();
-            const profileRecord = buildDataProfileResult(profilerData, stagedInputs, new Date().toISOString(), 
-            // Coverage measures the PROFILE, not the scan: the agent's own patterns are
-            // matched against the scanned tree, so a profile whose kinds leave most of
-            // the tree unmatched is visibly incomplete instead of reading as fresh.
-            computeCoverage(scan.files.map((f) => f.path), profilerData.kinds.map((kind) => kind.pathPattern)));
-            await ensureSearchIndex(deps.pool, analysisId, deps.embedding.dimensions);
-            const vectorStore = createVectorStore(deps.pool);
-            const indexName = searchIndexName(analysisId);
-            const entries = buildProfileIndexEntries({
-                analysisId,
-                kinds: profileRecord.kinds ?? [],
-                ...(profileRecord.axes ? { axes: profileRecord.axes } : {}),
-                scan,
-                files: profileRecord.files,
+            const profileRecord = buildDataProfileResult(submission, resolution, stagedInputs, new Date().toISOString());
+            await indexProfile({ deps, analysisId, session: runSession, record: profileRecord, scan, logger });
+            logProfileMonitoring(logger, {
+                resolution,
+                dimensions: resolution.dimensions,
+                probes: resolution.probes,
+                repairRounds: rounds - 1,
+                absorb: absorb.kind,
             });
-            // Batched: both interfaces already take arrays, and one round trip per entry is
-            // what made indexing 12 of the 39 minutes the motivating profile took.
-            for (let start = 0; start < entries.length; start += INDEX_BATCH_SIZE) {
-                const batch = entries.slice(start, start + INDEX_BATCH_SIZE);
-                const vectors = unwrapOrThrow(await deps.embedding.embed(batch.map((entry) => entry.text), runSession));
-                if (vectors.length !== batch.length)
-                    throw new Error(`data-profile: embedding returned ${vectors.length} vectors for ${batch.length} entries`);
-                unwrapOrThrow(await vectorStore.upsert({
-                    indexName,
-                    vectors,
-                    metadata: batch.map((entry) => ({ ...entry.metadata, text: entry.text })),
-                    ids: batch.map((entry) => entry.id),
-                }));
-            }
-            logger.info("indexed profile", {
-                entries: entries.length,
-                kinds: profileRecord.kinds?.length ?? 0,
-                stagedCount: stagedInputs.length,
-            });
-            if (profileRecord.coverage && profileRecord.coverage.unmatched > 0) {
-                logger.warn("profile kinds do not cover the scanned tree", {
-                    matched: profileRecord.coverage.matched,
-                    unmatched: profileRecord.coverage.unmatched,
-                    total: profileRecord.coverage.total,
-                });
-            }
-            // 5. Complete — store the FULL profiler finding plus the input signature for
+            // 6. Complete — store the FULL profiler finding plus the input signature for
             // staleness detection. The scratch tree is gone; this row is all that survives.
             if (!unwrapOrThrow(await completeDataProfile(deps.pool, analysisId, profileRecord))) {
                 logTerminalNoop(logger, analysisId, "completion");

package/dist/tools/bio/chembl.d.ts

@@ -32,7 +32,7 @@ export declare const chemblTool: import("../define-tool.js").Tool<{
     query?: string | undefined;
     searchType?: "target" | "smiles" | "compound" | undefined;
     chemblId?: string | undefined;
-    idType?: "target" | "compound" | undefined;
+    idType?: "target" | "compound" | "assay" | undefined;
     activityType?: string | undefined;
     limit?: number | undefined;
 }, ChemblOutput>;

package/dist/tools/bio/chembl.js

@@ -45,7 +45,9 @@ const inputSchema = z
         "(INHIBITOR, AGONIST, …), targetChemblId + targetName, moleculeChemblId. Curated mainly for clinical/approved molecules, so tool " +
         "compounds often have none.\n" +
         "'bioactivity' (chemblId + idType) — measured activity rows: the curated, QUOTABLE potency. → standardType, standardValue + " +
-        "standardUnits, pchemblValue (normalized -log10), assayChemblId, assayType, compoundChemblId, targetChemblId.\n" +
+        "standardUnits, pchemblValue (normalized -log10), assayChemblId, assayType, compoundChemblId, targetChemblId. With idType='assay' " +
+        "it reads ONE assay back by the assayChemblId that an earlier bioactivity row carried — every measurement made under the same " +
+        "protocol, which is how you compare compounds on like terms.\n" +
         "'targets' (query) — resolve a gene symbol or protein name to a ChEMBL target, producing the ID that bioactivity (idType='target') " +
         "and compounds (searchType='target') need. → targetChemblId, preferredName, targetType, organism, geneNames. Check `organism`: the " +
         "top hit for a human symbol may be a non-human ortholog."),
@@ -65,13 +67,13 @@ const inputSchema = z
         .string()
         .min(1)
         .optional()
-        .describe("Required for 'mechanism' and 'bioactivity'. A molecule ID ('CHEMBL25' = aspirin), or — for bioactivity with idType='target' — a target " +
-        "ID ('CHEMBL203' = EGFR). 'mechanism' takes a molecule ID only."),
+        .describe("Required for 'mechanism' and 'bioactivity'. A molecule ID ('CHEMBL25' = aspirin), or — for bioactivity — a target ID ('CHEMBL203' = " +
+        "EGFR) with idType='target', or an assay ID ('CHEMBL1217643') with idType='assay'. 'mechanism' takes a molecule ID only."),
     idType: z
-        .enum(["compound", "target"])
+        .enum(["compound", "target", "assay"])
         .optional()
-        .describe("Required for 'bioactivity' — which side of the activity table `chemblId` indexes. 'compound': everything that molecule was assayed " +
-        "against. 'target': every compound assayed against that target."),
+        .describe("Required for 'bioactivity' — which column of the activity table `chemblId` indexes. 'compound': everything that molecule was assayed " +
+        "against. 'target': every compound assayed against that target. 'assay': every measurement made under that one assay protocol."),
     activityType: z
         .string()
         .optional()
@@ -110,10 +112,13 @@ const inputSchema = z
 });
 export const chemblTool = defineTool({
     id: "chembl",
-    description: "ChEMBL — the manually curated database of drug-like bioactives (~2.4M compounds), the targets they were measured against, their mechanisms and " +
-        "their approval status. Five lookups; pick with `action`, which gives each one's params and return fields.\n" +
-        "IDs are resolved, never guessed: a `chemblId` comes from a prior action='compounds'/'drug' (molecules) or action='targets' (targets), or from " +
-        "pubchem action='crossrefs'. A compound or gene name is not a ChEMBL ID.\n" +
+    description: "ChEMBL — the manually curated database of drug-like bioactives of EMBL-EBI (~2.4M compounds), the targets they were measured against, their " +
+        "mechanisms and their approval status. Five lookups; pick with `action`, which gives each one's params and return fields.\n" +
+        "ACCEPTED IDENTIFIERS: free text for 'compounds', 'drug' and 'targets' — a compound name ('imatinib'), an indication ('melanoma'), a gene symbol " +
+        "('EGFR') or a SMILES string. A ChEMBL ID for 'mechanism' and 'bioactivity' — a molecule ID ('CHEMBL25'), a target ID ('CHEMBL203') or an assay " +
+        "ID ('CHEMBL1217643'), named by `idType`.\n" +
+        "IDs are resolved, never guessed: a `chemblId` comes from a prior action='compounds'/'drug' (molecules), action='targets' (targets), the " +
+        "assayChemblId of a bioactivity row (assays), or pubchem action='crossrefs'. A compound or gene name is not a ChEMBL ID.\n" +
         "Being curated, ChEMBL is what you quote from: prefer action='bioactivity' over pubchem action='assays' for any number you will cite, and resolve " +
         "compounds here first. If ChEMBL misses the compound, resolve it via pubchem action='compound' and bridge back with pubchem action='crossrefs'.\n" +
         "An empty array is valid no-data, not an error — do not retry the same call.",

package/dist/tools/bio/comptox.js

@@ -98,9 +98,13 @@ const inputSchema = z
 export function createComptoxTool(deps) {
     return defineTool({
         id: "comptox",
-        description: "EPA's CompTox (CTX) chemical-safety databases — four datasets behind `dataset`: 'toxcast' (in-vitro HTS bioactivity), 'hazard' (in-vivo: " +
+        description: "The CompTox Chemicals Dashboard (CTX) of the U.S. Environmental Protection Agency — the chemical-safety corpus that holds ToxCast, ToxValDB, " +
+            "the genotoxicity and cancer summaries, and the SEEM exposure predictions. Four datasets behind `dataset`: 'toxcast' (in-vitro HTS " +
+            "bioactivity), 'hazard' (in-vivo: " +
             "NOAEL/LOAEL/LD50, genotoxicity, cancer), 'chemical' (identity + physicochemical/ADMET), 'exposure' (SEEM predictions, toxicokinetics, uses, " +
             "products). See `dataset` for what each returns; all four resolve `query` to a DTXSID first.\n" +
+            "ACCEPTED IDENTIFIERS: a DTXSID ('DTXSID7020182') used directly, or a CASRN ('80-05-7'), a chemical name ('bisphenol A') or — for dataset " +
+            "'chemical' — an InChIKey, each resolved by EXACT match.\n" +
             "Requires EPA_CCTE_API_KEY — a missing key fails terminally: do NOT retry, tell the user the key needs configuring and proceed without EPA data.\n" +
             "found: false means the query did not resolve to a chemical; a present-but-empty section is likewise valid no-data. Do not retry either.\n" +
             "This is environmental/industrial-chemical data: for drug-like compounds prefer PubChem or ChEMBL — dataset 'chemical' returns " +

package/dist/tools/bio/drug-gene-interactions.js

@@ -111,8 +111,11 @@ export function createDrugGeneInteractionsTool(deps) {
     const logger = (deps.logger ?? createNoopLogger()).named("drug_gene_interactions");
     return defineTool({
         id: "drug_gene_interactions",
-        description: "Drug↔gene interactions across DGIdb, DrugBank and PharmGKB, in one call — 'what drugs hit these genes?' and 'what genes does this drug act " +
-            "on?'. See `direction` and `sources` for the modes.\n" +
+        description: "Drug↔gene interactions across DGIdb (the Drug Gene Interaction Database), DrugBank and PharmGKB, in one call — 'what drugs hit these " +
+            "genes?' and 'what genes does this drug act on?'. See `direction` and `sources` for the modes.\n" +
+            "ACCEPTED IDENTIFIERS: a HUGO gene symbol ('EGFR') for direction 'gene_to_drugs'; a drug name ('imatinib'), a DrugBank ID ('DB00619') or a " +
+            "DGIdb concept ID for 'drug_to_genes'. A brand name and a non-HUGO symbol match nothing — resolve the symbol with search_gene and the brand " +
+            "name with search_faers({action:'label'}) first.\n" +
             "It answers whether an interaction is KNOWN, by whom, and of what type. For quotable POTENCY (IC50/Ki) use chembl({action:'bioactivity'}) instead.\n" +
             "Verbose fields (`includeAttributes`, `includeDrugRecord`) are off by default — both cost a lot of context and are rarely what the question needs.\n" +
             "ALWAYS read `perSource` before concluding an interaction is unknown: 'no_data' means that corpus has nothing; 'unavailable' means it could " +