@inflexa-ai/harness 0.24.0 → 0.26.0

package/dist/state/data-profile.d.ts

@@ -5,98 +5,17 @@
  */
 import type { ResultAsync } from "neverthrow";
 import { type DbError } from "../lib/db-result.js";
-import type { Organism } from "../schemas/data-profile-schemas.js";
+import type { DataProfileLifecycleStatus, DataProfileResult } from "../contracts/data-profile.js";
 import type { Querier } from "./db.js";
 /**
- * One profiled input file's **drift signature**. `fileId` is a path identity the
- * embedder derives (two files at the same path share it regardless of content),
- * so `size` + `mtimeMs` are what let a consumer notice that the bytes behind a
- * path changed. The harness treats all three as opaque labels: it persists them
- * verbatim and never stats a source file or compares signatures itself.
+ * The persisted profile's shape lives in `contracts/` — it is read by consumers outside
+ * this package (a host route, a UI), and they must not import this module, which pulls in
+ * `pg` and the whole ledger query surface, to obtain a type. Re-exported here so ledger
+ * call sites keep one import.
  */
-export interface DataProfileInputFile {
-    fileId: string;
-    size: number;
-    mtimeMs: number;
-}
-/**
- * One profiled file's persisted record.
- *
- * `path` + `description` are the original pair every snapshot carries. Everything
- * below them is **optional on read** for the same reason `DataProfileResult.inputFiles`
- * is: a snapshot written before the record was widened carries only the pair, and a
- * reader must render that row rather than reject it. A writer fills in whatever the
- * profiler reported.
- */
-export interface DataProfileFile {
-    path: string;
-    description: string;
-    /** Semantic data type (count-matrix, variants, clinical-metadata, document, …). */
-    dataType?: string;
-    /** File format (CSV, h5ad, VCF, BAM, …). */
-    format?: string;
-    rows?: number | null;
-    cols?: number | null;
-    /** Searchable labels the profiler attached. */
-    tags?: string[];
-    /** Quality issues specific to this file. */
-    warnings?: string[];
-    /** Flat format-specific metrics (sparsity, medianLibrarySize, tiTvRatio, …). */
-    metrics?: Record<string, string | number | boolean>;
-}
-/** Dataset-wide quality findings the profiler recorded. */
-export interface DataProfileQualityAssessment {
-    concerns: string[];
-    strengths: string[];
-}
-/**
- * The profile snapshot `completeDataProfile` persists and `loadDataProfileStatus` reads back.
- *
- * This row is the **only** durable home of the profile: the profiler's `runs/data-profile/`
- * scratch tree is wiped when profiling completes, so there is no file to fall back on. It
- * therefore carries the profiler's full finding, not a summary of it — anything dropped here
- * is destroyed, and a downstream agent can only recover it by re-reading the raw input files.
- *
- * Every field past `summary` / `files` / `inputFileIds` / `profiledAt` is **optional on read**:
- * a snapshot written before the record was widened carries only those four, and a reader must
- * render it rather than throw. There is no parse at the read boundary (the jsonb column is cast
- * straight to this type), so optionality *is* the compatibility mechanism — exactly as it
- * already is for `inputFiles`.
- */
-export interface DataProfileResult {
-    summary: string;
-    files: DataProfileFile[];
-    /** Which files the profile covered — the audit record. */
-    inputFileIds: string[];
-    /**
-     * Whether the same bytes were covered — the drift comparand. Optional on read:
-     * a snapshot written before this field existed carries only `inputFileIds`, and
-     * a consumer treats the absence as drift (re-profiling repairs it), exactly as
-     * it already treats a wholly absent `result`.
-     */
-    inputFiles?: DataProfileInputFile[];
-    profiledAt: string;
-    /** Scientific domain (transcriptomics, cheminformatics, clinical, …). */
-    domain?: string;
-    /** Specific subtype within the domain (bulk-rna-seq, single-cell, LC-MS/MS, …). */
-    subtype?: string;
-    /**
-     * Subject organism with its provenance and confidence. Explicit `null` is a real,
-     * distinct value — the profiler looked and no input identified an organism — as
-     * opposed to absent, which means the snapshot predates this field.
-     */
-    organism?: Organism | null;
-    tissue?: string | null;
-    cellType?: string | null;
-    condition?: string | null;
-    /** Public dataset accessions found in any input (GSE/SRP/PRJNA/E-MTAB/phs/EGAS). */
-    accessions?: string[];
-    /** Conditions, groups, comparisons, replicates, pairing. */
-    experimentalDesign?: string;
-    qualityAssessment?: DataProfileQualityAssessment;
-}
+export type { DataProfileAxis, DataProfileCoverage, DataProfileFile, DataProfileInputFile, DataProfileInputSignature, DataProfileKind, DataProfileOrganism, DataProfileQualityAssessment, DataProfileLifecycleStatus, DataProfileResult, DataProfileSubjectSource, } from "../contracts/data-profile.js";
 export interface DataProfileStatus {
-    status: "pending" | "running" | "completed" | "failed";
+    status: DataProfileLifecycleStatus;
     error: string | null;
     startedAt: string | null;
     completedAt: string | null;

package/dist/state/index.d.ts

@@ -21,7 +21,7 @@ export type { ActiveSandboxRow } from "./active-sandboxes.js";
 export { insertPlan, upsertPlan, loadPlan } from "./plans.js";
 export type { InsertPlanInput, UpsertPlanInput } from "./plans.js";
 export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
-export type { DataProfileStatus, DataProfileResult, DataProfileInputFile, DataProfileFile, DataProfileQualityAssessment } from "./data-profile.js";
+export type { DataProfileStatus, DataProfileResult, DataProfileInputFile, DataProfileFile, DataProfileQualityAssessment, DataProfileKind, DataProfileAxis, DataProfileInputSignature, DataProfileCoverage, } from "./data-profile.js";
 export { queryRunCountsByAnalyses, queryThreadCountsByAnalyses, queryDataProfileStatusByAnalyses } from "./analyses-metrics.js";
 export { createAnalysisPurge } from "./purge-analysis.js";
 export type { AnalysisPurge, AnalysisPurgeDeps, AnalysisPurgeOutcome } from "./purge-analysis.js";

package/dist/tasks/data-profile-activity.d.ts

@@ -50,6 +50,14 @@ export type ActivityWrite = (part: unknown) => Promise<void>;
 export interface ProfileActivityEmitter {
     /** Reported before the sandbox is created — the longest wait in a profile, and the one that precedes the agent entirely. */
     sandboxInit(): Promise<void>;
+    /**
+     * Reported before the deterministic input scan, which runs between a ready sandbox
+     * and the agent's first turn. On a large tree it is the second-longest operation in
+     * a profile, and left unreported it would read as `Running data-profiler` for
+     * minutes before the agent had begun — the misreport `sandbox-init` exists to
+     * prevent.
+     */
+    scanning(): Promise<void>;
     /** Reported once the sandbox is ready, covering the gap before the agent's first tool call. */
     agentStarting(): Promise<void>;
     /**

package/dist/tasks/data-profile-activity.js

@@ -47,6 +47,7 @@ export function createProfileActivityEmitter(write, frame, logger) {
     };
     return {
         sandboxInit: () => emit("sandbox-init", "Starting sandbox"),
+        scanning: () => emit("executing", "Scanning input files"),
         agentStarting: () => emit("executing", "Running data-profiler"),
         forTool: (name, input, resolveDetail) => emit("executing", activityForTool(name, input, resolveDetail)),
         indexing: () => emit("indexing", "Indexing input descriptions for search"),

package/dist/tasks/data-profile-index.d.ts

@@ -0,0 +1,45 @@
+/**
+ * The profile's vector index, as a pure projection.
+ *
+ * The index is a function of the scan crossed with the submitted kinds: for each kind,
+ * one entry; for each entity, one entry templated from the kinds it participates in and
+ * its own axis value. It has no persisted representation of its own and nothing derives
+ * from it, so it can be rebuilt at any time by re-running the scan and reading `kinds`
+ * off the existing profile — no model call, no re-profile. Improving the template,
+ * adding a tier, or folding in entity attributes later are all re-projections.
+ *
+ * Two tiers, because they answer different queries. A query naming a KIND of data
+ * ("variant calls") wants one result carrying the set's count and path pattern; a
+ * per-entity tier would return five arbitrary subjects out of thousands of
+ * indistinguishable ones. A query naming an ENTITY ("PT0421") wants that entity. A
+ * per-file tier adds nothing: a file is identified by its kind and its entity, and its
+ * path is on the filesystem.
+ *
+ * `type: "input"` keeps its existing meaning for the entity tier, so searches written
+ * against it keep matching; `type: "input-kind"` is additive. Renaming `"input"` would
+ * have silently broken existing search on every new analysis.
+ */
+import type { InputScan } from "../input-scan/types.js";
+import type { DataProfileAxis, DataProfileFile, DataProfileKind } from "../state/data-profile.js";
+/** Entity entries built per analysis. Past this the entity tier is a liability, not a search aid. */
+export declare const MAX_ENTITY_ENTRIES = 20000;
+export interface ProfileIndexEntry {
+    readonly id: string;
+    readonly text: string;
+    readonly metadata: Record<string, unknown>;
+}
+export interface BuildProfileIndexArgs {
+    readonly analysisId: string;
+    readonly kinds: readonly DataProfileKind[];
+    readonly axes?: readonly DataProfileAxis[];
+    /** The scan the profile was taken against — the source of the entity value set. */
+    readonly scan?: InputScan;
+    /** Individually described files, indexed only when there are no kinds to index. */
+    readonly files?: readonly DataProfileFile[];
+}
+/**
+ * Build every index entry for a completed profile. Deterministic and model-free: the
+ * text is templated from the kind's own description and the entity's axis value, so
+ * indexing costs no LLM tokens whatever the dataset's size.
+ */
+export declare function buildProfileIndexEntries(args: BuildProfileIndexArgs): ProfileIndexEntry[];

package/dist/tasks/data-profile-index.js

@@ -0,0 +1,134 @@
+/**
+ * The profile's vector index, as a pure projection.
+ *
+ * The index is a function of the scan crossed with the submitted kinds: for each kind,
+ * one entry; for each entity, one entry templated from the kinds it participates in and
+ * its own axis value. It has no persisted representation of its own and nothing derives
+ * from it, so it can be rebuilt at any time by re-running the scan and reading `kinds`
+ * off the existing profile — no model call, no re-profile. Improving the template,
+ * adding a tier, or folding in entity attributes later are all re-projections.
+ *
+ * Two tiers, because they answer different queries. A query naming a KIND of data
+ * ("variant calls") wants one result carrying the set's count and path pattern; a
+ * per-entity tier would return five arbitrary subjects out of thousands of
+ * indistinguishable ones. A query naming an ENTITY ("PT0421") wants that entity. A
+ * per-file tier adds nothing: a file is identified by its kind and its entity, and its
+ * path is on the filesystem.
+ *
+ * `type: "input"` keeps its existing meaning for the entity tier, so searches written
+ * against it keep matching; `type: "input-kind"` is additive. Renaming `"input"` would
+ * have silently broken existing search on every new analysis.
+ */
+/** Entity entries built per analysis. Past this the entity tier is a liability, not a search aid. */
+export const MAX_ENTITY_ENTRIES = 20_000;
+/** Kind names named in one entity's entry text. */
+const MAX_KINDS_IN_ENTITY_TEXT = 6;
+function slug(value) {
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-|-$/g, "")
+        .slice(0, 60);
+}
+/**
+ * The entity value set: the largest distinct-value set any single variable position
+ * carries.
+ *
+ * Entities are what a dataset is ABOUT, and the position that varies most widely across
+ * one shape is what identifies them — 1171 subjects, against three timepoints and two
+ * replicates. Taking the widest position rather than the cross product is deliberate:
+ * the cross product is the file set, which is the tier this projection exists to avoid.
+ */
+function entityValues(scan) {
+    if (!scan)
+        return { values: [], distinct: 0 };
+    let best = [];
+    for (const shape of scan.manifest.shapes) {
+        const positions = scan.positionValues.get(shape.id);
+        if (!positions)
+            continue;
+        for (const position of shape.variablePositions) {
+            const values = positions[position.index];
+            if (values && values.length > best.length)
+                best = values;
+        }
+    }
+    return { values: [...best].sort((a, b) => a.localeCompare(b, "en")), distinct: best.length };
+}
+/**
+ * The axis label for an entity set. The agent labels axes; the scan counts values, so
+ * the axis whose cardinality matches the observed set is the one the agent was naming.
+ * With no match the entries still index under a neutral label rather than not at all.
+ */
+function entityAxisLabel(axes, distinct) {
+    if (!axes || axes.length === 0)
+        return "entity";
+    const exact = axes.find((axis) => axis.cardinality === distinct);
+    if (exact)
+        return exact.label;
+    const widest = [...axes].sort((a, b) => b.cardinality - a.cardinality)[0];
+    return widest?.label ?? "entity";
+}
+function kindEntry(analysisId, kind) {
+    const facts = [`${kind.count} files`, kind.format, kind.pathPattern].filter((f) => Boolean(f)).join(", ");
+    return {
+        id: `/${analysisId}/kind/${slug(kind.name)}`,
+        text: `${kind.name} — ${kind.description} One member is ${kind.memberRepresents}. (${facts})`,
+        metadata: {
+            type: "input-kind",
+            kind: kind.name,
+            pathPattern: kind.pathPattern,
+            count: kind.count,
+            ...(kind.format ? { format: kind.format } : {}),
+            ...(kind.axisLabels ? { axisLabels: kind.axisLabels } : {}),
+        },
+    };
+}
+function entityEntry(analysisId, label, value, kinds) {
+    const named = kinds.slice(0, MAX_KINDS_IN_ENTITY_TEXT);
+    const carried = named.map((kind) => `${kind.name} (${kind.memberRepresents})`).join("; ");
+    return {
+        id: `/${analysisId}/entity/${encodeURIComponent(value)}`,
+        text: `${label} ${value} — input data for this ${label}: ${carried}.`,
+        metadata: {
+            type: "input",
+            entity: value,
+            axis: label,
+            kinds: named.map((kind) => kind.name),
+        },
+    };
+}
+/** A file entry, for the degenerate case where a profile submitted no kinds at all. */
+function fileEntry(analysisId, file) {
+    return {
+        id: `/${analysisId}/${file.path}`,
+        text: `${file.path} — ${file.description}`,
+        metadata: {
+            type: "input",
+            ...(file.dataType ? { dataType: file.dataType } : {}),
+            ...(file.format ? { format: file.format } : {}),
+            ...(file.tags ? { tags: file.tags } : {}),
+        },
+    };
+}
+/**
+ * Build every index entry for a completed profile. Deterministic and model-free: the
+ * text is templated from the kind's own description and the entity's axis value, so
+ * indexing costs no LLM tokens whatever the dataset's size.
+ */
+export function buildProfileIndexEntries(args) {
+    const { analysisId, kinds, axes, scan, files } = args;
+    if (kinds.length === 0) {
+        // No kinds is a degenerate profile, but the notable files it did describe stay
+        // discoverable — the same losslessness the per-file fallback description had.
+        return (files ?? []).map((file) => fileEntry(analysisId, file));
+    }
+    const entries = kinds.map((kind) => kindEntry(analysisId, kind));
+    const { values, distinct } = entityValues(scan);
+    if (values.length > 1 && values.length <= MAX_ENTITY_ENTRIES) {
+        const label = entityAxisLabel(axes, distinct);
+        for (const value of values)
+            entries.push(entityEntry(analysisId, label, value, kinds));
+    }
+    return entries;
+}

package/dist/tasks/data-profile.d.ts

@@ -1,9 +1,10 @@
 /**
- * Data-profile DBOS workflow — profiles the already-staged input files, runs
- * the data-profiler sandbox agent, writes per-file metadata to
- * cortex_artifacts, and indexes descriptions into the pgvector store. Inputs
- * are staged under `data/inputs/` by the embedder before the run (see the data-profile-init spec);
- * the body assumes a populated tree and never downloads.
+ * Data-profile DBOS workflow — scans the already-staged input tree, runs the
+ * data-profiler sandbox agent over the resulting manifest, registers each staged
+ * file in cortex_artifacts, and indexes the profile's kinds and entities into the
+ * pgvector store. Inputs are staged under `data/inputs/` by the embedder before the
+ * run (see the data-profile-init spec); the body assumes a populated tree and never
+ * downloads.
  *
  * Recoverable: a crashed Cortex pod resumes the workflow from the DBOS step
  * cache. The run authorization is minted at the async edge (`triggerDataProfile`)
@@ -29,7 +30,7 @@ import type { SandboxClient } from "../sandbox/client.js";
 import type { WorkspaceFilesystem } from "../workspace/filesystem.js";
 import { type ResolveWorkspaceRoot } from "../workspace/paths.js";
 import { type ProfilerOutput } from "../schemas/data-profile-schemas.js";
-import { type DataProfileInputFile, type DataProfileResult } from "../state/index.js";
+import { type DataProfileCoverage, type DataProfileResult } from "../state/index.js";
 /** The body's construction-time deps — closed over at registration. */
 export interface DataProfileDeps extends EnvironmentStorePaths {
     /** Operational logging seam; omitted falls back to no-op. */
@@ -92,22 +93,6 @@ export interface DataProfileWorkflowInput {
      */
     readonly ownsMandate?: boolean;
 }
-/**
- * Build the drift-signature comparand a completed profile persists, or `undefined`
- * to omit it. `inputFiles` records whether the profiled files still hold the same
- * bytes; it sits beside the `inputFileIds` audit record (WHICH files) in the stored
- * `DataProfileResult`.
- *
- * A workflow input persisted before `StagedInput.mtimeMs` existed (recovered across
- * the deploy that added it) carries entries with NO `mtimeMs` despite the compile-time
- * type — the same recovered-legacy-input case `ownsMandate` handles. A per-entry
- * `{ …, mtimeMs: undefined }` would JSON.stringify to `{ fileId, size }` and violate
- * `DataProfileInputFile`, so the WHOLE signature is omitted when ANY entry lacks
- * `mtimeMs`: an absent comparand reads back as drift and buys one clean re-profile,
- * exactly as a wholly-absent `result` already does (see `DataProfileResult.inputFiles`).
- * The cast defeats the required-`number` type precisely to observe that legacy gap.
- */
-export declare function buildDriftSignature(stagedInputs: readonly StagedInput[]): DataProfileInputFile[] | undefined;
 /**
  * Project the profiler's structured output plus the staged manifest into the record
  * persisted on `cortex_analysis_state`.
@@ -124,7 +109,7 @@ export declare function buildDriftSignature(stagedInputs: readonly StagedInput[]
  * indistinguishable from a legacy snapshot that predates the field, which is exactly
  * the reading a consumer must already tolerate.
  */
-export declare function buildDataProfileResult(profile: ProfilerOutput, stagedInputs: readonly StagedInput[], profiledAt: string): DataProfileResult;
+export declare function buildDataProfileResult(profile: ProfilerOutput, stagedInputs: readonly StagedInput[], profiledAt: string, coverage?: DataProfileCoverage): DataProfileResult;
 /**
  * Register the data-profile workflow with DBOS. Returns the registered
  * callable so `triggerDataProfile` can dispatch via `DBOS.startWorkflow`.