@inflexa-ai/harness 0.23.0 → 0.25.0

package/dist/sandbox/k8s-client.js

@@ -12,7 +12,7 @@
  * both sides address one directory by construction.
  */
 import { relative as relativePath, sep } from "node:path";
-import { BatchV1Api, CoreV1Api, KubeConfig } from "@kubernetes/client-node";
+import { BatchV1Api, CoreV1Api, KubeConfig, } from "@kubernetes/client-node";
 import { ResultAsync, err, ok } from "neverthrow";
 import { trySandbox } from "./sandbox-error.js";
 import { buildMountPlan, buildSessionSubPaths } from "./mount-plan.js";
@@ -29,18 +29,24 @@ const POD_READY_TIMEOUT_MS = 5 * 60_000;
 const POD_POLL_INTERVAL_MS = 1_000;
 const MANAGED_BY_LABEL = "app.kubernetes.io/managed-by";
 const MANAGED_BY_VALUE = "cortex";
-const OWNER_WORKFLOW_LABEL = "cortex/owner-workflow-id";
+/**
+ * Annotation key of the owning DBOS workflow id. An annotation value has neither
+ * a length cap nor a charset restriction, so the id round-trips verbatim and can
+ * be handed back to `DBOS.getWorkflowStatus`. Deliberately not a label: nothing
+ * selects on it, and a label value could not hold it (see {@link sanitizeLabelValue}).
+ */
+const OWNER_WORKFLOW_ANNOTATION = "cortex/owner-workflow-id";
 const RUN_ID_LABEL = "cortex/run-id";
 const STEP_ID_LABEL = "cortex/step-id";
 const ANALYSIS_ID_LABEL = "cortex/analysis-id";
 /**
  * Coerce an arbitrary identifier into a valid K8s label value:
  * ≤63 chars, `[A-Za-z0-9._-]` only, starting and ending alphanumeric.
- * Workflow ids legitimately contain `:` (exec-id routing packs
- * `workflowId:stepId:fnId`), which is illegal in a label — without this an
- * otherwise-valid Job is rejected with a 422 at admission. Sanitization is a
- * no-op for already-valid ids, so DBOS-keyed lookups (adoption, reaper) match
- * unchanged; only the synthetic, non-DBOS data-profile id is rewritten.
+ * Without it one malformed value gets the whole Job rejected at admission.
+ *
+ * Lossy by construction: `:` becomes `-` and anything past 63 chars is dropped,
+ * neither of which is recoverable. No label value may therefore be used as a
+ * lookup key — see {@link OWNER_WORKFLOW_ANNOTATION}.
  */
 export function sanitizeLabelValue(value) {
     return value
@@ -49,6 +55,10 @@ export function sanitizeLabelValue(value) {
         .slice(0, 63)
         .replace(/[^A-Za-z0-9]+$/, "");
 }
+/** The owner workflow id recorded on a Job, or null if it records none. */
+function readOwnerWorkflowId(metadata) {
+    return metadata?.annotations?.[OWNER_WORKFLOW_ANNOTATION] ?? null;
+}
 function deleteJobIgnoreMissing(batchApi, namespace, name) {
     return trySandbox(async () => {
         await batchApi.deleteNamespacedJob({
@@ -230,11 +240,13 @@ function buildJobSpec(meta, config, identity) {
         metadata: {
             name: sandboxId,
             namespace: config.namespace,
+            annotations: {
+                [OWNER_WORKFLOW_ANNOTATION]: meta.childWorkflowId,
+            },
             labels: {
                 [MANAGED_BY_LABEL]: MANAGED_BY_VALUE,
                 role: "sandbox",
                 "cortex/sandbox-id": sandboxId,
-                [OWNER_WORKFLOW_LABEL]: sanitizeLabelValue(meta.childWorkflowId),
                 [STEP_ID_LABEL]: sanitizeLabelValue(meta.stepId),
                 ...attributionLabels,
             },
@@ -339,8 +351,8 @@ function waitForJobGone(batchApi, namespace, name) {
  * dead prior attempt under the same checkpointed name: delete it, wait for the
  * name to free, and recreate fresh.
  *
- * Adoption (and deletion) is **owner-guarded**: the existing Job's
- * `cortex/owner-workflow-id` label must match this step's `ownerWorkflowId`.
+ * Adoption (and deletion) is **owner-guarded**: the existing Job's recorded
+ * owner workflow id must match this step's `ownerWorkflowId`.
  * Only a recovery re-run carries the same checkpointed identity, so a mismatch
  * means an (astronomically rare) name collision with a *different* step —
  * adopting its pod would HMAC-fail every callback, and deleting its Job would
@@ -363,13 +375,13 @@ function createOrAdoptJob(batchApi, coreApi, namespace, sandboxId, ownerWorkflow
         const existingRead = await trySandbox(() => batchApi.readNamespacedJob({ namespace, name: sandboxId }), createFailed);
         if (existingRead.isErr())
             return err(existingRead.error);
-        const owner = existingRead.value.metadata?.labels?.[OWNER_WORKFLOW_LABEL];
+        const owner = readOwnerWorkflowId(existingRead.value.metadata);
         if (owner !== ownerWorkflowId) {
             return err({
                 type: "name_conflict",
                 op: "k8s.createSandbox",
                 sandboxId,
-                owner: owner ?? null,
+                owner,
             });
         }
         const terminal = await existingPodIsTerminal(coreApi, namespace, sandboxId);
@@ -396,7 +408,7 @@ export function createK8sSandboxOps(config) {
             return new ResultAsync((async () => {
                 const { sandboxId } = identity;
                 const job = buildJobSpec(meta, config, identity);
-                const adopted = await createOrAdoptJob(batchApi, coreApi, config.namespace, sandboxId, sanitizeLabelValue(meta.childWorkflowId), job);
+                const adopted = await createOrAdoptJob(batchApi, coreApi, config.namespace, sandboxId, meta.childWorkflowId, job);
                 if (adopted.isErr())
                     return err(adopted.error);
                 // A Job whose pod never schedules (quota rejection, no nodes, spot
@@ -452,40 +464,44 @@ export function createK8sSandboxOps(config) {
                 const ts = j.metadata?.creationTimestamp;
                 return {
                     sandboxId: labels["cortex/sandbox-id"] ?? j.metadata?.name ?? "",
-                    ownerWorkflowId: labels[OWNER_WORKFLOW_LABEL] ?? null,
+                    ownerWorkflowId: readOwnerWorkflowId(j.metadata),
                     createdAtMs: ts ? new Date(ts).getTime() : null,
                 };
             })
                 .filter((s) => s.sandboxId.length > 0));
         },
         isAlive(ref) {
-            return trySandbox(() => coreApi.listNamespacedPod({
-                namespace: config.namespace,
-                labelSelector: `cortex/sandbox-id=${ref.sandboxId}`,
-            }), (status, cause) => ({
-                type: "liveness_failed",
-                op: "k8s.isAlive",
-                sandboxId: ref.sandboxId,
-                status,
-                cause,
-            }))
-                .map((pods) => {
-                const pod = pods.items[0];
-                if (!pod)
-                    return { alive: false, oomKilled: false };
-                const phase = pod.status?.phase;
-                // `Pending` and `Running` are alive; `Succeeded`, `Failed`,
-                // `Unknown` are dead. `Pending` covers startup.
-                const alive = phase === "Pending" || phase === "Running";
-                const oomKilled = !alive &&
-                    (pod.status?.containerStatuses ?? []).some((cs) => cs.state?.terminated?.reason === "OOMKilled" || cs.lastState?.terminated?.reason === "OOMKilled");
-                return { alive, oomKilled };
-            })
-                .orElse((e) => 
-            // A 404 means the pod is gone — observably dead.
-            statusOf(e) === 404 ? ok({ alive: false, oomKilled: false }) : err(e));
+            return isAliveById(ref.sandboxId);
         },
+        isAliveById,
     };
+    function isAliveById(sandboxId) {
+        return trySandbox(() => coreApi.listNamespacedPod({
+            namespace: config.namespace,
+            labelSelector: `cortex/sandbox-id=${sandboxId}`,
+        }), (status, cause) => ({
+            type: "liveness_failed",
+            op: "k8s.isAlive",
+            sandboxId,
+            status,
+            cause,
+        }))
+            .map((pods) => {
+            const pod = pods.items[0];
+            if (!pod)
+                return { alive: false, oomKilled: false };
+            const phase = pod.status?.phase;
+            // `Pending` and `Running` are alive; `Succeeded`, `Failed`,
+            // `Unknown` are dead. `Pending` covers startup.
+            const alive = phase === "Pending" || phase === "Running";
+            const oomKilled = !alive &&
+                (pod.status?.containerStatuses ?? []).some((cs) => cs.state?.terminated?.reason === "OOMKilled" || cs.lastState?.terminated?.reason === "OOMKilled");
+            return { alive, oomKilled };
+        })
+            .orElse((e) => 
+        // A 404 means the pod is gone — observably dead.
+        statusOf(e) === 404 ? ok({ alive: false, oomKilled: false }) : err(e));
+    }
 }
 /**
  * Best-effort Job removal after a failed create. The original failure is what

package/dist/sandbox/reaper.d.ts

@@ -5,9 +5,10 @@
  * Distinct from the liveness watchdog (`sandbox/watchdog.ts`): the watchdog
  * sweeps registry→cluster and unblocks stuck recvs; the reaper sweeps
  * cluster→registry and garbage-collects. It lists every Cortex-managed sandbox
- * machine in the configured namespace, reads each machine's owner-workflow-id
- * label, and reaps any whose owning workflow is terminal/missing — tearing the
- * machine down and reconciling its step row.
+ * machine in the configured namespace, resolves each machine's recorded owner
+ * workflow, and reaps the ones whose owner is terminal — tearing the machine
+ * down and reconciling its step row. A machine whose owner does not resolve at
+ * all is torn down only once a liveness probe says it is dead.
  *
  * Why a sweep and not teardown-on-cancel: DBOS forbids running a step in a
  * cancelled workflow, so the body cannot tear down on the cancel path; and only
@@ -22,14 +23,18 @@ import type { Pool } from "pg";
 import type { Logger } from "../lib/logger.js";
 import type { SandboxClient } from "./client.js";
 import type { ManagedSandbox } from "./types.js";
-export type ReapDecision = "leave" | "reap";
+export type ReapDecision = "leave" | "reap" | "reap-if-dead";
 /**
- * Decide whether a managed machine is reapable. The owning workflow's status
- * is authoritative — DBOS writes it at enqueue, *before* the machine exists, so
- * a just-spawned machine is always observably in-flight and never mistaken for
- * an orphan. Only the no-status case (no owner label, or the workflow is gone)
- * falls back to a creation-time grace, the one window the status signal can't
- * cover.
+ * Decide whether a managed machine is reapable. A resolved owner status is
+ * authoritative in both directions — DBOS records it at enqueue, *before* the
+ * machine exists, so an in-flight workflow's machine is always observably
+ * in-flight and a terminal one's is always reapable.
+ *
+ * A missing status is authoritative for neither. It means the machine records
+ * no owner, or the id it records did not resolve, or DBOS pruned the row —
+ * and age cannot separate those from a genuine orphan. So the decision defers
+ * to a liveness probe: only an observably dead machine is torn down. Reaping a
+ * live machine on age alone is what this branch exists to prevent.
  */
 export declare function classifyManaged(sb: ManagedSandbox, status: {
     status: string;
@@ -38,11 +43,11 @@ export declare function classifyManaged(sb: ManagedSandbox, status: {
 export declare function terminalStepStatus(workflowStatus: string | null): "canceled" | "failed" | "completed";
 export interface ReaperDeps {
     pool: Pool;
-    sandboxClient: Pick<SandboxClient, "listManagedSandboxes" | "teardownById">;
+    sandboxClient: Pick<SandboxClient, "listManagedSandboxes" | "teardownById" | "isAliveById">;
     getStatus: (workflowId: string) => Promise<{
         status: string;
     } | null>;
-    /** Creation-time grace for label-less / orphaned machines. */
+    /** Creation-time grace before an unattributable machine is even probed. */
     graceMs?: number;
     /** Wall-clock ms; injected for tests. */
     nowMs?: () => number;
@@ -53,6 +58,12 @@ export interface ReapSummary {
     reapedCount: number;
     rowsReconciled: number;
     leftCount: number;
+    /**
+     * Machines past the grace whose owner did not resolve but which are still
+     * alive, so they were left standing. A non-zero value is a standing leak
+     * *and* a bug signal — it means owner ids are not round-tripping.
+     */
+    liveUnattributed: number;
 }
 /**
  * One cluster→registry sweep. Lists managed machines, classifies each against
@@ -62,7 +73,7 @@ export interface ReapSummary {
 export declare function reapOnce(deps: ReaperDeps): Promise<ReapSummary>;
 export interface RegisterReaperDeps {
     pool: Pool;
-    sandboxClient: Pick<SandboxClient, "listManagedSandboxes" | "teardownById">;
+    sandboxClient: Pick<SandboxClient, "listManagedSandboxes" | "teardownById" | "isAliveById">;
     graceMs?: number;
     logger?: Logger;
 }

package/dist/sandbox/reaper.js

@@ -5,9 +5,10 @@
  * Distinct from the liveness watchdog (`sandbox/watchdog.ts`): the watchdog
  * sweeps registry→cluster and unblocks stuck recvs; the reaper sweeps
  * cluster→registry and garbage-collects. It lists every Cortex-managed sandbox
- * machine in the configured namespace, reads each machine's owner-workflow-id
- * label, and reaps any whose owning workflow is terminal/missing — tearing the
- * machine down and reconciling its step row.
+ * machine in the configured namespace, resolves each machine's recorded owner
+ * workflow, and reaps the ones whose owner is terminal — tearing the machine
+ * down and reconciling its step row. A machine whose owner does not resolve at
+ * all is torn down only once a liveness probe says it is dead.
  *
  * Why a sweep and not teardown-on-cancel: DBOS forbids running a step in a
  * cancelled workflow, so the body cannot tear down on the cancel path; and only
@@ -33,20 +34,22 @@ const DEFAULT_GRACE_MS = 10 * 60_000;
  */
 const ACTIVE_WORKFLOW_STATUSES = new Set(["PENDING", "ENQUEUED", "RUNNING"]);
 /**
- * Decide whether a managed machine is reapable. The owning workflow's status
- * is authoritative — DBOS writes it at enqueue, *before* the machine exists, so
- * a just-spawned machine is always observably in-flight and never mistaken for
- * an orphan. Only the no-status case (no owner label, or the workflow is gone)
- * falls back to a creation-time grace, the one window the status signal can't
- * cover.
+ * Decide whether a managed machine is reapable. A resolved owner status is
+ * authoritative in both directions — DBOS records it at enqueue, *before* the
+ * machine exists, so an in-flight workflow's machine is always observably
+ * in-flight and a terminal one's is always reapable.
+ *
+ * A missing status is authoritative for neither. It means the machine records
+ * no owner, or the id it records did not resolve, or DBOS pruned the row —
+ * and age cannot separate those from a genuine orphan. So the decision defers
+ * to a liveness probe: only an observably dead machine is torn down. Reaping a
+ * live machine on age alone is what this branch exists to prevent.
  */
 export function classifyManaged(sb, status, nowMs, graceMs) {
-    if (status && ACTIVE_WORKFLOW_STATUSES.has(status.status))
-        return "leave";
     if (status)
-        return "reap";
+        return ACTIVE_WORKFLOW_STATUSES.has(status.status) ? "leave" : "reap";
     const ageMs = sb.createdAtMs == null ? Number.POSITIVE_INFINITY : nowMs - sb.createdAtMs;
-    return ageMs >= graceMs ? "reap" : "leave";
+    return ageMs >= graceMs ? "reap-if-dead" : "leave";
 }
 /** Map the owning workflow's terminal status to the step row's terminal status. */
 export function terminalStepStatus(workflowStatus) {
@@ -73,12 +76,41 @@ export async function reapOnce(deps) {
     let reapedCount = 0;
     let rowsReconciled = 0;
     let leftCount = 0;
+    let liveUnattributed = 0;
     for (const sb of managed) {
         const status = sb.ownerWorkflowId ? await deps.getStatus(sb.ownerWorkflowId) : null;
-        if (classifyManaged(sb, status, nowMs, graceMs) === "leave") {
+        const decision = classifyManaged(sb, status, nowMs, graceMs);
+        if (decision === "leave") {
             leftCount++;
             continue;
         }
+        if (decision === "reap-if-dead") {
+            // A recorded id that resolves to nothing is an anomaly worth naming: either
+            // DBOS pruned the row or the id never reached the machine intact.
+            if (sb.ownerWorkflowId) {
+                logger.warn("owner workflow id resolved to no workflow", { sandboxId: sb.sandboxId, ownerWorkflowId: sb.ownerWorkflowId });
+            }
+            let liveness;
+            try {
+                liveness = await deps.sandboxClient.isAliveById(sb.sandboxId);
+            }
+            catch (err) {
+                // A probe that cannot answer is not a dead machine. Leave it; the next
+                // sweep re-probes.
+                logger.warn("liveness probe threw — leaving this machine", { sandboxId: sb.sandboxId, ...logger.errorFields(err) });
+                leftCount++;
+                continue;
+            }
+            if (liveness.alive) {
+                logger.warn("machine has no resolvable owner but is still alive — left standing", {
+                    sandboxId: sb.sandboxId,
+                    ownerWorkflowId: sb.ownerWorkflowId,
+                });
+                leftCount++;
+                liveUnattributed++;
+                continue;
+            }
+        }
         try {
             await deps.sandboxClient.teardownById(sb.sandboxId);
             reapedCount++;
@@ -98,6 +130,7 @@ export async function reapOnce(deps) {
         reapedCount,
         rowsReconciled,
         leftCount,
+        liveUnattributed,
     };
     // Spread: an interface has no implicit index signature, so it needs widening to `LogFields`.
     logger.info("sweep completed", { ...summary });

package/dist/sandbox/types.d.ts

@@ -229,8 +229,8 @@ export interface CreateSandboxMeta {
     /** The first `execId` that will fire against this sandbox; nullable for
      *  early-create flows where the workflow mints the first execId later. */
     execId?: string | null;
-    /** Owning DBOS child workflow id (`"${parentRunId}-${N}"`). Stamped onto the
-     *  sandbox machine as the `cortex/owner-workflow-id` label so the reaper can
+    /** Owning DBOS child workflow id (`"${parentRunId}-${N}"`). Recorded verbatim
+     *  on the sandbox machine under `cortex/owner-workflow-id` so the reaper can
      *  map a cluster-side machine back to its workflow and check liveness. */
     childWorkflowId: string;
     /** Backend-specific extras carried through to the per-backend impl. */
@@ -270,9 +270,10 @@ export interface SandboxIdentity {
 /**
  * A sandbox machine the backend is running that Cortex *manages*
  * (`app.kubernetes.io/managed-by=cortex`), enumerated by the reaper for the
- * cluster→registry sweep (see the harness-sandbox-exec spec). The `ownerWorkflowId` comes from the
- * `cortex/owner-workflow-id` label and may be null for legacy/label-less
- * machines, which are only reaped past a creation-time grace.
+ * cluster→registry sweep (see the harness-sandbox-exec spec). `ownerWorkflowId`
+ * is the id as minted, so it is usable as a DBOS lookup key; null means the
+ * machine records no owner, and such a machine is reaped only past a
+ * creation-time grace and only when observably dead.
  */
 export interface ManagedSandbox {
     sandboxId: string;

package/dist/schemas/data-profile-schemas.d.ts

@@ -47,6 +47,53 @@ export declare const OrganismSchema: z.ZodObject<{
     notes: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
 export type Organism = z.infer<typeof OrganismSchema>;
+/** Kinds an output may carry. A tree needing more than this is not being grouped. */
+export declare const MAX_KINDS = 30;
+/** Axes an output may carry — an experimental design with more varying dimensions is vanishingly rare. */
+export declare const MAX_AXES = 8;
+/**
+ * Individually described files an output may carry. `files` holds NOTABLE SINGLETONS
+ * — the metadata sheet, the README, the paper, an outlier that fits no kind — not the
+ * dataset's contents.
+ */
+export declare const MAX_NOTABLE_FILES = 50;
+/**
+ * A repeating set of files that are the same sort of thing.
+ *
+ * A kind is a claim about MEANING, so only the agent can author one: the input scan
+ * reports shapes (these filenames differ only here) and cannot establish that the
+ * files are the same sort of thing. Kinds need not correspond one-to-one with observed
+ * shapes — one kind may span several shapes, and one shape may split into several
+ * kinds.
+ *
+ * A singleton is a kind of `count` 1. There is one concept with a count, not two
+ * concepts with a threshold between them, so degenerate datasets need no special case.
+ */
+export declare const KindSchema: z.ZodObject<{
+    name: z.ZodString;
+    memberRepresents: z.ZodString;
+    description: z.ZodString;
+    count: z.ZodNumber;
+    pathPattern: z.ZodString;
+    format: z.ZodOptional<z.ZodString>;
+    axisLabels: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export type ProfilerKind = z.infer<typeof KindSchema>;
+/**
+ * What varies across a kind's members.
+ *
+ * The scan reports that a filename position varies and which values it takes; what the
+ * variation IS — a subject, a timepoint, a treatment arm, a chromosome shard — is not
+ * derivable from the values, so the label is required and agent-supplied. An axis
+ * evident from a metadata sheet but not from filenames belongs here too.
+ */
+export declare const AxisSchema: z.ZodObject<{
+    label: z.ZodString;
+    cardinality: z.ZodNumber;
+    exampleValues: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    description: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export type ProfilerAxis = z.infer<typeof AxisSchema>;
 export declare const ProfilerOutputSchema: z.ZodObject<{
     files: z.ZodArray<z.ZodObject<{
         path: z.ZodString;
@@ -59,6 +106,21 @@ export declare const ProfilerOutputSchema: z.ZodObject<{
         warnings: z.ZodOptional<z.ZodArray<z.ZodString>>;
         metrics: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
     }, z.core.$strip>>;
+    kinds: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        memberRepresents: z.ZodString;
+        description: z.ZodString;
+        count: z.ZodNumber;
+        pathPattern: z.ZodString;
+        format: z.ZodOptional<z.ZodString>;
+        axisLabels: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strip>>;
+    axes: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        label: z.ZodString;
+        cardinality: z.ZodNumber;
+        exampleValues: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        description: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>>;
     analysisSummary: z.ZodString;
     domain: z.ZodString;
     subtype: z.ZodOptional<z.ZodString>;

package/dist/schemas/data-profile-schemas.js

@@ -24,7 +24,7 @@ export const ProfilerFileSchema = z.object({
     metrics: z
         .record(z.string(), z.union([z.string(), z.number(), z.boolean()]))
         .optional()
-        .describe("Format-specific profiling metrics as flat key-value pairs (e.g. sparsity, medianLibrarySize, tiTvRatio, gcContent, missingRate)"),
+        .describe("Identity-establishing facts as flat key-value pairs (e.g. sparsity, delimiter, normalizationState, missingRate)"),
 });
 /**
  * Where an extracted subject fact came from. Used so downstream agents can
@@ -48,8 +48,74 @@ export const OrganismSchema = z.object({
         "low: inferred from gene IDs or other indirect signals, OR sources disagree (in which case also record the conflict in qualityAssessment.concerns)."),
     notes: z.string().optional().describe("Brief note when source is 'inferred' or confidence is 'low' (one sentence)."),
 });
+/** Kinds an output may carry. A tree needing more than this is not being grouped. */
+export const MAX_KINDS = 30;
+/** Axes an output may carry — an experimental design with more varying dimensions is vanishingly rare. */
+export const MAX_AXES = 8;
+/**
+ * Individually described files an output may carry. `files` holds NOTABLE SINGLETONS
+ * — the metadata sheet, the README, the paper, an outlier that fits no kind — not the
+ * dataset's contents.
+ */
+export const MAX_NOTABLE_FILES = 50;
+/**
+ * A repeating set of files that are the same sort of thing.
+ *
+ * A kind is a claim about MEANING, so only the agent can author one: the input scan
+ * reports shapes (these filenames differ only here) and cannot establish that the
+ * files are the same sort of thing. Kinds need not correspond one-to-one with observed
+ * shapes — one kind may span several shapes, and one shape may split into several
+ * kinds.
+ *
+ * A singleton is a kind of `count` 1. There is one concept with a count, not two
+ * concepts with a threshold between them, so degenerate datasets need no special case.
+ */
+export const KindSchema = z.object({
+    name: z.string().describe("Short label for this set, e.g. 'per-patient variant calls', 'reference transcriptome'."),
+    memberRepresents: z
+        .string()
+        .describe("What ONE member of this set represents — 'one patient's somatic variant calls', 'one sequencing lane of one sample'. " +
+        "This is your grouping decision stated outright, and it is NOT the description: it cannot be answered by " +
+        "restating the shape the scan observed."),
+    description: z.string().describe("What this set contains and the role it plays in the analysis."),
+    count: z.number().int().describe("How many files are in this set."),
+    pathPattern: z
+        .string()
+        .describe("Glob matching this set's members, relative to the analysis root (e.g. 'data/inputs/vcf/*.vcf.gz'). " +
+        "Coverage is computed by matching this against the scanned tree, so a pattern that matches nothing reads as an uncovered kind."),
+    format: z.string().optional().describe("File format shared by the members (VCF, h5ad, CSV, …)."),
+    axisLabels: z
+        .array(z.string())
+        .max(MAX_AXES)
+        .optional()
+        .describe("Labels of the axes (see `axes`) that vary across this set's members — the design this kind participates in."),
+});
+/**
+ * What varies across a kind's members.
+ *
+ * The scan reports that a filename position varies and which values it takes; what the
+ * variation IS — a subject, a timepoint, a treatment arm, a chromosome shard — is not
+ * derivable from the values, so the label is required and agent-supplied. An axis
+ * evident from a metadata sheet but not from filenames belongs here too.
+ */
+export const AxisSchema = z.object({
+    label: z.string().describe("What this dimension is: 'patient', 'timepoint', 'treatment arm', 'chromosome shard', 'replicate'."),
+    cardinality: z.number().int().describe("How many distinct values this dimension takes."),
+    exampleValues: z.array(z.string()).max(20).optional().describe("A few observed values, for orientation."),
+    description: z.string().optional().describe("Anything a downstream planner needs about this dimension (ordering, pairing, imbalance)."),
+});
 export const ProfilerOutputSchema = z.object({
-    files: z.array(ProfilerFileSchema),
+    /**
+     * Notable singletons only. The workspace filesystem is the authoritative file list
+     * — `list_files`, `grep`, and the vector index all read the live tree — so a record
+     * per input file would store a stale duplicate of something that cannot go wrong.
+     */
+    files: z.array(ProfilerFileSchema).max(MAX_NOTABLE_FILES),
+    kinds: z
+        .array(KindSchema)
+        .max(MAX_KINDS)
+        .describe("The repeating sets this dataset is made of. Group; do not enumerate members — the cap is a hard bound, not a target."),
+    axes: z.array(AxisSchema).max(MAX_AXES).optional().describe("What varies across the kinds' members — the experimental design, as far as it is observable."),
     analysisSummary: z.string().describe("Narrative overview of the dataset — structure, content, analytical potential, and limitations"),
     domain: z
         .string()

package/dist/state/data-profile.d.ts

@@ -5,98 +5,17 @@
  */
 import type { ResultAsync } from "neverthrow";
 import { type DbError } from "../lib/db-result.js";
-import type { Organism } from "../schemas/data-profile-schemas.js";
+import type { DataProfileLifecycleStatus, DataProfileResult } from "../contracts/data-profile.js";
 import type { Querier } from "./db.js";
 /**
- * One profiled input file's **drift signature**. `fileId` is a path identity the
- * embedder derives (two files at the same path share it regardless of content),
- * so `size` + `mtimeMs` are what let a consumer notice that the bytes behind a
- * path changed. The harness treats all three as opaque labels: it persists them
- * verbatim and never stats a source file or compares signatures itself.
+ * The persisted profile's shape lives in `contracts/` — it is read by consumers outside
+ * this package (a host route, a UI), and they must not import this module, which pulls in
+ * `pg` and the whole ledger query surface, to obtain a type. Re-exported here so ledger
+ * call sites keep one import.
  */
-export interface DataProfileInputFile {
-    fileId: string;
-    size: number;
-    mtimeMs: number;
-}
-/**
- * One profiled file's persisted record.
- *
- * `path` + `description` are the original pair every snapshot carries. Everything
- * below them is **optional on read** for the same reason `DataProfileResult.inputFiles`
- * is: a snapshot written before the record was widened carries only the pair, and a
- * reader must render that row rather than reject it. A writer fills in whatever the
- * profiler reported.
- */
-export interface DataProfileFile {
-    path: string;
-    description: string;
-    /** Semantic data type (count-matrix, variants, clinical-metadata, document, …). */
-    dataType?: string;
-    /** File format (CSV, h5ad, VCF, BAM, …). */
-    format?: string;
-    rows?: number | null;
-    cols?: number | null;
-    /** Searchable labels the profiler attached. */
-    tags?: string[];
-    /** Quality issues specific to this file. */
-    warnings?: string[];
-    /** Flat format-specific metrics (sparsity, medianLibrarySize, tiTvRatio, …). */
-    metrics?: Record<string, string | number | boolean>;
-}
-/** Dataset-wide quality findings the profiler recorded. */
-export interface DataProfileQualityAssessment {
-    concerns: string[];
-    strengths: string[];
-}
-/**
- * The profile snapshot `completeDataProfile` persists and `loadDataProfileStatus` reads back.
- *
- * This row is the **only** durable home of the profile: the profiler's `runs/data-profile/`
- * scratch tree is wiped when profiling completes, so there is no file to fall back on. It
- * therefore carries the profiler's full finding, not a summary of it — anything dropped here
- * is destroyed, and a downstream agent can only recover it by re-reading the raw input files.
- *
- * Every field past `summary` / `files` / `inputFileIds` / `profiledAt` is **optional on read**:
- * a snapshot written before the record was widened carries only those four, and a reader must
- * render it rather than throw. There is no parse at the read boundary (the jsonb column is cast
- * straight to this type), so optionality *is* the compatibility mechanism — exactly as it
- * already is for `inputFiles`.
- */
-export interface DataProfileResult {
-    summary: string;
-    files: DataProfileFile[];
-    /** Which files the profile covered — the audit record. */
-    inputFileIds: string[];
-    /**
-     * Whether the same bytes were covered — the drift comparand. Optional on read:
-     * a snapshot written before this field existed carries only `inputFileIds`, and
-     * a consumer treats the absence as drift (re-profiling repairs it), exactly as
-     * it already treats a wholly absent `result`.
-     */
-    inputFiles?: DataProfileInputFile[];
-    profiledAt: string;
-    /** Scientific domain (transcriptomics, cheminformatics, clinical, …). */
-    domain?: string;
-    /** Specific subtype within the domain (bulk-rna-seq, single-cell, LC-MS/MS, …). */
-    subtype?: string;
-    /**
-     * Subject organism with its provenance and confidence. Explicit `null` is a real,
-     * distinct value — the profiler looked and no input identified an organism — as
-     * opposed to absent, which means the snapshot predates this field.
-     */
-    organism?: Organism | null;
-    tissue?: string | null;
-    cellType?: string | null;
-    condition?: string | null;
-    /** Public dataset accessions found in any input (GSE/SRP/PRJNA/E-MTAB/phs/EGAS). */
-    accessions?: string[];
-    /** Conditions, groups, comparisons, replicates, pairing. */
-    experimentalDesign?: string;
-    qualityAssessment?: DataProfileQualityAssessment;
-}
+export type { DataProfileAxis, DataProfileCoverage, DataProfileFile, DataProfileInputFile, DataProfileInputSignature, DataProfileKind, DataProfileOrganism, DataProfileQualityAssessment, DataProfileLifecycleStatus, DataProfileResult, DataProfileSubjectSource, } from "../contracts/data-profile.js";
 export interface DataProfileStatus {
-    status: "pending" | "running" | "completed" | "failed";
+    status: DataProfileLifecycleStatus;
     error: string | null;
     startedAt: string | null;
     completedAt: string | null;

package/dist/state/index.d.ts

@@ -21,7 +21,7 @@ export type { ActiveSandboxRow } from "./active-sandboxes.js";
 export { insertPlan, upsertPlan, loadPlan } from "./plans.js";
 export type { InsertPlanInput, UpsertPlanInput } from "./plans.js";
 export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
-export type { DataProfileStatus, DataProfileResult, DataProfileInputFile, DataProfileFile, DataProfileQualityAssessment } from "./data-profile.js";
+export type { DataProfileStatus, DataProfileResult, DataProfileInputFile, DataProfileFile, DataProfileQualityAssessment, DataProfileKind, DataProfileAxis, DataProfileInputSignature, DataProfileCoverage, } from "./data-profile.js";
 export { queryRunCountsByAnalyses, queryThreadCountsByAnalyses, queryDataProfileStatusByAnalyses } from "./analyses-metrics.js";
 export { createAnalysisPurge } from "./purge-analysis.js";
 export type { AnalysisPurge, AnalysisPurgeDeps, AnalysisPurgeOutcome } from "./purge-analysis.js";