@inflexa-ai/harness 0.28.1 → 0.29.0

package/dist/schemas/data-profile-schemas.js

@@ -1,31 +1,43 @@
 /**
- * Schemas for data profiling.
+ * The profiler's authoring contract.
  *
- * Defines the profiler structured output schema used by the data-profile
- * task and sandbox agent.
+ * What the agent submits is JUDGEMENT, never enumeration: operations on the scan's menu,
+ * what each resulting group means, and the dimensions it saw with the evidence it saw
+ * them in. Membership, counts, and display patterns are computed at resolution from the
+ * scanner's own templates, so no field here carries a count or a path pattern — those are
+ * unrepresentable rather than discouraged, because a discouraged field is a field that
+ * gets filled.
+ *
+ * Every array is bounded, and the bound is the enforcement: an over-long submission is a
+ * schema error the agent corrects, never a silent truncation. The governing invariant is
+ * that nothing the agent authors grows with the number of input files.
+ *
+ * The category enums are derived from the shipped vocabulary (`contracts/profile-vocabulary`),
+ * so the catalogue this validates against and the catalogue the prompt renders are one list.
  */
 import { z } from "zod";
-// ── Profiler structured output (sandbox agent) ─────────────────────
-/** Per-file metadata — includes path so the agent can discover files itself. */
-export const ProfilerFileSchema = z.object({
-    path: z.string().describe("File path relative to analysis root (e.g. data/inputs/{fileId}/counts.txt)"),
-    description: z.string().describe("What this file contains and its role in the analysis"),
-    dataType: z
-        .string()
-        .describe("Semantic data type: count-matrix, normalized-expression, variants, alignments, " +
-        "clinical-metadata, clinical-sdtm, clinical-adam, clinical-response, " +
-        "pharmacokinetic-data, adverse-events, safety-labs, " +
-        "molecular-structures, annotations, document (paper PDF, README, methods doc)"),
-    format: z.string().describe("File format (e.g. CSV, TSV, h5ad, MTX, VCF, BAM, SDF, FASTQ)"),
-    rows: z.number().nullish(),
-    cols: z.number().nullish(),
-    tags: z.array(z.string()).optional().describe("Searchable labels for downstream discovery"),
-    warnings: z.array(z.string()).optional().describe("Quality issues or concerns specific to this file"),
-    metrics: z
-        .record(z.string(), z.union([z.string(), z.number(), z.boolean()]))
-        .optional()
-        .describe("Identity-establishing facts as flat key-value pairs (e.g. sparsity, delimiter, normalizationState, missingRate)"),
-});
+import { DIMENSION_CATEGORY_IDS, DIMENSION_PROBE_IDS, GROUP_CATEGORY_IDS, GROUP_ROLE_IDS } from "../contracts/profile-vocabulary.js";
+// ── Bounds ─────────────────────────────────────────────────────────────────
+/** Operations one submission may carry — the menu's own bound. */
+export const MAX_OPERATIONS = 40;
+/** Groups one split may produce. A slot needing more is an identifier, and identifiers are never split. */
+export const MAX_SPLIT_GROUPS = 12;
+/** Slot values one split group may name. */
+export const MAX_SPLIT_VALUES = 50;
+export const MAX_MERGE_SETS = 12;
+/** Paths one explicit `group` may gather — the leftover aggregate, not the dataset. */
+export const MAX_EXPLICIT_PATHS = 50;
+/** Members of one group the agent may annotate individually. Never required to be exhaustive. */
+export const MAX_MEMBER_ANNOTATIONS = 25;
+/** Dataset-level dimensions. An experimental design with more varying dimensions is vanishingly rare. */
+export const MAX_DIMENSIONS = 12;
+export const MAX_OBSERVATIONS = 6;
+export const MAX_RECONCILIATIONS = 4;
+export const MAX_EXAMPLE_VALUES = 5;
+/** Files a `not-found` probe may name as searched — the probe-searchable set is bounded by design. */
+export const MAX_SEARCHED_FILES = 10;
+export const MAX_CAVEATS = 10;
+// ── Dataset identity ───────────────────────────────────────────────────────
 /**
  * Where an extracted subject fact came from. Used so downstream agents can
  * judge how much to trust a field and so this profiler can flag conflicts.
@@ -33,7 +45,7 @@ export const ProfilerFileSchema = z.object({
 export const SubjectSourceEnum = z.enum([
     "metadata", // sample-sheet/header column, internal manifest
     "document", // paper PDF, README, methods doc
-    "filename", // file or folder naming convention (GSExxx, "human_", etc.)
+    "filename", // file or folder naming convention
     "user-context", // user-supplied analysis context
     "inferred", // inferred from data content (e.g. gene symbol patterns)
 ]);
@@ -45,113 +57,286 @@ export const OrganismSchema = z.object({
         .enum(["high", "medium", "low"])
         .describe("high: explicit user-context statement, an organism column, or a paper/README statement. " +
         "medium: filename convention or accession lookup. " +
-        "low: inferred from gene IDs or other indirect signals, OR sources disagree (in which case also record the conflict in qualityAssessment.concerns)."),
+        "low: inferred from gene IDs or other indirect signals, OR sources disagree (in which case also record the conflict in caveats)."),
     notes: z.string().optional().describe("Brief note when source is 'inferred' or confidence is 'low' (one sentence)."),
 });
-/** Kinds an output may carry. A tree needing more than this is not being grouped. */
-export const MAX_KINDS = 30;
-/** Axes an output may carry — an experimental design with more varying dimensions is vanishingly rare. */
-export const MAX_AXES = 8;
+// ── Group annotations ──────────────────────────────────────────────────────
+export const MemberAnnotationSchema = z
+    .object({
+    path: z.string().describe("Path of the member, relative to the analysis root."),
+    note: z.string().describe("What is individually notable about THIS member — what you read in it, or why it stands out from its group."),
+})
+    .strict();
 /**
- * Individually described files an output may carry. `files` holds NOTABLE SINGLETONS
- * — the metadata sheet, the README, the paper, an outlier that fits no kind — not the
- * dataset's contents.
- */
-export const MAX_NOTABLE_FILES = 50;
-/**
- * A repeating set of files that are the same sort of thing.
- *
- * A kind is a claim about MEANING, so only the agent can author one: the input scan
- * reports shapes (these filenames differ only here) and cannot establish that the
- * files are the same sort of thing. Kinds need not correspond one-to-one with observed
- * shapes — one kind may span several shapes, and one shape may split into several
- * kinds.
+ * What a group IS, stated by the agent.
  *
- * A singleton is a kind of `count` 1. There is one concept with a count, not two
- * concepts with a threshold between them, so degenerate datasets need no special case.
+ * `.strict()` because the fields NOT here are the point: a count, a path pattern, and a
+ * member list are all computed at resolution from the operation's own membership, and a
+ * submission that carries one is rejected rather than quietly overridden.
  */
-export const KindSchema = z.object({
-    name: z.string().describe("Short label for this set, e.g. 'per-patient variant calls', 'reference transcriptome'."),
+export const GroupAnnotationSchema = z
+    .object({
+    name: z.string().describe("Short label for this group, e.g. 'per-subject somatic calls', 'the sample sheet'."),
     memberRepresents: z
         .string()
-        .describe("What ONE member of this set represents — 'one patient's somatic variant calls', 'one sequencing lane of one sample'. " +
-        "This is your grouping decision stated outright, and it is NOT the description: it cannot be answered by " +
-        "restating the shape the scan observed."),
-    description: z.string().describe("What this set contains and the role it plays in the analysis."),
-    count: z.number().int().describe("How many files are in this set."),
-    pathPattern: z
+        .describe("What ONE member of this group represents — 'one subject's somatic variant calls', 'one sequencing lane of one sample'. " +
+        "This is your grouping decision stated outright, and it is NOT the description: it cannot be answered by restating " +
+        "an observation the scan already made."),
+    description: z.string().describe("What this group contains and the role it plays in the analysis."),
+    role: z.enum(GROUP_ROLE_IDS).describe("The group's role in the dataset, from the shipped vocabulary."),
+    category: z
+        .enum(GROUP_CATEGORY_IDS)
+        .describe("The group's category, from the shipped vocabulary. Pick the most specific category that covers ALL members — " +
+        "members straddling two categories are usually a split you have not made yet. 'other' is allowed and monitored: " +
+        "a wrong specific category is worse than an honest 'other' with a good label."),
+    categoryLabel: z.string().optional().describe("Required when category is 'other': what this group actually is."),
+    subtype: z.string().optional().describe("Free refinement below the category (e.g. 'spatial', 'imaging MS', 'targeted panel')."),
+    categoryReason: z.string().optional().describe("Why you overrode a pre-suggested category, or chose one the anti-overlap notes make arguable."),
+    memberAnnotations: z
+        .array(MemberAnnotationSchema)
+        .max(MAX_MEMBER_ANNOTATIONS)
+        .optional()
+        .describe("Individually notable members. Never a member list — annotate the ones you read or that stand out, and no others."),
+})
+    .strict();
+// ── Menu operations ────────────────────────────────────────────────────────
+const UseOperationSchema = z
+    .object({
+    op: z.literal("use"),
+    setId: z.string().describe("A set id from the menu."),
+    group: GroupAnnotationSchema,
+})
+    .strict();
+const SplitBySlotSchema = z
+    .object({
+    kind: z.literal("slot"),
+    slotId: z.string().describe("A slot id of the named set. One group is resolved per distinct value of this slot."),
+    group: GroupAnnotationSchema.describe("The annotation every resulting group carries; the slot value distinguishes them."),
+})
+    .strict();
+const SplitByValuesSchema = z
+    .object({
+    kind: z.literal("values"),
+    slotId: z.string().describe("A slot id of the named set, whose values the mapping below assigns to groups."),
+    groups: z
+        .array(z
+        .object({
+        values: z.array(z.string()).min(1).max(MAX_SPLIT_VALUES).describe("Slot values, verbatim as the menu reported them."),
+        group: GroupAnnotationSchema,
+    })
+        .strict())
+        .min(2)
+        .max(MAX_SPLIT_GROUPS),
+    rest: GroupAnnotationSchema.optional().describe("Members whose slot value no mapping named. Omit only when the mapping is exhaustive."),
+})
+    .strict();
+const SplitOperationSchema = z
+    .object({
+    op: z.literal("split"),
+    setId: z.string(),
+    by: z.discriminatedUnion("kind", [SplitBySlotSchema, SplitByValuesSchema]),
+    reason: z
         .string()
-        .describe("Glob matching this set's members, relative to the analysis root (e.g. 'data/inputs/vcf/*.vcf.gz'). " +
-        "Coverage is computed by matching this against the scanned tree, so a pattern that matches nothing reads as an uncovered kind."),
-    format: z.string().optional().describe("File format shared by the members (VCF, h5ad, CSV, …)."),
-    axisLabels: z
+        .describe("Why these values are different substrates: would a downstream step TYPICALLY consume one value's files differently " +
+        "than another's? Identity slots (high-cardinality ids) are never split."),
+})
+    .strict();
+const MergeOperationSchema = z
+    .object({
+    op: z.literal("merge"),
+    setIds: z
         .array(z.string())
-        .max(MAX_AXES)
-        .optional()
-        .describe("Labels of the axes (see `axes`) that vary across this set's members — the design this kind participates in."),
-});
+        .min(2)
+        .max(MAX_MERGE_SETS)
+        .refine((ids) => new Set(ids).size === ids.length, { message: "Each set may be named once — a repeated id would claim its members twice." })
+        .describe("Distinct set ids from the menu, merged into one group."),
+    group: GroupAnnotationSchema,
+    reason: z.string().describe("Why these sets are one group despite instantiating different templates."),
+})
+    .strict();
+const GroupOperationSchema = z
+    .object({
+    op: z.literal("group"),
+    paths: z.array(z.string()).min(1).max(MAX_EXPLICIT_PATHS).describe("Paths relative to the analysis root, for files no set speaks for."),
+    group: GroupAnnotationSchema,
+})
+    .strict();
 /**
- * What varies across a kind's members.
+ * The four ways an agent expresses a grouping.
  *
- * The scan reports that a filename position varies and which values it takes; what the
- * variation IS — a subject, a timepoint, a treatment arm, a chromosome shard — is not
- * derivable from the values, so the label is required and agent-supplied. An axis
- * evident from a metadata sheet but not from filenames belongs here too.
+ * Operations, not freehand patterns: a mistyped value is a rejectable error at submit
+ * rather than a silent zero-match, and every run starts from the same menu, so two runs
+ * over one tree can only differ where the judgement differs.
  */
-export const AxisSchema = z.object({
-    label: z.string().describe("What this dimension is: 'patient', 'timepoint', 'treatment arm', 'chromosome shard', 'replicate'."),
-    cardinality: z.number().int().describe("How many distinct values this dimension takes."),
-    exampleValues: z.array(z.string()).max(20).optional().describe("A few observed values, for orientation."),
-    description: z.string().optional().describe("Anything a downstream planner needs about this dimension (ordering, pairing, imbalance)."),
-});
-export const ProfilerOutputSchema = z.object({
-    /**
-     * Notable singletons only. The workspace filesystem is the authoritative file list
-     * — `list_files`, `grep`, and the vector index all read the live tree — so a record
-     * per input file would store a stale duplicate of something that cannot go wrong.
-     */
-    files: z.array(ProfilerFileSchema).max(MAX_NOTABLE_FILES),
-    kinds: z
-        .array(KindSchema)
-        .max(MAX_KINDS)
-        .describe("The repeating sets this dataset is made of. Group; do not enumerate members — the cap is a hard bound, not a target."),
-    axes: z.array(AxisSchema).max(MAX_AXES).optional().describe("What varies across the kinds' members — the experimental design, as far as it is observable."),
-    analysisSummary: z.string().describe("Narrative overview of the dataset — structure, content, analytical potential, and limitations"),
+export const MenuOperationSchema = z.discriminatedUnion("op", [UseOperationSchema, SplitOperationSchema, MergeOperationSchema, GroupOperationSchema]);
+// ── Dimensions and their observations ──────────────────────────────────────
+const CheckedSchema = z
+    .object({
+    matched: z.number().int().min(0).describe("Values of this observation found in the other source."),
+    of: z.number().int().min(1).describe("Values compared."),
+})
+    .strict()
+    .describe("A measurement you actually performed. Omit the whole field when you did not perform it — its absence is 'unchecked', " +
+    "and there is no way to claim a check that did not happen.");
+const SlotObservationSchema = z
+    .object({
+    kind: z.literal("slot"),
+    setId: z.string(),
+    slotId: z.string().describe("The slot this dimension is seen in. Cardinality and values are computed from the scan, not asserted."),
+    note: z.string().optional(),
+})
+    .strict();
+const ColumnObservationSchema = z
+    .object({
+    kind: z.literal("column"),
+    path: z.string().describe("The file you read, relative to the analysis root."),
+    column: z.string().describe("The column name, verbatim."),
+    exampleValues: z.array(z.string()).min(1).max(MAX_EXAMPLE_VALUES).describe("Verbatim example values, so the citation is checkable against the file."),
+    distinctValues: z.number().int().min(1).optional().describe("Distinct values in the column, when you counted them."),
+    checked: CheckedSchema.optional(),
+    checkedAgainst: z.string().optional().describe("What you compared against, when `checked` is present."),
+    note: z.string().optional(),
+})
+    .strict();
+const DocumentObservationSchema = z
+    .object({
+    kind: z.literal("document"),
+    path: z.string().describe("The document or mapping file, relative to the analysis root."),
+    citation: z.string().describe("What the document says, close to verbatim."),
+    statesCardinality: z.number().int().min(1).optional().describe("The count the document states, when it states one."),
+    checked: CheckedSchema.optional(),
+    checkedAgainst: z.string().optional().describe("What you compared against, when `checked` is present."),
+    note: z.string().optional(),
+})
+    .strict();
+export const ObservationSchema = z.discriminatedUnion("kind", [SlotObservationSchema, ColumnObservationSchema, DocumentObservationSchema]);
+/**
+ * A dataset-level thing that varies, with where it was seen.
+ *
+ * At least one observation, always: a dimension without one is a claim, and a claim is
+ * what this schema exists to make unwritable. A slot binding is the ONLY way a group
+ * links to a dimension — there is no freehand "this group varies by X" field, so a
+ * per-subject dimension attached to a group whose template has no such slot cannot be
+ * written down.
+ *
+ * Naming a slot is not the same as promoting a dimension. Technical, single-set slots
+ * (shards, callers, lanes, read pairs) stay on the set; this list is reserved for
+ * biological or cross-set variation and must read as the design at a glance. A value
+ * constant across the dataset is not a dimension — it is an identity field.
+ */
+export const DimensionSchema = z
+    .object({
+    label: z.string().describe("What this dimension is, in the dataset's own words."),
+    category: z.enum(DIMENSION_CATEGORY_IDS).describe("The dimension's category, from the shipped vocabulary."),
+    categoryLabel: z.string().optional().describe("Required when category is 'other': what actually varies."),
+    scope: z
+        .enum(["biological", "technical"])
+        .optional()
+        .describe("Only read when category is 'other', whose scope the catalogue cannot derive. Defaults to technical. Ignored for every other category."),
+    description: z.string().optional().describe("What a downstream planner needs about this dimension — ordering, pairing, imbalance."),
+    observations: z
+        .array(ObservationSchema)
+        .min(1)
+        .max(MAX_OBSERVATIONS)
+        .describe("Where you saw it. At least one; a dimension without evidence is refused."),
+    reconciliations: z
+        .array(z
+        .object({
+        note: z.string().describe("What disagrees, and between which observations."),
+        delta: z.number().int().optional().describe("The numeric difference, when the disagreement is a count."),
+    })
+        .strict())
+        .max(MAX_RECONCILIATIONS)
+        .optional()
+        .describe("Disagreeing observations both stand. Record the delta here; do not pick a winner and do not report one canonical cardinality."),
+    nestsUnder: z
+        .object({
+        dimension: z.string().describe("The label of the dimension this one nests under."),
+        evidence: z.string().describe("Path structure or a mapping file's columns — what makes the nesting observable."),
+    })
+        .strict()
+        .optional(),
+    treatmentReason: z.string().optional().describe("Required when you deviate from the category's default treatment: why this dataset is the exception."),
+})
+    .strict();
+// ── Probe outcomes ─────────────────────────────────────────────────────────
+const ProbeFoundSchema = z
+    .object({
+    probe: z.enum(DIMENSION_PROBE_IDS),
+    outcome: z.literal("found"),
+    dimension: z.string().describe("The label of the dimension you recorded for it."),
+})
+    .strict();
+const ProbeNotFoundSchema = z
+    .object({
+    probe: z.enum(DIMENSION_PROBE_IDS),
+    outcome: z.literal("not-found"),
+    searched: z
+        .array(z.string())
+        .min(1)
+        .max(MAX_SEARCHED_FILES)
+        .describe("The files you looked in — metadata and documentation members, clinical and sample-annotation tables."),
+    reason: z.string().describe("Why it is not there. A pointer to where a near-miss landed instead is the most useful form."),
+})
+    .strict();
+const ProbeConstantSchema = z
+    .object({
+    probe: z.enum(DIMENSION_PROBE_IDS),
+    outcome: z.literal("found-but-constant"),
+    value: z.string().describe("The single value the attribute takes across the dataset."),
+    evidence: z.string().describe("Where you saw it."),
+})
+    .strict();
+const ProbeAttestedSchema = z
+    .object({
+    probe: z.enum(DIMENSION_PROBE_IDS),
+    outcome: z.literal("attested"),
+    citation: z.string().describe("What the prose says."),
+    path: z.string().describe("The document that says it."),
+})
+    .strict();
+/**
+ * One outcome per probe. "Not found after looking" is a correct, complete answer — that
+ * is what removes the completeness pressure that otherwise invents dimensions. An
+ * attested find is prose-only evidence and can never justify a split.
+ */
+export const ProbeReportSchema = z.discriminatedUnion("outcome", [ProbeFoundSchema, ProbeNotFoundSchema, ProbeConstantSchema, ProbeAttestedSchema]);
+// ── The submission ─────────────────────────────────────────────────────────
+export const ProfileSubmissionSchema = z
+    .object({
+    operations: z
+        .array(MenuOperationSchema)
+        .min(1)
+        .max(MAX_OPERATIONS)
+        .describe("Your grouping, as operations on the menu. Every kept file lands in exactly one group; whatever you leave unclaimed is swept visibly."),
+    dimensions: z
+        .array(DimensionSchema)
+        .max(MAX_DIMENSIONS)
+        .optional()
+        .describe("What varies across the dataset, each with the evidence you saw it in. An empty list is a correct answer for a simple tree."),
+    probes: z.array(ProbeReportSchema).max(DIMENSION_PROBE_IDS.length).optional().describe("One outcome per probe you were asked to look for."),
+    analysisSummary: z.string().describe("Narrative overview of the dataset — structure, content, analytical potential, and limitations."),
     domain: z
         .string()
-        .describe("Scientific domain (e.g. transcriptomics, proteomics, genomics, cheminformatics, clinical, imaging, metabolomics, multi-omics)"),
-    subtype: z.string().optional().describe("Specific subtype within domain (e.g. bulk-rna-seq, single-cell, LC-MS/MS, whole-genome-sequencing)"),
+        .describe("Scientific domain (e.g. transcriptomics, proteomics, genomics, cheminformatics, clinical, imaging, metabolomics, multi-omics)."),
+    subtype: z.string().optional().describe("Specific subtype within domain (e.g. bulk-rna-seq, single-cell, LC-MS/MS, whole-genome-sequencing)."),
     organism: OrganismSchema.nullable().describe("Subject organism, identified from any input (metadata files, paper PDFs, READMEs, filenames, accession patterns). " +
         "Set to null ONLY when no input identifies the organism — never guess. " +
-        "A null organism is acceptable; a wrong taxon ID is not. " +
-        "If inputs disagree on organism, pick the most-trusted source, set confidence accordingly, and add the conflict to qualityAssessment.concerns."),
-    tissue: z
-        .string()
-        .nullable()
-        .optional()
-        .describe("Subject tissue or anatomical site when applicable (e.g. 'rectal mucosal biopsy', 'liver', 'PBMC', 'whole blood'). null if not applicable or unknown."),
-    cellType: z
-        .string()
-        .nullable()
-        .optional()
-        .describe("Subject cell type when applicable (e.g. 'CD4+ T cells', 'iPSC-derived hepatocytes', 'primary hepatocytes'). null if not applicable or unknown."),
-    condition: z
-        .string()
-        .nullable()
-        .optional()
-        .describe("Disease, treatment, or perturbation under study (e.g. 'Ulcerative Colitis vs healthy controls', 'cisplatin dose-response', 'CRISPR knockout of TP53'). null if not applicable or unknown."),
+        "A null organism is acceptable; a wrong taxon ID is not."),
+    tissue: z.string().nullish().describe("Subject tissue or anatomical site when it is CONSTANT across the dataset. A varying one is a dimension."),
+    cellType: z.string().nullish().describe("Subject cell type when it is CONSTANT across the dataset. A varying one is a dimension."),
+    condition: z.string().nullish().describe("Disease, treatment, or perturbation under study, when constant. A varying one is a dimension."),
     accessions: z
         .array(z.string())
         .optional()
-        .describe("Public dataset accessions found in any input (filenames, metadata, paper). " +
+        .describe("Public dataset accessions found in any input. " +
         "GEO (GSE/GSM), SRA (SRP/SRR/SRX), BioProject (PRJNA/PRJEB/PRJDB), ArrayExpress (E-MTAB-xxxx), dbGaP (phs), EGA (EGAS/EGAD)."),
-    experimentalDesign: z.string().optional().describe("Description of experimental design — conditions, groups, comparisons, replicates, pairing"),
-    qualityAssessment: z
-        .object({
-        concerns: z
-            .array(z.string())
-            .describe("Dataset-wide quality issues (e.g. batch effects, sample imbalance, high missing rate, conflicting organism declarations across inputs)"),
-        strengths: z.array(z.string()).describe("Dataset strengths (e.g. deep coverage, balanced groups, low duplication)"),
-    })
-        .optional(),
-});
+    experimentalDesign: z.string().optional().describe("Conditions, groups, comparisons, replicates, pairing — the design as far as it is observable."),
+    caveats: z
+        .array(z.string())
+        .max(MAX_CAVEATS)
+        .optional()
+        .describe("What a planner must know before designing an analysis, in YOUR words. Companion gaps, incomplete crossings, and " +
+        "reconciliation deltas are computed and recorded separately — do not restate them here."),
+})
+    .strict();

package/dist/state/data-profile.d.ts

@@ -13,7 +13,7 @@ import type { Querier } from "./db.js";
  * `pg` and the whole ledger query surface, to obtain a type. Re-exported here so ledger
  * call sites keep one import.
  */
-export type { DataProfileAxis, DataProfileCoverage, DataProfileFile, DataProfileInputFile, DataProfileInputSignature, DataProfileKind, DataProfileOrganism, DataProfileQualityAssessment, DataProfileLifecycleStatus, DataProfileResult, DataProfileSubjectSource, } from "../contracts/data-profile.js";
+export type { DataProfileAxis, DataProfileChecked, DataProfileCompanionCompleteness, DataProfileCoverage, DataProfileDimension, DataProfileFile, DataProfileGroup, DataProfileGroupSlot, DataProfileInputSignature, DataProfileKind, DataProfileMemberAnnotation, DataProfileObservation, DataProfileOrganism, DataProfilePartition, DataProfileProbeReport, DataProfileQualityAssessment, DataProfileQuarantine, DataProfileRecipeStep, DataProfileLifecycleStatus, DataProfileResult, DataProfileSubjectSource, } from "../contracts/data-profile.js";
 export interface DataProfileStatus {
     status: DataProfileLifecycleStatus;
     error: string | null;

package/dist/state/index.d.ts

@@ -21,7 +21,7 @@ export type { ActiveSandboxRow } from "./active-sandboxes.js";
 export { insertPlan, upsertPlan, loadPlan } from "./plans.js";
 export type { InsertPlanInput, UpsertPlanInput } from "./plans.js";
 export { tryStartDataProfile, tryRetryDataProfile, tryRerunDataProfile, completeDataProfile, failDataProfile, expireStaleDataProfile, reconcileOrphanedDataProfile, clearDataProfile, loadDataProfileStatus, loadSeedInputFileIds, recordDataProfileWorkflowId, } from "./data-profile.js";
-export type { DataProfileStatus, DataProfileResult, DataProfileInputFile, DataProfileFile, DataProfileQualityAssessment, DataProfileKind, DataProfileAxis, DataProfileInputSignature, DataProfileCoverage, } from "./data-profile.js";
+export type { DataProfileStatus, DataProfileResult, DataProfileChecked, DataProfileCompanionCompleteness, DataProfileDimension, DataProfileGroup, DataProfileGroupSlot, DataProfileFile, DataProfileMemberAnnotation, DataProfileObservation, DataProfilePartition, DataProfileProbeReport, DataProfileQualityAssessment, DataProfileQuarantine, DataProfileRecipeStep, DataProfileKind, DataProfileAxis, DataProfileInputSignature, DataProfileCoverage, } from "./data-profile.js";
 export { queryRunCountsByAnalyses, queryThreadCountsByAnalyses, queryDataProfileStatusByAnalyses } from "./analyses-metrics.js";
 export { createAnalysisPurge } from "./purge-analysis.js";
 export type { AnalysisPurge, AnalysisPurgeDeps, AnalysisPurgeOutcome } from "./purge-analysis.js";

package/dist/state/vector-store.d.ts

@@ -39,6 +39,18 @@ export interface UpsertInput {
     readonly metadata?: readonly Record<string, unknown>[];
     readonly ids?: readonly string[];
 }
+/**
+ * A narrow delete surface: every entry of one index whose metadata `type` is named.
+ *
+ * `type` is the only dimension a caller may delete along, deliberately. The index holds
+ * several independently-owned tiers in one table, and a writer rebuilding its own tier
+ * has to be unable to reach another's — a general predicate would make that a review
+ * question rather than a type error.
+ */
+export interface DeleteByTypeInput {
+    readonly indexName: string;
+    readonly types: readonly string[];
+}
 export interface QueryInput {
     readonly indexName: string;
     readonly queryVector: readonly number[];
@@ -54,6 +66,7 @@ export interface VectorStore {
     createIndex(input: CreateIndexInput): ResultAsync<void, DbError>;
     buildIndex(input: BuildIndexInput): ResultAsync<void, DbError>;
     upsert(input: UpsertInput): ResultAsync<void, DbError>;
+    deleteByType(input: DeleteByTypeInput): ResultAsync<void, DbError>;
     query(input: QueryInput): ResultAsync<QueryResult[], DbError>;
 }
 export declare function createVectorStore(pool: Pool): VectorStore;

package/dist/state/vector-store.js

@@ -85,6 +85,14 @@ export function createVectorStore(pool) {
                 }
             });
         },
+        deleteByType({ indexName, types }) {
+            assertIdent(indexName);
+            if (types.length === 0)
+                return okAsync(undefined);
+            return tryMutation("vectorStore.deleteByType", async () => {
+                await pool.query(`DELETE FROM "${indexName}" WHERE metadata->>'type' = ANY($1::text[])`, [[...types]]);
+            });
+        },
         query({ indexName, queryVector, topK, filter }) {
             assertIdent(indexName);
             const literal = toVectorLiteral(queryVector);

package/dist/tasks/data-profile-absorb.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Incremental absorption — replaying a completed profile's recipe against a fresh scan.
+ *
+ * The recipe is keyed to scanner TEMPLATES, so it survives the scan that produced it: a
+ * tree that gained files for ten more subjects instantiates the same templates, and the
+ * operations that grouped it still say what its files are. Replaying them re-derives
+ * membership, counts, and the partition deterministically — no sandbox, no model — and the
+ * profile completes on arithmetic alone.
+ *
+ * Three outcomes past "there was nothing to replay". A replay claiming every kept file is a
+ * FULL absorb and completes the profile. A replay leaving files unclaimed is a PARTIAL
+ * absorb: those files are the delta, and the agent sees them alongside the resolution that
+ * already stands rather than an empty page. A recipe whose templates no longer resolve is
+ * STRANDED and falls back to a full re-profile — loudly, because silently absorbing the
+ * half that still resolves would persist a profile describing part of a tree while
+ * claiming to describe all of it.
+ *
+ * Group ids survive a replay because the reconstructed annotation carries the PERSISTED
+ * group name, which is what the original resolution derived the id from — including the
+ * `name — value` a slot split minted. That is checked rather than trusted: every group id
+ * the recipe named must be present in the replayed resolution, so a step that resolved to
+ * nothing strands the recipe instead of quietly deleting a group and renumbering its
+ * siblings out from under a dimension's `groupIds`.
+ *
+ * Dimensions are re-resolved, not copied. Their slot observations are persisted as
+ * template + slot position, so a replay binds each to whatever the FRESH scan has there
+ * and recomputes its cardinality and values — a scan-scoped slot id would name a
+ * different slot after the set order changed, and the numbers riding on it would describe
+ * some other slot entirely.
+ */
+import type { DataProfileResult } from "../contracts/data-profile.js";
+import type { SetMenu } from "../input-scan/menu.js";
+import type { DetectedSets } from "../input-scan/set-types.js";
+import type { MenuOperation } from "../schemas/data-profile-schemas.js";
+import { type ProfileResolution } from "./data-profile-resolve.js";
+/** Delta paths named in the repair briefing before the tail folds into a count. */
+export declare const MAX_DELTA_PATHS = 30;
+export type AbsorbKind = "none" | "full" | "partial" | "stranded";
+export type AbsorbOutcome = {
+    readonly kind: "none";
+} | {
+    readonly kind: "stranded";
+    readonly reason: string;
+} | {
+    readonly kind: "full";
+    readonly operations: readonly MenuOperation[];
+    readonly resolution: ProfileResolution;
+} | {
+    readonly kind: "partial";
+    readonly operations: readonly MenuOperation[];
+    readonly resolution: ProfileResolution;
+    /** Kept files the replayed recipe does not account for — what the agent is woken for. */
+    readonly delta: readonly string[];
+};
+/**
+ * Replay a completed profile's recipe against a fresh scan.
+ *
+ * Pure. Strandedness is decided both while rebuilding the operations — a template, slot,
+ * group id, or dimension binding that no longer resolves — and after resolving them, by
+ * checking that every group the recipe named came back. What remains is a fact about the
+ * tree having moved: files the recipe does not account for.
+ *
+ * The residue is replayed too. The recipe carries the paths the last resolution swept, so
+ * a file that profile already declined to classify re-sweeps deterministically and only a
+ * file NEW to the tree is delta — without that an unchanged tree would re-absorb as
+ * partial and wake the agent to re-judge what it already judged.
+ */
+export declare function absorbRecipe(prior: DataProfileResult | null | undefined, detected: DetectedSets, menu: SetMenu): AbsorbOutcome;
+/**
+ * The repair briefing for a partial absorb.
+ *
+ * It hands the agent the resolution that already stands, in fresh menu terms, plus the
+ * files it does not cover. A drift event that added one directory should cost one
+ * operation, not a re-authoring of a grouping nobody disputes.
+ */
+export declare function renderAbsorbDelta(outcome: Extract<AbsorbOutcome, {
+    kind: "partial";
+}>): string;