@inflexa-ai/harness 0.28.1 → 0.29.0

package/dist/tasks/data-profile-index.js

@@ -1,43 +1,127 @@
 /**
  * The profile's vector index, as a pure projection.
  *
- * The index is a function of the scan crossed with the submitted kinds: for each kind,
- * one entry; for each entity, one entry templated from the kinds it participates in and
- * its own axis value. It has no persisted representation of its own and nothing derives
- * from it, so it can be rebuilt at any time by re-running the scan and reading `kinds`
- * off the existing profile — no model call, no re-profile. Improving the template,
- * adding a tier, or folding in entity attributes later are all re-projections.
+ * The index is a function of the persisted profile crossed with the scan it was taken
+ * against. It has no durable representation of its own and nothing derives from it, so it
+ * can be rebuilt at any time from those two things — no model call, no re-profile.
  *
- * Two tiers, because they answer different queries. A query naming a KIND of data
- * ("variant calls") wants one result carrying the set's count and path pattern; a
- * per-entity tier would return five arbitrary subjects out of thousands of
- * indistinguishable ones. A query naming an ENTITY ("PT0421") wants that entity. A
- * per-file tier adds nothing: a file is identified by its kind and its entity, and its
- * path is on the filesystem.
+ * Three tiers, because they answer three different queries. A query naming a KIND of data
+ * ("variant calls") wants the group: its meaning, its count, its template. A query naming
+ * what VARIES ("timepoint") wants the dimension and the evidence behind it. A query naming
+ * a particular FILE wants that file — but only the members an agent wrote about
+ * individually are worth an entry, because a member entry templated from its group's text
+ * is a near-duplicate of the group entry, and thousands of them are recall noise rather
+ * than recall.
  *
- * `type: "input"` keeps its existing meaning for the entity tier, so searches written
- * against it keep matching; `type: "input-kind"` is additive. Renaming `"input"` would
- * have silently broken existing search on every new analysis.
+ * `type: "input"` keeps its existing meaning — one entry addressing one workspace path —
+ * so a search filtered to it matches whichever era wrote the profile it reads.
+ *
+ * Every file-addressed entry stamps its workspace path into `metadata.path`; pattern
+ * entries (group, dimension, kind, entity) never do. That presence is the discriminator
+ * `workspace_search` documents, so a consumer needs no id parsing to tell them apart.
  */
-/** Entity entries built per analysis. Past this the entity tier is a liability, not a search aid. */
+import { dimensionCardinalities, profileDimensions, profileFileRecords, profileGroups } from "../app/data-profile-view.js";
+/** Entity entries built for a legacy profile. Past this the entity tier is a liability, not a search aid. */
 export const MAX_ENTITY_ENTRIES = 20_000;
-/** Kind names named in one entity's entry text. */
-const MAX_KINDS_IN_ENTITY_TEXT = 6;
+/**
+ * Every metadata `type` a profile writes, in either era.
+ *
+ * The index is a pure PROJECTION, so re-indexing replaces rather than merges: an upsert
+ * keyed by entry id leaves the entries of a renamed group, a dropped dimension, or a
+ * de-annotated member behind forever, searchable and wrong. Clearing exactly these types
+ * clears exactly what a profile wrote — step outputs, summaries, and syntheses carry their
+ * own types and are untouched.
+ */
+export const PROFILE_INDEX_TYPES = ["input-group", "input-dimension", "input", "input-kind"];
+/** Group names named in one legacy entity entry's text. */
+const MAX_GROUPS_IN_ENTITY_TEXT = 6;
 function slug(value) {
-    return value
+    return (value
         .toLowerCase()
         .replace(/[^a-z0-9]+/g, "-")
         .replace(/^-|-$/g, "")
-        .slice(0, 60);
+        .slice(0, 60) || "unnamed");
+}
+function slotSummary(slot) {
+    return { id: slot.id, location: slot.location, tokenClass: slot.tokenClass, distinctValues: slot.distinctValues };
+}
+function groupEntry(analysisId, group) {
+    const category = group.categoryLabel ?? group.category;
+    const facts = [`${group.count} members`, `${group.fileCount} files`, group.displayPattern].join(", ");
+    return {
+        id: `/${analysisId}/group/${group.id}`,
+        text: `${group.name} — one member is ${group.memberRepresents}. ${group.description} ${category} held as ${group.role}. (${facts})`,
+        metadata: {
+            type: "input-group",
+            group: group.id,
+            name: group.name,
+            role: group.role,
+            category: group.category,
+            ...(group.categoryLabel ? { categoryLabel: group.categoryLabel } : {}),
+            pathPattern: group.displayPattern,
+            count: group.count,
+            fileCount: group.fileCount,
+            ...(group.slots && group.slots.length > 0 ? { slots: group.slots.map(slotSummary) } : {}),
+            ...(group.unclassified ? { unclassified: true } : {}),
+        },
+    };
+}
+/** One observation as a searchable clause — where the dimension was seen, and what was seen there. */
+function observationSummary(observation) {
+    if (observation.kind === "slot") {
+        const values = observation.sampleValues.length > 0 ? `, e.g. ${observation.sampleValues.join(", ")}` : "";
+        return `slot ${observation.slotId} (${observation.tokenClass}, ${observation.cardinality} values${values})`;
+    }
+    if (observation.kind === "column") {
+        const distinct = observation.distinctValues !== undefined ? `, ${observation.distinctValues} values` : "";
+        return `column ${observation.column} of ${observation.path}${distinct}`;
+    }
+    return `document ${observation.path} (${observation.citation})`;
+}
+function dimensionEntry(analysisId, dimension) {
+    const category = dimension.categoryLabel ?? dimension.category;
+    const description = dimension.description ? ` ${dimension.description}` : "";
+    const observations = dimension.observations.map(observationSummary).join("; ");
+    return {
+        id: `/${analysisId}/dimension/${slug(dimension.label)}`,
+        text: `${dimension.label} — ${category}, a ${dimension.scope} dimension of this dataset.${description} Observed as: ${observations}.`,
+        metadata: {
+            type: "input-dimension",
+            dimension: dimension.label,
+            category: dimension.category,
+            ...(dimension.categoryLabel ? { categoryLabel: dimension.categoryLabel } : {}),
+            scope: dimension.scope,
+            cardinalities: dimensionCardinalities(dimension),
+            groups: [...new Set(dimension.observations.flatMap((observation) => (observation.kind === "slot" ? observation.groupIds : [])))],
+        },
+    };
+}
+/**
+ * One annotated member. The id addresses the member's workspace path, so a search hit
+ * resolves to a file that exists rather than to a synthetic key.
+ */
+function memberEntry(analysisId, group, annotation) {
+    const format = group.formats.length === 1 ? group.formats[0].format : undefined;
+    return {
+        id: `/${analysisId}/${annotation.path}`,
+        text: `${annotation.path} — ${annotation.note} In ${group.name}, where one member is ${group.memberRepresents}.`,
+        metadata: {
+            type: "input",
+            path: annotation.path,
+            group: group.id,
+            groupName: group.name,
+            category: group.category,
+            ...(format ? { format } : {}),
+        },
+    };
 }
 /**
- * The entity value set: the largest distinct-value set any single variable position
- * carries.
+ * The entity value set of a legacy profile: the largest distinct-value set any single
+ * variable position carries.
  *
  * Entities are what a dataset is ABOUT, and the position that varies most widely across
- * one shape is what identifies them — 1171 subjects, against three timepoints and two
- * replicates. Taking the widest position rather than the cross product is deliberate:
- * the cross product is the file set, which is the tier this projection exists to avoid.
+ * one shape is what identifies them. Taking the widest position rather than the cross
+ * product is deliberate: the cross product is the file set.
  */
 function entityValues(scan) {
     if (!scan)
@@ -56,37 +140,36 @@ function entityValues(scan) {
     return { values: [...best].sort((a, b) => a.localeCompare(b, "en")), distinct: best.length };
 }
 /**
- * The axis label for an entity set. The agent labels axes; the scan counts values, so
- * the axis whose cardinality matches the observed set is the one the agent was naming.
- * With no match the entries still index under a neutral label rather than not at all.
+ * The dimension label for a legacy entity set. The agent labels dimensions; the scan
+ * counts values, so the dimension whose cardinality matches the observed set is the one
+ * the agent was naming.
  */
-function entityAxisLabel(axes, distinct) {
-    if (!axes || axes.length === 0)
+function entityDimensionLabel(dimensions, distinct) {
+    if (dimensions.length === 0)
         return "entity";
-    const exact = axes.find((axis) => axis.cardinality === distinct);
+    const exact = dimensions.find((dimension) => dimension.cardinalities.includes(distinct));
     if (exact)
         return exact.label;
-    const widest = [...axes].sort((a, b) => b.cardinality - a.cardinality)[0];
+    const widest = [...dimensions].sort((a, b) => Math.max(0, ...b.cardinalities) - Math.max(0, ...a.cardinalities))[0];
     return widest?.label ?? "entity";
 }
-function kindEntry(analysisId, kind) {
-    const facts = [`${kind.count} files`, kind.format, kind.pathPattern].filter((f) => Boolean(f)).join(", ");
+function kindEntry(analysisId, group) {
+    const facts = [`${group.count} files`, group.format, group.pattern].filter((fact) => Boolean(fact)).join(", ");
     return {
-        id: `/${analysisId}/kind/${slug(kind.name)}`,
-        text: `${kind.name} — ${kind.description} One member is ${kind.memberRepresents}. (${facts})`,
+        id: `/${analysisId}/kind/${slug(group.name)}`,
+        text: `${group.name} — ${group.description} One member is ${group.memberRepresents}. (${facts})`,
         metadata: {
             type: "input-kind",
-            kind: kind.name,
-            pathPattern: kind.pathPattern,
-            count: kind.count,
-            ...(kind.format ? { format: kind.format } : {}),
-            ...(kind.axisLabels ? { axisLabels: kind.axisLabels } : {}),
+            kind: group.name,
+            pathPattern: group.pattern,
+            count: group.count,
+            ...(group.format ? { format: group.format } : {}),
         },
     };
 }
-function entityEntry(analysisId, label, value, kinds) {
-    const named = kinds.slice(0, MAX_KINDS_IN_ENTITY_TEXT);
-    const carried = named.map((kind) => `${kind.name} (${kind.memberRepresents})`).join("; ");
+function entityEntry(analysisId, label, value, groups) {
+    const named = groups.slice(0, MAX_GROUPS_IN_ENTITY_TEXT);
+    const carried = named.map((group) => `${group.name} (${group.memberRepresents})`).join("; ");
     return {
         id: `/${analysisId}/entity/${encodeURIComponent(value)}`,
         text: `${label} ${value} — input data for this ${label}: ${carried}.`,
@@ -94,17 +177,18 @@ function entityEntry(analysisId, label, value, kinds) {
             type: "input",
             entity: value,
             axis: label,
-            kinds: named.map((kind) => kind.name),
+            kinds: named.map((group) => group.name),
         },
     };
 }
-/** A file entry, for the degenerate case where a profile submitted no kinds at all. */
+/** A file entry, for a legacy profile that resolved to no structure at all. */
 function fileEntry(analysisId, file) {
     return {
         id: `/${analysisId}/${file.path}`,
         text: `${file.path} — ${file.description}`,
         metadata: {
             type: "input",
+            path: file.path,
             ...(file.dataType ? { dataType: file.dataType } : {}),
             ...(file.format ? { format: file.format } : {}),
             ...(file.tags ? { tags: file.tags } : {}),
@@ -112,23 +196,40 @@ function fileEntry(analysisId, file) {
     };
 }
 /**
- * Build every index entry for a completed profile. Deterministic and model-free: the
- * text is templated from the kind's own description and the entity's axis value, so
- * indexing costs no LLM tokens whatever the dataset's size.
+ * A snapshot of the kinds era, indexed as that era indexed it: one entry per kind and one
+ * per entity value. Its groups were never resolved to a membership, so there is no
+ * annotated-member tier to write.
  */
-export function buildProfileIndexEntries(args) {
-    const { analysisId, kinds, axes, scan, files } = args;
-    if (kinds.length === 0) {
-        // No kinds is a degenerate profile, but the notable files it did describe stay
-        // discoverable — the same losslessness the per-file fallback description had.
-        return (files ?? []).map((file) => fileEntry(analysisId, file));
-    }
-    const entries = kinds.map((kind) => kindEntry(analysisId, kind));
+function legacyEntries(analysisId, result, scan) {
+    const groups = profileGroups(result);
+    if (groups.length === 0)
+        return profileFileRecords(result).map((file) => fileEntry(analysisId, file));
+    const entries = groups.map((group) => kindEntry(analysisId, group));
     const { values, distinct } = entityValues(scan);
     if (values.length > 1 && values.length <= MAX_ENTITY_ENTRIES) {
-        const label = entityAxisLabel(axes, distinct);
+        const label = entityDimensionLabel(profileDimensions(result), distinct);
         for (const value of values)
-            entries.push(entityEntry(analysisId, label, value, kinds));
+            entries.push(entityEntry(analysisId, label, value, groups));
+    }
+    return entries;
+}
+/**
+ * Build every index entry for a completed profile. Deterministic and model-free: each
+ * entry's text is templated from what the agent already wrote and what resolution already
+ * computed, so indexing costs no LLM tokens whatever the dataset's size.
+ */
+export function buildProfileIndexEntries(args) {
+    const { analysisId, result, scan } = args;
+    // Presence, not truthiness: a tree the scan kept nothing of resolves to `groups: []`,
+    // and that is a groups-era record with no groups, not a record from before groups.
+    if (result.groups === undefined)
+        return legacyEntries(analysisId, result, scan);
+    const entries = result.groups.map((group) => groupEntry(analysisId, group));
+    for (const dimension of result.dimensions ?? [])
+        entries.push(dimensionEntry(analysisId, dimension));
+    for (const group of result.groups) {
+        for (const annotation of group.memberAnnotations ?? [])
+            entries.push(memberEntry(analysisId, group, annotation));
     }
     return entries;
 }

package/dist/tasks/data-profile-resolve.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Submit-time resolution — the half of the authoring contract the agent does not write.
+ *
+ * The agent submits operations on the menu; this computes what they MEAN over the scanned
+ * tree: which members each group holds, how many files that is, what each group's slots
+ * take once a split has narrowed them, and whether the whole thing partitions the kept
+ * files. A count the resolution can compute is not the agent's to assert, so nothing here
+ * reads a number out of the submission.
+ *
+ * The partition is the load-bearing invariant: every kept file lands in exactly one
+ * group. Overlapping operations are an error returned for repair rather than resolved by
+ * precedence — precedence would silently pick a winner and the profile would still read
+ * as complete. Past the last repair round there is nobody left to return them to, so a
+ * still-contested file is removed from EVERY claimant and swept, recorded as a machine
+ * finding: unowned and counted once beats owned by a winner nobody chose. Files no
+ * operation claims are swept into the same visible `unclassified` group, so "how much did
+ * the profile cover" stops being a question and becomes a census.
+ */
+import type { SetMenu } from "../input-scan/menu.js";
+import type { DetectedSets } from "../input-scan/set-types.js";
+import type { DataProfileDimension, DataProfileGroup, DataProfilePartition, DataProfileProbeReport, DataProfileRecipeStep } from "../contracts/data-profile.js";
+import type { ProfileSubmission } from "../schemas/data-profile-schemas.js";
+/** The id the swept residue carries. Visible in the record and in the accounting. */
+export declare const UNCLASSIFIED_GROUP_ID = "unclassified";
+/**
+ * Sweep paths the recipe carries, so a replay can tell a file the last profile declined
+ * to classify from one that is new to the tree.
+ *
+ * Bounded because the record is a thin ledger and the row is detoasted by every reader.
+ * A sweep past the bound records a prefix and says so; its replay wakes the agent, which
+ * is the honest outcome for a profile that left that many files unaccounted for.
+ */
+export declare const MAX_UNCLASSIFIED_RECIPE_PATHS = 1000;
+/** Contested paths named in the accounting before the count stands alone. */
+export declare const MAX_CONTESTED_SAMPLE = 10;
+/** A resolved group plus the membership the record deliberately does not carry. */
+export interface ResolvedGroup extends DataProfileGroup {
+    /** Member paths — what the index projects from. The persisted record holds counts only. */
+    readonly memberPaths: readonly string[];
+}
+export interface ProfileResolution {
+    /** Repairable faults. A submission carrying any of these is handed back to the agent whole. */
+    readonly errors: readonly string[];
+    /** Kept members no operation claimed. Already swept into the `unclassified` group below. */
+    readonly unclaimed: readonly string[];
+    /** Kept members more than one operation claimed. Swept rather than awarded on the final round. */
+    readonly contested: readonly string[];
+    readonly groups: readonly ResolvedGroup[];
+    readonly dimensions: readonly DataProfileDimension[];
+    readonly probes: readonly DataProfileProbeReport[];
+    readonly partition: DataProfilePartition;
+    readonly recipe: readonly DataProfileRecipeStep[];
+}
+export interface ResolveOptions {
+    /**
+     * True when no repair round follows. A contested file is then swept rather than
+     * reported: there is nobody left to hand the error to, and leaving it with whichever
+     * operation ran first would be the precedence the partition forbids.
+     */
+    readonly finalRound?: boolean;
+}
+/**
+ * Resolve a submission against the scan it was authored over.
+ *
+ * Pure and total: it always returns a resolution, and the caller decides what to do with
+ * `errors` and `unclaimed`. A first round hands both back to the live agent; the repair
+ * round is a full resubmit, and after it the residue sweeps rather than blocking a
+ * profile that gates planning.
+ */
+export declare function resolveProfileSubmission(submission: ProfileSubmission, detected: DetectedSets, menu?: SetMenu, options?: ResolveOptions): ProfileResolution;
+/** The repair round's errors, phrased for the agent that must fix them in one resubmit. */
+export declare function formatResolutionErrors(resolution: ProfileResolution, unclaimedSample?: number): string;