@inflexa-ai/harness 0.28.1 → 0.29.0

package/dist/input-scan/detect-sets.js

@@ -0,0 +1,279 @@
+/**
+ * The detected-set pipeline: quarantine → markers → sibling clustering → template
+ * mining → assembly.
+ *
+ * Pure over paths, sizes, and the format already detected — no read, no container, no
+ * model. Each stage narrows what the next one has to explain: junk leaves before
+ * structure is observed, a marker-claimed subtree never reaches inference, and a
+ * clustered directory arrives at mining as one template with its entity already a
+ * slot.
+ *
+ * Assembly carries variance rather than splitting on it. Members that differ only in
+ * compression are one set whose wrapper census says so, and a token appearing in both
+ * a directory segment and the stem is cross-checked and reported as ONE identity slot,
+ * because reporting it twice would invent a second thing that varies.
+ */
+import { assessCompleteness, attachCompanions } from "./companions.js";
+import { buildTree, clusterChildren, computeSignatures } from "./clustering.js";
+import { applyMarkers } from "./markers.js";
+import { quarantine } from "./quarantine.js";
+import { selectReadouts } from "./readout-budget.js";
+import { catchAll, describeGroup, describeSlot, mineContext, slotToken } from "./templates.js";
+import { IDENTITY_CROSS_CHECK_RATIO, MAX_EXAMPLE_PATHS, MAX_LEFTOVER_SAMPLE, MAX_SETS_PER_CONTEXT, MIN_IDENTITY_TOKEN_LENGTH } from "./tuning.js";
+function fileCountOf(members) {
+    return members.reduce((total, member) => total + 1 + member.companions.length, 0);
+}
+function bytesOf(members) {
+    return members.reduce((total, member) => total + member.size + member.companions.reduce((sum, companion) => sum + companion.size, 0), 0);
+}
+function formatCensus(members) {
+    const counts = new Map();
+    for (const member of members)
+        counts.set(member.format, (counts.get(member.format) ?? 0) + 1);
+    return [...counts.entries()].map(([format, count]) => ({ format, count })).sort((a, b) => b.count - a.count || a.format.localeCompare(b.format, "en"));
+}
+function wrapperCensus(members) {
+    const counts = new Map();
+    for (const member of members) {
+        const wrapper = member.wrapper ?? "none";
+        counts.set(wrapper, (counts.get(wrapper) ?? 0) + 1);
+    }
+    return [...counts.entries()].map(([wrapper, count]) => ({ wrapper, count })).sort((a, b) => b.count - a.count || a.wrapper.localeCompare(b.wrapper, "en"));
+}
+function renderTemplate(segments, slots) {
+    return segments.map((segment) => (segment.kind === "literal" ? segment.text : slotToken(slots[segment.slot]))).join("");
+}
+/** The token value at a slot's position, with the literal affixes the template shows stripped off. */
+function coreAt(item, slot) {
+    const raw = item.tokens[slot.index]?.value ?? "";
+    const end = slot.suffix ? raw.length - slot.suffix.length : raw.length;
+    return raw.slice(slot.prefix?.length ?? 0, end);
+}
+/**
+ * Link a name slot to the directory slot it repeats.
+ *
+ * Containment rather than equality: affix recovery can take a character off the token,
+ * and a directory segment may carry decoration its stem twin lacks. Containment on its
+ * own is weak evidence, so it counts only where the contained token is long enough to
+ * be an identifier, or where the two positions have the same cardinality — a token
+ * that appears inside a segment AND varies with it one-for-one is that segment's
+ * identity, while one that merely appears inside it is a coincidence of naming.
+ */
+function crossCheckIdentity(group, nameSlots, directorySlots, directoryValues) {
+    if (group.kind !== "tokens" && group.kind !== "family")
+        return;
+    for (const slot of nameSlots) {
+        if (slot.distinctValues < 2)
+            continue;
+        for (let k = 0; k < directoryValues.length; k++) {
+            const values = directoryValues[k];
+            const paired = directorySlots[k].distinctValues === slot.distinctValues;
+            let matches = 0;
+            for (let i = 0; i < group.items.length; i++) {
+                const core = coreAt(group.items[i], slot);
+                const segment = values[i] ?? "";
+                const alike = core === segment ||
+                    (segment.includes(core) && (paired || core.length >= MIN_IDENTITY_TOKEN_LENGTH)) ||
+                    (core.includes(segment) && (paired || segment.length >= MIN_IDENTITY_TOKEN_LENGTH));
+                if (alike)
+                    matches++;
+            }
+            if (matches / group.items.length < IDENTITY_CROSS_CHECK_RATIO)
+                continue;
+            slot.sameAs = k;
+            slot.crossCheckMismatches = group.items.length - matches;
+            break;
+        }
+    }
+}
+function assembleMinedSet(group, path) {
+    const members = group.items.map((item) => item.file);
+    const { slots: nameSlots, segments: nameSegments } = describeGroup(group);
+    const directorySlots = [];
+    const directorySegments = [];
+    const directoryValues = [];
+    let variablePosition = 0;
+    for (const segment of path) {
+        if (segment.kind === "literal") {
+            directorySegments.push({ kind: "literal", text: `${segment.name}/` });
+            continue;
+        }
+        const at = variablePosition++;
+        const values = members.map((member) => member.varValues[at] ?? "");
+        const distinct = new Set(values);
+        if (distinct.size === 1) {
+            directorySegments.push({ kind: "literal", text: `${[...distinct][0]}/` });
+            continue;
+        }
+        const slot = describeSlot(values, "directory", segment.index);
+        if (slot.prefix)
+            directorySegments.push({ kind: "literal", text: slot.prefix });
+        directorySegments.push({ kind: "slot", slot: directorySlots.length });
+        if (slot.suffix)
+            directorySegments.push({ kind: "literal", text: slot.suffix });
+        directorySegments.push({ kind: "literal", text: "/" });
+        directorySlots.push(slot);
+        directoryValues.push(values);
+    }
+    crossCheckIdentity(group, nameSlots, directorySlots, directoryValues);
+    const slots = [...directorySlots, ...nameSlots];
+    const segments = [
+        ...directorySegments,
+        ...nameSegments.map((segment) => (segment.kind === "slot" ? { kind: "slot", slot: segment.slot + directorySlots.length } : segment)),
+    ];
+    const origin = group.kind === "tokens" ? "mined" : group.kind;
+    return { origin, pathTemplate: renderTemplate(segments, slots), segments, slots, members };
+}
+function finalizeSet(draft, index) {
+    const id = `set-${index + 1}`;
+    const slotIds = draft.slots.map((_, position) => `${id}.slot-${position + 1}`);
+    const slots = draft.slots.map((slot, position) => ({
+        id: slotIds[position],
+        location: slot.location,
+        index: slot.index,
+        tokenClass: slot.tokenClass,
+        ...(slot.width !== undefined ? { width: slot.width } : {}),
+        ...(slot.minLength !== undefined ? { minLength: slot.minLength, maxLength: slot.maxLength } : {}),
+        ...(slot.skeleton !== undefined ? { skeleton: slot.skeleton } : {}),
+        ...(slot.prefix !== undefined ? { prefix: slot.prefix } : {}),
+        ...(slot.suffix !== undefined ? { suffix: slot.suffix } : {}),
+        distinctValues: slot.distinctValues,
+        sampleValues: slot.sampleValues,
+        ...(slot.sameAs !== undefined ? { sameAsSlot: slotIds[slot.sameAs], crossCheckMismatches: slot.crossCheckMismatches ?? 0 } : {}),
+    }));
+    const segments = draft.segments.map((segment) => segment.kind === "literal" ? { kind: "literal", text: segment.text } : { kind: "slot", slotId: slotIds[segment.slot] });
+    const { members, completeness } = assessCompleteness(draft.members);
+    return {
+        set: {
+            id,
+            origin: draft.origin,
+            ...(draft.marker ? { marker: draft.marker } : {}),
+            pathTemplate: draft.pathTemplate,
+            segments,
+            memberCount: members.length,
+            fileCount: fileCountOf(members),
+            totalBytes: bytesOf(members),
+            formats: formatCensus(members),
+            wrappers: wrapperCensus(members),
+            slots,
+            completeness,
+            examplePaths: members.slice(0, MAX_EXAMPLE_PATHS).map((member) => member.path),
+            members,
+        },
+        values: draft.slots.map((slot, position) => [slotIds[position], slot.values]),
+        memberValues: draft.slots.map((slot, position) => [slotIds[position], slot.memberValues]),
+    };
+}
+/**
+ * Sample the leftovers across formats first, then by size.
+ *
+ * A metadata sheet, a README, and a paper are each one file of their own format, so a
+ * size-ordered sample would bury all three under whichever large files failed to
+ * group. Covering formats first is what makes the notable singletons visible.
+ */
+function sampleLeftovers(members) {
+    const bySize = [...members].sort((a, b) => b.size - a.size || a.path.localeCompare(b.path, "en"));
+    const picked = [];
+    const seen = new Set();
+    for (const member of bySize) {
+        if (picked.length >= MAX_LEFTOVER_SAMPLE)
+            break;
+        if (seen.has(member.format))
+            continue;
+        seen.add(member.format);
+        picked.push(member);
+    }
+    for (const member of bySize) {
+        if (picked.length >= MAX_LEFTOVER_SAMPLE)
+            break;
+        if (!picked.includes(member))
+            picked.push(member);
+    }
+    return picked.map((member) => ({ path: member.path, size: member.size, format: member.format }));
+}
+/** Observe the detected sets of a scanned tree. Pure — no I/O, no container, no model. */
+export function detectSets(files, options = {}) {
+    const { kept, summary } = quarantine(files);
+    const { units, unclaimed } = applyMarkers(attachCompanions(kept));
+    const drafts = [];
+    for (const unit of units) {
+        const template = `${unit.root ? `${unit.root}/` : ""}**`;
+        drafts.push({
+            origin: "marker",
+            marker: unit.label,
+            pathTemplate: template,
+            segments: [{ kind: "literal", text: template }],
+            slots: [],
+            members: unit.members,
+        });
+    }
+    const leftovers = [];
+    const emit = (context, path) => {
+        const hasDirectorySlot = path.some((segment) => segment.kind === "variable");
+        const mined = mineContext(context);
+        let sets = mined.sets;
+        let residue = mined.residue;
+        // The long tail of tiny sets inside a repeated directory shape is noise: the
+        // repetition is the structure, and the catch-all is where it belongs.
+        if (hasDirectorySlot && sets.length > MAX_SETS_PER_CONTEXT) {
+            const ordered = [...sets].sort((a, b) => b.items.length - a.items.length);
+            sets = ordered.slice(0, MAX_SETS_PER_CONTEXT);
+            residue = [...residue, ...ordered.slice(MAX_SETS_PER_CONTEXT).flatMap((group) => group.items)];
+        }
+        const caught = catchAll(residue, hasDirectorySlot);
+        for (const group of [...sets, ...caught.sets])
+            drafts.push(assembleMinedSet(group, path));
+        leftovers.push(...caught.rest.map((item) => item.file));
+    };
+    const descend = (entries, path) => {
+        const context = [];
+        for (const entry of entries)
+            for (const file of entry.node.files)
+                context.push({ ...file, varValues: entry.varValues });
+        if (context.length > 0)
+            emit(context, path);
+        const children = entries.flatMap((entry) => [...entry.node.children.values()].map((node) => ({ node, varValues: entry.varValues })));
+        if (children.length === 0)
+            return;
+        for (const cluster of clusterChildren(children, options.contentSimilarity)) {
+            if (cluster.names.length === 1) {
+                descend(cluster.entries, [...path, { kind: "literal", name: cluster.names[0] }]);
+                continue;
+            }
+            const next = cluster.entries.map((entry) => ({ node: entry.node, varValues: [...entry.varValues, entry.node.name] }));
+            descend(next, [...path, { kind: "variable", index: path.length }]);
+        }
+    };
+    descend([{ node: computeSignatures(buildTree(unclaimed)), varValues: [] }], []);
+    const ordered = drafts.sort((a, b) => b.members.length - a.members.length || bytesOf(b.members) - bytesOf(a.members) || a.pathTemplate.localeCompare(b.pathTemplate, "en"));
+    const slotValues = new Map();
+    const memberSlotValues = new Map();
+    const sets = [];
+    for (const [index, draft] of ordered.entries()) {
+        const finalized = finalizeSet(draft, index);
+        sets.push(finalized.set);
+        for (const [slotId, values] of finalized.values)
+            slotValues.set(slotId, values);
+        for (const [slotId, values] of finalized.memberValues)
+            memberSlotValues.set(slotId, values);
+    }
+    const coveredFiles = sets.reduce((total, set) => total + set.fileCount, 0);
+    const leftoverFiles = {
+        memberCount: leftovers.length,
+        fileCount: fileCountOf(leftovers),
+        totalBytes: bytesOf(leftovers),
+        sample: sampleLeftovers(leftovers),
+    };
+    return {
+        fileCount: files.length,
+        keptFileCount: kept.length,
+        coverage: kept.length === 0 ? 1 : coveredFiles / kept.length,
+        quarantine: summary,
+        sets,
+        leftovers: leftoverFiles,
+        leftoverMembers: leftovers,
+        readout: selectReadouts(sets, leftovers),
+        slotValues,
+        memberSlotValues,
+    };
+}

package/dist/input-scan/enrich.d.ts

@@ -1,30 +1,66 @@
 /**
- * Per-shape header readout — the one part of the scan that needs the sandbox.
+ * Per-shape header readout.
  *
- * A bounded prefix read compared against a magic table runs on the host; a DECODER
- * over user-supplied bytes does not. zlib, HDF5, and PDF parsers in the long-lived
- * multi-tenant harness process are exactly the exposure the ephemeral sandbox exists
- * to contain, so every decode below runs there.
+ * Split by what the readout needs. A header that sits at the front of the file —
+ * magic bytes, a `##` preamble, a delimited header row, a compressed prefix — is
+ * reachable from a bounded read, so it runs in this process over the workspace read
+ * seam (see `readout.ts`) and costs no sandbox round trip.
  *
- * Cost is O(shapes), not O(files): shapes are observed from names and sizes alone, so
- * enrichment decodes a bounded number of members per shape — roughly four files for
- * the 3513-file tree that motivated this capability. One exec covers every shape, so
- * even the round trip does not scale with the tree.
+ * A container that indexes its contents from a footer cannot be read that way, and
+ * running a container parser over user-supplied bytes in a long-lived multi-tenant
+ * process is the exposure the ephemeral sandbox exists to contain. Those readouts run
+ * from `decoder/container_readout.py`, a package asset delivered on the command line
+ * through the existing `runSandboxExec` path.
  *
- * Nothing ships in `sandbox-base` for this: the decoder is a Python program passed on
- * the command line through the existing `runSandboxExec` path, so the capability
- * releases on the harness's own path rather than on the image's manual one.
+ * Cost is O(shapes), not O(files): shapes are observed from names and sizes alone, so
+ * enrichment reads a bounded number of members per shape whatever the tree's size.
  */
+import type { AgentSession } from "../auth/types.js";
 import type { EmitFn } from "../tools/define-tool.js";
 import type { SandboxClient } from "../sandbox/client.js";
 import type { SandboxRef } from "../sandbox/types.js";
-import type { FileShape } from "./types.js";
+import type { WorkspaceFilesystem } from "../workspace/filesystem.js";
+import type { FileShape, HeaderReadout } from "./types.js";
 /** Members decoded per shape. One is enough to characterise a set whose names already match. */
 export declare const MEMBERS_DECODED_PER_SHAPE = 1;
+/**
+ * Prefix reads in flight at once. Each is a bounded read over the workspace seam; running
+ * them one at a time makes a menu of sixty leftovers sixty serial round trips, and running
+ * them all at once hands the seam an unbounded burst.
+ */
+export declare const READOUT_CONCURRENCY = 8;
+/**
+ * The formats whose header lives behind a whole-container parse: Parquet's schema is
+ * in its footer, the HDF5 families index objects across the file, and PDF and DOCX
+ * both resolve their contents through a trailer. Everything else reads from a prefix.
+ */
+export declare const SANDBOX_CONTAINER_FORMATS: ReadonlySet<string>;
+/** One file to read a header from, and what the scan already knows about its bytes. */
+export interface ReadoutTargetSpec {
+    readonly path: string;
+    readonly format: string;
+    readonly wrapper?: string;
+}
+export interface ReadHeadersArgs {
+    readonly targets: readonly ReadoutTargetSpec[];
+    readonly session: AgentSession;
+    readonly fs: WorkspaceFilesystem;
+    /** Omit the sandbox to skip container readouts; every other format still reads. */
+    readonly sandboxClient?: SandboxClient;
+    readonly sandbox?: SandboxRef;
+    /** Absolute in-sandbox path of the analysis root (`/{analysisId}`). */
+    readonly mountRoot: string;
+    readonly execId: string;
+    readonly deadlineMs: number;
+    readonly emit: EmitFn;
+}
 export interface EnrichShapesArgs {
     readonly shapes: readonly FileShape[];
-    readonly sandboxClient: SandboxClient;
-    readonly sandbox: SandboxRef;
+    readonly session: AgentSession;
+    readonly fs: WorkspaceFilesystem;
+    /** Omit the sandbox to skip container readouts; every other format still reads. */
+    readonly sandboxClient?: SandboxClient;
+    readonly sandbox?: SandboxRef;
     /** Absolute in-sandbox path of the analysis root (`/{analysisId}`). */
     readonly mountRoot: string;
     readonly execId: string;
@@ -32,10 +68,17 @@ export interface EnrichShapesArgs {
     readonly emit: EmitFn;
 }
 /**
- * Decode one member per shape and attach the readout.
+ * Read a header from each named file, keyed by path.
+ *
+ * A target whose readout failed carries an `unavailable` note rather than vanishing: the
+ * readout is enrichment, and a menu without it still carries every structural observation
+ * the agent's grouping rests on.
+ */
+export declare function readHeaders(args: ReadHeadersArgs): Promise<Map<string, HeaderReadout>>;
+/**
+ * Read one member per shape and attach the readout.
  *
- * A shape whose decode failed keeps its `unavailable` note rather than losing the
- * shape: the readout is enrichment, and a manifest without it still carries every
- * structural observation the agent's grouping rests on.
+ * A shape whose readout failed keeps its `unavailable` note rather than losing the
+ * shape, for the same reason.
  */
 export declare function enrichShapes(args: EnrichShapesArgs): Promise<FileShape[]>;