@inflexa-ai/harness 0.28.0 → 0.29.0

package/dist/input-scan/enrich.js

@@ -1,205 +1,133 @@
 /**
- * Per-shape header readout — the one part of the scan that needs the sandbox.
+ * Per-shape header readout.
  *
- * A bounded prefix read compared against a magic table runs on the host; a DECODER
- * over user-supplied bytes does not. zlib, HDF5, and PDF parsers in the long-lived
- * multi-tenant harness process are exactly the exposure the ephemeral sandbox exists
- * to contain, so every decode below runs there.
+ * Split by what the readout needs. A header that sits at the front of the file —
+ * magic bytes, a `##` preamble, a delimited header row, a compressed prefix — is
+ * reachable from a bounded read, so it runs in this process over the workspace read
+ * seam (see `readout.ts`) and costs no sandbox round trip.
  *
- * Cost is O(shapes), not O(files): shapes are observed from names and sizes alone, so
- * enrichment decodes a bounded number of members per shape — roughly four files for
- * the 3513-file tree that motivated this capability. One exec covers every shape, so
- * even the round trip does not scale with the tree.
+ * A container that indexes its contents from a footer cannot be read that way, and
+ * running a container parser over user-supplied bytes in a long-lived multi-tenant
+ * process is the exposure the ephemeral sandbox exists to contain. Those readouts run
+ * from `decoder/container_readout.py`, a package asset delivered on the command line
+ * through the existing `runSandboxExec` path.
  *
- * Nothing ships in `sandbox-base` for this: the decoder is a Python program passed on
- * the command line through the existing `runSandboxExec` path, so the capability
- * releases on the harness's own path rather than on the image's manual one.
+ * Cost is O(shapes), not O(files): shapes are observed from names and sizes alone, so
+ * enrichment reads a bounded number of members per shape whatever the tree's size.
  */
+import { readFile } from "node:fs/promises";
 import { runSandboxExec } from "../tools/workspace/run-exec.js";
+import { READOUT_PREFIX_BYTES, readPrefix } from "./readout.js";
 /** Members decoded per shape. One is enough to characterise a set whose names already match. */
 export const MEMBERS_DECODED_PER_SHAPE = 1;
-/** Wall-clock bound on the whole readout exec — it reads prefixes, nothing more. */
+/** Wall-clock bound on the container-decode exec. It opens footers, nothing more. */
 const ENRICH_TIMEOUT_SECONDS = 120;
 /**
- * The decoder, as a self-contained program. Reads a bounded prefix of each path and
- * prints one JSON object per line: `{"path": …, "fields": {…}}` or `{"path": …,
- * "unavailable": …}`. It never reads a file in full except a size-capped document,
- * whose page count lives in a trailer no prefix can reach.
+ * Prefix reads in flight at once. Each is a bounded read over the workspace seam; running
+ * them one at a time makes a menu of sixty leftovers sixty serial round trips, and running
+ * them all at once hands the seam an unbounded burst.
+ */
+export const READOUT_CONCURRENCY = 8;
+/** Characters of decoder stderr carried into a per-file reason. */
+const MAX_STDERR_CHARS = 160;
+/**
+ * The formats whose header lives behind a whole-container parse: Parquet's schema is
+ * in its footer, the HDF5 families index objects across the file, and PDF and DOCX
+ * both resolve their contents through a trailer. Everything else reads from a prefix.
+ */
+export const SANDBOX_CONTAINER_FORMATS = new Set(["parquet", "hdf5", "h5ad", "loom", "pdf", "docx"]);
+const CONTAINER_DECODER = new URL("./decoder/container_readout.py", import.meta.url);
+/**
+ * Read a header from each named file, keyed by path.
+ *
+ * A target whose readout failed carries an `unavailable` note rather than vanishing: the
+ * readout is enrichment, and a menu without it still carries every structural observation
+ * the agent's grouping rests on.
  */
-const DECODER = String.raw `
-import sys, json, zlib, io, os, csv
-
-PREFIX = 262144
-MEMBER = 1048576
-DOC_MAX = 33554432
-
-def prefix_bytes(path, n=PREFIX):
-    with open(path, "rb") as fh:
-        return fh.read(n)
-
-def gunzip_prefix(raw):
-    d = zlib.decompressobj(16 + zlib.MAX_WBITS)
-    try:
-        return d.decompress(raw, MEMBER)
-    except zlib.error:
-        return b""
-
-def sniff_delimiter(line):
-    counts = {d: line.count(d) for d in [",", "\t", ";", "|"]}
-    best = max(counts, key=lambda d: counts[d])
-    return best if counts[best] > 0 else None
-
-def text_fields(text):
-    lines = [l for l in text.splitlines() if l.strip() != ""]
-    if not lines:
-        return {}
-    head = lines[0]
-    if head.startswith("##fileformat=VCF") or head.startswith("##"):
-        fields = {}
-        for line in lines[:200]:
-            if line.startswith("##fileformat="):
-                fields["fileformat"] = line.split("=", 1)[1][:40]
-            elif line.startswith("##reference="):
-                fields["reference"] = line.split("=", 1)[1][:80]
-            elif line.startswith("#CHROM"):
-                cols = line.split("\t")
-                samples = cols[9:]
-                fields["sampleCount"] = len(samples)
-                fields["samples"] = ", ".join(samples[:5])[:120]
-                break
-        fields["metaLines"] = sum(1 for l in lines if l.startswith("##"))
-        return fields
-    if head.startswith(">"):
-        return {"firstRecord": head[1:80], "recordsInPrefix": sum(1 for l in lines if l.startswith(">"))}
-    if head.startswith("@") and len(lines) > 1 and not head.startswith("@HD"):
-        return {"firstRecord": head[1:80], "readLength": len(lines[1])}
-    if head.startswith("@HD") or head.startswith("@SQ"):
-        return {"referenceSequences": sum(1 for l in lines if l.startswith("@SQ")), "readGroups": sum(1 for l in lines if l.startswith("@RG"))}
-    delim = sniff_delimiter(head)
-    if delim:
-        columns = next(csv.reader([head], delimiter=delim))
-        return {
-            "delimiter": {",": "comma", "\t": "tab", ";": "semicolon", "|": "pipe"}[delim],
-            "columnCount": len(columns),
-            "columns": ", ".join(c.strip()[:24] for c in columns[:12])[:200],
-            "linesInPrefix": len(lines),
+export async function readHeaders(args) {
+    const readouts = new Map();
+    const containers = [];
+    const prefixed = [];
+    for (const target of args.targets) {
+        if (SANDBOX_CONTAINER_FORMATS.has(target.format))
+            containers.push(target);
+        else
+            prefixed.push(target);
+    }
+    let next = 0;
+    await Promise.all(Array.from({ length: Math.min(READOUT_CONCURRENCY, prefixed.length) }, async () => {
+        for (let i = next++; i < prefixed.length; i = next++) {
+            const target = prefixed[i];
+            readouts.set(target.path, await readFromPrefix(args, target));
         }
-    return {"firstLine": head[:120]}
-
-def hdf5_fields(path):
-    try:
-        import h5py
-    except Exception:
-        return {"unavailable": "h5py not installed in this sandbox"}
-    try:
-        with h5py.File(path, "r") as f:
-            keys = list(f.keys())[:20]
-            out = {"rootKeys": ", ".join(keys)[:200], "rootKeyCount": len(f.keys())}
-            for probe in ("X", "obs", "var", "matrix"):
-                if probe in f:
-                    node = f[probe]
-                    shape = getattr(node, "shape", None)
-                    if shape:
-                        out[probe + "Shape"] = "x".join(str(d) for d in shape)
-            return out
-    except Exception as exc:
-        return {"unavailable": "hdf5 open failed: " + str(exc)[:80]}
-
-def pdf_fields(path):
-    if os.path.getsize(path) > DOC_MAX:
-        return {"unavailable": "document too large to page-count within the scan's bound"}
-    try:
-        from pypdf import PdfReader
-        reader = PdfReader(path)
-        text = (reader.pages[0].extract_text() or "")[:200] if reader.pages else ""
-        return {"pageCount": len(reader.pages), "firstPageText": " ".join(text.split())[:200]}
-    except Exception as exc:
-        return {"unavailable": "pdf read failed: " + str(exc)[:80]}
-
-def docx_fields(path):
-    if os.path.getsize(path) > DOC_MAX:
-        return {"unavailable": "document too large to inspect within the scan's bound"}
-    try:
-        import zipfile, re
-        with zipfile.ZipFile(path) as z:
-            names = z.namelist()
-            out = {"parts": len(names)}
-            if "docProps/app.xml" in names:
-                app = z.read("docProps/app.xml").decode("utf8", "replace")
-                for tag in ("Pages", "Words"):
-                    m = re.search("<" + tag + ">(\\d+)</" + tag + ">", app)
-                    if m:
-                        out[tag.lower()] = int(m.group(1))
-            return out
-    except Exception as exc:
-        return {"unavailable": "docx read failed: " + str(exc)[:80]}
-
-def describe(path):
-    ext = path.lower()
-    if ext.endswith(".pdf"):
-        return pdf_fields(path)
-    if ext.endswith(".docx"):
-        return docx_fields(path)
-    if ext.endswith((".h5", ".hdf5", ".h5ad", ".loom")):
-        return hdf5_fields(path)
-    raw = prefix_bytes(path)
-    if raw[:2] == b"\x1f\x8b":
-        wrapper = "bgzip" if len(raw) > 13 and raw[3] & 4 and raw[12:14] == b"BC" else "gzip"
-        inner = gunzip_prefix(raw)
-        if inner[:4] == b"BAM\x01":
-            l_text = int.from_bytes(inner[4:8], "little")
-            header = inner[8:8 + min(l_text, 65536)].decode("utf8", "replace")
-            fields = text_fields(header)
-            fields["wrapper"] = wrapper
-            fields["container"] = "bam"
-            return fields
-        if inner[:4] == b"BCF\x02" or inner[:3] == b"BCF":
-            return {"wrapper": wrapper, "container": "bcf"}
-        fields = text_fields(inner.decode("utf8", "replace"))
-        fields["wrapper"] = wrapper
-        return fields
-    if raw[:8] == b"\x89HDF\r\n\x1a\n":
-        return hdf5_fields(path)
-    return text_fields(raw.decode("utf8", "replace"))
-
-for path in sys.argv[1:]:
-    try:
-        fields = describe(path)
-    except Exception as exc:
-        fields = {"unavailable": type(exc).__name__ + ": " + str(exc)[:80]}
-    note = fields.pop("unavailable", None)
-    record = {"path": path, "fields": fields}
-    if note:
-        record["unavailable"] = note
-    sys.stdout.write(json.dumps(record) + "\n")
-`;
+    }));
+    const { sandboxClient, sandbox } = args;
+    if (containers.length > 0 && sandboxClient && sandbox) {
+        for (const [path, readout] of await decodeContainers(args, { sandboxClient, sandbox }, containers))
+            readouts.set(path, readout);
+    }
+    return readouts;
+}
 /**
- * Decode one member per shape and attach the readout.
+ * Read one member per shape and attach the readout.
  *
- * A shape whose decode failed keeps its `unavailable` note rather than losing the
- * shape: the readout is enrichment, and a manifest without it still carries every
- * structural observation the agent's grouping rests on.
+ * A shape whose readout failed keeps its `unavailable` note rather than losing the
+ * shape, for the same reason.
  */
 export async function enrichShapes(args) {
-    const { shapes, mountRoot } = args;
-    const targets = shapes.map((shape) => ({ shape, paths: shape.examplePaths.slice(0, MEMBERS_DECODED_PER_SHAPE) })).filter((t) => t.paths.length > 0);
-    if (targets.length === 0)
-        return [...shapes];
-    const absolute = new Map();
-    for (const { shape, paths } of targets) {
-        for (const path of paths)
-            absolute.set(`${mountRoot}/${path}`, shape.id);
+    const targets = new Map();
+    for (const shape of args.shapes) {
+        const path = shape.examplePaths.slice(0, MEMBERS_DECODED_PER_SHAPE)[0];
+        if (path !== undefined && !targets.has(path))
+            targets.set(path, shape);
+    }
+    if (targets.size === 0)
+        return [...args.shapes];
+    const readouts = await readHeaders({
+        ...args,
+        targets: [...targets.entries()].map(([path, shape]) => ({ path, format: shape.format, ...(shape.wrapper ? { wrapper: shape.wrapper } : {}) })),
+    });
+    const byShape = new Map();
+    for (const [path, shape] of targets) {
+        const readout = readouts.get(path);
+        if (readout)
+            byShape.set(shape.id, readout);
     }
+    return args.shapes.map((shape) => {
+        const header = byShape.get(shape.id);
+        return header ? { ...shape, header } : shape;
+    });
+}
+async function readFromPrefix(args, target) {
+    const { path } = target;
+    const read = await args.fs
+        .readBytes({ session: args.session, path, length: READOUT_PREFIX_BYTES })
+        .unwrapOr({ kind: "read_failed" });
+    if (read.kind !== "ok")
+        return { path, fields: {}, unavailable: `prefix unreadable (${read.kind})` };
+    const readout = await readPrefix({ prefix: read.bytes, format: target.format, ...(target.wrapper ? { wrapper: target.wrapper } : {}) });
+    return { path, fields: readout.fields, ...(readout.unavailable ? { unavailable: readout.unavailable } : {}) };
+}
+async function decodeContainers(args, machine, containers) {
+    const readouts = new Map();
+    const source = await readFile(CONTAINER_DECODER, "utf8").catch(() => undefined);
+    if (source === undefined) {
+        for (const { path } of containers)
+            readouts.set(path, { path, fields: {}, unavailable: "container decoder is missing from this build" });
+        return readouts;
+    }
+    const byAbsolute = new Map();
+    for (const target of containers)
+        byAbsolute.set(`${args.mountRoot}/${target.path}`, target);
     const result = await runSandboxExec({
-        sandboxClient: args.sandboxClient,
-        sandbox: args.sandbox,
+        sandboxClient: machine.sandboxClient,
+        sandbox: machine.sandbox,
         execId: args.execId,
-        command: ["python3", "-c", DECODER, ...absolute.keys()],
+        command: ["python3", "-c", source, ...byAbsolute.keys()],
         timeoutSeconds: ENRICH_TIMEOUT_SECONDS,
         deadlineMs: args.deadlineMs,
         emit: args.emit,
     });
-    const readouts = new Map();
     for (const line of result.stdout.split("\n")) {
         const trimmed = line.trim();
         if (!trimmed.startsWith("{"))
@@ -211,17 +139,34 @@ export async function enrichShapes(args) {
         catch {
             continue;
         }
-        const shapeId = parsed.path ? absolute.get(parsed.path) : undefined;
-        if (!shapeId || !parsed.path)
+        const target = parsed.path ? byAbsolute.get(parsed.path) : undefined;
+        if (!target)
             continue;
-        readouts.set(shapeId, {
-            path: parsed.path.startsWith(`${mountRoot}/`) ? parsed.path.slice(mountRoot.length + 1) : parsed.path,
+        readouts.set(target.path, {
+            path: target.path,
             fields: parsed.fields ?? {},
             ...(parsed.unavailable ? { unavailable: parsed.unavailable } : {}),
         });
     }
-    return shapes.map((shape) => {
-        const header = readouts.get(shape.id);
-        return header ? { ...shape, header } : shape;
-    });
+    // A target the decoder said nothing about is told WHY, from the exec's own outcome: the
+    // decoder reports per path and exits 0 even when every path failed, so a silent gap is
+    // the exec itself having timed out, died, or never started — and "reported nothing" on
+    // its own is the one thing a reader cannot diagnose.
+    const failure = describeExecFailure(result);
+    for (const { path } of containers) {
+        if (!readouts.has(path))
+            readouts.set(path, { path, fields: {}, unavailable: failure });
+    }
+    return readouts;
+}
+function describeExecFailure(result) {
+    if (result.timedOut)
+        return `container decoder timed out after ${ENRICH_TIMEOUT_SECONDS}s`;
+    if (result.syntheticFailure)
+        return `container decoder did not complete: ${result.syntheticFailure.reason}`;
+    if (result.exitCode !== 0) {
+        const stderr = result.stderr.replace(/\s+/g, " ").trim().slice(0, MAX_STDERR_CHARS);
+        return `container decoder exited ${result.exitCode ?? "with no code"}${stderr ? `: ${stderr}` : ""}`;
+    }
+    return "container decoder reported nothing for this member";
 }

package/dist/input-scan/formats.d.ts

@@ -2,15 +2,15 @@
  * Format detection over a bounded byte prefix.
  *
  * A prefix read compared against a magic-byte table is NOT a decode: no parser
- * runs over the bytes, so this belongs in the harness process. Every actual
- * decoder — zlib, HDF5, PDF — stays in the sandbox (see `enrich.ts`), which is
- * the exposure the sandbox exists to contain.
+ * runs over the bytes, so this belongs in the harness process. So does every
+ * readout a bounded prefix suffices for (see `readout.ts`); a parser that must
+ * walk a whole container stays in the sandbox (see `enrich.ts`).
  *
  * Compression wrappers are reported ALONGSIDE the inner format, never in place of
  * it: `.vcf.gz` is a VCF, and a shape observed on the wrapper would present
  * unrelated data as mechanically alike. The wrapper comes from the bytes; the inner
- * format of a wrapped file comes from its extension chain, which the sandbox
- * header readout then confirms against the decompressed stream.
+ * format of a wrapped file comes from its extension chain, which the header
+ * readout then confirms against the decompressed prefix.
  *
  * The recognised list is a floor, not a closed set. An unrecognised format reports
  * `unknown` with its extension chain preserved, and still joins a shape — shape
@@ -18,6 +18,14 @@
  */
 /** Bytes read per file for detection. Enough for every magic below plus a text sniff line. */
 export declare const MAGIC_PREFIX_BYTES = 512;
+/**
+ * Every suffix the table above recognises, plus the compression wrappers.
+ *
+ * A dotted token outside this set belongs to the stem: `calls.somatic.vcf.gz` has a
+ * two-token extension chain and a two-token stem, and reading `somatic` as part of
+ * the extension would hide a categorical position inside what looks like a format.
+ */
+export declare const KNOWN_EXTENSIONS: ReadonlySet<string>;
 /** The extension chain of a basename, outermost last. `sample.vcf.gz` → `["vcf", "gz"]`. */
 export declare function extensionChain(fileName: string): string[];
 /** The extension chain with any compression wrappers stripped from the tail. */

package/dist/input-scan/formats.js

@@ -2,15 +2,15 @@
  * Format detection over a bounded byte prefix.
  *
  * A prefix read compared against a magic-byte table is NOT a decode: no parser
- * runs over the bytes, so this belongs in the harness process. Every actual
- * decoder — zlib, HDF5, PDF — stays in the sandbox (see `enrich.ts`), which is
- * the exposure the sandbox exists to contain.
+ * runs over the bytes, so this belongs in the harness process. So does every
+ * readout a bounded prefix suffices for (see `readout.ts`); a parser that must
+ * walk a whole container stays in the sandbox (see `enrich.ts`).
  *
  * Compression wrappers are reported ALONGSIDE the inner format, never in place of
  * it: `.vcf.gz` is a VCF, and a shape observed on the wrapper would present
  * unrelated data as mechanically alike. The wrapper comes from the bytes; the inner
- * format of a wrapped file comes from its extension chain, which the sandbox
- * header readout then confirms against the decompressed stream.
+ * format of a wrapped file comes from its extension chain, which the header
+ * readout then confirms against the decompressed prefix.
  *
  * The recognised list is a floor, not a closed set. An unrecognised format reports
  * `unknown` with its extension chain preserved, and still joins a shape — shape
@@ -110,6 +110,14 @@ const EXTENSION_FORMATS = {
     yml: "yaml",
     log: "text",
 };
+/**
+ * Every suffix the table above recognises, plus the compression wrappers.
+ *
+ * A dotted token outside this set belongs to the stem: `calls.somatic.vcf.gz` has a
+ * two-token extension chain and a two-token stem, and reading `somatic` as part of
+ * the extension would hide a categorical position inside what looks like a format.
+ */
+export const KNOWN_EXTENSIONS = new Set([...Object.keys(EXTENSION_FORMATS), ...WRAPPER_EXTENSIONS]);
 /**
  * `.bed` is PLINK's binary genotype table AND the interval format, and only the
  * bytes tell them apart (PLINK writes a three-byte magic).

package/dist/input-scan/index.d.ts

@@ -4,9 +4,15 @@
  * The scan observes; the profiler agent groups. Nothing exported here names a kind
  * or an axis (see the input-scan-manifest spec).
  */
-export { MAGIC_PREFIX_BYTES, detectFormat, extensionChain, innerExtensions, type DetectedFormat } from "./formats.js";
+export { KNOWN_EXTENSIONS, MAGIC_PREFIX_BYTES, detectFormat, extensionChain, innerExtensions, type DetectedFormat } from "./formats.js";
+export { detectSets, type DetectSetsOptions } from "./detect-sets.js";
+export { type ClusterEvidence, type ContentSimilarity } from "./clustering.js";
 export { MAX_SAMPLE_VALUES, MAX_SHAPES, nameStem, observeShapes, tokenizeStem, type ObservedShapes } from "./shapes.js";
-export { MAX_SCANNED_FILES, buildManifest, renderInputScanManifest, scanInputTree, type ScanInputTreeArgs } from "./scan.js";
-export { MEMBERS_DECODED_PER_SHAPE, enrichShapes, type EnrichShapesArgs } from "./enrich.js";
-export { computeCoverage, type ProfileCoverage } from "./coverage.js";
+export { MAX_SCANNED_FILES, buildManifest, formatBytes, renderInputScanManifest, scanInputTree, type ScanInputTreeArgs } from "./scan.js";
+export { MAX_CORRESPONDENCES, MAX_CORRESPONDENCE_SAMPLE, MAX_MENU_SETS, buildCorrespondences, buildSetMenu, renderSetMenu, type SetMenu, type SlotCorrespondence, type UnlistedSets, } from "./menu.js";
+export { readoutTargets, selectReadouts, type ReadoutTarget } from "./readout-budget.js";
+export { MEMBERS_DECODED_PER_SHAPE, SANDBOX_CONTAINER_FORMATS, enrichShapes, readHeaders, type EnrichShapesArgs, type ReadHeadersArgs, type ReadoutTargetSpec, } from "./enrich.js";
+export { READOUT_PREFIX_BYTES, READOUT_TEXT_BYTES, readPrefix, type PrefixReadout, type ReadoutFields } from "./readout.js";
 export type { FileShape, FormatCount, HeaderReadout, InputScan, InputScanManifest, PositionCooccurrence, ScannedFile, ShapeValueOverlap, UnstructuredEntry, UnstructuredFiles, VariablePosition, } from "./types.js";
+export type { CompanionFile, DetectedSet, DetectedSets, IncompleteMember, LeftoverFiles, MemberFile, QuarantineReason, QuarantineReasonCount, QuarantineSummary, ReadoutSelection, SetCompleteness, SetMember, SetOrigin, SetRepresentative, SetSlot, SlotLocation, TemplateSegment, WrapperCount, } from "./set-types.js";
+export type { SlotTokenClass } from "./tokens.js";

package/dist/input-scan/index.js

@@ -4,8 +4,11 @@
  * The scan observes; the profiler agent groups. Nothing exported here names a kind
  * or an axis (see the input-scan-manifest spec).
  */
-export { MAGIC_PREFIX_BYTES, detectFormat, extensionChain, innerExtensions } from "./formats.js";
+export { KNOWN_EXTENSIONS, MAGIC_PREFIX_BYTES, detectFormat, extensionChain, innerExtensions } from "./formats.js";
+export { detectSets } from "./detect-sets.js";
 export { MAX_SAMPLE_VALUES, MAX_SHAPES, nameStem, observeShapes, tokenizeStem } from "./shapes.js";
-export { MAX_SCANNED_FILES, buildManifest, renderInputScanManifest, scanInputTree } from "./scan.js";
-export { MEMBERS_DECODED_PER_SHAPE, enrichShapes } from "./enrich.js";
-export { computeCoverage } from "./coverage.js";
+export { MAX_SCANNED_FILES, buildManifest, formatBytes, renderInputScanManifest, scanInputTree } from "./scan.js";
+export { MAX_CORRESPONDENCES, MAX_CORRESPONDENCE_SAMPLE, MAX_MENU_SETS, buildCorrespondences, buildSetMenu, renderSetMenu, } from "./menu.js";
+export { readoutTargets, selectReadouts } from "./readout-budget.js";
+export { MEMBERS_DECODED_PER_SHAPE, SANDBOX_CONTAINER_FORMATS, enrichShapes, readHeaders, } from "./enrich.js";
+export { READOUT_PREFIX_BYTES, READOUT_TEXT_BYTES, readPrefix } from "./readout.js";

package/dist/input-scan/markers.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Stage 1 — a recognised marker claims its subtree.
+ *
+ * Some collections announce themselves. Where a marker file names the collection, it
+ * identifies it more precisely than its filenames ever could, so the claimed files
+ * skip inference entirely: mining a matrix triplet into name templates would report
+ * three unrelated singletons where the directory is one thing.
+ *
+ * The catalogue is a floor, not a closed set.
+ */
+import type { MemberFile } from "./set-types.js";
+export interface MarkerUnit {
+    readonly label: string;
+    readonly root: string;
+    readonly members: readonly MemberFile[];
+}
+/** The marker-claimed units of a tree, and the members no marker claimed. */
+export declare function applyMarkers(files: readonly MemberFile[]): {
+    units: MarkerUnit[];
+    unclaimed: MemberFile[];
+};

package/dist/input-scan/markers.js

@@ -0,0 +1,89 @@
+/**
+ * Stage 1 — a recognised marker claims its subtree.
+ *
+ * Some collections announce themselves. Where a marker file names the collection, it
+ * identifies it more precisely than its filenames ever could, so the claimed files
+ * skip inference entirely: mining a matrix triplet into name templates would report
+ * three unrelated singletons where the directory is one thing.
+ *
+ * The catalogue is a floor, not a closed set.
+ */
+import { basenameOf, dirnameOf } from "./tokens.js";
+/** A marker whose presence claims the directory it sits in, and nothing below it. */
+const DIRECTORY_MARKERS = [
+    {
+        label: "feature-barcode-matrix",
+        requires: [/^matrix\.mtx(\.gz)?$/, /^barcodes\.tsv(\.gz)?$/, /^(features|genes)\.tsv(\.gz)?$/],
+    },
+];
+/**
+ * A marker whose presence claims its whole subtree.
+ *
+ * A descriptor may need corroboration: a bare descriptor file can sit at the root of a
+ * tree it does not describe, and letting it claim everything below would swallow the
+ * dataset it happens to share a directory with.
+ */
+const SUBTREE_MARKERS = [
+    { label: "study-manifest", name: "meta_study.txt" },
+    { label: "dataset-descriptor", name: "dataset_description.json", corroborate: /^sub-/ },
+];
+function hasCorroboratingChild(files, root, pattern) {
+    const prefix = root === "" ? "" : `${root}/`;
+    return files.some((file) => file.path.startsWith(prefix) && pattern.test(file.path.slice(prefix.length).split("/")[0]));
+}
+/** The marker-claimed units of a tree, and the members no marker claimed. */
+export function applyMarkers(files) {
+    const byDirectory = new Map();
+    for (const file of files) {
+        const directory = dirnameOf(file.path);
+        const bucket = byDirectory.get(directory);
+        if (bucket)
+            bucket.push(file);
+        else
+            byDirectory.set(directory, [file]);
+    }
+    const subtrees = [];
+    const directories = [];
+    for (const [directory, members] of byDirectory) {
+        const names = members.map((member) => basenameOf(member.path));
+        for (const marker of DIRECTORY_MARKERS) {
+            if (marker.requires.every((pattern) => names.some((name) => pattern.test(name))))
+                directories.push({ label: marker.label, root: directory });
+        }
+        for (const marker of SUBTREE_MARKERS) {
+            if (!names.includes(marker.name))
+                continue;
+            if (marker.corroborate && !hasCorroboratingChild(files, directory, marker.corroborate))
+                continue;
+            subtrees.push({ label: marker.label, root: directory });
+        }
+    }
+    // An outer claim wins: a study that contains a descriptor is one study, not two units.
+    subtrees.sort((a, b) => a.root.length - b.root.length);
+    const outermost = [];
+    for (const candidate of subtrees) {
+        const covered = outermost.some((claim) => candidate.root === claim.root || candidate.root.startsWith(claim.root === "" ? "" : `${claim.root}/`));
+        if (!covered)
+            outermost.push(candidate);
+    }
+    const claimed = new Set();
+    const units = [];
+    for (const claim of outermost) {
+        const prefix = claim.root === "" ? "" : `${claim.root}/`;
+        const members = files.filter((file) => !claimed.has(file.path) && file.path.startsWith(prefix));
+        if (members.length === 0)
+            continue;
+        for (const member of members)
+            claimed.add(member.path);
+        units.push({ label: claim.label, root: claim.root, members });
+    }
+    for (const claim of directories) {
+        const members = (byDirectory.get(claim.root) ?? []).filter((file) => !claimed.has(file.path));
+        if (members.length === 0)
+            continue;
+        for (const member of members)
+            claimed.add(member.path);
+        units.push({ label: claim.label, root: claim.root, members });
+    }
+    return { units, unclaimed: files.filter((file) => !claimed.has(file.path)) };
+}