@inflexa-ai/harness 0.28.0 → 0.29.0

package/dist/execution/staged-input.d.ts

@@ -26,10 +26,10 @@ export interface StagedInput {
     readonly hash: string;
     readonly size: number;
     /**
-     * Last-modification time in epoch milliseconds. With `size` and `fileId` it forms
-     * the file's drift signature — the value a consumer compares against a completed
-     * profile's `inputFiles` to decide whether the same bytes were profiled, since a
-     * `fileId` is a path identity and survives an in-place content edit unchanged.
+     * Last-modification time in epoch milliseconds. With `size` and `fileId` it feeds
+     * the kept-files `inputSignature` a completed profile persists — what lets a
+     * consumer notice the bytes behind a path changed, since a `fileId` is a path
+     * identity and survives an in-place content edit unchanged.
      *
      * Embedder-supplied and opaque: the CLI reads it from the same `stat` that yields
      * `size`; a managed service supplies its object store's last-modified epoch. The

package/dist/index.d.ts

@@ -115,8 +115,12 @@ export { DATA_PROFILE_RUN_LITERAL } from "./contracts/data-profile.js";
 export { initCortexState } from "./state/init.js";
 export { upsertAnalysis } from "./state/analyses.js";
 export { clearDataProfile, loadDataProfileStatus, tryRetryDataProfile, reconcileOrphanedDataProfile } from "./state/data-profile.js";
-export type { DataProfileAxis, DataProfileCoverage, DataProfileFile, DataProfileInputFile, DataProfileInputSignature, DataProfileKind, DataProfileLifecycleStatus, DataProfileOrganism, DataProfileQualityAssessment, DataProfileResult, DataProfileSubjectSource, } from "./contracts/data-profile.js";
+export type { DataProfileAxis, DataProfileChecked, DataProfileCompanionCompleteness, DataProfileCoverage, DataProfileDimension, DataProfileFile, DataProfileGroup, DataProfileGroupSlot, DataProfileInputSignature, DataProfileKind, DataProfileLifecycleStatus, DataProfileMemberAnnotation, DataProfileObservation, DataProfileOrganism, DataProfilePartition, DataProfileProbeReport, DataProfileQualityAssessment, DataProfileQuarantine, DataProfileRecipeStep, DataProfileResult, DataProfileSubjectSource, } from "./contracts/data-profile.js";
 export type { DataProfileStatus } from "./state/data-profile.js";
+export { DIMENSION_CATEGORIES, DIMENSION_CATEGORY_IDS, DIMENSION_PROBES, DIMENSION_PROBE_IDS, GROUP_CATEGORIES, GROUP_CATEGORY_IDS, GROUP_ROLES, GROUP_ROLE_IDS, PROBE_OUTCOME_IDS, dimensionCategoryEntry, dimensionScope, groupCategoryEntry, } from "./contracts/profile-vocabulary.js";
+export type { DimensionCategory, DimensionCategoryEntry, DimensionProbe, DimensionProbeEntry, DimensionScope, DimensionTreatment, GroupCategory, GroupCategoryEntry, GroupRole, GroupRoleEntry, ProbeOutcome, } from "./contracts/profile-vocabulary.js";
+export { dimensionCardinalities, profileCaveats, profileDatasetFileCount, profileDimensions, profileFileRecords, profileGroups, } from "./app/data-profile-view.js";
+export type { ProfileDimensionView, ProfileGroupView } from "./app/data-profile-view.js";
 export { createAnalysisPurge } from "./state/purge-analysis.js";
 export type { AnalysisPurge, AnalysisPurgeDeps, AnalysisPurgeOutcome } from "./state/purge-analysis.js";
 export type { StagedInput } from "./execution/staged-input.js";

package/dist/index.js

@@ -204,6 +204,13 @@ export { upsertAnalysis } from "./state/analyses.js";
 // analysis's input set empties (deferring on a running workflow), so the UI
 // stops advertising a profile that describes files the analysis no longer has.
 export { clearDataProfile, loadDataProfileStatus, tryRetryDataProfile, reconcileOrphanedDataProfile } from "./state/data-profile.js";
+// The controlled vocabulary the profiler draws from, as data. A consumer rendering a
+// stored group needs the category's meaning, and the package owns the catalogue — a host
+// does not extend it.
+export { DIMENSION_CATEGORIES, DIMENSION_CATEGORY_IDS, DIMENSION_PROBES, DIMENSION_PROBE_IDS, GROUP_CATEGORIES, GROUP_CATEGORY_IDS, GROUP_ROLES, GROUP_ROLE_IDS, PROBE_OUTCOME_IDS, dimensionCategoryEntry, dimensionScope, groupCategoryEntry, } from "./contracts/profile-vocabulary.js";
+// Reading a persisted profile of either era through one shape — the projection every
+// consumer uses so that a row written before groups existed still renders.
+export { dimensionCardinalities, profileCaveats, profileDatasetFileCount, profileDimensions, profileFileRecords, profileGroups, } from "./app/data-profile-view.js";
 // Analysis reclamation — `purgeAnalysis` reclaims an analysis's whole Postgres
 // footprint from its id alone; workspace files stay the embedder's to dispose of.
 export { createAnalysisPurge } from "./state/purge-analysis.js";

package/dist/input-scan/clustering.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Stage 2 — sibling directories that agree become one template.
+ *
+ * A tree whose per-entity variation is expressed as one directory per entity is
+ * structure, not the absence of it: grouping within a directory sees one entity at a
+ * time and reports thousands of unrelated observations. Sibling directories whose
+ * contents agree on name template and format census cluster into instances of one
+ * template, and the directory segment becomes a slot like any other.
+ *
+ * Agreement is judged on three channels, strongest first: the masked name templates
+ * beneath each directory, the format census when the directory names are already
+ * skeleton-kin, and the immediate subdirectory layout. Content-schema agreement is a
+ * fourth channel this pipeline cannot read, so it is injected — see
+ * {@link ContentSimilarity}.
+ */
+import type { MemberFile } from "./set-types.js";
+export interface DirectoryNode {
+    readonly name: string;
+    readonly children: Map<string, DirectoryNode>;
+    readonly files: MemberFile[];
+    readonly fineSignature: Set<string>;
+    readonly coarseSignature: Set<string>;
+    readonly childNames: Set<string>;
+}
+/** One concrete directory, carrying the values of the template positions above it. */
+export interface DirectoryEntry {
+    readonly node: DirectoryNode;
+    readonly varValues: readonly string[];
+}
+/** What a content-schema comparison would be given to judge two candidate clusters by. */
+export interface ClusterEvidence {
+    readonly names: readonly string[];
+    readonly files: readonly MemberFile[];
+}
+/**
+ * Similarity from file contents — header fields, delimiters, column names.
+ *
+ * The extension point for read-based evidence: this pipeline is pure over paths and
+ * sizes, so a host that has header readouts injects the comparison rather than the
+ * pipeline reaching for a decoder. Returning `undefined` means "no opinion".
+ */
+export type ContentSimilarity = (a: ClusterEvidence, b: ClusterEvidence) => number | undefined;
+export declare function buildTree(files: readonly MemberFile[]): DirectoryNode;
+/**
+ * Postorder masked-template signatures: the fine one keys on name skeletons, the
+ * coarse one only on extensions. Both are capped — a wide directory needs no more
+ * keys than the cap to be compared by.
+ */
+export declare function computeSignatures(node: DirectoryNode): DirectoryNode;
+/**
+ * Cluster the child directories of a set of template-equivalent parents.
+ *
+ * Same-named children are one candidate by construction — they already occupy the same
+ * template position. Candidates then merge by agreement, majority first: the largest
+ * seeds a cluster, which is the rule that keeps a lone odd directory outside rather
+ * than dragging the cluster toward it. A second agglomerative pass merges whole
+ * clusters to a fixpoint, because the largest candidate can fail against each sibling
+ * individually while the siblings agree with each other.
+ */
+export declare function clusterChildren(entries: readonly DirectoryEntry[], contentSimilarity?: ContentSimilarity): {
+    names: string[];
+    entries: DirectoryEntry[];
+}[];

package/dist/input-scan/clustering.js

@@ -0,0 +1,208 @@
+/**
+ * Stage 2 — sibling directories that agree become one template.
+ *
+ * A tree whose per-entity variation is expressed as one directory per entity is
+ * structure, not the absence of it: grouping within a directory sees one entity at a
+ * time and reports thousands of unrelated observations. Sibling directories whose
+ * contents agree on name template and format census cluster into instances of one
+ * template, and the directory segment becomes a slot like any other.
+ *
+ * Agreement is judged on three channels, strongest first: the masked name templates
+ * beneath each directory, the format census when the directory names are already
+ * skeleton-kin, and the immediate subdirectory layout. Content-schema agreement is a
+ * fourth channel this pipeline cannot read, so it is injected — see
+ * {@link ContentSimilarity}.
+ */
+import { DATE_SYMBOL, DIGIT_SYMBOL, ID_SYMBOL, maskSegment, skeletonOf, tokenizeName } from "./tokens.js";
+import { COARSE_SIMILARITY, CONTAINMENT_SIMILARITY, FINE_SIMILARITY, MAX_SIGNATURE_KEYS } from "./tuning.js";
+function newNode(name) {
+    return { name, children: new Map(), files: [], fineSignature: new Set(), coarseSignature: new Set(), childNames: new Set() };
+}
+export function buildTree(files) {
+    const root = newNode("");
+    for (const file of files) {
+        const segments = file.path.split("/");
+        let node = root;
+        for (let i = 0; i < segments.length - 1; i++) {
+            const name = segments[i];
+            let child = node.children.get(name);
+            if (!child) {
+                child = newNode(name);
+                node.children.set(name, child);
+            }
+            node = child;
+        }
+        node.files.push(file);
+    }
+    return root;
+}
+function coarseKey(suffix) {
+    return suffix ? `*.${suffix}` : "*";
+}
+/**
+ * Postorder masked-template signatures: the fine one keys on name skeletons, the
+ * coarse one only on extensions. Both are capped — a wide directory needs no more
+ * keys than the cap to be compared by.
+ */
+export function computeSignatures(node) {
+    for (const file of node.files) {
+        const { tokens, suffix } = tokenizeName(file.name);
+        const key = skeletonOf(tokens) + (suffix ? ` ${suffix}` : "");
+        if (node.fineSignature.size < MAX_SIGNATURE_KEYS)
+            node.fineSignature.add(key);
+        node.coarseSignature.add(coarseKey(suffix));
+    }
+    for (const child of node.children.values()) {
+        computeSignatures(child);
+        node.childNames.add(child.name);
+        const masked = maskSegment(child.name);
+        for (const key of child.fineSignature)
+            if (node.fineSignature.size < MAX_SIGNATURE_KEYS)
+                node.fineSignature.add(`${masked}/${key}`);
+        for (const key of child.coarseSignature)
+            node.coarseSignature.add(`${masked}/${key}`);
+    }
+    return node;
+}
+function jaccard(a, b) {
+    if (a.size === 0 && b.size === 0)
+        return 1;
+    const [small, big] = a.size <= b.size ? [a, b] : [b, a];
+    let intersection = 0;
+    for (const key of small)
+        if (big.has(key))
+            intersection++;
+    return intersection / (a.size + b.size - intersection);
+}
+function containment(a, b) {
+    const [small, big] = a.size <= b.size ? [a, b] : [b, a];
+    if (small.size === 0)
+        return 0;
+    let intersection = 0;
+    for (const key of small)
+        if (big.has(key))
+            intersection++;
+    return intersection / small.size;
+}
+function hasMask(skeleton) {
+    return skeleton.includes(ID_SYMBOL) || skeleton.includes(DIGIT_SYMBOL) || skeleton.includes(DATE_SYMBOL);
+}
+/**
+ * Two directory names are kin when their masked forms agree, or when they agree up to
+ * the first separator and that head carries masked material — `s#` in `s#_intake` and
+ * `s#_followup` names the same entity position twice.
+ */
+function skeletonKin(a, b) {
+    if (a === b && hasMask(a))
+        return true;
+    const head = a.split(/[_\-.]/)[0];
+    return head === b.split(/[_\-.]/)[0] && hasMask(head);
+}
+function filesUnder(node, out = []) {
+    out.push(...node.files);
+    for (const child of node.children.values())
+        filesUnder(child, out);
+    return out;
+}
+function evidenceOf(candidate) {
+    const files = [];
+    for (const entry of candidate.entries)
+        filesUnder(entry.node, files);
+    return { names: candidate.names, files };
+}
+function agrees(a, b, contentSimilarity) {
+    if (jaccard(a.fine, b.fine) >= FINE_SIMILARITY)
+        return true;
+    if (skeletonKin(a.skeleton, b.skeleton)) {
+        if (jaccard(a.coarse, b.coarse) >= COARSE_SIMILARITY)
+            return true;
+        if (a.subdirs.size > 0 && b.subdirs.size > 0 && jaccard(a.subdirs, b.subdirs) >= COARSE_SIMILARITY)
+            return true;
+    }
+    if (a.fine.size >= 2 && b.fine.size >= 2 && containment(a.fine, b.fine) >= CONTAINMENT_SIMILARITY)
+        return true;
+    if (contentSimilarity) {
+        const score = contentSimilarity(evidenceOf(a), evidenceOf(b));
+        if (score !== undefined && score >= FINE_SIMILARITY)
+            return true;
+    }
+    return false;
+}
+function absorb(into, from) {
+    into.names.push(...from.names);
+    into.entries.push(...from.entries);
+    for (const key of from.fine)
+        if (into.fine.size < MAX_SIGNATURE_KEYS)
+            into.fine.add(key);
+    for (const key of from.coarse)
+        into.coarse.add(key);
+    for (const key of from.subdirs)
+        into.subdirs.add(key);
+}
+/**
+ * Cluster the child directories of a set of template-equivalent parents.
+ *
+ * Same-named children are one candidate by construction — they already occupy the same
+ * template position. Candidates then merge by agreement, majority first: the largest
+ * seeds a cluster, which is the rule that keeps a lone odd directory outside rather
+ * than dragging the cluster toward it. A second agglomerative pass merges whole
+ * clusters to a fixpoint, because the largest candidate can fail against each sibling
+ * individually while the siblings agree with each other.
+ */
+export function clusterChildren(entries, contentSimilarity) {
+    const byName = new Map();
+    for (const entry of entries) {
+        let candidate = byName.get(entry.node.name);
+        if (!candidate) {
+            candidate = {
+                names: [entry.node.name],
+                entries: [],
+                skeleton: maskSegment(entry.node.name),
+                fine: new Set(),
+                coarse: new Set(),
+                subdirs: new Set(),
+            };
+            byName.set(entry.node.name, candidate);
+        }
+        candidate.entries.push(entry);
+        for (const key of entry.node.fineSignature)
+            if (candidate.fine.size < MAX_SIGNATURE_KEYS)
+                candidate.fine.add(key);
+        for (const key of entry.node.coarseSignature)
+            candidate.coarse.add(key);
+        for (const key of entry.node.childNames)
+            candidate.subdirs.add(key);
+    }
+    const candidates = [...byName.values()].sort((a, b) => b.entries.length - a.entries.length || b.fine.size - a.fine.size);
+    const clusters = [];
+    for (const candidate of candidates) {
+        const host = clusters.find((cluster) => agrees(cluster, candidate, contentSimilarity));
+        if (host)
+            absorb(host, candidate);
+        else
+            clusters.push({
+                ...candidate,
+                names: [...candidate.names],
+                entries: [...candidate.entries],
+                fine: new Set(candidate.fine),
+                coarse: new Set(candidate.coarse),
+                subdirs: new Set(candidate.subdirs),
+            });
+    }
+    for (;;) {
+        let merged = false;
+        for (let i = 0; i < clusters.length && !merged; i++) {
+            for (let j = i + 1; j < clusters.length; j++) {
+                if (!agrees(clusters[i], clusters[j], contentSimilarity))
+                    continue;
+                absorb(clusters[i], clusters[j]);
+                clusters.splice(j, 1);
+                merged = true;
+                break;
+            }
+        }
+        if (!merged)
+            break;
+    }
+    return clusters.map((cluster) => ({ names: [...new Set(cluster.names)], entries: cluster.entries }));
+}

package/dist/input-scan/companions.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Companion attachment — an index, a checksum, or a dictionary is not data.
+ *
+ * A companion carries no meaning apart from the file it serves, so it is not a member
+ * and it is not a set: it attaches, and the member becomes the logical unit of a data
+ * file plus its helpers. That is what turns "N variant files, M of them indexed" from
+ * a claim into an arithmetic fact, and it keeps a directory of indexes from mining
+ * into a second set that mirrors the first.
+ */
+import type { MemberFile, SetCompleteness, SetMember } from "./set-types.js";
+import type { ScannedFile } from "./types.js";
+/**
+ * Fold companion files into the data files they serve.
+ *
+ * A companion whose data file is absent stays a member of its own: the attachment is
+ * evidence, and without the data file there is nothing to attach to.
+ */
+export declare function attachCompanions(files: readonly ScannedFile[]): MemberFile[];
+/**
+ * Per-member companion coverage.
+ *
+ * A suffix most of the set carries is expected, and a member without it is incomplete
+ * — named individually, because averaging the difference away hides the one member a
+ * downstream step will fail on. A suffix only some members carry is variation, and
+ * expects nothing.
+ */
+export declare function assessCompleteness(members: readonly MemberFile[]): {
+    members: SetMember[];
+    completeness: SetCompleteness;
+};

package/dist/input-scan/companions.js

@@ -0,0 +1,138 @@
+/**
+ * Companion attachment — an index, a checksum, or a dictionary is not data.
+ *
+ * A companion carries no meaning apart from the file it serves, so it is not a member
+ * and it is not a set: it attaches, and the member becomes the logical unit of a data
+ * file plus its helpers. That is what turns "N variant files, M of them indexed" from
+ * a claim into an arithmetic fact, and it keeps a directory of indexes from mining
+ * into a second set that mirrors the first.
+ */
+import { basenameOf } from "./tokens.js";
+import { EXPECTED_COMPANION_SHARE, MAX_INCOMPLETE_SAMPLE } from "./tuning.js";
+/** Suffixes that name a helper file. The list is a floor; growth is additive. */
+const COMPANION_SUFFIXES = [".bai", ".crai", ".tbi", ".csi", ".idx", ".fai", ".gzi", ".dict", ".md5", ".sha1", ".sha256"];
+/**
+ * Companions whose data file conventionally replaces the extension rather than
+ * extending it: `sample.bai` serves `sample.bam`, where `sample.bam.bai` serves the
+ * same file by extension.
+ */
+const REPLACED_EXTENSIONS = {
+    ".bai": ["bam"],
+    ".crai": ["cram"],
+    ".fai": ["fa", "fasta", "fna"],
+};
+function companionSuffixOf(path) {
+    const name = basenameOf(path).toLowerCase();
+    for (const suffix of COMPANION_SUFFIXES) {
+        // A file that is nothing but the suffix is a name, not a companion.
+        if (name.length > suffix.length && name.endsWith(suffix))
+            return suffix;
+    }
+    return undefined;
+}
+function attachmentTarget(path, suffix, present) {
+    const stripped = path.slice(0, path.length - suffix.length);
+    if (present.has(stripped))
+        return stripped;
+    for (const extension of REPLACED_EXTENSIONS[suffix] ?? []) {
+        const candidate = `${stripped}.${extension}`;
+        if (present.has(candidate))
+            return candidate;
+    }
+    return undefined;
+}
+/**
+ * Fold companion files into the data files they serve.
+ *
+ * A companion whose data file is absent stays a member of its own: the attachment is
+ * evidence, and without the data file there is nothing to attach to.
+ */
+export function attachCompanions(files) {
+    const present = new Set(files.map((file) => file.path));
+    const suffixes = new Map();
+    const targets = new Map();
+    for (const file of files) {
+        const suffix = companionSuffixOf(file.path);
+        if (!suffix)
+            continue;
+        const target = attachmentTarget(file.path, suffix, present);
+        if (target === undefined || target === file.path)
+            continue;
+        suffixes.set(file.path, suffix);
+        targets.set(file.path, target);
+    }
+    // `data.csv.md5` may itself be indexed; follow the chain to the file that is data.
+    const roots = new Map();
+    for (const path of targets.keys()) {
+        let current = path;
+        for (let hops = 0; hops < targets.size + 1; hops++) {
+            const next = targets.get(current);
+            if (next === undefined)
+                break;
+            current = next;
+        }
+        if (current !== path)
+            roots.set(path, current);
+    }
+    const attached = new Map();
+    for (const file of files) {
+        const root = roots.get(file.path);
+        if (root === undefined)
+            continue;
+        const bucket = attached.get(root);
+        const companion = { path: file.path, suffix: suffixes.get(file.path), size: file.size };
+        if (bucket)
+            bucket.push(companion);
+        else
+            attached.set(root, [companion]);
+    }
+    const members = [];
+    for (const file of files) {
+        if (roots.has(file.path))
+            continue;
+        const companions = (attached.get(file.path) ?? []).sort((a, b) => a.path.localeCompare(b.path, "en"));
+        members.push({
+            path: file.path,
+            name: basenameOf(file.path),
+            size: file.size,
+            format: file.format,
+            ...(file.wrapper ? { wrapper: file.wrapper } : {}),
+            companions,
+        });
+    }
+    return members;
+}
+/**
+ * Per-member companion coverage.
+ *
+ * A suffix most of the set carries is expected, and a member without it is incomplete
+ * — named individually, because averaging the difference away hides the one member a
+ * downstream step will fail on. A suffix only some members carry is variation, and
+ * expects nothing.
+ */
+export function assessCompleteness(members) {
+    const counts = new Map();
+    for (const member of members) {
+        for (const suffix of new Set(member.companions.map((companion) => companion.suffix))) {
+            counts.set(suffix, (counts.get(suffix) ?? 0) + 1);
+        }
+    }
+    const expectedCompanions = [...counts.entries()]
+        .filter(([, count]) => count > members.length * EXPECTED_COMPANION_SHARE)
+        .map(([suffix]) => suffix)
+        .sort((a, b) => a.localeCompare(b, "en"));
+    const assessed = members.map((member) => {
+        const carried = new Set(member.companions.map((companion) => companion.suffix));
+        return { ...member, missingCompanions: expectedCompanions.filter((suffix) => !carried.has(suffix)) };
+    });
+    const incomplete = assessed.filter((member) => member.missingCompanions.length > 0);
+    return {
+        members: assessed,
+        completeness: {
+            expectedCompanions,
+            completeMembers: assessed.length - incomplete.length,
+            incompleteMembers: incomplete.length,
+            incompleteSample: incomplete.slice(0, MAX_INCOMPLETE_SAMPLE).map((member) => ({ path: member.path, missingCompanions: member.missingCompanions })),
+        },
+    };
+}

package/dist/input-scan/decoder/container_readout.py

@@ -0,0 +1,171 @@
+"""Header readouts a bounded file prefix cannot produce.
+
+Parquet keeps its schema in a footer, the HDF5 families spread object structure
+across the file, and PDF and DOCX both index their contents from a trailer.
+Reading any of them means parsing the whole container, which is the exposure the
+sandbox exists to contain; every other readout the input scan needs is
+prefix-sufficient and runs in the harness process.
+
+Reads one path per argument and writes one JSON object per line to stdout:
+``{"path": ..., "fields": {...}}``, plus an ``"unavailable"`` key when the
+container could not be read.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import zipfile
+
+PARQUET_MAGIC = b"PAR1"
+HDF5_MAGIC = b"\x89HDF\r\n\x1a\n"
+
+PARQUET_SUFFIXES = (".parquet", ".pq")
+HDF5_SUFFIXES = (".h5", ".hdf5", ".h5ad", ".loom")
+
+MAX_FIELD_CHARS = 200
+MAX_NAME_CHARS = 24
+MAX_NAMES_REPORTED = 12
+MAX_ROOT_KEYS = 20
+MAX_REASON_CHARS = 80
+
+# A trailer-indexed document is read whole; past this it is not read at all.
+DOCUMENT_BYTE_CAP = 33_554_432
+
+HDF5_PROBES = ("X", "obs", "var", "matrix", "layers")
+
+
+def clip(text: str, limit: int = MAX_FIELD_CHARS) -> str:
+    return " ".join(str(text).split())[:limit]
+
+
+def reason(exc: BaseException) -> str:
+    return type(exc).__name__ + ": " + clip(str(exc), MAX_REASON_CHARS)
+
+
+def join_names(names: list) -> str:
+    return clip(", ".join(str(name)[:MAX_NAME_CHARS] for name in names[:MAX_NAMES_REPORTED]))
+
+
+def parquet_fields(path: str) -> dict:
+    try:
+        import pyarrow.parquet as pq
+    except ImportError:
+        return {"unavailable": "pyarrow is not installed in this sandbox"}
+    try:
+        handle = pq.ParquetFile(path)
+        schema = handle.schema_arrow
+        return {
+            "columnCount": len(schema.names),
+            "columns": join_names(list(schema.names)),
+            "columnTypes": join_names([field.type for field in schema]),
+            "rowCount": handle.metadata.num_rows,
+            "rowGroups": handle.metadata.num_row_groups,
+        }
+    except Exception as exc:
+        return {"unavailable": "parquet read failed: " + clip(str(exc), MAX_REASON_CHARS)}
+
+
+def hdf5_fields(path: str) -> dict:
+    try:
+        import h5py
+    except ImportError:
+        return {"unavailable": "h5py is not installed in this sandbox"}
+    try:
+        with h5py.File(path, "r") as handle:
+            keys = list(handle.keys())
+            fields = {"rootKeys": join_names(keys[:MAX_ROOT_KEYS]), "rootKeyCount": len(keys)}
+            for probe in HDF5_PROBES:
+                shape = getattr(handle.get(probe), "shape", None)
+                if shape:
+                    fields[probe + "Shape"] = "x".join(str(extent) for extent in shape)
+            return fields
+    except Exception as exc:
+        return {"unavailable": "hdf5 read failed: " + clip(str(exc), MAX_REASON_CHARS)}
+
+
+def pdf_fields(path: str) -> dict:
+    if os.path.getsize(path) > DOCUMENT_BYTE_CAP:
+        return {"unavailable": "document larger than the readout's byte cap"}
+    try:
+        from pypdf import PdfReader
+    except ImportError:
+        return {"unavailable": "pypdf is not installed in this sandbox"}
+    try:
+        pages = PdfReader(path).pages
+        text = pages[0].extract_text() or "" if len(pages) else ""
+        return {"pageCount": len(pages), "firstPageText": clip(text)}
+    except Exception as exc:
+        return {"unavailable": "pdf read failed: " + clip(str(exc), MAX_REASON_CHARS)}
+
+
+def docx_fields(path: str) -> dict:
+    if os.path.getsize(path) > DOCUMENT_BYTE_CAP:
+        return {"unavailable": "document larger than the readout's byte cap"}
+    try:
+        with zipfile.ZipFile(path) as archive:
+            names = archive.namelist()
+            fields = {"parts": len(names)}
+            if "docProps/app.xml" in names:
+                properties = archive.read("docProps/app.xml").decode("utf8", "replace")
+                for tag in ("Pages", "Words"):
+                    match = re.search(f"<{tag}>(\\d+)</{tag}>", properties)
+                    if match:
+                        fields[tag.lower()] = int(match.group(1))
+            return fields
+    except Exception as exc:
+        return {"unavailable": "docx read failed: " + clip(str(exc), MAX_REASON_CHARS)}
+
+
+def reader_for(path: str, magic: bytes):
+    """The reader a path belongs to, by extension first and container magic second."""
+    lowered = path.lower()
+    if lowered.endswith(PARQUET_SUFFIXES) or magic.startswith(PARQUET_MAGIC):
+        return parquet_fields
+    if lowered.endswith(HDF5_SUFFIXES) or magic.startswith(HDF5_MAGIC):
+        return hdf5_fields
+    if lowered.endswith(".pdf"):
+        return pdf_fields
+    if lowered.endswith(".docx"):
+        return docx_fields
+    return None
+
+
+def describe(path: str) -> dict:
+    with open(path, "rb") as handle:
+        magic = handle.read(len(HDF5_MAGIC))
+    reader = reader_for(path, magic)
+    if reader is None:
+        return {"unavailable": "not a container this decoder reads"}
+    return reader(path)
+
+
+def render(path: str) -> str:
+    try:
+        fields = describe(path)
+    except Exception as exc:
+        fields = {"unavailable": reason(exc)}
+    note = fields.pop("unavailable", None)
+    record = {"path": path, "fields": fields}
+    if note:
+        record["unavailable"] = note
+    try:
+        return json.dumps(record)
+    except Exception as exc:
+        return json.dumps({"path": path, "fields": {}, "unavailable": reason(exc)})
+
+
+def main(paths: list) -> int:
+    for path in paths:
+        try:
+            line = render(path)
+        except Exception as exc:
+            line = json.dumps({"path": str(path), "fields": {}, "unavailable": reason(exc)})
+        sys.stdout.write(line + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))