@inflexa-ai/harness 0.6.0 → 0.7.0

package/dist/tools/sandbox/list-available-refs.js

@@ -1,239 +1,44 @@
 /**
- * Reference-store discovery — two coordinated surfaces over the read-only
- * store at /mnt/refs:
+ * Reference-store discovery: `createListAvailableRefsTool`, a bounded, no-follow
+ * inventory of what is ACTUALLY provisioned in the reference store (managed
+ * installs and user-added files alike). Narrow with `path`/`category`, filter
+ * with `query`, cap with `limit`; every response carries `returned`/`total`/
+ * `hasMore` and the `categories` present, so a listing is never an unbounded
+ * dump. Symlinks are never followed; nothing can be downloaded at runtime.
  *
- * - `createListAvailableRefsTool` — the wired sandbox tool. A bounded, no-follow
- *   inventory of what is ACTUALLY mounted (managed installs, user-added files),
- *   scanned live inside the sandbox. Narrow with `path`/`category`, filter with
- *   `query`, cap with `limit`; every response carries `returned`/`total`/
- *   `hasMore` and the `categories` present, so a listing is never an unbounded
- *   dump. Symlinks are never followed; nothing can be downloaded at runtime.
- * - `queryRefs` — a pure query over a parsed `/mnt/refs/registry.json` manifest
- *   (`files.by_category`), filtering on fields the registry carries: `local_path`,
- *   `category`, `subtype`, `organism`, `tax_id`, `dataset`, `rows`.
+ * The store is read from the host filesystem at the embedder-supplied
+ * `refStorePath` and reported at `/mnt/refs`, the container path sandboxes mount
+ * it at — read where it lives, render where the caller will open it. Reading it
+ * host-side is what lets a planner ask what reference data exists before any
+ * sandbox is created; a step agent asking mid-run sees the same store, because
+ * the mount and this path are the same bytes.
+ *
+ * The scan reports paths; meaning is joined on afterwards from the install
+ * receipt and the catalog it names (organism, format, and each file's internal
+ * shape). Callers search by what the data IS, so the store's directory layout
+ * stays an installer detail rather than an interface anything encodes. Metadata
+ * only ever ENRICHES an entry — a file with no receipt and no catalog match is
+ * still returned, because a user-provided store is as valid as a managed one.
  */
-import { basename, join, posix as posixPath } from "node:path";
+import { lstat, readFile, readdir } from "node:fs/promises";
+import { join as hostJoin, posix as posixPath } from "node:path";
 import { ok } from "neverthrow";
 import { z } from "zod";
 import { REFERENCE_DATA_CATALOG } from "../../reference-data/catalog.js";
 import { parseReferenceInstallReceipt } from "../../reference-data/receipt.js";
 import { defineTool } from "../define-tool.js";
-import { runSandboxExec } from "../workspace/run-exec.js";
+/** Container path the ref store is mounted at; every reported path is rendered under it. */
 const REFS_ROOT = "/mnt/refs";
 const MAX_ENTRIES = 200;
 const MAX_SCANNED_ENTRIES = 2_000;
 const MAX_PATH_BYTES = 40_000;
 const MAX_ENVELOPE_BYTES = 64_000;
 const MAX_INPUT_PATH_BYTES = 4_096;
-/** Default number of file entries a registry listing (`queryRefs`) returns. */
-const DEFAULT_LIMIT = 40;
-const SCAN_SCRIPT = String.raw `
-import json, os, sys
-
-ROOT = sys.argv[1]
-RELATIVE = sys.argv[2]
-TARGET = ROOT if RELATIVE == "." else os.path.join(ROOT, *RELATIVE.split("/"))
-MAX_ENTRIES = int(sys.argv[3])
-MAX_SCANNED = int(sys.argv[4])
-MAX_PATH_BYTES = int(sys.argv[5])
-MAX_ENVELOPE_BYTES = int(sys.argv[6])
-
-def emit(value):
-    print(json.dumps(value, separators=(",", ":")))
-
-def capped_entries(path, limit):
-    """Inspect and sort at most limit names; probe one extra only to mark truncation."""
-    values = []
-    visited = 0
-    cut = False
-    try:
-        with os.scandir(path) as iterator:
-            for entry in iterator:
-                if visited >= limit:
-                    cut = True
-                    break
-                visited += 1
-                try:
-                    if entry.is_symlink():
-                        kind, size = "symlink", 0
-                    elif entry.is_file(follow_symlinks=False):
-                        kind, size = "file", entry.stat(follow_symlinks=False).st_size
-                    elif entry.is_dir(follow_symlinks=False):
-                        kind, size = "directory", 0
-                    else:
-                        kind, size = "other", 0
-                    values.append((entry.name, kind, size))
-                except OSError:
-                    continue
-    except OSError:
-        return [], False
-    values.sort(key=lambda value: value[0])
-    return values, cut
-
-if not os.path.isdir(ROOT) or os.path.islink(ROOT):
-    emit({"state": "unavailable", "entries": [], "scannedEntries": 0, "truncated": False})
-    raise SystemExit(0)
-
-relative = RELATIVE
-cursor = ROOT
-if relative != ".":
-    for part in relative.split(os.sep):
-        cursor = os.path.join(cursor, part)
-        if os.path.islink(cursor):
-            emit({"state": "symlink", "entries": [], "scannedEntries": 0, "truncated": False})
-            raise SystemExit(0)
-
-if not os.path.exists(TARGET):
-    emit({"state": "not_found", "entries": [], "scannedEntries": 0, "truncated": False})
-    raise SystemExit(0)
-if not os.path.isdir(TARGET):
-    emit({"state": "not_a_directory", "entries": [], "scannedEntries": 0, "truncated": False})
-    raise SystemExit(0)
-
-entries = []
-scanned = 0
-path_bytes = 0
-truncated = False
-
-children, children_cut = capped_entries(TARGET, min(MAX_ENTRIES + 2, MAX_SCANNED))
-truncated = children_cut
-
-for name, kind, size in children:
-    path = os.path.join(TARGET, name)
-    rel = os.path.relpath(path, ROOT).replace(os.sep, "/")
-    if rel == ".inflexa" or rel.startswith(".inflexa/") or rel == "registry.json":
-        continue
-    scanned += 1
-    rendered = "/mnt/refs/" + rel
-    item = None
-
-    if kind == "symlink":
-        item = {"path": rendered, "kind": "symlink", "bytes": 0}
-    elif kind == "file":
-        item = {"path": rendered, "kind": "file", "bytes": size}
-    elif kind == "directory":
-        total_bytes = 0
-        file_count = 0
-        file_types = set()
-        directory_truncated = False
-        stack = [path]
-        while stack and scanned < MAX_SCANNED:
-            current = stack.pop()
-            descendants, descendants_cut = capped_entries(current, MAX_SCANNED - scanned)
-            directory_truncated = directory_truncated or descendants_cut
-            child_directories = []
-            for child_name, child_kind, child_size in descendants:
-                scanned += 1
-                if child_kind == "directory":
-                    child_directories.append(os.path.join(current, child_name))
-                elif child_kind == "file":
-                    file_count += 1
-                    total_bytes += child_size
-                    suffix = os.path.splitext(child_name)[1].lower()
-                    if suffix:
-                        file_types.add(suffix)
-                if scanned >= MAX_SCANNED:
-                    directory_truncated = True
-                    break
-            stack.extend(reversed(child_directories))
-        item = {
-            "path": rendered,
-            "kind": "directory",
-            "bytes": total_bytes,
-            "fileCount": file_count,
-            "fileTypes": sorted(file_types)[:10],
-            "truncated": directory_truncated,
-        }
-        truncated = truncated or directory_truncated
-
-    if item is not None:
-        if len(entries) < MAX_ENTRIES and path_bytes + len(rendered.encode()) <= MAX_PATH_BYTES:
-            entries.append(item)
-            path_bytes += len(rendered.encode())
-        else:
-            truncated = True
-    if scanned >= MAX_SCANNED:
-        truncated = True
-        break
-
-receipts = []
-receipt_bytes = 0
-metadata_dir = os.path.join(ROOT, ".inflexa")
-receipts_dir = os.path.join(metadata_dir, "receipts")
-try:
-    if os.path.islink(metadata_dir) or os.path.islink(receipts_dir):
-        raise ValueError("symlinked metadata directory")
-    receipt_entries, _ = capped_entries(receipts_dir, 100)
-    for name, kind, _ in receipt_entries:
-        path = os.path.join(receipts_dir, name)
-        if kind != "file":
-            continue
-        # Per-file, so one unreadable or malformed receipt costs only its own labels.
-        # A shared try would abandon the loop and silently drop every receipt sorting
-        # after the bad one, quietly stripping metadata from datasets that are fine.
-        try:
-            with open(path, "rb") as handle:
-                raw = handle.read(16385)
-            if len(raw) > 16384 or receipt_bytes + len(raw) > 32768:
-                continue
-            parsed = json.loads(raw)
-        except Exception:
-            continue
-        receipts.append(parsed)
-        receipt_bytes += len(raw)
-except Exception:
-    pass
-
-legacy_entries = []
-try:
-    registry_path = os.path.join(ROOT, "registry.json")
-    if os.path.islink(registry_path):
-        raise ValueError("symlinked registry")
-    with open(registry_path, "rb") as handle:
-        registry = json.loads(handle.read(131073))
-    if isinstance(registry, dict):
-        by_category = registry.get("files", {}).get("by_category", {})
-        if isinstance(by_category, dict):
-            for category in sorted(by_category):
-                values = by_category.get(category)
-                if not isinstance(values, list):
-                    continue
-                for item in values:
-                    if len(legacy_entries) >= 200:
-                        break
-                    if isinstance(item, dict) and isinstance(item.get("local_path"), str):
-                        legacy_entries.append({
-                            "local_path": item["local_path"][:1000],
-                            "category": str(category)[:500],
-                            "dataset": str(item.get("dataset"))[:500] if item.get("dataset") is not None else None,
-                            "subtype": str(item.get("subtype"))[:500] if item.get("subtype") is not None else None,
-                            "organism": str(item.get("organism"))[:500] if item.get("organism") is not None else None,
-                            "rows": item.get("rows"),
-                            "endpoint": item.get("endpoint"),
-                        })
-except Exception:
-    pass
-
-payload = {
-    "state": "empty" if len(entries) == 0 else "populated",
-    "entries": entries,
-    "scannedEntries": scanned,
-    "truncated": truncated,
-    "receipts": receipts,
-    "legacyEntries": legacy_entries,
-}
-while len(json.dumps(payload, separators=(",", ":")).encode()) > MAX_ENVELOPE_BYTES:
-    payload["truncated"] = True
-    if payload["legacyEntries"]:
-        payload["legacyEntries"].pop()
-    elif payload["receipts"]:
-        payload["receipts"].pop()
-    elif payload["entries"]:
-        payload["entries"].pop()
-    else:
-        break
-emit(payload)
-`;
+const MAX_RECEIPT_FILES = 100;
+const MAX_RECEIPT_BYTES = 16_384;
+const MAX_RECEIPT_TOTAL_BYTES = 32_768;
+const MAX_REGISTRY_BYTES = 131_072;
+const MAX_LEGACY_ENTRIES = 200;
 const ListAvailableRefsInputSchema = z.object({
     path: z
         .string()
@@ -257,25 +62,6 @@ const ListAvailableRefsInputSchema = z.object({
         .optional()
         .describe(`Max entries to return (default ${MAX_ENTRIES}); the response carries \`returned\`, \`total\`, and \`hasMore\` so truncation is never silent.`),
 });
-const RawScanSchema = z.object({
-    state: z.enum(["unavailable", "empty", "populated", "not_found", "not_a_directory", "symlink"]),
-    entries: z.array(z.discriminatedUnion("kind", [
-        z.object({ path: z.string().startsWith(`${REFS_ROOT}/`), kind: z.literal("file"), bytes: z.number().nonnegative() }),
-        z.object({ path: z.string().startsWith(`${REFS_ROOT}/`), kind: z.literal("symlink"), bytes: z.literal(0) }),
-        z.object({
-            path: z.string().startsWith(`${REFS_ROOT}/`),
-            kind: z.literal("directory"),
-            bytes: z.number().nonnegative(),
-            fileCount: z.number().int().nonnegative(),
-            fileTypes: z.array(z.string()),
-            truncated: z.boolean(),
-        }),
-    ])),
-    scannedEntries: z.number().int().nonnegative(),
-    truncated: z.boolean(),
-    receipts: z.array(z.unknown()).optional(),
-    legacyEntries: z.array(z.unknown()).optional(),
-});
 function resolveRequestedPath(input) {
     if (input === undefined || input === "" || input === REFS_ROOT)
         return { ok: true, path: REFS_ROOT };
@@ -298,6 +84,287 @@ function resolveRequestedPath(input) {
         return { ok: false, reason: "Path is outside /mnt/refs." };
     return { ok: true, path: normalized };
 }
+/**
+ * Inspect at most `limit` names in one directory, sorted by name.
+ *
+ * Sorting BEFORE the cap (rather than capping the raw readdir order) keeps
+ * truncation deterministic: the same store yields the same prefix on every call,
+ * which a durably-cached step result has to be able to promise.
+ */
+async function cappedEntries(directory, limit) {
+    let names;
+    try {
+        names = (await readdir(directory, { withFileTypes: true })).map((entry) => entry.name);
+    }
+    catch {
+        return { values: [], cut: false };
+    }
+    names.sort();
+    const cut = names.length > limit;
+    const values = [];
+    for (const name of names.slice(0, limit)) {
+        try {
+            // lstat, never stat: a symlink is reported as a symlink and never resolved,
+            // so a link out of the store cannot smuggle in a foreign file's size or type.
+            const stats = await lstat(hostJoin(directory, name));
+            if (stats.isSymbolicLink())
+                values.push({ name, kind: "symlink", bytes: 0 });
+            else if (stats.isFile())
+                values.push({ name, kind: "file", bytes: stats.size });
+            else if (stats.isDirectory())
+                values.push({ name, kind: "directory", bytes: 0 });
+            else
+                values.push({ name, kind: "other", bytes: 0 });
+        }
+        catch {
+            continue;
+        }
+    }
+    return { values, cut };
+}
+/** Whether any component of `relative` beneath `root` is a symlink — the no-follow gate. */
+async function crossesSymlink(root, relative) {
+    let cursor = root;
+    for (const segment of relative.split("/")) {
+        cursor = hostJoin(cursor, segment);
+        try {
+            if ((await lstat(cursor)).isSymbolicLink())
+                return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    return false;
+}
+/** Recursively total a directory's files, bounded by the remaining scan budget. */
+async function summarizeDirectory(directory, scanned) {
+    let totalBytes = 0;
+    let fileCount = 0;
+    let truncated = false;
+    const fileTypes = new Set();
+    const stack = [directory];
+    while (stack.length > 0 && scanned < MAX_SCANNED_ENTRIES) {
+        const current = stack.pop();
+        const { values, cut } = await cappedEntries(current, MAX_SCANNED_ENTRIES - scanned);
+        truncated = truncated || cut;
+        const childDirectories = [];
+        for (const child of values) {
+            scanned += 1;
+            if (child.kind === "directory")
+                childDirectories.push(hostJoin(current, child.name));
+            else if (child.kind === "file") {
+                fileCount += 1;
+                totalBytes += child.bytes;
+                const suffix = posixPath.extname(child.name).toLowerCase();
+                if (suffix)
+                    fileTypes.add(suffix);
+            }
+            if (scanned >= MAX_SCANNED_ENTRIES) {
+                truncated = true;
+                break;
+            }
+        }
+        stack.push(...childDirectories.reverse());
+    }
+    return { bytes: totalBytes, fileCount, fileTypes: [...fileTypes].sort().slice(0, 10), truncated, scanned };
+}
+/** Install receipts written by the host installer under `<root>/.inflexa/receipts`. */
+async function readReceipts(root) {
+    const receiptsDirectory = hostJoin(root, ".inflexa", "receipts");
+    if (await crossesSymlink(root, ".inflexa"))
+        return [];
+    const { values } = await cappedEntries(receiptsDirectory, MAX_RECEIPT_FILES);
+    const receipts = [];
+    let totalBytes = 0;
+    for (const entry of values) {
+        if (entry.kind !== "file" || entry.bytes > MAX_RECEIPT_BYTES || totalBytes + entry.bytes > MAX_RECEIPT_TOTAL_BYTES)
+            continue;
+        // Per-file, so one unreadable or malformed receipt costs only its own labels.
+        // A shared try would abandon the loop and silently drop every receipt sorting
+        // after the bad one, quietly stripping metadata from datasets that are fine.
+        try {
+            receipts.push(JSON.parse(await readFile(hostJoin(receiptsDirectory, entry.name), "utf-8")));
+            totalBytes += entry.bytes;
+        }
+        catch {
+            continue;
+        }
+    }
+    return receipts;
+}
+/** Labels from a `registry.json` an externally-provisioned store may carry. */
+async function readLegacyEntries(root) {
+    const legacyEntries = [];
+    try {
+        if (await crossesSymlink(root, "registry.json"))
+            return [];
+        const registryPath = hostJoin(root, "registry.json");
+        if ((await lstat(registryPath)).size > MAX_REGISTRY_BYTES)
+            return [];
+        const registry = JSON.parse(await readFile(registryPath, "utf-8"));
+        if (!registry || typeof registry !== "object")
+            return [];
+        const byCategory = registry.files?.by_category;
+        if (!byCategory || typeof byCategory !== "object")
+            return [];
+        for (const category of Object.keys(byCategory).sort()) {
+            const values = byCategory[category];
+            if (!Array.isArray(values))
+                continue;
+            for (const item of values) {
+                if (legacyEntries.length >= MAX_LEGACY_ENTRIES)
+                    break;
+                if (item && typeof item === "object" && typeof item.local_path === "string") {
+                    legacyEntries.push({ ...item, category });
+                }
+            }
+        }
+    }
+    catch {
+        return legacyEntries;
+    }
+    return legacyEntries;
+}
+/**
+ * Collect FILES anywhere beneath `directory`, bounded by the scan budget.
+ *
+ * Searching by meaning only works if it reaches the files that carry meaning. A
+ * dataset's labels are joined onto its artifact at
+ * `managed/{id}/{version}/{artifact}`, so a listing of the store root — whose only
+ * entries are directories, which carry no such labels — can never match "regulon".
+ * Browsing wants the directory summary; searching wants the leaves.
+ */
+async function collectFiles(directory, relative, budget) {
+    const entries = [];
+    let truncated = false;
+    const stack = [{ path: directory, relative }];
+    while (stack.length > 0 && budget.scanned < MAX_SCANNED_ENTRIES) {
+        const current = stack.pop();
+        const { values, cut } = await cappedEntries(current.path, MAX_SCANNED_ENTRIES - budget.scanned);
+        truncated = truncated || cut;
+        for (const child of values) {
+            budget.scanned += 1;
+            const childRelative = current.relative === "." ? child.name : `${current.relative}/${child.name}`;
+            // Installer metadata is not reference data — it never becomes a search hit.
+            if (childRelative === ".inflexa" || childRelative.startsWith(".inflexa/") || childRelative === "registry.json")
+                continue;
+            if (child.kind === "directory") {
+                stack.push({ path: hostJoin(current.path, child.name), relative: childRelative });
+            }
+            else if (child.kind === "file" || child.kind === "symlink") {
+                if (entries.length >= MAX_ENTRIES) {
+                    truncated = true;
+                }
+                else {
+                    entries.push(child.kind === "symlink"
+                        ? { path: `${REFS_ROOT}/${childRelative}`, kind: "symlink", bytes: 0 }
+                        : { path: `${REFS_ROOT}/${childRelative}`, kind: "file", bytes: child.bytes });
+                }
+            }
+            if (budget.scanned >= MAX_SCANNED_ENTRIES) {
+                truncated = true;
+                break;
+            }
+        }
+    }
+    return { entries, truncated };
+}
+/**
+ * Scan the store beneath `root`, reporting every expected state as data.
+ *
+ * `recursive` selects the two modes: a browse lists immediate children (directories
+ * summarized), a search walks to the files, because only a file carries the labels a
+ * search matches on.
+ */
+async function scanStore(root, relative, recursive = false) {
+    const empty = () => ({ entries: [], scannedEntries: 0, truncated: false });
+    if (!root)
+        return { state: "unavailable", ...empty() };
+    try {
+        if (!(await lstat(root)).isDirectory())
+            return { state: "unavailable", ...empty() };
+    }
+    catch {
+        return { state: "unavailable", ...empty() };
+    }
+    if (relative !== "." && (await crossesSymlink(root, relative)))
+        return { state: "symlink", ...empty() };
+    const target = relative === "." ? root : hostJoin(root, ...relative.split("/"));
+    let targetStats;
+    try {
+        targetStats = await lstat(target);
+    }
+    catch {
+        return { state: "not_found", ...empty() };
+    }
+    if (!targetStats.isDirectory())
+        return { state: "not_a_directory", ...empty() };
+    if (recursive) {
+        const budget = { scanned: 0 };
+        const walked = await collectFiles(target, relative, budget);
+        return {
+            state: walked.entries.length === 0 ? "empty" : "populated",
+            entries: walked.entries,
+            scannedEntries: budget.scanned,
+            truncated: walked.truncated,
+            receipts: await readReceipts(root),
+            legacyEntries: await readLegacyEntries(root),
+        };
+    }
+    const entries = [];
+    let scanned = 0;
+    let pathBytes = 0;
+    let truncated = false;
+    const { values: children, cut } = await cappedEntries(target, Math.min(MAX_ENTRIES + 2, MAX_SCANNED_ENTRIES));
+    truncated = cut;
+    for (const child of children) {
+        const childRelative = relative === "." ? child.name : `${relative}/${child.name}`;
+        if (childRelative === ".inflexa" || childRelative.startsWith(".inflexa/") || childRelative === "registry.json")
+            continue;
+        scanned += 1;
+        const rendered = `${REFS_ROOT}/${childRelative}`;
+        let entry;
+        if (child.kind === "symlink")
+            entry = { path: rendered, kind: "symlink", bytes: 0 };
+        else if (child.kind === "file")
+            entry = { path: rendered, kind: "file", bytes: child.bytes };
+        else if (child.kind === "directory") {
+            const summary = await summarizeDirectory(hostJoin(target, child.name), scanned);
+            scanned = summary.scanned;
+            truncated = truncated || summary.truncated;
+            entry = {
+                path: rendered,
+                kind: "directory",
+                bytes: summary.bytes,
+                fileCount: summary.fileCount,
+                fileTypes: summary.fileTypes,
+                truncated: summary.truncated,
+            };
+        }
+        if (entry) {
+            if (entries.length < MAX_ENTRIES && pathBytes + Buffer.byteLength(rendered, "utf8") <= MAX_PATH_BYTES) {
+                entries.push(entry);
+                pathBytes += Buffer.byteLength(rendered, "utf8");
+            }
+            else {
+                truncated = true;
+            }
+        }
+        if (scanned >= MAX_SCANNED_ENTRIES) {
+            truncated = true;
+            break;
+        }
+    }
+    return {
+        state: entries.length === 0 ? "empty" : "populated",
+        entries,
+        scannedEntries: scanned,
+        truncated,
+        receipts: await readReceipts(root),
+        legacyEntries: await readLegacyEntries(root),
+    };
+}
 function stringValue(value) {
     return typeof value === "string" && value.length > 0 ? value : undefined;
 }
@@ -312,6 +379,9 @@ function enrichEntries(entries, raw) {
             continue;
         const dataset = REFERENCE_DATA_CATALOG.datasets.find((candidate) => candidate.id === receipt.datasetId && candidate.version === receipt.datasetVersion);
         for (const artifact of receipt.artifacts) {
+            // Per-artifact, not per-dataset: format and contents differ between files of the
+            // same dataset (a Seurat object and its Annoy index are read nothing alike).
+            const catalogArtifact = dataset?.artifacts.find((candidate) => candidate.path === artifact.path);
             metadata.set(`${REFS_ROOT}/managed/${receipt.datasetId}/${receipt.datasetVersion}/${artifact.path}`, {
                 datasetId: receipt.datasetId,
                 version: receipt.datasetVersion,
@@ -321,8 +391,11 @@ function enrichEntries(entries, raw) {
                         description: dataset.description,
                         sourceUrl: dataset.sourceUrl,
                         license: dataset.license.url ?? dataset.license.identifier,
+                        category: dataset.recommendation.group,
+                        ...(dataset.organism ? { organism: dataset.organism } : {}),
                     }
                     : {}),
+                ...(catalogArtifact ? { format: catalogArtifact.format, contents: catalogArtifact.contents } : {}),
             });
         }
     }
@@ -351,9 +424,23 @@ function enrichEntries(entries, raw) {
         return extra ? { ...entry, metadata: extra } : entry;
     });
 }
-/** Case-insensitive substring match over an inventory entry's path and joined metadata labels. */
+/**
+ * Case-insensitive substring match over an inventory entry's path and joined metadata labels.
+ * `description` and `contents` are in scope deliberately: a caller searching for what the data
+ * IS ("regulon", "tf-target") should hit a file whose name says only "CollecTRI".
+ */
 function entryMatchesQuery(entry, needle) {
-    const fields = [entry.path, entry.metadata?.datasetId, entry.metadata?.title, entry.metadata?.category, entry.metadata?.subtype, entry.metadata?.organism];
+    const fields = [
+        entry.path,
+        entry.metadata?.datasetId,
+        entry.metadata?.title,
+        entry.metadata?.description,
+        entry.metadata?.category,
+        entry.metadata?.subtype,
+        entry.metadata?.organism,
+        entry.metadata?.format,
+        entry.metadata?.contents,
+    ];
     return fields.some((field) => typeof field === "string" && field.toLowerCase().includes(needle));
 }
 /** The drill-in index: directory groups present in the scan, plus any category labels joined from metadata. */
@@ -369,7 +456,7 @@ function collectCategories(entries) {
 }
 function renderContent(path, scan, entries) {
     if (scan.state === "unavailable")
-        return "Reference store is not mounted at /mnt/refs. Reference data cannot be downloaded from sandbox workloads; provision it through the host setup flow.";
+        return "No reference store is provisioned. Reference data cannot be downloaded at runtime; provision it through the host setup flow.";
     if (scan.state === "not_found")
         return `Reference path does not exist: ${path}`;
     if (scan.state === "not_a_directory")
@@ -377,7 +464,7 @@ function renderContent(path, scan, entries) {
     if (scan.state === "symlink")
         return `Reference path crosses a symlink and was not scanned: ${path}`;
     if (scan.state === "empty")
-        return `Reference store path is mounted but contains no reference files: ${path}`;
+        return `Reference store path exists but contains no reference files: ${path}`;
     const lines = [`Reference files visible beneath ${path}:`];
     for (const entry of entries) {
         const labels = entry.metadata
@@ -391,26 +478,33 @@ function renderContent(path, scan, entries) {
                 ? `file, ${entry.bytes} bytes`
                 : "symlink (not followed)";
         lines.push(`- ${entry.path} (${details}${labels ? `; ${labels}` : ""})`);
+        // The deciding line: whether this file is the thing you need, and how to open it.
+        // Without it a caller is back to inferring both from the filename.
+        const shape = [entry.metadata?.format, entry.metadata?.contents].filter(Boolean).join(" — ");
+        if (shape)
+            lines.push(`    ${shape}`);
     }
     if (scan.truncated)
         lines.push(`Inventory truncated after scanning ${scan.scannedEntries} entries. Call list_available_refs with a narrower path to drill down.`);
     return lines.join("\n");
 }
-/** Create replay-safe reference discovery bound to the active sandbox. */
+/** Create reference discovery over the host-visible reference store. */
 export function createListAvailableRefsTool(deps) {
     return defineTool({
         id: "list_available_refs",
-        executionMode: "workflow",
-        description: "Inspect the reference files actually mounted read-only at /mnt/refs — the ONLY reference data that exists here; nothing can be downloaded at runtime. " +
-            "Returns managed and user-added files even when no manifest names them; valid receipts and legacy registry data only add labels (dataset, category, organism, subtype). Symlinks are never followed. " +
+        description: "Resolve reference data you need into a real path. Reports the reference data actually provisioned for this environment — the ONLY reference data that exists here; nothing can be downloaded at runtime. " +
+            "Search by what the data IS, not by where you expect it: the store's directory layout is an installer detail and is not a stable interface. " +
+            "Catalogued files come back with the organism, the format, and the file's internal shape (key columns, identifier space), so you can pick the right reader and the right species without opening the file first. " +
+            "User-added files are returned too, with whatever labels exist — often none. Symlinks are never followed. " +
+            "Paths are reported as the analysis environment sees them, so a returned path can be used verbatim in a script. " +
             "Results are ALWAYS bounded — prefer a targeted query over a full dump: " +
+            '`query` is the main entry point, a case-insensitive substring filter over each entry\'s path and its dataset/title/category/organism/format/contents labels (e.g. "regulon", "hallmark", "mouse"); ' +
             "`path` inspects one directory beneath /mnt/refs (a returned path, or store-relative) — drill in with it when a listing is truncated; " +
-            '`category` is shorthand for a top-level group (e.g. "atlas_singlecell", "msigdb"), and the groups present come back in `categories`; ' +
-            "`query` is a case-insensitive substring filter over each entry's path and its dataset/title/category/organism/subtype labels; " +
+            '`category` is shorthand for a top-level group (e.g. "pathways", "regulatory-networks"), and the groups present come back in `categories`; ' +
             "`limit` caps the entries returned. The response carries `returned`, `total`, and `hasMore`, so truncation is never silent. " +
-            "Provision additional files through the host and re-run in a sandbox with that store mounted.",
+            "An empty result means the data is genuinely absent — say so and proceed with what you have; do not guess a path. Provision additional files through the host setup flow.",
         inputSchema: ListAvailableRefsInputSchema,
-        execute: async ({ path, category, query, limit }, ctx) => {
+        execute: async ({ path, category, query, limit }) => {
             // `category` is shorthand for a top-level path; an explicit `path` wins.
             const requestedInput = path ?? category;
             const requested = resolveRequestedPath(requestedInput);
@@ -430,33 +524,11 @@ export function createListAvailableRefsTool(deps) {
                 };
                 return ok(response);
             }
-            const execId = `${deps.workflowId}:${deps.stepId}:${deps.nextFunctionId()}`;
-            const result = await runSandboxExec({
-                sandboxClient: deps.sandboxClient,
-                sandbox: deps.sandbox,
-                execId,
-                command: [
-                    "python3",
-                    "-c",
-                    SCAN_SCRIPT,
-                    deps.scanRoot ?? REFS_ROOT,
-                    posixPath.relative(REFS_ROOT, requested.path) || ".",
-                    String(MAX_ENTRIES),
-                    String(MAX_SCANNED_ENTRIES),
-                    String(MAX_PATH_BYTES),
-                    String(MAX_ENVELOPE_BYTES),
-                ],
-                timeoutSeconds: 20,
-                deadlineMs: deps.deadlineMs(),
-                emit: ctx.emit,
-                ...(deps.markExecActive ? { markExecActive: deps.markExecActive } : {}),
-            });
-            if (result.exitCode !== 0)
-                throw new Error(`Reference inventory command failed (${result.exitCode}): ${result.stderr}`);
-            if (Buffer.byteLength(result.stdout, "utf8") > MAX_ENVELOPE_BYTES + 1) {
-                throw new Error("Reference inventory response exceeded its output bound");
-            }
-            const raw = RawScanSchema.parse(JSON.parse(result.stdout));
+            // A query searches the leaves; a bare listing browses this level. Without the
+            // recursive pass a search from the store root only ever sees directories,
+            // which carry no dataset labels, so "find me a regulon network" finds nothing.
+            const needleGiven = (query?.trim() ?? "") !== "";
+            const raw = await scanStore(deps.refStorePath, posixPath.relative(REFS_ROOT, requested.path) || ".", needleGiven);
             const enriched = enrichEntries(raw.entries, raw);
             const categories = collectCategories(enriched);
             const needle = query?.trim().toLowerCase();
@@ -495,166 +567,3 @@ export function createListAvailableRefsTool(deps) {
         },
     });
 }
-const CATEGORY_LABELS = {
-    atlas_singlecell: "Single-cell reference atlases for label transfer (Pan-Cancer T cell — " +
-        "Zheng 2021, Pan-Cancer Myeloid — Cheng 2021, Tumor Immune Cell Atlas — " +
-        "Nieto 2021, Tabula Sapiens v2). Available as .h5ad (Python: anndata, " +
-        "scanpy, scvi-tools, symphonypy) and/or .rds (R: Seurat, ProjecTILs). " +
-        "Tabula Sapiens ships full + immune_subset variants. " +
-        "Use these for label transfer BEFORE falling back to marker genes.",
-    atlas_azimuth: "Azimuth references (Seurat-native): PBMC v1.0.0 and Tonsil v1.0.0. " +
-        "Each entry is a (ref.Rds, idx.annoy) pair stored under " +
-        "/mnt/refs/atlas_azimuth/{pbmc,tonsil}/. Load with " +
-        "Azimuth::LoadReference(path) where path is the directory containing " +
-        "ref.Rds + idx.annoy (e.g. '/mnt/refs/atlas_azimuth/pbmc'), then " +
-        "Azimuth::RunAzimuth(query, reference = path).",
-    atlas_projectils: "ProjecTILs reference atlases (Seurat .rds): human + mouse, CD8 + CD4. " +
-        "Use ProjecTILs::ProjecTILs.classifier() to assign every CD8 to one of " +
-        "the canonical TCF7+/GZMK+/TEX/TPEX/MAIT substates without re-deriving " +
-        "the classifier from scratch.",
-    celltypist_models: "Pre-staged CellTypist .pkl classifier models (Immune_All_Low backed by " +
-        "the Cross-Tissue Immune Cell Atlas; Immune_All_High; Pan_Fetal_Human; " +
-        "COVID19_Immune_Landscape). CELLTYPIST_FOLDER env var is preset, so " +
-        "celltypist.annotate_cells(model='Immune_All_Low') resolves with no " +
-        "network call.",
-    marker_panels: "Marker gene panels: PanglaoDB (community, parquet long-format), " +
-        "CellMarker 2.0 (tumor-context, parquet long-format), and the Cortex " +
-        "hand-curated panel for Stage 1 substates (TCF7+ memory CD8, exhausted " +
-        "CD8, effector-memory CD8, proliferating T, cDC1/cDC2/LAMP3+ DC, SPP1+ " +
-        "TAM, C1Q+ TAM, Tregs) — JSON with full provenance.",
-    gene_signatures: "Curated gene signatures from key tumor-immune papers: Sade-Feldman " +
-        "2018 exhausted CD8 (Table S2), Jerby-Arnon 2018 malignant resistance " +
-        "program (Table S1, Stage 3 classifier substrate), Tirosh 2016 AXL/MITF " +
-        "programs, Puram 2017 p-EMT (Table S5, HNSCC), Reactome cGAS-STING " +
-        "pathway (Stage 2.3 type-I IFN). JSON with PMID/DOI per entry.",
-    normal_reference: "Normal-tissue references for safety firing checks: GTEx v8 per-tissue " +
-        "median TPM (parquet) + sample attributes (parquet). Use to quantify " +
-        "off-target activation of any nominated classifier across healthy tissues.",
-    gene_mappings: "Gene ID Conversion Tables: Entrez/Ensembl/RefSeq/UniProt to symbol (NCBI); " +
-        "orthologs (Ensembl Compara) — file: orthologs_{tax_id}.parquet, columns: " +
-        "tax_id, ensembl_gene_id, entrez_id, symbol, relationship, other_tax_id, " +
-        "other_ensembl_gene_id, other_entrez_id, other_symbol; symmetric — to find " +
-        "human orthologs of Macaca fascicularis genes, open orthologs_9541.parquet " +
-        "and filter other_tax_id == '9606'; cross-division (e.g. yeast↔mouse) " +
-        "requires pivoting through human: yeast→human via orthologs_4896.parquet, " +
-        "then human→mouse via orthologs_9606.parquet",
-    omnipath: "OmniPath (interactions, regulons, annotations)",
-    reactome: "Reactome Pathways",
-    progeny: "PROGENy Pathway Weights (decoupler: source/target/weight/padj)",
-    collectri: "CollecTRI TF-Target Regulons (decoupler: source/target/mor)",
-    dorothea: "DoRothEA TF Regulons (decoupler: source/target/weight/confidence)",
-    lincs: "LINCS L1000 Drug Perturbation Consensus Signatures (up/down gene sets)",
-    hpa: "Human Protein Atlas (tissue RNA expression, target safety/secretome, full 107-column atlas)",
-    wikipathways: "WikiPathways Gene Sets",
-    msigdb: "MSigDB Gene Set Collections",
-    safety_targets: "Curated Off-Target Safety Panel (CSV + JSON: chembl_id, gene_symbol, uniprot, organ_system, severity, clinical_consequence)",
-};
-const CATEGORY_ORDER = [
-    "atlas_singlecell",
-    "atlas_azimuth",
-    "atlas_projectils",
-    "celltypist_models",
-    "marker_panels",
-    "gene_signatures",
-    "normal_reference",
-    "safety_targets",
-    "progeny",
-    "collectri",
-    "dorothea",
-    "omnipath",
-    "lincs",
-    "msigdb",
-    "wikipathways",
-    "reactome",
-    "hpa",
-    "gene_mappings",
-];
-/** The registry's categories in canonical order, unknown ones appended. */
-function orderedCategories(registry) {
-    const byCategory = registry.files.by_category;
-    const known = new Set(CATEGORY_ORDER);
-    return [...CATEGORY_ORDER.filter((c) => byCategory[c]), ...Object.keys(byCategory).filter((c) => !known.has(c))];
-}
-/** Case-insensitive substring match over the descriptive fields the registry carries. */
-function matchesQuery(category, entry, needle) {
-    const haystack = [category, entry.local_path, entry.dataset, entry.organism, entry.subtype, entry.tax_id].filter((v) => typeof v === "string" && v.length > 0);
-    return haystack.some((field) => field.toLowerCase().includes(needle));
-}
-/** One file line: `name: /mnt/refs/<local_path>  (rows, subtype, dataset)`. */
-function renderEntry(entry) {
-    const path = join(REFS_ROOT, entry.local_path);
-    const name = basename(entry.local_path);
-    const info = [entry.rows != null ? `${entry.rows.toLocaleString()} rows` : null, entry.subtype, entry.dataset].filter(Boolean).join(", ");
-    return `  ${name}: ${path}${info ? `  (${info})` : ""}`;
-}
-/**
- * Answer a refs query against a parsed registry. Pure — the tool's `execute` is
- * only the file read plus this call.
- */
-export function queryRefs(registry, { category, query, limit }) {
-    const categories = orderedCategories(registry);
-    // An unknown category is a caller mistake, not a failure — name the valid set
-    // rather than returning an empty listing with no explanation.
-    if (category && !categories.includes(category)) {
-        return {
-            available: true,
-            total: 0,
-            returned: 0,
-            hasMore: false,
-            categories,
-            content: `Unknown category "${category}". Valid categories: ${categories.join(", ")}.`,
-        };
-    }
-    const needle = query?.trim().toLowerCase();
-    const selected = category ? [category] : categories;
-    // Filter first so `total` is the honest count of matching files, then bound.
-    const matched = selected.map((c) => {
-        const entries = registry.files.by_category[c] ?? [];
-        return { category: c, entries: needle ? entries.filter((e) => matchesQuery(c, e, needle)) : entries };
-    });
-    const total = matched.reduce((n, m) => n + m.entries.length, 0);
-    if (total === 0) {
-        const scope = [category ? `category: ${category}` : null, query ? `query: "${query}"` : null].filter(Boolean).join(", ");
-        return {
-            available: true,
-            total: 0,
-            returned: 0,
-            hasMore: false,
-            categories,
-            content: `No reference files match this filter${scope ? ` (${scope})` : ""}. Nothing can be downloaded at runtime.`,
-        };
-    }
-    const cap = limit ?? DEFAULT_LIMIT;
-    const lines = [];
-    let returned = 0;
-    for (const { category: c, entries } of matched) {
-        if (entries.length === 0)
-            continue;
-        lines.push(`\n## ${CATEGORY_LABELS[c] ?? c}`);
-        const room = cap - returned;
-        if (room <= 0) {
-            lines.push(`  ${entries.length} file(s) — not shown (limit reached). Call with category: "${c}" to list them.`);
-            continue;
-        }
-        const shown = entries.slice(0, room);
-        returned += shown.length;
-        for (const entry of shown)
-            lines.push(renderEntry(entry));
-        const hidden = entries.length - shown.length;
-        if (hidden > 0)
-            lines.push(`  … and ${hidden} more in this category — call with category: "${c}" (and a higher \`limit\`) to list them.`);
-    }
-    const header = [
-        `Reference Store (build ${registry.build_id}, ${registry.generated_at})`,
-        `Mount: ${REFS_ROOT}`,
-        `Matching files: ${total} (showing ${returned})`,
-    ].join("\n");
-    return {
-        available: true,
-        total,
-        returned,
-        hasMore: returned < total,
-        categories,
-        content: header + "\n" + lines.join("\n"),
-    };
-}