@edgehero/pi-dispatch 0.1.2 → 0.2.0

package/src/materialize.mjs

@@ -11,46 +11,141 @@ const exec = promisify(execFile);
  * read-only `/job/pi/` directory the container mounts.
  *
  * This is security-critical: the content becomes the agent's SYSTEM PROMPT, and the repo is only
- * trusted at maintainer level. Three properties, each PROVEN with a hostile fixture in the tests:
+ * trusted at maintainer level. Four properties, the first three PROVEN with a hostile fixture:
  *
  *   1. NO SYMLINK FOLLOWING. A `.pi/APPEND_SYSTEM.md` symlinked to the worker's `.env` or
  *      `/etc/passwd` must never pull a host file into the prompt. We enumerate with `git ls-tree`
  *      and REJECT any entry that is not a regular blob (mode 100644): symlinks are 120000,
  *      submodules 160000. We never touch the working tree, so there is no link to follow.
- *   2. NO PATH TRAVERSAL. Every output path is re-derived from the git tree path and asserted to
- *      stay under the destination root; a crafted entry cannot escape `/job/pi/`.
+ *   2. NO PATH TRAVERSAL. Every output path is rebuilt from SEGMENTS THAT EACH MATCHED AN ANCHORED,
+ *      SEPARATOR-FREE CHARSET -- never from git's own path string. See classifyPiPath.
  *   3. NO EXECUTION. `git cat-file blob <oid>` dumps raw bytes by object id -- no working-tree
  *      checkout, no smudge/clean filters, no hooks, no diff drivers. Nothing in the repo runs.
+ *   4. BOUNDED. A repo skill is now materialised WHOLE (issue #60), so the file count and byte total
+ *      are repo-controlled where they used to be one file per skill. Every cap is decided from the
+ *      single `git ls-tree -r -l -z` listing, BEFORE the first `cat-file` and BEFORE the first
+ *      write -- CONST-BUDGET-BEFORE-TOKENS' ordering, one layer down. A breach REFUSES the job.
  *
  * The SHA is an input, resolved by the caller from a fresh default-branch API call -- NEVER a
  * webhook field, and NEVER the triggering (possibly fork) branch.
+ *
+ * WHY NOT `git cat-file --batch` (one process instead of N): it does preserve the by-oid,
+ * no-working-tree property, so that is not the objection. `promisify(execFile)` cannot drive it --
+ * it needs `spawn` plus a hand-rolled, binary-safe parser for the `<oid> <type> <size>\n<bytes>\n`
+ * framing, with partial-chunk handling and missing-object lines, which is ~60 lines of new bug
+ * surface at the one place in this codebase whose whole argument is "simple enough to prove". It
+ * would also replace a PER-FILE buffer bound with a shared one, undoing what makes maxFileBytes
+ * able to keep execFile's maxBuffer unreachable. The caps make the spawn cost a non-issue: 256
+ * files is a second or two, on a path that just did a network clone. Revisit if maxFiles ever rises
+ * past ~1000, or a profiler shows prepare dominated by spawns.
+ * WHY NOT `git archive | tar -x`: it would reconstruct output paths from the ARCHIVE's own strings,
+ * which is precisely the property property 2 exists to refuse.
  */
 
 const PI_DIR = ".pi";
 const APPEND_SYSTEM = `${PI_DIR}/APPEND_SYSTEM.md`;
-// A skill directory name: lowercase kebab/underscore, 1-64 chars, no dots (so no "..") and no
-// slashes. This is what makes a traversal name impossible at the source. Matched against the
-// CAPTURED segment only, never the whole path. The name charset itself is imported from
-// flow-gate.mjs (the exported single source of truth, issue #92); the path form stays local
-// because only the materialiser walks whole tree paths.
-const SKILL_PATH_RE = /^\.pi\/skills\/([a-z0-9](?:[a-z0-9_-]{0,62}[a-z0-9])?)\/SKILL\.md$/;
+const SKILLS_PREFIX = `${PI_DIR}/skills/`;
+const SKILL_FILE = "SKILL.md";
+
+/**
+ * The bounds on what one repo may put into one job's `/job/pi`. Frozen and exported so the tests
+ * name the same numbers the code enforces rather than restating them.
+ *
+ * maxFileBytes does double duty: at 1 MiB it makes the 16 MiB execFile maxBuffer on the `cat-file`
+ * path UNREACHABLE, which converts an opaque RangeError (retried as infrastructure, forever, on a
+ * repo that will overrun it every time) into a named policy refusal.
+ */
+export const PI_LIMITS = Object.freeze({
+	maxFiles: 256, // ~50 typical skills; also bounds cat-file spawns per job
+	maxFilesPerSkill: 64, // SKILL.md + references/ + scripts/ + assets/ runs to a handful
+	maxFileBytes: 1 << 20, // 1 MiB: ~250k words of markdown. Larger is a dataset, not a skill.
+	maxTotalBytes: 8 << 20, // 8 MiB: trivial beside the clone, and it bounds host disk under retention
+	maxTailSegments: 4, // references/x.md is 2, scripts/lib/util.sh is 3
+	maxOutRelChars: 200, // Windows MAX_PATH is 260 and jobDir is an OS temp path (~40-70)
+});
+
+/**
+ * The bound on the LISTING itself, and it is separate from PI_LIMITS on purpose: the caps above are
+ * computed FROM this buffer, so they cannot bound it. ~4000 records fit.
+ */
+const LS_TREE_MAX_BYTES = 1 << 20;
+
+/**
+ * ONE path segment BELOW the skill name: 1-64 chars, starts AND ends alphanumeric, with `.`, `_`
+ * and `-` allowed inside.
+ *
+ * The leading-alnum rule IS the traversal guard, and it needs no separate `!== ".."` test: `.` and
+ * `..` both LEAD with a dot, so neither can match. The trailing-alnum rule bars a trailing dot,
+ * which Windows silently strips (`foo.` opens `foo`) -- and this worker is cross-platform.
+ *
+ * Case-INSENSITIVE, unlike SKILL_NAME_RE, because `SKILL.md` and `README.md` are the whole point of
+ * issue #60. DELIBERATELY NO `u` FLAG: `/[a-z]/iu` additionally matches U+212A KELVIN SIGN and
+ * U+017F LATIN SMALL LETTER LONG S, so adding one would silently widen a security charset by two
+ * invisible codepoints. Verified by running it, not assumed.
+ *
+ * Same shape as import-pi.mjs's ENTRY_NAME_RE by design, and deliberately NOT imported from it:
+ * that module pulls in execFile and the npm package stager, and this one is on the security-critical
+ * read path. The duplication is pinned by a drift test in materialize.test.mjs.
+ */
+export const PI_SEGMENT_RE = /^[a-z0-9](?:[a-z0-9_.-]{0,62}[a-z0-9])?$/i;
+
+/**
+ * Windows reserved device names. `CON`, `NUL` and friends pass the charset above, and on Windows
+ * `writeFileSync(".../CON")` writes to a DEVICE: no file appears, no error is raised, and the skill
+ * silently arrives incomplete -- the exact silent-drop class issue #60 exists to remove. Matched on
+ * the part before the first dot, case-insensitively, because `con.md` is the device too.
+ */
+const WINDOWS_DEVICE_NAMES = new Set([
+	"con", "prn", "aux", "nul",
+	"com1", "com2", "com3", "com4", "com5", "com6", "com7", "com8", "com9",
+	"lpt1", "lpt2", "lpt3", "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9",
+]);
+
+function isWindowsDeviceName(segment) {
+	const stem = segment.split(".")[0].toLowerCase();
+	return WINDOWS_DEVICE_NAMES.has(stem);
+}
 
 /**
  * Classify a git tree path into the destination we will WRITE, or null to reject.
  *
- * Critically, the returned `outRel` is built from a FIXED TEMPLATE using the validated skill name,
- * never from the raw git path. gitshow-research proved `git ls-tree` can emit path strings
- * containing literal `../` segments (git does not sanitise tree-entry names), so deriving the
- * output path from git's string is unsafe even behind a containment check. The name is the only
- * attacker-influenced input, and it is validated to a charset that cannot express traversal.
+ * The output path is REBUILT from validated pieces, never taken from git's string. gitshow-research
+ * proved `git ls-tree` can emit path strings containing literal `../` segments (git does not
+ * sanitise tree-entry names), so deriving the output from git's string is unsafe even behind a
+ * containment check.
+ *
+ * Splitting on "/" first is what makes the rebuild safe, and it is stronger than the single regex it
+ * replaced: any OTHER separator (a backslash, a NUL, a CR) survives INSIDE a piece and is then
+ * refused by the anchored charset, and a literal `..` piece is refused by the leading-alnum rule.
+ * Only the validated pieces reach the join.
+ *
+ * Returns `{ outRel, skill }`; `skill` is null for the persona, which is a fixed path with no name
+ * and no subtree and so stays an EXACT match. Widening `.pi/*` would admit `settings.json`, and
+ * keeping a serviced repo's settings out of the runner is a constitutional property
+ * (INT-SDK-SESSION-OPTIONS: the runner uses SettingsManager.inMemory deliberately).
  */
 export function classifyPiPath(path) {
-	if (path === APPEND_SYSTEM) return { outRel: "pi/APPEND_SYSTEM.md" };
-	const m = SKILL_PATH_RE.exec(path);
-	if (!m) return null;
-	const name = m[1];
-	if (!SKILL_NAME_RE.test(name)) return null; // redundant with the capture group; belt and braces
-	return { outRel: `pi/skills/${name}/SKILL.md` };
+	if (path === APPEND_SYSTEM) return { outRel: "pi/APPEND_SYSTEM.md", skill: null };
+	if (!path.startsWith(SKILLS_PREFIX)) return null;
+
+	const segments = path.slice(SKILLS_PREFIX.length).split("/");
+	// Fewer than 2 means a blob AT `.pi/skills/<name>` rather than inside a skill directory.
+	if (segments.length < 2 || segments.length > 1 + PI_LIMITS.maxTailSegments) return null;
+
+	const [name, ...tail] = segments;
+	// The skill DIRECTORY name stays lowercase-only: SKILL_NAME_RE is the single source of truth
+	// (issue #92), shared with flow-gate.mjs, the outbox chain gate and the admin setup wizard. A
+	// name this module accepted but the gate rejected would be a skill that materialises and can
+	// never be chained.
+	if (!SKILL_NAME_RE.test(name)) return null;
+	for (const seg of tail) {
+		if (!PI_SEGMENT_RE.test(seg)) return null;
+		if (isWindowsDeviceName(seg)) return null;
+	}
+
+	const outRel = ["pi", "skills", name, ...tail].join("/");
+	if (outRel.length > PI_LIMITS.maxOutRelChars) return null;
+	return { outRel, skill: name };
 }
 
 /** Back-compat predicate used by callers/tests that only care whether a path is accepted. */
@@ -59,26 +154,123 @@ export function isAllowedPiPath(path) {
 }
 
 /**
- * Parse `git ls-tree -r -z` output into entries, keeping ONLY regular blobs (100644) at allowed
- * paths, each carrying its template-derived output path. Symlinks (120000), submodules (160000),
- * executables (100755), and anything outside the allowlist are dropped here -- the single choke
- * point for the reject-by-mode rule.
+ * Parse `git ls-tree -r -l -z` output into entries, keeping ONLY regular blobs (100644) at allowed
+ * paths, each carrying its rebuilt output path and its blob size. Symlinks (120000), submodules
+ * (160000), executables (100755), and anything outside the allowlist are dropped here -- the single
+ * choke point for the reject-by-mode rule.
+ *
+ * `100755` STAYS REJECTED, and it is a decision rather than an oversight. `/job` is mounted `:ro`
+ * and every file lands `0444`, so accepting the mode and writing `0444` anyway would accept what the
+ * repo asked for and silently strip it -- a no-op wearing a fix's clothes. Writing `0555` instead
+ * would grant "this path is execve-able" to repo bytes the worker placed there, which is not the
+ * worker's job. So a skill's scripts arrive non-executable and are invoked as `bash script.sh`; the
+ * failure is LOUD (Permission denied) and the agent recovers from it, which is the opposite of the
+ * silent class issue #60 is about. Keeping this gate byte-identical also keeps flow-gate.mjs's
+ * mirrored check honest and DES-AI-TRIGGER-FLOW-GATE's citation of it true.
+ *
+ * Returns `{ entries, skipped }`. `skipped` counts every enumerated record we did not keep.
  */
 export function selectEntries(lsTreeZ) {
 	const entries = [];
+	let skipped = 0;
 	for (const record of lsTreeZ.split("\0")) {
 		if (!record) continue;
-		// "<mode> <type> <oid>\t<path>"
+		// "<mode> <type> <oid> <size>\t<path>" -- the size column is space-padded, right-justified to
+		// a minimum width of 7 (git-ls-tree(1)), which the existing \s+ split already collapses.
 		const tab = record.indexOf("\t");
 		if (tab === -1) continue;
-		const [mode, type, oid] = record.slice(0, tab).split(/\s+/);
+		const [mode, type, oid, sizeField] = record.slice(0, tab).split(/\s+/);
 		const path = record.slice(tab + 1);
-		if (mode !== "100644" || type !== "blob") continue; // rejects symlink/submodule/exec
+		if (mode !== "100644" || type !== "blob") {
+			skipped++;
+			continue; // rejects symlink/submodule/exec
+		}
+		// Read the size AFTER the mode gate, never before: `-l` prints "-" for a tree or a commit, and
+		// those are exactly what the gate above has already dropped.
+		//
+		// A size we cannot read THROWS rather than defaulting to 0 or skipping. If `-l` were ever
+		// dropped from the argv, every sizeField would be `undefined`; skipping would then materialise
+		// NOTHING and report success, and defaulting to 0 would leave every byte cap unenforced. Both
+		// are silent. The message carries mode/type/oid and never the attacker-chosen path.
+		const size = Number(sizeField);
+		if (!Number.isSafeInteger(size) || size < 0) {
+			throw new Error(`git ls-tree: unreadable object size (is -l still in the argv?) for ${type} ${mode} ${oid}`);
+		}
 		const classified = classifyPiPath(path);
-		if (!classified) continue;
-		entries.push({ oid, path, outRel: classified.outRel });
+		if (!classified) {
+			skipped++;
+			continue;
+		}
+		entries.push({ oid, path, size, outRel: classified.outRel, skill: classified.skill });
+	}
+	return { entries, skipped };
+}
+
+/**
+ * Drop every entry under a `.pi/skills/<name>/` that declares no SKILL.md ANYWHERE beneath it.
+ *
+ * Verified against the pinned pi: loadSkillsFromDirInternal registers a skill only where a literal
+ * `SKILL.md` exists, and loose `.md` files load only at the skills ROOT (`includeRootFiles`). So a
+ * `.pi/skills/notes/` holding only `a.md` contributes nothing an agent can ever reference -- and
+ * materialising it would hand anyone with merge access a data-dump channel into the job container
+ * that never has to look like a skill.
+ *
+ * The test is "SKILL.md anywhere under <name>/", NOT "at <name>/SKILL.md", because pi keeps
+ * RECURSING while a directory has no SKILL.md: `.pi/skills/group/sub/SKILL.md` is a genuine,
+ * loadable skill. A root-only rule would re-create this issue's own bug one level down.
+ */
+export function keepOnlyDeclaredSkills(entries) {
+	const declared = new Set();
+	for (const e of entries) {
+		if (e.skill !== null && e.outRel.endsWith(`/${SKILL_FILE}`)) declared.add(e.skill);
+	}
+	const kept = entries.filter((e) => e.skill === null || declared.has(e.skill));
+	return { kept, skipped: entries.length - kept.length };
+}
+
+/**
+ * Decide every cap from the listing alone. Returns a reason string, or null to proceed.
+ *
+ * Reason tokens carry a `pi-` prefix on purpose: INT-RUN-HISTORY-FILE-CONTRACT's nested
+ * `session.reason` enum already contains a bare `too-large`, and two enums in one record sharing a
+ * token is how a reader misattributes a refusal.
+ */
+export function checkLimits(entries) {
+	if (entries.length > PI_LIMITS.maxFiles) return "pi-too-many-files";
+
+	const perSkill = new Map();
+	let total = 0;
+	for (const e of entries) {
+		if (e.size > PI_LIMITS.maxFileBytes) return "pi-file-too-large";
+		total += e.size;
+		if (total > PI_LIMITS.maxTotalBytes) return "pi-too-large";
+		if (e.skill !== null) {
+			const n = (perSkill.get(e.skill) ?? 0) + 1;
+			if (n > PI_LIMITS.maxFilesPerSkill) return "pi-too-many-files";
+			perSkill.set(e.skill, n);
+		}
+	}
+
+	// A case-insensitive host (APFS, NTFS) collapses `references/README.md` and
+	// `references/readme.md` -- two distinct git blobs -- onto ONE file. The second write would open
+	// an existing 0444 file O_WRONLY and EACCES, surfacing as an opaque throw that a retry cannot
+	// fix. Unreachable before issue #60 (one file per skill, and the name charset was lowercase-only)
+	// and reachable now, so it is refused here rather than discovered there. The same collapse
+	// applies between a FILE and a DIRECTORY prefix (`foo/Bar` beside `foo/bar/x.md`), which is why
+	// the ancestor prefixes are collected too.
+	const files = new Set();
+	const dirs = new Set();
+	for (const e of entries) {
+		const lower = e.outRel.toLowerCase();
+		if (files.has(lower)) return "pi-path-collision";
+		files.add(lower);
+		const parts = lower.split("/");
+		for (let i = 1; i < parts.length; i++) dirs.add(parts.slice(0, i).join("/"));
 	}
-	return entries;
+	for (const d of dirs) {
+		if (files.has(d)) return "pi-path-collision";
+	}
+	return null;
 }
 
 /**
@@ -97,30 +289,60 @@ function safeJoin(root, ...segments) {
 
 /**
  * Materialise `.pi/` at `sha` from the clone at `gitDir` into `destDir` (which becomes /job/pi).
- * Returns the list of relative paths written (under `pi/`), for logging.
+ *
+ * Returns `{ written, skipped }` -- `written` being the relative paths written, under `pi/` -- or
+ * `{ outcome: "policy", reason }` when a cap is breached. The refusal is a determinate POLICY result
+ * and therefore RETURNS rather than throws (CONST-RETRY-INFRA-ONLY): the same tree at the same sha
+ * breaches the same cap on every retry, so throwing would clone twice and land the job in the failed
+ * set with a message nobody maps to "prune your skill directory". Truncating instead was rejected
+ * outright -- a truncated skill IS the bug this change fixes, and which files survived would be
+ * decided by git's tree order, which no operator can predict.
+ *
+ * NOTHING IS WRITTEN before every cap has passed, so a refused job leaves no partial /job/pi behind.
  *
  * `git` is injected for tests; defaults to a thin wrapper over the real binary.
  */
 export async function materializePiDir({ gitDir, sha, destDir, git = defaultGit }) {
-	const lsTreeZ = await git(gitDir, ["ls-tree", "-r", "-z", sha, `${PI_DIR}/`]);
-	const entries = selectEntries(lsTreeZ);
+	let lsTreeZ;
+	try {
+		// `-l` adds the blob size, which is what lets every cap be decided here rather than after N
+		// reads. maxBuffer is scoped down from the default 16 MiB: this is a LISTING, and a `.pi/`
+		// whose listing alone overruns a megabyte is already past maxFiles by two orders of magnitude.
+		lsTreeZ = await git(gitDir, ["ls-tree", "-r", "-l", "-z", sha, `${PI_DIR}/`], { maxBuffer: LS_TREE_MAX_BYTES });
+	} catch (error) {
+		// A listing too large for the buffer is DETERMINATE -- the same tree overruns it every time --
+		// so it becomes the same policy refusal a counted breach gets, rather than an opaque RangeError
+		// retried as infrastructure. Narrow on purpose: every other git failure (a bad sha, a missing
+		// binary, an unreadable object store) still throws and is still retried.
+		if (error?.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER") {
+			return { outcome: "policy", reason: "pi-too-many-files" };
+		}
+		throw error;
+	}
+
+	const selected = selectEntries(lsTreeZ);
+	const declared = keepOnlyDeclaredSkills(selected.entries);
+	const skipped = selected.skipped + declared.skipped;
+
+	const breach = checkLimits(declared.kept);
+	if (breach) return { outcome: "policy", reason: breach };
 
 	const written = [];
-	for (const { oid, outRel } of entries) {
+	for (const { oid, outRel } of declared.kept) {
 		const content = await git(gitDir, ["cat-file", "blob", oid], { raw: true });
-		// outRel is TEMPLATE-derived from a validated name, never the raw git path -- so it cannot
-		// contain traversal. safeJoin re-checks containment as defence in depth, and splits the posix
-		// outRel into host path segments so it is correct on Windows too.
+		// outRel is REBUILT from validated segments, never the raw git path -- so it cannot contain
+		// traversal. safeJoin re-checks containment as defence in depth, and splits the posix outRel
+		// into host path segments so it is correct on Windows too.
 		const out = safeJoin(destDir, ...outRel.split("/"));
 		mkdirSync(dirname(out), { recursive: true });
 		writeFileSync(out, content, { mode: 0o444 });
 		// Report posix-style: this names a CONTAINER path (/job/pi/...), stable across host OSes.
 		written.push(outRel);
 	}
-	return written;
+	return { written, skipped };
 }
 
-async function defaultGit(gitDir, args, { raw = false } = {}) {
+async function defaultGit(gitDir, args, { raw = false, maxBuffer = 16 * 1024 * 1024 } = {}) {
 	// -c protecting against a hostile repo config: no hooks, no external filters, no pager.
 	const hardened = [
 		"-c",
@@ -134,7 +356,7 @@ async function defaultGit(gitDir, args, { raw = false } = {}) {
 	];
 	const { stdout } = await exec("git", hardened, {
 		encoding: raw ? "buffer" : "utf8",
-		maxBuffer: 16 * 1024 * 1024,
+		maxBuffer,
 	});
 	return stdout;
 }

package/src/outbox.mjs

@@ -157,6 +157,11 @@ export function makeCollectChain({ queue, enqueue = enqueueLocalJob, readFlowGat
 					// (INT-OUTBOX-CONTRACT's explicit-property-reads rule). Undefined stays undefined, so a parent with
 					// no image chains a child whose data is byte-identical to today's.
 					image: job.data?.image,
+					// The parent's injected skills follow the child, off validated JOB DATA and never off the
+					// request file (REQ-PER-TRIGGER-SKILLS). Same reason `image` does: a chained child runs the
+					// same operator's flows and, without them, would look up a skill that is not there, write a
+					// plausible report and exit 0. It is NOT part of chainedJobId, for the reason stated below.
+					skillsDir: job.data?.skillsDir,
 					chainDepth: childDepth,
 					parentJobId: job.id,
 					// chainedJobId deliberately does NOT take the image: the child's identity is (parent, flow, task).

package/src/packages.mjs

@@ -14,9 +14,12 @@
  * minor into every queued job becoming a no-op with NO signal -- the queue still reports success, the
  * worst failure class available. Pinning converts that into an operator-visible edit of a version string.
  *
- * Two directions, two error policies:
+ * Three directions, three error policies:
  *   - `parsePackagesFile` reads the OPERATOR's `pi-packages.json` before anything is staged. Pure and
  *     fs-free (mirrors triggers.mjs), fail-loud `configError` naming the offending package.
+ *   - `mergeHostPackages` folds in what host-pi.mjs discovered in the operator's own pi setup (issue #102).
+ *     Declared entries win; a discovered one that fails validation is DROPPED with a reason rather than
+ *     taking the declared set down with it.
  *   - `readStageManifest` reads the file the STAGER wrote, at job-wiring time. It NEVER throws: a corrupt
  *     manifest must degrade to "no staged packages", not crash the worker mid-queue.
  */
@@ -45,6 +48,23 @@ export const EXACT_VERSION_RE = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za
 /** An npm package name: lowercase, optionally `@scope/`-prefixed. */
 export const NPM_NAME_RE = /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/;
 
+/**
+ * The pi resource kinds a package may contribute by convention dir, when it carries no `pi` manifest.
+ * Lives here rather than in import-pi.mjs because host-pi.mjs needs the same list to decide whether a
+ * package the operator installed is a pi package at all, and a third module importing from import-pi.mjs
+ * would add an edge to the import-pi <-> packages cycle that only survives because both sides use their
+ * bindings at call time.
+ *
+ * This list is pi's, not ours: at the 0.80.7 pin `collectPackageResources` falls through to exactly these
+ * four directory names when `readPiManifest` returns null, so a package with no `pi` key and a `skills/`
+ * dir IS a pi package. See host-pi.mjs's PINNED_PI_NEEDLES for the assertion that keeps that true.
+ */
+export const RESOURCE_DIRS = ["extensions", "skills", "prompts", "themes"];
+
+/** Where a staged entry came from: the operator's file, or discovery of their pi setup. */
+export const FROM_DECLARED = "pi-packages";
+export const FROM_HOST = "host";
+
 /** npm's hard limit on a package name; a longer one could never have been published. */
 const MAX_NAME_LENGTH = 214;
 /** The staged dir name doubles as a path segment in the container, so it stays short and flat. */
@@ -145,6 +165,74 @@ function normalizePackage(entry, index, path, seenDirs) {
 	return { name, version, dir };
 }
 
+/** The `path` a discovered candidate reports as its origin, and the suffix stripped off its error text. */
+const DISCOVERED_AT = "your pi setup";
+
+/**
+ * Normalize ONE candidate discovered in the host's pi setup, returning `{ entry }` or `{ reason }` instead
+ * of throwing (issue #102).
+ *
+ * Deliberately a try/catch around the SAME `normalizePackage` the declared path uses. Every rule that
+ * guards a declared entry -- the admin block, the exact-version rule, the name charset, the dir length and
+ * the dir-collision check -- therefore reaches discovery by reuse rather than by a second implementation.
+ * A parallel implementation would eventually drift, and the rule it would drift away from is the one that
+ * keeps the admin package (which can enqueue paid jobs) out of every job container.
+ *
+ * Why a reason and not a throw: a declared pin is a promise the operator made, so failing it refuses the
+ * whole stage. A discovered package is an inference WE made, and taking the operator's declared set down
+ * because of it would be the wrong trade. A named skip is not a silent one.
+ */
+export function normalizeDiscoveredPackage(candidate, seenDirs = new Map()) {
+	try {
+		return { entry: normalizePackage(candidate, 0, DISCOVERED_AT, seenDirs) };
+	} catch (error) {
+		const suffix = `: ${DISCOVERED_AT}`;
+		const message = String(error?.message ?? error);
+		return { reason: message.endsWith(suffix) ? message.slice(0, -suffix.length) : message };
+	}
+}
+
+/**
+ * Merge the packages the operator DECLARED in `pi-packages.json` with the ones discovered in their pi setup.
+ * Returns `{ entries, overrides, dropped }` (issue #102).
+ *
+ * A declared entry WINS, matched by name, and keeps its position: that is what lets an operator pin an
+ * older version than their host happens to run, which is the whole reason `pi-packages.json` survives as a
+ * layer rather than being replaced by discovery. When the two disagree on the version, the shadowed host
+ * version is reported in `overrides` so the printed list can say so out loud -- an operator who pinned 2.0.1
+ * while running 2.3.0 should learn it here, not from a flow behaving differently in a job than it does
+ * interactively.
+ *
+ * Collisions are resolved the same direction: `seenDirs` is seeded with the declared dirs BEFORE any
+ * candidate is normalized, so `normalizePackage`'s own duplicate-dir refusal fires on the discovered side
+ * and the declared entry survives untouched.
+ */
+export function mergeHostPackages(declared = [], discovered = []) {
+	const entries = declared.map((entry) => ({ ...entry, from: FROM_DECLARED }));
+	const seenDirs = new Map(entries.map((entry) => [entry.dir, entry.name]));
+	const declaredByName = new Map(entries.map((entry) => [entry.name, entry]));
+	const overrides = [];
+	const dropped = [];
+
+	for (const candidate of discovered) {
+		const shadowing = declaredByName.get(candidate?.name);
+		if (shadowing) {
+			if (shadowing.version !== candidate.version) {
+				overrides.push({ name: shadowing.name, declared: shadowing.version, host: candidate.version });
+			}
+			continue;
+		}
+		const result = normalizeDiscoveredPackage(candidate, seenDirs);
+		if (result.reason) {
+			dropped.push({ name: candidate?.name ?? "(unnamed)", reason: result.reason });
+			continue;
+		}
+		entries.push({ ...result.entry, from: FROM_HOST });
+	}
+
+	return { entries, overrides, dropped };
+}
+
 /**
  * Read the stager's manifest from `<globalPiDir>/packages/packages.json`. Returns `{ stagedAt, packages }`
  * or `null` when there is nothing usable there. NEVER throws -- this runs on the job path, where a corrupt
@@ -153,6 +241,11 @@ function normalizePackage(entry, index, path, seenDirs) {
  * Entries are re-validated on the way in (the file is a host artifact that an operator may have hand-edited
  * between stage time and job time): a `dir` that is not a plain segment would otherwise flow straight into
  * a container path.
+ *
+ * `from` is a CLOSED enum with a default, never a pass-through (issue #102): anything that is not exactly
+ * "host" reads as "pi-packages", so a hand-edited receipt cannot inject a string that reaches a printed
+ * doctor line. A receipt written before #102 has no `from` at all and correctly reads as declared. Every
+ * other unknown key is still dropped, which is what keeps an older worker safe against a newer receipt.
  */
 export function readStageManifest({ globalPiDir, readFile = readFileSync, fileExists = existsSync } = {}) {
 	if (!globalPiDir) return null;
@@ -168,7 +261,7 @@ export function readStageManifest({ globalPiDir, readFile = readFileSync, fileEx
 			const { name, version, dir } = entry;
 			if (!isNonEmptyString(name) || !isNonEmptyString(version) || !isNonEmptyString(dir)) return null;
 			if (!ENTRY_NAME_RE.test(dir) || dir.length > MAX_DIR_LENGTH) return null;
-			packages.push({ name, version, dir });
+			packages.push({ name, version, dir, from: entry.from === FROM_HOST ? FROM_HOST : FROM_DECLARED });
 		}
 		return { stagedAt: typeof parsed.stagedAt === "string" ? parsed.stagedAt : null, packages };
 	} catch {

package/src/prepare-github.mjs

@@ -97,10 +97,13 @@ function isShaGone(error) {
  *   - `git(cwd, args, { env })`      transport for a single git invocation; default runs the real
  *                                     binary via execFile (no shell). Args arrive already hardened.
  *   - `resolveDefaultBranchSha(repo, token) => { sha }`  fresh default-branch tip (github-host).
- *   - `materialize({ gitDir, sha, destDir }) => string[]`  the .pi/ materialiser (git cat-file).
+ *   - `materialize({ gitDir, sha, destDir })`  the .pi/ materialiser (git cat-file). Returns
+ *                                     `{ written, skipped }`, or `{ outcome: "policy", reason }`
+ *                                     when the repo's .pi/ breaches a size cap (issue #60).
  *   - `writeFile` / `mkdir`          fs writes for the job inputs (default sync fs).
  *
- * On a determinate gone-SHA returns `{ outcome: "policy", reason: "sha-gone" }`. On success returns
+ * On a determinate gone-SHA returns `{ outcome: "policy", reason: "sha-gone" }`, and likewise on a
+ * materialiser cap breach (`pi-too-large` and friends). On success returns
  * `{ workspace, jobDir, sha, materialised }`.
  */
 export async function prepareGithubWorkspace(
@@ -165,7 +168,12 @@ export async function prepareGithubWorkspace(
 		}
 
 		// .pi/ is read by object id from the pinned SHA — symlink/submodule/exec safe, no working tree.
-		const materialised = await materialize({ gitDir: workspace, sha, destDir: jobDir });
+		const pi = await materialize({ gitDir: workspace, sha, destDir: jobDir });
+		// A repo whose .pi/ breaches a materialiser cap (issue #60) is a determinate refusal, returned
+		// BEFORE the session resolve and before prompt.md/event.json are written, so a refused job leaves
+		// no /job inputs behind at all. The processor's existing policy branch reserves no budget for it.
+		if (pi?.outcome === "policy") return pi;
+		const materialised = pi.written;
 
 		// Which transcript, if any, this job continues. The head ref for a pull/merge-request target is
 		// resolved from the FORGE, not the payload -- an issue_comment on a PR carries no head at all, and
@@ -188,7 +196,9 @@ export async function prepareGithubWorkspace(
 			// to interpolate, and they are what makes this job's branch differ from its sibling's. This is the
 			// SHARED forge preparer, so the gitlab/forgejo/azure builders receive the two keys and destructure
 			// them away -- harmless, and always undefined while replicas are github-only.
-			buildPrompt({ flow: job.flow, target: job.target, comment: job.trigger?.comment, resumed: session?.resume === true, replica: job.replica, replicas: job.replicas }),
+			// `review` rides beside `comment` and, like `replica`/`replicas`, is destructured away by the
+			// gitlab/forgejo/azure builders -- harmless, and always undefined while reviews are github-only.
+			buildPrompt({ flow: job.flow, target: job.target, comment: job.trigger?.comment, resumed: session?.resume === true, replica: job.replica, replicas: job.replicas, review: job.trigger?.review, instructions: job.instructions }),
 			{ mode: 0o444 },
 		);
 
@@ -207,6 +217,15 @@ export async function prepareGithubWorkspace(
 			repository: { full_name: job.repo },
 			...(job.target?.type === "pull_request" ? { pull_request: prEventBody(job.target) } : { issue: { number: job.target?.number, title: job.target?.title, body: job.target?.body } }),
 			...(job.trigger?.comment ? { comment: { body: job.trigger.comment.body, author_association: job.trigger.comment.author_association } } : {}),
+			// The invoking review, review-triggered jobs only (issue #66). Written as an explicit literal
+			// rather than a spread, the same discipline as `comment` above: this object is the subset's
+			// named fields, not whatever happened to reach the queue. Conditional for the same reason too --
+			// a PR job without a review must stay byte-identical to the one this file wrote before #66.
+			//
+			// `action` above is `submitted` (GitHub's word) while `matched.action` below is
+			// `review_submitted` (the triggers.json word). They differ on purpose; INT-CONTAINER-JOB-INPUTS
+			// says which is which.
+			...(job.trigger?.review ? { review: { id: job.trigger.review.id, body: job.trigger.review.body, state: job.trigger.review.state, author_association: job.trigger.review.author_association } } : {}),
 			sender: { id: job.trigger?.sender?.id },
 			...(job.trigger?.matched ? { matched: job.trigger.matched } : {}),
 		};

package/src/prepare-local.mjs

@@ -44,7 +44,12 @@ export async function prepareLocalWorkspace({ folder, task, jobDir, git = defaul
 	const outboxDir = join(jobDir, "outbox");
 	mkdirSync(outboxDir, { recursive: true });
 	// Instructions from HEAD, via the symlink-safe git materialiser, into /job/pi (mounted :ro).
-	const written = await materializePiDir({ gitDir: folder, sha, destDir: jobDir });
+	const pi = await materializePiDir({ gitDir: folder, sha, destDir: jobDir });
+	// A .pi/ over a materialiser cap (issue #60) refuses the job determinately, before prompt.md and
+	// event.json exist. Returned rather than thrown: the same tree breaches the same cap on every
+	// retry (CONST-RETRY-INFRA-ONLY), and the processor's policy branch spends nothing on it.
+	if (pi?.outcome === "policy") return pi;
+	const written = pi.written;
 
 	// The task the operator asked for. Plain data below the instructions.
 	writeFileSync(join(jobDir, "prompt.md"), String(task ?? ""), { mode: 0o444 });