failproofai 1.0.7 → 1.0.8

package/src/hooks/custom-hooks-loader.ts

@@ -17,7 +17,7 @@ import { readFile } from "node:fs/promises";
 import { pathToFileURL } from "node:url";
 import { createHash } from "node:crypto";
 import { hookLogWarn, hookLogError, hookLogInfo } from "./hook-logger";
-import { customPolicies, getCustomHooks, clearCustomHooks } from "./custom-hooks-registry";
+import { customPolicies, getCustomHooks, getSemanticRegistrations, clearCustomHooks } from "./custom-hooks-registry";
 import {
   findDistIndex,
   rewriteFileTree,
@@ -33,6 +33,8 @@ import type { CustomHook, PolicyCatalogEntry } from "./policy-types";
 import type { CloudManagedPolicyArtifact } from "./cloud-managed-policies";
 import type { ResolvedPack } from "./pack-manifest";
 import { customPoliciesDir, shimsDir } from "./fp-home";
+import { effectiveReviewerNames } from "./effective-reviewers";
+import { refusedAuthorityWarning, warnAuthority, withMergedAuthority } from "./policy-authority";
 
 const LOADING_KEY = "__FAILPROOFAI_LOADING_HOOKS__";
 
@@ -178,6 +180,8 @@ export interface PolicyLoadFailure {
   reason: string;
 }
 
+const warnedSemanticOutsidePack = new Set<string>();
+
 async function loadSingleFile(
   absPath: string,
   opts?: {
@@ -185,6 +189,12 @@ async function loadSingleFile(
     conventionScope?: "project" | "user";
     /** Cloud-managed policies pass their pinned digest for load-time re-verification. */
     verifyEntrySha?: string;
+    /**
+     * Not a pack artifact, so a `semanticPolicies.add` in it is never asked: a
+     * Jev check reaches a machine only through a pack's manifest. Said once per
+     * file, whether or not Jev is configured — the declaration is dead either way.
+     */
+    outsidePack?: boolean;
   },
 ): Promise<PolicyLoadFailure | null> {
   const g = globalThis as Record<string, unknown>;
@@ -220,7 +230,16 @@ async function loadSingleFile(
     const entryTmp = absPath + tmpSuffix;
     const fileUrl = pathToFileURL(entryTmp).href;
     const hooksBefore = getCustomHooks().length;
+    const semanticBefore = getSemanticRegistrations().length;
     await importWithDeadline(fileUrl);
+    const ignored = getSemanticRegistrations().slice(semanticBefore).map((d) => d?.name);
+    if (opts?.outsidePack && ignored.length > 0 && !warnedSemanticOutsidePack.has(absPath)) {
+      warnedSemanticOutsidePack.add(absPath);
+      hookLogWarn(
+        `${basename(absPath)}: semanticPolicies.add only takes effect in a pack published with \`failproofai publish\`; ` +
+          `${ignored.join(", ")} ${ignored.length === 1 ? "is" : "are"} never asked here`,
+      );
+    }
     if (policyModuleCache.size >= POLICY_MODULE_CACHE_MAX_ENTRIES) policyModuleCache.clear();
     policyModuleCache.set(absPath, {
       fingerprint,
@@ -448,9 +467,33 @@ export async function loadAllCustomHooks(
       `cloud-managed policies ${existing.id} and ${policy.id} have identical source, so they share ` +
         `one artifact and load as one policy; enforcing it if either asks to enforce`,
     );
-    if (existing.effect !== "enforce" && policy.effect === "enforce") {
-      cloudManagedByPath.set(key, policy);
+    // Authority resolves toward HARD, for the same reason: reviewable only if
+    // every assignment behind these bytes says so. Keeping one record's
+    // declaration let the order of active.json decide, so a team's reviewable
+    // assignment could clear what an org-wide hard one enforces.
+    //
+    // Judged against the checks THIS MACHINE can ask, which is what registration
+    // will judge these same declarations by. Against the builtin set instead, a
+    // `reviewedBy` naming a check an installed pack ships reads as unknown here,
+    // both assignments resolve hard, and the merge has no way to tell the
+    // reviewable one from the hard one. Read inside this branch, so a deployment
+    // with no duplicate artifact does not pay for the manifest read at all.
+    const knownReviewers = effectiveReviewerNames();
+    const winner = existing.effect !== "enforce" && policy.effect === "enforce" ? policy : existing;
+    const { merged, overruled, refused } = withMergedAuthority(winner, [existing, policy], knownReviewers);
+    if (overruled) {
+      warnAuthority(
+        `cloud-managed policies ${existing.id} and ${policy.id} share one artifact and do not all declare it ` +
+          `reviewable, so it stays hard`,
+      );
     }
+    // The merge hardened the record, so registration will not see the refused
+    // declaration and cannot report it. Said here instead, naming both
+    // assignments — which is more than registration could have said.
+    if (refused) {
+      warnAuthority(refusedAuthorityWarning(`cloud-managed policies ${existing.id} and ${policy.id}`, refused));
+    }
+    cloudManagedByPath.set(key, merged);
   }
 
   // Installed packs, keyed by artifact path — the same content-addressing, and
@@ -482,14 +525,46 @@ export async function loadAllCustomHooks(
    *
    * Winner precedence on a name both declare, because the merged record carries
    * the winner's id and version and there is no merging two different defaults.
+   *
+   * Except AUTHORITY, which resolves toward hard like the effect resolves toward
+   * enforce (`withMergedAuthority`): a name is reviewable only if every pack
+   * behind the artifact declares it reviewable. A pack whose manifest does not
+   * list the name at all counts as hard — its copy of the artifact registers
+   * the policy all the same, undeclared, and an undeclared pack policy is hard.
+   * Winner precedence here would let one pack's manifest make another pack's
+   * policy reviewable, which is the one thing a manifest may never do.
+   *
+   * `knownReviewers` is what makes that hold for a pack that ships BOTH tiers.
+   * Judged against the compiled-in set, a `reviewedBy` naming one of the pack's
+   * own checks is a name nothing here has, so the reviewable entry and its hard
+   * peer resolved alike and the merge could not tell them apart — while
+   * registration, which reads the manifest's own checks, honoured it. The set
+   * passed here is that same one.
    */
   const unionCatalog = (
     winner: PolicyCatalogEntry[],
     other: PolicyCatalogEntry[],
-  ): PolicyCatalogEntry[] => [
-    ...winner,
-    ...other.filter((p) => !winner.some((w) => w.name === p.name)),
-  ];
+    knownReviewers: ReadonlySet<string>,
+  ): { policies: PolicyCatalogEntry[]; overruled: string[]; refused: string[] } => {
+    /** Names a pack asked to be reviewable and did not get, for the warning. */
+    const overruled: string[] = [];
+    /** The refusals themselves, which registration will no longer see to report. */
+    const refused: string[] = [];
+    const harden = (entry: PolicyCatalogEntry, peer: PolicyCatalogEntry | undefined) => {
+      const r = withMergedAuthority(entry, [entry, peer ?? {}], knownReviewers);
+      if (r.overruled) overruled.push(entry.name);
+      if (r.refused) refused.push(refusedAuthorityWarning(entry.name, r.refused));
+      return r.merged;
+    };
+    return {
+      policies: [
+        ...winner.map((w) => harden(w, other.find((p) => p.name === w.name))),
+        ...other.filter((p) => !winner.some((w) => w.name === p.name)).map((p) => harden(p, undefined)),
+      ],
+      overruled,
+      refused,
+    };
+  };
 
   const packByPath = new Map<string, ResolvedPack>();
   /**
@@ -535,12 +610,24 @@ export async function loadAllCustomHooks(
     // being intersected away.
     const winner = existing.effect !== "enforce" && pack.effect === "enforce" ? pack : existing;
     const other = winner === existing ? pack : existing;
+    // Read here rather than at the top of the load: only a duplicate artifact
+    // needs it, and this is the branch that has one.
+    const catalog = unionCatalog(winner.policies, other.policies, effectiveReviewerNames());
     packByPath.set(key, {
       ...winner,
-      policies: unionCatalog(winner.policies, other.policies),
+      policies: catalog.policies,
       enabled: unionSelection(winner.enabled, other.enabled),
       clis: unionSelection(winner.clis, other.clis),
     });
+    if (catalog.overruled.length > 0) {
+      warnAuthority(
+        `packs ${existing.id} and ${pack.id} share one artifact and do not all declare ` +
+          `${catalog.overruled.join(", ")} reviewable, so ${catalog.overruled.length === 1 ? "it stays" : "they stay"} hard`,
+      );
+    }
+    // A refused `reviewable` is now hardened here rather than passed on to
+    // registration, so this is the only place left that can name the reason.
+    for (const message of catalog.refused) warnAuthority(message);
   }
 
   // 1. Explicit custom policy paths. Accept a string for callers/configs using
@@ -566,6 +653,7 @@ export async function loadAllCustomHooks(
         const failure = await loadSingleFile(absPath, {
           verifyEntrySha:
             cloudManaged?.sha256 ?? pack?.sha256,
+          outsidePack: !pack,
         });
         // Every id behind these bytes. One artifact, one import, one failure —
         // but as many fail-closed guards as there are packs depending on it.
@@ -646,7 +734,7 @@ export async function loadAllCustomHooks(
   for (const file of projectFiles) {
     loadedPaths.add(file);
     const hooksBefore = getCustomHooks().length;
-    await loadSingleFile(file, { conventionScope: projectScope });
+    await loadSingleFile(file, { conventionScope: projectScope, outsidePack: true });
     const newHooks = getCustomHooks().slice(hooksBefore);
     for (const hook of newHooks) {
       (hook as CustomHook & { __policyId?: string }).__policyId = conventionPolicyId(projectScope, basename(file), hook.name);
@@ -686,7 +774,7 @@ export async function loadAllCustomHooks(
   for (const file of userFiles) {
     loadedPaths.add(file);
     const hooksBefore = getCustomHooks().length;
-    await loadSingleFile(file, { conventionScope: "user" });
+    await loadSingleFile(file, { conventionScope: "user", outsidePack: true });
     const newHooks = getCustomHooks().slice(hooksBefore);
     for (const hook of newHooks) {
       (hook as CustomHook & { __policyId?: string }).__policyId = conventionPolicyId("user", basename(file), hook.name);

package/src/hooks/custom-hooks-registry.ts

@@ -4,9 +4,14 @@
  * from 'failproofai') and the hook handler (which reads from this file) share the
  * same in-process state when running within the same Node.js process.
  */
-import type { CustomHook } from "./policy-types";
+import type { CustomHook, SemanticPolicyDeclaration } from "./policy-types";
 
 const REGISTRY_KEY = "__failproofai_custom_hooks__";
+/**
+ * Beside the hook registry, on the same key namespace and cleared by the same
+ * call — the two are one registration pass and a build step reads both.
+ */
+const SEMANTIC_KEY = "__failproofai_semantic_policies__";
 
 function getRegistry(): CustomHook[] {
   const g = globalThis as Record<string, unknown>;
@@ -14,17 +19,56 @@ function getRegistry(): CustomHook[] {
   return g[REGISTRY_KEY] as CustomHook[];
 }
 
+function getSemanticRegistry(): SemanticPolicyDeclaration[] {
+  const g = globalThis as Record<string, unknown>;
+  if (!Array.isArray(g[SEMANTIC_KEY])) g[SEMANTIC_KEY] = [];
+  return g[SEMANTIC_KEY] as SemanticPolicyDeclaration[];
+}
+
 export const customPolicies = {
   add(hook: CustomHook): void {
     getRegistry().push(hook);
   },
 };
 
+/**
+ * Register a semantic (Jev) policy — a question set for the classifier, read by
+ * `failproofai publish` and emitted as the pack manifest's `semantic` array.
+ *
+ * A SEPARATE namespace from `customPolicies`, deliberately, rather than a
+ * `customPolicies.addSemantic`. A semantic policy has no `fn` and no `match`,
+ * and nothing about it ever executes on a user's machine: what installs is the
+ * declaration in the manifest, and the artifact's call here matters only at
+ * build time. Putting it behind the object whose entries return
+ * `allow()`/`deny()` invites exactly the shape confusion the manifest parser
+ * would then have to catch — a function on a semantic entry, a probe list on a
+ * regex one — and both mistakes would be silent until somebody read a manifest.
+ *
+ * Nothing is validated here. `parsePackSemanticPolicy` in `pack-manifest.ts`
+ * owns every rule, and it is applied at BUILD time (where the author can still
+ * fix it) and again at load time (where the bytes may have come from anywhere).
+ * Validating a third time in the setter would be a third copy of the rules.
+ */
+export const semanticPolicies = {
+  add(policy: SemanticPolicyDeclaration): void {
+    getSemanticRegistry().push(policy);
+  },
+};
+
 export function getCustomHooks(): CustomHook[] {
   return getRegistry();
 }
 
+export function getSemanticRegistrations(): SemanticPolicyDeclaration[] {
+  return getSemanticRegistry();
+}
+
 export function clearCustomHooks(): void {
   const g = globalThis as Record<string, unknown>;
   g[REGISTRY_KEY] = [];
+  // Both maps, on one call. A loader that cleared only the hooks would carry
+  // one entry file's semantic policies into the next file's build — and since
+  // the semantic half is read after the load rather than during it, the extras
+  // would be published under the wrong pack with nothing to notice.
+  g[SEMANTIC_KEY] = [];
 }

package/src/hooks/effective-reviewers.ts

@@ -0,0 +1,243 @@
+/**
+ * The semantic checks a `reviewedBy` may name ON THIS MACHINE.
+ *
+ * ## Why this is not simply `SEMANTIC_REVIEWER_NAMES`
+ *
+ * A pack that declares its own `semantic` entries replaces the compiled-in
+ * `SEMANTIC_POLICIES` wholesale (`semantic/pack-policies.ts`). So on that
+ * machine the sixteen builtin names are checks that will never be asked, and the
+ * pack's own names are the only ones that will — while
+ * `resolvePolicyAuthority`, judging against the builtin set, would read the
+ * pack's `reviewedBy: ["its-own-check"]` as naming a check "this build does not
+ * have" and downgrade the policy to `hard` with nothing but a warning in the
+ * hook log. That is a rewritten policy set arriving with its reviewability
+ * quietly removed: the deny half of the two-tier evaluator working exactly as
+ * documented while the clearing half cannot fire once — the failure
+ * `policy-reviewability.ts` was written to make audible, reintroduced one layer
+ * down.
+ *
+ * The names come from the MANIFEST, which is already where `authority` and
+ * `reviewedBy` themselves are read from, so this adds no new trust and imports
+ * no semantic module.
+ *
+ * ## Why it is a module of its own, and cached the way it is
+ *
+ * `registerPolicy` asks this for every policy that declares an authority — forty
+ * times per hook event — and the honest answer costs a digest verification per
+ * installed pack. So it is cached, and the cache is dropped by `clearPolicies()`:
+ * one read per registration pass, and the answer's lifetime is exactly the
+ * registry's. That is the right coupling rather than a convenient one — this set
+ * describes the policies sitting in the registry, and the handler rebuilds both
+ * together, so they cannot come to describe different machines.
+ *
+ * It lives beside `policy-registry.ts` rather than inside `pack-manifest.ts`
+ * because the registry may not depend on the whole manifest reader: several
+ * tests isolate themselves from the running machine's real packs by mocking
+ * `pack-manifest` down to `readInstalledPacks` and `hasInstalledPacks`, and a
+ * registration path reaching for a third export of that module breaks them all
+ * — loudly here, but it is the shape of a change that would later break them
+ * silently.
+ */
+import { readInstalledPacks, type ResolvedPack } from "./pack-manifest";
+import { SEMANTIC_REVIEWER_NAMES } from "./policy-authority";
+
+let cached: ReadonlySet<string> | null = null;
+let cachedContested: ReadonlyMap<string, string[]> = new Map();
+/** The agent the current registration pass is for; see {@link forgetEffectiveReviewerNames}. */
+let reviewerCli: string | undefined;
+
+/**
+ * The packs whose Jev checks take part for `cli`, filtered the way the regex
+ * half already is: an `observe` pack blocks nothing, and a pack scoped to other
+ * agents guards none here. Its checks would otherwise replace the questions and
+ * supply reviewers that clear enforce packs' denies on every agent. Both the
+ * reviewer set and the question set start from this list, so they agree.
+ */
+export function jevPacks<T extends Pick<ResolvedPack, "effect" | "clis">>(packs: ReadonlyArray<T>, cli?: string): T[] {
+  return packs.filter(
+    (p) => p.effect !== "observe" && !(cli && Array.isArray(p.clis) && p.clis.length > 0 && !p.clis.includes(cli)),
+  );
+}
+
+/**
+ * A stable serialization, for comparing two declarations of the same name.
+ * Key order is a manifest's business, not a difference in what is asked.
+ */
+function canonical(value: unknown): string {
+  if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
+  if (value !== null && typeof value === "object") {
+    const entries = Object.entries(value as Record<string, unknown>)
+      .filter(([, v]) => v !== undefined)
+      .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+    return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(",")}}`;
+  }
+  return JSON.stringify(value) ?? "null";
+}
+
+/**
+ * The semantic check names that more than one installed pack claims, with
+ * declarations that are not the same check — and which this machine must
+ * therefore honour for nobody. Keyed by name, valued with the packs that
+ * claimed it, for the warning.
+ *
+ * ## Why a contested name has to be refused rather than resolved
+ *
+ * The reviewer set is the whole protection: a policy is `reviewable` only when
+ * every name its `reviewedBy` lists is a check this machine can actually ask.
+ * The union across packs is deliberate and must keep working — the shipped
+ * configuration is exactly that, `FailproofAI/policies` naming checks that live
+ * in `FailproofAI/jev-policies` — but the union was AMBIGUOUS. Two packs may be
+ * installed at once, `semanticPoliciesFromPacks` keeps the first declaration of
+ * a duplicated name and drops the later one, and nothing tied a name to the pack
+ * that meant it. So a second pack declaring `production-infra-change` with a
+ * question that answers "no concern" to everything supplies the reviewer for the
+ * FIRST pack's policies: install it, and every policy reviewable by that name is
+ * cleared on every call. The pack doing it never declares a regex policy, and
+ * scoping reviewers per pack is not the fix — that would break the pairing the
+ * product ships.
+ *
+ * So the name leaves the set. The policies naming it resolve to `hard`, the
+ * question is not asked, and the pack that tried it gains nothing: stricter than
+ * either author asked for, with the regex deny simply standing. Nothing is
+ * failed closed over it — one policy being hard is a cost; a pack that denies
+ * every tool call is a different order of failure (`pack-failclosed.ts`).
+ *
+ * ## Why identical declarations are not a contest
+ *
+ * Content-addressed artifacts mean a fork or a re-publish arrives as two packs
+ * carrying the same semantic entries, and `custom-hooks-loader.ts` treats that
+ * shape as expected rather than as a corner. Both declarations are then the same
+ * question, so whichever the resolver keeps asks exactly what the other would
+ * have: nothing is ambiguous, and refusing the name would switch off the
+ * clearing half on a machine whose two packs agree to the byte.
+ *
+ * Pure, and takes the packs rather than reading them, so the two callers that
+ * already have them do not read `installed.json` again — and so both answer from
+ * the same list. They MUST agree: a name in the reviewer set whose question is
+ * another pack's is the bug this function exists to remove.
+ */
+export function contestedSemanticNames(
+  packs: ReadonlyArray<Pick<ResolvedPack, "id" | "semantic"> & { source?: string }>,
+): ReadonlyMap<string, string[]> {
+  const claims = new Map<string, { form: string; ids: string[] }>();
+  const contested = new Map<string, string[]>();
+  for (const pack of packs) {
+    for (const entry of pack.semantic ?? []) {
+      // Void, so not a claim either: see `isReservedClaim`.
+      if (isReservedClaim(pack, entry.name)) continue;
+      const form = canonical(entry);
+      const claim = claims.get(entry.name);
+      if (claim === undefined) {
+        claims.set(entry.name, { form, ids: [pack.id] });
+        continue;
+      }
+      claim.ids.push(pack.id);
+      // The array is the live one, so a third claimant is named too.
+      if (claim.form !== form) contested.set(entry.name, claim.ids);
+    }
+  }
+  return contested;
+}
+
+/**
+ * Installed from a FailproofAI repository. Read from `source`, which this CLI
+ * wrote from the repository it fetched, never from the pack's self-declared `id`.
+ */
+export function isFirstPartyPack(pack: { source?: string }): boolean {
+  return /^github:FailproofAI\//i.test(pack.source ?? "");
+}
+
+/**
+ * A builtin check name declared by a pack that is not FailproofAI's. Core and
+ * user policies name those checks in `reviewedBy`, so the third party's
+ * question would become the reviewer that clears them — an `instruct`-mode
+ * `destructive-deletion` beside the core pack turned `block-rm-rf` into a
+ * warning. The claim is void: never asked, never a reviewer, and never a
+ * second declaration that contests FailproofAI's own, which would switch off a
+ * deny the regex tier does not have. The reviewer set, the question set and
+ * the contest all apply it.
+ */
+export function isReservedClaim(pack: { source?: string }, name: string): boolean {
+  return SEMANTIC_REVIEWER_NAMES.has(name) && !isFirstPartyPack(pack);
+}
+
+/**
+ * The installed packs' semantic policy names when any pack declares some, and
+ * this build's compiled-in set otherwise.
+ *
+ * A name two packs claim differently is left out — see
+ * {@link contestedSemanticNames}. When that leaves nothing, the compiled-in set
+ * stands, which is not a softening but the same rule
+ * `semanticPoliciesFromPacks` applies to the QUESTIONS: a declaring pack whose
+ * every entry was unusable leaves the compiled-in semantic set live, so the
+ * names honoured here are the names of the questions that will actually be
+ * asked, and each of those is one of ours.
+ *
+ * Never throws: an unreadable manifest declares nothing, and the builtin set
+ * stands. That is the same fail-open posture every other reader of the pack
+ * manifest takes, and here it also fails in the safe direction — a pack's
+ * `reviewedBy` will not match a builtin name, so its policies stay `hard`
+ * rather than being cleared by a question nobody could read.
+ */
+export function effectiveReviewerNames(): ReadonlySet<string> {
+  if (cached) return cached;
+  let names: ReadonlySet<string> = SEMANTIC_REVIEWER_NAMES;
+  cachedContested = new Map();
+  try {
+    const packs = jevPacks(readInstalledPacks().packs, reviewerCli);
+    names = reviewerNamesFor(packs);
+    cachedContested = contestedSemanticNames(packs);
+  } catch {
+    // See above: silence here is the builtin set, not an empty one.
+  }
+  cached = names;
+  return names;
+}
+
+/** The contest behind {@link effectiveReviewerNames}' answer, so a refusal can name it. */
+export function contestedReviewerNames(): ReadonlyMap<string, string[]> {
+  effectiveReviewerNames();
+  return cachedContested;
+}
+
+/**
+ * A first-party pack declares Jev checks, so they REPLACE the compiled-in set.
+ * Anyone else's checks are ADDED to it: a stranger's one check must not switch
+ * off `credential-exfiltration` and turn every core reviewable policy hard.
+ * `semanticPoliciesFromPacks` applies the same rule to the questions.
+ */
+export function replacesBuiltinChecks(packs: ReadonlyArray<Pick<ResolvedPack, "semantic"> & { source?: string }>): boolean {
+  return packs.some((p) => isFirstPartyPack(p) && (p.semantic ?? []).length > 0);
+}
+
+/**
+ * The reviewer set for the packs taking part: see {@link replacesBuiltinChecks}. Pure.
+ *
+ * Manifest-only: it cannot see a check `semanticPoliciesFromPacks` drops for the
+ * question budget, because measuring questions means loading the semantic
+ * modules this hook-path file must not reach. That overcount is fail-safe — a
+ * dropped check is never answered, so it never clears — and the diagnostic
+ * (`surveyReviewableCoverage`) asks the resolver instead.
+ */
+export function reviewerNamesFor(
+  packs: ReadonlyArray<Pick<ResolvedPack, "id" | "semantic"> & { source?: string }>,
+): ReadonlySet<string> {
+  const contested = contestedSemanticNames(packs);
+  const declared = packs
+    .flatMap((p) => (p.semantic ?? []).filter((s) => !isReservedClaim(p, s.name)).map((s) => s.name))
+    .filter((name) => !contested.has(name));
+  // Nothing usable declared: the compiled-in set is the one being asked.
+  if (declared.length === 0) return SEMANTIC_REVIEWER_NAMES;
+  return new Set(replacesBuiltinChecks(packs) ? declared : [...SEMANTIC_REVIEWER_NAMES, ...declared]);
+}
+
+/**
+ * Drop the cached answer, and set the agent the next one is for. Called by
+ * `clearPolicies(cli)`, which every evaluation
+ * runs before it registers anything — so a pack installed under a long-lived
+ * warm worker is picked up on the next event rather than at the next restart.
+ */
+export function forgetEffectiveReviewerNames(cli?: string): void {
+  cached = null;
+  reviewerCli = cli;
+}

package/src/hooks/first-run-gate.ts

@@ -35,6 +35,11 @@ export const FIRST_RUN_EXEMPT_SUBCOMMANDS: readonly string[] = [
   // Same reason: a backfill is an explicit instruction about an already-set-up
   // machine, and interrupting it to offer setup answers a question nobody asked.
   "backfill",
+  // `jev setup` is configuration in its own right, and `jev status` / `test` /
+  // `remove` inspect or undo it. It is also the command most likely to be
+  // scripted with a key on stdin — a wizard in front of it would read that key
+  // as the answer to its first question.
+  "jev",
 ];
 
 export function shouldOfferFirstRun(args: readonly string[]): boolean {

package/src/hooks/flush-cli.ts

@@ -21,7 +21,7 @@
  */
 import { existsSync, mkdirSync, writeFileSync, readdirSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { failproofaiHome, spoolDir } from "./fp-home";
+import { customAgentsEventsDir, failedDir, failproofaiHome, spoolDir } from "./fp-home";
 import { readConfig } from "./fp-config";
 import { readIngestCredential } from "./collector-config";
 import { daemonServiceStatus, isDaemonSupportedPlatform } from "./daemon-service";
@@ -49,19 +49,41 @@ export function flushRequestPath(home?: string): string {
   return join(failproofaiHome(home), "state", "flush-request.json");
 }
 
-/** Every directory the collector spools batches into. */
+/** Every directory the collector spools batches into. The daemon writes its
+ *  own batches FLAT into state/spool and the SDKs into custom-agents/events
+ *  (`spool_dirs` in crates/fpai-collect/src/config.rs). Counting only
+ *  subdirectories of state/spool, as this once did, found nothing on every
+ *  machine, so flush always answered "everything already delivered". */
 function spoolDirs(home?: string): string[] {
   const root = spoolDir(home);
-  if (!existsSync(root)) return [];
+  const dirs = [root, customAgentsEventsDir(home)];
+  if (!existsSync(root)) return dirs;
   try {
-    return readdirSync(root, { withFileTypes: true })
-      .filter((e) => e.isDirectory() && e.name !== "failed")
-      .map((e) => join(root, e.name));
+    return dirs.concat(
+      readdirSync(root, { withFileTypes: true })
+        .filter((e) => e.isDirectory() && e.name !== "failed")
+        .map((e) => join(root, e.name)),
+    );
   } catch {
-    return [];
+    return dirs;
   }
 }
 
+/** Batches parked in state/failed after failed uploads, `.poison` excluded.
+ *  Flush does not resend these: the daemon retries them about hourly and
+ *  when it starts, so all this can do is not call them delivered. */
+function parkedBatches(home?: string): number {
+  try {
+    return readdirSync(failedDir(home)).filter((f) => f.endsWith(".jsonl")).length;
+  } catch {
+    return 0;
+  }
+}
+
+function parkedLine(n: number, home?: string): string {
+  return `${n} batch${n === 1 ? "" : "es"} parked in ${failedDir(home)} after failed uploads; the daemon retries them about hourly and when it restarts.`;
+}
+
 /** Batches awaiting delivery. `.tmp` files are half-written and not counted. */
 export function pendingBatches(home?: string): number {
   let n = 0;
@@ -138,8 +160,12 @@ export async function runFlushCommand(opts: FlushOptions = {}): Promise<FlushRes
   }
 
   const before = pendingBatches(home);
+  const parked = parkedBatches(home);
   if (before === 0) {
-    return { exitCode: 0, pending: 0, lines: ["Nothing spooled — everything already delivered."] };
+    const nothing = parked
+      ? ["Nothing spooled.", parkedLine(parked, home)]
+      : ["Nothing spooled — everything already delivered."];
+    return { exitCode: 0, pending: 0, lines: nothing };
   }
 
   const path = flushRequestPath(home);
@@ -149,6 +175,7 @@ export async function runFlushCommand(opts: FlushOptions = {}): Promise<FlushRes
   });
 
   lines.push(`${before} batch${before === 1 ? "" : "es"} spooled. Requested delivery.`);
+  if (parked) lines.push(parkedLine(parked, home));
 
   if (!wait) {
     lines.push("The daemon picks this up within a few seconds.");