failproofai 1.0.7-beta.2 → 1.0.8-beta.0

package/src/hooks/custom-hooks-loader.ts

@@ -17,7 +17,7 @@ import { readFile } from "node:fs/promises";
 import { pathToFileURL } from "node:url";
 import { createHash } from "node:crypto";
 import { hookLogWarn, hookLogError, hookLogInfo } from "./hook-logger";
-import { customPolicies, getCustomHooks, clearCustomHooks } from "./custom-hooks-registry";
+import { customPolicies, getCustomHooks, getSemanticRegistrations, clearCustomHooks } from "./custom-hooks-registry";
 import {
   findDistIndex,
   rewriteFileTree,
@@ -33,7 +33,8 @@ import type { CustomHook, PolicyCatalogEntry } from "./policy-types";
 import type { CloudManagedPolicyArtifact } from "./cloud-managed-policies";
 import type { ResolvedPack } from "./pack-manifest";
 import { customPoliciesDir, shimsDir } from "./fp-home";
-import { warnAuthority, withMergedAuthority } from "./policy-authority";
+import { effectiveReviewerNames } from "./effective-reviewers";
+import { refusedAuthorityWarning, warnAuthority, withMergedAuthority } from "./policy-authority";
 
 const LOADING_KEY = "__FAILPROOFAI_LOADING_HOOKS__";
 
@@ -179,6 +180,8 @@ export interface PolicyLoadFailure {
   reason: string;
 }
 
+const warnedSemanticOutsidePack = new Set<string>();
+
 async function loadSingleFile(
   absPath: string,
   opts?: {
@@ -186,6 +189,12 @@ async function loadSingleFile(
     conventionScope?: "project" | "user";
     /** Cloud-managed policies pass their pinned digest for load-time re-verification. */
     verifyEntrySha?: string;
+    /**
+     * Not a pack artifact, so a `semanticPolicies.add` in it is never asked: a
+     * Jev check reaches a machine only through a pack's manifest. Said once per
+     * file, whether or not Jev is configured — the declaration is dead either way.
+     */
+    outsidePack?: boolean;
   },
 ): Promise<PolicyLoadFailure | null> {
   const g = globalThis as Record<string, unknown>;
@@ -221,7 +230,16 @@ async function loadSingleFile(
     const entryTmp = absPath + tmpSuffix;
     const fileUrl = pathToFileURL(entryTmp).href;
     const hooksBefore = getCustomHooks().length;
+    const semanticBefore = getSemanticRegistrations().length;
     await importWithDeadline(fileUrl);
+    const ignored = getSemanticRegistrations().slice(semanticBefore).map((d) => d?.name);
+    if (opts?.outsidePack && ignored.length > 0 && !warnedSemanticOutsidePack.has(absPath)) {
+      warnedSemanticOutsidePack.add(absPath);
+      hookLogWarn(
+        `${basename(absPath)}: semanticPolicies.add only takes effect in a pack published with \`failproofai publish\`; ` +
+          `${ignored.join(", ")} ${ignored.length === 1 ? "is" : "are"} never asked here`,
+      );
+    }
     if (policyModuleCache.size >= POLICY_MODULE_CACHE_MAX_ENTRIES) policyModuleCache.clear();
     policyModuleCache.set(absPath, {
       fingerprint,
@@ -453,14 +471,28 @@ export async function loadAllCustomHooks(
     // every assignment behind these bytes says so. Keeping one record's
     // declaration let the order of active.json decide, so a team's reviewable
     // assignment could clear what an org-wide hard one enforces.
+    //
+    // Judged against the checks THIS MACHINE can ask, which is what registration
+    // will judge these same declarations by. Against the builtin set instead, a
+    // `reviewedBy` naming a check an installed pack ships reads as unknown here,
+    // both assignments resolve hard, and the merge has no way to tell the
+    // reviewable one from the hard one. Read inside this branch, so a deployment
+    // with no duplicate artifact does not pay for the manifest read at all.
+    const knownReviewers = effectiveReviewerNames();
     const winner = existing.effect !== "enforce" && policy.effect === "enforce" ? policy : existing;
-    const { merged, overruled } = withMergedAuthority(winner, [existing, policy]);
+    const { merged, overruled, refused } = withMergedAuthority(winner, [existing, policy], knownReviewers);
     if (overruled) {
       warnAuthority(
         `cloud-managed policies ${existing.id} and ${policy.id} share one artifact and do not all declare it ` +
           `reviewable, so it stays hard`,
       );
     }
+    // The merge hardened the record, so registration will not see the refused
+    // declaration and cannot report it. Said here instead, naming both
+    // assignments — which is more than registration could have said.
+    if (refused) {
+      warnAuthority(refusedAuthorityWarning(`cloud-managed policies ${existing.id} and ${policy.id}`, refused));
+    }
     cloudManagedByPath.set(key, merged);
   }
 
@@ -501,16 +533,27 @@ export async function loadAllCustomHooks(
    * the policy all the same, undeclared, and an undeclared pack policy is hard.
    * Winner precedence here would let one pack's manifest make another pack's
    * policy reviewable, which is the one thing a manifest may never do.
+   *
+   * `knownReviewers` is what makes that hold for a pack that ships BOTH tiers.
+   * Judged against the compiled-in set, a `reviewedBy` naming one of the pack's
+   * own checks is a name nothing here has, so the reviewable entry and its hard
+   * peer resolved alike and the merge could not tell them apart — while
+   * registration, which reads the manifest's own checks, honoured it. The set
+   * passed here is that same one.
    */
   const unionCatalog = (
     winner: PolicyCatalogEntry[],
     other: PolicyCatalogEntry[],
-  ): { policies: PolicyCatalogEntry[]; overruled: string[] } => {
+    knownReviewers: ReadonlySet<string>,
+  ): { policies: PolicyCatalogEntry[]; overruled: string[]; refused: string[] } => {
     /** Names a pack asked to be reviewable and did not get, for the warning. */
     const overruled: string[] = [];
+    /** The refusals themselves, which registration will no longer see to report. */
+    const refused: string[] = [];
     const harden = (entry: PolicyCatalogEntry, peer: PolicyCatalogEntry | undefined) => {
-      const r = withMergedAuthority(entry, [entry, peer ?? {}]);
+      const r = withMergedAuthority(entry, [entry, peer ?? {}], knownReviewers);
       if (r.overruled) overruled.push(entry.name);
+      if (r.refused) refused.push(refusedAuthorityWarning(entry.name, r.refused));
       return r.merged;
     };
     return {
@@ -519,6 +562,7 @@ export async function loadAllCustomHooks(
         ...other.filter((p) => !winner.some((w) => w.name === p.name)).map((p) => harden(p, undefined)),
       ],
       overruled,
+      refused,
     };
   };
 
@@ -566,7 +610,9 @@ export async function loadAllCustomHooks(
     // being intersected away.
     const winner = existing.effect !== "enforce" && pack.effect === "enforce" ? pack : existing;
     const other = winner === existing ? pack : existing;
-    const catalog = unionCatalog(winner.policies, other.policies);
+    // Read here rather than at the top of the load: only a duplicate artifact
+    // needs it, and this is the branch that has one.
+    const catalog = unionCatalog(winner.policies, other.policies, effectiveReviewerNames());
     packByPath.set(key, {
       ...winner,
       policies: catalog.policies,
@@ -579,6 +625,9 @@ export async function loadAllCustomHooks(
           `${catalog.overruled.join(", ")} reviewable, so ${catalog.overruled.length === 1 ? "it stays" : "they stay"} hard`,
       );
     }
+    // A refused `reviewable` is now hardened here rather than passed on to
+    // registration, so this is the only place left that can name the reason.
+    for (const message of catalog.refused) warnAuthority(message);
   }
 
   // 1. Explicit custom policy paths. Accept a string for callers/configs using
@@ -604,6 +653,7 @@ export async function loadAllCustomHooks(
         const failure = await loadSingleFile(absPath, {
           verifyEntrySha:
             cloudManaged?.sha256 ?? pack?.sha256,
+          outsidePack: !pack,
         });
         // Every id behind these bytes. One artifact, one import, one failure —
         // but as many fail-closed guards as there are packs depending on it.
@@ -684,7 +734,7 @@ export async function loadAllCustomHooks(
   for (const file of projectFiles) {
     loadedPaths.add(file);
     const hooksBefore = getCustomHooks().length;
-    await loadSingleFile(file, { conventionScope: projectScope });
+    await loadSingleFile(file, { conventionScope: projectScope, outsidePack: true });
     const newHooks = getCustomHooks().slice(hooksBefore);
     for (const hook of newHooks) {
       (hook as CustomHook & { __policyId?: string }).__policyId = conventionPolicyId(projectScope, basename(file), hook.name);
@@ -724,7 +774,7 @@ export async function loadAllCustomHooks(
   for (const file of userFiles) {
     loadedPaths.add(file);
     const hooksBefore = getCustomHooks().length;
-    await loadSingleFile(file, { conventionScope: "user" });
+    await loadSingleFile(file, { conventionScope: "user", outsidePack: true });
     const newHooks = getCustomHooks().slice(hooksBefore);
     for (const hook of newHooks) {
       (hook as CustomHook & { __policyId?: string }).__policyId = conventionPolicyId("user", basename(file), hook.name);

package/src/hooks/effective-reviewers.ts

@@ -38,15 +38,141 @@
  * — loudly here, but it is the shape of a change that would later break them
  * silently.
  */
-import { readInstalledPacks } from "./pack-manifest";
+import { readInstalledPacks, type ResolvedPack } from "./pack-manifest";
 import { SEMANTIC_REVIEWER_NAMES } from "./policy-authority";
 
 let cached: ReadonlySet<string> | null = null;
+let cachedContested: ReadonlyMap<string, string[]> = new Map();
+/** The agent the current registration pass is for; see {@link forgetEffectiveReviewerNames}. */
+let reviewerCli: string | undefined;
+
+/**
+ * The packs whose Jev checks take part for `cli`, filtered the way the regex
+ * half already is: an `observe` pack blocks nothing, and a pack scoped to other
+ * agents guards none here. Its checks would otherwise replace the questions and
+ * supply reviewers that clear enforce packs' denies on every agent. Both the
+ * reviewer set and the question set start from this list, so they agree.
+ */
+export function jevPacks<T extends Pick<ResolvedPack, "effect" | "clis">>(packs: ReadonlyArray<T>, cli?: string): T[] {
+  return packs.filter(
+    (p) => p.effect !== "observe" && !(cli && Array.isArray(p.clis) && p.clis.length > 0 && !p.clis.includes(cli)),
+  );
+}
+
+/**
+ * A stable serialization, for comparing two declarations of the same name.
+ * Key order is a manifest's business, not a difference in what is asked.
+ */
+function canonical(value: unknown): string {
+  if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
+  if (value !== null && typeof value === "object") {
+    const entries = Object.entries(value as Record<string, unknown>)
+      .filter(([, v]) => v !== undefined)
+      .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+    return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`).join(",")}}`;
+  }
+  return JSON.stringify(value) ?? "null";
+}
+
+/**
+ * The semantic check names that more than one installed pack claims, with
+ * declarations that are not the same check — and which this machine must
+ * therefore honour for nobody. Keyed by name, valued with the packs that
+ * claimed it, for the warning.
+ *
+ * ## Why a contested name has to be refused rather than resolved
+ *
+ * The reviewer set is the whole protection: a policy is `reviewable` only when
+ * every name its `reviewedBy` lists is a check this machine can actually ask.
+ * The union across packs is deliberate and must keep working — the shipped
+ * configuration is exactly that, `FailproofAI/policies` naming checks that live
+ * in `FailproofAI/jev-policies` — but the union was AMBIGUOUS. Two packs may be
+ * installed at once, `semanticPoliciesFromPacks` keeps the first declaration of
+ * a duplicated name and drops the later one, and nothing tied a name to the pack
+ * that meant it. So a second pack declaring `production-infra-change` with a
+ * question that answers "no concern" to everything supplies the reviewer for the
+ * FIRST pack's policies: install it, and every policy reviewable by that name is
+ * cleared on every call. The pack doing it never declares a regex policy, and
+ * scoping reviewers per pack is not the fix — that would break the pairing the
+ * product ships.
+ *
+ * So the name leaves the set. The policies naming it resolve to `hard`, the
+ * question is not asked, and the pack that tried it gains nothing: stricter than
+ * either author asked for, with the regex deny simply standing. Nothing is
+ * failed closed over it — one policy being hard is a cost; a pack that denies
+ * every tool call is a different order of failure (`pack-failclosed.ts`).
+ *
+ * ## Why identical declarations are not a contest
+ *
+ * Content-addressed artifacts mean a fork or a re-publish arrives as two packs
+ * carrying the same semantic entries, and `custom-hooks-loader.ts` treats that
+ * shape as expected rather than as a corner. Both declarations are then the same
+ * question, so whichever the resolver keeps asks exactly what the other would
+ * have: nothing is ambiguous, and refusing the name would switch off the
+ * clearing half on a machine whose two packs agree to the byte.
+ *
+ * Pure, and takes the packs rather than reading them, so the two callers that
+ * already have them do not read `installed.json` again — and so both answer from
+ * the same list. They MUST agree: a name in the reviewer set whose question is
+ * another pack's is the bug this function exists to remove.
+ */
+export function contestedSemanticNames(
+  packs: ReadonlyArray<Pick<ResolvedPack, "id" | "semantic"> & { source?: string }>,
+): ReadonlyMap<string, string[]> {
+  const claims = new Map<string, { form: string; ids: string[] }>();
+  const contested = new Map<string, string[]>();
+  for (const pack of packs) {
+    for (const entry of pack.semantic ?? []) {
+      // Void, so not a claim either: see `isReservedClaim`.
+      if (isReservedClaim(pack, entry.name)) continue;
+      const form = canonical(entry);
+      const claim = claims.get(entry.name);
+      if (claim === undefined) {
+        claims.set(entry.name, { form, ids: [pack.id] });
+        continue;
+      }
+      claim.ids.push(pack.id);
+      // The array is the live one, so a third claimant is named too.
+      if (claim.form !== form) contested.set(entry.name, claim.ids);
+    }
+  }
+  return contested;
+}
+
+/**
+ * Installed from a FailproofAI repository. Read from `source`, which this CLI
+ * wrote from the repository it fetched, never from the pack's self-declared `id`.
+ */
+export function isFirstPartyPack(pack: { source?: string }): boolean {
+  return /^github:FailproofAI\//i.test(pack.source ?? "");
+}
+
+/**
+ * A builtin check name declared by a pack that is not FailproofAI's. Core and
+ * user policies name those checks in `reviewedBy`, so the third party's
+ * question would become the reviewer that clears them — an `instruct`-mode
+ * `destructive-deletion` beside the core pack turned `block-rm-rf` into a
+ * warning. The claim is void: never asked, never a reviewer, and never a
+ * second declaration that contests FailproofAI's own, which would switch off a
+ * deny the regex tier does not have. The reviewer set, the question set and
+ * the contest all apply it.
+ */
+export function isReservedClaim(pack: { source?: string }, name: string): boolean {
+  return SEMANTIC_REVIEWER_NAMES.has(name) && !isFirstPartyPack(pack);
+}
 
 /**
  * The installed packs' semantic policy names when any pack declares some, and
  * this build's compiled-in set otherwise.
  *
+ * A name two packs claim differently is left out — see
+ * {@link contestedSemanticNames}. When that leaves nothing, the compiled-in set
+ * stands, which is not a softening but the same rule
+ * `semanticPoliciesFromPacks` applies to the QUESTIONS: a declaring pack whose
+ * every entry was unusable leaves the compiled-in semantic set live, so the
+ * names honoured here are the names of the questions that will actually be
+ * asked, and each of those is one of ours.
+ *
  * Never throws: an unreadable manifest declares nothing, and the builtin set
  * stands. That is the same fail-open posture every other reader of the pack
  * manifest takes, and here it also fails in the safe direction — a pack's
@@ -56,12 +182,11 @@ let cached: ReadonlySet<string> | null = null;
 export function effectiveReviewerNames(): ReadonlySet<string> {
   if (cached) return cached;
   let names: ReadonlySet<string> = SEMANTIC_REVIEWER_NAMES;
+  cachedContested = new Map();
   try {
-    const declared = readInstalledPacks().packs.flatMap((p) => (p.semantic ?? []).map((s) => s.name));
-    // Only when a pack actually declared some. A machine whose packs carry only
-    // the regex floor still runs the compiled-in semantic set, so its builtin
-    // reviewer names are the live ones.
-    if (declared.length > 0) names = new Set(declared);
+    const packs = jevPacks(readInstalledPacks().packs, reviewerCli);
+    names = reviewerNamesFor(packs);
+    cachedContested = contestedSemanticNames(packs);
   } catch {
     // See above: silence here is the builtin set, not an empty one.
   }
@@ -69,11 +194,50 @@ export function effectiveReviewerNames(): ReadonlySet<string> {
   return names;
 }
 
+/** The contest behind {@link effectiveReviewerNames}' answer, so a refusal can name it. */
+export function contestedReviewerNames(): ReadonlyMap<string, string[]> {
+  effectiveReviewerNames();
+  return cachedContested;
+}
+
+/**
+ * A first-party pack declares Jev checks, so they REPLACE the compiled-in set.
+ * Anyone else's checks are ADDED to it: a stranger's one check must not switch
+ * off `credential-exfiltration` and turn every core reviewable policy hard.
+ * `semanticPoliciesFromPacks` applies the same rule to the questions.
+ */
+export function replacesBuiltinChecks(packs: ReadonlyArray<Pick<ResolvedPack, "semantic"> & { source?: string }>): boolean {
+  return packs.some((p) => isFirstPartyPack(p) && (p.semantic ?? []).length > 0);
+}
+
+/**
+ * The reviewer set for the packs taking part: see {@link replacesBuiltinChecks}. Pure.
+ *
+ * Manifest-only: it cannot see a check `semanticPoliciesFromPacks` drops for the
+ * question budget, because measuring questions means loading the semantic
+ * modules this hook-path file must not reach. That overcount is fail-safe — a
+ * dropped check is never answered, so it never clears — and the diagnostic
+ * (`surveyReviewableCoverage`) asks the resolver instead.
+ */
+export function reviewerNamesFor(
+  packs: ReadonlyArray<Pick<ResolvedPack, "id" | "semantic"> & { source?: string }>,
+): ReadonlySet<string> {
+  const contested = contestedSemanticNames(packs);
+  const declared = packs
+    .flatMap((p) => (p.semantic ?? []).filter((s) => !isReservedClaim(p, s.name)).map((s) => s.name))
+    .filter((name) => !contested.has(name));
+  // Nothing usable declared: the compiled-in set is the one being asked.
+  if (declared.length === 0) return SEMANTIC_REVIEWER_NAMES;
+  return new Set(replacesBuiltinChecks(packs) ? declared : [...SEMANTIC_REVIEWER_NAMES, ...declared]);
+}
+
 /**
- * Drop the cached answer. Called by `clearPolicies()`, which every evaluation
+ * Drop the cached answer, and set the agent the next one is for. Called by
+ * `clearPolicies(cli)`, which every evaluation
  * runs before it registers anything — so a pack installed under a long-lived
  * warm worker is picked up on the next event rather than at the next restart.
  */
-export function forgetEffectiveReviewerNames(): void {
+export function forgetEffectiveReviewerNames(cli?: string): void {
   cached = null;
+  reviewerCli = cli;
 }

package/src/hooks/flush-cli.ts

@@ -21,7 +21,7 @@
  */
 import { existsSync, mkdirSync, writeFileSync, readdirSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { failproofaiHome, spoolDir } from "./fp-home";
+import { customAgentsEventsDir, failedDir, failproofaiHome, spoolDir } from "./fp-home";
 import { readConfig } from "./fp-config";
 import { readIngestCredential } from "./collector-config";
 import { daemonServiceStatus, isDaemonSupportedPlatform } from "./daemon-service";
@@ -49,19 +49,41 @@ export function flushRequestPath(home?: string): string {
   return join(failproofaiHome(home), "state", "flush-request.json");
 }
 
-/** Every directory the collector spools batches into. */
+/** Every directory the collector spools batches into. The daemon writes its
+ *  own batches FLAT into state/spool and the SDKs into custom-agents/events
+ *  (`spool_dirs` in crates/fpai-collect/src/config.rs). Counting only
+ *  subdirectories of state/spool, as this once did, found nothing on every
+ *  machine, so flush always answered "everything already delivered". */
 function spoolDirs(home?: string): string[] {
   const root = spoolDir(home);
-  if (!existsSync(root)) return [];
+  const dirs = [root, customAgentsEventsDir(home)];
+  if (!existsSync(root)) return dirs;
   try {
-    return readdirSync(root, { withFileTypes: true })
-      .filter((e) => e.isDirectory() && e.name !== "failed")
-      .map((e) => join(root, e.name));
+    return dirs.concat(
+      readdirSync(root, { withFileTypes: true })
+        .filter((e) => e.isDirectory() && e.name !== "failed")
+        .map((e) => join(root, e.name)),
+    );
   } catch {
-    return [];
+    return dirs;
   }
 }
 
+/** Batches parked in state/failed after failed uploads, `.poison` excluded.
+ *  Flush does not resend these: the daemon retries them about hourly and
+ *  when it starts, so all this can do is not call them delivered. */
+function parkedBatches(home?: string): number {
+  try {
+    return readdirSync(failedDir(home)).filter((f) => f.endsWith(".jsonl")).length;
+  } catch {
+    return 0;
+  }
+}
+
+function parkedLine(n: number, home?: string): string {
+  return `${n} batch${n === 1 ? "" : "es"} parked in ${failedDir(home)} after failed uploads; the daemon retries them about hourly and when it restarts.`;
+}
+
 /** Batches awaiting delivery. `.tmp` files are half-written and not counted. */
 export function pendingBatches(home?: string): number {
   let n = 0;
@@ -138,8 +160,12 @@ export async function runFlushCommand(opts: FlushOptions = {}): Promise<FlushRes
   }
 
   const before = pendingBatches(home);
+  const parked = parkedBatches(home);
   if (before === 0) {
-    return { exitCode: 0, pending: 0, lines: ["Nothing spooled — everything already delivered."] };
+    const nothing = parked
+      ? ["Nothing spooled.", parkedLine(parked, home)]
+      : ["Nothing spooled — everything already delivered."];
+    return { exitCode: 0, pending: 0, lines: nothing };
   }
 
   const path = flushRequestPath(home);
@@ -149,6 +175,7 @@ export async function runFlushCommand(opts: FlushOptions = {}): Promise<FlushRes
   });
 
   lines.push(`${before} batch${before === 1 ? "" : "es"} spooled. Requested delivery.`);
+  if (parked) lines.push(parkedLine(parked, home));
 
   if (!wait) {
     lines.push("The daemon picks this up within a few seconds.");