@luckydraw/cumulus 1.0.3 → 1.0.5

package/dist/lib/gateway.js

@@ -26,6 +26,7 @@ import { retrieve } from './retriever.js';
 import { SessionManager } from './session.js';
 import { StreamProcessor, shouldExternalizeUserInput, externalizeUserInput, } from './stream-processor.js';
 import { scaffoldFromTemplate } from './templates.js';
+import { applyCapabilityGates, compileDisallowedTools, getToolInventory, promptCapabilities, toolAvailable, } from './tool-inventory.js';
 import { getEagerTools } from './tools/index.js';
 // ─── Constants ───────────────────────────────────────────────────────────────
 const CUMULUS_DIR = process.env.CUMULUS_DIR || path.join(os.homedir(), '.cumulus');
@@ -151,11 +152,11 @@ To CHANGE a file, use the built-in Edit/Write tools.
 - Do NOT patch files by writing shell scripts (sed, awk, python, echo/cat heredocs, etc.). Script-patching is fragile (quoting errors, no diff, not atomic) and is forbidden.
 
 BACKGROUND WORK:
-Your turn runs in a subprocess that EXITS when the turn ends. Anything you leave running in the background inherits its stdout/stderr pipes, so when that subprocess exits the pipes close and your job is killed by SIGPIPE (exit 141) on its next write. This happens on EVERY turn, not occasionally.
-- To survive, redirect BOTH streams to a log file you can read on a later turn: \`long-command > /tmp/job.log 2>&1 &\`. That redirect is the entire cure.
-- \`nohup\` alone does NOT work (it ignores SIGHUP, the wrong signal, and only auto-redirects when stdout is a terminal — here it is a pipe). \`setsid\` alone does NOT work (nothing is signalling the process group). Neither is a substitute for redirecting.
-- If the job's completion should CONTINUE your work, arm BOTH legs before the turn ends: (1) a detached WATCHER that reports the outcome — exit code + log path — back to this thread, and (2) a schedule_trigger DEADLINE that fires if the watcher itself died. Never end a turn saying you will check back later — you get no turn of your own. Do NOT poll (no cron re-checks; each fire burns a full turn).
-- Retrieve the exact watcher one-liner with search_content("watcher recipe") BEFORE writing one — the watcher is itself background work with its own redirect requirement, and it is a script, not an agent: act on its report; do NOT send_to_agent a reply back to it.
+Your turn runs in a subprocess that EXITS when the turn ends, so a command you background yourself (\`cmd &\`) is killed as soon as you stop talking — on EVERY turn, not occasionally.
+- For work that outlasts your turn, use run_job(command, label). The gateway owns it, so it has no pipes to break and is not in your turn's process tree. When it exits you are woken with its exit code and the tail of its output.
+- Write the command plainly: no \`&\`, no \`> log 2>&1\`, no \`nohup\`/\`setsid\`. run_job already does all of that, and adding your own breaks the log capture.
+- Then END YOUR TURN. Do not wait for it and do not poll — you are woken automatically. Use list_jobs and job_log(id) to check on a running job, cancel_job(id) to stop one.
+- A command that finishes in a couple of minutes needs none of this — run it in the foreground and wait.
 
 TOOLS:
 - read_file: Read any file (text or PDF) — returns a summary + chunk table-of-contents and stores the content for future search. Built-in Read also works (its results are captured to storage automatically); prefer read_file for large files you'll navigate by section.
@@ -236,35 +237,20 @@ Example:
 3. **Never fix inline** — a bug unrelated to your current task gets its own todo, not a context switch
 4. **Reflect redirects** — update the todo list before executing on a user-directed change of plan
 5. **Debugging** — treat each hypothesis as a todo; test systematically rather than chasing the first suspicious lead`;
-/**
- * Watcher recipe (task 116): the copy-pasteable background-job continuation
- * one-liner, moved out of the per-turn template into the content store. The
- * template keeps the invariants (SIGPIPE mechanism, redirect cure, two-leg
- * rule, nohup/setsid negations) plus a search_content("watcher recipe")
- * pointer; this doc carries the worked example, seeded into every thread's
- * store with {threadName} resolved (task 109: the inject must address this
- * thread by exact name). Reads the gateway config through $CUMULUS_DIR, never
- * a hard-coded ~/.cumulus (task 111: sandboxed runs must fail closed).
+/*
+ * The watcher recipe (task 116) lived here: a copy-pasteable detached
+ * watcher + schedule_trigger deadline, seeded into every thread's content
+ * store. Deleted by task 139, not deprecated — `run_job` makes the mechanism
+ * it taught unnecessary, and two ways to do background work is precisely the
+ * shape Rule #8 forbids.
+ *
+ * It was also actively harmful where it could not be run: topic-free
+ * operational boilerplate is what survives the relevance floor when a query
+ * matches nothing else, so on an app's visitor thread it was retrieved for
+ * unrelated questions and taught a model to POST to the gateway carrying the
+ * admin key (task 134). Task 134 gated the seeding; this removes the thing
+ * being seeded.
  */
-export const WATCHER_RECIPE_TEMPLATE = `Watcher recipe — how a detached background job continues your work when it finishes (background job continuation).
-
-Use when a long-running command's completion should wake this thread. Arm BOTH legs before the turn ends.
-
-LEG 1 — WATCHER (wakes you the moment the job finishes). The job and its reporter run as one detached group with its OWN redirect — without it the watcher dies by SIGPIPE before it can report:
-
-{ npm run build > /tmp/job.log 2>&1; rc=$?; C=\${CUMULUS_DIR:-$HOME/.cumulus}/gateway.config.json; curl -s -X POST "http://127.0.0.1:$(jq -r '.port // 8090' $C)/api/agents/inject" -H "X-API-Key: $(jq -r '.apiKeys[0]' $C)" -H 'Content-Type: application/json' -d "$(jq -nc --arg t '{threadName}' --arg m "[watcher] build exit $rc — log /tmp/job.log. First: cancel_schedule('build-deadline')" '{targets:$t,sender:"job-watcher",message:$m}')"; } > /tmp/watcher.log 2>&1 &
-
-Replace \`npm run build\` with your command; keep every redirect exactly as shown. Fire on OUTCOME, not on success — the report carries the exit code and the log path so the woken turn can diagnose without re-running. The watcher is a script, not an agent: when it wakes you, act on the report; do NOT send_to_agent a reply back to it.
-
-LEG 2 — DEADLINE (guarantees a wake even if the watcher itself died; stored in thread config, so it survives gateway restarts, which the watcher does not):
-
-schedule_trigger({id: "build-deadline", trigger: "once", at: "<ISO 8601, generously past the expected finish>", message: "The build watcher never reported back. Check /tmp/job.log and whether the process is still alive."})
-
-cancel_schedule("build-deadline") as your first action when the watcher wakes you.`;
-/** Resolve the watcher recipe for a specific thread. */
-export function renderWatcherRecipe(threadName) {
-    return WATCHER_RECIPE_TEMPLATE.replace(/\{threadName\}/g, threadName);
-}
 /**
  * Committed budget for the resolved static template, in estimateTokens units
  * (task 117). The template reached ~3,400 tokens/turn by accretion — every
@@ -278,35 +264,26 @@ export function renderWatcherRecipe(threadName) {
  */
 export const STATIC_PROMPT_TOKEN_BUDGET = 1_800;
 /**
- * In-flight/done seeding memo. store()'s content-hash dedupe is check-then-write,
- * so two CONCURRENT seeds of the same thread would both pass the check and write
- * twice — the memo makes the second caller join the first's promise instead.
- * Cleared alongside the thread cache so a deleted-then-recreated thread re-seeds;
- * across process restarts the content-hash dedupe makes re-seeding free.
- */
-const seededRecipes = new Map();
-/**
- * Can this thread actually run the background-work recipe? (task 134)
+ * Can this thread actually run background work? (task 134, retained by 139)
  *
- * The watcher recipe IS a detached shell one-liner, so a thread whose merged
- * config denies `Bash` cannot execute it. Seeding it there buys nothing and
- * costs something real: the recipe is topic-free operational boilerplate, and
- * on a store holding a handful of an app's tool results it is what survives the
- * relevance floor whenever the user's question matches nothing else — measured
- * on cdda's visitor threads at 2 of 24 turns, and on turn 1 of every thread it
- * is the only stored item there is. What it then teaches a visitor's assistant
- * is an HTTP call carrying the gateway's admin key, to a model whose only
- * honest route is its own tools.
+ * Originally the gate on seeding the watcher recipe; now the gate on the
+ * `run_job` endpoint, which is the same question with more at stake. run_job
+ * is arbitrary shell execution reached over MCP, so without this a thread whose
+ * merged config denies `Bash` could run shell anyway — the bypass class task
+ * 136 closed for `read_file`.
  *
  * Same derived-predicate shape as task 110's isAgentSender: the condition that
- * decides whether the recipe is USEFUL is the condition that decides whether it
- * is seeded, so the two cannot drift apart. Deliberately NOT keyed on namespace
- * — see the task doc; namespace is a proxy, executability is the property, and
+ * decides whether background work is USEFUL is the condition that decides
+ * whether it is offered, so the two cannot drift apart. Deliberately NOT keyed
+ * on namespace — namespace is a proxy, executability is the property, and
  * keying on it would need an option threaded through six spawn paths that fails
  * OPEN when one is missed (task 113 is the recorded case of exactly that).
  */
 export function canRunBackgroundWork(config) {
-    return !(config.disallowedTools ?? []).includes('Bash');
+    // Task 137: delegates to toolAvailable so an ALLOWLIST is honoured too. Checking
+    // disallowedTools alone (as this did originally) reports true for a thread whose
+    // allowlist simply omits Bash — it would still be seeded a recipe it cannot run.
+    return toolAvailable('Bash', config);
 }
 /**
  * Root confinement for the `read_file` MCP tool (task 136).
@@ -334,38 +311,6 @@ export function readFileRootsForThread(config, threadCwd) {
         return undefined;
     return [threadCwd];
 }
-/**
- * Seed situational recipe docs into a thread's content store (task 116), gated
- * on the thread being able to run them (task 134).
- *
- * Called from sendMessage, NOT getOrCreateThread: the latter has 14 call sites,
- * several of them store-only paths (transcript ingest, history reads, thread
- * listing) that never spawn a turn, and it sees only the thread config —
- * a global `disallowedTools` would be invisible to a gate placed there.
- * The gate lives HERE rather than at the call site so a future caller cannot
- * reintroduce the seed by forgetting to check. Errors never fail the turn.
- */
-export function seedThreadRecipes(content, threadName, config) {
-    if (!canRunBackgroundWork(config))
-        return Promise.resolve();
-    const existing = seededRecipes.get(threadName);
-    if (existing)
-        return existing;
-    const seeding = (async () => {
-        try {
-            await content.store(renderWatcherRecipe(threadName), {
-                sourceType: 'reference',
-                summary: `Watcher recipe: detached background-job continuation one-liner (watcher + schedule_trigger deadline) for thread ${threadName}`,
-                metadata: { recipe: 'watcher' },
-            });
-        }
-        catch (err) {
-            console.error('[Gateway] Failed to seed thread recipes:', err);
-        }
-    })();
-    seededRecipes.set(threadName, seeding);
-    return seeding;
-}
 // ─── Thread cache ────────────────────────────────────────────────────────────
 const threadCache = new Map();
 // ─── Helper Functions ────────────────────────────────────────────────────────
@@ -682,8 +627,11 @@ export function generateSystemPrompt(count, tokens, sessionId, recentMessages, r
         .replace('{alwaysIncludeContext}', alwaysIncludeContext)
         .replace('{recentContext}', recentContext)
         .replace('{retrievedContext}', retrievedContext)
-        // Task 109: the watcher recipe must POST to /api/agents/inject with this thread's
-        // exact name. Global — a custom template may reference it more than once.
+        // Added by task 109 so the watcher recipe could address this thread by exact
+        // name. That recipe is gone (task 139) and the shipped template no longer uses
+        // the placeholder, but a CUSTOM template legitimately might — so the
+        // substitution stays and only the reason changes. Global: a template may
+        // reference it more than once.
         .replace(/\{threadName\}/g, threadName ?? 'this thread'));
 }
 /**
@@ -717,6 +665,35 @@ export function claudeModelArgs(threadPin, claudeModels) {
     const resolved = resolveClaudeModel(threadPin, claudeModels);
     return resolved ? ['--model', resolved] : [];
 }
+/**
+ * The full argv for a Claude CLI turn (task 137).
+ *
+ * Extracted from `sendMessage` for the same reason as `claudeModelArgs` and
+ * `resolveClaudeModel` above: the composition was only assertable by spawning a real
+ * subprocess, so nothing verified that a compiled allowlist actually reaches the spawn.
+ *
+ * Owns one measured invariant: `--disallowedTools` is VARIADIC, so it must come last.
+ * A positional argument after it is silently swallowed as extra deny rules — measured
+ * (`Permission deny rule "the" matches no known tool`). Append nothing after this call.
+ */
+export function claudeSpawnArgs(params) {
+    return appendDisallowedToolsArgs([
+        '--print',
+        '--verbose',
+        '--output-format',
+        'stream-json',
+        '--permission-mode',
+        'bypassPermissions',
+        '--mcp-config',
+        params.mcpConfigPath,
+        '--strict-mcp-config',
+        '--input-format',
+        'stream-json',
+        '--effort',
+        params.effort || 'high',
+        ...claudeModelArgs(params.claudeModel, params.claudeModels),
+    ], params.disallowedTools);
+}
 /**
  * Resolve which provider a thread's model id runs on (task 118) — pure, same
  * rationale as `resolveClaudeModel`: the decision must be assertable without
@@ -1029,15 +1006,13 @@ export async function getOrCreateThread(threadName, _basePath) {
     // config — the gate needs the merged one.
     return thread;
 }
-/** Clear a thread from the cache (and its recipe-seed memo, so recreation re-seeds) */
+/** Clear a thread from the cache */
 export function clearThreadCache(threadName) {
     if (threadName) {
         threadCache.delete(threadName);
-        seededRecipes.delete(threadName);
     }
     else {
         threadCache.clear();
-        seededRecipes.clear();
     }
 }
 // ─── Core Pipeline ───────────────────────────────────────────────────────────
@@ -1231,11 +1206,12 @@ export async function sendMessage(options) {
     const globalConfig = await loadGlobalConfig();
     const threadConfig = await loadThreadConfig(threadName);
     const mergedConfig = mergeConfigs(globalConfig, threadConfig);
-    // Task 116 puts the watcher recipe in the store rather than the per-turn
-    // template; task 134 gates it on this thread being able to run it. Here and
-    // not in getOrCreateThread: this is the single funnel every spawning path
-    // passes through, and the only place the MERGED config exists.
-    void seedThreadRecipes(thread.content, threadName, mergedConfig);
+    // Task 137: drop prompt sections whose tools this thread cannot reach. Computed ONCE
+    // here because three sites consume the template (the prompt, the 088 capture, and the
+    // direct-provider instructions) — task 113's lesson is that per-site derivation is how
+    // one of N paths silently keeps the old behaviour. Identity when nothing is gated, so
+    // every thread on this box is byte-unaffected.
+    const gatedPromptTemplate = applyCapabilityGates(systemPromptTemplate ?? SYSTEM_PROMPT_TEMPLATE, promptCapabilities(mergedConfig));
     const alwaysInclude = await readAlwaysIncludeFiles(mergedConfig, threadCwd);
     // ── Model resolution ──
     // Thread config `model` field determines the execution path:
@@ -1320,7 +1296,7 @@ export async function sendMessage(options) {
         systemPromptLength: 0,
     };
     // ── System prompt ──
-    const systemPrompt = generateSystemPrompt(stats.count, stats.totalTokens, thread.session.getSessionId(), recentConversation, retrievedContext, alwaysInclude.formattedContext, systemPromptTemplate, threadName);
+    const systemPrompt = generateSystemPrompt(stats.count, stats.totalTokens, thread.session.getSessionId(), recentConversation, retrievedContext, alwaysInclude.formattedContext, gatedPromptTemplate, threadName);
     // Update debug with system prompt breakdown
     const systemPromptTokens = estimateTokens(systemPrompt);
     const ragTokens = retrievalResult ? estimateTokens(retrievedContext) : 0;
@@ -1339,7 +1315,7 @@ export async function sendMessage(options) {
     // ── Prompt capture (task 088): assemble the observable sections ──
     // The pure template (empty components) isolates the instructions section.
     const promptCaptureEnabled = mergedConfig.promptCapture !== false;
-    const instructionsOnly = generateSystemPrompt(stats.count, stats.totalTokens, thread.session.getSessionId(), [], '', '', systemPromptTemplate, threadName);
+    const instructionsOnly = generateSystemPrompt(stats.count, stats.totalTokens, thread.session.getSessionId(), [], '', '', gatedPromptTemplate, threadName);
     // CLI shape by default; the HF branch overwrites path/instructions below.
     let promptCapturePath = 'cli';
     const promptCaptureSections = [
@@ -1544,7 +1520,7 @@ export async function sendMessage(options) {
         // Build system blocks with cache_control on static parts
         const systemBlocks = [];
         // Static instructions (cacheable)
-        const instructions = systemPromptTemplate || SYSTEM_PROMPT_TEMPLATE;
+        const instructions = gatedPromptTemplate;
         const modelIdentity = `You are ${threadModel}. You are NOT Claude, NOT Opus, NOT Sonnet, NOT Haiku. If asked what model you are, say "${threadModel}". `;
         const hfInstructionsText = modelIdentity +
             instructions
@@ -1788,7 +1764,13 @@ export async function sendMessage(options) {
                 tools: coreTools,
                 mcpTools: mcpToolEntries,
                 eagerAll: true, // HF models can't use ToolSearch — send all tools upfront
-                disallowedTools: mergedConfig.disallowedTools,
+                // Task 137: the allowlist must bind here too, or a visitor thread pinned to an
+                // HF/OpenAI model would silently ignore it — the fail-open, one-of-N-paths gap
+                // task 113 recorded. No probe needed on this path: the registry about to be
+                // handed to the loop IS the exact inventory.
+                disallowedTools: mergedConfig.allowedTools
+                    ? compileDisallowedTools(mergedConfig.allowedTools, [...coreTools, ...mcpToolEntries].map(t => t.definition.name), mergedConfig.disallowedTools)
+                    : mergedConfig.disallowedTools,
                 onToken: (() => {
                     // Filter suppresses <todo> and <thinking> blocks from reaching the frontend (unless debug)
                     const filter = onToken
@@ -1878,22 +1860,22 @@ export async function sendMessage(options) {
     }
     // ── Local execution: spawn Claude on this machine ──
     const mcpConfigPath = generateMcpConfig(thread.threadPath, thread.session.getSessionId(), threadName, sharedMcpPort, extraMcpServers, gatewayAgentsConfig, readFileRootsForThread(mergedConfig, threadCwd));
-    const args = appendDisallowedToolsArgs([
-        '--print',
-        '--verbose',
-        '--output-format',
-        'stream-json',
-        '--permission-mode',
-        'bypassPermissions',
-        '--mcp-config',
+    const resolvedClaudePath = claudePath || resolveClaudeCli();
+    // Task 137: compile an allowlist into --disallowedTools. The CLI's --allowedTools is an
+    // AUTO-APPROVE list and does not restrict (measured under three permission modes), and
+    // under --permission-mode bypassPermissions it is a guaranteed no-op — so deny-by-default
+    // has to be expressed as a denylist. Gated on the field being present so an unconstrained
+    // thread never pays for the inventory probe (a subprocess spawn) at all.
+    const effectiveDisallowedTools = mergedConfig.allowedTools
+        ? compileDisallowedTools(mergedConfig.allowedTools, await getToolInventory(resolvedClaudePath), mergedConfig.disallowedTools)
+        : mergedConfig.disallowedTools;
+    const args = claudeSpawnArgs({
         mcpConfigPath,
-        '--strict-mcp-config',
-        '--input-format',
-        'stream-json',
-        '--effort',
-        mergedConfig.effort || 'high',
-        ...claudeModelArgs(mergedConfig.claudeModel, options.claudeModels),
-    ], mergedConfig.disallowedTools);
+        effort: mergedConfig.effort,
+        claudeModel: mergedConfig.claudeModel,
+        claudeModels: options.claudeModels,
+        disallowedTools: effectiveDisallowedTools,
+    });
     // Filter out Claude env vars
     const cleanEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('CLAUDE') && key !== 'CLAUDECODE'));
     // Check if already aborted
@@ -1901,7 +1883,6 @@ export async function sendMessage(options) {
         cleanupMcpConfig(mcpConfigPath);
         throw new Error('Aborted');
     }
-    const resolvedClaudePath = claudePath || resolveClaudeCli();
     // Windows: spawn() can't directly exec .cmd/.bat shims (CVE-2024-27980).
     // Routing through cmd.exe is required for the npm-installed `claude.cmd`.
     const isWindowsShim = process.platform === 'win32' && /\.(cmd|bat|ps1)$/i.test(resolvedClaudePath);