clembot-doorman 0.1.0

package/doorman/cli/harness.mjs

@@ -0,0 +1,179 @@
+/**
+ * Runs INSIDE the sandbox container. One task, one agent loop, one JSON line out.
+ *
+ * This file is copied into both images verbatim, so the only thing that differs
+ * between arms is whether the candidate's install layer is present. It reads the
+ * task on stdin and prints exactly one metrics object on stdout as its last line.
+ *
+ * Success is decided by the task's own machine-checkable conditions, never by
+ * asking a model whether it did well. A model-judged pass is not comparable
+ * across arms: the judge sees a different transcript each time, and its leniency
+ * is one more thing varying between the two numbers you are trying to compare.
+ */
+
+const MODEL = process.env.DOORMAN_MODEL || 'claude-sonnet-5';
+const KEY = process.env.ANTHROPIC_API_KEY;
+// Derived from the money, not a constant. cli/cost.mjs works out how many
+// turns fit the ceiling and passes it in; 24 is only the fallback when this
+// file is run outside that orchestration.
+const MAX_TURNS = Number(process.env.DOORMAN_MAX_TURNS) || 24;
+
+/** Priced per million tokens. Unknown model means cost is reported as null. */
+const PRICES = {
+  'claude-sonnet-5': { in: 3, out: 15 },
+  'claude-opus-5': { in: 15, out: 75 },
+  'claude-haiku-4-5-20251001': { in: 1, out: 5 },
+};
+
+const files = new Map();
+
+const TOOLS = [
+  {
+    name: 'write_file',
+    description: 'Write text to a file in the working directory.',
+    input_schema: {
+      type: 'object',
+      properties: { path: { type: 'string' }, content: { type: 'string' } },
+      required: ['path', 'content'],
+    },
+  },
+  {
+    name: 'read_file',
+    description: 'Read a file previously written in this session.',
+    input_schema: {
+      type: 'object',
+      properties: { path: { type: 'string' } },
+      required: ['path'],
+    },
+  },
+  {
+    name: 'fetch_url',
+    description: 'HTTP GET a public URL and return the response body as text.',
+    input_schema: {
+      type: 'object',
+      properties: { url: { type: 'string' } },
+      required: ['url'],
+    },
+  },
+];
+
+async function callTool(name, input) {
+  if (name === 'write_file') {
+    files.set(String(input.path), String(input.content ?? ''));
+    return `wrote ${input.path} (${String(input.content ?? '').length} chars)`;
+  }
+  if (name === 'read_file') {
+    return files.has(String(input.path)) ? files.get(String(input.path)) : 'ENOENT';
+  }
+  if (name === 'fetch_url') {
+    try {
+      const r = await fetch(String(input.url), { redirect: 'follow' });
+      const t = await r.text();
+      return `HTTP ${r.status}\n${t.slice(0, 20000)}`;
+    } catch (e) {
+      // With --network none this is the expected path, and it is reported to the
+      // model rather than crashing the run: how an agent copes with a dead
+      // network is itself part of what the two arms are being compared on.
+      return `fetch failed: ${e.message}`;
+    }
+  }
+  return `unknown tool ${name}`;
+}
+
+/** Every condition must be machine-checkable. See the note at the top. */
+function checkSuccess(success) {
+  const failures = [];
+  const all = [...files.values()].join('\n');
+
+  if (success.file_exists) {
+    const want = Array.isArray(success.file_exists) ? success.file_exists : [success.file_exists];
+    for (const f of want) if (!files.has(f)) failures.push(`missing file ${f}`);
+  }
+  if (success.contains) {
+    const want = Array.isArray(success.contains) ? success.contains : [success.contains];
+    for (const s of want) {
+      if (!all.toLowerCase().includes(String(s).toLowerCase())) failures.push(`missing text "${s}"`);
+    }
+  }
+  if (success.sections) {
+    const n = Number(success.sections);
+    const found = (all.match(/^#{1,6} /gm) || []).length;
+    if (found < n) failures.push(`wanted ${n} sections, found ${found}`);
+  }
+  return failures;
+}
+
+async function main() {
+  const raw = await new Promise((res) => {
+    let b = ''; process.stdin.setEncoding('utf8');
+    process.stdin.on('data', (d) => { b += d; });
+    process.stdin.on('end', () => res(b));
+  });
+  const task = JSON.parse(raw);
+
+  if (!KEY) {
+    // Invariant 9: a missing key stops the run and says so. It never produces a
+    // number, because a fabricated benchmark is worse than no benchmark.
+    console.log(JSON.stringify({
+      success: false, blocked: true,
+      error: 'ANTHROPIC_API_KEY is not set inside the sandbox, so no run happened.',
+    }));
+    return;
+  }
+
+  const messages = [{ role: 'user', content: task.prompt }];
+  let turns = 0, toolCalls = 0, inTok = 0, outTok = 0, stop = 'max_turns';
+
+  while (turns < MAX_TURNS) {
+    turns++;
+    const res = await fetch('https://api.anthropic.com/v1/messages', {
+      method: 'POST',
+      headers: {
+        'content-type': 'application/json',
+        'x-api-key': KEY,
+        'anthropic-version': '2023-06-01',
+      },
+      body: JSON.stringify({ model: MODEL, max_tokens: 4096, temperature: 0, tools: TOOLS, messages }),
+    });
+    if (!res.ok) {
+      console.log(JSON.stringify({
+        success: false, turns, tool_calls: toolCalls,
+        error: `model call failed: HTTP ${res.status} ${(await res.text()).slice(0, 300)}`,
+      }));
+      return;
+    }
+    const body = await res.json();
+    inTok += body.usage?.input_tokens ?? 0;
+    outTok += body.usage?.output_tokens ?? 0;
+    messages.push({ role: 'assistant', content: body.content });
+
+    const uses = (body.content || []).filter((c) => c.type === 'tool_use');
+    if (!uses.length) { stop = 'end_turn'; break; }
+
+    const results = [];
+    for (const u of uses) {
+      toolCalls++;
+      results.push({ type: 'tool_result', tool_use_id: u.id, content: await callTool(u.name, u.input || {}) });
+    }
+    messages.push({ role: 'user', content: results });
+  }
+
+  const failures = checkSuccess(task.success || {});
+  const p = PRICES[MODEL];
+  console.log(JSON.stringify({
+    success: failures.length === 0,
+    failures,
+    turns,
+    tool_calls: toolCalls,
+    tokens: inTok + outTok,
+    input_tokens: inTok,
+    output_tokens: outTok,
+    cost_usd: p ? (inTok / 1e6) * p.in + (outTok / 1e6) * p.out : null,
+    stop_reason: stop,
+    files_written: [...files.keys()],
+  }));
+}
+
+main().catch((e) => {
+  console.log(JSON.stringify({ success: false, error: String(e && e.message ? e.message : e) }));
+});

package/doorman/cli/install.mjs

@@ -0,0 +1,175 @@
+/**
+ * `doorman install [target]` — install the gate, subagent, slash command,
+ * and skill into a target project.
+ *
+ * Cross-platform Node implementation of install.sh. Zero runtime dependencies.
+ *
+ * It copies:
+ *   - .claude/hooks/mcp-gate.sh
+ *   - agents/doorman.md
+ *   - commands/vet.md
+ *   - skills/doorman-guide/SKILL.md
+ *   - registry/allowlist.json & denylist.json (NEVER overwrites an existing registry)
+ *
+ * And safely inspects/wires .claude/settings.json without destructive clobbering.
+ */
+
+import { existsSync, mkdirSync, copyFileSync, readFileSync, writeFileSync } from 'node:fs';
+import { resolve, join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const ROOT = resolve(HERE, '..');
+
+export const GATE_HOOK_CONFIG = {
+  matcher: 'mcp__.*',
+  hooks: [{
+    type: 'command',
+    command: '$CLAUDE_PROJECT_DIR/.claude/hooks/mcp-gate.sh',
+    timeout: 5,
+  }],
+};
+
+export async function install(targetDir = process.cwd(), { dryRun = false } = {}) {
+  const absTarget = resolve(targetDir);
+  if (!existsSync(absTarget)) {
+    return { ok: false, why: `target directory does not exist: ${absTarget}` };
+  }
+
+  const copied = [];
+  const kept = [];
+  const warnings = [];
+
+  const copy = (srcRel, destRel) => {
+    const src = join(ROOT, srcRel);
+    const dest = join(absTarget, destRel);
+    if (!existsSync(src)) {
+      warnings.push(`source file not found: ${srcRel}`);
+      return;
+    }
+    if (dryRun) {
+      copied.push(`${srcRel} -> ${destRel}`);
+      return;
+    }
+    mkdirSync(dirname(dest), { recursive: true });
+    copyFileSync(src, dest);
+    copied.push(destRel);
+  };
+
+  // 1. Hooks, Subagent, Command
+  copy('.claude/hooks/mcp-gate.sh', '.claude/hooks/mcp-gate.sh');
+  copy('agents/doorman.md', '.claude/agents/doorman.md');
+  copy('commands/vet.md', '.claude/commands/vet.md');
+
+  // 2. Doorman Skill (both in .claude/skills and .agents/skills if applicable)
+  copy('skills/doorman-guide/SKILL.md', '.claude/skills/doorman-guide/SKILL.md');
+  if (existsSync(join(absTarget, '.agents'))) {
+    copy('skills/doorman-guide/SKILL.md', '.agents/skills/doorman-guide/SKILL.md');
+  }
+
+  // 3. Registry (never overwrite existing)
+  let registryStatus = 'copied';
+  const targetAllowlist = join(absTarget, 'registry', 'allowlist.json');
+  if (existsSync(targetAllowlist)) {
+    registryStatus = 'kept';
+    kept.push('registry/allowlist.json (existing trust list preserved)');
+  } else if (!dryRun) {
+    mkdirSync(join(absTarget, 'registry'), { recursive: true });
+    copyFileSync(join(ROOT, 'registry', 'allowlist.json'), targetAllowlist);
+    copyFileSync(join(ROOT, 'registry', 'denylist.json'), join(absTarget, 'registry', 'denylist.json'));
+    writeFileSync(join(absTarget, 'registry', 'ledger.jsonl'), '', 'utf8');
+    copied.push('registry/ (allowlist.json, denylist.json, ledger.jsonl)');
+  } else {
+    copied.push('registry/ (allowlist.json, denylist.json, ledger.jsonl)');
+  }
+
+  // 4. Inspect & wire settings.json
+  const settingsPath = join(absTarget, '.claude', 'settings.json');
+  let settingsWired = false;
+  let settingsStatus = 'missing';
+
+  if (existsSync(settingsPath)) {
+    try {
+      const raw = readFileSync(settingsPath, 'utf8');
+      if (raw.includes('mcp-gate.sh')) {
+        settingsWired = true;
+        settingsStatus = 'already-wired';
+      } else {
+        const settings = JSON.parse(raw);
+        if (!settings.hooks) settings.hooks = {};
+        if (!Array.isArray(settings.hooks.PreToolUse)) settings.hooks.PreToolUse = [];
+        settings.hooks.PreToolUse.push(GATE_HOOK_CONFIG);
+        if (!dryRun) {
+          writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + '\n', 'utf8');
+        }
+        settingsWired = true;
+        settingsStatus = 'wired';
+      }
+    } catch {
+      settingsStatus = 'unparseable';
+      warnings.push('.claude/settings.json exists but is not valid JSON. Wire hook manually.');
+    }
+  } else if (!dryRun) {
+    mkdirSync(dirname(settingsPath), { recursive: true });
+    const initialSettings = {
+      hooks: {
+        PreToolUse: [GATE_HOOK_CONFIG],
+      },
+    };
+    writeFileSync(settingsPath, JSON.stringify(initialSettings, null, 2) + '\n', 'utf8');
+    settingsWired = true;
+    settingsStatus = 'created-and-wired';
+  }
+
+  return {
+    ok: true,
+    target: absTarget,
+    dryRun,
+    copied,
+    kept,
+    warnings,
+    registryStatus,
+    settingsWired,
+    settingsStatus,
+  };
+}
+
+export function renderInstall(res) {
+  const L = [];
+  L.push('');
+  L.push(`doorman install -> ${res.target}`);
+  if (res.dryRun) L.push('  (dry run: nothing was written)');
+  L.push('');
+
+  L.push('## Files installed');
+  for (const c of res.copied) L.push(`  + ${c}`);
+  for (const k of res.kept) L.push(`  = ${k}`);
+  L.push('');
+
+  L.push('## Gate & Hook Wiring');
+  if (res.settingsWired) {
+    L.push(`  [PASS] Hook is wired in .claude/settings.json (${res.settingsStatus})`);
+  } else {
+    L.push(`  [WARN] Hook is NOT wired in .claude/settings.json (${res.settingsStatus})`);
+    L.push('         Add the PreToolUse hook manually to activate protection.');
+  }
+  L.push('');
+
+  L.push('## Registry Trust List');
+  if (res.registryStatus === 'kept') {
+    L.push('  Your existing trust list was preserved. What is trusted has not changed.');
+  } else {
+    L.push('  Initial registry deployed with baseline trust list.');
+    L.push('  Everything else your agent reaches for is blocked until vetted: /vet <url>');
+  }
+  L.push('');
+
+  if (res.warnings.length) {
+    L.push('## Warnings');
+    for (const w of res.warnings) L.push(`  ! ${w}`);
+    L.push('');
+  }
+
+  L.push('Done. Run `doorman doctor` to verify your installation.');
+  return L.join('\n');
+}

package/doorman/cli/needs.mjs

@@ -0,0 +1,116 @@
+/**
+ * `doorman needs` - the CLI half. Reads the history, fetches the free feed,
+ * and hands both to the pure functions in ../src/needs.mjs.
+ *
+ * The only request made here is the same anonymous `GET /feed` that `watch`
+ * makes. The prompts are read, matched and discarded locally. Nothing about
+ * this build is sent anywhere, which is the whole reason the expensive half of
+ * the product is the shared grade and the cheap half is the private fit.
+ */
+
+import { readFileSync, existsSync } from 'node:fs';
+import { inventoryFor } from '../src/inventory.mjs';
+import { installedKeys, serverKey } from './watch.mjs';
+import {
+  readPrompts, historyDirFor, suggest, renderNeeds, NEEDS,
+} from '../src/needs.mjs';
+
+export const DEFAULT_API = 'https://scorecard.wanessalabs.com';
+
+/**
+ * Candidates from a local `doorman discover` sweep, if one has been run.
+ *
+ * These are UNGRADED by construction: discover scans published text and never
+ * drives anything, and it writes `graded: false` on every row. They are
+ * included because a need with only ungraded options is still better
+ * information than a need with none, and every row says which it is.
+ */
+export function readCandidateFile(file) {
+  if (!file || !existsSync(file)) return [];
+  try {
+    const raw = JSON.parse(readFileSync(file, 'utf8'));
+    const rows = Array.isArray(raw) ? raw : (raw.candidates ?? []);
+    return rows.map((c) => ({ ...c, source: c.source ?? 'candidates' }));
+  } catch {
+    return [];
+  }
+}
+
+export async function needs({
+  root, api, historyDir, candidateFile, limit = 200, fetchImpl = fetch,
+} = {}) {
+  const inv = inventoryFor(root);
+  const installed = installedKeys(inv);
+
+  const dir = historyDir ?? historyDirFor(root);
+  let prompts = [];
+  let historyNote;
+  if (!dir) {
+    historyNote = 'No readable prompt history for this path. Claude Code keeps it under ' +
+      '~/.claude/projects/<path-with-dashes>; other harnesses keep none that doorman can read. ' +
+      'Pass --history DIR if yours lives elsewhere.';
+  } else {
+    prompts = readPrompts(dir);
+    historyNote = `history: ${dir}`;
+  }
+
+  // The feed is free and anonymous. A failure here is "could not measure",
+  // never a reason to invent a suggestion, so an empty candidate list flows
+  // straight through to a GAP line that says so.
+  let feed = [];
+  let feedNote = null;
+  try {
+    const url = new URL('/feed', api);
+    url.searchParams.set('limit', String(limit));
+    const res = await fetchImpl(url.toString(), { headers: { accept: 'application/json' } });
+    if (res.ok) {
+      const body = await res.json();
+      feed = (body.candidates ?? []).map((c) => ({ ...c, source: 'feed' }));
+    } else {
+      feedNote = `feed returned HTTP ${res.status}; candidates below are local only`;
+    }
+  } catch (e) {
+    feedNote = `feed unreachable (${e.message}); candidates below are local only`;
+  }
+
+  const candidates = [...feed, ...readCandidateFile(candidateFile)];
+  const installedUrls = new Set([...installed]);
+
+  const report = suggest({
+    prompts,
+    inventory: inv,
+    candidates,
+    // rankCandidates compares against whatever url a candidate carries, so the
+    // installed set has to be keyed the same way the inventory was.
+    installed: new Set([...candidates.map((c) => c.server_url).filter(Boolean)]
+      .filter((u) => installedUrls.has(serverKey(u)))),
+  });
+
+  return {
+    ...report,
+    api,
+    history_dir: dir,
+    history_note: historyNote,
+    feed_note: feedNote,
+    feed_rows: feed.length,
+    local_candidates: candidates.length - feed.length,
+    taxonomy: NEEDS.map((n) => n.id),
+    inventory: {
+      root: inv.root ?? root,
+      coverage: inv.coverage ?? { agents: 'unknown', skills: 'unknown', mcpServers: 'unknown' },
+      mcp_servers: (inv.mcpServers ?? []).length,
+    },
+  };
+}
+
+export function render(r) {
+  const head = [r.history_note, r.feed_note,
+    `feed: ${r.feed_rows} graded rows` + (r.local_candidates ? `, plus ${r.local_candidates} local` : ''),
+  ].filter(Boolean).join('\n  ');
+  let out = renderNeeds(r, { historyNote: head });
+  if (r.inventory.coverage.mcpServers !== 'read') {
+    out += '\n\n!! No MCP config was readable here, so "already covered" below is a' +
+           '\n   weaker claim than it looks: a server you have may be invisible to this run.';
+  }
+  return out;
+}

package/doorman/cli/report.mjs

@@ -0,0 +1,89 @@
+/**
+ * `doorman report <link>` — L1, the static implementation report.
+ *
+ * This layer already existed. `mcp-scorecard/runner/run.mjs --once --static-only`
+ * wraps the `mcpscore` CLI, runs the scan-only injection probe, and writes
+ * grade.json / report.md / recipe.md / transcripts.jsonl to an output directory.
+ * What was missing was a name and a stable interface, not the measurement.
+ *
+ * So this shells to it rather than reimplementing it. Invariant 2 in CLAUDE.md
+ * is about exactly this: two implementations drift, and the day they disagree,
+ * the one you trusted is whichever you happened to run.
+ *
+ * L1 needs NO model key. That matters more than it sounds: it means every
+ * candidate can get a real static report today, and the pipeline degrades to
+ * "measured statically, behaviourally unmeasured" rather than to nothing.
+ */
+
+import { execFile } from 'node:child_process';
+import { readFile, mkdir } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+/**
+ * Where the static runner lives.
+ *
+ * The first version hardcoded `../../mcp-scorecard/runner/run.mjs`, which is
+ * true inside this repo and false the moment the package is installed on its
+ * own. Since the whole point is that someone else installs this, the lookup
+ * now tries the places it could legitimately be and reports every one it
+ * checked when it finds none. A path that is right in the author's checkout
+ * and wrong everywhere else is the classic giveaway bug.
+ */
+const RUNNER_CANDIDATES = [
+  process.env.DOORMAN_SCORECARD_RUNNER,                       // explicit wins
+  path.resolve(HERE, '..', '..', 'mcp-scorecard', 'runner', 'run.mjs'),   // in-repo
+  path.resolve(HERE, '..', 'vendor', 'scorecard-runner', 'run.mjs'),      // vendored
+  path.resolve(process.cwd(), 'mcp-scorecard', 'runner', 'run.mjs'),      // cwd is the repo
+].filter(Boolean);
+
+const RUNNER = RUNNER_CANDIDATES.find((p) => existsSync(p)) ?? RUNNER_CANDIDATES[1];
+
+export async function report({ link, out, neededFor, log }) {
+  if (!existsSync(RUNNER)) {
+    return {
+      ok: false,
+      why:
+        'the scorecard static runner was not found. doorman report delegates the\n' +
+        'static layer rather than reimplementing it, so it needs that file.\n\n' +
+        'Looked in:\n' + RUNNER_CANDIDATES.map((p) => `  ${p}`).join('\n') + '\n\n' +
+        'Set DOORMAN_SCORECARD_RUNNER to its path, or run doorman from a clone of\n' +
+        'the repo, where the in-repo path resolves.',
+    };
+  }
+
+  await mkdir(out, { recursive: true });
+  log(`static layer via ${path.relative(process.cwd(), RUNNER)}`);
+  log(`out: ${out}`);
+
+  const args = [
+    RUNNER, '--once',
+    '--server', link,
+    '--static-only',
+    '--out', out,
+  ];
+  if (neededFor) args.push('--needed-for', neededFor);
+
+  const r = await new Promise((res) =>
+    execFile(process.execPath, args, { maxBuffer: 32 * 1024 * 1024 }, (err, stdout, stderr) =>
+      res({ ok: !err, stdout: stdout ?? '', stderr: stderr ?? '' })));
+
+  for (const line of r.stdout.split('\n').filter(Boolean)) log('  ' + line);
+
+  const gradePath = path.join(out, 'grade.json');
+  if (!existsSync(gradePath)) {
+    return {
+      ok: false,
+      why: 'the runner produced no grade.json',
+      detail: (r.stderr || r.stdout).slice(-1500),
+      hint:
+        'The most common cause is that `mcpscore` is not on PATH. It is a Python ' +
+        'console script: pip install mcpscore, then export the interpreter Scripts/ dir.',
+    };
+  }
+
+  const grade = JSON.parse(await readFile(gradePath, 'utf8'));
+  return { ok: true, grade, out, files: ['grade.json', 'report.md', 'recipe.md', 'transcripts.jsonl'] };
+}