clembot-doorman 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +17 -0
- package/LICENSE +21 -0
- package/README.md +951 -0
- package/WALKTHROUGH.md +224 -0
- package/doorman/.claude/hooks/mcp-gate.sh +205 -0
- package/doorman/.claude/settings.json +16 -0
- package/doorman/.claude-plugin/plugin.json +22 -0
- package/doorman/.mcp.json +24 -0
- package/doorman/README.md +259 -0
- package/doorman/agents/doorman.md +104 -0
- package/doorman/cli/agents.mjs +128 -0
- package/doorman/cli/allow.mjs +128 -0
- package/doorman/cli/cost.mjs +119 -0
- package/doorman/cli/discover.mjs +265 -0
- package/doorman/cli/doctor.mjs +282 -0
- package/doorman/cli/doorman.mjs +345 -0
- package/doorman/cli/eval.mjs +320 -0
- package/doorman/cli/harness.mjs +179 -0
- package/doorman/cli/install.mjs +175 -0
- package/doorman/cli/needs.mjs +116 -0
- package/doorman/cli/report.mjs +89 -0
- package/doorman/cli/sandbox.mjs +177 -0
- package/doorman/cli/task.mjs +239 -0
- package/doorman/cli/verdict.mjs +199 -0
- package/doorman/cli/watch.mjs +218 -0
- package/doorman/commands/doorman.md +116 -0
- package/doorman/commands/vet.md +69 -0
- package/doorman/hooks/hooks.json +30 -0
- package/doorman/install.sh +186 -0
- package/doorman/package.json +38 -0
- package/doorman/recipes/README.md +36 -0
- package/doorman/recipes/deepwiki.md +10 -0
- package/doorman/recipes/planted-bad.md +27 -0
- package/doorman/recipes/scorecard.md +10 -0
- package/doorman/registry/allowlist.json +37 -0
- package/doorman/registry/denylist.json +23 -0
- package/doorman/registry/ledger.jsonl +1 -0
- package/doorman/scripts/poller.mjs +292 -0
- package/doorman/scripts/resolve-cli.sh +58 -0
- package/doorman/scripts/vet.mjs +190 -0
- package/doorman/skills/doorman-guide/SKILL.md +69 -0
- package/doorman/src/budget.mjs +236 -0
- package/doorman/src/candidate.mjs +132 -0
- package/doorman/src/fit-review.mjs +255 -0
- package/doorman/src/injection.mjs +189 -0
- package/doorman/src/instructions.mjs +134 -0
- package/doorman/src/inventory.mjs +411 -0
- package/doorman/src/llm.mjs +87 -0
- package/doorman/src/needs.mjs +491 -0
- package/doorman/src/note.mjs +213 -0
- package/doorman/src/reviews.mjs +120 -0
- package/doorman/src/scorecard.mjs +123 -0
- package/doorman/src/vet.mjs +174 -0
- package/package.json +54 -0
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The review note. One markdown file, one page, the human approval surface.
|
|
3
|
+
*
|
|
4
|
+
* ── Nothing auto-approves ────────────────────────────────────────────────────
|
|
5
|
+
*
|
|
6
|
+
* `status: pending` is the resting state and this module only ever writes that.
|
|
7
|
+
* A person flips it to `approved` or `denied` by editing the frontmatter, and
|
|
8
|
+
* the poller acts on the flip. Nothing here writes `approved`, ever.
|
|
9
|
+
*
|
|
10
|
+
* ── It never loses a report ──────────────────────────────────────────────────
|
|
11
|
+
*
|
|
12
|
+
* If the vault path is unset or unwritable the note goes to `registry/reviews/`
|
|
13
|
+
* with a warning. A review that vanished because a path was wrong is worse than
|
|
14
|
+
* a review in the wrong folder.
|
|
15
|
+
*
|
|
16
|
+
* ── It only ever creates and appends ─────────────────────────────────────────
|
|
17
|
+
*
|
|
18
|
+
* The vault belongs to the human. This writes new notes and appends to the
|
|
19
|
+
* decision log of notes it wrote. It never edits or deletes anything else.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
23
|
+
import { dirname, join } from 'node:path';
|
|
24
|
+
import { candidateSlug } from './candidate.mjs';
|
|
25
|
+
|
|
26
|
+
/** A page. Same reasoning as the scorecard's report: enforced, not aspirational. */
|
|
27
|
+
export const MAX_NOTE_LINES = 72;
|
|
28
|
+
|
|
29
|
+
/** Where notes go inside the vault. */
|
|
30
|
+
export const VAULT_SUBDIR = join('clembot-doorman', 'reviews');
|
|
31
|
+
|
|
32
|
+
function yamlString(v) {
|
|
33
|
+
if (v === null || v === undefined) return 'null';
|
|
34
|
+
const s = String(v);
|
|
35
|
+
// Quote anything YAML would misread. A description with a colon in it is the
|
|
36
|
+
// common case and it silently truncates the value otherwise.
|
|
37
|
+
return /[:#\n"']/.test(s) ? JSON.stringify(s) : s;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Render the note.
|
|
42
|
+
*
|
|
43
|
+
* @param {object} r a runVet result
|
|
44
|
+
* @returns {{ filename: string, body: string, truncated: boolean }}
|
|
45
|
+
*/
|
|
46
|
+
export function renderNote(r, { now = () => new Date() } = {}) {
|
|
47
|
+
const c = r.candidate;
|
|
48
|
+
const date = r.reviewed ?? now().toISOString().slice(0, 10);
|
|
49
|
+
const g = r.grade;
|
|
50
|
+
|
|
51
|
+
const front = [
|
|
52
|
+
'---',
|
|
53
|
+
`candidate: ${yamlString(c.id)}`,
|
|
54
|
+
`type: ${c.type}`,
|
|
55
|
+
`fit: ${r.fit ? r.fit.verdict : 'null'}`,
|
|
56
|
+
`owner: ${yamlString(r.fit && r.fit.owner)}`,
|
|
57
|
+
`grade: ${yamlString(g && g.grade)}`,
|
|
58
|
+
`cost_usdc: ${r.cost_usdc ?? 0}`,
|
|
59
|
+
`evidence_hash: ${yamlString(g && g.evidence_sha256)}`,
|
|
60
|
+
'status: pending',
|
|
61
|
+
`reviewed: ${date}`,
|
|
62
|
+
'---',
|
|
63
|
+
'',
|
|
64
|
+
];
|
|
65
|
+
|
|
66
|
+
const head = [`# ${c.id}`, ''];
|
|
67
|
+
|
|
68
|
+
const tldr = ['## Verdict', ''];
|
|
69
|
+
if (r.fit) {
|
|
70
|
+
tldr.push(`**${r.fit.verdict}**. ${r.fit.rationale}`);
|
|
71
|
+
if (r.fit.verdict === 'fits' && r.fit.owner) {
|
|
72
|
+
tldr.push('', `Scoped to \`${r.fit.owner}\`.`);
|
|
73
|
+
}
|
|
74
|
+
} else {
|
|
75
|
+
tldr.push('No fit review ran.');
|
|
76
|
+
}
|
|
77
|
+
tldr.push('');
|
|
78
|
+
|
|
79
|
+
const overlap = ['## Overlap', ''];
|
|
80
|
+
if (r.fit && r.fit.overlaps.length) {
|
|
81
|
+
overlap.push('| Kind | Name | Already covers |', '|---|---|---|');
|
|
82
|
+
for (const o of r.fit.overlaps) overlap.push(`| ${o.kind} | \`${o.name}\` | ${o.why} |`);
|
|
83
|
+
} else {
|
|
84
|
+
overlap.push('Nothing in the current inventory covers this.');
|
|
85
|
+
}
|
|
86
|
+
overlap.push('');
|
|
87
|
+
|
|
88
|
+
const placement = ['## Placement', ''];
|
|
89
|
+
if (r.fit && r.fit.verdict === 'fits') {
|
|
90
|
+
placement.push(`\`${r.fit.owner}\` gets the tool. No other subagent is changed.`);
|
|
91
|
+
placement.push('');
|
|
92
|
+
placement.push('> Scoping here is **recorded, not enforced**. The gate matches every');
|
|
93
|
+
placement.push('> `mcp__*` call and reads only the tool name, so it cannot tell which');
|
|
94
|
+
placement.push('> subagent is calling. Treat `owner` as the intent, not a boundary.');
|
|
95
|
+
} else if (r.fit && r.fit.verdict === 'needs-new-subagent') {
|
|
96
|
+
placement.push('No existing subagent is the right home. A new one would have to be');
|
|
97
|
+
placement.push('created before this is worth adding.');
|
|
98
|
+
} else {
|
|
99
|
+
placement.push('Not placed: the fit review stopped before placement.');
|
|
100
|
+
}
|
|
101
|
+
placement.push('');
|
|
102
|
+
|
|
103
|
+
const scorecard = ['## Scorecard', ''];
|
|
104
|
+
if (g) {
|
|
105
|
+
scorecard.push(`**${g.grade} ${g.score}/100** against \`${g.model}\`.`);
|
|
106
|
+
const L = g.layers ?? {};
|
|
107
|
+
scorecard.push('', `Static ${fmt(L.static_pct)} · behavioural ${fmt(L.behavioral_pct)} · guidance ${fmt(L.guidance_pct)}`);
|
|
108
|
+
if (g.hard_fail) scorecard.push('', `**Hard fail:** ${g.hard_fail}`);
|
|
109
|
+
const worst = (g.grade_json && g.grade_json.worst_failure_modes) || [];
|
|
110
|
+
if (worst.length) {
|
|
111
|
+
scorecard.push('');
|
|
112
|
+
for (const w of worst.slice(0, 3)) scorecard.push(`- ${w}`);
|
|
113
|
+
}
|
|
114
|
+
if (r.transcripts) scorecard.push('', `[Replay the tape](${r.transcripts})`);
|
|
115
|
+
} else if (r.scan) {
|
|
116
|
+
scorecard.push('**behavioral grade: n/a - no tools to probe.**');
|
|
117
|
+
scorecard.push('', `Scanned ${r.scan.scanned_chars} chars of instruction text from \`${r.scan.source}\`${r.scan.truncated ? ' (TRUNCATED, so this is a partial scan)' : ''}.`);
|
|
118
|
+
scorecard.push('', r.scan.hits.length
|
|
119
|
+
? `${r.scan.hard} hard, ${r.scan.steering} steering:`
|
|
120
|
+
: 'No pattern fired.');
|
|
121
|
+
for (const fmode of r.scan.failure_modes.slice(0, 4)) scorecard.push(`- ${fmode}`);
|
|
122
|
+
} else if (r.audit_id) {
|
|
123
|
+
scorecard.push('Queued, not yet graded. There is no grade here and no estimate of one.');
|
|
124
|
+
if (r.transcripts) scorecard.push('', `Tape (once it exists): ${r.transcripts}`);
|
|
125
|
+
} else {
|
|
126
|
+
scorecard.push('Not graded. The fit review stopped before the paid phase, which is');
|
|
127
|
+
scorecard.push('the point: a redundant candidate costs nothing.');
|
|
128
|
+
}
|
|
129
|
+
scorecard.push('');
|
|
130
|
+
|
|
131
|
+
const recipe = ['## Recipe', ''];
|
|
132
|
+
const recipeText = (g && g.recipe_md) || '';
|
|
133
|
+
if (recipeText) recipe.push(...recipeText.trim().split('\n'));
|
|
134
|
+
else recipe.push('_No recipe drafted._');
|
|
135
|
+
recipe.push('');
|
|
136
|
+
|
|
137
|
+
const log = [
|
|
138
|
+
'## Decision log', '',
|
|
139
|
+
`- ${date} · review written by the doorman · status \`pending\``,
|
|
140
|
+
'',
|
|
141
|
+
];
|
|
142
|
+
|
|
143
|
+
// Trim the recipe FIRST, per the brief. It is the only section whose loss
|
|
144
|
+
// costs context rather than a claim.
|
|
145
|
+
let body = [...front, ...head, ...tldr, ...overlap, ...placement, ...scorecard, ...recipe, ...log];
|
|
146
|
+
let truncated = false;
|
|
147
|
+
while (body.length > MAX_NOTE_LINES && recipe.length > 3) {
|
|
148
|
+
recipe.splice(recipe.length - 2, 1);
|
|
149
|
+
truncated = true;
|
|
150
|
+
body = [...front, ...head, ...tldr, ...overlap, ...placement, ...scorecard, ...recipe, ...log];
|
|
151
|
+
}
|
|
152
|
+
if (truncated) {
|
|
153
|
+
recipe.splice(recipe.length - 1, 0, '', '_Recipe trimmed to hold one page. Full text is on the audit._');
|
|
154
|
+
body = [...front, ...head, ...tldr, ...overlap, ...placement, ...scorecard, ...recipe, ...log];
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
filename: `${candidateSlug(c)}-${date}.md`,
|
|
159
|
+
body: body.join('\n').replace(/\n{3,}/g, '\n\n') + '\n',
|
|
160
|
+
truncated,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Write it. Vault first, registry fallback second, never nowhere.
|
|
166
|
+
*
|
|
167
|
+
* @returns {{ path: string, fellBack: boolean, warning: string|null }}
|
|
168
|
+
*/
|
|
169
|
+
export function writeNote(r, { vaultPath, registryDir, fs, now } = {}) {
|
|
170
|
+
const io = fs ?? { existsSync, mkdirSync, writeFileSync };
|
|
171
|
+
const { filename, body, truncated } = renderNote(r, { now });
|
|
172
|
+
|
|
173
|
+
const targets = [];
|
|
174
|
+
if (vaultPath) targets.push({ dir: join(vaultPath, VAULT_SUBDIR), vault: true });
|
|
175
|
+
if (registryDir) targets.push({ dir: join(registryDir, 'reviews'), vault: false });
|
|
176
|
+
|
|
177
|
+
let warning = null;
|
|
178
|
+
for (const t of targets) {
|
|
179
|
+
try {
|
|
180
|
+
io.mkdirSync(t.dir, { recursive: true });
|
|
181
|
+
const path = join(t.dir, filename);
|
|
182
|
+
io.writeFileSync(path, body, 'utf8');
|
|
183
|
+
return { path, fellBack: !t.vault, truncated, warning };
|
|
184
|
+
} catch (e) {
|
|
185
|
+
warning = `could not write to ${t.dir} (${e.message})`;
|
|
186
|
+
if (t.vault) {
|
|
187
|
+
warning += '; falling back to the registry so the report is not lost';
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
throw new Error('nowhere to write the review note. ' + (warning ?? 'no target given'));
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Append one line to a note's decision log.
|
|
196
|
+
*
|
|
197
|
+
* The ONLY mutation this module makes to an existing file, and it is additive.
|
|
198
|
+
* A note whose log cannot be found is left alone and reported, rather than
|
|
199
|
+
* having a log section invented at the end of somebody's document.
|
|
200
|
+
*/
|
|
201
|
+
export function appendDecision(path, line, { fs } = {}) {
|
|
202
|
+
const io = fs ?? { readFileSync, appendFileSync };
|
|
203
|
+
const text = io.readFileSync(path, 'utf8');
|
|
204
|
+
if (!/^## Decision log$/m.test(text)) {
|
|
205
|
+
return { appended: false, reason: 'no "## Decision log" heading in that note' };
|
|
206
|
+
}
|
|
207
|
+
io.appendFileSync(path, `- ${line}\n`, 'utf8');
|
|
208
|
+
return { appended: true };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function fmt(v) {
|
|
212
|
+
return v === null || v === undefined ? '_not measured_' : `${v}%`;
|
|
213
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turning an approved note into a registry entry.
|
|
3
|
+
*
|
|
4
|
+
* Split out of the poller so it can be tested without a filesystem, a clock or
|
|
5
|
+
* a network. The poller reads notes and writes files; this decides what should
|
|
6
|
+
* happen, and decides nothing else.
|
|
7
|
+
*
|
|
8
|
+
* ── The rule that outranks the human ─────────────────────────────────────────
|
|
9
|
+
*
|
|
10
|
+
* A hard-failed server is refused even when the note says `approved`. Not
|
|
11
|
+
* because the human is wrong, but because "approved" on a note and "safe to put
|
|
12
|
+
* in the file the gate reads" are different claims, and a hard fail means the
|
|
13
|
+
* server was caught doing the one thing that disqualifies it outright. The
|
|
14
|
+
* refusal is written back into the note's decision log so it is visible where
|
|
15
|
+
* the decision was made, not buried in a terminal.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { parseFrontmatter } from './inventory.mjs';
|
|
19
|
+
|
|
20
|
+
export const APPROVED = 'approved';
|
|
21
|
+
export const DENIED = 'denied';
|
|
22
|
+
export const PENDING = 'pending';
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Decide what a single note should cause.
|
|
26
|
+
*
|
|
27
|
+
* @param {object} note { path, frontmatter }
|
|
28
|
+
* @param {object} registry { allow: {servers}, deny: {servers} }
|
|
29
|
+
* @returns {{ action, key, reason, entry? }}
|
|
30
|
+
* action: 'allow' | 'deny' | 'refuse' | 'skip'
|
|
31
|
+
*/
|
|
32
|
+
export function decideForNote(note, registry = {}) {
|
|
33
|
+
const fm = note.frontmatter ?? {};
|
|
34
|
+
const status = (fm.status ?? '').trim();
|
|
35
|
+
const candidate = fm.candidate;
|
|
36
|
+
const key = fm.owner && fm.owner !== 'null' ? fm.owner : null;
|
|
37
|
+
|
|
38
|
+
if (status === PENDING || status === '') {
|
|
39
|
+
return { action: 'skip', reason: 'still pending; a human has not decided' };
|
|
40
|
+
}
|
|
41
|
+
if (status === DENIED) {
|
|
42
|
+
return {
|
|
43
|
+
action: 'deny',
|
|
44
|
+
key: fm.mcp_name ?? null,
|
|
45
|
+
candidate,
|
|
46
|
+
reason: 'denied by hand in the note',
|
|
47
|
+
entry: baseEntry(fm),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
if (status !== APPROVED) {
|
|
51
|
+
return { action: 'skip', reason: `unrecognised status "${status}"; refusing to guess` };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
if (fm.type && fm.type !== 'mcp-server') {
|
|
55
|
+
return {
|
|
56
|
+
action: 'skip',
|
|
57
|
+
reason: `type "${fm.type}" is not an MCP server, so it has no place in the gate's registry`,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// The refusal. Deliberately AFTER the approval check, so the log line records
|
|
62
|
+
// that a human said yes and the rule said no.
|
|
63
|
+
const grade = fm.grade;
|
|
64
|
+
const hardFail = fm.hard_fail && fm.hard_fail !== 'null' ? fm.hard_fail : null;
|
|
65
|
+
if (grade === 'F' || hardFail) {
|
|
66
|
+
return {
|
|
67
|
+
action: 'refuse',
|
|
68
|
+
candidate,
|
|
69
|
+
reason: hardFail
|
|
70
|
+
? `approved by hand, but REFUSED: hard fail (${hardFail})`
|
|
71
|
+
: 'approved by hand, but REFUSED: grade F',
|
|
72
|
+
entry: baseEntry(fm),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (!grade || grade === 'null') {
|
|
77
|
+
return {
|
|
78
|
+
action: 'refuse',
|
|
79
|
+
candidate,
|
|
80
|
+
reason: 'approved by hand, but REFUSED: there is no grade on this note',
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const already = (registry.allow?.servers ?? {})[fm.mcp_name ?? ''];
|
|
85
|
+
if (already && already.audit_id === fm.audit_id) {
|
|
86
|
+
return { action: 'skip', reason: 'already allowlisted from this same audit' };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return {
|
|
90
|
+
action: 'allow',
|
|
91
|
+
key: fm.mcp_name ?? null,
|
|
92
|
+
candidate,
|
|
93
|
+
assigned_to: key,
|
|
94
|
+
reason: `approved by hand, grade ${grade}`,
|
|
95
|
+
entry: baseEntry(fm),
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function baseEntry(fm) {
|
|
100
|
+
return {
|
|
101
|
+
decision: 'allow',
|
|
102
|
+
url: fm.candidate ?? null,
|
|
103
|
+
grade: fm.grade === 'null' ? null : (fm.grade ?? null),
|
|
104
|
+
audit_id: fm.audit_id ?? null,
|
|
105
|
+
evidence_sha256: fm.evidence_hash === 'null' ? null : (fm.evidence_hash ?? null),
|
|
106
|
+
/* `assigned_to`, NOT `owner`. The poller already uses `owner` for the
|
|
107
|
+
scorecard allowlist tenant, and one word meaning two things in one file
|
|
108
|
+
is how a scoping rule quietly stops meaning anything. */
|
|
109
|
+
assigned_to: fm.owner === 'null' ? null : (fm.owner ?? null),
|
|
110
|
+
graded_at: fm.reviewed ?? null,
|
|
111
|
+
note: 'Approved by hand from a review note. Scoping is recorded, not enforced: ' +
|
|
112
|
+
'the gate cannot tell which subagent is calling.',
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Read a note file's frontmatter into the shape decideForNote wants. */
|
|
117
|
+
export function readNote(path, text) {
|
|
118
|
+
const fm = parseFrontmatter(text);
|
|
119
|
+
return { path, frontmatter: fm ?? {}, readable: Boolean(fm) };
|
|
120
|
+
}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The paid client. The only thing here that can spend money.
|
|
3
|
+
*
|
|
4
|
+
* Injected everywhere so tests can pass a stub that THROWS if it is touched.
|
|
5
|
+
* That is the shape of the guarantee: "no scorecard call happened" is only
|
|
6
|
+
* checkable if there is one object that would have made it.
|
|
7
|
+
*
|
|
8
|
+
* It is also never CONSTRUCTED on a path that must not pay. A skill or a repo
|
|
9
|
+
* has no tools to drive, so `runVet` does not build one at all rather than
|
|
10
|
+
* building one and remembering not to call it.
|
|
11
|
+
*
|
|
12
|
+
* ## A budget is required, including for the free reads
|
|
13
|
+
*
|
|
14
|
+
* `scorecardClient` refuses to exist without one. Making it optional would mean
|
|
15
|
+
* there is a way to obtain an enqueue-capable client with no spend cap, and a
|
|
16
|
+
* guarantee with an opt-out is a default. `cached()` and `price()` cost nothing
|
|
17
|
+
* and still travel with the cap, because the cap is a property of the client,
|
|
18
|
+
* not a step someone has to remember.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
export class ScorecardError extends Error {}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* @param {object} opts
|
|
25
|
+
* @param {string} opts.api base url, e.g. https://scorecard.wanessalabs.com
|
|
26
|
+
* @param {object} opts.budget from openBudget(). REQUIRED. See above.
|
|
27
|
+
* @param {Function} [opts.fetch] injectable
|
|
28
|
+
*/
|
|
29
|
+
export function scorecardClient({ api, budget, fetch: f = fetch, timeoutMs = 30_000 } = {}) {
|
|
30
|
+
if (!api) throw new ScorecardError('scorecardClient needs an api base url');
|
|
31
|
+
if (!budget || typeof budget.isOpen !== 'function') {
|
|
32
|
+
throw new ScorecardError(
|
|
33
|
+
'scorecardClient needs a budget. Build one with openBudget(). This is ' +
|
|
34
|
+
'not optional: a client with no spend cap is a client that can spend ' +
|
|
35
|
+
'without limit, and an optional guarantee is a default.',
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
const base = api.replace(/\/+$/, '');
|
|
39
|
+
|
|
40
|
+
const call = async (path, init) => {
|
|
41
|
+
const ac = new AbortController();
|
|
42
|
+
const timer = setTimeout(() => ac.abort(), timeoutMs);
|
|
43
|
+
try {
|
|
44
|
+
const res = await f(base + path, { ...init, signal: ac.signal });
|
|
45
|
+
return res;
|
|
46
|
+
} finally {
|
|
47
|
+
clearTimeout(timer);
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
return {
|
|
52
|
+
api: base,
|
|
53
|
+
|
|
54
|
+
/** The cheap read. Returns null when the server has never been graded. */
|
|
55
|
+
async cached(serverUrl) {
|
|
56
|
+
const res = await call('/grade?server=' + encodeURIComponent(serverUrl));
|
|
57
|
+
if (res.status === 404) return null;
|
|
58
|
+
if (!res.ok) throw new ScorecardError(`cached lookup failed: HTTP ${res.status}`);
|
|
59
|
+
const j = await res.json();
|
|
60
|
+
return j.graded ? j : null;
|
|
61
|
+
},
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* What an audit costs, asked rather than assumed.
|
|
65
|
+
*
|
|
66
|
+
* A missing price is NOT zero. When the service cannot be reached or gives
|
|
67
|
+
* an answer this client does not understand, `price_usdc` comes back null
|
|
68
|
+
* and `openBudget().reserve()` refuses it. That is deliberate: defaulting
|
|
69
|
+
* an unknown price to zero passes every cap forever.
|
|
70
|
+
*/
|
|
71
|
+
async price() {
|
|
72
|
+
const res = await call('/price');
|
|
73
|
+
if (!res.ok) {
|
|
74
|
+
return { payment_required: null, price_usdc: null, known: false,
|
|
75
|
+
why: `GET /price returned HTTP ${res.status}` };
|
|
76
|
+
}
|
|
77
|
+
const j = await res.json().catch(() => null);
|
|
78
|
+
if (!j || typeof j.price_usdc !== 'number') {
|
|
79
|
+
return { payment_required: j?.payment_required ?? null, price_usdc: null, known: false,
|
|
80
|
+
why: 'the service did not state a decimal price' };
|
|
81
|
+
}
|
|
82
|
+
return { ...j, known: true };
|
|
83
|
+
},
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The paid write. Requires an OPEN permit from the budget this client was
|
|
87
|
+
* built with, and the permit is checked here rather than trusted, so a
|
|
88
|
+
* fabricated or already-spent one buys nothing.
|
|
89
|
+
*/
|
|
90
|
+
async enqueue({ url, name, needed_for, owner = 'doorman', permit }) {
|
|
91
|
+
if (!budget.isOpen(permit)) {
|
|
92
|
+
throw new ScorecardError(
|
|
93
|
+
'enqueue() needs an open budget permit. Call budget.reserve({ ' +
|
|
94
|
+
'price_usdc, server }) first. A permit is single-use: one that was ' +
|
|
95
|
+
'already settled or released cannot buy a second audit.',
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
const res = await call('/grade', {
|
|
99
|
+
method: 'POST',
|
|
100
|
+
headers: { 'content-type': 'application/json', 'x-owner': owner },
|
|
101
|
+
body: JSON.stringify([{ url, name, needed_for }]),
|
|
102
|
+
});
|
|
103
|
+
if (!res.ok) {
|
|
104
|
+
const body = await res.text().catch(() => '');
|
|
105
|
+
throw new ScorecardError(`enqueue failed: HTTP ${res.status} ${body.slice(0, 200)}`);
|
|
106
|
+
}
|
|
107
|
+
const j = await res.json();
|
|
108
|
+
const a = (j.audits ?? [])[0];
|
|
109
|
+
if (!a?.audit_id) throw new ScorecardError('enqueue returned no audit id');
|
|
110
|
+
return a;
|
|
111
|
+
},
|
|
112
|
+
|
|
113
|
+
async audit(id) {
|
|
114
|
+
const res = await call('/grade/' + encodeURIComponent(id));
|
|
115
|
+
if (!res.ok) throw new ScorecardError(`audit read failed: HTTP ${res.status}`);
|
|
116
|
+
return res.json();
|
|
117
|
+
},
|
|
118
|
+
|
|
119
|
+
transcriptsUrl(id) {
|
|
120
|
+
return `${base}/grade/${id}/transcripts`;
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two-phase vet. Fit first, money second.
|
|
3
|
+
*
|
|
4
|
+
* Two questions, in this order, and the order is the whole feature:
|
|
5
|
+
*
|
|
6
|
+
* 1. Does MY system need this? free, local, one model call
|
|
7
|
+
* 2. Can I afford to find out? free, local, a spend cap
|
|
8
|
+
* 3. Is it trustworthy? paid, external, the scorecard
|
|
9
|
+
*
|
|
10
|
+
* Question 2 exists because an agent that can spend on your behalf while you
|
|
11
|
+
* are asleep is only as safe as the thing that says no. The price is DISCOVERED
|
|
12
|
+
* from the service, never assumed: a client that defaults an unknown price to
|
|
13
|
+
* zero passes every cap it has, forever.
|
|
14
|
+
*
|
|
15
|
+
* A `redundant` or `out-of-scope` verdict RETURNS BEFORE the scorecard client
|
|
16
|
+
* is constructed. Not before it is called: before it exists. A path that cannot
|
|
17
|
+
* spend money is easier to prove than a path that remembers not to.
|
|
18
|
+
*
|
|
19
|
+
* Everything is injected, so the whole flow runs offline in tests with a
|
|
20
|
+
* scorecard stub that throws on contact.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { detectCandidate, mayBeGraded } from './candidate.mjs';
|
|
24
|
+
import { fitReview } from './fit-review.mjs';
|
|
25
|
+
import { gatherInventory, renderInventory, findInventoryRoot } from './inventory.mjs';
|
|
26
|
+
import { fetchInstructions } from './instructions.mjs';
|
|
27
|
+
import { sniffInstructions } from './injection.mjs';
|
|
28
|
+
|
|
29
|
+
/** Verdicts that stop the flow before anything is paid for. */
|
|
30
|
+
export const STOP_VERDICTS = ['redundant', 'out-of-scope'];
|
|
31
|
+
|
|
32
|
+
/** Owner used when the caller does not supply one. */
|
|
33
|
+
export const DEFAULT_OWNER = 'doorman';
|
|
34
|
+
|
|
35
|
+
export class VetError extends Error {}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* @param {string} input url, repo, or path
|
|
39
|
+
* @param {object} deps
|
|
40
|
+
* @param {object} deps.llm for the fit review
|
|
41
|
+
* @param {Function} [deps.makeScorecard] () => client. NOT called on a stop path.
|
|
42
|
+
* @param {object} [deps.inventory] pre-gathered; otherwise read from disk
|
|
43
|
+
* @param {string} [deps.needed_for]
|
|
44
|
+
* @param {string} [deps.type] force the candidate type
|
|
45
|
+
* @param {Function} [deps.fetch] for instruction text
|
|
46
|
+
* @param {Function} [deps.now]
|
|
47
|
+
*/
|
|
48
|
+
export async function runVet(input, deps = {}) {
|
|
49
|
+
const {
|
|
50
|
+
llm, makeScorecard, budget, needed_for, type,
|
|
51
|
+
inventoryRoot, registryDir, fetch: f, now = () => new Date(),
|
|
52
|
+
} = deps;
|
|
53
|
+
|
|
54
|
+
const candidate = detectCandidate(input, type ? { type } : undefined);
|
|
55
|
+
candidate.needed_for = needed_for;
|
|
56
|
+
|
|
57
|
+
const inventory = deps.inventory ?? gatherInventory({
|
|
58
|
+
root: inventoryRoot ?? findInventoryRoot(process.cwd()),
|
|
59
|
+
registryDir,
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
const result = {
|
|
63
|
+
candidate,
|
|
64
|
+
reviewed: now().toISOString().slice(0, 10),
|
|
65
|
+
fit: null,
|
|
66
|
+
scan: null,
|
|
67
|
+
grade: null,
|
|
68
|
+
audit_id: null,
|
|
69
|
+
transcripts: null,
|
|
70
|
+
cost_usdc: 0,
|
|
71
|
+
paid: false,
|
|
72
|
+
price: null,
|
|
73
|
+
budget: budget
|
|
74
|
+
? { per_run_usdc: budget.perRunUsdc, per_day_usdc: budget.perDayUsdc,
|
|
75
|
+
spent_today_usdc: budget.spentToday(), remaining_usdc: budget.remainingToday() }
|
|
76
|
+
: null,
|
|
77
|
+
stopped_at: null,
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
// ── Phase 1: fit. Free. ────────────────────────────────────────────────────
|
|
81
|
+
result.fit = await fitReview(candidate, inventory, { llm, renderInventory });
|
|
82
|
+
|
|
83
|
+
if (STOP_VERDICTS.includes(result.fit.verdict)) {
|
|
84
|
+
// The point of the whole feature. No client, no request, no charge.
|
|
85
|
+
result.stopped_at = 'fit';
|
|
86
|
+
return result;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// ── Phase 2a: a skill or a repo is scanned, never graded. ──────────────────
|
|
90
|
+
if (!mayBeGraded(candidate.type)) {
|
|
91
|
+
const text = await fetchInstructions(candidate, f ? { fetch: f } : undefined);
|
|
92
|
+
result.scan = {
|
|
93
|
+
...sniffInstructions(text.text, candidate.type === 'repo' ? 'README.md' : 'SKILL.md'),
|
|
94
|
+
source: text.source,
|
|
95
|
+
bytes: text.bytes,
|
|
96
|
+
truncated: text.truncated,
|
|
97
|
+
};
|
|
98
|
+
// Said in the result rather than left for the reader to infer from a null.
|
|
99
|
+
result.grade = null;
|
|
100
|
+
result.behavioral = 'n/a - no tools to probe';
|
|
101
|
+
result.stopped_at = 'scan';
|
|
102
|
+
return result;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// ── Phase 2b: an MCP server that the system actually needs. Paid. ──────────
|
|
106
|
+
if (!makeScorecard) {
|
|
107
|
+
throw new VetError(
|
|
108
|
+
'fit passed and this candidate is gradeable, but no scorecard client was ' +
|
|
109
|
+
'supplied. Refusing to report a verdict with no grade behind it.',
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
if (!budget) {
|
|
113
|
+
throw new VetError(
|
|
114
|
+
'fit passed and this candidate is gradeable, but no budget was supplied. ' +
|
|
115
|
+
'Refusing to reach a paid path with no spend cap. Build one with ' +
|
|
116
|
+
'openBudget() and pass it as deps.budget.',
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
const scorecard = makeScorecard();
|
|
120
|
+
|
|
121
|
+
// The cache is free, and checking it BEFORE reserving means an exhausted
|
|
122
|
+
// budget still answers from cache rather than refusing a question that costs
|
|
123
|
+
// nothing. Refusing a free read to enforce a spend cap would be theatre.
|
|
124
|
+
const cached = await scorecard.cached(candidate.id);
|
|
125
|
+
if (cached) {
|
|
126
|
+
result.grade = cached;
|
|
127
|
+
result.audit_id = cached.audit_id;
|
|
128
|
+
result.transcripts = scorecard.transcriptsUrl(cached.audit_id);
|
|
129
|
+
result.stopped_at = 'cached';
|
|
130
|
+
return result; // a cache hit is still not a purchase
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// ── Phase 2c: the money. Discover the price, reserve, then spend. ──────────
|
|
134
|
+
result.price = await scorecard.price();
|
|
135
|
+
if (!result.price.known) {
|
|
136
|
+
result.stopped_at = 'price-unknown';
|
|
137
|
+
result.why = result.price.why;
|
|
138
|
+
return result; // an unknown price is not a free one
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
let permit;
|
|
142
|
+
try {
|
|
143
|
+
permit = budget.reserve({
|
|
144
|
+
price_usdc: result.price.price_usdc,
|
|
145
|
+
server: candidate.id,
|
|
146
|
+
});
|
|
147
|
+
} catch (e) {
|
|
148
|
+
// A refusal, not a failure. The caller gets a result to print, and the
|
|
149
|
+
// reason, rather than a stack trace.
|
|
150
|
+
result.stopped_at = 'budget';
|
|
151
|
+
result.why = e.message;
|
|
152
|
+
return result;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
try {
|
|
156
|
+
const queued = await scorecard.enqueue({
|
|
157
|
+
url: candidate.id, name: candidate.host ?? undefined, needed_for, permit,
|
|
158
|
+
});
|
|
159
|
+
budget.settle(permit, { audit_id: queued.audit_id });
|
|
160
|
+
result.audit_id = queued.audit_id;
|
|
161
|
+
result.transcripts = scorecard.transcriptsUrl(queued.audit_id);
|
|
162
|
+
result.cost_usdc = permit.price_usdc;
|
|
163
|
+
result.paid = permit.price_usdc > 0; // a free call is not a purchase
|
|
164
|
+
result.stopped_at = 'queued';
|
|
165
|
+
} catch (e) {
|
|
166
|
+
// The call did not happen, so the headroom goes back. Releasing on the
|
|
167
|
+
// error path is the whole reason reserve and settle are separate steps.
|
|
168
|
+
budget.release(permit, 'enqueue failed: ' + e.message);
|
|
169
|
+
throw e;
|
|
170
|
+
}
|
|
171
|
+
result.budget.spent_today_usdc = budget.spentToday();
|
|
172
|
+
result.budget.remaining_usdc = budget.remainingToday();
|
|
173
|
+
return result;
|
|
174
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "clembot-doorman",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "The Package Manager & Security Doorman for Claude Code: inspect your build, recommend vetted MCPs from prompt history, and block rogue tools before context.",
|
|
5
|
+
"homepage": "https://clembot-doorman.wanessalabs.com",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"bin": {
|
|
8
|
+
"doorman": "doorman/cli/doorman.mjs"
|
|
9
|
+
},
|
|
10
|
+
"scripts": {
|
|
11
|
+
"doctor": "node doorman/cli/doorman.mjs doctor",
|
|
12
|
+
"test": "node doorman/test/run.mjs && node doorman/test/cli-run.mjs",
|
|
13
|
+
"test:all": "node doorman/test/run.mjs && node doorman/test/cli-run.mjs && bash doorman/test-gate.sh"
|
|
14
|
+
},
|
|
15
|
+
"engines": {
|
|
16
|
+
"node": ">=20"
|
|
17
|
+
},
|
|
18
|
+
"license": "MIT",
|
|
19
|
+
"repository": {
|
|
20
|
+
"type": "git",
|
|
21
|
+
"url": "git+https://github.com/clemenswan/clembot-doorman.git"
|
|
22
|
+
},
|
|
23
|
+
"comment": [
|
|
24
|
+
"Root package so the CLI installs from a clone:",
|
|
25
|
+
" git clone https://github.com/clemenswan/clembot-doorman && npm i -g ./clembot-doorman",
|
|
26
|
+
"",
|
|
27
|
+
"It deliberately declares NO dependencies. This is a giveaway: every",
|
|
28
|
+
"dependency is one more thing that can fail to install on somebody else's",
|
|
29
|
+
"machine, and a gate that fails to start is a gate that fails open.",
|
|
30
|
+
"",
|
|
31
|
+
"The bin points into doorman/cli/ rather than duplicating it, and",
|
|
32
|
+
"report.mjs resolves the scorecard runner at runtime rather than assuming",
|
|
33
|
+
"the author's directory layout."
|
|
34
|
+
],
|
|
35
|
+
"files": [
|
|
36
|
+
"doorman/cli",
|
|
37
|
+
"doorman/src",
|
|
38
|
+
"doorman/registry",
|
|
39
|
+
"doorman/hooks",
|
|
40
|
+
"doorman/agents",
|
|
41
|
+
"doorman/commands",
|
|
42
|
+
"doorman/skills",
|
|
43
|
+
"doorman/scripts",
|
|
44
|
+
"doorman/recipes",
|
|
45
|
+
"doorman/.claude",
|
|
46
|
+
"doorman/.claude-plugin",
|
|
47
|
+
"doorman/.mcp.json",
|
|
48
|
+
"doorman/install.sh",
|
|
49
|
+
"doorman/README.md",
|
|
50
|
+
"doorman/package.json",
|
|
51
|
+
".claude-plugin",
|
|
52
|
+
"WALKTHROUGH.md"
|
|
53
|
+
]
|
|
54
|
+
}
|