clembot-doorman 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +17 -0
- package/LICENSE +21 -0
- package/README.md +951 -0
- package/WALKTHROUGH.md +224 -0
- package/doorman/.claude/hooks/mcp-gate.sh +205 -0
- package/doorman/.claude/settings.json +16 -0
- package/doorman/.claude-plugin/plugin.json +22 -0
- package/doorman/.mcp.json +24 -0
- package/doorman/README.md +259 -0
- package/doorman/agents/doorman.md +104 -0
- package/doorman/cli/agents.mjs +128 -0
- package/doorman/cli/allow.mjs +128 -0
- package/doorman/cli/cost.mjs +119 -0
- package/doorman/cli/discover.mjs +265 -0
- package/doorman/cli/doctor.mjs +282 -0
- package/doorman/cli/doorman.mjs +345 -0
- package/doorman/cli/eval.mjs +320 -0
- package/doorman/cli/harness.mjs +179 -0
- package/doorman/cli/install.mjs +175 -0
- package/doorman/cli/needs.mjs +116 -0
- package/doorman/cli/report.mjs +89 -0
- package/doorman/cli/sandbox.mjs +177 -0
- package/doorman/cli/task.mjs +239 -0
- package/doorman/cli/verdict.mjs +199 -0
- package/doorman/cli/watch.mjs +218 -0
- package/doorman/commands/doorman.md +116 -0
- package/doorman/commands/vet.md +69 -0
- package/doorman/hooks/hooks.json +30 -0
- package/doorman/install.sh +186 -0
- package/doorman/package.json +38 -0
- package/doorman/recipes/README.md +36 -0
- package/doorman/recipes/deepwiki.md +10 -0
- package/doorman/recipes/planted-bad.md +27 -0
- package/doorman/recipes/scorecard.md +10 -0
- package/doorman/registry/allowlist.json +37 -0
- package/doorman/registry/denylist.json +23 -0
- package/doorman/registry/ledger.jsonl +1 -0
- package/doorman/scripts/poller.mjs +292 -0
- package/doorman/scripts/resolve-cli.sh +58 -0
- package/doorman/scripts/vet.mjs +190 -0
- package/doorman/skills/doorman-guide/SKILL.md +69 -0
- package/doorman/src/budget.mjs +236 -0
- package/doorman/src/candidate.mjs +132 -0
- package/doorman/src/fit-review.mjs +255 -0
- package/doorman/src/injection.mjs +189 -0
- package/doorman/src/instructions.mjs +134 -0
- package/doorman/src/inventory.mjs +411 -0
- package/doorman/src/llm.mjs +87 -0
- package/doorman/src/needs.mjs +491 -0
- package/doorman/src/note.mjs +213 -0
- package/doorman/src/reviews.mjs +120 -0
- package/doorman/src/scorecard.mjs +123 -0
- package/doorman/src/vet.mjs +174 -0
- package/package.json +54 -0
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* /vet, as a program.
|
|
4
|
+
*
|
|
5
|
+
* node scripts/vet.mjs <url|repo|path> --needed-for "what you want it for"
|
|
6
|
+
* node scripts/vet.mjs <...> --type repo force the candidate type
|
|
7
|
+
* node scripts/vet.mjs <...> --dry-run fit only, never touch the scorecard
|
|
8
|
+
*
|
|
9
|
+
* Fit first, money second. A redundant candidate returns before a scorecard
|
|
10
|
+
* client is constructed, which is why the slash command is a thin wrapper over
|
|
11
|
+
* this rather than prose that asks an agent to remember the order.
|
|
12
|
+
*
|
|
13
|
+
* Env: ANTHROPIC_API_KEY, DOORMAN_MODEL, DOORMAN_INVENTORY_ROOT,
|
|
14
|
+
* DOORMAN_VAULT_PATH, SCORECARD_API,
|
|
15
|
+
* DOORMAN_MAX_USDC_PER_RUN, DOORMAN_MAX_USDC_PER_DAY. See env.example.
|
|
16
|
+
*
|
|
17
|
+
* Never calls process.exit(): on Node 25 / Windows that trips a libuv assertion
|
|
18
|
+
* during teardown and replaces the real exit code with 127.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { dirname, join } from 'node:path';
|
|
22
|
+
import { fileURLToPath } from 'node:url';
|
|
23
|
+
import { anthropicClient, MissingApiKeyError } from '../src/llm.mjs';
|
|
24
|
+
import { openBudget, DEFAULT_PER_DAY_USDC, DEFAULT_PER_RUN_USDC } from '../src/budget.mjs';
|
|
25
|
+
import { scorecardClient } from '../src/scorecard.mjs';
|
|
26
|
+
import { runVet } from '../src/vet.mjs';
|
|
27
|
+
import { writeNote } from '../src/note.mjs';
|
|
28
|
+
import { findInventoryRoot, gatherInventory } from '../src/inventory.mjs';
|
|
29
|
+
|
|
30
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
31
|
+
const REGISTRY = join(HERE, '..', 'registry');
|
|
32
|
+
|
|
33
|
+
const args = parse(process.argv.slice(2));
|
|
34
|
+
const target = args._[0];
|
|
35
|
+
|
|
36
|
+
if (!target || args.help) {
|
|
37
|
+
console.log(`
|
|
38
|
+
doorman vet
|
|
39
|
+
|
|
40
|
+
node scripts/vet.mjs <url|repo|path> [options]
|
|
41
|
+
|
|
42
|
+
--needed-for TXT what you want it for. Seeds the fit review and the probes.
|
|
43
|
+
--type TYPE mcp-server | skill | repo. Overrides detection.
|
|
44
|
+
--dry-run fit review only. Never constructs a scorecard client.
|
|
45
|
+
--no-note print the result, write no note.
|
|
46
|
+
--max-usdc N per-run spend cap. Default DOORMAN_MAX_USDC_PER_RUN, else 1.
|
|
47
|
+
--max-usdc-day N per-day spend cap. Default DOORMAN_MAX_USDC_PER_DAY, else 5.
|
|
48
|
+
|
|
49
|
+
The price is read from the service, never assumed. If it cannot be read the
|
|
50
|
+
run stops without spending: an unknown price is not a free one.
|
|
51
|
+
|
|
52
|
+
Fit runs first and is free. The paid grade only fires if fit passes AND the
|
|
53
|
+
candidate is an MCP server. A redundant candidate costs nothing.
|
|
54
|
+
`);
|
|
55
|
+
process.exitCode = target ? 0 : 1;
|
|
56
|
+
} else {
|
|
57
|
+
await main().catch((e) => {
|
|
58
|
+
console.error('\n' + (e instanceof MissingApiKeyError ? e.message : `error: ${e.message}`) + '\n');
|
|
59
|
+
process.exitCode = 1;
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function main() {
|
|
64
|
+
const root = process.env.DOORMAN_INVENTORY_ROOT || findInventoryRoot(process.cwd());
|
|
65
|
+
const inventory = gatherInventory({ root, registryDir: REGISTRY });
|
|
66
|
+
|
|
67
|
+
console.log(`\n candidate ${target}`);
|
|
68
|
+
console.log(` inventory ${root ?? 'NONE FOUND'} - ${inventory.agents.length} agents, ` +
|
|
69
|
+
`${inventory.skills.length} skills, ${inventory.mcpServers.length} mcp servers`);
|
|
70
|
+
for (const n of inventory.notes) console.log(` gap: ${n}`);
|
|
71
|
+
|
|
72
|
+
const llm = anthropicClient({
|
|
73
|
+
apiKey: process.env.ANTHROPIC_API_KEY,
|
|
74
|
+
model: process.env.DOORMAN_MODEL,
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
const budget = args['dry-run'] ? undefined : openBudget({
|
|
78
|
+
ledgerPath: join(REGISTRY, 'spend.ndjson'),
|
|
79
|
+
perRunUsdc: num(args['max-usdc'], process.env.DOORMAN_MAX_USDC_PER_RUN, DEFAULT_PER_RUN_USDC),
|
|
80
|
+
perDayUsdc: num(args['max-usdc-day'], process.env.DOORMAN_MAX_USDC_PER_DAY, DEFAULT_PER_DAY_USDC),
|
|
81
|
+
});
|
|
82
|
+
if (budget) {
|
|
83
|
+
console.log(` budget ${budget.spentToday()} spent today, ${budget.remainingToday()} of ` +
|
|
84
|
+
`${budget.perDayUsdc} left. Per-run cap ${budget.perRunUsdc}.`);
|
|
85
|
+
if (budget.damagedLines) {
|
|
86
|
+
console.log(` WARNING ${budget.damagedLines} unreadable line(s) in the spend ledger. ` +
|
|
87
|
+
'Today may be under-counted.');
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const api = process.env.SCORECARD_API;
|
|
92
|
+
const makeScorecard = args['dry-run']
|
|
93
|
+
? undefined
|
|
94
|
+
: () => {
|
|
95
|
+
if (!api) throw new Error('SCORECARD_API is not set, so the paid grade cannot run');
|
|
96
|
+
return scorecardClient({ api, budget });
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
const result = await runVet(target, {
|
|
100
|
+
llm, makeScorecard, budget, inventory,
|
|
101
|
+
needed_for: args['needed-for'],
|
|
102
|
+
type: args.type,
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
report(result);
|
|
106
|
+
|
|
107
|
+
if (!args['no-note']) {
|
|
108
|
+
const w = writeNote(result, {
|
|
109
|
+
vaultPath: process.env.DOORMAN_VAULT_PATH,
|
|
110
|
+
registryDir: REGISTRY,
|
|
111
|
+
});
|
|
112
|
+
if (w.warning) console.log(`\n WARNING ${w.warning}`);
|
|
113
|
+
console.log(`\n note ${w.path}${w.fellBack ? ' (fallback, not the vault)' : ''}`);
|
|
114
|
+
console.log(' status: pending. Nothing is allowlisted until you flip it.');
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** One blank line. Written this way so no anchor in this file needs an escape. */
|
|
119
|
+
const BLANK = String.fromCharCode(10);
|
|
120
|
+
|
|
121
|
+
function report(r) {
|
|
122
|
+
const f = r.fit;
|
|
123
|
+
console.log(`\n FIT ${f.verdict.toUpperCase()}${f.owner ? ' -> ' + f.owner : ''}`);
|
|
124
|
+
console.log(` ${f.rationale}`);
|
|
125
|
+
if (f.overlaps.length) {
|
|
126
|
+
console.log('\n Already covered by:');
|
|
127
|
+
for (const o of f.overlaps) console.log(` - ${o.kind} ${o.name}: ${o.why}`);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (r.stopped_at === 'fit') {
|
|
131
|
+
console.log('\n STOPPED before the paid grade. $0.00 spent.');
|
|
132
|
+
console.log(' Nothing was sent to the scorecard, and no client was built.');
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
if (r.stopped_at === 'budget') {
|
|
136
|
+
console.log(BLANK + ' REFUSED the spend cap said no. $0.00 spent.');
|
|
137
|
+
console.log(' ' + r.why);
|
|
138
|
+
console.log(' Raise it with --max-usdc / --max-usdc-day if you mean to.');
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
if (r.stopped_at === 'price-unknown') {
|
|
142
|
+
console.log(BLANK + ' REFUSED could not read the price, so nothing was bought.');
|
|
143
|
+
console.log(' ' + r.why);
|
|
144
|
+
console.log(' An unknown price is not a free one.');
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
if (r.stopped_at === 'scan') {
|
|
148
|
+
console.log(`\n SCAN ${r.scan.hard} hard, ${r.scan.steering} steering ` +
|
|
149
|
+
`over ${r.scan.scanned_chars} chars${r.scan.truncated ? ' (TRUNCATED)' : ''}`);
|
|
150
|
+
console.log(' behavioral grade: n/a - no tools to probe');
|
|
151
|
+
for (const m of r.scan.failure_modes.slice(0, 3)) console.log(` - ${m}`);
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
if (r.stopped_at === 'queued') {
|
|
155
|
+
console.log(BLANK + ` SPENT ${r.cost_usdc} USDC` +
|
|
156
|
+
(r.cost_usdc === 0 ? ' (the service states it is free right now)' : ''));
|
|
157
|
+
}
|
|
158
|
+
if (r.grade) {
|
|
159
|
+
console.log(`\n GRADE ${r.grade.grade} ${r.grade.score}/100 (cached)`);
|
|
160
|
+
if (r.grade.hard_fail) console.log(` HARD FAIL: ${r.grade.hard_fail}`);
|
|
161
|
+
console.log(` tape: ${r.transcripts}`);
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
console.log(`\n QUEUED audit ${r.audit_id}`);
|
|
165
|
+
console.log(' No grade yet, and no estimate of one. Run the probe runner.');
|
|
166
|
+
console.log(` tape (once it exists): ${r.transcripts}`);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** First finite number among a CLI flag, an env var, and a default. */
|
|
170
|
+
function num(...candidates) {
|
|
171
|
+
for (const c of candidates) {
|
|
172
|
+
const n = typeof c === 'number' ? c : Number(c);
|
|
173
|
+
if (c !== undefined && c !== true && c !== '' && Number.isFinite(n)) return n;
|
|
174
|
+
}
|
|
175
|
+
return undefined;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function parse(argv) {
|
|
179
|
+
const out = { _: [] };
|
|
180
|
+
for (let i = 0; i < argv.length; i++) {
|
|
181
|
+
const a = argv[i];
|
|
182
|
+
if (a.startsWith('--')) {
|
|
183
|
+
const k = a.slice(2);
|
|
184
|
+
const next = argv[i + 1];
|
|
185
|
+
if (next && !next.startsWith('--')) { out[k] = next; i++; }
|
|
186
|
+
else out[k] = true;
|
|
187
|
+
} else out._.push(a);
|
|
188
|
+
}
|
|
189
|
+
return out;
|
|
190
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: doorman-guide
|
|
3
|
+
description: Package manager and security gate for agent tools. Inspects your build, analyzes prompt history to recommend safe MCP servers, audits candidate tools for prompt injections, runs local fit reviews against existing skills, and blocks rogue tools at the gate.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Doorman Skill
|
|
7
|
+
|
|
8
|
+
The package manager and security doorman for AI agents.
|
|
9
|
+
|
|
10
|
+
Use this skill when:
|
|
11
|
+
- Setting up a net-new agent build or inspecting what tools your project can currently reach.
|
|
12
|
+
- Recommending or discovering new MCP tools and skills based on real user prompt history.
|
|
13
|
+
- Evaluating a candidate MCP server, GitHub repository, or new skill before installing it.
|
|
14
|
+
- Checking whether a proposed capability is already covered by existing skills or subagents.
|
|
15
|
+
- Verifying the local offline security hook (`PreToolUse` gate).
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## Capabilities & Commands
|
|
20
|
+
|
|
21
|
+
### 1. Inspect Your Build (`doorman doctor`)
|
|
22
|
+
```bash
|
|
23
|
+
doorman doctor
|
|
24
|
+
```
|
|
25
|
+
- **L0 check**: 100% local, read-only, costs zero tokens.
|
|
26
|
+
- Reports active harness (Claude Code, Cursor, Windsurf, Copilot, Gemini), reachable MCP servers, subagents, and gate wiring status.
|
|
27
|
+
- Calculates agent-to-tool exposure ratios to stop tool bloat.
|
|
28
|
+
|
|
29
|
+
### 2. Discover Needs from Prompts (`doorman needs`)
|
|
30
|
+
```bash
|
|
31
|
+
doorman needs
|
|
32
|
+
```
|
|
33
|
+
- Reads local session transcripts (`~/.claude/projects/`).
|
|
34
|
+
- Strips tool outputs, compaction summaries, and slash-command expansions to isolate real human asks.
|
|
35
|
+
- Categorizes needs across 12 taxonomies (`docs-lookup`, `web-search`, `database`, `browser-automation`, `cloud-deploy`, `observability`, `payments`, `comms`, `design-assets`, `knowledge-base`, `code-host`, `data-files`).
|
|
36
|
+
- Suppresses recommendations for needs already satisfied (`COVERED`).
|
|
37
|
+
- Pairs unmet needs with pre-vetted Grade A/B candidates from `scorecard.wanessalabs.com/feed` (`worth-measuring`), flags malicious ones (`blocked`), or notes ecosystem gaps (`GAP`).
|
|
38
|
+
|
|
39
|
+
### 3. Two-Phase Fit Review (`scripts/vet.mjs --dry-run`)
|
|
40
|
+
```bash
|
|
41
|
+
node scripts/vet.mjs <candidate-url-or-repo> --dry-run
|
|
42
|
+
```
|
|
43
|
+
- **Fit first, money second**: Ingests YAML frontmatter (`name`, `description`) of all installed skills and agents.
|
|
44
|
+
- If an existing skill already handles the ask, it returns `REDUNDANT` and cites the exact skill name.
|
|
45
|
+
- Stops immediately at Phase 1 ($0.00 spent) before contacting any external scorecard.
|
|
46
|
+
|
|
47
|
+
### 4. Static Protocol & Injection Scan (`doorman report <url>`)
|
|
48
|
+
```bash
|
|
49
|
+
doorman report <url>
|
|
50
|
+
```
|
|
51
|
+
- Scans MCP tool descriptions for prompt injection, hidden steering ads, and protocol violations.
|
|
52
|
+
- Caught WebZum's 6,290-character in-context steering ad on live internet.
|
|
53
|
+
- Caps hostile tools at Grade F.
|
|
54
|
+
|
|
55
|
+
### 5. Monitor Live Feed (`doorman watch`)
|
|
56
|
+
```bash
|
|
57
|
+
doorman watch --all
|
|
58
|
+
```
|
|
59
|
+
- Streams newly graded candidates from the public feed.
|
|
60
|
+
- Categorizes them against local inventory: `already-installed`, `blocked`, `unreviewed`.
|
|
61
|
+
- Warns if an installed server suffers a security demotion.
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
## Guiding Principles
|
|
66
|
+
|
|
67
|
+
1. **Say No to Redundant Tools**: A tool that duplicates an existing skill costs tokens, pollutes context, and increases hallucination risk. Always run the fit review first.
|
|
68
|
+
2. **Never Grade Documentation as Proof**: Documentation describes what authors wish was true. Doorman measures what tools actually do when an agent executes them.
|
|
69
|
+
3. **Fail Closed**: If a tool is unapproved or cannot be verified, the gate blocks it at exit 2. Never bypass the security hook.
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The spend cap. The doorman's second refusal, after fit.
|
|
3
|
+
*
|
|
4
|
+
* Fit answers "does my system need this at all". This answers "can I afford to
|
|
5
|
+
* find out". They are different refusals and both have to be structural,
|
|
6
|
+
* because an agent that can spend money on your behalf while you are asleep is
|
|
7
|
+
* only as safe as the thing that says no.
|
|
8
|
+
*
|
|
9
|
+
* ## The guarantee, and why it is a permit
|
|
10
|
+
*
|
|
11
|
+
* `scorecard.enqueue()` will not spend without a **permit**, and a permit is
|
|
12
|
+
* single-use. That is stronger than checking a number before calling: a check
|
|
13
|
+
* can be skipped by a new code path, forgotten in a refactor, or bypassed by a
|
|
14
|
+
* caller that does not know it exists. A required argument cannot.
|
|
15
|
+
*
|
|
16
|
+
* const permit = budget.reserve({ price_usdc, server }); // may throw
|
|
17
|
+
* await scorecard.enqueue({ ..., permit }); // throws without
|
|
18
|
+
* budget.settle(permit, { audit_id });
|
|
19
|
+
*
|
|
20
|
+
* Same shape as invariant 13: a path that CANNOT spend is easier to prove than
|
|
21
|
+
* a path that remembers not to.
|
|
22
|
+
*
|
|
23
|
+
* ## An unknown price is not free
|
|
24
|
+
*
|
|
25
|
+
* `reserve()` refuses a null, undefined, negative or non-finite price. The
|
|
26
|
+
* tempting default is 0, and 0 passes every cap forever. This is the same
|
|
27
|
+
* mistake as scoring an unmeasured layer zero, pointed at money instead of at
|
|
28
|
+
* a grade.
|
|
29
|
+
*
|
|
30
|
+
* ## Reservations count against the cap until they are released
|
|
31
|
+
*
|
|
32
|
+
* The ledger is written BEFORE the call, not after. If the process dies
|
|
33
|
+
* mid-request, the ledger shows a reservation that never settled, and today's
|
|
34
|
+
* spend is over-counted rather than under-counted. Over-counting refuses a call
|
|
35
|
+
* you could have afforded; under-counting spends money you did not have. Only
|
|
36
|
+
* one of those is recoverable by waiting until tomorrow.
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs';
|
|
40
|
+
import { dirname } from 'node:path';
|
|
41
|
+
|
|
42
|
+
export class BudgetError extends Error {}
|
|
43
|
+
export class BudgetExceededError extends BudgetError {}
|
|
44
|
+
|
|
45
|
+
/** Deliberately small. A cap you have to raise on purpose is doing its job. */
|
|
46
|
+
export const DEFAULT_PER_RUN_USDC = 1;
|
|
47
|
+
export const DEFAULT_PER_DAY_USDC = 5;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* @param {object} opts
|
|
51
|
+
* @param {string} [opts.ledgerPath] NDJSON, append-only. Omit for in-memory.
|
|
52
|
+
* @param {number} [opts.perRunUsdc]
|
|
53
|
+
* @param {number} [opts.perDayUsdc]
|
|
54
|
+
* @param {Function} [opts.now] injectable clock
|
|
55
|
+
* @param {Function} [opts.randomId] injectable id, for deterministic tests
|
|
56
|
+
*/
|
|
57
|
+
export function openBudget(opts = {}) {
|
|
58
|
+
const {
|
|
59
|
+
ledgerPath,
|
|
60
|
+
perRunUsdc = DEFAULT_PER_RUN_USDC,
|
|
61
|
+
perDayUsdc = DEFAULT_PER_DAY_USDC,
|
|
62
|
+
now = () => new Date(),
|
|
63
|
+
randomId = () => 'permit_' + Math.random().toString(36).slice(2, 10),
|
|
64
|
+
} = opts;
|
|
65
|
+
|
|
66
|
+
for (const [k, v] of Object.entries({ perRunUsdc, perDayUsdc })) {
|
|
67
|
+
if (!Number.isFinite(v) || v < 0) {
|
|
68
|
+
throw new BudgetError(`${k} must be a finite number >= 0, got ${v}`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if (perRunUsdc > perDayUsdc) {
|
|
72
|
+
// Not a hypothetical: it means one call can exceed the daily cap, so the
|
|
73
|
+
// daily cap silently does nothing. Better to refuse the configuration.
|
|
74
|
+
throw new BudgetError(
|
|
75
|
+
`perRunUsdc (${perRunUsdc}) is above perDayUsdc (${perDayUsdc}), so the ` +
|
|
76
|
+
'daily cap could never bind. Fix the configuration rather than shipping ' +
|
|
77
|
+
'a limit that does not limit.',
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** In-memory mirror. The file is the record; this avoids re-reading per call. */
|
|
82
|
+
const loaded = ledgerPath ? readLedger(ledgerPath) : { entries: [], skipped: 0 };
|
|
83
|
+
const entries = loaded.entries;
|
|
84
|
+
const open = new Map(); // permit id -> reservation, unsettled
|
|
85
|
+
|
|
86
|
+
const write = (entry) => {
|
|
87
|
+
entries.push(entry);
|
|
88
|
+
if (!ledgerPath) return;
|
|
89
|
+
mkdirSync(dirname(ledgerPath), { recursive: true });
|
|
90
|
+
appendFileSync(ledgerPath, JSON.stringify(entry) + '\n', 'utf8');
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
const today = () => now().toISOString().slice(0, 10);
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Committed plus outstanding, for today only.
|
|
97
|
+
*
|
|
98
|
+
* A `release` cancels its own `reserve`. A `settle` does not: settling means
|
|
99
|
+
* the money went out, so it stays counted.
|
|
100
|
+
*/
|
|
101
|
+
const spentToday = () => {
|
|
102
|
+
const day = today();
|
|
103
|
+
const released = new Set(
|
|
104
|
+
entries.filter((e) => e.event === 'release').map((e) => e.permit),
|
|
105
|
+
);
|
|
106
|
+
return round4(
|
|
107
|
+
entries
|
|
108
|
+
.filter((e) => e.event === 'reserve' && e.ts.slice(0, 10) === day)
|
|
109
|
+
.filter((e) => !released.has(e.permit))
|
|
110
|
+
.reduce((a, e) => a + e.price_usdc, 0),
|
|
111
|
+
);
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
return {
|
|
115
|
+
perRunUsdc,
|
|
116
|
+
perDayUsdc,
|
|
117
|
+
ledgerPath: ledgerPath ?? null,
|
|
118
|
+
spentToday,
|
|
119
|
+
remainingToday: () => round4(Math.max(0, perDayUsdc - spentToday())),
|
|
120
|
+
entries: () => entries.slice(),
|
|
121
|
+
/** Unparseable ledger lines found at open. Reported, never silently zero. */
|
|
122
|
+
damagedLines: loaded.skipped,
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Claim headroom for one paid call. Throws rather than returning a falsy
|
|
126
|
+
* value: a caller that ignores a return value must not end up spending.
|
|
127
|
+
* @returns {{id: string, price_usdc: number, server: string, ts: string}}
|
|
128
|
+
*/
|
|
129
|
+
reserve({ price_usdc, server = '(unnamed)' } = {}) {
|
|
130
|
+
if (typeof price_usdc !== 'number' || !Number.isFinite(price_usdc)) {
|
|
131
|
+
throw new BudgetError(
|
|
132
|
+
'reserve() needs a numeric price_usdc. An unknown price is not a free ' +
|
|
133
|
+
'one: refusing to spend against a number nobody supplied. Read the ' +
|
|
134
|
+
'price from the 402 challenge, or set it in config.',
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
if (price_usdc < 0) throw new BudgetError('price_usdc cannot be negative');
|
|
138
|
+
|
|
139
|
+
if (price_usdc > perRunUsdc) {
|
|
140
|
+
throw new BudgetExceededError(
|
|
141
|
+
`this call costs ${price_usdc} USDC and the per-run cap is ${perRunUsdc}. ` +
|
|
142
|
+
'Nothing was spent.',
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
const after = round4(spentToday() + price_usdc);
|
|
146
|
+
if (after > perDayUsdc) {
|
|
147
|
+
throw new BudgetExceededError(
|
|
148
|
+
`this call costs ${price_usdc} USDC, today's spend is already ` +
|
|
149
|
+
`${spentToday()}, and the daily cap is ${perDayUsdc}. Nothing was spent.`,
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const permit = {
|
|
154
|
+
id: randomId(),
|
|
155
|
+
price_usdc,
|
|
156
|
+
server,
|
|
157
|
+
ts: now().toISOString(),
|
|
158
|
+
};
|
|
159
|
+
open.set(permit.id, permit);
|
|
160
|
+
write({
|
|
161
|
+
event: 'reserve', permit: permit.id, price_usdc,
|
|
162
|
+
server, ts: permit.ts,
|
|
163
|
+
});
|
|
164
|
+
return permit;
|
|
165
|
+
},
|
|
166
|
+
|
|
167
|
+
/** The call happened. The money is spent and stays counted. */
|
|
168
|
+
settle(permit, { audit_id = null } = {}) {
|
|
169
|
+
assertOpen(open, permit);
|
|
170
|
+
open.delete(permit.id);
|
|
171
|
+
write({
|
|
172
|
+
event: 'settle', permit: permit.id, price_usdc: permit.price_usdc,
|
|
173
|
+
server: permit.server, audit_id, ts: now().toISOString(),
|
|
174
|
+
});
|
|
175
|
+
return permit;
|
|
176
|
+
},
|
|
177
|
+
|
|
178
|
+
/** The call did not happen. Give the headroom back, with a reason. */
|
|
179
|
+
release(permit, reason = 'unspecified') {
|
|
180
|
+
assertOpen(open, permit);
|
|
181
|
+
open.delete(permit.id);
|
|
182
|
+
write({
|
|
183
|
+
event: 'release', permit: permit.id, price_usdc: permit.price_usdc,
|
|
184
|
+
server: permit.server, reason, ts: now().toISOString(),
|
|
185
|
+
});
|
|
186
|
+
return permit;
|
|
187
|
+
},
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Used by the paid client. A permit is valid exactly once and only while
|
|
191
|
+
* open, so a replayed or already-settled permit cannot buy a second audit.
|
|
192
|
+
*/
|
|
193
|
+
isOpen: (permit) => Boolean(permit?.id) && open.has(permit.id),
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function assertOpen(open, permit) {
|
|
198
|
+
if (!permit?.id) throw new BudgetError('not a permit');
|
|
199
|
+
if (!open.has(permit.id)) {
|
|
200
|
+
throw new BudgetError(
|
|
201
|
+
`permit ${permit.id} is not open: it was already settled or released. ` +
|
|
202
|
+
'Reserve a new one rather than reusing this.',
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Read an NDJSON ledger, skipping lines that do not parse.
|
|
209
|
+
*
|
|
210
|
+
* A corrupt line is skipped rather than thrown on, because a half-written line
|
|
211
|
+
* from a killed process must not lock the doorman out of spending forever. It
|
|
212
|
+
* is counted in `skipped` so the CLI can say the ledger is damaged instead of
|
|
213
|
+
* quietly under-counting the day.
|
|
214
|
+
*
|
|
215
|
+
* @returns {{entries: object[], skipped: number}}
|
|
216
|
+
*/
|
|
217
|
+
export function readLedger(path) {
|
|
218
|
+
if (!existsSync(path)) return { entries: [], skipped: 0 };
|
|
219
|
+
const out = [];
|
|
220
|
+
let skipped = 0;
|
|
221
|
+
for (const line of readFileSync(path, 'utf8').split('\n')) {
|
|
222
|
+
if (!line.trim()) continue;
|
|
223
|
+
try {
|
|
224
|
+
const e = JSON.parse(line);
|
|
225
|
+
if (e && typeof e.price_usdc === 'number' && typeof e.ts === 'string') out.push(e);
|
|
226
|
+
else skipped++;
|
|
227
|
+
} catch {
|
|
228
|
+
skipped++;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return { entries: out, skipped };
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function round4(n) {
|
|
235
|
+
return Math.round(n * 10000) / 10000;
|
|
236
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What kind of thing are we being asked to vet?
|
|
3
|
+
*
|
|
4
|
+
* ── Syntactic, and offline ───────────────────────────────────────────────────
|
|
5
|
+
*
|
|
6
|
+
* The obvious implementation probes the URL and sees whether it speaks MCP.
|
|
7
|
+
* This does not, for two reasons. CI has to run with no network, and a probe
|
|
8
|
+
* would mean connecting to an unvetted endpoint as the FIRST thing we do,
|
|
9
|
+
* before anything has decided the candidate is worth touching at all.
|
|
10
|
+
*
|
|
11
|
+
* So detection reads the string. It will occasionally be wrong, which is why
|
|
12
|
+
* `--type` exists and why the type is printed in the report.
|
|
13
|
+
*
|
|
14
|
+
* ── Why the type decides the money ───────────────────────────────────────────
|
|
15
|
+
*
|
|
16
|
+
* Only `mcp-server` may reach the paid scorecard. A skill or a repo has no
|
|
17
|
+
* tools to drive, so a behavioural grade is not a thing that can exist for it,
|
|
18
|
+
* and the report says `behavioral grade: n/a` rather than implying one ran.
|
|
19
|
+
* `mayBeGraded()` is the single place that rule lives.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
export const CANDIDATE_TYPES = ['mcp-server', 'skill', 'repo'];
|
|
23
|
+
|
|
24
|
+
/** Only this type may ever be sent to the paid scorecard. */
|
|
25
|
+
export function mayBeGraded(type) {
|
|
26
|
+
return type === 'mcp-server';
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export class CandidateError extends Error {}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @param {string} input a url, or a local path to a markdown file
|
|
33
|
+
* @param {object} [opts] `{ type }` forces the answer
|
|
34
|
+
* @returns {{ id, type, kind, host, forced }}
|
|
35
|
+
*/
|
|
36
|
+
export function detectCandidate(input, { type } = {}) {
|
|
37
|
+
const raw = String(input ?? '').trim();
|
|
38
|
+
if (!raw) throw new CandidateError('no candidate given');
|
|
39
|
+
|
|
40
|
+
if (type !== undefined) {
|
|
41
|
+
if (!CANDIDATE_TYPES.includes(type)) {
|
|
42
|
+
throw new CandidateError(
|
|
43
|
+
`unknown --type ${JSON.stringify(type)}; must be one of ` + CANDIDATE_TYPES.join(', '),
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
return { id: raw, type, host: hostOf(raw), forced: true };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// A local path. Only markdown is a skill; anything else we decline to guess
|
|
50
|
+
// about rather than sending a directory to the grader.
|
|
51
|
+
if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(raw)) {
|
|
52
|
+
if (/\.md$/i.test(raw)) return { id: raw, type: 'skill', host: null, forced: false };
|
|
53
|
+
throw new CandidateError(
|
|
54
|
+
`"${raw}" is not a url and is not a .md file, so its type cannot be inferred. ` +
|
|
55
|
+
'Pass --type mcp-server|skill|repo.',
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
let url;
|
|
60
|
+
try {
|
|
61
|
+
url = new URL(raw);
|
|
62
|
+
} catch {
|
|
63
|
+
throw new CandidateError(`"${raw}" is not a valid url`);
|
|
64
|
+
}
|
|
65
|
+
if (url.protocol !== 'https:' && url.protocol !== 'http:') {
|
|
66
|
+
throw new CandidateError(`unsupported scheme ${url.protocol} in "${raw}"`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const host = url.hostname.toLowerCase();
|
|
70
|
+
const path = url.pathname.replace(/\/+$/, '');
|
|
71
|
+
|
|
72
|
+
// A gist is a skill. A raw .md anywhere is a skill.
|
|
73
|
+
if (host === 'gist.github.com' || /\.md$/i.test(path)) {
|
|
74
|
+
return { id: raw, type: 'skill', host, forced: false };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (host === 'github.com' || host === 'www.github.com') {
|
|
78
|
+
const segments = path.split('/').filter(Boolean);
|
|
79
|
+
// /owner/repo/blob/... points at a file, which is a skill, not a repo.
|
|
80
|
+
if (segments.length >= 4 && (segments[2] === 'blob' || segments[2] === 'raw')) {
|
|
81
|
+
return { id: raw, type: 'skill', host, forced: false };
|
|
82
|
+
}
|
|
83
|
+
if (segments.length >= 2) {
|
|
84
|
+
return { id: raw, type: 'repo', host, forced: false };
|
|
85
|
+
}
|
|
86
|
+
// github.com with one segment or none is a user or the site itself.
|
|
87
|
+
throw new CandidateError(
|
|
88
|
+
`"${raw}" points at github.com but names no repository. ` +
|
|
89
|
+
'Pass --type, or give a full owner/repo url.',
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Everything else that is an http(s) endpoint is treated as an MCP server.
|
|
94
|
+
// This is the only branch that can lead to a paid grade, and it is the
|
|
95
|
+
// default rather than a special case because that is what people paste.
|
|
96
|
+
return { id: raw, type: 'mcp-server', host, forced: false };
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function hostOf(raw) {
|
|
100
|
+
try {
|
|
101
|
+
return new URL(raw).hostname.toLowerCase();
|
|
102
|
+
} catch {
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* A filesystem-safe slug for the review note filename.
|
|
109
|
+
*
|
|
110
|
+
* Uses the FULL hostname for a url. The scorecard's poller learned this the
|
|
111
|
+
* hard way: keying on the first label collapsed every `mcp.<vendor>.com` onto
|
|
112
|
+
* the key `mcp`, so allowlisting one vendor silently allowlisted another. A
|
|
113
|
+
* filename is not a trust decision, but two candidates sharing a note file
|
|
114
|
+
* would overwrite each other's evidence, which is bad in the same direction.
|
|
115
|
+
*/
|
|
116
|
+
export function candidateSlug(candidate) {
|
|
117
|
+
const raw = candidate.id ?? String(candidate);
|
|
118
|
+
let base;
|
|
119
|
+
try {
|
|
120
|
+
const u = new URL(raw);
|
|
121
|
+
base = u.hostname + u.pathname;
|
|
122
|
+
} catch {
|
|
123
|
+
base = raw.replace(/^.*[\\/]/, ''); // a local path: keep the filename
|
|
124
|
+
}
|
|
125
|
+
const slug = base
|
|
126
|
+
.toLowerCase()
|
|
127
|
+
.replace(/\.md$/, '')
|
|
128
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
129
|
+
.replace(/^-+|-+$/g, '')
|
|
130
|
+
.slice(0, 80);
|
|
131
|
+
return slug || 'candidate';
|
|
132
|
+
}
|