create-agent-rig 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +105 -1
- package/README.md +92 -3
- package/package.json +4 -3
- package/packages/cli/dist/commands/memory.js +123 -0
- package/packages/cli/dist/commands/setup.js +45 -0
- package/packages/cli/dist/index.js +107 -3
- package/packages/cli/dist/lib/subsystems.js +269 -0
- package/packages/cli/dist/lib/version.js +15 -0
- package/packages/cli/dist/policy/benchmark/corpus.js +165 -0
- package/packages/cli/dist/policy/core/coverage.js +253 -0
- package/packages/cli/dist/policy/core/decision-record.js +130 -44
- package/packages/cli/dist/policy/core/declaration.js +58 -17
- package/packages/cli/dist/policy/core/evidence-matrix.js +94 -0
- package/packages/cli/dist/policy/core/probe.js +442 -0
- package/packages/cli/dist/policy/core/validation.js +194 -1
- package/packages/cli/dist/policy/core/vocabulary.js +70 -3
- package/packages/cli/dist/policy/harness/claude.js +9 -1
- package/packages/cli/dist/policy/harness/codex.js +48 -1
- package/packages/cli/dist/policy/harness/shared-hooks.js +18 -0
- package/packages/cli/dist/policy/index.js +9 -2
- package/templates/agent-os/stack/aws-cdk/.claude/agents/cdk-diff-reviewer.md +2 -0
- package/templates/agent-os/stack/aws-cdk/.codex/agents/cdk-diff-reviewer.toml +2 -0
- package/templates/agent-os/subagent-routing.json +32 -0
- package/templates/agent-os/universal/.agents/skills/loop/SKILL.md +41 -4
- package/templates/agent-os/universal/.claude/agents/code-reviewer.md +2 -0
- package/templates/agent-os/universal/.claude/agents/prose-reviewer.md +2 -0
- package/templates/agent-os/universal/.claude/agents/security-scanner.md +2 -0
- package/templates/agent-os/universal/.claude/agents/test-writer.md +2 -0
- package/templates/agent-os/universal/.claude/hooks/guard-subagent-model.mjs +234 -0
- package/templates/agent-os/universal/.claude/hooks/lib/edit-input.mjs +75 -32
- package/templates/agent-os/universal/.claude/hooks/warn-subagent-routing.mjs +120 -0
- package/templates/agent-os/universal/.claude/rules/workflow.md +5 -0
- package/templates/agent-os/universal/.claude/scripts/preflight.mjs +27 -3
- package/templates/agent-os/universal/.claude/scripts/queue/gate-rounds.mjs +70 -2
- package/templates/agent-os/universal/.claude/scripts/queue/index.mjs +12 -4
- package/templates/agent-os/universal/.claude/scripts/unattended-flag.mjs +64 -1
- package/templates/agent-os/universal/.claude/settings.json +16 -0
- package/templates/agent-os/universal/.claude/skills/loop/SKILL.md +41 -4
- package/templates/agent-os/universal/.codex/agents/code-reviewer.toml +2 -0
- package/templates/agent-os/universal/.codex/agents/prose-reviewer.toml +2 -0
- package/templates/agent-os/universal/.codex/agents/security-scanner.toml +2 -0
- package/templates/agent-os/universal/.codex/agents/test-writer.toml +2 -0
- package/templates/agent-os/universal/.codex/config.toml +3 -0
- package/templates/agent-os/universal/docs/decisions/codex-adapter.md +31 -5
- package/templates/agent-os/universal/docs/decisions/subagent-routing.md +142 -0
- package/templates/agent-os/universal/layers.json +4 -0
- package/templates/hash-history.json +8 -4
- package/templates/release-ledger.json +2 -1
- package/templates/skeleton/node-service/services/api/test/artifact.test.ts +3 -4
- package/templates/skeleton/node-service/services/api/test/package-manager.test.ts +40 -0
- package/templates/skeleton/node-service/services/api/test/package-manager.ts +51 -0
- package/templates/skeleton/node-service/services/api/test/static-dir.test.ts +9 -8
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
// The machine-scoped subsystem manifest (ADR-RP-002 R6, RP-147): installation
|
|
2
|
+
// metadata, not a service registry. One writer (`setup`; `upgrade` re-runs the
|
|
3
|
+
// same derivation), one scope (the user's configuration root), one entry per
|
|
4
|
+
// subsystem carrying the resolved executable invocation and the contract major
|
|
5
|
+
// it must answer. No search path, no fallback chain, no PATH scan: the
|
|
6
|
+
// executable is derived from the one root the owner declared, and every
|
|
7
|
+
// substantive call performs the `--version --json` handshake first.
|
|
8
|
+
//
|
|
9
|
+
// Handshake results carry no path — a status is machine JSON a consumer may
|
|
10
|
+
// print, and the manifest itself is the only place a location is written.
|
|
11
|
+
import { statSync } from 'node:fs';
|
|
12
|
+
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
13
|
+
import { randomBytes } from 'node:crypto';
|
|
14
|
+
import path from 'node:path';
|
|
15
|
+
export const SUBSYSTEMS_SCHEMA_VERSION = 1;
|
|
16
|
+
export const MEMORY_CONTRACT_MAJOR = 1;
|
|
17
|
+
const MEMORY_EXECUTABLE_REL = ['shared-memory', 'memory.mjs'];
|
|
18
|
+
const HANDSHAKE_TIMEOUT_MS = 15_000;
|
|
19
|
+
export class SubsystemsError extends Error {
|
|
20
|
+
code;
|
|
21
|
+
constructor(code, message) {
|
|
22
|
+
super(message);
|
|
23
|
+
this.name = 'SubsystemsError';
|
|
24
|
+
this.code = code;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Exactly one location per platform. An unset root is refused, not guessed:
|
|
29
|
+
* a manifest written under a guessed home is a manifest nothing else reads.
|
|
30
|
+
*/
|
|
31
|
+
export function subsystemsManifestPath(env, platform) {
|
|
32
|
+
if (platform === 'win32') {
|
|
33
|
+
const root = env.APPDATA;
|
|
34
|
+
if (!root)
|
|
35
|
+
throw new SubsystemsError('config-root-unavailable', 'APPDATA is not set');
|
|
36
|
+
return path.join(root, 'create-agent-rig', 'subsystems.json');
|
|
37
|
+
}
|
|
38
|
+
const home = env.HOME;
|
|
39
|
+
if (!home)
|
|
40
|
+
throw new SubsystemsError('config-root-unavailable', 'HOME is not set');
|
|
41
|
+
return path.join(home, '.config', 'create-agent-rig', 'subsystems.json');
|
|
42
|
+
}
|
|
43
|
+
/** The path module of the DECLARED platform, so an entry's encoding is pinned by the platform argument, not by the host. */
|
|
44
|
+
const pathFor = (platform) => (platform === 'win32' ? path.win32 : path.posix);
|
|
45
|
+
export function memoryExecutablePath(memoryRoot, platform) {
|
|
46
|
+
return pathFor(platform).join(memoryRoot, ...MEMORY_EXECUTABLE_REL);
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The pure half of the derivation: root → invocation, in the declared
|
|
50
|
+
* platform's encoding. The root is the single declared input
|
|
51
|
+
* (`setup --memory-root`), so it must be absolute under that platform — a
|
|
52
|
+
* relative root would make the entry mean a different executable from every
|
|
53
|
+
* working directory. No filesystem here, so both encodings are pinned by tests
|
|
54
|
+
* on any host.
|
|
55
|
+
*/
|
|
56
|
+
export function memoryInvocation(input, platform) {
|
|
57
|
+
if (!pathFor(platform).isAbsolute(input.memoryRoot))
|
|
58
|
+
throw new SubsystemsError('memory-root-relative', 'the Memory root must be an absolute path');
|
|
59
|
+
return [input.nodeExecutable, memoryExecutablePath(input.memoryRoot, platform)];
|
|
60
|
+
}
|
|
61
|
+
function isFile(file) {
|
|
62
|
+
try {
|
|
63
|
+
return statSync(file).isFile();
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return false;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The whole derivation: the invocation above, plus the check that the
|
|
71
|
+
* executable is already there on this host — the manifest records what
|
|
72
|
+
* exists, it does not promise what an install will bring. So `platform` is
|
|
73
|
+
* the host's here; the pure half is what a foreign platform's encoding is
|
|
74
|
+
* tested through.
|
|
75
|
+
*/
|
|
76
|
+
export function deriveMemoryEntry(input, platform) {
|
|
77
|
+
const invocation = memoryInvocation(input, platform);
|
|
78
|
+
if (!isFile(invocation[1]))
|
|
79
|
+
throw new SubsystemsError('memory-executable-absent', `no ${MEMORY_EXECUTABLE_REL.join('/')} under the declared Memory root`);
|
|
80
|
+
return {
|
|
81
|
+
memoryRoot: input.memoryRoot,
|
|
82
|
+
invocation,
|
|
83
|
+
contractMajor: MEMORY_CONTRACT_MAJOR,
|
|
84
|
+
memoryRef: input.memoryRef,
|
|
85
|
+
installedVersion: input.installedVersion,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
const majorOf = (contractVersion) => {
|
|
89
|
+
const match = /^(\d+)\.\d+$/.exec(contractVersion);
|
|
90
|
+
return match ? Number(match[1]) : null;
|
|
91
|
+
};
|
|
92
|
+
/**
|
|
93
|
+
* `--version --json` and its classification. The consumer compares the
|
|
94
|
+
* contract major alone; a foreign major is the consumer's refusal (exit 4 in
|
|
95
|
+
* `setup`), never something Memory emits. A failure payload is
|
|
96
|
+
* INTEGRATION-FAILED, never absence: the executable answered.
|
|
97
|
+
*/
|
|
98
|
+
export async function handshake(entry, run, options = {}) {
|
|
99
|
+
const [file, script] = entry.invocation;
|
|
100
|
+
// Absence is the runner's answer (ENOENT), not a pre-check: the executable
|
|
101
|
+
// the manifest names is exactly what this call must exercise.
|
|
102
|
+
const result = await run(file, [script, '--version', '--json'], {
|
|
103
|
+
timeoutMs: options.timeoutMs ?? HANDSHAKE_TIMEOUT_MS,
|
|
104
|
+
});
|
|
105
|
+
if (result.spawnError) {
|
|
106
|
+
if (result.spawnError.code === 'ENOENT')
|
|
107
|
+
return { status: 'unsupported', reason: 'absent' };
|
|
108
|
+
return { status: 'integration-failed', reason: 'invalid-payload' };
|
|
109
|
+
}
|
|
110
|
+
let payload;
|
|
111
|
+
try {
|
|
112
|
+
payload = JSON.parse(result.stdout);
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
return { status: 'integration-failed', reason: 'invalid-payload' };
|
|
116
|
+
}
|
|
117
|
+
if (typeof payload !== 'object' || payload === null)
|
|
118
|
+
return { status: 'integration-failed', reason: 'invalid-payload' };
|
|
119
|
+
const record = payload;
|
|
120
|
+
// The executable answered, and answered a failure — a broken VERSION file is
|
|
121
|
+
// the Memory-side shape (RP-19). The manifest's promise about what is
|
|
122
|
+
// installed no longer holds, so this is `manifest-stale`, never absence.
|
|
123
|
+
if (result.code !== 0 || record.result === 'integration-failed')
|
|
124
|
+
return { status: 'integration-failed', reason: 'manifest-stale' };
|
|
125
|
+
const { name, version, contractVersion } = record;
|
|
126
|
+
if (typeof version !== 'string' || typeof contractVersion !== 'string')
|
|
127
|
+
return { status: 'integration-failed', reason: 'invalid-payload' };
|
|
128
|
+
if (name !== 'memory')
|
|
129
|
+
return { status: 'integration-failed', reason: 'manifest-stale' };
|
|
130
|
+
const major = majorOf(contractVersion);
|
|
131
|
+
if (major === null)
|
|
132
|
+
return { status: 'integration-failed', reason: 'manifest-stale' };
|
|
133
|
+
if (major !== MEMORY_CONTRACT_MAJOR)
|
|
134
|
+
return { status: 'foreign-major', contractVersion, requiredMajor: MEMORY_CONTRACT_MAJOR };
|
|
135
|
+
return { status: 'ok', version, contractVersion };
|
|
136
|
+
}
|
|
137
|
+
/** Stable bytes: fixed key order, two-space indent, trailing newline. */
|
|
138
|
+
export function serializeSubsystemsManifest(manifest) {
|
|
139
|
+
const memory = manifest.entries.memory;
|
|
140
|
+
const ordered = {
|
|
141
|
+
schemaVersion: SUBSYSTEMS_SCHEMA_VERSION,
|
|
142
|
+
entries: {
|
|
143
|
+
memory: {
|
|
144
|
+
memoryRoot: memory.memoryRoot,
|
|
145
|
+
invocation: [memory.invocation[0], memory.invocation[1]],
|
|
146
|
+
contractMajor: MEMORY_CONTRACT_MAJOR,
|
|
147
|
+
memoryRef: memory.memoryRef,
|
|
148
|
+
installedVersion: memory.installedVersion,
|
|
149
|
+
},
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
return `${JSON.stringify(ordered, null, 2)}\n`;
|
|
153
|
+
}
|
|
154
|
+
const isString = (value) => typeof value === 'string';
|
|
155
|
+
export function parseSubsystemsManifest(text) {
|
|
156
|
+
let parsed;
|
|
157
|
+
try {
|
|
158
|
+
parsed = JSON.parse(text);
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
163
|
+
if (typeof parsed !== 'object' || parsed === null)
|
|
164
|
+
return null;
|
|
165
|
+
const root = parsed;
|
|
166
|
+
if (root.schemaVersion !== SUBSYSTEMS_SCHEMA_VERSION)
|
|
167
|
+
return null;
|
|
168
|
+
const entries = root.entries;
|
|
169
|
+
if (typeof entries !== 'object' || entries === null)
|
|
170
|
+
return null;
|
|
171
|
+
const memory = entries.memory;
|
|
172
|
+
if (typeof memory !== 'object' || memory === null)
|
|
173
|
+
return null;
|
|
174
|
+
const m = memory;
|
|
175
|
+
const invocation = m.invocation;
|
|
176
|
+
if (!isString(m.memoryRoot) ||
|
|
177
|
+
!Array.isArray(invocation) ||
|
|
178
|
+
invocation.length !== 2 ||
|
|
179
|
+
!invocation.every(isString) ||
|
|
180
|
+
m.contractMajor !== MEMORY_CONTRACT_MAJOR ||
|
|
181
|
+
!(m.memoryRef === null || isString(m.memoryRef)) ||
|
|
182
|
+
!isString(m.installedVersion))
|
|
183
|
+
return null;
|
|
184
|
+
return {
|
|
185
|
+
schemaVersion: SUBSYSTEMS_SCHEMA_VERSION,
|
|
186
|
+
entries: {
|
|
187
|
+
memory: {
|
|
188
|
+
memoryRoot: m.memoryRoot,
|
|
189
|
+
// `every(isString)` above proved both elements are strings, but
|
|
190
|
+
// Array.prototype.every does not narrow a tuple's element type.
|
|
191
|
+
invocation: [invocation[0], invocation[1]],
|
|
192
|
+
contractMajor: MEMORY_CONTRACT_MAJOR,
|
|
193
|
+
// Checked above as `null` or a string; the `||` chain does not narrow `m.memoryRef`.
|
|
194
|
+
memoryRef: m.memoryRef,
|
|
195
|
+
installedVersion: m.installedVersion,
|
|
196
|
+
},
|
|
197
|
+
},
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
/** `null` when absent — no evidence; a present file that does not parse is an error, not silence. */
|
|
201
|
+
export async function readSubsystemsManifest(file) {
|
|
202
|
+
let text;
|
|
203
|
+
try {
|
|
204
|
+
text = await readFile(file, 'utf8');
|
|
205
|
+
}
|
|
206
|
+
catch (error) {
|
|
207
|
+
if (error.code === 'ENOENT')
|
|
208
|
+
return null;
|
|
209
|
+
throw new SubsystemsError('manifest-unreadable', `cannot read ${path.basename(file)}`);
|
|
210
|
+
}
|
|
211
|
+
const manifest = parseSubsystemsManifest(text);
|
|
212
|
+
if (manifest === null)
|
|
213
|
+
throw new SubsystemsError('manifest-unreadable', `${path.basename(file)} is not a schemaVersion ${SUBSYSTEMS_SCHEMA_VERSION} subsystem manifest`);
|
|
214
|
+
return manifest;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Atomic and idempotent: the bytes land in a sibling temp file and are renamed
|
|
218
|
+
* over the target, so a reader never sees a torn manifest and a re-run of the
|
|
219
|
+
* same derivation produces no diff of its own.
|
|
220
|
+
*/
|
|
221
|
+
export async function writeSubsystemsManifest(file, manifest) {
|
|
222
|
+
const dir = path.dirname(file);
|
|
223
|
+
await mkdir(dir, { recursive: true });
|
|
224
|
+
const temp = path.join(dir, `.${path.basename(file)}.tmp-${process.pid}-${randomBytes(4).toString('hex')}`);
|
|
225
|
+
try {
|
|
226
|
+
await writeFile(temp, serializeSubsystemsManifest(manifest));
|
|
227
|
+
await rename(temp, file);
|
|
228
|
+
}
|
|
229
|
+
catch (error) {
|
|
230
|
+
// See packages/cli/test/subsystems.test.ts › "removes its temp file and
|
|
231
|
+
// rethrows when the rename is refused".
|
|
232
|
+
await rm(temp, { force: true });
|
|
233
|
+
throw error;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* What `upgrade` re-runs: the same derivation from the recorded root, the same
|
|
238
|
+
* handshake, and a rewrite only when the handshake is ok. A manifest that is
|
|
239
|
+
* absent is left absent (`setup` is the one writer that creates it); one whose
|
|
240
|
+
* executable no longer answers is left untouched and the result says why.
|
|
241
|
+
*/
|
|
242
|
+
export async function refreshSubsystems(options) {
|
|
243
|
+
const manifest = await readSubsystemsManifest(options.file);
|
|
244
|
+
if (manifest === null)
|
|
245
|
+
return 'absent';
|
|
246
|
+
const previous = manifest.entries.memory;
|
|
247
|
+
let entry;
|
|
248
|
+
try {
|
|
249
|
+
entry = deriveMemoryEntry({
|
|
250
|
+
memoryRoot: previous.memoryRoot,
|
|
251
|
+
nodeExecutable: options.nodeExecutable,
|
|
252
|
+
memoryRef: previous.memoryRef,
|
|
253
|
+
installedVersion: previous.installedVersion,
|
|
254
|
+
}, options.platform);
|
|
255
|
+
}
|
|
256
|
+
catch (error) {
|
|
257
|
+
if (error instanceof SubsystemsError && error.code === 'memory-executable-absent')
|
|
258
|
+
return { status: 'unsupported', reason: 'absent' };
|
|
259
|
+
throw error;
|
|
260
|
+
}
|
|
261
|
+
const result = await handshake(entry, options.run);
|
|
262
|
+
if (result.status !== 'ok')
|
|
263
|
+
return result;
|
|
264
|
+
await writeSubsystemsManifest(options.file, {
|
|
265
|
+
schemaVersion: SUBSYSTEMS_SCHEMA_VERSION,
|
|
266
|
+
entries: { memory: { ...entry, installedVersion: result.version } },
|
|
267
|
+
});
|
|
268
|
+
return 'refreshed';
|
|
269
|
+
}
|
|
@@ -13,3 +13,18 @@ export async function packageVersion() {
|
|
|
13
13
|
const pkg = JSON.parse(await readFile(pkgPath, 'utf8'));
|
|
14
14
|
return pkg.version;
|
|
15
15
|
}
|
|
16
|
+
/**
|
|
17
|
+
* The version of the command contract (`docs/command-contract.md`) this rig
|
|
18
|
+
* implements. Independent of the package version: many releases ship against
|
|
19
|
+
* one contract, and a consumer compares this field's major alone.
|
|
20
|
+
*/
|
|
21
|
+
export const RIG_CONTRACT_VERSION = '1.0';
|
|
22
|
+
/** What `--version --json` answers — the handshake object of the contract. */
|
|
23
|
+
export async function rigHandshake() {
|
|
24
|
+
return {
|
|
25
|
+
schemaVersion: 1,
|
|
26
|
+
name: 'create-agent-rig',
|
|
27
|
+
version: await packageVersion(),
|
|
28
|
+
contractVersion: RIG_CONTRACT_VERSION,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { CAPABILITY_STATES } from '../core/vocabulary.js';
|
|
2
|
+
export const BENCHMARK_CLASSIFICATIONS = Object.freeze([
|
|
3
|
+
'equivalent',
|
|
4
|
+
'intentional-degradation',
|
|
5
|
+
'unsupported',
|
|
6
|
+
]);
|
|
7
|
+
const MEASUREMENT_KINDS = Object.freeze([
|
|
8
|
+
'capability-probe',
|
|
9
|
+
'guard-enforcement',
|
|
10
|
+
'compatibility-rejection',
|
|
11
|
+
]);
|
|
12
|
+
const REQUIRED_SCENARIO_IDS = [
|
|
13
|
+
'real-wiring',
|
|
14
|
+
'unwired-enforcement',
|
|
15
|
+
'disabled-enforcement',
|
|
16
|
+
'narrowed-enforcement',
|
|
17
|
+
'bypassed-enforcement',
|
|
18
|
+
'unreadable-input',
|
|
19
|
+
'hook-input',
|
|
20
|
+
'protected-rulebook',
|
|
21
|
+
'widening',
|
|
22
|
+
'foreign-major',
|
|
23
|
+
];
|
|
24
|
+
const requiredScenarioIds = new Set(REQUIRED_SCENARIO_IDS);
|
|
25
|
+
const scenario = (id, fixture, mutation, action, measurementKind, expectedStates) => Object.freeze({
|
|
26
|
+
id,
|
|
27
|
+
fixture,
|
|
28
|
+
mutation,
|
|
29
|
+
action,
|
|
30
|
+
measurementKind,
|
|
31
|
+
expectedStates: Object.freeze([...expectedStates]),
|
|
32
|
+
});
|
|
33
|
+
export const BENCHMARK_CORPUS = Object.freeze({
|
|
34
|
+
contractMajor: 1,
|
|
35
|
+
scenarios: Object.freeze([
|
|
36
|
+
scenario('real-wiring', 'universal-rig', 'none', 'exercise protected write', 'capability-probe', ['SUPPORTED']),
|
|
37
|
+
scenario('unwired-enforcement', 'universal-rig', 'remove hook registration', 'exercise protected write', 'capability-probe', ['UNSUPPORTED']),
|
|
38
|
+
scenario('disabled-enforcement', 'universal-rig', 'disable all hooks', 'exercise protected write', 'capability-probe', ['UNSUPPORTED']),
|
|
39
|
+
scenario('narrowed-enforcement', 'universal-rig', 'narrow matcher', 'exercise omitted protected write', 'capability-probe', ['DEGRADED']),
|
|
40
|
+
scenario('bypassed-enforcement', 'universal-rig', 'bypass guard command', 'exercise protected write', 'capability-probe', ['INTEGRATION-FAILED']),
|
|
41
|
+
scenario('unreadable-input', 'universal-rig', 'make snapshot unreadable', 'exercise protected write', 'capability-probe', ['INTEGRATION-FAILED']),
|
|
42
|
+
scenario('hook-input', 'universal-rig', 'make tool_input unreadable', 'exercise protected write', 'guard-enforcement', ['SUPPORTED']),
|
|
43
|
+
scenario('protected-rulebook', 'universal-rig', 'none', 'edit protected rulebook', 'guard-enforcement', ['SUPPORTED']),
|
|
44
|
+
scenario('widening', 'universal-rig', 'widen authorization allow-list', 'attempt unauthorized widening', 'guard-enforcement', ['SUPPORTED']),
|
|
45
|
+
scenario('foreign-major', 'contracts/session-messaging/v1/fixtures/negative/envelope-foreign-major.json', 'foreign schema major', 'check policy compatibility', 'compatibility-rejection', ['UNSUPPORTED']),
|
|
46
|
+
]),
|
|
47
|
+
});
|
|
48
|
+
const canonicalScenarios = new Map(BENCHMARK_CORPUS.scenarios.map((entry) => [entry.id, entry]));
|
|
49
|
+
export const classifyCapability = (state) => {
|
|
50
|
+
switch (state) {
|
|
51
|
+
case 'SUPPORTED':
|
|
52
|
+
return { classification: 'equivalent', integrationFailed: false };
|
|
53
|
+
case 'DEGRADED':
|
|
54
|
+
return { classification: 'intentional-degradation', integrationFailed: false };
|
|
55
|
+
case 'UNSUPPORTED':
|
|
56
|
+
return { classification: 'unsupported', integrationFailed: false };
|
|
57
|
+
case 'INTEGRATION-FAILED':
|
|
58
|
+
return { classification: 'unsupported', integrationFailed: true };
|
|
59
|
+
default:
|
|
60
|
+
throw new Error(`Unknown capability state: ${String(state)}`);
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
const isOwnDataRecord = (value) => {
|
|
64
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
65
|
+
return false;
|
|
66
|
+
const prototype = Object.getPrototypeOf(value);
|
|
67
|
+
if (prototype !== Object.prototype && prototype !== null)
|
|
68
|
+
return false;
|
|
69
|
+
return Object.values(Object.getOwnPropertyDescriptors(value)).every((descriptor) => descriptor.enumerable === true && 'value' in descriptor);
|
|
70
|
+
};
|
|
71
|
+
const isDataArray = (value) => {
|
|
72
|
+
if (!Array.isArray(value) || Object.getPrototypeOf(value) !== Array.prototype)
|
|
73
|
+
return false;
|
|
74
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
75
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, String(index));
|
|
76
|
+
if (descriptor === undefined || descriptor.enumerable !== true || !('value' in descriptor))
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
return true;
|
|
80
|
+
};
|
|
81
|
+
const hasOnlyKeys = (value, keys) => {
|
|
82
|
+
const actual = Object.keys(value);
|
|
83
|
+
return actual.length === keys.length && keys.every((key) => Object.hasOwn(value, key));
|
|
84
|
+
};
|
|
85
|
+
const nonBlank = (value) => typeof value === 'string' && value.trim() !== '';
|
|
86
|
+
const invalidCorpus = (reason) => {
|
|
87
|
+
throw new Error(`Invalid benchmark corpus: ${reason}`);
|
|
88
|
+
};
|
|
89
|
+
export const parseBenchmarkCorpus = (input) => {
|
|
90
|
+
if (!isOwnDataRecord(input) || !hasOnlyKeys(input, ['contractMajor', 'scenarios'])) {
|
|
91
|
+
return invalidCorpus('must be a plain corpus with contractMajor and scenarios');
|
|
92
|
+
}
|
|
93
|
+
if (input.contractMajor !== 1)
|
|
94
|
+
return invalidCorpus('unsupported contract major; expected 1');
|
|
95
|
+
if (!isDataArray(input.scenarios))
|
|
96
|
+
return invalidCorpus('scenarios must be a data array');
|
|
97
|
+
if (input.scenarios.length !== REQUIRED_SCENARIO_IDS.length) {
|
|
98
|
+
return invalidCorpus('must include every required scenario exactly once');
|
|
99
|
+
}
|
|
100
|
+
const ids = new Set();
|
|
101
|
+
const scenarios = [];
|
|
102
|
+
for (const value of input.scenarios) {
|
|
103
|
+
if (!isOwnDataRecord(value) ||
|
|
104
|
+
!hasOnlyKeys(value, [
|
|
105
|
+
'id',
|
|
106
|
+
'fixture',
|
|
107
|
+
'mutation',
|
|
108
|
+
'action',
|
|
109
|
+
'measurementKind',
|
|
110
|
+
'expectedStates',
|
|
111
|
+
])) {
|
|
112
|
+
return invalidCorpus('each scenario must have only id, fixture, mutation, action, measurementKind, and expectedStates');
|
|
113
|
+
}
|
|
114
|
+
const { id, fixture, mutation, action, measurementKind, expectedStates } = value;
|
|
115
|
+
if (!nonBlank(id) || !requiredScenarioIds.has(id))
|
|
116
|
+
return invalidCorpus(`unknown scenario ${String(id)}`);
|
|
117
|
+
if (ids.has(id))
|
|
118
|
+
return invalidCorpus(`scenario ${id} is duplicated`);
|
|
119
|
+
if (!nonBlank(fixture) || !nonBlank(mutation) || !nonBlank(action)) {
|
|
120
|
+
return invalidCorpus(`scenario ${id} has empty metadata`);
|
|
121
|
+
}
|
|
122
|
+
if (typeof measurementKind !== 'string' ||
|
|
123
|
+
!MEASUREMENT_KINDS.includes(measurementKind)) {
|
|
124
|
+
return invalidCorpus(`scenario ${id} has an unknown measurement kind`);
|
|
125
|
+
}
|
|
126
|
+
if (!isDataArray(expectedStates) || expectedStates.length === 0) {
|
|
127
|
+
return invalidCorpus(`scenario ${id} must name expected capability states`);
|
|
128
|
+
}
|
|
129
|
+
const states = new Set();
|
|
130
|
+
for (const expectedState of expectedStates) {
|
|
131
|
+
if (typeof expectedState !== 'string' ||
|
|
132
|
+
!CAPABILITY_STATES.includes(expectedState)) {
|
|
133
|
+
return invalidCorpus(`scenario ${id} has an unknown capability state`);
|
|
134
|
+
}
|
|
135
|
+
if (states.has(expectedState))
|
|
136
|
+
return invalidCorpus(`scenario ${id} repeats a capability state`);
|
|
137
|
+
states.add(expectedState);
|
|
138
|
+
}
|
|
139
|
+
const canonical = canonicalScenarios.get(id);
|
|
140
|
+
if (fixture !== canonical.fixture ||
|
|
141
|
+
mutation !== canonical.mutation ||
|
|
142
|
+
action !== canonical.action ||
|
|
143
|
+
measurementKind !== canonical.measurementKind) {
|
|
144
|
+
return invalidCorpus(`scenario ${id} changes its canonical metadata`);
|
|
145
|
+
}
|
|
146
|
+
if (expectedStates.length !== canonical.expectedStates.length ||
|
|
147
|
+
expectedStates.some((state, index) => state !== canonical.expectedStates[index])) {
|
|
148
|
+
return invalidCorpus(`scenario ${id} changes its expected capability states`);
|
|
149
|
+
}
|
|
150
|
+
ids.add(id);
|
|
151
|
+
scenarios.push(Object.freeze({
|
|
152
|
+
id,
|
|
153
|
+
fixture,
|
|
154
|
+
mutation,
|
|
155
|
+
action,
|
|
156
|
+
measurementKind: measurementKind,
|
|
157
|
+
expectedStates: Object.freeze([...states]),
|
|
158
|
+
}));
|
|
159
|
+
}
|
|
160
|
+
for (const id of REQUIRED_SCENARIO_IDS) {
|
|
161
|
+
if (!ids.has(id))
|
|
162
|
+
return invalidCorpus(`required scenario ${id} is missing`);
|
|
163
|
+
}
|
|
164
|
+
return Object.freeze({ contractMajor: 1, scenarios: Object.freeze(scenarios) });
|
|
165
|
+
};
|