@event4u/agent-config 7.0.1 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +8 -1
- package/CHANGELOG.md +99 -0
- package/README.md +4 -2
- package/dist/agent-src/commands/council/default.md +28 -20
- package/dist/agent-src/commands/evals/fix.json +5 -1
- package/dist/agent-src/commands/fix/comments.md +101 -0
- package/dist/agent-src/commands/fix.md +5 -2
- package/dist/agent-src/contexts/execution/auto-dispatch-classification.md +65 -0
- package/dist/agent-src/contexts/execution/auto-orchestration-activation.md +68 -0
- package/dist/agent-src/contexts/execution/host-capability-manifest.md +68 -0
- package/dist/agent-src/contexts/execution/orchestration-benchmark-gate.md +62 -0
- package/dist/agent-src/contexts/execution/orchestration-telemetry.md +59 -0
- package/dist/agent-src/contexts/execution/roadmap-process-loop.md +8 -3
- package/dist/agent-src/contexts/execution/subagent-response-contract.md +56 -0
- package/dist/agent-src/contexts/execution/subagent-routing.md +57 -0
- package/dist/agent-src/contexts/execution/subagent-spawn-contract.md +54 -0
- package/dist/agent-src/contexts/execution/subagent-steering.md +71 -0
- package/dist/agent-src/contexts/execution/verify-budget.md +50 -0
- package/dist/agent-src/rules/delegation-policy.md +84 -0
- package/dist/agent-src/rules/design-fidelity.md +98 -0
- package/dist/agent-src/rules/domain-safety-pii.md +22 -0
- package/dist/agent-src/rules/legal-safety-floor.md +144 -0
- package/dist/agent-src/rules/lethal-trifecta-guard.md +25 -22
- package/dist/agent-src/scripts/archive_completed_roadmaps.ts +102 -15
- package/dist/agent-src/scripts/update_roadmap_progress.ts +52 -7
- package/dist/agent-src/skills/ai-council/SKILL.md +28 -1
- package/dist/agent-src/skills/contract-review/SKILL.md +124 -0
- package/dist/agent-src/skills/contract-review/evals/triggers.json +17 -0
- package/dist/agent-src/skills/dpa-review/SKILL.md +129 -0
- package/dist/agent-src/skills/dpa-review/evals/triggers.json +17 -0
- package/dist/agent-src/skills/laravel/SKILL.md +1 -0
- package/dist/agent-src/skills/legal-intake-triage/SKILL.md +127 -0
- package/dist/agent-src/skills/legal-intake-triage/evals/triggers.json +17 -0
- package/dist/agent-src/skills/legal-practice-profile/SKILL.md +100 -0
- package/dist/agent-src/skills/legal-practice-profile/evals/triggers.json +17 -0
- package/dist/agent-src/skills/nda-triage/SKILL.md +113 -0
- package/dist/agent-src/skills/nda-triage/evals/triggers.json +17 -0
- package/dist/agent-src/skills/reasoning-orchestrator/SKILL.md +36 -9
- package/dist/agent-src/skills/recursive-verification/SKILL.md +186 -0
- package/dist/agent-src/skills/roadmap-management/SKILL.md +17 -0
- package/dist/agent-src/skills/subagent-orchestration/SKILL.md +14 -0
- package/dist/agent-src/templates/agent-settings.md +64 -19
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +278 -23
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +12 -4
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +50 -8
- package/dist/discovery/trust-report.md +7 -5
- package/dist/discovery/workspaces.json +34 -6
- package/dist/mcp/registry-manifest.json +3 -3
- package/dist/router.json +1 -1
- package/dist/server/schemas/settings.js +14 -0
- package/dist/server/schemas/settings.js.map +1 -1
- package/docs/architecture.md +4 -4
- package/docs/benchmark.md +46 -0
- package/docs/catalog.md +18 -6
- package/docs/command-flows.md +7 -1
- package/docs/contracts/ai-council-config.md +17 -10
- package/docs/contracts/audit-log-v1.md +7 -0
- package/docs/contracts/auto-orchestration-v1.md +70 -0
- package/docs/contracts/command-clusters.md +1 -1
- package/docs/contracts/discovery-manifest.schema.json +2 -2
- package/docs/decisions/ADR-013-discovery-frontmatter-contract.md +19 -0
- package/docs/decisions/ADR-020-global-only-consumer-scope.md +13 -6
- package/docs/decisions/ADR-049-configuration-trust-boundary.md +11 -1
- package/docs/decisions/ADR-093-ai-council-config-user-global.md +10 -4
- package/docs/decisions/ADR-104-ai-council-config-global-only.md +119 -0
- package/docs/decisions/ADR-105-automatic-subagent-orchestration.md +102 -0
- package/docs/decisions/ADR-106-recursive-verification-benchmark-gate.md +126 -0
- package/docs/decisions/ADR-107-legal-domain-pack-adoption.md +51 -0
- package/docs/decisions/ADR-108-open-source-forever-no-commercial-tier.md +46 -0
- package/docs/decisions/INDEX.md +5 -0
- package/docs/getting-started.md +2 -2
- package/docs/guidelines/agent-infra/domain-eval-anti-pattern.md +53 -0
- package/docs/guidelines/agent-infra/domain-pack-architecture.md +55 -0
- package/package.json +1 -1
- package/src/config/agent-settings.template.yml +72 -3
- package/src/config/discovery/packs.yml +13 -0
- package/src/config/discovery/workspaces.yml +7 -0
- package/src/scripts/_lib/auto_dispatch.ts +95 -0
- package/src/scripts/_lib/host_capability.ts +59 -0
- package/src/scripts/_lib/orchestration_gate.ts +98 -0
- package/src/scripts/_lib/subagent_bundle.ts +154 -0
- package/src/scripts/_lib/subagent_response.ts +82 -0
- package/src/scripts/_lib/subagent_routing.ts +81 -0
- package/src/scripts/_lib/subagent_spawn.ts +97 -0
- package/src/scripts/_lib/subagent_steering.ts +64 -0
- package/src/scripts/_lib/verify_budget.ts +54 -0
- package/src/scripts/ai_council/advisors.ts +4 -2
- package/src/scripts/ai_council/clients.ts +7 -1
- package/src/scripts/ai_council/config.ts +22 -17
- package/src/scripts/bench_ab_v2_run.ts +127 -0
- package/src/scripts/bench_ab_v2_stats.ts +39 -0
- package/src/scripts/check_council_config_location.ts +6 -5
- package/src/scripts/lint_discovery_vocabulary.ts +6 -0
- package/src/scripts/lint_framework_leakage_allowlist.json +21 -0
- package/src/scripts/lint_legal_pack.ts +139 -0
- package/src/scripts/probe_projection_fidelity.ts +7 -1
- package/src/scripts/rdp_gate_classify.ts +131 -0
- package/src/scripts/rdp_quality_eval.ts +437 -0
- package/src/scripts/schemas/agent-settings.schema.json +11 -0
- package/src/scripts/skill_linter.ts +3 -0
|
@@ -67,14 +67,20 @@ type Dict = Record<string, unknown>;
|
|
|
67
67
|
interface ArmSpec {
|
|
68
68
|
setting_sources: string | null;
|
|
69
69
|
inject: string | null; // None | "rdp" | "placebo"
|
|
70
|
+
recursive?: boolean; // ADR-106 D₂ arm: depth-bounded attempt→critic→re-attempt loop
|
|
70
71
|
}
|
|
71
72
|
|
|
72
73
|
// Arm -> (setting_sources, inject) where inject ∈ {None, "rdp", "placebo"}.
|
|
74
|
+
// `package-recursive` (ADR-106's D₂) reuses the real-plugin `package` config but
|
|
75
|
+
// runs the recursion loop instead of a single --print. It only executes when
|
|
76
|
+
// explicitly selected (`--arms package-recursive`); the four default arms are
|
|
77
|
+
// untouched, so existing runs and their golden-parity outputs are unchanged.
|
|
73
78
|
const ARMS: Record<string, ArmSpec> = {
|
|
74
79
|
vanilla: { setting_sources: 'project,local', inject: null },
|
|
75
80
|
package: { setting_sources: null, inject: null },
|
|
76
81
|
'package-rdp': { setting_sources: null, inject: 'rdp' },
|
|
77
82
|
placebo: { setting_sources: 'project,local', inject: 'placebo' },
|
|
83
|
+
'package-recursive': { setting_sources: null, inject: null, recursive: true },
|
|
78
84
|
};
|
|
79
85
|
|
|
80
86
|
/**
|
|
@@ -165,10 +171,14 @@ interface RunOneOpts {
|
|
|
165
171
|
timeout: number;
|
|
166
172
|
placebo_chars: number;
|
|
167
173
|
sp_dir: string;
|
|
174
|
+
max_depth?: number | null; // recursion arm only — hard cap on correction rounds (default 1)
|
|
168
175
|
}
|
|
169
176
|
|
|
170
177
|
export function run_one(task: Dict, arm: string, opts: RunOneOpts): Dict {
|
|
171
178
|
const spec = ARMS[arm] as ArmSpec;
|
|
179
|
+
if (spec.recursive) {
|
|
180
|
+
return run_one_recursive(task, opts);
|
|
181
|
+
}
|
|
172
182
|
const [clone, fixture] = reset_fixture(task);
|
|
173
183
|
const sp_text = injected_text(spec.inject, opts.placebo_chars);
|
|
174
184
|
let sp_file: string | null = null;
|
|
@@ -198,6 +208,123 @@ export function run_one(task: Dict, arm: string, opts: RunOneOpts): Dict {
|
|
|
198
208
|
};
|
|
199
209
|
}
|
|
200
210
|
|
|
211
|
+
export interface RecursiveAttempt {
|
|
212
|
+
run: Dict;
|
|
213
|
+
score: scoring.ScoreResultV2;
|
|
214
|
+
output?: string; // pair-capture: changed-file contents at this depth (for human-preference judging)
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* ADR-106 D₂ arm — depth-bounded recursive self-verification.
|
|
219
|
+
*
|
|
220
|
+
* Loop: attempt → critic verdict → conditional re-attempt, depth = hard compute
|
|
221
|
+
* cap. The critic is the DETERMINISTIC v2 scorer (accept iff the attempt is both
|
|
222
|
+
* capability- AND discipline-passing) — cheapest critic, no extra model call,
|
|
223
|
+
* fully testable. A model-based / cross-vendor critic is a later option (see
|
|
224
|
+
* road-to-recursive-verification.md Phase 4).
|
|
225
|
+
*
|
|
226
|
+
* `attemptFn(depth, priorVerdict)` is an injectable seam: the default runs the
|
|
227
|
+
* real `package`-config agent (prior verdict threaded as the system prompt on
|
|
228
|
+
* re-attempts) and scores it; tests inject a scripted attemptFn to exercise the
|
|
229
|
+
* loop control flow with NO live model call.
|
|
230
|
+
*
|
|
231
|
+
* Verifies the loop's CONTROL FLOW (stop conditions, depth cap, verdict
|
|
232
|
+
* threading). Whether recursion actually lifts capability/discipline is the
|
|
233
|
+
* empirical question the live `bench:ab` run answers — not asserted here.
|
|
234
|
+
*/
|
|
235
|
+
export function run_one_recursive(
|
|
236
|
+
task: Dict,
|
|
237
|
+
opts: RunOneOpts,
|
|
238
|
+
attemptFn?: (depth: number, priorVerdict: string | null) => RecursiveAttempt,
|
|
239
|
+
onAttempt?: (depth: number, attempt: RecursiveAttempt) => void,
|
|
240
|
+
): Dict {
|
|
241
|
+
const maxDepth = Math.max(0, opts.max_depth ?? 1);
|
|
242
|
+
const makeAttempt =
|
|
243
|
+
attemptFn ??
|
|
244
|
+
((depth: number, priorVerdict: string | null): RecursiveAttempt => {
|
|
245
|
+
const [clone, fixture] = reset_fixture(task);
|
|
246
|
+
let sp_file: string | null = null;
|
|
247
|
+
if (priorVerdict) {
|
|
248
|
+
sp_file = path.join(opts.sp_dir, `.sp-recursive-${depth}.txt`);
|
|
249
|
+
fs.writeFileSync(sp_file, priorVerdict, { encoding: 'utf-8' });
|
|
250
|
+
}
|
|
251
|
+
const run = v1.run_live(task, clone, opts.timeout, {
|
|
252
|
+
sysprompt_file: sp_file,
|
|
253
|
+
setting_sources: (ARMS['package'] as ArmSpec).setting_sources,
|
|
254
|
+
max_budget: opts.max_budget,
|
|
255
|
+
model: opts.model,
|
|
256
|
+
}) as unknown as Dict;
|
|
257
|
+
const score = scoring.score_task_v2(task, {
|
|
258
|
+
fixture_root: fixture,
|
|
259
|
+
clone_root: clone,
|
|
260
|
+
transcript: String(run['transcript'] ?? ''),
|
|
261
|
+
});
|
|
262
|
+
// Capture the attempt's output (changed-file contents) for the
|
|
263
|
+
// human-preference pair, before the next depth resets the clone.
|
|
264
|
+
const output = score.files_changed
|
|
265
|
+
.map((f: string) => {
|
|
266
|
+
try {
|
|
267
|
+
return `--- ${f} ---\n${fs.readFileSync(path.join(clone, f), 'utf-8')}`;
|
|
268
|
+
} catch {
|
|
269
|
+
return `--- ${f} --- (unreadable)`;
|
|
270
|
+
}
|
|
271
|
+
})
|
|
272
|
+
.join('\n\n')
|
|
273
|
+
.slice(0, 4000);
|
|
274
|
+
return { run, score, output };
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
const accepts = (s: scoring.ScoreResultV2): boolean => Boolean(s.capability_pass) && Boolean(s.discipline_pass);
|
|
278
|
+
|
|
279
|
+
let depth = 0;
|
|
280
|
+
let prior: RecursiveAttempt | null = null;
|
|
281
|
+
let attempt = makeAttempt(0, null);
|
|
282
|
+
onAttempt?.(0, attempt);
|
|
283
|
+
let stop_reason: string;
|
|
284
|
+
let last_verdict_len = 0;
|
|
285
|
+
for (;;) {
|
|
286
|
+
const s = attempt.score;
|
|
287
|
+
if (accepts(s)) {
|
|
288
|
+
stop_reason = 'accept';
|
|
289
|
+
break;
|
|
290
|
+
}
|
|
291
|
+
if (depth >= maxDepth) {
|
|
292
|
+
stop_reason = 'max_depth';
|
|
293
|
+
break;
|
|
294
|
+
}
|
|
295
|
+
if (
|
|
296
|
+
prior !== null &&
|
|
297
|
+
s.discipline_score === prior.score.discipline_score &&
|
|
298
|
+
Boolean(s.capability_pass) === Boolean(prior.score.capability_pass)
|
|
299
|
+
) {
|
|
300
|
+
stop_reason = 'no_progress';
|
|
301
|
+
break;
|
|
302
|
+
}
|
|
303
|
+
depth += 1;
|
|
304
|
+
const verdict =
|
|
305
|
+
`Prior attempt (depth ${depth - 1}) did not pass: ` +
|
|
306
|
+
`capability=${s.capability_pass}, discipline=${s.discipline_score}. ` +
|
|
307
|
+
`Revise to satisfy the unmet acceptance criteria; change only what the task requires.`;
|
|
308
|
+
last_verdict_len = verdict.length;
|
|
309
|
+
prior = attempt;
|
|
310
|
+
attempt = makeAttempt(depth, verdict);
|
|
311
|
+
onAttempt?.(depth, attempt);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
const { run, score } = attempt;
|
|
315
|
+
return {
|
|
316
|
+
errored: _pyBool(run['errored']),
|
|
317
|
+
reason: run['reason'] ?? null,
|
|
318
|
+
capability_pass: score.capability_pass,
|
|
319
|
+
discipline_score: new PyFloat(score.discipline_score),
|
|
320
|
+
discipline_pass: score.discipline_pass,
|
|
321
|
+
metrics: trajectory_metrics(run, score),
|
|
322
|
+
injected_chars: last_verdict_len,
|
|
323
|
+
depth_reached: depth,
|
|
324
|
+
stop_reason,
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
|
|
201
328
|
export function main(argv: string[] | null = null): number {
|
|
202
329
|
const args = parse_args(argv ?? process.argv.slice(2));
|
|
203
330
|
|
|
@@ -41,6 +41,11 @@ const COMPARISONS: Array<[string, string, string]> = [
|
|
|
41
41
|
['package', 'vanilla', 'package lift'],
|
|
42
42
|
['package-rdp', 'package', 'RDP lift'],
|
|
43
43
|
['package', 'placebo', 'attribution (content vs length)'],
|
|
44
|
+
// ADR-106: recursion's NOVEL lift over rules-only (D₂ − D₁). The analyse()
|
|
45
|
+
// loop is arm-guarded (`arms.includes(t) && arms.includes(b)`), so this row
|
|
46
|
+
// renders ONLY when a `package-recursive` arm is present — existing runs
|
|
47
|
+
// (no such arm) are byte-identical, golden parity preserved.
|
|
48
|
+
['package-recursive', 'package', 'recursion novel lift (D₂ − D₁)'],
|
|
44
49
|
];
|
|
45
50
|
|
|
46
51
|
// ── CPython math.erf / math.comb ports ────────────────────────────────────
|
|
@@ -333,6 +338,40 @@ export function compare(records: Dict[], arm_t: string, arm_b: string): Dict {
|
|
|
333
338
|
};
|
|
334
339
|
}
|
|
335
340
|
|
|
341
|
+
/**
|
|
342
|
+
* Recursion's NOVEL discipline lift over rules-only (ADR-106).
|
|
343
|
+
*
|
|
344
|
+
* Compares the `package-recursive` arm (D₂ = rules + recursion) against
|
|
345
|
+
* `package` (D₁ = rules only) via the existing pure `compare()`, so recursion's
|
|
346
|
+
* marginal lift `D₂ − D₁` is a measured fact, never an assumed redundancy.
|
|
347
|
+
*
|
|
348
|
+
* ADDITIVE: does NOT touch the rendered COMPARISONS path — existing CLI output
|
|
349
|
+
* (and its golden-parity tests) are unaffected. Returns `null` when no
|
|
350
|
+
* `package-recursive` arm is present in the records, so a future renderer can
|
|
351
|
+
* skip the block cleanly for non-recursion runs.
|
|
352
|
+
*
|
|
353
|
+
* NOTE (Phase 2b live-integration, NOT done here): wiring this into the rendered
|
|
354
|
+
* stats output (a guarded COMPARISONS entry + a golden-parity fixture refresh)
|
|
355
|
+
* and producing the `package-recursive` arm itself (a multi-call recursion loop
|
|
356
|
+
* the single-`--print` runner does not yet support) require the live bench and
|
|
357
|
+
* are deferred to the supervised Phase 3 session.
|
|
358
|
+
*/
|
|
359
|
+
export const RECURSIVE_ARM = 'package-recursive';
|
|
360
|
+
export const RECURSIVE_BASELINE_ARM = 'package';
|
|
361
|
+
|
|
362
|
+
export function recursiveNovelLift(records: Dict[]): Dict | null {
|
|
363
|
+
const hasRecursion = records.some((rec) => {
|
|
364
|
+
const runs = _dictOr(rec['arms'])[RECURSIVE_ARM];
|
|
365
|
+
return Array.isArray(runs) && runs.length > 0;
|
|
366
|
+
});
|
|
367
|
+
if (!hasRecursion) return null;
|
|
368
|
+
return {
|
|
369
|
+
label: 'recursion novel lift (D₂ − D₁)',
|
|
370
|
+
arms: [RECURSIVE_ARM, RECURSIVE_BASELINE_ARM],
|
|
371
|
+
...compare(records, RECURSIVE_ARM, RECURSIVE_BASELINE_ARM),
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
|
|
336
375
|
export function mean_tokens_by_arm(records: Dict[], arms: string[]): Dict {
|
|
337
376
|
// Mean total tokens per arm over non-errored runs (the cost axis, L10).
|
|
338
377
|
// Truncated/errored runs are excluded — their token count is capped by the
|
|
@@ -9,11 +9,12 @@
|
|
|
9
9
|
* and the same `<!-- council-config-allowed -->` escape pragma. No behaviour
|
|
10
10
|
* changes — latent bugs replicated.
|
|
11
11
|
*
|
|
12
|
-
* Per ADR-093 the council reads a dedicated
|
|
13
|
-
*
|
|
14
|
-
* (`~/.event4u/agent-config/settings/.ai-council.yml`)
|
|
15
|
-
*
|
|
16
|
-
*
|
|
12
|
+
* Per ADR-104 (superseding ADR-093) the council reads a dedicated
|
|
13
|
+
* `.ai-council.yml` resolved ALWAYS from the user-global location
|
|
14
|
+
* (`~/.event4u/agent-config/settings/.ai-council.yml`) — the project tree
|
|
15
|
+
* is never searched (the only escape is the explicit `$AI_COUNCIL_CONFIG`
|
|
16
|
+
* path). Keys are top-level in that file; the legacy `ai_council.*` block
|
|
17
|
+
* under `.agent-settings.yml` was removed in Phase 0.
|
|
17
18
|
*
|
|
18
19
|
* What it flags, in the council command/skill surfaces + the config contract:
|
|
19
20
|
*
|
|
@@ -29,6 +29,8 @@ const PACKS_YML = path.join(REPO_ROOT, 'src', 'config', 'discovery', 'packs.yml'
|
|
|
29
29
|
const ADR_WORKSPACES: ReadonlySet<string> = new Set<string>([
|
|
30
30
|
'engineering', 'product', 'finance', 'founder', 'gtm', 'ops',
|
|
31
31
|
'small-business', 'construction', 'agent-config-maintainer',
|
|
32
|
+
// road-to-legal-pack Phase 0 (ADR-013 amendment, same PR).
|
|
33
|
+
'legal',
|
|
32
34
|
]);
|
|
33
35
|
const ADR_PACKS: ReadonlySet<string> = new Set<string>([
|
|
34
36
|
'engineering-base', 'php', 'laravel', 'symfony', 'javascript',
|
|
@@ -40,6 +42,10 @@ const ADR_PACKS: ReadonlySet<string> = new Set<string>([
|
|
|
40
42
|
'memory', 'analytics', 'product-reasoning',
|
|
41
43
|
// road-to-image-brand-typography Phase B (ADR-013 amendment, same PR).
|
|
42
44
|
'brand',
|
|
45
|
+
// Pre-existing drift restore: present in packs.yml but missing from this set.
|
|
46
|
+
'ai-image', 'analysis-workbench',
|
|
47
|
+
// road-to-legal-pack Phase 0 (ADR-013 amendment, same PR).
|
|
48
|
+
'legal',
|
|
43
49
|
]);
|
|
44
50
|
|
|
45
51
|
// ADR-010 non-overlap reservations.
|
|
@@ -577,6 +577,27 @@
|
|
|
577
577
|
24
|
|
578
578
|
],
|
|
579
579
|
"reason": "Single Laravel-shape example for per-module agents directory (road-to-configurable-modules Phase D)"
|
|
580
|
+
},
|
|
581
|
+
{
|
|
582
|
+
"file": "src/skills/code-review/SKILL.md",
|
|
583
|
+
"lines": [
|
|
584
|
+
214
|
|
585
|
+
],
|
|
586
|
+
"reason": "Cross-stack formatter enumeration (ECS, Prettier, Ruff, gofmt) — multi-ecosystem documentation"
|
|
587
|
+
},
|
|
588
|
+
{
|
|
589
|
+
"file": "src/skills/context-document/SKILL.md",
|
|
590
|
+
"lines": [
|
|
591
|
+
82
|
|
592
|
+
],
|
|
593
|
+
"reason": "Laravel example path inside a stack-agnostic module-context table"
|
|
594
|
+
},
|
|
595
|
+
{
|
|
596
|
+
"file": "src/skills/iconography/SKILL.md",
|
|
597
|
+
"lines": [
|
|
598
|
+
29
|
|
599
|
+
],
|
|
600
|
+
"reason": "Cross-stack UI-stack enumeration (React, Blade, Tailwind)"
|
|
580
601
|
}
|
|
581
602
|
]
|
|
582
603
|
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* Lint the legal pack's deterministic safety-floor backstops.
|
|
4
|
+
*
|
|
5
|
+
* The `legal-safety-floor` rule is prose; this linter makes three of its
|
|
6
|
+
* Iron-Law elements machine-checkable over every `packs: [legal]` skill —
|
|
7
|
+
* so they are governance, not just prompt instructions (road-to-legal-pack
|
|
8
|
+
* Phase 1, the "deterministic hardening = differentiator" thesis):
|
|
9
|
+
*
|
|
10
|
+
* 1.1 disclaimer-presence — the exact attorney-review line must appear in
|
|
11
|
+
* the skill body (a skill that drafts legal work must instruct the model
|
|
12
|
+
* to emit it).
|
|
13
|
+
* 1.2 jurisdiction-tag — the skill must reference the machine-checkable
|
|
14
|
+
* `Jurisdiction:` tag (jurisdiction-honesty made deterministic).
|
|
15
|
+
* 1.3 freshness — any legal-pack artefact that DECLARES `last_verified` /
|
|
16
|
+
* `freshness_window` frontmatter must use a valid shape and not be stale
|
|
17
|
+
* (no-op when absent; the mechanism for when positions carry dates).
|
|
18
|
+
*
|
|
19
|
+
* Scope is the legal pack only — it never touches the other 250+ skills.
|
|
20
|
+
* Exit codes: 0 all legal-pack skills compliant, 1 one or more violations.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import * as fs from 'node:fs';
|
|
24
|
+
import * as path from 'node:path';
|
|
25
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
26
|
+
|
|
27
|
+
const QUIET = process.argv.includes('--quiet');
|
|
28
|
+
|
|
29
|
+
const _HERE = fileURLToPath(import.meta.url);
|
|
30
|
+
const REPO = path.resolve(path.dirname(_HERE), '..', '..');
|
|
31
|
+
const SKILLS_DIR = path.join(REPO, 'src', 'skills');
|
|
32
|
+
|
|
33
|
+
// The exact line the floor requires in every legal-pack deliverable.
|
|
34
|
+
const ATTORNEY_LINE = 'Attorney review required on material use';
|
|
35
|
+
const JURISDICTION_TAG = 'Jurisdiction:';
|
|
36
|
+
const FRESHNESS_WINDOW_RE = /^\d+\s+(day|days|month|months|year|years)$/;
|
|
37
|
+
|
|
38
|
+
// D4 — definitive-legal-language blocklist (the floor bans these; this is the
|
|
39
|
+
// deterministic backstop over skill bodies). Narrow on purpose; the floor's
|
|
40
|
+
// prompt layer is the primary enforcement.
|
|
41
|
+
const DEFINITIVE_PATTERNS: readonly (readonly [RegExp, string])[] = [
|
|
42
|
+
[/\bthis (contract|nda|agreement|dpa|clause) is (valid|enforceable|compliant|legal|illegal)\b/i, 'definitive validity/compliance verdict'],
|
|
43
|
+
[/\b(is|are) (fully |now )?gdpr[- ]compliant\b/i, 'definitive GDPR-compliance claim'],
|
|
44
|
+
[/\byou are (legally )?required to\b/i, 'definitive legal obligation'],
|
|
45
|
+
[/\byou will (win|lose|prevail)\b/i, 'outcome prediction'],
|
|
46
|
+
[/\byour chances (of success )?are\b/i, 'success prognosis'],
|
|
47
|
+
];
|
|
48
|
+
// A line is exempt when it is clearly a negative example / guidance / quote —
|
|
49
|
+
// so "Do NOT say 'this contract is valid'" or "Forbidden: …" never false-positive.
|
|
50
|
+
const NEG_EXAMPLE_RE = /(never|avoid|forbidden|do not|don'?t|instead|rather than|not say|→|⚠️)/i;
|
|
51
|
+
|
|
52
|
+
function _print(msg: string): void {
|
|
53
|
+
if (!QUIET) process.stdout.write(`${msg}\n`);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function _readText(p: string): string {
|
|
57
|
+
return fs.readFileSync(p, 'utf-8');
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Return [frontmatterText, body] — frontmatter null when no `---` fence. */
|
|
61
|
+
function _splitFrontmatter(content: string): [string | null, string] {
|
|
62
|
+
if (!content.startsWith('---\n')) return [null, content];
|
|
63
|
+
const end = content.indexOf('\n---', 4);
|
|
64
|
+
if (end === -1) return [null, content];
|
|
65
|
+
return [content.slice(4, end), content.slice(end + 4)];
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** True when the frontmatter `packs:` block lists `legal`. */
|
|
69
|
+
function _isLegalPack(fm: string): boolean {
|
|
70
|
+
// Match a `packs:` block then a `- legal` item before the next top-level key.
|
|
71
|
+
const m = /(^|\n)packs:\s*\n((?:\s*-\s*[^\n]+\n?)+)/.exec(fm);
|
|
72
|
+
if (!m) return false;
|
|
73
|
+
return /(^|\n)\s*-\s*legal\s*$/m.test(m[2] ?? '');
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
interface Violation {
|
|
77
|
+
file: string;
|
|
78
|
+
rule: string;
|
|
79
|
+
msg: string;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function lintLegalPack(skillsDir: string = SKILLS_DIR): Violation[] {
|
|
83
|
+
const violations: Violation[] = [];
|
|
84
|
+
let dirs: string[] = [];
|
|
85
|
+
try {
|
|
86
|
+
dirs = fs.readdirSync(skillsDir).sort();
|
|
87
|
+
} catch {
|
|
88
|
+
return violations;
|
|
89
|
+
}
|
|
90
|
+
for (const d of dirs) {
|
|
91
|
+
const file = path.join(skillsDir, d, 'SKILL.md');
|
|
92
|
+
if (!fs.existsSync(file)) continue;
|
|
93
|
+
const content = _readText(file);
|
|
94
|
+
const [fm, body] = _splitFrontmatter(content);
|
|
95
|
+
if (fm === null || !_isLegalPack(fm)) continue;
|
|
96
|
+
const rel = path.relative(REPO, file);
|
|
97
|
+
|
|
98
|
+
// 1.1 disclaimer-presence
|
|
99
|
+
if (!body.includes(ATTORNEY_LINE)) {
|
|
100
|
+
violations.push({ file: rel, rule: 'disclaimer-presence', msg: `missing the attorney-review line ("${ATTORNEY_LINE}")` });
|
|
101
|
+
}
|
|
102
|
+
// 1.2 jurisdiction-tag
|
|
103
|
+
if (!body.includes(JURISDICTION_TAG)) {
|
|
104
|
+
violations.push({ file: rel, rule: 'jurisdiction-tag', msg: `missing the machine-checkable "${JURISDICTION_TAG}" tag` });
|
|
105
|
+
}
|
|
106
|
+
// 1.3 freshness — only when declared
|
|
107
|
+
const fw = /(^|\n)freshness_window:\s*"?([^"\n]+)"?/.exec(fm);
|
|
108
|
+
if (fw && !FRESHNESS_WINDOW_RE.test((fw[2] ?? '').trim())) {
|
|
109
|
+
violations.push({ file: rel, rule: 'freshness', msg: `freshness_window "${fw[2]}" is not a valid "<N> days|months|years" shape` });
|
|
110
|
+
}
|
|
111
|
+
// 4.1 — no definitive legal language (line-scoped, skips negative examples)
|
|
112
|
+
for (const line of body.split('\n')) {
|
|
113
|
+
if (NEG_EXAMPLE_RE.test(line)) continue;
|
|
114
|
+
for (const [re, label] of DEFINITIVE_PATTERNS) {
|
|
115
|
+
if (re.test(line)) {
|
|
116
|
+
violations.push({ file: rel, rule: 'definitive-language', msg: `${label}: ${line.trim().slice(0, 80)}` });
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return violations;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export function main(): number {
|
|
125
|
+
const violations = lintLegalPack();
|
|
126
|
+
if (violations.length === 0) {
|
|
127
|
+
_print('✅ legal pack — all legal-pack skills carry the attorney-review line + Jurisdiction tag');
|
|
128
|
+
return 0;
|
|
129
|
+
}
|
|
130
|
+
_print(`❌ legal pack — ${violations.length} violation(s):`);
|
|
131
|
+
for (const v of violations) {
|
|
132
|
+
_print(` - [${v.rule}] ${v.file}: ${v.msg}`);
|
|
133
|
+
}
|
|
134
|
+
return 1;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
|
|
138
|
+
process.exit(main());
|
|
139
|
+
}
|
|
@@ -154,7 +154,13 @@ function check_entry(entry: Record<string, unknown>): EntryResult {
|
|
|
154
154
|
for (const [tool, spec] of Object.entries(checks)) {
|
|
155
155
|
const result: CheckResult = { status: 'pass', details: [] };
|
|
156
156
|
const expect_present = 'present' in spec ? Boolean(spec['present']) : true;
|
|
157
|
-
|
|
157
|
+
// Honor an explicit per-tool `location` from the fixture when present
|
|
158
|
+
// (e.g. a renamed command whose projection stem differs from its source
|
|
159
|
+
// stem); otherwise derive the path from the artefact source.
|
|
160
|
+
const explicitLoc = spec['location'] as string | undefined;
|
|
161
|
+
const pathLocated = typeof explicitLoc === 'string'
|
|
162
|
+
? (fs.existsSync(path.join(ROOT, explicitLoc)) ? path.join(ROOT, explicitLoc) : null)
|
|
163
|
+
: locate(tool, out.type, entry['source'] as string);
|
|
158
164
|
|
|
159
165
|
if (tool === 'windsurf' && spec['concatenated_in']) {
|
|
160
166
|
const fp = path.join(ROOT, spec['concatenated_in'] as string);
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* RDP gate-time classification validation (road-to-rdp-frontier-polish Phase 1).
|
|
4
|
+
*
|
|
5
|
+
* The council's blocker on "keep-scoped" (2026-06-22): scoping the orchestrator
|
|
6
|
+
* to interdependent-multi-step work via agent self-assessment is only honest if
|
|
7
|
+
* the agent can classify a task AT GATE TIME — from the prompt alone, before
|
|
8
|
+
* doing the work. This script measures exactly that: the standard host classifies
|
|
9
|
+
* each labelled corpus prompt as multi-step vs single-turn, scored against the
|
|
10
|
+
* ground-truth `mechanism` label. High accuracy → the self-assessment gate is a
|
|
11
|
+
* real control, not vibes. Low accuracy → keep-scoped is not implementable.
|
|
12
|
+
*
|
|
13
|
+
* Direct Messages API via fetch (no SDK dep), reuses the key gate + cost
|
|
14
|
+
* estimate from skill_trigger_eval.ts. Dry-run by default; --confirm to spend
|
|
15
|
+
* (+ RDP_EVAL_ALLOW_NONTTY=1 on a non-tty caller that has confirmed cost).
|
|
16
|
+
*
|
|
17
|
+
* Usage: ./scripts-run src/scripts/rdp_gate_classify --corpus <file> --confirm
|
|
18
|
+
*/
|
|
19
|
+
import * as fs from 'node:fs';
|
|
20
|
+
import { estimate_cost, load_anthropic_key } from './skill_trigger_eval.js';
|
|
21
|
+
|
|
22
|
+
/** stdout helper — the lint config forbids `console.log` (allows warn/error). */
|
|
23
|
+
const emit = (s: string): void => {
|
|
24
|
+
process.stdout.write(`${s}\n`);
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const ANTHROPIC_URL = 'https://api.anthropic.com/v1/messages';
|
|
28
|
+
const ANTHROPIC_VERSION = '2023-06-01';
|
|
29
|
+
|
|
30
|
+
const GATE_SYSTEM = `You are an agent deciding, at the START of a task (before doing any work), whether
|
|
31
|
+
to engage a heavyweight multi-step reasoning orchestrator. You see ONLY the task
|
|
32
|
+
prompt. Classify the task into exactly one bucket using the rdp-gate criterion:
|
|
33
|
+
|
|
34
|
+
- "multi-step": interdependent multi-step work where the ordering and handoffs
|
|
35
|
+
between steps matter (e.g. schema → API → job → UI; a migration with stages; a
|
|
36
|
+
cross-cutting refactor). The orchestrator helps here.
|
|
37
|
+
- "single-turn": a single analytical or lookup question, a naming/explain/yes-no
|
|
38
|
+
ask, or a one-shot edit — work that does NOT decompose into ordered
|
|
39
|
+
interdependent steps. The orchestrator over-processes here.
|
|
40
|
+
|
|
41
|
+
Respond with STRICT JSON only: {"bucket":"multi-step"|"single-turn","confidence":"high"|"medium"|"low"}`;
|
|
42
|
+
|
|
43
|
+
interface Slot {
|
|
44
|
+
n: string;
|
|
45
|
+
slug: string;
|
|
46
|
+
mechanism?: string;
|
|
47
|
+
band: string;
|
|
48
|
+
prompt: string;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function classify(apiKey: string, model: string, prompt: string): Promise<{ bucket: string; confidence: string } | null> {
|
|
52
|
+
const resp = await fetch(ANTHROPIC_URL, {
|
|
53
|
+
method: 'POST',
|
|
54
|
+
headers: {
|
|
55
|
+
'x-api-key': apiKey,
|
|
56
|
+
'anthropic-version': ANTHROPIC_VERSION,
|
|
57
|
+
'content-type': 'application/json',
|
|
58
|
+
},
|
|
59
|
+
body: JSON.stringify({
|
|
60
|
+
model,
|
|
61
|
+
max_tokens: 60,
|
|
62
|
+
system: GATE_SYSTEM,
|
|
63
|
+
messages: [{ role: 'user', content: `Task prompt:\n${prompt}` }],
|
|
64
|
+
}),
|
|
65
|
+
});
|
|
66
|
+
if (!resp.ok) throw new Error(`API ${resp.status}: ${(await resp.text()).slice(0, 300)}`);
|
|
67
|
+
const data = (await resp.json()) as {
|
|
68
|
+
content?: Array<{ type?: string; text?: string }>;
|
|
69
|
+
usage?: { input_tokens?: number; output_tokens?: number };
|
|
70
|
+
};
|
|
71
|
+
const text = (data.content ?? []).filter((b) => b.type === 'text').map((b) => b.text ?? '').join('');
|
|
72
|
+
const m = text.match(/\{[\s\S]*\}/);
|
|
73
|
+
if (!m) return null;
|
|
74
|
+
try {
|
|
75
|
+
return JSON.parse(m[0]) as { bucket: string; confidence: string };
|
|
76
|
+
} catch {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function expectedBucket(mechanism: string | undefined): string {
|
|
82
|
+
return mechanism === 'multi-stage' ? 'multi-step' : 'single-turn';
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function main(): Promise<number> {
|
|
86
|
+
const argv = process.argv.slice(2);
|
|
87
|
+
const args: Record<string, string | boolean> = {};
|
|
88
|
+
for (let i = 0; i < argv.length; i++) {
|
|
89
|
+
const t = argv[i];
|
|
90
|
+
if (t === undefined) continue;
|
|
91
|
+
if (t === '--confirm') args.confirm = true;
|
|
92
|
+
else if (t.startsWith('--')) {
|
|
93
|
+
const next = argv[i + 1];
|
|
94
|
+
if (next && !next.startsWith('--')) { args[t.slice(2)] = next; i++; } else args[t.slice(2)] = true;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
const corpusPath = (args.corpus as string) ?? '';
|
|
98
|
+
const model = (args.model as string) ?? 'claude-haiku-4-5-20251001';
|
|
99
|
+
if (!corpusPath) throw new Error('--corpus <file> required');
|
|
100
|
+
const slots = (JSON.parse(fs.readFileSync(corpusPath, 'utf-8')) as { slots: Slot[] }).slots;
|
|
101
|
+
|
|
102
|
+
const est = slots.reduce((p) => p + estimate_cost(model, 250, 60), 0);
|
|
103
|
+
emit(`rdp-gate-classify · ${slots.length} prompts · ${model} · ~$${est.toFixed(4)}`);
|
|
104
|
+
if (!args.confirm) {
|
|
105
|
+
emit('DRY-RUN — no spend. Re-run with --confirm.');
|
|
106
|
+
return 0;
|
|
107
|
+
}
|
|
108
|
+
if (!process.stdin.isTTY && process.env.RDP_EVAL_ALLOW_NONTTY !== '1') {
|
|
109
|
+
throw new Error('Non-tty: set RDP_EVAL_ALLOW_NONTTY=1 only after confirming cost.');
|
|
110
|
+
}
|
|
111
|
+
const apiKey = load_anthropic_key();
|
|
112
|
+
let correct = 0;
|
|
113
|
+
const rows: Array<Record<string, unknown>> = [];
|
|
114
|
+
for (const s of slots) {
|
|
115
|
+
const r = await classify(apiKey, model, s.prompt);
|
|
116
|
+
const exp = expectedBucket(s.mechanism);
|
|
117
|
+
const got = r?.bucket ?? 'parse-error';
|
|
118
|
+
const ok = got === exp;
|
|
119
|
+
if (ok) correct++;
|
|
120
|
+
console.error(` ${s.n} ${s.slug}: expected ${exp} · got ${got} (${r?.confidence ?? '-'}) ${ok ? 'OK' : 'MISS'}`);
|
|
121
|
+
rows.push({ slot: s.n, slug: s.slug, expected: exp, got, confidence: r?.confidence ?? null, ok });
|
|
122
|
+
}
|
|
123
|
+
const acc = Math.round((correct / slots.length) * 1000) / 10;
|
|
124
|
+
const out = (args.results as string) ?? '/tmp/rdp-gate-classify.json';
|
|
125
|
+
fs.writeFileSync(out, JSON.stringify({ model, n: slots.length, correct, accuracy_pct: acc, rows }, null, 2), 'utf-8');
|
|
126
|
+
emit(`\nGATE-TIME CLASSIFICATION ACCURACY: ${correct}/${slots.length} = ${acc}% (model ${model})`);
|
|
127
|
+
emit(`Results: ${out}`);
|
|
128
|
+
return 0;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
main().then((c) => process.exit(c), (e) => { console.error(e instanceof Error ? e.message : String(e)); process.exit(1); });
|