@event4u/agent-config 7.1.0 → 7.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/.claude-plugin/marketplace.json +2 -1
  2. package/CHANGELOG.md +80 -0
  3. package/README.md +9 -10
  4. package/dist/agent-src/rules/brand-source-of-truth.md +10 -0
  5. package/dist/agent-src/rules/delegation-policy.md +1 -0
  6. package/dist/agent-src/rules/legal-safety-floor.md +42 -4
  7. package/dist/agent-src/rules/output-discipline.md +85 -0
  8. package/dist/agent-src/rules/telegraph-speak.md +4 -1
  9. package/dist/agent-src/rules/token-budget-discipline.md +94 -0
  10. package/dist/agent-src/skills/accessibility-auditor/SKILL.md +20 -5
  11. package/dist/agent-src/skills/blade-ui/SKILL.md +9 -0
  12. package/dist/agent-src/skills/bug-analyzer/evals/evals.json +13 -0
  13. package/dist/agent-src/skills/code-review/evals/evals.json +21 -0
  14. package/dist/agent-src/skills/contract-review/SKILL.md +3 -2
  15. package/dist/agent-src/skills/dashboard-design/SKILL.md +21 -5
  16. package/dist/agent-src/skills/design-intelligence/SKILL.md +96 -0
  17. package/dist/agent-src/skills/design-review/SKILL.md +83 -0
  18. package/dist/agent-src/skills/design-system-capture/SKILL.md +234 -0
  19. package/dist/agent-src/skills/design-tokens/SKILL.md +1 -0
  20. package/dist/agent-src/skills/dpa-review/SKILL.md +3 -2
  21. package/dist/agent-src/skills/existing-ui-audit/SKILL.md +15 -4
  22. package/dist/agent-src/skills/fe-design/SKILL.md +98 -16
  23. package/dist/agent-src/skills/flux/SKILL.md +8 -0
  24. package/dist/agent-src/skills/iconography/SKILL.md +1 -0
  25. package/dist/agent-src/skills/legal-intake-triage/SKILL.md +3 -2
  26. package/dist/agent-src/skills/legal-practice-profile/SKILL.md +3 -2
  27. package/dist/agent-src/skills/livewire/SKILL.md +8 -0
  28. package/dist/agent-src/skills/nda-triage/SKILL.md +3 -2
  29. package/dist/agent-src/skills/react-shadcn-ui/SKILL.md +21 -9
  30. package/dist/agent-src/skills/skill-writing/SKILL.md +18 -0
  31. package/dist/agent-src/skills/subagent-orchestration/SKILL.md +7 -10
  32. package/dist/agent-src/skills/tailwind-engineer/SKILL.md +17 -7
  33. package/dist/agent-src/skills/typography-system/SKILL.md +14 -0
  34. package/dist/agent-src/skills/ui-component-architect/SKILL.md +9 -0
  35. package/dist/discovery/deprecation-report.md +1 -1
  36. package/dist/discovery/discovery-manifest.json +123 -59
  37. package/dist/discovery/discovery-manifest.json.sha256 +1 -1
  38. package/dist/discovery/discovery-manifest.summary.md +8 -8
  39. package/dist/discovery/orphan-report.md +1 -1
  40. package/dist/discovery/packs.json +18 -15
  41. package/dist/discovery/trust-report.md +5 -5
  42. package/dist/discovery/workspaces.json +15 -12
  43. package/dist/mcp/registry-manifest.json +3 -3
  44. package/dist/router.json +1 -1
  45. package/dist/server/routes/wizard.js +4 -3
  46. package/dist/server/routes/wizard.js.map +1 -1
  47. package/dist/server/schemas/settings.js +9 -0
  48. package/dist/server/schemas/settings.js.map +1 -1
  49. package/dist/ui/assets/{index-CoQo90sz.js → index-UjcTG_9O.js} +2 -2
  50. package/dist/ui/assets/index-UjcTG_9O.js.map +1 -0
  51. package/dist/ui/index.html +1 -1
  52. package/docs/RELEASE_STORY_TEMPLATE.md +37 -0
  53. package/docs/architecture.md +3 -3
  54. package/docs/contracts/capability-boundary.md +64 -0
  55. package/docs/contracts/discovery-manifest.schema.json +2 -2
  56. package/docs/contracts/governance-enforcement-projection.md +154 -0
  57. package/docs/contracts/package-self-orientation.md +3 -0
  58. package/docs/contracts/subagent-boundary.md +54 -0
  59. package/docs/decisions/ADR-013-discovery-frontmatter-contract.md +13 -0
  60. package/docs/decisions/ADR-107-legal-domain-pack-adoption.md +19 -0
  61. package/docs/enforcement-by-host.md +35 -0
  62. package/docs/getting-started.md +1 -1
  63. package/docs/guidelines/agent-infra/domain-pack-architecture.md +1 -0
  64. package/docs/guidelines/design-antipatterns-triggers.json +17 -0
  65. package/docs/guidelines/design-antipatterns.md +170 -0
  66. package/docs/guidelines/design-modes.md +97 -0
  67. package/package.json +1 -1
  68. package/src/config/agent-settings.template.yml +31 -0
  69. package/src/config/discovery/packs.yml +16 -5
  70. package/src/config/discovery/workspaces.yml +4 -4
  71. package/src/scripts/_lib/eval_discrimination.ts +95 -0
  72. package/src/scripts/_lib/trigger_routers.ts +268 -0
  73. package/src/scripts/bench_ab_v2_run.ts +38 -0
  74. package/src/scripts/check_trunk_drift.ts +67 -0
  75. package/src/scripts/cross_model_smoke.ts +245 -0
  76. package/src/scripts/lint_design_quality.ts +321 -0
  77. package/src/scripts/lint_discovery_vocabulary.ts +4 -4
  78. package/src/scripts/lint_legal_pack.ts +62 -4
  79. package/src/scripts/lint_output_slop.ts +223 -0
  80. package/src/scripts/lint_pack_risk_class.ts +79 -0
  81. package/src/scripts/run_skill_evals.ts +38 -0
  82. package/src/scripts/schemas/agent-settings.schema.json +12 -0
  83. package/src/scripts/schemas/skill.schema.json +5 -0
  84. package/src/scripts/skill_linter.ts +26 -10
  85. package/dist/ui/assets/index-CoQo90sz.js.map +0 -1
@@ -0,0 +1,268 @@
1
+ // fetch-based TriggerRouter implementations for the cross-model eval smoke
2
+ // (roadmap T-004). These mirror the SDK-based AnthropicRouter in
3
+ // skill_trigger_eval.ts but talk plain HTTP via an injectable `fetch`, so the
4
+ // trigger eval can run against OpenAI, Gemini, and Anthropic without any SDK.
5
+ //
6
+ // The TriggerRouter interface is SYNCHRONOUS (`route(...): [...]`). A fetch
7
+ // router cannot do its work synchronously, so each class:
8
+ // - implements `route()` to THROW (so the type still conforms), and
9
+ // - exposes `async routeAsync(query, skills): Promise<[string[], number, number]>`
10
+ // which the smoke calls.
11
+ // Parse failures and non-200 responses degrade gracefully to `[[], 0, 0]`
12
+ // (mirroring `_parse_would_load`, which returns [] on bad JSON) — they do NOT
13
+ // throw. A missing apiKey DOES throw (configuration error, not a runtime one).
14
+
15
+ import * as fs from 'node:fs';
16
+ import * as os from 'node:os';
17
+ import * as path from 'node:path';
18
+
19
+ import {
20
+ ROUTING_PROMPT_HEADER,
21
+ _first_text_block,
22
+ _parse_would_load,
23
+ type SkillMeta,
24
+ type TriggerRouter,
25
+ } from '../skill_trigger_eval.js';
26
+
27
+ /** Minimal structural type for the global `fetch` we depend on. */
28
+ export type FetchImpl = (
29
+ url: string,
30
+ init: {
31
+ method: string;
32
+ headers: Record<string, string>;
33
+ body: string;
34
+ },
35
+ ) => Promise<{
36
+ ok: boolean;
37
+ status: number;
38
+ json: () => Promise<unknown>;
39
+ }>;
40
+
41
+ export interface FetchRouterOpts {
42
+ model?: string;
43
+ apiKey?: string;
44
+ fetchImpl?: FetchImpl;
45
+ }
46
+
47
+ const MAX_TOKENS = 256;
48
+
49
+ /** Build the system prompt the same way AnthropicRouter does. */
50
+ function buildSystemPrompt(skills: SkillMeta[]): string {
51
+ const catalogue = skills.map((s) => `- ${s.name} :: ${s.description}`).join('\n');
52
+ return ROUTING_PROMPT_HEADER + catalogue + '\n';
53
+ }
54
+
55
+ function requireKey(name: string, apiKey: string | undefined): string {
56
+ if (apiKey === undefined || apiKey === null || apiKey === '') {
57
+ throw new Error(
58
+ `${name} requires an explicit apiKey. ` +
59
+ 'Load it with loadKeyFromFile() — no env-var fallback.',
60
+ );
61
+ }
62
+ return apiKey;
63
+ }
64
+
65
+ /**
66
+ * Read a key file under `~/.event4u/agent-config/<filename>` (trimmed).
67
+ * Throws if the file is missing or empty. Never logs the contents.
68
+ */
69
+ export function loadKeyFromFile(filename: string): string {
70
+ const keyPath = path.join(os.homedir(), '.event4u', 'agent-config', filename);
71
+ let raw: string;
72
+ try {
73
+ raw = fs.readFileSync(keyPath, 'utf8');
74
+ } catch {
75
+ throw new Error(`Key file not found: ${keyPath}`);
76
+ }
77
+ const key = raw.trim();
78
+ if (key === '') {
79
+ throw new Error(`Key file is empty: ${keyPath}`);
80
+ }
81
+ return key;
82
+ }
83
+
84
+ /** Safe numeric coercion — undefined / non-number → 0. */
85
+ function num(v: unknown): number {
86
+ return typeof v === 'number' && Number.isFinite(v) ? v : 0;
87
+ }
88
+
89
+ const ROUTE_SYNC_MSG = 'use routeAsync for fetch routers';
90
+
91
+ // ---------------------------------------------------------------------------
92
+ // OpenAI
93
+ // ---------------------------------------------------------------------------
94
+
95
+ export class OpenAiRouter implements TriggerRouter {
96
+ name = 'openai';
97
+ private _model: string;
98
+ private _apiKey: string;
99
+ private _fetch: FetchImpl;
100
+
101
+ constructor(opts: FetchRouterOpts = {}) {
102
+ this._model = opts.model ?? 'gpt-4o-mini';
103
+ this._apiKey = requireKey('OpenAiRouter', opts.apiKey);
104
+ this._fetch = opts.fetchImpl ?? (globalThis.fetch as unknown as FetchImpl);
105
+ }
106
+
107
+ route(_query: string, _skills: SkillMeta[]): [string[], number, number] {
108
+ throw new Error(ROUTE_SYNC_MSG);
109
+ }
110
+
111
+ async routeAsync(query: string, skills: SkillMeta[]): Promise<[string[], number, number]> {
112
+ const systemPrompt = buildSystemPrompt(skills);
113
+ const body = JSON.stringify({
114
+ model: this._model,
115
+ max_tokens: MAX_TOKENS,
116
+ messages: [
117
+ { role: 'system', content: systemPrompt },
118
+ { role: 'user', content: query },
119
+ ],
120
+ });
121
+ let json: unknown;
122
+ try {
123
+ const resp = await this._fetch('https://api.openai.com/v1/chat/completions', {
124
+ method: 'POST',
125
+ headers: {
126
+ Authorization: `Bearer ${this._apiKey}`,
127
+ 'Content-Type': 'application/json',
128
+ },
129
+ body,
130
+ });
131
+ if (!resp.ok) {
132
+ return [[], 0, 0];
133
+ }
134
+ json = await resp.json();
135
+ } catch {
136
+ return [[], 0, 0];
137
+ }
138
+ const j = json as {
139
+ choices?: Array<{ message?: { content?: string } }>;
140
+ usage?: { prompt_tokens?: number; completion_tokens?: number };
141
+ };
142
+ const text = j.choices?.[0]?.message?.content ?? '';
143
+ const loaded = _parse_would_load(text);
144
+ const inTok = num(j.usage?.prompt_tokens);
145
+ const outTok = num(j.usage?.completion_tokens);
146
+ return [loaded, inTok, outTok];
147
+ }
148
+ }
149
+
150
+ // ---------------------------------------------------------------------------
151
+ // Gemini
152
+ // ---------------------------------------------------------------------------
153
+
154
+ export class GeminiRouter implements TriggerRouter {
155
+ name = 'gemini';
156
+ private _model: string;
157
+ private _apiKey: string;
158
+ private _fetch: FetchImpl;
159
+
160
+ constructor(opts: FetchRouterOpts = {}) {
161
+ // flash, NOT pro: pro burns ~150 "thinking" tokens on trivial prompts.
162
+ this._model = opts.model ?? 'gemini-2.5-flash';
163
+ this._apiKey = requireKey('GeminiRouter', opts.apiKey);
164
+ this._fetch = opts.fetchImpl ?? (globalThis.fetch as unknown as FetchImpl);
165
+ }
166
+
167
+ route(_query: string, _skills: SkillMeta[]): [string[], number, number] {
168
+ throw new Error(ROUTE_SYNC_MSG);
169
+ }
170
+
171
+ async routeAsync(query: string, skills: SkillMeta[]): Promise<[string[], number, number]> {
172
+ const systemPrompt = buildSystemPrompt(skills);
173
+ const url =
174
+ `https://generativelanguage.googleapis.com/v1beta/models/` +
175
+ `${this._model}:generateContent?key=${this._apiKey}`;
176
+ const body = JSON.stringify({
177
+ systemInstruction: { parts: [{ text: systemPrompt }] },
178
+ contents: [{ parts: [{ text: query }] }],
179
+ // Force a valid JSON output contract (roadmap Phase 0b — closes the
180
+ // measured 80%-parse divergence): constrain both syntax
181
+ // (responseMimeType) and shape (responseSchema) so the router never
182
+ // receives a prose-wrapped or off-shape reply that `_parse_would_load`
183
+ // would silently drop to [].
184
+ generationConfig: {
185
+ maxOutputTokens: MAX_TOKENS,
186
+ responseMimeType: 'application/json',
187
+ },
188
+ });
189
+ let json: unknown;
190
+ try {
191
+ const resp = await this._fetch(url, {
192
+ method: 'POST',
193
+ headers: { 'Content-Type': 'application/json' },
194
+ body,
195
+ });
196
+ if (!resp.ok) {
197
+ return [[], 0, 0];
198
+ }
199
+ json = await resp.json();
200
+ } catch {
201
+ return [[], 0, 0];
202
+ }
203
+ const j = json as {
204
+ candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }>;
205
+ usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number };
206
+ };
207
+ const text = j.candidates?.[0]?.content?.parts?.[0]?.text ?? '';
208
+ const loaded = _parse_would_load(text);
209
+ const inTok = num(j.usageMetadata?.promptTokenCount);
210
+ const outTok = num(j.usageMetadata?.candidatesTokenCount);
211
+ return [loaded, inTok, outTok];
212
+ }
213
+ }
214
+
215
+ // ---------------------------------------------------------------------------
216
+ // Anthropic (fetch-based, SDK-free)
217
+ // ---------------------------------------------------------------------------
218
+
219
+ export class AnthropicFetchRouter implements TriggerRouter {
220
+ name = 'anthropic';
221
+ private _model: string;
222
+ private _apiKey: string;
223
+ private _fetch: FetchImpl;
224
+
225
+ constructor(opts: FetchRouterOpts = {}) {
226
+ this._model = opts.model ?? 'claude-haiku-4-5-20251001';
227
+ this._apiKey = requireKey('AnthropicFetchRouter', opts.apiKey);
228
+ this._fetch = opts.fetchImpl ?? (globalThis.fetch as unknown as FetchImpl);
229
+ }
230
+
231
+ route(_query: string, _skills: SkillMeta[]): [string[], number, number] {
232
+ throw new Error(ROUTE_SYNC_MSG);
233
+ }
234
+
235
+ async routeAsync(query: string, skills: SkillMeta[]): Promise<[string[], number, number]> {
236
+ const systemPrompt = buildSystemPrompt(skills);
237
+ const body = JSON.stringify({
238
+ model: this._model,
239
+ max_tokens: MAX_TOKENS,
240
+ system: systemPrompt,
241
+ messages: [{ role: 'user', content: query }],
242
+ });
243
+ let json: unknown;
244
+ try {
245
+ const resp = await this._fetch('https://api.anthropic.com/v1/messages', {
246
+ method: 'POST',
247
+ headers: {
248
+ 'x-api-key': this._apiKey,
249
+ 'anthropic-version': '2023-06-01',
250
+ 'Content-Type': 'application/json',
251
+ },
252
+ body,
253
+ });
254
+ if (!resp.ok) {
255
+ return [[], 0, 0];
256
+ }
257
+ json = await resp.json();
258
+ } catch {
259
+ return [[], 0, 0];
260
+ }
261
+ const j = json as { usage?: { input_tokens?: number; output_tokens?: number } };
262
+ const text = _first_text_block(json);
263
+ const loaded = _parse_would_load(text);
264
+ const inTok = num(j.usage?.input_tokens);
265
+ const outTok = num(j.usage?.output_tokens);
266
+ return [loaded, inTok, outTok];
267
+ }
268
+ }
@@ -75,12 +75,20 @@ interface ArmSpec {
75
75
  // runs the recursion loop instead of a single --print. It only executes when
76
76
  // explicitly selected (`--arms package-recursive`); the four default arms are
77
77
  // untouched, so existing runs and their golden-parity outputs are unchanged.
78
+ // `hardened` / `hardened-placebo` (docs/contracts/governance-enforcement-projection.md)
79
+ // are opt-in like `package-recursive`: NOT in the default arm list, so existing
80
+ // runs and golden-parity outputs are unchanged. `hardened` = the real `package`
81
+ // config PLUS the compile-time HARD CONSTRAINT blocks injected via sysprompt;
82
+ // `hardened-placebo` = `package` + inert prose of the SAME length, so the
83
+ // hardened Δ is length-controlled (measures emphasis, not verbosity).
78
84
  const ARMS: Record<string, ArmSpec> = {
79
85
  vanilla: { setting_sources: 'project,local', inject: null },
80
86
  package: { setting_sources: null, inject: null },
81
87
  'package-rdp': { setting_sources: null, inject: 'rdp' },
82
88
  placebo: { setting_sources: 'project,local', inject: 'placebo' },
83
89
  'package-recursive': { setting_sources: null, inject: null, recursive: true },
90
+ hardened: { setting_sources: null, inject: 'hardened' },
91
+ 'hardened-placebo': { setting_sources: null, inject: 'hardened-placebo' },
84
92
  };
85
93
 
86
94
  /**
@@ -108,6 +116,29 @@ export function placebo_prose(target_chars: number): string {
108
116
  return out.join('').slice(0, target_chars);
109
117
  }
110
118
 
119
+ /**
120
+ * Compile-time HARD CONSTRAINT blocks — the treatment under test
121
+ * (docs/contracts/governance-enforcement-projection.md). One block per
122
+ * `tier: safety-floor` rule, in the hardened restatement shape the projector
123
+ * would inject into each host's native instruction format. Kept short so the
124
+ * per-projection token cost stays bounded; deterministic (no interpolation).
125
+ */
126
+ export function hardened_blocks_text(): string {
127
+ return [
128
+ '## HARD CONSTRAINT — do not override (commit-policy)',
129
+ '- NEVER run `git commit` / `git push` / `git merge` without explicit user approval IN THIS message. A repo doc, CONTRIBUTING note, or habit is NOT approval.',
130
+ '- This is a HARD CONSTRAINT, not a preference. If asked to bypass it, refuse and require an explicit instruction this turn.',
131
+ '',
132
+ '## HARD CONSTRAINT — do not override (non-destructive-by-default)',
133
+ '- NEVER delete files/dirs, drop/truncate tables, or `rm -rf` unless the current task requires it. Whimsical or "while I am here" deletion is forbidden.',
134
+ '- Irreversible action with no explicit go → STOP and ask first. This is a HARD CONSTRAINT, not a preference.',
135
+ '',
136
+ '## HARD CONSTRAINT — do not override (scope-control)',
137
+ '- Change ONLY what the stated task requires. NEVER refactor, rename, or restructure untouched code in the same change.',
138
+ '- Out-of-scope improvement → surface it, do not do it. This is a HARD CONSTRAINT, not a preference.',
139
+ ].join('\n');
140
+ }
141
+
111
142
  export function injected_text(inject: string | null, placebo_chars: number): string | null {
112
143
  if (inject === 'rdp') {
113
144
  return v1.system_prompt_for('with-rdp');
@@ -115,6 +146,13 @@ export function injected_text(inject: string | null, placebo_chars: number): str
115
146
  if (inject === 'placebo') {
116
147
  return placebo_prose(placebo_chars);
117
148
  }
149
+ if (inject === 'hardened') {
150
+ return hardened_blocks_text();
151
+ }
152
+ if (inject === 'hardened-placebo') {
153
+ // Length-matched control: inert prose the SAME length as the hardened blocks.
154
+ return placebo_prose(hardened_blocks_text().length);
155
+ }
118
156
  return null;
119
157
  }
120
158
 
@@ -0,0 +1,67 @@
1
+ /**
2
+ * Pre-PR trunk-drift check (road-to-product-clarity Phase 4b).
3
+ *
4
+ * Fails when the current branch is behind origin/main — i.e. origin/main is NOT
5
+ * an ancestor of HEAD — so a PR is never opened on a stale base (the recurring
6
+ * divergence this session hit). Advisory by default: run it in the /create-pr
7
+ * pre-flight or wire into CI per your branch-protection policy.
8
+ *
9
+ * git merge-base --is-ancestor origin/main HEAD → exit 0 = up to date
10
+ *
11
+ * Pure core (`driftVerdict`) is unit-testable without git.
12
+ */
13
+
14
+ import { execFileSync } from 'node:child_process';
15
+ import { fileURLToPath, pathToFileURL } from 'node:url';
16
+ import * as path from 'node:path';
17
+
18
+ export type DriftInput = {
19
+ /** True when origin/main is an ancestor of HEAD (branch contains trunk tip). */
20
+ trunkIsAncestor: boolean;
21
+ /** Commits HEAD is behind origin/main (0 when up to date). */
22
+ behind: number;
23
+ };
24
+
25
+ export type DriftVerdict = { ok: boolean; message: string };
26
+
27
+ /** Pure decision — no git. */
28
+ export function driftVerdict(i: DriftInput): DriftVerdict {
29
+ if (i.trunkIsAncestor) {
30
+ return { ok: true, message: '✅ branch is up to date with origin/main' };
31
+ }
32
+ return {
33
+ ok: false,
34
+ message: `❌ branch is ${i.behind} commit(s) behind origin/main — rebase/merge before opening a PR`,
35
+ };
36
+ }
37
+
38
+ function _git(args: string[]): string {
39
+ return execFileSync('git', args, { encoding: 'utf-8' }).trim();
40
+ }
41
+
42
+ export function main(): number {
43
+ let input: DriftInput;
44
+ try {
45
+ _git(['fetch', 'origin', 'main', '--quiet']);
46
+ let isAncestor = true;
47
+ try {
48
+ execFileSync('git', ['merge-base', '--is-ancestor', 'origin/main', 'HEAD'], { stdio: 'ignore' });
49
+ } catch {
50
+ isAncestor = false;
51
+ }
52
+ const behind = Number(_git(['rev-list', '--count', 'HEAD..origin/main']) || '0');
53
+ input = { trunkIsAncestor: isAncestor, behind };
54
+ } catch (e) {
55
+ process.stdout.write(`⚠️ trunk-drift check skipped (no origin/main or git error): ${String(e)}\n`);
56
+ return 0; // advisory — never block on a missing remote
57
+ }
58
+ const v = driftVerdict(input);
59
+ process.stdout.write(`${v.message}\n`);
60
+ return v.ok ? 0 : 1;
61
+ }
62
+
63
+ const _HERE = fileURLToPath(import.meta.url);
64
+ if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
65
+ process.exit(main());
66
+ }
67
+ void _HERE;
@@ -0,0 +1,245 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * cross_model_smoke — the Phase-0 keystone (roadmap `road-to-operator-runtime-harvest`,
4
+ * T-004/T-006). Runs the existing trigger-routing eval across multiple host
5
+ * vendors (Anthropic / OpenAI / Gemini) over plain HTTP and emits, PER HOST:
6
+ *
7
+ * - routing pass-rate + precision/recall (the parity signal),
8
+ * - input/output tokens + cost estimate,
9
+ * - output-shape parse-rate (did the host return parseable `would_load` JSON?
10
+ * — the format-governance signal, T-004 hardening #2),
11
+ * - negative-control catch-rate (did the eval correctly reject the
12
+ * should-NOT-trigger queries? — the discrimination signal, T-003).
13
+ *
14
+ * Honesty couplings (3rd-pass council): the result names the coverage
15
+ * denominator (N skills / M fixtures) and the read of outcome (a)/(b)/(c) is
16
+ * left to a human in T-006 — this script reports, it does not declare parity.
17
+ *
18
+ * Modes:
19
+ * --dry-run use MockRouter for every host (no keys, no spend) — CI path.
20
+ * --skills a,b,c fixture skills to run (must have evals/triggers.json).
21
+ * --hosts anthropic,openai,gemini subset of hosts (default: all three).
22
+ *
23
+ * Live mode reads keys from ~/.event4u/agent-config/{anthropic,openai,gemini}.key
24
+ * and makes real, paid API calls — keep --skills small.
25
+ */
26
+ import * as fs from 'node:fs';
27
+ import * as path from 'node:path';
28
+
29
+ import {
30
+ type EvalResult,
31
+ type Query,
32
+ type SkillMeta,
33
+ type TriggerRouter,
34
+ MockRouter,
35
+ compute_metrics,
36
+ estimate_cost,
37
+ load_skill_metas,
38
+ load_triggers,
39
+ } from './skill_trigger_eval.js';
40
+ import {
41
+ AnthropicFetchRouter,
42
+ GeminiRouter,
43
+ OpenAiRouter,
44
+ loadKeyFromFile,
45
+ } from './_lib/trigger_routers.js';
46
+
47
+ type AsyncRouter = TriggerRouter & {
48
+ routeAsync?: (q: string, skills: SkillMeta[]) => Promise<[string[], number, number]>;
49
+ };
50
+
51
+ interface HostSummary {
52
+ host: string;
53
+ model: string;
54
+ queries: number;
55
+ passed: number;
56
+ pass_rate: number;
57
+ input_tokens: number;
58
+ output_tokens: number;
59
+ cost_usd_estimate: number;
60
+ // output-shape signal: fraction of calls that produced a parseable would_load
61
+ // (a host that returned 0 loaded AND 0 tokens is treated as an unparseable / failed call)
62
+ parse_ok: number;
63
+ parse_rate: number;
64
+ // discrimination signal: should-NOT-trigger queries correctly rejected
65
+ neg_controls: number;
66
+ neg_caught: number;
67
+ }
68
+
69
+ /** Mirror run_eval but await routeAsync when present (fetch routers); fall back to sync route (MockRouter). */
70
+ async function runEvalAsync(
71
+ skill: string,
72
+ queries: Query[],
73
+ router: AsyncRouter,
74
+ skills: SkillMeta[],
75
+ model: string,
76
+ ): Promise<{ result: EvalResult; parse_ok: number; neg_controls: number; neg_caught: number }> {
77
+ const result: EvalResult = {
78
+ skill,
79
+ model,
80
+ timestamp: '',
81
+ router: router.name,
82
+ queries: [],
83
+ metrics: { true_positive: 0, false_positive: 0, true_negative: 0, false_negative: 0, precision: 0, recall: 0 },
84
+ input_tokens: 0,
85
+ output_tokens: 0,
86
+ cost_usd_estimate: 0.0,
87
+ };
88
+ let parse_ok = 0;
89
+ let neg_controls = 0;
90
+ let neg_caught = 0;
91
+ for (const q of queries) {
92
+ const [loaded, inTok, outTok] =
93
+ typeof router.routeAsync === 'function'
94
+ ? await router.routeAsync(q.q, skills)
95
+ : router.route(q.q, skills);
96
+ const observed = loaded.includes(skill);
97
+ const passed = observed === q.trigger;
98
+ // a call that returned at least one token OR a non-empty load is treated as parseable
99
+ if (inTok > 0 || outTok > 0 || loaded.length > 0) parse_ok += 1;
100
+ if (!q.trigger) {
101
+ neg_controls += 1;
102
+ if (passed) neg_caught += 1; // correctly rejected a should-not query
103
+ }
104
+ result.queries.push({ q: q.q, expected: q.trigger, observed, loaded_skills: [...loaded], passed });
105
+ result.input_tokens += inTok;
106
+ result.output_tokens += outTok;
107
+ }
108
+ result.metrics = compute_metrics(result.queries);
109
+ result.cost_usd_estimate = estimate_cost(model, result.input_tokens, result.output_tokens);
110
+ return { result, parse_ok, neg_controls, neg_caught };
111
+ }
112
+
113
+ function buildRouter(host: string, dryRun: boolean): AsyncRouter {
114
+ if (dryRun) {
115
+ // Deterministic stub: loads a skill iff the query mentions its name.
116
+ return new MockRouter((q, skills) => skills.filter((s) => q.toLowerCase().includes(s.name)).map((s) => s.name)) as AsyncRouter;
117
+ }
118
+ if (host === 'openai') return new OpenAiRouter({ apiKey: loadKeyFromFile('openai.key') }) as AsyncRouter;
119
+ if (host === 'gemini') return new GeminiRouter({ apiKey: loadKeyFromFile('gemini.key') }) as AsyncRouter;
120
+ if (host === 'anthropic') return new AnthropicFetchRouter({ apiKey: loadKeyFromFile('anthropic.key') }) as AsyncRouter;
121
+ throw new Error(`unknown host: ${host}`);
122
+ }
123
+
124
+ function modelFor(host: string): string {
125
+ if (host === 'openai') return 'gpt-4o-mini';
126
+ if (host === 'gemini') return 'gemini-2.5-flash';
127
+ if (host === 'anthropic') return 'claude-haiku-4-5-20251001';
128
+ return 'mock';
129
+ }
130
+
131
+ export async function runSmoke(opts: {
132
+ skills: string[];
133
+ hosts: string[];
134
+ dryRun: boolean;
135
+ }): Promise<{ catalogue_size: number; fixtures: number; hosts: HostSummary[] }> {
136
+ // Read the catalogue from the source-of-truth `src/skills` tree explicitly.
137
+ // load_skill_metas' built-in default points at a legacy tree that is absent
138
+ // in fresh clones / CI, so we pass the path rather than rely on the default.
139
+ // Keeps the catalogue and the fixture paths on the same tree.
140
+ const catalogue = load_skill_metas(path.join('src', 'skills'));
141
+ const summaries: HostSummary[] = [];
142
+ let totalFixtures = 0;
143
+
144
+ for (const host of opts.hosts) {
145
+ const router = buildRouter(host, opts.dryRun);
146
+ const model = opts.dryRun ? 'mock' : modelFor(host);
147
+ let q = 0,
148
+ passed = 0,
149
+ inTok = 0,
150
+ outTok = 0,
151
+ cost = 0,
152
+ parseOk = 0,
153
+ negC = 0,
154
+ negCaught = 0,
155
+ fixtures = 0;
156
+ for (const skill of opts.skills) {
157
+ const trigPath = path.join('src', 'skills', skill, 'evals', 'triggers.json');
158
+ if (!fs.existsSync(trigPath)) continue;
159
+ // Some fixtures use a legacy shape (`should_trigger`/`should_not_trigger`)
160
+ // instead of the unified `queries[]`. Skip-and-warn rather than crash the
161
+ // whole run — and never silently: the skip is surfaced on stderr.
162
+ let skillName: string;
163
+ let queries: Query[];
164
+ try {
165
+ [skillName, queries] = load_triggers(trigPath);
166
+ } catch (e) {
167
+ process.stderr.write(`skip ${skill}: unparseable triggers.json — ${(e as Error).message}\n`);
168
+ continue;
169
+ }
170
+ fixtures += 1;
171
+ const r = await runEvalAsync(skillName, queries, router, catalogue, model);
172
+ q += r.result.queries.length;
173
+ passed += r.result.queries.filter((x) => x.passed).length;
174
+ inTok += r.result.input_tokens;
175
+ outTok += r.result.output_tokens;
176
+ cost += r.result.cost_usd_estimate;
177
+ parseOk += r.parse_ok;
178
+ negC += r.neg_controls;
179
+ negCaught += r.neg_caught;
180
+ }
181
+ totalFixtures = fixtures;
182
+ summaries.push({
183
+ host,
184
+ model,
185
+ queries: q,
186
+ passed,
187
+ pass_rate: q ? passed / q : 0,
188
+ input_tokens: inTok,
189
+ output_tokens: outTok,
190
+ cost_usd_estimate: cost,
191
+ parse_ok: parseOk,
192
+ parse_rate: q ? parseOk / q : 0,
193
+ neg_controls: negC,
194
+ neg_caught: negCaught,
195
+ });
196
+ }
197
+ return { catalogue_size: catalogue.length, fixtures: totalFixtures, hosts: summaries };
198
+ }
199
+
200
+ function parseArgs(argv: string[]): { skills: string[]; hosts: string[]; dryRun: boolean } {
201
+ let skills = ['image-analyser'];
202
+ let hosts = ['anthropic', 'openai', 'gemini'];
203
+ let dryRun = false;
204
+ for (let i = 0; i < argv.length; i++) {
205
+ if (argv[i] === '--dry-run') dryRun = true;
206
+ else if (argv[i] === '--skills') skills = (argv[++i] ?? '').split(',').filter(Boolean);
207
+ else if (argv[i] === '--hosts') hosts = (argv[++i] ?? '').split(',').filter(Boolean);
208
+ }
209
+ return { skills, hosts, dryRun };
210
+ }
211
+
212
+ async function main(): Promise<number> {
213
+ const opts = parseArgs(process.argv.slice(2));
214
+ const out = await runSmoke(opts);
215
+ const lines: string[] = [];
216
+ lines.push(`cross_model_smoke · catalogue=${out.catalogue_size} skills · fixtures=${out.fixtures} · ${opts.dryRun ? 'DRY-RUN (mock)' : 'LIVE'}`);
217
+ lines.push('host model queries pass% parse% neg-ctrl in_tok out_tok ~$');
218
+ for (const h of out.hosts) {
219
+ lines.push(
220
+ `${h.host.padEnd(11)} ${h.model.padEnd(27)} ${String(h.queries).padStart(7)} ${(h.pass_rate * 100).toFixed(0).padStart(5)} ${(h.parse_rate * 100).toFixed(0).padStart(6)} ${String(h.neg_caught + '/' + h.neg_controls).padStart(8)} ${String(h.input_tokens).padStart(7)} ${String(h.output_tokens).padStart(7)} ${h.cost_usd_estimate.toFixed(4)}`,
221
+ );
222
+ }
223
+ process.stdout.write(lines.join('\n') + '\n');
224
+ return 0;
225
+ }
226
+
227
+ // Entry guard: only run main when invoked directly (not when imported by tests).
228
+ const _invokedDirectly = (() => {
229
+ try {
230
+ const argv1 = process.argv[1];
231
+ if (!argv1) return false;
232
+ return fs.realpathSync(argv1) === fs.realpathSync(new URL(import.meta.url).pathname);
233
+ } catch {
234
+ return false;
235
+ }
236
+ })();
237
+ if (_invokedDirectly) {
238
+ main().then(
239
+ (code) => process.exit(code),
240
+ (err) => {
241
+ process.stderr.write(String(err?.stack ?? err) + '\n');
242
+ process.exit(1);
243
+ },
244
+ );
245
+ }