pantheon-opencode 1.6.3-beta.10 → 1.6.3-beta.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +39 -6
  2. package/README.md +6 -0
  3. package/bin/pantheon-init.mjs +15 -27
  4. package/package.json +1 -1
  5. package/plugin.json +1 -1
  6. package/pyproject.toml +1 -1
  7. package/scripts/decision-router-budget.mjs +102 -2
  8. package/scripts/decision-router-v2-budget.mjs +2 -2
  9. package/scripts/decision-router.mjs +82 -25
  10. package/scripts/install/decision-router-config.mjs +12 -4
  11. package/scripts/install/interactive.mjs +44 -27
  12. package/scripts/install/opencode.mjs +6 -0
  13. package/scripts/jev-live-probe.mjs +110 -84
  14. package/scripts/skill-search.mjs +242 -0
  15. package/src/agents/aphrodite.md +34 -1
  16. package/src/agents/apollo.md +227 -0
  17. package/src/agents/athena.md +24 -0
  18. package/src/agents/demeter.md +33 -0
  19. package/src/agents/gaia.md +24 -0
  20. package/src/agents/hephaestus.md +24 -0
  21. package/src/agents/hermes.md +33 -0
  22. package/src/agents/iris.md +24 -0
  23. package/src/agents/mnemosyne.md +25 -3
  24. package/src/agents/nyx.md +53 -22
  25. package/src/agents/prometheus.md +242 -1
  26. package/src/agents/talos.md +24 -0
  27. package/src/agents/themis.md +24 -0
  28. package/src/agents/zeus.md +36 -20
  29. package/src/pantheon/jev-decision-tool.ts +69 -11
  30. package/src/pantheon/jev-decisions.ts +68 -5
  31. package/src/pantheon/jev-quality-score.ts +1 -1
  32. package/src/pantheon/jev-rank-verify.ts +22 -7
  33. package/src/pantheon/jev-risk-gate.ts +1 -0
  34. package/src/pantheon/local-decision-budget.ts +169 -2
  35. package/src/pantheon/local-decision-router.ts +10 -0
  36. package/src/pantheon/local-decision-usage-evidence.internal.ts +2 -2
  37. package/src/pantheon/pantheon-decision-catalog.ts +1 -1
  38. package/src/pantheon/v2-decision-budget.ts +4 -2
  39. package/src/plugins/tui/dist/tui-v2.js +128 -25
  40. package/src/plugins/tui/dist/tui-v2.js.map +1 -1
  41. package/src/plugins/tui/dist/tui.js +1 -1
  42. package/src/plugins/tui/package-lock.json +2 -2
  43. package/src/plugins/tui/package.json +1 -1
  44. package/src/plugins/tui/src/whats-new-highlights.ts +59 -5
  45. package/src/plugins/tui/src/whats-new-open.ts +41 -1
  46. package/src/plugins/tui/src/whats-new.tsx +61 -8
@@ -56,8 +56,8 @@ export async function promptReview(
56
56
  /**
57
57
  * Collect interactive setup choices before the installer begins any writes.
58
58
  *
59
- * @param {{ components?: string[], defaultComponents?: string[], noMcp?: boolean, prompts?: typeof clack, isTTY?: boolean, getDecisionRouterConfig?: (target: string) => unknown }} [options]
60
- * @returns {Promise<{ components?: Set<string>, target?: string, decisionRouterConfig?: object, canceled?: boolean, cancelPoint?: string }>}
59
+ * @param {{ components?: string[], defaultComponents?: string[], noMcp?: boolean, prompts?: typeof clack, isTTY?: boolean, env?: Record<string, string | undefined>, getDecisionRouterConfig?: (target: string) => unknown }} [options]
60
+ * @returns {Promise<{ components?: Set<string>, target?: string, decisionRouterConfig?: object, decisionModelActivation?: object | null, decisionModelDeferredSetup?: object | null, canceled?: boolean, cancelPoint?: string }>}
61
61
  *
62
62
  * `cancelPoint` names the setup choice where the user stopped (`components` or
63
63
  * `target`), so the caller can print one explainable line and persist the same
@@ -67,6 +67,7 @@ export async function runInteractiveInstall({
67
67
  components = [],
68
68
  defaultComponents = [],
69
69
  noMcp = false,
70
+ env = process.env,
70
71
  getDecisionRouterConfig = () => DEFAULT_DECISION_ROUTER_CONFIG,
71
72
  prompts = clack,
72
73
  isTTY = isTerminal(),
@@ -138,7 +139,7 @@ export async function runInteractiveInstall({
138
139
  // that setup-model writes.
139
140
  const jevActivation = await prompts.select({
140
141
  message:
141
- 'Configure the Jev decision model now? (advisory tool for Zeus; one write: selection, consent, caps, period — USD caps are required)',
142
+ 'Configure the Jev decision model now? (Console, OpenRouter, and TypeSafe activate here; Cloudflare is provisioned after install)',
142
143
  options: [
143
144
  {
144
145
  label: 'Skip (default)',
@@ -148,7 +149,7 @@ export async function runInteractiveInstall({
148
149
  {
149
150
  label: 'OpenCode Console Jev (stock)',
150
151
  value: 'opencode-console-jev/jev-1.13',
151
- hint: consoleJevHint(process.env),
152
+ hint: consoleJevHint(env),
152
153
  },
153
154
  {
154
155
  label: 'OpenRouter-hosted Jev',
@@ -158,12 +159,12 @@ export async function runInteractiveInstall({
158
159
  {
159
160
  label: 'TypeSafe Jev (direct)',
160
161
  value: 'typesafe-jev/jev-1.14.0',
161
- hint: typesafeJevHint(process.env),
162
+ hint: typesafeJevHint(env),
162
163
  },
163
164
  {
164
- label: 'Cloudflare Clef',
165
+ label: 'Cloudflare Clef (provision after install)',
165
166
  value: 'cloudflare-workers-jev/clef',
166
- hint: cloudflareJevHint(process.env),
167
+ hint: cloudflareJevHint(env),
167
168
  },
168
169
  ],
169
170
  initialValue: 'skip',
@@ -177,7 +178,8 @@ export async function runInteractiveInstall({
177
178
  // file the user sources (the host reads these at plugin registration from
178
179
  // its own environment — the installer never edits shell profiles silently).
179
180
  const jevFlags = await prompts.multiselect({
180
- message: 'Decision layer extras (all advisory/shadow; none change behavior by default):',
181
+ message:
182
+ 'Decision layer extras (shadow gates are observe-only; specialist access is explicit):',
181
183
  options: [
182
184
  {
183
185
  label: 'J1 risk-score shadow gate',
@@ -191,7 +193,7 @@ export async function runInteractiveInstall({
191
193
  },
192
194
  {
193
195
  label: 'Specialist callers: athena + themis (explicit questions)',
194
- value: 'CALLERS=athena,themis',
196
+ value: 'PANTHEON_JEV_SPECIALIST_CALLERS=athena,themis',
195
197
  hint: 'Allowlisted specialists may ask bounded Jev questions mid-task; presets stay Zeus-only',
196
198
  },
197
199
  ],
@@ -201,26 +203,23 @@ export async function runInteractiveInstall({
201
203
  if (cancelPrompt(prompts, jevFlags, 'decisionFlags')) {
202
204
  return { canceled: true, cancelPoint: 'decisionFlags' }
203
205
  }
204
- if (Array.isArray(jevFlags) && jevFlags.length > 0) {
205
- const flagsDir = join(process.env.XDG_CONFIG_HOME || `${process.env.HOME}/.config`, 'pantheon')
206
- mkdirSync(flagsDir, { recursive: true, mode: 0o700 })
207
- const flagsPath = join(flagsDir, 'decision-flags.env')
208
- const lines = jevFlags.map((flag) => `export ${flag}`).join('\n')
209
- writeFileSync(
210
- flagsPath,
211
- `# Pantheon decision layer — source from your shell profile
212
- ${lines}
213
- `,
214
- )
215
- }
216
- // Only known targets activate; anything else (a stale stub, a renamed
217
- // option) degrades to skip instead of configuring a wrong route.
218
- const KNOWN_JEV_TARGETS = new Set(['opencode-console-jev/jev-1.13', 'typesafe-jev/jev-1.14.0'])
206
+ // Cloudflare needs an account-specific host provider entry. It remains an
207
+ // explicit deferred choice; setup-model must receive --account-id before any
208
+ // route is enabled.
209
+ const KNOWN_JEV_TARGETS = new Set([
210
+ 'opencode-console-jev/jev-1.13',
211
+ 'openrouter-jev/typesafe/jev-1.13',
212
+ 'typesafe-jev/jev-1.14.0',
213
+ ])
214
+ const cloudflareDeferred = jevActivation === 'cloudflare-workers-jev/clef'
219
215
  const activationChoice =
220
216
  typeof jevActivation === 'string' && KNOWN_JEV_TARGETS.has(jevActivation)
221
217
  ? jevActivation
222
218
  : 'skip'
223
219
  let decisionModelActivation = null
220
+ const decisionModelDeferredSetup = cloudflareDeferred
221
+ ? { target: 'cloudflare-workers-jev/clef' }
222
+ : null
224
223
  if (activationChoice !== 'skip') {
225
224
  // Caps prompts degrade to the recommended defaults when the prompt
226
225
  // implementation has no text input (stubbed tests, restricted TTYs).
@@ -247,11 +246,28 @@ ${lines}
247
246
  }
248
247
  }
249
248
 
249
+ // Empty/absent caller allowlist is intentionally Zeus-only. Persist that
250
+ // default too, so rerunning the wizard cannot leave a prior allowlist active.
251
+ const selectedFlags = Array.isArray(jevFlags) ? jevFlags : []
252
+ const hasCallerFlag = selectedFlags.some((flag) =>
253
+ flag.startsWith('PANTHEON_JEV_SPECIALIST_CALLERS='),
254
+ )
255
+ const flagsDir = join(env.XDG_CONFIG_HOME || `${env.HOME}/.config`, 'pantheon')
256
+ mkdirSync(flagsDir, { recursive: true, mode: 0o700 })
257
+ const flagsPath = join(flagsDir, 'decision-flags.env')
258
+ const lines = selectedFlags.map((flag) => `export ${flag}`)
259
+ if (!hasCallerFlag) lines.push('export PANTHEON_JEV_SPECIALIST_CALLERS=')
260
+ writeFileSync(
261
+ flagsPath,
262
+ `# Pantheon decision layer — source from your shell profile\n${lines.join('\n')}\n`,
263
+ )
264
+
250
265
  return {
251
266
  components: selectedComponents,
252
267
  target,
253
268
  decisionRouterConfig: createDecisionRouterConfig(DECISION_ROUTER_PROVIDERS[0], false),
254
269
  decisionModelActivation,
270
+ decisionModelDeferredSetup,
255
271
  }
256
272
  }
257
273
 
@@ -273,7 +289,8 @@ function typesafeJevHint(env = process.env) {
273
289
  }
274
290
 
275
291
  function cloudflareJevHint(env = process.env) {
276
- return env.CLOUDFLARE_API_TOKEN
277
- ? '✓ CLOUDFLARE_API_TOKEN detected — pass --account-id at setup-model'
278
- : 'Workers AI-scoped CLOUDFLARE_API_TOKEN + accountId; setup-model merges the entry'
292
+ const tokenHint = env.CLOUDFLARE_API_TOKEN
293
+ ? 'CLOUDFLARE_API_TOKEN detected'
294
+ : 'requires a Workers AI-scoped CLOUDFLARE_API_TOKEN'
295
+ return `${tokenHint}; not activated by this wizard. After install run: npx pantheon-opencode decision-router setup-model cloudflare-workers-jev/clef --account-id=<32-hex-id> --monthly-usd 5 --per-decision-usd 0.10`
279
296
  }
@@ -1646,6 +1646,8 @@ async function installPass(
1646
1646
  )
1647
1647
  /** Wizard-confirmed Jev activation (target + caps), surfaced on the result. */
1648
1648
  let decisionModelActivation = null
1649
+ /** Cloudflare needs a user-supplied account ID, so the wizard defers it. */
1650
+ let decisionModelDeferredSetup = null
1649
1651
  const stats = summary.opencode
1650
1652
  // Per-call baseline. `stats` is the module-level cumulative summary, so the
1651
1653
  // consistency guard must compare against THIS call's delta, not the total.
@@ -1804,6 +1806,7 @@ async function installPass(
1804
1806
  },
1805
1807
  prompts: opts.deps?.prompts,
1806
1808
  isTTY: opts.deps?.isTTY,
1809
+ env: opts.deps?.env ?? opts.env ?? process.env,
1807
1810
  })
1808
1811
 
1809
1812
  // Resolve the location from the SELECTION before any cancel return: a
@@ -1840,6 +1843,7 @@ async function installPass(
1840
1843
  decisionRouterConfig = migrateDecisionRouterConfig(result.decisionRouterConfig)
1841
1844
  opts.decisionRouterConfig = decisionRouterConfig
1842
1845
  decisionModelActivation = result.decisionModelActivation ?? null
1846
+ decisionModelDeferredSetup = result.decisionModelDeferredSetup ?? null
1843
1847
  if (!dryRun && _lastRun) _lastRun.target = target
1844
1848
  }
1845
1849
 
@@ -3453,6 +3457,7 @@ async function installPass(
3453
3457
  configAdvisories,
3454
3458
  v2Capabilities: version === 'v2' ? v2Capabilities() : null,
3455
3459
  decisionModelActivation,
3460
+ decisionModelDeferredSetup,
3456
3461
  pluginOptions: v2PluginOptions,
3457
3462
  }
3458
3463
  : {
@@ -3460,6 +3465,7 @@ async function installPass(
3460
3465
  applied: true,
3461
3466
  reason: 'applied',
3462
3467
  decisionModelActivation,
3468
+ decisionModelDeferredSetup,
3463
3469
  target,
3464
3470
  runtimeEffects,
3465
3471
  openCodeVersion: version,
@@ -1,25 +1,39 @@
1
1
  #!/usr/bin/env node
2
2
  /**
3
3
  * jev-live-probe.mjs — operator-run live measurement for the Jev decision
4
- * route (calibration ping + live scorecard).
4
+ * route (config-only route check + live scorecard).
5
5
  *
6
- * Talks to the HOST's decisions API through `opencode api` (the same route
7
- * the plugin uses), so no credentials cross this script and every call is
8
- * billed to the user's configured route. Operator-run by design: this is the
9
- * consented measurement the J2 scorecard gate requires (offline harness
10
- * measures plumbing; THIS measures the live model).
6
+ * The ping resolves the configured route without contacting a provider. The
7
+ * scorecard talks to the HOST's decisions API through `opencode api` (the same
8
+ * route the plugin uses), so no credentials cross this script and each live
9
+ * call is billed to the user's configured route. Operator-run by design: the
10
+ * J2 scorecard is the consented live-model measurement; the offline harness
11
+ * measures plumbing only.
11
12
  *
12
13
  * Usage:
13
- * node scripts/jev-live-probe.mjs --ping # 1 call, exit 0 if reachable
14
+ * node scripts/jev-live-probe.mjs --ping # config-only route resolution; no provider call
14
15
  * node scripts/jev-live-probe.mjs --scorecard # N scenarios, dated scorecard
15
16
  */
16
17
 
17
18
  import { spawnSync } from 'node:child_process'
18
19
  import { writeFileSync } from 'node:fs'
19
- import { join } from 'node:path'
20
+ import { pathToFileURL } from 'node:url'
21
+ import {
22
+ DECISION_ROUTER_MODELS,
23
+ readUserDecisionRouterConfig,
24
+ } from './install/decision-router-config.mjs'
20
25
 
21
26
  const HOST_TIMEOUT_MS = 30_000
22
27
 
28
+ /** Config provider IDs translated to the host's native Decisions provider IDs. */
29
+ export const HOST_PROVIDERS = Object.freeze({
30
+ 'typesafe-jev': 'typesafe-ai',
31
+ 'opencode-console-jev': 'opencode',
32
+ 'openrouter-jev': 'openrouter',
33
+ 'vercel-gateway-jev': 'vercel-ai-gateway',
34
+ 'cloudflare-workers-jev': 'cloudflare-workers-ai',
35
+ })
36
+
23
37
  /** Live scenarios: state → the skill a correct picker would choose. */
24
38
  const SCENARIOS = [
25
39
  { state: 'Investigate why the payment webhook drops events under load', expect: 'apollo' },
@@ -40,7 +54,7 @@ const SCENARIOS = [
40
54
  { state: 'Migrate the open issues to the new milestone labels via the API', expect: 'iris' },
41
55
  { state: 'Decide which stale session summaries are worth keeping', expect: 'mnemosyne' },
42
56
  { state: 'Fuzz the input parser with malformed payloads to find crashes', expect: 'nyx' },
43
- { state: 'Verify the export module against the compliance checklist', expect: 'talos' },
57
+ { state: 'Fix the typo that makes the CLI parse --verbose as the output path', expect: 'talos' },
44
58
  { state: 'Polish the onboarding screen layout and micro-interactions', expect: 'aphrodite' },
45
59
  ]
46
60
 
@@ -67,7 +81,7 @@ function callDecisions(modelRef, state) {
67
81
  { value: 'aphrodite', description: 'UI and frontend craft' },
68
82
  { value: 'mnemosyne', description: 'memory curation and recall' },
69
83
  { value: 'nyx', description: 'chaos and adversarial testing' },
70
- { value: 'talos', description: 'validation and compliance checks' },
84
+ { value: 'talos', description: 'hotfix express lane - small direct fixes, no ceremony' },
71
85
  { value: 'none', description: 'no specialist fits' },
72
86
  ],
73
87
  },
@@ -98,87 +112,99 @@ function answerChoice(data) {
98
112
  return pick?.choice ?? null
99
113
  }
100
114
 
101
- const args = process.argv.slice(2)
102
-
103
- if (args.includes('--ping')) {
104
- const modelRef = { providerID: 'opencode', id: 'jev-1.13-free' }
105
- const r = callDecisions(modelRef, 'Calibration ping: is the Jev decision route reachable?')
106
- if (r.ok) {
107
- const cost = r.data?.cost ?? r.data?.data?.cost
108
- console.log(`✅ reachable — cost: ${JSON.stringify(cost)}`)
109
- process.exit(0)
115
+ /** Resolve the configured explicit route without probing the provider. */
116
+ export function resolveConfiguredRoute(config) {
117
+ if (!config?.enabled || config.selection?.mode !== 'explicit') {
118
+ throw new Error('the router must be enabled with an explicit selection')
110
119
  }
111
- console.error(`❌ unreachable: ${r.raw}`)
112
- process.exit(1)
120
+ const { provider, model } = config.selection
121
+ const hostProvider = HOST_PROVIDERS[provider]
122
+ if (
123
+ !Object.hasOwn(HOST_PROVIDERS, provider) ||
124
+ !DECISION_ROUTER_MODELS[provider]?.includes(model)
125
+ ) {
126
+ throw new Error(`unknown or unsupported selection: ${provider}/${model}`)
127
+ }
128
+ return { providerID: hostProvider, id: model }
113
129
  }
114
130
 
115
- if (args.includes('--scorecard')) {
116
- // Route from the user's decision-router config (explicit selection only).
117
- const { readFileSync } = await import('node:fs')
118
- const { homedir } = await import('node:os')
119
- const xdg = process.env.XDG_CONFIG_HOME?.startsWith('/')
120
- ? process.env.XDG_CONFIG_HOME
121
- : join(homedir(), '.config')
122
- const cfg = JSON.parse(readFileSync(join(xdg, 'pantheon', 'decision-router.json'), 'utf8'))
123
- if (!cfg.enabled || cfg.selection.mode !== 'explicit') {
124
- console.error('the router must be enabled with an explicit selection for a live run')
125
- process.exit(1)
126
- }
127
- // Same host-provider mapping the plugin route resolver uses (subset of
128
- // NATIVE_JEV_HOST_PROVIDERS — keep in sync when the matrix expands).
129
- const HOST_PROVIDERS = {
130
- 'typesafe-jev': 'typesafe-ai',
131
- 'opencode-console-jev': 'opencode',
132
- 'openrouter-jev': 'openrouter',
133
- 'vercel-gateway-jev': 'vercel-ai-gateway',
134
- 'cloudflare-workers-jev': 'cloudflare-workers-ai',
135
- }
136
- const hostProvider = HOST_PROVIDERS[cfg.selection.provider]
137
- if (!hostProvider) {
138
- console.error(`unknown selection provider: ${cfg.selection.provider}`)
139
- process.exit(1)
131
+ /** Report the configured route only; deliberately makes no Decisions call. */
132
+ export function pingConfiguredRoute(config, { stdout = console.log } = {}) {
133
+ const modelRef = resolveConfiguredRoute(config)
134
+ stdout(`✅ configured route: ${modelRef.providerID}/${modelRef.id} (no live provider call)`)
135
+ return modelRef
136
+ }
137
+
138
+ function main(args = process.argv.slice(2)) {
139
+ if (args.includes('--ping')) {
140
+ try {
141
+ pingConfiguredRoute(readUserDecisionRouterConfig())
142
+ process.exit(0)
143
+ } catch (error) {
144
+ console.error(
145
+ `❌ route unresolved: ${error instanceof Error ? error.message : String(error)}`,
146
+ )
147
+ process.exit(1)
148
+ }
140
149
  }
141
- const modelRef = { providerID: hostProvider, id: cfg.selection.model }
142
150
 
143
- const rows = []
144
- let totalCost = 0
145
- for (const scenario of SCENARIOS) {
146
- const r = callDecisions(modelRef, scenario.state)
147
- if (!r.ok) {
148
- rows.push({ state: scenario.state, expect: scenario.expect, got: `ERROR: ${r.raw}` })
149
- continue
151
+ if (args.includes('--scorecard')) {
152
+ // Route from the user's decision-router config (explicit selection only).
153
+ let cfg
154
+ let modelRef
155
+ try {
156
+ cfg = readUserDecisionRouterConfig()
157
+ modelRef = resolveConfiguredRoute(cfg)
158
+ } catch (error) {
159
+ console.error(error instanceof Error ? error.message : String(error))
160
+ process.exit(1)
161
+ }
162
+
163
+ const rows = []
164
+ let totalCost = 0
165
+ for (const scenario of SCENARIOS) {
166
+ const r = callDecisions(modelRef, scenario.state)
167
+ if (!r.ok) {
168
+ rows.push({ state: scenario.state, expect: scenario.expect, got: `ERROR: ${r.raw}` })
169
+ continue
170
+ }
171
+ const cost = r.data?.cost ?? 0
172
+ totalCost += Number(cost) || 0
173
+ rows.push({ state: scenario.state, expect: scenario.expect, got: answerChoice(r.data), cost })
150
174
  }
151
- const cost = r.data?.cost ?? 0
152
- totalCost += Number(cost) || 0
153
- rows.push({ state: scenario.state, expect: scenario.expect, got: answerChoice(r.data), cost })
175
+
176
+ const date = new Date().toISOString().slice(0, 10)
177
+ const out = [
178
+ `# Live J2 scorecard — ${date}`,
179
+ '',
180
+ `Route: \`${modelRef.providerID}/${modelRef.id}\`. Operator-run via \`opencode api\``,
181
+ '(consented route, billed to the user). 12 scenarios; agreement between the',
182
+ 'live model and the expected pick is the metric — this measures the MODEL,',
183
+ 'unlike the offline harness that measures the plumbing.',
184
+ '',
185
+ '| state (truncated) | expect | live | cost |',
186
+ '| --- | --- | --- | --- |',
187
+ ...rows.map(
188
+ (r) =>
189
+ `| ${r.state.slice(0, 48)}${r.state.length > 48 ? '…' : ''} | ${r.expect} | ${r.got ?? '—'} | ${r.cost ?? '—'} |`,
190
+ ),
191
+ '',
192
+ `Total cost: $${totalCost.toFixed(6)}. Honest caveat: one run is a smoke`,
193
+ 'measurement, not a calibration — a claim about model quality needs',
194
+ 'repeated runs and a bigger scenario set.',
195
+ '',
196
+ ].join('\n')
197
+ const outPath = `bench/results/j2-live-scorecard-${date}.md`
198
+ writeFileSync(outPath, out)
199
+ console.log(`✅ live scorecard written: ${outPath} (total $${totalCost.toFixed(6)})`)
200
+ process.exit(0)
154
201
  }
155
202
 
156
- const date = new Date().toISOString().slice(0, 10)
157
- const out = [
158
- `# Live J2 scorecard — ${date}`,
159
- '',
160
- `Route: \`${modelRef.providerID}/${modelRef.id}\`. Operator-run via \`opencode api\``,
161
- '(consented route, billed to the user). 12 scenarios; agreement between the',
162
- 'live model and the expected pick is the metric — this measures the MODEL,',
163
- 'unlike the offline harness that measures the plumbing.',
164
- '',
165
- '| state (truncated) | expect | live | cost |',
166
- '| --- | --- | --- | --- |',
167
- ...rows.map(
168
- (r) =>
169
- `| ${r.state.slice(0, 48)}${r.state.length > 48 ? '…' : ''} | ${r.expect} | ${r.got ?? '—'} | ${r.cost ?? '—'} |`,
170
- ),
171
- '',
172
- `Total cost: $${totalCost.toFixed(6)}. Honest caveat: one run is a smoke`,
173
- 'measurement, not a calibration — a claim about model quality needs',
174
- 'repeated runs and a bigger scenario set.',
175
- '',
176
- ].join('\n')
177
- const outPath = `bench/results/j2-live-scorecard-${date}.md`
178
- writeFileSync(outPath, out)
179
- console.log(`✅ live scorecard written: ${outPath} (total $${totalCost.toFixed(6)})`)
180
- process.exit(0)
203
+ console.error('usage: node scripts/jev-live-probe.mjs --ping | --scorecard')
204
+ process.exit(1)
181
205
  }
182
206
 
183
- console.error('usage: node scripts/jev-live-probe.mjs --ping | --scorecard')
184
- process.exit(1)
207
+ const invokedDirectly =
208
+ process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
209
+
210
+ if (invokedDirectly) main()
@@ -0,0 +1,242 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * skill-search.mjs — read-only search over the open skills ecosystem
4
+ * (skills.sh via the `skills` CLI) for the discovery pipeline documented in
5
+ * `docs/reference/native-decisions.md` and `src/agents/zeus.md`.
6
+ *
7
+ * The ladder this tool serves: installed skills first (one Jev
8
+ * `mode: "skills"` call answers over the local pool); only when nothing fits
9
+ * does the agent search the ecosystem, rank what came back with one typed
10
+ * Jev choice question, and PROPOSE the project-scoped install to the user —
11
+ * Pantheon never installs, and third-party skills are untrusted instructions.
12
+ *
13
+ * The `skills find` CLI prints ANSI-colored human text; this wrapper strips
14
+ * the escapes and emits the parseable contract (`owner/repo@skill`,
15
+ * install count, skills.sh URL) as JSON — the same contract the agent-side
16
+ * inline parse follows. Read-only: it never adds, updates, or removes
17
+ * skills, and no credentials cross it.
18
+ *
19
+ * Usage:
20
+ * node scripts/skill-search.mjs "<query>" [--limit 5] [--owner <owner>] [--raw]
21
+ */
22
+
23
+ import { spawnSync } from 'node:child_process'
24
+ import { pathToFileURL } from 'node:url'
25
+
26
+ const FIND_TIMEOUT_MS = 60_000
27
+ const DEFAULT_LIMIT = 5
28
+ /** SGR escape sequences are matched via the ESC codepoint (lint-clean, same semantics). */
29
+ const ANSI_SGR = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g')
30
+ const SKILL_LINE = /^([A-Za-z0-9._-]+\/[A-Za-z0-9._-]+@[A-Za-z0-9._-]+)\s+(\d+)\s+installs$/
31
+ const URL_LINE = /^└?\s*(https:\/\/skills\.sh\/\S+)$/
32
+
33
+ /** Remove SGR escape sequences so the text lines become parseable. */
34
+ export function stripAnsi(text) {
35
+ return String(text).replace(ANSI_SGR, '')
36
+ }
37
+
38
+ /**
39
+ * Parse `skills find` output into ordered entries {skill, installs, url?}.
40
+ * `limit` (default 5) caps the result; entries keep the CLI's printed order
41
+ * (the ecosystem's own ranking), never re-sorted here.
42
+ */
43
+ export function parseSkillSearchOutput(raw, { limit = DEFAULT_LIMIT } = {}) {
44
+ const lines = stripAnsi(raw)
45
+ .split('\n')
46
+ .map((line) => line.trim())
47
+ .filter((line) => line.length > 0)
48
+ const found = []
49
+ for (const line of lines) {
50
+ const skillMatch = line.match(SKILL_LINE)
51
+ if (skillMatch !== null) {
52
+ if (found.length >= limit) continue
53
+ found.push({ skill: skillMatch[1], installs: Number(skillMatch[2]), url: undefined })
54
+ continue
55
+ }
56
+ const urlMatch = line.match(URL_LINE)
57
+ if (urlMatch !== null && found.length > 0) {
58
+ const last = found[found.length - 1]
59
+ if (last.url === undefined) last.url = urlMatch[1]
60
+ }
61
+ }
62
+ return found
63
+ }
64
+
65
+ /** Run the ecosystem search once and return parsed entries. */
66
+ export function searchSkills(query, { limit = DEFAULT_LIMIT, owner, spawn = spawnSync } = {}) {
67
+ const args = ['skills', 'find', String(query)]
68
+ if (owner !== undefined) args.push('--owner', String(owner))
69
+ const result = spawn('npx', args, {
70
+ encoding: 'utf8',
71
+ timeout: FIND_TIMEOUT_MS,
72
+ stdio: ['ignore', 'pipe', 'pipe'],
73
+ })
74
+ if (result.error !== undefined) {
75
+ return { ok: false, error: `skills search failed to start: ${result.error.message}` }
76
+ }
77
+ const raw = `${result.stdout ?? ''}`
78
+ if (result.status !== 0 && raw.trim() === '') {
79
+ return { ok: false, error: `skills search exited ${result.status}` }
80
+ }
81
+ return { ok: true, entries: parseSkillSearchOutput(raw, { limit }), raw }
82
+ }
83
+
84
+ const GITHUB_TIMEOUT_MS = 30_000
85
+ const ANTHROPIC_SKILLS_REPO = 'anthropics/skills'
86
+
87
+ /**
88
+ * Tier 1: the canonical Anthropic catalog. Highest trust (curated upstream),
89
+ * checked first so the canonical `pdf`/`docx`/`xlsx`/`mcp-builder` skills
90
+ * outrank the third-party clones that skills.sh's ranking can surface.
91
+ * Matches query tokens against catalog entry names; returns one entry per
92
+ * matched name.
93
+ */
94
+ export function searchAnthropicCatalog(query, { limit = DEFAULT_LIMIT, spawn = spawnSync } = {}) {
95
+ const result = spawn(
96
+ 'gh',
97
+ ['api', `repos/${ANTHROPIC_SKILLS_REPO}/contents/skills`, '--jq', '.[].name'],
98
+ { encoding: 'utf8', timeout: GITHUB_TIMEOUT_MS, stdio: ['ignore', 'pipe', 'pipe'] },
99
+ )
100
+ if (result.error !== undefined || result.status !== 0) {
101
+ return { ok: false, error: 'anthropic catalog lookup failed (gh api)' }
102
+ }
103
+ const tokens = String(query)
104
+ .toLowerCase()
105
+ .split(/[^a-z0-9]+/)
106
+ .filter((token) => token.length > 2)
107
+ const names = String(result.stdout ?? '')
108
+ .split('\n')
109
+ .map((name) => name.trim())
110
+ .filter((name) => name.length > 0)
111
+ const matched = names.filter((name) => {
112
+ const lower = name.toLowerCase()
113
+ return tokens.some((token) => lower.includes(token))
114
+ })
115
+ return {
116
+ ok: true,
117
+ entries: matched.slice(0, limit).map((name) => ({
118
+ skill: `${ANTHROPIC_SKILLS_REPO}@${name}`,
119
+ installs: null,
120
+ url: `https://github.com/${ANTHROPIC_SKILLS_REPO}/tree/main/skills/${name}`,
121
+ source: 'anthropic',
122
+ })),
123
+ }
124
+ }
125
+
126
+ /**
127
+ * Tier 3: raw GitHub code search over every `SKILL.md` (the widest net —
128
+ * orders of magnitude beyond the skills.sh index, and the weakest trust:
129
+ * every hit is unreviewed third-party instructions).
130
+ */
131
+ export function searchGithubSkills(query, { limit = DEFAULT_LIMIT, spawn = spawnSync } = {}) {
132
+ const terms = String(query)
133
+ .split(/\s+/)
134
+ .filter((term) => term.length > 1)
135
+ .slice(0, 3)
136
+ .map((term) => encodeURIComponent(term))
137
+ .join('+')
138
+ if (terms === '') return { ok: true, entries: [] }
139
+ const result = spawn(
140
+ 'gh',
141
+ ['api', `search/code?q=filename:SKILL.md+${terms}&per_page=${Math.min(limit, 10)}`],
142
+ { encoding: 'utf8', timeout: GITHUB_TIMEOUT_MS, stdio: ['ignore', 'pipe', 'pipe'] },
143
+ )
144
+ if (result.error !== undefined || result.status !== 0) {
145
+ return { ok: false, error: 'github code search failed (gh api)' }
146
+ }
147
+ let parsed
148
+ try {
149
+ parsed = JSON.parse(String(result.stdout ?? '{}'))
150
+ } catch {
151
+ return { ok: false, error: 'github code search returned non-JSON output' }
152
+ }
153
+ const items = Array.isArray(parsed.items) ? parsed.items : []
154
+ return {
155
+ ok: true,
156
+ entries: items.slice(0, limit).map((item) => {
157
+ const repo = String(item?.repository?.full_name ?? 'unknown/repo')
158
+ const path = String(item?.path ?? 'SKILL.md')
159
+ const name = path.split('/')[Math.max(0, path.split('/').length - 2)] ?? 'skill'
160
+ return {
161
+ skill: `${repo}@${name}`,
162
+ installs: null,
163
+ url: `https://github.com/${repo}/blob/main/${path}`,
164
+ source: 'github',
165
+ }
166
+ }),
167
+ }
168
+ }
169
+
170
+ /**
171
+ * The trust cascade: anthropic (canonical) → skills-sh (install-count
172
+ * signal, unreviewed) → github (widest, least trusted). Tier order is the
173
+ * recommendation order; callers may narrow with `sources`.
174
+ */
175
+ export const SKILL_SEARCH_SOURCES = Object.freeze(['anthropic', 'skills-sh', 'github'])
176
+
177
+ export function searchSkillSources(
178
+ query,
179
+ { limit = DEFAULT_LIMIT, owner, sources = SKILL_SEARCH_SOURCES, spawn = spawnSync } = {},
180
+ ) {
181
+ const perTier = Math.max(1, Math.ceil(limit / sources.length))
182
+ const out = []
183
+ const failures = []
184
+ for (const source of sources) {
185
+ const result =
186
+ source === 'anthropic'
187
+ ? searchAnthropicCatalog(query, { limit: perTier, spawn })
188
+ : source === 'skills-sh'
189
+ ? searchSkills(query, { limit: perTier, owner, spawn })
190
+ : source === 'github'
191
+ ? searchGithubSkills(query, { limit: perTier, spawn })
192
+ : { ok: false, error: `unknown source: ${source}` }
193
+ if (result.ok) out.push(...result.entries.map((entry) => ({ ...entry, source })))
194
+ else failures.push(`${source}: ${result.error}`)
195
+ }
196
+ return { ok: failures.length < sources.length, entries: out.slice(0, limit), failures }
197
+ }
198
+
199
+ function main(args = process.argv.slice(2)) {
200
+ const get = (flag) => {
201
+ const i = args.indexOf(flag)
202
+ return i >= 0 ? args[i + 1] : undefined
203
+ }
204
+ const query = args.find((a) => !a.startsWith('--'))
205
+ if (query === undefined) {
206
+ console.error(
207
+ 'usage: node scripts/skill-search.mjs "<query>" [--limit 5] [--sources anthropic,skills-sh,github] [--owner <owner>] [--raw]',
208
+ )
209
+ process.exit(1)
210
+ }
211
+ const sourcesArg = get('--sources')
212
+ const sources =
213
+ sourcesArg === undefined
214
+ ? SKILL_SEARCH_SOURCES
215
+ : sourcesArg
216
+ .split(',')
217
+ .map((s) => s.trim())
218
+ .filter((s) => s.length > 0)
219
+ const result = searchSkillSources(query, {
220
+ limit: Number(get('--limit') ?? DEFAULT_LIMIT),
221
+ owner: get('--owner'),
222
+ sources,
223
+ })
224
+ if (args.includes('--raw')) {
225
+ const skillsSh = result.entries.filter((e) => e.source === 'skills-sh')
226
+ console.log(JSON.stringify(skillsSh, null, 1))
227
+ return
228
+ }
229
+ if (result.failures.length > 0) {
230
+ for (const failure of result.failures) console.error(`warning: ${failure}`)
231
+ }
232
+ if (result.entries.length === 0 && result.failures.length === sources.length) {
233
+ console.error('all sources failed')
234
+ process.exit(1)
235
+ }
236
+ console.log(JSON.stringify(result.entries, null, 1))
237
+ }
238
+
239
+ const invokedDirectly =
240
+ process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
241
+
242
+ if (invokedDirectly) main()