pantheon-opencode 1.6.3-beta.10 → 1.6.3-beta.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -6
- package/README.md +6 -0
- package/bin/pantheon-init.mjs +15 -27
- package/package.json +1 -1
- package/plugin.json +1 -1
- package/pyproject.toml +1 -1
- package/scripts/decision-router-budget.mjs +102 -2
- package/scripts/decision-router-v2-budget.mjs +2 -2
- package/scripts/decision-router.mjs +82 -25
- package/scripts/install/decision-router-config.mjs +12 -4
- package/scripts/install/interactive.mjs +44 -27
- package/scripts/install/opencode.mjs +6 -0
- package/scripts/jev-live-probe.mjs +110 -84
- package/scripts/skill-search.mjs +242 -0
- package/src/agents/aphrodite.md +34 -1
- package/src/agents/apollo.md +227 -0
- package/src/agents/athena.md +24 -0
- package/src/agents/demeter.md +33 -0
- package/src/agents/gaia.md +24 -0
- package/src/agents/hephaestus.md +24 -0
- package/src/agents/hermes.md +33 -0
- package/src/agents/iris.md +24 -0
- package/src/agents/mnemosyne.md +25 -3
- package/src/agents/nyx.md +53 -22
- package/src/agents/prometheus.md +242 -1
- package/src/agents/talos.md +24 -0
- package/src/agents/themis.md +24 -0
- package/src/agents/zeus.md +36 -20
- package/src/pantheon/jev-decision-tool.ts +9 -0
- package/src/pantheon/jev-risk-gate.ts +1 -0
- package/src/pantheon/local-decision-budget.ts +169 -2
- package/src/pantheon/local-decision-usage-evidence.internal.ts +2 -2
- package/src/pantheon/v2-decision-budget.ts +4 -2
- package/src/plugins/tui/dist/tui-v2.js +3 -3
- package/src/plugins/tui/dist/tui.js +1 -1
- package/src/plugins/tui/package-lock.json +2 -2
- package/src/plugins/tui/package.json +1 -1
|
@@ -56,8 +56,8 @@ export async function promptReview(
|
|
|
56
56
|
/**
|
|
57
57
|
* Collect interactive setup choices before the installer begins any writes.
|
|
58
58
|
*
|
|
59
|
-
* @param {{ components?: string[], defaultComponents?: string[], noMcp?: boolean, prompts?: typeof clack, isTTY?: boolean, getDecisionRouterConfig?: (target: string) => unknown }} [options]
|
|
60
|
-
* @returns {Promise<{ components?: Set<string>, target?: string, decisionRouterConfig?: object, canceled?: boolean, cancelPoint?: string }>}
|
|
59
|
+
* @param {{ components?: string[], defaultComponents?: string[], noMcp?: boolean, prompts?: typeof clack, isTTY?: boolean, env?: Record<string, string | undefined>, getDecisionRouterConfig?: (target: string) => unknown }} [options]
|
|
60
|
+
* @returns {Promise<{ components?: Set<string>, target?: string, decisionRouterConfig?: object, decisionModelActivation?: object | null, decisionModelDeferredSetup?: object | null, canceled?: boolean, cancelPoint?: string }>}
|
|
61
61
|
*
|
|
62
62
|
* `cancelPoint` names the setup choice where the user stopped (`components` or
|
|
63
63
|
* `target`), so the caller can print one explainable line and persist the same
|
|
@@ -67,6 +67,7 @@ export async function runInteractiveInstall({
|
|
|
67
67
|
components = [],
|
|
68
68
|
defaultComponents = [],
|
|
69
69
|
noMcp = false,
|
|
70
|
+
env = process.env,
|
|
70
71
|
getDecisionRouterConfig = () => DEFAULT_DECISION_ROUTER_CONFIG,
|
|
71
72
|
prompts = clack,
|
|
72
73
|
isTTY = isTerminal(),
|
|
@@ -138,7 +139,7 @@ export async function runInteractiveInstall({
|
|
|
138
139
|
// that setup-model writes.
|
|
139
140
|
const jevActivation = await prompts.select({
|
|
140
141
|
message:
|
|
141
|
-
'Configure the Jev decision model now? (
|
|
142
|
+
'Configure the Jev decision model now? (Console, OpenRouter, and TypeSafe activate here; Cloudflare is provisioned after install)',
|
|
142
143
|
options: [
|
|
143
144
|
{
|
|
144
145
|
label: 'Skip (default)',
|
|
@@ -148,7 +149,7 @@ export async function runInteractiveInstall({
|
|
|
148
149
|
{
|
|
149
150
|
label: 'OpenCode Console Jev (stock)',
|
|
150
151
|
value: 'opencode-console-jev/jev-1.13',
|
|
151
|
-
hint: consoleJevHint(
|
|
152
|
+
hint: consoleJevHint(env),
|
|
152
153
|
},
|
|
153
154
|
{
|
|
154
155
|
label: 'OpenRouter-hosted Jev',
|
|
@@ -158,12 +159,12 @@ export async function runInteractiveInstall({
|
|
|
158
159
|
{
|
|
159
160
|
label: 'TypeSafe Jev (direct)',
|
|
160
161
|
value: 'typesafe-jev/jev-1.14.0',
|
|
161
|
-
hint: typesafeJevHint(
|
|
162
|
+
hint: typesafeJevHint(env),
|
|
162
163
|
},
|
|
163
164
|
{
|
|
164
|
-
label: 'Cloudflare Clef',
|
|
165
|
+
label: 'Cloudflare Clef (provision after install)',
|
|
165
166
|
value: 'cloudflare-workers-jev/clef',
|
|
166
|
-
hint: cloudflareJevHint(
|
|
167
|
+
hint: cloudflareJevHint(env),
|
|
167
168
|
},
|
|
168
169
|
],
|
|
169
170
|
initialValue: 'skip',
|
|
@@ -177,7 +178,8 @@ export async function runInteractiveInstall({
|
|
|
177
178
|
// file the user sources (the host reads these at plugin registration from
|
|
178
179
|
// its own environment — the installer never edits shell profiles silently).
|
|
179
180
|
const jevFlags = await prompts.multiselect({
|
|
180
|
-
message:
|
|
181
|
+
message:
|
|
182
|
+
'Decision layer extras (shadow gates are observe-only; specialist access is explicit):',
|
|
181
183
|
options: [
|
|
182
184
|
{
|
|
183
185
|
label: 'J1 risk-score shadow gate',
|
|
@@ -191,7 +193,7 @@ export async function runInteractiveInstall({
|
|
|
191
193
|
},
|
|
192
194
|
{
|
|
193
195
|
label: 'Specialist callers: athena + themis (explicit questions)',
|
|
194
|
-
value: '
|
|
196
|
+
value: 'PANTHEON_JEV_SPECIALIST_CALLERS=athena,themis',
|
|
195
197
|
hint: 'Allowlisted specialists may ask bounded Jev questions mid-task; presets stay Zeus-only',
|
|
196
198
|
},
|
|
197
199
|
],
|
|
@@ -201,26 +203,23 @@ export async function runInteractiveInstall({
|
|
|
201
203
|
if (cancelPrompt(prompts, jevFlags, 'decisionFlags')) {
|
|
202
204
|
return { canceled: true, cancelPoint: 'decisionFlags' }
|
|
203
205
|
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
`,
|
|
214
|
-
)
|
|
215
|
-
}
|
|
216
|
-
// Only known targets activate; anything else (a stale stub, a renamed
|
|
217
|
-
// option) degrades to skip instead of configuring a wrong route.
|
|
218
|
-
const KNOWN_JEV_TARGETS = new Set(['opencode-console-jev/jev-1.13', 'typesafe-jev/jev-1.14.0'])
|
|
206
|
+
// Cloudflare needs an account-specific host provider entry. It remains an
|
|
207
|
+
// explicit deferred choice; setup-model must receive --account-id before any
|
|
208
|
+
// route is enabled.
|
|
209
|
+
const KNOWN_JEV_TARGETS = new Set([
|
|
210
|
+
'opencode-console-jev/jev-1.13',
|
|
211
|
+
'openrouter-jev/typesafe/jev-1.13',
|
|
212
|
+
'typesafe-jev/jev-1.14.0',
|
|
213
|
+
])
|
|
214
|
+
const cloudflareDeferred = jevActivation === 'cloudflare-workers-jev/clef'
|
|
219
215
|
const activationChoice =
|
|
220
216
|
typeof jevActivation === 'string' && KNOWN_JEV_TARGETS.has(jevActivation)
|
|
221
217
|
? jevActivation
|
|
222
218
|
: 'skip'
|
|
223
219
|
let decisionModelActivation = null
|
|
220
|
+
const decisionModelDeferredSetup = cloudflareDeferred
|
|
221
|
+
? { target: 'cloudflare-workers-jev/clef' }
|
|
222
|
+
: null
|
|
224
223
|
if (activationChoice !== 'skip') {
|
|
225
224
|
// Caps prompts degrade to the recommended defaults when the prompt
|
|
226
225
|
// implementation has no text input (stubbed tests, restricted TTYs).
|
|
@@ -247,11 +246,28 @@ ${lines}
|
|
|
247
246
|
}
|
|
248
247
|
}
|
|
249
248
|
|
|
249
|
+
// Empty/absent caller allowlist is intentionally Zeus-only. Persist that
|
|
250
|
+
// default too, so rerunning the wizard cannot leave a prior allowlist active.
|
|
251
|
+
const selectedFlags = Array.isArray(jevFlags) ? jevFlags : []
|
|
252
|
+
const hasCallerFlag = selectedFlags.some((flag) =>
|
|
253
|
+
flag.startsWith('PANTHEON_JEV_SPECIALIST_CALLERS='),
|
|
254
|
+
)
|
|
255
|
+
const flagsDir = join(env.XDG_CONFIG_HOME || `${env.HOME}/.config`, 'pantheon')
|
|
256
|
+
mkdirSync(flagsDir, { recursive: true, mode: 0o700 })
|
|
257
|
+
const flagsPath = join(flagsDir, 'decision-flags.env')
|
|
258
|
+
const lines = selectedFlags.map((flag) => `export ${flag}`)
|
|
259
|
+
if (!hasCallerFlag) lines.push('export PANTHEON_JEV_SPECIALIST_CALLERS=')
|
|
260
|
+
writeFileSync(
|
|
261
|
+
flagsPath,
|
|
262
|
+
`# Pantheon decision layer — source from your shell profile\n${lines.join('\n')}\n`,
|
|
263
|
+
)
|
|
264
|
+
|
|
250
265
|
return {
|
|
251
266
|
components: selectedComponents,
|
|
252
267
|
target,
|
|
253
268
|
decisionRouterConfig: createDecisionRouterConfig(DECISION_ROUTER_PROVIDERS[0], false),
|
|
254
269
|
decisionModelActivation,
|
|
270
|
+
decisionModelDeferredSetup,
|
|
255
271
|
}
|
|
256
272
|
}
|
|
257
273
|
|
|
@@ -273,7 +289,8 @@ function typesafeJevHint(env = process.env) {
|
|
|
273
289
|
}
|
|
274
290
|
|
|
275
291
|
function cloudflareJevHint(env = process.env) {
|
|
276
|
-
|
|
277
|
-
? '
|
|
278
|
-
: 'Workers AI-scoped CLOUDFLARE_API_TOKEN
|
|
292
|
+
const tokenHint = env.CLOUDFLARE_API_TOKEN
|
|
293
|
+
? 'CLOUDFLARE_API_TOKEN detected'
|
|
294
|
+
: 'requires a Workers AI-scoped CLOUDFLARE_API_TOKEN'
|
|
295
|
+
return `${tokenHint}; not activated by this wizard. After install run: npx pantheon-opencode decision-router setup-model cloudflare-workers-jev/clef --account-id=<32-hex-id> --monthly-usd 5 --per-decision-usd 0.10`
|
|
279
296
|
}
|
|
@@ -1646,6 +1646,8 @@ async function installPass(
|
|
|
1646
1646
|
)
|
|
1647
1647
|
/** Wizard-confirmed Jev activation (target + caps), surfaced on the result. */
|
|
1648
1648
|
let decisionModelActivation = null
|
|
1649
|
+
/** Cloudflare needs a user-supplied account ID, so the wizard defers it. */
|
|
1650
|
+
let decisionModelDeferredSetup = null
|
|
1649
1651
|
const stats = summary.opencode
|
|
1650
1652
|
// Per-call baseline. `stats` is the module-level cumulative summary, so the
|
|
1651
1653
|
// consistency guard must compare against THIS call's delta, not the total.
|
|
@@ -1804,6 +1806,7 @@ async function installPass(
|
|
|
1804
1806
|
},
|
|
1805
1807
|
prompts: opts.deps?.prompts,
|
|
1806
1808
|
isTTY: opts.deps?.isTTY,
|
|
1809
|
+
env: opts.deps?.env ?? opts.env ?? process.env,
|
|
1807
1810
|
})
|
|
1808
1811
|
|
|
1809
1812
|
// Resolve the location from the SELECTION before any cancel return: a
|
|
@@ -1840,6 +1843,7 @@ async function installPass(
|
|
|
1840
1843
|
decisionRouterConfig = migrateDecisionRouterConfig(result.decisionRouterConfig)
|
|
1841
1844
|
opts.decisionRouterConfig = decisionRouterConfig
|
|
1842
1845
|
decisionModelActivation = result.decisionModelActivation ?? null
|
|
1846
|
+
decisionModelDeferredSetup = result.decisionModelDeferredSetup ?? null
|
|
1843
1847
|
if (!dryRun && _lastRun) _lastRun.target = target
|
|
1844
1848
|
}
|
|
1845
1849
|
|
|
@@ -3453,6 +3457,7 @@ async function installPass(
|
|
|
3453
3457
|
configAdvisories,
|
|
3454
3458
|
v2Capabilities: version === 'v2' ? v2Capabilities() : null,
|
|
3455
3459
|
decisionModelActivation,
|
|
3460
|
+
decisionModelDeferredSetup,
|
|
3456
3461
|
pluginOptions: v2PluginOptions,
|
|
3457
3462
|
}
|
|
3458
3463
|
: {
|
|
@@ -3460,6 +3465,7 @@ async function installPass(
|
|
|
3460
3465
|
applied: true,
|
|
3461
3466
|
reason: 'applied',
|
|
3462
3467
|
decisionModelActivation,
|
|
3468
|
+
decisionModelDeferredSetup,
|
|
3463
3469
|
target,
|
|
3464
3470
|
runtimeEffects,
|
|
3465
3471
|
openCodeVersion: version,
|
|
@@ -1,25 +1,39 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
3
|
* jev-live-probe.mjs — operator-run live measurement for the Jev decision
|
|
4
|
-
* route (
|
|
4
|
+
* route (config-only route check + live scorecard).
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
6
|
+
* The ping resolves the configured route without contacting a provider. The
|
|
7
|
+
* scorecard talks to the HOST's decisions API through `opencode api` (the same
|
|
8
|
+
* route the plugin uses), so no credentials cross this script and each live
|
|
9
|
+
* call is billed to the user's configured route. Operator-run by design: the
|
|
10
|
+
* J2 scorecard is the consented live-model measurement; the offline harness
|
|
11
|
+
* measures plumbing only.
|
|
11
12
|
*
|
|
12
13
|
* Usage:
|
|
13
|
-
* node scripts/jev-live-probe.mjs --ping #
|
|
14
|
+
* node scripts/jev-live-probe.mjs --ping # config-only route resolution; no provider call
|
|
14
15
|
* node scripts/jev-live-probe.mjs --scorecard # N scenarios, dated scorecard
|
|
15
16
|
*/
|
|
16
17
|
|
|
17
18
|
import { spawnSync } from 'node:child_process'
|
|
18
19
|
import { writeFileSync } from 'node:fs'
|
|
19
|
-
import {
|
|
20
|
+
import { pathToFileURL } from 'node:url'
|
|
21
|
+
import {
|
|
22
|
+
DECISION_ROUTER_MODELS,
|
|
23
|
+
readUserDecisionRouterConfig,
|
|
24
|
+
} from './install/decision-router-config.mjs'
|
|
20
25
|
|
|
21
26
|
const HOST_TIMEOUT_MS = 30_000
|
|
22
27
|
|
|
28
|
+
/** Config provider IDs translated to the host's native Decisions provider IDs. */
|
|
29
|
+
export const HOST_PROVIDERS = Object.freeze({
|
|
30
|
+
'typesafe-jev': 'typesafe-ai',
|
|
31
|
+
'opencode-console-jev': 'opencode',
|
|
32
|
+
'openrouter-jev': 'openrouter',
|
|
33
|
+
'vercel-gateway-jev': 'vercel-ai-gateway',
|
|
34
|
+
'cloudflare-workers-jev': 'cloudflare-workers-ai',
|
|
35
|
+
})
|
|
36
|
+
|
|
23
37
|
/** Live scenarios: state → the skill a correct picker would choose. */
|
|
24
38
|
const SCENARIOS = [
|
|
25
39
|
{ state: 'Investigate why the payment webhook drops events under load', expect: 'apollo' },
|
|
@@ -40,7 +54,7 @@ const SCENARIOS = [
|
|
|
40
54
|
{ state: 'Migrate the open issues to the new milestone labels via the API', expect: 'iris' },
|
|
41
55
|
{ state: 'Decide which stale session summaries are worth keeping', expect: 'mnemosyne' },
|
|
42
56
|
{ state: 'Fuzz the input parser with malformed payloads to find crashes', expect: 'nyx' },
|
|
43
|
-
{ state: '
|
|
57
|
+
{ state: 'Fix the typo that makes the CLI parse --verbose as the output path', expect: 'talos' },
|
|
44
58
|
{ state: 'Polish the onboarding screen layout and micro-interactions', expect: 'aphrodite' },
|
|
45
59
|
]
|
|
46
60
|
|
|
@@ -67,7 +81,7 @@ function callDecisions(modelRef, state) {
|
|
|
67
81
|
{ value: 'aphrodite', description: 'UI and frontend craft' },
|
|
68
82
|
{ value: 'mnemosyne', description: 'memory curation and recall' },
|
|
69
83
|
{ value: 'nyx', description: 'chaos and adversarial testing' },
|
|
70
|
-
{ value: 'talos', description: '
|
|
84
|
+
{ value: 'talos', description: 'hotfix express lane - small direct fixes, no ceremony' },
|
|
71
85
|
{ value: 'none', description: 'no specialist fits' },
|
|
72
86
|
],
|
|
73
87
|
},
|
|
@@ -98,87 +112,99 @@ function answerChoice(data) {
|
|
|
98
112
|
return pick?.choice ?? null
|
|
99
113
|
}
|
|
100
114
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
if (
|
|
104
|
-
|
|
105
|
-
const r = callDecisions(modelRef, 'Calibration ping: is the Jev decision route reachable?')
|
|
106
|
-
if (r.ok) {
|
|
107
|
-
const cost = r.data?.cost ?? r.data?.data?.cost
|
|
108
|
-
console.log(`✅ reachable — cost: ${JSON.stringify(cost)}`)
|
|
109
|
-
process.exit(0)
|
|
115
|
+
/** Resolve the configured explicit route without probing the provider. */
|
|
116
|
+
export function resolveConfiguredRoute(config) {
|
|
117
|
+
if (!config?.enabled || config.selection?.mode !== 'explicit') {
|
|
118
|
+
throw new Error('the router must be enabled with an explicit selection')
|
|
110
119
|
}
|
|
111
|
-
|
|
112
|
-
|
|
120
|
+
const { provider, model } = config.selection
|
|
121
|
+
const hostProvider = HOST_PROVIDERS[provider]
|
|
122
|
+
if (
|
|
123
|
+
!Object.hasOwn(HOST_PROVIDERS, provider) ||
|
|
124
|
+
!DECISION_ROUTER_MODELS[provider]?.includes(model)
|
|
125
|
+
) {
|
|
126
|
+
throw new Error(`unknown or unsupported selection: ${provider}/${model}`)
|
|
127
|
+
}
|
|
128
|
+
return { providerID: hostProvider, id: model }
|
|
113
129
|
}
|
|
114
130
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
const
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
if (
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
'vercel-gateway-jev': 'vercel-ai-gateway',
|
|
134
|
-
'cloudflare-workers-jev': 'cloudflare-workers-ai',
|
|
135
|
-
}
|
|
136
|
-
const hostProvider = HOST_PROVIDERS[cfg.selection.provider]
|
|
137
|
-
if (!hostProvider) {
|
|
138
|
-
console.error(`unknown selection provider: ${cfg.selection.provider}`)
|
|
139
|
-
process.exit(1)
|
|
131
|
+
/** Report the configured route only; deliberately makes no Decisions call. */
|
|
132
|
+
export function pingConfiguredRoute(config, { stdout = console.log } = {}) {
|
|
133
|
+
const modelRef = resolveConfiguredRoute(config)
|
|
134
|
+
stdout(`✅ configured route: ${modelRef.providerID}/${modelRef.id} (no live provider call)`)
|
|
135
|
+
return modelRef
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function main(args = process.argv.slice(2)) {
|
|
139
|
+
if (args.includes('--ping')) {
|
|
140
|
+
try {
|
|
141
|
+
pingConfiguredRoute(readUserDecisionRouterConfig())
|
|
142
|
+
process.exit(0)
|
|
143
|
+
} catch (error) {
|
|
144
|
+
console.error(
|
|
145
|
+
`❌ route unresolved: ${error instanceof Error ? error.message : String(error)}`,
|
|
146
|
+
)
|
|
147
|
+
process.exit(1)
|
|
148
|
+
}
|
|
140
149
|
}
|
|
141
|
-
const modelRef = { providerID: hostProvider, id: cfg.selection.model }
|
|
142
150
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
151
|
+
if (args.includes('--scorecard')) {
|
|
152
|
+
// Route from the user's decision-router config (explicit selection only).
|
|
153
|
+
let cfg
|
|
154
|
+
let modelRef
|
|
155
|
+
try {
|
|
156
|
+
cfg = readUserDecisionRouterConfig()
|
|
157
|
+
modelRef = resolveConfiguredRoute(cfg)
|
|
158
|
+
} catch (error) {
|
|
159
|
+
console.error(error instanceof Error ? error.message : String(error))
|
|
160
|
+
process.exit(1)
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const rows = []
|
|
164
|
+
let totalCost = 0
|
|
165
|
+
for (const scenario of SCENARIOS) {
|
|
166
|
+
const r = callDecisions(modelRef, scenario.state)
|
|
167
|
+
if (!r.ok) {
|
|
168
|
+
rows.push({ state: scenario.state, expect: scenario.expect, got: `ERROR: ${r.raw}` })
|
|
169
|
+
continue
|
|
170
|
+
}
|
|
171
|
+
const cost = r.data?.cost ?? 0
|
|
172
|
+
totalCost += Number(cost) || 0
|
|
173
|
+
rows.push({ state: scenario.state, expect: scenario.expect, got: answerChoice(r.data), cost })
|
|
150
174
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
175
|
+
|
|
176
|
+
const date = new Date().toISOString().slice(0, 10)
|
|
177
|
+
const out = [
|
|
178
|
+
`# Live J2 scorecard — ${date}`,
|
|
179
|
+
'',
|
|
180
|
+
`Route: \`${modelRef.providerID}/${modelRef.id}\`. Operator-run via \`opencode api\``,
|
|
181
|
+
'(consented route, billed to the user). 12 scenarios; agreement between the',
|
|
182
|
+
'live model and the expected pick is the metric — this measures the MODEL,',
|
|
183
|
+
'unlike the offline harness that measures the plumbing.',
|
|
184
|
+
'',
|
|
185
|
+
'| state (truncated) | expect | live | cost |',
|
|
186
|
+
'| --- | --- | --- | --- |',
|
|
187
|
+
...rows.map(
|
|
188
|
+
(r) =>
|
|
189
|
+
`| ${r.state.slice(0, 48)}${r.state.length > 48 ? '…' : ''} | ${r.expect} | ${r.got ?? '—'} | ${r.cost ?? '—'} |`,
|
|
190
|
+
),
|
|
191
|
+
'',
|
|
192
|
+
`Total cost: $${totalCost.toFixed(6)}. Honest caveat: one run is a smoke`,
|
|
193
|
+
'measurement, not a calibration — a claim about model quality needs',
|
|
194
|
+
'repeated runs and a bigger scenario set.',
|
|
195
|
+
'',
|
|
196
|
+
].join('\n')
|
|
197
|
+
const outPath = `bench/results/j2-live-scorecard-${date}.md`
|
|
198
|
+
writeFileSync(outPath, out)
|
|
199
|
+
console.log(`✅ live scorecard written: ${outPath} (total $${totalCost.toFixed(6)})`)
|
|
200
|
+
process.exit(0)
|
|
154
201
|
}
|
|
155
202
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
`# Live J2 scorecard — ${date}`,
|
|
159
|
-
'',
|
|
160
|
-
`Route: \`${modelRef.providerID}/${modelRef.id}\`. Operator-run via \`opencode api\``,
|
|
161
|
-
'(consented route, billed to the user). 12 scenarios; agreement between the',
|
|
162
|
-
'live model and the expected pick is the metric — this measures the MODEL,',
|
|
163
|
-
'unlike the offline harness that measures the plumbing.',
|
|
164
|
-
'',
|
|
165
|
-
'| state (truncated) | expect | live | cost |',
|
|
166
|
-
'| --- | --- | --- | --- |',
|
|
167
|
-
...rows.map(
|
|
168
|
-
(r) =>
|
|
169
|
-
`| ${r.state.slice(0, 48)}${r.state.length > 48 ? '…' : ''} | ${r.expect} | ${r.got ?? '—'} | ${r.cost ?? '—'} |`,
|
|
170
|
-
),
|
|
171
|
-
'',
|
|
172
|
-
`Total cost: $${totalCost.toFixed(6)}. Honest caveat: one run is a smoke`,
|
|
173
|
-
'measurement, not a calibration — a claim about model quality needs',
|
|
174
|
-
'repeated runs and a bigger scenario set.',
|
|
175
|
-
'',
|
|
176
|
-
].join('\n')
|
|
177
|
-
const outPath = `bench/results/j2-live-scorecard-${date}.md`
|
|
178
|
-
writeFileSync(outPath, out)
|
|
179
|
-
console.log(`✅ live scorecard written: ${outPath} (total $${totalCost.toFixed(6)})`)
|
|
180
|
-
process.exit(0)
|
|
203
|
+
console.error('usage: node scripts/jev-live-probe.mjs --ping | --scorecard')
|
|
204
|
+
process.exit(1)
|
|
181
205
|
}
|
|
182
206
|
|
|
183
|
-
|
|
184
|
-
process.
|
|
207
|
+
const invokedDirectly =
|
|
208
|
+
process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
|
|
209
|
+
|
|
210
|
+
if (invokedDirectly) main()
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* skill-search.mjs — read-only search over the open skills ecosystem
|
|
4
|
+
* (skills.sh via the `skills` CLI) for the discovery pipeline documented in
|
|
5
|
+
* `docs/reference/native-decisions.md` and `src/agents/zeus.md`.
|
|
6
|
+
*
|
|
7
|
+
* The ladder this tool serves: installed skills first (one Jev
|
|
8
|
+
* `mode: "skills"` call answers over the local pool); only when nothing fits
|
|
9
|
+
* does the agent search the ecosystem, rank what came back with one typed
|
|
10
|
+
* Jev choice question, and PROPOSE the project-scoped install to the user —
|
|
11
|
+
* Pantheon never installs, and third-party skills are untrusted instructions.
|
|
12
|
+
*
|
|
13
|
+
* The `skills find` CLI prints ANSI-colored human text; this wrapper strips
|
|
14
|
+
* the escapes and emits the parseable contract (`owner/repo@skill`,
|
|
15
|
+
* install count, skills.sh URL) as JSON — the same contract the agent-side
|
|
16
|
+
* inline parse follows. Read-only: it never adds, updates, or removes
|
|
17
|
+
* skills, and no credentials cross it.
|
|
18
|
+
*
|
|
19
|
+
* Usage:
|
|
20
|
+
* node scripts/skill-search.mjs "<query>" [--limit 5] [--owner <owner>] [--raw]
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { spawnSync } from 'node:child_process'
|
|
24
|
+
import { pathToFileURL } from 'node:url'
|
|
25
|
+
|
|
26
|
+
const FIND_TIMEOUT_MS = 60_000
|
|
27
|
+
const DEFAULT_LIMIT = 5
|
|
28
|
+
/** SGR escape sequences are matched via the ESC codepoint (lint-clean, same semantics). */
|
|
29
|
+
const ANSI_SGR = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g')
|
|
30
|
+
const SKILL_LINE = /^([A-Za-z0-9._-]+\/[A-Za-z0-9._-]+@[A-Za-z0-9._-]+)\s+(\d+)\s+installs$/
|
|
31
|
+
const URL_LINE = /^└?\s*(https:\/\/skills\.sh\/\S+)$/
|
|
32
|
+
|
|
33
|
+
/** Remove SGR escape sequences so the text lines become parseable. */
|
|
34
|
+
export function stripAnsi(text) {
|
|
35
|
+
return String(text).replace(ANSI_SGR, '')
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Parse `skills find` output into ordered entries {skill, installs, url?}.
|
|
40
|
+
* `limit` (default 5) caps the result; entries keep the CLI's printed order
|
|
41
|
+
* (the ecosystem's own ranking), never re-sorted here.
|
|
42
|
+
*/
|
|
43
|
+
export function parseSkillSearchOutput(raw, { limit = DEFAULT_LIMIT } = {}) {
|
|
44
|
+
const lines = stripAnsi(raw)
|
|
45
|
+
.split('\n')
|
|
46
|
+
.map((line) => line.trim())
|
|
47
|
+
.filter((line) => line.length > 0)
|
|
48
|
+
const found = []
|
|
49
|
+
for (const line of lines) {
|
|
50
|
+
const skillMatch = line.match(SKILL_LINE)
|
|
51
|
+
if (skillMatch !== null) {
|
|
52
|
+
if (found.length >= limit) continue
|
|
53
|
+
found.push({ skill: skillMatch[1], installs: Number(skillMatch[2]), url: undefined })
|
|
54
|
+
continue
|
|
55
|
+
}
|
|
56
|
+
const urlMatch = line.match(URL_LINE)
|
|
57
|
+
if (urlMatch !== null && found.length > 0) {
|
|
58
|
+
const last = found[found.length - 1]
|
|
59
|
+
if (last.url === undefined) last.url = urlMatch[1]
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return found
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Run the ecosystem search once and return parsed entries. */
|
|
66
|
+
export function searchSkills(query, { limit = DEFAULT_LIMIT, owner, spawn = spawnSync } = {}) {
|
|
67
|
+
const args = ['skills', 'find', String(query)]
|
|
68
|
+
if (owner !== undefined) args.push('--owner', String(owner))
|
|
69
|
+
const result = spawn('npx', args, {
|
|
70
|
+
encoding: 'utf8',
|
|
71
|
+
timeout: FIND_TIMEOUT_MS,
|
|
72
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
73
|
+
})
|
|
74
|
+
if (result.error !== undefined) {
|
|
75
|
+
return { ok: false, error: `skills search failed to start: ${result.error.message}` }
|
|
76
|
+
}
|
|
77
|
+
const raw = `${result.stdout ?? ''}`
|
|
78
|
+
if (result.status !== 0 && raw.trim() === '') {
|
|
79
|
+
return { ok: false, error: `skills search exited ${result.status}` }
|
|
80
|
+
}
|
|
81
|
+
return { ok: true, entries: parseSkillSearchOutput(raw, { limit }), raw }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const GITHUB_TIMEOUT_MS = 30_000
|
|
85
|
+
const ANTHROPIC_SKILLS_REPO = 'anthropics/skills'
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Tier 1: the canonical Anthropic catalog. Highest trust (curated upstream),
|
|
89
|
+
* checked first so the canonical `pdf`/`docx`/`xlsx`/`mcp-builder` skills
|
|
90
|
+
* outrank the third-party clones that skills.sh's ranking can surface.
|
|
91
|
+
* Matches query tokens against catalog entry names; returns one entry per
|
|
92
|
+
* matched name.
|
|
93
|
+
*/
|
|
94
|
+
export function searchAnthropicCatalog(query, { limit = DEFAULT_LIMIT, spawn = spawnSync } = {}) {
|
|
95
|
+
const result = spawn(
|
|
96
|
+
'gh',
|
|
97
|
+
['api', `repos/${ANTHROPIC_SKILLS_REPO}/contents/skills`, '--jq', '.[].name'],
|
|
98
|
+
{ encoding: 'utf8', timeout: GITHUB_TIMEOUT_MS, stdio: ['ignore', 'pipe', 'pipe'] },
|
|
99
|
+
)
|
|
100
|
+
if (result.error !== undefined || result.status !== 0) {
|
|
101
|
+
return { ok: false, error: 'anthropic catalog lookup failed (gh api)' }
|
|
102
|
+
}
|
|
103
|
+
const tokens = String(query)
|
|
104
|
+
.toLowerCase()
|
|
105
|
+
.split(/[^a-z0-9]+/)
|
|
106
|
+
.filter((token) => token.length > 2)
|
|
107
|
+
const names = String(result.stdout ?? '')
|
|
108
|
+
.split('\n')
|
|
109
|
+
.map((name) => name.trim())
|
|
110
|
+
.filter((name) => name.length > 0)
|
|
111
|
+
const matched = names.filter((name) => {
|
|
112
|
+
const lower = name.toLowerCase()
|
|
113
|
+
return tokens.some((token) => lower.includes(token))
|
|
114
|
+
})
|
|
115
|
+
return {
|
|
116
|
+
ok: true,
|
|
117
|
+
entries: matched.slice(0, limit).map((name) => ({
|
|
118
|
+
skill: `${ANTHROPIC_SKILLS_REPO}@${name}`,
|
|
119
|
+
installs: null,
|
|
120
|
+
url: `https://github.com/${ANTHROPIC_SKILLS_REPO}/tree/main/skills/${name}`,
|
|
121
|
+
source: 'anthropic',
|
|
122
|
+
})),
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Tier 3: raw GitHub code search over every `SKILL.md` (the widest net —
|
|
128
|
+
* orders of magnitude beyond the skills.sh index, and the weakest trust:
|
|
129
|
+
* every hit is unreviewed third-party instructions).
|
|
130
|
+
*/
|
|
131
|
+
export function searchGithubSkills(query, { limit = DEFAULT_LIMIT, spawn = spawnSync } = {}) {
|
|
132
|
+
const terms = String(query)
|
|
133
|
+
.split(/\s+/)
|
|
134
|
+
.filter((term) => term.length > 1)
|
|
135
|
+
.slice(0, 3)
|
|
136
|
+
.map((term) => encodeURIComponent(term))
|
|
137
|
+
.join('+')
|
|
138
|
+
if (terms === '') return { ok: true, entries: [] }
|
|
139
|
+
const result = spawn(
|
|
140
|
+
'gh',
|
|
141
|
+
['api', `search/code?q=filename:SKILL.md+${terms}&per_page=${Math.min(limit, 10)}`],
|
|
142
|
+
{ encoding: 'utf8', timeout: GITHUB_TIMEOUT_MS, stdio: ['ignore', 'pipe', 'pipe'] },
|
|
143
|
+
)
|
|
144
|
+
if (result.error !== undefined || result.status !== 0) {
|
|
145
|
+
return { ok: false, error: 'github code search failed (gh api)' }
|
|
146
|
+
}
|
|
147
|
+
let parsed
|
|
148
|
+
try {
|
|
149
|
+
parsed = JSON.parse(String(result.stdout ?? '{}'))
|
|
150
|
+
} catch {
|
|
151
|
+
return { ok: false, error: 'github code search returned non-JSON output' }
|
|
152
|
+
}
|
|
153
|
+
const items = Array.isArray(parsed.items) ? parsed.items : []
|
|
154
|
+
return {
|
|
155
|
+
ok: true,
|
|
156
|
+
entries: items.slice(0, limit).map((item) => {
|
|
157
|
+
const repo = String(item?.repository?.full_name ?? 'unknown/repo')
|
|
158
|
+
const path = String(item?.path ?? 'SKILL.md')
|
|
159
|
+
const name = path.split('/')[Math.max(0, path.split('/').length - 2)] ?? 'skill'
|
|
160
|
+
return {
|
|
161
|
+
skill: `${repo}@${name}`,
|
|
162
|
+
installs: null,
|
|
163
|
+
url: `https://github.com/${repo}/blob/main/${path}`,
|
|
164
|
+
source: 'github',
|
|
165
|
+
}
|
|
166
|
+
}),
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* The trust cascade: anthropic (canonical) → skills-sh (install-count
|
|
172
|
+
* signal, unreviewed) → github (widest, least trusted). Tier order is the
|
|
173
|
+
* recommendation order; callers may narrow with `sources`.
|
|
174
|
+
*/
|
|
175
|
+
export const SKILL_SEARCH_SOURCES = Object.freeze(['anthropic', 'skills-sh', 'github'])
|
|
176
|
+
|
|
177
|
+
export function searchSkillSources(
|
|
178
|
+
query,
|
|
179
|
+
{ limit = DEFAULT_LIMIT, owner, sources = SKILL_SEARCH_SOURCES, spawn = spawnSync } = {},
|
|
180
|
+
) {
|
|
181
|
+
const perTier = Math.max(1, Math.ceil(limit / sources.length))
|
|
182
|
+
const out = []
|
|
183
|
+
const failures = []
|
|
184
|
+
for (const source of sources) {
|
|
185
|
+
const result =
|
|
186
|
+
source === 'anthropic'
|
|
187
|
+
? searchAnthropicCatalog(query, { limit: perTier, spawn })
|
|
188
|
+
: source === 'skills-sh'
|
|
189
|
+
? searchSkills(query, { limit: perTier, owner, spawn })
|
|
190
|
+
: source === 'github'
|
|
191
|
+
? searchGithubSkills(query, { limit: perTier, spawn })
|
|
192
|
+
: { ok: false, error: `unknown source: ${source}` }
|
|
193
|
+
if (result.ok) out.push(...result.entries.map((entry) => ({ ...entry, source })))
|
|
194
|
+
else failures.push(`${source}: ${result.error}`)
|
|
195
|
+
}
|
|
196
|
+
return { ok: failures.length < sources.length, entries: out.slice(0, limit), failures }
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function main(args = process.argv.slice(2)) {
|
|
200
|
+
const get = (flag) => {
|
|
201
|
+
const i = args.indexOf(flag)
|
|
202
|
+
return i >= 0 ? args[i + 1] : undefined
|
|
203
|
+
}
|
|
204
|
+
const query = args.find((a) => !a.startsWith('--'))
|
|
205
|
+
if (query === undefined) {
|
|
206
|
+
console.error(
|
|
207
|
+
'usage: node scripts/skill-search.mjs "<query>" [--limit 5] [--sources anthropic,skills-sh,github] [--owner <owner>] [--raw]',
|
|
208
|
+
)
|
|
209
|
+
process.exit(1)
|
|
210
|
+
}
|
|
211
|
+
const sourcesArg = get('--sources')
|
|
212
|
+
const sources =
|
|
213
|
+
sourcesArg === undefined
|
|
214
|
+
? SKILL_SEARCH_SOURCES
|
|
215
|
+
: sourcesArg
|
|
216
|
+
.split(',')
|
|
217
|
+
.map((s) => s.trim())
|
|
218
|
+
.filter((s) => s.length > 0)
|
|
219
|
+
const result = searchSkillSources(query, {
|
|
220
|
+
limit: Number(get('--limit') ?? DEFAULT_LIMIT),
|
|
221
|
+
owner: get('--owner'),
|
|
222
|
+
sources,
|
|
223
|
+
})
|
|
224
|
+
if (args.includes('--raw')) {
|
|
225
|
+
const skillsSh = result.entries.filter((e) => e.source === 'skills-sh')
|
|
226
|
+
console.log(JSON.stringify(skillsSh, null, 1))
|
|
227
|
+
return
|
|
228
|
+
}
|
|
229
|
+
if (result.failures.length > 0) {
|
|
230
|
+
for (const failure of result.failures) console.error(`warning: ${failure}`)
|
|
231
|
+
}
|
|
232
|
+
if (result.entries.length === 0 && result.failures.length === sources.length) {
|
|
233
|
+
console.error('all sources failed')
|
|
234
|
+
process.exit(1)
|
|
235
|
+
}
|
|
236
|
+
console.log(JSON.stringify(result.entries, null, 1))
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const invokedDirectly =
|
|
240
|
+
process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href
|
|
241
|
+
|
|
242
|
+
if (invokedDirectly) main()
|