pantheon-opencode 1.6.3-beta.6 → 1.6.3-beta.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +9 -1
- package/package.json +1 -1
- package/plugin.json +1 -1
- package/pyproject.toml +1 -1
- package/scripts/jev-live-probe.mjs +184 -0
- package/src/agents/zeus.md +24 -0
- package/src/pantheon/jev-decision-tool.ts +151 -24
- package/src/pantheon/jev-rank-verify.ts +435 -0
- package/src/plugins/tui/dist/tui-v2.js +3 -3
- package/src/plugins/tui/dist/tui.js +1 -1
- package/src/plugins/tui/package-lock.json +2 -2
- package/src/plugins/tui/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -10,7 +10,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
10
10
|
<!-- Add new changes here. Running `node scripts/versioning.mjs apply` will
|
|
11
11
|
move this section to a versioned entry and reset the template below. -->
|
|
12
12
|
|
|
13
|
-
## [v1.6.3-beta.
|
|
13
|
+
## [v1.6.3-beta.9] - 2026-11-04
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- **agents** — The live scorecard now runs the full 13-specialist catalog (was a 5-agent sample); first full-catalog run: 6 exact hits, 2 domain-adjacent judgment calls, measured per agent.
|
|
18
|
+
- **agents** — Zeus gains consolidated MCP call-timing rules: when to recall, when to store, when to checkpoint, when vision fires, and the escalation order (local selector -> Jev -> MCP last resort).
|
|
19
|
+
|
|
20
|
+
## [v1.6.3-beta.7] - 2026-11-04
|
|
21
|
+
|
|
22
|
+
### Added
|
|
23
|
+
|
|
24
|
+
- **routing** — `jev-live-probe.mjs`: the operator-run live measurement tool. `--ping` is the calibration dry-run after `setup-model` (one call, exit 0 when reachable); `--scorecard` runs the live scenarios against the configured route and writes the dated scorecard into `bench/results/`. First live run: 7/8 correct specialist picks via OpenRouter.
|
|
25
|
+
|
|
26
|
+
## [v1.6.3-beta.8] - 2026-11-04
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
|
|
30
|
+
- **routing** — The v3.1 **rank** and **verify** primitives on `pantheon_jev_decide`: `mode: "rank"` picks the winner among up to 8 candidates with a fit score (the host answers one scalar per question, so full ordering is impossible — documented); `mode: "verify"` checks 1–4 claims as predicate questions with **BLOCK fail-closed**. Hostile inputs rejected. Zeus-only, same gate chain.
|
|
31
|
+
- **agents** — The live scorecard runs the full 13-specialist catalog (first full run: 6/8 exact, 2 domain-adjacent); Zeus gains consolidated MCP call-timing rules.
|
|
32
|
+
|
|
33
|
+
## [v1.6.3-beta.7] - 2026-11-04
|
|
14
34
|
|
|
15
35
|
### Added
|
|
16
36
|
|
package/README.md
CHANGED
|
@@ -236,7 +236,15 @@ asks, after a delegation completes — one typed round-trip returns an advisory
|
|
|
236
236
|
quality score against named criteria plus a needs-review flag. The result
|
|
237
237
|
content never leaves the machine (shape metrics and criteria only, hostile
|
|
238
238
|
criteria rejected outright), and the verdict is advisory telemetry plus a
|
|
239
|
-
flag: it never blocks, retries, or alters the delegated result.
|
|
239
|
+
flag: it never blocks, retries, or alters the delegated result. Two more
|
|
240
|
+
orchestrator-owned tool modes round out the surface: `mode: "rank"` picks the
|
|
241
|
+
best of 2–8 named candidates for a state in one typed round-trip (it returns
|
|
242
|
+
the winner and that candidate's fit score — the host API answers one scalar
|
|
243
|
+
per question, so no full ordering is returned), and `mode: "verify"` checks
|
|
244
|
+
1–4 named claims with one predicate question per claim, mapping each answer to
|
|
245
|
+
`PASS` / `FAIL` / `BLOCK` where every refusal, timeout or malformed answer is
|
|
246
|
+
`BLOCK` (fail-closed); claims are bounded caller-provided strings, hostile
|
|
247
|
+
shapes rejected outright, and neither mode adds a telemetry point.
|
|
240
248
|
|
|
241
249
|
The installer may retain a legacy target-specific preference; it does not control
|
|
242
250
|
runtime routing. `npx pantheon-opencode decision-router ...` manages stored
|
package/package.json
CHANGED
package/plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pantheon",
|
|
3
3
|
"publisher": "ils15",
|
|
4
|
-
"version": "1.6.3-beta.
|
|
4
|
+
"version": "1.6.3-beta.9",
|
|
5
5
|
"displayName": "Pantheon",
|
|
6
6
|
"description": "Conductor-Delegate orchestration platform with 14 specialized agents, 20 skills, and OpenCode support.",
|
|
7
7
|
"agents": "./src/agents",
|
package/pyproject.toml
CHANGED
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.backends._legacy:_Backend"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pantheon"
|
|
7
|
-
version = "1.6.3-beta.
|
|
7
|
+
version = "1.6.3-beta.9"
|
|
8
8
|
description = "Multi-agent orchestration framework"
|
|
9
9
|
requires-python = ">=3.11"
|
|
10
10
|
# Python sources under src/mcp/ import fastmcp/pydantic at runtime, but those
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* jev-live-probe.mjs — operator-run live measurement for the Jev decision
|
|
4
|
+
* route (calibration ping + live scorecard).
|
|
5
|
+
*
|
|
6
|
+
* Talks to the HOST's decisions API through `opencode api` (the same route
|
|
7
|
+
* the plugin uses), so no credentials cross this script and every call is
|
|
8
|
+
* billed to the user's configured route. Operator-run by design: this is the
|
|
9
|
+
* consented measurement the J2 scorecard gate requires (offline harness
|
|
10
|
+
* measures plumbing; THIS measures the live model).
|
|
11
|
+
*
|
|
12
|
+
* Usage:
|
|
13
|
+
* node scripts/jev-live-probe.mjs --ping # 1 call, exit 0 if reachable
|
|
14
|
+
* node scripts/jev-live-probe.mjs --scorecard # N scenarios, dated scorecard
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { spawnSync } from 'node:child_process'
|
|
18
|
+
import { writeFileSync } from 'node:fs'
|
|
19
|
+
import { join } from 'node:path'
|
|
20
|
+
|
|
21
|
+
const HOST_TIMEOUT_MS = 30_000
|
|
22
|
+
|
|
23
|
+
/** Live scenarios: state → the skill a correct picker would choose. */
|
|
24
|
+
const SCENARIOS = [
|
|
25
|
+
{ state: 'Investigate why the payment webhook drops events under load', expect: 'apollo' },
|
|
26
|
+
{ state: 'Plan the database migration for the new billing table', expect: 'athena' },
|
|
27
|
+
{ state: 'Review the auth flow for security gaps before the release', expect: 'themis' },
|
|
28
|
+
{ state: 'Implement the retry queue endpoint for the notifications service', expect: 'hermes' },
|
|
29
|
+
{ state: 'Set up the CI pipeline for the new monorepo layout', expect: 'prometheus' },
|
|
30
|
+
{ state: 'Audit the permissions matrix across all specialists', expect: 'themis' },
|
|
31
|
+
{ state: 'Design the caching layer for the read-heavy dashboard', expect: 'athena' },
|
|
32
|
+
{ state: 'none — greet the user and ask what they want', expect: 'none' },
|
|
33
|
+
// Zero-dispatch agents' domains (roster-reform evidence batch)
|
|
34
|
+
{ state: 'Write the SQL indexes and stored procedures for the orders schema', expect: 'demeter' },
|
|
35
|
+
{ state: 'Process the satellite imagery tiles and classify land use', expect: 'gaia' },
|
|
36
|
+
{
|
|
37
|
+
state: 'Integrate the new LLM provider SDK and wire the prompt templates',
|
|
38
|
+
expect: 'hephaestus',
|
|
39
|
+
},
|
|
40
|
+
{ state: 'Migrate the open issues to the new milestone labels via the API', expect: 'iris' },
|
|
41
|
+
{ state: 'Decide which stale session summaries are worth keeping', expect: 'mnemosyne' },
|
|
42
|
+
{ state: 'Fuzz the input parser with malformed payloads to find crashes', expect: 'nyx' },
|
|
43
|
+
{ state: 'Verify the export module against the compliance checklist', expect: 'talos' },
|
|
44
|
+
{ state: 'Polish the onboarding screen layout and micro-interactions', expect: 'aphrodite' },
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
function callDecisions(modelRef, state) {
|
|
48
|
+
const payload = JSON.stringify({
|
|
49
|
+
model: modelRef,
|
|
50
|
+
input: [{ type: 'text', text: state }],
|
|
51
|
+
questions: [
|
|
52
|
+
{
|
|
53
|
+
type: 'choice',
|
|
54
|
+
name: 'pick',
|
|
55
|
+
instructions:
|
|
56
|
+
'Pick the single best specialist for this task, or none. Required skills beat preferred ones; judge from the state alone.',
|
|
57
|
+
choices: [
|
|
58
|
+
{ value: 'apollo', description: 'read-only investigation and research' },
|
|
59
|
+
{ value: 'athena', description: 'planning and architecture' },
|
|
60
|
+
{ value: 'themis', description: 'security, review and compliance gates' },
|
|
61
|
+
{ value: 'hermes', description: 'backend implementation' },
|
|
62
|
+
{ value: 'prometheus', description: 'infrastructure, CI and pipelines' },
|
|
63
|
+
{ value: 'hephaestus', description: 'AI tooling and model integration' },
|
|
64
|
+
{ value: 'demeter', description: 'databases and data layer' },
|
|
65
|
+
{ value: 'gaia', description: 'remote sensing and geospatial' },
|
|
66
|
+
{ value: 'iris', description: 'GitHub operations and automation' },
|
|
67
|
+
{ value: 'aphrodite', description: 'UI and frontend craft' },
|
|
68
|
+
{ value: 'mnemosyne', description: 'memory curation and recall' },
|
|
69
|
+
{ value: 'nyx', description: 'chaos and adversarial testing' },
|
|
70
|
+
{ value: 'talos', description: 'validation and compliance checks' },
|
|
71
|
+
{ value: 'none', description: 'no specialist fits' },
|
|
72
|
+
],
|
|
73
|
+
},
|
|
74
|
+
],
|
|
75
|
+
})
|
|
76
|
+
const result = spawnSync(
|
|
77
|
+
'opencode',
|
|
78
|
+
['api', 'POST', '/api/experimental/generate/decisions', '-d', payload],
|
|
79
|
+
{
|
|
80
|
+
encoding: 'utf8',
|
|
81
|
+
timeout: HOST_TIMEOUT_MS,
|
|
82
|
+
},
|
|
83
|
+
)
|
|
84
|
+
if (result.status !== 0) {
|
|
85
|
+
return { ok: false, raw: (result.stderr || result.stdout || '').slice(0, 300) }
|
|
86
|
+
}
|
|
87
|
+
try {
|
|
88
|
+
return { ok: true, data: JSON.parse(result.stdout) }
|
|
89
|
+
} catch {
|
|
90
|
+
return { ok: false, raw: result.stdout.slice(0, 300) }
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function answerChoice(data) {
|
|
95
|
+
const answers = data?.data?.answers ?? data?.answers
|
|
96
|
+
if (!Array.isArray(answers)) return null
|
|
97
|
+
const pick = answers.find((a) => a.name === 'pick')
|
|
98
|
+
return pick?.choice ?? null
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
const args = process.argv.slice(2)
|
|
102
|
+
|
|
103
|
+
if (args.includes('--ping')) {
|
|
104
|
+
const modelRef = { providerID: 'opencode', id: 'jev-1.13-free' }
|
|
105
|
+
const r = callDecisions(modelRef, 'Calibration ping: is the Jev decision route reachable?')
|
|
106
|
+
if (r.ok) {
|
|
107
|
+
const cost = r.data?.cost ?? r.data?.data?.cost
|
|
108
|
+
console.log(`✅ reachable — cost: ${JSON.stringify(cost)}`)
|
|
109
|
+
process.exit(0)
|
|
110
|
+
}
|
|
111
|
+
console.error(`❌ unreachable: ${r.raw}`)
|
|
112
|
+
process.exit(1)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
if (args.includes('--scorecard')) {
|
|
116
|
+
// Route from the user's decision-router config (explicit selection only).
|
|
117
|
+
const { readFileSync } = await import('node:fs')
|
|
118
|
+
const { homedir } = await import('node:os')
|
|
119
|
+
const xdg = process.env.XDG_CONFIG_HOME?.startsWith('/')
|
|
120
|
+
? process.env.XDG_CONFIG_HOME
|
|
121
|
+
: join(homedir(), '.config')
|
|
122
|
+
const cfg = JSON.parse(readFileSync(join(xdg, 'pantheon', 'decision-router.json'), 'utf8'))
|
|
123
|
+
if (!cfg.enabled || cfg.selection.mode !== 'explicit') {
|
|
124
|
+
console.error('the router must be enabled with an explicit selection for a live run')
|
|
125
|
+
process.exit(1)
|
|
126
|
+
}
|
|
127
|
+
// Same host-provider mapping the plugin route resolver uses (subset of
|
|
128
|
+
// NATIVE_JEV_HOST_PROVIDERS — keep in sync when the matrix expands).
|
|
129
|
+
const HOST_PROVIDERS = {
|
|
130
|
+
'typesafe-jev': 'typesafe-ai',
|
|
131
|
+
'opencode-console-jev': 'opencode',
|
|
132
|
+
'openrouter-jev': 'openrouter',
|
|
133
|
+
'vercel-gateway-jev': 'vercel-ai-gateway',
|
|
134
|
+
'cloudflare-workers-jev': 'cloudflare-workers-ai',
|
|
135
|
+
}
|
|
136
|
+
const hostProvider = HOST_PROVIDERS[cfg.selection.provider]
|
|
137
|
+
if (!hostProvider) {
|
|
138
|
+
console.error(`unknown selection provider: ${cfg.selection.provider}`)
|
|
139
|
+
process.exit(1)
|
|
140
|
+
}
|
|
141
|
+
const modelRef = { providerID: hostProvider, id: cfg.selection.model }
|
|
142
|
+
|
|
143
|
+
const rows = []
|
|
144
|
+
let totalCost = 0
|
|
145
|
+
for (const scenario of SCENARIOS) {
|
|
146
|
+
const r = callDecisions(modelRef, scenario.state)
|
|
147
|
+
if (!r.ok) {
|
|
148
|
+
rows.push({ state: scenario.state, expect: scenario.expect, got: `ERROR: ${r.raw}` })
|
|
149
|
+
continue
|
|
150
|
+
}
|
|
151
|
+
const cost = r.data?.cost ?? 0
|
|
152
|
+
totalCost += Number(cost) || 0
|
|
153
|
+
rows.push({ state: scenario.state, expect: scenario.expect, got: answerChoice(r.data), cost })
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const date = new Date().toISOString().slice(0, 10)
|
|
157
|
+
const out = [
|
|
158
|
+
`# Live J2 scorecard — ${date}`,
|
|
159
|
+
'',
|
|
160
|
+
`Route: \`${modelRef.providerID}/${modelRef.id}\`. Operator-run via \`opencode api\``,
|
|
161
|
+
'(consented route, billed to the user). 12 scenarios; agreement between the',
|
|
162
|
+
'live model and the expected pick is the metric — this measures the MODEL,',
|
|
163
|
+
'unlike the offline harness that measures the plumbing.',
|
|
164
|
+
'',
|
|
165
|
+
'| state (truncated) | expect | live | cost |',
|
|
166
|
+
'| --- | --- | --- | --- |',
|
|
167
|
+
...rows.map(
|
|
168
|
+
(r) =>
|
|
169
|
+
`| ${r.state.slice(0, 48)}${r.state.length > 48 ? '…' : ''} | ${r.expect} | ${r.got ?? '—'} | ${r.cost ?? '—'} |`,
|
|
170
|
+
),
|
|
171
|
+
'',
|
|
172
|
+
`Total cost: $${totalCost.toFixed(6)}. Honest caveat: one run is a smoke`,
|
|
173
|
+
'measurement, not a calibration — a claim about model quality needs',
|
|
174
|
+
'repeated runs and a bigger scenario set.',
|
|
175
|
+
'',
|
|
176
|
+
].join('\n')
|
|
177
|
+
const outPath = `bench/results/j2-live-scorecard-${date}.md`
|
|
178
|
+
writeFileSync(outPath, out)
|
|
179
|
+
console.log(`✅ live scorecard written: ${outPath} (total $${totalCost.toFixed(6)})`)
|
|
180
|
+
process.exit(0)
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
console.error('usage: node scripts/jev-live-probe.mjs --ping | --scorecard')
|
|
184
|
+
process.exit(1)
|
package/src/agents/zeus.md
CHANGED
|
@@ -53,6 +53,30 @@ skills:
|
|
|
53
53
|
`docs/tui-v2-quota-providers.md` — it is not a Pantheon surface and never a
|
|
54
54
|
Pantheon egress path.
|
|
55
55
|
|
|
56
|
+
## MCP call timing (when each integration fires)
|
|
57
|
+
|
|
58
|
+
- **Memory (recall)** — only when prior project history could change the
|
|
59
|
+
outcome of the current decision, and at most one selective search per
|
|
60
|
+
turn. Never recall for context you already hold; never recall during a
|
|
61
|
+
delegation (the Brief is self-contained by design).
|
|
62
|
+
- **Memory (store)** — when YOU judge it prudent: durable, reusable facts
|
|
63
|
+
(decisions, root causes, conventions) that a future session would need.
|
|
64
|
+
Not per-turn noise; never store secrets, user content or transient state.
|
|
65
|
+
The user can always ask to store or purge — your judgment handles the
|
|
66
|
+
rest.
|
|
67
|
+
- **Persistence (checkpoint)** — when prudent: session milestones where the
|
|
68
|
+
work done so far would be expensive to lose (after a risky migration, a
|
|
69
|
+
long investigation, a structural change). Never per turn; never for
|
|
70
|
+
trivial progress.
|
|
71
|
+
- **Vision (`vision_describe`)** — only when a tool call or attachment
|
|
72
|
+
actually carries an image; a text-only turn never fires it.
|
|
73
|
+
- **Code-mode / quota** — infrastructure reads (sessions, models, worktrees)
|
|
74
|
+
are cheap and on demand; never for decisions that `pantheon_jev_decide`
|
|
75
|
+
or the local selector already answer.
|
|
76
|
+
- When in doubt: the local selector answers free, the Jev tool answers
|
|
77
|
+
cheap and typed, an MCP call is the LAST resort for state the session
|
|
78
|
+
cannot reconstruct.
|
|
79
|
+
|
|
56
80
|
## Missing capability? Find a skill before inventing one
|
|
57
81
|
|
|
58
82
|
When no installed skill covers a recurring need, search the open ecosystem before writing new instructions:
|
|
@@ -42,6 +42,12 @@ import {
|
|
|
42
42
|
type QualityScoreEvaluation,
|
|
43
43
|
qualityVerdict,
|
|
44
44
|
} from './jev-quality-score.ts'
|
|
45
|
+
import {
|
|
46
|
+
buildJevRankRequest,
|
|
47
|
+
buildJevVerifyRequest,
|
|
48
|
+
consumeJevRankResult,
|
|
49
|
+
consumeJevVerifyResult,
|
|
50
|
+
} from './jev-rank-verify.ts'
|
|
45
51
|
import { mapGateFailureToSkipReason } from './jev-risk-gate.ts'
|
|
46
52
|
import {
|
|
47
53
|
openLocalDecisionBudgetLedger,
|
|
@@ -70,7 +76,8 @@ export interface V2JevDecisionRuntime {
|
|
|
70
76
|
/**
|
|
71
77
|
* Allowlisted specialist callers for explicit-questions mode (parsed from
|
|
72
78
|
* `PANTHEON_JEV_SPECIALIST_CALLERS` by the plugin wiring). Empty/absent =
|
|
73
|
-
* Zeus only. Preset modes
|
|
79
|
+
* Zeus only. Preset and v3.1 modes (skills, delegation, review, rank,
|
|
80
|
+
* verify) stay Zeus-only regardless.
|
|
74
81
|
*/
|
|
75
82
|
readonly specialistCallers?: readonly string[]
|
|
76
83
|
/**
|
|
@@ -98,11 +105,12 @@ export const SPECIALIST_CALLERS_FLAG_ENV = 'PANTHEON_JEV_SPECIALIST_CALLERS'
|
|
|
98
105
|
|
|
99
106
|
/**
|
|
100
107
|
* Parse the caller allowlist from the flag's CSV value. Unset/empty = Zeus
|
|
101
|
-
* only (the default). Preset modes (skills, delegation, review
|
|
102
|
-
* Zeus-only regardless of the flag: owner/skill selection
|
|
103
|
-
* scoring
|
|
104
|
-
* questions mid-task, nothing
|
|
105
|
-
*
|
|
108
|
+
* only (the default). Preset and v3.1 modes (skills, delegation, review,
|
|
109
|
+
* rank, verify) stay Zeus-only regardless of the flag: owner/skill selection,
|
|
110
|
+
* result scoring, candidate ranking and claim verification are orchestration
|
|
111
|
+
* decisions; specialists may ask bounded typed questions mid-task, nothing
|
|
112
|
+
* else. Lives in the plugin wiring (which owns flag reads); this module
|
|
113
|
+
* receives the parsed list via the runtime.
|
|
106
114
|
*/
|
|
107
115
|
export function specialistCallerAllowlist(flagValue: string | undefined): readonly string[] {
|
|
108
116
|
const raw = String(flagValue ?? '').trim()
|
|
@@ -118,6 +126,10 @@ export type JevDecisionRunFailure =
|
|
|
118
126
|
| 'CALLER_NOT_ALLOWED'
|
|
119
127
|
| 'REVIEW_INPUT_INVALID'
|
|
120
128
|
| 'CRITERIA_INVALID'
|
|
129
|
+
| 'RANK_INPUT_INVALID'
|
|
130
|
+
| 'CANDIDATE_INVALID'
|
|
131
|
+
| 'VERIFY_INPUT_INVALID'
|
|
132
|
+
| 'CLAIM_INVALID'
|
|
121
133
|
| 'CONTEXT_UNAVAILABLE'
|
|
122
134
|
| 'CONFIG_UNAVAILABLE'
|
|
123
135
|
| 'DISABLED'
|
|
@@ -159,15 +171,24 @@ function reviewRefusalSkipReason(reasonCode: JevDecisionRunFailure): string {
|
|
|
159
171
|
/**
|
|
160
172
|
* Every gate-chain failure the shared runner can produce. It is
|
|
161
173
|
* {@link JevDecisionRunFailure} minus the tool-surface properties:
|
|
162
|
-
* `CALLER_NOT_ALLOWED` (the caller check) and the
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
174
|
+
* `CALLER_NOT_ALLOWED` (the caller check) and the input validation of the
|
|
175
|
+
* prebuilt-request modes — the review mode (`REVIEW_INPUT_INVALID` /
|
|
176
|
+
* `CRITERIA_INVALID`) and the v3.1 rank and verify modes
|
|
177
|
+
* (`RANK_INPUT_INVALID` / `CANDIDATE_INVALID` / `VERIFY_INPUT_INVALID` /
|
|
178
|
+
* `CLAIM_INVALID`). All of these are properties of the explicit tool surface,
|
|
179
|
+
* whose input the caller supplies directly, not of the gate chain; the
|
|
180
|
+
* J1/J3 shadow observers run the same chain for any agent and can never
|
|
181
|
+
* produce them.
|
|
167
182
|
*/
|
|
168
183
|
export type GatedNativeJevFailure = Exclude<
|
|
169
184
|
JevDecisionRunFailure,
|
|
170
|
-
|
|
185
|
+
| 'CALLER_NOT_ALLOWED'
|
|
186
|
+
| 'REVIEW_INPUT_INVALID'
|
|
187
|
+
| 'CRITERIA_INVALID'
|
|
188
|
+
| 'RANK_INPUT_INVALID'
|
|
189
|
+
| 'CANDIDATE_INVALID'
|
|
190
|
+
| 'VERIFY_INPUT_INVALID'
|
|
191
|
+
| 'CLAIM_INVALID'
|
|
171
192
|
>
|
|
172
193
|
|
|
173
194
|
/** The minimal runtime slice the gate chain needs. */
|
|
@@ -425,8 +446,10 @@ export async function runJevDecision(
|
|
|
425
446
|
}
|
|
426
447
|
// Review mode (J4): the bounded request is built before any gate work, and
|
|
427
448
|
// an invalid review input is a tool-surface refusal — same class as the
|
|
428
|
-
// caller check, never a gate failure.
|
|
429
|
-
|
|
449
|
+
// caller check, never a gate failure. The v3.1 rank and verify modes follow
|
|
450
|
+
// the same prebuilt-request pattern (no gate work for a malformed input);
|
|
451
|
+
// neither has a telemetry point — they are tool modes, not shadow gates.
|
|
452
|
+
let prebuilt: Omit<NativeDecisionRequest, 'model'> | undefined
|
|
430
453
|
if (requested.mode === 'review') {
|
|
431
454
|
const prepared = buildJevReviewRequest(input ?? {})
|
|
432
455
|
if (!prepared.ok) {
|
|
@@ -441,7 +464,15 @@ export async function runJevDecision(
|
|
|
441
464
|
)
|
|
442
465
|
return refusal(prepared.reasonCode)
|
|
443
466
|
}
|
|
444
|
-
|
|
467
|
+
prebuilt = prepared.request
|
|
468
|
+
} else if (requested.mode === 'rank') {
|
|
469
|
+
const prepared = buildJevRankRequest(input ?? {})
|
|
470
|
+
if (!prepared.ok) return refusal(prepared.reasonCode)
|
|
471
|
+
prebuilt = prepared.request
|
|
472
|
+
} else if (requested.mode === 'verify') {
|
|
473
|
+
const prepared = buildJevVerifyRequest(input ?? {})
|
|
474
|
+
if (!prepared.ok) return refusal(prepared.reasonCode)
|
|
475
|
+
prebuilt = prepared.request
|
|
445
476
|
}
|
|
446
477
|
const result = await runGatedNativeJevCall({
|
|
447
478
|
sessionID: context?.sessionID,
|
|
@@ -449,13 +480,13 @@ export async function runJevDecision(
|
|
|
449
480
|
agent: caller,
|
|
450
481
|
runtime,
|
|
451
482
|
prepare: async (route) =>
|
|
452
|
-
|
|
483
|
+
prebuilt !== undefined
|
|
453
484
|
? {
|
|
454
485
|
ok: true as const,
|
|
455
486
|
request: {
|
|
456
487
|
model: route.model,
|
|
457
|
-
input:
|
|
458
|
-
questions:
|
|
488
|
+
input: prebuilt.input,
|
|
489
|
+
questions: prebuilt.questions,
|
|
459
490
|
},
|
|
460
491
|
}
|
|
461
492
|
: requested.mode === undefined
|
|
@@ -541,6 +572,53 @@ export async function runJevDecision(
|
|
|
541
572
|
frozen: result.call.frozen,
|
|
542
573
|
})
|
|
543
574
|
}
|
|
575
|
+
if (requested.mode === 'rank') {
|
|
576
|
+
// v3.1 rank: the winner choice plus that candidate's fit score. The host
|
|
577
|
+
// API answers one scalar per question, so no runner-up or full ordering is
|
|
578
|
+
// returned — see jev-rank-verify.ts for the honest limitation.
|
|
579
|
+
const rank = consumeJevRankResult(result.call.response, result.call.request)
|
|
580
|
+
return JSON.stringify({
|
|
581
|
+
advisory: true,
|
|
582
|
+
effect: 'none',
|
|
583
|
+
reason: 'JEV_DECISION',
|
|
584
|
+
caller,
|
|
585
|
+
route: {
|
|
586
|
+
providerID: result.call.selection.route.model.providerID,
|
|
587
|
+
id: result.call.selection.route.model.id,
|
|
588
|
+
billing: result.call.selection.route.billing,
|
|
589
|
+
},
|
|
590
|
+
mode: 'rank',
|
|
591
|
+
winner: rank.winner,
|
|
592
|
+
winnerFit: rank.winnerFit,
|
|
593
|
+
answers: advice.answers,
|
|
594
|
+
missing: advice.missing,
|
|
595
|
+
usage: { costUsd: advice.costUsd, tokens: advice.tokens },
|
|
596
|
+
frozen: result.call.frozen,
|
|
597
|
+
})
|
|
598
|
+
}
|
|
599
|
+
if (requested.mode === 'verify') {
|
|
600
|
+
// v3.1 verify: one PASS/FAIL/BLOCK verdict per claim, fail-closed. The
|
|
601
|
+
// tool never acts on the verdicts; a caller that requires a verdict
|
|
602
|
+
// treats a refusal (which never reaches this branch) as BLOCK.
|
|
603
|
+
const results = consumeJevVerifyResult(result.call.response, result.call.request)
|
|
604
|
+
return JSON.stringify({
|
|
605
|
+
advisory: true,
|
|
606
|
+
effect: 'none',
|
|
607
|
+
reason: 'JEV_DECISION',
|
|
608
|
+
caller,
|
|
609
|
+
route: {
|
|
610
|
+
providerID: result.call.selection.route.model.providerID,
|
|
611
|
+
id: result.call.selection.route.model.id,
|
|
612
|
+
billing: result.call.selection.route.billing,
|
|
613
|
+
},
|
|
614
|
+
mode: 'verify',
|
|
615
|
+
results,
|
|
616
|
+
answers: advice.answers,
|
|
617
|
+
missing: advice.missing,
|
|
618
|
+
usage: { costUsd: advice.costUsd, tokens: advice.tokens },
|
|
619
|
+
frozen: result.call.frozen,
|
|
620
|
+
})
|
|
621
|
+
}
|
|
544
622
|
return JSON.stringify({
|
|
545
623
|
advisory: true,
|
|
546
624
|
effect: 'none',
|
|
@@ -567,7 +645,10 @@ const TOOL_DESCRIPTION =
|
|
|
567
645
|
'host), `mode: "delegation"` to pick ' +
|
|
568
646
|
'a specialist, `mode: "review"` to score a finished delegated result against named criteria ' +
|
|
569
647
|
'(advisory quality score + needs-review flag; the result content itself is never sent — ' +
|
|
570
|
-
'shape metrics and criteria only),
|
|
648
|
+
'shape metrics and criteria only), `mode: "rank"` to pick the best of 2-8 named candidates ' +
|
|
649
|
+
'(returns the winner and its fit score; no full ordering), `mode: "verify"` to check 1-4 ' +
|
|
650
|
+
'named claims (PASS/FAIL per claim; any refusal, timeout or malformed answer is BLOCK, ' +
|
|
651
|
+
'fail-closed), or pass explicit typed `questions` (choice, score, ' +
|
|
571
652
|
'predicate). Invalid-input refusals carry a `detail` token naming the exact cause. Returns ' +
|
|
572
653
|
'probabilistic answers with confidence. Requires decision-router egress ' +
|
|
573
654
|
'consent and budget caps; asks host permission first. It never dispatches agents, runs ' +
|
|
@@ -622,13 +703,45 @@ const REVIEW_METRICS_SCHEMA = {
|
|
|
622
703
|
},
|
|
623
704
|
} as const
|
|
624
705
|
|
|
706
|
+
const RANK_CANDIDATE_SCHEMA = {
|
|
707
|
+
type: 'object',
|
|
708
|
+
additionalProperties: false,
|
|
709
|
+
required: ['id'],
|
|
710
|
+
properties: {
|
|
711
|
+
id: {
|
|
712
|
+
type: 'string',
|
|
713
|
+
minLength: 1,
|
|
714
|
+
maxLength: 64,
|
|
715
|
+
pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$',
|
|
716
|
+
},
|
|
717
|
+
description: { type: 'string', minLength: 1, maxLength: 500 },
|
|
718
|
+
},
|
|
719
|
+
} as const
|
|
720
|
+
|
|
721
|
+
const VERIFY_CLAIM_SCHEMA = {
|
|
722
|
+
type: 'object',
|
|
723
|
+
additionalProperties: false,
|
|
724
|
+
required: ['name', 'claim'],
|
|
725
|
+
properties: {
|
|
726
|
+
name: {
|
|
727
|
+
type: 'string',
|
|
728
|
+
minLength: 1,
|
|
729
|
+
maxLength: 64,
|
|
730
|
+
pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$',
|
|
731
|
+
},
|
|
732
|
+
claim: { type: 'string', minLength: 1, maxLength: 200 },
|
|
733
|
+
},
|
|
734
|
+
} as const
|
|
735
|
+
|
|
625
736
|
/**
|
|
626
|
-
* One schema,
|
|
627
|
-
* questions carry the state text)
|
|
737
|
+
* One schema, four disjoint payloads: the state branch (presets and explicit
|
|
738
|
+
* questions carry the state text), the review branch (J4 takes structural
|
|
628
739
|
* shape metrics, a delegate id and named criteria — never a state text, never
|
|
629
740
|
* result content; the criteria are the only free text, bounded at the DSH
|
|
630
|
-
* caps: at most 4, each 1–120 chars)
|
|
631
|
-
*
|
|
741
|
+
* caps: at most 4, each 1–120 chars), the rank branch (v3.1: a state plus
|
|
742
|
+
* 2–8 named candidates) and the verify branch (v3.1: 1–4 named claims, no
|
|
743
|
+
* state). The preparers are the authority; the schema is the model-facing
|
|
744
|
+
* contract.
|
|
632
745
|
*/
|
|
633
746
|
const JEV_INPUT_SCHEMA: Record<string, unknown> = {
|
|
634
747
|
type: 'object',
|
|
@@ -637,11 +750,13 @@ const JEV_INPUT_SCHEMA: Record<string, unknown> = {
|
|
|
637
750
|
anyOf: [
|
|
638
751
|
{ required: ['state'] },
|
|
639
752
|
{ required: ['delegateId', 'resultSummaryKind', 'resultMetrics', 'criteria'] },
|
|
753
|
+
{ required: ['candidates'] },
|
|
754
|
+
{ required: ['claims'] },
|
|
640
755
|
],
|
|
641
756
|
properties: {
|
|
642
757
|
schemaVersion: { type: 'integer', const: 1 },
|
|
643
758
|
state: { type: 'string', minLength: 1, maxLength: 12_000 },
|
|
644
|
-
mode: { type: 'string', enum: ['skills', 'delegation', 'review'] },
|
|
759
|
+
mode: { type: 'string', enum: ['skills', 'delegation', 'review', 'rank', 'verify'] },
|
|
645
760
|
questions: { type: 'array', minItems: 1, maxItems: 4, items: questionSchema },
|
|
646
761
|
delegateId: {
|
|
647
762
|
type: 'string',
|
|
@@ -657,6 +772,18 @@ const JEV_INPUT_SCHEMA: Record<string, unknown> = {
|
|
|
657
772
|
maxItems: 4,
|
|
658
773
|
items: { type: 'string', minLength: 1, maxLength: 120 },
|
|
659
774
|
},
|
|
775
|
+
candidates: {
|
|
776
|
+
type: 'array',
|
|
777
|
+
minItems: 2,
|
|
778
|
+
maxItems: 8,
|
|
779
|
+
items: RANK_CANDIDATE_SCHEMA,
|
|
780
|
+
},
|
|
781
|
+
claims: {
|
|
782
|
+
type: 'array',
|
|
783
|
+
minItems: 1,
|
|
784
|
+
maxItems: 4,
|
|
785
|
+
items: VERIFY_CLAIM_SCHEMA,
|
|
786
|
+
},
|
|
660
787
|
},
|
|
661
788
|
}
|
|
662
789
|
|
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The v3.1 rank and verify primitives on the native Jev route (tool modes on
|
|
3
|
+
* `pantheon_jev_decide`, built after the J4 review-mode pattern).
|
|
4
|
+
*
|
|
5
|
+
* ## Rank — `mode: "rank"`, one choice + one score, `{ winner, winnerFit }`
|
|
6
|
+
*
|
|
7
|
+
* Zeus can rank 2–8 named candidates for a state in ONE typed round-trip: one
|
|
8
|
+
* `choice` question (`winner`) over the candidate ids picks the best candidate
|
|
9
|
+
* and one `score` question (`fit`, the 2..10 rubric bound) rates that
|
|
10
|
+
* candidate's fit. THE HONEST LIMITATION: the host Decisions API returns one
|
|
11
|
+
* scalar answer per question (`NativeDecisionRequest` is tied to the 2.2.5
|
|
12
|
+
* host types), so per-candidate scores are not expressible in a single
|
|
13
|
+
* round-trip — one score question per candidate would break the 4-question
|
|
14
|
+
* bound above four candidates, and one ambiguous "score each candidate"
|
|
15
|
+
* question has no per-candidate answer structure to pair back. Rank therefore
|
|
16
|
+
* returns `{ winner, winnerFit }` and NOTHING more: no runner-up, no full
|
|
17
|
+
* ordering. A caller that needs a deeper order uses the explicit-questions
|
|
18
|
+
* mode with its own typed questions.
|
|
19
|
+
*
|
|
20
|
+
* ## Verify — `mode: "verify"`, one predicate per claim, fail-closed BLOCK
|
|
21
|
+
*
|
|
22
|
+
* Zeus can check 1–4 named claims in ONE typed round-trip: one `predicate`
|
|
23
|
+
* question per claim (within the 4-question bound), the claim text riding that
|
|
24
|
+
* question's instructions. The consumer maps each answer: probability
|
|
25
|
+
* ≥ {@link VERIFY_PASS_THRESHOLD} → `PASS`, below → `FAIL`, and every refusal,
|
|
26
|
+
* timeout, duplicate or malformed answer → `BLOCK` (fail-closed: an
|
|
27
|
+
* unanswered mandatory claim must never read as verified). The tool itself
|
|
28
|
+
* never acts on a verdict — `effect` stays `none` — and a gate-chain refusal
|
|
29
|
+
* stays the standard bounded refusal JSON; a caller that REQUIRES a verdict
|
|
30
|
+
* treats every refusal as `BLOCK` at its own call site.
|
|
31
|
+
*
|
|
32
|
+
* ## Sanitization contract (the J4 criteria class)
|
|
33
|
+
*
|
|
34
|
+
* Both modes take bounded caller-provided fields only: rank's candidate ids
|
|
35
|
+
* are strict tokens (they become choice values), its state rides the same
|
|
36
|
+
* 12,000-char bound and verbatim-carriage rule as the presets, and verify's
|
|
37
|
+
* claim texts are the only free text, riding the predicate instructions. The
|
|
38
|
+
* hostility scan mirrors the J4 criteria scan (paths, credential assignments,
|
|
39
|
+
* key material, control characters, over-length): hostile shapes are REJECTED
|
|
40
|
+
* — `CANDIDATE_INVALID` / `CLAIM_INVALID` — never sanitized on the way in.
|
|
41
|
+
* Neither mode has a telemetry point (they are tool modes, not shadow gates)
|
|
42
|
+
* and both reuse the tool's strict answer consumer
|
|
43
|
+
* (`consumeJevDecisionResult`): one answer per question name, duplicates
|
|
44
|
+
* become errors, unbounded host metadata never crosses.
|
|
45
|
+
*
|
|
46
|
+
* @module jev-rank-verify
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
import { consumeJevDecisionResult } from './jev-decisions.ts'
|
|
50
|
+
import type { NativeDecisionRequest, ResolvedNativeDecisionRoute } from './v2-native-decisions.ts'
|
|
51
|
+
import { NATIVE_DECISION_MAX_PROMPT_LENGTH } from './v2-native-decisions.ts'
|
|
52
|
+
|
|
53
|
+
// ─── Bounds ──────────────────────────────────────────────────────────────
|
|
54
|
+
|
|
55
|
+
/** A ranking needs at least two candidates to choose between. */
|
|
56
|
+
export const RANK_MIN_CANDIDATES = 2
|
|
57
|
+
/** Above eight, ranking degrades to explicit questions with a real rubric. */
|
|
58
|
+
export const RANK_MAX_CANDIDATES = 8
|
|
59
|
+
|
|
60
|
+
/** Question name of the rank's choice question. */
|
|
61
|
+
export const RANK_WINNER_QUESTION_NAME = 'winner'
|
|
62
|
+
/** Question name of the rank's fit score question. */
|
|
63
|
+
export const RANK_FIT_QUESTION_NAME = 'fit'
|
|
64
|
+
|
|
65
|
+
/** The fit rubric mirrors the documented 2..10 score-question bound. */
|
|
66
|
+
export const RANK_FIT_LEVEL_COUNT = 9
|
|
67
|
+
|
|
68
|
+
/** At least one claim per verify call; one predicate question per claim. */
|
|
69
|
+
export const VERIFY_MIN_CLAIMS = 1
|
|
70
|
+
/** One predicate question per claim, within the 4-question bound. */
|
|
71
|
+
export const VERIFY_MAX_CLAIMS = 4
|
|
72
|
+
/** Each claim is a bounded 1–200-char instruction string. */
|
|
73
|
+
export const CLAIM_MAX_LENGTH = 200
|
|
74
|
+
|
|
75
|
+
/** Predicate probability at or above which a claim counts as PASS. */
|
|
76
|
+
export const VERIFY_PASS_THRESHOLD = 0.5
|
|
77
|
+
|
|
78
|
+
const CANDIDATE_ID_MAX_LENGTH = 64
|
|
79
|
+
const CLAIM_NAME_MAX_LENGTH = 64
|
|
80
|
+
const DESCRIPTION_MAX_LENGTH = 500
|
|
81
|
+
|
|
82
|
+
/** Candidate ids become choice values: strict bounded tokens only. */
|
|
83
|
+
const TOKEN_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/
|
|
84
|
+
|
|
85
|
+
const RANK_FIT_LABELS: readonly string[] = Object.freeze([
|
|
86
|
+
'unusable',
|
|
87
|
+
'poor',
|
|
88
|
+
'weak',
|
|
89
|
+
'fair',
|
|
90
|
+
'adequate',
|
|
91
|
+
'good',
|
|
92
|
+
'strong',
|
|
93
|
+
'excellent',
|
|
94
|
+
'exemplary',
|
|
95
|
+
])
|
|
96
|
+
|
|
97
|
+
const WINNER_INSTRUCTIONS =
|
|
98
|
+
'Choose the single best candidate from the listed options for the described state. ' +
|
|
99
|
+
'Answer with exactly one candidate id. Advisory only; nothing is dispatched or executed.'
|
|
100
|
+
|
|
101
|
+
const FIT_INSTRUCTIONS =
|
|
102
|
+
'Rate how well the candidate you chose as the best fits the described state. ' +
|
|
103
|
+
'Higher means a better fit; judge from the state and the candidate descriptions alone. ' +
|
|
104
|
+
'Advisory only; nothing is enforced.'
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* The verify request's only input part: a fixed template with NO free-text
|
|
108
|
+
* slot — the claims ride the predicate questions' instructions, so the state
|
|
109
|
+
* part cannot carry them twice (or be mistaken for a content field).
|
|
110
|
+
*/
|
|
111
|
+
const VERIFY_STATE_TEMPLATE =
|
|
112
|
+
'Advisory claim verification. Each claim is stated in its own question below; ' +
|
|
113
|
+
'assess each claim on its own terms from the claim text alone. Nothing is enforced.'
|
|
114
|
+
|
|
115
|
+
// ─── Hostility scan (mirrors the J4 criteria scan) ───────────────────────
|
|
116
|
+
|
|
117
|
+
/** Path separators in any position: caller prose is never a location. */
|
|
118
|
+
const PATH_PATTERN = /[/\\]/
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Control characters (C0 and DEL — newlines and tabs included): these are
|
|
122
|
+
* single-line prose fields. An explicit code-point scan, not a regex, so the
|
|
123
|
+
* hostile-input check stays lint-clean without a rule suppression.
|
|
124
|
+
*/
|
|
125
|
+
function hasControlCharacter(value: string): boolean {
|
|
126
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
127
|
+
const code = value.charCodeAt(index)
|
|
128
|
+
if (code <= 0x1f || code === 0x7f) return true
|
|
129
|
+
}
|
|
130
|
+
return false
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Key material shapes: PEM headers, AWS/SDK/Git token literals, key files. */
|
|
134
|
+
const KEY_MATERIAL_PATTERNS: readonly RegExp[] = Object.freeze([
|
|
135
|
+
/-----BEGIN/,
|
|
136
|
+
/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/,
|
|
137
|
+
/\bsk-[A-Za-z0-9_-]{16,}\b/,
|
|
138
|
+
/\bgh[pousr]_[A-Za-z0-9]{20,}\b/,
|
|
139
|
+
/\b[a-z0-9._-]+\.(?:key|pem|p12|pfx|kdbx)\b/i,
|
|
140
|
+
])
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Credential ASSIGNMENT shapes only: naming a concept is prose ("handles
|
|
144
|
+
* secrets without echoing them" stays admissible); binding a value to it
|
|
145
|
+
* ("password=hunter2") is a credential shape and is rejected.
|
|
146
|
+
*/
|
|
147
|
+
const CREDENTIAL_ASSIGNMENT_PATTERN =
|
|
148
|
+
/\b(?:password|passwd|secret|token|api[-_]?key|credential)s?\s*[:=]\s*\S+/i
|
|
149
|
+
|
|
150
|
+
function isHostileProse(value: string): boolean {
|
|
151
|
+
if (PATH_PATTERN.test(value)) return true
|
|
152
|
+
if (hasControlCharacter(value)) return true
|
|
153
|
+
if (KEY_MATERIAL_PATTERNS.some((pattern) => pattern.test(value))) return true
|
|
154
|
+
return CREDENTIAL_ASSIGNMENT_PATTERN.test(value)
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function isBoundedNonBlankString(value: unknown, max: number): value is string {
|
|
158
|
+
return typeof value === 'string' && value.trim().length > 0 && value.length <= max
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
162
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// ─── Rank: request building ──────────────────────────────────────────────
|
|
166
|
+
|
|
167
|
+
const RANK_ENVELOPE_KEYS: ReadonlySet<string> = new Set([
|
|
168
|
+
'schemaVersion',
|
|
169
|
+
'mode',
|
|
170
|
+
'state',
|
|
171
|
+
'candidates',
|
|
172
|
+
])
|
|
173
|
+
|
|
174
|
+
export type JevRankBuildFailure = 'RANK_INPUT_INVALID' | 'CANDIDATE_INVALID' | 'STATE_INVALID'
|
|
175
|
+
|
|
176
|
+
export type JevRankBuild =
|
|
177
|
+
| { readonly ok: true; readonly request: Omit<NativeDecisionRequest, 'model'> }
|
|
178
|
+
| { readonly ok: false; readonly reasonCode: JevRankBuildFailure }
|
|
179
|
+
|
|
180
|
+
export type JevRankPreparation =
|
|
181
|
+
| { readonly ok: true; readonly request: NativeDecisionRequest }
|
|
182
|
+
| { readonly ok: false; readonly reasonCode: JevRankBuildFailure }
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Validate the rank tool input and build the model-less request. The state is
|
|
186
|
+
* carried verbatim under the same bound and rule as the presets; the
|
|
187
|
+
* candidates ride the winner question's choice values (ids, strict tokens)
|
|
188
|
+
* and choice descriptions (bounded prose, hostility-scanned). Unknown
|
|
189
|
+
* envelope keys — `questions`, review fields, anything else — refuse the
|
|
190
|
+
* preparation: no second payload slot exists.
|
|
191
|
+
*/
|
|
192
|
+
export function buildJevRankRequest(input: unknown): JevRankBuild {
|
|
193
|
+
if (!isRecord(input)) return { ok: false, reasonCode: 'RANK_INPUT_INVALID' }
|
|
194
|
+
for (const key of Object.keys(input)) {
|
|
195
|
+
if (!RANK_ENVELOPE_KEYS.has(key)) return { ok: false, reasonCode: 'RANK_INPUT_INVALID' }
|
|
196
|
+
}
|
|
197
|
+
const state = input.state
|
|
198
|
+
if (!isBoundedNonBlankString(state, NATIVE_DECISION_MAX_PROMPT_LENGTH)) {
|
|
199
|
+
return { ok: false, reasonCode: 'STATE_INVALID' }
|
|
200
|
+
}
|
|
201
|
+
const rawCandidates = input.candidates
|
|
202
|
+
// A MISSING candidates field is a malformed rank; a PRESENT-but-hostile
|
|
203
|
+
// candidates array keeps its precise CANDIDATE_INVALID (the review-mode
|
|
204
|
+
// discipline).
|
|
205
|
+
if (rawCandidates === undefined) return { ok: false, reasonCode: 'RANK_INPUT_INVALID' }
|
|
206
|
+
if (!Array.isArray(rawCandidates)) return { ok: false, reasonCode: 'CANDIDATE_INVALID' }
|
|
207
|
+
if (rawCandidates.length < RANK_MIN_CANDIDATES || rawCandidates.length > RANK_MAX_CANDIDATES) {
|
|
208
|
+
return { ok: false, reasonCode: 'CANDIDATE_INVALID' }
|
|
209
|
+
}
|
|
210
|
+
const seen = new Set<string>()
|
|
211
|
+
const choices: Array<{ value: string; description?: string }> = []
|
|
212
|
+
for (const raw of rawCandidates) {
|
|
213
|
+
if (!isRecord(raw)) return { ok: false, reasonCode: 'CANDIDATE_INVALID' }
|
|
214
|
+
const extraKeys = Object.keys(raw).filter((key) => key !== 'id' && key !== 'description')
|
|
215
|
+
if (extraKeys.length > 0) return { ok: false, reasonCode: 'CANDIDATE_INVALID' }
|
|
216
|
+
const id = raw.id
|
|
217
|
+
if (
|
|
218
|
+
typeof id !== 'string' ||
|
|
219
|
+
id.length > CANDIDATE_ID_MAX_LENGTH ||
|
|
220
|
+
!TOKEN_PATTERN.test(id) ||
|
|
221
|
+
seen.has(id)
|
|
222
|
+
) {
|
|
223
|
+
return { ok: false, reasonCode: 'CANDIDATE_INVALID' }
|
|
224
|
+
}
|
|
225
|
+
seen.add(id)
|
|
226
|
+
const description = raw.description
|
|
227
|
+
if (description === undefined) {
|
|
228
|
+
choices.push({ value: id })
|
|
229
|
+
} else if (
|
|
230
|
+
isBoundedNonBlankString(description, DESCRIPTION_MAX_LENGTH) &&
|
|
231
|
+
!isHostileProse(description)
|
|
232
|
+
) {
|
|
233
|
+
choices.push({ value: id, description })
|
|
234
|
+
} else {
|
|
235
|
+
return { ok: false, reasonCode: 'CANDIDATE_INVALID' }
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
const request = {
|
|
239
|
+
input: Object.freeze([Object.freeze({ type: 'text' as const, text: state })]),
|
|
240
|
+
questions: Object.freeze([
|
|
241
|
+
Object.freeze({
|
|
242
|
+
type: 'choice' as const,
|
|
243
|
+
name: RANK_WINNER_QUESTION_NAME,
|
|
244
|
+
instructions: WINNER_INSTRUCTIONS,
|
|
245
|
+
choices: Object.freeze(choices.map((choice) => Object.freeze(choice))),
|
|
246
|
+
}),
|
|
247
|
+
Object.freeze({
|
|
248
|
+
type: 'score' as const,
|
|
249
|
+
name: RANK_FIT_QUESTION_NAME,
|
|
250
|
+
instructions: FIT_INSTRUCTIONS,
|
|
251
|
+
levels: Object.freeze(RANK_FIT_LABELS.map((label) => Object.freeze({ label }))),
|
|
252
|
+
}),
|
|
253
|
+
]),
|
|
254
|
+
} satisfies Omit<NativeDecisionRequest, 'model'>
|
|
255
|
+
return { ok: true, request: Object.freeze(request) }
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** `prepareJevPresetRequest` contract: route plus tool input in, request out. */
|
|
259
|
+
export function prepareJevRankRequest(
|
|
260
|
+
route: ResolvedNativeDecisionRoute,
|
|
261
|
+
input: unknown,
|
|
262
|
+
): JevRankPreparation {
|
|
263
|
+
const built = buildJevRankRequest(input)
|
|
264
|
+
if (!built.ok) return built
|
|
265
|
+
return {
|
|
266
|
+
ok: true,
|
|
267
|
+
request: {
|
|
268
|
+
model: route.model,
|
|
269
|
+
input: built.request.input,
|
|
270
|
+
questions: built.request.questions,
|
|
271
|
+
},
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// ─── Verify: request building ────────────────────────────────────────────
|
|
276
|
+
|
|
277
|
+
const VERIFY_ENVELOPE_KEYS: ReadonlySet<string> = new Set(['schemaVersion', 'mode', 'claims'])
|
|
278
|
+
|
|
279
|
+
export type JevVerifyBuildFailure = 'VERIFY_INPUT_INVALID' | 'CLAIM_INVALID'
|
|
280
|
+
|
|
281
|
+
export type JevVerifyBuild =
|
|
282
|
+
| { readonly ok: true; readonly request: Omit<NativeDecisionRequest, 'model'> }
|
|
283
|
+
| { readonly ok: false; readonly reasonCode: JevVerifyBuildFailure }
|
|
284
|
+
|
|
285
|
+
export type JevVerifyPreparation =
|
|
286
|
+
| { readonly ok: true; readonly request: NativeDecisionRequest }
|
|
287
|
+
| { readonly ok: false; readonly reasonCode: JevVerifyBuildFailure }
|
|
288
|
+
|
|
289
|
+
function verifyInstructions(claim: string): string {
|
|
290
|
+
return (
|
|
291
|
+
'Assess the following claim on its own terms and answer the probability that it holds. ' +
|
|
292
|
+
`Claim: "${claim}". Higher means the claim is more likely to hold. ` +
|
|
293
|
+
'This is advisory verification; nothing is enforced.'
|
|
294
|
+
)
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* Validate the verify tool input and build the model-less request: one
|
|
299
|
+
* predicate question per claim, named after the claim, the claim text riding
|
|
300
|
+
* that question's instructions. The verify mode takes NO state text — any
|
|
301
|
+
* smuggled slot (`state`, `candidates`, review fields) refuses the
|
|
302
|
+
* preparation `VERIFY_INPUT_INVALID`; hostile claim shapes keep the precise
|
|
303
|
+
* `CLAIM_INVALID`.
|
|
304
|
+
*/
|
|
305
|
+
export function buildJevVerifyRequest(input: unknown): JevVerifyBuild {
|
|
306
|
+
if (!isRecord(input)) return { ok: false, reasonCode: 'VERIFY_INPUT_INVALID' }
|
|
307
|
+
for (const key of Object.keys(input)) {
|
|
308
|
+
if (!VERIFY_ENVELOPE_KEYS.has(key)) return { ok: false, reasonCode: 'VERIFY_INPUT_INVALID' }
|
|
309
|
+
}
|
|
310
|
+
const rawClaims = input.claims
|
|
311
|
+
if (rawClaims === undefined) return { ok: false, reasonCode: 'VERIFY_INPUT_INVALID' }
|
|
312
|
+
if (!Array.isArray(rawClaims)) return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
313
|
+
if (rawClaims.length < VERIFY_MIN_CLAIMS || rawClaims.length > VERIFY_MAX_CLAIMS) {
|
|
314
|
+
return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
315
|
+
}
|
|
316
|
+
const seen = new Set<string>()
|
|
317
|
+
const questions: Array<{
|
|
318
|
+
type: 'predicate'
|
|
319
|
+
name: string
|
|
320
|
+
instructions: string
|
|
321
|
+
}> = []
|
|
322
|
+
for (const raw of rawClaims) {
|
|
323
|
+
if (!isRecord(raw)) return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
324
|
+
const extraKeys = Object.keys(raw).filter((key) => key !== 'name' && key !== 'claim')
|
|
325
|
+
if (extraKeys.length > 0) return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
326
|
+
const name = raw.name
|
|
327
|
+
if (
|
|
328
|
+
typeof name !== 'string' ||
|
|
329
|
+
name.length > CLAIM_NAME_MAX_LENGTH ||
|
|
330
|
+
!TOKEN_PATTERN.test(name) ||
|
|
331
|
+
seen.has(name)
|
|
332
|
+
) {
|
|
333
|
+
return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
334
|
+
}
|
|
335
|
+
const claim = raw.claim
|
|
336
|
+
if (!isBoundedNonBlankString(claim, CLAIM_MAX_LENGTH) || isHostileProse(claim)) {
|
|
337
|
+
return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
338
|
+
}
|
|
339
|
+
seen.add(name)
|
|
340
|
+
questions.push({ type: 'predicate', name, instructions: verifyInstructions(claim) })
|
|
341
|
+
}
|
|
342
|
+
const request = {
|
|
343
|
+
input: Object.freeze([Object.freeze({ type: 'text' as const, text: VERIFY_STATE_TEMPLATE })]),
|
|
344
|
+
questions: Object.freeze(questions.map((question) => Object.freeze(question))),
|
|
345
|
+
} satisfies Omit<NativeDecisionRequest, 'model'>
|
|
346
|
+
return { ok: true, request: Object.freeze(request) }
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** `prepareJevPresetRequest` contract: route plus tool input in, request out. */
|
|
350
|
+
export function prepareJevVerifyRequest(
|
|
351
|
+
route: ResolvedNativeDecisionRoute,
|
|
352
|
+
input: unknown,
|
|
353
|
+
): JevVerifyPreparation {
|
|
354
|
+
const built = buildJevVerifyRequest(input)
|
|
355
|
+
if (!built.ok) return built
|
|
356
|
+
return {
|
|
357
|
+
ok: true,
|
|
358
|
+
request: {
|
|
359
|
+
model: route.model,
|
|
360
|
+
input: built.request.input,
|
|
361
|
+
questions: built.request.questions,
|
|
362
|
+
},
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
// ─── Consumption ─────────────────────────────────────────────────────────
|
|
367
|
+
|
|
368
|
+
export interface JevRankVerdict {
|
|
369
|
+
/**
|
|
370
|
+
* The host's chosen candidate id; `null` when the answer is missing,
|
|
371
|
+
* mistyped, duplicated, an error, or outside the prepared candidate ids.
|
|
372
|
+
*/
|
|
373
|
+
readonly winner: string | null
|
|
374
|
+
/**
|
|
375
|
+
* The host's fit score for the winner, on the prepared 2..10 rubric;
|
|
376
|
+
* `null` under the same failures.
|
|
377
|
+
*/
|
|
378
|
+
readonly winnerFit: number | null
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Consume a host Decisions response against the prepared rank request.
|
|
383
|
+
* Strict validation is borrowed wholesale from the tool's consumer
|
|
384
|
+
* (`consumeJevDecisionResult`): an answer counts only when its name and type
|
|
385
|
+
* match the prepared questions, duplicates become errors, and anything else
|
|
386
|
+
* is `null`, never a guess.
|
|
387
|
+
*/
|
|
388
|
+
export function consumeJevRankResult(
|
|
389
|
+
response: unknown,
|
|
390
|
+
request: NativeDecisionRequest,
|
|
391
|
+
): JevRankVerdict {
|
|
392
|
+
const advice = consumeJevDecisionResult(response, request)
|
|
393
|
+
let winner: string | null = null
|
|
394
|
+
let winnerFit: number | null = null
|
|
395
|
+
for (const answer of advice.answers) {
|
|
396
|
+
if (answer.type === 'error') continue
|
|
397
|
+
if (answer.type === 'choice' && answer.name === RANK_WINNER_QUESTION_NAME) {
|
|
398
|
+
winner = answer.choice ?? null
|
|
399
|
+
} else if (answer.type === 'score' && answer.name === RANK_FIT_QUESTION_NAME) {
|
|
400
|
+
winnerFit = answer.score ?? null
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
return Object.freeze({ winner, winnerFit })
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/**
|
|
407
|
+
* The verdict of one verified claim. `BLOCK` is the fail-closed answer: a
|
|
408
|
+
* host refusal, timeout, duplicate or malformed reply must never read as
|
|
409
|
+
* verified (or as refuted — an unverifiable claim is unverifiable, not
|
|
410
|
+
* false).
|
|
411
|
+
*/
|
|
412
|
+
export type VerifyClaimVerdict = 'PASS' | 'FAIL' | 'BLOCK'
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* Consume a host Decisions response against the prepared verify request:
|
|
416
|
+
* one verdict per claim name. Probability ≥ {@link VERIFY_PASS_THRESHOLD} is
|
|
417
|
+
* `PASS`, below is `FAIL`; error answers, missing answers and any unusable
|
|
418
|
+
* response shape are `BLOCK` — fail-closed.
|
|
419
|
+
*/
|
|
420
|
+
export function consumeJevVerifyResult(
|
|
421
|
+
response: unknown,
|
|
422
|
+
request: NativeDecisionRequest,
|
|
423
|
+
): Readonly<Record<string, VerifyClaimVerdict>> {
|
|
424
|
+
const advice = consumeJevDecisionResult(response, request)
|
|
425
|
+
const verdicts: Record<string, VerifyClaimVerdict> = {}
|
|
426
|
+
for (const question of request.questions) {
|
|
427
|
+
const answer = advice.answers.find((candidate) => candidate.name === question.name)
|
|
428
|
+
if (answer !== undefined && answer.type === 'predicate') {
|
|
429
|
+
verdicts[question.name] = (answer.probability ?? 0) >= VERIFY_PASS_THRESHOLD ? 'PASS' : 'FAIL'
|
|
430
|
+
} else {
|
|
431
|
+
verdicts[question.name] = 'BLOCK'
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
return Object.freeze(verdicts)
|
|
435
|
+
}
|
|
@@ -330,7 +330,7 @@ async function readChangelogForBundle(bundleUrl) {
|
|
|
330
330
|
}
|
|
331
331
|
/** The ambient build-time version, or null outside a tsdown build. */
|
|
332
332
|
function embeddedVersion() {
|
|
333
|
-
return "1.6.3-beta.
|
|
333
|
+
return "1.6.3-beta.9";
|
|
334
334
|
}
|
|
335
335
|
/**
|
|
336
336
|
* Resolve the running Pantheon version for the what's-new window: the plugin
|
|
@@ -3649,7 +3649,7 @@ function Sidebar(props) {
|
|
|
3649
3649
|
setProp(_el$12, "width", "100%");
|
|
3650
3650
|
setProp(_el$12, "padding", 1);
|
|
3651
3651
|
setProp(_el$13, "attributes", 1);
|
|
3652
|
-
insert(_el$13, `Pantheon v1.6.3-beta.
|
|
3652
|
+
insert(_el$13, `Pantheon v1.6.3-beta.9`);
|
|
3653
3653
|
insert(_el$14, () => presetHeaderLine(model.preset().name ?? "", model.provider()));
|
|
3654
3654
|
insert(_el$12, createComponent(Show, {
|
|
3655
3655
|
get when() {
|
|
@@ -4370,7 +4370,7 @@ function HomeStatus(props) {
|
|
|
4370
4370
|
const muted = () => props.context.theme?.text?.muted ?? props.context.theme?.text?.base;
|
|
4371
4371
|
return (() => {
|
|
4372
4372
|
var _el$72 = createElement("text");
|
|
4373
|
-
insert(_el$72, () => `Pantheon v1.6.3-beta.
|
|
4373
|
+
insert(_el$72, () => `Pantheon v1.6.3-beta.9${preset().name ? ` ▸ ${preset().name}` : ""}`);
|
|
4374
4374
|
effect((_$p) => setProp(_el$72, "fg", muted(), _$p));
|
|
4375
4375
|
return _el$72;
|
|
4376
4376
|
})();
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pantheon-tui",
|
|
3
|
-
"version": "1.6.3-beta.
|
|
3
|
+
"version": "1.6.3-beta.9",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "pantheon-tui",
|
|
9
|
-
"version": "1.6.3-beta.
|
|
9
|
+
"version": "1.6.3-beta.9",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@opencode-ai/plugin": "1.18.33",
|