pantheon-opencode 1.6.3-beta.2 → 1.6.3-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.global.md +1 -1
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +143 -0
- package/README.md +21 -3
- package/README.pt-BR.md +1 -1
- package/bin/pantheon-init.mjs +46 -28
- package/package.json +1 -1
- package/plugin.json +2 -2
- package/pyproject.toml +1 -1
- package/scripts/decision-router-budget.mjs +103 -3
- package/scripts/decision-router-v2-budget.mjs +2 -2
- package/scripts/decision-router.mjs +88 -6
- package/scripts/install/cli-ui.mjs +2 -1
- package/scripts/install/decision-router-config.mjs +53 -0
- package/scripts/install/interactive.mjs +103 -9
- package/scripts/install/opencode.mjs +101 -30
- package/scripts/install/plugin.mjs +17 -0
- package/scripts/jev-live-probe.mjs +212 -0
- package/scripts/skill-search.mjs +273 -0
- package/scripts/sync-tui.mjs +11 -0
- package/skills-lock.json +1 -0
- package/src/agents/aphrodite.md +34 -1
- package/src/agents/apollo.md +227 -0
- package/src/agents/athena.md +26 -0
- package/src/agents/demeter.md +33 -0
- package/src/agents/gaia.md +24 -0
- package/src/agents/hephaestus.md +24 -0
- package/src/agents/hermes.md +33 -0
- package/src/agents/iris.md +24 -0
- package/src/agents/mnemosyne.md +25 -3
- package/src/agents/nyx.md +50 -22
- package/src/agents/prometheus.md +242 -1
- package/src/agents/talos.md +24 -0
- package/src/agents/themis.md +24 -0
- package/src/agents/zeus.md +67 -6
- package/src/pantheon/j-telemetry-file.ts +96 -0
- package/src/pantheon/j1-approvals.ts +123 -0
- package/src/pantheon/jev-decision-tool.ts +268 -34
- package/src/pantheon/jev-decisions.ts +92 -8
- package/src/pantheon/jev-escalation.ts +1 -1
- package/src/pantheon/jev-quality-score.ts +7 -2
- package/src/pantheon/jev-rank-verify.ts +661 -0
- package/src/pantheon/jev-risk-gate.ts +75 -8
- package/src/pantheon/local-decision-budget.ts +170 -3
- package/src/pantheon/local-decision-router.ts +16 -0
- package/src/pantheon/local-decision-turn.ts +39 -4
- package/src/pantheon/local-decision-usage-evidence.internal.ts +2 -2
- package/src/pantheon/pantheon-decision-catalog.ts +9 -2
- package/src/pantheon/v2-decision-budget.ts +4 -2
- package/src/plugin-v2.ts +94 -7
- package/src/plugins/tui/dist/server-v2.d.ts +14 -6
- package/src/plugins/tui/dist/server-v2.d.ts.map +1 -1
- package/src/plugins/tui/dist/server-v2.js +16 -9
- package/src/plugins/tui/dist/server-v2.js.map +1 -1
- package/src/plugins/tui/dist/tui-v2.d.ts.map +1 -1
- package/src/plugins/tui/dist/tui-v2.js +560 -4
- package/src/plugins/tui/dist/tui-v2.js.map +1 -1
- package/src/plugins/tui/dist/tui.js +1 -1
- package/src/plugins/tui/package-lock.json +2 -2
- package/src/plugins/tui/package.json +1 -1
- package/src/plugins/tui/src/index-v2.tsx +6 -0
- package/src/plugins/tui/src/server-v2.ts +14 -7
- package/src/plugins/tui/src/whats-new-highlights.ts +317 -0
- package/src/plugins/tui/src/whats-new-open.ts +120 -0
- package/src/plugins/tui/src/whats-new-state.ts +128 -0
- package/src/plugins/tui/src/whats-new.tsx +172 -0
- package/src/routing.yml +1 -0
- package/src/skills/README.md +1 -1
- package/src/skills/prompt-refinery/SKILL.md +41 -0
|
@@ -0,0 +1,661 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The v3.1 rank and verify primitives on the native Jev route (tool modes on
|
|
3
|
+
* `pantheon_jev_decide`, built after the J4 review-mode pattern).
|
|
4
|
+
*
|
|
5
|
+
* ## Rank — `mode: "rank"`, one choice + one score, `{ winner, winnerFit }`
|
|
6
|
+
*
|
|
7
|
+
* Zeus can rank 2–8 named candidates for a state in ONE typed round-trip: one
|
|
8
|
+
* `choice` question (`winner`) over the candidate ids picks the best candidate
|
|
9
|
+
* and one `score` question (`fit`, the 2..10 rubric bound) rates that
|
|
10
|
+
* candidate's fit. THE HONEST LIMITATION: the host Decisions API returns one
|
|
11
|
+
* scalar answer per question (`NativeDecisionRequest` is tied to the 2.2.5
|
|
12
|
+
* host types), so per-candidate scores are not expressible in a single
|
|
13
|
+
* round-trip — one score question per candidate would break the 4-question
|
|
14
|
+
* bound above four candidates, and one ambiguous "score each candidate"
|
|
15
|
+
* question has no per-candidate answer structure to pair back. Rank therefore
|
|
16
|
+
* returns `{ winner, winnerFit }` and NOTHING more: no runner-up, no full
|
|
17
|
+
* ordering. A caller that needs a deeper order uses the explicit-questions
|
|
18
|
+
* mode with its own typed questions.
|
|
19
|
+
*
|
|
20
|
+
* ## Verify — `mode: "verify"`, one predicate per claim, fail-closed BLOCK
|
|
21
|
+
*
|
|
22
|
+
* Zeus can check 1–4 named claims in ONE typed round-trip: one `predicate`
|
|
23
|
+
* question per claim (within the 4-question bound), the claim text riding that
|
|
24
|
+
* question's instructions. The consumer maps each answer: probability
|
|
25
|
+
* > {@link VERIFY_PASS_THRESHOLD} (0.65) → `PASS`; 0.35–0.65 → `UNVERIFIABLE`; below →
|
|
26
|
+
* `FAIL`; and every refusal,
|
|
27
|
+
* timeout, duplicate or malformed answer → `BLOCK` (fail-closed: an
|
|
28
|
+
* unanswered mandatory claim must never read as verified). The tool itself
|
|
29
|
+
* never acts on a verdict — `effect` stays `none` — and a gate-chain refusal
|
|
30
|
+
* stays the standard bounded refusal JSON; a caller that REQUIRES a verdict
|
|
31
|
+
* treats every refusal as `BLOCK` at its own call site.
|
|
32
|
+
*
|
|
33
|
+
* ## Sanitization contract (the J4 criteria class)
|
|
34
|
+
*
|
|
35
|
+
* Both modes take bounded caller-provided fields only: rank's candidate ids
|
|
36
|
+
* are strict tokens (they become choice values), its state rides the same
|
|
37
|
+
* 12,000-char bound and verbatim-carriage rule as the presets, and verify's
|
|
38
|
+
* claim texts are the only free text, riding the predicate instructions. The
|
|
39
|
+
* hostility scan mirrors the J4 criteria scan (paths, credential assignments,
|
|
40
|
+
* key material, control characters, over-length): hostile shapes are REJECTED
|
|
41
|
+
* — `CANDIDATE_INVALID` / `CLAIM_INVALID` — never sanitized on the way in.
|
|
42
|
+
* Neither mode has a telemetry point (they are tool modes, not shadow gates)
|
|
43
|
+
* and both reuse the tool's strict answer consumer
|
|
44
|
+
* (`consumeJevDecisionResult`): one answer per question name, duplicates
|
|
45
|
+
* become errors, unbounded host metadata never crosses.
|
|
46
|
+
*
|
|
47
|
+
* @module jev-rank-verify
|
|
48
|
+
*/
|
|
49
|
+
|
|
50
|
+
import { consumeJevDecisionResult, JEV_DECISION_MAX_QUESTIONS } from './jev-decisions.ts'
|
|
51
|
+
import type { NativeDecisionRequest, ResolvedNativeDecisionRoute } from './v2-native-decisions.ts'
|
|
52
|
+
import { NATIVE_DECISION_MAX_PROMPT_LENGTH } from './v2-native-decisions.ts'
|
|
53
|
+
|
|
54
|
+
// ─── Bounds ──────────────────────────────────────────────────────────────
|
|
55
|
+
|
|
56
|
+
/** A ranking needs at least two candidates to choose between. */
|
|
57
|
+
export const RANK_MIN_CANDIDATES = 2
|
|
58
|
+
/** Mirrors the typesafe_rank reference contract (1-20 accepted upstream). */
|
|
59
|
+
export const RANK_MAX_CANDIDATES = 20
|
|
60
|
+
|
|
61
|
+
/** Question name of the rank's choice question. */
|
|
62
|
+
export const RANK_WINNER_QUESTION_NAME = 'winner'
|
|
63
|
+
/** Question name of the rank's fit score question. */
|
|
64
|
+
export const RANK_FIT_QUESTION_NAME = 'fit'
|
|
65
|
+
|
|
66
|
+
/** The fit rubric mirrors the documented 2..10 score-question bound. */
|
|
67
|
+
export const RANK_FIT_LEVEL_COUNT = 9
|
|
68
|
+
|
|
69
|
+
/** At least one claim per verify call; one predicate question per claim. */
|
|
70
|
+
export const VERIFY_MIN_CLAIMS = 1
|
|
71
|
+
/** One predicate question per claim, within the 4-question bound. */
|
|
72
|
+
export const VERIFY_MAX_CLAIMS = 4
|
|
73
|
+
/** Each claim is a bounded 1–200-char instruction string. */
|
|
74
|
+
export const CLAIM_MAX_LENGTH = 200
|
|
75
|
+
|
|
76
|
+
/** Predicate probability at or above which a claim counts as PASS. */
|
|
77
|
+
/**
|
|
78
|
+
* PASS lives strictly ABOVE the UNVERIFIABLE band (QA beta.20: with PASS at
|
|
79
|
+
* 0.5 the documented 0.35-0.65 band was unreachable at its top half — 0.50
|
|
80
|
+
* and 0.65 returned PASS). Verdicts: < 0.35 FAIL, 0.35-0.65 UNVERIFIABLE,
|
|
81
|
+
* > 0.65 PASS.
|
|
82
|
+
*/
|
|
83
|
+
export const VERIFY_PASS_THRESHOLD = 0.65
|
|
84
|
+
|
|
85
|
+
const CANDIDATE_ID_MAX_LENGTH = 64
|
|
86
|
+
const CLAIM_NAME_MAX_LENGTH = 64
|
|
87
|
+
const DESCRIPTION_MAX_LENGTH = 500
|
|
88
|
+
|
|
89
|
+
/** Candidate ids become choice values: strict bounded tokens only. */
|
|
90
|
+
const TOKEN_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/
|
|
91
|
+
|
|
92
|
+
const RANK_FIT_LABELS: readonly string[] = Object.freeze([
|
|
93
|
+
'unusable',
|
|
94
|
+
'poor',
|
|
95
|
+
'weak',
|
|
96
|
+
'fair',
|
|
97
|
+
'adequate',
|
|
98
|
+
'good',
|
|
99
|
+
'strong',
|
|
100
|
+
'excellent',
|
|
101
|
+
'exemplary',
|
|
102
|
+
])
|
|
103
|
+
|
|
104
|
+
const WINNER_INSTRUCTIONS =
|
|
105
|
+
'Choose the single best candidate from the listed options for the described state. ' +
|
|
106
|
+
'Answer with exactly one candidate id. Advisory only; nothing is dispatched or executed.'
|
|
107
|
+
|
|
108
|
+
const FIT_INSTRUCTIONS =
|
|
109
|
+
'Rate how well the candidate you chose as the best fits the described state. ' +
|
|
110
|
+
'Higher means a better fit; judge from the state and the candidate descriptions alone. ' +
|
|
111
|
+
'Advisory only; nothing is enforced.'
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* The verify request's only input part: a fixed template with NO free-text
|
|
115
|
+
* slot — the claims ride the predicate questions' instructions, so the state
|
|
116
|
+
* part cannot carry them twice (or be mistaken for a content field).
|
|
117
|
+
*/
|
|
118
|
+
const VERIFY_STATE_TEMPLATE =
|
|
119
|
+
'Advisory claim verification. Each claim is stated in its own question below; ' +
|
|
120
|
+
'assess each claim on its own terms from the claim text alone. Nothing is enforced.'
|
|
121
|
+
|
|
122
|
+
// ─── Hostility scan (mirrors the J4 criteria scan) ───────────────────────
|
|
123
|
+
|
|
124
|
+
/** Path separators in any position: caller prose is never a location. */
|
|
125
|
+
const PATH_PATTERN = /[/\\]/
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Control characters (C0 and DEL — newlines and tabs included): these are
|
|
129
|
+
* single-line prose fields. An explicit code-point scan, not a regex, so the
|
|
130
|
+
* hostile-input check stays lint-clean without a rule suppression.
|
|
131
|
+
*/
|
|
132
|
+
function hasControlCharacter(value: string): boolean {
|
|
133
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
134
|
+
const code = value.charCodeAt(index)
|
|
135
|
+
if (code <= 0x1f || code === 0x7f) return true
|
|
136
|
+
}
|
|
137
|
+
return false
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Key material shapes: PEM headers, AWS/SDK/Git token literals, key files. */
|
|
141
|
+
const KEY_MATERIAL_PATTERNS: readonly RegExp[] = Object.freeze([
|
|
142
|
+
/-----BEGIN/,
|
|
143
|
+
/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/,
|
|
144
|
+
/\bsk-[A-Za-z0-9_-]{16,}\b/,
|
|
145
|
+
/\bgh[pousr]_[A-Za-z0-9]{20,}\b/,
|
|
146
|
+
/\b[a-z0-9._-]+\.(?:key|pem|p12|pfx|kdbx)\b/i,
|
|
147
|
+
])
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Credential ASSIGNMENT shapes only: naming a concept is prose ("handles
|
|
151
|
+
* secrets without echoing them" stays admissible); binding a value to it
|
|
152
|
+
* ("password=hunter2") is a credential shape and is rejected.
|
|
153
|
+
*/
|
|
154
|
+
const CREDENTIAL_ASSIGNMENT_PATTERN =
|
|
155
|
+
/\b(?:password|passwd|secret|token|api[-_]?key|credential)s?\s*[:=]\s*\S+/i
|
|
156
|
+
|
|
157
|
+
function isHostileProse(value: string): boolean {
|
|
158
|
+
if (PATH_PATTERN.test(value)) return true
|
|
159
|
+
if (hasControlCharacter(value)) return true
|
|
160
|
+
if (KEY_MATERIAL_PATTERNS.some((pattern) => pattern.test(value))) return true
|
|
161
|
+
return CREDENTIAL_ASSIGNMENT_PATTERN.test(value)
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function isBoundedNonBlankString(value: unknown, max: number): value is string {
|
|
165
|
+
return typeof value === 'string' && value.trim().length > 0 && value.length <= max
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
169
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// ─── Rank: request building ──────────────────────────────────────────────
|
|
173
|
+
|
|
174
|
+
export const RANK_ENVELOPE_KEYS: ReadonlySet<string> = new Set([
|
|
175
|
+
'schemaVersion',
|
|
176
|
+
'mode',
|
|
177
|
+
'state',
|
|
178
|
+
'candidates',
|
|
179
|
+
])
|
|
180
|
+
|
|
181
|
+
export type JevRankBuildFailure = 'RANK_INPUT_INVALID' | 'CANDIDATE_INVALID' | 'STATE_INVALID'
|
|
182
|
+
|
|
183
|
+
/** Per-cause token attached to a rank build refusal (e.g. the accepted count range). */
|
|
184
|
+
export type JevRankBuildDetail = 'RANK_CANDIDATE_COUNT_RANGE'
|
|
185
|
+
|
|
186
|
+
export type JevRankBuild =
|
|
187
|
+
| { readonly ok: true; readonly request: Omit<NativeDecisionRequest, 'model'> }
|
|
188
|
+
| {
|
|
189
|
+
readonly ok: false
|
|
190
|
+
readonly reasonCode: JevRankBuildFailure
|
|
191
|
+
readonly detail?: JevRankBuildDetail
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
export type JevRankPreparation =
|
|
195
|
+
| { readonly ok: true; readonly request: NativeDecisionRequest }
|
|
196
|
+
| {
|
|
197
|
+
readonly ok: false
|
|
198
|
+
readonly reasonCode: JevRankBuildFailure
|
|
199
|
+
readonly detail?: JevRankBuildDetail
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Validate the rank tool input and build the model-less request. The state is
|
|
204
|
+
* carried verbatim under the same bound and rule as the presets; the
|
|
205
|
+
* candidates ride the winner question's choice values (ids, strict tokens)
|
|
206
|
+
* and choice descriptions (bounded prose, hostility-scanned). Unknown
|
|
207
|
+
* envelope keys — `questions`, review fields, anything else — refuse the
|
|
208
|
+
* preparation: no second payload slot exists.
|
|
209
|
+
*/
|
|
210
|
+
export function buildJevRankRequest(input: unknown): JevRankBuild {
|
|
211
|
+
if (!isRecord(input)) return { ok: false, reasonCode: 'RANK_INPUT_INVALID' }
|
|
212
|
+
for (const key of Object.keys(input)) {
|
|
213
|
+
if (!RANK_ENVELOPE_KEYS.has(key)) return { ok: false, reasonCode: 'RANK_INPUT_INVALID' }
|
|
214
|
+
}
|
|
215
|
+
const state = input.state
|
|
216
|
+
if (!isBoundedNonBlankString(state, NATIVE_DECISION_MAX_PROMPT_LENGTH)) {
|
|
217
|
+
return { ok: false, reasonCode: 'STATE_INVALID' }
|
|
218
|
+
}
|
|
219
|
+
const rawCandidates = input.candidates
|
|
220
|
+
// A MISSING candidates field is a malformed rank; a PRESENT-but-hostile
|
|
221
|
+
// candidates array keeps its precise CANDIDATE_INVALID (the review-mode
|
|
222
|
+
// discipline).
|
|
223
|
+
if (rawCandidates === undefined) return { ok: false, reasonCode: 'RANK_INPUT_INVALID' }
|
|
224
|
+
if (!Array.isArray(rawCandidates)) return { ok: false, reasonCode: 'CANDIDATE_INVALID' }
|
|
225
|
+
if (rawCandidates.length < RANK_MIN_CANDIDATES || rawCandidates.length > RANK_MAX_CANDIDATES) {
|
|
226
|
+
return { ok: false, reasonCode: 'CANDIDATE_INVALID', detail: 'RANK_CANDIDATE_COUNT_RANGE' }
|
|
227
|
+
}
|
|
228
|
+
const seen = new Set<string>()
|
|
229
|
+
const choices: Array<{ value: string; description?: string }> = []
|
|
230
|
+
for (const raw of rawCandidates) {
|
|
231
|
+
if (!isRecord(raw)) return { ok: false, reasonCode: 'CANDIDATE_INVALID' }
|
|
232
|
+
const extraKeys = Object.keys(raw).filter((key) => key !== 'id' && key !== 'description')
|
|
233
|
+
if (extraKeys.length > 0) return { ok: false, reasonCode: 'CANDIDATE_INVALID' }
|
|
234
|
+
const id = raw.id
|
|
235
|
+
if (
|
|
236
|
+
typeof id !== 'string' ||
|
|
237
|
+
id.length > CANDIDATE_ID_MAX_LENGTH ||
|
|
238
|
+
!TOKEN_PATTERN.test(id) ||
|
|
239
|
+
seen.has(id)
|
|
240
|
+
) {
|
|
241
|
+
return { ok: false, reasonCode: 'CANDIDATE_INVALID' }
|
|
242
|
+
}
|
|
243
|
+
seen.add(id)
|
|
244
|
+
const description = raw.description
|
|
245
|
+
if (description === undefined) {
|
|
246
|
+
choices.push({ value: id })
|
|
247
|
+
} else if (
|
|
248
|
+
isBoundedNonBlankString(description, DESCRIPTION_MAX_LENGTH) &&
|
|
249
|
+
!isHostileProse(description)
|
|
250
|
+
) {
|
|
251
|
+
choices.push({ value: id, description })
|
|
252
|
+
} else {
|
|
253
|
+
return { ok: false, reasonCode: 'CANDIDATE_INVALID' }
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
const request = {
|
|
257
|
+
input: Object.freeze([Object.freeze({ type: 'text' as const, text: state })]),
|
|
258
|
+
questions: Object.freeze([
|
|
259
|
+
Object.freeze({
|
|
260
|
+
type: 'choice' as const,
|
|
261
|
+
name: RANK_WINNER_QUESTION_NAME,
|
|
262
|
+
instructions: WINNER_INSTRUCTIONS,
|
|
263
|
+
choices: Object.freeze(choices.map((choice) => Object.freeze(choice))),
|
|
264
|
+
}),
|
|
265
|
+
Object.freeze({
|
|
266
|
+
type: 'score' as const,
|
|
267
|
+
name: RANK_FIT_QUESTION_NAME,
|
|
268
|
+
instructions: FIT_INSTRUCTIONS,
|
|
269
|
+
levels: Object.freeze(RANK_FIT_LABELS.map((label) => Object.freeze({ label }))),
|
|
270
|
+
}),
|
|
271
|
+
]),
|
|
272
|
+
} satisfies Omit<NativeDecisionRequest, 'model'>
|
|
273
|
+
return { ok: true, request: Object.freeze(request) }
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** `prepareJevPresetRequest` contract: route plus tool input in, request out. */
|
|
277
|
+
export function prepareJevRankRequest(
|
|
278
|
+
route: ResolvedNativeDecisionRoute,
|
|
279
|
+
input: unknown,
|
|
280
|
+
): JevRankPreparation {
|
|
281
|
+
const built = buildJevRankRequest(input)
|
|
282
|
+
if (!built.ok) return built
|
|
283
|
+
return {
|
|
284
|
+
ok: true,
|
|
285
|
+
request: {
|
|
286
|
+
model: route.model,
|
|
287
|
+
input: built.request.input,
|
|
288
|
+
questions: built.request.questions,
|
|
289
|
+
},
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// ─── Order: top-K ordering (2-4 candidates, one fit question each) ───────
|
|
294
|
+
|
|
295
|
+
export const ORDER_ENVELOPE_KEYS: ReadonlySet<string> = new Set([
|
|
296
|
+
'schemaVersion',
|
|
297
|
+
'mode',
|
|
298
|
+
'state',
|
|
299
|
+
'candidates',
|
|
300
|
+
])
|
|
301
|
+
|
|
302
|
+
export type JevOrderBuildFailure =
|
|
303
|
+
| 'ORDER_INPUT_INVALID'
|
|
304
|
+
| 'ORDER_CANDIDATE_INVALID'
|
|
305
|
+
| 'STATE_INVALID'
|
|
306
|
+
|
|
307
|
+
export type JevOrderBuildDetail = 'ORDER_CANDIDATE_COUNT_RANGE'
|
|
308
|
+
|
|
309
|
+
export type JevOrderBuild =
|
|
310
|
+
| {
|
|
311
|
+
readonly ok: true
|
|
312
|
+
readonly request: Omit<NativeDecisionRequest, 'model'>
|
|
313
|
+
/** The caller's candidate ids in question order (QA beta.20: the consumer must map scores back to THESE, not positional c1..cN). */
|
|
314
|
+
readonly candidates: readonly string[]
|
|
315
|
+
}
|
|
316
|
+
| {
|
|
317
|
+
readonly ok: false
|
|
318
|
+
readonly reasonCode: JevOrderBuildFailure
|
|
319
|
+
readonly detail?: JevOrderBuildDetail
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
export type JevOrderPreparation =
|
|
323
|
+
| {
|
|
324
|
+
readonly ok: true
|
|
325
|
+
readonly request: NativeDecisionRequest
|
|
326
|
+
readonly candidates: readonly string[]
|
|
327
|
+
}
|
|
328
|
+
| {
|
|
329
|
+
readonly ok: false
|
|
330
|
+
readonly reasonCode: JevOrderBuildFailure
|
|
331
|
+
readonly detail?: JevOrderBuildDetail
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
const ORDER_MAX_CANDIDATES = JEV_DECISION_MAX_QUESTIONS // one score question per candidate
|
|
335
|
+
|
|
336
|
+
const ORDER_STATE_TEMPLATE =
|
|
337
|
+
'Advisory ordering. The caller supplied a task and 2-4 candidates; ' +
|
|
338
|
+
'each question scores ONE candidate for fit against the stated task. ' +
|
|
339
|
+
'Score each candidate on its own terms. Nothing is enforced.'
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Build the order request: one SCORE question per candidate (the host answers
|
|
343
|
+
* one scalar per question, so 2-4 candidates is the honest ceiling). Ids ride
|
|
344
|
+
* the candidate validation reused from rank; question names are positional
|
|
345
|
+
* (`fit_1`..) so no id pattern can collide with the question-name grammar.
|
|
346
|
+
*/
|
|
347
|
+
export function buildJevOrderRequest(input: unknown): JevOrderBuild {
|
|
348
|
+
if (!isRecord(input)) return { ok: false, reasonCode: 'ORDER_INPUT_INVALID' }
|
|
349
|
+
for (const key of Object.keys(input)) {
|
|
350
|
+
if (!ORDER_ENVELOPE_KEYS.has(key)) return { ok: false, reasonCode: 'ORDER_INPUT_INVALID' }
|
|
351
|
+
}
|
|
352
|
+
const state = input.state
|
|
353
|
+
if (!isBoundedNonBlankString(state, NATIVE_DECISION_MAX_PROMPT_LENGTH) || isHostileProse(state)) {
|
|
354
|
+
return { ok: false, reasonCode: 'STATE_INVALID' }
|
|
355
|
+
}
|
|
356
|
+
const rawCandidates = input.candidates
|
|
357
|
+
if (rawCandidates === undefined) return { ok: false, reasonCode: 'ORDER_INPUT_INVALID' }
|
|
358
|
+
if (!Array.isArray(rawCandidates)) return { ok: false, reasonCode: 'ORDER_CANDIDATE_INVALID' }
|
|
359
|
+
if (rawCandidates.length < 2 || rawCandidates.length > ORDER_MAX_CANDIDATES) {
|
|
360
|
+
return {
|
|
361
|
+
ok: false,
|
|
362
|
+
reasonCode: 'ORDER_CANDIDATE_INVALID',
|
|
363
|
+
detail: 'ORDER_CANDIDATE_COUNT_RANGE',
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
const seen = new Set<string>()
|
|
367
|
+
const candidates: Array<{ id: string; description?: string }> = []
|
|
368
|
+
for (const raw of rawCandidates) {
|
|
369
|
+
if (!isRecord(raw)) return { ok: false, reasonCode: 'ORDER_CANDIDATE_INVALID' }
|
|
370
|
+
const extraKeys = Object.keys(raw).filter((key) => key !== 'id' && key !== 'description')
|
|
371
|
+
if (extraKeys.length > 0) return { ok: false, reasonCode: 'ORDER_CANDIDATE_INVALID' }
|
|
372
|
+
const id = raw.id
|
|
373
|
+
if (typeof id !== 'string' || id.length < 1 || id.length > 64) {
|
|
374
|
+
return { ok: false, reasonCode: 'ORDER_CANDIDATE_INVALID' }
|
|
375
|
+
}
|
|
376
|
+
if (seen.has(id)) return { ok: false, reasonCode: 'ORDER_CANDIDATE_INVALID' }
|
|
377
|
+
seen.add(id)
|
|
378
|
+
const description = raw.description
|
|
379
|
+
if (description !== undefined) {
|
|
380
|
+
if (!isBoundedNonBlankString(description, 500) || isHostileProse(description)) {
|
|
381
|
+
return { ok: false, reasonCode: 'ORDER_CANDIDATE_INVALID' }
|
|
382
|
+
}
|
|
383
|
+
candidates.push({ id, description })
|
|
384
|
+
} else {
|
|
385
|
+
candidates.push({ id })
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
const levels = Array.from({ length: 9 }, (_, index) => ({
|
|
389
|
+
label: String(index + 2),
|
|
390
|
+
}))
|
|
391
|
+
const questions = candidates.map((candidate, index) => ({
|
|
392
|
+
type: 'score' as const,
|
|
393
|
+
name: `fit_${index + 1}`,
|
|
394
|
+
instructions:
|
|
395
|
+
`Score how well candidate "${candidate.id}"` +
|
|
396
|
+
(candidate.description ? ` (${candidate.description})` : '') +
|
|
397
|
+
` fits the task. 2 = poor fit, 10 = exact fit.`,
|
|
398
|
+
levels,
|
|
399
|
+
}))
|
|
400
|
+
const request = {
|
|
401
|
+
input: Object.freeze([
|
|
402
|
+
Object.freeze({ type: 'text' as const, text: `${ORDER_STATE_TEMPLATE} Task: ${state}` }),
|
|
403
|
+
]),
|
|
404
|
+
questions: Object.freeze(questions.map((question) => Object.freeze(question))),
|
|
405
|
+
} satisfies Omit<NativeDecisionRequest, 'model'>
|
|
406
|
+
return {
|
|
407
|
+
ok: true,
|
|
408
|
+
request: Object.freeze(request),
|
|
409
|
+
candidates: Object.freeze(candidates.map((c) => c.id)),
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
export function prepareJevOrderRequest(
|
|
414
|
+
route: ResolvedNativeDecisionRoute,
|
|
415
|
+
input: unknown,
|
|
416
|
+
): JevOrderPreparation {
|
|
417
|
+
const built = buildJevOrderRequest(input)
|
|
418
|
+
if (!built.ok) return built
|
|
419
|
+
return {
|
|
420
|
+
ok: true,
|
|
421
|
+
request: {
|
|
422
|
+
model: route.model,
|
|
423
|
+
input: built.request.input,
|
|
424
|
+
questions: built.request.questions,
|
|
425
|
+
},
|
|
426
|
+
candidates: built.candidates,
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/** One ordered entry: the candidate id and its fit score (null when the host answered nothing usable). */
|
|
431
|
+
export interface JevOrderEntry {
|
|
432
|
+
readonly id: string
|
|
433
|
+
readonly score: number | null
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
/**
|
|
437
|
+
* Consume the order result: map each positional score answer back to its
|
|
438
|
+
* candidate, sort by score descending (nulls last, stable), and return the
|
|
439
|
+
* ordering. A missing/invalid score stays visible as null — never guessed.
|
|
440
|
+
*/
|
|
441
|
+
export function consumeJevOrderResult(
|
|
442
|
+
response: unknown,
|
|
443
|
+
request: NativeDecisionRequest,
|
|
444
|
+
candidates: readonly string[],
|
|
445
|
+
): ReadonlyArray<JevOrderEntry> {
|
|
446
|
+
const advice = consumeJevDecisionResult(response, request)
|
|
447
|
+
const entries: Array<{ id: string; score: number | null; order: number }> = []
|
|
448
|
+
request.questions.forEach((question, index) => {
|
|
449
|
+
const answer = advice.answers.find((candidate) => candidate.name === question.name)
|
|
450
|
+
const score =
|
|
451
|
+
answer !== undefined && answer.type === 'score' && typeof answer.score === 'number'
|
|
452
|
+
? answer.score
|
|
453
|
+
: null
|
|
454
|
+
// QA beta.20: map back to the CALLER's candidate id — positional c1..cN
|
|
455
|
+
// told the user nothing.
|
|
456
|
+
entries.push({ id: candidates[index] ?? `c${index + 1}`, score, order: index })
|
|
457
|
+
})
|
|
458
|
+
return Object.freeze(
|
|
459
|
+
entries
|
|
460
|
+
.sort((left, right) => {
|
|
461
|
+
if (left.score === null && right.score === null) return left.order - right.order
|
|
462
|
+
if (left.score === null) return 1
|
|
463
|
+
if (right.score === null) return -1
|
|
464
|
+
return right.score - left.score || left.order - right.order
|
|
465
|
+
})
|
|
466
|
+
.map(({ id, score }) => ({ id, score })),
|
|
467
|
+
)
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// ─── Verify: request building ────────────────────────────────────────────
|
|
471
|
+
|
|
472
|
+
export const VERIFY_ENVELOPE_KEYS: ReadonlySet<string> = new Set([
|
|
473
|
+
'schemaVersion',
|
|
474
|
+
'mode',
|
|
475
|
+
'claims',
|
|
476
|
+
// Optional evidence: the environment/context the claims are judged against
|
|
477
|
+
// (field report beta.13 §6 — without it, environment-dependent claims are
|
|
478
|
+
// unanswerable and the honest verdict is UNVERIFIABLE, not FAIL).
|
|
479
|
+
'state',
|
|
480
|
+
])
|
|
481
|
+
|
|
482
|
+
/** The verdict band treated as "the model could not tell" (neither PASS nor FAIL). */
|
|
483
|
+
export const VERIFY_UNVERIFIABLE_LOW = 0.35
|
|
484
|
+
export const VERIFY_UNVERIFIABLE_HIGH = 0.65
|
|
485
|
+
|
|
486
|
+
export type JevVerifyBuildFailure = 'VERIFY_INPUT_INVALID' | 'CLAIM_INVALID'
|
|
487
|
+
|
|
488
|
+
export type JevVerifyBuild =
|
|
489
|
+
| { readonly ok: true; readonly request: Omit<NativeDecisionRequest, 'model'> }
|
|
490
|
+
| { readonly ok: false; readonly reasonCode: JevVerifyBuildFailure }
|
|
491
|
+
|
|
492
|
+
export type JevVerifyPreparation =
|
|
493
|
+
| { readonly ok: true; readonly request: NativeDecisionRequest }
|
|
494
|
+
| { readonly ok: false; readonly reasonCode: JevVerifyBuildFailure }
|
|
495
|
+
|
|
496
|
+
function verifyInstructions(claim: string): string {
|
|
497
|
+
return (
|
|
498
|
+
'Assess the following claim on its own terms and answer the probability that it holds. ' +
|
|
499
|
+
`Claim: "${claim}". Higher means the claim is more likely to hold. ` +
|
|
500
|
+
'This is advisory verification; nothing is enforced.'
|
|
501
|
+
)
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
/**
|
|
505
|
+
* Validate the verify tool input and build the model-less request: one
|
|
506
|
+
* predicate question per claim, named after the claim, the claim text riding
|
|
507
|
+
* that question's instructions. The verify mode takes NO state text — any
|
|
508
|
+
* smuggled slot (`state`, `candidates`, review fields) refuses the
|
|
509
|
+
* preparation `VERIFY_INPUT_INVALID`; hostile claim shapes keep the precise
|
|
510
|
+
* `CLAIM_INVALID`.
|
|
511
|
+
*/
|
|
512
|
+
export function buildJevVerifyRequest(input: unknown): JevVerifyBuild {
|
|
513
|
+
if (!isRecord(input)) return { ok: false, reasonCode: 'VERIFY_INPUT_INVALID' }
|
|
514
|
+
for (const key of Object.keys(input)) {
|
|
515
|
+
if (!VERIFY_ENVELOPE_KEYS.has(key)) return { ok: false, reasonCode: 'VERIFY_INPUT_INVALID' }
|
|
516
|
+
}
|
|
517
|
+
const rawClaims = input.claims
|
|
518
|
+
if (rawClaims === undefined) return { ok: false, reasonCode: 'VERIFY_INPUT_INVALID' }
|
|
519
|
+
if (!Array.isArray(rawClaims)) return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
520
|
+
if (rawClaims.length < VERIFY_MIN_CLAIMS || rawClaims.length > VERIFY_MAX_CLAIMS) {
|
|
521
|
+
return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
522
|
+
}
|
|
523
|
+
const seen = new Set<string>()
|
|
524
|
+
const questions: Array<{
|
|
525
|
+
type: 'predicate'
|
|
526
|
+
name: string
|
|
527
|
+
instructions: string
|
|
528
|
+
}> = []
|
|
529
|
+
for (const raw of rawClaims) {
|
|
530
|
+
if (!isRecord(raw)) return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
531
|
+
const extraKeys = Object.keys(raw).filter((key) => key !== 'name' && key !== 'claim')
|
|
532
|
+
if (extraKeys.length > 0) return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
533
|
+
const name = raw.name
|
|
534
|
+
if (
|
|
535
|
+
typeof name !== 'string' ||
|
|
536
|
+
name.length > CLAIM_NAME_MAX_LENGTH ||
|
|
537
|
+
!TOKEN_PATTERN.test(name) ||
|
|
538
|
+
seen.has(name)
|
|
539
|
+
) {
|
|
540
|
+
return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
541
|
+
}
|
|
542
|
+
const claim = raw.claim
|
|
543
|
+
if (!isBoundedNonBlankString(claim, CLAIM_MAX_LENGTH) || isHostileProse(claim)) {
|
|
544
|
+
return { ok: false, reasonCode: 'CLAIM_INVALID' }
|
|
545
|
+
}
|
|
546
|
+
seen.add(name)
|
|
547
|
+
questions.push({ type: 'predicate', name, instructions: verifyInstructions(claim) })
|
|
548
|
+
}
|
|
549
|
+
// Optional evidence: the caller's state rides as the input text the claims
|
|
550
|
+
// are judged against; without it the template keeps claims self-contained.
|
|
551
|
+
const rawState = (input as Record<string, unknown>).state
|
|
552
|
+
let evidence = VERIFY_STATE_TEMPLATE
|
|
553
|
+
if (rawState !== undefined) {
|
|
554
|
+
if (
|
|
555
|
+
!isBoundedNonBlankString(rawState, NATIVE_DECISION_MAX_PROMPT_LENGTH) ||
|
|
556
|
+
isHostileProse(rawState)
|
|
557
|
+
) {
|
|
558
|
+
return { ok: false, reasonCode: 'VERIFY_INPUT_INVALID' }
|
|
559
|
+
}
|
|
560
|
+
evidence = `${VERIFY_STATE_TEMPLATE} Evidence/context supplied by the caller, to be trusted as given: ${rawState}`
|
|
561
|
+
}
|
|
562
|
+
const request = {
|
|
563
|
+
input: Object.freeze([Object.freeze({ type: 'text' as const, text: evidence })]),
|
|
564
|
+
questions: Object.freeze(questions.map((question) => Object.freeze(question))),
|
|
565
|
+
} satisfies Omit<NativeDecisionRequest, 'model'>
|
|
566
|
+
return { ok: true, request: Object.freeze(request) }
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/** `prepareJevPresetRequest` contract: route plus tool input in, request out. */
|
|
570
|
+
export function prepareJevVerifyRequest(
|
|
571
|
+
route: ResolvedNativeDecisionRoute,
|
|
572
|
+
input: unknown,
|
|
573
|
+
): JevVerifyPreparation {
|
|
574
|
+
const built = buildJevVerifyRequest(input)
|
|
575
|
+
if (!built.ok) return built
|
|
576
|
+
return {
|
|
577
|
+
ok: true,
|
|
578
|
+
request: {
|
|
579
|
+
model: route.model,
|
|
580
|
+
input: built.request.input,
|
|
581
|
+
questions: built.request.questions,
|
|
582
|
+
},
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
// ─── Consumption ─────────────────────────────────────────────────────────
|
|
587
|
+
|
|
588
|
+
export interface JevRankVerdict {
|
|
589
|
+
/**
|
|
590
|
+
* The host's chosen candidate id; `null` when the answer is missing,
|
|
591
|
+
* mistyped, duplicated, an error, or outside the prepared candidate ids.
|
|
592
|
+
*/
|
|
593
|
+
readonly winner: string | null
|
|
594
|
+
/**
|
|
595
|
+
* The host's fit score for the winner, on the prepared 2..10 rubric;
|
|
596
|
+
* `null` under the same failures.
|
|
597
|
+
*/
|
|
598
|
+
readonly winnerFit: number | null
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
/**
|
|
602
|
+
* Consume a host Decisions response against the prepared rank request.
|
|
603
|
+
* Strict validation is borrowed wholesale from the tool's consumer
|
|
604
|
+
* (`consumeJevDecisionResult`): an answer counts only when its name and type
|
|
605
|
+
* match the prepared questions, duplicates become errors, and anything else
|
|
606
|
+
* is `null`, never a guess.
|
|
607
|
+
*/
|
|
608
|
+
export function consumeJevRankResult(
|
|
609
|
+
response: unknown,
|
|
610
|
+
request: NativeDecisionRequest,
|
|
611
|
+
): JevRankVerdict {
|
|
612
|
+
const advice = consumeJevDecisionResult(response, request)
|
|
613
|
+
let winner: string | null = null
|
|
614
|
+
let winnerFit: number | null = null
|
|
615
|
+
for (const answer of advice.answers) {
|
|
616
|
+
if (answer.type === 'error') continue
|
|
617
|
+
if (answer.type === 'choice' && answer.name === RANK_WINNER_QUESTION_NAME) {
|
|
618
|
+
winner = answer.choice ?? null
|
|
619
|
+
} else if (answer.type === 'score' && answer.name === RANK_FIT_QUESTION_NAME) {
|
|
620
|
+
winnerFit = answer.score ?? null
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
return Object.freeze({ winner, winnerFit })
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
/**
|
|
627
|
+
* The verdict of one verified claim. `BLOCK` is the fail-closed answer: a
|
|
628
|
+
* host refusal, timeout, duplicate or malformed reply must never read as
|
|
629
|
+
* verified (or as refuted — an unverifiable claim is unverifiable, not
|
|
630
|
+
* false).
|
|
631
|
+
*/
|
|
632
|
+
export type VerifyClaimVerdict = 'PASS' | 'FAIL' | 'UNVERIFIABLE' | 'BLOCK'
|
|
633
|
+
|
|
634
|
+
/**
|
|
635
|
+
* Consume a host Decisions response against the prepared verify request:
|
|
636
|
+
* one verdict per claim name. Probability > {@link VERIFY_PASS_THRESHOLD} is
|
|
637
|
+
* `PASS`, below is `FAIL`; error answers, missing answers and any unusable
|
|
638
|
+
* response shape are `BLOCK` — fail-closed.
|
|
639
|
+
*/
|
|
640
|
+
export function consumeJevVerifyResult(
|
|
641
|
+
response: unknown,
|
|
642
|
+
request: NativeDecisionRequest,
|
|
643
|
+
): Readonly<Record<string, VerifyClaimVerdict>> {
|
|
644
|
+
const advice = consumeJevDecisionResult(response, request)
|
|
645
|
+
const verdicts: Record<string, VerifyClaimVerdict> = {}
|
|
646
|
+
for (const question of request.questions) {
|
|
647
|
+
const answer = advice.answers.find((candidate) => candidate.name === question.name)
|
|
648
|
+
if (answer !== undefined && answer.type === 'predicate') {
|
|
649
|
+
const probability = answer.probability ?? 0
|
|
650
|
+
verdicts[question.name] =
|
|
651
|
+
probability > VERIFY_PASS_THRESHOLD
|
|
652
|
+
? 'PASS'
|
|
653
|
+
: probability >= VERIFY_UNVERIFIABLE_LOW
|
|
654
|
+
? 'UNVERIFIABLE'
|
|
655
|
+
: 'FAIL'
|
|
656
|
+
} else {
|
|
657
|
+
verdicts[question.name] = 'BLOCK'
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
return Object.freeze(verdicts)
|
|
661
|
+
}
|