@tangle-network/agent-eval 0.205.1 → 0.206.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/CHANGELOG.md +9 -0
  2. package/dist/analyst/index.d.ts +1 -1
  3. package/dist/analyst/index.js +1 -1
  4. package/dist/{benchmark-command-DuDTN5ID.js → benchmark-command-DtP94cOB.js} +2 -2
  5. package/dist/{benchmark-command-DuDTN5ID.js.map → benchmark-command-DtP94cOB.js.map} +1 -1
  6. package/dist/{claude-jsonl-DoQJBCVR.js → claude-jsonl-CG3JUbTJ.js} +8 -4
  7. package/dist/claude-jsonl-CG3JUbTJ.js.map +1 -0
  8. package/dist/cli.js +1 -1
  9. package/dist/{index-BoVEyQwH.d.ts → index-CbBfavbT.d.ts} +5 -3
  10. package/dist/{index-BoVEyQwH.d.ts.map → index-CbBfavbT.d.ts.map} +1 -1
  11. package/dist/index.d.ts +1 -1
  12. package/dist/index.js +1 -1
  13. package/dist/meta-eval/index.d.ts +6 -0
  14. package/dist/meta-eval/index.d.ts.map +1 -1
  15. package/dist/meta-eval/index.js +15 -1
  16. package/dist/meta-eval/index.js.map +1 -1
  17. package/dist/openapi.json +1 -1
  18. package/dist/rl.d.ts +12 -12
  19. package/dist/rl.d.ts.map +1 -1
  20. package/dist/rl.js +17 -17
  21. package/dist/rl.js.map +1 -1
  22. package/dist/rollout/index.d.ts +2 -2
  23. package/dist/rollout/index.js +3 -3
  24. package/dist/{rollout-BdCoWV9n.js → rollout-l9w3Gdvh.js} +2 -2
  25. package/dist/{rollout-BdCoWV9n.js.map → rollout-l9w3Gdvh.js.map} +1 -1
  26. package/dist/supervisor-run/index.js +1 -1
  27. package/dist/systemone-protocol-DKoi1NdK.d.ts +132 -0
  28. package/dist/systemone-protocol-DKoi1NdK.d.ts.map +1 -0
  29. package/dist/systemone-protocol.d.ts +2 -0
  30. package/dist/systemone-protocol.js +214 -0
  31. package/dist/systemone-protocol.js.map +1 -0
  32. package/dist/{jev-review-Cw06AKuF.d.ts → systemone-review-DGTKkLMG.d.ts} +16 -16
  33. package/dist/systemone-review-DGTKkLMG.d.ts.map +1 -0
  34. package/dist/{jev-review-DZEjDzlF.js → systemone-review-DTClwgmE.js} +11 -11
  35. package/dist/systemone-review-DTClwgmE.js.map +1 -0
  36. package/dist/systemone.d.ts +60 -0
  37. package/dist/systemone.d.ts.map +1 -0
  38. package/dist/{jev.js → systemone.js} +16 -16
  39. package/dist/systemone.js.map +1 -0
  40. package/docs/changes/jev-evidence-review.md +1 -1
  41. package/docs/changes/systemone-provider-neutral.md +36 -0
  42. package/docs/public-api.md +93 -68
  43. package/docs/rollout.md +9 -1
  44. package/docs/{jev-behavior-review.md → systemone-behavior-review.md} +17 -17
  45. package/docs/{jev-integrations.md → systemone-integrations.md} +4 -4
  46. package/docs/{jev-replay.md → systemone-replay.md} +6 -6
  47. package/docs/{jev.md → systemone.md} +84 -33
  48. package/package.json +9 -9
  49. package/dist/claude-jsonl-DoQJBCVR.js.map +0 -1
  50. package/dist/jev-protocol-BQLxwbqD.d.ts +0 -99
  51. package/dist/jev-protocol-BQLxwbqD.d.ts.map +0 -1
  52. package/dist/jev-protocol.d.ts +0 -2
  53. package/dist/jev-protocol.js +0 -114
  54. package/dist/jev-protocol.js.map +0 -1
  55. package/dist/jev-review-Cw06AKuF.d.ts.map +0 -1
  56. package/dist/jev-review-DZEjDzlF.js.map +0 -1
  57. package/dist/jev.d.ts +0 -60
  58. package/dist/jev.d.ts.map +0 -1
  59. package/dist/jev.js.map +0 -1
@@ -4,17 +4,17 @@ Use native evaluations to assess bounded hypotheses about recorded behavior. Do
4
4
  classifier whether an agent is "malicious" or a whole run is "safe". Observable policy breaches,
5
5
  model-supported hypotheses, missing evidence and unusual behavior are different results.
6
6
 
7
- The optional `/jev` helpers are ordinary functions:
7
+ The optional `/systemone` helpers are ordinary functions:
8
8
 
9
- - `prepareJevReview`: validate and freeze caller-defined native questions, evidence references,
9
+ - `prepareSystemOneReview`: validate and freeze caller-defined native questions, evidence references,
10
10
  coverage and decision thresholds before spending. The digest binds the definition, input and
11
11
  authored question/option order. Optional undefined object fields are absent on the wire.
12
- - `assessJevReview`: validate native answers and interpret designated alternative masses as
12
+ - `assessSystemOneReview`: validate native answers and interpret designated alternative masses as
13
13
  `supported`, `refuted` or `unresolved`. Re-scoring makes no network call.
14
- - `jevReviewFindings`: map that report into the existing `AnalystFinding` contract. It keeps
14
+ - `systemOneReviewFindings`: map that report into the existing `AnalystFinding` contract. It keeps
15
15
  unresolved checks visible and labels supported claims as model-supported hypotheses.
16
16
 
17
- These helpers do not constrain `jevEvaluator`, `jevJudge`, `jevAnalyst`, or the provider-independent
17
+ These helpers do not constrain `systemOneEvaluator`, `systemOneJudge`, `systemOneAnalyst`, or the provider-independent
18
18
  `createEvaluator`/`asJudge`/`asAnalyst` APIs. Use an ordinary custom mapper for different policies.
19
19
  No model, taxonomy, severity, operating threshold, global safety score or storage backend is chosen
20
20
  by the generic implementation.
@@ -26,16 +26,16 @@ Supply the native request on each invocation. This example's `question`, `eviden
26
26
 
27
27
  ```ts
28
28
  import {
29
- prepareJevReview, assessJevReview, jevEvaluator,
30
- } from '@tangle-network/agent-eval/jev'
29
+ prepareSystemOneReview, assessSystemOneReview, systemOneEvaluator,
30
+ } from '@tangle-network/agent-eval/systemone'
31
31
 
32
- const evaluate = jevEvaluator({
32
+ const evaluate = systemOneEvaluator({
33
33
  evaluate: nativeTransport, // One Router /v1/systemone attempt; preserve signal/idempotency key.
34
34
  receipt: readRouterReceipt,
35
35
  maximumCharge: enforcedMaximumCharge,
36
36
  })
37
37
 
38
- const review = prepareJevReview({
38
+ const review = prepareSystemOneReview({
39
39
  version: policy.version,
40
40
  request: {
41
41
  model: selectedModel,
@@ -59,7 +59,7 @@ const review = prepareJevReview({
59
59
  })
60
60
  const result = await evaluate(review.request, executionContext)
61
61
  await persistAuthorizedObservation({ review, result })
62
- const report = assessJevReview(review, result.value)
62
+ const report = assessSystemOneReview(review, result.value)
63
63
  ```
64
64
 
65
65
  The example assumes a native Choice question with `supported`, `refuted` and `insufficient`
@@ -71,8 +71,8 @@ not a requirement imposed on other evaluators.
71
71
 
72
72
  ## Optional behavior recipe
73
73
 
74
- `examples/jev-behavior-review.ts` provides replaceable question definitions and
75
- `behaviorReviewAnalyst`, which composes the existing `jevAnalyst`. Copy/adapt the recipe in an
74
+ `examples/systemone-behavior-review.ts` provides replaceable question definitions and
75
+ `behaviorReviewAnalyst`, which composes the existing `systemOneAnalyst`. Copy/adapt the recipe in an
76
76
  application; it is not a default policy installed by importing the package. Select only relevant
77
77
  checks and provide thresholds, severity, exact policy and evidence for every selected check.
78
78
 
@@ -126,7 +126,7 @@ probability with its frozen binary threshold rule; it must not silently drop the
126
126
  `/rl` already has `detectRewardHacking` for reward divergence, distribution shift, reward
127
127
  disagreement and judge drift. Those diagnostic scores are not probabilities. Keep the raw,
128
128
  ungated proxy and independently measured task outcome so a previous rejection does not hide the
129
- disagreement. Do not substitute the Jev monitor as the only independent outcome.
129
+ disagreement. Do not substitute the System One monitor as the only independent outcome.
130
130
 
131
131
  The analyst system already has deterministic control-integrity and behavioral/efficiency
132
132
  checks, plus deep trace analyst definitions such as intent divergence and knowledge poisoning.
@@ -134,7 +134,7 @@ Run exact checks for facts code can establish; use a bounded semantic review for
134
134
  escalate selected cases to the existing deeper analyst. None is replaced by this recipe.
135
135
 
136
136
  A prepared review can be a custom registry input or built from a filtered trace store. Existing
137
- `asAnalyst`/`jevAnalyst` forward the shared paid-call account, cancellation and deadline. The
137
+ `asAnalyst`/`systemOneAnalyst` forward the shared paid-call account, cancellation and deadline. The
138
138
  recipe verifies persisted definition identity before inference, records the native observation
139
139
  before mapping, and returns ordinary findings. Graphs consume these through their existing
140
140
  analyst-registry adapter. No new supervisor, scheduler, ledger, or trace database is introduced.
@@ -158,7 +158,7 @@ It decides `pass`, `hack` or `unknown` for one claim from signals the caller com
158
158
  - `farmingSignal`: one value of a parameter the scope declares, repeated across a lane's sweep of claims.
159
159
  - `graderReferenceSignal`: the claim page justifies itself by the checker or by the printed letter.
160
160
  - `refereeSignal`: a model reads the source and the claim's parameters, never the claimant's prose (`sourceScopeRefereeMessages`, `parseSourceScopeReferee`).
161
- - `jevSignal`: typed questions over the same brief (`claimIntegrityJevReview`). The caller supplies calibrated thresholds.
161
+ - `systemOneSignal`: typed questions over the same brief (`claimIntegritySystemOneReview`). The caller supplies calibrated thresholds.
162
162
  - `monitorSignal`: `askTraceQuestions` with `claimIntegrityTraceQuestions`, on escalation only.
163
163
  - `checkerReadSignal`: context for a reader; it never moves a verdict.
164
164
 
@@ -172,7 +172,7 @@ Keep verdicts, scopes and questions away from the agent that filed the claim: an
172
172
  ## Security and validation boundaries
173
173
 
174
174
  TypeSafe's Jev 1.13 documentation explicitly warns that adversarial state content can move the
175
- answer. Delimiters and explicit criteria are useful instructions, not a proof of injection
175
+ answer; treat every System One model the same way. Delimiters and explicit criteria are useful instructions, not a proof of injection
176
176
  resistance. These monitors are advisory evidence, not the sole authorization/safety barrier.
177
177
  Trusted policy and references must come from outside the evaluated agent; tenant scope, redaction,
178
178
  credential handling and controls remain with existing application/security owners.
@@ -192,7 +192,7 @@ attacks, not only the worker under a fixed monitor.
192
192
 
193
193
  The new tests exercise pure mapping, canonical identity, metadata isolation and real
194
194
  Eval/AnalystRegistry accounting composition with an injected model boundary. They do not prove
195
- that Jev detects these behaviors correctly or is robust against adversarial prompts. No live
195
+ that a System One model detects these behaviors correctly or is robust against adversarial prompts. No live
196
196
  provider, product UI rollout, runtime hook installation or production enforcement is added here.
197
197
 
198
198
  Sources reviewed: https://docs.typesafe.ai/confidence and
@@ -1,7 +1,7 @@
1
1
  # Ten useful integrations, one evaluator
2
2
 
3
- Jev evaluates bounded questions. Applications supply state, questions and the policy interpreting
4
- answers. jevEvaluator handles the native model; createEvaluator supports other classifiers.
3
+ A System One model evaluates bounded questions. Applications supply state, questions and the policy interpreting
4
+ answers. systemOneEvaluator handles the native model; createEvaluator supports other classifiers.
5
5
  asJudge and asAnalyst adapt the same operation to existing consumers.
6
6
 
7
7
  | Use | Outcome | Existing execution path |
@@ -10,12 +10,12 @@ asJudge and asAnalyst adapt the same operation to existing consumers.
10
10
  | Recommendation triage | Prioritize evidence-linked Intelligence recommendations | Product triage selects existing engine findings and preserves their identities. |
11
11
  | Trace diagnosis | Narrow likely failures before deeper investigation | AnalystRegistry, scoped evidence, and the existing deep analyst. |
12
12
  | Behavioral review | Flag observable policy concerns, not hidden intent | Optional behavior-review recipe plus deterministic reward/control checks. |
13
- | Rubric judging | Compare product or agent outputs | jevJudge/asJudge with caller mappings and independent final assessment. |
13
+ | Rubric judging | Compare product or agent outputs | systemOneJudge/asJudge with caller mappings and independent final assessment. |
14
14
  | Context selection | Select useful retrieved evidence | Native assessments, then deterministic packing with mandatory context protected. |
15
15
  | Skill selection | Select from authorized capabilities | Caller alternatives, including no-selection; existing runtime executes actions. |
16
16
  | Next verification | Choose a test, retrieval or review that resolves uncertainty | Existing graph checkpoints and awaited boundaries, not new permissions. |
17
17
  | Candidate selection | Choose among generated plans or artifacts | Existing matrix/graph, exact checks first, semantic assessment second. |
18
- | Configuration experiments | Compare questions, context, models and thresholds | examples/jev-decision-benchmark.ts using runAgentMatrix and cost receipts. |
18
+ | Configuration experiments | Compare questions, context, models and thresholds | examples/systemone-decision-benchmark.ts using runAgentMatrix and cost receipts. |
19
19
 
20
20
  These are compositions, not ten new frameworks. Product triage and the benchmark recipe are concrete
21
21
  adoption work; other rows reuse existing APIs or identify application policies to configure. An API
@@ -1,4 +1,4 @@
1
- # Inspect recorded Jev evidence without new inference
1
+ # Inspect recorded System One evidence without new inference
2
2
 
3
3
  The historical decoder from #897 is already the protocol owner. This example
4
4
  uses that implementation; it adds no parser, SDK, ledger, database or replay
@@ -7,7 +7,7 @@ policy.
7
7
  From a repository checkout with dependencies installed:
8
8
 
9
9
  ```sh
10
- pnpm exec tsx examples/jev-replay.ts request.json response.json
10
+ pnpm exec tsx examples/systemone-replay.ts request.json response.json
11
11
  ```
12
12
 
13
13
  Supply the original native request and response as separate JSON files. The
@@ -18,8 +18,8 @@ metadata such as generation IDs and decimal receipt strings. Neither input file
18
18
  is rewritten. This is offline inspection, not replay of a paid provider request.
19
19
 
20
20
  Historical null state and null score levels remain readable through
21
- `parseJevRecordedRequest`. They are still invalid for new paid calls through
22
- `parseJevRequest`. Do not cast a historical request to the live type, or change
21
+ `parseSystemOneRecordedRequest`. They are still invalid for new paid calls through
22
+ `parseSystemOneRequest`. Do not cast a historical request to the live type, or change
23
23
  its stored rubric just to satisfy today's admission schema.
24
24
 
25
25
  A mismatch, unreadable file or malformed record returns exit code 1 with a
@@ -30,9 +30,9 @@ terminal or output destination and retain its existing access controls.
30
30
  The same helper is importable from the example:
31
31
 
32
32
  ```ts
33
- import { inspectJevFiles } from './examples/jev-replay'
33
+ import { inspectSystemOneFiles } from './examples/systemone-replay'
34
34
 
35
- const observation = await inspectJevFiles('request.json', 'response.json')
35
+ const observation = await inspectSystemOneFiles('request.json', 'response.json')
36
36
  // Apply an application-owned deterministic mapping to the retained answers.
37
37
  ```
38
38
 
@@ -1,35 +1,50 @@
1
- # Typed evaluations: products, judges, and analysts
1
+ # System One evaluations: products, judges, and analysts
2
+
3
+ System One is a typed decision protocol: a request carries `state` and named questions (noul
4
+ yes/no, choice, or score rubric), and the answer is a probability for every allowed option.
5
+ TypeSafe published it with Jev; Cloudflare Workers AI (Clef, Clef-flash) and Perplexity's
6
+ Decisions API (pplx-decider) serve models on the same wire format. This package owns what a
7
+ valid result means. It names no endpoint, credential or model. Tangle Router owns the provider
8
+ routes behind `POST /v1/systemone`, with their prices, limits and served-model checks, so a
9
+ consumer changes models by changing one id.
2
10
 
3
11
  Configure a transport once. Supply your own model, state, and questions on every
4
12
  call. Interpret the answers in ordinary code; no campaign or agent is required.
5
13
 
6
14
  ```ts
7
- import { TypeSafeClient, noul, choice } from '@typesafe-ai/sdk'
8
- import { jevEvaluator } from '@tangle-network/agent-eval/jev'
9
-
10
- const client = new TypeSafeClient({
11
- baseURL: routerOrigin, // Origin without /v1; omit to use TypeSafe directly.
12
- apiKey: scopedRouterKey,
13
- retry: { maxRetries: 0 },
14
- })
15
- const evaluate = jevEvaluator({
16
- evaluate: (request, { signal, idempotencyKey }) => client.systemOne(request, {
17
- signal,
18
- retry: { maxRetries: 0 },
19
- headers: { 'Idempotency-Key': idempotencyKey },
20
- }),
15
+ import { systemOneEvaluator } from '@tangle-network/agent-eval/systemone'
16
+
17
+ const evaluate = systemOneEvaluator({
18
+ evaluate: async (request, { signal, idempotencyKey }) => {
19
+ const response = await fetch(`${routerOrigin}/v1/systemone`, {
20
+ method: 'POST',
21
+ signal,
22
+ headers: {
23
+ Authorization: `Bearer ${scopedRouterKey}`,
24
+ 'Content-Type': 'application/json',
25
+ 'Idempotency-Key': idempotencyKey,
26
+ },
27
+ body: JSON.stringify(request),
28
+ })
29
+ if (!response.ok) throw new Error(`System One request failed: ${response.status}`)
30
+ return response.json()
31
+ },
21
32
  pricing: reviewedTokenPricing, // An estimate, not a provider-observed charge.
22
33
  })
23
34
 
24
35
  const result = await evaluate({
25
- model: selectedModel,
36
+ model: selectedModel, // a native_evaluation id from Router's /v1/models
26
37
  state: { requirements, evidence },
27
38
  questions: {
28
- supported: noul('Does the evidence support the claim?'),
29
- next: choice({ task: 'Choose the next step using the evidence' }, {
30
- inspect: { when: 'Evidence is missing or contradictory' },
31
- finish: { when: 'The requirements are satisfied' },
32
- }),
39
+ supported: { type: 'noul', instructions: 'Does the evidence support the claim?' },
40
+ next: {
41
+ type: 'choice',
42
+ instructions: 'Choose the next step using the evidence',
43
+ criteria: {
44
+ inspect: 'Evidence is missing or contradictory',
45
+ finish: 'The requirements are satisfied',
46
+ },
47
+ },
33
48
  },
34
49
  }, { signal, costLedger })
35
50
 
@@ -38,21 +53,56 @@ result.value.answers.next.probabilities.inspect // number
38
53
  result.receipt // Existing CostLedger receipt, separate from native answers.
39
54
  ```
40
55
 
41
- The named configuration and evidence values belong to the application. Plain JSON
42
- questions work too; SDK builders are optional. Structured criteria, null instructions and choice or noul
56
+ The named configuration and evidence values belong to the application. TypeSafe's SDK
57
+ builders produce the same JSON and are optional. Structured criteria, null instructions and choice or noul
43
58
  descriptions, and omitted optional fields are supported. `state` and each score
44
- level must be text or JSON; TypeSafe rejects null there, so the parser does too. Static TypeScript questions preserve
59
+ level must be text or JSON; TypeSafe rejects null there, so the parser does too. Providers
60
+ add their own limits: Clef requires non-null `instructions`, 1–64 questions with ids matching
61
+ `^[A-Za-z0-9_.-]{1,100}$`, and 2–255 choice options. Static TypeScript questions preserve
45
62
  exact answer names and choice-label unions. Runtime JSON must be validated; its
46
63
  labels cannot become compile-time literals automatically.
47
64
 
48
- ## Any classifier, not just Jev
65
+ ## Providers, rounding and confidence
66
+
67
+ The wire format is the same for every provider; how each reports its numbers is not. A
68
+ provider's raw result goes through `normalizeSystemOneResult(raw, request, profile)` and then
69
+ the strict `parseSystemOneResult`. The route owner supplies the profile.
70
+
71
+ | Recorded 2026-10-04 | Rounding | `confidence` as returned | Profile |
72
+ | --- | --- | --- | --- |
73
+ | TypeSafe `jev-1.13.0` | 2 decimals. Probabilities sum to 1; the score is not always their expectation (8.79 against 8.86). | `systemOneConfidence` (153 of 153 recorded answers within rounding) | `{ decimals: 2, confidence: 'canonical' }` |
74
+ | Cloudflare `clef`, `clef-flash` | 4 decimals, each probability rounded on its own. Wrapped in Workers AI's `{ result, success, errors, messages }`. | (Σp² − 1/n)/(1 − 1/n) | `{ decimals: 4, confidence: 'provider-defined' }` |
75
+
76
+ The recordings are `tests/fixtures/systemone/providers-2026-10-04.json`, checked by
77
+ `tests/systemone-providers.test.ts`. Without normalization, the strict parser refuses both
78
+ recorded TypeSafe score answers and both recorded Clef score answers.
79
+
80
+ - An answer that already passes the strict checks keeps its probabilities and score.
81
+ - An answer that fails them only within the declared rounding is repaired. A distribution whose
82
+ sum is more than 1e-4 from 1 is divided by its sum. A score becomes the expectation of the
83
+ distribution.
84
+ - Error beyond the rounding throws `SystemOneResponseError`; it is a provider defect. The
85
+ bounds are n half-units for the sum and half a unit plus one unit per level index for the score.
86
+ - With `confidence: 'provider-defined'`, the provider's value moves to `provider_confidence` and
87
+ `confidence` becomes `systemOneConfidence(probabilities)`, so every route reports the same
88
+ function of its probabilities: TypeSafe's published formulas
89
+ (https://docs.typesafe.ai/confidence).
90
+ - Noul answers never change. `noul` is the probability of yes for every provider.
91
+
92
+ Probabilities are not made equivalent across models. A Clef 0.8 and a Jev 0.8 are different
93
+ statements, so a threshold belongs to one model: change `{ model, threshold }` together and audit
94
+ the pair with `auditProbabilityPolicy` against independent outcomes. `brierScore` and
95
+ `calibrationFromPairs` (with `range: { lo: 0, hi: 1 }`; `binning: 'equal-frequency'` for
96
+ equal-mass bins) in `/meta-eval` report calibration on the same pairs.
97
+
98
+ ## Any classifier, not only System One
49
99
 
50
100
  `createEvaluator`, `asJudge`, and `asAnalyst` are provider-independent exports from
51
- `@tangle-network/agent-eval/evaluation` (also re-exported from `/jev`).
101
+ `@tangle-network/agent-eval/evaluation` (also re-exported from `/systemone`).
52
102
  `createEvaluator({ execute, receipt })` meters a typed asynchronous function.
53
- It does not require a Jev response, probability output, prompt, or provider SDK.
103
+ It does not require a System One response, probability output, prompt, or provider SDK.
54
104
 
55
- `jevEvaluator` adds native protocol validation and optional `acceptModel(requested,
105
+ `systemOneEvaluator` adds native protocol validation and optional `acceptModel(requested,
56
106
  served)` policy. It preserves the reported model instead of guessing alias rules.
57
107
  Router independently enforces its supported model and pricing policy.
58
108
 
@@ -69,13 +119,13 @@ and analyst context. `map(value, input, analystContext)` creates evidence-backed
69
119
  findings. Use the existing `makeFinding` factory and real event/span/artifact references.
70
120
  An empty finding list is valid; an inference failure is not a clean review.
71
121
 
72
- The optional `jevJudge` and `jevAnalyst` conveniences accept static questions or a
122
+ The optional `systemOneJudge` and `systemOneAnalyst` conveniences accept static questions or a
73
123
  question-producing function. State renderers may be async. Judge renderers receive
74
124
  the evaluation context; analyst renderers and finding mappers receive the effective
75
125
  cancellation/deadline signal. Rendering must honor that signal during external work.
76
126
  Both conveniences accept a `record` callback for the complete observation.
77
127
 
78
- For a numeric rubric, `jevJudge` defaults to `normalizedJevScore`: equally spaced
128
+ For a numeric rubric, `systemOneJudge` defaults to `normalizedSystemOneScore`: equally spaced
79
129
  score expectations on [0,1], noul values unchanged, and an optional weighted mean.
80
130
  It emits no canned notes. Supply `map` for choice utilities, different scales, or
81
131
  critical-failure policies. Dynamic questions and custom mappings require explicit
@@ -98,7 +148,7 @@ none is supplied, an explicit `budgetUsd` creates an account for that invocation
98
148
  Otherwise the evaluator keeps its configured ledger, including its cumulative cap
99
149
  across direct calls, judges, and analysts. An absent analyst budget never creates a
100
150
  fresh unlimited account that overrides the configured one. This precedence is the
101
- same for Jev and other classifiers. A per-analysis budget replaces the default
151
+ same for System One and other classifiers. A per-analysis budget replaces the default
102
152
  account; it is not an additional child limit. Pass the shared run ledger explicitly
103
153
  when the whole run must conserve one authority. Only trusted host code should choose
104
154
  or replace that authority.
@@ -132,10 +182,11 @@ confidence as demonstrated calibration or a classification as proof of root caus
132
182
 
133
183
  ## Checks
134
184
 
135
- Run `pnpm exec vitest run tests/jev*.test.ts tests/evaluation-authority.test.ts`,
185
+ Run `pnpm exec vitest run tests/systemone*.test.ts tests/evaluation-authority.test.ts`,
136
186
  `pnpm typecheck`, `pnpm build`, and `pnpm verify:package`. These are contract and
137
187
  execution checks, not live-provider quality or deployment evidence. The product-integrity
138
188
  rubric lives in `examples/`, not the generic implementation, and is not a default policy.
139
189
 
140
190
  Protocol: https://docs.typesafe.ai/api
141
- SDK: https://github.com/typesafe-ai/typesafe-sdk-js
191
+ Clef: https://developers.cloudflare.com/workers-ai/models/clef/
192
+ SDK (optional builders): https://github.com/typesafe-ai/typesafe-sdk-js
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tangle-network/agent-eval",
3
- "version": "0.205.1",
3
+ "version": "0.206.0",
4
4
  "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
5
5
  "homepage": "https://github.com/tangle-network/agent-eval#readme",
6
6
  "repository": {
@@ -19,10 +19,10 @@
19
19
  "import": "./dist/index.js",
20
20
  "default": "./dist/index.js"
21
21
  },
22
- "./jev": {
23
- "types": "./dist/jev.d.ts",
24
- "import": "./dist/jev.js",
25
- "default": "./dist/jev.js"
22
+ "./systemone": {
23
+ "types": "./dist/systemone.d.ts",
24
+ "import": "./dist/systemone.js",
25
+ "default": "./dist/systemone.js"
26
26
  },
27
27
  "./profile-cell": {
28
28
  "types": "./dist/profile-cell.d.ts",
@@ -177,10 +177,10 @@
177
177
  "import": "./dist/evaluation.js",
178
178
  "default": "./dist/evaluation.js"
179
179
  },
180
- "./jev/protocol": {
181
- "types": "./dist/jev-protocol.d.ts",
182
- "import": "./dist/jev-protocol.js",
183
- "default": "./dist/jev-protocol.js"
180
+ "./systemone/protocol": {
181
+ "types": "./dist/systemone-protocol.d.ts",
182
+ "import": "./dist/systemone-protocol.js",
183
+ "default": "./dist/systemone-protocol.js"
184
184
  }
185
185
  },
186
186
  "bin": {
@@ -1 +0,0 @@
1
- {"version":3,"file":"claude-jsonl-DoQJBCVR.js","names":[],"sources":["../src/rollout/readers/claude-jsonl.ts"],"sourcesContent":["/**\n * Backfill reader over Claude Code project transcripts\n * (~/.claude/projects/<cwd-slug>/<sessionId>.jsonl) → canonical\n * chat-with-tools messages plus per-session token usage.\n *\n * Transcript lines consumed: type:\"user\" (string content or content blocks —\n * text + tool_result) and type:\"assistant\" (content blocks — thinking, text,\n * tool_use; message.usage carries tokens). Sidechain lines (isSidechain=true,\n * subagent threads) are separate invocations and are excluded from the main\n * transcript. Everything else (queue-operation, attachment, last-prompt…) is\n * transport metadata, not conversation.\n */\n\nimport { readdir, readFile } from 'node:fs/promises'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport type { ChatMessage, ChatToolCall } from '../schema'\n\nexport const DEFAULT_CLAUDE_PROJECTS_DIR = join(homedir(), '.claude', 'projects')\n\n/** Claude Code's project-directory slug for a working directory. */\nexport function claudeProjectSlug(cwd: string): string {\n return cwd.replace(/[^a-zA-Z0-9-]/g, '-')\n}\n\nexport interface ClaudeTranscriptRef {\n sessionId: string\n path: string\n}\n\n/** Transcript files recorded for sessions launched from `cwd`. */\nexport async function findClaudeTranscripts(\n cwd: string,\n projectsDir: string = DEFAULT_CLAUDE_PROJECTS_DIR,\n): Promise<ClaudeTranscriptRef[]> {\n const dir = join(projectsDir, claudeProjectSlug(cwd))\n const names = await readdir(dir).catch(() => [])\n return names\n .filter((n) => n.endsWith('.jsonl'))\n .sort()\n .map((n) => ({ sessionId: n.replace(/\\.jsonl$/, ''), path: join(dir, n) }))\n}\n\nexport interface ClaudeUsageTotals {\n tokensIn: number | null\n tokensOut: number | null\n cacheRead: number | null\n cacheWrite: number | null\n}\n\nexport interface ClaudeTranscript {\n messages: ChatMessage[]\n usage: ClaudeUsageTotals\n /** Timestamp of the first conversation line; null = empty transcript. */\n startedAt: string | null\n endedAt: string | null\n model: string | null\n /** Every parse or usage gap retained beside this semantic projection. */\n gaps: string[]\n}\n\nconst isRecord = (v: unknown): v is Record<string, unknown> =>\n typeof v === 'object' && v !== null && !Array.isArray(v)\n\n/**\n * One conversation line of a transcript, still in Claude Code's own shape.\n *\n * This is the single line-level parse of the format. `readClaudeTranscript`\n * projects it to canonical messages + usage; the supervision-tree reader\n * (`src/supervisor-run/claude-code-reader.ts`) projects the SAME entries to\n * spawn/settle/steer instants. Two projections, one parser — a second\n * transcript parser is how the two views silently disagree.\n */\nexport interface ClaudeEntry {\n readonly type: 'user' | 'assistant'\n /** ISO instant of the line; null when the line carried none. */\n readonly timestamp: string | null\n /** The Anthropic message body (`role`, `content`, `model`, `usage`). */\n readonly message: Record<string, unknown>\n /** Claude Code's structured tool result, when the line carries one. */\n readonly toolUseResult: unknown\n /** True on subagent threads — a separate invocation, not this transcript's turn. */\n readonly isSidechain: boolean\n /** Subagent id Claude Code stamps on sidechain lines; null on main-thread lines. */\n readonly agentId: string | null\n}\n\n/** Parse transcript jsonl text into conversation lines. Non-conversation lines are dropped. */\nexport function parseClaudeEntries(raw: string, onGap?: (gap: string) => void): ClaudeEntry[] {\n const out: ClaudeEntry[] = []\n for (const [index, line] of raw.split('\\n').entries()) {\n if (!line.trim()) continue\n const gap = (reason: string): void => {\n const detail = `line ${index + 1}: ${reason}`\n if (onGap) onGap(detail)\n else throw new Error(`Claude transcript ${detail}`)\n }\n let parsed: unknown\n try {\n parsed = JSON.parse(line)\n } catch {\n gap('malformed JSON')\n continue\n }\n if (!isRecord(parsed)) {\n gap('JSON value is not an object')\n continue\n }\n const entry = parsed\n if (entry.type !== 'user' && entry.type !== 'assistant') continue\n const message = entry.message\n if (!isRecord(message)) {\n gap('conversation message is missing')\n continue\n }\n out.push({\n type: entry.type,\n timestamp: typeof entry.timestamp === 'string' ? entry.timestamp : null,\n message,\n toolUseResult: entry.toolUseResult,\n isSidechain: entry.isSidechain === true,\n agentId: typeof entry.agentId === 'string' ? entry.agentId : null,\n })\n }\n return out\n}\n\nexport interface ReadClaudeTranscriptOptions {\n /**\n * Read the sidechain (subagent) thread instead of skipping it. Subagent\n * transcripts under `<session>/subagents/agent-<id>.jsonl` are sidechain\n * lines end to end, so their usage is invisible without this.\n */\n readonly includeSidechain?: boolean\n /** Parser diagnostics from the retained source; used by the shared reader. */\n readonly sourceGaps?: readonly string[]\n}\n\nfunction blockText(content: unknown): string {\n if (typeof content === 'string') return content\n if (!Array.isArray(content)) return ''\n return content\n .filter(\n (b): b is Record<string, unknown> =>\n isRecord(b) && b.type === 'text' && typeof b.text === 'string',\n )\n .map((b) => b.text as string)\n .join('\\n')\n}\n\n/** Parse one transcript jsonl into canonical messages + usage totals. */\nexport async function readClaudeTranscript(\n path: string,\n options: ReadClaudeTranscriptOptions = {},\n): Promise<ClaudeTranscript> {\n const gaps: string[] = []\n const entries = parseClaudeEntries(await readFile(path, 'utf8'), (gap) => gaps.push(gap))\n return transcriptFromEntries(entries, { ...options, sourceGaps: gaps })\n}\n\n/** The messages+usage projection of already-parsed entries. */\nexport function transcriptFromEntries(\n entries: readonly ClaudeEntry[],\n options: ReadClaudeTranscriptOptions = {},\n): ClaudeTranscript {\n const wantSidechain = options.includeSidechain === true\n const messages: ChatMessage[] = []\n const usage: ClaudeUsageTotals = { tokensIn: 0, tokensOut: 0, cacheRead: 0, cacheWrite: 0 }\n const gaps = [...(options.sourceGaps ?? [])]\n let usageObserved = false\n let startedAt: string | null = null\n let endedAt: string | null = null\n let model: string | null = null\n // Claude Code writes one jsonl line PER CONTENT BLOCK of an API message,\n // repeating message.id and usage on each — merge blocks into one canonical\n // assistant turn and count usage once per API message id.\n let lastAssistantApiId: string | null = null\n let lastAssistantIndex = -1\n\n for (const entry of entries) {\n if (entry.isSidechain !== wantSidechain) continue\n const message = entry.message\n if (entry.timestamp !== null) {\n if (startedAt === null) startedAt = entry.timestamp\n endedAt = entry.timestamp\n }\n\n if (entry.type === 'user') {\n lastAssistantApiId = null\n lastAssistantIndex = -1\n const content = message.content\n if (typeof content === 'string') {\n messages.push({ role: 'user', content })\n continue\n }\n if (!Array.isArray(content)) continue\n // A user line may interleave tool_result blocks (answers to the prior\n // assistant tool_use) with plain text; preserve order.\n let userText = ''\n for (const block of content) {\n if (!isRecord(block)) continue\n if (block.type === 'tool_result' && typeof block.tool_use_id === 'string') {\n messages.push({\n role: 'tool',\n tool_call_id: block.tool_use_id,\n content:\n blockText(block.content) || (typeof block.content === 'string' ? block.content : ''),\n })\n } else if (block.type === 'text' && typeof block.text === 'string') {\n userText += (userText.length > 0 ? '\\n' : '') + block.text\n }\n }\n if (userText.length > 0) messages.push({ role: 'user', content: userText })\n continue\n }\n\n // assistant\n if (typeof message.model === 'string') model = message.model\n const apiId = typeof message.id === 'string' ? message.id : null\n const continuesTurn = apiId !== null && apiId === lastAssistantApiId && lastAssistantIndex >= 0\n const msgUsage = message.usage\n if (!continuesTurn) {\n usageObserved = true\n const fields = [\n ['tokensIn', 'input_tokens'],\n ['tokensOut', 'output_tokens'],\n ['cacheRead', 'cache_read_input_tokens'],\n ['cacheWrite', 'cache_creation_input_tokens'],\n ] as const\n for (const [target, source] of fields) {\n const value = isRecord(msgUsage) ? msgUsage[source] : undefined\n if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {\n usage[target] = null\n gaps.push(`assistant ${apiId ?? 'without id'}: ${source} unavailable`)\n } else if (usage[target] !== null) {\n usage[target] += value\n }\n }\n }\n const content = message.content\n if (!Array.isArray(content)) continue\n let reasoning = ''\n let text = ''\n const toolCalls: ChatToolCall[] = []\n for (const block of content) {\n if (!isRecord(block)) continue\n if (\n block.type === 'thinking' &&\n typeof block.thinking === 'string' &&\n block.thinking.length > 0\n ) {\n reasoning += (reasoning.length > 0 ? '\\n' : '') + block.thinking\n } else if (block.type === 'text' && typeof block.text === 'string') {\n text += (text.length > 0 ? '\\n' : '') + block.text\n } else if (block.type === 'tool_use' && typeof block.id === 'string') {\n toolCalls.push({\n id: block.id,\n type: 'function',\n function: {\n name: typeof block.name === 'string' ? block.name : 'unknown',\n arguments: JSON.stringify(block.input ?? {}),\n },\n })\n }\n }\n if (reasoning.length === 0 && text.length === 0 && toolCalls.length === 0) continue\n if (continuesTurn) {\n const prev = messages[lastAssistantIndex]!\n if (text.length > 0) prev.content = prev.content === null ? text : `${prev.content}\\n${text}`\n if (reasoning.length > 0) {\n prev.reasoning_content =\n prev.reasoning_content === undefined\n ? reasoning\n : `${prev.reasoning_content}\\n${reasoning}`\n }\n if (toolCalls.length > 0) prev.tool_calls = [...(prev.tool_calls ?? []), ...toolCalls]\n continue\n }\n messages.push({\n role: 'assistant',\n content: text.length > 0 ? text : null,\n ...(reasoning.length > 0 ? { reasoning_content: reasoning } : {}),\n ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),\n })\n lastAssistantApiId = apiId\n lastAssistantIndex = messages.length - 1\n }\n\n if (!usageObserved) {\n usage.tokensIn = null\n usage.tokensOut = null\n usage.cacheRead = null\n usage.cacheWrite = null\n }\n return { messages, usage, startedAt, endedAt, model, gaps }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAkBA,MAAa,8BAA8B,KAAK,QAAQ,GAAG,WAAW,UAAU;;AAGhF,SAAgB,kBAAkB,KAAqB;CACrD,OAAO,IAAI,QAAQ,kBAAkB,GAAG;AAC1C;;AAQA,eAAsB,sBACpB,KACA,cAAsB,6BACU;CAChC,MAAM,MAAM,KAAK,aAAa,kBAAkB,GAAG,CAAC;CAEpD,QAAO,MADa,QAAQ,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,EAAA,CAE5C,QAAQ,MAAM,EAAE,SAAS,QAAQ,CAAC,CAAC,CACnC,KAAK,CAAC,CACN,KAAK,OAAO;EAAE,WAAW,EAAE,QAAQ,YAAY,EAAE;EAAG,MAAM,KAAK,KAAK,CAAC;CAAE,EAAE;AAC9E;AAoBA,MAAM,YAAY,MAChB,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;;AA0BzD,SAAgB,mBAAmB,KAAa,OAA8C;CAC5F,MAAM,MAAqB,CAAC;CAC5B,KAAK,MAAM,CAAC,OAAO,SAAS,IAAI,MAAM,IAAI,CAAC,CAAC,QAAQ,GAAG;EACrD,IAAI,CAAC,KAAK,KAAK,GAAG;EAClB,MAAM,OAAO,WAAyB;GACpC,MAAM,SAAS,QAAQ,QAAQ,EAAE,IAAI;GACrC,IAAI,OAAO,MAAM,MAAM;QAClB,MAAM,IAAI,MAAM,qBAAqB,QAAQ;EACpD;EACA,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,IAAI;EAC1B,QAAQ;GACN,IAAI,gBAAgB;GACpB;EACF;EACA,IAAI,CAAC,SAAS,MAAM,GAAG;GACrB,IAAI,6BAA6B;GACjC;EACF;EACA,MAAM,QAAQ;EACd,IAAI,MAAM,SAAS,UAAU,MAAM,SAAS,aAAa;EACzD,MAAM,UAAU,MAAM;EACtB,IAAI,CAAC,SAAS,OAAO,GAAG;GACtB,IAAI,iCAAiC;GACrC;EACF;EACA,IAAI,KAAK;GACP,MAAM,MAAM;GACZ,WAAW,OAAO,MAAM,cAAc,WAAW,MAAM,YAAY;GACnE;GACA,eAAe,MAAM;GACrB,aAAa,MAAM,gBAAgB;GACnC,SAAS,OAAO,MAAM,YAAY,WAAW,MAAM,UAAU;EAC/D,CAAC;CACH;CACA,OAAO;AACT;AAaA,SAAS,UAAU,SAA0B;CAC3C,IAAI,OAAO,YAAY,UAAU,OAAO;CACxC,IAAI,CAAC,MAAM,QAAQ,OAAO,GAAG,OAAO;CACpC,OAAO,QACJ,QACE,MACC,SAAS,CAAC,KAAK,EAAE,SAAS,UAAU,OAAO,EAAE,SAAS,QAC1D,CAAC,CACA,KAAK,MAAM,EAAE,IAAc,CAAC,CAC5B,KAAK,IAAI;AACd;;AAGA,eAAsB,qBACpB,MACA,UAAuC,CAAC,GACb;CAC3B,MAAM,OAAiB,CAAC;CAExB,OAAO,sBADS,mBAAmB,MAAM,SAAS,MAAM,MAAM,IAAI,QAAQ,KAAK,KAAK,GAAG,CACpD,GAAG;EAAE,GAAG;EAAS,YAAY;CAAK,CAAC;AACxE;;AAGA,SAAgB,sBACd,SACA,UAAuC,CAAC,GACtB;CAClB,MAAM,gBAAgB,QAAQ,qBAAqB;CACnD,MAAM,WAA0B,CAAC;CACjC,MAAM,QAA2B;EAAE,UAAU;EAAG,WAAW;EAAG,WAAW;EAAG,YAAY;CAAE;CAC1F,MAAM,OAAO,CAAC,GAAI,QAAQ,cAAc,CAAC,CAAE;CAC3C,IAAI,gBAAgB;CACpB,IAAI,YAA2B;CAC/B,IAAI,UAAyB;CAC7B,IAAI,QAAuB;CAI3B,IAAI,qBAAoC;CACxC,IAAI,qBAAqB;CAEzB,KAAK,MAAM,SAAS,SAAS;EAC3B,IAAI,MAAM,gBAAgB,eAAe;EACzC,MAAM,UAAU,MAAM;EACtB,IAAI,MAAM,cAAc,MAAM;GAC5B,IAAI,cAAc,MAAM,YAAY,MAAM;GAC1C,UAAU,MAAM;EAClB;EAEA,IAAI,MAAM,SAAS,QAAQ;GACzB,qBAAqB;GACrB,qBAAqB;GACrB,MAAM,UAAU,QAAQ;GACxB,IAAI,OAAO,YAAY,UAAU;IAC/B,SAAS,KAAK;KAAE,MAAM;KAAQ;IAAQ,CAAC;IACvC;GACF;GACA,IAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;GAG7B,IAAI,WAAW;GACf,KAAK,MAAM,SAAS,SAAS;IAC3B,IAAI,CAAC,SAAS,KAAK,GAAG;IACtB,IAAI,MAAM,SAAS,iBAAiB,OAAO,MAAM,gBAAgB,UAC/D,SAAS,KAAK;KACZ,MAAM;KACN,cAAc,MAAM;KACpB,SACE,UAAU,MAAM,OAAO,MAAM,OAAO,MAAM,YAAY,WAAW,MAAM,UAAU;IACrF,CAAC;SACI,IAAI,MAAM,SAAS,UAAU,OAAO,MAAM,SAAS,UACxD,aAAa,SAAS,SAAS,IAAI,OAAO,MAAM,MAAM;GAE1D;GACA,IAAI,SAAS,SAAS,GAAG,SAAS,KAAK;IAAE,MAAM;IAAQ,SAAS;GAAS,CAAC;GAC1E;EACF;EAGA,IAAI,OAAO,QAAQ,UAAU,UAAU,QAAQ,QAAQ;EACvD,MAAM,QAAQ,OAAO,QAAQ,OAAO,WAAW,QAAQ,KAAK;EAC5D,MAAM,gBAAgB,UAAU,QAAQ,UAAU,sBAAsB,sBAAsB;EAC9F,MAAM,WAAW,QAAQ;EACzB,IAAI,CAAC,eAAe;GAClB,gBAAgB;GAOhB,KAAK,MAAM,CAAC,QAAQ,WAAW;IAL7B,CAAC,YAAY,cAAc;IAC3B,CAAC,aAAa,eAAe;IAC7B,CAAC,aAAa,yBAAyB;IACvC,CAAC,cAAc,6BAA6B;GAEV,GAAG;IACrC,MAAM,QAAQ,SAAS,QAAQ,IAAI,SAAS,UAAU,KAAA;IACtD,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GAAG;KACrE,MAAM,UAAU;KAChB,KAAK,KAAK,aAAa,SAAS,aAAa,IAAI,OAAO,aAAa;IACvE,OAAO,IAAI,MAAM,YAAY,MAC3B,MAAM,WAAW;GAErB;EACF;EACA,MAAM,UAAU,QAAQ;EACxB,IAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;EAC7B,IAAI,YAAY;EAChB,IAAI,OAAO;EACX,MAAM,YAA4B,CAAC;EACnC,KAAK,MAAM,SAAS,SAAS;GAC3B,IAAI,CAAC,SAAS,KAAK,GAAG;GACtB,IACE,MAAM,SAAS,cACf,OAAO,MAAM,aAAa,YAC1B,MAAM,SAAS,SAAS,GAExB,cAAc,UAAU,SAAS,IAAI,OAAO,MAAM,MAAM;QACnD,IAAI,MAAM,SAAS,UAAU,OAAO,MAAM,SAAS,UACxD,SAAS,KAAK,SAAS,IAAI,OAAO,MAAM,MAAM;QACzC,IAAI,MAAM,SAAS,cAAc,OAAO,MAAM,OAAO,UAC1D,UAAU,KAAK;IACb,IAAI,MAAM;IACV,MAAM;IACN,UAAU;KACR,MAAM,OAAO,MAAM,SAAS,WAAW,MAAM,OAAO;KACpD,WAAW,KAAK,UAAU,MAAM,SAAS,CAAC,CAAC;IAC7C;GACF,CAAC;EAEL;EACA,IAAI,UAAU,WAAW,KAAK,KAAK,WAAW,KAAK,UAAU,WAAW,GAAG;EAC3E,IAAI,eAAe;GACjB,MAAM,OAAO,SAAS;GACtB,IAAI,KAAK,SAAS,GAAG,KAAK,UAAU,KAAK,YAAY,OAAO,OAAO,GAAG,KAAK,QAAQ,IAAI;GACvF,IAAI,UAAU,SAAS,GACrB,KAAK,oBACH,KAAK,sBAAsB,KAAA,IACvB,YACA,GAAG,KAAK,kBAAkB,IAAI;GAEtC,IAAI,UAAU,SAAS,GAAG,KAAK,aAAa,CAAC,GAAI,KAAK,cAAc,CAAC,GAAI,GAAG,SAAS;GACrF;EACF;EACA,SAAS,KAAK;GACZ,MAAM;GACN,SAAS,KAAK,SAAS,IAAI,OAAO;GAClC,GAAI,UAAU,SAAS,IAAI,EAAE,mBAAmB,UAAU,IAAI,CAAC;GAC/D,GAAI,UAAU,SAAS,IAAI,EAAE,YAAY,UAAU,IAAI,CAAC;EAC1D,CAAC;EACD,qBAAqB;EACrB,qBAAqB,SAAS,SAAS;CACzC;CAEA,IAAI,CAAC,eAAe;EAClB,MAAM,WAAW;EACjB,MAAM,YAAY;EAClB,MAAM,YAAY;EAClB,MAAM,aAAa;CACrB;CACA,OAAO;EAAE;EAAU;EAAO;EAAW;EAAS;EAAO;CAAK;AAC5D"}
@@ -1,99 +0,0 @@
1
- //#region src/jev-protocol.d.ts
2
- /**
3
- * Structural equivalents of TypeSafe's published request schema
4
- * (https://api.typesafe.ai/openapi.json); callers can use its builders unchanged.
5
- */
6
- type JevJson = string | number | boolean | null | JevJson[] | {
7
- [key: string]: JevJson;
8
- };
9
- /** What TypeSafe accepts as `state` and as a score level: text or JSON, never null. */
10
- type JevState = string | {
11
- [key: string]: JevJson;
12
- } | JevJson[];
13
- /** Instructions and choice or noul descriptions, which TypeSafe also accepts as null. */
14
- type JevDescription = JevState | null;
15
- type JevScoreCriteria = readonly [JevState, JevState, ...JevState[]];
16
- type JevQuestion = {
17
- type: 'noul';
18
- instructions?: JevDescription;
19
- criteria?: {
20
- true?: JevDescription;
21
- false?: JevDescription;
22
- } | null;
23
- } | {
24
- type: 'score';
25
- instructions?: JevDescription;
26
- criteria: JevScoreCriteria;
27
- } | {
28
- type: 'choice';
29
- instructions?: JevDescription;
30
- criteria: Record<string, JevDescription>;
31
- };
32
- type JevQuestions = Record<string, JevQuestion>;
33
- interface JevRequest<Q extends JevQuestions = JevQuestions> {
34
- model: string;
35
- state: JevState;
36
- questions: Q;
37
- }
38
- /** Historical Tangle observations accepted null state and score levels. Not a live request. */
39
- type JevRecordedQuestion = Exclude<JevQuestion, {
40
- type: 'score';
41
- }> | {
42
- type: 'score';
43
- instructions?: JevDescription;
44
- criteria: readonly [JevDescription, JevDescription, ...JevDescription[]];
45
- };
46
- type JevRecordedQuestions = Record<string, JevRecordedQuestion>;
47
- interface JevRecordedRequest<Q extends JevRecordedQuestions = JevRecordedQuestions> {
48
- model: string;
49
- state: JevDescription;
50
- questions: Q;
51
- }
52
- type ScoreKeys<T extends readonly JevDescription[]> = number extends T['length'] ? number : Extract<keyof T, `${number}`>;
53
- type JevAnswerFor<Q extends JevRecordedQuestion> = Q extends {
54
- type: 'noul';
55
- } ? {
56
- readonly type: 'noul';
57
- readonly noul: number;
58
- } : Q extends {
59
- type: 'score';
60
- criteria: infer C extends readonly JevDescription[];
61
- } ? {
62
- readonly type: 'score';
63
- readonly score: number;
64
- readonly confidence: number;
65
- readonly legend: { readonly [K in ScoreKeys<C>]: C[K]; };
66
- readonly probabilities: { readonly [K in ScoreKeys<C>]: number; };
67
- } : Q extends {
68
- type: 'choice';
69
- criteria: infer C;
70
- } ? {
71
- readonly type: 'choice';
72
- readonly choice: keyof C & string;
73
- readonly confidence: number;
74
- readonly probabilities: { readonly [K in keyof C]: number; };
75
- } : never;
76
- type JevAnswer = JevAnswerFor<JevQuestion>;
77
- interface JevResult<Q extends JevRecordedQuestions = JevQuestions> {
78
- readonly model: string;
79
- readonly answers: { readonly [K in keyof Q]: JevAnswerFor<Q[K]>; };
80
- readonly usage: {
81
- readonly input_tokens: number;
82
- readonly output_tokens: number;
83
- };
84
- }
85
- declare class JevResponseError extends Error {
86
- constructor(message: string);
87
- }
88
- /** Validate named questions without a state, as a judge does before it has input. */
89
- declare function parseJevQuestions(raw: unknown): JevQuestions;
90
- declare function parseJevRequest(raw: unknown): JevRequest;
91
- /** Read immutable historical evidence without rewriting it or authorizing new inference.
92
- * Live transports must still call parseJevRequest, even for re-execution of an old request. */
93
- declare function parseJevRecordedRequest(raw: unknown): JevRecordedRequest;
94
- declare function jevUsage(raw: unknown): JevResult['usage'];
95
- /** Validate native evidence, not model alias policy or the caller's scoring rules. */
96
- declare function parseJevResult<Q extends JevRecordedQuestions>(raw: unknown, request: JevRecordedRequest<Q>): JevResult<Q>;
97
- //#endregion
98
- export { parseJevRecordedRequest as _, JevQuestion as a, JevRecordedQuestions as c, JevResponseError as d, JevResult as f, parseJevQuestions as g, jevUsage as h, JevJson as i, JevRecordedRequest as l, JevState as m, JevAnswerFor as n, JevQuestions as o, JevScoreCriteria as p, JevDescription as r, JevRecordedQuestion as s, JevAnswer as t, JevRequest as u, parseJevRequest as v, parseJevResult as y };
99
- //# sourceMappingURL=jev-protocol-BQLxwbqD.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"jev-protocol-BQLxwbqD.d.ts","names":[],"sources":["../src/jev-protocol.ts"],"mappings":";;;;;KAOY,6CAA6C;GAAe,cAAc;;;KAE1E;GAAuB,cAAc;IAAY;;KAEjD,iBAAiB;KACjB,6BAA6B,UAAU,aAAa;KACpD;EAEN;EACA,eAAe;EACf;IAAa,OAAO;IAAgB,QAAQ;;;EAE5C;EAAe,eAAe;EAAgB,UAAU;;EACxD;EAAgB,eAAe;EAAgB,UAAU,eAAe;;KAClE,eAAe,eAAe;UACzB,WAAW,UAAU,eAAe;EACnD;EACA,OAAO;EACP,WAAW;;;KAID,sBACR,QAAQ;EAAe;;EAErB;EACA,eAAe;EACf,oBAAoB,gBAAgB,mBAAmB;;KAEjD,uBAAuB,eAAe;UACjC,mBAAmB,UAAU,uBAAuB;EACnE;EACA,OAAO;EACP,WAAW;;KAGR,UAAU,mBAAmB,mCAAmC,uBAEjE,cAAc;KACN,aAAa,UAAU,uBAAuB;EAAY;;WACvD;WAAuB;IAClC;EAAY;EAAe,gBAAgB,mBAAmB;;WAEjD;WACA;WACA;WACA,oBAAoB,KAAK,UAAU,KAAK,EAAE;WAC1C,2BAA2B,KAAK,UAAU;IAErD;EAAY;EAAgB,gBAAgB;;WAE/B;WACA,cAAc;WACd;WACA,2BAA2B,WAAW;;KAG7C,YAAY,aAAa;UACpB,UAAU,UAAU,uBAAuB;WACjD;WACA,qBAAqB,WAAW,IAAI,aAAa,EAAE;WACnD;aAAkB;aAA+B;;;cAG/C,yBAAyB;EACpC,YAAY;;;iBA6BE,kBAAkB,eAAe;iBAyCjC,gBAAgB,eAAe;;;iBAM/B,wBAAwB,eAAe;iBAevC,SAAS,eAAe;;iBA2BxB,eAAe,UAAU,sBACvC,cACA,SAAS,mBAAmB,KAC3B,UAAU"}
@@ -1,2 +0,0 @@
1
- import { _ as parseJevRecordedRequest, a as JevQuestion, c as JevRecordedQuestions, d as JevResponseError, f as JevResult, g as parseJevQuestions, h as jevUsage, i as JevJson, l as JevRecordedRequest, m as JevState, n as JevAnswerFor, o as JevQuestions, p as JevScoreCriteria, r as JevDescription, s as JevRecordedQuestion, t as JevAnswer, u as JevRequest, v as parseJevRequest, y as parseJevResult } from "./jev-protocol-BQLxwbqD.js";
2
- export { JevAnswer, JevAnswerFor, JevDescription, JevJson, JevQuestion, JevQuestions, JevRecordedQuestion, JevRecordedQuestions, JevRecordedRequest, JevRequest, JevResponseError, JevResult, JevScoreCriteria, JevState, jevUsage, parseJevQuestions, parseJevRecordedRequest, parseJevRequest, parseJevResult };