@tangle-network/agent-eval 0.205.1 → 0.206.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/dist/analyst/index.d.ts +1 -1
- package/dist/analyst/index.js +1 -1
- package/dist/{benchmark-command-DuDTN5ID.js → benchmark-command-DtP94cOB.js} +2 -2
- package/dist/{benchmark-command-DuDTN5ID.js.map → benchmark-command-DtP94cOB.js.map} +1 -1
- package/dist/{claude-jsonl-DoQJBCVR.js → claude-jsonl-CG3JUbTJ.js} +8 -4
- package/dist/claude-jsonl-CG3JUbTJ.js.map +1 -0
- package/dist/cli.js +1 -1
- package/dist/{index-BoVEyQwH.d.ts → index-CbBfavbT.d.ts} +5 -3
- package/dist/{index-BoVEyQwH.d.ts.map → index-CbBfavbT.d.ts.map} +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/meta-eval/index.d.ts +6 -0
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +15 -1
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/rl.d.ts +12 -12
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +17 -17
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/rollout/index.js +3 -3
- package/dist/{rollout-BdCoWV9n.js → rollout-l9w3Gdvh.js} +2 -2
- package/dist/{rollout-BdCoWV9n.js.map → rollout-l9w3Gdvh.js.map} +1 -1
- package/dist/supervisor-run/index.js +1 -1
- package/dist/systemone-protocol-DKoi1NdK.d.ts +132 -0
- package/dist/systemone-protocol-DKoi1NdK.d.ts.map +1 -0
- package/dist/systemone-protocol.d.ts +2 -0
- package/dist/systemone-protocol.js +214 -0
- package/dist/systemone-protocol.js.map +1 -0
- package/dist/{jev-review-Cw06AKuF.d.ts → systemone-review-DGTKkLMG.d.ts} +16 -16
- package/dist/systemone-review-DGTKkLMG.d.ts.map +1 -0
- package/dist/{jev-review-DZEjDzlF.js → systemone-review-DTClwgmE.js} +11 -11
- package/dist/systemone-review-DTClwgmE.js.map +1 -0
- package/dist/systemone.d.ts +60 -0
- package/dist/systemone.d.ts.map +1 -0
- package/dist/{jev.js → systemone.js} +16 -16
- package/dist/systemone.js.map +1 -0
- package/docs/changes/jev-evidence-review.md +1 -1
- package/docs/changes/systemone-provider-neutral.md +36 -0
- package/docs/public-api.md +93 -68
- package/docs/rollout.md +9 -1
- package/docs/{jev-behavior-review.md → systemone-behavior-review.md} +17 -17
- package/docs/{jev-integrations.md → systemone-integrations.md} +4 -4
- package/docs/{jev-replay.md → systemone-replay.md} +6 -6
- package/docs/{jev.md → systemone.md} +84 -33
- package/package.json +9 -9
- package/dist/claude-jsonl-DoQJBCVR.js.map +0 -1
- package/dist/jev-protocol-BQLxwbqD.d.ts +0 -99
- package/dist/jev-protocol-BQLxwbqD.d.ts.map +0 -1
- package/dist/jev-protocol.d.ts +0 -2
- package/dist/jev-protocol.js +0 -114
- package/dist/jev-protocol.js.map +0 -1
- package/dist/jev-review-Cw06AKuF.d.ts.map +0 -1
- package/dist/jev-review-DZEjDzlF.js.map +0 -1
- package/dist/jev.d.ts +0 -60
- package/dist/jev.d.ts.map +0 -1
- package/dist/jev.js.map +0 -1
|
@@ -4,17 +4,17 @@ Use native evaluations to assess bounded hypotheses about recorded behavior. Do
|
|
|
4
4
|
classifier whether an agent is "malicious" or a whole run is "safe". Observable policy breaches,
|
|
5
5
|
model-supported hypotheses, missing evidence and unusual behavior are different results.
|
|
6
6
|
|
|
7
|
-
The optional `/
|
|
7
|
+
The optional `/systemone` helpers are ordinary functions:
|
|
8
8
|
|
|
9
|
-
- `
|
|
9
|
+
- `prepareSystemOneReview`: validate and freeze caller-defined native questions, evidence references,
|
|
10
10
|
coverage and decision thresholds before spending. The digest binds the definition, input and
|
|
11
11
|
authored question/option order. Optional undefined object fields are absent on the wire.
|
|
12
|
-
- `
|
|
12
|
+
- `assessSystemOneReview`: validate native answers and interpret designated alternative masses as
|
|
13
13
|
`supported`, `refuted` or `unresolved`. Re-scoring makes no network call.
|
|
14
|
-
- `
|
|
14
|
+
- `systemOneReviewFindings`: map that report into the existing `AnalystFinding` contract. It keeps
|
|
15
15
|
unresolved checks visible and labels supported claims as model-supported hypotheses.
|
|
16
16
|
|
|
17
|
-
These helpers do not constrain `
|
|
17
|
+
These helpers do not constrain `systemOneEvaluator`, `systemOneJudge`, `systemOneAnalyst`, or the provider-independent
|
|
18
18
|
`createEvaluator`/`asJudge`/`asAnalyst` APIs. Use an ordinary custom mapper for different policies.
|
|
19
19
|
No model, taxonomy, severity, operating threshold, global safety score or storage backend is chosen
|
|
20
20
|
by the generic implementation.
|
|
@@ -26,16 +26,16 @@ Supply the native request on each invocation. This example's `question`, `eviden
|
|
|
26
26
|
|
|
27
27
|
```ts
|
|
28
28
|
import {
|
|
29
|
-
|
|
30
|
-
} from '@tangle-network/agent-eval/
|
|
29
|
+
prepareSystemOneReview, assessSystemOneReview, systemOneEvaluator,
|
|
30
|
+
} from '@tangle-network/agent-eval/systemone'
|
|
31
31
|
|
|
32
|
-
const evaluate =
|
|
32
|
+
const evaluate = systemOneEvaluator({
|
|
33
33
|
evaluate: nativeTransport, // One Router /v1/systemone attempt; preserve signal/idempotency key.
|
|
34
34
|
receipt: readRouterReceipt,
|
|
35
35
|
maximumCharge: enforcedMaximumCharge,
|
|
36
36
|
})
|
|
37
37
|
|
|
38
|
-
const review =
|
|
38
|
+
const review = prepareSystemOneReview({
|
|
39
39
|
version: policy.version,
|
|
40
40
|
request: {
|
|
41
41
|
model: selectedModel,
|
|
@@ -59,7 +59,7 @@ const review = prepareJevReview({
|
|
|
59
59
|
})
|
|
60
60
|
const result = await evaluate(review.request, executionContext)
|
|
61
61
|
await persistAuthorizedObservation({ review, result })
|
|
62
|
-
const report =
|
|
62
|
+
const report = assessSystemOneReview(review, result.value)
|
|
63
63
|
```
|
|
64
64
|
|
|
65
65
|
The example assumes a native Choice question with `supported`, `refuted` and `insufficient`
|
|
@@ -71,8 +71,8 @@ not a requirement imposed on other evaluators.
|
|
|
71
71
|
|
|
72
72
|
## Optional behavior recipe
|
|
73
73
|
|
|
74
|
-
`examples/
|
|
75
|
-
`behaviorReviewAnalyst`, which composes the existing `
|
|
74
|
+
`examples/systemone-behavior-review.ts` provides replaceable question definitions and
|
|
75
|
+
`behaviorReviewAnalyst`, which composes the existing `systemOneAnalyst`. Copy/adapt the recipe in an
|
|
76
76
|
application; it is not a default policy installed by importing the package. Select only relevant
|
|
77
77
|
checks and provide thresholds, severity, exact policy and evidence for every selected check.
|
|
78
78
|
|
|
@@ -126,7 +126,7 @@ probability with its frozen binary threshold rule; it must not silently drop the
|
|
|
126
126
|
`/rl` already has `detectRewardHacking` for reward divergence, distribution shift, reward
|
|
127
127
|
disagreement and judge drift. Those diagnostic scores are not probabilities. Keep the raw,
|
|
128
128
|
ungated proxy and independently measured task outcome so a previous rejection does not hide the
|
|
129
|
-
disagreement. Do not substitute the
|
|
129
|
+
disagreement. Do not substitute the System One monitor as the only independent outcome.
|
|
130
130
|
|
|
131
131
|
The analyst system already has deterministic control-integrity and behavioral/efficiency
|
|
132
132
|
checks, plus deep trace analyst definitions such as intent divergence and knowledge poisoning.
|
|
@@ -134,7 +134,7 @@ Run exact checks for facts code can establish; use a bounded semantic review for
|
|
|
134
134
|
escalate selected cases to the existing deeper analyst. None is replaced by this recipe.
|
|
135
135
|
|
|
136
136
|
A prepared review can be a custom registry input or built from a filtered trace store. Existing
|
|
137
|
-
`asAnalyst`/`
|
|
137
|
+
`asAnalyst`/`systemOneAnalyst` forward the shared paid-call account, cancellation and deadline. The
|
|
138
138
|
recipe verifies persisted definition identity before inference, records the native observation
|
|
139
139
|
before mapping, and returns ordinary findings. Graphs consume these through their existing
|
|
140
140
|
analyst-registry adapter. No new supervisor, scheduler, ledger, or trace database is introduced.
|
|
@@ -158,7 +158,7 @@ It decides `pass`, `hack` or `unknown` for one claim from signals the caller com
|
|
|
158
158
|
- `farmingSignal`: one value of a parameter the scope declares, repeated across a lane's sweep of claims.
|
|
159
159
|
- `graderReferenceSignal`: the claim page justifies itself by the checker or by the printed letter.
|
|
160
160
|
- `refereeSignal`: a model reads the source and the claim's parameters, never the claimant's prose (`sourceScopeRefereeMessages`, `parseSourceScopeReferee`).
|
|
161
|
-
- `
|
|
161
|
+
- `systemOneSignal`: typed questions over the same brief (`claimIntegritySystemOneReview`). The caller supplies calibrated thresholds.
|
|
162
162
|
- `monitorSignal`: `askTraceQuestions` with `claimIntegrityTraceQuestions`, on escalation only.
|
|
163
163
|
- `checkerReadSignal`: context for a reader; it never moves a verdict.
|
|
164
164
|
|
|
@@ -172,7 +172,7 @@ Keep verdicts, scopes and questions away from the agent that filed the claim: an
|
|
|
172
172
|
## Security and validation boundaries
|
|
173
173
|
|
|
174
174
|
TypeSafe's Jev 1.13 documentation explicitly warns that adversarial state content can move the
|
|
175
|
-
answer. Delimiters and explicit criteria are useful instructions, not a proof of injection
|
|
175
|
+
answer; treat every System One model the same way. Delimiters and explicit criteria are useful instructions, not a proof of injection
|
|
176
176
|
resistance. These monitors are advisory evidence, not the sole authorization/safety barrier.
|
|
177
177
|
Trusted policy and references must come from outside the evaluated agent; tenant scope, redaction,
|
|
178
178
|
credential handling and controls remain with existing application/security owners.
|
|
@@ -192,7 +192,7 @@ attacks, not only the worker under a fixed monitor.
|
|
|
192
192
|
|
|
193
193
|
The new tests exercise pure mapping, canonical identity, metadata isolation and real
|
|
194
194
|
Eval/AnalystRegistry accounting composition with an injected model boundary. They do not prove
|
|
195
|
-
that
|
|
195
|
+
that a System One model detects these behaviors correctly or is robust against adversarial prompts. No live
|
|
196
196
|
provider, product UI rollout, runtime hook installation or production enforcement is added here.
|
|
197
197
|
|
|
198
198
|
Sources reviewed: https://docs.typesafe.ai/confidence and
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Ten useful integrations, one evaluator
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
answers.
|
|
3
|
+
A System One model evaluates bounded questions. Applications supply state, questions and the policy interpreting
|
|
4
|
+
answers. systemOneEvaluator handles the native model; createEvaluator supports other classifiers.
|
|
5
5
|
asJudge and asAnalyst adapt the same operation to existing consumers.
|
|
6
6
|
|
|
7
7
|
| Use | Outcome | Existing execution path |
|
|
@@ -10,12 +10,12 @@ asJudge and asAnalyst adapt the same operation to existing consumers.
|
|
|
10
10
|
| Recommendation triage | Prioritize evidence-linked Intelligence recommendations | Product triage selects existing engine findings and preserves their identities. |
|
|
11
11
|
| Trace diagnosis | Narrow likely failures before deeper investigation | AnalystRegistry, scoped evidence, and the existing deep analyst. |
|
|
12
12
|
| Behavioral review | Flag observable policy concerns, not hidden intent | Optional behavior-review recipe plus deterministic reward/control checks. |
|
|
13
|
-
| Rubric judging | Compare product or agent outputs |
|
|
13
|
+
| Rubric judging | Compare product or agent outputs | systemOneJudge/asJudge with caller mappings and independent final assessment. |
|
|
14
14
|
| Context selection | Select useful retrieved evidence | Native assessments, then deterministic packing with mandatory context protected. |
|
|
15
15
|
| Skill selection | Select from authorized capabilities | Caller alternatives, including no-selection; existing runtime executes actions. |
|
|
16
16
|
| Next verification | Choose a test, retrieval or review that resolves uncertainty | Existing graph checkpoints and awaited boundaries, not new permissions. |
|
|
17
17
|
| Candidate selection | Choose among generated plans or artifacts | Existing matrix/graph, exact checks first, semantic assessment second. |
|
|
18
|
-
| Configuration experiments | Compare questions, context, models and thresholds | examples/
|
|
18
|
+
| Configuration experiments | Compare questions, context, models and thresholds | examples/systemone-decision-benchmark.ts using runAgentMatrix and cost receipts. |
|
|
19
19
|
|
|
20
20
|
These are compositions, not ten new frameworks. Product triage and the benchmark recipe are concrete
|
|
21
21
|
adoption work; other rows reuse existing APIs or identify application policies to configure. An API
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Inspect recorded
|
|
1
|
+
# Inspect recorded System One evidence without new inference
|
|
2
2
|
|
|
3
3
|
The historical decoder from #897 is already the protocol owner. This example
|
|
4
4
|
uses that implementation; it adds no parser, SDK, ledger, database or replay
|
|
@@ -7,7 +7,7 @@ policy.
|
|
|
7
7
|
From a repository checkout with dependencies installed:
|
|
8
8
|
|
|
9
9
|
```sh
|
|
10
|
-
pnpm exec tsx examples/
|
|
10
|
+
pnpm exec tsx examples/systemone-replay.ts request.json response.json
|
|
11
11
|
```
|
|
12
12
|
|
|
13
13
|
Supply the original native request and response as separate JSON files. The
|
|
@@ -18,8 +18,8 @@ metadata such as generation IDs and decimal receipt strings. Neither input file
|
|
|
18
18
|
is rewritten. This is offline inspection, not replay of a paid provider request.
|
|
19
19
|
|
|
20
20
|
Historical null state and null score levels remain readable through
|
|
21
|
-
`
|
|
22
|
-
`
|
|
21
|
+
`parseSystemOneRecordedRequest`. They are still invalid for new paid calls through
|
|
22
|
+
`parseSystemOneRequest`. Do not cast a historical request to the live type, or change
|
|
23
23
|
its stored rubric just to satisfy today's admission schema.
|
|
24
24
|
|
|
25
25
|
A mismatch, unreadable file or malformed record returns exit code 1 with a
|
|
@@ -30,9 +30,9 @@ terminal or output destination and retain its existing access controls.
|
|
|
30
30
|
The same helper is importable from the example:
|
|
31
31
|
|
|
32
32
|
```ts
|
|
33
|
-
import {
|
|
33
|
+
import { inspectSystemOneFiles } from './examples/systemone-replay'
|
|
34
34
|
|
|
35
|
-
const observation = await
|
|
35
|
+
const observation = await inspectSystemOneFiles('request.json', 'response.json')
|
|
36
36
|
// Apply an application-owned deterministic mapping to the retained answers.
|
|
37
37
|
```
|
|
38
38
|
|
|
@@ -1,35 +1,50 @@
|
|
|
1
|
-
#
|
|
1
|
+
# System One evaluations: products, judges, and analysts
|
|
2
|
+
|
|
3
|
+
System One is a typed decision protocol: a request carries `state` and named questions (noul
|
|
4
|
+
yes/no, choice, or score rubric), and the answer is a probability for every allowed option.
|
|
5
|
+
TypeSafe published it with Jev; Cloudflare Workers AI (Clef, Clef-flash) and Perplexity's
|
|
6
|
+
Decisions API (pplx-decider) serve models on the same wire format. This package owns what a
|
|
7
|
+
valid result means. It names no endpoint, credential or model. Tangle Router owns the provider
|
|
8
|
+
routes behind `POST /v1/systemone`, with their prices, limits and served-model checks, so a
|
|
9
|
+
consumer changes models by changing one id.
|
|
2
10
|
|
|
3
11
|
Configure a transport once. Supply your own model, state, and questions on every
|
|
4
12
|
call. Interpret the answers in ordinary code; no campaign or agent is required.
|
|
5
13
|
|
|
6
14
|
```ts
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
15
|
+
import { systemOneEvaluator } from '@tangle-network/agent-eval/systemone'
|
|
16
|
+
|
|
17
|
+
const evaluate = systemOneEvaluator({
|
|
18
|
+
evaluate: async (request, { signal, idempotencyKey }) => {
|
|
19
|
+
const response = await fetch(`${routerOrigin}/v1/systemone`, {
|
|
20
|
+
method: 'POST',
|
|
21
|
+
signal,
|
|
22
|
+
headers: {
|
|
23
|
+
Authorization: `Bearer ${scopedRouterKey}`,
|
|
24
|
+
'Content-Type': 'application/json',
|
|
25
|
+
'Idempotency-Key': idempotencyKey,
|
|
26
|
+
},
|
|
27
|
+
body: JSON.stringify(request),
|
|
28
|
+
})
|
|
29
|
+
if (!response.ok) throw new Error(`System One request failed: ${response.status}`)
|
|
30
|
+
return response.json()
|
|
31
|
+
},
|
|
21
32
|
pricing: reviewedTokenPricing, // An estimate, not a provider-observed charge.
|
|
22
33
|
})
|
|
23
34
|
|
|
24
35
|
const result = await evaluate({
|
|
25
|
-
model: selectedModel,
|
|
36
|
+
model: selectedModel, // a native_evaluation id from Router's /v1/models
|
|
26
37
|
state: { requirements, evidence },
|
|
27
38
|
questions: {
|
|
28
|
-
supported: noul
|
|
29
|
-
next:
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
39
|
+
supported: { type: 'noul', instructions: 'Does the evidence support the claim?' },
|
|
40
|
+
next: {
|
|
41
|
+
type: 'choice',
|
|
42
|
+
instructions: 'Choose the next step using the evidence',
|
|
43
|
+
criteria: {
|
|
44
|
+
inspect: 'Evidence is missing or contradictory',
|
|
45
|
+
finish: 'The requirements are satisfied',
|
|
46
|
+
},
|
|
47
|
+
},
|
|
33
48
|
},
|
|
34
49
|
}, { signal, costLedger })
|
|
35
50
|
|
|
@@ -38,21 +53,56 @@ result.value.answers.next.probabilities.inspect // number
|
|
|
38
53
|
result.receipt // Existing CostLedger receipt, separate from native answers.
|
|
39
54
|
```
|
|
40
55
|
|
|
41
|
-
The named configuration and evidence values belong to the application.
|
|
42
|
-
|
|
56
|
+
The named configuration and evidence values belong to the application. TypeSafe's SDK
|
|
57
|
+
builders produce the same JSON and are optional. Structured criteria, null instructions and choice or noul
|
|
43
58
|
descriptions, and omitted optional fields are supported. `state` and each score
|
|
44
|
-
level must be text or JSON; TypeSafe rejects null there, so the parser does too.
|
|
59
|
+
level must be text or JSON; TypeSafe rejects null there, so the parser does too. Providers
|
|
60
|
+
add their own limits: Clef requires non-null `instructions`, 1–64 questions with ids matching
|
|
61
|
+
`^[A-Za-z0-9_.-]{1,100}$`, and 2–255 choice options. Static TypeScript questions preserve
|
|
45
62
|
exact answer names and choice-label unions. Runtime JSON must be validated; its
|
|
46
63
|
labels cannot become compile-time literals automatically.
|
|
47
64
|
|
|
48
|
-
##
|
|
65
|
+
## Providers, rounding and confidence
|
|
66
|
+
|
|
67
|
+
The wire format is the same for every provider; how each reports its numbers is not. A
|
|
68
|
+
provider's raw result goes through `normalizeSystemOneResult(raw, request, profile)` and then
|
|
69
|
+
the strict `parseSystemOneResult`. The route owner supplies the profile.
|
|
70
|
+
|
|
71
|
+
| Recorded 2026-10-04 | Rounding | `confidence` as returned | Profile |
|
|
72
|
+
| --- | --- | --- | --- |
|
|
73
|
+
| TypeSafe `jev-1.13.0` | 2 decimals. Probabilities sum to 1; the score is not always their expectation (8.79 against 8.86). | `systemOneConfidence` (153 of 153 recorded answers within rounding) | `{ decimals: 2, confidence: 'canonical' }` |
|
|
74
|
+
| Cloudflare `clef`, `clef-flash` | 4 decimals, each probability rounded on its own. Wrapped in Workers AI's `{ result, success, errors, messages }`. | (Σp² − 1/n)/(1 − 1/n) | `{ decimals: 4, confidence: 'provider-defined' }` |
|
|
75
|
+
|
|
76
|
+
The recordings are `tests/fixtures/systemone/providers-2026-10-04.json`, checked by
|
|
77
|
+
`tests/systemone-providers.test.ts`. Without normalization, the strict parser refuses both
|
|
78
|
+
recorded TypeSafe score answers and both recorded Clef score answers.
|
|
79
|
+
|
|
80
|
+
- An answer that already passes the strict checks keeps its probabilities and score.
|
|
81
|
+
- An answer that fails them only within the declared rounding is repaired. A distribution whose
|
|
82
|
+
sum is more than 1e-4 from 1 is divided by its sum. A score becomes the expectation of the
|
|
83
|
+
distribution.
|
|
84
|
+
- Error beyond the rounding throws `SystemOneResponseError`; it is a provider defect. The
|
|
85
|
+
bounds are n half-units for the sum and half a unit plus one unit per level index for the score.
|
|
86
|
+
- With `confidence: 'provider-defined'`, the provider's value moves to `provider_confidence` and
|
|
87
|
+
`confidence` becomes `systemOneConfidence(probabilities)`, so every route reports the same
|
|
88
|
+
function of its probabilities: TypeSafe's published formulas
|
|
89
|
+
(https://docs.typesafe.ai/confidence).
|
|
90
|
+
- Noul answers never change. `noul` is the probability of yes for every provider.
|
|
91
|
+
|
|
92
|
+
Probabilities are not made equivalent across models. A Clef 0.8 and a Jev 0.8 are different
|
|
93
|
+
statements, so a threshold belongs to one model: change `{ model, threshold }` together and audit
|
|
94
|
+
the pair with `auditProbabilityPolicy` against independent outcomes. `brierScore` and
|
|
95
|
+
`calibrationFromPairs` (with `range: { lo: 0, hi: 1 }`; `binning: 'equal-frequency'` for
|
|
96
|
+
equal-mass bins) in `/meta-eval` report calibration on the same pairs.
|
|
97
|
+
|
|
98
|
+
## Any classifier, not only System One
|
|
49
99
|
|
|
50
100
|
`createEvaluator`, `asJudge`, and `asAnalyst` are provider-independent exports from
|
|
51
|
-
`@tangle-network/agent-eval/evaluation` (also re-exported from `/
|
|
101
|
+
`@tangle-network/agent-eval/evaluation` (also re-exported from `/systemone`).
|
|
52
102
|
`createEvaluator({ execute, receipt })` meters a typed asynchronous function.
|
|
53
|
-
It does not require a
|
|
103
|
+
It does not require a System One response, probability output, prompt, or provider SDK.
|
|
54
104
|
|
|
55
|
-
`
|
|
105
|
+
`systemOneEvaluator` adds native protocol validation and optional `acceptModel(requested,
|
|
56
106
|
served)` policy. It preserves the reported model instead of guessing alias rules.
|
|
57
107
|
Router independently enforces its supported model and pricing policy.
|
|
58
108
|
|
|
@@ -69,13 +119,13 @@ and analyst context. `map(value, input, analystContext)` creates evidence-backed
|
|
|
69
119
|
findings. Use the existing `makeFinding` factory and real event/span/artifact references.
|
|
70
120
|
An empty finding list is valid; an inference failure is not a clean review.
|
|
71
121
|
|
|
72
|
-
The optional `
|
|
122
|
+
The optional `systemOneJudge` and `systemOneAnalyst` conveniences accept static questions or a
|
|
73
123
|
question-producing function. State renderers may be async. Judge renderers receive
|
|
74
124
|
the evaluation context; analyst renderers and finding mappers receive the effective
|
|
75
125
|
cancellation/deadline signal. Rendering must honor that signal during external work.
|
|
76
126
|
Both conveniences accept a `record` callback for the complete observation.
|
|
77
127
|
|
|
78
|
-
For a numeric rubric, `
|
|
128
|
+
For a numeric rubric, `systemOneJudge` defaults to `normalizedSystemOneScore`: equally spaced
|
|
79
129
|
score expectations on [0,1], noul values unchanged, and an optional weighted mean.
|
|
80
130
|
It emits no canned notes. Supply `map` for choice utilities, different scales, or
|
|
81
131
|
critical-failure policies. Dynamic questions and custom mappings require explicit
|
|
@@ -98,7 +148,7 @@ none is supplied, an explicit `budgetUsd` creates an account for that invocation
|
|
|
98
148
|
Otherwise the evaluator keeps its configured ledger, including its cumulative cap
|
|
99
149
|
across direct calls, judges, and analysts. An absent analyst budget never creates a
|
|
100
150
|
fresh unlimited account that overrides the configured one. This precedence is the
|
|
101
|
-
same for
|
|
151
|
+
same for System One and other classifiers. A per-analysis budget replaces the default
|
|
102
152
|
account; it is not an additional child limit. Pass the shared run ledger explicitly
|
|
103
153
|
when the whole run must conserve one authority. Only trusted host code should choose
|
|
104
154
|
or replace that authority.
|
|
@@ -132,10 +182,11 @@ confidence as demonstrated calibration or a classification as proof of root caus
|
|
|
132
182
|
|
|
133
183
|
## Checks
|
|
134
184
|
|
|
135
|
-
Run `pnpm exec vitest run tests/
|
|
185
|
+
Run `pnpm exec vitest run tests/systemone*.test.ts tests/evaluation-authority.test.ts`,
|
|
136
186
|
`pnpm typecheck`, `pnpm build`, and `pnpm verify:package`. These are contract and
|
|
137
187
|
execution checks, not live-provider quality or deployment evidence. The product-integrity
|
|
138
188
|
rubric lives in `examples/`, not the generic implementation, and is not a default policy.
|
|
139
189
|
|
|
140
190
|
Protocol: https://docs.typesafe.ai/api
|
|
141
|
-
|
|
191
|
+
Clef: https://developers.cloudflare.com/workers-ai/models/clef/
|
|
192
|
+
SDK (optional builders): https://github.com/typesafe-ai/typesafe-sdk-js
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-eval",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.206.0",
|
|
4
4
|
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-eval#readme",
|
|
6
6
|
"repository": {
|
|
@@ -19,10 +19,10 @@
|
|
|
19
19
|
"import": "./dist/index.js",
|
|
20
20
|
"default": "./dist/index.js"
|
|
21
21
|
},
|
|
22
|
-
"./
|
|
23
|
-
"types": "./dist/
|
|
24
|
-
"import": "./dist/
|
|
25
|
-
"default": "./dist/
|
|
22
|
+
"./systemone": {
|
|
23
|
+
"types": "./dist/systemone.d.ts",
|
|
24
|
+
"import": "./dist/systemone.js",
|
|
25
|
+
"default": "./dist/systemone.js"
|
|
26
26
|
},
|
|
27
27
|
"./profile-cell": {
|
|
28
28
|
"types": "./dist/profile-cell.d.ts",
|
|
@@ -177,10 +177,10 @@
|
|
|
177
177
|
"import": "./dist/evaluation.js",
|
|
178
178
|
"default": "./dist/evaluation.js"
|
|
179
179
|
},
|
|
180
|
-
"./
|
|
181
|
-
"types": "./dist/
|
|
182
|
-
"import": "./dist/
|
|
183
|
-
"default": "./dist/
|
|
180
|
+
"./systemone/protocol": {
|
|
181
|
+
"types": "./dist/systemone-protocol.d.ts",
|
|
182
|
+
"import": "./dist/systemone-protocol.js",
|
|
183
|
+
"default": "./dist/systemone-protocol.js"
|
|
184
184
|
}
|
|
185
185
|
},
|
|
186
186
|
"bin": {
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"claude-jsonl-DoQJBCVR.js","names":[],"sources":["../src/rollout/readers/claude-jsonl.ts"],"sourcesContent":["/**\n * Backfill reader over Claude Code project transcripts\n * (~/.claude/projects/<cwd-slug>/<sessionId>.jsonl) → canonical\n * chat-with-tools messages plus per-session token usage.\n *\n * Transcript lines consumed: type:\"user\" (string content or content blocks —\n * text + tool_result) and type:\"assistant\" (content blocks — thinking, text,\n * tool_use; message.usage carries tokens). Sidechain lines (isSidechain=true,\n * subagent threads) are separate invocations and are excluded from the main\n * transcript. Everything else (queue-operation, attachment, last-prompt…) is\n * transport metadata, not conversation.\n */\n\nimport { readdir, readFile } from 'node:fs/promises'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport type { ChatMessage, ChatToolCall } from '../schema'\n\nexport const DEFAULT_CLAUDE_PROJECTS_DIR = join(homedir(), '.claude', 'projects')\n\n/** Claude Code's project-directory slug for a working directory. */\nexport function claudeProjectSlug(cwd: string): string {\n return cwd.replace(/[^a-zA-Z0-9-]/g, '-')\n}\n\nexport interface ClaudeTranscriptRef {\n sessionId: string\n path: string\n}\n\n/** Transcript files recorded for sessions launched from `cwd`. */\nexport async function findClaudeTranscripts(\n cwd: string,\n projectsDir: string = DEFAULT_CLAUDE_PROJECTS_DIR,\n): Promise<ClaudeTranscriptRef[]> {\n const dir = join(projectsDir, claudeProjectSlug(cwd))\n const names = await readdir(dir).catch(() => [])\n return names\n .filter((n) => n.endsWith('.jsonl'))\n .sort()\n .map((n) => ({ sessionId: n.replace(/\\.jsonl$/, ''), path: join(dir, n) }))\n}\n\nexport interface ClaudeUsageTotals {\n tokensIn: number | null\n tokensOut: number | null\n cacheRead: number | null\n cacheWrite: number | null\n}\n\nexport interface ClaudeTranscript {\n messages: ChatMessage[]\n usage: ClaudeUsageTotals\n /** Timestamp of the first conversation line; null = empty transcript. */\n startedAt: string | null\n endedAt: string | null\n model: string | null\n /** Every parse or usage gap retained beside this semantic projection. */\n gaps: string[]\n}\n\nconst isRecord = (v: unknown): v is Record<string, unknown> =>\n typeof v === 'object' && v !== null && !Array.isArray(v)\n\n/**\n * One conversation line of a transcript, still in Claude Code's own shape.\n *\n * This is the single line-level parse of the format. `readClaudeTranscript`\n * projects it to canonical messages + usage; the supervision-tree reader\n * (`src/supervisor-run/claude-code-reader.ts`) projects the SAME entries to\n * spawn/settle/steer instants. Two projections, one parser — a second\n * transcript parser is how the two views silently disagree.\n */\nexport interface ClaudeEntry {\n readonly type: 'user' | 'assistant'\n /** ISO instant of the line; null when the line carried none. */\n readonly timestamp: string | null\n /** The Anthropic message body (`role`, `content`, `model`, `usage`). */\n readonly message: Record<string, unknown>\n /** Claude Code's structured tool result, when the line carries one. */\n readonly toolUseResult: unknown\n /** True on subagent threads — a separate invocation, not this transcript's turn. */\n readonly isSidechain: boolean\n /** Subagent id Claude Code stamps on sidechain lines; null on main-thread lines. */\n readonly agentId: string | null\n}\n\n/** Parse transcript jsonl text into conversation lines. Non-conversation lines are dropped. */\nexport function parseClaudeEntries(raw: string, onGap?: (gap: string) => void): ClaudeEntry[] {\n const out: ClaudeEntry[] = []\n for (const [index, line] of raw.split('\\n').entries()) {\n if (!line.trim()) continue\n const gap = (reason: string): void => {\n const detail = `line ${index + 1}: ${reason}`\n if (onGap) onGap(detail)\n else throw new Error(`Claude transcript ${detail}`)\n }\n let parsed: unknown\n try {\n parsed = JSON.parse(line)\n } catch {\n gap('malformed JSON')\n continue\n }\n if (!isRecord(parsed)) {\n gap('JSON value is not an object')\n continue\n }\n const entry = parsed\n if (entry.type !== 'user' && entry.type !== 'assistant') continue\n const message = entry.message\n if (!isRecord(message)) {\n gap('conversation message is missing')\n continue\n }\n out.push({\n type: entry.type,\n timestamp: typeof entry.timestamp === 'string' ? entry.timestamp : null,\n message,\n toolUseResult: entry.toolUseResult,\n isSidechain: entry.isSidechain === true,\n agentId: typeof entry.agentId === 'string' ? entry.agentId : null,\n })\n }\n return out\n}\n\nexport interface ReadClaudeTranscriptOptions {\n /**\n * Read the sidechain (subagent) thread instead of skipping it. Subagent\n * transcripts under `<session>/subagents/agent-<id>.jsonl` are sidechain\n * lines end to end, so their usage is invisible without this.\n */\n readonly includeSidechain?: boolean\n /** Parser diagnostics from the retained source; used by the shared reader. */\n readonly sourceGaps?: readonly string[]\n}\n\nfunction blockText(content: unknown): string {\n if (typeof content === 'string') return content\n if (!Array.isArray(content)) return ''\n return content\n .filter(\n (b): b is Record<string, unknown> =>\n isRecord(b) && b.type === 'text' && typeof b.text === 'string',\n )\n .map((b) => b.text as string)\n .join('\\n')\n}\n\n/** Parse one transcript jsonl into canonical messages + usage totals. */\nexport async function readClaudeTranscript(\n path: string,\n options: ReadClaudeTranscriptOptions = {},\n): Promise<ClaudeTranscript> {\n const gaps: string[] = []\n const entries = parseClaudeEntries(await readFile(path, 'utf8'), (gap) => gaps.push(gap))\n return transcriptFromEntries(entries, { ...options, sourceGaps: gaps })\n}\n\n/** The messages+usage projection of already-parsed entries. */\nexport function transcriptFromEntries(\n entries: readonly ClaudeEntry[],\n options: ReadClaudeTranscriptOptions = {},\n): ClaudeTranscript {\n const wantSidechain = options.includeSidechain === true\n const messages: ChatMessage[] = []\n const usage: ClaudeUsageTotals = { tokensIn: 0, tokensOut: 0, cacheRead: 0, cacheWrite: 0 }\n const gaps = [...(options.sourceGaps ?? [])]\n let usageObserved = false\n let startedAt: string | null = null\n let endedAt: string | null = null\n let model: string | null = null\n // Claude Code writes one jsonl line PER CONTENT BLOCK of an API message,\n // repeating message.id and usage on each — merge blocks into one canonical\n // assistant turn and count usage once per API message id.\n let lastAssistantApiId: string | null = null\n let lastAssistantIndex = -1\n\n for (const entry of entries) {\n if (entry.isSidechain !== wantSidechain) continue\n const message = entry.message\n if (entry.timestamp !== null) {\n if (startedAt === null) startedAt = entry.timestamp\n endedAt = entry.timestamp\n }\n\n if (entry.type === 'user') {\n lastAssistantApiId = null\n lastAssistantIndex = -1\n const content = message.content\n if (typeof content === 'string') {\n messages.push({ role: 'user', content })\n continue\n }\n if (!Array.isArray(content)) continue\n // A user line may interleave tool_result blocks (answers to the prior\n // assistant tool_use) with plain text; preserve order.\n let userText = ''\n for (const block of content) {\n if (!isRecord(block)) continue\n if (block.type === 'tool_result' && typeof block.tool_use_id === 'string') {\n messages.push({\n role: 'tool',\n tool_call_id: block.tool_use_id,\n content:\n blockText(block.content) || (typeof block.content === 'string' ? block.content : ''),\n })\n } else if (block.type === 'text' && typeof block.text === 'string') {\n userText += (userText.length > 0 ? '\\n' : '') + block.text\n }\n }\n if (userText.length > 0) messages.push({ role: 'user', content: userText })\n continue\n }\n\n // assistant\n if (typeof message.model === 'string') model = message.model\n const apiId = typeof message.id === 'string' ? message.id : null\n const continuesTurn = apiId !== null && apiId === lastAssistantApiId && lastAssistantIndex >= 0\n const msgUsage = message.usage\n if (!continuesTurn) {\n usageObserved = true\n const fields = [\n ['tokensIn', 'input_tokens'],\n ['tokensOut', 'output_tokens'],\n ['cacheRead', 'cache_read_input_tokens'],\n ['cacheWrite', 'cache_creation_input_tokens'],\n ] as const\n for (const [target, source] of fields) {\n const value = isRecord(msgUsage) ? msgUsage[source] : undefined\n if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {\n usage[target] = null\n gaps.push(`assistant ${apiId ?? 'without id'}: ${source} unavailable`)\n } else if (usage[target] !== null) {\n usage[target] += value\n }\n }\n }\n const content = message.content\n if (!Array.isArray(content)) continue\n let reasoning = ''\n let text = ''\n const toolCalls: ChatToolCall[] = []\n for (const block of content) {\n if (!isRecord(block)) continue\n if (\n block.type === 'thinking' &&\n typeof block.thinking === 'string' &&\n block.thinking.length > 0\n ) {\n reasoning += (reasoning.length > 0 ? '\\n' : '') + block.thinking\n } else if (block.type === 'text' && typeof block.text === 'string') {\n text += (text.length > 0 ? '\\n' : '') + block.text\n } else if (block.type === 'tool_use' && typeof block.id === 'string') {\n toolCalls.push({\n id: block.id,\n type: 'function',\n function: {\n name: typeof block.name === 'string' ? block.name : 'unknown',\n arguments: JSON.stringify(block.input ?? {}),\n },\n })\n }\n }\n if (reasoning.length === 0 && text.length === 0 && toolCalls.length === 0) continue\n if (continuesTurn) {\n const prev = messages[lastAssistantIndex]!\n if (text.length > 0) prev.content = prev.content === null ? text : `${prev.content}\\n${text}`\n if (reasoning.length > 0) {\n prev.reasoning_content =\n prev.reasoning_content === undefined\n ? reasoning\n : `${prev.reasoning_content}\\n${reasoning}`\n }\n if (toolCalls.length > 0) prev.tool_calls = [...(prev.tool_calls ?? []), ...toolCalls]\n continue\n }\n messages.push({\n role: 'assistant',\n content: text.length > 0 ? text : null,\n ...(reasoning.length > 0 ? { reasoning_content: reasoning } : {}),\n ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),\n })\n lastAssistantApiId = apiId\n lastAssistantIndex = messages.length - 1\n }\n\n if (!usageObserved) {\n usage.tokensIn = null\n usage.tokensOut = null\n usage.cacheRead = null\n usage.cacheWrite = null\n }\n return { messages, usage, startedAt, endedAt, model, gaps }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAkBA,MAAa,8BAA8B,KAAK,QAAQ,GAAG,WAAW,UAAU;;AAGhF,SAAgB,kBAAkB,KAAqB;CACrD,OAAO,IAAI,QAAQ,kBAAkB,GAAG;AAC1C;;AAQA,eAAsB,sBACpB,KACA,cAAsB,6BACU;CAChC,MAAM,MAAM,KAAK,aAAa,kBAAkB,GAAG,CAAC;CAEpD,QAAO,MADa,QAAQ,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,EAAA,CAE5C,QAAQ,MAAM,EAAE,SAAS,QAAQ,CAAC,CAAC,CACnC,KAAK,CAAC,CACN,KAAK,OAAO;EAAE,WAAW,EAAE,QAAQ,YAAY,EAAE;EAAG,MAAM,KAAK,KAAK,CAAC;CAAE,EAAE;AAC9E;AAoBA,MAAM,YAAY,MAChB,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;;AA0BzD,SAAgB,mBAAmB,KAAa,OAA8C;CAC5F,MAAM,MAAqB,CAAC;CAC5B,KAAK,MAAM,CAAC,OAAO,SAAS,IAAI,MAAM,IAAI,CAAC,CAAC,QAAQ,GAAG;EACrD,IAAI,CAAC,KAAK,KAAK,GAAG;EAClB,MAAM,OAAO,WAAyB;GACpC,MAAM,SAAS,QAAQ,QAAQ,EAAE,IAAI;GACrC,IAAI,OAAO,MAAM,MAAM;QAClB,MAAM,IAAI,MAAM,qBAAqB,QAAQ;EACpD;EACA,IAAI;EACJ,IAAI;GACF,SAAS,KAAK,MAAM,IAAI;EAC1B,QAAQ;GACN,IAAI,gBAAgB;GACpB;EACF;EACA,IAAI,CAAC,SAAS,MAAM,GAAG;GACrB,IAAI,6BAA6B;GACjC;EACF;EACA,MAAM,QAAQ;EACd,IAAI,MAAM,SAAS,UAAU,MAAM,SAAS,aAAa;EACzD,MAAM,UAAU,MAAM;EACtB,IAAI,CAAC,SAAS,OAAO,GAAG;GACtB,IAAI,iCAAiC;GACrC;EACF;EACA,IAAI,KAAK;GACP,MAAM,MAAM;GACZ,WAAW,OAAO,MAAM,cAAc,WAAW,MAAM,YAAY;GACnE;GACA,eAAe,MAAM;GACrB,aAAa,MAAM,gBAAgB;GACnC,SAAS,OAAO,MAAM,YAAY,WAAW,MAAM,UAAU;EAC/D,CAAC;CACH;CACA,OAAO;AACT;AAaA,SAAS,UAAU,SAA0B;CAC3C,IAAI,OAAO,YAAY,UAAU,OAAO;CACxC,IAAI,CAAC,MAAM,QAAQ,OAAO,GAAG,OAAO;CACpC,OAAO,QACJ,QACE,MACC,SAAS,CAAC,KAAK,EAAE,SAAS,UAAU,OAAO,EAAE,SAAS,QAC1D,CAAC,CACA,KAAK,MAAM,EAAE,IAAc,CAAC,CAC5B,KAAK,IAAI;AACd;;AAGA,eAAsB,qBACpB,MACA,UAAuC,CAAC,GACb;CAC3B,MAAM,OAAiB,CAAC;CAExB,OAAO,sBADS,mBAAmB,MAAM,SAAS,MAAM,MAAM,IAAI,QAAQ,KAAK,KAAK,GAAG,CACpD,GAAG;EAAE,GAAG;EAAS,YAAY;CAAK,CAAC;AACxE;;AAGA,SAAgB,sBACd,SACA,UAAuC,CAAC,GACtB;CAClB,MAAM,gBAAgB,QAAQ,qBAAqB;CACnD,MAAM,WAA0B,CAAC;CACjC,MAAM,QAA2B;EAAE,UAAU;EAAG,WAAW;EAAG,WAAW;EAAG,YAAY;CAAE;CAC1F,MAAM,OAAO,CAAC,GAAI,QAAQ,cAAc,CAAC,CAAE;CAC3C,IAAI,gBAAgB;CACpB,IAAI,YAA2B;CAC/B,IAAI,UAAyB;CAC7B,IAAI,QAAuB;CAI3B,IAAI,qBAAoC;CACxC,IAAI,qBAAqB;CAEzB,KAAK,MAAM,SAAS,SAAS;EAC3B,IAAI,MAAM,gBAAgB,eAAe;EACzC,MAAM,UAAU,MAAM;EACtB,IAAI,MAAM,cAAc,MAAM;GAC5B,IAAI,cAAc,MAAM,YAAY,MAAM;GAC1C,UAAU,MAAM;EAClB;EAEA,IAAI,MAAM,SAAS,QAAQ;GACzB,qBAAqB;GACrB,qBAAqB;GACrB,MAAM,UAAU,QAAQ;GACxB,IAAI,OAAO,YAAY,UAAU;IAC/B,SAAS,KAAK;KAAE,MAAM;KAAQ;IAAQ,CAAC;IACvC;GACF;GACA,IAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;GAG7B,IAAI,WAAW;GACf,KAAK,MAAM,SAAS,SAAS;IAC3B,IAAI,CAAC,SAAS,KAAK,GAAG;IACtB,IAAI,MAAM,SAAS,iBAAiB,OAAO,MAAM,gBAAgB,UAC/D,SAAS,KAAK;KACZ,MAAM;KACN,cAAc,MAAM;KACpB,SACE,UAAU,MAAM,OAAO,MAAM,OAAO,MAAM,YAAY,WAAW,MAAM,UAAU;IACrF,CAAC;SACI,IAAI,MAAM,SAAS,UAAU,OAAO,MAAM,SAAS,UACxD,aAAa,SAAS,SAAS,IAAI,OAAO,MAAM,MAAM;GAE1D;GACA,IAAI,SAAS,SAAS,GAAG,SAAS,KAAK;IAAE,MAAM;IAAQ,SAAS;GAAS,CAAC;GAC1E;EACF;EAGA,IAAI,OAAO,QAAQ,UAAU,UAAU,QAAQ,QAAQ;EACvD,MAAM,QAAQ,OAAO,QAAQ,OAAO,WAAW,QAAQ,KAAK;EAC5D,MAAM,gBAAgB,UAAU,QAAQ,UAAU,sBAAsB,sBAAsB;EAC9F,MAAM,WAAW,QAAQ;EACzB,IAAI,CAAC,eAAe;GAClB,gBAAgB;GAOhB,KAAK,MAAM,CAAC,QAAQ,WAAW;IAL7B,CAAC,YAAY,cAAc;IAC3B,CAAC,aAAa,eAAe;IAC7B,CAAC,aAAa,yBAAyB;IACvC,CAAC,cAAc,6BAA6B;GAEV,GAAG;IACrC,MAAM,QAAQ,SAAS,QAAQ,IAAI,SAAS,UAAU,KAAA;IACtD,IAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GAAG;KACrE,MAAM,UAAU;KAChB,KAAK,KAAK,aAAa,SAAS,aAAa,IAAI,OAAO,aAAa;IACvE,OAAO,IAAI,MAAM,YAAY,MAC3B,MAAM,WAAW;GAErB;EACF;EACA,MAAM,UAAU,QAAQ;EACxB,IAAI,CAAC,MAAM,QAAQ,OAAO,GAAG;EAC7B,IAAI,YAAY;EAChB,IAAI,OAAO;EACX,MAAM,YAA4B,CAAC;EACnC,KAAK,MAAM,SAAS,SAAS;GAC3B,IAAI,CAAC,SAAS,KAAK,GAAG;GACtB,IACE,MAAM,SAAS,cACf,OAAO,MAAM,aAAa,YAC1B,MAAM,SAAS,SAAS,GAExB,cAAc,UAAU,SAAS,IAAI,OAAO,MAAM,MAAM;QACnD,IAAI,MAAM,SAAS,UAAU,OAAO,MAAM,SAAS,UACxD,SAAS,KAAK,SAAS,IAAI,OAAO,MAAM,MAAM;QACzC,IAAI,MAAM,SAAS,cAAc,OAAO,MAAM,OAAO,UAC1D,UAAU,KAAK;IACb,IAAI,MAAM;IACV,MAAM;IACN,UAAU;KACR,MAAM,OAAO,MAAM,SAAS,WAAW,MAAM,OAAO;KACpD,WAAW,KAAK,UAAU,MAAM,SAAS,CAAC,CAAC;IAC7C;GACF,CAAC;EAEL;EACA,IAAI,UAAU,WAAW,KAAK,KAAK,WAAW,KAAK,UAAU,WAAW,GAAG;EAC3E,IAAI,eAAe;GACjB,MAAM,OAAO,SAAS;GACtB,IAAI,KAAK,SAAS,GAAG,KAAK,UAAU,KAAK,YAAY,OAAO,OAAO,GAAG,KAAK,QAAQ,IAAI;GACvF,IAAI,UAAU,SAAS,GACrB,KAAK,oBACH,KAAK,sBAAsB,KAAA,IACvB,YACA,GAAG,KAAK,kBAAkB,IAAI;GAEtC,IAAI,UAAU,SAAS,GAAG,KAAK,aAAa,CAAC,GAAI,KAAK,cAAc,CAAC,GAAI,GAAG,SAAS;GACrF;EACF;EACA,SAAS,KAAK;GACZ,MAAM;GACN,SAAS,KAAK,SAAS,IAAI,OAAO;GAClC,GAAI,UAAU,SAAS,IAAI,EAAE,mBAAmB,UAAU,IAAI,CAAC;GAC/D,GAAI,UAAU,SAAS,IAAI,EAAE,YAAY,UAAU,IAAI,CAAC;EAC1D,CAAC;EACD,qBAAqB;EACrB,qBAAqB,SAAS,SAAS;CACzC;CAEA,IAAI,CAAC,eAAe;EAClB,MAAM,WAAW;EACjB,MAAM,YAAY;EAClB,MAAM,YAAY;EAClB,MAAM,aAAa;CACrB;CACA,OAAO;EAAE;EAAU;EAAO;EAAW;EAAS;EAAO;CAAK;AAC5D"}
|
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
//#region src/jev-protocol.d.ts
|
|
2
|
-
/**
|
|
3
|
-
* Structural equivalents of TypeSafe's published request schema
|
|
4
|
-
* (https://api.typesafe.ai/openapi.json); callers can use its builders unchanged.
|
|
5
|
-
*/
|
|
6
|
-
type JevJson = string | number | boolean | null | JevJson[] | {
|
|
7
|
-
[key: string]: JevJson;
|
|
8
|
-
};
|
|
9
|
-
/** What TypeSafe accepts as `state` and as a score level: text or JSON, never null. */
|
|
10
|
-
type JevState = string | {
|
|
11
|
-
[key: string]: JevJson;
|
|
12
|
-
} | JevJson[];
|
|
13
|
-
/** Instructions and choice or noul descriptions, which TypeSafe also accepts as null. */
|
|
14
|
-
type JevDescription = JevState | null;
|
|
15
|
-
type JevScoreCriteria = readonly [JevState, JevState, ...JevState[]];
|
|
16
|
-
type JevQuestion = {
|
|
17
|
-
type: 'noul';
|
|
18
|
-
instructions?: JevDescription;
|
|
19
|
-
criteria?: {
|
|
20
|
-
true?: JevDescription;
|
|
21
|
-
false?: JevDescription;
|
|
22
|
-
} | null;
|
|
23
|
-
} | {
|
|
24
|
-
type: 'score';
|
|
25
|
-
instructions?: JevDescription;
|
|
26
|
-
criteria: JevScoreCriteria;
|
|
27
|
-
} | {
|
|
28
|
-
type: 'choice';
|
|
29
|
-
instructions?: JevDescription;
|
|
30
|
-
criteria: Record<string, JevDescription>;
|
|
31
|
-
};
|
|
32
|
-
type JevQuestions = Record<string, JevQuestion>;
|
|
33
|
-
interface JevRequest<Q extends JevQuestions = JevQuestions> {
|
|
34
|
-
model: string;
|
|
35
|
-
state: JevState;
|
|
36
|
-
questions: Q;
|
|
37
|
-
}
|
|
38
|
-
/** Historical Tangle observations accepted null state and score levels. Not a live request. */
|
|
39
|
-
type JevRecordedQuestion = Exclude<JevQuestion, {
|
|
40
|
-
type: 'score';
|
|
41
|
-
}> | {
|
|
42
|
-
type: 'score';
|
|
43
|
-
instructions?: JevDescription;
|
|
44
|
-
criteria: readonly [JevDescription, JevDescription, ...JevDescription[]];
|
|
45
|
-
};
|
|
46
|
-
type JevRecordedQuestions = Record<string, JevRecordedQuestion>;
|
|
47
|
-
interface JevRecordedRequest<Q extends JevRecordedQuestions = JevRecordedQuestions> {
|
|
48
|
-
model: string;
|
|
49
|
-
state: JevDescription;
|
|
50
|
-
questions: Q;
|
|
51
|
-
}
|
|
52
|
-
type ScoreKeys<T extends readonly JevDescription[]> = number extends T['length'] ? number : Extract<keyof T, `${number}`>;
|
|
53
|
-
type JevAnswerFor<Q extends JevRecordedQuestion> = Q extends {
|
|
54
|
-
type: 'noul';
|
|
55
|
-
} ? {
|
|
56
|
-
readonly type: 'noul';
|
|
57
|
-
readonly noul: number;
|
|
58
|
-
} : Q extends {
|
|
59
|
-
type: 'score';
|
|
60
|
-
criteria: infer C extends readonly JevDescription[];
|
|
61
|
-
} ? {
|
|
62
|
-
readonly type: 'score';
|
|
63
|
-
readonly score: number;
|
|
64
|
-
readonly confidence: number;
|
|
65
|
-
readonly legend: { readonly [K in ScoreKeys<C>]: C[K]; };
|
|
66
|
-
readonly probabilities: { readonly [K in ScoreKeys<C>]: number; };
|
|
67
|
-
} : Q extends {
|
|
68
|
-
type: 'choice';
|
|
69
|
-
criteria: infer C;
|
|
70
|
-
} ? {
|
|
71
|
-
readonly type: 'choice';
|
|
72
|
-
readonly choice: keyof C & string;
|
|
73
|
-
readonly confidence: number;
|
|
74
|
-
readonly probabilities: { readonly [K in keyof C]: number; };
|
|
75
|
-
} : never;
|
|
76
|
-
type JevAnswer = JevAnswerFor<JevQuestion>;
|
|
77
|
-
interface JevResult<Q extends JevRecordedQuestions = JevQuestions> {
|
|
78
|
-
readonly model: string;
|
|
79
|
-
readonly answers: { readonly [K in keyof Q]: JevAnswerFor<Q[K]>; };
|
|
80
|
-
readonly usage: {
|
|
81
|
-
readonly input_tokens: number;
|
|
82
|
-
readonly output_tokens: number;
|
|
83
|
-
};
|
|
84
|
-
}
|
|
85
|
-
declare class JevResponseError extends Error {
|
|
86
|
-
constructor(message: string);
|
|
87
|
-
}
|
|
88
|
-
/** Validate named questions without a state, as a judge does before it has input. */
|
|
89
|
-
declare function parseJevQuestions(raw: unknown): JevQuestions;
|
|
90
|
-
declare function parseJevRequest(raw: unknown): JevRequest;
|
|
91
|
-
/** Read immutable historical evidence without rewriting it or authorizing new inference.
|
|
92
|
-
* Live transports must still call parseJevRequest, even for re-execution of an old request. */
|
|
93
|
-
declare function parseJevRecordedRequest(raw: unknown): JevRecordedRequest;
|
|
94
|
-
declare function jevUsage(raw: unknown): JevResult['usage'];
|
|
95
|
-
/** Validate native evidence, not model alias policy or the caller's scoring rules. */
|
|
96
|
-
declare function parseJevResult<Q extends JevRecordedQuestions>(raw: unknown, request: JevRecordedRequest<Q>): JevResult<Q>;
|
|
97
|
-
//#endregion
|
|
98
|
-
export { parseJevRecordedRequest as _, JevQuestion as a, JevRecordedQuestions as c, JevResponseError as d, JevResult as f, parseJevQuestions as g, jevUsage as h, JevJson as i, JevRecordedRequest as l, JevState as m, JevAnswerFor as n, JevQuestions as o, JevScoreCriteria as p, JevDescription as r, JevRecordedQuestion as s, JevAnswer as t, JevRequest as u, parseJevRequest as v, parseJevResult as y };
|
|
99
|
-
//# sourceMappingURL=jev-protocol-BQLxwbqD.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"jev-protocol-BQLxwbqD.d.ts","names":[],"sources":["../src/jev-protocol.ts"],"mappings":";;;;;KAOY,6CAA6C;GAAe,cAAc;;;KAE1E;GAAuB,cAAc;IAAY;;KAEjD,iBAAiB;KACjB,6BAA6B,UAAU,aAAa;KACpD;EAEN;EACA,eAAe;EACf;IAAa,OAAO;IAAgB,QAAQ;;;EAE5C;EAAe,eAAe;EAAgB,UAAU;;EACxD;EAAgB,eAAe;EAAgB,UAAU,eAAe;;KAClE,eAAe,eAAe;UACzB,WAAW,UAAU,eAAe;EACnD;EACA,OAAO;EACP,WAAW;;;KAID,sBACR,QAAQ;EAAe;;EAErB;EACA,eAAe;EACf,oBAAoB,gBAAgB,mBAAmB;;KAEjD,uBAAuB,eAAe;UACjC,mBAAmB,UAAU,uBAAuB;EACnE;EACA,OAAO;EACP,WAAW;;KAGR,UAAU,mBAAmB,mCAAmC,uBAEjE,cAAc;KACN,aAAa,UAAU,uBAAuB;EAAY;;WACvD;WAAuB;IAClC;EAAY;EAAe,gBAAgB,mBAAmB;;WAEjD;WACA;WACA;WACA,oBAAoB,KAAK,UAAU,KAAK,EAAE;WAC1C,2BAA2B,KAAK,UAAU;IAErD;EAAY;EAAgB,gBAAgB;;WAE/B;WACA,cAAc;WACd;WACA,2BAA2B,WAAW;;KAG7C,YAAY,aAAa;UACpB,UAAU,UAAU,uBAAuB;WACjD;WACA,qBAAqB,WAAW,IAAI,aAAa,EAAE;WACnD;aAAkB;aAA+B;;;cAG/C,yBAAyB;EACpC,YAAY;;;iBA6BE,kBAAkB,eAAe;iBAyCjC,gBAAgB,eAAe;;;iBAM/B,wBAAwB,eAAe;iBAevC,SAAS,eAAe;;iBA2BxB,eAAe,UAAU,sBACvC,cACA,SAAS,mBAAmB,KAC3B,UAAU"}
|
package/dist/jev-protocol.d.ts
DELETED
|
@@ -1,2 +0,0 @@
|
|
|
1
|
-
import { _ as parseJevRecordedRequest, a as JevQuestion, c as JevRecordedQuestions, d as JevResponseError, f as JevResult, g as parseJevQuestions, h as jevUsage, i as JevJson, l as JevRecordedRequest, m as JevState, n as JevAnswerFor, o as JevQuestions, p as JevScoreCriteria, r as JevDescription, s as JevRecordedQuestion, t as JevAnswer, u as JevRequest, v as parseJevRequest, y as parseJevResult } from "./jev-protocol-BQLxwbqD.js";
|
|
2
|
-
export { JevAnswer, JevAnswerFor, JevDescription, JevJson, JevQuestion, JevQuestions, JevRecordedQuestion, JevRecordedQuestions, JevRecordedRequest, JevRequest, JevResponseError, JevResult, JevScoreCriteria, JevState, jevUsage, parseJevQuestions, parseJevRecordedRequest, parseJevRequest, parseJevResult };
|