harness-reflex 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +103 -23
- package/USER_GUIDE.md +121 -43
- package/dist/adapters/agy.js +1 -1
- package/dist/adapters/agy.js.map +1 -1
- package/dist/adapters/codex.d.ts +133 -4
- package/dist/adapters/codex.js +325 -2
- package/dist/adapters/codex.js.map +1 -1
- package/dist/backend/defaults.d.ts +31 -0
- package/dist/backend/defaults.js +32 -0
- package/dist/backend/defaults.js.map +1 -0
- package/dist/backend/jev.js +6 -5
- package/dist/backend/jev.js.map +1 -1
- package/dist/backend/mock.d.ts +12 -1
- package/dist/backend/mock.js +34 -0
- package/dist/backend/mock.js.map +1 -1
- package/dist/backend/simple-jev.js +5 -4
- package/dist/backend/simple-jev.js.map +1 -1
- package/dist/backend/types.d.ts +34 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/cli/index.js +5 -3
- package/dist/cli/index.js.map +1 -1
- package/dist/core/closing.d.ts +90 -0
- package/dist/core/closing.js +111 -0
- package/dist/core/closing.js.map +1 -0
- package/dist/core/evidence.d.ts +119 -0
- package/dist/core/evidence.js +209 -0
- package/dist/core/evidence.js.map +1 -0
- package/dist/core/judge.d.ts +221 -0
- package/dist/core/judge.js +251 -0
- package/dist/core/judge.js.map +1 -0
- package/dist/core/orchestrate-closing.d.ts +237 -0
- package/dist/core/orchestrate-closing.js +250 -0
- package/dist/core/orchestrate-closing.js.map +1 -0
- package/dist/core/orchestrate-correction.d.ts +209 -0
- package/dist/core/orchestrate-correction.js +141 -0
- package/dist/core/orchestrate-correction.js.map +1 -0
- package/dist/core/orchestrate-host.d.ts +225 -0
- package/dist/core/orchestrate-host.js +80 -0
- package/dist/core/orchestrate-host.js.map +1 -0
- package/dist/core/orchestrate-stagnation.d.ts +259 -0
- package/dist/core/orchestrate-stagnation.js +270 -0
- package/dist/core/orchestrate-stagnation.js.map +1 -0
- package/dist/core/orchestrate.d.ts +357 -0
- package/dist/core/orchestrate.js +418 -0
- package/dist/core/orchestrate.js.map +1 -0
- package/dist/core/policy.d.ts +122 -0
- package/dist/core/policy.js +102 -0
- package/dist/core/policy.js.map +1 -0
- package/dist/core/prune-plan.d.ts +141 -0
- package/dist/core/prune-plan.js +390 -0
- package/dist/core/prune-plan.js.map +1 -0
- package/dist/core/score.d.ts +2 -2
- package/dist/core/score.js.map +1 -1
- package/dist/core/text.d.ts +36 -0
- package/dist/core/text.js +66 -0
- package/dist/core/text.js.map +1 -0
- package/dist/hook/cli.d.ts +9 -0
- package/dist/hook/cli.js +52 -0
- package/dist/hook/cli.js.map +1 -0
- package/dist/hook/contract.d.ts +125 -0
- package/dist/hook/contract.js +159 -0
- package/dist/hook/contract.js.map +1 -0
- package/dist/hook/decision.d.ts +97 -0
- package/dist/hook/decision.js +955 -0
- package/dist/hook/decision.js.map +1 -0
- package/dist/hook/memory-store.d.ts +81 -0
- package/dist/hook/memory-store.js +157 -0
- package/dist/hook/memory-store.js.map +1 -0
- package/dist/hook/payload.d.ts +55 -0
- package/dist/hook/payload.js +129 -0
- package/dist/hook/payload.js.map +1 -0
- package/dist/hook/run.d.ts +48 -0
- package/dist/hook/run.js +218 -0
- package/dist/hook/run.js.map +1 -0
- package/dist/hook/store-port.d.ts +72 -0
- package/dist/hook/store-port.js +2 -0
- package/dist/hook/store-port.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/ports/audit.d.ts +18 -0
- package/dist/ports/audit.js +11 -0
- package/dist/ports/audit.js.map +1 -0
- package/dist/ports/clock.d.ts +14 -0
- package/dist/ports/clock.js +11 -0
- package/dist/ports/clock.js.map +1 -0
- package/dist/ports/decision.d.ts +75 -0
- package/dist/ports/decision.js +2 -0
- package/dist/ports/decision.js.map +1 -0
- package/dist/ports/index.d.ts +14 -0
- package/dist/ports/index.js +2 -0
- package/dist/ports/index.js.map +1 -0
- package/dist/ports/log.d.ts +22 -0
- package/dist/ports/log.js +12 -0
- package/dist/ports/log.js.map +1 -0
- package/dist/ports/state.d.ts +46 -0
- package/dist/ports/state.js +13 -0
- package/dist/ports/state.js.map +1 -0
- package/dist/state/clock.d.ts +40 -0
- package/dist/state/clock.js +43 -0
- package/dist/state/clock.js.map +1 -0
- package/dist/state/file-store.d.ts +340 -0
- package/dist/state/file-store.js +849 -0
- package/dist/state/file-store.js.map +1 -0
- package/dist/state/paths.d.ts +41 -0
- package/dist/state/paths.js +73 -0
- package/dist/state/paths.js.map +1 -0
- package/docs/USER_GUIDE.en.md +79 -36
- package/docs/claude-code.md +281 -0
- package/docs/design/v0.2.0-harness-plugins.md +381 -0
- package/docs/design/v0.3.0-orchestration-unification.md +580 -0
- package/docs/dsh-plugin.md +54 -54
- package/docs/release-checklist-v0.1.0.md +32 -21
- package/docs/release-handbook-porting-notes.md +11 -11
- package/docs/release-handbook.md +55 -30
- package/docs/release-notes-v0.1.0.md +27 -14
- package/docs/release-notes-v0.3.0.md +152 -0
- package/docs/research/claude-code-harness.md +607 -0
- package/docs/research/pi-harness.md +621 -0
- package/package.json +16 -7
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The deterministic pre-trigger, the judgment call, and the cost accounting.
|
|
3
|
+
*
|
|
4
|
+
* The expensive path of every capability is one System 1 judgment. This module
|
|
5
|
+
* owns the cheap path that decides whether a judgment is worth buying, the
|
|
6
|
+
* once-per-turn call itself (memoized by the window it is about), and the
|
|
7
|
+
* bookkeeping that bills exactly one judgment per call. The transport arrives as
|
|
8
|
+
* one `buy(step)` callback and never as the service itself; the clock is a
|
|
9
|
+
* {@link ClockPort}; neither is imported from a host.
|
|
10
|
+
*
|
|
11
|
+
* @module core/judge
|
|
12
|
+
*/
|
|
13
|
+
import type { StagnationCheckResponse } from '../backend/types.js';
|
|
14
|
+
import type { ClockPort } from '../ports/clock.js';
|
|
15
|
+
import type { StagnationBackendPort } from '../ports/decision.js';
|
|
16
|
+
import type { JudgmentState, JudgmentStep } from './orchestrate-host.js';
|
|
17
|
+
import type { PolicySnapshot } from './policy.js';
|
|
18
|
+
/** One step of one agent's detection window. */
|
|
19
|
+
export interface ObservedStep {
|
|
20
|
+
/** `tool:argumentsDigest` — the identity a repetition is measured on. */
|
|
21
|
+
signature: string;
|
|
22
|
+
tool: string;
|
|
23
|
+
input: string;
|
|
24
|
+
output: string;
|
|
25
|
+
error: string;
|
|
26
|
+
failed: boolean;
|
|
27
|
+
turn: number;
|
|
28
|
+
at: number;
|
|
29
|
+
/** Whether the step was a legitimate wait query rather than progress. */
|
|
30
|
+
wait?: boolean;
|
|
31
|
+
}
|
|
32
|
+
/** Why a deterministic pre-trigger wants a judgment. */
|
|
33
|
+
export interface StagnationSuspicion {
|
|
34
|
+
suspicious: boolean;
|
|
35
|
+
/** The pattern the trigger suspects, before System 1 confirms it. */
|
|
36
|
+
pattern?: 'repetition' | 'oscillation' | 'failing_retry';
|
|
37
|
+
/** Human-readable trigger facts, for the audit record. */
|
|
38
|
+
note: string;
|
|
39
|
+
/** How many steps the pattern was measured on. */
|
|
40
|
+
repeats: number;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Count the identical trailing repeats of one signature.
|
|
44
|
+
* @param steps - observed steps, oldest first.
|
|
45
|
+
* @param signature - the identity to count.
|
|
46
|
+
* @returns the length of the trailing run of that signature.
|
|
47
|
+
*/
|
|
48
|
+
export declare function countTrailingRepeats(steps: readonly ObservedStep[], signature: string): number;
|
|
49
|
+
/**
|
|
50
|
+
* Count the trailing consecutive failures.
|
|
51
|
+
* @param steps - observed steps, oldest first.
|
|
52
|
+
* @returns the length of the trailing run of failing steps.
|
|
53
|
+
*/
|
|
54
|
+
export declare function countTrailingFailures(steps: readonly ObservedStep[]): number;
|
|
55
|
+
/**
|
|
56
|
+
* Whether the window's tail alternates between exactly two call identities.
|
|
57
|
+
*
|
|
58
|
+
* `A B A B` is the cheapest loop to miss and among the most wasteful to run:
|
|
59
|
+
* neither call repeats consecutively, so a consecutive-repeat trigger never
|
|
60
|
+
* fires.
|
|
61
|
+
* @param steps - observed steps, oldest first.
|
|
62
|
+
* @param window - how many trailing steps to inspect.
|
|
63
|
+
* @returns whether the tail oscillates.
|
|
64
|
+
*/
|
|
65
|
+
export declare function detectOscillation(steps: readonly ObservedStep[], window: number): boolean;
|
|
66
|
+
/**
|
|
67
|
+
* Decide, locally and deterministically, whether a judgment is worth spending.
|
|
68
|
+
*
|
|
69
|
+
* This is the cost gate of the whole surface: the expensive path is a System 1
|
|
70
|
+
* call, and it only opens when the *cheap* path already sees a pattern. A single
|
|
71
|
+
* distinctive call is never judged, however large its output.
|
|
72
|
+
*
|
|
73
|
+
* @param steps - observed steps, oldest first.
|
|
74
|
+
* @param options - thresholds, window, and the current call identity.
|
|
75
|
+
* @returns the suspicion, with `suspicious: false` when nothing justifies a call.
|
|
76
|
+
*/
|
|
77
|
+
export declare function suspicionOf(steps: readonly ObservedStep[], options: {
|
|
78
|
+
repeatThreshold: number;
|
|
79
|
+
failureThreshold: number;
|
|
80
|
+
window: number;
|
|
81
|
+
signature: string;
|
|
82
|
+
}): StagnationSuspicion;
|
|
83
|
+
/** The verdict that stands in when no judgment could be spent. */
|
|
84
|
+
export declare const EMPTY_VERDICT: StagnationCheckResponse;
|
|
85
|
+
/** What one judgment cost, charged exactly once per judgment. */
|
|
86
|
+
export interface JudgmentCharge {
|
|
87
|
+
judgments: number;
|
|
88
|
+
memoHits: number;
|
|
89
|
+
localLatencyMs: number;
|
|
90
|
+
remoteLatencyMs: number;
|
|
91
|
+
judgeTokens: number;
|
|
92
|
+
}
|
|
93
|
+
/** One token and latency account of a mode or a capability. */
|
|
94
|
+
export interface InterventionCostBucket {
|
|
95
|
+
/** Decisions planned. */
|
|
96
|
+
plans: number;
|
|
97
|
+
/** Decisions whose plan was executed. */
|
|
98
|
+
applied: number;
|
|
99
|
+
/** System 1 judgments paid for. */
|
|
100
|
+
judgments: number;
|
|
101
|
+
/** Judgments served from the per-turn verdict memo instead of a new call. */
|
|
102
|
+
memoHits: number;
|
|
103
|
+
/** Milliseconds spent in local, deterministic work. */
|
|
104
|
+
localLatencyMs: number;
|
|
105
|
+
/** Milliseconds spent waiting for System 1. */
|
|
106
|
+
remoteLatencyMs: number;
|
|
107
|
+
/** Tokens the judgments reasoned about. */
|
|
108
|
+
judgeTokens: number;
|
|
109
|
+
/** Tokens of the payloads decisions were made about. */
|
|
110
|
+
originalTokens: number;
|
|
111
|
+
/** Tokens the candidates would carry. */
|
|
112
|
+
candidateTokens: number;
|
|
113
|
+
/** Tokens the candidates would have saved. */
|
|
114
|
+
potentialSavedTokens: number;
|
|
115
|
+
/** Tokens actually removed from the model-visible surface. */
|
|
116
|
+
actualSavedTokens: number;
|
|
117
|
+
/** Tokens actually added to the model-visible surface. */
|
|
118
|
+
injectedTokens: number;
|
|
119
|
+
}
|
|
120
|
+
/** A fresh, zeroed cost bucket. */
|
|
121
|
+
export declare function emptyBucket(): InterventionCostBucket;
|
|
122
|
+
/** One charge against a mode's and a capability's account. */
|
|
123
|
+
export interface CostCharge {
|
|
124
|
+
judgments?: number;
|
|
125
|
+
memoHits?: number;
|
|
126
|
+
localLatencyMs?: number;
|
|
127
|
+
remoteLatencyMs?: number;
|
|
128
|
+
judgeTokens?: number;
|
|
129
|
+
originalTokens?: number;
|
|
130
|
+
candidateTokens?: number;
|
|
131
|
+
potentialSavedTokens?: number;
|
|
132
|
+
actualSavedTokens?: number;
|
|
133
|
+
injectedTokens?: number;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Add one charge to one bucket.
|
|
137
|
+
*
|
|
138
|
+
* Every counter has exactly one writer, so a reader can add the parts up without
|
|
139
|
+
* meeting a number twice.
|
|
140
|
+
* @param bucket - the bucket to bill.
|
|
141
|
+
* @param plans - decisions planned by this charge.
|
|
142
|
+
* @param applied - decisions applied by this charge.
|
|
143
|
+
* @param charge - the token/latency/decision counts.
|
|
144
|
+
*/
|
|
145
|
+
export declare function applyCharge(bucket: InterventionCostBucket, plans: number, applied: number, charge: CostCharge): void;
|
|
146
|
+
/** The state one judgment memo and the two budgets live in. */
|
|
147
|
+
export interface VerdictMemoState extends JudgmentState {
|
|
148
|
+
verdictTurn: number;
|
|
149
|
+
/** Identity of the window the cached verdict was about. */
|
|
150
|
+
verdictKey: string;
|
|
151
|
+
verdict?: StagnationCheckResponse;
|
|
152
|
+
}
|
|
153
|
+
/** The outcome of one judgment request. */
|
|
154
|
+
export interface JudgmentOutcome {
|
|
155
|
+
useful: boolean;
|
|
156
|
+
paid: boolean;
|
|
157
|
+
reused: boolean;
|
|
158
|
+
judgeTokens: number;
|
|
159
|
+
localLatencyMs: number;
|
|
160
|
+
remoteLatencyMs: number;
|
|
161
|
+
skip?: string;
|
|
162
|
+
response: StagnationCheckResponse;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* The memo key of one judgment: the question plus the trailing window it is
|
|
166
|
+
* asked about.
|
|
167
|
+
* @param goal - the wait/context question, when one was asked.
|
|
168
|
+
* @param steps - the window.
|
|
169
|
+
* @param repeatThreshold - how many trailing signatures identify the window.
|
|
170
|
+
* @returns a stable key.
|
|
171
|
+
*/
|
|
172
|
+
export declare function judgmentKey(goal: string | undefined, steps: readonly ObservedStep[], repeatThreshold: number): string;
|
|
173
|
+
/**
|
|
174
|
+
* Ask for one stagnation verdict, reusing this turn's answer when the window is
|
|
175
|
+
* the same one.
|
|
176
|
+
*
|
|
177
|
+
* The question is the trailing window plus the context it is asked in, so the
|
|
178
|
+
* answer is cached against both: the same episode is never judged twice, while
|
|
179
|
+
* a *different* episode or a different wait context in the same turn is a new
|
|
180
|
+
* question — and therefore subject to the turn's budget.
|
|
181
|
+
*
|
|
182
|
+
* The purchase is this function's, not the host's: `buy` is called only after
|
|
183
|
+
* the memo missed, after both budgets admitted the question, and after
|
|
184
|
+
* {@link spendJudgment} booked the slot. A host that was handed the transport
|
|
185
|
+
* instead would have to reproduce that order, and the one place a wrong answer is
|
|
186
|
+
* a model-visible regression is the place the type would stop helping.
|
|
187
|
+
*
|
|
188
|
+
* @param args - state, policy, the host's `buy`, clock, window, question, and
|
|
189
|
+
* abort signal.
|
|
190
|
+
* @returns the outcome, including why a judgment was refused.
|
|
191
|
+
*/
|
|
192
|
+
export declare function runJudgment(args: {
|
|
193
|
+
state: VerdictMemoState;
|
|
194
|
+
policy: PolicySnapshot;
|
|
195
|
+
/** One purchase: ask the service about one step, and report what it cost. */
|
|
196
|
+
buy?: (step: JudgmentStep) => Promise<StagnationCheckResponse>;
|
|
197
|
+
/**
|
|
198
|
+
* The pre-g-046 channel: a service handed over whole, instead of a purchase.
|
|
199
|
+
*
|
|
200
|
+
* Kept **only** for the frozen parity snapshot
|
|
201
|
+
* (`packages/dsh-harness-reflex/src/baseline/interventions.ts`), which is the
|
|
202
|
+
* oracle of the refactor and may not be touched. Every live caller passes
|
|
203
|
+
* {@link buy}; the snapshot's `#judge` still passes its composition's service,
|
|
204
|
+
* and the parity run therefore keeps this channel exercised.
|
|
205
|
+
*
|
|
206
|
+
* @deprecated Pass `buy`. Nothing added from now on should use this.
|
|
207
|
+
*/
|
|
208
|
+
backend?: StagnationBackendPort;
|
|
209
|
+
clock: ClockPort;
|
|
210
|
+
steps: readonly ObservedStep[];
|
|
211
|
+
goal?: string;
|
|
212
|
+
signal?: AbortSignal;
|
|
213
|
+
}): Promise<JudgmentOutcome>;
|
|
214
|
+
/**
|
|
215
|
+
* The charge of one judgment, so every judgment is billed exactly once.
|
|
216
|
+
* @param verdict - the outcome of the judgment call.
|
|
217
|
+
* @param started - clock reading at hook entry.
|
|
218
|
+
* @param finished - clock reading now.
|
|
219
|
+
* @returns the charge.
|
|
220
|
+
*/
|
|
221
|
+
export declare function judgmentCharge(verdict: Pick<JudgmentOutcome, 'paid' | 'reused' | 'judgeTokens' | 'localLatencyMs' | 'remoteLatencyMs'>, started: number, finished: number): JudgmentCharge;
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import { spendJudgment, usefulVerdict } from './policy.js';
|
|
2
|
+
import { estimateTokens } from './text.js';
|
|
3
|
+
/**
|
|
4
|
+
* Count the identical trailing repeats of one signature.
|
|
5
|
+
* @param steps - observed steps, oldest first.
|
|
6
|
+
* @param signature - the identity to count.
|
|
7
|
+
* @returns the length of the trailing run of that signature.
|
|
8
|
+
*/
|
|
9
|
+
export function countTrailingRepeats(steps, signature) {
|
|
10
|
+
let count = 0;
|
|
11
|
+
for (let index = steps.length - 1; index >= 0; index -= 1) {
|
|
12
|
+
if (steps[index].signature !== signature)
|
|
13
|
+
break;
|
|
14
|
+
count += 1;
|
|
15
|
+
}
|
|
16
|
+
return count;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Count the trailing consecutive failures.
|
|
20
|
+
* @param steps - observed steps, oldest first.
|
|
21
|
+
* @returns the length of the trailing run of failing steps.
|
|
22
|
+
*/
|
|
23
|
+
export function countTrailingFailures(steps) {
|
|
24
|
+
let count = 0;
|
|
25
|
+
for (let index = steps.length - 1; index >= 0; index -= 1) {
|
|
26
|
+
if (!steps[index].failed)
|
|
27
|
+
break;
|
|
28
|
+
count += 1;
|
|
29
|
+
}
|
|
30
|
+
return count;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Whether the window's tail alternates between exactly two call identities.
|
|
34
|
+
*
|
|
35
|
+
* `A B A B` is the cheapest loop to miss and among the most wasteful to run:
|
|
36
|
+
* neither call repeats consecutively, so a consecutive-repeat trigger never
|
|
37
|
+
* fires.
|
|
38
|
+
* @param steps - observed steps, oldest first.
|
|
39
|
+
* @param window - how many trailing steps to inspect.
|
|
40
|
+
* @returns whether the tail oscillates.
|
|
41
|
+
*/
|
|
42
|
+
export function detectOscillation(steps, window) {
|
|
43
|
+
const tail = steps.slice(Math.max(0, steps.length - window));
|
|
44
|
+
if (tail.length < 4)
|
|
45
|
+
return false;
|
|
46
|
+
const signatures = new Set(tail.map((step) => step.signature));
|
|
47
|
+
if (signatures.size !== 2)
|
|
48
|
+
return false;
|
|
49
|
+
for (let index = 2; index < tail.length; index += 1) {
|
|
50
|
+
if (tail[index].signature !== tail[index - 2].signature)
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Decide, locally and deterministically, whether a judgment is worth spending.
|
|
57
|
+
*
|
|
58
|
+
* This is the cost gate of the whole surface: the expensive path is a System 1
|
|
59
|
+
* call, and it only opens when the *cheap* path already sees a pattern. A single
|
|
60
|
+
* distinctive call is never judged, however large its output.
|
|
61
|
+
*
|
|
62
|
+
* @param steps - observed steps, oldest first.
|
|
63
|
+
* @param options - thresholds, window, and the current call identity.
|
|
64
|
+
* @returns the suspicion, with `suspicious: false` when nothing justifies a call.
|
|
65
|
+
*/
|
|
66
|
+
export function suspicionOf(steps, options) {
|
|
67
|
+
const window = steps.slice(Math.max(0, steps.length - options.window));
|
|
68
|
+
const repeats = countTrailingRepeats(window, options.signature);
|
|
69
|
+
if (repeats >= options.repeatThreshold) {
|
|
70
|
+
return { suspicious: true, pattern: 'repetition', note: `identical call repeated ${repeats}x`, repeats };
|
|
71
|
+
}
|
|
72
|
+
const failures = countTrailingFailures(window);
|
|
73
|
+
if (failures >= options.failureThreshold) {
|
|
74
|
+
return { suspicious: true, pattern: 'failing_retry', note: `${failures} consecutive failing steps`, repeats: failures };
|
|
75
|
+
}
|
|
76
|
+
if (detectOscillation(window, options.window)) {
|
|
77
|
+
return { suspicious: true, pattern: 'oscillation', note: `two calls alternating over ${window.length} steps`, repeats: window.length };
|
|
78
|
+
}
|
|
79
|
+
return { suspicious: false, note: 'no local pattern', repeats: 0 };
|
|
80
|
+
}
|
|
81
|
+
/** The verdict that stands in when no judgment could be spent. */
|
|
82
|
+
export const EMPTY_VERDICT = { isStuck: false, confidence: 0, latencyMs: 0 };
|
|
83
|
+
/** A fresh, zeroed cost bucket. */
|
|
84
|
+
export function emptyBucket() {
|
|
85
|
+
return {
|
|
86
|
+
plans: 0,
|
|
87
|
+
applied: 0,
|
|
88
|
+
judgments: 0,
|
|
89
|
+
memoHits: 0,
|
|
90
|
+
localLatencyMs: 0,
|
|
91
|
+
remoteLatencyMs: 0,
|
|
92
|
+
judgeTokens: 0,
|
|
93
|
+
originalTokens: 0,
|
|
94
|
+
candidateTokens: 0,
|
|
95
|
+
potentialSavedTokens: 0,
|
|
96
|
+
actualSavedTokens: 0,
|
|
97
|
+
injectedTokens: 0
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Add one charge to one bucket.
|
|
102
|
+
*
|
|
103
|
+
* Every counter has exactly one writer, so a reader can add the parts up without
|
|
104
|
+
* meeting a number twice.
|
|
105
|
+
* @param bucket - the bucket to bill.
|
|
106
|
+
* @param plans - decisions planned by this charge.
|
|
107
|
+
* @param applied - decisions applied by this charge.
|
|
108
|
+
* @param charge - the token/latency/decision counts.
|
|
109
|
+
*/
|
|
110
|
+
export function applyCharge(bucket, plans, applied, charge) {
|
|
111
|
+
bucket.plans += plans;
|
|
112
|
+
bucket.applied += applied;
|
|
113
|
+
bucket.judgments += charge.judgments ?? 0;
|
|
114
|
+
bucket.memoHits += charge.memoHits ?? 0;
|
|
115
|
+
bucket.localLatencyMs += charge.localLatencyMs ?? 0;
|
|
116
|
+
bucket.remoteLatencyMs += charge.remoteLatencyMs ?? 0;
|
|
117
|
+
bucket.judgeTokens += charge.judgeTokens ?? 0;
|
|
118
|
+
bucket.originalTokens += charge.originalTokens ?? 0;
|
|
119
|
+
bucket.candidateTokens += charge.candidateTokens ?? 0;
|
|
120
|
+
bucket.potentialSavedTokens += charge.potentialSavedTokens ?? 0;
|
|
121
|
+
bucket.actualSavedTokens += charge.actualSavedTokens ?? 0;
|
|
122
|
+
bucket.injectedTokens += charge.injectedTokens ?? 0;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* The memo key of one judgment: the question plus the trailing window it is
|
|
126
|
+
* asked about.
|
|
127
|
+
* @param goal - the wait/context question, when one was asked.
|
|
128
|
+
* @param steps - the window.
|
|
129
|
+
* @param repeatThreshold - how many trailing signatures identify the window.
|
|
130
|
+
* @returns a stable key.
|
|
131
|
+
*/
|
|
132
|
+
export function judgmentKey(goal, steps, repeatThreshold) {
|
|
133
|
+
return `${goal ?? ''}\u0000${steps
|
|
134
|
+
.slice(-Math.max(1, repeatThreshold))
|
|
135
|
+
.map((step) => step.signature)
|
|
136
|
+
.join('|')}`;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* The one purchase a judgment makes, from whichever channel the caller used.
|
|
140
|
+
* @param buy - the current channel: one call, one step.
|
|
141
|
+
* @param backend - the deprecated channel: the service itself.
|
|
142
|
+
* @returns the purchase to run.
|
|
143
|
+
* @throws Error when a caller supplied neither, which is a programming error.
|
|
144
|
+
*/
|
|
145
|
+
function purchaseOf(buy, backend) {
|
|
146
|
+
if (buy !== undefined)
|
|
147
|
+
return buy;
|
|
148
|
+
if (backend === undefined) {
|
|
149
|
+
throw new Error('runJudgment needs a purchase: pass buy(step)');
|
|
150
|
+
}
|
|
151
|
+
return (step) => backend.check(step.goal, step.history);
|
|
152
|
+
}
|
|
153
|
+
/** The zero-cost verdict of a judgment that could not be spent. */
|
|
154
|
+
function noJudgment(skip) {
|
|
155
|
+
return {
|
|
156
|
+
useful: false,
|
|
157
|
+
paid: false,
|
|
158
|
+
reused: false,
|
|
159
|
+
judgeTokens: 0,
|
|
160
|
+
localLatencyMs: 0,
|
|
161
|
+
remoteLatencyMs: 0,
|
|
162
|
+
skip,
|
|
163
|
+
response: EMPTY_VERDICT
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Ask for one stagnation verdict, reusing this turn's answer when the window is
|
|
168
|
+
* the same one.
|
|
169
|
+
*
|
|
170
|
+
* The question is the trailing window plus the context it is asked in, so the
|
|
171
|
+
* answer is cached against both: the same episode is never judged twice, while
|
|
172
|
+
* a *different* episode or a different wait context in the same turn is a new
|
|
173
|
+
* question — and therefore subject to the turn's budget.
|
|
174
|
+
*
|
|
175
|
+
* The purchase is this function's, not the host's: `buy` is called only after
|
|
176
|
+
* the memo missed, after both budgets admitted the question, and after
|
|
177
|
+
* {@link spendJudgment} booked the slot. A host that was handed the transport
|
|
178
|
+
* instead would have to reproduce that order, and the one place a wrong answer is
|
|
179
|
+
* a model-visible regression is the place the type would stop helping.
|
|
180
|
+
*
|
|
181
|
+
* @param args - state, policy, the host's `buy`, clock, window, question, and
|
|
182
|
+
* abort signal.
|
|
183
|
+
* @returns the outcome, including why a judgment was refused.
|
|
184
|
+
*/
|
|
185
|
+
export async function runJudgment(args) {
|
|
186
|
+
const { state, policy, clock, steps, goal, signal } = args;
|
|
187
|
+
const buy = purchaseOf(args.buy, args.backend);
|
|
188
|
+
const key = judgmentKey(goal, steps, policy.repeatThreshold);
|
|
189
|
+
const cached = state.verdictTurn === state.turn && state.verdictKey === key ? state.verdict : undefined;
|
|
190
|
+
if (cached) {
|
|
191
|
+
return {
|
|
192
|
+
useful: usefulVerdict(cached, policy),
|
|
193
|
+
paid: false,
|
|
194
|
+
reused: true,
|
|
195
|
+
judgeTokens: 0,
|
|
196
|
+
localLatencyMs: 0,
|
|
197
|
+
remoteLatencyMs: 0,
|
|
198
|
+
response: cached
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
if (state.judgmentsCount >= policy.maxJudgmentsPerTurn)
|
|
202
|
+
return noJudgment('judgment-turn-cap');
|
|
203
|
+
if (state.judgmentsSession >= policy.maxJudgmentsPerSession)
|
|
204
|
+
return noJudgment('judgment-session-cap');
|
|
205
|
+
if (signal?.aborted)
|
|
206
|
+
return noJudgment('aborted');
|
|
207
|
+
spendJudgment(state, policy);
|
|
208
|
+
const history = steps.map((step) => ({
|
|
209
|
+
tool: step.tool,
|
|
210
|
+
input: step.input,
|
|
211
|
+
...(step.output ? { output: step.output } : {}),
|
|
212
|
+
...(step.error ? { error: step.error } : {}),
|
|
213
|
+
timestamp: step.at
|
|
214
|
+
}));
|
|
215
|
+
const judgeTokens = history.reduce((sum, step) => sum + estimateTokens(`${step.tool}${step.input}${step.output ?? ''}${step.error ?? ''}`), 0);
|
|
216
|
+
const started = clock.now();
|
|
217
|
+
const response = await buy({ goal, history });
|
|
218
|
+
const remoteLatencyMs = Number.isFinite(response.latencyMs) ? response.latencyMs : 0;
|
|
219
|
+
state.verdict = response;
|
|
220
|
+
state.verdictTurn = state.turn;
|
|
221
|
+
state.verdictKey = key;
|
|
222
|
+
return {
|
|
223
|
+
useful: usefulVerdict(response, policy),
|
|
224
|
+
paid: true,
|
|
225
|
+
reused: false,
|
|
226
|
+
judgeTokens,
|
|
227
|
+
localLatencyMs: Math.max(0, clock.now() - started - remoteLatencyMs),
|
|
228
|
+
remoteLatencyMs,
|
|
229
|
+
response
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* The charge of one judgment, so every judgment is billed exactly once.
|
|
234
|
+
* @param verdict - the outcome of the judgment call.
|
|
235
|
+
* @param started - clock reading at hook entry.
|
|
236
|
+
* @param finished - clock reading now.
|
|
237
|
+
* @returns the charge.
|
|
238
|
+
*/
|
|
239
|
+
export function judgmentCharge(verdict, started, finished) {
|
|
240
|
+
if (!verdict.paid) {
|
|
241
|
+
return { judgments: 0, memoHits: verdict.reused ? 1 : 0, localLatencyMs: 0, remoteLatencyMs: 0, judgeTokens: 0 };
|
|
242
|
+
}
|
|
243
|
+
return {
|
|
244
|
+
judgments: 1,
|
|
245
|
+
memoHits: 0,
|
|
246
|
+
localLatencyMs: Math.max(verdict.localLatencyMs, Math.max(0, finished - started - verdict.remoteLatencyMs)),
|
|
247
|
+
remoteLatencyMs: verdict.remoteLatencyMs,
|
|
248
|
+
judgeTokens: verdict.judgeTokens
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
//# sourceMappingURL=judge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/core/judge.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AA4B3C;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAA8B,EAAE,SAAiB;IACpF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1D,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,SAAS,KAAK,SAAS;YAAE,MAAM;QACjD,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAA8B;IAClE,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,KAAK,CAAE,CAAC,MAAM;YAAE,MAAM;QACjC,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAA8B,EAAE,MAAc;IAC9E,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;IAC7D,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAClC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAC/D,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACpD,IAAI,IAAI,CAAC,KAAK,CAAE,CAAC,SAAS,KAAK,IAAI,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,SAAS;YAAE,OAAO,KAAK,CAAC;IAC1E,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,WAAW,CACzB,KAA8B,EAC9B,OAAiG;IAEjG,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACvE,MAAM,OAAO,GAAG,oBAAoB,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAChE,IAAI,OAAO,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;QACvC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,2BAA2B,OAAO,GAAG,EAAE,OAAO,EAAE,CAAC;IAC3G,CAAC;IACD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAC/C,IAAI,QAAQ,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;QACzC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,GAAG,QAAQ,4BAA4B,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC1H,CAAC;IACD,IAAI,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9C,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,8BAA8B,MAAM,CAAC,MAAM,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IACzI,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;AACrE,CAAC;AAED,kEAAkE;AAClE,MAAM,CAAC,MAAM,aAAa,GAA4B,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AAuCtG,mCAAmC;AACnC,MAAM,UAAU,WAAW;IACzB,OAAO;QACL,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,CAAC;QACV,SAAS,EAAE,CAAC;QACZ,QAAQ,EAAE,CAAC;QACX,cAAc,EAAE,CAAC;QACjB,eAAe,EAAE,CAAC;QAClB,WAAW,EAAE,CAAC;QACd,cAAc,EAAE,CAAC;QACjB,eAAe,EAAE,CAAC;QAClB,oBAAoB,EAAE,CAAC;QACvB,iBAAiB,EAAE,CAAC;QACpB,cAAc,EAAE,CAAC;KAClB,CAAC;AACJ,CAAC;AAgBD;;;;;;;;;GASG;AACH,MAAM,UAAU,WAAW,CACzB,MAA8B,EAC9B,KAAa,EACb,OAAe,EACf,MAAkB;IAElB,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC;IACtB,MAAM,CAAC,OAAO,IAAI,OAAO,CAAC;IAC1B,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;IAC1C,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;IACxC,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;IACpD,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC,eAAe,IAAI,CAAC,CAAC;IACtD,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;IAC9C,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;IACpD,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC,eAAe,IAAI,CAAC,CAAC;IACtD,MAAM,CAAC,oBAAoB,IAAI,MAAM,CAAC,oBAAoB,IAAI,CAAC,CAAC;IAChE,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,iBAAiB,IAAI,CAAC,CAAC;IAC1D,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;AACtD,CAAC;AAsBD;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CAAC,IAAwB,EAAE,KAA8B,EAAE,eAAuB;IAC3G,OAAO,GAAG,IAAI,IAAI,EAAE,SAAS,KAAK;SAC/B,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;SACpC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC;SAC7B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,UAAU,CACjB,GAA2E,EAC3E,OAA0C;IAE1C,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,GAAG,CAAC;IAClC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;AAC1D,CAAC;AAED,mEAAmE;AACnE,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO;QACL,MAAM,EAAE,KAAK;QACb,IAAI,EAAE,KAAK;QACX,MAAM,EAAE,KAAK;QACb,WAAW,EAAE,CAAC;QACd,cAAc,EAAE,CAAC;QACjB,eAAe,EAAE,CAAC;QAClB,IAAI;QACJ,QAAQ,EAAE,aAAa;KACxB,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAqBjC;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC3D,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IACxG,IAAI,MAAM,EAAE,CAAC;QACX,OAAO;YACL,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC;YACrC,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,IAAI;YACZ,WAAW,EAAE,CAAC;YACd,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,QAAQ,EAAE,MAAM;SACjB,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,cAAc,IAAI,MAAM,CAAC,mBAAmB;QAAE,OAAO,UAAU,CAAC,mBAAmB,CAAC,CAAC;IAC/F,IAAI,KAAK,CAAC,gBAAgB,IAAI,MAAM,CAAC,sBAAsB;QAAE,OAAO,UAAU,CAAC,sBAAsB,CAAC,CAAC;IACvG,IAAI,MAAM,EAAE,OAAO;QAAE,OAAO,UAAU,CAAC,SAAS,CAAC,CAAC;IAClD,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAE7B,MAAM,OAAO,GAAiB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACjD,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,SAAS,EAAE,IAAI,CAAC,EAAE;KACnB,CAAC,CAAC,CAAC;IACJ,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAChC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,cAAc,CAAC,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,EACvG,CAAC,CACF,CAAC;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9C,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,KAAK,CAAC,OAAO,GAAG,QAAQ,CAAC;IACzB,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC;IAC/B,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC;IACvB,OAAO;QACL,MAAM,EAAE,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC;QACvC,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,KAAK;QACb,WAAW;QACX,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,EAAE,GAAG,OAAO,GAAG,eAAe,CAAC;QACpE,eAAe;QACf,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC5B,OAAwG,EACxG,OAAe,EACf,QAAgB;IAEhB,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;IACnH,CAAC;IACD,OAAO;QACL,SAAS,EAAE,CAAC;QACZ,QAAQ,EAAE,CAAC;QACX,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;QAC3G,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,WAAW,EAAE,OAAO,CAAC,WAAW;KACjC,CAAC;AACJ,CAAC","sourcesContent":["/**\n * The deterministic pre-trigger, the judgment call, and the cost accounting.\n *\n * The expensive path of every capability is one System 1 judgment. This module\n * owns the cheap path that decides whether a judgment is worth buying, the\n * once-per-turn call itself (memoized by the window it is about), and the\n * bookkeeping that bills exactly one judgment per call. The transport arrives as\n * one `buy(step)` callback and never as the service itself; the clock is a\n * {@link ClockPort}; neither is imported from a host.\n *\n * @module core/judge\n */\nimport type { ActionStep, StagnationCheckResponse } from '../backend/types.js';\nimport type { ClockPort } from '../ports/clock.js';\nimport type { StagnationBackendPort } from '../ports/decision.js';\nimport type { JudgmentState, JudgmentStep } from './orchestrate-host.js';\nimport type { PolicySnapshot } from './policy.js';\nimport { spendJudgment, usefulVerdict } from './policy.js';\nimport { estimateTokens } from './text.js';\n\n/** One step of one agent's detection window. */\nexport interface ObservedStep {\n /** `tool:argumentsDigest` — the identity a repetition is measured on. */\n signature: string;\n tool: string;\n input: string;\n output: string;\n error: string;\n failed: boolean;\n turn: number;\n at: number;\n /** Whether the step was a legitimate wait query rather than progress. */\n wait?: boolean;\n}\n\n/** Why a deterministic pre-trigger wants a judgment. */\nexport interface StagnationSuspicion {\n suspicious: boolean;\n /** The pattern the trigger suspects, before System 1 confirms it. */\n pattern?: 'repetition' | 'oscillation' | 'failing_retry';\n /** Human-readable trigger facts, for the audit record. */\n note: string;\n /** How many steps the pattern was measured on. */\n repeats: number;\n}\n\n/**\n * Count the identical trailing repeats of one signature.\n * @param steps - observed steps, oldest first.\n * @param signature - the identity to count.\n * @returns the length of the trailing run of that signature.\n */\nexport function countTrailingRepeats(steps: readonly ObservedStep[], signature: string): number {\n let count = 0;\n for (let index = steps.length - 1; index >= 0; index -= 1) {\n if (steps[index]!.signature !== signature) break;\n count += 1;\n }\n return count;\n}\n\n/**\n * Count the trailing consecutive failures.\n * @param steps - observed steps, oldest first.\n * @returns the length of the trailing run of failing steps.\n */\nexport function countTrailingFailures(steps: readonly ObservedStep[]): number {\n let count = 0;\n for (let index = steps.length - 1; index >= 0; index -= 1) {\n if (!steps[index]!.failed) break;\n count += 1;\n }\n return count;\n}\n\n/**\n * Whether the window's tail alternates between exactly two call identities.\n *\n * `A B A B` is the cheapest loop to miss and among the most wasteful to run:\n * neither call repeats consecutively, so a consecutive-repeat trigger never\n * fires.\n * @param steps - observed steps, oldest first.\n * @param window - how many trailing steps to inspect.\n * @returns whether the tail oscillates.\n */\nexport function detectOscillation(steps: readonly ObservedStep[], window: number): boolean {\n const tail = steps.slice(Math.max(0, steps.length - window));\n if (tail.length < 4) return false;\n const signatures = new Set(tail.map((step) => step.signature));\n if (signatures.size !== 2) return false;\n for (let index = 2; index < tail.length; index += 1) {\n if (tail[index]!.signature !== tail[index - 2]!.signature) return false;\n }\n return true;\n}\n\n/**\n * Decide, locally and deterministically, whether a judgment is worth spending.\n *\n * This is the cost gate of the whole surface: the expensive path is a System 1\n * call, and it only opens when the *cheap* path already sees a pattern. A single\n * distinctive call is never judged, however large its output.\n *\n * @param steps - observed steps, oldest first.\n * @param options - thresholds, window, and the current call identity.\n * @returns the suspicion, with `suspicious: false` when nothing justifies a call.\n */\nexport function suspicionOf(\n steps: readonly ObservedStep[],\n options: { repeatThreshold: number; failureThreshold: number; window: number; signature: string }\n): StagnationSuspicion {\n const window = steps.slice(Math.max(0, steps.length - options.window));\n const repeats = countTrailingRepeats(window, options.signature);\n if (repeats >= options.repeatThreshold) {\n return { suspicious: true, pattern: 'repetition', note: `identical call repeated ${repeats}x`, repeats };\n }\n const failures = countTrailingFailures(window);\n if (failures >= options.failureThreshold) {\n return { suspicious: true, pattern: 'failing_retry', note: `${failures} consecutive failing steps`, repeats: failures };\n }\n if (detectOscillation(window, options.window)) {\n return { suspicious: true, pattern: 'oscillation', note: `two calls alternating over ${window.length} steps`, repeats: window.length };\n }\n return { suspicious: false, note: 'no local pattern', repeats: 0 };\n}\n\n/** The verdict that stands in when no judgment could be spent. */\nexport const EMPTY_VERDICT: StagnationCheckResponse = { isStuck: false, confidence: 0, latencyMs: 0 };\n\n/** What one judgment cost, charged exactly once per judgment. */\nexport interface JudgmentCharge {\n judgments: number;\n memoHits: number;\n localLatencyMs: number;\n remoteLatencyMs: number;\n judgeTokens: number;\n}\n\n/** One token and latency account of a mode or a capability. */\nexport interface InterventionCostBucket {\n /** Decisions planned. */\n plans: number;\n /** Decisions whose plan was executed. */\n applied: number;\n /** System 1 judgments paid for. */\n judgments: number;\n /** Judgments served from the per-turn verdict memo instead of a new call. */\n memoHits: number;\n /** Milliseconds spent in local, deterministic work. */\n localLatencyMs: number;\n /** Milliseconds spent waiting for System 1. */\n remoteLatencyMs: number;\n /** Tokens the judgments reasoned about. */\n judgeTokens: number;\n /** Tokens of the payloads decisions were made about. */\n originalTokens: number;\n /** Tokens the candidates would carry. */\n candidateTokens: number;\n /** Tokens the candidates would have saved. */\n potentialSavedTokens: number;\n /** Tokens actually removed from the model-visible surface. */\n actualSavedTokens: number;\n /** Tokens actually added to the model-visible surface. */\n injectedTokens: number;\n}\n\n/** A fresh, zeroed cost bucket. */\nexport function emptyBucket(): InterventionCostBucket {\n return {\n plans: 0,\n applied: 0,\n judgments: 0,\n memoHits: 0,\n localLatencyMs: 0,\n remoteLatencyMs: 0,\n judgeTokens: 0,\n originalTokens: 0,\n candidateTokens: 0,\n potentialSavedTokens: 0,\n actualSavedTokens: 0,\n injectedTokens: 0\n };\n}\n\n/** One charge against a mode's and a capability's account. */\nexport interface CostCharge {\n judgments?: number;\n memoHits?: number;\n localLatencyMs?: number;\n remoteLatencyMs?: number;\n judgeTokens?: number;\n originalTokens?: number;\n candidateTokens?: number;\n potentialSavedTokens?: number;\n actualSavedTokens?: number;\n injectedTokens?: number;\n}\n\n/**\n * Add one charge to one bucket.\n *\n * Every counter has exactly one writer, so a reader can add the parts up without\n * meeting a number twice.\n * @param bucket - the bucket to bill.\n * @param plans - decisions planned by this charge.\n * @param applied - decisions applied by this charge.\n * @param charge - the token/latency/decision counts.\n */\nexport function applyCharge(\n bucket: InterventionCostBucket,\n plans: number,\n applied: number,\n charge: CostCharge\n): void {\n bucket.plans += plans;\n bucket.applied += applied;\n bucket.judgments += charge.judgments ?? 0;\n bucket.memoHits += charge.memoHits ?? 0;\n bucket.localLatencyMs += charge.localLatencyMs ?? 0;\n bucket.remoteLatencyMs += charge.remoteLatencyMs ?? 0;\n bucket.judgeTokens += charge.judgeTokens ?? 0;\n bucket.originalTokens += charge.originalTokens ?? 0;\n bucket.candidateTokens += charge.candidateTokens ?? 0;\n bucket.potentialSavedTokens += charge.potentialSavedTokens ?? 0;\n bucket.actualSavedTokens += charge.actualSavedTokens ?? 0;\n bucket.injectedTokens += charge.injectedTokens ?? 0;\n}\n\n/** The state one judgment memo and the two budgets live in. */\nexport interface VerdictMemoState extends JudgmentState {\n verdictTurn: number;\n /** Identity of the window the cached verdict was about. */\n verdictKey: string;\n verdict?: StagnationCheckResponse;\n}\n\n/** The outcome of one judgment request. */\nexport interface JudgmentOutcome {\n useful: boolean;\n paid: boolean;\n reused: boolean;\n judgeTokens: number;\n localLatencyMs: number;\n remoteLatencyMs: number;\n skip?: string;\n response: StagnationCheckResponse;\n}\n\n/**\n * The memo key of one judgment: the question plus the trailing window it is\n * asked about.\n * @param goal - the wait/context question, when one was asked.\n * @param steps - the window.\n * @param repeatThreshold - how many trailing signatures identify the window.\n * @returns a stable key.\n */\nexport function judgmentKey(goal: string | undefined, steps: readonly ObservedStep[], repeatThreshold: number): string {\n return `${goal ?? ''}\\u0000${steps\n .slice(-Math.max(1, repeatThreshold))\n .map((step) => step.signature)\n .join('|')}`;\n}\n\n/**\n * The one purchase a judgment makes, from whichever channel the caller used.\n * @param buy - the current channel: one call, one step.\n * @param backend - the deprecated channel: the service itself.\n * @returns the purchase to run.\n * @throws Error when a caller supplied neither, which is a programming error.\n */\nfunction purchaseOf(\n buy: ((step: JudgmentStep) => Promise<StagnationCheckResponse>) | undefined,\n backend: StagnationBackendPort | undefined\n): (step: JudgmentStep) => Promise<StagnationCheckResponse> {\n if (buy !== undefined) return buy;\n if (backend === undefined) {\n throw new Error('runJudgment needs a purchase: pass buy(step)');\n }\n return (step) => backend.check(step.goal, step.history);\n}\n\n/** The zero-cost verdict of a judgment that could not be spent. */\nfunction noJudgment(skip: string): JudgmentOutcome {\n return {\n useful: false,\n paid: false,\n reused: false,\n judgeTokens: 0,\n localLatencyMs: 0,\n remoteLatencyMs: 0,\n skip,\n response: EMPTY_VERDICT\n };\n}\n\n/**\n * Ask for one stagnation verdict, reusing this turn's answer when the window is\n * the same one.\n *\n * The question is the trailing window plus the context it is asked in, so the\n * answer is cached against both: the same episode is never judged twice, while\n * a *different* episode or a different wait context in the same turn is a new\n * question — and therefore subject to the turn's budget.\n *\n * The purchase is this function's, not the host's: `buy` is called only after\n * the memo missed, after both budgets admitted the question, and after\n * {@link spendJudgment} booked the slot. A host that was handed the transport\n * instead would have to reproduce that order, and the one place a wrong answer is\n * a model-visible regression is the place the type would stop helping.\n *\n * @param args - state, policy, the host's `buy`, clock, window, question, and\n * abort signal.\n * @returns the outcome, including why a judgment was refused.\n */\nexport async function runJudgment(args: {\n state: VerdictMemoState;\n policy: PolicySnapshot;\n /** One purchase: ask the service about one step, and report what it cost. */\n buy?: (step: JudgmentStep) => Promise<StagnationCheckResponse>;\n /**\n * The pre-g-046 channel: a service handed over whole, instead of a purchase.\n *\n * Kept **only** for the frozen parity snapshot\n * (`packages/dsh-harness-reflex/src/baseline/interventions.ts`), which is the\n * oracle of the refactor and may not be touched. Every live caller passes\n * {@link buy}; the snapshot's `#judge` still passes its composition's service,\n * and the parity run therefore keeps this channel exercised.\n *\n * @deprecated Pass `buy`. Nothing added from now on should use this.\n */\n backend?: StagnationBackendPort;\n clock: ClockPort;\n steps: readonly ObservedStep[];\n goal?: string;\n signal?: AbortSignal;\n}): Promise<JudgmentOutcome> {\n const { state, policy, clock, steps, goal, signal } = args;\n const buy = purchaseOf(args.buy, args.backend);\n const key = judgmentKey(goal, steps, policy.repeatThreshold);\n const cached = state.verdictTurn === state.turn && state.verdictKey === key ? state.verdict : undefined;\n if (cached) {\n return {\n useful: usefulVerdict(cached, policy),\n paid: false,\n reused: true,\n judgeTokens: 0,\n localLatencyMs: 0,\n remoteLatencyMs: 0,\n response: cached\n };\n }\n if (state.judgmentsCount >= policy.maxJudgmentsPerTurn) return noJudgment('judgment-turn-cap');\n if (state.judgmentsSession >= policy.maxJudgmentsPerSession) return noJudgment('judgment-session-cap');\n if (signal?.aborted) return noJudgment('aborted');\n spendJudgment(state, policy);\n\n const history: ActionStep[] = steps.map((step) => ({\n tool: step.tool,\n input: step.input,\n ...(step.output ? { output: step.output } : {}),\n ...(step.error ? { error: step.error } : {}),\n timestamp: step.at\n }));\n const judgeTokens = history.reduce(\n (sum, step) => sum + estimateTokens(`${step.tool}${step.input}${step.output ?? ''}${step.error ?? ''}`),\n 0\n );\n const started = clock.now();\n const response = await buy({ goal, history });\n const remoteLatencyMs = Number.isFinite(response.latencyMs) ? response.latencyMs : 0;\n state.verdict = response;\n state.verdictTurn = state.turn;\n state.verdictKey = key;\n return {\n useful: usefulVerdict(response, policy),\n paid: true,\n reused: false,\n judgeTokens,\n localLatencyMs: Math.max(0, clock.now() - started - remoteLatencyMs),\n remoteLatencyMs,\n response\n };\n}\n\n/**\n * The charge of one judgment, so every judgment is billed exactly once.\n * @param verdict - the outcome of the judgment call.\n * @param started - clock reading at hook entry.\n * @param finished - clock reading now.\n * @returns the charge.\n */\nexport function judgmentCharge(\n verdict: Pick<JudgmentOutcome, 'paid' | 'reused' | 'judgeTokens' | 'localLatencyMs' | 'remoteLatencyMs'>,\n started: number,\n finished: number\n): JudgmentCharge {\n if (!verdict.paid) {\n return { judgments: 0, memoHits: verdict.reused ? 1 : 0, localLatencyMs: 0, remoteLatencyMs: 0, judgeTokens: 0 };\n }\n return {\n judgments: 1,\n memoHits: 0,\n localLatencyMs: Math.max(verdict.localLatencyMs, Math.max(0, finished - started - verdict.remoteLatencyMs)),\n remoteLatencyMs: verdict.remoteLatencyMs,\n judgeTokens: verdict.judgeTokens\n };\n}\n"]}
|