@causyne/agent 1.10.21 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/native-skills/media/professional-document-publishing/SKILL.md +44 -0
- package/native-skills/media/screenplay-and-shotlist-forge/SKILL.md +42 -0
- package/package.json +1 -1
- package/src/acp/client.js +235 -0
- package/src/acp/client.ts +282 -0
- package/src/acp/peerProfiles.js +122 -0
- package/src/acp/peerProfiles.ts +128 -0
- package/src/acp/protocol.js +54 -0
- package/src/acp/protocol.ts +434 -0
- package/src/acp/server.js +609 -0
- package/src/acp/server.ts +676 -0
- package/src/agentLoop.d.ts +43 -0
- package/src/agentLoop.js +142 -8
- package/src/commands.js +203 -0
- package/src/compiledSkills.js +10 -0
- package/src/documentEngine/antiSlopAuditor.js +156 -0
- package/src/documentEngine/antiSlopAuditor.ts +221 -0
- package/src/documentEngine/documentForge.js +443 -0
- package/src/documentEngine/documentForge.ts +519 -0
- package/src/documentEngine/screenplayFormatter.js +336 -0
- package/src/documentEngine/screenplayFormatter.ts +407 -0
- package/src/entrypoint.js +13 -0
- package/src/execSafe.js +10 -13
- package/src/execSafe.ts +25 -0
- package/src/modelRouter.js +56 -123
- package/src/modelRouter.ts +134 -0
- package/src/observer/advisory.js +164 -0
- package/src/observer/advisory.ts +171 -0
- package/src/observer/coldObserver.js +462 -0
- package/src/observer/coldObserver.ts +540 -0
- package/src/observer/evidence.js +435 -0
- package/src/observer/evidence.ts +499 -0
- package/src/perception.js +303 -118
- package/src/perception.ts +387 -0
- package/src/procSignal.js +15 -16
- package/src/procSignal.ts +31 -0
- package/src/promptMentions.js +170 -0
- package/src/promptMentions.ts +212 -0
- package/src/sandbox.js +1 -1
- package/src/textDiff.js +117 -109
- package/src/textDiff.ts +169 -0
- package/src/toolPacks.js +5 -2
- package/src/tools/documentForge.js +215 -0
- package/src/tools/documentForge.ts +265 -0
- package/src/tools/peerAgent.js +102 -0
- package/src/tools/peerAgent.ts +116 -0
- package/src/tools/registry.js +3 -1
- package/src/tui/completion.js +97 -41
- package/src/tui/prompt.js +17 -17
- package/src/uncertainty.js +93 -205
- package/src/uncertainty.ts +255 -0
- package/src/utils/paths.js +15 -18
- package/src/utils/paths.ts +36 -0
- package/src/vfeTracker.js +122 -181
- package/src/vfeTracker.ts +242 -0
package/src/uncertainty.js
CHANGED
|
@@ -1,94 +1,8 @@
|
|
|
1
1
|
import { failureClass } from './planCache.js';
|
|
2
|
-
|
|
3
|
-
// Step-level uncertainty, computed from signals the loop already holds.
|
|
4
|
-
//
|
|
5
|
-
// WHAT THIS IS NOT, AND WHY.
|
|
6
|
-
//
|
|
7
|
-
// The research spec asked for CATTS-style gating: sample N candidate actions,
|
|
8
|
-
// measure entropy and the top-1/top-2 margin across them, and escalate when the
|
|
9
|
-
// samples disagree. Two findings killed that design.
|
|
10
|
-
//
|
|
11
|
-
// 1. The CATTS and OPPO papers are NOT in the research corpus. Asked
|
|
12
|
-
// directly, it reported their absence three times. The spec cited them
|
|
13
|
-
// second-hand, so their thresholds were never available to us.
|
|
14
|
-
// 2. More decisively: without token logprobs, every method in that family
|
|
15
|
-
// measures disagreement by SAMPLING N completions. Causyne is BYOK across
|
|
16
|
-
// ten providers and several expose no logprobs at all. Sampling N times to
|
|
17
|
-
// decide whether to spend more is self-defeating — the gate would cost
|
|
18
|
-
// more than the thing it gates.
|
|
19
|
-
//
|
|
20
|
-
// Asked which substitutes exist when logprobs are unavailable, the corpus named
|
|
21
|
-
// four: majority voting over N samples, an auxiliary verifier model, cross-tool
|
|
22
|
-
// execution matching, and verbalized self-critique. The first two each cost an
|
|
23
|
-
// extra model call. **The last two do not**, and they are what this module is
|
|
24
|
-
// built from, together with execution signals the loop already tracks for free.
|
|
25
|
-
//
|
|
26
|
-
// WHAT THE CORPUS THEN CONFIRMED (research round 3, questions A1-A7).
|
|
27
|
-
//
|
|
28
|
-
// "Dynamic Adaptive Reasoning: Intent-Aware Scheduling" (IARS) detects its
|
|
29
|
-
// reasoning states — Exploring, Confirming, Stuck/Ambiguous, Near-Answer — with
|
|
30
|
-
// one of two pipelines: a small trained classifier, OR **a text-based heuristic
|
|
31
|
-
// of keyword matching plus structural repetition analysis**, run as a parallel
|
|
32
|
-
// stream analyser with negligible cost and no logprobs. That second pipeline is
|
|
33
|
-
// this module. The thrash detector below is its "structural repetition"; the
|
|
34
|
-
// hedge patterns are its "keyword matching". So this is an implementation of a
|
|
35
|
-
// published detector, not an extrapolation.
|
|
36
|
-
//
|
|
37
|
-
// IARS issues four directives — [DEEPER_THINK], [STREAMLINE_THINK],
|
|
38
|
-
// [CONCLUDE_NOW], [RETRY_SUBPROBLEM] — and couples them to decoding:
|
|
39
|
-
// DEEPER_THINK raises temperature to widen the search; CONCLUDE_NOW drops into
|
|
40
|
-
// fast decoding. The paper states the DIRECTION of those changes and NOT the
|
|
41
|
-
// magnitudes ("present, but not stated"). The delta below is therefore ours.
|
|
42
|
-
//
|
|
43
|
-
// Two things the corpus ruled OUT, so nobody re-proposes them:
|
|
44
|
-
// * REVERSE ("Generate, but Verify") gates on P(</UN>) > tau with tau = 0.003
|
|
45
|
-
// (LLaVA) / 0.01 (Qwen2.5-VL). It needs a model FINE-TUNED to emit that
|
|
46
|
-
// token — unavailable over BYOK — and its measured overhead is 3.05x tokens.
|
|
47
|
-
// A gate that triples the bill is the failure this module exists to avoid.
|
|
48
|
-
// * Verbalized self-confidence (a model rating its own confidence) is NOT
|
|
49
|
-
// evaluated or calibrated anywhere in the corpus. The hedge patterns here
|
|
50
|
-
// are used as IARS-style Stuck/Ambiguous keyword detection, never as a
|
|
51
|
-
// calibrated confidence score, and they cannot cross the gate alone.
|
|
52
|
-
//
|
|
53
|
-
// HONEST LIMITS:
|
|
54
|
-
// * No single signal can move the score past the escalation threshold alone.
|
|
55
|
-
// * The WEIGHTS are engineering judgement, not published constants. The
|
|
56
|
-
// corpus validates the SIGNALS (IARS, OBLIVION's query-to-buffer cosine
|
|
57
|
-
// similarity) but publishes no weights for combining them. They are named
|
|
58
|
-
// constants so measured values can replace them without touching logic.
|
|
59
|
-
|
|
60
|
-
/**
|
|
61
|
-
* @typedef {Object} StepSignals
|
|
62
|
-
* @property {number} [consecutiveFailures] Failures in a row, from the loop.
|
|
63
|
-
* @property {any} [lastResult] The previous tool result, if any.
|
|
64
|
-
* @property {string} [lastTool] Name of the previous tool.
|
|
65
|
-
* @property {string} [pendingTool] The tool about to be called.
|
|
66
|
-
* @property {string[]}[recentTools] Recent tool names, oldest first.
|
|
67
|
-
* @property {number} [planSimilarity] 0..1 plan-cache match, if any.
|
|
68
|
-
* @property {number} [lessonSimilarity] 0..1 Cortex retrieval match, if any.
|
|
69
|
-
* @property {string} [assistantText] The model's own last message.
|
|
70
|
-
* @property {number} [logprobMargin] top1-top2 in nats, when a provider exposes it.
|
|
71
|
-
* @property {boolean} [vfePlateau] Whether variational free energy has plateaued.
|
|
72
|
-
*/
|
|
73
|
-
|
|
74
|
-
/**
|
|
75
|
-
* @typedef {Object} Assessment
|
|
76
|
-
* @property {number} uncertainty 0 (settled) .. 1 (lost).
|
|
77
|
-
* @property {'confident'|'contested'|'stuck'} level
|
|
78
|
-
* @property {string[]} signals Which contributors fired, for the audit trail.
|
|
79
|
-
* @property {string} directive Prompt text to inject, or '' to stay silent.
|
|
80
|
-
* @property {number} temperatureDelta Added to the task temperature for this
|
|
81
|
-
* step. IARS couples its escalation directive to a wider
|
|
82
|
-
* search; the direction is the paper's, the magnitude is
|
|
83
|
-
* ours (see STUCK_TEMPERATURE_DELTA).
|
|
84
|
-
*/
|
|
85
|
-
|
|
86
2
|
/** Above this, the step is contested enough to be worth a reflection directive. */
|
|
87
3
|
export const CONTESTED_AT = 0.45;
|
|
88
|
-
|
|
89
4
|
/** Above this, the agent is not converging and should re-plan rather than retry. */
|
|
90
5
|
export const STUCK_AT = 0.75;
|
|
91
|
-
|
|
92
6
|
/**
|
|
93
7
|
* How much wider the search gets when the agent is stuck.
|
|
94
8
|
*
|
|
@@ -98,7 +12,6 @@ export const STUCK_AT = 0.75;
|
|
|
98
12
|
* stuck agent needs a different route, not a random one.
|
|
99
13
|
*/
|
|
100
14
|
export const STUCK_TEMPERATURE_DELTA = 0.2;
|
|
101
|
-
|
|
102
15
|
/**
|
|
103
16
|
* No single signal may cross CONTESTED_AT alone.
|
|
104
17
|
*
|
|
@@ -107,165 +20,140 @@ export const STUCK_TEMPERATURE_DELTA = 0.2;
|
|
|
107
20
|
* itself would fire reflection on ordinary hedging language.
|
|
108
21
|
*/
|
|
109
22
|
const MAX_SINGLE_CONTRIBUTION = 0.4;
|
|
110
|
-
|
|
111
23
|
const WEIGHTS = {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
24
|
+
consecutiveFailure: 0.25, // per failure, capped below
|
|
25
|
+
// Weighted ABOVE a bare failure count on purpose. Three failures say
|
|
26
|
+
// "something is wrong"; a `not_found` class says WHAT — and a diagnosis is
|
|
27
|
+
// worth more than a symptom. With this at 0.15, three failures on a wrong
|
|
28
|
+
// route scored 0.60 and never reached the retry directive that exists for
|
|
29
|
+
// exactly that case.
|
|
30
|
+
hardFailureClass: 0.30, // a failure that says the ROUTE is wrong
|
|
31
|
+
thrash: 0.30, // the same tool three times running
|
|
32
|
+
vfePlateau: 0.35, // Active Inference: error plateau with zero variance (stuck in local minimum)
|
|
33
|
+
noPriorArt: 0.05, // nothing in plan cache or Cortex to lean on
|
|
34
|
+
hedging: 0.20, // the model's own language says it is unsure
|
|
35
|
+
logprobMargin: 0.25 // only when a provider actually exposes it
|
|
124
36
|
};
|
|
125
|
-
|
|
126
37
|
/**
|
|
127
38
|
* Phrases in which a model reports its own uncertainty.
|
|
128
39
|
*
|
|
129
40
|
* Deliberately narrow. A broad list turns ordinary careful writing into a
|
|
130
41
|
* signal, and a gate that fires on caution is a gate that fires always.
|
|
131
|
-
* @type {RegExp}
|
|
132
42
|
*/
|
|
133
43
|
const HEDGE = /\b(i am not sure|i'm not sure|not certain|unclear (?:why|what|how)|might be|may be wrong|hard to tell|cannot determine|let me guess|assuming that|if that is the case|unsure)\b/i;
|
|
134
|
-
|
|
135
44
|
/**
|
|
136
45
|
* Failure classes that mean the ROUTE is wrong rather than the moment.
|
|
137
46
|
*
|
|
138
47
|
* `network` and `rate_limited` are absent on purpose: retrying is the correct
|
|
139
48
|
* response to those, and treating them as uncertainty would trigger a re-plan
|
|
140
49
|
* for a problem that fixes itself.
|
|
141
|
-
* @type {ReadonlySet<string>}
|
|
142
50
|
*/
|
|
143
51
|
const ROUTE_IS_WRONG = new Set(['not_found', 'permission', 'invalid_args', 'syntax']);
|
|
144
|
-
|
|
145
52
|
/**
|
|
146
53
|
* Is the agent calling the same tool over and over?
|
|
147
54
|
*
|
|
148
55
|
* Repetition is the cheapest honest uncertainty signal there is: a converging
|
|
149
56
|
* agent moves on. Three of the same in a row is the threshold because two is a
|
|
150
57
|
* legitimate read-then-write pair.
|
|
151
|
-
*
|
|
152
|
-
* @param {string[]} recent
|
|
153
|
-
* @param {string} pending
|
|
154
|
-
* @returns {boolean}
|
|
155
58
|
*/
|
|
156
59
|
export function isThrashing(recent, pending) {
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
60
|
+
const tail = [...(recent || [])];
|
|
61
|
+
if (pending)
|
|
62
|
+
tail.push(pending);
|
|
63
|
+
if (tail.length < 3)
|
|
64
|
+
return false;
|
|
65
|
+
const last3 = tail.slice(-3);
|
|
66
|
+
return last3.every((t) => t && t === last3[0]);
|
|
162
67
|
}
|
|
163
|
-
|
|
164
68
|
/**
|
|
165
69
|
* Score a step before committing to it.
|
|
166
70
|
*
|
|
167
71
|
* Every input is something the loop already has. Nothing here calls a model,
|
|
168
72
|
* samples a completion, or touches the network — which is the entire point.
|
|
169
|
-
*
|
|
170
|
-
* @param {StepSignals} signals
|
|
171
|
-
* @returns {Assessment}
|
|
172
73
|
*/
|
|
173
74
|
export function assessStep(signals = {}) {
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
level,
|
|
221
|
-
signals: fired,
|
|
222
|
-
directive: directiveFor(level, fired),
|
|
223
|
-
temperatureDelta: level === 'stuck' ? STUCK_TEMPERATURE_DELTA : 0
|
|
224
|
-
};
|
|
75
|
+
const fired = [];
|
|
76
|
+
let score = 0;
|
|
77
|
+
/** Add a contributor, capped so no single one can decide the outcome. */
|
|
78
|
+
const add = (amount, name) => {
|
|
79
|
+
const capped = Math.min(amount, MAX_SINGLE_CONTRIBUTION);
|
|
80
|
+
if (capped <= 0)
|
|
81
|
+
return;
|
|
82
|
+
score += capped;
|
|
83
|
+
fired.push(name);
|
|
84
|
+
};
|
|
85
|
+
const failures = Number(signals.consecutiveFailures) || 0;
|
|
86
|
+
if (failures > 0)
|
|
87
|
+
add(Math.min(failures, 3) * WEIGHTS.consecutiveFailure, `failures:${failures}`);
|
|
88
|
+
if (signals.lastResult) {
|
|
89
|
+
const why = failureClass(signals.lastResult);
|
|
90
|
+
if (why && ROUTE_IS_WRONG.has(why))
|
|
91
|
+
add(WEIGHTS.hardFailureClass, `route_wrong:${why}`);
|
|
92
|
+
}
|
|
93
|
+
if (isThrashing(signals.recentTools, signals.pendingTool)) {
|
|
94
|
+
add(WEIGHTS.thrash, `thrash:${signals.pendingTool || 'repeat'}`);
|
|
95
|
+
}
|
|
96
|
+
if (signals.vfePlateau) {
|
|
97
|
+
add(WEIGHTS.vfePlateau, 'vfe_plateau');
|
|
98
|
+
}
|
|
99
|
+
const bestPrior = Math.max(Number(signals.planSimilarity) || 0, Number(signals.lessonSimilarity) || 0);
|
|
100
|
+
if (bestPrior === 0)
|
|
101
|
+
add(WEIGHTS.noPriorArt, 'no_prior_art');
|
|
102
|
+
if (typeof signals.assistantText === 'string' && HEDGE.test(signals.assistantText)) {
|
|
103
|
+
add(WEIGHTS.hedging, 'model_hedged');
|
|
104
|
+
}
|
|
105
|
+
// Only when the provider genuinely reports it. A missing margin is NOT
|
|
106
|
+
// evidence of confidence, so its absence contributes nothing in either
|
|
107
|
+
// direction.
|
|
108
|
+
if (typeof signals.logprobMargin === 'number' && Number.isFinite(signals.logprobMargin)) {
|
|
109
|
+
if (signals.logprobMargin < 0.5)
|
|
110
|
+
add(WEIGHTS.logprobMargin, 'logprob_margin_narrow');
|
|
111
|
+
}
|
|
112
|
+
const uncertainty = Math.min(1, score);
|
|
113
|
+
const level = uncertainty >= STUCK_AT ? 'stuck' : uncertainty >= CONTESTED_AT ? 'contested' : 'confident';
|
|
114
|
+
return {
|
|
115
|
+
uncertainty,
|
|
116
|
+
level,
|
|
117
|
+
signals: fired,
|
|
118
|
+
directive: directiveFor(level, fired),
|
|
119
|
+
temperatureDelta: level === 'stuck' ? STUCK_TEMPERATURE_DELTA : 0
|
|
120
|
+
};
|
|
225
121
|
}
|
|
226
|
-
|
|
227
122
|
/**
|
|
228
123
|
* What to say to the model, if anything.
|
|
229
124
|
*
|
|
230
125
|
* Returns '' for a confident step. That empty string is the whole economic
|
|
231
126
|
* argument: reflecting on every step is what the gating literature exists to
|
|
232
127
|
* prevent, and the directive costs tokens on every step it appears in.
|
|
233
|
-
*
|
|
234
|
-
* @param {string} level
|
|
235
|
-
* @param {string[]} fired
|
|
236
|
-
* @returns {string}
|
|
237
128
|
*/
|
|
238
129
|
export function directiveFor(level, fired = []) {
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
130
|
+
if (level === 'confident')
|
|
131
|
+
return '';
|
|
132
|
+
const because = fired.length ? ` (${fired.join(', ')})` : '';
|
|
133
|
+
if (level === 'stuck') {
|
|
134
|
+
// Paradigm shift: when VFE plateau occurs, standard re-planning isn't enough;
|
|
135
|
+
// the agent is stuck in an active-inference local minimum.
|
|
136
|
+
if (fired.includes('vfe_plateau')) {
|
|
137
|
+
return `[PARADIGM SHIFT REQUIRED — persistent error plateau with zero convergence${because}. `
|
|
138
|
+
+ 'Do NOT retry previous step variations or tweak command arguments. '
|
|
139
|
+
+ 'Invalidate your operational hypothesis, state in one line what fundamental assumption was wrong, '
|
|
140
|
+
+ 'and switch to a completely different tool, perspective, or strategy.]';
|
|
141
|
+
}
|
|
142
|
+
// IARS separates [RETRY_SUBPROBLEM] from a full re-plan. A wrong ROUTE
|
|
143
|
+
// with no thrashing means this one sub-problem needs a different approach;
|
|
144
|
+
// thrashing means the plan itself is not converging.
|
|
145
|
+
const thrashing = fired.some((f) => f.startsWith('thrash:'));
|
|
146
|
+
if (!thrashing) {
|
|
147
|
+
return `[RETRY_SUBPROBLEM — the last approach to THIS step failed in a way that says the route was wrong${because}. `
|
|
148
|
+
+ 'Keep the overall plan. Solve this one step by a different route. '
|
|
149
|
+
+ 'If a tool reported "not found", that is evidence about the ROUTE, not proof the thing is absent.]';
|
|
150
|
+
}
|
|
151
|
+
return `[RE-PLAN — the last few steps are not converging${because}. `
|
|
152
|
+
+ 'Do NOT retry the same route. State in one line what you now believe is actually true, '
|
|
153
|
+
+ 'name the assumption that turned out to be wrong, and choose a different approach. '
|
|
154
|
+
+ 'If a tool reported "not found", that is evidence about the ROUTE, not proof the thing is absent.]';
|
|
261
155
|
}
|
|
262
|
-
return `[
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
+ 'If a tool reported "not found", that is evidence about the ROUTE, not proof the thing is absent.]';
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
return `[CHECK YOURSELF — this step looks contested${because}. `
|
|
269
|
-
+ 'Before calling the tool, say in one line what you expect it to return. '
|
|
270
|
-
+ 'If the result contradicts that expectation, treat the expectation as the thing that was wrong.]';
|
|
156
|
+
return `[CHECK YOURSELF — this step looks contested${because}. `
|
|
157
|
+
+ 'Before calling the tool, say in one line what you expect it to return. '
|
|
158
|
+
+ 'If the result contradicts that expectation, treat the expectation as the thing that was wrong.]';
|
|
271
159
|
}
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import { failureClass } from './planCache.js';
|
|
2
|
+
|
|
3
|
+
// Step-level uncertainty, computed from signals the loop already holds.
|
|
4
|
+
//
|
|
5
|
+
// WHAT THIS IS NOT, AND WHY.
|
|
6
|
+
//
|
|
7
|
+
// The research spec asked for CATTS-style gating: sample N candidate actions,
|
|
8
|
+
// measure entropy and the top-1/top-2 margin across them, and escalate when the
|
|
9
|
+
// samples disagree. Two findings killed that design.
|
|
10
|
+
//
|
|
11
|
+
// 1. The CATTS and OPPO papers are NOT in the research corpus. Asked
|
|
12
|
+
// directly, it reported their absence three times. The spec cited them
|
|
13
|
+
// second-hand, so their thresholds were never available to us.
|
|
14
|
+
// 2. More decisively: without token logprobs, every method in that family
|
|
15
|
+
// measures disagreement by SAMPLING N completions. Causyne is BYOK across
|
|
16
|
+
// ten providers and several expose no logprobs at all. Sampling N times to
|
|
17
|
+
// decide whether to spend more is self-defeating — the gate would cost
|
|
18
|
+
// more than the thing it gates.
|
|
19
|
+
//
|
|
20
|
+
// Asked which substitutes exist when logprobs are unavailable, the corpus named
|
|
21
|
+
// four: majority voting over N samples, an auxiliary verifier model, cross-tool
|
|
22
|
+
// execution matching, and verbalized self-critique. The first two each cost an
|
|
23
|
+
// extra model call. **The last two do not**, and they are what this module is
|
|
24
|
+
// built from, together with execution signals the loop already tracks for free.
|
|
25
|
+
//
|
|
26
|
+
// WHAT THE CORPUS THEN CONFIRMED (research round 3, questions A1-A7).
|
|
27
|
+
//
|
|
28
|
+
// "Dynamic Adaptive Reasoning: Intent-Aware Scheduling" (IARS) detects its
|
|
29
|
+
// reasoning states — Exploring, Confirming, Stuck/Ambiguous, Near-Answer — with
|
|
30
|
+
// one of two pipelines: a small trained classifier, OR **a text-based heuristic
|
|
31
|
+
// of keyword matching plus structural repetition analysis**, run as a parallel
|
|
32
|
+
// stream analyser with negligible cost and no logprobs. That second pipeline is
|
|
33
|
+
// this module. The thrash detector below is its "structural repetition"; the
|
|
34
|
+
// hedge patterns are its "keyword matching". So this is an implementation of a
|
|
35
|
+
// published detector, not an extrapolation.
|
|
36
|
+
//
|
|
37
|
+
// IARS issues four directives — [DEEPER_THINK], [STREAMLINE_THINK],
|
|
38
|
+
// [CONCLUDE_NOW], [RETRY_SUBPROBLEM] — and couples them to decoding:
|
|
39
|
+
// DEEPER_THINK raises temperature to widen the search; CONCLUDE_NOW drops into
|
|
40
|
+
// fast decoding. The paper states the DIRECTION of those changes and NOT the
|
|
41
|
+
// magnitudes ("present, but not stated"). The delta below is therefore ours.
|
|
42
|
+
//
|
|
43
|
+
// Two things the corpus ruled OUT, so nobody re-proposes them:
|
|
44
|
+
// * REVERSE ("Generate, but Verify") gates on P(</UN>) > tau with tau = 0.003
|
|
45
|
+
// (LLaVA) / 0.01 (Qwen2.5-VL). It needs a model FINE-TUNED to emit that
|
|
46
|
+
// token — unavailable over BYOK — and its measured overhead is 3.05x tokens.
|
|
47
|
+
// A gate that triples the bill is the failure this module exists to avoid.
|
|
48
|
+
// * Verbalized self-confidence (a model rating its own confidence) is NOT
|
|
49
|
+
// evaluated or calibrated anywhere in the corpus. The hedge patterns here
|
|
50
|
+
// are used as IARS-style Stuck/Ambiguous keyword detection, never as a
|
|
51
|
+
// calibrated confidence score, and they cannot cross the gate alone.
|
|
52
|
+
//
|
|
53
|
+
// HONEST LIMITS:
|
|
54
|
+
// * No single signal can move the score past the escalation threshold alone.
|
|
55
|
+
// * The WEIGHTS are engineering judgement, not published constants. The
|
|
56
|
+
// corpus validates the SIGNALS (IARS, OBLIVION's query-to-buffer cosine
|
|
57
|
+
// similarity) but publishes no weights for combining them. They are named
|
|
58
|
+
// constants so measured values can replace them without touching logic.
|
|
59
|
+
|
|
60
|
+
export interface StepSignals {
|
|
61
|
+
consecutiveFailures?: number | undefined;
|
|
62
|
+
lastResult?: unknown;
|
|
63
|
+
lastTool?: string | undefined;
|
|
64
|
+
pendingTool?: string | undefined;
|
|
65
|
+
recentTools?: string[] | undefined;
|
|
66
|
+
planSimilarity?: number | undefined;
|
|
67
|
+
lessonSimilarity?: number | undefined;
|
|
68
|
+
assistantText?: string | undefined;
|
|
69
|
+
logprobMargin?: number | undefined;
|
|
70
|
+
vfePlateau?: boolean | undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export type UncertaintyLevel = 'confident' | 'contested' | 'stuck';
|
|
74
|
+
|
|
75
|
+
export interface Assessment {
|
|
76
|
+
uncertainty: number;
|
|
77
|
+
level: UncertaintyLevel;
|
|
78
|
+
signals: string[];
|
|
79
|
+
directive: string;
|
|
80
|
+
temperatureDelta: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Above this, the step is contested enough to be worth a reflection directive. */
|
|
84
|
+
export const CONTESTED_AT = 0.45;
|
|
85
|
+
|
|
86
|
+
/** Above this, the agent is not converging and should re-plan rather than retry. */
|
|
87
|
+
export const STUCK_AT = 0.75;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* How much wider the search gets when the agent is stuck.
|
|
91
|
+
*
|
|
92
|
+
* IARS: [DEEPER_THINK] / [RETRY_SUBPROBLEM] raise decoding temperature to
|
|
93
|
+
* escape a local minimum. The paper gives the direction and no number
|
|
94
|
+
* ("present, but not stated"), so this value is unmeasured. Kept small: a
|
|
95
|
+
* stuck agent needs a different route, not a random one.
|
|
96
|
+
*/
|
|
97
|
+
export const STUCK_TEMPERATURE_DELTA = 0.2;
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* No single signal may cross CONTESTED_AT alone.
|
|
101
|
+
*
|
|
102
|
+
* This is the rule that keeps a badly-calibrated signal from driving the gate.
|
|
103
|
+
* Verbalized confidence in particular is unreliable enough that acting on it by
|
|
104
|
+
* itself would fire reflection on ordinary hedging language.
|
|
105
|
+
*/
|
|
106
|
+
const MAX_SINGLE_CONTRIBUTION = 0.4;
|
|
107
|
+
|
|
108
|
+
const WEIGHTS = {
|
|
109
|
+
consecutiveFailure: 0.25, // per failure, capped below
|
|
110
|
+
// Weighted ABOVE a bare failure count on purpose. Three failures say
|
|
111
|
+
// "something is wrong"; a `not_found` class says WHAT — and a diagnosis is
|
|
112
|
+
// worth more than a symptom. With this at 0.15, three failures on a wrong
|
|
113
|
+
// route scored 0.60 and never reached the retry directive that exists for
|
|
114
|
+
// exactly that case.
|
|
115
|
+
hardFailureClass: 0.30, // a failure that says the ROUTE is wrong
|
|
116
|
+
thrash: 0.30, // the same tool three times running
|
|
117
|
+
vfePlateau: 0.35, // Active Inference: error plateau with zero variance (stuck in local minimum)
|
|
118
|
+
noPriorArt: 0.05, // nothing in plan cache or Cortex to lean on
|
|
119
|
+
hedging: 0.20, // the model's own language says it is unsure
|
|
120
|
+
logprobMargin: 0.25 // only when a provider actually exposes it
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Phrases in which a model reports its own uncertainty.
|
|
125
|
+
*
|
|
126
|
+
* Deliberately narrow. A broad list turns ordinary careful writing into a
|
|
127
|
+
* signal, and a gate that fires on caution is a gate that fires always.
|
|
128
|
+
*/
|
|
129
|
+
const HEDGE = /\b(i am not sure|i'm not sure|not certain|unclear (?:why|what|how)|might be|may be wrong|hard to tell|cannot determine|let me guess|assuming that|if that is the case|unsure)\b/i;
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Failure classes that mean the ROUTE is wrong rather than the moment.
|
|
133
|
+
*
|
|
134
|
+
* `network` and `rate_limited` are absent on purpose: retrying is the correct
|
|
135
|
+
* response to those, and treating them as uncertainty would trigger a re-plan
|
|
136
|
+
* for a problem that fixes itself.
|
|
137
|
+
*/
|
|
138
|
+
const ROUTE_IS_WRONG = new Set(['not_found', 'permission', 'invalid_args', 'syntax']);
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Is the agent calling the same tool over and over?
|
|
142
|
+
*
|
|
143
|
+
* Repetition is the cheapest honest uncertainty signal there is: a converging
|
|
144
|
+
* agent moves on. Three of the same in a row is the threshold because two is a
|
|
145
|
+
* legitimate read-then-write pair.
|
|
146
|
+
*/
|
|
147
|
+
export function isThrashing(recent?: string[] | null, pending?: string | null): boolean {
|
|
148
|
+
const tail = [...(recent || [])];
|
|
149
|
+
if (pending) tail.push(pending);
|
|
150
|
+
if (tail.length < 3) return false;
|
|
151
|
+
const last3 = tail.slice(-3);
|
|
152
|
+
return last3.every((t) => t && t === last3[0]);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Score a step before committing to it.
|
|
157
|
+
*
|
|
158
|
+
* Every input is something the loop already has. Nothing here calls a model,
|
|
159
|
+
* samples a completion, or touches the network — which is the entire point.
|
|
160
|
+
*/
|
|
161
|
+
export function assessStep(signals: StepSignals = {}): Assessment {
|
|
162
|
+
const fired: string[] = [];
|
|
163
|
+
let score = 0;
|
|
164
|
+
|
|
165
|
+
/** Add a contributor, capped so no single one can decide the outcome. */
|
|
166
|
+
const add = (amount: number, name: string): void => {
|
|
167
|
+
const capped = Math.min(amount, MAX_SINGLE_CONTRIBUTION);
|
|
168
|
+
if (capped <= 0) return;
|
|
169
|
+
score += capped;
|
|
170
|
+
fired.push(name);
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
const failures = Number(signals.consecutiveFailures) || 0;
|
|
174
|
+
if (failures > 0) add(Math.min(failures, 3) * WEIGHTS.consecutiveFailure, `failures:${failures}`);
|
|
175
|
+
|
|
176
|
+
if (signals.lastResult) {
|
|
177
|
+
const why = failureClass(signals.lastResult);
|
|
178
|
+
if (why && ROUTE_IS_WRONG.has(why)) add(WEIGHTS.hardFailureClass, `route_wrong:${why}`);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (isThrashing(signals.recentTools, signals.pendingTool)) {
|
|
182
|
+
add(WEIGHTS.thrash, `thrash:${signals.pendingTool || 'repeat'}`);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
if (signals.vfePlateau) {
|
|
186
|
+
add(WEIGHTS.vfePlateau, 'vfe_plateau');
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const bestPrior = Math.max(Number(signals.planSimilarity) || 0, Number(signals.lessonSimilarity) || 0);
|
|
190
|
+
if (bestPrior === 0) add(WEIGHTS.noPriorArt, 'no_prior_art');
|
|
191
|
+
|
|
192
|
+
if (typeof signals.assistantText === 'string' && HEDGE.test(signals.assistantText)) {
|
|
193
|
+
add(WEIGHTS.hedging, 'model_hedged');
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Only when the provider genuinely reports it. A missing margin is NOT
|
|
197
|
+
// evidence of confidence, so its absence contributes nothing in either
|
|
198
|
+
// direction.
|
|
199
|
+
if (typeof signals.logprobMargin === 'number' && Number.isFinite(signals.logprobMargin)) {
|
|
200
|
+
if (signals.logprobMargin < 0.5) add(WEIGHTS.logprobMargin, 'logprob_margin_narrow');
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const uncertainty = Math.min(1, score);
|
|
204
|
+
const level: UncertaintyLevel = uncertainty >= STUCK_AT ? 'stuck' : uncertainty >= CONTESTED_AT ? 'contested' : 'confident';
|
|
205
|
+
|
|
206
|
+
return {
|
|
207
|
+
uncertainty,
|
|
208
|
+
level,
|
|
209
|
+
signals: fired,
|
|
210
|
+
directive: directiveFor(level, fired),
|
|
211
|
+
temperatureDelta: level === 'stuck' ? STUCK_TEMPERATURE_DELTA : 0
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* What to say to the model, if anything.
|
|
217
|
+
*
|
|
218
|
+
* Returns '' for a confident step. That empty string is the whole economic
|
|
219
|
+
* argument: reflecting on every step is what the gating literature exists to
|
|
220
|
+
* prevent, and the directive costs tokens on every step it appears in.
|
|
221
|
+
*/
|
|
222
|
+
export function directiveFor(level: string, fired: string[] = []): string {
|
|
223
|
+
if (level === 'confident') return '';
|
|
224
|
+
|
|
225
|
+
const because = fired.length ? ` (${fired.join(', ')})` : '';
|
|
226
|
+
|
|
227
|
+
if (level === 'stuck') {
|
|
228
|
+
// Paradigm shift: when VFE plateau occurs, standard re-planning isn't enough;
|
|
229
|
+
// the agent is stuck in an active-inference local minimum.
|
|
230
|
+
if (fired.includes('vfe_plateau')) {
|
|
231
|
+
return `[PARADIGM SHIFT REQUIRED — persistent error plateau with zero convergence${because}. `
|
|
232
|
+
+ 'Do NOT retry previous step variations or tweak command arguments. '
|
|
233
|
+
+ 'Invalidate your operational hypothesis, state in one line what fundamental assumption was wrong, '
|
|
234
|
+
+ 'and switch to a completely different tool, perspective, or strategy.]';
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// IARS separates [RETRY_SUBPROBLEM] from a full re-plan. A wrong ROUTE
|
|
238
|
+
// with no thrashing means this one sub-problem needs a different approach;
|
|
239
|
+
// thrashing means the plan itself is not converging.
|
|
240
|
+
const thrashing = fired.some((f) => f.startsWith('thrash:'));
|
|
241
|
+
if (!thrashing) {
|
|
242
|
+
return `[RETRY_SUBPROBLEM — the last approach to THIS step failed in a way that says the route was wrong${because}. `
|
|
243
|
+
+ 'Keep the overall plan. Solve this one step by a different route. '
|
|
244
|
+
+ 'If a tool reported "not found", that is evidence about the ROUTE, not proof the thing is absent.]';
|
|
245
|
+
}
|
|
246
|
+
return `[RE-PLAN — the last few steps are not converging${because}. `
|
|
247
|
+
+ 'Do NOT retry the same route. State in one line what you now believe is actually true, '
|
|
248
|
+
+ 'name the assumption that turned out to be wrong, and choose a different approach. '
|
|
249
|
+
+ 'If a tool reported "not found", that is evidence about the ROUTE, not proof the thing is absent.]';
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
return `[CHECK YOURSELF — this step looks contested${because}. `
|
|
253
|
+
+ 'Before calling the tool, say in one line what you expect it to return. '
|
|
254
|
+
+ 'If the result contradicts that expectation, treat the expectation as the thing that was wrong.]';
|
|
255
|
+
}
|