agent-nuvira 3.5.0 → 3.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -13
- package/dist/agent-sdk/src/scaffold.d.ts +1 -1
- package/dist/agent-sdk/src/scaffold.js +1 -1
- package/dist/cli/chat.d.ts +159 -6
- package/dist/cli/chat.d.ts.map +1 -1
- package/dist/cli/chat.js +583 -35
- package/dist/cli/chat.js.map +1 -1
- package/dist/cli/duties.d.ts +11 -0
- package/dist/cli/duties.d.ts.map +1 -1
- package/dist/cli/duties.js +31 -0
- package/dist/cli/duties.js.map +1 -1
- package/dist/cli/edit.d.ts.map +1 -1
- package/dist/cli/edit.js +3 -2
- package/dist/cli/edit.js.map +1 -1
- package/dist/cli/execute.d.ts.map +1 -1
- package/dist/cli/execute.js +10 -3
- package/dist/cli/execute.js.map +1 -1
- package/dist/cli/loop-executor.d.ts +16 -0
- package/dist/cli/loop-executor.d.ts.map +1 -1
- package/dist/cli/loop-executor.js +9 -0
- package/dist/cli/loop-executor.js.map +1 -1
- package/dist/cli/models.d.ts.map +1 -1
- package/dist/cli/models.js +55 -0
- package/dist/cli/models.js.map +1 -1
- package/dist/cli/plan.d.ts.map +1 -1
- package/dist/cli/plan.js +18 -6
- package/dist/cli/plan.js.map +1 -1
- package/dist/cli/run.d.ts.map +1 -1
- package/dist/cli/run.js +23 -8
- package/dist/cli/run.js.map +1 -1
- package/dist/cli/ui/answer-view.d.ts +38 -0
- package/dist/cli/ui/answer-view.d.ts.map +1 -0
- package/dist/cli/ui/answer-view.js +72 -0
- package/dist/cli/ui/answer-view.js.map +1 -0
- package/dist/cli/ui/cards.d.ts +61 -0
- package/dist/cli/ui/cards.d.ts.map +1 -0
- package/dist/cli/ui/cards.js +141 -0
- package/dist/cli/ui/cards.js.map +1 -0
- package/dist/cli/ui/highlight.d.ts +32 -0
- package/dist/cli/ui/highlight.d.ts.map +1 -0
- package/dist/cli/ui/highlight.js +261 -0
- package/dist/cli/ui/highlight.js.map +1 -0
- package/dist/cli/ui/index.d.ts +21 -0
- package/dist/cli/ui/index.d.ts.map +1 -0
- package/dist/cli/ui/index.js +14 -0
- package/dist/cli/ui/index.js.map +1 -0
- package/dist/cli/ui/markdown.d.ts +90 -0
- package/dist/cli/ui/markdown.d.ts.map +1 -0
- package/dist/cli/ui/markdown.js +292 -0
- package/dist/cli/ui/markdown.js.map +1 -0
- package/dist/cli/ui/presenter.d.ts +147 -0
- package/dist/cli/ui/presenter.d.ts.map +1 -0
- package/dist/cli/ui/presenter.js +313 -0
- package/dist/cli/ui/presenter.js.map +1 -0
- package/dist/cli/ui/render.d.ts +52 -0
- package/dist/cli/ui/render.d.ts.map +1 -0
- package/dist/cli/ui/render.js +384 -0
- package/dist/cli/ui/render.js.map +1 -0
- package/dist/cli/ui/theme.d.ts +36 -0
- package/dist/cli/ui/theme.d.ts.map +1 -0
- package/dist/cli/ui/theme.js +92 -0
- package/dist/cli/ui/theme.js.map +1 -0
- package/dist/inference/model-entitlement.d.ts +10 -0
- package/dist/inference/model-entitlement.d.ts.map +1 -1
- package/dist/inference/model-entitlement.js +1 -1
- package/dist/inference/model-entitlement.js.map +1 -1
- package/dist/learning/auto-router.d.ts +38 -1
- package/dist/learning/auto-router.d.ts.map +1 -1
- package/dist/learning/auto-router.js +139 -8
- package/dist/learning/auto-router.js.map +1 -1
- package/dist/learning/duty-lease.d.ts +163 -0
- package/dist/learning/duty-lease.d.ts.map +1 -0
- package/dist/learning/duty-lease.js +301 -0
- package/dist/learning/duty-lease.js.map +1 -0
- package/dist/learning/model-first-router.d.ts +14 -0
- package/dist/learning/model-first-router.d.ts.map +1 -1
- package/dist/learning/model-first-router.js +54 -17
- package/dist/learning/model-first-router.js.map +1 -1
- package/dist/learning/model-warmup.d.ts +92 -0
- package/dist/learning/model-warmup.d.ts.map +1 -1
- package/dist/learning/model-warmup.js +137 -2
- package/dist/learning/model-warmup.js.map +1 -1
- package/dist/learning/reasoning-trace.d.ts +21 -1
- package/dist/learning/reasoning-trace.d.ts.map +1 -1
- package/dist/learning/reasoning-trace.js.map +1 -1
- package/dist/learning/resilient-call.d.ts.map +1 -1
- package/dist/learning/resilient-call.js +70 -4
- package/dist/learning/resilient-call.js.map +1 -1
- package/dist/learning/retry-patience.d.ts +139 -0
- package/dist/learning/retry-patience.d.ts.map +1 -0
- package/dist/learning/retry-patience.js +169 -0
- package/dist/learning/retry-patience.js.map +1 -0
- package/dist/tools/process-liveness.d.ts +227 -0
- package/dist/tools/process-liveness.d.ts.map +1 -0
- package/dist/tools/process-liveness.js +486 -0
- package/dist/tools/process-liveness.js.map +1 -0
- package/dist/tools/registry.d.ts.map +1 -1
- package/dist/tools/registry.js +4 -2
- package/dist/tools/registry.js.map +1 -1
- package/dist/tools/run-registry.d.ts +282 -0
- package/dist/tools/run-registry.d.ts.map +1 -0
- package/dist/tools/run-registry.js +544 -0
- package/dist/tools/run-registry.js.map +1 -0
- package/dist/tools/run-terminal.d.ts +100 -2
- package/dist/tools/run-terminal.d.ts.map +1 -1
- package/dist/tools/run-terminal.js +841 -67
- package/dist/tools/run-terminal.js.map +1 -1
- package/dist/tools/tool-loop.d.ts +87 -3
- package/dist/tools/tool-loop.d.ts.map +1 -1
- package/dist/tools/tool-loop.js +375 -6
- package/dist/tools/tool-loop.js.map +1 -1
- package/dist/web-dashboard/chat-console.d.ts +25 -0
- package/dist/web-dashboard/chat-console.d.ts.map +1 -1
- package/dist/web-dashboard/chat-console.js +16 -1
- package/dist/web-dashboard/chat-console.js.map +1 -1
- package/dist/web-dashboard/server.d.ts.map +1 -1
- package/dist/web-dashboard/server.js +7 -1
- package/dist/web-dashboard/server.js.map +1 -1
- package/dist/web-dashboard/src/types.d.ts +32 -1
- package/dist/web-dashboard/src/types.d.ts.map +1 -1
- package/dist/web-dashboard/task-registry.d.ts +98 -0
- package/dist/web-dashboard/task-registry.d.ts.map +1 -0
- package/dist/web-dashboard/task-registry.js +159 -0
- package/dist/web-dashboard/task-registry.js.map +1 -0
- package/dist/web-dashboard/task-runner.d.ts +155 -5
- package/dist/web-dashboard/task-runner.d.ts.map +1 -1
- package/dist/web-dashboard/task-runner.js +505 -14
- package/dist/web-dashboard/task-runner.js.map +1 -1
- package/package.json +4 -1
- package/src/web-dashboard/public/assets/index-DycGBWwy.js +236 -0
- package/src/web-dashboard/public/assets/index-DycGBWwy.js.map +1 -0
- package/src/web-dashboard/public/index.html +1 -1
- package/src/web-dashboard/public/assets/index-yky0dw5e.js +0 -236
- package/src/web-dashboard/public/assets/index-yky0dw5e.js.map +0 -1
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Blip patience — how long to STAY on the pair that just failed, and whether
|
|
3
|
+
* walking away is even a remedy.
|
|
4
|
+
*
|
|
5
|
+
* THE OBSERVED PROBLEM THIS ENCODES. A cloud provider does not go down for a
|
|
6
|
+
* turn: it returns 200, then a few empty/5xx responses for a handful of seconds,
|
|
7
|
+
* then serves normally again (measured repeatedly on `deepseek`). Two opposite
|
|
8
|
+
* mistakes are available, and the codebase has made both:
|
|
9
|
+
*
|
|
10
|
+
* - FAIL FAST: give up after seconds and fold onto the next candidate — which
|
|
11
|
+
* for that blip means silently dropping from a strong model to a weak one.
|
|
12
|
+
* The result is answered, badly, by a model the user did not choose.
|
|
13
|
+
* - WAIT BLINDLY: sit out a long fixed backoff on a pair that is actually
|
|
14
|
+
* dead, and make every user of a broken provider pay for it.
|
|
15
|
+
*
|
|
16
|
+
* A timeout cannot tell those apart, so a fixed schedule is the wrong shape
|
|
17
|
+
* either way. What CAN tell them apart is the evidence already in hand:
|
|
18
|
+
*
|
|
19
|
+
* 1. THE ALTERNATIVES. Walking away is only a remedy when it does not cost
|
|
20
|
+
* quality. If every alternative sits in a LOWER capability band, then
|
|
21
|
+
* "fail over" means "de-escalate", and for a blip that is the worse trade —
|
|
22
|
+
* so we wait instead. If a same-or-better band is reachable, failing over is
|
|
23
|
+
* free and waiting is pure waste.
|
|
24
|
+
* 2. THE PAIR'S TRACK RECORD. A pair that has served real traffic with a low
|
|
25
|
+
* error rate earns patience; a never-proven id does not.
|
|
26
|
+
* 3. THE SHAPE OF THE FAILURE. One failing step is a blip. The SAME pair
|
|
27
|
+
* failing two steps in a row is not — patience collapses there, whatever
|
|
28
|
+
* the alternatives look like.
|
|
29
|
+
*
|
|
30
|
+
* And the schedule's SHAPE matters as much as its total: a 2-second blink is
|
|
31
|
+
* caught by a retry at t≈1s, not at t=5s, so the early delays are dense and the
|
|
32
|
+
* late ones sparse ({@link TRANSIENT_BLIP_DELAYS_MS} → attempts at ≈1, 3, 7, 15,
|
|
33
|
+
* 30, 60s). That rides out both a blink and a real minute-long spike.
|
|
34
|
+
*
|
|
35
|
+
* PURE AND DETERMINISTIC. This module reads no clock, no registry and no config:
|
|
36
|
+
* the caller gathers the evidence (it has the registry and the ranked pool) and
|
|
37
|
+
* gets back a plan it can log. That is what makes the policy reviewable and
|
|
38
|
+
* testable one row at a time — see the table in the tests.
|
|
39
|
+
*/
|
|
40
|
+
import { bandRank, capabilityClassOf } from './model-capability.js';
|
|
41
|
+
/**
|
|
42
|
+
* Transient (server/network/timeout) backoff, dense early and sparse late:
|
|
43
|
+
* attempts at ≈1s, 3s, 7s, 15s, 30s, 60s. Used exactly when WAITING beats
|
|
44
|
+
* walking away — no alternative, or only degrading ones.
|
|
45
|
+
*/
|
|
46
|
+
export const TRANSIENT_BLIP_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 15_000, 30_000];
|
|
47
|
+
/**
|
|
48
|
+
* Transient backoff when a same-or-better-band candidate IS reachable: a short
|
|
49
|
+
* courtesy retry (two blinks' worth, <2s) and then the walk. Spending longer is
|
|
50
|
+
* waste — the replacement costs nothing in quality.
|
|
51
|
+
*/
|
|
52
|
+
export const TRANSIENT_FREE_ALTERNATIVE_DELAYS_MS = [500, 1_500];
|
|
53
|
+
/**
|
|
54
|
+
* Transient backoff for a pair we have no reason to trust and no alternative to:
|
|
55
|
+
* a token wait (≈5s), then the honest failure.
|
|
56
|
+
*/
|
|
57
|
+
export const TRANSIENT_LAST_RESORT_DELAYS_MS = [1_000, 4_000];
|
|
58
|
+
/** Empty-response patience for a proven pair — the "final verse" case. */
|
|
59
|
+
export const EMPTY_PATIENCE_MS = [2_000, 5_000];
|
|
60
|
+
/**
|
|
61
|
+
* Empty-response patience when EVERY alternative would de-escalate. Longer, and
|
|
62
|
+
* deliberately so: this is the one situation where the choice is literally
|
|
63
|
+
* "wait a few more seconds" vs "answer with a weaker model".
|
|
64
|
+
*/
|
|
65
|
+
export const EMPTY_PATIENCE_WAIT_BEATS_FOLDING_MS = [2_000, 5_000, 10_000];
|
|
66
|
+
/**
|
|
67
|
+
* Error rate at or below which a verified pair counts as a proven track record.
|
|
68
|
+
* Above it the pair has failed roughly 1 call in 5, which is not something a
|
|
69
|
+
* blip explains away.
|
|
70
|
+
*/
|
|
71
|
+
export const PROVEN_ERROR_RATE_MAX = 0.2;
|
|
72
|
+
/** Is the registry evidence strong enough to grant a pair extra patience? */
|
|
73
|
+
function hasProvenTrackRecord(evidence) {
|
|
74
|
+
if (!evidence?.verified)
|
|
75
|
+
return false;
|
|
76
|
+
if ((evidence.errorRate ?? 0) > PROVEN_ERROR_RATE_MAX)
|
|
77
|
+
return false;
|
|
78
|
+
// A `probe` (catalog listing) is not evidence about answers; `telemetry` and
|
|
79
|
+
// `spot-check` are.
|
|
80
|
+
return evidence.source !== 'probe';
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Decide how patient to be with the pair that just failed, and say why.
|
|
84
|
+
*
|
|
85
|
+
* The whole policy, in order (see the tests for each row):
|
|
86
|
+
*
|
|
87
|
+
* 1. The pair already failed an earlier step of this turn → NOT a blip. One
|
|
88
|
+
* token wait, no empty patience.
|
|
89
|
+
* 2. Nothing to gain by patience: a same-or-better band is reachable → short
|
|
90
|
+
* transient retry; empty patched only for a proven pair (a good pair's
|
|
91
|
+
* first blink is still worth two short waits).
|
|
92
|
+
* 3. A PROVEN pair facing only degrading alternatives (or none) → long, dense
|
|
93
|
+
* transient patience, and extended empty patience. This is the row that
|
|
94
|
+
* stops a blinking strong model from being replaced by a weak one.
|
|
95
|
+
* 4. An UNPROVEN pair with nowhere better to go → a token wait; the walk (or
|
|
96
|
+
* the honest failure) is the remedy, not optimism about a stranger.
|
|
97
|
+
*/
|
|
98
|
+
export function planBlipPatience(input) {
|
|
99
|
+
const currentBand = capabilityClassOf(input.current.model);
|
|
100
|
+
const currentRank = bandRank(currentBand);
|
|
101
|
+
let bestRank = -1;
|
|
102
|
+
let bestAlternativeBand;
|
|
103
|
+
for (const alternative of input.alternatives) {
|
|
104
|
+
if (!alternative?.model)
|
|
105
|
+
continue;
|
|
106
|
+
const band = capabilityClassOf(alternative.model);
|
|
107
|
+
const rank = bandRank(band);
|
|
108
|
+
if (rank > bestRank) {
|
|
109
|
+
bestRank = rank;
|
|
110
|
+
bestAlternativeBand = band;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const hasAlternative = bestRank >= 0;
|
|
114
|
+
const sameOrBetterReachable = bestRank >= currentRank;
|
|
115
|
+
const foldingWouldDegrade = hasAlternative && !sameOrBetterReachable;
|
|
116
|
+
const proven = hasProvenTrackRecord(input.evidence) || input.deliveredWorkThisTurn === true;
|
|
117
|
+
// 1. Two failing steps in a row is a pattern, not a blip.
|
|
118
|
+
if (input.failedEarlierStep) {
|
|
119
|
+
return {
|
|
120
|
+
transientDelaysMs: TRANSIENT_LAST_RESORT_DELAYS_MS,
|
|
121
|
+
emptyResponseDelaysMs: [],
|
|
122
|
+
bestAlternativeBand,
|
|
123
|
+
foldingWouldDegrade: false,
|
|
124
|
+
reason: `${input.current.provider}/${input.current.model} failed an earlier step too — not a blip, so no patience`,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
// 2. A same-or-better candidate is reachable: failing over costs no quality.
|
|
128
|
+
if (sameOrBetterReachable) {
|
|
129
|
+
return {
|
|
130
|
+
transientDelaysMs: TRANSIENT_FREE_ALTERNATIVE_DELAYS_MS,
|
|
131
|
+
emptyResponseDelaysMs: proven ? EMPTY_PATIENCE_MS : [],
|
|
132
|
+
bestAlternativeBand,
|
|
133
|
+
foldingWouldDegrade: false,
|
|
134
|
+
reason: `a ${bestAlternativeBand}-band candidate is reachable — failing over costs no quality`,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
// 3. Waiting beats folding down a band.
|
|
138
|
+
if (foldingWouldDegrade && proven) {
|
|
139
|
+
return {
|
|
140
|
+
transientDelaysMs: TRANSIENT_BLIP_DELAYS_MS,
|
|
141
|
+
emptyResponseDelaysMs: EMPTY_PATIENCE_WAIT_BEATS_FOLDING_MS,
|
|
142
|
+
bestAlternativeBand,
|
|
143
|
+
foldingWouldDegrade: true,
|
|
144
|
+
reason: `waiting on ${input.current.provider}/${input.current.model} (${currentBand}) rather than ` +
|
|
145
|
+
`de-escalating to ${bestAlternativeBand} — a blip is not a reason to downgrade`,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
// 3b. Nowhere else to go, but the pair has a track record: waiting is the only remedy.
|
|
149
|
+
if (!hasAlternative && proven) {
|
|
150
|
+
return {
|
|
151
|
+
transientDelaysMs: TRANSIENT_BLIP_DELAYS_MS,
|
|
152
|
+
emptyResponseDelaysMs: EMPTY_PATIENCE_WAIT_BEATS_FOLDING_MS,
|
|
153
|
+
bestAlternativeBand,
|
|
154
|
+
foldingWouldDegrade: false,
|
|
155
|
+
reason: `no alternative is configured — waiting on ${input.current.provider}/${input.current.model} is the only remedy`,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
// 4. Unproven pair: a token wait, then the walk or the honest failure.
|
|
159
|
+
return {
|
|
160
|
+
transientDelaysMs: TRANSIENT_LAST_RESORT_DELAYS_MS,
|
|
161
|
+
emptyResponseDelaysMs: hasAlternative ? [] : EMPTY_PATIENCE_MS,
|
|
162
|
+
bestAlternativeBand,
|
|
163
|
+
foldingWouldDegrade,
|
|
164
|
+
reason: hasAlternative
|
|
165
|
+
? `${input.current.provider}/${input.current.model} has no track record — the walk is the remedy`
|
|
166
|
+
: `nothing is proven for ${input.current.provider}/${input.current.model} and there is no alternative`,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
//# sourceMappingURL=retry-patience.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry-patience.js","sourceRoot":"","sources":["../../src/learning/retry-patience.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAuB,MAAM,uBAAuB,CAAC;AAEzF;;;;GAIG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAU,CAAC;AAE9F;;;;GAIG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAU,CAAC;AAE1E;;;GAGG;AACH,MAAM,CAAC,MAAM,+BAA+B,GAAG,CAAC,KAAK,EAAE,KAAK,CAAU,CAAC;AAEvE,0EAA0E;AAC1E,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,KAAK,EAAE,KAAK,CAAU,CAAC;AAEzD;;;;GAIG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAU,CAAC;AAEpF;;;;GAIG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAiDzC,6EAA6E;AAC7E,SAAS,oBAAoB,CAAC,QAAsC;IAClE,IAAI,CAAC,QAAQ,EAAE,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtC,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC,GAAG,qBAAqB;QAAE,OAAO,KAAK,CAAC;IACpE,6EAA6E;IAC7E,oBAAoB;IACpB,OAAO,QAAQ,CAAC,MAAM,KAAK,OAAO,CAAC;AACrC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAoB;IACnD,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC3D,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;IAE1C,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,IAAI,mBAA+C,CAAC;IACpD,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;QAC7C,IAAI,CAAC,WAAW,EAAE,KAAK;YAAE,SAAS;QAClC,MAAM,IAAI,GAAG,iBAAiB,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;YACpB,QAAQ,GAAG,IAAI,CAAC;YAChB,mBAAmB,GAAG,IAAI,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,QAAQ,IAAI,CAAC,CAAC;IACrC,MAAM,qBAAqB,GAAG,QAAQ,IAAI,WAAW,CAAC;IACtD,MAAM,mBAAmB,GAAG,cAAc,IAAI,CAAC,qBAAqB,CAAC;IACrE,MAAM,MAAM,GAAG,oBAAoB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,qBAAqB,KAAK,IAAI,CAAC;IAE5F,0DAA0D;IAC1D,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;QAC5B,OAAO;YACL,iBAAiB,EAAE,+BAA+B;YAClD,qBAAqB,EAAE,EAAE;YACzB,mBAAmB;YACnB,mBAAmB,EAAE,KAAK;YAC1B,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,0DAA0D;SACnH,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,IAAI,qBAAqB,EAAE,CAAC;QAC1B,OAAO;YACL,iBAAiB,EAAE,oCAAoC;YACvD,qBAAqB,EAAE,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE;YACtD,mBAAmB;YACnB,mBAAmB,EAAE,KAAK;YAC1B,MAAM,EAAE,KAAK,mBAAmB,8DAA8D;SAC/F,CAAC;IACJ,CAAC;IAED,wCAAwC;IACxC,IAAI,mBAAmB,IAAI,MAAM,EAAE,CAAC;QAClC,OAAO;YACL,iBAAiB,EAAE,wBAAwB;YAC3C,qBAAqB,EAAE,oCAAoC;YAC3D,mBAAmB;YACnB,mBAAmB,EAAE,IAAI;YACzB,MAAM,EACJ,cAAc,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,KAAK,WAAW,gBAAgB;gBAC3F,oBAAoB,mBAAmB,wCAAwC;SAClF,CAAC;IACJ,CAAC;IAED,uFAAuF;IACvF,IAAI,CAAC,cAAc,IAAI,MAAM,EAAE,CAAC;QAC9B,OAAO;YACL,iBAAiB,EAAE,wBAAwB;YAC3C,qBAAqB,EAAE,oCAAoC;YAC3D,mBAAmB;YACnB,mBAAmB,EAAE,KAAK;YAC1B,MAAM,EAAE,6CAA6C,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,qBAAqB;SACxH,CAAC;IACJ,CAAC;IAED,uEAAuE;IACvE,OAAO;QACL,iBAAiB,EAAE,+BAA+B;QAClD,qBAAqB,EAAE,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,iBAAiB;QAC9D,mBAAmB;QACnB,mBAAmB;QACnB,MAAM,EAAE,cAAc;YACpB,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,+CAA+C;YACjG,CAAC,CAAC,yBAAyB,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,8BAA8B;KACzG,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process liveness — is a long-running command WORKING, or is it HUNG?
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS. Handing a command to monitoring instead of killing it at a
|
|
5
|
+
* clock (see `run-terminal.ts`) removed the false failure, but it left the
|
|
6
|
+
* opposite hole open: a command that is genuinely STUCK — waiting on a prompt
|
|
7
|
+
* with no stdin, holding a lock, blocked on a socket that will never answer —
|
|
8
|
+
* now runs forever, and "it is still running" is not a diagnosis. Silence alone
|
|
9
|
+
* cannot tell the two apart, because a real test suite can be quiet for minutes
|
|
10
|
+
* while it works. The signal that separates them is CPU:
|
|
11
|
+
*
|
|
12
|
+
* - producing output → it is doing something
|
|
13
|
+
* - burning CPU with no output → it is computing (a build, a solver)
|
|
14
|
+
* - NO output AND NO CPU for a long time → it is waiting for something that is
|
|
15
|
+
* never coming. That is a hang, and it is a FACT about the process, not a
|
|
16
|
+
* guess from a stopwatch.
|
|
17
|
+
*
|
|
18
|
+
* So this module samples the process TREE (the shell `spawn(..., { shell: true })`
|
|
19
|
+
* creates plus everything under it) for cumulative CPU time, and classifies a
|
|
20
|
+
* run into `working` / `stalled` / `hung` with the evidence attached. Callers
|
|
21
|
+
* decide what to do with the verdict; the one rule this module enforces is that
|
|
22
|
+
* `hung` requires POSITIVE evidence (measurable CPU that is not advancing) —
|
|
23
|
+
* when CPU cannot be measured, a long silence is reported as `stalled`, never as
|
|
24
|
+
* `hung`, because an unmeasurable process must not be reaped on a guess.
|
|
25
|
+
*
|
|
26
|
+
* Platform: on POSIX the table comes from `ps` (macOS + Linux), the same tool
|
|
27
|
+
* `src/cli/process-control.ts` already relies on. On Windows it comes from ONE
|
|
28
|
+
* `Get-CimInstance Win32_Process` query — which reports `ParentProcessId` AND
|
|
29
|
+
* `KernelModeTime` + `UserModeTime` (CPU, in 100ns units) — so the same rule
|
|
30
|
+
* works there instead of degrading to silence-only: the tree walk and the
|
|
31
|
+
* zero-CPU evidence are both available, with `wmic` as the fallback the rest of
|
|
32
|
+
* this repo already uses.
|
|
33
|
+
*
|
|
34
|
+
* One Windows semantic difference is worth stating plainly, because `kill`
|
|
35
|
+
* implies something it cannot deliver there: Windows has no graceful SIGTERM.
|
|
36
|
+
* `process.kill(pid, 'SIGTERM')` is an immediate termination, so the
|
|
37
|
+
* SIGTERM→grace→SIGKILL ladder is a POSIX-only refinement.
|
|
38
|
+
*/
|
|
39
|
+
/** How a long run is doing: the state a clock alone cannot distinguish. */
|
|
40
|
+
export type Liveness = 'finished' | 'starting' | 'working' | 'stalled' | 'hung';
|
|
41
|
+
/** The thresholds the classification is made from (ms). */
|
|
42
|
+
export interface LivenessThresholds {
|
|
43
|
+
/** Silence after which a run is worth *reporting* as stalled. */
|
|
44
|
+
stallQuietMs: number;
|
|
45
|
+
/** Silence after which, WITH no CPU, a run is HUNG. */
|
|
46
|
+
hangQuietMs: number;
|
|
47
|
+
/** CPU ms over one sample window that counts as "it is computing". */
|
|
48
|
+
hangCpuFloorMs: number;
|
|
49
|
+
}
|
|
50
|
+
export declare const DEFAULT_LIVENESS_THRESHOLDS: LivenessThresholds;
|
|
51
|
+
/**
|
|
52
|
+
* The thresholds, with env overrides — read at CALL time, not module load, so a
|
|
53
|
+
* test (or an operator) can change them without re-importing.
|
|
54
|
+
*
|
|
55
|
+
* `NUVIRA_HANG_QUIET_MS` is the one that matters operationally: it is how long a
|
|
56
|
+
* command may be silent AND idle before the agent is told it is hung.
|
|
57
|
+
*/
|
|
58
|
+
export declare function livenessThresholds(env?: NodeJS.ProcessEnv): LivenessThresholds;
|
|
59
|
+
/**
|
|
60
|
+
* Parse a `ps -o time=` value into milliseconds of cumulative CPU.
|
|
61
|
+
*
|
|
62
|
+
* Formats seen in the wild: `MM:SS.cc` (macOS), `HH:MM:SS` (Linux), and
|
|
63
|
+
* `DD-HH:MM:SS` (BSD, for a long-lived process). A value that does not parse is
|
|
64
|
+
* `null` — the caller must treat "no reading" differently from "no CPU", or a
|
|
65
|
+
* formatting change would read as a hang.
|
|
66
|
+
*/
|
|
67
|
+
export declare function parseCpuTime(text: string): number | null;
|
|
68
|
+
export interface ProcessTreeSample {
|
|
69
|
+
/** The root pid was found in the process table. */
|
|
70
|
+
found: boolean;
|
|
71
|
+
/** Cumulative CPU (ms) of the root AND every descendant; null = unmeasurable. */
|
|
72
|
+
cpuMs: number | null;
|
|
73
|
+
/** The tree, root first, children after (deepest last) — the kill order. */
|
|
74
|
+
pids: number[];
|
|
75
|
+
}
|
|
76
|
+
/** One row of the process table. */
|
|
77
|
+
export interface ProcessRow {
|
|
78
|
+
ppid: number;
|
|
79
|
+
/** Cumulative CPU (user+sys) ms of THIS process; null = unmeasurable. */
|
|
80
|
+
cpuMs: number | null;
|
|
81
|
+
/** The process line, when the platform reports one (used for identity checks). */
|
|
82
|
+
command: string | null;
|
|
83
|
+
/**
|
|
84
|
+
* When the OS says this process STARTED, as an opaque per-platform token —
|
|
85
|
+
* `lstart` text on POSIX, Windows file-time ticks. `null` = unreadable.
|
|
86
|
+
*
|
|
87
|
+
* It exists to answer one question a pid cannot: is this the SAME process?
|
|
88
|
+
* A pid is a number the kernel reuses (Windows recycles aggressively, Linux
|
|
89
|
+
* wraps at a few million), so `pid === 1234` is not identity — it is a hint.
|
|
90
|
+
* This is a VALUE the kernel will not hand to two different processes, which
|
|
91
|
+
* is what makes it a witness. Kept as the raw token rather than a parsed date
|
|
92
|
+
* on purpose: two reads of the same process produce the same token, and
|
|
93
|
+
* comparing tokens cannot be skewed by a clock step, a timezone change or a
|
|
94
|
+
* rounding error. Unknown stays `null`, never a fabricated value.
|
|
95
|
+
*/
|
|
96
|
+
startedAtText: string | null;
|
|
97
|
+
}
|
|
98
|
+
export type ProcessTable = Map<number, ProcessRow>;
|
|
99
|
+
/**
|
|
100
|
+
* Parse a POSIX process table: `ps -eo pid=,ppid=,time=,lstart=,command=` — and
|
|
101
|
+
* the older `…,command=` shape, which parses identically minus the start token.
|
|
102
|
+
*
|
|
103
|
+
* The command column is LAST on purpose: it may contain spaces and anything else,
|
|
104
|
+
* so everything after the known fields is taken verbatim.
|
|
105
|
+
*/
|
|
106
|
+
export declare function parsePosixProcessTable(stdout: string): ProcessTable;
|
|
107
|
+
/**
|
|
108
|
+
* Parse the Windows probe: `<pid>|<ppid>|<kernel 100ns>|<user 100ns>|<command>`.
|
|
109
|
+
*
|
|
110
|
+
* The pipe-delimited shape is the one `src/cli/process-control.ts` already uses
|
|
111
|
+
* for the same reason: it survives command lines that contain spaces, commas or
|
|
112
|
+
* quotes. A process whose CPU counters are unreadable (`KernelModeTime` is null
|
|
113
|
+
* for some system processes) yields an EMPTY field, which must stay
|
|
114
|
+
* "unmeasurable" — not zero — or an unreadable counter would read as a hang.
|
|
115
|
+
*
|
|
116
|
+
* 100ns units are converted to ms (÷10 000), the same unit `ps` reports in.
|
|
117
|
+
*/
|
|
118
|
+
export declare function parseWindowsProcessTable(stdout: string): ProcessTable;
|
|
119
|
+
/**
|
|
120
|
+
* Parse the `wmic … /format:csv` fallback.
|
|
121
|
+
*
|
|
122
|
+
* Header-driven, because wmic emits its columns in QUERY order and the values
|
|
123
|
+
* themselves can contain commas — so the header is the only reliable index.
|
|
124
|
+
*/
|
|
125
|
+
export declare function parseWmicProcessTable(stdout: string): ProcessTable;
|
|
126
|
+
/**
|
|
127
|
+
* One process table read, for a caller that needs the SAME view several times.
|
|
128
|
+
*
|
|
129
|
+
* A sample tick asks about every monitored run and then signals some of them, and
|
|
130
|
+
* on Windows each read costs a PowerShell start — so the tick reads the table ONCE
|
|
131
|
+
* and threads it through, instead of N times.
|
|
132
|
+
*
|
|
133
|
+
* Deliberately NOT a time-based cache. A cached table can answer about a snapshot
|
|
134
|
+
* taken before a process existed, and the answer it gives is "not found" — which
|
|
135
|
+
* a caller reads as "it is gone". That is the one direction this module must never
|
|
136
|
+
* guess in.
|
|
137
|
+
*/
|
|
138
|
+
export declare function processTableSnapshot(): ProcessTable | null;
|
|
139
|
+
/**
|
|
140
|
+
* Sample the process tree rooted at `pid`: whether it exists, what it has cost
|
|
141
|
+
* in CPU so far, and every pid in it.
|
|
142
|
+
*
|
|
143
|
+
* The tree is the point. `spawn(command, { shell: true })` means the REAL work
|
|
144
|
+
* is a child of the shell we hold the handle to, so a sample (and a stop) that
|
|
145
|
+
* only looked at the root would report the shell's own ~0 CPU while the build
|
|
146
|
+
* underneath burned a core — the exact false hang this must not produce.
|
|
147
|
+
*/
|
|
148
|
+
export declare function processTreeOf(table: ProcessTable, pid: number): ProcessTreeSample;
|
|
149
|
+
export declare function sampleProcessTree(pid: number, table?: ProcessTable | null): ProcessTreeSample;
|
|
150
|
+
/**
|
|
151
|
+
* Is this pid running? `null` means THIS PLATFORM CANNOT TELL.
|
|
152
|
+
*
|
|
153
|
+
* The distinction is load-bearing for a run recovered from disk: "not found"
|
|
154
|
+
* must never be read as "finished" on a platform where the process table is not
|
|
155
|
+
* readable, or a restart would declare live work dead (and a dead run's recorded
|
|
156
|
+
* exit code would be reported as its verdict).
|
|
157
|
+
*/
|
|
158
|
+
export declare function processAlive(pid: number): boolean | null;
|
|
159
|
+
/**
|
|
160
|
+
* `processAlive`, answered from a table the caller ALREADY read — so a tick that
|
|
161
|
+
* samples every monitored run (and reconciles the recovered ones) costs one
|
|
162
|
+
* process listing, not one per run.
|
|
163
|
+
*/
|
|
164
|
+
export declare function tableAlive(table: ProcessTable | null, pid: number): boolean | null;
|
|
165
|
+
/**
|
|
166
|
+
* One process's row, cross-platform: `null` when the platform cannot read the
|
|
167
|
+
* table at all, `{ found: false }` when the pid is simply not there.
|
|
168
|
+
*
|
|
169
|
+
* The distinction matters to a caller that has to decide whether a pid is still
|
|
170
|
+
* OURS: "gone" is an answer, "cannot tell" is not.
|
|
171
|
+
*/
|
|
172
|
+
export declare function lookupProcess(pid: number): {
|
|
173
|
+
found: boolean;
|
|
174
|
+
command: string | null;
|
|
175
|
+
startedAtText: string | null;
|
|
176
|
+
} | null;
|
|
177
|
+
export interface LivenessInput {
|
|
178
|
+
/** The run still exists (the registry has not seen it exit). */
|
|
179
|
+
alive: boolean;
|
|
180
|
+
/** Bytes produced since the run started. */
|
|
181
|
+
bytes: number;
|
|
182
|
+
/** ms since the last output byte (or since the start when nothing came). */
|
|
183
|
+
quietMs: number;
|
|
184
|
+
/** ms since the run started. */
|
|
185
|
+
ageMs: number;
|
|
186
|
+
/** CPU ms now, and at the previous sample (null = unmeasurable). */
|
|
187
|
+
cpuMs: number | null;
|
|
188
|
+
prevCpuMs: number | null;
|
|
189
|
+
/** ms between the two CPU samples. */
|
|
190
|
+
sampleWindowMs: number;
|
|
191
|
+
}
|
|
192
|
+
export interface LivenessVerdict {
|
|
193
|
+
verdict: Liveness;
|
|
194
|
+
/** The evidence, in words — this is what the agent and the operator read. */
|
|
195
|
+
why: string;
|
|
196
|
+
}
|
|
197
|
+
/** `45s` / `3m 05s` — how long, for a human reading a message. */
|
|
198
|
+
export declare function formatSpan(ms: number): string;
|
|
199
|
+
/**
|
|
200
|
+
* Classify a run from its evidence. Deterministic and pure, so the decision can
|
|
201
|
+
* be asserted directly instead of inferred from a message.
|
|
202
|
+
*
|
|
203
|
+
* Order matters, and it is deliberately "innocent until proven hung":
|
|
204
|
+
* output recently → working; CPU advancing → working; only then, silence PLUS a
|
|
205
|
+
* measurable absence of CPU → hung. Anything we cannot prove lands in `stalled`,
|
|
206
|
+
* which is reported but never reaped.
|
|
207
|
+
*/
|
|
208
|
+
export declare function classifyLiveness(input: LivenessInput, thresholds?: LivenessThresholds): LivenessVerdict;
|
|
209
|
+
/**
|
|
210
|
+
* Stop a process TREE, deepest first — the shell we spawned and the real work
|
|
211
|
+
* under it.
|
|
212
|
+
*
|
|
213
|
+
* A plain `child.kill()` reaches only the shell, which is why a "stopped" hung
|
|
214
|
+
* build can keep running: the shell dies, the compiler it launched does not.
|
|
215
|
+
* Best-effort throughout — a pid that exited between the sample and the signal
|
|
216
|
+
* must not throw, and a failed signal falls back to the child handle.
|
|
217
|
+
*
|
|
218
|
+
* On Windows the tree comes from WMI (`Get-CimInstance`), so this genuinely
|
|
219
|
+
* reaches grandchildren there too; if that probe is unavailable the tree is just
|
|
220
|
+
* the root, which is the old behaviour rather than a regression. Note that a
|
|
221
|
+
* Windows "signal" takes effect immediately — there is no graceful SIGTERM to
|
|
222
|
+
* follow with a SIGKILL.
|
|
223
|
+
*/
|
|
224
|
+
export declare function killProcessTree(pid: number, signal: NodeJS.Signals, child?: {
|
|
225
|
+
kill(signal?: NodeJS.Signals | number): boolean;
|
|
226
|
+
}, table?: ProcessTable | null): number[];
|
|
227
|
+
//# sourceMappingURL=process-liveness.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"process-liveness.d.ts","sourceRoot":"","sources":["../../src/tools/process-liveness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAIH,2EAA2E;AAC3E,MAAM,MAAM,QAAQ,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;AAEhF,2DAA2D;AAC3D,MAAM,WAAW,kBAAkB;IACjC,iEAAiE;IACjE,YAAY,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,eAAO,MAAM,2BAA2B,EAAE,kBAIzC,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,kBAAkB,CAU3F;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CASxD;AAED,MAAM,WAAW,iBAAiB;IAChC,mDAAmD;IACnD,KAAK,EAAE,OAAO,CAAC;IACf,iFAAiF;IACjF,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,4EAA4E;IAC5E,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,oCAAoC;AACpC,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,yEAAyE;IACzE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,kFAAkF;IAClF,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB;;;;;;;;;;;;OAYG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,MAAM,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;AAanD;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAiBnE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAyBrE;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,CAgClE;AAiED;;;;;;;;;;;GAWG;AACH,wBAAgB,oBAAoB,IAAI,YAAY,GAAG,IAAI,CAE1D;AAED;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,GAAG,iBAAiB,CA2CjF;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,YAAY,GAAG,IAAI,GAAG,iBAAiB,CAK7F;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAExD;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAIlF;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,GAAG,EAAE,MAAM,GACV;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,CAQjF;AAED,MAAM,WAAW,aAAa;IAC5B,gEAAgE;IAChE,KAAK,EAAE,OAAO,CAAC;IACf,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,OAAO,EAAE,MAAM,CAAC;IAChB,gCAAgC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,oEAAoE;IACpE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,QAAQ,CAAC;IAClB,6EAA6E;IAC7E,GAAG,EAAE,MAAM,CAAC;CACb;AAED,kEAAkE;AAClE,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAO7C;AAED;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,aAAa,EACpB,UAAU,GAAE,kBAAgD,GAC3D,eAAe,CAmDjB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,eAAe,CAC7B,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,CAAC,OAAO,EACtB,KAAK,CAAC,EAAE;IAAE,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,OAAO,GAAG,MAAM,GAAG,OAAO,CAAA;CAAE,EAC3D,KAAK,CAAC,EAAE,YAAY,GAAG,IAAI,GAC1B,MAAM,EAAE,CAyBV"}
|