instar 1.3.672 → 1.3.674
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +113 -11
- package/dist/commands/server.js.map +1 -1
- package/dist/core/inboundLossRouting.d.ts +30 -0
- package/dist/core/inboundLossRouting.d.ts.map +1 -0
- package/dist/core/inboundLossRouting.js +35 -0
- package/dist/core/inboundLossRouting.js.map +1 -0
- package/dist/monitoring/EnforcedTerminationWatchdog.d.ts +91 -0
- package/dist/monitoring/EnforcedTerminationWatchdog.d.ts.map +1 -0
- package/dist/monitoring/EnforcedTerminationWatchdog.js +172 -0
- package/dist/monitoring/EnforcedTerminationWatchdog.js.map +1 -0
- package/dist/monitoring/enforcedTermination.d.ts +105 -0
- package/dist/monitoring/enforcedTermination.d.ts.map +1 -0
- package/dist/monitoring/enforcedTermination.js +105 -0
- package/dist/monitoring/enforcedTermination.js.map +1 -0
- package/dist/monitoring/enforcedTerminationWiring.d.ts +18 -0
- package/dist/monitoring/enforcedTerminationWiring.d.ts.map +1 -0
- package/dist/monitoring/enforcedTerminationWiring.js +80 -0
- package/dist/monitoring/enforcedTerminationWiring.js.map +1 -0
- package/dist/monitoring/guardManifest.d.ts.map +1 -1
- package/dist/monitoring/guardManifest.js +10 -0
- package/dist/monitoring/guardManifest.js.map +1 -1
- package/dist/server/AgentServer.d.ts +3 -0
- package/dist/server/AgentServer.d.ts.map +1 -1
- package/dist/server/AgentServer.js +1 -0
- package/dist/server/AgentServer.js.map +1 -1
- package/dist/server/routes.d.ts +3 -0
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +8 -0
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +47 -47
- package/upgrades/1.3.673.md +50 -0
- package/upgrades/1.3.674.md +43 -0
- package/upgrades/side-effects/enforced-termination-watchdog.md +95 -0
- package/upgrades/side-effects/inbound-delivery-sacred.md +62 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* F3 (Inbound Delivery Is Sacred) — pure routing plan for inbound-queue loss
|
|
3
|
+
* notices. A lost inbound user message must reach the user who sent it (each
|
|
4
|
+
* loss item's `sessionKey` IS the topic id they messaged from) OR, if it has no
|
|
5
|
+
* resolvable destination, be surfaced LOUDLY — never silently expired. This pure
|
|
6
|
+
* function decides the routing; server.ts does the actual notify()/loud-surface.
|
|
7
|
+
* Constitution: "The User Experience Is the Product" → sub-standard #3 Inbound
|
|
8
|
+
* Delivery Is Sacred. Spec: docs/specs/inbound-delivery-sacred.md.
|
|
9
|
+
*/
|
|
10
|
+
export interface InboundLossRoutePlan {
|
|
11
|
+
/** Per-ORIGINATING-topic notice — each affected topic + how many of its
|
|
12
|
+
* messages were lost (delivered IN that topic, the proven path). */
|
|
13
|
+
perTopic: Array<{
|
|
14
|
+
topicId: number;
|
|
15
|
+
count: number;
|
|
16
|
+
}>;
|
|
17
|
+
/** Count of lost items whose sessionKey is not a resolvable numeric topic.
|
|
18
|
+
* These fall back to the attention topic; if that is unset too, they MUST be
|
|
19
|
+
* surfaced loudly (the one seam where a loss could otherwise go silent). */
|
|
20
|
+
unresolved: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Group inbound loss items by their originating topic. Pure: same input → same
|
|
24
|
+
* output. A sessionKey that is a positive finite number is a topic id; anything
|
|
25
|
+
* else (empty, non-numeric, legacy single-file key) is counted as `unresolved`.
|
|
26
|
+
*/
|
|
27
|
+
export declare function planInboundLossNotices(items: ReadonlyArray<{
|
|
28
|
+
sessionKey: string;
|
|
29
|
+
}>): InboundLossRoutePlan;
|
|
30
|
+
//# sourceMappingURL=inboundLossRouting.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inboundLossRouting.d.ts","sourceRoot":"","sources":["../../src/core/inboundLossRouting.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,oBAAoB;IACnC;yEACqE;IACrE,QAAQ,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpD;;iFAE6E;IAC7E,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,aAAa,CAAC;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC,GAC3C,oBAAoB,CAkBtB"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* F3 (Inbound Delivery Is Sacred) — pure routing plan for inbound-queue loss
|
|
3
|
+
* notices. A lost inbound user message must reach the user who sent it (each
|
|
4
|
+
* loss item's `sessionKey` IS the topic id they messaged from) OR, if it has no
|
|
5
|
+
* resolvable destination, be surfaced LOUDLY — never silently expired. This pure
|
|
6
|
+
* function decides the routing; server.ts does the actual notify()/loud-surface.
|
|
7
|
+
* Constitution: "The User Experience Is the Product" → sub-standard #3 Inbound
|
|
8
|
+
* Delivery Is Sacred. Spec: docs/specs/inbound-delivery-sacred.md.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Group inbound loss items by their originating topic. Pure: same input → same
|
|
12
|
+
* output. A sessionKey that is a positive finite number is a topic id; anything
|
|
13
|
+
* else (empty, non-numeric, legacy single-file key) is counted as `unresolved`.
|
|
14
|
+
*/
|
|
15
|
+
export function planInboundLossNotices(items) {
|
|
16
|
+
const byTopic = new Map();
|
|
17
|
+
let unresolved = 0;
|
|
18
|
+
for (const it of items) {
|
|
19
|
+
const tid = Number(it.sessionKey);
|
|
20
|
+
if (Number.isFinite(tid) && tid > 0) {
|
|
21
|
+
byTopic.set(tid, (byTopic.get(tid) ?? 0) + 1);
|
|
22
|
+
}
|
|
23
|
+
else {
|
|
24
|
+
unresolved++;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return {
|
|
28
|
+
// Deterministic order (ascending topic id) for stable notices + tests.
|
|
29
|
+
perTopic: [...byTopic.entries()]
|
|
30
|
+
.sort((a, b) => a[0] - b[0])
|
|
31
|
+
.map(([topicId, count]) => ({ topicId, count })),
|
|
32
|
+
unresolved,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=inboundLossRouting.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inboundLossRouting.js","sourceRoot":"","sources":["../../src/core/inboundLossRouting.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAYH;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CACpC,KAA4C;IAE5C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;QAClC,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,UAAU,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO;QACL,uEAAuE;QACvE,QAAQ,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;aAC7B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAClD,UAAU;KACX,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EnforcedTerminationWatchdog — the external hard-stop loop for autonomous runs
|
|
3
|
+
* that overrun their budget. Wires the pure decision core (computeOverrun +
|
|
4
|
+
* TerminationConfirmer) to injected, side-effecting dependencies so the loop is
|
|
5
|
+
* unit-testable with fakes and the real (session-killing) actuation lives behind
|
|
6
|
+
* one interface. Spec: docs/specs/enforced-termination-watchdog.md.
|
|
7
|
+
*
|
|
8
|
+
* Constitution: "The User Experience Is the Product" → sub-standard #2 Enforced
|
|
9
|
+
* Termination — "Structure beats Willpower" applied to the END of work. The
|
|
10
|
+
* counterweight to "An Autonomous Run Must Outlive Its Session": that keeps a run
|
|
11
|
+
* alive across vessel events; THIS keeps a run from outliving its budget.
|
|
12
|
+
*
|
|
13
|
+
* Dark/dryRun-first rollout (mirrors AutonomousLivenessReconciler): ships with
|
|
14
|
+
* `enabled` omitted from config defaults (resolves live only on a dev agent) and
|
|
15
|
+
* `dryRun` defaulting true (logs would-terminate, actuates nothing) until a
|
|
16
|
+
* deliberate flip. The actuation is deliberately NOT in this class — it is the
|
|
17
|
+
* injected `terminate` actuator, so this orchestration never imports a session
|
|
18
|
+
* killer and stays provable in isolation.
|
|
19
|
+
*/
|
|
20
|
+
import { type AutonomousRunSnapshot, type EnforcedTerminationConfig, type OverrunReason } from './enforcedTermination.js';
|
|
21
|
+
export interface EnforcedTerminationDeps {
|
|
22
|
+
/** Returns the current durable snapshot of every autonomous run. */
|
|
23
|
+
listRuns: () => AutonomousRunSnapshot[];
|
|
24
|
+
/**
|
|
25
|
+
* Durably terminate a run so neither the liveness-reconciler nor the resume
|
|
26
|
+
* queue revives it (delete state file + record operator-stop + cancel resume +
|
|
27
|
+
* clear endedMidWork + killSession). Returns true on a clean actuation. Only
|
|
28
|
+
* ever called for a TWO-tick-confirmed overrun, and never in dryRun.
|
|
29
|
+
*/
|
|
30
|
+
terminate: (topicId: string, reason: OverrunReason) => Promise<boolean>;
|
|
31
|
+
/** Append one audit row per transition. Must never throw into the loop. */
|
|
32
|
+
audit: (row: EnforcedTerminationAuditRow) => void;
|
|
33
|
+
/** Monotonic clock (injectable for tests). */
|
|
34
|
+
now?: () => number;
|
|
35
|
+
}
|
|
36
|
+
export interface EnforcedTerminationOptions extends Partial<EnforcedTerminationConfig> {
|
|
37
|
+
enabled: boolean;
|
|
38
|
+
dryRun?: boolean;
|
|
39
|
+
/** Consecutive overrun ticks required before a kill. Default 2. */
|
|
40
|
+
confirmThreshold?: number;
|
|
41
|
+
/** Per-window cap on actuations; a flapping detector gives up LOUDLY. Default 5. */
|
|
42
|
+
maxTerminationsPerWindow?: number;
|
|
43
|
+
tickIntervalSec?: number;
|
|
44
|
+
}
|
|
45
|
+
export interface EnforcedTerminationAuditRow {
|
|
46
|
+
ts: number;
|
|
47
|
+
topicId: string;
|
|
48
|
+
event: 'overrun-detected' | 'terminate-pending' | 'terminated' | 'would-terminate' | 'terminate-failed' | 'cap-exceeded';
|
|
49
|
+
reason?: OverrunReason;
|
|
50
|
+
dryRun: boolean;
|
|
51
|
+
}
|
|
52
|
+
export interface EnforcedTerminationGuardStatus {
|
|
53
|
+
enabled: boolean;
|
|
54
|
+
dryRun: boolean;
|
|
55
|
+
graceSeconds: number;
|
|
56
|
+
absoluteCeilingSeconds: number;
|
|
57
|
+
lastTickAt: number | null;
|
|
58
|
+
pending: string[];
|
|
59
|
+
terminatedCount: number;
|
|
60
|
+
wouldTerminateCount: number;
|
|
61
|
+
capExceededCount: number;
|
|
62
|
+
}
|
|
63
|
+
export declare class EnforcedTerminationWatchdog {
|
|
64
|
+
private readonly deps;
|
|
65
|
+
private readonly opts;
|
|
66
|
+
private readonly cfg;
|
|
67
|
+
private readonly confirmer;
|
|
68
|
+
private readonly now;
|
|
69
|
+
private timer;
|
|
70
|
+
private lastTickAt;
|
|
71
|
+
private terminatedCount;
|
|
72
|
+
private wouldTerminateCount;
|
|
73
|
+
private capExceededCount;
|
|
74
|
+
private readonly actuationTimes;
|
|
75
|
+
private readonly windowMs;
|
|
76
|
+
constructor(deps: EnforcedTerminationDeps, opts: EnforcedTerminationOptions);
|
|
77
|
+
/** Start the unref'd tick loop. No-op when disabled. */
|
|
78
|
+
start(): void;
|
|
79
|
+
stop(): void;
|
|
80
|
+
/**
|
|
81
|
+
* One reconcile pass. Pure-core decides overrun; the confirmer enforces the
|
|
82
|
+
* two-tick rule; only a CONFIRMED overrun actuates (and never in dryRun). Every
|
|
83
|
+
* predicate failure is swallowed toward NO actuation (fail-safe direction).
|
|
84
|
+
*/
|
|
85
|
+
tick(): Promise<void>;
|
|
86
|
+
guardStatus(): EnforcedTerminationGuardStatus;
|
|
87
|
+
private get dryRun();
|
|
88
|
+
private withinCap;
|
|
89
|
+
private safeAudit;
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=EnforcedTerminationWatchdog.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"EnforcedTerminationWatchdog.d.ts","sourceRoot":"","sources":["../../src/monitoring/EnforcedTerminationWatchdog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,EAIL,KAAK,qBAAqB,EAC1B,KAAK,yBAAyB,EAC9B,KAAK,aAAa,EACnB,MAAM,0BAA0B,CAAC;AAElC,MAAM,WAAW,uBAAuB;IACtC,oEAAoE;IACpE,QAAQ,EAAE,MAAM,qBAAqB,EAAE,CAAC;IACxC;;;;;OAKG;IACH,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACxE,2EAA2E;IAC3E,KAAK,EAAE,CAAC,GAAG,EAAE,2BAA2B,KAAK,IAAI,CAAC;IAClD,8CAA8C;IAC9C,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,0BAA2B,SAAQ,OAAO,CAAC,yBAAyB,CAAC;IACpF,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,mEAAmE;IACnE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oFAAoF;IACpF,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,2BAA2B;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EACD,kBAAkB,GAClB,mBAAmB,GACnB,YAAY,GACZ,iBAAiB,GACjB,kBAAkB,GAClB,cAAc,CAAC;IACnB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,2BAA2B;IAepC,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAfvB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA4B;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAuB;IACjD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,KAAK,CAA+C;IAE5D,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,gBAAgB,CAAK;IAE7B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAgB;IAC/C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;gBAGf,IAAI,EAAE,uBAAuB,EAC7B,IAAI,EAAE,0BAA0B;IAanD,wDAAwD;IACxD,KAAK,IAAI,IAAI;IAYb,IAAI,IAAI,IAAI;IAOZ;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAoE3B,WAAW,IAAI,8BAA8B;IAc7C,OAAO,KAAK,MAAM,GAEjB;IAED,OAAO,CAAC,SAAS;IASjB,OAAO,CAAC,SAAS;CAOlB"}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EnforcedTerminationWatchdog — the external hard-stop loop for autonomous runs
|
|
3
|
+
* that overrun their budget. Wires the pure decision core (computeOverrun +
|
|
4
|
+
* TerminationConfirmer) to injected, side-effecting dependencies so the loop is
|
|
5
|
+
* unit-testable with fakes and the real (session-killing) actuation lives behind
|
|
6
|
+
* one interface. Spec: docs/specs/enforced-termination-watchdog.md.
|
|
7
|
+
*
|
|
8
|
+
* Constitution: "The User Experience Is the Product" → sub-standard #2 Enforced
|
|
9
|
+
* Termination — "Structure beats Willpower" applied to the END of work. The
|
|
10
|
+
* counterweight to "An Autonomous Run Must Outlive Its Session": that keeps a run
|
|
11
|
+
* alive across vessel events; THIS keeps a run from outliving its budget.
|
|
12
|
+
*
|
|
13
|
+
* Dark/dryRun-first rollout (mirrors AutonomousLivenessReconciler): ships with
|
|
14
|
+
* `enabled` omitted from config defaults (resolves live only on a dev agent) and
|
|
15
|
+
* `dryRun` defaulting true (logs would-terminate, actuates nothing) until a
|
|
16
|
+
* deliberate flip. The actuation is deliberately NOT in this class — it is the
|
|
17
|
+
* injected `terminate` actuator, so this orchestration never imports a session
|
|
18
|
+
* killer and stays provable in isolation.
|
|
19
|
+
*/
|
|
20
|
+
import { computeOverrun, TerminationConfirmer, DEFAULT_ENFORCED_TERMINATION_CONFIG, } from './enforcedTermination.js';
|
|
21
|
+
export class EnforcedTerminationWatchdog {
|
|
22
|
+
deps;
|
|
23
|
+
opts;
|
|
24
|
+
cfg;
|
|
25
|
+
confirmer;
|
|
26
|
+
now;
|
|
27
|
+
timer = null;
|
|
28
|
+
lastTickAt = null;
|
|
29
|
+
terminatedCount = 0;
|
|
30
|
+
wouldTerminateCount = 0;
|
|
31
|
+
capExceededCount = 0;
|
|
32
|
+
// Window of actuation timestamps for the per-window cap.
|
|
33
|
+
actuationTimes = [];
|
|
34
|
+
windowMs;
|
|
35
|
+
constructor(deps, opts) {
|
|
36
|
+
this.deps = deps;
|
|
37
|
+
this.opts = opts;
|
|
38
|
+
this.cfg = {
|
|
39
|
+
graceSeconds: opts.graceSeconds ?? DEFAULT_ENFORCED_TERMINATION_CONFIG.graceSeconds,
|
|
40
|
+
absoluteCeilingSeconds: opts.absoluteCeilingSeconds ?? DEFAULT_ENFORCED_TERMINATION_CONFIG.absoluteCeilingSeconds,
|
|
41
|
+
maxIterations: opts.maxIterations,
|
|
42
|
+
};
|
|
43
|
+
this.confirmer = new TerminationConfirmer(opts.confirmThreshold ?? 2);
|
|
44
|
+
this.now = deps.now ?? Date.now;
|
|
45
|
+
this.windowMs = (opts.tickIntervalSec ?? 60) * 1000 * 60; // ~60 ticks
|
|
46
|
+
}
|
|
47
|
+
/** Start the unref'd tick loop. No-op when disabled. */
|
|
48
|
+
start() {
|
|
49
|
+
if (!this.opts.enabled || this.timer)
|
|
50
|
+
return;
|
|
51
|
+
const intervalMs = (this.opts.tickIntervalSec ?? 60) * 1000;
|
|
52
|
+
this.timer = setInterval(() => void this.tick().catch(() => {
|
|
53
|
+
// @silent-fallback-ok: tick() handles its own errors and fails safe toward
|
|
54
|
+
// NO actuation; a thrown tick must not crash the unref'd timer loop.
|
|
55
|
+
}), intervalMs);
|
|
56
|
+
if (typeof this.timer.unref === 'function') {
|
|
57
|
+
this.timer.unref();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
stop() {
|
|
61
|
+
if (this.timer) {
|
|
62
|
+
clearInterval(this.timer);
|
|
63
|
+
this.timer = null;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* One reconcile pass. Pure-core decides overrun; the confirmer enforces the
|
|
68
|
+
* two-tick rule; only a CONFIRMED overrun actuates (and never in dryRun). Every
|
|
69
|
+
* predicate failure is swallowed toward NO actuation (fail-safe direction).
|
|
70
|
+
*/
|
|
71
|
+
async tick() {
|
|
72
|
+
if (!this.opts.enabled)
|
|
73
|
+
return;
|
|
74
|
+
const t = this.now();
|
|
75
|
+
this.lastTickAt = t;
|
|
76
|
+
let runs;
|
|
77
|
+
try {
|
|
78
|
+
runs = this.deps.listRuns();
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
// @silent-fallback-ok: can't read state → do nothing. Fail-safe is the
|
|
82
|
+
// SAFE direction for a session-killer (never kill on uncertainty); a louder
|
|
83
|
+
// surface here would risk noise on a transient read while changing nothing.
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
const overrun = new Map();
|
|
87
|
+
for (const r of runs) {
|
|
88
|
+
let reason = null;
|
|
89
|
+
try {
|
|
90
|
+
reason = computeOverrun(r, this.cfg, t);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
// @silent-fallback-ok: a malformed snapshot is not an overrun → skip it
|
|
94
|
+
// (fail-safe: never classify-to-kill on a parse error).
|
|
95
|
+
reason = null;
|
|
96
|
+
}
|
|
97
|
+
if (reason) {
|
|
98
|
+
overrun.set(r.topicId, reason);
|
|
99
|
+
this.safeAudit({ ts: t, topicId: r.topicId, event: 'overrun-detected', reason, dryRun: this.dryRun });
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
const confirmed = this.confirmer.reconcile(overrun.keys());
|
|
103
|
+
// Topics overrun but not yet confirmed are terminate-pending.
|
|
104
|
+
for (const topic of this.confirmer.pendingTopics()) {
|
|
105
|
+
this.safeAudit({ ts: t, topicId: topic, event: 'terminate-pending', reason: overrun.get(topic), dryRun: this.dryRun });
|
|
106
|
+
}
|
|
107
|
+
for (const topic of confirmed) {
|
|
108
|
+
const reason = overrun.get(topic);
|
|
109
|
+
if (this.dryRun) {
|
|
110
|
+
this.wouldTerminateCount++;
|
|
111
|
+
this.safeAudit({ ts: t, topicId: topic, event: 'would-terminate', reason, dryRun: true });
|
|
112
|
+
this.confirmer.clear(topic);
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
if (!this.withinCap(t)) {
|
|
116
|
+
this.capExceededCount++;
|
|
117
|
+
this.safeAudit({ ts: t, topicId: topic, event: 'cap-exceeded', reason, dryRun: false });
|
|
118
|
+
continue; // give up LOUDLY (audited) rather than kill-loop; keep pending
|
|
119
|
+
}
|
|
120
|
+
let ok = false;
|
|
121
|
+
try {
|
|
122
|
+
ok = await this.deps.terminate(topic, reason);
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
ok = false;
|
|
126
|
+
}
|
|
127
|
+
if (ok) {
|
|
128
|
+
this.terminatedCount++;
|
|
129
|
+
this.actuationTimes.push(t);
|
|
130
|
+
this.confirmer.clear(topic);
|
|
131
|
+
this.safeAudit({ ts: t, topicId: topic, event: 'terminated', reason, dryRun: false });
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
this.safeAudit({ ts: t, topicId: topic, event: 'terminate-failed', reason, dryRun: false });
|
|
135
|
+
// leave pending so a later tick retries (subject to the cap)
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
guardStatus() {
|
|
140
|
+
return {
|
|
141
|
+
enabled: this.opts.enabled,
|
|
142
|
+
dryRun: this.dryRun,
|
|
143
|
+
graceSeconds: this.cfg.graceSeconds,
|
|
144
|
+
absoluteCeilingSeconds: this.cfg.absoluteCeilingSeconds,
|
|
145
|
+
lastTickAt: this.lastTickAt,
|
|
146
|
+
pending: this.confirmer.pendingTopics(),
|
|
147
|
+
terminatedCount: this.terminatedCount,
|
|
148
|
+
wouldTerminateCount: this.wouldTerminateCount,
|
|
149
|
+
capExceededCount: this.capExceededCount,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
get dryRun() {
|
|
153
|
+
return this.opts.dryRun ?? true;
|
|
154
|
+
}
|
|
155
|
+
withinCap(now) {
|
|
156
|
+
const cap = this.opts.maxTerminationsPerWindow ?? 5;
|
|
157
|
+
// prune old actuations outside the window
|
|
158
|
+
while (this.actuationTimes.length && now - this.actuationTimes[0] > this.windowMs) {
|
|
159
|
+
this.actuationTimes.shift();
|
|
160
|
+
}
|
|
161
|
+
return this.actuationTimes.length < cap;
|
|
162
|
+
}
|
|
163
|
+
safeAudit(row) {
|
|
164
|
+
try {
|
|
165
|
+
this.deps.audit(row);
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
/* the audit sink must never endanger the loop */
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
//# sourceMappingURL=EnforcedTerminationWatchdog.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"EnforcedTerminationWatchdog.js","sourceRoot":"","sources":["../../src/monitoring/EnforcedTerminationWatchdog.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,EACL,cAAc,EACd,oBAAoB,EACpB,mCAAmC,GAIpC,MAAM,0BAA0B,CAAC;AAsDlC,MAAM,OAAO,2BAA2B;IAenB;IACA;IAfF,GAAG,CAA4B;IAC/B,SAAS,CAAuB;IAChC,GAAG,CAAe;IAC3B,KAAK,GAA0C,IAAI,CAAC;IAEpD,UAAU,GAAkB,IAAI,CAAC;IACjC,eAAe,GAAG,CAAC,CAAC;IACpB,mBAAmB,GAAG,CAAC,CAAC;IACxB,gBAAgB,GAAG,CAAC,CAAC;IAC7B,yDAAyD;IACxC,cAAc,GAAa,EAAE,CAAC;IAC9B,QAAQ,CAAS;IAElC,YACmB,IAA6B,EAC7B,IAAgC;QADhC,SAAI,GAAJ,IAAI,CAAyB;QAC7B,SAAI,GAAJ,IAAI,CAA4B;QAEjD,IAAI,CAAC,GAAG,GAAG;YACT,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,mCAAmC,CAAC,YAAY;YACnF,sBAAsB,EACpB,IAAI,CAAC,sBAAsB,IAAI,mCAAmC,CAAC,sBAAsB;YAC3F,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC,CAAC;QACF,IAAI,CAAC,SAAS,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC,CAAC;QACtE,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QAChC,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,YAAY;IACxE,CAAC;IAED,wDAAwD;IACxD,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QAC7C,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;YACzD,2EAA2E;YAC3E,qEAAqE;QACvE,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAChB,IAAI,OAAQ,IAAI,CAAC,KAAgC,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YACtE,IAAI,CAAC,KAA+B,CAAC,KAAK,EAAE,CAAC;QAChD,CAAC;IACH,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAC/B,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QAEpB,IAAI,IAA6B,CAAC;QAClC,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,uEAAuE;YACvE,4EAA4E;YAC5E,4EAA4E;YAC5E,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,IAAI,MAAM,GAAyB,IAAI,CAAC;YACxC,IAAI,CAAC;gBACH,MAAM,GAAG,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC1C,CAAC;YAAC,MAAM,CAAC;gBACP,wEAAwE;gBACxE,wDAAwD;gBACxD,MAAM,GAAG,IAAI,CAAC;YAChB,CAAC;YACD,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC/B,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACxG,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,8DAA8D;QAC9D,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,EAAE,CAAC;YACnD,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACzH,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YACnC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC3B,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC1F,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC5B,SAAS;YACX,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvB,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACxB,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBACxF,SAAS,CAAC,+DAA+D;YAC3E,CAAC;YACD,IAAI,EAAE,GAAG,KAAK,CAAC;YACf,IAAI,CAAC;gBACH,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAChD,CAAC;YAAC,MAAM,CAAC;gBACP,EAAE,GAAG,KAAK,CAAC;YACb,CAAC;YACD,IAAI,EAAE,EAAE,CAAC;gBACP,IAAI,CAAC,eAAe,EAAE,CAAC;gBACvB,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC5B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC5B,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YACxF,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5F,6DAA6D;YAC/D,CAAC;QACH,CAAC;IACH,CAAC;IAED,WAAW;QACT,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO;YAC1B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY;YACnC,sBAAsB,EAAE,IAAI,CAAC,GAAG,CAAC,sBAAsB;YACvD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE;YACvC,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;SACxC,CAAC;IACJ,CAAC;IAED,IAAY,MAAM;QAChB,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC;IAClC,CAAC;IAEO,SAAS,CAAC,GAAW;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,wBAAwB,IAAI,CAAC,CAAC;QACpD,0CAA0C;QAC1C,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClF,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC9B,CAAC;QACD,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG,GAAG,CAAC;IAC1C,CAAC;IAEO,SAAS,CAAC,GAAgC;QAChD,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,iDAAiD;QACnD,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Enforced Termination — the PURE decision core for the external hard-stop on
|
|
3
|
+
* autonomous runs. Spec: docs/specs/enforced-termination-watchdog.md.
|
|
4
|
+
* Constitution: "The User Experience Is the Product" → sub-standard #2 Enforced
|
|
5
|
+
* Termination; an instance of "Structure beats Willpower" applied to the END of
|
|
6
|
+
* work. Earned from 2026-06-25 (topic 27515 ran ~46h on a 24h budget).
|
|
7
|
+
*
|
|
8
|
+
* This module is intentionally dependency-free (no sessions, no fs, no clock of
|
|
9
|
+
* its own) so the overrun predicate and the two-phase confirm are unit-tested
|
|
10
|
+
* deterministically. The watchdog that actuates a kill (delete state file +
|
|
11
|
+
* record operator-stop + cancel resume + clear endedMidWork + killSession) wires
|
|
12
|
+
* THIS core to real deps — the safety-critical actuation is deliberately kept
|
|
13
|
+
* out of the pure logic so the decision boundary is provable in isolation.
|
|
14
|
+
*/
|
|
15
|
+
/** A durable snapshot of one autonomous run, parsed from its state frontmatter. */
|
|
16
|
+
export interface AutonomousRunSnapshot {
|
|
17
|
+
/** Telegram topic id (string key of the run). */
|
|
18
|
+
topicId: string;
|
|
19
|
+
/** Parsed `started_at` epoch ms, or null when absent/unparseable. */
|
|
20
|
+
startedAtMs: number | null;
|
|
21
|
+
/**
|
|
22
|
+
* Fallback clock for the absolute ceiling ONLY — the run state file's mtime
|
|
23
|
+
* (ms). Used when `startedAtMs` is null so an unparseable timestamp still has
|
|
24
|
+
* a bounded ceiling (fail TOWARD termination, never toward run-forever).
|
|
25
|
+
*/
|
|
26
|
+
fileMtimeMs: number;
|
|
27
|
+
/** `duration_seconds` budget. null or 0 ⇒ UNBOUNDED (no time budget). */
|
|
28
|
+
durationSeconds: number | null;
|
|
29
|
+
/** Current iteration count (for the optional iteration ceiling). */
|
|
30
|
+
iteration: number;
|
|
31
|
+
/** Whether the run is marked active. */
|
|
32
|
+
active: boolean;
|
|
33
|
+
/** Whether the run is paused (operator pause / move). Paused ⇒ never terminated. */
|
|
34
|
+
paused: boolean;
|
|
35
|
+
/**
|
|
36
|
+
* True when the run is mid cross-machine move (`move_suspended_at`/`moved_to`
|
|
37
|
+
* present). The destination re-evaluates; this watchdog must NOT kill it.
|
|
38
|
+
*/
|
|
39
|
+
moveSuspended: boolean;
|
|
40
|
+
}
|
|
41
|
+
export interface EnforcedTerminationConfig {
|
|
42
|
+
/**
|
|
43
|
+
* Grace past the time budget before the watchdog fires, letting the
|
|
44
|
+
* cooperative in-hook duration check win the normal case. Default 120s.
|
|
45
|
+
*/
|
|
46
|
+
graceSeconds: number;
|
|
47
|
+
/**
|
|
48
|
+
* Absolute hard ceiling from the run's start. Fires even for an UNBOUNDED run
|
|
49
|
+
* or an unparseable `started_at` (via file mtime). Default 26h.
|
|
50
|
+
*/
|
|
51
|
+
absoluteCeilingSeconds: number;
|
|
52
|
+
/** Optional iteration ceiling. Unset ⇒ no iteration-based termination. */
|
|
53
|
+
maxIterations?: number;
|
|
54
|
+
}
|
|
55
|
+
export declare const DEFAULT_ENFORCED_TERMINATION_CONFIG: EnforcedTerminationConfig;
|
|
56
|
+
export type OverrunReason = {
|
|
57
|
+
kind: 'time-budget';
|
|
58
|
+
elapsedSeconds: number;
|
|
59
|
+
budgetSeconds: number;
|
|
60
|
+
} | {
|
|
61
|
+
kind: 'absolute-ceiling';
|
|
62
|
+
elapsedSeconds: number;
|
|
63
|
+
ceilingSeconds: number;
|
|
64
|
+
clock: 'started_at' | 'file-mtime';
|
|
65
|
+
} | {
|
|
66
|
+
kind: 'iteration-ceiling';
|
|
67
|
+
iteration: number;
|
|
68
|
+
max: number;
|
|
69
|
+
};
|
|
70
|
+
/**
|
|
71
|
+
* Decide whether a run has PROVABLY overrun its budget. Pure: same inputs →
|
|
72
|
+
* same output. Returns the first matching reason, or null when the run is
|
|
73
|
+
* within budget / not eligible.
|
|
74
|
+
*
|
|
75
|
+
* Eligibility gates (return null): not active, paused, or mid-move. A run that
|
|
76
|
+
* is not eligible is NEVER terminated by this watchdog — the reaper handles
|
|
77
|
+
* idle/pressure; this fires ONLY on a genuine budget overrun.
|
|
78
|
+
*/
|
|
79
|
+
export declare function computeOverrun(run: AutonomousRunSnapshot, cfg: EnforcedTerminationConfig, nowMs: number): OverrunReason | null;
|
|
80
|
+
/**
|
|
81
|
+
* Two-phase confirm — a topic is only terminated after it is observed overrun
|
|
82
|
+
* on TWO consecutive reconcile() ticks. This absorbs a clock blip, an in-flight
|
|
83
|
+
* cooperative stop landing between ticks, and a run that just completed. Mirrors
|
|
84
|
+
* SessionReaper's mark-pending / re-confirm pattern.
|
|
85
|
+
*
|
|
86
|
+
* Stateful but pure of side effects: it only tracks per-topic consecutive
|
|
87
|
+
* overrun counts. A topic that drops out of the overrun set is forgotten (its
|
|
88
|
+
* pending count resets), so a transient overrun never accumulates toward a kill.
|
|
89
|
+
*/
|
|
90
|
+
export declare class TerminationConfirmer {
|
|
91
|
+
private readonly pending;
|
|
92
|
+
private readonly threshold;
|
|
93
|
+
constructor(confirmThreshold?: number);
|
|
94
|
+
/**
|
|
95
|
+
* Feed the set of currently-overrun topic ids; returns the topics CONFIRMED
|
|
96
|
+
* for termination this tick (overrun for `threshold` consecutive ticks).
|
|
97
|
+
* Topics no longer overrun are cleared.
|
|
98
|
+
*/
|
|
99
|
+
reconcile(overrunTopicIds: Iterable<string>): string[];
|
|
100
|
+
/** Drop a topic's pending state once it has been actuated (or cancelled). */
|
|
101
|
+
clear(topicId: string): void;
|
|
102
|
+
/** Topics currently marked terminate-pending (overrun but not yet confirmed). */
|
|
103
|
+
pendingTopics(): string[];
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=enforcedTermination.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"enforcedTermination.d.ts","sourceRoot":"","sources":["../../src/monitoring/enforcedTermination.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,mFAAmF;AACnF,MAAM,WAAW,qBAAqB;IACpC,iDAAiD;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,qEAAqE;IACrE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,yEAAyE;IACzE,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,oEAAoE;IACpE,SAAS,EAAE,MAAM,CAAC;IAClB,wCAAwC;IACxC,MAAM,EAAE,OAAO,CAAC;IAChB,oFAAoF;IACpF,MAAM,EAAE,OAAO,CAAC;IAChB;;;OAGG;IACH,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,yBAAyB;IACxC;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,sBAAsB,EAAE,MAAM,CAAC;IAC/B,0EAA0E;IAC1E,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,eAAO,MAAM,mCAAmC,EAAE,yBAGjD,CAAC;AAEF,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,cAAc,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE,GACtE;IACE,IAAI,EAAE,kBAAkB,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,YAAY,GAAG,YAAY,CAAC;CACpC,GACD;IAAE,IAAI,EAAE,mBAAmB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;AAElE;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAC5B,GAAG,EAAE,qBAAqB,EAC1B,GAAG,EAAE,yBAAyB,EAC9B,KAAK,EAAE,MAAM,GACZ,aAAa,GAAG,IAAI,CAiCtB;AAED;;;;;;;;;GASG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA6B;IACrD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,gBAAgB,SAAI;IAIhC;;;;OAIG;IACH,SAAS,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE;IAetD,6EAA6E;IAC7E,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAI5B,iFAAiF;IACjF,aAAa,IAAI,MAAM,EAAE;CAG1B"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Enforced Termination — the PURE decision core for the external hard-stop on
|
|
3
|
+
* autonomous runs. Spec: docs/specs/enforced-termination-watchdog.md.
|
|
4
|
+
* Constitution: "The User Experience Is the Product" → sub-standard #2 Enforced
|
|
5
|
+
* Termination; an instance of "Structure beats Willpower" applied to the END of
|
|
6
|
+
* work. Earned from 2026-06-25 (topic 27515 ran ~46h on a 24h budget).
|
|
7
|
+
*
|
|
8
|
+
* This module is intentionally dependency-free (no sessions, no fs, no clock of
|
|
9
|
+
* its own) so the overrun predicate and the two-phase confirm are unit-tested
|
|
10
|
+
* deterministically. The watchdog that actuates a kill (delete state file +
|
|
11
|
+
* record operator-stop + cancel resume + clear endedMidWork + killSession) wires
|
|
12
|
+
* THIS core to real deps — the safety-critical actuation is deliberately kept
|
|
13
|
+
* out of the pure logic so the decision boundary is provable in isolation.
|
|
14
|
+
*/
|
|
15
|
+
export const DEFAULT_ENFORCED_TERMINATION_CONFIG = {
|
|
16
|
+
graceSeconds: 120,
|
|
17
|
+
absoluteCeilingSeconds: 26 * 60 * 60, // 26h
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Decide whether a run has PROVABLY overrun its budget. Pure: same inputs →
|
|
21
|
+
* same output. Returns the first matching reason, or null when the run is
|
|
22
|
+
* within budget / not eligible.
|
|
23
|
+
*
|
|
24
|
+
* Eligibility gates (return null): not active, paused, or mid-move. A run that
|
|
25
|
+
* is not eligible is NEVER terminated by this watchdog — the reaper handles
|
|
26
|
+
* idle/pressure; this fires ONLY on a genuine budget overrun.
|
|
27
|
+
*/
|
|
28
|
+
export function computeOverrun(run, cfg, nowMs) {
|
|
29
|
+
if (!run.active || run.paused || run.moveSuspended)
|
|
30
|
+
return null;
|
|
31
|
+
// Iteration ceiling (optional, explicit opt-in).
|
|
32
|
+
if (cfg.maxIterations != null && cfg.maxIterations > 0 && run.iteration >= cfg.maxIterations) {
|
|
33
|
+
return { kind: 'iteration-ceiling', iteration: run.iteration, max: cfg.maxIterations };
|
|
34
|
+
}
|
|
35
|
+
// Time budget — only for a BOUNDED run with a parseable start. The grace
|
|
36
|
+
// window gives the cooperative in-hook check first right of termination.
|
|
37
|
+
if (run.startedAtMs != null && run.durationSeconds != null && run.durationSeconds > 0) {
|
|
38
|
+
const elapsed = (nowMs - run.startedAtMs) / 1000;
|
|
39
|
+
if (elapsed >= run.durationSeconds + cfg.graceSeconds) {
|
|
40
|
+
return { kind: 'time-budget', elapsedSeconds: elapsed, budgetSeconds: run.durationSeconds };
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
// Absolute ceiling — the backstop that covers the holes the in-hook check
|
|
44
|
+
// can't: an UNBOUNDED run (no duration), and an UNPARSEABLE started_at (uses
|
|
45
|
+
// file mtime). Fail TOWARD a bounded stop, never toward run-forever.
|
|
46
|
+
const clockMs = run.startedAtMs ?? run.fileMtimeMs;
|
|
47
|
+
const clock = run.startedAtMs != null ? 'started_at' : 'file-mtime';
|
|
48
|
+
const elapsedAbs = (nowMs - clockMs) / 1000;
|
|
49
|
+
if (elapsedAbs >= cfg.absoluteCeilingSeconds) {
|
|
50
|
+
return {
|
|
51
|
+
kind: 'absolute-ceiling',
|
|
52
|
+
elapsedSeconds: elapsedAbs,
|
|
53
|
+
ceilingSeconds: cfg.absoluteCeilingSeconds,
|
|
54
|
+
clock,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Two-phase confirm — a topic is only terminated after it is observed overrun
|
|
61
|
+
* on TWO consecutive reconcile() ticks. This absorbs a clock blip, an in-flight
|
|
62
|
+
* cooperative stop landing between ticks, and a run that just completed. Mirrors
|
|
63
|
+
* SessionReaper's mark-pending / re-confirm pattern.
|
|
64
|
+
*
|
|
65
|
+
* Stateful but pure of side effects: it only tracks per-topic consecutive
|
|
66
|
+
* overrun counts. A topic that drops out of the overrun set is forgotten (its
|
|
67
|
+
* pending count resets), so a transient overrun never accumulates toward a kill.
|
|
68
|
+
*/
|
|
69
|
+
export class TerminationConfirmer {
|
|
70
|
+
pending = new Map();
|
|
71
|
+
threshold;
|
|
72
|
+
constructor(confirmThreshold = 2) {
|
|
73
|
+
this.threshold = Math.max(1, confirmThreshold);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Feed the set of currently-overrun topic ids; returns the topics CONFIRMED
|
|
77
|
+
* for termination this tick (overrun for `threshold` consecutive ticks).
|
|
78
|
+
* Topics no longer overrun are cleared.
|
|
79
|
+
*/
|
|
80
|
+
reconcile(overrunTopicIds) {
|
|
81
|
+
const overrun = new Set(overrunTopicIds);
|
|
82
|
+
const confirmed = [];
|
|
83
|
+
for (const topic of overrun) {
|
|
84
|
+
const next = (this.pending.get(topic) ?? 0) + 1;
|
|
85
|
+
this.pending.set(topic, next);
|
|
86
|
+
if (next >= this.threshold)
|
|
87
|
+
confirmed.push(topic);
|
|
88
|
+
}
|
|
89
|
+
// Forget topics that are no longer overrun (reset their streak).
|
|
90
|
+
for (const topic of [...this.pending.keys()]) {
|
|
91
|
+
if (!overrun.has(topic))
|
|
92
|
+
this.pending.delete(topic);
|
|
93
|
+
}
|
|
94
|
+
return confirmed;
|
|
95
|
+
}
|
|
96
|
+
/** Drop a topic's pending state once it has been actuated (or cancelled). */
|
|
97
|
+
clear(topicId) {
|
|
98
|
+
this.pending.delete(topicId);
|
|
99
|
+
}
|
|
100
|
+
/** Topics currently marked terminate-pending (overrun but not yet confirmed). */
|
|
101
|
+
pendingTopics() {
|
|
102
|
+
return [...this.pending.entries()].filter(([, n]) => n < this.threshold).map(([t]) => t);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=enforcedTermination.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"enforcedTermination.js","sourceRoot":"","sources":["../../src/monitoring/enforcedTermination.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AA4CH,MAAM,CAAC,MAAM,mCAAmC,GAA8B;IAC5E,YAAY,EAAE,GAAG;IACjB,sBAAsB,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,MAAM;CAC7C,CAAC;AAYF;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAC5B,GAA0B,EAC1B,GAA8B,EAC9B,KAAa;IAEb,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,aAAa;QAAE,OAAO,IAAI,CAAC;IAEhE,iDAAiD;IACjD,IAAI,GAAG,CAAC,aAAa,IAAI,IAAI,IAAI,GAAG,CAAC,aAAa,GAAG,CAAC,IAAI,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;QAC7F,OAAO,EAAE,IAAI,EAAE,mBAAmB,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE,CAAC;IACzF,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,IAAI,GAAG,CAAC,WAAW,IAAI,IAAI,IAAI,GAAG,CAAC,eAAe,IAAI,IAAI,IAAI,GAAG,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;QACtF,MAAM,OAAO,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC;QACjD,IAAI,OAAO,IAAI,GAAG,CAAC,eAAe,GAAG,GAAG,CAAC,YAAY,EAAE,CAAC;YACtD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,CAAC,eAAe,EAAE,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,6EAA6E;IAC7E,qEAAqE;IACrE,MAAM,OAAO,GAAG,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,WAAW,CAAC;IACnD,MAAM,KAAK,GAAgC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC;IACjG,MAAM,UAAU,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC;IAC5C,IAAI,UAAU,IAAI,GAAG,CAAC,sBAAsB,EAAE,CAAC;QAC7C,OAAO;YACL,IAAI,EAAE,kBAAkB;YACxB,cAAc,EAAE,UAAU;YAC1B,cAAc,EAAE,GAAG,CAAC,sBAAsB;YAC1C,KAAK;SACN,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,OAAO,oBAAoB;IACd,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpC,SAAS,CAAS;IAEnC,YAAY,gBAAgB,GAAG,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,eAAiC;QACzC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,CAAC;QACzC,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YAC9B,IAAI,IAAI,IAAI,IAAI,CAAC,SAAS;gBAAE,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpD,CAAC;QACD,iEAAiE;QACjE,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;gBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,6EAA6E;IAC7E,KAAK,CAAC,OAAe;QACnB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAED,iFAAiF;IACjF,aAAa;QACX,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3F,CAAC;CACF"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { AutonomousRunSnapshot } from './enforcedTermination.js';
|
|
2
|
+
import type { EnforcedTerminationAuditRow } from './EnforcedTerminationWatchdog.js';
|
|
3
|
+
/**
|
|
4
|
+
* Build the `listRuns` provider: read every per-topic autonomous run file and
|
|
5
|
+
* project it into the pure-core snapshot. Legacy single-file jobs (topic === null)
|
|
6
|
+
* are skipped — the watchdog only governs per-topic runs (the only ones with a
|
|
7
|
+
* resolvable session to terminate). Each snapshot is durable-state only (file
|
|
8
|
+
* frontmatter + mtime), so the overrun decision survives a server restart.
|
|
9
|
+
*/
|
|
10
|
+
export declare function buildEnforcedTerminationListRuns(stateDir: string): () => AutonomousRunSnapshot[];
|
|
11
|
+
/**
|
|
12
|
+
* Build the audit sink: append one JSON line per transition to
|
|
13
|
+
* `logs/enforced-termination.jsonl`. Bounded rotation at ~5MB so the audit can
|
|
14
|
+
* never grow unbounded. Wrapped by the watchdog's safeAudit so a write error
|
|
15
|
+
* never reaches the loop, but we also swallow here for defense in depth.
|
|
16
|
+
*/
|
|
17
|
+
export declare function buildEnforcedTerminationAudit(logsDir: string): (row: EnforcedTerminationAuditRow) => void;
|
|
18
|
+
//# sourceMappingURL=enforcedTerminationWiring.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"enforcedTerminationWiring.d.ts","sourceRoot":"","sources":["../../src/monitoring/enforcedTerminationWiring.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACtE,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAEpF;;;;;;GAMG;AACH,wBAAgB,gCAAgC,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,qBAAqB,EAAE,CA8BhG;AAED;;;;;GAKG;AACH,wBAAgB,6BAA6B,CAAC,OAAO,EAAE,MAAM,GAAG,CAAC,GAAG,EAAE,2BAA2B,KAAK,IAAI,CAiBzG"}
|