@telora/daemon 0.22.4 → 0.22.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build-info.json +5 -3
- package/dist/backends/index.d.ts.map +1 -1
- package/dist/backends/index.js +2 -0
- package/dist/backends/index.js.map +1 -1
- package/dist/ci/ci-sensor-loop.d.ts +36 -2
- package/dist/ci/ci-sensor-loop.d.ts.map +1 -1
- package/dist/ci/ci-sensor-loop.js +48 -9
- package/dist/ci/ci-sensor-loop.js.map +1 -1
- package/dist/ci/ci-status.d.ts.map +1 -1
- package/dist/ci/ci-status.js +1 -0
- package/dist/ci/ci-status.js.map +1 -1
- package/dist/completion/detector.d.ts +9 -0
- package/dist/completion/detector.d.ts.map +1 -1
- package/dist/completion/detector.js +18 -0
- package/dist/completion/detector.js.map +1 -1
- package/dist/completion/event-phases.d.ts +24 -8
- package/dist/completion/event-phases.d.ts.map +1 -1
- package/dist/completion/event-phases.js +81 -45
- package/dist/completion/event-phases.js.map +1 -1
- package/dist/completion/event.d.ts +9 -0
- package/dist/completion/event.d.ts.map +1 -1
- package/dist/completion/event.js +4 -0
- package/dist/completion/event.js.map +1 -1
- package/dist/completion/exit-classification.d.ts +2 -2
- package/dist/completion/exit-classification.d.ts.map +1 -1
- package/dist/completion/exit-classification.js +2 -0
- package/dist/completion/exit-classification.js.map +1 -1
- package/dist/completion/index.js +1 -1
- package/dist/completion/liveness-firing.d.ts +27 -0
- package/dist/completion/liveness-firing.d.ts.map +1 -0
- package/dist/completion/liveness-firing.js +79 -0
- package/dist/completion/liveness-firing.js.map +1 -0
- package/dist/completion/stdin-turn-tracker.d.ts +19 -0
- package/dist/completion/stdin-turn-tracker.d.ts.map +1 -0
- package/dist/completion/stdin-turn-tracker.js +57 -0
- package/dist/completion/stdin-turn-tracker.js.map +1 -0
- package/dist/completion/supervision-firing-record.d.ts +190 -0
- package/dist/completion/supervision-firing-record.d.ts.map +1 -0
- package/dist/completion/supervision-firing-record.js +232 -0
- package/dist/completion/supervision-firing-record.js.map +1 -0
- package/dist/completion/team-lifecycle.d.ts.map +1 -1
- package/dist/completion/team-lifecycle.js +3 -0
- package/dist/completion/team-lifecycle.js.map +1 -1
- package/dist/completion/team-phase.d.ts.map +1 -1
- package/dist/completion/team-phase.js +2 -1
- package/dist/completion/team-phase.js.map +1 -1
- package/dist/directive/directive-dispatch.d.ts +7 -1
- package/dist/directive/directive-dispatch.d.ts.map +1 -1
- package/dist/directive/directive-dispatch.js +75 -12
- package/dist/directive/directive-dispatch.js.map +1 -1
- package/dist/directive/directive-executor.d.ts +1 -1
- package/dist/directive/directive-executor.d.ts.map +1 -1
- package/dist/directive/directive-executor.js +10 -6
- package/dist/directive/directive-executor.js.map +1 -1
- package/dist/directive/directive-queue.d.ts +35 -9
- package/dist/directive/directive-queue.d.ts.map +1 -1
- package/dist/directive/directive-queue.js +15 -8
- package/dist/directive/directive-queue.js.map +1 -1
- package/dist/directive/directive-verify-hold-gate.d.ts +20 -0
- package/dist/directive/directive-verify-hold-gate.d.ts.map +1 -0
- package/dist/directive/directive-verify-hold-gate.js +47 -0
- package/dist/directive/directive-verify-hold-gate.js.map +1 -0
- package/dist/drift/staleness-sensor.d.ts +62 -67
- package/dist/drift/staleness-sensor.d.ts.map +1 -1
- package/dist/drift/staleness-sensor.js +233 -68
- package/dist/drift/staleness-sensor.js.map +1 -1
- package/dist/drift/staleness.d.ts +8 -0
- package/dist/drift/staleness.d.ts.map +1 -1
- package/dist/drift/staleness.js +7 -1
- package/dist/drift/staleness.js.map +1 -1
- package/dist/focus/focus-engine.js +1 -1
- package/dist/focus/focus-engine.js.map +1 -1
- package/dist/focus/focus-executor.d.ts.map +1 -1
- package/dist/focus/focus-executor.js +12 -6
- package/dist/focus/focus-executor.js.map +1 -1
- package/dist/focus/focus-team-state.d.ts.map +1 -1
- package/dist/focus/focus-team-state.js +1 -0
- package/dist/focus/focus-team-state.js.map +1 -1
- package/dist/focus/focus-worktree-state.d.ts.map +1 -1
- package/dist/focus/focus-worktree-state.js +4 -2
- package/dist/focus/focus-worktree-state.js.map +1 -1
- package/dist/focus/listener-spawn.d.ts +59 -6
- package/dist/focus/listener-spawn.d.ts.map +1 -1
- package/dist/focus/listener-spawn.js +164 -39
- package/dist/focus/listener-spawn.js.map +1 -1
- package/dist/focus/listener.d.ts.map +1 -1
- package/dist/focus/listener.js +20 -11
- package/dist/focus/listener.js.map +1 -1
- package/dist/focus/team-actionable-work.d.ts +90 -0
- package/dist/focus/team-actionable-work.d.ts.map +1 -0
- package/dist/focus/team-actionable-work.js +113 -0
- package/dist/focus/team-actionable-work.js.map +1 -0
- package/dist/merge/focus-merge.d.ts.map +1 -1
- package/dist/merge/focus-merge.js +2 -1
- package/dist/merge/focus-merge.js.map +1 -1
- package/dist/nexus/return.d.ts +60 -1
- package/dist/nexus/return.d.ts.map +1 -1
- package/dist/nexus/return.js +78 -2
- package/dist/nexus/return.js.map +1 -1
- package/dist/queries/schemas.d.ts +61 -0
- package/dist/queries/schemas.d.ts.map +1 -1
- package/dist/queries/schemas.js +12 -2
- package/dist/queries/schemas.js.map +1 -1
- package/dist/shell/heartbeat-metadata.d.ts +91 -0
- package/dist/shell/heartbeat-metadata.d.ts.map +1 -0
- package/dist/shell/heartbeat-metadata.js +169 -0
- package/dist/shell/heartbeat-metadata.js.map +1 -0
- package/dist/shell/heartbeat.d.ts +7 -22
- package/dist/shell/heartbeat.d.ts.map +1 -1
- package/dist/shell/heartbeat.js +120 -89
- package/dist/shell/heartbeat.js.map +1 -1
- package/dist/shell/model-resolution.d.ts +54 -0
- package/dist/shell/model-resolution.d.ts.map +1 -1
- package/dist/shell/model-resolution.js +50 -0
- package/dist/shell/model-resolution.js.map +1 -1
- package/dist/shell/unified-shell.d.ts.map +1 -1
- package/dist/shell/unified-shell.js +21 -12
- package/dist/shell/unified-shell.js.map +1 -1
- package/dist/spawn/helpers.d.ts.map +1 -1
- package/dist/spawn/helpers.js +6 -1
- package/dist/spawn/helpers.js.map +1 -1
- package/dist/spawn/model-routing.d.ts +14 -19
- package/dist/spawn/model-routing.d.ts.map +1 -1
- package/dist/spawn/model-routing.js +57 -39
- package/dist/spawn/model-routing.js.map +1 -1
- package/dist/spawn/process-spawn.d.ts +32 -1
- package/dist/spawn/process-spawn.d.ts.map +1 -1
- package/dist/spawn/process-spawn.js +43 -5
- package/dist/spawn/process-spawn.js.map +1 -1
- package/dist/spawn/spawn-cooldown.d.ts.map +1 -1
- package/dist/spawn/spawn-cooldown.js +2 -1
- package/dist/spawn/spawn-cooldown.js.map +1 -1
- package/dist/spawn/worktree-session.d.ts +53 -5
- package/dist/spawn/worktree-session.d.ts.map +1 -1
- package/dist/spawn/worktree-session.js +61 -29
- package/dist/spawn/worktree-session.js.map +1 -1
- package/dist/testing/fixtures/index.js +1 -1
- package/dist/types/focus.d.ts +12 -2
- package/dist/types/focus.d.ts.map +1 -1
- package/dist/types/index.js +1 -1
- package/dist/types/session.d.ts +1 -1
- package/dist/types/session.d.ts.map +1 -1
- package/dist/update/auto-update-idle.d.ts +22 -0
- package/dist/update/auto-update-idle.d.ts.map +1 -1
- package/dist/update/auto-update-idle.js +8 -1
- package/dist/update/auto-update-idle.js.map +1 -1
- package/dist/update/auto-update.d.ts +0 -6
- package/dist/update/auto-update.d.ts.map +1 -1
- package/dist/update/auto-update.js +20 -6
- package/dist/update/auto-update.js.map +1 -1
- package/dist/update/version-check.d.ts +73 -0
- package/dist/update/version-check.d.ts.map +1 -1
- package/dist/update/version-check.js +107 -11
- package/dist/update/version-check.js.map +1 -1
- package/dist/worktree/verify-interlock.d.ts +34 -0
- package/dist/worktree/verify-interlock.d.ts.map +1 -1
- package/dist/worktree/verify-interlock.js +27 -1
- package/dist/worktree/verify-interlock.js.map +1 -1
- package/package.json +3 -3
|
@@ -1,55 +1,5 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Propagation-staleness sensor (focus: "Propagation Staleness Is Invisible").
|
|
3
|
-
*
|
|
4
|
-
* The differential instrument this focus is named for: it computes
|
|
5
|
-
* newest-available MINUS actually-running for a tracked target. Its first (and
|
|
6
|
-
* directly observable) target is the running daemon vs the Gitea npm registry:
|
|
7
|
-
*
|
|
8
|
-
* running = ~/.telora/daemon.meta.json .version (this daemon's own build)
|
|
9
|
-
* newest = the warm version cache .latestVersion (Gitea registry dist-tags.latest,
|
|
10
|
-
* kept fresh by startVersionCacheRefreshLoop)
|
|
11
|
-
*
|
|
12
|
-
* CLOSING the gap, not reporting it. The daemon-version gap is machine-closable:
|
|
13
|
-
* the safe-seam auto-update controller (auto-update.ts) already installs a newer
|
|
14
|
-
* registry version at the next batch-safe idle seam. So when the gap is beyond
|
|
15
|
-
* threshold this sensor DEFERS to that closer -- it does NOT tell a human "the
|
|
16
|
-
* daemon is stale, restart it". Telling a human about a gap the machine is
|
|
17
|
-
* already closing converts machine signal into human chore, and firing on a
|
|
18
|
-
* healthy daemon that is merely deferring to its next safe seam is exactly the
|
|
19
|
-
* failure mode this focus exists to remove.
|
|
20
|
-
*
|
|
21
|
-
* A human is involved ONLY where the machine genuinely cannot act: auto-update
|
|
22
|
-
* is disabled for this daemon (installMode=npx, or --no-auto-update /
|
|
23
|
-
* TELORA_AUTO_UPDATE=0, reflected as autoUpdateEnabled=false in daemon.meta.json).
|
|
24
|
-
* The operator opted out of automation, so the machine cannot close the gap --
|
|
25
|
-
* and even then the deliverable is a READY ACTION handed over (the exact
|
|
26
|
-
* `npm install -g` command + restart), never an inbox item that merely says a
|
|
27
|
-
* gap exists. That handover auto-resolves once the running version catches up,
|
|
28
|
-
* and is retired immediately if auto-update is re-enabled (the machine reclaims
|
|
29
|
-
* closure).
|
|
30
|
-
*
|
|
31
|
-
* Every tick leaves a trace (delegated-to-auto-update / handed-over /
|
|
32
|
-
* back-in-sync / "measured, within threshold" / "UNMEASURED"), so an ABSENT
|
|
33
|
-
* trace positively means "measured and in sync" -- never "not measured". Wedged
|
|
34
|
-
* auto-update (enabled but not running) is made visible by the SEPARATE
|
|
35
|
-
* auto-update decision heartbeat (auto-update.ts): an absent [auto-update] trace
|
|
36
|
-
* identifies a controller that is not running. That is not re-derived here.
|
|
37
|
-
*
|
|
38
|
-
* Pure decision (computeDaemonVersionDifferential / decideStalenessAction) +
|
|
39
|
-
* injected IO, mirroring ci-sensor-loop.ts, so the loop is unit-testable without
|
|
40
|
-
* a live registry or DB. Registered in LOOP_REGISTRY under the opt-out guard
|
|
41
|
-
* TELORA_STALENESS_SENSOR_LOOP.
|
|
42
|
-
*
|
|
43
|
-
* The two infra-owned targets observed 2026-07-24 (npmjs mirror serving a stale
|
|
44
|
-
* version; Opus 5 seed merged-but-maybe-not-applied) are NOT computable from
|
|
45
|
-
* this repo (no prod DB access; infra alone publishes to npmjs). They are
|
|
46
|
-
* recorded, with their operator-facing close action, in
|
|
47
|
-
* docs/runbook-propagation-staleness-targets.md -- the sensor deliberately does
|
|
48
|
-
* NOT attempt those infra actions.
|
|
49
|
-
*
|
|
50
|
-
* @module staleness-sensor
|
|
51
|
-
*/
|
|
52
1
|
import type { DaemonConfig } from '../types/index.js';
|
|
2
|
+
import { type VersionDistance } from '../update/version-check.js';
|
|
53
3
|
/**
|
|
54
4
|
* escalation_kind for a stale daemon that the machine CANNOT auto-close (auto-update
|
|
55
5
|
* disabled). Carries a ready action, not a bare "gap exists" notice.
|
|
@@ -64,12 +14,7 @@ export declare const DAEMON_VERSION_TARGET = "daemon-version";
|
|
|
64
14
|
* failure (0.20.21 vs 0.20.28 = 7 patches) is well past this floor.
|
|
65
15
|
*/
|
|
66
16
|
export declare const DEFAULT_STALENESS_PATCH_THRESHOLD = 3;
|
|
67
|
-
|
|
68
|
-
export interface VersionDistance {
|
|
69
|
-
major: number;
|
|
70
|
-
minor: number;
|
|
71
|
-
patch: number;
|
|
72
|
-
}
|
|
17
|
+
export type { VersionDistance };
|
|
73
18
|
/** The computed newest-vs-running differential for a tracked target. */
|
|
74
19
|
export interface StalenessDifferential {
|
|
75
20
|
target: string;
|
|
@@ -92,8 +37,11 @@ export declare function computeDaemonVersionDifferential(running: string | null,
|
|
|
92
37
|
* What the sensor should do with the current differential.
|
|
93
38
|
* - `delegate` -- gap is beyond threshold and the machine closes it itself
|
|
94
39
|
* (auto-update enabled): defer to that closer, no human.
|
|
95
|
-
* - `hand-over` -- gap is beyond threshold
|
|
96
|
-
* (auto-update disabled
|
|
40
|
+
* - `hand-over` -- gap is beyond threshold and a human signal is owed: either
|
|
41
|
+
* the machine cannot act (auto-update disabled -> a ready
|
|
42
|
+
* action), OR the machine SHOULD act but the gap has persisted
|
|
43
|
+
* past the wedge window without the running version advancing
|
|
44
|
+
* (auto-update wedged -> a needs_operator signal).
|
|
97
45
|
* - `resolve` -- an open handover exists but the machine is now closing the
|
|
98
46
|
* gap (back within threshold, OR auto-update re-enabled).
|
|
99
47
|
* - `none` -- nothing to do.
|
|
@@ -101,12 +49,30 @@ export declare function computeDaemonVersionDifferential(running: string | null,
|
|
|
101
49
|
export type StalenessAction = 'delegate' | 'hand-over' | 'resolve' | 'none';
|
|
102
50
|
/**
|
|
103
51
|
* Pure decision. The pivot is `canMachineClose`: when the auto-update controller
|
|
104
|
-
* owns closing the daemon-version gap, the sensor
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
52
|
+
* owns closing the daemon-version gap, the sensor normally DEFERS and never hands
|
|
53
|
+
* a human a report (and retires any stale handover). Two things pull a human in:
|
|
54
|
+
* the machine genuinely cannot act (`!canMachineClose`), OR the machine can act
|
|
55
|
+
* but the SAME beyond-threshold gap has persisted past the wedge window without
|
|
56
|
+
* the running version advancing (`wedged`) -- a closer that is enabled but not
|
|
57
|
+
* applying is a failed close, a legitimate operator signal, not an in-flight one
|
|
58
|
+
* to stay silent about. A null differential (unmeasurable) never acts.
|
|
108
59
|
*/
|
|
109
|
-
export declare function decideStalenessAction(diff: StalenessDifferential | null, hasOpen: boolean, canMachineClose: boolean): StalenessAction;
|
|
60
|
+
export declare function decideStalenessAction(diff: StalenessDifferential | null, hasOpen: boolean, canMachineClose: boolean, wedged?: boolean): StalenessAction;
|
|
61
|
+
/**
|
|
62
|
+
* Record that `causeKey` is currently a beyond-threshold, machine-closable gap
|
|
63
|
+
* and report whether it has persisted at/past `thresholdMs`. Prunes any other
|
|
64
|
+
* key so the store stays single-entry (a new cause_key -- running advanced or a
|
|
65
|
+
* new newest -- starts a fresh clock). Pure over the injected store, so a unit
|
|
66
|
+
* test drives it with a fresh Map and a fixed clock.
|
|
67
|
+
*/
|
|
68
|
+
export declare function recordAndCheckGapWedged(store: Map<string, number>, causeKey: string, nowMs: number, thresholdMs: number): boolean;
|
|
69
|
+
/**
|
|
70
|
+
* Default wedge window in hours: how long a machine-closable, beyond-threshold
|
|
71
|
+
* gap may persist without the running version advancing before the closer is
|
|
72
|
+
* treated as wedged. Kept ABOVE the auto-update drain threshold (6h default) so a
|
|
73
|
+
* legitimately-busy daemon that will drain-and-update on its own is never flagged.
|
|
74
|
+
*/
|
|
75
|
+
export declare const DEFAULT_STALENESS_WEDGE_THRESHOLD_HOURS = 12;
|
|
110
76
|
/** Minimal open-escalation row shape the sensor's dedup / resolve reads. */
|
|
111
77
|
interface StalenessEscalationRow {
|
|
112
78
|
id: string;
|
|
@@ -126,10 +92,24 @@ export declare function isStalenessRefileSuppressed(rows: readonly StalenessEsca
|
|
|
126
92
|
running: string;
|
|
127
93
|
newest: string;
|
|
128
94
|
}): boolean;
|
|
95
|
+
/**
|
|
96
|
+
* Why a human is owed a signal for a beyond-threshold gap:
|
|
97
|
+
* - `auto_update_disabled` -- the machine cannot act; hand a READY ACTION.
|
|
98
|
+
* - `auto_update_wedged` -- the machine SHOULD act but the gap has persisted
|
|
99
|
+
* past the wedge window without the running version
|
|
100
|
+
* advancing; the closer is enabled but not applying.
|
|
101
|
+
*/
|
|
102
|
+
export type StalenessHandoverReason = 'auto_update_disabled' | 'auto_update_wedged';
|
|
129
103
|
/** Injected IO for the sensor tick. Defaults reach for the real implementations. */
|
|
130
104
|
export interface StalenessSensorDeps {
|
|
131
105
|
readRunningVersion: () => string | null;
|
|
132
106
|
readNewestAvailable: () => string | null;
|
|
107
|
+
/** The registry the `newest` reading was consulted from (warm cache), or null
|
|
108
|
+
* (legacy cache with no registry) -- used to ATTRIBUTE the reading. */
|
|
109
|
+
readReadingRegistry: () => string | null;
|
|
110
|
+
/** The registry npm resolves for an install of @telora/daemon -- the reading
|
|
111
|
+
* must attribute to THIS for its agreement to count. */
|
|
112
|
+
readInstallRegistry: () => string;
|
|
133
113
|
patchThreshold: number;
|
|
134
114
|
/**
|
|
135
115
|
* True when the auto-update controller can close the daemon-version gap itself.
|
|
@@ -141,14 +121,28 @@ export interface StalenessSensorDeps {
|
|
|
141
121
|
running: string;
|
|
142
122
|
newest: string;
|
|
143
123
|
}) => Promise<boolean>;
|
|
144
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* Hand a human a signal for a beyond-threshold gap: a READY ACTION when the
|
|
126
|
+
* machine cannot act (reason auto_update_disabled), or a WEDGED-CLOSER signal
|
|
127
|
+
* when auto-update is enabled but not applying (reason auto_update_wedged).
|
|
128
|
+
*/
|
|
145
129
|
handOverReadyAction: (params: {
|
|
146
130
|
organizationId: string;
|
|
147
131
|
productId: string;
|
|
148
132
|
diff: StalenessDifferential;
|
|
149
133
|
readyCommand: string;
|
|
134
|
+
reason: StalenessHandoverReason;
|
|
135
|
+
registry: string;
|
|
150
136
|
}) => Promise<void>;
|
|
151
137
|
resolveStalenessEscalations: () => Promise<number>;
|
|
138
|
+
/**
|
|
139
|
+
* Whether a machine-closable, beyond-threshold gap for `causeKey` has persisted
|
|
140
|
+
* past the wedge window (auto-update enabled but the running version not
|
|
141
|
+
* advancing). Records first-seen on the way, so persistence is a code/state
|
|
142
|
+
* fact -- a test drives it with a stub returning true/false.
|
|
143
|
+
*/
|
|
144
|
+
checkGapWedged: (causeKey: string, nowMs: number) => boolean;
|
|
145
|
+
now: () => number;
|
|
152
146
|
}
|
|
153
147
|
export interface StalenessSensorTickCounts {
|
|
154
148
|
/** True when the differential could be computed (both versions known). */
|
|
@@ -157,8 +151,10 @@ export interface StalenessSensorTickCounts {
|
|
|
157
151
|
behind: boolean;
|
|
158
152
|
/** True when the machine (auto-update) owns closing the measured gap this tick. */
|
|
159
153
|
delegatedToAutoUpdate: boolean;
|
|
160
|
-
/**
|
|
154
|
+
/** Handovers filed this tick (auto-update disabled OR wedged). */
|
|
161
155
|
handoversToOperator: number;
|
|
156
|
+
/** Of the handovers, those filed because auto-update is enabled but WEDGED. */
|
|
157
|
+
wedgedSignalsFiled: number;
|
|
162
158
|
escalationsResolved: number;
|
|
163
159
|
}
|
|
164
160
|
/**
|
|
@@ -172,5 +168,4 @@ export interface StalenessSensorTickCounts {
|
|
|
172
168
|
* Every path emits a trace so an absent signal means "measured and in sync".
|
|
173
169
|
*/
|
|
174
170
|
export declare function runStalenessSensorTick(config: DaemonConfig, deps?: StalenessSensorDeps): Promise<StalenessSensorTickCounts>;
|
|
175
|
-
export {};
|
|
176
171
|
//# sourceMappingURL=staleness-sensor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"staleness-sensor.d.ts","sourceRoot":"","sources":["../../src/drift/staleness-sensor.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"staleness-sensor.d.ts","sourceRoot":"","sources":["../../src/drift/staleness-sensor.ts"],"names":[],"mappings":"AAmEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAItD,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,4BAA4B,CAAC;AAUpC;;;GAGG;AACH,eAAO,MAAM,iCAAiC,sBAAsB,CAAC;AAErE,gEAAgE;AAChE,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,iCAAiC,IAAI,CAAC;AAEnD,YAAY,EAAE,eAAe,EAAE,CAAC;AAEhC,wEAAwE;AACxE,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,eAAe,CAAC;IAC1B,wGAAwG;IACxG,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED;;;;;GAKG;AACH,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,MAAM,GAAG,IAAI,EACtB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,cAAc,GAAE,MAA0C,GACzD,qBAAqB,GAAG,IAAI,CAO9B;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,eAAe,GAAG,UAAU,GAAG,WAAW,GAAG,SAAS,GAAG,MAAM,CAAC;AAE5E;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,qBAAqB,GAAG,IAAI,EAClC,OAAO,EAAE,OAAO,EAChB,eAAe,EAAE,OAAO,EACxB,MAAM,UAAQ,GACb,eAAe,CAmBjB;AAYD;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC1B,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,GAClB,OAAO,CAKT;AAED;;;;;GAKG;AACH,eAAO,MAAM,uCAAuC,KAAK,CAAC;AA0C1D,4EAA4E;AAC5E,UAAU,sBAAsB;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC3C;AAWD;;;;;;;GAOG;AACH,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,SAAS,sBAAsB,EAAE,EACvC,MAAM,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC1C,OAAO,CAUT;AAmCD;;;;;;GAMG;AACH,MAAM,MAAM,uBAAuB,GAAG,sBAAsB,GAAG,oBAAoB,CAAC;AA4DpF,oFAAoF;AACpF,MAAM,WAAW,mBAAmB;IAClC,kBAAkB,EAAE,MAAM,MAAM,GAAG,IAAI,CAAC;IACxC,mBAAmB,EAAE,MAAM,MAAM,GAAG,IAAI,CAAC;IACzC;4EACwE;IACxE,mBAAmB,EAAE,MAAM,MAAM,GAAG,IAAI,CAAC;IACzC;6DACyD;IACzD,mBAAmB,EAAE,MAAM,MAAM,CAAC;IAClC,cAAc,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,kBAAkB,EAAE,MAAM,OAAO,CAAC;IAClC,0BAA0B,EAAE,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9F;;;;OAIG;IACH,mBAAmB,EAAE,CAAC,MAAM,EAAE;QAC5B,cAAc,EAAE,MAAM,CAAC;QACvB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,qBAAqB,CAAC;QAC5B,YAAY,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,uBAAuB,CAAC;QAChC,QAAQ,EAAE,MAAM,CAAC;KAClB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpB,2BAA2B,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;IACnD;;;;;OAKG;IACH,cAAc,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;IAC7D,GAAG,EAAE,MAAM,MAAM,CAAC;CACnB;AAiBD,MAAM,WAAW,yBAAyB;IACxC,0EAA0E;IAC1E,QAAQ,EAAE,OAAO,CAAC;IAClB,yEAAyE;IACzE,MAAM,EAAE,OAAO,CAAC;IAChB,mFAAmF;IACnF,qBAAqB,EAAE,OAAO,CAAC;IAC/B,kEAAkE;IAClE,mBAAmB,EAAE,MAAM,CAAC;IAC5B,+EAA+E;IAC/E,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAuED;;;;;;;;;GASG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,YAAY,EACpB,IAAI,GAAE,mBAAiC,GACtC,OAAO,CAAC,yBAAyB,CAAC,CA4EpC"}
|
|
@@ -1,10 +1,72 @@
|
|
|
1
1
|
// PINNED-BY-BEGIN (generated by gen:pinned-by -- do not hand-edit; checked by lint:pinned-by)
|
|
2
2
|
// Pinned by: packages/daemon/src/drift/staleness-sensor.test.ts
|
|
3
3
|
// PINNED-BY-END
|
|
4
|
+
/**
|
|
5
|
+
* Propagation-staleness sensor (focus: "Propagation Staleness Is Invisible").
|
|
6
|
+
*
|
|
7
|
+
* The differential instrument this focus is named for: it computes
|
|
8
|
+
* newest-available MINUS actually-running for a tracked target. Its first (and
|
|
9
|
+
* directly observable) target is the running daemon vs the INSTALL registry --
|
|
10
|
+
* the registry npm resolves for `npm install -g @telora/daemon@latest`, which is
|
|
11
|
+
* npmjs or Gitea or a mirror depending on the host's npm config, NOT a fixed host:
|
|
12
|
+
*
|
|
13
|
+
* running = ~/.telora/daemon.meta.json .version (this daemon's own build)
|
|
14
|
+
* newest = the warm version cache .latestVersion (dist-tags.latest from the
|
|
15
|
+
* registry checkForUpdates resolved, kept fresh by
|
|
16
|
+
* startVersionCacheRefreshLoop)
|
|
17
|
+
*
|
|
18
|
+
* ATTRIBUTED readings only. The cache records WHICH registry produced `newest`;
|
|
19
|
+
* a reading is trusted only when that registry is the one npm resolves for an
|
|
20
|
+
* install. A reading from any other registry (an operator's TELORA_REGISTRY_URL
|
|
21
|
+
* override, a legacy cache with no registry) is reported UNMEASURED, never
|
|
22
|
+
* "in sync" -- a host cannot certify the installed version is current by reading
|
|
23
|
+
* a registry the installer does not use.
|
|
24
|
+
*
|
|
25
|
+
* CLOSING the gap, not reporting it. The daemon-version gap is machine-closable:
|
|
26
|
+
* the safe-seam auto-update controller (auto-update.ts) already installs a newer
|
|
27
|
+
* registry version at the next batch-safe idle seam. So when the gap is beyond
|
|
28
|
+
* threshold this sensor DEFERS to that closer -- it does NOT tell a human "the
|
|
29
|
+
* daemon is stale, restart it". Telling a human about a gap the machine is
|
|
30
|
+
* already closing converts machine signal into human chore, and firing on a
|
|
31
|
+
* healthy daemon that is merely deferring to its next safe seam is exactly the
|
|
32
|
+
* failure mode this focus exists to remove.
|
|
33
|
+
*
|
|
34
|
+
* A human is involved ONLY where the machine genuinely cannot act: auto-update
|
|
35
|
+
* is disabled for this daemon (installMode=npx, or --no-auto-update /
|
|
36
|
+
* TELORA_AUTO_UPDATE=0, reflected as autoUpdateEnabled=false in daemon.meta.json).
|
|
37
|
+
* The operator opted out of automation, so the machine cannot close the gap --
|
|
38
|
+
* and even then the deliverable is a READY ACTION handed over (the exact
|
|
39
|
+
* `npm install -g` command + restart), never an inbox item that merely says a
|
|
40
|
+
* gap exists. That handover auto-resolves once the running version catches up,
|
|
41
|
+
* and is retired immediately if auto-update is re-enabled (the machine reclaims
|
|
42
|
+
* closure).
|
|
43
|
+
*
|
|
44
|
+
* Every tick leaves a trace (delegated-to-auto-update / handed-over /
|
|
45
|
+
* back-in-sync / "measured, within threshold" / "UNMEASURED"), so an ABSENT
|
|
46
|
+
* trace positively means "measured and in sync" -- never "not measured". Wedged
|
|
47
|
+
* auto-update (enabled but not running) is made visible by the SEPARATE
|
|
48
|
+
* auto-update decision heartbeat (auto-update.ts): an absent [auto-update] trace
|
|
49
|
+
* identifies a controller that is not running. That is not re-derived here.
|
|
50
|
+
*
|
|
51
|
+
* Pure decision (computeDaemonVersionDifferential / decideStalenessAction) +
|
|
52
|
+
* injected IO, mirroring ci-sensor-loop.ts, so the loop is unit-testable without
|
|
53
|
+
* a live registry or DB. Registered in LOOP_REGISTRY under the opt-out guard
|
|
54
|
+
* TELORA_STALENESS_SENSOR_LOOP.
|
|
55
|
+
*
|
|
56
|
+
* The two infra-owned targets observed 2026-07-24 (npmjs mirror serving a stale
|
|
57
|
+
* version; Opus 5 seed merged-but-maybe-not-applied) are NOT computable from
|
|
58
|
+
* this repo (no prod DB access; infra alone publishes to npmjs). They are
|
|
59
|
+
* recorded, with their operator-facing close action, in
|
|
60
|
+
* docs/runbook-propagation-staleness-targets.md -- the sensor deliberately does
|
|
61
|
+
* NOT attempt those infra actions.
|
|
62
|
+
*
|
|
63
|
+
* @module staleness-sensor
|
|
64
|
+
*/
|
|
65
|
+
import { hostname as osHostname } from 'node:os';
|
|
4
66
|
import { configForProduct } from '../shell/config.js';
|
|
5
67
|
import { readFreshVersionCache } from './staleness.js';
|
|
6
68
|
import { readDaemonMeta } from '../shell/daemon-process.js';
|
|
7
|
-
import { isNewerVersion,
|
|
69
|
+
import { isNewerVersion, versionDistance, resolveInstallRegistryUrl, isReadingAttributable, } from '../update/version-check.js';
|
|
8
70
|
import { createEscalation, markEscalationResolved } from '../queries/escalations.js';
|
|
9
71
|
import { callApi } from '../queries/shared.js';
|
|
10
72
|
import { daemonStalenessCauseKey } from '../queries/cause-key.js';
|
|
@@ -25,18 +87,6 @@ export const DAEMON_VERSION_TARGET = 'daemon-version';
|
|
|
25
87
|
* failure (0.20.21 vs 0.20.28 = 7 patches) is well past this floor.
|
|
26
88
|
*/
|
|
27
89
|
export const DEFAULT_STALENESS_PATCH_THRESHOLD = 3;
|
|
28
|
-
/** Component-wise (newest - running), clamped at 0 per component. */
|
|
29
|
-
function versionDistance(running, newest) {
|
|
30
|
-
const r = parseVersionCore(running);
|
|
31
|
-
const n = parseVersionCore(newest);
|
|
32
|
-
if (!r || !n)
|
|
33
|
-
return { major: 0, minor: 0, patch: 0 };
|
|
34
|
-
return {
|
|
35
|
-
major: Math.max(0, n[0] - r[0]),
|
|
36
|
-
minor: Math.max(0, n[1] - r[1]),
|
|
37
|
-
patch: Math.max(0, n[2] - r[2]),
|
|
38
|
-
};
|
|
39
|
-
}
|
|
40
90
|
/**
|
|
41
91
|
* Compute the daemon-vs-registry differential. Returns null when it cannot be
|
|
42
92
|
* MEASURED (either version missing) -- the caller must then emit an explicit
|
|
@@ -53,16 +103,24 @@ export function computeDaemonVersionDifferential(running, newest, patchThreshold
|
|
|
53
103
|
}
|
|
54
104
|
/**
|
|
55
105
|
* Pure decision. The pivot is `canMachineClose`: when the auto-update controller
|
|
56
|
-
* owns closing the daemon-version gap, the sensor
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
106
|
+
* owns closing the daemon-version gap, the sensor normally DEFERS and never hands
|
|
107
|
+
* a human a report (and retires any stale handover). Two things pull a human in:
|
|
108
|
+
* the machine genuinely cannot act (`!canMachineClose`), OR the machine can act
|
|
109
|
+
* but the SAME beyond-threshold gap has persisted past the wedge window without
|
|
110
|
+
* the running version advancing (`wedged`) -- a closer that is enabled but not
|
|
111
|
+
* applying is a failed close, a legitimate operator signal, not an in-flight one
|
|
112
|
+
* to stay silent about. A null differential (unmeasurable) never acts.
|
|
60
113
|
*/
|
|
61
|
-
export function decideStalenessAction(diff, hasOpen, canMachineClose) {
|
|
114
|
+
export function decideStalenessAction(diff, hasOpen, canMachineClose, wedged = false) {
|
|
62
115
|
if (!diff)
|
|
63
116
|
return 'none';
|
|
64
117
|
if (diff.beyondThreshold) {
|
|
65
118
|
if (canMachineClose) {
|
|
119
|
+
if (wedged) {
|
|
120
|
+
// Enabled but not applying: the running version has not advanced past the
|
|
121
|
+
// wedge window. Hand a human a signal once (dedup by cause_key).
|
|
122
|
+
return hasOpen ? 'none' : 'hand-over';
|
|
123
|
+
}
|
|
66
124
|
// Auto-update closes this at the next safe seam. Defer to it. If a stale
|
|
67
125
|
// handover from a prior disabled period is still open, retire it -- the
|
|
68
126
|
// machine has reclaimed closure.
|
|
@@ -74,6 +132,51 @@ export function decideStalenessAction(diff, hasOpen, canMachineClose) {
|
|
|
74
132
|
// Within threshold / in sync: retire any open handover, else stay quiet.
|
|
75
133
|
return hasOpen ? 'resolve' : 'none';
|
|
76
134
|
}
|
|
135
|
+
/**
|
|
136
|
+
* First-seen wall-clock (ms) per beyond-threshold, machine-closable cause_key,
|
|
137
|
+
* for wedge detection. Module state: it persists for the life of THIS running
|
|
138
|
+
* daemon -- which is exactly the wedged case (a closer that is not applying never
|
|
139
|
+
* restarts), while a successful update restarts the process and clears it. A
|
|
140
|
+
* genuine catch-up changes the running version and thus the cause_key, resetting
|
|
141
|
+
* the clock on its own.
|
|
142
|
+
*/
|
|
143
|
+
const gapFirstSeen = new Map();
|
|
144
|
+
/**
|
|
145
|
+
* Record that `causeKey` is currently a beyond-threshold, machine-closable gap
|
|
146
|
+
* and report whether it has persisted at/past `thresholdMs`. Prunes any other
|
|
147
|
+
* key so the store stays single-entry (a new cause_key -- running advanced or a
|
|
148
|
+
* new newest -- starts a fresh clock). Pure over the injected store, so a unit
|
|
149
|
+
* test drives it with a fresh Map and a fixed clock.
|
|
150
|
+
*/
|
|
151
|
+
export function recordAndCheckGapWedged(store, causeKey, nowMs, thresholdMs) {
|
|
152
|
+
for (const k of [...store.keys()])
|
|
153
|
+
if (k !== causeKey)
|
|
154
|
+
store.delete(k);
|
|
155
|
+
const firstSeen = store.get(causeKey) ?? nowMs;
|
|
156
|
+
if (!store.has(causeKey))
|
|
157
|
+
store.set(causeKey, firstSeen);
|
|
158
|
+
return nowMs - firstSeen >= thresholdMs;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Default wedge window in hours: how long a machine-closable, beyond-threshold
|
|
162
|
+
* gap may persist without the running version advancing before the closer is
|
|
163
|
+
* treated as wedged. Kept ABOVE the auto-update drain threshold (6h default) so a
|
|
164
|
+
* legitimately-busy daemon that will drain-and-update on its own is never flagged.
|
|
165
|
+
*/
|
|
166
|
+
export const DEFAULT_STALENESS_WEDGE_THRESHOLD_HOURS = 12;
|
|
167
|
+
/**
|
|
168
|
+
* The configured wedge window in ms. Read straight from process.env (the knob is
|
|
169
|
+
* declared in daemon-core env-config for discoverability, but read directly here
|
|
170
|
+
* so the daemon does not depend on a daemon-core republish to honor it), with a
|
|
171
|
+
* safe fallback to {@link DEFAULT_STALENESS_WEDGE_THRESHOLD_HOURS} on absent /
|
|
172
|
+
* non-numeric / non-positive input.
|
|
173
|
+
*/
|
|
174
|
+
function loadWedgeThresholdMs() {
|
|
175
|
+
const raw = process.env.TELORA_STALENESS_WEDGE_THRESHOLD_HOURS;
|
|
176
|
+
const hours = raw !== undefined && raw.trim() !== '' ? Number(raw) : DEFAULT_STALENESS_WEDGE_THRESHOLD_HOURS;
|
|
177
|
+
const safe = Number.isFinite(hours) && hours > 0 ? hours : DEFAULT_STALENESS_WEDGE_THRESHOLD_HOURS;
|
|
178
|
+
return safe * 3_600_000;
|
|
179
|
+
}
|
|
77
180
|
function formatDistance(d) {
|
|
78
181
|
return `${d.major}.${d.minor}.${d.patch}`;
|
|
79
182
|
}
|
|
@@ -155,12 +258,30 @@ async function defaultResolveStalenessEscalations() {
|
|
|
155
258
|
return resolved;
|
|
156
259
|
}
|
|
157
260
|
/**
|
|
158
|
-
* Hand a human a
|
|
159
|
-
* cannot auto-close
|
|
160
|
-
*
|
|
261
|
+
* Hand a human a signal for a stale daemon: a READY ACTION when the machine
|
|
262
|
+
* cannot auto-close (`auto_update_disabled`), or a WEDGED-CLOSER signal when
|
|
263
|
+
* auto-update is enabled but has not advanced the running version past the wedge
|
|
264
|
+
* window (`auto_update_wedged`). Both carry running/newest/distance and share the
|
|
265
|
+
* cause_key dedup + auto-resolve path, so either auto-resolves once the running
|
|
266
|
+
* version catches up.
|
|
161
267
|
*/
|
|
162
268
|
async function defaultHandOverReadyAction(params) {
|
|
163
|
-
const { diff, readyCommand } = params;
|
|
269
|
+
const { diff, readyCommand, reason, registry } = params;
|
|
270
|
+
const description = reason === 'auto_update_wedged'
|
|
271
|
+
? `Auto-update appears WEDGED on this daemon (host ${osHostname()}): it is ENABLED ` +
|
|
272
|
+
`but the running version has not advanced past v${diff.running} while the install ` +
|
|
273
|
+
`registry ${registry} serves v${diff.newest} (behind by ${formatDistance(diff.distance)}), and ` +
|
|
274
|
+
`the gap has persisted past the wedge window. A fix has shipped but is not running. ` +
|
|
275
|
+
`Check the [auto-update] trace in ~/.telora/daemon.log and restart the daemon; if ` +
|
|
276
|
+
`it still will not advance, install manually:\n\n ${readyCommand}\n\n` +
|
|
277
|
+
`This resolves automatically once the running version catches up.`
|
|
278
|
+
: `Ready action -- update this daemon (host ${osHostname()}):\n\n ${readyCommand}\n\n` +
|
|
279
|
+
`Auto-update is disabled on this daemon (running from an npx cache, or ` +
|
|
280
|
+
`--no-auto-update / TELORA_AUTO_UPDATE=0), so the machine cannot close the ` +
|
|
281
|
+
`daemon-version gap itself: running v${diff.running} while the install registry ` +
|
|
282
|
+
`${registry} serves v${diff.newest} (behind by ${formatDistance(diff.distance)}). Running the ` +
|
|
283
|
+
`command above closes it. This resolves automatically once the running version ` +
|
|
284
|
+
`catches up, or if auto-update is re-enabled.`;
|
|
164
285
|
await createEscalation({
|
|
165
286
|
organizationId: params.organizationId,
|
|
166
287
|
issueId: null,
|
|
@@ -168,20 +289,16 @@ async function defaultHandOverReadyAction(params) {
|
|
|
168
289
|
disposition: ESCALATION_DISPOSITIONS.NEEDS_OPERATOR,
|
|
169
290
|
productId: params.productId,
|
|
170
291
|
escalationKind: PROPAGATION_STALE_ESCALATION_KIND,
|
|
171
|
-
description
|
|
172
|
-
`Auto-update is disabled on this daemon (running from an npx cache, or ` +
|
|
173
|
-
`--no-auto-update / TELORA_AUTO_UPDATE=0), so the machine cannot close the ` +
|
|
174
|
-
`daemon-version gap itself: running v${diff.running} while the Gitea registry ` +
|
|
175
|
-
`serves v${diff.newest} (behind by ${formatDistance(diff.distance)}). Running the ` +
|
|
176
|
-
`command above closes it. This resolves automatically once the running version ` +
|
|
177
|
-
`catches up, or if auto-update is re-enabled.`,
|
|
292
|
+
description,
|
|
178
293
|
metadata: {
|
|
179
294
|
target: diff.target,
|
|
180
295
|
running_version: diff.running,
|
|
181
296
|
newest_version: diff.newest,
|
|
182
297
|
distance: diff.distance,
|
|
298
|
+
registry,
|
|
183
299
|
ready_command: readyCommand,
|
|
184
|
-
reason
|
|
300
|
+
reason,
|
|
301
|
+
host: osHostname(),
|
|
185
302
|
product_id: params.productId,
|
|
186
303
|
// cause_key (running+newest) suppresses a re-file for the SAME gap across
|
|
187
304
|
// ticks AND after an operator dismissal; a new `newest` is a new cause key.
|
|
@@ -192,12 +309,72 @@ async function defaultHandOverReadyAction(params) {
|
|
|
192
309
|
const defaultDeps = {
|
|
193
310
|
readRunningVersion: () => readDaemonMeta()?.version ?? null,
|
|
194
311
|
readNewestAvailable: () => readFreshVersionCache()?.latestVersion ?? null,
|
|
312
|
+
readReadingRegistry: () => readFreshVersionCache()?.registryUrl ?? null,
|
|
313
|
+
readInstallRegistry: () => resolveInstallRegistryUrl(),
|
|
195
314
|
patchThreshold: DEFAULT_STALENESS_PATCH_THRESHOLD,
|
|
196
315
|
canMachineCloseGap: defaultCanMachineCloseGap,
|
|
197
316
|
hasOpenStalenessEscalation: defaultHasOpenStalenessEscalation,
|
|
198
317
|
handOverReadyAction: defaultHandOverReadyAction,
|
|
199
318
|
resolveStalenessEscalations: defaultResolveStalenessEscalations,
|
|
319
|
+
checkGapWedged: (causeKey, nowMs) => recordAndCheckGapWedged(gapFirstSeen, causeKey, nowMs, loadWedgeThresholdMs()),
|
|
320
|
+
now: Date.now,
|
|
200
321
|
};
|
|
322
|
+
/**
|
|
323
|
+
* Act on the decided staleness action (hand-over / delegate / resolve / none)
|
|
324
|
+
* and emit its trace, mutating `counts`. Extracted verbatim from
|
|
325
|
+
* runStalenessSensorTick so that function stays within the per-function size
|
|
326
|
+
* limit -- a cohesive "apply the decision" step, no behavior change.
|
|
327
|
+
*/
|
|
328
|
+
async function applyStalenessAction(args) {
|
|
329
|
+
const { action, diff, pc, canMachineClose, installRegistry, counts, deps } = args;
|
|
330
|
+
if (action === 'hand-over') {
|
|
331
|
+
const readyCommand = readyUpdateCommand(diff.newest);
|
|
332
|
+
// canMachineClose + hand-over can only mean wedged (decideStalenessAction);
|
|
333
|
+
// !canMachineClose means auto-update is disabled.
|
|
334
|
+
const reason = canMachineClose ? 'auto_update_wedged' : 'auto_update_disabled';
|
|
335
|
+
await deps.handOverReadyAction({
|
|
336
|
+
organizationId: pc.organizationId,
|
|
337
|
+
productId: pc.productId,
|
|
338
|
+
diff,
|
|
339
|
+
readyCommand,
|
|
340
|
+
reason,
|
|
341
|
+
registry: installRegistry,
|
|
342
|
+
});
|
|
343
|
+
counts.handoversToOperator++;
|
|
344
|
+
if (reason === 'auto_update_wedged')
|
|
345
|
+
counts.wedgedSignalsFiled++;
|
|
346
|
+
log.warn(reason === 'auto_update_wedged'
|
|
347
|
+
? `[staleness] daemon behind registry ${installRegistry} (running v${diff.running} < newest ` +
|
|
348
|
+
`v${diff.newest}, behind ${formatDistance(diff.distance)}) and auto-update is ENABLED but ` +
|
|
349
|
+
`WEDGED (not advancing past the wedge window) -- filed a needs_operator signal.`
|
|
350
|
+
: `[staleness] daemon behind registry ${installRegistry} (running v${diff.running} < newest ` +
|
|
351
|
+
`v${diff.newest}, behind ${formatDistance(diff.distance)}) and auto-update is DISABLED -- ` +
|
|
352
|
+
`the machine cannot close it. Handed over ready action: ${readyCommand}`);
|
|
353
|
+
}
|
|
354
|
+
else if (action === 'delegate') {
|
|
355
|
+
log.info(`[staleness] daemon behind registry ${installRegistry} (running v${diff.running} < newest ` +
|
|
356
|
+
`v${diff.newest}, behind ${formatDistance(diff.distance)}); auto-update owns closure at the ` +
|
|
357
|
+
`next safe seam -- deferring, no operator action.`);
|
|
358
|
+
}
|
|
359
|
+
else if (action === 'resolve') {
|
|
360
|
+
counts.escalationsResolved += await deps.resolveStalenessEscalations();
|
|
361
|
+
log.info(diff.beyondThreshold
|
|
362
|
+
? `[staleness] auto-update re-enabled while behind (v${diff.running}); the machine now ` +
|
|
363
|
+
`owns closure -- retired the operator handover.`
|
|
364
|
+
: `[staleness] daemon back in sync with registry ${installRegistry} (v${diff.running}); ` +
|
|
365
|
+
`retired the operator handover.`);
|
|
366
|
+
}
|
|
367
|
+
else {
|
|
368
|
+
// Positive in-sync (or within-threshold) trace: silence must never mean
|
|
369
|
+
// "not measured". This is the heartbeat that makes an absent signal
|
|
370
|
+
// diagnostic. Names the registry the reading was attributed to.
|
|
371
|
+
log.info(`[staleness] measured against registry ${installRegistry}: running v${diff.running} vs ` +
|
|
372
|
+
`newest v${diff.newest} -- ` +
|
|
373
|
+
(diff.behind
|
|
374
|
+
? `behind by ${formatDistance(diff.distance)} (within threshold).`
|
|
375
|
+
: 'in sync.'));
|
|
376
|
+
}
|
|
377
|
+
}
|
|
201
378
|
/**
|
|
202
379
|
* One propagation-staleness sensor tick. Computes the global daemon-vs-registry
|
|
203
380
|
* differential once (the running daemon has a single version). When the gap is
|
|
@@ -214,6 +391,7 @@ export async function runStalenessSensorTick(config, deps = defaultDeps) {
|
|
|
214
391
|
behind: false,
|
|
215
392
|
delegatedToAutoUpdate: false,
|
|
216
393
|
handoversToOperator: 0,
|
|
394
|
+
wedgedSignalsFiled: 0,
|
|
217
395
|
escalationsResolved: 0,
|
|
218
396
|
};
|
|
219
397
|
try {
|
|
@@ -227,57 +405,44 @@ export async function runStalenessSensorTick(config, deps = defaultDeps) {
|
|
|
227
405
|
`cannot compute the propagation gap -- the version cache may not be warm yet.`);
|
|
228
406
|
return counts;
|
|
229
407
|
}
|
|
408
|
+
// Attribution gate: the `newest` reading must be attributable to the registry
|
|
409
|
+
// npm resolves for an install. A reading from any other registry (a
|
|
410
|
+
// TELORA_REGISTRY_URL override, a legacy cache with no registry) is UNMEASURED
|
|
411
|
+
// -- reporting "in sync" here would certify the installed version against a
|
|
412
|
+
// registry the installer does not use (the mars false-negative, on the
|
|
413
|
+
// platform). Silence still means "measured and in sync", never "not measured".
|
|
414
|
+
const readingRegistry = deps.readReadingRegistry();
|
|
415
|
+
const installRegistry = deps.readInstallRegistry();
|
|
416
|
+
if (!isReadingAttributable(readingRegistry, installRegistry)) {
|
|
417
|
+
log.warn(`[staleness] daemon-version differential UNMEASURED: the newest reading (v${diff.newest}) ` +
|
|
418
|
+
`came from registry ${readingRegistry ?? 'unknown'}, but an install of @telora/daemon ` +
|
|
419
|
+
`resolves ${installRegistry} -- cannot attribute the reading to the install registry, ` +
|
|
420
|
+
`so NOT asserting in sync.`);
|
|
421
|
+
return counts; // measured stays false -- unattributable is UNMEASURED.
|
|
422
|
+
}
|
|
230
423
|
counts.measured = true;
|
|
231
424
|
counts.behind = diff.behind;
|
|
232
425
|
const anchor = config.products[0];
|
|
233
426
|
if (!anchor) {
|
|
234
|
-
log.info(`[staleness] measured: running v${diff.running} vs
|
|
427
|
+
log.info(`[staleness] measured against registry ${installRegistry}: running v${diff.running} vs ` +
|
|
428
|
+
`newest v${diff.newest} ` +
|
|
235
429
|
`(${diff.behind ? `behind ${formatDistance(diff.distance)}` : 'in sync'}); ` +
|
|
236
430
|
`no product configured to anchor a handover.`);
|
|
237
431
|
return counts;
|
|
238
432
|
}
|
|
239
433
|
const pc = configForProduct(config, anchor);
|
|
240
434
|
const canMachineClose = deps.canMachineCloseGap();
|
|
435
|
+
// Wedge check only where the machine SHOULD be closing the gap: enabled but
|
|
436
|
+
// the same beyond-threshold gap has persisted past the wedge window without
|
|
437
|
+
// the running version advancing. Records first-seen as a side effect.
|
|
438
|
+
const causeKey = daemonStalenessCauseKey(diff.running, diff.newest);
|
|
439
|
+
const wedged = diff.beyondThreshold && canMachineClose ? deps.checkGapWedged(causeKey, deps.now()) : false;
|
|
241
440
|
const hasOpen = await deps.hasOpenStalenessEscalation({ running: diff.running, newest: diff.newest });
|
|
242
|
-
const action = decideStalenessAction(diff, hasOpen, canMachineClose);
|
|
243
|
-
if (diff.beyondThreshold && canMachineClose) {
|
|
441
|
+
const action = decideStalenessAction(diff, hasOpen, canMachineClose, wedged);
|
|
442
|
+
if (diff.beyondThreshold && canMachineClose && !wedged) {
|
|
244
443
|
counts.delegatedToAutoUpdate = true;
|
|
245
444
|
}
|
|
246
|
-
|
|
247
|
-
const readyCommand = readyUpdateCommand(diff.newest);
|
|
248
|
-
await deps.handOverReadyAction({
|
|
249
|
-
organizationId: pc.organizationId,
|
|
250
|
-
productId: pc.productId,
|
|
251
|
-
diff,
|
|
252
|
-
readyCommand,
|
|
253
|
-
});
|
|
254
|
-
counts.handoversToOperator++;
|
|
255
|
-
log.warn(`[staleness] daemon behind registry (running v${diff.running} < newest v${diff.newest}, ` +
|
|
256
|
-
`behind ${formatDistance(diff.distance)}) and auto-update is DISABLED -- the machine ` +
|
|
257
|
-
`cannot close it. Handed over ready action: ${readyCommand}`);
|
|
258
|
-
}
|
|
259
|
-
else if (action === 'delegate') {
|
|
260
|
-
log.info(`[staleness] daemon behind registry (running v${diff.running} < newest v${diff.newest}, ` +
|
|
261
|
-
`behind ${formatDistance(diff.distance)}); auto-update owns closure at the next safe ` +
|
|
262
|
-
`seam -- deferring, no operator action.`);
|
|
263
|
-
}
|
|
264
|
-
else if (action === 'resolve') {
|
|
265
|
-
counts.escalationsResolved += await deps.resolveStalenessEscalations();
|
|
266
|
-
log.info(diff.beyondThreshold
|
|
267
|
-
? `[staleness] auto-update re-enabled while behind (v${diff.running}); the machine now ` +
|
|
268
|
-
`owns closure -- retired the operator handover.`
|
|
269
|
-
: `[staleness] daemon back in sync with registry (v${diff.running}); ` +
|
|
270
|
-
`retired the operator handover.`);
|
|
271
|
-
}
|
|
272
|
-
else {
|
|
273
|
-
// Positive in-sync (or within-threshold) trace: silence must never mean
|
|
274
|
-
// "not measured". This is the heartbeat that makes an absent signal
|
|
275
|
-
// diagnostic.
|
|
276
|
-
log.info(`[staleness] measured: running v${diff.running} vs newest v${diff.newest} -- ` +
|
|
277
|
-
(diff.behind
|
|
278
|
-
? `behind by ${formatDistance(diff.distance)} (within threshold).`
|
|
279
|
-
: 'in sync.'));
|
|
280
|
-
}
|
|
445
|
+
await applyStalenessAction({ action, diff, pc, canMachineClose, installRegistry, counts, deps });
|
|
281
446
|
}
|
|
282
447
|
catch (err) {
|
|
283
448
|
log.warn(`[staleness] tick failed: ${err.message}`);
|