@evomap/evolver 1.89.20 → 2.0.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/README.md +27 -564
  2. package/bin/evolver-llm-proxy.js +3 -0
  3. package/bin/evolver-mcp.js +2 -0
  4. package/bin/evolver-proxy.js +3 -0
  5. package/bin/evolver.js +4 -0
  6. package/index.js +1 -3586
  7. package/package.json +32 -62
  8. package/CONTRIBUTING.md +0 -19
  9. package/LICENSE +0 -641
  10. package/README.ja-JP.md +0 -521
  11. package/README.ko-KR.md +0 -520
  12. package/README.zh-CN.md +0 -531
  13. package/SKILL.md +0 -365
  14. package/assets/cover.png +0 -0
  15. package/assets/gep/genes.seed.json +0 -496
  16. package/conformance/savings-core/constants.json +0 -30
  17. package/conformance/savings-core/golden-vectors.json +0 -333
  18. package/scripts/a2a_export.js +0 -63
  19. package/scripts/a2a_ingest.js +0 -79
  20. package/scripts/a2a_promote.js +0 -118
  21. package/scripts/analyze_by_skill.js +0 -121
  22. package/scripts/build_binaries.js +0 -479
  23. package/scripts/check-changelog.js +0 -166
  24. package/scripts/extract_log.js +0 -85
  25. package/scripts/generate_history.js +0 -75
  26. package/scripts/gep_append_event.js +0 -96
  27. package/scripts/gep_personality_report.js +0 -234
  28. package/scripts/human_report.js +0 -147
  29. package/scripts/recall-verify-report.js +0 -234
  30. package/scripts/recover_loop.js +0 -61
  31. package/scripts/refresh_stars_badge.js +0 -168
  32. package/scripts/seed-merchants.js +0 -91
  33. package/scripts/skill2recipes.js +0 -118
  34. package/scripts/suggest_version.js +0 -89
  35. package/scripts/validate-modules.js +0 -38
  36. package/scripts/validate-suite.js +0 -78
  37. package/skills/_meta/SKILL.md +0 -41
  38. package/skills/index.json +0 -14
  39. package/src/adapters/claudeCode.js +0 -194
  40. package/src/adapters/codex.js +0 -216
  41. package/src/adapters/cursor.js +0 -91
  42. package/src/adapters/hookAdapter.js +0 -469
  43. package/src/adapters/kiro.js +0 -195
  44. package/src/adapters/opencode.js +0 -326
  45. package/src/adapters/scripts/_lockPaths.js +0 -74
  46. package/src/adapters/scripts/_memoryFiltering.js +0 -35
  47. package/src/adapters/scripts/_runtimePaths.js +0 -440
  48. package/src/adapters/scripts/evolver-session-end.js +0 -321
  49. package/src/adapters/scripts/evolver-session-start.js +0 -587
  50. package/src/adapters/scripts/evolver-signal-detect.js +0 -98
  51. package/src/adapters/scripts/evolver-task-recall.js +0 -173
  52. package/src/atp/atpExecute.js +0 -283
  53. package/src/atp/atpTaskPickup.js +0 -233
  54. package/src/atp/autoBuyer.js +0 -382
  55. package/src/atp/autoDeliver.js +0 -215
  56. package/src/atp/cli.js +0 -354
  57. package/src/atp/cliAutobuyPrompt.js +0 -154
  58. package/src/atp/consumerAgent.js +0 -157
  59. package/src/atp/defaultHandler.js +0 -69
  60. package/src/atp/heartbeatSignalsHandler.js +0 -254
  61. package/src/atp/hubClient.js +0 -317
  62. package/src/atp/index.js +0 -38
  63. package/src/atp/merchantAgent.js +0 -118
  64. package/src/atp/protocol.js +0 -41
  65. package/src/atp/questionComposer.js +0 -133
  66. package/src/atp/serviceHelper.js +0 -92
  67. package/src/canary.js +0 -13
  68. package/src/config.js +0 -351
  69. package/src/evolve/guards.js +0 -1
  70. package/src/evolve/pipeline/collect.js +0 -1
  71. package/src/evolve/pipeline/dispatch.js +0 -1
  72. package/src/evolve/pipeline/enrich.js +0 -1
  73. package/src/evolve/pipeline/hub.js +0 -1
  74. package/src/evolve/pipeline/select.js +0 -1
  75. package/src/evolve/pipeline/signals.js +0 -1
  76. package/src/evolve/utils.js +0 -1
  77. package/src/evolve.js +0 -1
  78. package/src/experiment/agentRunner.js +0 -229
  79. package/src/experiment/cli.js +0 -159
  80. package/src/experiment/comparison.js +0 -233
  81. package/src/experiment/metrics.js +0 -75
  82. package/src/forceUpdate.js +0 -991
  83. package/src/gep/a2a.js +0 -173
  84. package/src/gep/a2aProtocol.js +0 -1
  85. package/src/gep/analyzer.js +0 -35
  86. package/src/gep/antiAbuseTelemetry.js +0 -1
  87. package/src/gep/assetCallLog.js +0 -197
  88. package/src/gep/assetStore.js +0 -723
  89. package/src/gep/assets.js +0 -36
  90. package/src/gep/autoDistillConv.js +0 -1
  91. package/src/gep/autoDistillLlm.js +0 -1
  92. package/src/gep/bridge.js +0 -138
  93. package/src/gep/candidateEval.js +0 -1
  94. package/src/gep/candidates.js +0 -1
  95. package/src/gep/claimNudge.js +0 -121
  96. package/src/gep/cliContracts.js +0 -1190
  97. package/src/gep/contentHash.js +0 -1
  98. package/src/gep/conversationDistiller.js +0 -1
  99. package/src/gep/conversationSniffer.js +0 -1
  100. package/src/gep/crypto.js +0 -1
  101. package/src/gep/curriculum.js +0 -1
  102. package/src/gep/deviceId.js +0 -1
  103. package/src/gep/directoryClient.js +0 -115
  104. package/src/gep/envFingerprint.js +0 -1
  105. package/src/gep/epigenetics.js +0 -1
  106. package/src/gep/execBridge.js +0 -1
  107. package/src/gep/executionTrace.js +0 -291
  108. package/src/gep/explore.js +0 -1
  109. package/src/gep/featureFlags.js +0 -121
  110. package/src/gep/gitOps.js +0 -265
  111. package/src/gep/hash.js +0 -1
  112. package/src/gep/hostErrorClassifier.js +0 -34
  113. package/src/gep/hubFetch.js +0 -1
  114. package/src/gep/hubReview.js +0 -1
  115. package/src/gep/hubSearch.js +0 -1
  116. package/src/gep/hubVerify.js +0 -1
  117. package/src/gep/idleScheduler.js +0 -400
  118. package/src/gep/issueReporter.js +0 -416
  119. package/src/gep/learningSignals.js +0 -1
  120. package/src/gep/llmReview.js +0 -92
  121. package/src/gep/localStateAwareness.js +0 -243
  122. package/src/gep/mailboxTransport.js +0 -119
  123. package/src/gep/memoryGraph.js +0 -1
  124. package/src/gep/memoryGraphAdapter.js +0 -1
  125. package/src/gep/mutation.js +0 -1
  126. package/src/gep/narrativeMemory.js +0 -1
  127. package/src/gep/oauthLogin.js +0 -181
  128. package/src/gep/openPRRegistry.js +0 -1
  129. package/src/gep/paths.js +0 -522
  130. package/src/gep/personality.js +0 -1
  131. package/src/gep/policyCheck.js +0 -1
  132. package/src/gep/portable.js +0 -103
  133. package/src/gep/privacyClient.js +0 -235
  134. package/src/gep/prompt.js +0 -1
  135. package/src/gep/questionGenerator.js +0 -518
  136. package/src/gep/recallInject.js +0 -1
  137. package/src/gep/recallVerifier.js +0 -1
  138. package/src/gep/reflection.js +0 -1
  139. package/src/gep/sanitize.js +0 -264
  140. package/src/gep/savingsCore.js +0 -1
  141. package/src/gep/schemas/capsule.js +0 -170
  142. package/src/gep/schemas/gene.js +0 -154
  143. package/src/gep/schemas/index.js +0 -8
  144. package/src/gep/schemas/protocol.js +0 -51
  145. package/src/gep/schemas/task.js +0 -74
  146. package/src/gep/selector.js +0 -1
  147. package/src/gep/selfPR.js +0 -469
  148. package/src/gep/signals.js +0 -776
  149. package/src/gep/skill2gep.js +0 -1056
  150. package/src/gep/skill2gepAudit.js +0 -303
  151. package/src/gep/skill2recipes.js +0 -511
  152. package/src/gep/skillDistiller.js +0 -1
  153. package/src/gep/skillPublisher.js +0 -358
  154. package/src/gep/solidify.js +0 -1
  155. package/src/gep/strategy.js +0 -1
  156. package/src/gep/taskReceiver.js +0 -575
  157. package/src/gep/tokenSavings.js +0 -1
  158. package/src/gep/trajectoryExport.js +0 -1
  159. package/src/gep/validationReport.js +0 -55
  160. package/src/gep/validator/index.js +0 -411
  161. package/src/gep/validator/reporter.js +0 -210
  162. package/src/gep/validator/sandboxExecutor.js +0 -480
  163. package/src/gep/validator/stakeBootstrap.js +0 -357
  164. package/src/gep/workspaceKeychain.js +0 -1
  165. package/src/ops/cleanup.js +0 -80
  166. package/src/ops/commentary.js +0 -60
  167. package/src/ops/health_check.js +0 -104
  168. package/src/ops/index.js +0 -11
  169. package/src/ops/innovation.js +0 -67
  170. package/src/ops/lifecycle.js +0 -798
  171. package/src/ops/self_repair.js +0 -76
  172. package/src/ops/skills_monitor.js +0 -147
  173. package/src/ops/trigger.js +0 -33
  174. package/src/proxy/clientSettings.js +0 -405
  175. package/src/proxy/envelope.js +0 -59
  176. package/src/proxy/extensions/dmHandler.js +0 -45
  177. package/src/proxy/extensions/sessionHandler.js +0 -141
  178. package/src/proxy/extensions/skillUpdater.js +0 -64
  179. package/src/proxy/extensions/traceControl.js +0 -1
  180. package/src/proxy/index.js +0 -1395
  181. package/src/proxy/inject.js +0 -1
  182. package/src/proxy/lifecycle/manager.js +0 -1568
  183. package/src/proxy/mailbox/state.js +0 -207
  184. package/src/proxy/mailbox/store.js +0 -590
  185. package/src/proxy/router/cache_passthrough.js +0 -26
  186. package/src/proxy/router/features.js +0 -84
  187. package/src/proxy/router/gemini_route.js +0 -154
  188. package/src/proxy/router/messages_route.js +0 -535
  189. package/src/proxy/router/model_router.js +0 -113
  190. package/src/proxy/router/models_route.js +0 -52
  191. package/src/proxy/router/ollama_route.js +0 -103
  192. package/src/proxy/router/responses_route.js +0 -170
  193. package/src/proxy/router/vertex_route.js +0 -110
  194. package/src/proxy/server/http.js +0 -363
  195. package/src/proxy/server/routes.js +0 -558
  196. package/src/proxy/server/settings.js +0 -115
  197. package/src/proxy/sync/engine.js +0 -179
  198. package/src/proxy/sync/inbound.js +0 -211
  199. package/src/proxy/sync/outbound.js +0 -320
  200. package/src/proxy/task/monitor.js +0 -131
  201. package/src/proxy/trace/extractor.js +0 -1
  202. package/src/proxy/trace/usage.js +0 -1
  203. package/src/webui/client/clientJs/assets.js +0 -111
  204. package/src/webui/client/clientJs/bootstrap.js +0 -92
  205. package/src/webui/client/clientJs/common.js +0 -77
  206. package/src/webui/client/clientJs/i18n.js +0 -366
  207. package/src/webui/client/clientJs/index.js +0 -35
  208. package/src/webui/client/clientJs/interactions.js +0 -351
  209. package/src/webui/client/clientJs/overview.js +0 -152
  210. package/src/webui/client/clientJs/personality.js +0 -285
  211. package/src/webui/client/clientJs/pipelines.js +0 -330
  212. package/src/webui/client/indexHtml.js +0 -221
  213. package/src/webui/client/static.js +0 -23
  214. package/src/webui/client/stylesCss.js +0 -639
  215. package/src/webui/client/vendor/README.md +0 -15
  216. package/src/webui/client/vendor/echarts.min.js +0 -45
  217. package/src/webui/index.js +0 -14
  218. package/src/webui/observer/assets.js +0 -146
  219. package/src/webui/observer/index.js +0 -37
  220. package/src/webui/observer/interactions.js +0 -127
  221. package/src/webui/observer/jsonl.js +0 -75
  222. package/src/webui/observer/paths.js +0 -46
  223. package/src/webui/observer/personality.js +0 -43
  224. package/src/webui/observer/pipelineEvents.js +0 -58
  225. package/src/webui/observer/redact.js +0 -63
  226. package/src/webui/observer/runs.js +0 -356
  227. package/src/webui/observer/safety.js +0 -57
  228. package/src/webui/observer/skills.js +0 -70
  229. package/src/webui/observer/status.js +0 -71
  230. package/src/webui/server/http.js +0 -138
  231. package/src/webui/server/routes.js +0 -41
@@ -1,1568 +0,0 @@
1
- 'use strict';
2
-
3
- const fs = require('fs');
4
- const path = require('path');
5
- const { PROXY_PROTOCOL_VERSION } = require('../mailbox/store');
6
- const { readMailboxStateFile } = require('../mailbox/state');
7
- const { buildEnvelope } = require('../envelope');
8
- const crypto = require('crypto');
9
- const {
10
- hubFetch,
11
- hubUnreachableBackoffMs,
12
- isHubUnreachableError,
13
- readHubResponseJson,
14
- readHubResponseText,
15
- sanitizeHubResponseForLog,
16
- throwIfHubUnreachableResponse,
17
- } = require('../../gep/hubFetch');
18
- const { getEvomapPath } = require('../../gep/paths');
19
- // last_update transit (PR #188): proxy heartbeat ferries a pending
20
- // force_update outcome to the hub, then clears the state file on 2xx.
21
- // Proxy DOES run the upgrade now (PR #188 follow-up, HIGH bug): the
22
- // original comment "Proxy itself never runs the upgrade — telemetry-only
23
- // here" reflected pre-fix behaviour. Pure proxy-mode nodes (EVOMAP_PROXY=1,
24
- // no evolve loop) never traversed a2aProtocol.js sendHeartbeat, so the
25
- // canonical `_maybeTriggerForceUpdateFromHeartbeat` block at
26
- // a2aProtocol.js:2304 never fired for them — Hub could push force_update
27
- // forever with no upgrade attempt and no EvolverUpgradeAttempt row. The
28
- // proxy heartbeat (200 with force_update, AND 426 with force_update in the
29
- // error envelope) must mirror that logic. reportForceUpdateOutcome writes
30
- // the state file the next heartbeat will pick up via body.last_update.
31
- const {
32
- readPendingLastUpdate,
33
- clearLastUpdateOnAck,
34
- reportForceUpdateOutcome,
35
- } = require('../../gep/a2aProtocol');
36
-
37
- // Hub's nodeId regex; mirror of src/gep/a2aProtocol.js so a malformed
38
- // legacy file can never feed garbage into the hello payload.
39
- const NODE_ID_RE = /^node_[a-f0-9]{12,32}$/;
40
-
41
- const DEFAULT_HEARTBEAT_INTERVAL = 360_000;
42
- // Heartbeat backoff ceiling. Was 30min; reporter (#544) showed that
43
- // a single transient failure could park the loop at 30min and feel
44
- // indistinguishable from a daemon that had crashed. 15min sits above
45
- // `DEFAULT_HEARTBEAT_INTERVAL` (6min) so the `interval * 2^failures`
46
- // growth still has headroom — capping below the interval would invert
47
- // the backoff and make failures retry FASTER than success ticks.
48
- // 15min ≈ 2.5× the default, giving one full doubling step before park.
49
- const HEARTBEAT_BACKOFF_CAP_MS = 15 * 60_000;
50
- const HELLO_TIMEOUT = 15_000;
51
- const HEARTBEAT_TIMEOUT = 10_000;
52
- const MAX_REAUTH_ATTEMPTS = 2;
53
- // First failure = 2 min, subsequent consecutive failures double up to ~4h.
54
- // Aligned with a2aProtocol.js Round-9 reduction (was 30 min, caused
55
- // "idle-death" for proxy-mode users: one benign 401 silenced the node for 30
56
- // min, triggering stagnation kills and manual restart loops).
57
- const REAUTH_BACKOFF_BASE_MS = 2 * 60_000;
58
- const REAUTH_BACKOFF_MAX_MS = 4 * 60 * 60_000;
59
-
60
- // Wall-clock drift detector tunables. Mirrors DRIFT_CHECK_MS /
61
- // DRIFT_SLEEP_THRESHOLD_MS / DRIFT_LONG_SLEEP_THRESHOLD_MS in
62
- // src/gep/a2aProtocol.js. setTimeout / setInterval fire on libuv's
63
- // monotonic clock, which freezes while the host is suspended -- so a
64
- // laptop closed for hours and reopened would not trigger any heartbeat
65
- // tick until the next scheduled time, which under exponential backoff
66
- // can sit at HEARTBEAT_BACKOFF_CAP_MS (15 min). Sampling Date.now()
67
- // (wall clock) every DRIFT_CHECK_MS lets us detect the jump and
68
- // immediately poke the heartbeat so recovery does not have to wait for
69
- // the next natural tick. Long-sleep gap also clears reauth backoff:
70
- // hub-side state we cached is almost certainly stale after a 30min+
71
- // suspend, so force a clean retry path on wake instead of carrying the
72
- // pre-sleep penalty through. R10 (#544).
73
- const DRIFT_CHECK_MS = 30 * 1000;
74
- const DRIFT_SLEEP_THRESHOLD_MS = 90 * 1000;
75
- const DRIFT_LONG_SLEEP_THRESHOLD_MS = 30 * 60_000;
76
-
77
- // Heartbeat-driven force_update lifecycle tracking. Mirrors
78
- // `_forceUpdateInFlight` / `_forceUpdateLastAttemptAt` /
79
- // `_getForceUpdateRetryCooldownMs` in src/gep/a2aProtocol.js so the proxy
80
- // path uses the same in-flight + cooldown contract as the canonical path.
81
- // Module-level (not instance-level) so multiple LifecycleManager instances
82
- // in the same process serialize through one upgrade attempt — matches
83
- // a2aProtocol.js's module-level guard. Process-local is sufficient: the
84
- // proxy daemon runs in a single process and any sibling process would
85
- // have its own require-cached state; cross-process serialization is the
86
- // hub's job via directive_id dedup, not the client's.
87
- let _proxyForceUpdateInFlight = false;
88
- let _proxyForceUpdateLastAttemptAt = 0;
89
- function _getProxyForceUpdateRetryCooldownMs() {
90
- // Share the env var with a2aProtocol.js: an operator who sets
91
- // EVOLVER_FORCE_UPDATE_RETRY_COOLDOWN_MS=0 in a test or production tune
92
- // expects BOTH code paths to honour it. Default 15min matches
93
- // a2aProtocol.js exactly.
94
- const v = Number(process.env.EVOLVER_FORCE_UPDATE_RETRY_COOLDOWN_MS);
95
- if (Number.isFinite(v) && v >= 0) return v;
96
- return 15 * 60 * 1000;
97
- }
98
-
99
- let _cachedFingerprint = null;
100
- function _getEnvFingerprint() {
101
- if (_cachedFingerprint) return _cachedFingerprint;
102
- try {
103
- const { captureEnvFingerprint } = require('../../gep/envFingerprint');
104
- _cachedFingerprint = captureEnvFingerprint();
105
- } catch {
106
- _cachedFingerprint = {
107
- platform: process.platform,
108
- arch: process.arch,
109
- node_version: process.version,
110
- };
111
- }
112
- return _cachedFingerprint;
113
- }
114
-
115
- // Recover a node_id persisted by the legacy GEP path
116
- // (`src/gep/a2aProtocol.js` writes ~/.evomap/node_id, falling back to
117
- // `<install>/.evomap_node_id` when the home dir isn't writable). Without
118
- // this fallback, a daemon whose MailboxStore was created AFTER the legacy
119
- // GEP file (any install upgrading from pre-lifecycle to lifecycle, or any
120
- // state.json wiped without also wiping the legacy file) mints a fresh
121
- // `node_${randomBytes(6)}` identity in hello(), which the hub registers
122
- // as a *new* A2ANode under the same owner — the original (with stake,
123
- // reputation, aliases) gets silently abandoned. Mirror the writer's two
124
- // candidates in the same order as `_loadPersistedNodeId` so both code
125
- // paths land on the single identity.
126
- //
127
- // Resolve both paths on every call:
128
- // - getEvomapPath() reads EVOLVER_HOME (and falls through to os.homedir())
129
- // at call time, so tests and privileged-drop daemons can flip the
130
- // resolved location without monkey-patching globals.
131
- // - The install-root path uses __dirname so it's stable across cwd changes.
132
- function _readLegacyNodeId() {
133
- const candidates = [
134
- getEvomapPath('node_id'),
135
- path.resolve(__dirname, '..', '..', '..', '.evomap_node_id'),
136
- ];
137
- for (const file of candidates) {
138
- try {
139
- if (!fs.existsSync(file)) continue;
140
- const raw = fs.readFileSync(file, 'utf8').trim();
141
- if (NODE_ID_RE.test(raw)) return raw;
142
- } catch {
143
- // Unreadable / racing writer — try the next location.
144
- }
145
- }
146
- return null;
147
- }
148
-
149
- // Mirror of src/gep/a2aProtocol.js `_persistNodeId`. Pure-proxy daemons
150
- // (EVOMAP_PROXY=1, no a2aProtocol heartbeat thread) mint their own
151
- // node_id and ONLY persist it to MailboxStore state.json. The legacy
152
- // `~/.evomap/node_id` file never gets written, so:
153
- //
154
- // 1. `_shortNodeIdForStatePath` in a2aProtocol.js (used by the proxy
155
- // heartbeat to pick the per-node `force_update_last.<suffix>.json`
156
- // path) falls all the way through to 'anon' — every proxy node on
157
- // the same EVOLVER_HOME would collide on the same state file.
158
- // 2. A mixed-mode install (legacy evolve loop ran once, then user
159
- // switched to proxy mode) is even worse: the legacy file holds a
160
- // DIFFERENT id than the one in MailboxStore. The proxy heartbeats
161
- // with body.node_id = its OWN id while writing
162
- // `force_update_last.<legacy-suffix>.json`. The hub-side upgrade
163
- // attempt row gets attributed to the wrong node.
164
- //
165
- // Calling this helper from hello() after the nodeId is resolved unifies
166
- // the two persistence paths onto a single identity. Atomic write
167
- // (per-pid tmp + rename) mirrors `_persistNodeSecret` in a2aProtocol.js;
168
- // 0o600 mode keeps the file owner-read-only on POSIX (silently ignored
169
- // on Windows, where %USERPROFILE% isolation is the only protection).
170
- //
171
- // Idempotent: if the file already holds the same id, we skip the write
172
- // to avoid an inode churn on every hello tick. If it holds a DIFFERENT
173
- // valid id, we still overwrite — the proxy's MailboxStore wins because
174
- // that is the id the hub already knows us by (any rotation away from
175
- // the legacy id was a deliberate operator action). The only way to
176
- // re-seed a legacy id back onto a proxy install is to clear
177
- // MailboxStore state.json (`evolver reset-local-secret`).
178
- function _persistLegacyNodeId(id) {
179
- if (!id || !NODE_ID_RE.test(id)) return;
180
- const targets = [
181
- getEvomapPath('node_id'),
182
- path.resolve(__dirname, '..', '..', '..', '.evomap_node_id'),
183
- ];
184
- // Try targets in order until one succeeds, matching the read order in
185
- // _readLegacyNodeId. We only need ONE persistent copy; once the home
186
- // path takes the write, the install-root path is unused.
187
- for (const file of targets) {
188
- try {
189
- // Skip if the file already matches — common steady-state path,
190
- // saves a syscall storm under heartbeat backoff doubling.
191
- try {
192
- if (fs.existsSync(file)) {
193
- const existing = fs.readFileSync(file, 'utf8').trim();
194
- if (existing === id) return;
195
- }
196
- } catch {
197
- // Unreadable -- treat as missing and try to write.
198
- }
199
- const dir = path.dirname(file);
200
- try {
201
- if (!fs.existsSync(dir)) {
202
- fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
203
- }
204
- } catch (_) {
205
- // mkdir failed (read-only fs, EPERM under sandboxing). Skip
206
- // this candidate; the next one (install-root .evomap_node_id)
207
- // may still work.
208
- continue;
209
- }
210
- // Atomic write: a sibling evolver process (mixed-mode upgrade, two
211
- // proxy daemons started by hand) could otherwise race on this
212
- // path and leave a half-written file. Matches the pattern in
213
- // a2aProtocol.js `_persistNodeSecret`.
214
- const tmp = file + '.' + process.pid + '.tmp';
215
- fs.writeFileSync(tmp, id, { encoding: 'utf8', mode: 0o600 });
216
- fs.renameSync(tmp, file);
217
- return;
218
- } catch {
219
- // Best-effort: continue to the next candidate. If both fail (no
220
- // home, no writable install root) we accept the legacy file is
221
- // unavailable — the proxy will still function, it just cannot
222
- // unify state-file suffixes with a co-resident a2aProtocol path.
223
- }
224
- }
225
- }
226
-
227
- // Heartbeat-driven force_update trigger for proxy-mode nodes. Mirrors
228
- // `_maybeTriggerForceUpdateFromHeartbeat` in src/gep/a2aProtocol.js
229
- // (search there for that name to compare). Pure proxy-mode deployments
230
- // (EVOMAP_PROXY=1) never run the evolve run() loop or sendHeartbeat, so
231
- // without this trigger Hub can push force_update on every heartbeat
232
- // forever and the node will keep heartbeating on the old version — which
233
- // is exactly what shipped before PR #188's H1 fix.
234
- //
235
- // Drives `executeForceUpdate` directly, gated by an in-flight lock + a
236
- // cooldown on failures so we do not hammer npm/degit on every tick. After
237
- // the attempt, persists the outcome via `reportForceUpdateOutcome` so the
238
- // next heartbeat carries it as body.last_update — that's the path that
239
- // finally writes a row to the hub's EvolverUpgradeAttempt table.
240
- //
241
- // Logger is injected (not a console fallback) so tests can capture the
242
- // upgrade-path stderr without polluting CI output. The logger contract
243
- // matches what the LifecycleManager already uses.
244
- function _maybeTriggerForceUpdateFromHeartbeat(forceUpdate, logger) {
245
- if (!forceUpdate || typeof forceUpdate !== 'object') return;
246
- if (_proxyForceUpdateInFlight) return;
247
- const nowMs = Date.now();
248
- if (
249
- _proxyForceUpdateLastAttemptAt &&
250
- (nowMs - _proxyForceUpdateLastAttemptAt) < _getProxyForceUpdateRetryCooldownMs()
251
- ) {
252
- // A recent attempt already ran and either succeeded (process exited
253
- // and we wouldn't be here on the post-restart heartbeat — see the
254
- // FORCE_UPDATE_NOOP path in forceUpdate.js / reportForceUpdateOutcome
255
- // status="skipped") or failed. Back off.
256
- return;
257
- }
258
- _proxyForceUpdateInFlight = true;
259
- _proxyForceUpdateLastAttemptAt = nowMs;
260
-
261
- // Capture from_version BEFORE executeForceUpdate runs. A successful
262
- // upgrade calls process.exit(78); the post-restart heartbeat reads the
263
- // state file from a fresh process where require('package.json').version
264
- // is the NEW version — so snapshot the CURRENTLY running version now.
265
- let fromVersion = '';
266
- try {
267
- fromVersion = String((require('../../../package.json') || {}).version || '');
268
- } catch (_) { /* best-effort */ }
269
-
270
- // Kick off in a microtask so the heartbeat promise chain can still
271
- // complete (log + return {ok:true}) before the long-running upgrade
272
- // takes over the process. Matches a2aProtocol.js exactly.
273
- Promise.resolve().then(() => {
274
- let updated = false;
275
- let noop = false;
276
- let busy = false;
277
- let thrownErr = null;
278
- // Structured failure object ({ ok:false, code, detail }) when the upgrader
279
- // RETURNED a failure; forwarded to reportForceUpdateOutcome so the hub gets
280
- // the precise branch code instead of "executeForceUpdate returned false".
281
- // Hoisted out of the try because `result` is block-scoped there.
282
- let failureResult = null;
283
- try {
284
- const mod = require('../../forceUpdate');
285
- const result = mod.executeForceUpdate(forceUpdate);
286
- // Sentinel === comparison: executeForceUpdate returns the
287
- // FORCE_UPDATE_NOOP symbol when the install is already at the
288
- // required version. We must NOT treat that as "success" — doing so
289
- // would (a) write a phantom {status:"success", from==to} row to
290
- // EvolverUpgradeAttempt, and (b) trigger an exit(78) restart with
291
- // nothing to restart for. The hub schema accepts status="skipped".
292
- noop = (result === mod.FORCE_UPDATE_NOOP);
293
- // FORCE_UPDATE_BUSY: another caller (e.g. a2aProtocol heartbeat
294
- // trigger or an evolve tick) already holds the module-level
295
- // _inFlight mutex in forceUpdate.js. Defensive only — the
296
- // instance-level _proxyForceUpdateInFlight gate above and the
297
- // single-caller property of pure proxy mode make BUSY unreachable
298
- // in practice. If it does fire (mixed-mode regression, future
299
- // additional caller, etc.), the other caller owns the telemetry:
300
- // we MUST NOT write a state file or exit(78). Mirrors
301
- // src/gep/a2aProtocol.js (search FORCE_UPDATE_BUSY).
302
- busy = (result === mod.FORCE_UPDATE_BUSY);
303
- updated = (result === true);
304
- // Inline the failure-shape check rather than calling mod.isForceUpdateFailure:
305
- // keeps this robust against partial test mocks of forceUpdate that stub
306
- // executeForceUpdate but omit the helper (a missing-function throw here
307
- // would otherwise demote a real success to "failed").
308
- failureResult = (result && typeof result === 'object' && result.ok === false) ? result : null;
309
- } catch (e) {
310
- thrownErr = e;
311
- try {
312
- logger.warn(`[ForceUpdate] proxy heartbeat-trigger failed (non-fatal): ${e && e.message || e}`);
313
- } catch (_) { /* logger broken; non-fatal */ }
314
- updated = false;
315
- } finally {
316
- _proxyForceUpdateInFlight = false;
317
- }
318
- if (busy) {
319
- _proxyForceUpdateLastAttemptAt = 0;
320
- try {
321
- logger.log('[ForceUpdate] proxy heartbeat-trigger observed BUSY (concurrent invocation). Skipping telemetry; in-flight caller owns the outcome.');
322
- } catch (_) { /* logger broken; non-fatal */ }
323
- return;
324
- }
325
- // Persist outcome via the shared helper so the heartbeat-thread
326
- // trigger and the proxy trigger stay in lockstep on payload assembly
327
- // + validation. The next heartbeat reads this file and ferries it as
328
- // body.last_update — same contract as the canonical path.
329
- try {
330
- reportForceUpdateOutcome(forceUpdate, {
331
- updated: updated,
332
- noop: noop,
333
- error: thrownErr,
334
- failure: failureResult,
335
- fromVersion: fromVersion,
336
- });
337
- } catch (e) {
338
- try {
339
- logger.warn(`[ForceUpdate] proxy reportForceUpdateOutcome failed (non-fatal): ${e && e.message || e}`);
340
- } catch (_) { /* logger broken; non-fatal */ }
341
- }
342
- if (updated) {
343
- try { logger.log('[ForceUpdate] Update complete (proxy heartbeat-trigger). Exiting for restart...'); } catch (_) {}
344
- try { process.exit(78); } catch (_) {}
345
- } else if (noop) {
346
- _proxyForceUpdateLastAttemptAt = 0;
347
- try {
348
- logger.log('[ForceUpdate] No-op (proxy heartbeat-trigger): already at required version. Skipping restart.');
349
- } catch (_) {}
350
- } else {
351
- try {
352
- logger.warn('[ForceUpdate] proxy heartbeat-trigger failed. Will retry after cooldown (' +
353
- Math.round(_getProxyForceUpdateRetryCooldownMs() / 60000) + 'min).');
354
- } catch (_) {}
355
- }
356
- });
357
- }
358
-
359
- class AuthError extends Error {
360
- constructor(message, statusCode) {
361
- super(message);
362
- this.name = 'AuthError';
363
- this.statusCode = statusCode;
364
- }
365
- }
366
-
367
- function parseNodeSecretVersion(value) {
368
- const n = Number(value);
369
- return Number.isInteger(n) && n > 0 ? n : null;
370
- }
371
-
372
- const NODE_SECRET_RE = /^[a-f0-9]{64}$/i;
373
- const NODE_SECRET_SUPPRESSION_RE = /^sha256:[a-f0-9]{64}$/i;
374
- const SECRET_DIVERGENCE_ERROR = 'secret_diverged_cleared';
375
- const SECRET_DIVERGENCE_REASON_CODES = [
376
- 'node_secret_invalid',
377
- 'invalid_secret',
378
- 'rotation_requires_current_secret',
379
- ];
380
-
381
- function validNodeSecret(value) {
382
- return typeof value === 'string' && NODE_SECRET_RE.test(value);
383
- }
384
-
385
- function getEnvNodeSecret() {
386
- return ((process.env.A2A_NODE_SECRET || process.env.EVOMAP_NODE_SECRET || '').trim() || null);
387
- }
388
-
389
- function fingerprintNodeSecret(secret) {
390
- if (!validNodeSecret(secret)) return null;
391
- const normalized = String(secret).trim().toLowerCase();
392
- return 'sha256:' + crypto.createHash('sha256').update(normalized).digest('hex');
393
- }
394
-
395
- function truthyState(value) {
396
- const v = String(value || '').trim().toLowerCase();
397
- return v === '1' || v === 'true' || v === 'yes' || v === 'on';
398
- }
399
-
400
- function matchSecretDivergenceReason(value) {
401
- const reason = String(value || '').trim().toLowerCase();
402
- if (!reason) return null;
403
- return SECRET_DIVERGENCE_REASON_CODES.find((code) => reason.includes(code)) || null;
404
- }
405
-
406
- function extractSecretDivergenceReason(data) {
407
- const candidates = [
408
- data?.payload?.reason,
409
- data?.payload?.error,
410
- data?.reason,
411
- data?.error,
412
- ];
413
- for (const value of candidates) {
414
- const reason = matchSecretDivergenceReason(value);
415
- if (reason) return reason;
416
- }
417
- return null;
418
- }
419
-
420
- function isSecretDivergenceError(error) {
421
- return error === SECRET_DIVERGENCE_ERROR || Boolean(matchSecretDivergenceReason(error));
422
- }
423
-
424
- function safeHubErrorCode(data, statusCode) {
425
- const candidates = [
426
- data?.error,
427
- data?.reason,
428
- data?.payload?.error,
429
- data?.payload?.reason,
430
- ];
431
- for (const value of candidates) {
432
- if (typeof value !== 'string') continue;
433
- const code = value.trim();
434
- if (/^[a-z][a-z0-9_:-]{0,79}$/i.test(code)) return code;
435
- }
436
- return `http_${statusCode}`;
437
- }
438
-
439
- function normalizeNodeSecretEnvSuppression(value) {
440
- const raw = String(value || '').trim();
441
- if (!raw) return null;
442
- if (truthyState(raw)) return 'true';
443
- const lower = raw.toLowerCase();
444
- return NODE_SECRET_SUPPRESSION_RE.test(lower) ? lower : null;
445
- }
446
-
447
- class LifecycleManager {
448
- constructor({ hubUrl, store, logger, getTaskMeta } = {}) {
449
- this.hubUrl = (hubUrl || process.env.A2A_HUB_URL || '').replace(/\/+$/, '');
450
- this.store = store;
451
- this.logger = logger || console;
452
- this.getTaskMeta = getTaskMeta || null;
453
- this._heartbeatTimer = null;
454
- this._running = false;
455
- this._startedAt = null;
456
- this._lastHeartbeatTickAt = 0;
457
- this._consecutiveFailures = 0;
458
- this._reauthInProgress = false;
459
- this._helloRateLimitUntil = 0;
460
- this._reauthBackoffUntil = 0;
461
- this._consecutiveReauthFailures = 0;
462
- this._driftInterval = null;
463
- this._lastDriftCheckAt = 0;
464
- this._hubUnreachableFailures = 0;
465
- this._hubUnreachableUntil = 0;
466
- this._envSecretSuppressionMarker = normalizeNodeSecretEnvSuppression(this.store && this.store.getState
467
- ? this.store.getState('node_secret_env_suppressed')
468
- : null);
469
- this._envSuppressionClearedForSecret = null;
470
- this._suppressEnvSecret = Boolean(this._envSecretSuppressionMarker && getEnvNodeSecret());
471
-
472
- // H4 fix: persist the legacy node_id file as soon as the in-memory
473
- // node_id is known, NOT only after a successful hello(). The original
474
- // code persisted only in hello() (see ~L390-398) — but proxy-mode boot
475
- // can fire `reportForceUpdateOutcome` BEFORE hello() returns:
476
- //
477
- // - First-tick heartbeat hits 426 → executeForceUpdate() → exit(78)
478
- // all happens BEFORE the hello() response is processed.
479
- // - enrich.js force_update path can fire during the same window.
480
- //
481
- // `_shortNodeIdForStatePath` in a2aProtocol.js then picks the
482
- // state-file suffix from `_cachedNodeId` (never set in proxy mode —
483
- // only getNodeId() sets it, and proxy never calls getNodeId()) or the
484
- // legacy ~/.evomap/node_id file. With both empty, it falls through
485
- // to 'anon', and the outcome lands at `force_update_last.anon.json`.
486
- // Next boot's hello() writes the real id, the heartbeat reads
487
- // `force_update_last.<8hex>.json`, the anon file is orphaned and the
488
- // outcome is silently lost.
489
- //
490
- // Persisting at construction closes the window. _persistLegacyNodeId
491
- // early-returns on invalid input (NODE_ID_RE gate, same regex as the
492
- // hello() path uses) so a malformed/empty store value is a no-op,
493
- // and it is idempotent on matching content so the cost is one
494
- // existsSync + one readFileSync per construction. We keep the
495
- // existing post-hello call as a safety net in case hello() mints or
496
- // mutates the id.
497
- try {
498
- const earlyNodeId = this.store && this.store.getState
499
- ? this.store.getState('node_id')
500
- : null;
501
- if (earlyNodeId && NODE_ID_RE.test(earlyNodeId)) {
502
- _persistLegacyNodeId(earlyNodeId);
503
- }
504
- } catch (e) {
505
- // Best-effort: persistence failure must never break construction.
506
- // Logger may not exist if tests passed undefined; guard the call.
507
- try {
508
- this.logger.warn(
509
- `[lifecycle] early persist of legacy node_id failed (non-fatal): ${e && e.message || e}`
510
- );
511
- } catch (_) { /* logger broken; non-fatal */ }
512
- }
513
- }
514
-
515
- get nodeId() {
516
- return this.store.getState('node_id');
517
- }
518
-
519
- get nodeSecret() {
520
- return this._resolveNodeSecret();
521
- }
522
-
523
- get nodeSecretVersion() {
524
- this._resolveNodeSecret();
525
- const storeVersion = parseNodeSecretVersion(this.store.getState('node_secret_version'));
526
- const storeSecret = this.store.getState('node_secret') || null;
527
- const storeSource = this.store.getState('node_secret_source') || null;
528
- const envSecret = this._effectiveEnvNodeSecret();
529
- const envVersion = parseNodeSecretVersion(process.env.A2A_NODE_SECRET_VERSION || process.env.EVOMAP_NODE_SECRET_VERSION);
530
- const validStoreSecret = validNodeSecret(storeSecret);
531
- if (this._suppressEnvSecret) return validStoreSecret ? storeVersion : null;
532
- if (storeSource === 'hub_rotate' && validStoreSecret) return storeVersion;
533
- if (envSecret) {
534
- if (envVersion) return envVersion;
535
- return storeSecret === envSecret ? storeVersion : null;
536
- }
537
- return validStoreSecret ? storeVersion : null;
538
- }
539
-
540
- /**
541
- * Resolve the active node_secret with conflict reconciliation between the
542
- * persistent MailboxStore and `process.env.A2A_NODE_SECRET`.
543
- *
544
- * Two opposite failure modes shape this logic:
545
- *
546
- * #529 (env-fresh, store-stale): operator exports a freshly minted
547
- * secret in A2A_NODE_SECRET (e.g. from .env), but the MailboxStore
548
- * still holds a long-stale value. The store value would otherwise
549
- * win and produce a 403 -> rotate -> 30-min backoff loop.
550
- *
551
- * "store-fresh, env-stale": process A rotates the secret via /a2a/hello,
552
- * so the store holds the value the hub now recognises. Process A then
553
- * restarts (typical: daemon respawn after upgrade or crash). The shell
554
- * it inherits its env from still exports the *previous* value of
555
- * A2A_NODE_SECRET. Without source-tracking we would treat this as
556
- * env-vs-store conflict, env-wins, and silently overwrite the
557
- * hub-recognised secret with a stale shell value -- exactly the loop
558
- * #529 was meant to fix, just symmetrical.
559
- *
560
- * Resolution: track *who wrote* the store value. When the hub returns a
561
- * rotated secret (`hello`), we tag the store entry with
562
- * `node_secret_source = 'hub_rotate'`. On conflict we honour that tag:
563
- *
564
- * source=hub_rotate -> store wins (recent rotation; env is stale)
565
- * source missing/'env_seed' -> env wins (legacy / first-boot bootstrap)
566
- *
567
- * Single-source mode (only one of store/env present) is unchanged.
568
- * @returns {string|null}
569
- */
570
- _resolveNodeSecret() {
571
- const envSecret = this._effectiveEnvNodeSecret();
572
- const envUpdatedAfterSuppression = this._envSuppressionClearedForSecret === envSecret;
573
- const storeSecret = this.store.getState('node_secret') || null;
574
- const storeSource = this.store.getState('node_secret_source') || null;
575
-
576
- if (
577
- envSecret &&
578
- envUpdatedAfterSuppression &&
579
- validNodeSecret(envSecret) &&
580
- (!storeSecret || envSecret === storeSecret) &&
581
- !(storeSecret && envSecret === storeSecret && storeSource === 'hub_rotate')
582
- ) {
583
- this._syncEnvNodeSecretToStore(envSecret);
584
- return envSecret;
585
- }
586
-
587
- if (envSecret && storeSecret && envSecret !== storeSecret) {
588
- // Store value came from a successful hub rotation -> trust it.
589
- // The env var is necessarily stale: it was captured by the parent
590
- // shell before the rotation and a child process cannot mutate it
591
- // back into its parent.
592
- if (storeSource === 'hub_rotate' && validNodeSecret(storeSecret) && !envUpdatedAfterSuppression) {
593
- if (!this._storeSourceLogged) {
594
- this._storeSourceLogged = true;
595
- this.logger.warn(
596
- '[lifecycle] A2A_NODE_SECRET env var differs from MailboxStore; ' +
597
- 'store value originated from a hub rotation, treating env as stale. ' +
598
- 'After a manual web reset, run `evolver reset-local-secret` to clear local ' +
599
- 'secret and env suppression state before relying only on updated env vars.'
600
- );
601
- }
602
- return storeSecret;
603
- }
604
- if (validNodeSecret(envSecret)) {
605
- const diskRotated = this._latestDiskHubRotatedSecret();
606
- if (diskRotated) {
607
- this._setNodeSecretState({
608
- secret: diskRotated.secret,
609
- version: diskRotated.version ? String(diskRotated.version) : '',
610
- source: 'hub_rotate',
611
- envSuppressed: this.store.getState('node_secret_env_suppressed') || '',
612
- });
613
- if (!this._storeSourceLogged) {
614
- this._storeSourceLogged = true;
615
- this.logger.warn(
616
- '[lifecycle] MailboxStore memory differs from disk hub-rotated node_secret; ' +
617
- 'using disk value and treating env as stale.'
618
- );
619
- }
620
- return diskRotated.secret;
621
- }
622
- this._syncEnvNodeSecretToStore(envSecret);
623
- if (!this._envOverrideLogged) {
624
- this._envOverrideLogged = true;
625
- this.logger.warn(
626
- '[lifecycle] A2A_NODE_SECRET env var differs from MailboxStore; using env value and syncing store. ' +
627
- 'Clear ~/.evomap/mailbox/state.json or unset A2A_NODE_SECRET to silence this warning.'
628
- );
629
- }
630
- return envSecret;
631
- }
632
- // env var malformed -- ignore it, fall back to store
633
- return storeSecret;
634
- }
635
-
636
- return storeSecret || envSecret || null;
637
- }
638
-
639
- _latestDiskHubRotatedSecret() {
640
- const file = this.store && this.store._stateFile;
641
- if (!file) return null;
642
- const state = readMailboxStateFile(file);
643
- const secret = typeof state?.node_secret === 'string' ? state.node_secret.trim() : null;
644
- if (!validNodeSecret(secret) || state?.node_secret_source !== 'hub_rotate') return null;
645
- return {
646
- secret,
647
- version: parseNodeSecretVersion(state.node_secret_version),
648
- };
649
- }
650
-
651
- _syncEnvNodeSecretToStore(envSecret) {
652
- const envVersion = parseNodeSecretVersion(process.env.A2A_NODE_SECRET_VERSION || process.env.EVOMAP_NODE_SECRET_VERSION);
653
- this._setNodeSecretState({
654
- secret: envSecret,
655
- version: envVersion ? String(envVersion) : '',
656
- source: 'env_seed',
657
- envSuppressed: '',
658
- });
659
- this._clearEnvSecretSuppression();
660
- }
661
-
662
- _setNodeSecretState({ secret = '', version = '', source = '', envSuppressed = '' } = {}) {
663
- if (this.store && typeof this.store.setNodeSecretState === 'function') {
664
- this.store.setNodeSecretState({ secret, version, source, envSuppressed });
665
- return;
666
- }
667
- this.store.setState('node_secret', secret);
668
- this.store.setState('node_secret_version', version ? String(version) : '');
669
- this.store.setState('node_secret_source', source);
670
- this.store.setState('node_secret_env_suppressed', envSuppressed);
671
- }
672
-
673
- _effectiveEnvNodeSecret() {
674
- const envSecret = getEnvNodeSecret();
675
- return this._isEnvNodeSecretSuppressed(envSecret) ? null : envSecret;
676
- }
677
-
678
- _isEnvNodeSecretSuppressed(envSecret) {
679
- this._envSuppressionClearedForSecret = null;
680
- const persistedMarker = normalizeNodeSecretEnvSuppression(this.store && this.store.getState
681
- ? this.store.getState('node_secret_env_suppressed')
682
- : null);
683
- const marker = persistedMarker || this._envSecretSuppressionMarker;
684
- if (!marker) {
685
- this._envSecretSuppressionMarker = null;
686
- this._suppressEnvSecret = false;
687
- return false;
688
- }
689
- this._envSecretSuppressionMarker = marker;
690
- if (marker === 'true') {
691
- this._suppressEnvSecret = Boolean(envSecret);
692
- return Boolean(envSecret);
693
- }
694
- const envFingerprint = fingerprintNodeSecret(envSecret);
695
- if (!envFingerprint) {
696
- this._suppressEnvSecret = false;
697
- return false;
698
- }
699
- if (envFingerprint === marker) {
700
- this._suppressEnvSecret = true;
701
- return true;
702
- }
703
- this._clearEnvSecretSuppression();
704
- this._envSuppressionClearedForSecret = envSecret;
705
- return false;
706
- }
707
-
708
- _markCurrentEnvSecretSuppressed() {
709
- const marker = fingerprintNodeSecret(getEnvNodeSecret());
710
- if (!marker) {
711
- this._clearEnvSecretSuppression();
712
- return;
713
- }
714
- try { this.store.setState('node_secret_env_suppressed', marker); } catch { /* best-effort */ }
715
- this._envSecretSuppressionMarker = marker;
716
- this._suppressEnvSecret = true;
717
- }
718
-
719
- _clearEnvSecretSuppression() {
720
- try { this.store.setState('node_secret_env_suppressed', ''); } catch { /* best-effort */ }
721
- this._envSecretSuppressionMarker = null;
722
- this._suppressEnvSecret = false;
723
- }
724
-
725
- _buildHeaders() {
726
- const headers = { 'Content-Type': 'application/json' };
727
- const secret = this.nodeSecret;
728
- if (secret) headers['Authorization'] = 'Bearer ' + secret;
729
- const secretVersion = this.nodeSecretVersion;
730
- if (secretVersion) headers['X-EvoMap-Node-Secret-Version'] = String(secretVersion);
731
- headers['x-correlation-id'] = crypto.randomUUID();
732
- return headers;
733
- }
734
-
735
- _hubUnreachableWaitMs() {
736
- return Math.max(0, this._hubUnreachableUntil - Date.now());
737
- }
738
-
739
- _recordHubReachable() {
740
- this._hubUnreachableFailures = 0;
741
- this._hubUnreachableUntil = 0;
742
- }
743
-
744
- _recordHubUnreachable(err) {
745
- this._hubUnreachableFailures += 1;
746
- const retryAfterMs = hubUnreachableBackoffMs(this._hubUnreachableFailures);
747
- this._hubUnreachableUntil = Date.now() + retryAfterMs;
748
- this.logger.warn?.(
749
- `[lifecycle] Hub unreachable; backing off for ${Math.ceil(retryAfterMs / 1000)}s: ` +
750
- `${err && err.message || err}`
751
- );
752
- return retryAfterMs;
753
- }
754
-
755
- async hello({ rotateSecret = false } = {}) {
756
- if (!this.hubUrl) return { ok: false, error: 'no_hub_url' };
757
-
758
- if (this._helloRateLimitUntil > Date.now()) {
759
- const waitSec = Math.ceil((this._helloRateLimitUntil - Date.now()) / 1000);
760
- this.logger.warn(`[lifecycle] hello suppressed: rate limited for ${waitSec}s`);
761
- return { ok: false, error: 'hello_rate_limit_active', waitSec };
762
- }
763
-
764
- const waitMs = this._hubUnreachableWaitMs();
765
- if (waitMs > 0) {
766
- return {
767
- ok: false,
768
- error: 'hub_unreachable_backoff',
769
- retryAfterMs: waitMs,
770
- };
771
- }
772
-
773
- const endpoint = `${this.hubUrl}/a2a/hello`;
774
- const nodeId = this.store.getState('node_id')
775
- || _readLegacyNodeId()
776
- || `node_${crypto.randomBytes(6).toString('hex')}`;
777
-
778
- const payload = { capabilities: {} };
779
- if (rotateSecret) payload.rotate_secret = true;
780
-
781
- const fp = _getEnvFingerprint();
782
-
783
- const body = {
784
- ...buildEnvelope('hello', payload, nodeId),
785
- env_fingerprint: fp,
786
- };
787
-
788
- try {
789
- const res = await hubFetch(endpoint, {
790
- method: 'POST',
791
- headers: this._buildHeaders(),
792
- body: JSON.stringify(body),
793
- signal: AbortSignal.timeout(HELLO_TIMEOUT),
794
- });
795
- await throwIfHubUnreachableResponse(res, 'lifecycle hello');
796
- this._recordHubReachable();
797
- if (!res.ok) {
798
- const errText = await readHubResponseText(res).catch(() => '');
799
- let errData = {};
800
- try { errData = errText ? JSON.parse(errText) : {}; } catch (_) { /* non-JSON API error */ }
801
- const errMsg = safeHubErrorCode(errData, res.status);
802
- if (res.status === 429) {
803
- const retryAfter = parseInt(res.headers.get('retry-after') || '3600', 10);
804
- this._helloRateLimitUntil = Date.now() + retryAfter * 1000;
805
- this.logger.error(`[lifecycle] hello rate limited (429): retry after ${retryAfter}s`);
806
- return { ok: false, error: 'hello_rate_limited', retryAfter };
807
- }
808
- const divergenceReason = (res.status === 401 || res.status === 403)
809
- ? extractSecretDivergenceReason(errData)
810
- : null;
811
- if (divergenceReason) {
812
- this.logger.warn(
813
- `[lifecycle] hello rejected secret rotation (${res.status}, reason=${divergenceReason}); ` +
814
- 'local secret divergence detected'
815
- );
816
- return {
817
- ok: false,
818
- error: SECRET_DIVERGENCE_ERROR,
819
- reason: divergenceReason,
820
- statusCode: res.status,
821
- };
822
- }
823
- const safeErrText = errText ? sanitizeHubResponseForLog(errText) : errMsg;
824
- this.logger.error(`[lifecycle] hello HTTP ${res.status}: ${safeErrText}`);
825
- return { ok: false, error: errMsg, statusCode: res.status };
826
- }
827
-
828
- const data = await readHubResponseJson(res);
829
-
830
- if (data?.payload?.status === 'rejected') {
831
- const divergenceReason = extractSecretDivergenceReason(data);
832
- if (divergenceReason) {
833
- this.logger.warn(
834
- `[lifecycle] hello rejected secret rotation (reason=${divergenceReason}); ` +
835
- 'local secret divergence detected'
836
- );
837
- return {
838
- ok: false,
839
- error: SECRET_DIVERGENCE_ERROR,
840
- reason: divergenceReason,
841
- response: data,
842
- };
843
- }
844
- const reason = data.payload.reason || 'unknown';
845
- const safeReason = sanitizeHubResponseForLog(reason);
846
- this.logger.error(`[lifecycle] hello rejected: ${safeReason}`);
847
- const error = String(reason || '').startsWith('node_id_already_claimed')
848
- ? reason
849
- : 'hello_rejected';
850
- return { ok: false, error, response: data };
851
- }
852
-
853
- const secret = data?.payload?.node_secret || data?.node_secret || null;
854
- const secretVersion = parseNodeSecretVersion(data?.payload?.node_secret_version || data?.node_secret_version);
855
- if (secret && validNodeSecret(secret)) {
856
- // Hub just handed us a fresh secret. Whatever sits in
857
- // A2A_NODE_SECRET is now older than the store, so suppress the
858
- // env-wins reconciliation in _resolveNodeSecret for the rest of
859
- // this process. Without this, the very next _buildHeaders call
860
- // (e.g. the verification heartbeat in reAuthenticate) would see
861
- // env vs store as a conflict, treat the env value as authoritative,
862
- // and overwrite the freshly rotated secret with the stale one,
863
- // re-creating the auth loop the previous patch fixed (see #529
864
- // and the Bugbot review on PR #22).
865
- if (getEnvNodeSecret() && getEnvNodeSecret() !== secret) {
866
- this._markCurrentEnvSecretSuppressed();
867
- } else {
868
- this._clearEnvSecretSuppression();
869
- }
870
- this._setNodeSecretState({
871
- secret,
872
- version: secretVersion ? String(secretVersion) : '',
873
- source: 'hub_rotate',
874
- envSuppressed: this.store.getState('node_secret_env_suppressed') || '',
875
- });
876
- this.logger.log('[lifecycle] new node_secret stored from hello response');
877
- } else if (secretVersion) {
878
- this.store.setState('node_secret_version', String(secretVersion));
879
- } else {
880
- this.store.setState('node_secret_version', '');
881
- }
882
-
883
- this.store.setState('node_id', nodeId);
884
- // Unify proxy node_id with the legacy GEP file. Without this, the
885
- // proxy-only fast path (EVOMAP_PROXY=1) never seeds
886
- // ~/.evomap/node_id and `_shortNodeIdForStatePath` in a2aProtocol
887
- // (used to pick the per-node `force_update_last.<suffix>.json`
888
- // path for upgrade telemetry) falls through to 'anon' — every
889
- // proxy node under the same EVOLVER_HOME would collide on the
890
- // same state file. In a mixed-mode install where the legacy file
891
- // holds a DIFFERENT (stale) id, the helper overwrites it so the
892
- // state-file suffix matches `this.nodeId` — the id the hub sees
893
- // in body.node_id. We persist AFTER hello succeeds so a rejected
894
- // first-boot mint never commits to disk; on a rejection the next
895
- // tick will mint fresh again (existing behaviour).
896
- try {
897
- _persistLegacyNodeId(nodeId);
898
- } catch (e) {
899
- // Best-effort: persistence failure must never break hello. Log
900
- // and move on — the proxy still functions, the state-file
901
- // suffix just falls back to 'anon' until the next successful
902
- // hello retries the write.
903
- this.logger.warn(`[lifecycle] failed to persist legacy node_id (non-fatal): ${e && e.message || e}`);
904
- }
905
- this.logger.log(`[lifecycle] hello OK, node_id=${nodeId}${rotateSecret ? ' (secret rotated)' : ''}`);
906
- return { ok: true, nodeId, response: data };
907
- } catch (err) {
908
- if (isHubUnreachableError(err)) {
909
- const retryAfterMs = this._recordHubUnreachable(err);
910
- return {
911
- ok: false,
912
- error: 'hub_unreachable',
913
- detail: err.message,
914
- retryAfterMs,
915
- };
916
- }
917
- this.logger.error(`[lifecycle] hello failed: ${err.message}`);
918
- return { ok: false, error: err.message };
919
- }
920
- }
921
-
922
- /**
923
- * Re-authenticate after 403: rotate secret via hello, then verify with a
924
- * heartbeat. Returns true if auth is restored, false otherwise.
925
- *
926
- * Recovery sequence (issue EvoMap/evolver#529):
927
- * attempt 1 -> hello with current Bearer + rotate_secret=true
928
- * (works when the stale secret is still recognised)
929
- * attempt 2 -> drop the bearer locally, hello WITHOUT Authorization
930
- * + rotate_secret=true. If the node is owned by someone
931
- * else, hub returns node_id_already_claimed; we surface a
932
- * manual-reset hint to the user instead of churning forever.
933
- */
934
- async reAuthenticate() {
935
- if (this._reauthInProgress) return false;
936
- if (this._reauthBackoffUntil > Date.now()) {
937
- const waitSec = Math.ceil((this._reauthBackoffUntil - Date.now()) / 1000);
938
- this.logger.warn(`[lifecycle] re-auth suppressed: backoff active for ${waitSec}s`);
939
- return false;
940
- }
941
- this._reauthInProgress = true;
942
- let manualResetRequired = false;
943
- let hubUnreachable = false;
944
- let droppedDivergedSecret = false;
945
- try {
946
- for (let attempt = 1; attempt <= MAX_REAUTH_ATTEMPTS; attempt++) {
947
- this.logger.warn(`[lifecycle] re-auth attempt ${attempt}/${MAX_REAUTH_ATTEMPTS}: rotating secret via hello...`);
948
- const helloResult = await this.hello({ rotateSecret: true });
949
- if (!helloResult.ok) {
950
- this.logger.error(`[lifecycle] re-auth hello failed: ${helloResult.error}`);
951
- if (helloResult.error === 'hello_rate_limited' || helloResult.error === 'hello_rate_limit_active') break;
952
- // Hub link is down (WAF HTML, network error, timeout) -- NOT an auth
953
- // failure. Bail without arming the re-auth backoff: otherwise a
954
- // transient outage that arrives mid-rotate would burn both attempts
955
- // (attempt 2's hello short-circuits on the hub-unreachable window)
956
- // and suppress genuine auth recovery for up to REAUTH_BACKOFF_MAX_MS,
957
- // even though a JSON 401/403 retry would succeed once the hub is
958
- // reachable again. The hub-unreachable window already gates re-entry.
959
- if (helloResult.error === 'hub_unreachable' || helloResult.error === 'hub_unreachable_backoff') {
960
- hubUnreachable = true;
961
- break;
962
- }
963
- if (isSecretDivergenceError(helloResult.error)) {
964
- // The hub has explicitly rejected our current secret as diverged
965
- // from its record. Drop store/env-derived auth once, then spend
966
- // the second attempt on an unauthenticated rotate hello instead
967
- // of backing off while still presenting the same stale Bearer.
968
- if (attempt < MAX_REAUTH_ATTEMPTS && !droppedDivergedSecret) {
969
- this._dropLocalSecret(SECRET_DIVERGENCE_ERROR);
970
- droppedDivergedSecret = true;
971
- continue;
972
- }
973
- break;
974
- }
975
- if (typeof helloResult.error === 'string' && helloResult.error.startsWith('node_id_already_claimed')) {
976
- // Hub does not believe we own this nodeId. Our locally cached
977
- // secret(s) are useless. Drop them so attempt 2 retries WITHOUT
978
- // a Bearer (lenient hello path). If even unauthenticated rotate
979
- // is rejected, only a manual reset can recover.
980
- if (attempt < MAX_REAUTH_ATTEMPTS) {
981
- this._dropLocalSecret('node_id_already_claimed');
982
- continue;
983
- }
984
- manualResetRequired = true;
985
- break;
986
- }
987
- continue;
988
- }
989
- const newSecret = helloResult.response?.payload?.node_secret;
990
- if (!newSecret) {
991
- this.logger.error('[lifecycle] re-auth: hub did not return a new secret (rotate may not have taken effect)');
992
- break;
993
- }
994
- const hbResult = await this.heartbeat({ _skipReauth: true });
995
- if (hbResult.ok) {
996
- this.logger.log('[lifecycle] re-auth succeeded: heartbeat confirmed with new secret');
997
- this._consecutiveReauthFailures = 0;
998
- // Note: _envOverrideLogged is intentionally NOT reset here.
999
- // The successful hello path above already set _suppressEnvSecret=true,
1000
- // which means _resolveNodeSecret will never hit the env-vs-store
1001
- // conflict branch again in this process, so the warning would never
1002
- // fire a second time anyway. Resetting the flag was misleading.
1003
- return true;
1004
- }
1005
- // Same as the hello path: if the verification heartbeat fails because
1006
- // the hub link dropped (not an auth rejection), defer without arming
1007
- // the re-auth backoff.
1008
- if (hbResult.error === 'hub_unreachable' || hbResult.error === 'hub_unreachable_backoff') {
1009
- hubUnreachable = true;
1010
- break;
1011
- }
1012
- this.logger.warn(`[lifecycle] re-auth attempt ${attempt}: heartbeat still failing after rotate`);
1013
- }
1014
- if (hubUnreachable) {
1015
- this.logger.warn('[lifecycle] re-auth deferred: hub unreachable (no re-auth backoff armed)');
1016
- return false;
1017
- }
1018
- if (manualResetRequired) {
1019
- this._emitManualResetNeeded();
1020
- }
1021
- this._consecutiveReauthFailures += 1;
1022
- const backoffMs = Math.min(
1023
- REAUTH_BACKOFF_BASE_MS * Math.pow(2, this._consecutiveReauthFailures - 1),
1024
- REAUTH_BACKOFF_MAX_MS
1025
- );
1026
- const backoffMin = Math.round(backoffMs / 60_000);
1027
- this.logger.error(
1028
- `[lifecycle] re-auth exhausted all attempts (failure #${this._consecutiveReauthFailures}), ` +
1029
- `backing off for ${backoffMin} minutes`
1030
- );
1031
- this._reauthBackoffUntil = Date.now() + backoffMs;
1032
- return false;
1033
- } finally {
1034
- this._reauthInProgress = false;
1035
- }
1036
- }
1037
-
1038
- /**
1039
- * Drop the cached node_secret in MailboxStore AND signal the env-var path to
1040
- * skip its value for this process lifetime. Used when the hub explicitly
1041
- * disowns our claim.
1042
- * @param {string} reason - log tag describing why we are dropping it.
1043
- */
1044
- _dropLocalSecret(reason) {
1045
- this.logger.warn(`[lifecycle] dropping cached node_secret (reason=${reason}); next hello will run unauthenticated`);
1046
- const diskRotated = this._latestDiskHubRotatedSecret();
1047
- const storeSecret = this.store.getState('node_secret') || null;
1048
- if (diskRotated && diskRotated.secret !== storeSecret) {
1049
- try {
1050
- this._setNodeSecretState({
1051
- secret: diskRotated.secret,
1052
- version: diskRotated.version ? String(diskRotated.version) : '',
1053
- source: 'hub_rotate',
1054
- envSuppressed: this.store.getState('node_secret_env_suppressed') || '',
1055
- });
1056
- this.logger.warn('[lifecycle] local in-memory node_secret is stale; preserving newer disk hub-rotated secret');
1057
- return;
1058
- } catch (err) {
1059
- const reason = err && (err.code || err.name) ? (err.code || err.name) : 'error';
1060
- this.logger.warn(`[lifecycle] failed to sync newer disk hub-rotated node_secret into memory (reason=${reason})`);
1061
- }
1062
- }
1063
- try {
1064
- this._setNodeSecretState({
1065
- secret: '',
1066
- version: '',
1067
- source: '',
1068
- envSuppressed: this.store.getState('node_secret_env_suppressed') || '',
1069
- });
1070
- } catch (err) {
1071
- const reason = err && (err.code || err.name || err.message) ? (err.code || err.name || err.message) : 'error';
1072
- this.logger.warn(`[lifecycle] failed to clear local node_secret state (reason=${reason})`);
1073
- }
1074
- // Suppress only the exact env secret that was just proven stale. If no
1075
- // env secret is present, do not leave a marker that blocks a future reset.
1076
- this._markCurrentEnvSecretSuppressed();
1077
- }
1078
-
1079
- _emitManualResetNeeded() {
1080
- try {
1081
- this.store.writeInbound({
1082
- type: 'system',
1083
- priority: 'high',
1084
- channel: 'evomap-hub',
1085
- payload: {
1086
- action: 'manual_secret_reset_required',
1087
- message:
1088
- 'Hub disowns this node_id (node_id_already_claimed). Local node_secret and the current node secret env value are invalid. Visit https://evomap.ai/account, click "Reset Secret" on the agent card, run `evolver reset-local-secret` to clear local secret and env suppression state, then update A2A_NODE_SECRET/EVOMAP_NODE_SECRET and restart proxy. Only changing env before clearing suppression state can keep an older local marker active.',
1089
- docs_url: 'https://evomap.ai/account',
1090
- },
1091
- });
1092
- } catch (err) {
1093
- this.logger.warn(`[lifecycle] failed to emit manual_secret_reset_required event: ${err.message}`);
1094
- }
1095
- }
1096
-
1097
- async heartbeat({ _skipReauth = false } = {}) {
1098
- // Wrap the entire body — including pre-fetch helpers (hello,
1099
- // getTaskMeta, store.countPending, env_fingerprint) — in a single
1100
- // try/catch. Reporter (#544) showed a single throw from any of
1101
- // these (store corrupt, hello rejecting, fingerprint failing under
1102
- // sandboxing) used to escape and kill `tick()`, leaving the
1103
- // daemon "alive but never heartbeating". The post-fetch path is
1104
- // unchanged; only the boundary moved earlier.
1105
- if (!this.hubUrl) return { ok: false, error: 'no_hub_url' };
1106
- try {
1107
- const waitMs = this._hubUnreachableWaitMs();
1108
- if (waitMs > 0) {
1109
- return {
1110
- ok: false,
1111
- error: 'hub_unreachable_backoff',
1112
- retryAfterMs: waitMs,
1113
- };
1114
- }
1115
-
1116
- const nodeId = this.nodeId;
1117
- if (!nodeId) {
1118
- const helloResult = await this.hello();
1119
- if (!helloResult.ok) return helloResult;
1120
- }
1121
-
1122
- const endpoint = `${this.hubUrl}/a2a/heartbeat`;
1123
- const taskMeta = typeof this.getTaskMeta === 'function' ? this.getTaskMeta() : {};
1124
- const fp = _getEnvFingerprint();
1125
- const secretVersion = this.nodeSecretVersion;
1126
- const body = {
1127
- node_id: this.nodeId,
1128
- sender_id: this.nodeId,
1129
- evolver_version: fp.evolver_version || PROXY_PROTOCOL_VERSION,
1130
- env_fingerprint: fp,
1131
- meta: {
1132
- proxy_version: PROXY_PROTOCOL_VERSION,
1133
- proxy_protocol_version: PROXY_PROTOCOL_VERSION,
1134
- outbound_pending: this.store.countPending({ direction: 'outbound' }),
1135
- inbound_pending: this.store.countPending({ direction: 'inbound' }),
1136
- ...taskMeta,
1137
- },
1138
- };
1139
- if (secretVersion) {
1140
- body.node_secret_version = secretVersion;
1141
- body.meta.node_secret_version = secretVersion;
1142
- }
1143
-
1144
- try {
1145
- const cfg = require('../../config');
1146
- if (cfg.antiAbuseTelemetryMode && cfg.antiAbuseTelemetryMode() === 'heartbeat') {
1147
- const { buildHeartbeatAntiAbuseTelemetry } = require('../../gep/antiAbuseTelemetry');
1148
- body.meta.anti_abuse = buildHeartbeatAntiAbuseTelemetry({
1149
- source: 'evolver-proxy',
1150
- nodeId: this.nodeId,
1151
- envFingerprint: fp,
1152
- taskMeta: body.meta,
1153
- // This heartbeat is sent FROM the running proxy — ground truth,
1154
- // not env sniffing (this process usually has neither EVOMAP_PROXY
1155
- // nor EVOMAP_PROXY_PORT set, which would misreport false).
1156
- proxyPortConfigured: true,
1157
- });
1158
- }
1159
- } catch (e) {
1160
- this.logger.warn(`[AntiAbuseTelemetry] failed to build heartbeat summary: ${e && e.message || e}`);
1161
- }
1162
-
1163
- // Attach any pending force_update outcome so the hub-side
1164
- // EvolverUpgradeAttempt table gets a row. Captured in a local so the
1165
- // post-2xx clear matches identity (rotation-safe — see
1166
- // _clearLastUpdateStateIfMatches). Never let telemetry throw.
1167
- let capturedLastUpdate = null;
1168
- try {
1169
- const pending = readPendingLastUpdate();
1170
- if (pending) {
1171
- body.last_update = pending;
1172
- capturedLastUpdate = pending;
1173
- }
1174
- } catch (e) {
1175
- this.logger.warn(`[lifecycle] readPendingLastUpdate failed (non-fatal): ${e && e.message || e}`);
1176
- }
1177
-
1178
- const res = await hubFetch(endpoint, {
1179
- method: 'POST',
1180
- headers: this._buildHeaders(),
1181
- body: JSON.stringify(body),
1182
- signal: AbortSignal.timeout(HEARTBEAT_TIMEOUT),
1183
- });
1184
-
1185
- await throwIfHubUnreachableResponse(res, 'lifecycle heartbeat');
1186
- this._recordHubReachable();
1187
-
1188
- if (res.status === 403 || res.status === 401) {
1189
- this._consecutiveFailures++;
1190
- const errText = await readHubResponseText(res).catch(() => '');
1191
- const safeErrText = sanitizeHubResponseForLog(errText);
1192
- this.logger.error(`[lifecycle] heartbeat auth failed (${res.status}): ${safeErrText}`);
1193
- if (!_skipReauth) {
1194
- const recovered = await this.reAuthenticate();
1195
- if (recovered) {
1196
- this._consecutiveFailures = 0;
1197
- return { ok: true, recovered: true };
1198
- }
1199
- }
1200
- return { ok: false, error: `auth_failed_${res.status}`, statusCode: res.status };
1201
- }
1202
-
1203
- if (!res.ok) {
1204
- const errText = await readHubResponseText(res).catch(() => '');
1205
- const safeErrText = sanitizeHubResponseForLog(errText);
1206
- // 426 Upgrade Required: hub emits this when our evolver_version is
1207
- // below the minimum version it requires. The body is JSON of shape
1208
- // `{ error: 'evolver_min_version_required', force_update: {...} }`
1209
- // (see hub `src/routes/a2a/_middleware.js`). Pre-fix this fell
1210
- // through to the generic `http_426` error and the proxy never
1211
- // attempted the upgrade — defeating the very mechanism that 426
1212
- // exists to drive. Mirror the 200+force_update path: parse the
1213
- // body, fire executeForceUpdate (which writes the state file via
1214
- // reportForceUpdateOutcome), and let the next heartbeat carry the
1215
- // attempt as body.last_update. Still return an error so the
1216
- // caller's failure counter ticks and the loop backs off.
1217
- if (res.status === 426) {
1218
- let parsed = null;
1219
- try { parsed = JSON.parse(errText); } catch (_) { /* body not JSON */ }
1220
- const fu = parsed && parsed.force_update;
1221
- if (fu && typeof fu === 'object') {
1222
- this.logger.warn(
1223
- `[lifecycle] heartbeat HTTP 426 with force_update directive (required=${
1224
- fu.required_version || '?'
1225
- }) — triggering executeForceUpdate`
1226
- );
1227
- _maybeTriggerForceUpdateFromHeartbeat(fu, this.logger);
1228
- } else {
1229
- this.logger.warn(
1230
- `[lifecycle] heartbeat HTTP 426 without parseable force_update payload: ${safeErrText}`
1231
- );
1232
- }
1233
- }
1234
- // Hub 400 circuit breaker (mirrors a2aProtocol.js sendHeartbeat
1235
- // ~L2376-2411): if last_update was attached this tick and the hub
1236
- // rejected the body with 400 AND the rejection names the
1237
- // last_update field, the state file is poisoning every heartbeat
1238
- // (e.g. downgrade-then-upgrade left a payload the new hub schema
1239
- // rejects, or a manual edit corrupted the JSON shape). The proxy
1240
- // path used to lack this breaker entirely, so a single bad payload
1241
- // would block telemetry forever -- every retry re-sends the same
1242
- // poison and re-fails with 400. Single-strike (no counter): the
1243
- // 400 + last_update substring pair is unambiguous enough that
1244
- // waiting for repeats just delays recovery. Scope intentionally
1245
- // narrowed to 400-only (NOT any 4xx): 401/403 are auth errors
1246
- // (handled above), 404/405/409 etc. are hub-routing problems that
1247
- // are not the payload's fault. The existing _consecutiveFailures
1248
- // backoff is preserved -- the breaker runs BEFORE the early
1249
- // return so the file is cleared, and then the normal failure
1250
- // path continues unchanged.
1251
- if (res.status === 400 && capturedLastUpdate) {
1252
- const errorText = 'http_400: ' + errText;
1253
- if (/last[_-]?update/i.test(errorText)) {
1254
- // Bypass any rate-limited warn helper: this is a critical
1255
- // recovery signal that must surface even if other ForceUpdate
1256
- // warns fired recently.
1257
- this.logger.warn(
1258
- '[lifecycle] hub 400 with last_update attached (error names last_update); ' +
1259
- 'clearing poisoning state file.'
1260
- );
1261
- try {
1262
- clearLastUpdateOnAck(capturedLastUpdate);
1263
- } catch (e) {
1264
- this.logger.warn(`[lifecycle] clearLastUpdateOnAck failed (non-fatal): ${e && e.message || e}`);
1265
- }
1266
- }
1267
- }
1268
- this._consecutiveFailures++;
1269
- this.logger.error(`[lifecycle] heartbeat HTTP ${res.status}: ${safeErrText}`);
1270
- return { ok: false, error: `http_${res.status}`, statusCode: res.status };
1271
- }
1272
-
1273
- const data = await readHubResponseJson(res);
1274
-
1275
- this._consecutiveFailures = 0;
1276
- this.store.setState('last_heartbeat_at', new Date().toISOString());
1277
-
1278
- // Semantic parity with a2aProtocol.js sendHeartbeat: a 2xx with
1279
- // `{ok:false}` or `status:'unknown_node'` is NOT a hub-side persist,
1280
- // so the state file must survive for the next heartbeat to retry
1281
- // (unknown_node triggers a re-hello below).
1282
- //
1283
- // PR #188 follow-up (HIGH H1-client): the hub now writes a top-level
1284
- // `last_update_ack: { ok, reason? }` whenever the request carried a
1285
- // last_update payload. Gate the clear on the ack so we do not unlink
1286
- // the only evidence of the upgrade attempt when the hub's
1287
- // fire-and-forget persist throws / dedup-misses / schema-rejects /
1288
- // bypass-path returns false. Backward compat: an old hub that has not
1289
- // yet rolled out the ack writer falls back to the original bare-2xx
1290
- // semantics so this client keeps working against pre-rollout hubs.
1291
- // See src/gep/a2aProtocol.js sendHeartbeat for the canonical comment.
1292
- const hubAccepted = !(data && data.ok === false) && data?.status !== 'unknown_node';
1293
- if (capturedLastUpdate) {
1294
- const ack = data && data.last_update_ack;
1295
- const hasAck = ack && typeof ack === 'object';
1296
- let shouldClear;
1297
- if (hasAck) {
1298
- shouldClear = ack.ok === true
1299
- || ack.reason === 'duplicate'
1300
- || ack.reason === 'invalid';
1301
- if (ack.reason === 'failed') {
1302
- this.logger.warn('[lifecycle] hub last_update_ack=failed; ' +
1303
- 'keeping state file for retry on next heartbeat.');
1304
- } else if (ack.reason === 'invalid') {
1305
- this.logger.warn('[lifecycle] hub last_update_ack=invalid; ' +
1306
- 'clearing state file (retry will not help).');
1307
- }
1308
- } else {
1309
- shouldClear = hubAccepted;
1310
- }
1311
- if (shouldClear) {
1312
- try {
1313
- clearLastUpdateOnAck(capturedLastUpdate);
1314
- } catch (e) {
1315
- this.logger.warn(`[lifecycle] clearLastUpdateOnAck failed (non-fatal): ${e && e.message || e}`);
1316
- }
1317
- }
1318
- }
1319
-
1320
- if (data?.status === 'unknown_node') {
1321
- this.logger.warn('[lifecycle] Node unknown, re-registering...');
1322
- await this.hello();
1323
- }
1324
-
1325
- // PR #188 H1 fix: 200 with a `force_update` directive must drive
1326
- // executeForceUpdate the same way a2aProtocol.js does for
1327
- // non-proxy nodes (see a2aProtocol.js:2292-2305 and
1328
- // _maybeTriggerForceUpdateFromHeartbeat). Pure proxy-mode nodes
1329
- // never enter the evolve run() loop, so the consumeForceUpdate
1330
- // path never fires for them — without this block the hub could
1331
- // push force_update forever with zero upgrade attempts and zero
1332
- // EvolverUpgradeAttempt rows. The helper is in-flight + cooldown
1333
- // gated; placing the call here (post-events, pre-min_version
1334
- // banner) means a single response carrying both events AND a
1335
- // force_update still processes the events first.
1336
- if (data && data.force_update && typeof data.force_update === 'object') {
1337
- this.logger.log(
1338
- '[ForceUpdate] Hub requires update to ' +
1339
- (data.force_update.required_version || '?') +
1340
- ' -- reason: ' + (data.force_update.reason || 'unspecified')
1341
- );
1342
- _maybeTriggerForceUpdateFromHeartbeat(data.force_update, this.logger);
1343
- }
1344
-
1345
- if (Array.isArray(data?.events) && data.events.length > 0) {
1346
- this.store.writeInboundBatch(
1347
- data.events.map(e => ({
1348
- type: e.type || 'hub_event',
1349
- payload: e,
1350
- channel: 'evomap-hub',
1351
- }))
1352
- );
1353
- }
1354
-
1355
- if (data?.min_proxy_version && this._shouldUpgrade(data.min_proxy_version)) {
1356
- this.store.writeInbound({
1357
- type: 'system',
1358
- payload: {
1359
- action: 'proxy_upgrade_required',
1360
- min_version: data.min_proxy_version,
1361
- current_version: PROXY_PROTOCOL_VERSION,
1362
- upgrade_url: data.upgrade_url || null,
1363
- message: data.upgrade_message || 'Proxy version is below the minimum required by Hub.',
1364
- },
1365
- channel: 'evomap-hub',
1366
- priority: 'high',
1367
- });
1368
- this.logger.warn(`[lifecycle] Hub requires proxy >= ${data.min_proxy_version}, current: ${PROXY_PROTOCOL_VERSION}`);
1369
- }
1370
-
1371
- return { ok: true, response: data };
1372
- } catch (err) {
1373
- if (isHubUnreachableError(err)) {
1374
- this._consecutiveFailures++;
1375
- const retryAfterMs = this._recordHubUnreachable(err);
1376
- return {
1377
- ok: false,
1378
- error: 'hub_unreachable',
1379
- detail: err.message,
1380
- retryAfterMs,
1381
- };
1382
- }
1383
- this._consecutiveFailures++;
1384
- this.logger.error(`[lifecycle] heartbeat failed (${this._consecutiveFailures}): ${err.message}`);
1385
- return { ok: false, error: err.message };
1386
- }
1387
- }
1388
-
1389
- startHeartbeatLoop(intervalMs) {
1390
- if (this._running) return;
1391
- this._running = true;
1392
- this._startedAt = Date.now();
1393
- this._heartbeatInterval = Math.max(30_000, intervalMs || DEFAULT_HEARTBEAT_INTERVAL);
1394
- // Generation counter: every poke / stop bumps this. Tick captures
1395
- // its gen at entry; if it doesn't match on resume, the tick refuses
1396
- // to schedule its own next timer (a fresher path already owns it).
1397
- this._heartbeatGen = (this._heartbeatGen || 0) + 1;
1398
- this._heartbeatTick(this._heartbeatGen);
1399
- this._startDriftDetector();
1400
- }
1401
-
1402
- // Sample wall-clock every DRIFT_CHECK_MS so macOS sleep / hypervisor
1403
- // pause / debugger break is detected and the heartbeat loop is poked
1404
- // back into action without waiting for the (possibly 15-min) backoff
1405
- // timer to fire on libuv's monotonic clock. R10 (#544).
1406
- _startDriftDetector() {
1407
- if (this._driftInterval) return;
1408
- this._lastDriftCheckAt = Date.now();
1409
- this._driftInterval = setInterval(() => {
1410
- // Wrap the whole body in try/catch -- this is a setInterval
1411
- // callback; any throw escaping it kills the detector itself,
1412
- // which is the bug we're protecting against.
1413
- try {
1414
- if (!this._running) return;
1415
- const now = Date.now();
1416
- const gap = now - this._lastDriftCheckAt;
1417
- this._lastDriftCheckAt = now;
1418
- if (gap > DRIFT_SLEEP_THRESHOLD_MS) {
1419
- try {
1420
- this.logger.warn(
1421
- `[lifecycle] wall-clock jump detected (+${Math.round(gap / 1000)}s); ` +
1422
- 'likely sleep/wake or process suspension, poking heartbeat'
1423
- );
1424
- } catch (_) { /* logger broken; detector must still poke */ }
1425
- // Long-sleep recovery: the hub-side cached state we carried
1426
- // through the suspend is almost certainly stale. Clear reauth
1427
- // backoff so the next tick can try a clean recovery path
1428
- // instead of sitting out a pre-sleep penalty for up to 4h.
1429
- if (gap > DRIFT_LONG_SLEEP_THRESHOLD_MS) {
1430
- this._consecutiveReauthFailures = 0;
1431
- this._reauthBackoffUntil = 0;
1432
- try {
1433
- this.logger.warn(
1434
- `[lifecycle] long sleep (+${Math.round(gap / 60_000)}min) cleared reauth backoff`
1435
- );
1436
- } catch (_) { /* logger broken; non-fatal */ }
1437
- }
1438
- this.pokeHeartbeatLoop();
1439
- }
1440
- } catch (err) {
1441
- try { this.logger.error(`[lifecycle] drift detector threw: ${err && err.message}`); }
1442
- catch (_) { /* never let the detector escape */ }
1443
- }
1444
- }, DRIFT_CHECK_MS);
1445
- // Don't keep the event loop alive on behalf of the detector alone --
1446
- // matches the unref() used on _heartbeatTimer.
1447
- if (this._driftInterval.unref) this._driftInterval.unref();
1448
- }
1449
-
1450
- async _heartbeatTick(myGen) {
1451
- if (!this._running) return;
1452
- this._lastHeartbeatTickAt = Date.now();
1453
- // Defence-in-depth: even with heartbeat() now fully wrapped (see
1454
- // its body), an unforeseen synchronous throw inside the awaited
1455
- // path or a defective stub passed in tests would still bubble
1456
- // into this closure as a rejected promise and cancel the next
1457
- // setTimeout. Catching here guarantees the loop schedules its
1458
- // own next tick under all conditions short of `stopHeartbeatLoop`.
1459
- try {
1460
- await this.heartbeat();
1461
- } catch (err) {
1462
- this._consecutiveFailures++;
1463
- this.logger.error(`[lifecycle] heartbeat tick threw (${this._consecutiveFailures}): ${err && err.message}`);
1464
- }
1465
- if (!this._running) return;
1466
- // Generation guard: a poke or stop fired while we were awaiting
1467
- // `heartbeat()`. The fresher path already armed its own timer (or
1468
- // tore the loop down); scheduling here would orphan the new timer
1469
- // or fork the loop into two concurrent ticks (Bugbot #147 finding
1470
- // 2026-05-28).
1471
- if (myGen !== undefined && myGen !== this._heartbeatGen) return;
1472
- // 15min cap (was 30min, then briefly 5min). Must stay above
1473
- // `DEFAULT_HEARTBEAT_INTERVAL` (6min) or the exponential branch
1474
- // retries faster than the success branch (Bugbot #147 finding).
1475
- const interval = this._heartbeatInterval || DEFAULT_HEARTBEAT_INTERVAL;
1476
- const hubWaitMs = this._hubUnreachableWaitMs();
1477
- // While a hub-unreachable window is active, wake exactly when it elapses
1478
- // (1s floor to avoid a busy 0ms reschedule), mirroring SyncEngine's
1479
- // `Math.max(1_000, retryAfterMs)`. A 30s floor over-waited up to ~30s past
1480
- // the window's end, delaying recovery once the hub was reachable again.
1481
- const backoff = hubWaitMs > 0
1482
- ? Math.max(1_000, hubWaitMs)
1483
- : this._consecutiveFailures > 0
1484
- ? Math.min(interval * Math.pow(2, this._consecutiveFailures), HEARTBEAT_BACKOFF_CAP_MS)
1485
- : interval;
1486
- const armedGen = this._heartbeatGen;
1487
- this._heartbeatTimer = setTimeout(() => this._heartbeatTick(armedGen), backoff);
1488
- if (this._heartbeatTimer.unref) this._heartbeatTimer.unref();
1489
- }
1490
-
1491
- /// Reset the heartbeat schedule immediately. Call when an external
1492
- /// signal (Hub push, tab visibility, manual `evolver heartbeat now`)
1493
- /// indicates the next tick should not wait out its current backoff.
1494
- /// No-op when the loop is not running.
1495
- pokeHeartbeatLoop() {
1496
- if (!this._running) return;
1497
- if (this._heartbeatTimer) {
1498
- clearTimeout(this._heartbeatTimer);
1499
- this._heartbeatTimer = null;
1500
- }
1501
- this._consecutiveFailures = 0;
1502
- // Bump generation: any in-flight `_heartbeatTick` from before the
1503
- // poke will see its captured gen mismatch on resume and skip its
1504
- // tail-`setTimeout`, so we don't end up with two concurrent timers.
1505
- this._heartbeatGen = (this._heartbeatGen || 0) + 1;
1506
- const myGen = this._heartbeatGen;
1507
- // setImmediate-equivalent: defer to the next event-loop turn so
1508
- // re-entrant pokes (e.g. from a Hub event handler) don't run
1509
- // synchronously inside the caller's frame.
1510
- this._heartbeatTimer = setTimeout(() => this._heartbeatTick(myGen), 0);
1511
- if (this._heartbeatTimer.unref) this._heartbeatTimer.unref();
1512
- }
1513
-
1514
- stopHeartbeatLoop() {
1515
- this._running = false;
1516
- if (this._heartbeatTimer) {
1517
- clearTimeout(this._heartbeatTimer);
1518
- this._heartbeatTimer = null;
1519
- }
1520
- if (this._driftInterval) {
1521
- clearInterval(this._driftInterval);
1522
- this._driftInterval = null;
1523
- }
1524
- }
1525
-
1526
- getHeartbeatStats() {
1527
- return {
1528
- running: this._running,
1529
- intervalMs: this._heartbeatInterval || DEFAULT_HEARTBEAT_INTERVAL,
1530
- uptimeMs: this._startedAt ? Date.now() - this._startedAt : 0,
1531
- consecutiveFailures: this._consecutiveFailures,
1532
- lastTickAt: this._lastHeartbeatTickAt,
1533
- };
1534
- }
1535
-
1536
- _shouldUpgrade(minVersion) {
1537
- // parseInt strips trailing non-digit chars in prerelease segments like
1538
- // `1-beta`, so `0.1.1-beta.1`.split('.')[2] -> `1-beta` -> parseInt = 1.
1539
- // Using Number() here returned NaN and was treated as 0, under-counting
1540
- // prerelease minimums. See community PR #516.
1541
- const parse = (v) => String(v || '0.0.0').split('.').map((part) => parseInt(part, 10));
1542
- const min = parse(minVersion);
1543
- const cur = parse(PROXY_PROTOCOL_VERSION);
1544
- for (let i = 0; i < 3; i++) {
1545
- if ((cur[i] || 0) < (min[i] || 0)) return true;
1546
- if ((cur[i] || 0) > (min[i] || 0)) return false;
1547
- }
1548
- return false;
1549
- }
1550
- }
1551
-
1552
- module.exports = {
1553
- LifecycleManager,
1554
- AuthError,
1555
- DEFAULT_HEARTBEAT_INTERVAL,
1556
- HEARTBEAT_BACKOFF_CAP_MS,
1557
- // Test hooks behind `_testing` to mirror the namespacing used by
1558
- // a2aProtocol.js — production callers must not accidentally tweak the
1559
- // proxy force_update lifecycle state.
1560
- _testing: {
1561
- // Reset proxy heartbeat-driven force_update state. Avoids cooldown
1562
- // leakage between sibling tests that share one process.
1563
- _resetProxyForceUpdateStateForTesting: function () {
1564
- _proxyForceUpdateInFlight = false;
1565
- _proxyForceUpdateLastAttemptAt = 0;
1566
- },
1567
- },
1568
- };