@evomap/evolver 1.90.0 → 2.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -564
- package/bin/evolver-llm-proxy.js +3 -0
- package/bin/evolver-mcp.js +2 -0
- package/bin/evolver-proxy.js +3 -0
- package/bin/evolver.js +4 -0
- package/index.js +1 -3670
- package/package.json +32 -62
- package/CONTRIBUTING.md +0 -19
- package/LICENSE +0 -641
- package/README.ja-JP.md +0 -521
- package/README.ko-KR.md +0 -520
- package/README.zh-CN.md +0 -531
- package/SKILL.md +0 -365
- package/assets/cover.png +0 -0
- package/assets/gep/genes.seed.json +0 -496
- package/conformance/savings-core/constants.json +0 -30
- package/conformance/savings-core/golden-vectors.json +0 -333
- package/scripts/a2a_export.js +0 -63
- package/scripts/a2a_ingest.js +0 -79
- package/scripts/a2a_promote.js +0 -118
- package/scripts/analyze_by_skill.js +0 -121
- package/scripts/build_binaries.js +0 -479
- package/scripts/check-changelog.js +0 -166
- package/scripts/extract_log.js +0 -85
- package/scripts/generate_history.js +0 -75
- package/scripts/gep_append_event.js +0 -96
- package/scripts/gep_personality_report.js +0 -234
- package/scripts/human_report.js +0 -147
- package/scripts/recall-verify-report.js +0 -234
- package/scripts/recover_loop.js +0 -61
- package/scripts/refresh_stars_badge.js +0 -168
- package/scripts/seed-merchants.js +0 -91
- package/scripts/skill2recipes.js +0 -118
- package/scripts/suggest_version.js +0 -89
- package/scripts/validate-modules.js +0 -38
- package/scripts/validate-suite.js +0 -78
- package/src/adapters/claudeCode.js +0 -194
- package/src/adapters/codex.js +0 -216
- package/src/adapters/cursor.js +0 -91
- package/src/adapters/hookAdapter.js +0 -469
- package/src/adapters/kiro.js +0 -195
- package/src/adapters/opencode.js +0 -326
- package/src/adapters/scripts/_lockPaths.js +0 -74
- package/src/adapters/scripts/_memoryFiltering.js +0 -35
- package/src/adapters/scripts/_runtimePaths.js +0 -440
- package/src/adapters/scripts/evolver-session-end.js +0 -321
- package/src/adapters/scripts/evolver-session-start.js +0 -587
- package/src/adapters/scripts/evolver-signal-detect.js +0 -98
- package/src/adapters/scripts/evolver-task-recall.js +0 -173
- package/src/atp/atpExecute.js +0 -283
- package/src/atp/atpTaskPickup.js +0 -233
- package/src/atp/autoBuyer.js +0 -382
- package/src/atp/autoDeliver.js +0 -215
- package/src/atp/cli.js +0 -354
- package/src/atp/cliAutobuyPrompt.js +0 -154
- package/src/atp/consumerAgent.js +0 -157
- package/src/atp/defaultHandler.js +0 -69
- package/src/atp/heartbeatSignalsHandler.js +0 -254
- package/src/atp/hubClient.js +0 -317
- package/src/atp/index.js +0 -38
- package/src/atp/merchantAgent.js +0 -118
- package/src/atp/protocol.js +0 -41
- package/src/atp/questionComposer.js +0 -133
- package/src/atp/serviceHelper.js +0 -92
- package/src/canary.js +0 -13
- package/src/config.js +0 -351
- package/src/evolve/guards.js +0 -1
- package/src/evolve/pipeline/collect.js +0 -1
- package/src/evolve/pipeline/dispatch.js +0 -1
- package/src/evolve/pipeline/enrich.js +0 -1
- package/src/evolve/pipeline/hub.js +0 -1
- package/src/evolve/pipeline/select.js +0 -1
- package/src/evolve/pipeline/signals.js +0 -1
- package/src/evolve/utils.js +0 -1
- package/src/evolve.js +0 -1
- package/src/experiment/agentRunner.js +0 -229
- package/src/experiment/cli.js +0 -159
- package/src/experiment/comparison.js +0 -233
- package/src/experiment/metrics.js +0 -75
- package/src/forceUpdate.js +0 -991
- package/src/gep/a2a.js +0 -173
- package/src/gep/a2aProtocol.js +0 -1
- package/src/gep/analyzer.js +0 -35
- package/src/gep/antiAbuseTelemetry.js +0 -1
- package/src/gep/assetCallLog.js +0 -197
- package/src/gep/assetStore.js +0 -723
- package/src/gep/assets.js +0 -36
- package/src/gep/autoDistillConv.js +0 -1
- package/src/gep/autoDistillLlm.js +0 -1
- package/src/gep/bridge.js +0 -138
- package/src/gep/candidateEval.js +0 -1
- package/src/gep/candidates.js +0 -1
- package/src/gep/claimNudge.js +0 -121
- package/src/gep/cliContracts.js +0 -1190
- package/src/gep/contentHash.js +0 -1
- package/src/gep/conversationDistiller.js +0 -1
- package/src/gep/conversationSniffer.js +0 -1
- package/src/gep/crypto.js +0 -1
- package/src/gep/curriculum.js +0 -1
- package/src/gep/deviceId.js +0 -1
- package/src/gep/directoryClient.js +0 -115
- package/src/gep/envFingerprint.js +0 -1
- package/src/gep/epigenetics.js +0 -1
- package/src/gep/execBridge.js +0 -1
- package/src/gep/executionTrace.js +0 -291
- package/src/gep/explore.js +0 -1
- package/src/gep/featureFlags.js +0 -121
- package/src/gep/gitOps.js +0 -265
- package/src/gep/hash.js +0 -1
- package/src/gep/hostErrorClassifier.js +0 -34
- package/src/gep/hubFetch.js +0 -1
- package/src/gep/hubReview.js +0 -1
- package/src/gep/hubSearch.js +0 -1
- package/src/gep/hubVerify.js +0 -1
- package/src/gep/idleScheduler.js +0 -400
- package/src/gep/issueReporter.js +0 -416
- package/src/gep/learningSignals.js +0 -1
- package/src/gep/llmReview.js +0 -92
- package/src/gep/localStateAwareness.js +0 -243
- package/src/gep/mailboxTransport.js +0 -119
- package/src/gep/memoryGraph.js +0 -1
- package/src/gep/memoryGraphAdapter.js +0 -1
- package/src/gep/mutation.js +0 -1
- package/src/gep/narrativeMemory.js +0 -1
- package/src/gep/oauthLogin.js +0 -181
- package/src/gep/openPRRegistry.js +0 -1
- package/src/gep/paths.js +0 -522
- package/src/gep/personality.js +0 -1
- package/src/gep/policyCheck.js +0 -1
- package/src/gep/portable.js +0 -103
- package/src/gep/privacyClient.js +0 -235
- package/src/gep/prompt.js +0 -1
- package/src/gep/questionGenerator.js +0 -518
- package/src/gep/recallInject.js +0 -1
- package/src/gep/recallVerifier.js +0 -1
- package/src/gep/reflection.js +0 -1
- package/src/gep/sanitize.js +0 -264
- package/src/gep/savingsCore.js +0 -1
- package/src/gep/schemas/capsule.js +0 -170
- package/src/gep/schemas/gene.js +0 -154
- package/src/gep/schemas/index.js +0 -8
- package/src/gep/schemas/protocol.js +0 -51
- package/src/gep/schemas/task.js +0 -74
- package/src/gep/selector.js +0 -1
- package/src/gep/selfPR.js +0 -469
- package/src/gep/signals.js +0 -776
- package/src/gep/skill2gep.js +0 -1056
- package/src/gep/skill2gepAudit.js +0 -303
- package/src/gep/skill2recipes.js +0 -511
- package/src/gep/skillDistiller.js +0 -1
- package/src/gep/skillPublisher.js +0 -358
- package/src/gep/solidify.js +0 -1
- package/src/gep/strategy.js +0 -1
- package/src/gep/taskReceiver.js +0 -575
- package/src/gep/tokenSavings.js +0 -1
- package/src/gep/trajectoryExport.js +0 -1
- package/src/gep/validationReport.js +0 -55
- package/src/gep/validator/index.js +0 -411
- package/src/gep/validator/reporter.js +0 -210
- package/src/gep/validator/sandboxExecutor.js +0 -480
- package/src/gep/validator/stakeBootstrap.js +0 -357
- package/src/gep/workspaceKeychain.js +0 -1
- package/src/ops/cleanup.js +0 -80
- package/src/ops/commentary.js +0 -60
- package/src/ops/health_check.js +0 -104
- package/src/ops/index.js +0 -11
- package/src/ops/innovation.js +0 -67
- package/src/ops/lifecycle.js +0 -798
- package/src/ops/self_repair.js +0 -76
- package/src/ops/skills_monitor.js +0 -147
- package/src/ops/trigger.js +0 -33
- package/src/proxy/clientSettings.js +0 -405
- package/src/proxy/envelope.js +0 -59
- package/src/proxy/extensions/dmHandler.js +0 -45
- package/src/proxy/extensions/sessionHandler.js +0 -141
- package/src/proxy/extensions/skillUpdater.js +0 -64
- package/src/proxy/extensions/traceControl.js +0 -1
- package/src/proxy/index.js +0 -1395
- package/src/proxy/inject.js +0 -1
- package/src/proxy/lifecycle/manager.js +0 -1568
- package/src/proxy/mailbox/state.js +0 -207
- package/src/proxy/mailbox/store.js +0 -602
- package/src/proxy/router/cache_passthrough.js +0 -26
- package/src/proxy/router/features.js +0 -84
- package/src/proxy/router/gemini_route.js +0 -154
- package/src/proxy/router/messages_route.js +0 -535
- package/src/proxy/router/model_router.js +0 -113
- package/src/proxy/router/models_route.js +0 -52
- package/src/proxy/router/ollama_route.js +0 -103
- package/src/proxy/router/responses_route.js +0 -170
- package/src/proxy/router/vertex_route.js +0 -110
- package/src/proxy/server/http.js +0 -363
- package/src/proxy/server/routes.js +0 -558
- package/src/proxy/server/settings.js +0 -115
- package/src/proxy/sync/engine.js +0 -179
- package/src/proxy/sync/inbound.js +0 -211
- package/src/proxy/sync/outbound.js +0 -361
- package/src/proxy/task/monitor.js +0 -131
- package/src/proxy/trace/extractor.js +0 -1
- package/src/proxy/trace/usage.js +0 -1
- package/src/solo/breaker.js +0 -25
- package/src/solo/gitGuard.js +0 -65
- package/src/webui/client/clientJs/assets.js +0 -111
- package/src/webui/client/clientJs/bootstrap.js +0 -92
- package/src/webui/client/clientJs/common.js +0 -77
- package/src/webui/client/clientJs/i18n.js +0 -366
- package/src/webui/client/clientJs/index.js +0 -35
- package/src/webui/client/clientJs/interactions.js +0 -351
- package/src/webui/client/clientJs/overview.js +0 -152
- package/src/webui/client/clientJs/personality.js +0 -285
- package/src/webui/client/clientJs/pipelines.js +0 -330
- package/src/webui/client/indexHtml.js +0 -221
- package/src/webui/client/static.js +0 -23
- package/src/webui/client/stylesCss.js +0 -639
- package/src/webui/client/vendor/README.md +0 -15
- package/src/webui/client/vendor/echarts.min.js +0 -45
- package/src/webui/index.js +0 -14
- package/src/webui/observer/assets.js +0 -146
- package/src/webui/observer/index.js +0 -37
- package/src/webui/observer/interactions.js +0 -127
- package/src/webui/observer/jsonl.js +0 -75
- package/src/webui/observer/paths.js +0 -46
- package/src/webui/observer/personality.js +0 -43
- package/src/webui/observer/pipelineEvents.js +0 -58
- package/src/webui/observer/redact.js +0 -63
- package/src/webui/observer/runs.js +0 -356
- package/src/webui/observer/safety.js +0 -57
- package/src/webui/observer/skills.js +0 -70
- package/src/webui/observer/status.js +0 -71
- package/src/webui/server/http.js +0 -138
- package/src/webui/server/routes.js +0 -41
|
@@ -1,1568 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
const fs = require('fs');
|
|
4
|
-
const path = require('path');
|
|
5
|
-
const { PROXY_PROTOCOL_VERSION } = require('../mailbox/store');
|
|
6
|
-
const { readMailboxStateFile } = require('../mailbox/state');
|
|
7
|
-
const { buildEnvelope } = require('../envelope');
|
|
8
|
-
const crypto = require('crypto');
|
|
9
|
-
const {
|
|
10
|
-
hubFetch,
|
|
11
|
-
hubUnreachableBackoffMs,
|
|
12
|
-
isHubUnreachableError,
|
|
13
|
-
readHubResponseJson,
|
|
14
|
-
readHubResponseText,
|
|
15
|
-
sanitizeHubResponseForLog,
|
|
16
|
-
throwIfHubUnreachableResponse,
|
|
17
|
-
} = require('../../gep/hubFetch');
|
|
18
|
-
const { getEvomapPath } = require('../../gep/paths');
|
|
19
|
-
// last_update transit (PR #188): proxy heartbeat ferries a pending
|
|
20
|
-
// force_update outcome to the hub, then clears the state file on 2xx.
|
|
21
|
-
// Proxy DOES run the upgrade now (PR #188 follow-up, HIGH bug): the
|
|
22
|
-
// original comment "Proxy itself never runs the upgrade — telemetry-only
|
|
23
|
-
// here" reflected pre-fix behaviour. Pure proxy-mode nodes (EVOMAP_PROXY=1,
|
|
24
|
-
// no evolve loop) never traversed a2aProtocol.js sendHeartbeat, so the
|
|
25
|
-
// canonical `_maybeTriggerForceUpdateFromHeartbeat` block at
|
|
26
|
-
// a2aProtocol.js:2304 never fired for them — Hub could push force_update
|
|
27
|
-
// forever with no upgrade attempt and no EvolverUpgradeAttempt row. The
|
|
28
|
-
// proxy heartbeat (200 with force_update, AND 426 with force_update in the
|
|
29
|
-
// error envelope) must mirror that logic. reportForceUpdateOutcome writes
|
|
30
|
-
// the state file the next heartbeat will pick up via body.last_update.
|
|
31
|
-
const {
|
|
32
|
-
readPendingLastUpdate,
|
|
33
|
-
clearLastUpdateOnAck,
|
|
34
|
-
reportForceUpdateOutcome,
|
|
35
|
-
} = require('../../gep/a2aProtocol');
|
|
36
|
-
|
|
37
|
-
// Hub's nodeId regex; mirror of src/gep/a2aProtocol.js so a malformed
|
|
38
|
-
// legacy file can never feed garbage into the hello payload.
|
|
39
|
-
const NODE_ID_RE = /^node_[a-f0-9]{12,32}$/;
|
|
40
|
-
|
|
41
|
-
const DEFAULT_HEARTBEAT_INTERVAL = 360_000;
|
|
42
|
-
// Heartbeat backoff ceiling. Was 30min; reporter (#544) showed that
|
|
43
|
-
// a single transient failure could park the loop at 30min and feel
|
|
44
|
-
// indistinguishable from a daemon that had crashed. 15min sits above
|
|
45
|
-
// `DEFAULT_HEARTBEAT_INTERVAL` (6min) so the `interval * 2^failures`
|
|
46
|
-
// growth still has headroom — capping below the interval would invert
|
|
47
|
-
// the backoff and make failures retry FASTER than success ticks.
|
|
48
|
-
// 15min ≈ 2.5× the default, giving one full doubling step before park.
|
|
49
|
-
const HEARTBEAT_BACKOFF_CAP_MS = 15 * 60_000;
|
|
50
|
-
const HELLO_TIMEOUT = 15_000;
|
|
51
|
-
const HEARTBEAT_TIMEOUT = 10_000;
|
|
52
|
-
const MAX_REAUTH_ATTEMPTS = 2;
|
|
53
|
-
// First failure = 2 min, subsequent consecutive failures double up to ~4h.
|
|
54
|
-
// Aligned with a2aProtocol.js Round-9 reduction (was 30 min, caused
|
|
55
|
-
// "idle-death" for proxy-mode users: one benign 401 silenced the node for 30
|
|
56
|
-
// min, triggering stagnation kills and manual restart loops).
|
|
57
|
-
const REAUTH_BACKOFF_BASE_MS = 2 * 60_000;
|
|
58
|
-
const REAUTH_BACKOFF_MAX_MS = 4 * 60 * 60_000;
|
|
59
|
-
|
|
60
|
-
// Wall-clock drift detector tunables. Mirrors DRIFT_CHECK_MS /
|
|
61
|
-
// DRIFT_SLEEP_THRESHOLD_MS / DRIFT_LONG_SLEEP_THRESHOLD_MS in
|
|
62
|
-
// src/gep/a2aProtocol.js. setTimeout / setInterval fire on libuv's
|
|
63
|
-
// monotonic clock, which freezes while the host is suspended -- so a
|
|
64
|
-
// laptop closed for hours and reopened would not trigger any heartbeat
|
|
65
|
-
// tick until the next scheduled time, which under exponential backoff
|
|
66
|
-
// can sit at HEARTBEAT_BACKOFF_CAP_MS (15 min). Sampling Date.now()
|
|
67
|
-
// (wall clock) every DRIFT_CHECK_MS lets us detect the jump and
|
|
68
|
-
// immediately poke the heartbeat so recovery does not have to wait for
|
|
69
|
-
// the next natural tick. Long-sleep gap also clears reauth backoff:
|
|
70
|
-
// hub-side state we cached is almost certainly stale after a 30min+
|
|
71
|
-
// suspend, so force a clean retry path on wake instead of carrying the
|
|
72
|
-
// pre-sleep penalty through. R10 (#544).
|
|
73
|
-
const DRIFT_CHECK_MS = 30 * 1000;
|
|
74
|
-
const DRIFT_SLEEP_THRESHOLD_MS = 90 * 1000;
|
|
75
|
-
const DRIFT_LONG_SLEEP_THRESHOLD_MS = 30 * 60_000;
|
|
76
|
-
|
|
77
|
-
// Heartbeat-driven force_update lifecycle tracking. Mirrors
|
|
78
|
-
// `_forceUpdateInFlight` / `_forceUpdateLastAttemptAt` /
|
|
79
|
-
// `_getForceUpdateRetryCooldownMs` in src/gep/a2aProtocol.js so the proxy
|
|
80
|
-
// path uses the same in-flight + cooldown contract as the canonical path.
|
|
81
|
-
// Module-level (not instance-level) so multiple LifecycleManager instances
|
|
82
|
-
// in the same process serialize through one upgrade attempt — matches
|
|
83
|
-
// a2aProtocol.js's module-level guard. Process-local is sufficient: the
|
|
84
|
-
// proxy daemon runs in a single process and any sibling process would
|
|
85
|
-
// have its own require-cached state; cross-process serialization is the
|
|
86
|
-
// hub's job via directive_id dedup, not the client's.
|
|
87
|
-
let _proxyForceUpdateInFlight = false;
|
|
88
|
-
let _proxyForceUpdateLastAttemptAt = 0;
|
|
89
|
-
function _getProxyForceUpdateRetryCooldownMs() {
|
|
90
|
-
// Share the env var with a2aProtocol.js: an operator who sets
|
|
91
|
-
// EVOLVER_FORCE_UPDATE_RETRY_COOLDOWN_MS=0 in a test or production tune
|
|
92
|
-
// expects BOTH code paths to honour it. Default 15min matches
|
|
93
|
-
// a2aProtocol.js exactly.
|
|
94
|
-
const v = Number(process.env.EVOLVER_FORCE_UPDATE_RETRY_COOLDOWN_MS);
|
|
95
|
-
if (Number.isFinite(v) && v >= 0) return v;
|
|
96
|
-
return 15 * 60 * 1000;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
let _cachedFingerprint = null;
|
|
100
|
-
function _getEnvFingerprint() {
|
|
101
|
-
if (_cachedFingerprint) return _cachedFingerprint;
|
|
102
|
-
try {
|
|
103
|
-
const { captureEnvFingerprint } = require('../../gep/envFingerprint');
|
|
104
|
-
_cachedFingerprint = captureEnvFingerprint();
|
|
105
|
-
} catch {
|
|
106
|
-
_cachedFingerprint = {
|
|
107
|
-
platform: process.platform,
|
|
108
|
-
arch: process.arch,
|
|
109
|
-
node_version: process.version,
|
|
110
|
-
};
|
|
111
|
-
}
|
|
112
|
-
return _cachedFingerprint;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
// Recover a node_id persisted by the legacy GEP path
|
|
116
|
-
// (`src/gep/a2aProtocol.js` writes ~/.evomap/node_id, falling back to
|
|
117
|
-
// `<install>/.evomap_node_id` when the home dir isn't writable). Without
|
|
118
|
-
// this fallback, a daemon whose MailboxStore was created AFTER the legacy
|
|
119
|
-
// GEP file (any install upgrading from pre-lifecycle to lifecycle, or any
|
|
120
|
-
// state.json wiped without also wiping the legacy file) mints a fresh
|
|
121
|
-
// `node_${randomBytes(6)}` identity in hello(), which the hub registers
|
|
122
|
-
// as a *new* A2ANode under the same owner — the original (with stake,
|
|
123
|
-
// reputation, aliases) gets silently abandoned. Mirror the writer's two
|
|
124
|
-
// candidates in the same order as `_loadPersistedNodeId` so both code
|
|
125
|
-
// paths land on the single identity.
|
|
126
|
-
//
|
|
127
|
-
// Resolve both paths on every call:
|
|
128
|
-
// - getEvomapPath() reads EVOLVER_HOME (and falls through to os.homedir())
|
|
129
|
-
// at call time, so tests and privileged-drop daemons can flip the
|
|
130
|
-
// resolved location without monkey-patching globals.
|
|
131
|
-
// - The install-root path uses __dirname so it's stable across cwd changes.
|
|
132
|
-
function _readLegacyNodeId() {
|
|
133
|
-
const candidates = [
|
|
134
|
-
getEvomapPath('node_id'),
|
|
135
|
-
path.resolve(__dirname, '..', '..', '..', '.evomap_node_id'),
|
|
136
|
-
];
|
|
137
|
-
for (const file of candidates) {
|
|
138
|
-
try {
|
|
139
|
-
if (!fs.existsSync(file)) continue;
|
|
140
|
-
const raw = fs.readFileSync(file, 'utf8').trim();
|
|
141
|
-
if (NODE_ID_RE.test(raw)) return raw;
|
|
142
|
-
} catch {
|
|
143
|
-
// Unreadable / racing writer — try the next location.
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
return null;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
// Mirror of src/gep/a2aProtocol.js `_persistNodeId`. Pure-proxy daemons
|
|
150
|
-
// (EVOMAP_PROXY=1, no a2aProtocol heartbeat thread) mint their own
|
|
151
|
-
// node_id and ONLY persist it to MailboxStore state.json. The legacy
|
|
152
|
-
// `~/.evomap/node_id` file never gets written, so:
|
|
153
|
-
//
|
|
154
|
-
// 1. `_shortNodeIdForStatePath` in a2aProtocol.js (used by the proxy
|
|
155
|
-
// heartbeat to pick the per-node `force_update_last.<suffix>.json`
|
|
156
|
-
// path) falls all the way through to 'anon' — every proxy node on
|
|
157
|
-
// the same EVOLVER_HOME would collide on the same state file.
|
|
158
|
-
// 2. A mixed-mode install (legacy evolve loop ran once, then user
|
|
159
|
-
// switched to proxy mode) is even worse: the legacy file holds a
|
|
160
|
-
// DIFFERENT id than the one in MailboxStore. The proxy heartbeats
|
|
161
|
-
// with body.node_id = its OWN id while writing
|
|
162
|
-
// `force_update_last.<legacy-suffix>.json`. The hub-side upgrade
|
|
163
|
-
// attempt row gets attributed to the wrong node.
|
|
164
|
-
//
|
|
165
|
-
// Calling this helper from hello() after the nodeId is resolved unifies
|
|
166
|
-
// the two persistence paths onto a single identity. Atomic write
|
|
167
|
-
// (per-pid tmp + rename) mirrors `_persistNodeSecret` in a2aProtocol.js;
|
|
168
|
-
// 0o600 mode keeps the file owner-read-only on POSIX (silently ignored
|
|
169
|
-
// on Windows, where %USERPROFILE% isolation is the only protection).
|
|
170
|
-
//
|
|
171
|
-
// Idempotent: if the file already holds the same id, we skip the write
|
|
172
|
-
// to avoid an inode churn on every hello tick. If it holds a DIFFERENT
|
|
173
|
-
// valid id, we still overwrite — the proxy's MailboxStore wins because
|
|
174
|
-
// that is the id the hub already knows us by (any rotation away from
|
|
175
|
-
// the legacy id was a deliberate operator action). The only way to
|
|
176
|
-
// re-seed a legacy id back onto a proxy install is to clear
|
|
177
|
-
// MailboxStore state.json (`evolver reset-local-secret`).
|
|
178
|
-
function _persistLegacyNodeId(id) {
|
|
179
|
-
if (!id || !NODE_ID_RE.test(id)) return;
|
|
180
|
-
const targets = [
|
|
181
|
-
getEvomapPath('node_id'),
|
|
182
|
-
path.resolve(__dirname, '..', '..', '..', '.evomap_node_id'),
|
|
183
|
-
];
|
|
184
|
-
// Try targets in order until one succeeds, matching the read order in
|
|
185
|
-
// _readLegacyNodeId. We only need ONE persistent copy; once the home
|
|
186
|
-
// path takes the write, the install-root path is unused.
|
|
187
|
-
for (const file of targets) {
|
|
188
|
-
try {
|
|
189
|
-
// Skip if the file already matches — common steady-state path,
|
|
190
|
-
// saves a syscall storm under heartbeat backoff doubling.
|
|
191
|
-
try {
|
|
192
|
-
if (fs.existsSync(file)) {
|
|
193
|
-
const existing = fs.readFileSync(file, 'utf8').trim();
|
|
194
|
-
if (existing === id) return;
|
|
195
|
-
}
|
|
196
|
-
} catch {
|
|
197
|
-
// Unreadable -- treat as missing and try to write.
|
|
198
|
-
}
|
|
199
|
-
const dir = path.dirname(file);
|
|
200
|
-
try {
|
|
201
|
-
if (!fs.existsSync(dir)) {
|
|
202
|
-
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
203
|
-
}
|
|
204
|
-
} catch (_) {
|
|
205
|
-
// mkdir failed (read-only fs, EPERM under sandboxing). Skip
|
|
206
|
-
// this candidate; the next one (install-root .evomap_node_id)
|
|
207
|
-
// may still work.
|
|
208
|
-
continue;
|
|
209
|
-
}
|
|
210
|
-
// Atomic write: a sibling evolver process (mixed-mode upgrade, two
|
|
211
|
-
// proxy daemons started by hand) could otherwise race on this
|
|
212
|
-
// path and leave a half-written file. Matches the pattern in
|
|
213
|
-
// a2aProtocol.js `_persistNodeSecret`.
|
|
214
|
-
const tmp = file + '.' + process.pid + '.tmp';
|
|
215
|
-
fs.writeFileSync(tmp, id, { encoding: 'utf8', mode: 0o600 });
|
|
216
|
-
fs.renameSync(tmp, file);
|
|
217
|
-
return;
|
|
218
|
-
} catch {
|
|
219
|
-
// Best-effort: continue to the next candidate. If both fail (no
|
|
220
|
-
// home, no writable install root) we accept the legacy file is
|
|
221
|
-
// unavailable — the proxy will still function, it just cannot
|
|
222
|
-
// unify state-file suffixes with a co-resident a2aProtocol path.
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
// Heartbeat-driven force_update trigger for proxy-mode nodes. Mirrors
|
|
228
|
-
// `_maybeTriggerForceUpdateFromHeartbeat` in src/gep/a2aProtocol.js
|
|
229
|
-
// (search there for that name to compare). Pure proxy-mode deployments
|
|
230
|
-
// (EVOMAP_PROXY=1) never run the evolve run() loop or sendHeartbeat, so
|
|
231
|
-
// without this trigger Hub can push force_update on every heartbeat
|
|
232
|
-
// forever and the node will keep heartbeating on the old version — which
|
|
233
|
-
// is exactly what shipped before PR #188's H1 fix.
|
|
234
|
-
//
|
|
235
|
-
// Drives `executeForceUpdate` directly, gated by an in-flight lock + a
|
|
236
|
-
// cooldown on failures so we do not hammer npm/degit on every tick. After
|
|
237
|
-
// the attempt, persists the outcome via `reportForceUpdateOutcome` so the
|
|
238
|
-
// next heartbeat carries it as body.last_update — that's the path that
|
|
239
|
-
// finally writes a row to the hub's EvolverUpgradeAttempt table.
|
|
240
|
-
//
|
|
241
|
-
// Logger is injected (not a console fallback) so tests can capture the
|
|
242
|
-
// upgrade-path stderr without polluting CI output. The logger contract
|
|
243
|
-
// matches what the LifecycleManager already uses.
|
|
244
|
-
function _maybeTriggerForceUpdateFromHeartbeat(forceUpdate, logger) {
|
|
245
|
-
if (!forceUpdate || typeof forceUpdate !== 'object') return;
|
|
246
|
-
if (_proxyForceUpdateInFlight) return;
|
|
247
|
-
const nowMs = Date.now();
|
|
248
|
-
if (
|
|
249
|
-
_proxyForceUpdateLastAttemptAt &&
|
|
250
|
-
(nowMs - _proxyForceUpdateLastAttemptAt) < _getProxyForceUpdateRetryCooldownMs()
|
|
251
|
-
) {
|
|
252
|
-
// A recent attempt already ran and either succeeded (process exited
|
|
253
|
-
// and we wouldn't be here on the post-restart heartbeat — see the
|
|
254
|
-
// FORCE_UPDATE_NOOP path in forceUpdate.js / reportForceUpdateOutcome
|
|
255
|
-
// status="skipped") or failed. Back off.
|
|
256
|
-
return;
|
|
257
|
-
}
|
|
258
|
-
_proxyForceUpdateInFlight = true;
|
|
259
|
-
_proxyForceUpdateLastAttemptAt = nowMs;
|
|
260
|
-
|
|
261
|
-
// Capture from_version BEFORE executeForceUpdate runs. A successful
|
|
262
|
-
// upgrade calls process.exit(78); the post-restart heartbeat reads the
|
|
263
|
-
// state file from a fresh process where require('package.json').version
|
|
264
|
-
// is the NEW version — so snapshot the CURRENTLY running version now.
|
|
265
|
-
let fromVersion = '';
|
|
266
|
-
try {
|
|
267
|
-
fromVersion = String((require('../../../package.json') || {}).version || '');
|
|
268
|
-
} catch (_) { /* best-effort */ }
|
|
269
|
-
|
|
270
|
-
// Kick off in a microtask so the heartbeat promise chain can still
|
|
271
|
-
// complete (log + return {ok:true}) before the long-running upgrade
|
|
272
|
-
// takes over the process. Matches a2aProtocol.js exactly.
|
|
273
|
-
Promise.resolve().then(() => {
|
|
274
|
-
let updated = false;
|
|
275
|
-
let noop = false;
|
|
276
|
-
let busy = false;
|
|
277
|
-
let thrownErr = null;
|
|
278
|
-
// Structured failure object ({ ok:false, code, detail }) when the upgrader
|
|
279
|
-
// RETURNED a failure; forwarded to reportForceUpdateOutcome so the hub gets
|
|
280
|
-
// the precise branch code instead of "executeForceUpdate returned false".
|
|
281
|
-
// Hoisted out of the try because `result` is block-scoped there.
|
|
282
|
-
let failureResult = null;
|
|
283
|
-
try {
|
|
284
|
-
const mod = require('../../forceUpdate');
|
|
285
|
-
const result = mod.executeForceUpdate(forceUpdate);
|
|
286
|
-
// Sentinel === comparison: executeForceUpdate returns the
|
|
287
|
-
// FORCE_UPDATE_NOOP symbol when the install is already at the
|
|
288
|
-
// required version. We must NOT treat that as "success" — doing so
|
|
289
|
-
// would (a) write a phantom {status:"success", from==to} row to
|
|
290
|
-
// EvolverUpgradeAttempt, and (b) trigger an exit(78) restart with
|
|
291
|
-
// nothing to restart for. The hub schema accepts status="skipped".
|
|
292
|
-
noop = (result === mod.FORCE_UPDATE_NOOP);
|
|
293
|
-
// FORCE_UPDATE_BUSY: another caller (e.g. a2aProtocol heartbeat
|
|
294
|
-
// trigger or an evolve tick) already holds the module-level
|
|
295
|
-
// _inFlight mutex in forceUpdate.js. Defensive only — the
|
|
296
|
-
// instance-level _proxyForceUpdateInFlight gate above and the
|
|
297
|
-
// single-caller property of pure proxy mode make BUSY unreachable
|
|
298
|
-
// in practice. If it does fire (mixed-mode regression, future
|
|
299
|
-
// additional caller, etc.), the other caller owns the telemetry:
|
|
300
|
-
// we MUST NOT write a state file or exit(78). Mirrors
|
|
301
|
-
// src/gep/a2aProtocol.js (search FORCE_UPDATE_BUSY).
|
|
302
|
-
busy = (result === mod.FORCE_UPDATE_BUSY);
|
|
303
|
-
updated = (result === true);
|
|
304
|
-
// Inline the failure-shape check rather than calling mod.isForceUpdateFailure:
|
|
305
|
-
// keeps this robust against partial test mocks of forceUpdate that stub
|
|
306
|
-
// executeForceUpdate but omit the helper (a missing-function throw here
|
|
307
|
-
// would otherwise demote a real success to "failed").
|
|
308
|
-
failureResult = (result && typeof result === 'object' && result.ok === false) ? result : null;
|
|
309
|
-
} catch (e) {
|
|
310
|
-
thrownErr = e;
|
|
311
|
-
try {
|
|
312
|
-
logger.warn(`[ForceUpdate] proxy heartbeat-trigger failed (non-fatal): ${e && e.message || e}`);
|
|
313
|
-
} catch (_) { /* logger broken; non-fatal */ }
|
|
314
|
-
updated = false;
|
|
315
|
-
} finally {
|
|
316
|
-
_proxyForceUpdateInFlight = false;
|
|
317
|
-
}
|
|
318
|
-
if (busy) {
|
|
319
|
-
_proxyForceUpdateLastAttemptAt = 0;
|
|
320
|
-
try {
|
|
321
|
-
logger.log('[ForceUpdate] proxy heartbeat-trigger observed BUSY (concurrent invocation). Skipping telemetry; in-flight caller owns the outcome.');
|
|
322
|
-
} catch (_) { /* logger broken; non-fatal */ }
|
|
323
|
-
return;
|
|
324
|
-
}
|
|
325
|
-
// Persist outcome via the shared helper so the heartbeat-thread
|
|
326
|
-
// trigger and the proxy trigger stay in lockstep on payload assembly
|
|
327
|
-
// + validation. The next heartbeat reads this file and ferries it as
|
|
328
|
-
// body.last_update — same contract as the canonical path.
|
|
329
|
-
try {
|
|
330
|
-
reportForceUpdateOutcome(forceUpdate, {
|
|
331
|
-
updated: updated,
|
|
332
|
-
noop: noop,
|
|
333
|
-
error: thrownErr,
|
|
334
|
-
failure: failureResult,
|
|
335
|
-
fromVersion: fromVersion,
|
|
336
|
-
});
|
|
337
|
-
} catch (e) {
|
|
338
|
-
try {
|
|
339
|
-
logger.warn(`[ForceUpdate] proxy reportForceUpdateOutcome failed (non-fatal): ${e && e.message || e}`);
|
|
340
|
-
} catch (_) { /* logger broken; non-fatal */ }
|
|
341
|
-
}
|
|
342
|
-
if (updated) {
|
|
343
|
-
try { logger.log('[ForceUpdate] Update complete (proxy heartbeat-trigger). Exiting for restart...'); } catch (_) {}
|
|
344
|
-
try { process.exit(78); } catch (_) {}
|
|
345
|
-
} else if (noop) {
|
|
346
|
-
_proxyForceUpdateLastAttemptAt = 0;
|
|
347
|
-
try {
|
|
348
|
-
logger.log('[ForceUpdate] No-op (proxy heartbeat-trigger): already at required version. Skipping restart.');
|
|
349
|
-
} catch (_) {}
|
|
350
|
-
} else {
|
|
351
|
-
try {
|
|
352
|
-
logger.warn('[ForceUpdate] proxy heartbeat-trigger failed. Will retry after cooldown (' +
|
|
353
|
-
Math.round(_getProxyForceUpdateRetryCooldownMs() / 60000) + 'min).');
|
|
354
|
-
} catch (_) {}
|
|
355
|
-
}
|
|
356
|
-
});
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
class AuthError extends Error {
|
|
360
|
-
constructor(message, statusCode) {
|
|
361
|
-
super(message);
|
|
362
|
-
this.name = 'AuthError';
|
|
363
|
-
this.statusCode = statusCode;
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
function parseNodeSecretVersion(value) {
|
|
368
|
-
const n = Number(value);
|
|
369
|
-
return Number.isInteger(n) && n > 0 ? n : null;
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
const NODE_SECRET_RE = /^[a-f0-9]{64}$/i;
|
|
373
|
-
const NODE_SECRET_SUPPRESSION_RE = /^sha256:[a-f0-9]{64}$/i;
|
|
374
|
-
const SECRET_DIVERGENCE_ERROR = 'secret_diverged_cleared';
|
|
375
|
-
const SECRET_DIVERGENCE_REASON_CODES = [
|
|
376
|
-
'node_secret_invalid',
|
|
377
|
-
'invalid_secret',
|
|
378
|
-
'rotation_requires_current_secret',
|
|
379
|
-
];
|
|
380
|
-
|
|
381
|
-
function validNodeSecret(value) {
|
|
382
|
-
return typeof value === 'string' && NODE_SECRET_RE.test(value);
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
function getEnvNodeSecret() {
|
|
386
|
-
return ((process.env.A2A_NODE_SECRET || process.env.EVOMAP_NODE_SECRET || '').trim() || null);
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
function fingerprintNodeSecret(secret) {
|
|
390
|
-
if (!validNodeSecret(secret)) return null;
|
|
391
|
-
const normalized = String(secret).trim().toLowerCase();
|
|
392
|
-
return 'sha256:' + crypto.createHash('sha256').update(normalized).digest('hex');
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
function truthyState(value) {
|
|
396
|
-
const v = String(value || '').trim().toLowerCase();
|
|
397
|
-
return v === '1' || v === 'true' || v === 'yes' || v === 'on';
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
function matchSecretDivergenceReason(value) {
|
|
401
|
-
const reason = String(value || '').trim().toLowerCase();
|
|
402
|
-
if (!reason) return null;
|
|
403
|
-
return SECRET_DIVERGENCE_REASON_CODES.find((code) => reason.includes(code)) || null;
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
function extractSecretDivergenceReason(data) {
|
|
407
|
-
const candidates = [
|
|
408
|
-
data?.payload?.reason,
|
|
409
|
-
data?.payload?.error,
|
|
410
|
-
data?.reason,
|
|
411
|
-
data?.error,
|
|
412
|
-
];
|
|
413
|
-
for (const value of candidates) {
|
|
414
|
-
const reason = matchSecretDivergenceReason(value);
|
|
415
|
-
if (reason) return reason;
|
|
416
|
-
}
|
|
417
|
-
return null;
|
|
418
|
-
}
|
|
419
|
-
|
|
420
|
-
function isSecretDivergenceError(error) {
|
|
421
|
-
return error === SECRET_DIVERGENCE_ERROR || Boolean(matchSecretDivergenceReason(error));
|
|
422
|
-
}
|
|
423
|
-
|
|
424
|
-
function safeHubErrorCode(data, statusCode) {
|
|
425
|
-
const candidates = [
|
|
426
|
-
data?.error,
|
|
427
|
-
data?.reason,
|
|
428
|
-
data?.payload?.error,
|
|
429
|
-
data?.payload?.reason,
|
|
430
|
-
];
|
|
431
|
-
for (const value of candidates) {
|
|
432
|
-
if (typeof value !== 'string') continue;
|
|
433
|
-
const code = value.trim();
|
|
434
|
-
if (/^[a-z][a-z0-9_:-]{0,79}$/i.test(code)) return code;
|
|
435
|
-
}
|
|
436
|
-
return `http_${statusCode}`;
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
function normalizeNodeSecretEnvSuppression(value) {
|
|
440
|
-
const raw = String(value || '').trim();
|
|
441
|
-
if (!raw) return null;
|
|
442
|
-
if (truthyState(raw)) return 'true';
|
|
443
|
-
const lower = raw.toLowerCase();
|
|
444
|
-
return NODE_SECRET_SUPPRESSION_RE.test(lower) ? lower : null;
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
class LifecycleManager {
|
|
448
|
-
constructor({ hubUrl, store, logger, getTaskMeta } = {}) {
|
|
449
|
-
this.hubUrl = (hubUrl || process.env.A2A_HUB_URL || '').replace(/\/+$/, '');
|
|
450
|
-
this.store = store;
|
|
451
|
-
this.logger = logger || console;
|
|
452
|
-
this.getTaskMeta = getTaskMeta || null;
|
|
453
|
-
this._heartbeatTimer = null;
|
|
454
|
-
this._running = false;
|
|
455
|
-
this._startedAt = null;
|
|
456
|
-
this._lastHeartbeatTickAt = 0;
|
|
457
|
-
this._consecutiveFailures = 0;
|
|
458
|
-
this._reauthInProgress = false;
|
|
459
|
-
this._helloRateLimitUntil = 0;
|
|
460
|
-
this._reauthBackoffUntil = 0;
|
|
461
|
-
this._consecutiveReauthFailures = 0;
|
|
462
|
-
this._driftInterval = null;
|
|
463
|
-
this._lastDriftCheckAt = 0;
|
|
464
|
-
this._hubUnreachableFailures = 0;
|
|
465
|
-
this._hubUnreachableUntil = 0;
|
|
466
|
-
this._envSecretSuppressionMarker = normalizeNodeSecretEnvSuppression(this.store && this.store.getState
|
|
467
|
-
? this.store.getState('node_secret_env_suppressed')
|
|
468
|
-
: null);
|
|
469
|
-
this._envSuppressionClearedForSecret = null;
|
|
470
|
-
this._suppressEnvSecret = Boolean(this._envSecretSuppressionMarker && getEnvNodeSecret());
|
|
471
|
-
|
|
472
|
-
// H4 fix: persist the legacy node_id file as soon as the in-memory
|
|
473
|
-
// node_id is known, NOT only after a successful hello(). The original
|
|
474
|
-
// code persisted only in hello() (see ~L390-398) — but proxy-mode boot
|
|
475
|
-
// can fire `reportForceUpdateOutcome` BEFORE hello() returns:
|
|
476
|
-
//
|
|
477
|
-
// - First-tick heartbeat hits 426 → executeForceUpdate() → exit(78)
|
|
478
|
-
// all happens BEFORE the hello() response is processed.
|
|
479
|
-
// - enrich.js force_update path can fire during the same window.
|
|
480
|
-
//
|
|
481
|
-
// `_shortNodeIdForStatePath` in a2aProtocol.js then picks the
|
|
482
|
-
// state-file suffix from `_cachedNodeId` (never set in proxy mode —
|
|
483
|
-
// only getNodeId() sets it, and proxy never calls getNodeId()) or the
|
|
484
|
-
// legacy ~/.evomap/node_id file. With both empty, it falls through
|
|
485
|
-
// to 'anon', and the outcome lands at `force_update_last.anon.json`.
|
|
486
|
-
// Next boot's hello() writes the real id, the heartbeat reads
|
|
487
|
-
// `force_update_last.<8hex>.json`, the anon file is orphaned and the
|
|
488
|
-
// outcome is silently lost.
|
|
489
|
-
//
|
|
490
|
-
// Persisting at construction closes the window. _persistLegacyNodeId
|
|
491
|
-
// early-returns on invalid input (NODE_ID_RE gate, same regex as the
|
|
492
|
-
// hello() path uses) so a malformed/empty store value is a no-op,
|
|
493
|
-
// and it is idempotent on matching content so the cost is one
|
|
494
|
-
// existsSync + one readFileSync per construction. We keep the
|
|
495
|
-
// existing post-hello call as a safety net in case hello() mints or
|
|
496
|
-
// mutates the id.
|
|
497
|
-
try {
|
|
498
|
-
const earlyNodeId = this.store && this.store.getState
|
|
499
|
-
? this.store.getState('node_id')
|
|
500
|
-
: null;
|
|
501
|
-
if (earlyNodeId && NODE_ID_RE.test(earlyNodeId)) {
|
|
502
|
-
_persistLegacyNodeId(earlyNodeId);
|
|
503
|
-
}
|
|
504
|
-
} catch (e) {
|
|
505
|
-
// Best-effort: persistence failure must never break construction.
|
|
506
|
-
// Logger may not exist if tests passed undefined; guard the call.
|
|
507
|
-
try {
|
|
508
|
-
this.logger.warn(
|
|
509
|
-
`[lifecycle] early persist of legacy node_id failed (non-fatal): ${e && e.message || e}`
|
|
510
|
-
);
|
|
511
|
-
} catch (_) { /* logger broken; non-fatal */ }
|
|
512
|
-
}
|
|
513
|
-
}
|
|
514
|
-
|
|
515
|
-
get nodeId() {
|
|
516
|
-
return this.store.getState('node_id');
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
get nodeSecret() {
|
|
520
|
-
return this._resolveNodeSecret();
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
get nodeSecretVersion() {
|
|
524
|
-
this._resolveNodeSecret();
|
|
525
|
-
const storeVersion = parseNodeSecretVersion(this.store.getState('node_secret_version'));
|
|
526
|
-
const storeSecret = this.store.getState('node_secret') || null;
|
|
527
|
-
const storeSource = this.store.getState('node_secret_source') || null;
|
|
528
|
-
const envSecret = this._effectiveEnvNodeSecret();
|
|
529
|
-
const envVersion = parseNodeSecretVersion(process.env.A2A_NODE_SECRET_VERSION || process.env.EVOMAP_NODE_SECRET_VERSION);
|
|
530
|
-
const validStoreSecret = validNodeSecret(storeSecret);
|
|
531
|
-
if (this._suppressEnvSecret) return validStoreSecret ? storeVersion : null;
|
|
532
|
-
if (storeSource === 'hub_rotate' && validStoreSecret) return storeVersion;
|
|
533
|
-
if (envSecret) {
|
|
534
|
-
if (envVersion) return envVersion;
|
|
535
|
-
return storeSecret === envSecret ? storeVersion : null;
|
|
536
|
-
}
|
|
537
|
-
return validStoreSecret ? storeVersion : null;
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
/**
|
|
541
|
-
* Resolve the active node_secret with conflict reconciliation between the
|
|
542
|
-
* persistent MailboxStore and `process.env.A2A_NODE_SECRET`.
|
|
543
|
-
*
|
|
544
|
-
* Two opposite failure modes shape this logic:
|
|
545
|
-
*
|
|
546
|
-
* #529 (env-fresh, store-stale): operator exports a freshly minted
|
|
547
|
-
* secret in A2A_NODE_SECRET (e.g. from .env), but the MailboxStore
|
|
548
|
-
* still holds a long-stale value. The store value would otherwise
|
|
549
|
-
* win and produce a 403 -> rotate -> 30-min backoff loop.
|
|
550
|
-
*
|
|
551
|
-
* "store-fresh, env-stale": process A rotates the secret via /a2a/hello,
|
|
552
|
-
* so the store holds the value the hub now recognises. Process A then
|
|
553
|
-
* restarts (typical: daemon respawn after upgrade or crash). The shell
|
|
554
|
-
* it inherits its env from still exports the *previous* value of
|
|
555
|
-
* A2A_NODE_SECRET. Without source-tracking we would treat this as
|
|
556
|
-
* env-vs-store conflict, env-wins, and silently overwrite the
|
|
557
|
-
* hub-recognised secret with a stale shell value -- exactly the loop
|
|
558
|
-
* #529 was meant to fix, just symmetrical.
|
|
559
|
-
*
|
|
560
|
-
* Resolution: track *who wrote* the store value. When the hub returns a
|
|
561
|
-
* rotated secret (`hello`), we tag the store entry with
|
|
562
|
-
* `node_secret_source = 'hub_rotate'`. On conflict we honour that tag:
|
|
563
|
-
*
|
|
564
|
-
* source=hub_rotate -> store wins (recent rotation; env is stale)
|
|
565
|
-
* source missing/'env_seed' -> env wins (legacy / first-boot bootstrap)
|
|
566
|
-
*
|
|
567
|
-
* Single-source mode (only one of store/env present) is unchanged.
|
|
568
|
-
* @returns {string|null}
|
|
569
|
-
*/
|
|
570
|
-
_resolveNodeSecret() {
|
|
571
|
-
const envSecret = this._effectiveEnvNodeSecret();
|
|
572
|
-
const envUpdatedAfterSuppression = this._envSuppressionClearedForSecret === envSecret;
|
|
573
|
-
const storeSecret = this.store.getState('node_secret') || null;
|
|
574
|
-
const storeSource = this.store.getState('node_secret_source') || null;
|
|
575
|
-
|
|
576
|
-
if (
|
|
577
|
-
envSecret &&
|
|
578
|
-
envUpdatedAfterSuppression &&
|
|
579
|
-
validNodeSecret(envSecret) &&
|
|
580
|
-
(!storeSecret || envSecret === storeSecret) &&
|
|
581
|
-
!(storeSecret && envSecret === storeSecret && storeSource === 'hub_rotate')
|
|
582
|
-
) {
|
|
583
|
-
this._syncEnvNodeSecretToStore(envSecret);
|
|
584
|
-
return envSecret;
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
if (envSecret && storeSecret && envSecret !== storeSecret) {
|
|
588
|
-
// Store value came from a successful hub rotation -> trust it.
|
|
589
|
-
// The env var is necessarily stale: it was captured by the parent
|
|
590
|
-
// shell before the rotation and a child process cannot mutate it
|
|
591
|
-
// back into its parent.
|
|
592
|
-
if (storeSource === 'hub_rotate' && validNodeSecret(storeSecret) && !envUpdatedAfterSuppression) {
|
|
593
|
-
if (!this._storeSourceLogged) {
|
|
594
|
-
this._storeSourceLogged = true;
|
|
595
|
-
this.logger.warn(
|
|
596
|
-
'[lifecycle] A2A_NODE_SECRET env var differs from MailboxStore; ' +
|
|
597
|
-
'store value originated from a hub rotation, treating env as stale. ' +
|
|
598
|
-
'After a manual web reset, run `evolver reset-local-secret` to clear local ' +
|
|
599
|
-
'secret and env suppression state before relying only on updated env vars.'
|
|
600
|
-
);
|
|
601
|
-
}
|
|
602
|
-
return storeSecret;
|
|
603
|
-
}
|
|
604
|
-
if (validNodeSecret(envSecret)) {
|
|
605
|
-
const diskRotated = this._latestDiskHubRotatedSecret();
|
|
606
|
-
if (diskRotated) {
|
|
607
|
-
this._setNodeSecretState({
|
|
608
|
-
secret: diskRotated.secret,
|
|
609
|
-
version: diskRotated.version ? String(diskRotated.version) : '',
|
|
610
|
-
source: 'hub_rotate',
|
|
611
|
-
envSuppressed: this.store.getState('node_secret_env_suppressed') || '',
|
|
612
|
-
});
|
|
613
|
-
if (!this._storeSourceLogged) {
|
|
614
|
-
this._storeSourceLogged = true;
|
|
615
|
-
this.logger.warn(
|
|
616
|
-
'[lifecycle] MailboxStore memory differs from disk hub-rotated node_secret; ' +
|
|
617
|
-
'using disk value and treating env as stale.'
|
|
618
|
-
);
|
|
619
|
-
}
|
|
620
|
-
return diskRotated.secret;
|
|
621
|
-
}
|
|
622
|
-
this._syncEnvNodeSecretToStore(envSecret);
|
|
623
|
-
if (!this._envOverrideLogged) {
|
|
624
|
-
this._envOverrideLogged = true;
|
|
625
|
-
this.logger.warn(
|
|
626
|
-
'[lifecycle] A2A_NODE_SECRET env var differs from MailboxStore; using env value and syncing store. ' +
|
|
627
|
-
'Clear ~/.evomap/mailbox/state.json or unset A2A_NODE_SECRET to silence this warning.'
|
|
628
|
-
);
|
|
629
|
-
}
|
|
630
|
-
return envSecret;
|
|
631
|
-
}
|
|
632
|
-
// env var malformed -- ignore it, fall back to store
|
|
633
|
-
return storeSecret;
|
|
634
|
-
}
|
|
635
|
-
|
|
636
|
-
return storeSecret || envSecret || null;
|
|
637
|
-
}
|
|
638
|
-
|
|
639
|
-
_latestDiskHubRotatedSecret() {
|
|
640
|
-
const file = this.store && this.store._stateFile;
|
|
641
|
-
if (!file) return null;
|
|
642
|
-
const state = readMailboxStateFile(file);
|
|
643
|
-
const secret = typeof state?.node_secret === 'string' ? state.node_secret.trim() : null;
|
|
644
|
-
if (!validNodeSecret(secret) || state?.node_secret_source !== 'hub_rotate') return null;
|
|
645
|
-
return {
|
|
646
|
-
secret,
|
|
647
|
-
version: parseNodeSecretVersion(state.node_secret_version),
|
|
648
|
-
};
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
_syncEnvNodeSecretToStore(envSecret) {
|
|
652
|
-
const envVersion = parseNodeSecretVersion(process.env.A2A_NODE_SECRET_VERSION || process.env.EVOMAP_NODE_SECRET_VERSION);
|
|
653
|
-
this._setNodeSecretState({
|
|
654
|
-
secret: envSecret,
|
|
655
|
-
version: envVersion ? String(envVersion) : '',
|
|
656
|
-
source: 'env_seed',
|
|
657
|
-
envSuppressed: '',
|
|
658
|
-
});
|
|
659
|
-
this._clearEnvSecretSuppression();
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
_setNodeSecretState({ secret = '', version = '', source = '', envSuppressed = '' } = {}) {
|
|
663
|
-
if (this.store && typeof this.store.setNodeSecretState === 'function') {
|
|
664
|
-
this.store.setNodeSecretState({ secret, version, source, envSuppressed });
|
|
665
|
-
return;
|
|
666
|
-
}
|
|
667
|
-
this.store.setState('node_secret', secret);
|
|
668
|
-
this.store.setState('node_secret_version', version ? String(version) : '');
|
|
669
|
-
this.store.setState('node_secret_source', source);
|
|
670
|
-
this.store.setState('node_secret_env_suppressed', envSuppressed);
|
|
671
|
-
}
|
|
672
|
-
|
|
673
|
-
_effectiveEnvNodeSecret() {
|
|
674
|
-
const envSecret = getEnvNodeSecret();
|
|
675
|
-
return this._isEnvNodeSecretSuppressed(envSecret) ? null : envSecret;
|
|
676
|
-
}
|
|
677
|
-
|
|
678
|
-
_isEnvNodeSecretSuppressed(envSecret) {
|
|
679
|
-
this._envSuppressionClearedForSecret = null;
|
|
680
|
-
const persistedMarker = normalizeNodeSecretEnvSuppression(this.store && this.store.getState
|
|
681
|
-
? this.store.getState('node_secret_env_suppressed')
|
|
682
|
-
: null);
|
|
683
|
-
const marker = persistedMarker || this._envSecretSuppressionMarker;
|
|
684
|
-
if (!marker) {
|
|
685
|
-
this._envSecretSuppressionMarker = null;
|
|
686
|
-
this._suppressEnvSecret = false;
|
|
687
|
-
return false;
|
|
688
|
-
}
|
|
689
|
-
this._envSecretSuppressionMarker = marker;
|
|
690
|
-
if (marker === 'true') {
|
|
691
|
-
this._suppressEnvSecret = Boolean(envSecret);
|
|
692
|
-
return Boolean(envSecret);
|
|
693
|
-
}
|
|
694
|
-
const envFingerprint = fingerprintNodeSecret(envSecret);
|
|
695
|
-
if (!envFingerprint) {
|
|
696
|
-
this._suppressEnvSecret = false;
|
|
697
|
-
return false;
|
|
698
|
-
}
|
|
699
|
-
if (envFingerprint === marker) {
|
|
700
|
-
this._suppressEnvSecret = true;
|
|
701
|
-
return true;
|
|
702
|
-
}
|
|
703
|
-
this._clearEnvSecretSuppression();
|
|
704
|
-
this._envSuppressionClearedForSecret = envSecret;
|
|
705
|
-
return false;
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
_markCurrentEnvSecretSuppressed() {
|
|
709
|
-
const marker = fingerprintNodeSecret(getEnvNodeSecret());
|
|
710
|
-
if (!marker) {
|
|
711
|
-
this._clearEnvSecretSuppression();
|
|
712
|
-
return;
|
|
713
|
-
}
|
|
714
|
-
try { this.store.setState('node_secret_env_suppressed', marker); } catch { /* best-effort */ }
|
|
715
|
-
this._envSecretSuppressionMarker = marker;
|
|
716
|
-
this._suppressEnvSecret = true;
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
_clearEnvSecretSuppression() {
|
|
720
|
-
try { this.store.setState('node_secret_env_suppressed', ''); } catch { /* best-effort */ }
|
|
721
|
-
this._envSecretSuppressionMarker = null;
|
|
722
|
-
this._suppressEnvSecret = false;
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
_buildHeaders() {
|
|
726
|
-
const headers = { 'Content-Type': 'application/json' };
|
|
727
|
-
const secret = this.nodeSecret;
|
|
728
|
-
if (secret) headers['Authorization'] = 'Bearer ' + secret;
|
|
729
|
-
const secretVersion = this.nodeSecretVersion;
|
|
730
|
-
if (secretVersion) headers['X-EvoMap-Node-Secret-Version'] = String(secretVersion);
|
|
731
|
-
headers['x-correlation-id'] = crypto.randomUUID();
|
|
732
|
-
return headers;
|
|
733
|
-
}
|
|
734
|
-
|
|
735
|
-
_hubUnreachableWaitMs() {
|
|
736
|
-
return Math.max(0, this._hubUnreachableUntil - Date.now());
|
|
737
|
-
}
|
|
738
|
-
|
|
739
|
-
_recordHubReachable() {
|
|
740
|
-
this._hubUnreachableFailures = 0;
|
|
741
|
-
this._hubUnreachableUntil = 0;
|
|
742
|
-
}
|
|
743
|
-
|
|
744
|
-
_recordHubUnreachable(err) {
|
|
745
|
-
this._hubUnreachableFailures += 1;
|
|
746
|
-
const retryAfterMs = hubUnreachableBackoffMs(this._hubUnreachableFailures);
|
|
747
|
-
this._hubUnreachableUntil = Date.now() + retryAfterMs;
|
|
748
|
-
this.logger.warn?.(
|
|
749
|
-
`[lifecycle] Hub unreachable; backing off for ${Math.ceil(retryAfterMs / 1000)}s: ` +
|
|
750
|
-
`${err && err.message || err}`
|
|
751
|
-
);
|
|
752
|
-
return retryAfterMs;
|
|
753
|
-
}
|
|
754
|
-
|
|
755
|
-
async hello({ rotateSecret = false } = {}) {
|
|
756
|
-
if (!this.hubUrl) return { ok: false, error: 'no_hub_url' };
|
|
757
|
-
|
|
758
|
-
if (this._helloRateLimitUntil > Date.now()) {
|
|
759
|
-
const waitSec = Math.ceil((this._helloRateLimitUntil - Date.now()) / 1000);
|
|
760
|
-
this.logger.warn(`[lifecycle] hello suppressed: rate limited for ${waitSec}s`);
|
|
761
|
-
return { ok: false, error: 'hello_rate_limit_active', waitSec };
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
const waitMs = this._hubUnreachableWaitMs();
|
|
765
|
-
if (waitMs > 0) {
|
|
766
|
-
return {
|
|
767
|
-
ok: false,
|
|
768
|
-
error: 'hub_unreachable_backoff',
|
|
769
|
-
retryAfterMs: waitMs,
|
|
770
|
-
};
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
const endpoint = `${this.hubUrl}/a2a/hello`;
|
|
774
|
-
const nodeId = this.store.getState('node_id')
|
|
775
|
-
|| _readLegacyNodeId()
|
|
776
|
-
|| `node_${crypto.randomBytes(6).toString('hex')}`;
|
|
777
|
-
|
|
778
|
-
const payload = { capabilities: {} };
|
|
779
|
-
if (rotateSecret) payload.rotate_secret = true;
|
|
780
|
-
|
|
781
|
-
const fp = _getEnvFingerprint();
|
|
782
|
-
|
|
783
|
-
const body = {
|
|
784
|
-
...buildEnvelope('hello', payload, nodeId),
|
|
785
|
-
env_fingerprint: fp,
|
|
786
|
-
};
|
|
787
|
-
|
|
788
|
-
try {
|
|
789
|
-
const res = await hubFetch(endpoint, {
|
|
790
|
-
method: 'POST',
|
|
791
|
-
headers: this._buildHeaders(),
|
|
792
|
-
body: JSON.stringify(body),
|
|
793
|
-
signal: AbortSignal.timeout(HELLO_TIMEOUT),
|
|
794
|
-
});
|
|
795
|
-
await throwIfHubUnreachableResponse(res, 'lifecycle hello');
|
|
796
|
-
this._recordHubReachable();
|
|
797
|
-
if (!res.ok) {
|
|
798
|
-
const errText = await readHubResponseText(res).catch(() => '');
|
|
799
|
-
let errData = {};
|
|
800
|
-
try { errData = errText ? JSON.parse(errText) : {}; } catch (_) { /* non-JSON API error */ }
|
|
801
|
-
const errMsg = safeHubErrorCode(errData, res.status);
|
|
802
|
-
if (res.status === 429) {
|
|
803
|
-
const retryAfter = parseInt(res.headers.get('retry-after') || '3600', 10);
|
|
804
|
-
this._helloRateLimitUntil = Date.now() + retryAfter * 1000;
|
|
805
|
-
this.logger.error(`[lifecycle] hello rate limited (429): retry after ${retryAfter}s`);
|
|
806
|
-
return { ok: false, error: 'hello_rate_limited', retryAfter };
|
|
807
|
-
}
|
|
808
|
-
const divergenceReason = (res.status === 401 || res.status === 403)
|
|
809
|
-
? extractSecretDivergenceReason(errData)
|
|
810
|
-
: null;
|
|
811
|
-
if (divergenceReason) {
|
|
812
|
-
this.logger.warn(
|
|
813
|
-
`[lifecycle] hello rejected secret rotation (${res.status}, reason=${divergenceReason}); ` +
|
|
814
|
-
'local secret divergence detected'
|
|
815
|
-
);
|
|
816
|
-
return {
|
|
817
|
-
ok: false,
|
|
818
|
-
error: SECRET_DIVERGENCE_ERROR,
|
|
819
|
-
reason: divergenceReason,
|
|
820
|
-
statusCode: res.status,
|
|
821
|
-
};
|
|
822
|
-
}
|
|
823
|
-
const safeErrText = errText ? sanitizeHubResponseForLog(errText) : errMsg;
|
|
824
|
-
this.logger.error(`[lifecycle] hello HTTP ${res.status}: ${safeErrText}`);
|
|
825
|
-
return { ok: false, error: errMsg, statusCode: res.status };
|
|
826
|
-
}
|
|
827
|
-
|
|
828
|
-
const data = await readHubResponseJson(res);
|
|
829
|
-
|
|
830
|
-
if (data?.payload?.status === 'rejected') {
|
|
831
|
-
const divergenceReason = extractSecretDivergenceReason(data);
|
|
832
|
-
if (divergenceReason) {
|
|
833
|
-
this.logger.warn(
|
|
834
|
-
`[lifecycle] hello rejected secret rotation (reason=${divergenceReason}); ` +
|
|
835
|
-
'local secret divergence detected'
|
|
836
|
-
);
|
|
837
|
-
return {
|
|
838
|
-
ok: false,
|
|
839
|
-
error: SECRET_DIVERGENCE_ERROR,
|
|
840
|
-
reason: divergenceReason,
|
|
841
|
-
response: data,
|
|
842
|
-
};
|
|
843
|
-
}
|
|
844
|
-
const reason = data.payload.reason || 'unknown';
|
|
845
|
-
const safeReason = sanitizeHubResponseForLog(reason);
|
|
846
|
-
this.logger.error(`[lifecycle] hello rejected: ${safeReason}`);
|
|
847
|
-
const error = String(reason || '').startsWith('node_id_already_claimed')
|
|
848
|
-
? reason
|
|
849
|
-
: 'hello_rejected';
|
|
850
|
-
return { ok: false, error, response: data };
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
const secret = data?.payload?.node_secret || data?.node_secret || null;
|
|
854
|
-
const secretVersion = parseNodeSecretVersion(data?.payload?.node_secret_version || data?.node_secret_version);
|
|
855
|
-
if (secret && validNodeSecret(secret)) {
|
|
856
|
-
// Hub just handed us a fresh secret. Whatever sits in
|
|
857
|
-
// A2A_NODE_SECRET is now older than the store, so suppress the
|
|
858
|
-
// env-wins reconciliation in _resolveNodeSecret for the rest of
|
|
859
|
-
// this process. Without this, the very next _buildHeaders call
|
|
860
|
-
// (e.g. the verification heartbeat in reAuthenticate) would see
|
|
861
|
-
// env vs store as a conflict, treat the env value as authoritative,
|
|
862
|
-
// and overwrite the freshly rotated secret with the stale one,
|
|
863
|
-
// re-creating the auth loop the previous patch fixed (see #529
|
|
864
|
-
// and the Bugbot review on PR #22).
|
|
865
|
-
if (getEnvNodeSecret() && getEnvNodeSecret() !== secret) {
|
|
866
|
-
this._markCurrentEnvSecretSuppressed();
|
|
867
|
-
} else {
|
|
868
|
-
this._clearEnvSecretSuppression();
|
|
869
|
-
}
|
|
870
|
-
this._setNodeSecretState({
|
|
871
|
-
secret,
|
|
872
|
-
version: secretVersion ? String(secretVersion) : '',
|
|
873
|
-
source: 'hub_rotate',
|
|
874
|
-
envSuppressed: this.store.getState('node_secret_env_suppressed') || '',
|
|
875
|
-
});
|
|
876
|
-
this.logger.log('[lifecycle] new node_secret stored from hello response');
|
|
877
|
-
} else if (secretVersion) {
|
|
878
|
-
this.store.setState('node_secret_version', String(secretVersion));
|
|
879
|
-
} else {
|
|
880
|
-
this.store.setState('node_secret_version', '');
|
|
881
|
-
}
|
|
882
|
-
|
|
883
|
-
this.store.setState('node_id', nodeId);
|
|
884
|
-
// Unify proxy node_id with the legacy GEP file. Without this, the
|
|
885
|
-
// proxy-only fast path (EVOMAP_PROXY=1) never seeds
|
|
886
|
-
// ~/.evomap/node_id and `_shortNodeIdForStatePath` in a2aProtocol
|
|
887
|
-
// (used to pick the per-node `force_update_last.<suffix>.json`
|
|
888
|
-
// path for upgrade telemetry) falls through to 'anon' — every
|
|
889
|
-
// proxy node under the same EVOLVER_HOME would collide on the
|
|
890
|
-
// same state file. In a mixed-mode install where the legacy file
|
|
891
|
-
// holds a DIFFERENT (stale) id, the helper overwrites it so the
|
|
892
|
-
// state-file suffix matches `this.nodeId` — the id the hub sees
|
|
893
|
-
// in body.node_id. We persist AFTER hello succeeds so a rejected
|
|
894
|
-
// first-boot mint never commits to disk; on a rejection the next
|
|
895
|
-
// tick will mint fresh again (existing behaviour).
|
|
896
|
-
try {
|
|
897
|
-
_persistLegacyNodeId(nodeId);
|
|
898
|
-
} catch (e) {
|
|
899
|
-
// Best-effort: persistence failure must never break hello. Log
|
|
900
|
-
// and move on — the proxy still functions, the state-file
|
|
901
|
-
// suffix just falls back to 'anon' until the next successful
|
|
902
|
-
// hello retries the write.
|
|
903
|
-
this.logger.warn(`[lifecycle] failed to persist legacy node_id (non-fatal): ${e && e.message || e}`);
|
|
904
|
-
}
|
|
905
|
-
this.logger.log(`[lifecycle] hello OK, node_id=${nodeId}${rotateSecret ? ' (secret rotated)' : ''}`);
|
|
906
|
-
return { ok: true, nodeId, response: data };
|
|
907
|
-
} catch (err) {
|
|
908
|
-
if (isHubUnreachableError(err)) {
|
|
909
|
-
const retryAfterMs = this._recordHubUnreachable(err);
|
|
910
|
-
return {
|
|
911
|
-
ok: false,
|
|
912
|
-
error: 'hub_unreachable',
|
|
913
|
-
detail: err.message,
|
|
914
|
-
retryAfterMs,
|
|
915
|
-
};
|
|
916
|
-
}
|
|
917
|
-
this.logger.error(`[lifecycle] hello failed: ${err.message}`);
|
|
918
|
-
return { ok: false, error: err.message };
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
|
-
|
|
922
|
-
/**
|
|
923
|
-
* Re-authenticate after 403: rotate secret via hello, then verify with a
|
|
924
|
-
* heartbeat. Returns true if auth is restored, false otherwise.
|
|
925
|
-
*
|
|
926
|
-
* Recovery sequence (issue EvoMap/evolver#529):
|
|
927
|
-
* attempt 1 -> hello with current Bearer + rotate_secret=true
|
|
928
|
-
* (works when the stale secret is still recognised)
|
|
929
|
-
* attempt 2 -> drop the bearer locally, hello WITHOUT Authorization
|
|
930
|
-
* + rotate_secret=true. If the node is owned by someone
|
|
931
|
-
* else, hub returns node_id_already_claimed; we surface a
|
|
932
|
-
* manual-reset hint to the user instead of churning forever.
|
|
933
|
-
*/
|
|
934
|
-
async reAuthenticate() {
|
|
935
|
-
if (this._reauthInProgress) return false;
|
|
936
|
-
if (this._reauthBackoffUntil > Date.now()) {
|
|
937
|
-
const waitSec = Math.ceil((this._reauthBackoffUntil - Date.now()) / 1000);
|
|
938
|
-
this.logger.warn(`[lifecycle] re-auth suppressed: backoff active for ${waitSec}s`);
|
|
939
|
-
return false;
|
|
940
|
-
}
|
|
941
|
-
this._reauthInProgress = true;
|
|
942
|
-
let manualResetRequired = false;
|
|
943
|
-
let hubUnreachable = false;
|
|
944
|
-
let droppedDivergedSecret = false;
|
|
945
|
-
try {
|
|
946
|
-
for (let attempt = 1; attempt <= MAX_REAUTH_ATTEMPTS; attempt++) {
|
|
947
|
-
this.logger.warn(`[lifecycle] re-auth attempt ${attempt}/${MAX_REAUTH_ATTEMPTS}: rotating secret via hello...`);
|
|
948
|
-
const helloResult = await this.hello({ rotateSecret: true });
|
|
949
|
-
if (!helloResult.ok) {
|
|
950
|
-
this.logger.error(`[lifecycle] re-auth hello failed: ${helloResult.error}`);
|
|
951
|
-
if (helloResult.error === 'hello_rate_limited' || helloResult.error === 'hello_rate_limit_active') break;
|
|
952
|
-
// Hub link is down (WAF HTML, network error, timeout) -- NOT an auth
|
|
953
|
-
// failure. Bail without arming the re-auth backoff: otherwise a
|
|
954
|
-
// transient outage that arrives mid-rotate would burn both attempts
|
|
955
|
-
// (attempt 2's hello short-circuits on the hub-unreachable window)
|
|
956
|
-
// and suppress genuine auth recovery for up to REAUTH_BACKOFF_MAX_MS,
|
|
957
|
-
// even though a JSON 401/403 retry would succeed once the hub is
|
|
958
|
-
// reachable again. The hub-unreachable window already gates re-entry.
|
|
959
|
-
if (helloResult.error === 'hub_unreachable' || helloResult.error === 'hub_unreachable_backoff') {
|
|
960
|
-
hubUnreachable = true;
|
|
961
|
-
break;
|
|
962
|
-
}
|
|
963
|
-
if (isSecretDivergenceError(helloResult.error)) {
|
|
964
|
-
// The hub has explicitly rejected our current secret as diverged
|
|
965
|
-
// from its record. Drop store/env-derived auth once, then spend
|
|
966
|
-
// the second attempt on an unauthenticated rotate hello instead
|
|
967
|
-
// of backing off while still presenting the same stale Bearer.
|
|
968
|
-
if (attempt < MAX_REAUTH_ATTEMPTS && !droppedDivergedSecret) {
|
|
969
|
-
this._dropLocalSecret(SECRET_DIVERGENCE_ERROR);
|
|
970
|
-
droppedDivergedSecret = true;
|
|
971
|
-
continue;
|
|
972
|
-
}
|
|
973
|
-
break;
|
|
974
|
-
}
|
|
975
|
-
if (typeof helloResult.error === 'string' && helloResult.error.startsWith('node_id_already_claimed')) {
|
|
976
|
-
// Hub does not believe we own this nodeId. Our locally cached
|
|
977
|
-
// secret(s) are useless. Drop them so attempt 2 retries WITHOUT
|
|
978
|
-
// a Bearer (lenient hello path). If even unauthenticated rotate
|
|
979
|
-
// is rejected, only a manual reset can recover.
|
|
980
|
-
if (attempt < MAX_REAUTH_ATTEMPTS) {
|
|
981
|
-
this._dropLocalSecret('node_id_already_claimed');
|
|
982
|
-
continue;
|
|
983
|
-
}
|
|
984
|
-
manualResetRequired = true;
|
|
985
|
-
break;
|
|
986
|
-
}
|
|
987
|
-
continue;
|
|
988
|
-
}
|
|
989
|
-
const newSecret = helloResult.response?.payload?.node_secret;
|
|
990
|
-
if (!newSecret) {
|
|
991
|
-
this.logger.error('[lifecycle] re-auth: hub did not return a new secret (rotate may not have taken effect)');
|
|
992
|
-
break;
|
|
993
|
-
}
|
|
994
|
-
const hbResult = await this.heartbeat({ _skipReauth: true });
|
|
995
|
-
if (hbResult.ok) {
|
|
996
|
-
this.logger.log('[lifecycle] re-auth succeeded: heartbeat confirmed with new secret');
|
|
997
|
-
this._consecutiveReauthFailures = 0;
|
|
998
|
-
// Note: _envOverrideLogged is intentionally NOT reset here.
|
|
999
|
-
// The successful hello path above already set _suppressEnvSecret=true,
|
|
1000
|
-
// which means _resolveNodeSecret will never hit the env-vs-store
|
|
1001
|
-
// conflict branch again in this process, so the warning would never
|
|
1002
|
-
// fire a second time anyway. Resetting the flag was misleading.
|
|
1003
|
-
return true;
|
|
1004
|
-
}
|
|
1005
|
-
// Same as the hello path: if the verification heartbeat fails because
|
|
1006
|
-
// the hub link dropped (not an auth rejection), defer without arming
|
|
1007
|
-
// the re-auth backoff.
|
|
1008
|
-
if (hbResult.error === 'hub_unreachable' || hbResult.error === 'hub_unreachable_backoff') {
|
|
1009
|
-
hubUnreachable = true;
|
|
1010
|
-
break;
|
|
1011
|
-
}
|
|
1012
|
-
this.logger.warn(`[lifecycle] re-auth attempt ${attempt}: heartbeat still failing after rotate`);
|
|
1013
|
-
}
|
|
1014
|
-
if (hubUnreachable) {
|
|
1015
|
-
this.logger.warn('[lifecycle] re-auth deferred: hub unreachable (no re-auth backoff armed)');
|
|
1016
|
-
return false;
|
|
1017
|
-
}
|
|
1018
|
-
if (manualResetRequired) {
|
|
1019
|
-
this._emitManualResetNeeded();
|
|
1020
|
-
}
|
|
1021
|
-
this._consecutiveReauthFailures += 1;
|
|
1022
|
-
const backoffMs = Math.min(
|
|
1023
|
-
REAUTH_BACKOFF_BASE_MS * Math.pow(2, this._consecutiveReauthFailures - 1),
|
|
1024
|
-
REAUTH_BACKOFF_MAX_MS
|
|
1025
|
-
);
|
|
1026
|
-
const backoffMin = Math.round(backoffMs / 60_000);
|
|
1027
|
-
this.logger.error(
|
|
1028
|
-
`[lifecycle] re-auth exhausted all attempts (failure #${this._consecutiveReauthFailures}), ` +
|
|
1029
|
-
`backing off for ${backoffMin} minutes`
|
|
1030
|
-
);
|
|
1031
|
-
this._reauthBackoffUntil = Date.now() + backoffMs;
|
|
1032
|
-
return false;
|
|
1033
|
-
} finally {
|
|
1034
|
-
this._reauthInProgress = false;
|
|
1035
|
-
}
|
|
1036
|
-
}
|
|
1037
|
-
|
|
1038
|
-
/**
|
|
1039
|
-
* Drop the cached node_secret in MailboxStore AND signal the env-var path to
|
|
1040
|
-
* skip its value for this process lifetime. Used when the hub explicitly
|
|
1041
|
-
* disowns our claim.
|
|
1042
|
-
* @param {string} reason - log tag describing why we are dropping it.
|
|
1043
|
-
*/
|
|
1044
|
-
_dropLocalSecret(reason) {
|
|
1045
|
-
this.logger.warn(`[lifecycle] dropping cached node_secret (reason=${reason}); next hello will run unauthenticated`);
|
|
1046
|
-
const diskRotated = this._latestDiskHubRotatedSecret();
|
|
1047
|
-
const storeSecret = this.store.getState('node_secret') || null;
|
|
1048
|
-
if (diskRotated && diskRotated.secret !== storeSecret) {
|
|
1049
|
-
try {
|
|
1050
|
-
this._setNodeSecretState({
|
|
1051
|
-
secret: diskRotated.secret,
|
|
1052
|
-
version: diskRotated.version ? String(diskRotated.version) : '',
|
|
1053
|
-
source: 'hub_rotate',
|
|
1054
|
-
envSuppressed: this.store.getState('node_secret_env_suppressed') || '',
|
|
1055
|
-
});
|
|
1056
|
-
this.logger.warn('[lifecycle] local in-memory node_secret is stale; preserving newer disk hub-rotated secret');
|
|
1057
|
-
return;
|
|
1058
|
-
} catch (err) {
|
|
1059
|
-
const reason = err && (err.code || err.name) ? (err.code || err.name) : 'error';
|
|
1060
|
-
this.logger.warn(`[lifecycle] failed to sync newer disk hub-rotated node_secret into memory (reason=${reason})`);
|
|
1061
|
-
}
|
|
1062
|
-
}
|
|
1063
|
-
try {
|
|
1064
|
-
this._setNodeSecretState({
|
|
1065
|
-
secret: '',
|
|
1066
|
-
version: '',
|
|
1067
|
-
source: '',
|
|
1068
|
-
envSuppressed: this.store.getState('node_secret_env_suppressed') || '',
|
|
1069
|
-
});
|
|
1070
|
-
} catch (err) {
|
|
1071
|
-
const reason = err && (err.code || err.name || err.message) ? (err.code || err.name || err.message) : 'error';
|
|
1072
|
-
this.logger.warn(`[lifecycle] failed to clear local node_secret state (reason=${reason})`);
|
|
1073
|
-
}
|
|
1074
|
-
// Suppress only the exact env secret that was just proven stale. If no
|
|
1075
|
-
// env secret is present, do not leave a marker that blocks a future reset.
|
|
1076
|
-
this._markCurrentEnvSecretSuppressed();
|
|
1077
|
-
}
|
|
1078
|
-
|
|
1079
|
-
_emitManualResetNeeded() {
|
|
1080
|
-
try {
|
|
1081
|
-
this.store.writeInbound({
|
|
1082
|
-
type: 'system',
|
|
1083
|
-
priority: 'high',
|
|
1084
|
-
channel: 'evomap-hub',
|
|
1085
|
-
payload: {
|
|
1086
|
-
action: 'manual_secret_reset_required',
|
|
1087
|
-
message:
|
|
1088
|
-
'Hub disowns this node_id (node_id_already_claimed). Local node_secret and the current node secret env value are invalid. Visit https://evomap.ai/account, click "Reset Secret" on the agent card, run `evolver reset-local-secret` to clear local secret and env suppression state, then update A2A_NODE_SECRET/EVOMAP_NODE_SECRET and restart proxy. Only changing env before clearing suppression state can keep an older local marker active.',
|
|
1089
|
-
docs_url: 'https://evomap.ai/account',
|
|
1090
|
-
},
|
|
1091
|
-
});
|
|
1092
|
-
} catch (err) {
|
|
1093
|
-
this.logger.warn(`[lifecycle] failed to emit manual_secret_reset_required event: ${err.message}`);
|
|
1094
|
-
}
|
|
1095
|
-
}
|
|
1096
|
-
|
|
1097
|
-
async heartbeat({ _skipReauth = false } = {}) {
|
|
1098
|
-
// Wrap the entire body — including pre-fetch helpers (hello,
|
|
1099
|
-
// getTaskMeta, store.countPending, env_fingerprint) — in a single
|
|
1100
|
-
// try/catch. Reporter (#544) showed a single throw from any of
|
|
1101
|
-
// these (store corrupt, hello rejecting, fingerprint failing under
|
|
1102
|
-
// sandboxing) used to escape and kill `tick()`, leaving the
|
|
1103
|
-
// daemon "alive but never heartbeating". The post-fetch path is
|
|
1104
|
-
// unchanged; only the boundary moved earlier.
|
|
1105
|
-
if (!this.hubUrl) return { ok: false, error: 'no_hub_url' };
|
|
1106
|
-
try {
|
|
1107
|
-
const waitMs = this._hubUnreachableWaitMs();
|
|
1108
|
-
if (waitMs > 0) {
|
|
1109
|
-
return {
|
|
1110
|
-
ok: false,
|
|
1111
|
-
error: 'hub_unreachable_backoff',
|
|
1112
|
-
retryAfterMs: waitMs,
|
|
1113
|
-
};
|
|
1114
|
-
}
|
|
1115
|
-
|
|
1116
|
-
const nodeId = this.nodeId;
|
|
1117
|
-
if (!nodeId) {
|
|
1118
|
-
const helloResult = await this.hello();
|
|
1119
|
-
if (!helloResult.ok) return helloResult;
|
|
1120
|
-
}
|
|
1121
|
-
|
|
1122
|
-
const endpoint = `${this.hubUrl}/a2a/heartbeat`;
|
|
1123
|
-
const taskMeta = typeof this.getTaskMeta === 'function' ? this.getTaskMeta() : {};
|
|
1124
|
-
const fp = _getEnvFingerprint();
|
|
1125
|
-
const secretVersion = this.nodeSecretVersion;
|
|
1126
|
-
const body = {
|
|
1127
|
-
node_id: this.nodeId,
|
|
1128
|
-
sender_id: this.nodeId,
|
|
1129
|
-
evolver_version: fp.evolver_version || PROXY_PROTOCOL_VERSION,
|
|
1130
|
-
env_fingerprint: fp,
|
|
1131
|
-
meta: {
|
|
1132
|
-
proxy_version: PROXY_PROTOCOL_VERSION,
|
|
1133
|
-
proxy_protocol_version: PROXY_PROTOCOL_VERSION,
|
|
1134
|
-
outbound_pending: this.store.countPending({ direction: 'outbound' }),
|
|
1135
|
-
inbound_pending: this.store.countPending({ direction: 'inbound' }),
|
|
1136
|
-
...taskMeta,
|
|
1137
|
-
},
|
|
1138
|
-
};
|
|
1139
|
-
if (secretVersion) {
|
|
1140
|
-
body.node_secret_version = secretVersion;
|
|
1141
|
-
body.meta.node_secret_version = secretVersion;
|
|
1142
|
-
}
|
|
1143
|
-
|
|
1144
|
-
try {
|
|
1145
|
-
const cfg = require('../../config');
|
|
1146
|
-
if (cfg.antiAbuseTelemetryMode && cfg.antiAbuseTelemetryMode() === 'heartbeat') {
|
|
1147
|
-
const { buildHeartbeatAntiAbuseTelemetry } = require('../../gep/antiAbuseTelemetry');
|
|
1148
|
-
body.meta.anti_abuse = buildHeartbeatAntiAbuseTelemetry({
|
|
1149
|
-
source: 'evolver-proxy',
|
|
1150
|
-
nodeId: this.nodeId,
|
|
1151
|
-
envFingerprint: fp,
|
|
1152
|
-
taskMeta: body.meta,
|
|
1153
|
-
// This heartbeat is sent FROM the running proxy — ground truth,
|
|
1154
|
-
// not env sniffing (this process usually has neither EVOMAP_PROXY
|
|
1155
|
-
// nor EVOMAP_PROXY_PORT set, which would misreport false).
|
|
1156
|
-
proxyPortConfigured: true,
|
|
1157
|
-
});
|
|
1158
|
-
}
|
|
1159
|
-
} catch (e) {
|
|
1160
|
-
this.logger.warn(`[AntiAbuseTelemetry] failed to build heartbeat summary: ${e && e.message || e}`);
|
|
1161
|
-
}
|
|
1162
|
-
|
|
1163
|
-
// Attach any pending force_update outcome so the hub-side
|
|
1164
|
-
// EvolverUpgradeAttempt table gets a row. Captured in a local so the
|
|
1165
|
-
// post-2xx clear matches identity (rotation-safe — see
|
|
1166
|
-
// _clearLastUpdateStateIfMatches). Never let telemetry throw.
|
|
1167
|
-
let capturedLastUpdate = null;
|
|
1168
|
-
try {
|
|
1169
|
-
const pending = readPendingLastUpdate();
|
|
1170
|
-
if (pending) {
|
|
1171
|
-
body.last_update = pending;
|
|
1172
|
-
capturedLastUpdate = pending;
|
|
1173
|
-
}
|
|
1174
|
-
} catch (e) {
|
|
1175
|
-
this.logger.warn(`[lifecycle] readPendingLastUpdate failed (non-fatal): ${e && e.message || e}`);
|
|
1176
|
-
}
|
|
1177
|
-
|
|
1178
|
-
const res = await hubFetch(endpoint, {
|
|
1179
|
-
method: 'POST',
|
|
1180
|
-
headers: this._buildHeaders(),
|
|
1181
|
-
body: JSON.stringify(body),
|
|
1182
|
-
signal: AbortSignal.timeout(HEARTBEAT_TIMEOUT),
|
|
1183
|
-
});
|
|
1184
|
-
|
|
1185
|
-
await throwIfHubUnreachableResponse(res, 'lifecycle heartbeat');
|
|
1186
|
-
this._recordHubReachable();
|
|
1187
|
-
|
|
1188
|
-
if (res.status === 403 || res.status === 401) {
|
|
1189
|
-
this._consecutiveFailures++;
|
|
1190
|
-
const errText = await readHubResponseText(res).catch(() => '');
|
|
1191
|
-
const safeErrText = sanitizeHubResponseForLog(errText);
|
|
1192
|
-
this.logger.error(`[lifecycle] heartbeat auth failed (${res.status}): ${safeErrText}`);
|
|
1193
|
-
if (!_skipReauth) {
|
|
1194
|
-
const recovered = await this.reAuthenticate();
|
|
1195
|
-
if (recovered) {
|
|
1196
|
-
this._consecutiveFailures = 0;
|
|
1197
|
-
return { ok: true, recovered: true };
|
|
1198
|
-
}
|
|
1199
|
-
}
|
|
1200
|
-
return { ok: false, error: `auth_failed_${res.status}`, statusCode: res.status };
|
|
1201
|
-
}
|
|
1202
|
-
|
|
1203
|
-
if (!res.ok) {
|
|
1204
|
-
const errText = await readHubResponseText(res).catch(() => '');
|
|
1205
|
-
const safeErrText = sanitizeHubResponseForLog(errText);
|
|
1206
|
-
// 426 Upgrade Required: hub emits this when our evolver_version is
|
|
1207
|
-
// below the minimum version it requires. The body is JSON of shape
|
|
1208
|
-
// `{ error: 'evolver_min_version_required', force_update: {...} }`
|
|
1209
|
-
// (see hub `src/routes/a2a/_middleware.js`). Pre-fix this fell
|
|
1210
|
-
// through to the generic `http_426` error and the proxy never
|
|
1211
|
-
// attempted the upgrade — defeating the very mechanism that 426
|
|
1212
|
-
// exists to drive. Mirror the 200+force_update path: parse the
|
|
1213
|
-
// body, fire executeForceUpdate (which writes the state file via
|
|
1214
|
-
// reportForceUpdateOutcome), and let the next heartbeat carry the
|
|
1215
|
-
// attempt as body.last_update. Still return an error so the
|
|
1216
|
-
// caller's failure counter ticks and the loop backs off.
|
|
1217
|
-
if (res.status === 426) {
|
|
1218
|
-
let parsed = null;
|
|
1219
|
-
try { parsed = JSON.parse(errText); } catch (_) { /* body not JSON */ }
|
|
1220
|
-
const fu = parsed && parsed.force_update;
|
|
1221
|
-
if (fu && typeof fu === 'object') {
|
|
1222
|
-
this.logger.warn(
|
|
1223
|
-
`[lifecycle] heartbeat HTTP 426 with force_update directive (required=${
|
|
1224
|
-
fu.required_version || '?'
|
|
1225
|
-
}) — triggering executeForceUpdate`
|
|
1226
|
-
);
|
|
1227
|
-
_maybeTriggerForceUpdateFromHeartbeat(fu, this.logger);
|
|
1228
|
-
} else {
|
|
1229
|
-
this.logger.warn(
|
|
1230
|
-
`[lifecycle] heartbeat HTTP 426 without parseable force_update payload: ${safeErrText}`
|
|
1231
|
-
);
|
|
1232
|
-
}
|
|
1233
|
-
}
|
|
1234
|
-
// Hub 400 circuit breaker (mirrors a2aProtocol.js sendHeartbeat
|
|
1235
|
-
// ~L2376-2411): if last_update was attached this tick and the hub
|
|
1236
|
-
// rejected the body with 400 AND the rejection names the
|
|
1237
|
-
// last_update field, the state file is poisoning every heartbeat
|
|
1238
|
-
// (e.g. downgrade-then-upgrade left a payload the new hub schema
|
|
1239
|
-
// rejects, or a manual edit corrupted the JSON shape). The proxy
|
|
1240
|
-
// path used to lack this breaker entirely, so a single bad payload
|
|
1241
|
-
// would block telemetry forever -- every retry re-sends the same
|
|
1242
|
-
// poison and re-fails with 400. Single-strike (no counter): the
|
|
1243
|
-
// 400 + last_update substring pair is unambiguous enough that
|
|
1244
|
-
// waiting for repeats just delays recovery. Scope intentionally
|
|
1245
|
-
// narrowed to 400-only (NOT any 4xx): 401/403 are auth errors
|
|
1246
|
-
// (handled above), 404/405/409 etc. are hub-routing problems that
|
|
1247
|
-
// are not the payload's fault. The existing _consecutiveFailures
|
|
1248
|
-
// backoff is preserved -- the breaker runs BEFORE the early
|
|
1249
|
-
// return so the file is cleared, and then the normal failure
|
|
1250
|
-
// path continues unchanged.
|
|
1251
|
-
if (res.status === 400 && capturedLastUpdate) {
|
|
1252
|
-
const errorText = 'http_400: ' + errText;
|
|
1253
|
-
if (/last[_-]?update/i.test(errorText)) {
|
|
1254
|
-
// Bypass any rate-limited warn helper: this is a critical
|
|
1255
|
-
// recovery signal that must surface even if other ForceUpdate
|
|
1256
|
-
// warns fired recently.
|
|
1257
|
-
this.logger.warn(
|
|
1258
|
-
'[lifecycle] hub 400 with last_update attached (error names last_update); ' +
|
|
1259
|
-
'clearing poisoning state file.'
|
|
1260
|
-
);
|
|
1261
|
-
try {
|
|
1262
|
-
clearLastUpdateOnAck(capturedLastUpdate);
|
|
1263
|
-
} catch (e) {
|
|
1264
|
-
this.logger.warn(`[lifecycle] clearLastUpdateOnAck failed (non-fatal): ${e && e.message || e}`);
|
|
1265
|
-
}
|
|
1266
|
-
}
|
|
1267
|
-
}
|
|
1268
|
-
this._consecutiveFailures++;
|
|
1269
|
-
this.logger.error(`[lifecycle] heartbeat HTTP ${res.status}: ${safeErrText}`);
|
|
1270
|
-
return { ok: false, error: `http_${res.status}`, statusCode: res.status };
|
|
1271
|
-
}
|
|
1272
|
-
|
|
1273
|
-
const data = await readHubResponseJson(res);
|
|
1274
|
-
|
|
1275
|
-
this._consecutiveFailures = 0;
|
|
1276
|
-
this.store.setState('last_heartbeat_at', new Date().toISOString());
|
|
1277
|
-
|
|
1278
|
-
// Semantic parity with a2aProtocol.js sendHeartbeat: a 2xx with
|
|
1279
|
-
// `{ok:false}` or `status:'unknown_node'` is NOT a hub-side persist,
|
|
1280
|
-
// so the state file must survive for the next heartbeat to retry
|
|
1281
|
-
// (unknown_node triggers a re-hello below).
|
|
1282
|
-
//
|
|
1283
|
-
// PR #188 follow-up (HIGH H1-client): the hub now writes a top-level
|
|
1284
|
-
// `last_update_ack: { ok, reason? }` whenever the request carried a
|
|
1285
|
-
// last_update payload. Gate the clear on the ack so we do not unlink
|
|
1286
|
-
// the only evidence of the upgrade attempt when the hub's
|
|
1287
|
-
// fire-and-forget persist throws / dedup-misses / schema-rejects /
|
|
1288
|
-
// bypass-path returns false. Backward compat: an old hub that has not
|
|
1289
|
-
// yet rolled out the ack writer falls back to the original bare-2xx
|
|
1290
|
-
// semantics so this client keeps working against pre-rollout hubs.
|
|
1291
|
-
// See src/gep/a2aProtocol.js sendHeartbeat for the canonical comment.
|
|
1292
|
-
const hubAccepted = !(data && data.ok === false) && data?.status !== 'unknown_node';
|
|
1293
|
-
if (capturedLastUpdate) {
|
|
1294
|
-
const ack = data && data.last_update_ack;
|
|
1295
|
-
const hasAck = ack && typeof ack === 'object';
|
|
1296
|
-
let shouldClear;
|
|
1297
|
-
if (hasAck) {
|
|
1298
|
-
shouldClear = ack.ok === true
|
|
1299
|
-
|| ack.reason === 'duplicate'
|
|
1300
|
-
|| ack.reason === 'invalid';
|
|
1301
|
-
if (ack.reason === 'failed') {
|
|
1302
|
-
this.logger.warn('[lifecycle] hub last_update_ack=failed; ' +
|
|
1303
|
-
'keeping state file for retry on next heartbeat.');
|
|
1304
|
-
} else if (ack.reason === 'invalid') {
|
|
1305
|
-
this.logger.warn('[lifecycle] hub last_update_ack=invalid; ' +
|
|
1306
|
-
'clearing state file (retry will not help).');
|
|
1307
|
-
}
|
|
1308
|
-
} else {
|
|
1309
|
-
shouldClear = hubAccepted;
|
|
1310
|
-
}
|
|
1311
|
-
if (shouldClear) {
|
|
1312
|
-
try {
|
|
1313
|
-
clearLastUpdateOnAck(capturedLastUpdate);
|
|
1314
|
-
} catch (e) {
|
|
1315
|
-
this.logger.warn(`[lifecycle] clearLastUpdateOnAck failed (non-fatal): ${e && e.message || e}`);
|
|
1316
|
-
}
|
|
1317
|
-
}
|
|
1318
|
-
}
|
|
1319
|
-
|
|
1320
|
-
if (data?.status === 'unknown_node') {
|
|
1321
|
-
this.logger.warn('[lifecycle] Node unknown, re-registering...');
|
|
1322
|
-
await this.hello();
|
|
1323
|
-
}
|
|
1324
|
-
|
|
1325
|
-
// PR #188 H1 fix: 200 with a `force_update` directive must drive
|
|
1326
|
-
// executeForceUpdate the same way a2aProtocol.js does for
|
|
1327
|
-
// non-proxy nodes (see a2aProtocol.js:2292-2305 and
|
|
1328
|
-
// _maybeTriggerForceUpdateFromHeartbeat). Pure proxy-mode nodes
|
|
1329
|
-
// never enter the evolve run() loop, so the consumeForceUpdate
|
|
1330
|
-
// path never fires for them — without this block the hub could
|
|
1331
|
-
// push force_update forever with zero upgrade attempts and zero
|
|
1332
|
-
// EvolverUpgradeAttempt rows. The helper is in-flight + cooldown
|
|
1333
|
-
// gated; placing the call here (post-events, pre-min_version
|
|
1334
|
-
// banner) means a single response carrying both events AND a
|
|
1335
|
-
// force_update still processes the events first.
|
|
1336
|
-
if (data && data.force_update && typeof data.force_update === 'object') {
|
|
1337
|
-
this.logger.log(
|
|
1338
|
-
'[ForceUpdate] Hub requires update to ' +
|
|
1339
|
-
(data.force_update.required_version || '?') +
|
|
1340
|
-
' -- reason: ' + (data.force_update.reason || 'unspecified')
|
|
1341
|
-
);
|
|
1342
|
-
_maybeTriggerForceUpdateFromHeartbeat(data.force_update, this.logger);
|
|
1343
|
-
}
|
|
1344
|
-
|
|
1345
|
-
if (Array.isArray(data?.events) && data.events.length > 0) {
|
|
1346
|
-
this.store.writeInboundBatch(
|
|
1347
|
-
data.events.map(e => ({
|
|
1348
|
-
type: e.type || 'hub_event',
|
|
1349
|
-
payload: e,
|
|
1350
|
-
channel: 'evomap-hub',
|
|
1351
|
-
}))
|
|
1352
|
-
);
|
|
1353
|
-
}
|
|
1354
|
-
|
|
1355
|
-
if (data?.min_proxy_version && this._shouldUpgrade(data.min_proxy_version)) {
|
|
1356
|
-
this.store.writeInbound({
|
|
1357
|
-
type: 'system',
|
|
1358
|
-
payload: {
|
|
1359
|
-
action: 'proxy_upgrade_required',
|
|
1360
|
-
min_version: data.min_proxy_version,
|
|
1361
|
-
current_version: PROXY_PROTOCOL_VERSION,
|
|
1362
|
-
upgrade_url: data.upgrade_url || null,
|
|
1363
|
-
message: data.upgrade_message || 'Proxy version is below the minimum required by Hub.',
|
|
1364
|
-
},
|
|
1365
|
-
channel: 'evomap-hub',
|
|
1366
|
-
priority: 'high',
|
|
1367
|
-
});
|
|
1368
|
-
this.logger.warn(`[lifecycle] Hub requires proxy >= ${data.min_proxy_version}, current: ${PROXY_PROTOCOL_VERSION}`);
|
|
1369
|
-
}
|
|
1370
|
-
|
|
1371
|
-
return { ok: true, response: data };
|
|
1372
|
-
} catch (err) {
|
|
1373
|
-
if (isHubUnreachableError(err)) {
|
|
1374
|
-
this._consecutiveFailures++;
|
|
1375
|
-
const retryAfterMs = this._recordHubUnreachable(err);
|
|
1376
|
-
return {
|
|
1377
|
-
ok: false,
|
|
1378
|
-
error: 'hub_unreachable',
|
|
1379
|
-
detail: err.message,
|
|
1380
|
-
retryAfterMs,
|
|
1381
|
-
};
|
|
1382
|
-
}
|
|
1383
|
-
this._consecutiveFailures++;
|
|
1384
|
-
this.logger.error(`[lifecycle] heartbeat failed (${this._consecutiveFailures}): ${err.message}`);
|
|
1385
|
-
return { ok: false, error: err.message };
|
|
1386
|
-
}
|
|
1387
|
-
}
|
|
1388
|
-
|
|
1389
|
-
startHeartbeatLoop(intervalMs) {
|
|
1390
|
-
if (this._running) return;
|
|
1391
|
-
this._running = true;
|
|
1392
|
-
this._startedAt = Date.now();
|
|
1393
|
-
this._heartbeatInterval = Math.max(30_000, intervalMs || DEFAULT_HEARTBEAT_INTERVAL);
|
|
1394
|
-
// Generation counter: every poke / stop bumps this. Tick captures
|
|
1395
|
-
// its gen at entry; if it doesn't match on resume, the tick refuses
|
|
1396
|
-
// to schedule its own next timer (a fresher path already owns it).
|
|
1397
|
-
this._heartbeatGen = (this._heartbeatGen || 0) + 1;
|
|
1398
|
-
this._heartbeatTick(this._heartbeatGen);
|
|
1399
|
-
this._startDriftDetector();
|
|
1400
|
-
}
|
|
1401
|
-
|
|
1402
|
-
// Sample wall-clock every DRIFT_CHECK_MS so macOS sleep / hypervisor
|
|
1403
|
-
// pause / debugger break is detected and the heartbeat loop is poked
|
|
1404
|
-
// back into action without waiting for the (possibly 15-min) backoff
|
|
1405
|
-
// timer to fire on libuv's monotonic clock. R10 (#544).
|
|
1406
|
-
_startDriftDetector() {
|
|
1407
|
-
if (this._driftInterval) return;
|
|
1408
|
-
this._lastDriftCheckAt = Date.now();
|
|
1409
|
-
this._driftInterval = setInterval(() => {
|
|
1410
|
-
// Wrap the whole body in try/catch -- this is a setInterval
|
|
1411
|
-
// callback; any throw escaping it kills the detector itself,
|
|
1412
|
-
// which is the bug we're protecting against.
|
|
1413
|
-
try {
|
|
1414
|
-
if (!this._running) return;
|
|
1415
|
-
const now = Date.now();
|
|
1416
|
-
const gap = now - this._lastDriftCheckAt;
|
|
1417
|
-
this._lastDriftCheckAt = now;
|
|
1418
|
-
if (gap > DRIFT_SLEEP_THRESHOLD_MS) {
|
|
1419
|
-
try {
|
|
1420
|
-
this.logger.warn(
|
|
1421
|
-
`[lifecycle] wall-clock jump detected (+${Math.round(gap / 1000)}s); ` +
|
|
1422
|
-
'likely sleep/wake or process suspension, poking heartbeat'
|
|
1423
|
-
);
|
|
1424
|
-
} catch (_) { /* logger broken; detector must still poke */ }
|
|
1425
|
-
// Long-sleep recovery: the hub-side cached state we carried
|
|
1426
|
-
// through the suspend is almost certainly stale. Clear reauth
|
|
1427
|
-
// backoff so the next tick can try a clean recovery path
|
|
1428
|
-
// instead of sitting out a pre-sleep penalty for up to 4h.
|
|
1429
|
-
if (gap > DRIFT_LONG_SLEEP_THRESHOLD_MS) {
|
|
1430
|
-
this._consecutiveReauthFailures = 0;
|
|
1431
|
-
this._reauthBackoffUntil = 0;
|
|
1432
|
-
try {
|
|
1433
|
-
this.logger.warn(
|
|
1434
|
-
`[lifecycle] long sleep (+${Math.round(gap / 60_000)}min) cleared reauth backoff`
|
|
1435
|
-
);
|
|
1436
|
-
} catch (_) { /* logger broken; non-fatal */ }
|
|
1437
|
-
}
|
|
1438
|
-
this.pokeHeartbeatLoop();
|
|
1439
|
-
}
|
|
1440
|
-
} catch (err) {
|
|
1441
|
-
try { this.logger.error(`[lifecycle] drift detector threw: ${err && err.message}`); }
|
|
1442
|
-
catch (_) { /* never let the detector escape */ }
|
|
1443
|
-
}
|
|
1444
|
-
}, DRIFT_CHECK_MS);
|
|
1445
|
-
// Don't keep the event loop alive on behalf of the detector alone --
|
|
1446
|
-
// matches the unref() used on _heartbeatTimer.
|
|
1447
|
-
if (this._driftInterval.unref) this._driftInterval.unref();
|
|
1448
|
-
}
|
|
1449
|
-
|
|
1450
|
-
async _heartbeatTick(myGen) {
|
|
1451
|
-
if (!this._running) return;
|
|
1452
|
-
this._lastHeartbeatTickAt = Date.now();
|
|
1453
|
-
// Defence-in-depth: even with heartbeat() now fully wrapped (see
|
|
1454
|
-
// its body), an unforeseen synchronous throw inside the awaited
|
|
1455
|
-
// path or a defective stub passed in tests would still bubble
|
|
1456
|
-
// into this closure as a rejected promise and cancel the next
|
|
1457
|
-
// setTimeout. Catching here guarantees the loop schedules its
|
|
1458
|
-
// own next tick under all conditions short of `stopHeartbeatLoop`.
|
|
1459
|
-
try {
|
|
1460
|
-
await this.heartbeat();
|
|
1461
|
-
} catch (err) {
|
|
1462
|
-
this._consecutiveFailures++;
|
|
1463
|
-
this.logger.error(`[lifecycle] heartbeat tick threw (${this._consecutiveFailures}): ${err && err.message}`);
|
|
1464
|
-
}
|
|
1465
|
-
if (!this._running) return;
|
|
1466
|
-
// Generation guard: a poke or stop fired while we were awaiting
|
|
1467
|
-
// `heartbeat()`. The fresher path already armed its own timer (or
|
|
1468
|
-
// tore the loop down); scheduling here would orphan the new timer
|
|
1469
|
-
// or fork the loop into two concurrent ticks (Bugbot #147 finding
|
|
1470
|
-
// 2026-05-28).
|
|
1471
|
-
if (myGen !== undefined && myGen !== this._heartbeatGen) return;
|
|
1472
|
-
// 15min cap (was 30min, then briefly 5min). Must stay above
|
|
1473
|
-
// `DEFAULT_HEARTBEAT_INTERVAL` (6min) or the exponential branch
|
|
1474
|
-
// retries faster than the success branch (Bugbot #147 finding).
|
|
1475
|
-
const interval = this._heartbeatInterval || DEFAULT_HEARTBEAT_INTERVAL;
|
|
1476
|
-
const hubWaitMs = this._hubUnreachableWaitMs();
|
|
1477
|
-
// While a hub-unreachable window is active, wake exactly when it elapses
|
|
1478
|
-
// (1s floor to avoid a busy 0ms reschedule), mirroring SyncEngine's
|
|
1479
|
-
// `Math.max(1_000, retryAfterMs)`. A 30s floor over-waited up to ~30s past
|
|
1480
|
-
// the window's end, delaying recovery once the hub was reachable again.
|
|
1481
|
-
const backoff = hubWaitMs > 0
|
|
1482
|
-
? Math.max(1_000, hubWaitMs)
|
|
1483
|
-
: this._consecutiveFailures > 0
|
|
1484
|
-
? Math.min(interval * Math.pow(2, this._consecutiveFailures), HEARTBEAT_BACKOFF_CAP_MS)
|
|
1485
|
-
: interval;
|
|
1486
|
-
const armedGen = this._heartbeatGen;
|
|
1487
|
-
this._heartbeatTimer = setTimeout(() => this._heartbeatTick(armedGen), backoff);
|
|
1488
|
-
if (this._heartbeatTimer.unref) this._heartbeatTimer.unref();
|
|
1489
|
-
}
|
|
1490
|
-
|
|
1491
|
-
/// Reset the heartbeat schedule immediately. Call when an external
|
|
1492
|
-
/// signal (Hub push, tab visibility, manual `evolver heartbeat now`)
|
|
1493
|
-
/// indicates the next tick should not wait out its current backoff.
|
|
1494
|
-
/// No-op when the loop is not running.
|
|
1495
|
-
pokeHeartbeatLoop() {
|
|
1496
|
-
if (!this._running) return;
|
|
1497
|
-
if (this._heartbeatTimer) {
|
|
1498
|
-
clearTimeout(this._heartbeatTimer);
|
|
1499
|
-
this._heartbeatTimer = null;
|
|
1500
|
-
}
|
|
1501
|
-
this._consecutiveFailures = 0;
|
|
1502
|
-
// Bump generation: any in-flight `_heartbeatTick` from before the
|
|
1503
|
-
// poke will see its captured gen mismatch on resume and skip its
|
|
1504
|
-
// tail-`setTimeout`, so we don't end up with two concurrent timers.
|
|
1505
|
-
this._heartbeatGen = (this._heartbeatGen || 0) + 1;
|
|
1506
|
-
const myGen = this._heartbeatGen;
|
|
1507
|
-
// setImmediate-equivalent: defer to the next event-loop turn so
|
|
1508
|
-
// re-entrant pokes (e.g. from a Hub event handler) don't run
|
|
1509
|
-
// synchronously inside the caller's frame.
|
|
1510
|
-
this._heartbeatTimer = setTimeout(() => this._heartbeatTick(myGen), 0);
|
|
1511
|
-
if (this._heartbeatTimer.unref) this._heartbeatTimer.unref();
|
|
1512
|
-
}
|
|
1513
|
-
|
|
1514
|
-
stopHeartbeatLoop() {
|
|
1515
|
-
this._running = false;
|
|
1516
|
-
if (this._heartbeatTimer) {
|
|
1517
|
-
clearTimeout(this._heartbeatTimer);
|
|
1518
|
-
this._heartbeatTimer = null;
|
|
1519
|
-
}
|
|
1520
|
-
if (this._driftInterval) {
|
|
1521
|
-
clearInterval(this._driftInterval);
|
|
1522
|
-
this._driftInterval = null;
|
|
1523
|
-
}
|
|
1524
|
-
}
|
|
1525
|
-
|
|
1526
|
-
getHeartbeatStats() {
|
|
1527
|
-
return {
|
|
1528
|
-
running: this._running,
|
|
1529
|
-
intervalMs: this._heartbeatInterval || DEFAULT_HEARTBEAT_INTERVAL,
|
|
1530
|
-
uptimeMs: this._startedAt ? Date.now() - this._startedAt : 0,
|
|
1531
|
-
consecutiveFailures: this._consecutiveFailures,
|
|
1532
|
-
lastTickAt: this._lastHeartbeatTickAt,
|
|
1533
|
-
};
|
|
1534
|
-
}
|
|
1535
|
-
|
|
1536
|
-
_shouldUpgrade(minVersion) {
|
|
1537
|
-
// parseInt strips trailing non-digit chars in prerelease segments like
|
|
1538
|
-
// `1-beta`, so `0.1.1-beta.1`.split('.')[2] -> `1-beta` -> parseInt = 1.
|
|
1539
|
-
// Using Number() here returned NaN and was treated as 0, under-counting
|
|
1540
|
-
// prerelease minimums. See community PR #516.
|
|
1541
|
-
const parse = (v) => String(v || '0.0.0').split('.').map((part) => parseInt(part, 10));
|
|
1542
|
-
const min = parse(minVersion);
|
|
1543
|
-
const cur = parse(PROXY_PROTOCOL_VERSION);
|
|
1544
|
-
for (let i = 0; i < 3; i++) {
|
|
1545
|
-
if ((cur[i] || 0) < (min[i] || 0)) return true;
|
|
1546
|
-
if ((cur[i] || 0) > (min[i] || 0)) return false;
|
|
1547
|
-
}
|
|
1548
|
-
return false;
|
|
1549
|
-
}
|
|
1550
|
-
}
|
|
1551
|
-
|
|
1552
|
-
module.exports = {
|
|
1553
|
-
LifecycleManager,
|
|
1554
|
-
AuthError,
|
|
1555
|
-
DEFAULT_HEARTBEAT_INTERVAL,
|
|
1556
|
-
HEARTBEAT_BACKOFF_CAP_MS,
|
|
1557
|
-
// Test hooks behind `_testing` to mirror the namespacing used by
|
|
1558
|
-
// a2aProtocol.js — production callers must not accidentally tweak the
|
|
1559
|
-
// proxy force_update lifecycle state.
|
|
1560
|
-
_testing: {
|
|
1561
|
-
// Reset proxy heartbeat-driven force_update state. Avoids cooldown
|
|
1562
|
-
// leakage between sibling tests that share one process.
|
|
1563
|
-
_resetProxyForceUpdateStateForTesting: function () {
|
|
1564
|
-
_proxyForceUpdateInFlight = false;
|
|
1565
|
-
_proxyForceUpdateLastAttemptAt = 0;
|
|
1566
|
-
},
|
|
1567
|
-
},
|
|
1568
|
-
};
|