instar 1.3.194 → 1.3.196
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +32 -3
- package/dist/commands/server.js.map +1 -1
- package/dist/core/HttpLeaseTransport.d.ts +21 -0
- package/dist/core/HttpLeaseTransport.d.ts.map +1 -1
- package/dist/core/HttpLeaseTransport.js +58 -1
- package/dist/core/HttpLeaseTransport.js.map +1 -1
- package/dist/core/LeaseCoordinator.d.ts +38 -0
- package/dist/core/LeaseCoordinator.d.ts.map +1 -1
- package/dist/core/LeaseCoordinator.js +36 -0
- package/dist/core/LeaseCoordinator.js.map +1 -1
- package/dist/core/MultiMachineCoordinator.d.ts +39 -0
- package/dist/core/MultiMachineCoordinator.d.ts.map +1 -1
- package/dist/core/MultiMachineCoordinator.js +130 -4
- package/dist/core/MultiMachineCoordinator.js.map +1 -1
- package/dist/core/PostUpdateMigrator.js +1 -1
- package/dist/core/seamlessnessConfig.d.ts +2 -0
- package/dist/core/seamlessnessConfig.d.ts.map +1 -1
- package/dist/core/seamlessnessConfig.js +13 -0
- package/dist/core/seamlessnessConfig.js.map +1 -1
- package/dist/core/types.d.ts +9 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/core/uncaughtExceptionPolicy.d.ts.map +1 -1
- package/dist/core/uncaughtExceptionPolicy.js +11 -0
- package/dist/core/uncaughtExceptionPolicy.js.map +1 -1
- package/dist/scaffold/templates.js +1 -1
- package/dist/server/AgentServer.d.ts +6 -0
- package/dist/server/AgentServer.d.ts.map +1 -1
- package/dist/server/AgentServer.js +1 -0
- package/dist/server/AgentServer.js.map +1 -1
- package/dist/server/machineRoutes.d.ts +8 -0
- package/dist/server/machineRoutes.d.ts.map +1 -1
- package/dist/server/machineRoutes.js +12 -0
- package/dist/server/machineRoutes.js.map +1 -1
- package/package.json +1 -1
- package/scripts/check-spec-review-link.mjs +63 -0
- package/src/data/builtin-manifest.json +22 -22
- package/src/scaffold/templates.ts +1 -1
- package/src/templates/scripts/convergence-check.sh +20 -0
- package/upgrades/1.3.195.md +33 -0
- package/upgrades/1.3.196.md +27 -0
- package/upgrades/side-effects/fix-activepull-solo-demotion.md +75 -0
- package/upgrades/side-effects/multi-machine-robust-lease-propagation.md +120 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-06-
|
|
5
|
-
"instarVersion": "1.3.
|
|
4
|
+
"generatedAt": "2026-06-02T02:41:08.992Z",
|
|
5
|
+
"instarVersion": "1.3.196",
|
|
6
6
|
"entryCount": 193,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"domain": "identity",
|
|
12
12
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
13
13
|
"installedPath": ".instar/hooks/instar/session-start.sh",
|
|
14
|
-
"contentHash": "
|
|
14
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
15
15
|
"since": "2025-01-01"
|
|
16
16
|
},
|
|
17
17
|
"hook:dangerous-command-guard": {
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"domain": "safety",
|
|
21
21
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
22
22
|
"installedPath": ".instar/hooks/instar/dangerous-command-guard.sh",
|
|
23
|
-
"contentHash": "
|
|
23
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
24
24
|
"since": "2025-01-01"
|
|
25
25
|
},
|
|
26
26
|
"hook:grounding-before-messaging": {
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
"domain": "safety",
|
|
30
30
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
31
31
|
"installedPath": ".instar/hooks/instar/grounding-before-messaging.sh",
|
|
32
|
-
"contentHash": "
|
|
32
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
33
33
|
"since": "2025-01-01"
|
|
34
34
|
},
|
|
35
35
|
"hook:compaction-recovery": {
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
"domain": "identity",
|
|
39
39
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
40
40
|
"installedPath": ".instar/hooks/instar/compaction-recovery.sh",
|
|
41
|
-
"contentHash": "
|
|
41
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
42
42
|
"since": "2025-01-01"
|
|
43
43
|
},
|
|
44
44
|
"hook:external-operation-gate": {
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"domain": "safety",
|
|
48
48
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
49
49
|
"installedPath": ".instar/hooks/instar/external-operation-gate.js",
|
|
50
|
-
"contentHash": "
|
|
50
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
51
51
|
"since": "2025-01-01"
|
|
52
52
|
},
|
|
53
53
|
"hook:deferral-detector": {
|
|
@@ -56,7 +56,7 @@
|
|
|
56
56
|
"domain": "safety",
|
|
57
57
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
58
58
|
"installedPath": ".instar/hooks/instar/deferral-detector.js",
|
|
59
|
-
"contentHash": "
|
|
59
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
60
60
|
"since": "2025-01-01"
|
|
61
61
|
},
|
|
62
62
|
"hook:post-action-reflection": {
|
|
@@ -65,7 +65,7 @@
|
|
|
65
65
|
"domain": "evolution",
|
|
66
66
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
67
67
|
"installedPath": ".instar/hooks/instar/post-action-reflection.js",
|
|
68
|
-
"contentHash": "
|
|
68
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
69
69
|
"since": "2025-01-01"
|
|
70
70
|
},
|
|
71
71
|
"hook:external-communication-guard": {
|
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
"domain": "safety",
|
|
75
75
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
76
76
|
"installedPath": ".instar/hooks/instar/external-communication-guard.js",
|
|
77
|
-
"contentHash": "
|
|
77
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
78
78
|
"since": "2025-01-01"
|
|
79
79
|
},
|
|
80
80
|
"hook:scope-coherence-collector": {
|
|
@@ -83,7 +83,7 @@
|
|
|
83
83
|
"domain": "coherence",
|
|
84
84
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
85
85
|
"installedPath": ".instar/hooks/instar/scope-coherence-collector.js",
|
|
86
|
-
"contentHash": "
|
|
86
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
87
87
|
"since": "2025-01-01"
|
|
88
88
|
},
|
|
89
89
|
"hook:scope-coherence-checkpoint": {
|
|
@@ -92,7 +92,7 @@
|
|
|
92
92
|
"domain": "coherence",
|
|
93
93
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
94
94
|
"installedPath": ".instar/hooks/instar/scope-coherence-checkpoint.js",
|
|
95
|
-
"contentHash": "
|
|
95
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
96
96
|
"since": "2025-01-01"
|
|
97
97
|
},
|
|
98
98
|
"hook:free-text-guard": {
|
|
@@ -101,7 +101,7 @@
|
|
|
101
101
|
"domain": "safety",
|
|
102
102
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
103
103
|
"installedPath": ".instar/hooks/instar/free-text-guard.sh",
|
|
104
|
-
"contentHash": "
|
|
104
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
105
105
|
"since": "2025-01-01"
|
|
106
106
|
},
|
|
107
107
|
"hook:claim-intercept": {
|
|
@@ -110,7 +110,7 @@
|
|
|
110
110
|
"domain": "coherence",
|
|
111
111
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
112
112
|
"installedPath": ".instar/hooks/instar/claim-intercept.js",
|
|
113
|
-
"contentHash": "
|
|
113
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
114
114
|
"since": "2025-01-01"
|
|
115
115
|
},
|
|
116
116
|
"hook:claim-intercept-response": {
|
|
@@ -119,7 +119,7 @@
|
|
|
119
119
|
"domain": "coherence",
|
|
120
120
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
121
121
|
"installedPath": ".instar/hooks/instar/claim-intercept-response.js",
|
|
122
|
-
"contentHash": "
|
|
122
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
123
123
|
"since": "2025-01-01"
|
|
124
124
|
},
|
|
125
125
|
"hook:stop-gate-router": {
|
|
@@ -128,7 +128,7 @@
|
|
|
128
128
|
"domain": "safety",
|
|
129
129
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
130
130
|
"installedPath": ".instar/hooks/instar/stop-gate-router.js",
|
|
131
|
-
"contentHash": "
|
|
131
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
132
132
|
"since": "2025-01-01"
|
|
133
133
|
},
|
|
134
134
|
"hook:auto-approve-permissions": {
|
|
@@ -137,7 +137,7 @@
|
|
|
137
137
|
"domain": "safety",
|
|
138
138
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
139
139
|
"installedPath": ".instar/hooks/instar/auto-approve-permissions.js",
|
|
140
|
-
"contentHash": "
|
|
140
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
141
141
|
"since": "2025-01-01"
|
|
142
142
|
},
|
|
143
143
|
"job:health-check": {
|
|
@@ -753,7 +753,7 @@
|
|
|
753
753
|
"type": "route-group",
|
|
754
754
|
"domain": "coordination",
|
|
755
755
|
"sourcePath": "src/server/machineRoutes.ts",
|
|
756
|
-
"contentHash": "
|
|
756
|
+
"contentHash": "a8b151ac8857cb3c161ed123cb175b400db9ad6d0bb18fe8def72a33fa1444ac",
|
|
757
757
|
"since": "2025-01-01"
|
|
758
758
|
},
|
|
759
759
|
"route-group:secret-drop": {
|
|
@@ -1089,7 +1089,7 @@
|
|
|
1089
1089
|
"type": "template",
|
|
1090
1090
|
"domain": "operations",
|
|
1091
1091
|
"sourcePath": "src/templates/scripts/convergence-check.sh",
|
|
1092
|
-
"contentHash": "
|
|
1092
|
+
"contentHash": "b0475554bde6729d2d9a1f8005dcada6a8e4c767203d8293c09a212f9bf0a196",
|
|
1093
1093
|
"since": "2025-01-01"
|
|
1094
1094
|
},
|
|
1095
1095
|
"template:git-sync-gate.sh": {
|
|
@@ -1457,7 +1457,7 @@
|
|
|
1457
1457
|
"type": "subsystem",
|
|
1458
1458
|
"domain": "server",
|
|
1459
1459
|
"sourcePath": "src/server/AgentServer.ts",
|
|
1460
|
-
"contentHash": "
|
|
1460
|
+
"contentHash": "fde1c526b156577e0ad447d179760c9a0b095b051b6af0fd56b879278f504c38",
|
|
1461
1461
|
"since": "2025-01-01"
|
|
1462
1462
|
},
|
|
1463
1463
|
"subsystem:session-manager": {
|
|
@@ -1489,7 +1489,7 @@
|
|
|
1489
1489
|
"type": "subsystem",
|
|
1490
1490
|
"domain": "updates",
|
|
1491
1491
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
1492
|
-
"contentHash": "
|
|
1492
|
+
"contentHash": "1cd3546e774a6045c3b8c1481a1aa2acfbbb330232f7d3f1353d0f92aca6b1bc",
|
|
1493
1493
|
"since": "2025-01-01"
|
|
1494
1494
|
},
|
|
1495
1495
|
"subsystem:scheduler": {
|
|
@@ -1545,7 +1545,7 @@
|
|
|
1545
1545
|
"type": "subsystem",
|
|
1546
1546
|
"domain": "coordination",
|
|
1547
1547
|
"sourcePath": "src/core/MultiMachineCoordinator.ts",
|
|
1548
|
-
"contentHash": "
|
|
1548
|
+
"contentHash": "63ce6fcd9564b1e547e2b0ad6153e43293566dac6a6db6f17ffb8cbf28b28894",
|
|
1549
1549
|
"since": "2025-01-01"
|
|
1550
1550
|
},
|
|
1551
1551
|
"subsystem:backup-manager": {
|
|
@@ -435,7 +435,7 @@ This routes feedback to the Instar maintainers automatically. Valid types: \`bug
|
|
|
435
435
|
- **A handoff feels like a compaction pause, not amnesia** — the new machine resumes via CONTINUATION (picks up the thread, no re-greeting). Planned handoff = current context; hard failover = as-of-last-sync, and if my context is partial I say so honestly ("picking this back up from the other machine").
|
|
436
436
|
- Read mesh/sync state, never guess it: \`curl -H "Authorization: Bearer $AUTH" http://localhost:${port}/health\` → \`multiMachine.syncStatus\` (\`leaseHolder, leaseEpoch, holdsLease, splitBrainState, awakeMachineCount\`); \`instar doctor\` shows the same.
|
|
437
437
|
- A genuinely **unresolvable split-brain** surfaces as ONE Attention-queue item with a Y/N decision ("demote machine X?"), deduped per partition episode — I present it to the user, I don't silently pick.
|
|
438
|
-
- Dials under \`.instar/config.json\` → \`multiMachine\` (ingressHeartbeatMs, leaseTtlMs, liveTailMaxStalenessMs, handoffAckTimeoutMs, …); a nonsensical combo is rejected at startup, not run silently.
|
|
438
|
+
- Dials under \`.instar/config.json\` → \`multiMachine\` (ingressHeartbeatMs, leaseTtlMs, leasePullIntervalMs, liveTailMaxStalenessMs, handoffAckTimeoutMs, …); a nonsensical combo is rejected at startup, not run silently.
|
|
439
439
|
|
|
440
440
|
**Multi-Machine Session Pool (active-active — spread conversations across machines)** — The longer arc beyond one-awake-machine: with the pool enabled I run conversations across ALL my machines at once and can MOVE a conversation between them. Ships DARK behind \`multiMachine.sessionPool.stage\` (default 'dark'); a single-machine agent is a no-op.
|
|
441
441
|
- **See the pool:** the **Machines tab** in the dashboard, or \`curl -H "Authorization: Bearer $AUTH" http://localhost:${port}/pool\` → which machine is the router ("dispatcher") + every machine's nickname, hardware, online status, load, and clock-skew status.
|
|
@@ -149,6 +149,26 @@ if echo "$CONTENT" | grep -qiE "(i used to (think|believe|feel|assume)|back when
|
|
|
149
149
|
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
|
150
150
|
fi
|
|
151
151
|
|
|
152
|
+
# 8. SPEC-REVIEW LINK — a spec handed over for review must carry a rendered
|
|
153
|
+
# ELI16 tunnel link the operator can open. Structure > Willpower: the link is
|
|
154
|
+
# guaranteed by skills/spec-converge/scripts/publish-spec-review.mjs. Narrow by
|
|
155
|
+
# design (this blocks the send): fires only on an unambiguous spec handoff —
|
|
156
|
+
# a docs/specs/*.md reference, OR a GitHub PR mentioned with the word "spec" —
|
|
157
|
+
# in an explicit review/approval handoff, with no /view/ link present. An
|
|
158
|
+
# ordinary code-PR mention does not fire.
|
|
159
|
+
SPEC_HANDOFF=""
|
|
160
|
+
if echo "$CONTENT" | grep -qiE "docs/specs/[^ )]+\.md"; then
|
|
161
|
+
SPEC_HANDOFF="yes"
|
|
162
|
+
elif echo "$CONTENT" | grep -qiE "github\.com/[^ )]+/pull/[0-9]+" && echo "$CONTENT" | grep -qiE "\bspec(s|ification)?\b"; then
|
|
163
|
+
SPEC_HANDOFF="yes"
|
|
164
|
+
fi
|
|
165
|
+
if [ -n "$SPEC_HANDOFF" ] \
|
|
166
|
+
&& echo "$CONTENT" | grep -qiE "\b(review|approv|sign[- ]?off|take a look|for your|ready for)\b" \
|
|
167
|
+
&& ! echo "$CONTENT" | grep -qiE "/view/[0-9a-f-]{8,}"; then
|
|
168
|
+
ISSUES+=("SPEC_REVIEW_LINK: You're handing a spec over for review with no rendered ELI16 tunnel link the operator can open. Deliver spec reviews via skills/spec-converge/scripts/publish-spec-review.mjs — it renders the ELI16, verifies the link (HTTP 200), and includes it. Don't send a spec for review without the rendered link.")
|
|
169
|
+
ISSUE_COUNT=$((ISSUE_COUNT + 1))
|
|
170
|
+
fi
|
|
171
|
+
|
|
152
172
|
# Output results
|
|
153
173
|
if [ "$ISSUE_COUNT" -gt "0" ]; then
|
|
154
174
|
echo "=== CONVERGENCE CHECK: ${ISSUE_COUNT} ISSUE(S) FOUND ==="
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: minor -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
Multi-machine agents now actively **pull** the lease that decides which machine is "awake," instead of only waiting to be told.
|
|
9
|
+
|
|
10
|
+
Previously a standby machine learned about a takeover only when the awake machine could push a message to it. On a quiet network, or one where traffic only flows one direction (a common home-network situation), that push might never arrive — so a standby could be left blind, and two machines could each believe they were in charge without noticing each other.
|
|
11
|
+
|
|
12
|
+
Now every machine, on a steady cadence and regardless of whether the other looks alive, asks its peers for their current lease. If it learns a newer one, it steps down automatically. If it discovers a genuine tie — both machines holding the same generation of the lease — it records that as a contested state visible on the dashboard rather than silently carrying on. This is the robustness-under-poor-conditions half of the cross-machine work, and it ships build-and-test only: nothing in a live deployment is switched on by this change.
|
|
13
|
+
|
|
14
|
+
The session-pool rollout control was also tightened so it can only roll a stage back (on a failed end-to-end check for the running build), never advance on its own — advancing stays a deliberate operator action.
|
|
15
|
+
|
|
16
|
+
## What to Tell Your User
|
|
17
|
+
|
|
18
|
+
- **No more blind standby**: "If I'm running on more than one of your machines, the backup now actively checks in with the others on its own schedule — so even on a quiet or one-way home network it won't get stuck thinking it's in charge when it isn't."
|
|
19
|
+
- **Honest about confusion**: "If two of my machines ever end up both believing they're the active one, I now surface that as a flagged, contested state you can see — instead of quietly double-acting."
|
|
20
|
+
- **Nothing to set up**: "This is on by default with sensible timing and changes nothing about how you talk to me; it only makes the multi-machine setup steadier when conditions are bad."
|
|
21
|
+
|
|
22
|
+
## Summary of New Capabilities
|
|
23
|
+
|
|
24
|
+
- An active lease-pull path on the machine-to-machine channel (a new authenticated pull endpoint plus a pull method on the lease transport) so a standby can ask for a peer's current lease instead of only being pushed to.
|
|
25
|
+
- A constant-cadence, jittered standby pull loop that runs independent of holder liveness, auto-demotes a fenced holder on a newer lease, and surfaces a same-epoch split-brain near-silently to the sync status / dashboard.
|
|
26
|
+
- A configurable pull cadence with a startup-validated bound (pull at least once per lease lifetime), resolved alongside the other cross-machine timing knobs and noted in the agent's own capability docs.
|
|
27
|
+
- A revert-only rollout reconcile tick for the session pool, with the running commit identifier resolved from git at boot.
|
|
28
|
+
|
|
29
|
+
## Evidence
|
|
30
|
+
|
|
31
|
+
- 108 tests green across the touched area and all three tiers: unit (lease transport pull cases, lease-coordinator pull accessors and same-epoch masking, the pull-cadence config invariant, the coordinator pull-loop contested surface), integration (the new pull endpoint: serve, empty, and unauthenticated), and end-to-end (a two-machine partition produces two awake machines, and on heal the active pull converges back to one — over the real lease components and a mock HTTP wire). Type-check clean.
|
|
32
|
+
- Test-writing caught a real setup bug (a lease builder's third argument is the nonce, not the epoch) before it reached an assertion.
|
|
33
|
+
- Spec: docs/specs/MULTI-MACHINE-ROBUST-LEASE-PROPAGATION-SPEC.md (approved). Side-effects review: upgrades/side-effects/multi-machine-robust-lease-propagation.md.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: patch -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
Fixes a crash-loop regression introduced with the multi-machine active-pull feature.
|
|
9
|
+
|
|
10
|
+
On a machine running by itself (no other live machine in its mesh), the active-pull loop was re-checking the "who's awake" role every few seconds and, whenever the short-lived internal lease briefly lapsed between renewals, wrongly flipped the machine to a standby (read-only) state. A follow-up write then crashed the server, and it restarted into the same loop. Because the server is also the messaging relay, the operator saw silence during the loop.
|
|
11
|
+
|
|
12
|
+
The fix: the active-pull loop now only re-evaluates the awake/standby role when it has actually heard a lease from a peer. A machine on its own is never demoted by the pull loop on a transient lease lapse — its normal heartbeat keeps renewing the lease as before. The genuine multi-machine behavior (a backup that pulls a newer lease from a live peer still steps down) is unchanged.
|
|
13
|
+
|
|
14
|
+
This pairs with the earlier hardening that made a read-only/standby write non-fatal, so both the wrong state and the crash it could cause are now closed.
|
|
15
|
+
|
|
16
|
+
## What to Tell Your User
|
|
17
|
+
|
|
18
|
+
Nothing to do. If you run an agent on a single machine, this removes a rare crash-restart loop that could make it go quiet for a while; multi-machine setups behave the same as before, just without the spurious solo demotion.
|
|
19
|
+
|
|
20
|
+
## Summary of New Capabilities
|
|
21
|
+
|
|
22
|
+
- The active lease-pull loop only reconciles the awake/standby role when a peer lease was actually observed, so a solo machine is never demoted (and never driven read-only) by the pull loop on a transient self-lease lapse. Regression test added; the real peer-driven demotion path is retained and tested.
|
|
23
|
+
|
|
24
|
+
## Evidence
|
|
25
|
+
|
|
26
|
+
- `tsc --noEmit` clean; 45 multi-machine unit tests green, including a new regression (a solo holder whose self-lease lapses with no peer observed stays awake and never goes read-only) and the retained same-epoch contested-split-brain test (feature intact).
|
|
27
|
+
- Root cause of the 2026-06-02 echo crash-loop incident. Spec: docs/specs/MULTI-MACHINE-ROBUST-LEASE-PROPAGATION-SPEC.md (approved). Side-effects review: upgrades/side-effects/fix-activepull-solo-demotion.md.
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# Side-Effects Review — Fix: active-pull must not demote a solo machine on a transient lease lapse
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `fix-activepull-solo-demotion`
|
|
4
|
+
**Date:** `2026-06-02`
|
|
5
|
+
**Author:** `echo`
|
|
6
|
+
**Spec:** `docs/specs/MULTI-MACHINE-ROBUST-LEASE-PROPAGATION-SPEC.md` (approved: true) — corrects a regression in the active-pull it introduced (#668).
|
|
7
|
+
**Parent principle:** `Cross-Machine Coherence — One Agent, Robust Under Degraded Conditions`
|
|
8
|
+
|
|
9
|
+
## Summary of the change
|
|
10
|
+
|
|
11
|
+
A one-condition fix to `MultiMachineCoordinator.tickLeasePull`: it now only calls
|
|
12
|
+
`reconcileRoleToLease('lease-pull')` + `surfacePullDiscoveredSplitBrain()` when a peer
|
|
13
|
+
lease was **actually observed** (`leaseCoordinator.observedPeerLease()` non-null).
|
|
14
|
+
|
|
15
|
+
### Why (live incident, 2026-06-02)
|
|
16
|
+
|
|
17
|
+
The active-pull loop (shipped in #668) reconciled the machine's lease role
|
|
18
|
+
**unconditionally** every `leasePullIntervalMs` (~5s). On a solo machine — one with no
|
|
19
|
+
reachable peers (the only registry peer was dead 3 days, no `lastKnownUrl`, so `peers()`
|
|
20
|
+
was empty) — every brief lapse of the 60s self-lease between renewals made
|
|
21
|
+
`holdsLease()` return false, so the pull loop flipped the machine to **standby** →
|
|
22
|
+
`StateManager` read-only → a standby write threw and (pre-#673) crashed the server in a
|
|
23
|
+
launchd restart loop (lease epoch climbed into the thousands). The crashing server is the
|
|
24
|
+
Telegram relay, so the operator saw silence. #673 already stopped the *crash* (read-only
|
|
25
|
+
write is now an isolated/recoverable uncaught-exception class); this PR removes the *root
|
|
26
|
+
cause* (the spurious demotion itself).
|
|
27
|
+
|
|
28
|
+
## Decision-point inventory
|
|
29
|
+
|
|
30
|
+
- `tickLeasePull` reconcile vs skip: reconcile only when `observedPeerLease()` is non-null.
|
|
31
|
+
Pull is for LEARNING from peers; with no peer signal there is nothing to reconcile.
|
|
32
|
+
Role/renewal for a solo holder stays with the heartbeat `tickLease` (which RE-ACQUIRES,
|
|
33
|
+
not merely demotes).
|
|
34
|
+
|
|
35
|
+
## 1. Over-correction risk
|
|
36
|
+
Does gating on an observed peer break the real feature (a standby pulling a higher-epoch
|
|
37
|
+
holder must still demote)? No — that path records the holder's lease via `recordObserved`,
|
|
38
|
+
so `observedPeerLease()` is non-null and the reconcile runs exactly as before. Covered by
|
|
39
|
+
the retained "same-epoch contested" test.
|
|
40
|
+
|
|
41
|
+
## 2. Under-correction risk
|
|
42
|
+
A solo holder whose lease lapses is no longer demoted by the *pull* loop, but the 2-min
|
|
43
|
+
heartbeat `tickLease` still renews/re-acquires it — so role management is preserved, just
|
|
44
|
+
moved off the aggressive 5s pull cadence that caused the DoS.
|
|
45
|
+
|
|
46
|
+
## 3. Level-of-abstraction fit
|
|
47
|
+
The fix sits in the pull tick (the loop that caused the regression), gating on the
|
|
48
|
+
coordinator's existing `observedPeerLease()` accessor — no new surface.
|
|
49
|
+
|
|
50
|
+
## 4. Signal vs Authority
|
|
51
|
+
The pull remains a SIGNAL (observe peer leases); the FencedLease epoch/validity checks
|
|
52
|
+
remain the authority for who holds. The fix stops the signal loop from ACTING (demoting)
|
|
53
|
+
in the absence of any signal.
|
|
54
|
+
|
|
55
|
+
## 5. External surfaces
|
|
56
|
+
None. No route/config/schema change. `leasePullIntervalMs` unchanged.
|
|
57
|
+
|
|
58
|
+
## 6. Interactions with existing primitives
|
|
59
|
+
Composes with #673 (read-only-write no longer fatal) as defense-in-depth: #673 prevents
|
|
60
|
+
the crash; this prevents the wrong state that led to it. The heartbeat `tickLease` and the
|
|
61
|
+
acquire path are untouched.
|
|
62
|
+
|
|
63
|
+
## 7. Rollback cost
|
|
64
|
+
Trivial + safe: a single `if (observedPeerLease())` guard. Reverting restores the prior
|
|
65
|
+
(buggy) unconditional reconcile.
|
|
66
|
+
|
|
67
|
+
## Migration parity
|
|
68
|
+
None — pure code logic change, reaches existing agents on the normal dist update.
|
|
69
|
+
|
|
70
|
+
## Tests
|
|
71
|
+
`tsc --noEmit` clean. 45 multi-machine tests green, including a NEW regression
|
|
72
|
+
(`MultiMachineCoordinator-leasePull` — a solo holder whose self-lease lapses with no peer
|
|
73
|
+
observed stays awake + never goes read-only) and the retained same-epoch-contested test
|
|
74
|
+
(feature intact). Operational note: echo was reverted to solo mode during the incident
|
|
75
|
+
(machine identity moved aside); re-pairing its mesh is a separate, operator-gated step.
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# Side-Effects Review — Multi-Machine Robust Lease Propagation (active PULL)
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `multi-machine-robust-lease-propagation`
|
|
4
|
+
**Date:** `2026-06-01`
|
|
5
|
+
**Author:** `echo`
|
|
6
|
+
**Second-pass reviewer:** `independent lessons-aware convergence review (2 passes → converged)`
|
|
7
|
+
**Spec:** `docs/specs/MULTI-MACHINE-ROBUST-LEASE-PROPAGATION-SPEC.md` (approved: true, approved-with-change)
|
|
8
|
+
**Parent principle:** `Cross-Machine Coherence — One Agent, Robust Under Degraded Conditions`
|
|
9
|
+
|
|
10
|
+
## Summary of the change
|
|
11
|
+
|
|
12
|
+
Adds an **active lease PULL** path to the cross-machine fenced-lease layer, so a
|
|
13
|
+
standby is no longer blind on a quiet or one-way (NAT) network where the holder's
|
|
14
|
+
broadcast never arrives. The transport gains a `pullPeer`/`pullAllPeers` over the
|
|
15
|
+
authenticated channel (`POST /api/lease/pull`, machine-auth is body-hash based so a
|
|
16
|
+
signed empty body authenticates cleanly); the coordinator runs a constant-cadence,
|
|
17
|
+
jittered pull loop independent of holder liveness, folds a pulled higher-epoch lease
|
|
18
|
+
(auto-demoting a fenced holder) and surfaces a same-epoch contested split-brain
|
|
19
|
+
Near-Silently (logs + `getSyncStatus().splitBrainState='contested'`), which the
|
|
20
|
+
registry `awakeMachineCount` misses in a git-less mesh.
|
|
21
|
+
|
|
22
|
+
This is **build + test only**: the loop arms only when a pull-capable transport is
|
|
23
|
+
attached and multi-machine is configured; no production lease, stage, or config is
|
|
24
|
+
flipped by this change. The session-pool rollout `StageAdvancer` is now wired to a
|
|
25
|
+
**revert-only** reconcile tick (it can demote a live stage on a red E2E for the current
|
|
26
|
+
commit; promotion stays operator-triggered) and its commit SHA is boot-cached from git.
|
|
27
|
+
|
|
28
|
+
## Decision-point inventory
|
|
29
|
+
|
|
30
|
+
- `HttpLeaseTransport.pullPeer` returns the peer lease / `null` (network error, non-ok,
|
|
31
|
+
or no lease are all advisory `null` — never thrown); a successful pull (even one
|
|
32
|
+
carrying no lease) marks the medium reachable (bidirectional `isReachable`).
|
|
33
|
+
- `LeaseCoordinator.effectiveView` fold: a pulled lease overrides only when STRICTLY
|
|
34
|
+
higher-epoch (same-epoch self-issued wins the tie) — the existing rule, unchanged.
|
|
35
|
+
- `surfacePullDiscoveredSplitBrain` contested vs clear: we still hold AND a raw observed
|
|
36
|
+
peer lease names a different holder at our epoch → latch `contested` (rising-edge log).
|
|
37
|
+
- `StageAdvancer.reconcile` revert vs hold: red E2E for the CURRENT commit → revert one
|
|
38
|
+
stage; otherwise hold (promotion is never automatic).
|
|
39
|
+
|
|
40
|
+
## 1. Over-block / over-act risk
|
|
41
|
+
The pull loop never ACTS on roles — it only folds observed leases and reconciles role to
|
|
42
|
+
`holdsLease()`. A pulled higher epoch demotes us (correct fencing); a same-epoch tie does
|
|
43
|
+
NOT demote (it surfaces as contested, the genuinely-unresolvable case the operator owns).
|
|
44
|
+
The reconcile tick can only demote a stage on a verified red, never advance.
|
|
45
|
+
|
|
46
|
+
## 2. Under-act risk
|
|
47
|
+
A quiet/one-way network previously left a standby blind (the bug this closes). The
|
|
48
|
+
constant cadence — independent of whether the holder looks alive — is the anti-blinding
|
|
49
|
+
guarantee; the startup invariant `leasePullIntervalMs < leaseTtlMs` ensures at least one
|
|
50
|
+
pull per lease lifetime.
|
|
51
|
+
|
|
52
|
+
## 3. Level-of-abstraction fit
|
|
53
|
+
The pull lives at the transport (wire) + coordinator (lease authority) layers — the same
|
|
54
|
+
layers that own broadcast/observe — not bolted onto a sentinel or prompt. The route is a
|
|
55
|
+
sibling of the existing `POST /api/lease` receive route.
|
|
56
|
+
|
|
57
|
+
## 4. Signal vs Authority
|
|
58
|
+
The pull is a LEARNING signal (folds observed state); the FencedLease epoch/signature
|
|
59
|
+
checks remain the sole authority for who holds. A pull cannot grant authority — only the
|
|
60
|
+
CAS/fence can. Split-brain surfacing is observability only (no auto-demote on a tie).
|
|
61
|
+
|
|
62
|
+
## 5. External surfaces
|
|
63
|
+
Adds `POST /api/lease/pull` (machine-auth gated; returns the responder's effective-view
|
|
64
|
+
lease, which may re-serve a third machine — there is intentionally NO holder==responder
|
|
65
|
+
guard since the puller re-verifies via `acceptTunnelLease`). No new outbound external
|
|
66
|
+
calls beyond peer pulls over the already-authenticated machine channel.
|
|
67
|
+
|
|
68
|
+
## 6. Interactions with existing primitives
|
|
69
|
+
`pullPeer`/`pullAllPeers` are OPTIONAL on the `LeaseTransport` interface, so existing
|
|
70
|
+
mock/git-only transports remain valid and `canPullPeers()` is false there (loop no-ops).
|
|
71
|
+
`leasePullIntervalMs` joins the resolved seamlessness config with a startup invariant,
|
|
72
|
+
de-conflicted from the existing `standbyPullIntervalMs` (git pull) by name and purpose.
|
|
73
|
+
The pull-tick reconcile reuses `reconcileRoleToLease`; the contested flag ORs into the
|
|
74
|
+
existing `getSyncStatus` split-brain computation.
|
|
75
|
+
|
|
76
|
+
## 7. Rollback cost
|
|
77
|
+
All additive + default-safe. The pull loop only arms with a pull-capable transport +
|
|
78
|
+
multi-machine config; `stop()` clears the timer (stopped-guard prevents resurrection from
|
|
79
|
+
an in-flight tick's re-arm). Disabling = set `leasePullIntervalMs` ≥ `leaseTtlMs` (caught
|
|
80
|
+
at startup) or run a git-only transport. No production lease/stage state is changed by
|
|
81
|
+
this commit.
|
|
82
|
+
|
|
83
|
+
## Migration parity
|
|
84
|
+
- `leasePullIntervalMs` is a code-defaulted (5s) optional knob resolved + validated in
|
|
85
|
+
`seamlessnessConfig` (matches its siblings — not seeded into config.json), so existing
|
|
86
|
+
agents get the default with no migration; a nonsensical value is rejected at startup.
|
|
87
|
+
- The CLAUDE.md dials note gains `leasePullIntervalMs` in BOTH the generator
|
|
88
|
+
(`templates.ts`) and the migrator (`PostUpdateMigrator.ts` `migrateClaudeMd`) so new
|
|
89
|
+
and existing agents both learn the dial.
|
|
90
|
+
|
|
91
|
+
## Tests
|
|
92
|
+
108 green across the touched area + 3 tiers: unit (`HttpLeaseTransport` pull cases,
|
|
93
|
+
`LeaseCoordinator` pull accessors + same-epoch masking, `seamlessnessConfig`
|
|
94
|
+
leasePullIntervalMs invariant, `MultiMachineCoordinator-leasePull` loop→contested),
|
|
95
|
+
integration (`machine-routes` `POST /api/lease/pull` serve/null/401), e2e
|
|
96
|
+
(`multi-machine-lease-split-brain` partition→two-awake→pull-converge-to-one over real
|
|
97
|
+
lease components + mock HTTP). `tsc --noEmit` clean. Test-writing caught a real bug: the
|
|
98
|
+
`buildAcquisition` 3rd arg is the NONCE not the epoch (the first draft built epoch-1
|
|
99
|
+
leases when it meant epoch-5), and the same-epoch split-brain only forms when the peer
|
|
100
|
+
lease appears AFTER local acquisition.
|
|
101
|
+
|
|
102
|
+
## Follow-up (CI shard 4/4, post-merge-with-main)
|
|
103
|
+
|
|
104
|
+
`initializeLease()` → `startLeasePullLoop()` called `leaseCoordinator.canPullPeers()`
|
|
105
|
+
unconditionally, which threw for an injected coordinator that does not implement the
|
|
106
|
+
pull API — the existing `multi-machine-coordinator.test.ts` partial double (`fakeLease`)
|
|
107
|
+
lacks it, and so would any build predating active-pull. Hardened the loop start with a
|
|
108
|
+
`typeof this.leaseCoordinator.canPullPeers === 'function'` guard so such a coordinator
|
|
109
|
+
simply skips the loop (the intended git-only-mesh / partial-double behavior) instead of
|
|
110
|
+
crashing. Caught by the full CI unit matrix (shard 4/4, both Node 20 and 22) after the
|
|
111
|
+
merge with main brought the existing coordinator test into scope; 28 multi-machine tests
|
|
112
|
+
green, `tsc --noEmit` clean.
|
|
113
|
+
|
|
114
|
+
Also regenerated the generated `src/data/builtin-manifest.json` (via
|
|
115
|
+
`npm run generate:manifest`): an earlier revert to drop version churn from the PR diff
|
|
116
|
+
left it stale at v1.3.154, but `builtin-manifest.test.ts` ("is up-to-date with current
|
|
117
|
+
source") compares against the regenerated manifest for current source (now v1.3.192
|
|
118
|
+
post-merge). No behavior change — it is a generated data file. (`esm-compliance.test.ts`'s
|
|
119
|
+
local `require()` findings are byte-identical to main and present on main, so they are
|
|
120
|
+
pre-existing and pass in CI — not introduced here.)
|