instar 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +26 -3
- package/dist/cli.js.map +1 -1
- package/dist/lifeline/ServerSupervisor.d.ts.map +1 -1
- package/dist/lifeline/ServerSupervisor.js +23 -1
- package/dist/lifeline/ServerSupervisor.js.map +1 -1
- package/dist/lifeline/TelegramLifeline.d.ts +15 -0
- package/dist/lifeline/TelegramLifeline.d.ts.map +1 -1
- package/dist/lifeline/TelegramLifeline.js +107 -8
- package/dist/lifeline/TelegramLifeline.js.map +1 -1
- package/dist/lifeline/rateLimitState.d.ts.map +1 -1
- package/dist/lifeline/rateLimitState.js +8 -1
- package/dist/lifeline/rateLimitState.js.map +1 -1
- package/dist/memory/NativeModuleHealer.d.ts.map +1 -1
- package/dist/memory/NativeModuleHealer.js +8 -1
- package/dist/memory/NativeModuleHealer.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +32 -32
- package/upgrades/1.1.3.md +124 -0
- package/upgrades/side-effects/lifeline-version-skew-recovery.md +180 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-05-
|
|
5
|
-
"instarVersion": "1.1.
|
|
4
|
+
"generatedAt": "2026-05-21T00:22:35.171Z",
|
|
5
|
+
"instarVersion": "1.1.3",
|
|
6
6
|
"entryCount": 190,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -752,7 +752,7 @@
|
|
|
752
752
|
"type": "cli-command",
|
|
753
753
|
"domain": "setup",
|
|
754
754
|
"sourcePath": "src/cli.ts",
|
|
755
|
-
"contentHash": "
|
|
755
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
756
756
|
"since": "2025-01-01"
|
|
757
757
|
},
|
|
758
758
|
"cli:setup": {
|
|
@@ -760,7 +760,7 @@
|
|
|
760
760
|
"type": "cli-command",
|
|
761
761
|
"domain": "setup",
|
|
762
762
|
"sourcePath": "src/cli.ts",
|
|
763
|
-
"contentHash": "
|
|
763
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
764
764
|
"since": "2025-01-01"
|
|
765
765
|
},
|
|
766
766
|
"cli:add": {
|
|
@@ -768,7 +768,7 @@
|
|
|
768
768
|
"type": "cli-command",
|
|
769
769
|
"domain": "setup",
|
|
770
770
|
"sourcePath": "src/cli.ts",
|
|
771
|
-
"contentHash": "
|
|
771
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
772
772
|
"since": "2025-01-01"
|
|
773
773
|
},
|
|
774
774
|
"cli:backup": {
|
|
@@ -776,7 +776,7 @@
|
|
|
776
776
|
"type": "cli-command",
|
|
777
777
|
"domain": "operations",
|
|
778
778
|
"sourcePath": "src/cli.ts",
|
|
779
|
-
"contentHash": "
|
|
779
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
780
780
|
"since": "2025-01-01"
|
|
781
781
|
},
|
|
782
782
|
"cli:git": {
|
|
@@ -784,7 +784,7 @@
|
|
|
784
784
|
"type": "cli-command",
|
|
785
785
|
"domain": "coordination",
|
|
786
786
|
"sourcePath": "src/cli.ts",
|
|
787
|
-
"contentHash": "
|
|
787
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
788
788
|
"since": "2025-01-01"
|
|
789
789
|
},
|
|
790
790
|
"cli:memory": {
|
|
@@ -792,7 +792,7 @@
|
|
|
792
792
|
"type": "cli-command",
|
|
793
793
|
"domain": "memory",
|
|
794
794
|
"sourcePath": "src/cli.ts",
|
|
795
|
-
"contentHash": "
|
|
795
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
796
796
|
"since": "2025-01-01"
|
|
797
797
|
},
|
|
798
798
|
"cli:knowledge": {
|
|
@@ -800,7 +800,7 @@
|
|
|
800
800
|
"type": "cli-command",
|
|
801
801
|
"domain": "memory",
|
|
802
802
|
"sourcePath": "src/cli.ts",
|
|
803
|
-
"contentHash": "
|
|
803
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
804
804
|
"since": "2025-01-01"
|
|
805
805
|
},
|
|
806
806
|
"cli:semantic": {
|
|
@@ -808,7 +808,7 @@
|
|
|
808
808
|
"type": "cli-command",
|
|
809
809
|
"domain": "memory",
|
|
810
810
|
"sourcePath": "src/cli.ts",
|
|
811
|
-
"contentHash": "
|
|
811
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
812
812
|
"since": "2025-01-01"
|
|
813
813
|
},
|
|
814
814
|
"cli:intent": {
|
|
@@ -816,7 +816,7 @@
|
|
|
816
816
|
"type": "cli-command",
|
|
817
817
|
"domain": "intent",
|
|
818
818
|
"sourcePath": "src/cli.ts",
|
|
819
|
-
"contentHash": "
|
|
819
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
820
820
|
"since": "2025-01-01"
|
|
821
821
|
},
|
|
822
822
|
"cli:feedback": {
|
|
@@ -824,7 +824,7 @@
|
|
|
824
824
|
"type": "cli-command",
|
|
825
825
|
"domain": "feedback",
|
|
826
826
|
"sourcePath": "src/cli.ts",
|
|
827
|
-
"contentHash": "
|
|
827
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
828
828
|
"since": "2025-01-01"
|
|
829
829
|
},
|
|
830
830
|
"cli:server": {
|
|
@@ -832,7 +832,7 @@
|
|
|
832
832
|
"type": "cli-command",
|
|
833
833
|
"domain": "server",
|
|
834
834
|
"sourcePath": "src/cli.ts",
|
|
835
|
-
"contentHash": "
|
|
835
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
836
836
|
"since": "2025-01-01"
|
|
837
837
|
},
|
|
838
838
|
"cli:status": {
|
|
@@ -840,7 +840,7 @@
|
|
|
840
840
|
"type": "cli-command",
|
|
841
841
|
"domain": "monitoring",
|
|
842
842
|
"sourcePath": "src/cli.ts",
|
|
843
|
-
"contentHash": "
|
|
843
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
844
844
|
"since": "2025-01-01"
|
|
845
845
|
},
|
|
846
846
|
"cli:user": {
|
|
@@ -848,7 +848,7 @@
|
|
|
848
848
|
"type": "cli-command",
|
|
849
849
|
"domain": "users",
|
|
850
850
|
"sourcePath": "src/cli.ts",
|
|
851
|
-
"contentHash": "
|
|
851
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
852
852
|
"since": "2025-01-01"
|
|
853
853
|
},
|
|
854
854
|
"cli:relationship": {
|
|
@@ -856,7 +856,7 @@
|
|
|
856
856
|
"type": "cli-command",
|
|
857
857
|
"domain": "relationships",
|
|
858
858
|
"sourcePath": "src/cli.ts",
|
|
859
|
-
"contentHash": "
|
|
859
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
860
860
|
"since": "2025-01-01"
|
|
861
861
|
},
|
|
862
862
|
"cli:job": {
|
|
@@ -864,7 +864,7 @@
|
|
|
864
864
|
"type": "cli-command",
|
|
865
865
|
"domain": "scheduling",
|
|
866
866
|
"sourcePath": "src/cli.ts",
|
|
867
|
-
"contentHash": "
|
|
867
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
868
868
|
"since": "2025-01-01"
|
|
869
869
|
},
|
|
870
870
|
"cli:lifeline": {
|
|
@@ -872,7 +872,7 @@
|
|
|
872
872
|
"type": "cli-command",
|
|
873
873
|
"domain": "communication",
|
|
874
874
|
"sourcePath": "src/cli.ts",
|
|
875
|
-
"contentHash": "
|
|
875
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
876
876
|
"since": "2025-01-01"
|
|
877
877
|
},
|
|
878
878
|
"cli:list": {
|
|
@@ -880,7 +880,7 @@
|
|
|
880
880
|
"type": "cli-command",
|
|
881
881
|
"domain": "monitoring",
|
|
882
882
|
"sourcePath": "src/cli.ts",
|
|
883
|
-
"contentHash": "
|
|
883
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
884
884
|
"since": "2025-01-01"
|
|
885
885
|
},
|
|
886
886
|
"cli:autostart": {
|
|
@@ -888,7 +888,7 @@
|
|
|
888
888
|
"type": "cli-command",
|
|
889
889
|
"domain": "operations",
|
|
890
890
|
"sourcePath": "src/cli.ts",
|
|
891
|
-
"contentHash": "
|
|
891
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
892
892
|
"since": "2025-01-01"
|
|
893
893
|
},
|
|
894
894
|
"cli:migrate": {
|
|
@@ -896,7 +896,7 @@
|
|
|
896
896
|
"type": "cli-command",
|
|
897
897
|
"domain": "updates",
|
|
898
898
|
"sourcePath": "src/cli.ts",
|
|
899
|
-
"contentHash": "
|
|
899
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
900
900
|
"since": "2025-01-01"
|
|
901
901
|
},
|
|
902
902
|
"cli:machines": {
|
|
@@ -904,7 +904,7 @@
|
|
|
904
904
|
"type": "cli-command",
|
|
905
905
|
"domain": "coordination",
|
|
906
906
|
"sourcePath": "src/cli.ts",
|
|
907
|
-
"contentHash": "
|
|
907
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
908
908
|
"since": "2025-01-01"
|
|
909
909
|
},
|
|
910
910
|
"cli:whoami": {
|
|
@@ -912,7 +912,7 @@
|
|
|
912
912
|
"type": "cli-command",
|
|
913
913
|
"domain": "identity",
|
|
914
914
|
"sourcePath": "src/cli.ts",
|
|
915
|
-
"contentHash": "
|
|
915
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
916
916
|
"since": "2025-01-01"
|
|
917
917
|
},
|
|
918
918
|
"cli:pair": {
|
|
@@ -920,7 +920,7 @@
|
|
|
920
920
|
"type": "cli-command",
|
|
921
921
|
"domain": "coordination",
|
|
922
922
|
"sourcePath": "src/cli.ts",
|
|
923
|
-
"contentHash": "
|
|
923
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
924
924
|
"since": "2025-01-01"
|
|
925
925
|
},
|
|
926
926
|
"cli:join": {
|
|
@@ -928,7 +928,7 @@
|
|
|
928
928
|
"type": "cli-command",
|
|
929
929
|
"domain": "coordination",
|
|
930
930
|
"sourcePath": "src/cli.ts",
|
|
931
|
-
"contentHash": "
|
|
931
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
932
932
|
"since": "2025-01-01"
|
|
933
933
|
},
|
|
934
934
|
"cli:wakeup": {
|
|
@@ -936,7 +936,7 @@
|
|
|
936
936
|
"type": "cli-command",
|
|
937
937
|
"domain": "coordination",
|
|
938
938
|
"sourcePath": "src/cli.ts",
|
|
939
|
-
"contentHash": "
|
|
939
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
940
940
|
"since": "2025-01-01"
|
|
941
941
|
},
|
|
942
942
|
"cli:leave": {
|
|
@@ -944,7 +944,7 @@
|
|
|
944
944
|
"type": "cli-command",
|
|
945
945
|
"domain": "coordination",
|
|
946
946
|
"sourcePath": "src/cli.ts",
|
|
947
|
-
"contentHash": "
|
|
947
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
948
948
|
"since": "2025-01-01"
|
|
949
949
|
},
|
|
950
950
|
"cli:doctor": {
|
|
@@ -952,7 +952,7 @@
|
|
|
952
952
|
"type": "cli-command",
|
|
953
953
|
"domain": "monitoring",
|
|
954
954
|
"sourcePath": "src/cli.ts",
|
|
955
|
-
"contentHash": "
|
|
955
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
956
956
|
"since": "2025-01-01"
|
|
957
957
|
},
|
|
958
958
|
"cli:review": {
|
|
@@ -960,7 +960,7 @@
|
|
|
960
960
|
"type": "cli-command",
|
|
961
961
|
"domain": "monitoring",
|
|
962
962
|
"sourcePath": "src/cli.ts",
|
|
963
|
-
"contentHash": "
|
|
963
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
964
964
|
"since": "2025-01-01"
|
|
965
965
|
},
|
|
966
966
|
"cli:nuke": {
|
|
@@ -968,7 +968,7 @@
|
|
|
968
968
|
"type": "cli-command",
|
|
969
969
|
"domain": "operations",
|
|
970
970
|
"sourcePath": "src/cli.ts",
|
|
971
|
-
"contentHash": "
|
|
971
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
972
972
|
"since": "2025-01-01"
|
|
973
973
|
},
|
|
974
974
|
"cli:playbook": {
|
|
@@ -976,7 +976,7 @@
|
|
|
976
976
|
"type": "cli-command",
|
|
977
977
|
"domain": "context",
|
|
978
978
|
"sourcePath": "src/cli.ts",
|
|
979
|
-
"contentHash": "
|
|
979
|
+
"contentHash": "ef620787cb2d40a51831b00ba7071e67aa0d4cac86027dc3a04635d344608d29",
|
|
980
980
|
"since": "2025-01-01"
|
|
981
981
|
},
|
|
982
982
|
"template:build-stop-hook.sh": {
|
|
@@ -1496,7 +1496,7 @@
|
|
|
1496
1496
|
"type": "subsystem",
|
|
1497
1497
|
"domain": "communication",
|
|
1498
1498
|
"sourcePath": "src/lifeline/TelegramLifeline.ts",
|
|
1499
|
-
"contentHash": "
|
|
1499
|
+
"contentHash": "45cd2b449e34134cae67859e79e378ccc131f481b4d5701faf26913b8a3a26e2",
|
|
1500
1500
|
"since": "2025-01-01"
|
|
1501
1501
|
},
|
|
1502
1502
|
"subsystem:orphan-process-reaper": {
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- bump: patch -->
|
|
4
|
+
<!-- patch = bug fixes, refactors, test additions, doc updates -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
**fix(lifeline): close the silent-drop version-skew failure class.**
|
|
9
|
+
|
|
10
|
+
Five interlocking fixes that together address the 2026-05-19 →
|
|
11
|
+
2026-05-20 b2lead-insights field incident (21h silent Telegram ingress
|
|
12
|
+
drop after a server auto-update from v1.0.13 → v1.1.0 left the
|
|
13
|
+
lifeline on v1.0.13; server's `/internal/telegram-forward` rejected
|
|
14
|
+
the lifeline with HTTP 426 and the lifeline's drop-after-3-failures
|
|
15
|
+
replay policy silently lost user messages).
|
|
16
|
+
|
|
17
|
+
1. **`rateLimitState.decide` — `versionSkew` bucket bypasses
|
|
18
|
+
cooldown.** A hard incompatibility cannot be cured by waiting; the
|
|
19
|
+
per-minute cooldown wedged every restart attempt. The daily cap
|
|
20
|
+
(3 per 24h) and storm detection remain as backstops.
|
|
21
|
+
|
|
22
|
+
2. **Replay-loop drop policy is gated on a version-skew episode
|
|
23
|
+
flag.** When `forwardToServer` raises `ForwardVersionSkewError`,
|
|
24
|
+
the lifeline sets `versionSkewActive = true` and sends one
|
|
25
|
+
user-visible Telegram alert ("ingress paused: version skew
|
|
26
|
+
detected, your messages are not lost"). The replay loop checks
|
|
27
|
+
the flag BEFORE the `MAX_REPLAY_FAILURES` drop check and re-queues
|
|
28
|
+
without incrementing the failure counter. Cleared on the next
|
|
29
|
+
successful forward.
|
|
30
|
+
|
|
31
|
+
3. **`instar lifeline restart` CLI uses the correct service label.**
|
|
32
|
+
Was `com.instar.<projectName>.lifeline` — different domain AND
|
|
33
|
+
different suffix from what `installMacOSLaunchAgent` writes
|
|
34
|
+
(`ai.instar.<projectName>`). `launchctl kickstart` always failed,
|
|
35
|
+
silently fell back to pkill.
|
|
36
|
+
|
|
37
|
+
4. **pkill fallback escalates to SIGKILL; lock-acquire recognizes
|
|
38
|
+
wedged sleeping processes.** SIGTERM, 3s grace, SIGKILL by name
|
|
39
|
+
pattern in the CLI. In `acquireLockFile`, 'S' (sleeping) state
|
|
40
|
+
with lock-write age > 5 min is treated as a wedged lifeline:
|
|
41
|
+
SIGTERM → 3s poll → SIGKILL → take the lock.
|
|
42
|
+
|
|
43
|
+
5. **`npm rebuild` for better-sqlite3 uses `--build-from-source` and
|
|
44
|
+
`--ignore-scripts`.** Two call sites: `ServerSupervisor`
|
|
45
|
+
preflight rebuild and `NativeModuleHealer.healBetterSqlite3Sync`.
|
|
46
|
+
Without `--build-from-source`, npm can install a cached prebuilt
|
|
47
|
+
with the same wrong ABI and exit 0 — producing the "rebuild
|
|
48
|
+
succeeded but module still fails to load" pattern in the b2lead
|
|
49
|
+
logs.
|
|
50
|
+
|
|
51
|
+
Files touched (excluding tests / docs / upgrade notes):
|
|
52
|
+
- `src/lifeline/rateLimitState.ts` — `decide()`
|
|
53
|
+
- `src/lifeline/TelegramLifeline.ts` — instance state,
|
|
54
|
+
`handleVersionSkew`, `forwardToServer`, `replayQueue`,
|
|
55
|
+
`acquireLockFile`
|
|
56
|
+
- `src/cli.ts` — `lifelineCmd.command('restart')`
|
|
57
|
+
- `src/lifeline/ServerSupervisor.ts` — preflight rebuild argv
|
|
58
|
+
- `src/memory/NativeModuleHealer.ts` — in-line rebuild argv
|
|
59
|
+
|
|
60
|
+
## Evidence
|
|
61
|
+
|
|
62
|
+
**Repro of the original failure** (b2lead-insights, 2026-05-19 evening
|
|
63
|
+
→ 2026-05-20 ~17:50 UTC):
|
|
64
|
+
|
|
65
|
+
1. Server auto-updated v1.0.13 → v1.1.0.
|
|
66
|
+
2. Lifeline kept running v1.0.13.
|
|
67
|
+
3. User Telegram messages → lifeline forwards to server → server
|
|
68
|
+
returns HTTP 426 with `{ok: false, upgradeRequired: true,
|
|
69
|
+
serverVersion: '1.1.0', action: 'restart', reason: 'major-minor-mismatch'}`.
|
|
70
|
+
4. Lifeline catches `ForwardVersionSkewError`, calls
|
|
71
|
+
`handleVersionSkew` → `initiateRestart('versionSkew', ...)`.
|
|
72
|
+
5. `RestartOrchestrator.maybeRestart` consults `rateLimitState.decide`.
|
|
73
|
+
`elapsed < WATCHDOG_COOLDOWN_MS` → returns `{allowed: false, reason:
|
|
74
|
+
'cooldown-active'}`. No restart.
|
|
75
|
+
6. Lifeline returns false from `forwardToServer`. Replay loop
|
|
76
|
+
increments `replayFailures`. After 3 attempts → drop.
|
|
77
|
+
7. 40+ "server-down" notifications + at least 2 explicitly-logged
|
|
78
|
+
dropped messages (`tg-6577`, `tg-6580`). No user-visible alert.
|
|
79
|
+
|
|
80
|
+
**Observed before:** `rateLimitState.ts:112` had no special-case for
|
|
81
|
+
`versionSkew` — the cooldown check fired for all buckets. The replay
|
|
82
|
+
loop at `TelegramLifeline.ts:1394` checked `failures >=
|
|
83
|
+
MAX_REPLAY_FAILURES` unconditionally. The CLI restart command at
|
|
84
|
+
`cli.ts:1639` hardcoded `com.instar.${projectName}.lifeline`.
|
|
85
|
+
`ServerSupervisor.ts:751` and `NativeModuleHealer.ts:397` both ran
|
|
86
|
+
`npm rebuild better-sqlite3` without `--build-from-source`.
|
|
87
|
+
|
|
88
|
+
**Observed after:**
|
|
89
|
+
- 26 new + extended unit tests (8 new in
|
|
90
|
+
`tests/unit/lifeline/version-skew-recovery.test.ts`, 1 new in
|
|
91
|
+
`tests/unit/lifeline/rateLimitState.test.ts`) cover all five paths.
|
|
92
|
+
- Source assertions verify the supervisor rebuild argv includes
|
|
93
|
+
`--build-from-source` AND `--ignore-scripts`, the healer in-line
|
|
94
|
+
rebuild includes both flags, the CLI restart command builds
|
|
95
|
+
`ai.instar.${...}` and never `com.instar.${...}.lifeline`, the
|
|
96
|
+
pkill fallback contains both `-TERM` and `-KILL`, the
|
|
97
|
+
`versionSkewActive` flag is set in `handleVersionSkew`, cleared in
|
|
98
|
+
`forwardToServer` success, checked in `replayQueue` BEFORE the
|
|
99
|
+
drop branch, and the lock-acquire path recognizes wedged-S.
|
|
100
|
+
|
|
101
|
+
Spec: `docs/specs/lifeline-version-skew-recovery.md`
|
|
102
|
+
ELI16: `docs/specs/lifeline-version-skew-recovery.eli16.md`
|
|
103
|
+
Side-effects review: `upgrades/side-effects/lifeline-version-skew-recovery.md`
|
|
104
|
+
|
|
105
|
+
## What to Tell Your User
|
|
106
|
+
|
|
107
|
+
- **You will never silently lose a message during an auto-update
|
|
108
|
+
again**: if my server and my Telegram lifeline temporarily get out
|
|
109
|
+
of sync after an update, I'll send you one clear note that ingress
|
|
110
|
+
is paused, keep all your messages safely in the queue, and replay
|
|
111
|
+
them as soon as the lifeline catches up.
|
|
112
|
+
- **Recovery is automatic**: the bug class that needed a manual
|
|
113
|
+
restart to clear is now self-healing. No operator intervention.
|
|
114
|
+
|
|
115
|
+
## Summary of New Capabilities
|
|
116
|
+
|
|
117
|
+
| Capability | How to Use |
|
|
118
|
+
|-----------|-----------|
|
|
119
|
+
| Version-skew cooldown bypass | automatic |
|
|
120
|
+
| Version-skew user alert (one per episode) | automatic |
|
|
121
|
+
| No-drop policy during version-skew episodes | automatic |
|
|
122
|
+
| CLI lifeline restart resolves correct service | automatic via `instar lifeline restart` |
|
|
123
|
+
| Wedged-lifeline lock recovery | automatic |
|
|
124
|
+
| Force-from-source native rebuilds | automatic |
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# Side-Effects Review — Lifeline version-skew recovery
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `lifeline-version-skew-recovery`
|
|
4
|
+
**Date:** `2026-05-20`
|
|
5
|
+
**Author:** Echo (instar developer agent)
|
|
6
|
+
**Trigger:** 2026-05-19 → 2026-05-20 b2lead-insights field incident (21h silent
|
|
7
|
+
Telegram ingress drop after server auto-update from v1.0.13 → v1.1.0;
|
|
8
|
+
lifeline stayed on v1.0.13; HTTP 426 from `/internal/telegram-forward`
|
|
9
|
+
went to 3-attempts-then-drop replay policy; restart attempts blocked by
|
|
10
|
+
cooldown; manual operator recovery required).
|
|
11
|
+
|
|
12
|
+
## Summary of the change
|
|
13
|
+
|
|
14
|
+
Five interlocking fixes that together close the silent-drop failure
|
|
15
|
+
class. All independently revertable.
|
|
16
|
+
|
|
17
|
+
1. **Rate-limit cooldown bypass for `versionSkew` bucket.** A
|
|
18
|
+
no-op-against-mismatch wait is replaced with allow-but-rate-limit
|
|
19
|
+
(daily cap still applies). One-line condition change in
|
|
20
|
+
`decide()`.
|
|
21
|
+
|
|
22
|
+
2. **Drop policy guards on `versionSkewActive` instance flag.** Three
|
|
23
|
+
new state operations:
|
|
24
|
+
- `handleVersionSkew` sets `versionSkewActive = true` and fires
|
|
25
|
+
one user-visible alert (dedupe via `versionSkewAlertSentAt`).
|
|
26
|
+
- `forwardToServer` success clears the flag + dedupe timestamp.
|
|
27
|
+
- `replayQueue` re-queues without incrementing failures while
|
|
28
|
+
the flag is set.
|
|
29
|
+
|
|
30
|
+
3. **CLI service label fix.** `com.instar.<name>.lifeline` →
|
|
31
|
+
`ai.instar.<name>`. Matches the plist label written by
|
|
32
|
+
`installMacOSLaunchAgent`. Stops the always-fall-back-to-pkill path.
|
|
33
|
+
|
|
34
|
+
4. **pkill fallback escalates + lock-acquire recognizes wedged-S.**
|
|
35
|
+
CLI: SIGTERM, 3s grace, SIGKILL by pattern. Lock-acquire: detects
|
|
36
|
+
sleeping ('S') state with lock-write age > 5 min as a wedged
|
|
37
|
+
lifeline, sends its own SIGTERM → 3s poll → SIGKILL sequence.
|
|
38
|
+
|
|
39
|
+
5. **`--build-from-source` + `--ignore-scripts` on every native
|
|
40
|
+
rebuild.** Two call sites: ServerSupervisor preflight rebuild
|
|
41
|
+
(line 751) and NativeModuleHealer.healBetterSqlite3Sync
|
|
42
|
+
(line 397). The Remediator-orchestrated path already had these
|
|
43
|
+
flags (NativeModuleHealer.healBetterSqlite3FromRemediator).
|
|
44
|
+
|
|
45
|
+
Files touched (excluding tests / docs):
|
|
46
|
+
- `src/lifeline/rateLimitState.ts` — `decide()` cooldown bypass
|
|
47
|
+
- `src/lifeline/TelegramLifeline.ts` — instance state, `handleVersionSkew`,
|
|
48
|
+
`forwardToServer`, `replayQueue`, `acquireLockFile`
|
|
49
|
+
- `src/cli.ts` — label fix + pkill escalation
|
|
50
|
+
- `src/lifeline/ServerSupervisor.ts` — rebuild flags
|
|
51
|
+
- `src/memory/NativeModuleHealer.ts` — rebuild flags
|
|
52
|
+
|
|
53
|
+
## Decision matrix
|
|
54
|
+
|
|
55
|
+
### Over-block
|
|
56
|
+
|
|
57
|
+
| Scenario | Outcome |
|
|
58
|
+
|----------|---------|
|
|
59
|
+
| Lifeline at SAME version as server (no skew) | No change — `versionSkewActive` stays false |
|
|
60
|
+
| Lifeline at OLD version, server unchanged (no auto-update) | No change — no 426, no skew |
|
|
61
|
+
| Genuine transient 5xx | Drop-after-3 still fires (versionSkewActive is false) |
|
|
62
|
+
| `ForwardServerBootError` (server starting) | Drop-after-3 still fires — known limitation; this PR doesn't try to also branch on serverBoot. Out of scope. Justified: serverBoot is short-lived (seconds), version skew is steady-state until restart. |
|
|
63
|
+
| Lifeline restart loop after fix | Daily cap (3-per-24h) plus storm detection bound the loop |
|
|
64
|
+
|
|
65
|
+
No over-block. The new authority only fires when ALL of:
|
|
66
|
+
1. The forward threw `ForwardVersionSkewError`,
|
|
67
|
+
2. The server version in the body differs from the lifeline's,
|
|
68
|
+
3. The body's `upgradeRequired === true`.
|
|
69
|
+
|
|
70
|
+
That's the same gate that already triggers the restart request, so we
|
|
71
|
+
haven't widened the surface — just made the consequences during the
|
|
72
|
+
window correct.
|
|
73
|
+
|
|
74
|
+
### Under-block
|
|
75
|
+
|
|
76
|
+
| Scenario | Outcome |
|
|
77
|
+
|----------|---------|
|
|
78
|
+
| Lifeline version "skews" because of a malformed 426 body | Guarded by existing `body.upgradeRequired !== true` and `serverVersion === this.lifelineVersion` checks — they were already in `handleVersionSkew` |
|
|
79
|
+
| Lifeline tries to forge a skew to bypass cooldown | The 426 must come from the SERVER (loopback addr only on a privileged port); the lifeline cannot self-trigger the body shape |
|
|
80
|
+
| Two version-skew episodes in <24h (daily cap reached) | Falls through to existing `version-skew-daily-cap` denial — observability path remains intact |
|
|
81
|
+
| Stuck-lock detection: live healthy lifeline misidentified as wedged | Requires lock-age > 5 min AND 'S' state — a healthy lifeline writes the lock, immediately starts polling Telegram (lots of 'R' time), and answers heartbeats. Misidentification window is narrow but possible during long Telegram long-poll waits. Acceptable: false positive triggers SIGTERM → 3s grace → SIGKILL of a healthy lifeline → launchd respawns it within seconds. Worst case: ~5s downtime + a fresh boot. Far better than the 21h+ silent drop the current code permits. |
|
|
82
|
+
| Forced rebuild with `--build-from-source` fails because node-gyp toolchain absent | Already-existing failure path: spawnSync returns non-zero, healer logs error, falls back to existing escalation. No new silent failure. |
|
|
83
|
+
|
|
84
|
+
### Level of abstraction fit
|
|
85
|
+
|
|
86
|
+
- Rate-limit bypass — at the decision-point that already knows about
|
|
87
|
+
buckets. Right layer.
|
|
88
|
+
- Drop policy — at the replay loop that already has the failure-counter
|
|
89
|
+
state. Right layer. Could have lived in `forwardToServer` instead,
|
|
90
|
+
but the queue is what owns "should this message be dropped" — the
|
|
91
|
+
forward path only owns "did this single attempt succeed".
|
|
92
|
+
- CLI label — at the kickstart callsite. One source of truth.
|
|
93
|
+
- pkill escalation — at the CLI fallback. The CLI is the place where
|
|
94
|
+
"user typed `instar lifeline restart`" gets translated to a system
|
|
95
|
+
action.
|
|
96
|
+
- Lock-acquire — at the lock-acquire function. The detector and the
|
|
97
|
+
recovery primitive are both bounded mechanics in the same scope.
|
|
98
|
+
- Rebuild flags — at the rebuild callsites. No new abstraction.
|
|
99
|
+
|
|
100
|
+
No over-engineering. Each change is at the layer that already owns
|
|
101
|
+
the concern.
|
|
102
|
+
|
|
103
|
+
### Signal-vs-authority compliance
|
|
104
|
+
|
|
105
|
+
- `rateLimitState.decide` — returns signal. No authority change.
|
|
106
|
+
- `versionSkewActive` flag — pure state signal. Replay loop is the
|
|
107
|
+
authority that decides re-queue vs drop.
|
|
108
|
+
- `handleVersionSkew` — already had authority over "request a restart".
|
|
109
|
+
Now also: sets the flag and sends an alert. Both are bounded
|
|
110
|
+
mechanics with clear preconditions, not judgmental gates.
|
|
111
|
+
- CLI label — mechanical change.
|
|
112
|
+
- pkill escalation — bounded recovery (fixed 3s grace, single SIGKILL).
|
|
113
|
+
- Lock-acquire stuck-S — bounded recovery (fixed 5min threshold for
|
|
114
|
+
age, 3s SIGTERM grace, single SIGKILL).
|
|
115
|
+
- Rebuild flags — mechanical change.
|
|
116
|
+
|
|
117
|
+
Compliant. No new judgmental gates; new authorities are bounded
|
|
118
|
+
recovery primitives.
|
|
119
|
+
|
|
120
|
+
### Interactions with adjacent systems
|
|
121
|
+
|
|
122
|
+
| System | Interaction | Risk |
|
|
123
|
+
|--------|-------------|------|
|
|
124
|
+
| `MessageQueue` | Replay loop re-queues messages during skew (same enqueue path used for transient failures) | None — queue persistence semantics unchanged |
|
|
125
|
+
| `notifyMessageDropped` | NOT called during version-skew episodes (drop is suppressed) | Strictly improves — these messages will deliver |
|
|
126
|
+
| `DegradationReporter` | Still fires for non-skew drops; version-skew episode itself emits a degradation via the existing path | None |
|
|
127
|
+
| `RestartOrchestrator` | Receives a `versionSkew` request that's now more likely to be granted (cooldown bypass) | Storm detection + daily cap remain as backstops |
|
|
128
|
+
| `ServerSupervisor.preflightSelfHeal` | Rebuild flag change is a strict improvement | None |
|
|
129
|
+
| Remediator (Tier 1+) | Not yet shipped; in-line healer is the steady-state path. When Remediator ships, its rebuild already used `--build-from-source` — no conflict | None |
|
|
130
|
+
| `NativeModuleHealer.invokeFromRemediator` | Unchanged (already had the flag) | None |
|
|
131
|
+
| `LifelineHealthWatchdog` | Tracks `consecutiveForwardFailures`; this PR clears it on success (existing behavior). No change to the watchdog's trip conditions | None |
|
|
132
|
+
| `WatchdogTriggers.versionSkew` | The flag-set + alert-send happens BEFORE `initiateRestart`. If the restart is rate-limited, the alert still fires — good (user knows ingress is paused even while the lifeline is being patient about restarts) | None |
|
|
133
|
+
| `acquireLockFile` callers | The added SIGTERM-then-SIGKILL path runs in the boot-startup window. New lifelines that find a wedged old one will spend up to 3s waiting for graceful exit. Acceptable | None |
|
|
134
|
+
| Existing tests | All 18 `rateLimitState.test.ts` + 14 `lifeline-shadow-install-self-heal.test.ts` + 9 `NativeModuleHealer.test.ts` tests pass unchanged | Verified |
|
|
135
|
+
|
|
136
|
+
### Rollback cost
|
|
137
|
+
|
|
138
|
+
- Each fix is independently revertable.
|
|
139
|
+
- No persistent state schema changes; the new lifeline instance
|
|
140
|
+
fields (`versionSkewActive`, `versionSkewAlertSentAt`) are not
|
|
141
|
+
serialized to disk.
|
|
142
|
+
- No migration entry; existing in-the-wild agents pick up the fix on
|
|
143
|
+
their next `npx instar` update like any other patch.
|
|
144
|
+
- Reverting the rebuild flag returns to the pre-existing flaky
|
|
145
|
+
rebuild behavior — agents that had drifted to "rebuild succeeded
|
|
146
|
+
but still fails" on the old code path continue exactly as they
|
|
147
|
+
did.
|
|
148
|
+
|
|
149
|
+
## Acceptance criteria (test-mapped)
|
|
150
|
+
|
|
151
|
+
1. `tests/unit/lifeline/rateLimitState.test.ts > versionSkew bucket
|
|
152
|
+
BYPASSES the cooldown` — asserts the new bypass.
|
|
153
|
+
2. `tests/unit/lifeline/version-skew-recovery.test.ts > CLI service
|
|
154
|
+
label` — asserts `ai.instar.${...}` is in source and
|
|
155
|
+
`com.instar.${...}.lifeline` is gone.
|
|
156
|
+
3. `tests/unit/lifeline/version-skew-recovery.test.ts > pkill
|
|
157
|
+
fallback escalates to SIGKILL` — asserts both signal calls
|
|
158
|
+
present in the restart command source.
|
|
159
|
+
4. `tests/unit/lifeline/version-skew-recovery.test.ts >
|
|
160
|
+
ServerSupervisor preflight rebuild passes --build-from-source` —
|
|
161
|
+
asserts the rebuild argv on the supervisor side.
|
|
162
|
+
5. `tests/unit/lifeline/version-skew-recovery.test.ts >
|
|
163
|
+
NativeModuleHealer in-line rebuild passes --build-from-source` —
|
|
164
|
+
asserts EVERY rebuild call in the healer source has the flag.
|
|
165
|
+
6. `tests/unit/lifeline/version-skew-recovery.test.ts > lifeline
|
|
166
|
+
source guards drop with versionSkewActive` — asserts the source
|
|
167
|
+
order (skew bypass before drop check).
|
|
168
|
+
7. `tests/unit/lifeline/version-skew-recovery.test.ts >
|
|
169
|
+
handleVersionSkew sets the active flag + alert dedupe`.
|
|
170
|
+
8. `tests/unit/lifeline/version-skew-recovery.test.ts >
|
|
171
|
+
forwardToServer success clears the version-skew episode flag`.
|
|
172
|
+
9. `tests/unit/lifeline/version-skew-recovery.test.ts > lock-acquire
|
|
173
|
+
treats sleeping (S) state > 5 min as recoverable`.
|
|
174
|
+
|
|
175
|
+
## Forward note
|
|
176
|
+
|
|
177
|
+
If the auto-update flow eventually grows a "tell the lifeline to
|
|
178
|
+
restart" step, the failure class this PR addresses becomes
|
|
179
|
+
impossible-by-construction. Until then, this is the steady-state
|
|
180
|
+
recovery for the window between server-update and lifeline-restart.
|