instar 1.3.501 → 1.3.503
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +616 -9
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +48 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/BackupManager.d.ts.map +1 -1
- package/dist/core/BackupManager.js +7 -0
- package/dist/core/BackupManager.js.map +1 -1
- package/dist/core/CoherenceJournal.d.ts +1 -1
- package/dist/core/CoherenceJournal.d.ts.map +1 -1
- package/dist/core/CoherenceJournal.js +5 -1
- package/dist/core/CoherenceJournal.js.map +1 -1
- package/dist/core/DeliverMessageHandler.d.ts +13 -1
- package/dist/core/DeliverMessageHandler.d.ts.map +1 -1
- package/dist/core/DeliverMessageHandler.js +7 -0
- package/dist/core/DeliverMessageHandler.js.map +1 -1
- package/dist/core/MachinePoolRegistry.d.ts +3 -0
- package/dist/core/MachinePoolRegistry.d.ts.map +1 -1
- package/dist/core/MachinePoolRegistry.js +1 -0
- package/dist/core/MachinePoolRegistry.js.map +1 -1
- package/dist/core/OwnerSuspectBreaker.d.ts +37 -0
- package/dist/core/OwnerSuspectBreaker.d.ts.map +1 -1
- package/dist/core/OwnerSuspectBreaker.js +58 -0
- package/dist/core/OwnerSuspectBreaker.js.map +1 -1
- package/dist/core/OwnershipReconciler.d.ts +102 -0
- package/dist/core/OwnershipReconciler.d.ts.map +1 -0
- package/dist/core/OwnershipReconciler.js +185 -0
- package/dist/core/OwnershipReconciler.js.map +1 -0
- package/dist/core/PendingInboundStore.d.ts +310 -0
- package/dist/core/PendingInboundStore.d.ts.map +1 -0
- package/dist/core/PendingInboundStore.js +715 -0
- package/dist/core/PendingInboundStore.js.map +1 -0
- package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
- package/dist/core/PostUpdateMigrator.js +13 -0
- package/dist/core/PostUpdateMigrator.js.map +1 -1
- package/dist/core/QueueDrainLoop.d.ts +247 -0
- package/dist/core/QueueDrainLoop.d.ts.map +1 -0
- package/dist/core/QueueDrainLoop.js +0 -0
- package/dist/core/QueueDrainLoop.js.map +1 -0
- package/dist/core/SessionOwnership.d.ts +16 -1
- package/dist/core/SessionOwnership.d.ts.map +1 -1
- package/dist/core/SessionOwnership.js +14 -0
- package/dist/core/SessionOwnership.js.map +1 -1
- package/dist/core/SessionRouter.d.ts +36 -2
- package/dist/core/SessionRouter.d.ts.map +1 -1
- package/dist/core/SessionRouter.js +34 -9
- package/dist/core/SessionRouter.js.map +1 -1
- package/dist/core/devGatedFeatures.d.ts.map +1 -1
- package/dist/core/devGatedFeatures.js +10 -0
- package/dist/core/devGatedFeatures.js.map +1 -1
- package/dist/core/inboundQueueBootSweep.d.ts +61 -0
- package/dist/core/inboundQueueBootSweep.d.ts.map +1 -0
- package/dist/core/inboundQueueBootSweep.js +207 -0
- package/dist/core/inboundQueueBootSweep.js.map +1 -0
- package/dist/core/inboundQueueConfig.d.ts +78 -0
- package/dist/core/inboundQueueConfig.d.ts.map +1 -0
- package/dist/core/inboundQueueConfig.js +135 -0
- package/dist/core/inboundQueueConfig.js.map +1 -0
- package/dist/core/types.d.ts +23 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/messaging/TelegramAdapter.d.ts +4 -0
- package/dist/messaging/TelegramAdapter.d.ts.map +1 -1
- package/dist/messaging/TelegramAdapter.js +12 -0
- package/dist/messaging/TelegramAdapter.js.map +1 -1
- package/dist/monitoring/SessionReaper.d.ts +5 -0
- package/dist/monitoring/SessionReaper.d.ts.map +1 -1
- package/dist/monitoring/SessionReaper.js +22 -2
- package/dist/monitoring/SessionReaper.js.map +1 -1
- package/dist/monitoring/guardManifest.d.ts.map +1 -1
- package/dist/monitoring/guardManifest.js +27 -0
- package/dist/monitoring/guardManifest.js.map +1 -1
- package/dist/scaffold/templates.d.ts.map +1 -1
- package/dist/scaffold/templates.js +6 -0
- package/dist/scaffold/templates.js.map +1 -1
- package/dist/server/AgentServer.d.ts +2 -0
- package/dist/server/AgentServer.d.ts.map +1 -1
- package/dist/server/AgentServer.js +1 -0
- package/dist/server/AgentServer.js.map +1 -1
- package/dist/server/routes.d.ts +4 -0
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +37 -1
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/scripts/lint-guard-manifest.js +2 -0
- package/src/data/builtin-manifest.json +65 -65
- package/src/scaffold/templates.ts +6 -0
- package/upgrades/1.3.502.md +75 -0
- package/upgrades/1.3.503.md +61 -0
- package/upgrades/side-effects/1.3.492.md +20 -0
- package/upgrades/side-effects/durable-inbound-message-queue.md +213 -0
- package/upgrades/side-effects/multi-machine-seamlessness-ws13.md +122 -0
|
@@ -46,6 +46,21 @@ export interface OwnerSuspectBreakerOpts {
|
|
|
46
46
|
suspectForMs: number;
|
|
47
47
|
marks: number;
|
|
48
48
|
}) => void;
|
|
49
|
+
/**
|
|
50
|
+
* Breaker-close hook (Durable Inbound Message Queue §3.2 — NEW CODE, named):
|
|
51
|
+
* fires when `recordSuccess` closes an OPEN episode. The queue drain wires
|
|
52
|
+
* this to deliver held rows instantly when the owner recovers.
|
|
53
|
+
*/
|
|
54
|
+
onClose?: (machineId: string) => void;
|
|
55
|
+
/** Flap-accounting threshold (§4.4): suspect-episodes-per-hour at/above which
|
|
56
|
+
* a flap-episode opens. Default 6. */
|
|
57
|
+
flapThresholdPerHour?: number;
|
|
58
|
+
/** ONE attention item per flap-episode open (nickname resolution is the
|
|
59
|
+
* caller's job — this passes the machine id). */
|
|
60
|
+
reportFlapEpisode?: (info: {
|
|
61
|
+
machineId: string;
|
|
62
|
+
episodesLastHour: number;
|
|
63
|
+
}) => void;
|
|
49
64
|
}
|
|
50
65
|
export declare class OwnerSuspectBreaker {
|
|
51
66
|
private readonly ttlMs;
|
|
@@ -55,6 +70,15 @@ export declare class OwnerSuspectBreaker {
|
|
|
55
70
|
private suspectUntil;
|
|
56
71
|
/** Per-peer episode accounting (created on first mark, deleted on success). */
|
|
57
72
|
private episodes;
|
|
73
|
+
/**
|
|
74
|
+
* Flap accounting (§4.4) — per-peer episode-OPEN timestamps within the last
|
|
75
|
+
* hour. Deliberately SURVIVES `recordSuccess` (the per-episode latch does
|
|
76
|
+
* not — that reset was exactly the round-2 hold-budget bug). In-memory:
|
|
77
|
+
* a restart resets it and it re-trips within an hour — stated in the spec.
|
|
78
|
+
*/
|
|
79
|
+
private episodeOpens;
|
|
80
|
+
/** Open flap-episodes (machineId → opened-at). Closes after 30 min calm. */
|
|
81
|
+
private flapOpenSince;
|
|
58
82
|
constructor(opts?: OwnerSuspectBreakerOpts);
|
|
59
83
|
private log;
|
|
60
84
|
/** Delivery retries to this peer exhausted — open its suspect window.
|
|
@@ -75,5 +99,18 @@ export declare class OwnerSuspectBreaker {
|
|
|
75
99
|
isSuspect(machineId: string): boolean;
|
|
76
100
|
/** A delivery reached the peer — close the episode and re-arm. */
|
|
77
101
|
recordSuccess(machineId: string): void;
|
|
102
|
+
private recordEpisodeOpen;
|
|
103
|
+
/**
|
|
104
|
+
* Is a flap-episode open for this machine (§4.4)? While true, the hold
|
|
105
|
+
* verdict forces `failover` — a chronically flapping machine gets no hold
|
|
106
|
+
* at all until it calms (rate below threshold for 30 min re-arms).
|
|
107
|
+
*/
|
|
108
|
+
isFlapping(machineId: string): boolean;
|
|
109
|
+
/** Surfaced in /pool/queue (§4.4). */
|
|
110
|
+
flapState(): Array<{
|
|
111
|
+
machineId: string;
|
|
112
|
+
episodesLastHour: number;
|
|
113
|
+
flapping: boolean;
|
|
114
|
+
}>;
|
|
78
115
|
}
|
|
79
116
|
//# sourceMappingURL=OwnerSuspectBreaker.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OwnerSuspectBreaker.d.ts","sourceRoot":"","sources":["../../src/core/OwnerSuspectBreaker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAIH,MAAM,WAAW,uBAAuB;IACtC,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+FAA+F;IAC/F,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAC/B,+EAA+E;IAC/E,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"OwnerSuspectBreaker.d.ts","sourceRoot":"","sources":["../../src/core/OwnerSuspectBreaker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAIH,MAAM,WAAW,uBAAuB;IACtC,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,+FAA+F;IAC/F,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAC/B,+EAA+E;IAC/E,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAC7F;;;;OAIG;IACH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;IACtC;2CACuC;IACvC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;sDACkD;IAClD,iBAAiB,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACrF;AAKD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA0B;IAC/C,kEAAkE;IAClE,OAAO,CAAC,YAAY,CAA6B;IACjD,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAA0C;IAC1D;;;;;OAKG;IACH,OAAO,CAAC,YAAY,CAA+B;IACnD,4EAA4E;IAC5E,OAAO,CAAC,aAAa,CAA6B;gBAEtC,IAAI,GAAE,uBAA4B;IAM9C,OAAO,CAAC,GAAG;IAIX;;;;;;;;;;;;4DAYwD;IACxD,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAuBpC,mFAAmF;IACnF,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAKrC,kEAAkE;IAClE,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IActC,OAAO,CAAC,iBAAiB;IAczB;;;;OAIG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAetC,sCAAsC;IACtC,SAAS,IAAI,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,gBAAgB,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,OAAO,CAAA;KAAE,CAAC;CAUvF"}
|
|
@@ -44,6 +44,15 @@ export class OwnerSuspectBreaker {
|
|
|
44
44
|
suspectUntil = new Map();
|
|
45
45
|
/** Per-peer episode accounting (created on first mark, deleted on success). */
|
|
46
46
|
episodes = new Map();
|
|
47
|
+
/**
|
|
48
|
+
* Flap accounting (§4.4) — per-peer episode-OPEN timestamps within the last
|
|
49
|
+
* hour. Deliberately SURVIVES `recordSuccess` (the per-episode latch does
|
|
50
|
+
* not — that reset was exactly the round-2 hold-budget bug). In-memory:
|
|
51
|
+
* a restart resets it and it re-trips within an hour — stated in the spec.
|
|
52
|
+
*/
|
|
53
|
+
episodeOpens = new Map();
|
|
54
|
+
/** Open flap-episodes (machineId → opened-at). Closes after 30 min calm. */
|
|
55
|
+
flapOpenSince = new Map();
|
|
47
56
|
constructor(opts = {}) {
|
|
48
57
|
this.opts = opts;
|
|
49
58
|
this.ttlMs = opts.suspectTtlMs ?? DEFAULT_SUSPECT_TTL_MS;
|
|
@@ -80,6 +89,7 @@ export class OwnerSuspectBreaker {
|
|
|
80
89
|
const f = latch.recordFailure();
|
|
81
90
|
if (f.firstOfEpisode) {
|
|
82
91
|
this.log(`owner ${machineId} SUSPECT (delivery retries exhausted) — short-circuiting its sessions' dispatches for ${Math.round(this.ttlMs / 1000)}s windows until a delivery succeeds`);
|
|
92
|
+
this.recordEpisodeOpen(machineId);
|
|
83
93
|
}
|
|
84
94
|
if (f.shouldSignal) {
|
|
85
95
|
this.log(`owner ${machineId} suspect for ${Math.round(f.failingForMs / 60_000)}min (${f.failures} marks) — signaling once; half-open re-probes continue`);
|
|
@@ -100,7 +110,55 @@ export class OwnerSuspectBreaker {
|
|
|
100
110
|
if (s.recovered)
|
|
101
111
|
this.log(`owner ${machineId} recovered after ${s.failures} suspect mark(s)`);
|
|
102
112
|
this.episodes.delete(machineId);
|
|
113
|
+
// §3.2 event trigger: an OPEN episode just closed — held rows deliver now.
|
|
114
|
+
this.opts.onClose?.(machineId);
|
|
103
115
|
}
|
|
104
116
|
}
|
|
117
|
+
// ── Flap accounting (§4.4) ──────────────────────────────────────────
|
|
118
|
+
recordEpisodeOpen(machineId) {
|
|
119
|
+
const now = this.now();
|
|
120
|
+
const hourAgo = now - 3600_000;
|
|
121
|
+
const opens = (this.episodeOpens.get(machineId) ?? []).filter((t) => t > hourAgo);
|
|
122
|
+
opens.push(now);
|
|
123
|
+
this.episodeOpens.set(machineId, opens);
|
|
124
|
+
const threshold = this.opts.flapThresholdPerHour ?? 6;
|
|
125
|
+
if (opens.length >= threshold && !this.flapOpenSince.has(machineId)) {
|
|
126
|
+
this.flapOpenSince.set(machineId, now);
|
|
127
|
+
this.log(`owner ${machineId} FLAPPING (${opens.length} suspect episodes in the last hour) — holds disabled for it until calm`);
|
|
128
|
+
this.opts.reportFlapEpisode?.({ machineId, episodesLastHour: opens.length });
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Is a flap-episode open for this machine (§4.4)? While true, the hold
|
|
133
|
+
* verdict forces `failover` — a chronically flapping machine gets no hold
|
|
134
|
+
* at all until it calms (rate below threshold for 30 min re-arms).
|
|
135
|
+
*/
|
|
136
|
+
isFlapping(machineId) {
|
|
137
|
+
const since = this.flapOpenSince.get(machineId);
|
|
138
|
+
if (since === undefined)
|
|
139
|
+
return false;
|
|
140
|
+
const now = this.now();
|
|
141
|
+
const opens = (this.episodeOpens.get(machineId) ?? []).filter((t) => t > now - 3600_000);
|
|
142
|
+
const threshold = this.opts.flapThresholdPerHour ?? 6;
|
|
143
|
+
const lastOpen = opens.length > 0 ? Math.max(...opens) : 0;
|
|
144
|
+
// Close when the rate stays below threshold for 30 min.
|
|
145
|
+
if (opens.length < threshold && now - lastOpen > 30 * 60_000) {
|
|
146
|
+
this.flapOpenSince.delete(machineId);
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
return true;
|
|
150
|
+
}
|
|
151
|
+
/** Surfaced in /pool/queue (§4.4). */
|
|
152
|
+
flapState() {
|
|
153
|
+
const now = this.now();
|
|
154
|
+
const out = [];
|
|
155
|
+
for (const [machineId, opens] of this.episodeOpens) {
|
|
156
|
+
const recent = opens.filter((t) => t > now - 3600_000);
|
|
157
|
+
if (recent.length === 0 && !this.flapOpenSince.has(machineId))
|
|
158
|
+
continue;
|
|
159
|
+
out.push({ machineId, episodesLastHour: recent.length, flapping: this.isFlapping(machineId) });
|
|
160
|
+
}
|
|
161
|
+
return out;
|
|
162
|
+
}
|
|
105
163
|
}
|
|
106
164
|
//# sourceMappingURL=OwnerSuspectBreaker.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OwnerSuspectBreaker.js","sourceRoot":"","sources":["../../src/core/OwnerSuspectBreaker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"OwnerSuspectBreaker.js","sourceRoot":"","sources":["../../src/core/OwnerSuspectBreaker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAyB/D,MAAM,sBAAsB,GAAG,MAAM,CAAC;AACtC,MAAM,uBAAuB,GAAG,EAAE,GAAG,MAAM,CAAC;AAE5C,MAAM,OAAO,mBAAmB;IACb,KAAK,CAAS;IACd,GAAG,CAAe;IAClB,IAAI,CAA0B;IAC/C,kEAAkE;IAC1D,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IACjD,+EAA+E;IACvE,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAC;IAC1D;;;;;OAKG;IACK,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;IACnD,4EAA4E;IACpE,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAElD,YAAY,OAAgC,EAAE;QAC5C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,YAAY,IAAI,sBAAsB,CAAC;QACzD,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IAClC,CAAC;IAEO,GAAG,CAAC,CAAS;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;;;;;;;;4DAYwD;IACxD,WAAW,CAAC,SAAiB;QAC3B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC;QACD,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,IAAI,mBAAmB,CAAC;gBAC9B,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,uBAAuB;gBACjE,GAAG,EAAE,IAAI,CAAC,GAAG;aACd,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;QACD,MAAM,CAAC,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC;QAChC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG,CAAC,SAAS,SAAS,yFAAyF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACxL,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC;QACD,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,SAAS,SAAS,gBAAgB,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,wDAAwD,CAAC,CAAC;YAC1J,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IAED,mFAAmF;IACnF,SAAS,CAAC,SAAiB;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/C,OAAO,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IACnD,CAAC;IAED,kEAAkE;IAClE,aAAa,CAAC,SAAiB;QAC7B,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC3C,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,GAAG,KAAK,CAAC,aAAa,EAAE,CAAC;YAChC,IAAI,CAAC,CAAC,SAAS;gBAAE,IAAI,CAAC,GAAG,CAAC,SAAS,SAAS,oBAAoB,CAAC,CAAC,QAAQ,kBAAkB,CAAC,CAAC;YAC9F,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAChC,2EAA2E;YAC3E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,SAAS,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,uEAAuE;IAE/D,iBAAiB,CAAC,SAAiB;QACzC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,GAAG,QAAQ,CAAC;QAC/B,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC;QAClF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,oBAAoB,IAAI,CAAC,CAAC;QACtD,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACpE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YACvC,IAAI,CAAC,GAAG,CAAC,SAAS,SAAS,cAAc,KAAK,CAAC,MAAM,wEAAwE,CAAC,CAAC;YAC/H,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,gBAAgB,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,UAAU,CAAC,SAAiB;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QACtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC,CAAC;QACzF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,oBAAoB,IAAI,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,wDAAwD;QACxD,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,IAAI,GAAG,GAAG,QAAQ,GAAG,EAAE,GAAG,MAAM,EAAE,CAAC;YAC7D,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACrC,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sCAAsC;IACtC,SAAS;QACP,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,GAAG,GAA8E,EAAE,CAAC;QAC1F,KAAK,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACnD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,GAAG,QAAQ,CAAC,CAAC;YACvD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC;gBAAE,SAAS;YACxE,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACjG,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF"}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OwnershipReconciler — WS1.3 of MULTI-MACHINE-SEAMLESSNESS-SPEC: a pin that
|
|
3
|
+
* disagrees with ownership is a first-class PENDING state with a BOUNDED
|
|
4
|
+
* convergence path, instead of an indefinite divergence that waits for an
|
|
5
|
+
* inbound message that may never route (the 2026-06-12 live incident: topic
|
|
6
|
+
* 13481 sat owner=Mini / pin=Laptop for hours; the closeout reaper attacked the
|
|
7
|
+
* working session the whole time).
|
|
8
|
+
*
|
|
9
|
+
* Each machine runs its own reconciler over LOCAL replicated state (pins are
|
|
10
|
+
* router-write-only on this machine's disk; ownership reads are the local CAS
|
|
11
|
+
* registry view) — never a mesh call on the tick path. Convergence is the
|
|
12
|
+
* cooperative FSM handoff whenever the owner is alive:
|
|
13
|
+
*
|
|
14
|
+
* owner sees conflicting pin → (debounce, safe point, bounded) → CAS
|
|
15
|
+
* transfer→(pin target) → target's reconciler sees transferring-to-me → CAS
|
|
16
|
+
* claim → active. Claim-before-release; exactly-one-owner by the epoch fence.
|
|
17
|
+
*
|
|
18
|
+
* The FORCE path exists only for a PROVABLY DEAD owner: pool capacity offline
|
|
19
|
+
* with last-seen older than the death-evidence bound AND quorum membership
|
|
20
|
+
* (Phase C: online > total/2 for N-machine pools; a 2-machine pool degrades to
|
|
21
|
+
* the surviving machine when its peer is provably dark — documented, since
|
|
22
|
+
* majority-of-2 cannot lose a member). A reachable-but-slow owner is NEVER
|
|
23
|
+
* force-claimed — that direction of error (stealing a live conversation
|
|
24
|
+
* mid-turn) is the round-2 adversarial finding this design closed.
|
|
25
|
+
*
|
|
26
|
+
* Provenance: the pin store is written ONLY by the authenticated router
|
|
27
|
+
* transfer path on this machine — a peer cannot write this machine's pin file,
|
|
28
|
+
* and journal-replicated placement entries are evidence, never triggers. The
|
|
29
|
+
* reconciler acts only on its OWN pin store (L15: reach ≠ authority).
|
|
30
|
+
*
|
|
31
|
+
* Posture (spec §7): the reconciler is per-machine BY DESIGN (each machine
|
|
32
|
+
* reconciles its own view; the CAS registry is the shared arbiter). Ships DARK
|
|
33
|
+
* behind multiMachine.seamlessness.ws13Reconcile with dryRun=true default
|
|
34
|
+
* (logs intended actions without CAS). Single-machine pools: strict no-op —
|
|
35
|
+
* the tick returns before touching any machinery when fewer than 2 machines
|
|
36
|
+
* are registered.
|
|
37
|
+
*/
|
|
38
|
+
import type { SessionOwnershipRegistry, CasResult } from './SessionOwnershipRegistry.js';
|
|
39
|
+
import type { TopicPlacementPinStore } from './TopicPlacementPinStore.js';
|
|
40
|
+
export interface ReconcilerMachineView {
|
|
41
|
+
machineId: string;
|
|
42
|
+
online: boolean;
|
|
43
|
+
/** ms since epoch of the machine's last self-reported heartbeat (0 = never). */
|
|
44
|
+
lastSeenMs: number;
|
|
45
|
+
}
|
|
46
|
+
export interface OwnershipReconcilerDeps {
|
|
47
|
+
/** Dark flag — read live each tick. */
|
|
48
|
+
enabled: () => boolean;
|
|
49
|
+
/** Dry-run — log intended actions, never CAS. Read live. */
|
|
50
|
+
dryRun: () => boolean;
|
|
51
|
+
selfMachineId: string;
|
|
52
|
+
pinStore: TopicPlacementPinStore;
|
|
53
|
+
ownership: SessionOwnershipRegistry;
|
|
54
|
+
/** All REGISTERED machines (online and not). [] or [self] → strict no-op. */
|
|
55
|
+
machines: () => ReconcilerMachineView[];
|
|
56
|
+
/**
|
|
57
|
+
* Safe-point check: is the topic's local session mid-turn right now?
|
|
58
|
+
* Busy → defer the cooperative release until idle OR until safePointDeadlineMs
|
|
59
|
+
* has elapsed since the conflict was first observed (the spec's bounded
|
|
60
|
+
* "next safe point": end of turn or T seconds, whichever first).
|
|
61
|
+
*/
|
|
62
|
+
isTopicBusy: (sessionKey: string) => boolean;
|
|
63
|
+
/** Journal pairing (§3.3): every landed CAS emits a placement entry. */
|
|
64
|
+
emitPlacement: (sessionKey: string, r: CasResult & {
|
|
65
|
+
ok: true;
|
|
66
|
+
}, reason: 'reconcile-transfer' | 'reconcile-claim' | 'reconcile-force-claim' | 'reconcile-adopt') => void;
|
|
67
|
+
/** Pin must be stable this long before the owner acts (flap debounce). */
|
|
68
|
+
debounceMs?: number;
|
|
69
|
+
/** Bounded safe-point wait: after this, transfer even if the session is busy. */
|
|
70
|
+
safePointDeadlineMs?: number;
|
|
71
|
+
/** Owner-death evidence: offline AND lastSeen older than this. */
|
|
72
|
+
deathEvidenceMs?: number;
|
|
73
|
+
now?: () => number;
|
|
74
|
+
logger?: (msg: string) => void;
|
|
75
|
+
}
|
|
76
|
+
export interface ReconcileTickReport {
|
|
77
|
+
examined: number;
|
|
78
|
+
transfers: number;
|
|
79
|
+
claims: number;
|
|
80
|
+
forceClaims: number;
|
|
81
|
+
adoptions: number;
|
|
82
|
+
deferredBusy: number;
|
|
83
|
+
deferredDebounce: number;
|
|
84
|
+
deferredNoEvidence: number;
|
|
85
|
+
dryRun: boolean;
|
|
86
|
+
skipped?: 'disabled' | 'single-machine';
|
|
87
|
+
}
|
|
88
|
+
export declare class OwnershipReconciler {
|
|
89
|
+
private readonly d;
|
|
90
|
+
/** First time we observed each topic's CURRENT conflict (cleared on convergence). */
|
|
91
|
+
private readonly conflictSince;
|
|
92
|
+
constructor(deps: OwnershipReconcilerDeps);
|
|
93
|
+
private now;
|
|
94
|
+
private log;
|
|
95
|
+
/**
|
|
96
|
+
* One reconcile pass. Deterministic over local state; bounded by the number
|
|
97
|
+
* of pinned topics. Safe to call on any cadence.
|
|
98
|
+
*/
|
|
99
|
+
tick(): ReconcileTickReport;
|
|
100
|
+
private act;
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=OwnershipReconciler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"OwnershipReconciler.d.ts","sourceRoot":"","sources":["../../src/core/OwnershipReconciler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,KAAK,EAAE,wBAAwB,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAEzF,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAE1E,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,gFAAgF;IAChF,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,uBAAuB;IACtC,uCAAuC;IACvC,OAAO,EAAE,MAAM,OAAO,CAAC;IACvB,4DAA4D;IAC5D,MAAM,EAAE,MAAM,OAAO,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,sBAAsB,CAAC;IACjC,SAAS,EAAE,wBAAwB,CAAC;IACpC,6EAA6E;IAC7E,QAAQ,EAAE,MAAM,qBAAqB,EAAE,CAAC;IACxC;;;;;OAKG;IACH,WAAW,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,OAAO,CAAC;IAC7C,wEAAwE;IACxE,aAAa,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,GAAG;QAAE,EAAE,EAAE,IAAI,CAAA;KAAE,EAAE,MAAM,EAAE,oBAAoB,GAAG,iBAAiB,GAAG,uBAAuB,GAAG,iBAAiB,KAAK,IAAI,CAAC;IACzK,0EAA0E;IAC1E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iFAAiF;IACjF,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kEAAkE;IAClE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,UAAU,GAAG,gBAAgB,CAAC;CACzC;AAMD,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAA0B;IAC5C,qFAAqF;IACrF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA6B;gBAE/C,IAAI,EAAE,uBAAuB;IAIzC,OAAO,CAAC,GAAG;IAGX,OAAO,CAAC,GAAG;IAIX;;;OAGG;IACH,IAAI,IAAI,mBAAmB;IAiG3B,OAAO,CAAC,GAAG;CA4BZ"}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OwnershipReconciler — WS1.3 of MULTI-MACHINE-SEAMLESSNESS-SPEC: a pin that
|
|
3
|
+
* disagrees with ownership is a first-class PENDING state with a BOUNDED
|
|
4
|
+
* convergence path, instead of an indefinite divergence that waits for an
|
|
5
|
+
* inbound message that may never route (the 2026-06-12 live incident: topic
|
|
6
|
+
* 13481 sat owner=Mini / pin=Laptop for hours; the closeout reaper attacked the
|
|
7
|
+
* working session the whole time).
|
|
8
|
+
*
|
|
9
|
+
* Each machine runs its own reconciler over LOCAL replicated state (pins are
|
|
10
|
+
* router-write-only on this machine's disk; ownership reads are the local CAS
|
|
11
|
+
* registry view) — never a mesh call on the tick path. Convergence is the
|
|
12
|
+
* cooperative FSM handoff whenever the owner is alive:
|
|
13
|
+
*
|
|
14
|
+
* owner sees conflicting pin → (debounce, safe point, bounded) → CAS
|
|
15
|
+
* transfer→(pin target) → target's reconciler sees transferring-to-me → CAS
|
|
16
|
+
* claim → active. Claim-before-release; exactly-one-owner by the epoch fence.
|
|
17
|
+
*
|
|
18
|
+
* The FORCE path exists only for a PROVABLY DEAD owner: pool capacity offline
|
|
19
|
+
* with last-seen older than the death-evidence bound AND quorum membership
|
|
20
|
+
* (Phase C: online > total/2 for N-machine pools; a 2-machine pool degrades to
|
|
21
|
+
* the surviving machine when its peer is provably dark — documented, since
|
|
22
|
+
* majority-of-2 cannot lose a member). A reachable-but-slow owner is NEVER
|
|
23
|
+
* force-claimed — that direction of error (stealing a live conversation
|
|
24
|
+
* mid-turn) is the round-2 adversarial finding this design closed.
|
|
25
|
+
*
|
|
26
|
+
* Provenance: the pin store is written ONLY by the authenticated router
|
|
27
|
+
* transfer path on this machine — a peer cannot write this machine's pin file,
|
|
28
|
+
* and journal-replicated placement entries are evidence, never triggers. The
|
|
29
|
+
* reconciler acts only on its OWN pin store (L15: reach ≠ authority).
|
|
30
|
+
*
|
|
31
|
+
* Posture (spec §7): the reconciler is per-machine BY DESIGN (each machine
|
|
32
|
+
* reconciles its own view; the CAS registry is the shared arbiter). Ships DARK
|
|
33
|
+
* behind multiMachine.seamlessness.ws13Reconcile with dryRun=true default
|
|
34
|
+
* (logs intended actions without CAS). Single-machine pools: strict no-op —
|
|
35
|
+
* the tick returns before touching any machinery when fewer than 2 machines
|
|
36
|
+
* are registered.
|
|
37
|
+
*/
|
|
38
|
+
const DEFAULT_DEBOUNCE_MS = 30_000;
|
|
39
|
+
const DEFAULT_SAFE_POINT_DEADLINE_MS = 120_000;
|
|
40
|
+
const DEFAULT_DEATH_EVIDENCE_MS = 180_000;
|
|
41
|
+
export class OwnershipReconciler {
|
|
42
|
+
d;
|
|
43
|
+
/** First time we observed each topic's CURRENT conflict (cleared on convergence). */
|
|
44
|
+
conflictSince = new Map();
|
|
45
|
+
constructor(deps) {
|
|
46
|
+
this.d = deps;
|
|
47
|
+
}
|
|
48
|
+
now() {
|
|
49
|
+
return (this.d.now ?? Date.now)();
|
|
50
|
+
}
|
|
51
|
+
log(msg) {
|
|
52
|
+
try {
|
|
53
|
+
this.d.logger?.(`[OwnershipReconciler] ${msg}`);
|
|
54
|
+
}
|
|
55
|
+
catch { /* observability never gates */ }
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* One reconcile pass. Deterministic over local state; bounded by the number
|
|
59
|
+
* of pinned topics. Safe to call on any cadence.
|
|
60
|
+
*/
|
|
61
|
+
tick() {
|
|
62
|
+
const report = {
|
|
63
|
+
examined: 0, transfers: 0, claims: 0, forceClaims: 0, adoptions: 0,
|
|
64
|
+
deferredBusy: 0, deferredDebounce: 0, deferredNoEvidence: 0,
|
|
65
|
+
dryRun: this.d.dryRun(),
|
|
66
|
+
};
|
|
67
|
+
if (!this.d.enabled()) {
|
|
68
|
+
report.skipped = 'disabled';
|
|
69
|
+
return report;
|
|
70
|
+
}
|
|
71
|
+
const machines = this.d.machines();
|
|
72
|
+
if (machines.length < 2) {
|
|
73
|
+
// Spec invariant 6: single-machine strict no-op — no machinery entered.
|
|
74
|
+
report.skipped = 'single-machine';
|
|
75
|
+
return report;
|
|
76
|
+
}
|
|
77
|
+
const now = this.now();
|
|
78
|
+
const self = this.d.selfMachineId;
|
|
79
|
+
const byId = new Map(machines.map((m) => [m.machineId, m]));
|
|
80
|
+
for (const [sessionKey, pin] of Object.entries(this.d.pinStore.all())) {
|
|
81
|
+
if (!pin.pinned || !pin.preferredMachine)
|
|
82
|
+
continue;
|
|
83
|
+
const rec = this.d.ownership.read(sessionKey);
|
|
84
|
+
const owner = rec && rec.status !== 'released' ? rec.ownerMachineId : null;
|
|
85
|
+
if (owner === pin.preferredMachine && rec?.status === 'active') {
|
|
86
|
+
this.conflictSince.delete(sessionKey);
|
|
87
|
+
continue; // converged
|
|
88
|
+
}
|
|
89
|
+
report.examined++;
|
|
90
|
+
if (!this.conflictSince.has(sessionKey))
|
|
91
|
+
this.conflictSince.set(sessionKey, now);
|
|
92
|
+
const since = this.conflictSince.get(sessionKey);
|
|
93
|
+
// ── Case B: a transfer is mid-flight TO ME → claim (completes the handoff).
|
|
94
|
+
if (rec?.status === 'transferring' && rec.transferTo === self) {
|
|
95
|
+
this.act(report, 'claims', sessionKey, { type: 'claim', machineId: self }, 'reconcile-claim');
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
if (rec?.status === 'transferring')
|
|
99
|
+
continue; // someone else's claim to make
|
|
100
|
+
// ── Case D: no live record and the pin names ME → adopt (place→claim).
|
|
101
|
+
if ((!rec || rec.status === 'released') && pin.preferredMachine === self) {
|
|
102
|
+
const placed = this.act(report, 'adoptions', sessionKey, { type: 'place', machineId: self }, 'reconcile-adopt');
|
|
103
|
+
if (placed)
|
|
104
|
+
this.act(report, 'adoptions', sessionKey, { type: 'claim', machineId: self }, 'reconcile-adopt', /*countOnce*/ true);
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
if (!rec || rec.status === 'released')
|
|
108
|
+
continue;
|
|
109
|
+
// From here the record is active with owner ≠ pin target.
|
|
110
|
+
// ── Case A: I am the live owner → cooperative transfer at a safe point.
|
|
111
|
+
if (owner === self) {
|
|
112
|
+
// Flap debounce: the pin must be stable before the owner gives up custody.
|
|
113
|
+
const pinAgeMs = now - Date.parse(pin.updatedAt);
|
|
114
|
+
if (Number.isFinite(pinAgeMs) && pinAgeMs < (this.d.debounceMs ?? DEFAULT_DEBOUNCE_MS)) {
|
|
115
|
+
report.deferredDebounce++;
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
// Bounded safe point: wait for idle, but never past the deadline.
|
|
119
|
+
const busy = this.d.isTopicBusy(sessionKey);
|
|
120
|
+
const pastDeadline = now - since >= (this.d.safePointDeadlineMs ?? DEFAULT_SAFE_POINT_DEADLINE_MS);
|
|
121
|
+
if (busy && !pastDeadline) {
|
|
122
|
+
report.deferredBusy++;
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
this.act(report, 'transfers', sessionKey, { type: 'transfer', to: pin.preferredMachine }, 'reconcile-transfer');
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
// ── Case C: the pin names ME but a DEAD machine holds the record → force.
|
|
129
|
+
if (pin.preferredMachine === self) {
|
|
130
|
+
const ownerView = owner ? byId.get(owner) : undefined;
|
|
131
|
+
const ownerProvablyDead = !!ownerView
|
|
132
|
+
&& !ownerView.online
|
|
133
|
+
&& (now - ownerView.lastSeenMs) >= (this.d.deathEvidenceMs ?? DEFAULT_DEATH_EVIDENCE_MS);
|
|
134
|
+
// Owner not even REGISTERED → it cannot heartbeat or hold a lease here;
|
|
135
|
+
// treat as dead only with the same age discipline (no view = no liveness
|
|
136
|
+
// proof; lastSeenMs 0 satisfies the age bound trivially, which is right:
|
|
137
|
+
// a machine the pool has never seen cannot be the live owner).
|
|
138
|
+
const ownerUnknown = !!owner && !ownerView;
|
|
139
|
+
// Quorum (Phase C — N machines): act only from the majority partition.
|
|
140
|
+
// total ≤ 2 degrades to "the surviving machine proceeds against a
|
|
141
|
+
// provably dark peer" (majority-of-2 cannot lose a member; documented).
|
|
142
|
+
const online = machines.filter((m) => m.online).length;
|
|
143
|
+
const inQuorum = machines.length <= 2 || online * 2 > machines.length;
|
|
144
|
+
if ((ownerProvablyDead || ownerUnknown) && inQuorum) {
|
|
145
|
+
this.act(report, 'forceClaims', sessionKey, { type: 'force-claim', machineId: self }, 'reconcile-force-claim');
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
// Alive (or merely slow / unproven) owner: its own reconciler runs
|
|
149
|
+
// Case A. We surface the pending state and wait — never steal.
|
|
150
|
+
report.deferredNoEvidence++;
|
|
151
|
+
}
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
// Neither owner nor pin target: not my move this tick.
|
|
155
|
+
}
|
|
156
|
+
return report;
|
|
157
|
+
}
|
|
158
|
+
act(report, counter, sessionKey, action, reason, countOnce = false) {
|
|
159
|
+
if (this.d.dryRun()) {
|
|
160
|
+
this.log(`DRY-RUN would ${action.type} ${sessionKey} (${reason})`);
|
|
161
|
+
if (!countOnce)
|
|
162
|
+
report[counter]++;
|
|
163
|
+
return false; // dry-run never lands a CAS, so chained steps stop here
|
|
164
|
+
}
|
|
165
|
+
const r = this.d.ownership.cas(action, {
|
|
166
|
+
sessionKey,
|
|
167
|
+
sender: this.d.selfMachineId,
|
|
168
|
+
nonce: `${this.d.selfMachineId}:${reason}:${sessionKey}:${this.now()}`,
|
|
169
|
+
});
|
|
170
|
+
if (r.ok) {
|
|
171
|
+
if (!countOnce)
|
|
172
|
+
report[counter]++;
|
|
173
|
+
this.log(`${action.type} landed for ${sessionKey} (${reason}, epoch ${r.record.ownershipEpoch})`);
|
|
174
|
+
try {
|
|
175
|
+
this.d.emitPlacement(sessionKey, r, reason);
|
|
176
|
+
}
|
|
177
|
+
catch { /* §3.3 pairing is observability — never endangers the CAS */ }
|
|
178
|
+
this.conflictSince.delete(sessionKey);
|
|
179
|
+
return true;
|
|
180
|
+
}
|
|
181
|
+
this.log(`${action.type} rejected for ${sessionKey} (${'reason' in r ? r.reason : 'unknown'}) — will re-evaluate next tick`);
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
//# sourceMappingURL=OwnershipReconciler.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"OwnershipReconciler.js","sourceRoot":"","sources":["../../src/core/OwnershipReconciler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAuDH,MAAM,mBAAmB,GAAG,MAAM,CAAC;AACnC,MAAM,8BAA8B,GAAG,OAAO,CAAC;AAC/C,MAAM,yBAAyB,GAAG,OAAO,CAAC;AAE1C,MAAM,OAAO,mBAAmB;IACb,CAAC,CAA0B;IAC5C,qFAAqF;IACpE,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE3D,YAAY,IAA6B;QACvC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;IAChB,CAAC;IAEO,GAAG;QACT,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,CAAC;IACO,GAAG,CAAC,GAAW;QACrB,IAAI,CAAC;YAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,+BAA+B,CAAC,CAAC;IACpG,CAAC;IAED;;;OAGG;IACH,IAAI;QACF,MAAM,MAAM,GAAwB;YAClC,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC;YAClE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC;YAC3D,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE;SACxB,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YACtB,MAAM,CAAC,OAAO,GAAG,UAAU,CAAC;YAC5B,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACnC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,wEAAwE;YACxE,MAAM,CAAC,OAAO,GAAG,gBAAgB,CAAC;YAClC,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5D,KAAK,MAAM,CAAC,UAAU,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;YACtE,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,gBAAgB;gBAAE,SAAS;YACnD,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC9C,MAAM,KAAK,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;YAC3E,IAAI,KAAK,KAAK,GAAG,CAAC,gBAAgB,IAAI,GAAG,EAAE,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/D,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBACtC,SAAS,CAAC,YAAY;YACxB,CAAC;YACD,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC;gBAAE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YACjF,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,UAAU,CAAE,CAAC;YAElD,6EAA6E;YAC7E,IAAI,GAAG,EAAE,MAAM,KAAK,cAAc,IAAI,GAAG,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;gBAC9D,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,iBAAiB,CAAC,CAAC;gBAC9F,SAAS;YACX,CAAC;YACD,IAAI,GAAG,EAAE,MAAM,KAAK,cAAc;gBAAE,SAAS,CAAC,+BAA+B;YAE7E,wEAAwE;YACxE,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,UAAU,CAAC,IAAI,GAAG,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;gBACzE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,iBAAiB,CAAC,CAAC;gBAChH,IAAI,MAAM;oBAAE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,iBAAiB,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;gBACjI,SAAS;YACX,CAAC;YACD,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,UAAU;gBAAE,SAAS;YAEhD,0DAA0D;YAC1D,yEAAyE;YACzE,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;gBACnB,2EAA2E;gBAC3E,MAAM,QAAQ,GAAG,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACjD,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,IAAI,mBAAmB,CAAC,EAAE,CAAC;oBACvF,MAAM,CAAC,gBAAgB,EAAE,CAAC;oBAC1B,SAAS;gBACX,CAAC;gBACD,kEAAkE;gBAClE,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;gBAC5C,MAAM,YAAY,GAAG,GAAG,GAAG,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,IAAI,8BAA8B,CAAC,CAAC;gBACnG,IAAI,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAC1B,MAAM,CAAC,YAAY,EAAE,CAAC;oBACtB,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,GAAG,CAAC,gBAAgB,EAAE,EAAE,oBAAoB,CAAC,CAAC;gBAChH,SAAS;YACX,CAAC;YAED,2EAA2E;YAC3E,IAAI,GAAG,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACtD,MAAM,iBAAiB,GAAG,CAAC,CAAC,SAAS;uBAChC,CAAC,SAAS,CAAC,MAAM;uBACjB,CAAC,GAAG,GAAG,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,IAAI,yBAAyB,CAAC,CAAC;gBAC3F,wEAAwE;gBACxE,yEAAyE;gBACzE,yEAAyE;gBACzE,+DAA+D;gBAC/D,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC;gBAC3C,uEAAuE;gBACvE,kEAAkE;gBAClE,wEAAwE;gBACxE,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;gBACvD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACtE,IAAI,CAAC,iBAAiB,IAAI,YAAY,CAAC,IAAI,QAAQ,EAAE,CAAC;oBACpD,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,uBAAuB,CAAC,CAAC;gBACjH,CAAC;qBAAM,CAAC;oBACN,mEAAmE;oBACnE,+DAA+D;oBAC/D,MAAM,CAAC,kBAAkB,EAAE,CAAC;gBAC9B,CAAC;gBACD,SAAS;YACX,CAAC;YACD,uDAAuD;QACzD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,GAAG,CACT,MAA2B,EAC3B,OAA6D,EAC7D,UAAkB,EAClB,MAAsD,EACtD,MAA8F,EAC9F,SAAS,GAAG,KAAK;QAEjB,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,IAAI,IAAI,UAAU,KAAK,MAAM,GAAG,CAAC,CAAC;YACnE,IAAI,CAAC,SAAS;gBAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,OAAO,KAAK,CAAC,CAAC,wDAAwD;QACxE,CAAC;QACD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE;YACrC,UAAU;YACV,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,aAAa;YAC5B,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,aAAa,IAAI,MAAM,IAAI,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;SACvE,CAAC,CAAC;QACH,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;YACT,IAAI,CAAC,SAAS;gBAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,eAAe,UAAU,KAAK,MAAM,WAAW,CAAC,CAAC,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC;YAClG,IAAI,CAAC;gBAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,6DAA6D,CAAC,CAAC;YAC5H,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;YACtC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,iBAAiB,UAAU,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,gCAAgC,CAAC,CAAC;QAC7H,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
|