instar 1.3.501 → 1.3.503
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +616 -9
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +48 -0
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/BackupManager.d.ts.map +1 -1
- package/dist/core/BackupManager.js +7 -0
- package/dist/core/BackupManager.js.map +1 -1
- package/dist/core/CoherenceJournal.d.ts +1 -1
- package/dist/core/CoherenceJournal.d.ts.map +1 -1
- package/dist/core/CoherenceJournal.js +5 -1
- package/dist/core/CoherenceJournal.js.map +1 -1
- package/dist/core/DeliverMessageHandler.d.ts +13 -1
- package/dist/core/DeliverMessageHandler.d.ts.map +1 -1
- package/dist/core/DeliverMessageHandler.js +7 -0
- package/dist/core/DeliverMessageHandler.js.map +1 -1
- package/dist/core/MachinePoolRegistry.d.ts +3 -0
- package/dist/core/MachinePoolRegistry.d.ts.map +1 -1
- package/dist/core/MachinePoolRegistry.js +1 -0
- package/dist/core/MachinePoolRegistry.js.map +1 -1
- package/dist/core/OwnerSuspectBreaker.d.ts +37 -0
- package/dist/core/OwnerSuspectBreaker.d.ts.map +1 -1
- package/dist/core/OwnerSuspectBreaker.js +58 -0
- package/dist/core/OwnerSuspectBreaker.js.map +1 -1
- package/dist/core/OwnershipReconciler.d.ts +102 -0
- package/dist/core/OwnershipReconciler.d.ts.map +1 -0
- package/dist/core/OwnershipReconciler.js +185 -0
- package/dist/core/OwnershipReconciler.js.map +1 -0
- package/dist/core/PendingInboundStore.d.ts +310 -0
- package/dist/core/PendingInboundStore.d.ts.map +1 -0
- package/dist/core/PendingInboundStore.js +715 -0
- package/dist/core/PendingInboundStore.js.map +1 -0
- package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
- package/dist/core/PostUpdateMigrator.js +13 -0
- package/dist/core/PostUpdateMigrator.js.map +1 -1
- package/dist/core/QueueDrainLoop.d.ts +247 -0
- package/dist/core/QueueDrainLoop.d.ts.map +1 -0
- package/dist/core/QueueDrainLoop.js +0 -0
- package/dist/core/QueueDrainLoop.js.map +1 -0
- package/dist/core/SessionOwnership.d.ts +16 -1
- package/dist/core/SessionOwnership.d.ts.map +1 -1
- package/dist/core/SessionOwnership.js +14 -0
- package/dist/core/SessionOwnership.js.map +1 -1
- package/dist/core/SessionRouter.d.ts +36 -2
- package/dist/core/SessionRouter.d.ts.map +1 -1
- package/dist/core/SessionRouter.js +34 -9
- package/dist/core/SessionRouter.js.map +1 -1
- package/dist/core/devGatedFeatures.d.ts.map +1 -1
- package/dist/core/devGatedFeatures.js +10 -0
- package/dist/core/devGatedFeatures.js.map +1 -1
- package/dist/core/inboundQueueBootSweep.d.ts +61 -0
- package/dist/core/inboundQueueBootSweep.d.ts.map +1 -0
- package/dist/core/inboundQueueBootSweep.js +207 -0
- package/dist/core/inboundQueueBootSweep.js.map +1 -0
- package/dist/core/inboundQueueConfig.d.ts +78 -0
- package/dist/core/inboundQueueConfig.d.ts.map +1 -0
- package/dist/core/inboundQueueConfig.js +135 -0
- package/dist/core/inboundQueueConfig.js.map +1 -0
- package/dist/core/types.d.ts +23 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/messaging/TelegramAdapter.d.ts +4 -0
- package/dist/messaging/TelegramAdapter.d.ts.map +1 -1
- package/dist/messaging/TelegramAdapter.js +12 -0
- package/dist/messaging/TelegramAdapter.js.map +1 -1
- package/dist/monitoring/SessionReaper.d.ts +5 -0
- package/dist/monitoring/SessionReaper.d.ts.map +1 -1
- package/dist/monitoring/SessionReaper.js +22 -2
- package/dist/monitoring/SessionReaper.js.map +1 -1
- package/dist/monitoring/guardManifest.d.ts.map +1 -1
- package/dist/monitoring/guardManifest.js +27 -0
- package/dist/monitoring/guardManifest.js.map +1 -1
- package/dist/scaffold/templates.d.ts.map +1 -1
- package/dist/scaffold/templates.js +6 -0
- package/dist/scaffold/templates.js.map +1 -1
- package/dist/server/AgentServer.d.ts +2 -0
- package/dist/server/AgentServer.d.ts.map +1 -1
- package/dist/server/AgentServer.js +1 -0
- package/dist/server/AgentServer.js.map +1 -1
- package/dist/server/routes.d.ts +4 -0
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +37 -1
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/scripts/lint-guard-manifest.js +2 -0
- package/src/data/builtin-manifest.json +65 -65
- package/src/scaffold/templates.ts +6 -0
- package/upgrades/1.3.502.md +75 -0
- package/upgrades/1.3.503.md +61 -0
- package/upgrades/side-effects/1.3.492.md +20 -0
- package/upgrades/side-effects/durable-inbound-message-queue.md +213 -0
- package/upgrades/side-effects/multi-machine-seamlessness-ws13.md +122 -0
package/dist/commands/server.js
CHANGED
|
@@ -366,6 +366,20 @@ let _topicResumeMap = null;
|
|
|
366
366
|
// is byte-identical to today's always-local dispatch. Set once in startServer().
|
|
367
367
|
let _sessionRouter = null;
|
|
368
368
|
let _sessionPoolStage = () => 'dark';
|
|
369
|
+
// ── Durable Inbound Message Queue (docs/specs/durable-inbound-message-queue.md) ──
|
|
370
|
+
// The custody engine (null = feature dark / gate failed / invariants violated —
|
|
371
|
+
// every consumer treats null as "refused → today's fall-through").
|
|
372
|
+
let _inboundQueue = null;
|
|
373
|
+
// The store the unconditional boot sweep opened when the queue will run this
|
|
374
|
+
// boot — adopted by the engine construction (one open handle, single-writer).
|
|
375
|
+
let _sweptInboundStore = null;
|
|
376
|
+
// The drain's local-delivery tail (§3.1 via:'drain') — assigned inside
|
|
377
|
+
// wireTelegramRouting where the session primitives live in scope.
|
|
378
|
+
let _drainLocalDeliver = null;
|
|
379
|
+
// Emergency-stop integration (§3.6): marks a topic stopped for the drain's
|
|
380
|
+
// pass/batch/chokepoint consults AND settles its custody (terminal
|
|
381
|
+
// operator-stop + PIS cleanup + loss report). Set with the engine.
|
|
382
|
+
let _inboundQueueStop = null;
|
|
369
383
|
/** This machine's mesh id — lets the inbound dispatch tell a REMOTE placement
|
|
370
384
|
* (forward/spawn on another machine → must NOT also dispatch locally) from a
|
|
371
385
|
* self placement. Set once in startServer()'s mesh block. */
|
|
@@ -1746,17 +1760,42 @@ getAttentionTopicId) {
|
|
|
1746
1760
|
}
|
|
1747
1761
|
}
|
|
1748
1762
|
if (_sessionRouter && _sessionPoolStage() !== 'dark') {
|
|
1763
|
+
// Ordering gate (Durable Inbound Message Queue §2.3): a live message for
|
|
1764
|
+
// a session with queued custody enqueues BEHIND the existing entries —
|
|
1765
|
+
// injecting it now would deliver out of order. Gated on a live engine.
|
|
1766
|
+
if (_inboundQueue) {
|
|
1767
|
+
try {
|
|
1768
|
+
if (_inboundQueue.hasQueued(String(topicId))) {
|
|
1769
|
+
const ord = _inboundQueue.enqueueLive({
|
|
1770
|
+
sessionKey: String(topicId),
|
|
1771
|
+
messageId: String(msg.id),
|
|
1772
|
+
payload: text,
|
|
1773
|
+
senderEnvelope: { userId: telegramUserId || undefined, firstName: pipeline.sender.firstName },
|
|
1774
|
+
topicMetadata: _topicPinStore?.asTopicMetadata(String(topicId)),
|
|
1775
|
+
}, 'ordering-behind-queued');
|
|
1776
|
+
if (ord.result === 'queued' || ord.result === 'already-queued') {
|
|
1777
|
+
console.log(`[inbound-queue] topic ${topicId} msg ${msg.id} queued behind existing entries (ordering)`);
|
|
1778
|
+
return;
|
|
1779
|
+
}
|
|
1780
|
+
// refused → fall through to route() — delivery beats both loss and
|
|
1781
|
+
// silence; the ordering violation is counted by the engine.
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
catch { /* gate is best-effort; route() owns the message */ }
|
|
1785
|
+
}
|
|
1749
1786
|
try {
|
|
1750
1787
|
const outcome = await _sessionRouter.route({
|
|
1751
1788
|
sessionKey: String(topicId),
|
|
1752
1789
|
messageId: String(msg.id),
|
|
1753
1790
|
payload: text,
|
|
1754
1791
|
topicMetadata: _topicPinStore?.asTopicMetadata(String(topicId)),
|
|
1792
|
+
// §2.2: sender identity captured at ingress, persisted with custody.
|
|
1793
|
+
senderEnvelope: { userId: telegramUserId || undefined, firstName: pipeline.sender.firstName },
|
|
1755
1794
|
});
|
|
1756
1795
|
// Routing-decision observability — the live transfer path is otherwise a black
|
|
1757
1796
|
// box (the recognizer logs its pin, but route()'s actual placement/forward
|
|
1758
1797
|
// decision was invisible; that hid the bug below from the first live test).
|
|
1759
|
-
console.log(`[session-pool] route topic ${topicId} → action=${outcome.action} owner=${outcome.owner ?? '?'} self=${_meshSelfId ?? '?'}`);
|
|
1798
|
+
console.log(`[session-pool] route topic ${topicId} → action=${outcome.action} owner=${outcome.owner ?? '?'} self=${_meshSelfId ?? '?'} acked=${outcome.acked}`);
|
|
1760
1799
|
// Short-circuit local dispatch whenever the session ended up on ANOTHER machine
|
|
1761
1800
|
// (forward/duplicate, OR a fresh remote 'spawned'/'owner-dead-replaced'). Before
|
|
1762
1801
|
// this, only 'forwarded'/'duplicate' were caught, so a just-moved topic was
|
|
@@ -1766,10 +1805,30 @@ getAttentionTopicId) {
|
|
|
1766
1805
|
console.log(`[session-pool] topic ${topicId} handled by owner ${outcome.owner ?? '?'} (${outcome.action}) — not dispatching locally`);
|
|
1767
1806
|
return;
|
|
1768
1807
|
}
|
|
1769
|
-
//
|
|
1770
|
-
//
|
|
1808
|
+
// Custody-ack short-circuit (§2.2): a queued/placement-blocked verdict
|
|
1809
|
+
// whose enqueue COMMITTED (acked) is the queue's message now — no local
|
|
1810
|
+
// fall-through. Un-custodied (refused/off/dry-run) keeps today's
|
|
1811
|
+
// fall-through. Wiring pins assert both directions.
|
|
1812
|
+
if ((outcome.action === 'queued' || outcome.action === 'placement-blocked') && outcome.acked) {
|
|
1813
|
+
console.log(`[inbound-queue] topic ${topicId} msg ${msg.id} in durable custody (${outcome.detail ?? outcome.action}) — drain will deliver`);
|
|
1814
|
+
return;
|
|
1815
|
+
}
|
|
1816
|
+
// 'handled-locally' / 'spawned'(self) / 'owner-dead-replaced'(self) /
|
|
1817
|
+
// un-acked 'queued'/'placement-blocked' → fall through to local dispatch.
|
|
1771
1818
|
}
|
|
1772
1819
|
catch (err) {
|
|
1820
|
+
// Route-throw fail-open is CUSTODY-AWARE (§2.2): a per-MESSAGE point
|
|
1821
|
+
// read against the store — a committed non-terminal row for THIS
|
|
1822
|
+
// message means the queue owns it (skip local dispatch); no row (or
|
|
1823
|
+
// engine dark) → today's fall-through. A point-read ERROR fails OPEN
|
|
1824
|
+
// to fall-through — the bounded duplicate window, §5-enumerated.
|
|
1825
|
+
try {
|
|
1826
|
+
if (_inboundQueue?.hasCommittedRow(String(topicId), String(msg.id))) {
|
|
1827
|
+
console.warn(`[session-pool] route error for topic ${topicId} but custody is committed — not dispatching locally: ${err instanceof Error ? err.message : String(err)}`);
|
|
1828
|
+
return;
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1831
|
+
catch { /* point-read error → fail open (fall through) */ }
|
|
1773
1832
|
console.warn(`[session-pool] route error for topic ${topicId} — falling back to local dispatch: ${err instanceof Error ? err.message : String(err)}`);
|
|
1774
1833
|
}
|
|
1775
1834
|
}
|
|
@@ -1893,6 +1952,99 @@ getAttentionTopicId) {
|
|
|
1893
1952
|
});
|
|
1894
1953
|
}
|
|
1895
1954
|
};
|
|
1955
|
+
// ── Durable Inbound Message Queue: the drain's local-delivery tail ──
|
|
1956
|
+
// (§3.1 via:'drain'.) Built from the SAME primitives as the live tail above
|
|
1957
|
+
// (injectTelegramMessage / respawnSessionForTopic / spawnSessionForTopic +
|
|
1958
|
+
// the spawningTopics guard) with the drain contract's divergences, each
|
|
1959
|
+
// enumerated in the spec: bypasses the intercept stack (a stored message is
|
|
1960
|
+
// DATA — this function never re-interprets commands or re-binds operators),
|
|
1961
|
+
// bypasses the ingress ledger (the receipt is the at-most-once authority),
|
|
1962
|
+
// suppresses the per-message "✓ Delivered" confirmation, AWAITS the tail
|
|
1963
|
+
// through the receipt write, injects with the STORED sender envelope, and
|
|
1964
|
+
// paces same-session runs at 1s.
|
|
1965
|
+
_drainLocalDeliver = async (dmsg, handover) => {
|
|
1966
|
+
const topicId = Number(dmsg.sessionKey);
|
|
1967
|
+
if (!Number.isFinite(topicId))
|
|
1968
|
+
return { kind: 'failed', error: new Error(`non-numeric sessionKey ${dmsg.sessionKey}`) };
|
|
1969
|
+
if (!_sessionRouter)
|
|
1970
|
+
return { kind: 'un-routable', reason: 'router-not-constructed' };
|
|
1971
|
+
// Routing first — ownership may have moved while the entry waited.
|
|
1972
|
+
const outcome = await _sessionRouter.route({
|
|
1973
|
+
sessionKey: dmsg.sessionKey,
|
|
1974
|
+
messageId: dmsg.messageId,
|
|
1975
|
+
payload: dmsg.payload,
|
|
1976
|
+
topicMetadata: dmsg.topicMetadata
|
|
1977
|
+
?? _topicPinStore?.asTopicMetadata(dmsg.sessionKey),
|
|
1978
|
+
senderEnvelope: dmsg.senderEnvelope,
|
|
1979
|
+
});
|
|
1980
|
+
if (outcome.action === 'forwarded' && outcome.detail === 'sender-rejected') {
|
|
1981
|
+
return { kind: 'sender-rejected' };
|
|
1982
|
+
}
|
|
1983
|
+
if (isRemotelyHandled(outcome, _meshSelfId)) {
|
|
1984
|
+
return { kind: 'remote-delivered' };
|
|
1985
|
+
}
|
|
1986
|
+
if (outcome.action === 'queued' || outcome.action === 'placement-blocked') {
|
|
1987
|
+
// The router re-queued our own entry ('already-queued' against the
|
|
1988
|
+
// claimed row) — un-routable: release + backoff + attempts++ (§3.1).
|
|
1989
|
+
return { kind: 'un-routable', reason: outcome.detail ?? outcome.action };
|
|
1990
|
+
}
|
|
1991
|
+
// Local delivery ('handled-locally' / self 'owner-dead-replaced').
|
|
1992
|
+
const sender = dmsg.senderEnvelope ?? undefined;
|
|
1993
|
+
const senderFirstName = sender?.firstName ?? 'User';
|
|
1994
|
+
const senderUserId = typeof sender?.userId === 'number' ? sender.userId : Number(sender?.userId ?? 0) || 0;
|
|
1995
|
+
const resolvedUser = senderUserId && userManager ? userManager.resolveFromTelegramUserId(senderUserId) : null;
|
|
1996
|
+
const targetSession = telegram.getSessionForTopic(topicId);
|
|
1997
|
+
const topicName = telegram.getTopicName(topicId) || `topic-${topicId}`;
|
|
1998
|
+
if (targetSession && sessionManager.isSessionAlive(targetSession)) {
|
|
1999
|
+
// Direct inject: receipt FIRST (the §3.4 handover point), stop re-check,
|
|
2000
|
+
// then the inject. A caught inject error AFTER the receipt is
|
|
2001
|
+
// local-delivered+injectError (never silent loss).
|
|
2002
|
+
if (!handover.commitReceipt())
|
|
2003
|
+
return { kind: 'handover-refused' };
|
|
2004
|
+
if (handover.stopRecheck())
|
|
2005
|
+
return { kind: 'stopped-before-inject' };
|
|
2006
|
+
try {
|
|
2007
|
+
sessionManager.injectTelegramMessage(targetSession, topicId, dmsg.payload, topicName, senderFirstName, senderUserId, Number(dmsg.messageId.replace(/^\D*/, '')) || undefined);
|
|
2008
|
+
telegram.trackMessageInjection(topicId, targetSession, dmsg.payload);
|
|
2009
|
+
// 1s inter-inject pacing for same-session runs (§3.1, pinned round-6).
|
|
2010
|
+
await new Promise((r) => setTimeout(r, 1000));
|
|
2011
|
+
return { kind: 'local-delivered' };
|
|
2012
|
+
}
|
|
2013
|
+
catch (err) {
|
|
2014
|
+
return { kind: 'local-delivered', injectError: err instanceof Error ? err.message : String(err) };
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
// Respawn / auto-spawn path. The spawn-in-progress guard maps to
|
|
2018
|
+
// un-routable (§3.1) — never a silent skip.
|
|
2019
|
+
if (spawningTopics.has(topicId)) {
|
|
2020
|
+
return { kind: 'un-routable', reason: 'spawn-in-progress' };
|
|
2021
|
+
}
|
|
2022
|
+
if (!handover.commitReceipt())
|
|
2023
|
+
return { kind: 'handover-refused' };
|
|
2024
|
+
if (handover.stopRecheck())
|
|
2025
|
+
return { kind: 'stopped-before-inject' };
|
|
2026
|
+
spawningTopics.add(topicId);
|
|
2027
|
+
try {
|
|
2028
|
+
if (targetSession) {
|
|
2029
|
+
// Dead session — respawn, AWAITED through the spawn+inject (the PIS
|
|
2030
|
+
// record is written inside, AFTER our receipt — the round-7 order pin).
|
|
2031
|
+
await respawnSessionForTopic(sessionManager, telegram, targetSession, topicId, dmsg.payload, topicMemory, resolvedUser ?? undefined);
|
|
2032
|
+
}
|
|
2033
|
+
else {
|
|
2034
|
+
const newSessionName = await spawnSessionForTopic(sessionManager, telegram, topicName, topicId, dmsg.payload, topicMemory, resolvedUser ?? undefined);
|
|
2035
|
+
telegram.registerTopicSession(topicId, newSessionName, topicName);
|
|
2036
|
+
}
|
|
2037
|
+
return { kind: 'local-delivered' };
|
|
2038
|
+
}
|
|
2039
|
+
catch (err) {
|
|
2040
|
+
// Receipt already committed — the at-most-once authority forbids a
|
|
2041
|
+
// re-inject; honest disposition: delivered-unconfirmed + report (§3.4).
|
|
2042
|
+
return { kind: 'local-delivered', injectError: err instanceof Error ? err.message : String(err) };
|
|
2043
|
+
}
|
|
2044
|
+
finally {
|
|
2045
|
+
spawningTopics.delete(topicId);
|
|
2046
|
+
}
|
|
2047
|
+
};
|
|
1896
2048
|
}
|
|
1897
2049
|
/**
|
|
1898
2050
|
* Wire WhatsApp message routing: incoming messages → Claude sessions.
|
|
@@ -5885,6 +6037,104 @@ export async function startServer(options) {
|
|
|
5885
6037
|
// lifeline to restart the server in a tight loop.
|
|
5886
6038
|
await sessionManager.purgeDeadSessions();
|
|
5887
6039
|
sessionManager.startMonitoring();
|
|
6040
|
+
// ── Durable Inbound Message Queue: unconditional boot sweep (§5.3) ──
|
|
6041
|
+
// MUST run BEFORE recoverPendingInjects (boot ordering, spec §3.4): the
|
|
6042
|
+
// sweep consults injection receipts and vetoes PIS records for
|
|
6043
|
+
// operator-stop rows — running PIS recovery first would replay an inject
|
|
6044
|
+
// the operator stopped. Keyed on store-file existence; fail-open
|
|
6045
|
+
// (quarantine) on a corrupt store; gate-expires all custody with a NAMED
|
|
6046
|
+
// reason when the drain will not run this boot. The mesh-identity gate is
|
|
6047
|
+
// resolved later (the mesh block) — when the wiring there finds no
|
|
6048
|
+
// identity, it runs the same expire-all with reason no-mesh-identity
|
|
6049
|
+
// before any drain could have started.
|
|
6050
|
+
try {
|
|
6051
|
+
const sweepMod = await import('../core/inboundQueueBootSweep.js');
|
|
6052
|
+
const pisMod = await import('../core/PendingInjectStore.js');
|
|
6053
|
+
const sweepPis = new pisMod.PendingInjectStore(path.join(config.stateDir, 'state'));
|
|
6054
|
+
const pisRecordsForTopic = (sessionKey) => sweepPis.list().records.filter((r) => String(r.telegramTopicId ?? '') === sessionKey);
|
|
6055
|
+
const qRaw = config.multiMachine?.sessionPool?.inboundQueue;
|
|
6056
|
+
const poolRaw = config.multiMachine?.sessionPool;
|
|
6057
|
+
const queueWillRun = qRaw?.enabled !== true ? { run: false, gateReason: 'feature-disabled' }
|
|
6058
|
+
: qRaw?.dryRun !== false ? { run: false, gateReason: 'dry-run' }
|
|
6059
|
+
: (poolRaw?.enabled !== true || (poolRaw?.stage ?? 'dark') === 'dark') ? { run: false, gateReason: 'pool-dark' }
|
|
6060
|
+
: { run: true };
|
|
6061
|
+
const sweepRes = sweepMod.runInboundQueueBootSweep({
|
|
6062
|
+
stateDir: config.stateDir,
|
|
6063
|
+
agentId: config.projectName ?? 'agent',
|
|
6064
|
+
queueWillRun,
|
|
6065
|
+
hasPisRecord: (sk) => pisRecordsForTopic(sk).length > 0,
|
|
6066
|
+
clearPisRecord: (sk) => { for (const r of pisRecordsForTopic(sk))
|
|
6067
|
+
sweepPis.clear(r.tmuxSession); },
|
|
6068
|
+
reportLoss: (items, reason) => {
|
|
6069
|
+
const topics = [...new Set(items.map((i) => i.sessionKey))].join(', ');
|
|
6070
|
+
notify('SUMMARY', 'inbound-queue', `I didn't get to ${items.length} queued message(s) (${reason}; topics: ${topics}) — resend anything still needed.`);
|
|
6071
|
+
},
|
|
6072
|
+
reportPossiblyNotInjected: (items) => {
|
|
6073
|
+
const topics = [...new Set(items.map((i) => i.sessionKey))].join(', ');
|
|
6074
|
+
notify('SUMMARY', 'inbound-queue', `${items.length} message(s) may not have been injected before a crash (topics: ${topics}) — if a message went unanswered, resend it.`);
|
|
6075
|
+
},
|
|
6076
|
+
raiseAttention: (title, body) => notify('IMMEDIATE', 'inbound-queue', `${title}: ${body}`),
|
|
6077
|
+
log: (line) => console.log(pc.dim(` ${line}`)),
|
|
6078
|
+
nowMs: () => Date.now(),
|
|
6079
|
+
});
|
|
6080
|
+
_sweptInboundStore = sweepRes.store;
|
|
6081
|
+
if (sweepRes.storePresent) {
|
|
6082
|
+
console.log(pc.dim(` [inbound-queue] boot sweep: gateExpired=${sweepRes.gateExpired} recovered=${sweepRes.recoveredToQueued} ` +
|
|
6083
|
+
`delivered=${sweepRes.settledDelivered} pni=${sweepRes.possiblyNotInjected} pisVetoed=${sweepRes.pisVetoed} quarantined=${sweepRes.quarantined}`));
|
|
6084
|
+
}
|
|
6085
|
+
// Engine-never-constructed backstop (second-pass concern 1 — the
|
|
6086
|
+
// no-mesh-identity gate, made unreachable-proof): the sweep ran with
|
|
6087
|
+
// run:true expecting the mesh block to construct the drain. If, 90s into
|
|
6088
|
+
// boot, the swept store is still unadopted (no mesh identity, or the
|
|
6089
|
+
// construction threw), expire its custody with the NAMED reason and
|
|
6090
|
+
// close the handle — custody is NEVER silently stranded (§5.3), through
|
|
6091
|
+
// ANY non-construction path. One-shot; a no-op when the engine is live.
|
|
6092
|
+
if (sweepRes.store) {
|
|
6093
|
+
const orphanTimer = setTimeout(() => {
|
|
6094
|
+
if (!_sweptInboundStore || _inboundQueue)
|
|
6095
|
+
return;
|
|
6096
|
+
try {
|
|
6097
|
+
const store = _sweptInboundStore;
|
|
6098
|
+
const nowIso = new Date().toISOString();
|
|
6099
|
+
const rows = store.listNonTerminal();
|
|
6100
|
+
const dropped = [];
|
|
6101
|
+
for (const row of rows) {
|
|
6102
|
+
if (store.transition(row.enqueue_seq, row.state, 'expired', { nowIso, terminalReason: 'gate:no-mesh-identity' })) {
|
|
6103
|
+
dropped.push(row.session_key);
|
|
6104
|
+
}
|
|
6105
|
+
}
|
|
6106
|
+
if (dropped.length > 0) {
|
|
6107
|
+
notify('SUMMARY', 'inbound-queue', `I didn't get to ${dropped.length} queued message(s) (the queue is enabled but this machine has no mesh identity, so the drain never started; topics: ${[...new Set(dropped)].join(', ')}) — resend anything still needed.`);
|
|
6108
|
+
}
|
|
6109
|
+
store.close();
|
|
6110
|
+
_sweptInboundStore = null;
|
|
6111
|
+
console.warn('[inbound-queue] swept store never adopted by an engine — custody gate-expired (no-mesh-identity) and store closed');
|
|
6112
|
+
}
|
|
6113
|
+
catch (e) {
|
|
6114
|
+
console.warn(`[inbound-queue] orphan-store backstop failed (next boot's sweep retries): ${e instanceof Error ? e.message : String(e)}`);
|
|
6115
|
+
}
|
|
6116
|
+
}, 90_000);
|
|
6117
|
+
orphanTimer.unref?.();
|
|
6118
|
+
}
|
|
6119
|
+
}
|
|
6120
|
+
catch (err) {
|
|
6121
|
+
// The sweep is a recovery backstop — its own failure must never block
|
|
6122
|
+
// boot. The next boot retries; rows are durable.
|
|
6123
|
+
console.error(`[server] inbound-queue boot sweep failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
6124
|
+
}
|
|
6125
|
+
// Hold-for-stability effective-state getter (§4.2) — registered on the
|
|
6126
|
+
// UNCONDITIONAL boot path (same path as the sweep above), as a closure
|
|
6127
|
+
// over the effective-state computation: always-failover default ⇒
|
|
6128
|
+
// enabled:false, so the orphaned-config case (hold on, queue off/dark)
|
|
6129
|
+
// derives /guards off-runtime-divergent instead of on-unverified.
|
|
6130
|
+
try {
|
|
6131
|
+
guardRegistry.register('multiMachine.sessionPool.holdForStability.enabled', () => ({
|
|
6132
|
+
enabled: config.multiMachine?.sessionPool?.holdForStability?.enabled === true &&
|
|
6133
|
+
_inboundQueue !== null,
|
|
6134
|
+
lastTickAt: Date.now(),
|
|
6135
|
+
}));
|
|
6136
|
+
}
|
|
6137
|
+
catch { /* posture observability never blocks boot */ }
|
|
5888
6138
|
// Pending-inject recovery (finding 8d300555): re-deliver initial messages
|
|
5889
6139
|
// orphaned by the previous server process dying in the spawn→ready→inject
|
|
5890
6140
|
// window (the auto-updater restart race). Runs AFTER the purge so dead
|
|
@@ -8649,6 +8899,13 @@ export async function startServer(options) {
|
|
|
8649
8899
|
semanticMemory?.checkpoint();
|
|
8650
8900
|
}
|
|
8651
8901
|
catch { /* non-critical */ }
|
|
8902
|
+
// Durable Inbound Message Queue §6: sleep-shift backoff deadlines by the
|
|
8903
|
+
// nap span + the nap clamp (stale rows expire reported). Low wake
|
|
8904
|
+
// confidence → the conservative branch (clamp without shift).
|
|
8905
|
+
try {
|
|
8906
|
+
void _inboundQueue?.onWake(event.sleepDurationSeconds * 1000, event.lowConfidence ? 'low' : 'high');
|
|
8907
|
+
}
|
|
8908
|
+
catch { /* the backstop tick covers a missed wake trigger */ }
|
|
8652
8909
|
// Re-validate tmux sessions
|
|
8653
8910
|
try {
|
|
8654
8911
|
const tmuxPath = detectTmuxPath();
|
|
@@ -9027,6 +9284,13 @@ export async function startServer(options) {
|
|
|
9027
9284
|
}
|
|
9028
9285
|
return null; // Normal messages pass through
|
|
9029
9286
|
};
|
|
9287
|
+
// Durable Inbound Message Queue §3.6: emergency stop reaches custody.
|
|
9288
|
+
telegram.onSentinelStopCustody = (topicId) => {
|
|
9289
|
+
try {
|
|
9290
|
+
_inboundQueueStop?.(String(topicId));
|
|
9291
|
+
}
|
|
9292
|
+
catch { /* best-effort */ }
|
|
9293
|
+
};
|
|
9030
9294
|
telegram.onSentinelKillSession = (sessionName) => {
|
|
9031
9295
|
// Save resume UUID before killing so respawn can --resume
|
|
9032
9296
|
if (_topicResumeMap) {
|
|
@@ -11117,6 +11381,19 @@ export async function startServer(options) {
|
|
|
11117
11381
|
return null; /* @silent-fallback-ok — pool not wired yet → rule inert */
|
|
11118
11382
|
}
|
|
11119
11383
|
},
|
|
11384
|
+
// WS1.3: pin-conflict do-not-act — a pin naming THIS machine while the
|
|
11385
|
+
// owner is elsewhere means the reconciler is bringing the topic back;
|
|
11386
|
+
// the closeout holds instead of attacking the session the pin wants here.
|
|
11387
|
+
topicPinnedHere: (topicId) => {
|
|
11388
|
+
try {
|
|
11389
|
+
const self = _meshSelfId;
|
|
11390
|
+
const pin = _topicPinStore?.get(String(topicId));
|
|
11391
|
+
return !!self && !!pin && pin.pinned && pin.preferredMachine === self;
|
|
11392
|
+
}
|
|
11393
|
+
catch {
|
|
11394
|
+
return false; /* @silent-fallback-ok — no pin signal → hold not applied, closeout behaves as before */
|
|
11395
|
+
}
|
|
11396
|
+
},
|
|
11120
11397
|
audit: reaperAuditSink(config.stateDir),
|
|
11121
11398
|
},
|
|
11122
11399
|
// developmentAgent gate (standard_development_agent_dark_feature_gate):
|
|
@@ -11615,6 +11892,28 @@ export async function startServer(options) {
|
|
|
11615
11892
|
loadAvg: osMod.loadavg()[0],
|
|
11616
11893
|
quotaState: selfQuotaState(),
|
|
11617
11894
|
guardPosture: selfGuardPosture(),
|
|
11895
|
+
// Durable Inbound Message Queue §5.1: depth + oldest + tenure +
|
|
11896
|
+
// bounded top-K — the survivor's loss-SUSPECTED item, capped
|
|
11897
|
+
// re-placement arm, and supersede-dedupe key all read these.
|
|
11898
|
+
// Absent while the queue is dark — depth honestly unknown.
|
|
11899
|
+
...(() => {
|
|
11900
|
+
try {
|
|
11901
|
+
if (!_inboundQueue)
|
|
11902
|
+
return {};
|
|
11903
|
+
const snap = _inboundQueue.snapshot();
|
|
11904
|
+
return {
|
|
11905
|
+
inboundQueue: {
|
|
11906
|
+
queueDepth: snap.counts.queued + snap.counts.claimed,
|
|
11907
|
+
oldestQueuedAt: snap.counts.oldestQueuedAt,
|
|
11908
|
+
tenure: snap.tenure,
|
|
11909
|
+
topK: _inboundQueue.topKSessionDepths(10),
|
|
11910
|
+
},
|
|
11911
|
+
};
|
|
11912
|
+
}
|
|
11913
|
+
catch {
|
|
11914
|
+
return {};
|
|
11915
|
+
}
|
|
11916
|
+
})(),
|
|
11618
11917
|
});
|
|
11619
11918
|
const hbApi = machineHeartbeat?.api;
|
|
11620
11919
|
if (hbApi) {
|
|
@@ -11712,6 +12011,72 @@ export async function startServer(options) {
|
|
|
11712
12011
|
}
|
|
11713
12012
|
},
|
|
11714
12013
|
};
|
|
12014
|
+
// ── WS1.3 ownership reconciler (MULTI-MACHINE-SEAMLESSNESS-SPEC) ──
|
|
12015
|
+
// Bounded pin/owner convergence on a tick, replacing "wait for an
|
|
12016
|
+
// inbound message that delivery may never route" (the 2026-06-12 stuck
|
|
12017
|
+
// transfer-back). Dark + dry-run defaults; strict single-machine no-op
|
|
12018
|
+
// inside the module. Phase C: quorum logic is N-machine from day one.
|
|
12019
|
+
if (_topicPinStore && _meshSelfId) {
|
|
12020
|
+
const ws13Cfg = () => (config.multiMachine?.seamlessness ?? {});
|
|
12021
|
+
const { OwnershipReconciler } = await import('../core/OwnershipReconciler.js');
|
|
12022
|
+
const reconciler = new OwnershipReconciler({
|
|
12023
|
+
enabled: () => ws13Cfg().ws13Reconcile === true,
|
|
12024
|
+
dryRun: () => ws13Cfg().ws13DryRun !== false,
|
|
12025
|
+
selfMachineId: _meshSelfId,
|
|
12026
|
+
pinStore: _topicPinStore,
|
|
12027
|
+
ownership: ownReg,
|
|
12028
|
+
machines: () => {
|
|
12029
|
+
try {
|
|
12030
|
+
return (machinePoolRegistry?.getCapacities() ?? []).map((c) => ({
|
|
12031
|
+
machineId: c.machineId,
|
|
12032
|
+
online: !!c.online,
|
|
12033
|
+
lastSeenMs: c.selfReportedLastSeen ? Date.parse(c.selfReportedLastSeen) || 0 : 0,
|
|
12034
|
+
}));
|
|
12035
|
+
}
|
|
12036
|
+
catch {
|
|
12037
|
+
return []; /* @silent-fallback-ok — no pool view → module's single-machine strict no-op */
|
|
12038
|
+
}
|
|
12039
|
+
},
|
|
12040
|
+
isTopicBusy: (sessionKey) => {
|
|
12041
|
+
try {
|
|
12042
|
+
// Conservative safe-point signal: an inbound for this topic is
|
|
12043
|
+
// mid-processing. The bounded deadline in the module keeps a
|
|
12044
|
+
// false-busy from deferring forever.
|
|
12045
|
+
return currentInboundByTopic?.has(sessionKey) ?? false;
|
|
12046
|
+
}
|
|
12047
|
+
catch {
|
|
12048
|
+
return false; /* @silent-fallback-ok — unknown busy state defers to the module's bounded deadline */
|
|
12049
|
+
}
|
|
12050
|
+
},
|
|
12051
|
+
emitPlacement: (sessionKey, r, reason) => {
|
|
12052
|
+
try {
|
|
12053
|
+
const topicNum = Number(sessionKey);
|
|
12054
|
+
if (Number.isFinite(topicNum)) {
|
|
12055
|
+
state.getCoherenceJournal()?.emitPlacement(topicNum, {
|
|
12056
|
+
owner: r.record.ownerMachineId ?? '',
|
|
12057
|
+
epoch: r.record.ownershipEpoch,
|
|
12058
|
+
reason: 'reconcile',
|
|
12059
|
+
});
|
|
12060
|
+
}
|
|
12061
|
+
}
|
|
12062
|
+
catch { /* §3.3 pairing is observability — never endangers the CAS */ }
|
|
12063
|
+
},
|
|
12064
|
+
logger: (m) => console.log(pc.dim(` ${m}`)),
|
|
12065
|
+
});
|
|
12066
|
+
const tickMs = Math.max(5000, ws13Cfg().ws13TickMs ?? 30000);
|
|
12067
|
+
const ws13Timer = setInterval(() => {
|
|
12068
|
+
try {
|
|
12069
|
+
const rep = reconciler.tick();
|
|
12070
|
+
if (!rep.skipped && (rep.transfers || rep.claims || rep.forceClaims || rep.adoptions)) {
|
|
12071
|
+
console.log(pc.dim(` [OwnershipReconciler] tick: t=${rep.transfers} c=${rep.claims} f=${rep.forceClaims} a=${rep.adoptions}${rep.dryRun ? ' (dry-run)' : ''}`));
|
|
12072
|
+
}
|
|
12073
|
+
}
|
|
12074
|
+
catch (err) {
|
|
12075
|
+
console.error('[OwnershipReconciler] tick failed:', err instanceof Error ? err.message : String(err));
|
|
12076
|
+
}
|
|
12077
|
+
}, tickMs);
|
|
12078
|
+
ws13Timer.unref?.();
|
|
12079
|
+
}
|
|
11715
12080
|
// Coherence journal §3.3: the emit is a thin wrapper at every CAS call
|
|
11716
12081
|
// site — `reason` is caller knowledge (cas() is a storage primitive and
|
|
11717
12082
|
// cannot know WHY). A mesh-applied action records the coarse reason;
|
|
@@ -11719,6 +12084,15 @@ export async function startServer(options) {
|
|
|
11719
12084
|
// cas-pairing lint (scripts/lint-cas-emit-placement.js) fails CI on
|
|
11720
12085
|
// any cas( call site missing this pairing.
|
|
11721
12086
|
const emitPlacement = (sessionKey, r, reason, prevOwner) => {
|
|
12087
|
+
// Durable Inbound Message Queue §3.2 event trigger: an ownership
|
|
12088
|
+
// transition for a session with queued entries makes its head due
|
|
12089
|
+
// now (scoped reset + a drain pass). Fire-and-forget; the engine
|
|
12090
|
+
// no-ops when the session has nothing queued.
|
|
12091
|
+
try {
|
|
12092
|
+
if (r?.ok)
|
|
12093
|
+
void _inboundQueue?.onOwnershipTransition(sessionKey);
|
|
12094
|
+
}
|
|
12095
|
+
catch { /* trigger is best-effort; the backstop tick covers it */ }
|
|
11722
12096
|
try {
|
|
11723
12097
|
if (!coherenceJournal || !r?.ok)
|
|
11724
12098
|
return;
|
|
@@ -11774,6 +12148,15 @@ export async function startServer(options) {
|
|
|
11774
12148
|
const deliverMessageHandler = deliverMod.createDeliverMessageHandler({
|
|
11775
12149
|
ownerEpochOf: (s) => ownReg.read(s)?.ownershipEpoch ?? null,
|
|
11776
12150
|
recordReceipt: (messageId, session) => {
|
|
12151
|
+
// Durable Inbound Message Queue §3.4 remote path: the queue store's
|
|
12152
|
+
// remote receipt (canonical-id keyed, carries the `injected` marker
|
|
12153
|
+
// that makes peer-crash-between-receipt-and-inject boot-detectable —
|
|
12154
|
+
// loss window 6). Recorded ALONGSIDE the existing ledger receipt;
|
|
12155
|
+
// engine dark → the prior posture, named in the spec's skew note.
|
|
12156
|
+
try {
|
|
12157
|
+
_inboundQueue?.recordRemoteReceipt(session, messageId);
|
|
12158
|
+
}
|
|
12159
|
+
catch { /* receipt best-effort; ledger is authoritative for dedupe */ }
|
|
11777
12160
|
if (messageLedger)
|
|
11778
12161
|
return messageLedger.record(messageId, { platform: 'mesh', topic: session }).firstSeen;
|
|
11779
12162
|
if (deliverSeenFallback.has(messageId))
|
|
@@ -11781,6 +12164,23 @@ export async function startServer(options) {
|
|
|
11781
12164
|
deliverSeenFallback.add(messageId);
|
|
11782
12165
|
return true;
|
|
11783
12166
|
},
|
|
12167
|
+
// §3.4 sender re-validation (per-machine registries can diverge during
|
|
12168
|
+
// a deauthorization): a carried envelope whose userId no longer
|
|
12169
|
+
// resolves on THIS machine NACKs `sender-rejected`. Envelope absent
|
|
12170
|
+
// (old peer / live local frame) → not consulted.
|
|
12171
|
+
validateSender: (envelope) => {
|
|
12172
|
+
const uid = Number(envelope.userId);
|
|
12173
|
+
if (!Number.isFinite(uid) || uid === 0)
|
|
12174
|
+
return true;
|
|
12175
|
+
// THIS machine's registry (file-backed; the telegram block's
|
|
12176
|
+
// instance is scoped there — same files, same truth).
|
|
12177
|
+
try {
|
|
12178
|
+
return new UserManager(config.stateDir, config.users).resolveFromTelegramUserId(uid) !== null;
|
|
12179
|
+
}
|
|
12180
|
+
catch {
|
|
12181
|
+
return true;
|
|
12182
|
+
}
|
|
12183
|
+
},
|
|
11784
12184
|
// Owner-side bridge (§L4 handoff): a forwarded message landed → spawn/resume
|
|
11785
12185
|
// the local session for the topic so the conversation continues on THIS machine.
|
|
11786
12186
|
// Only fires for a FIRST-seen forwarded deliverMessage (the ledger dedupes
|
|
@@ -11824,8 +12224,17 @@ export async function startServer(options) {
|
|
|
11824
12224
|
if (existing && sessionManager.isSessionAlive(existing)) {
|
|
11825
12225
|
if (text) {
|
|
11826
12226
|
console.log(pc.green(` [session-pool] owner-side inject for forwarded topic ${topicId} → ${existing}`));
|
|
11827
|
-
|
|
11828
|
-
|
|
12227
|
+
// Loss window 6 (Durable Inbound Message Queue §3.4 remote):
|
|
12228
|
+
// flip the receipt's injected marker on success; a CAUGHT
|
|
12229
|
+
// failure reports at error time — never silent.
|
|
12230
|
+
try {
|
|
12231
|
+
sessionManager.injectTelegramMessage(existing, topicId, text, tg.getTopicName?.(topicId) ?? undefined);
|
|
12232
|
+
tg.trackMessageInjection(topicId, existing, text);
|
|
12233
|
+
_inboundQueue?.markRemoteInjected(cmd.session, cmd.messageId);
|
|
12234
|
+
}
|
|
12235
|
+
catch (err) {
|
|
12236
|
+
_inboundQueue?.reportPeerInjectError(cmd.session, cmd.messageId, err instanceof Error ? err.message : String(err));
|
|
12237
|
+
}
|
|
11829
12238
|
}
|
|
11830
12239
|
return;
|
|
11831
12240
|
}
|
|
@@ -11859,8 +12268,13 @@ export async function startServer(options) {
|
|
|
11859
12268
|
.then((name) => {
|
|
11860
12269
|
tg.registerTopicSession(topicId, name, spawnName);
|
|
11861
12270
|
console.log(pc.green(` [session-pool] owner-side resume for forwarded topic ${topicId} → ${name}`));
|
|
12271
|
+
// Loss window 6: the forwarded message reached a real session.
|
|
12272
|
+
_inboundQueue?.markRemoteInjected(cmd.session, cmd.messageId);
|
|
11862
12273
|
})
|
|
11863
|
-
.catch((err) =>
|
|
12274
|
+
.catch((err) => {
|
|
12275
|
+
console.warn(` [session-pool] owner-side resume failed for topic ${topicId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
12276
|
+
_inboundQueue?.reportPeerInjectError(cmd.session, cmd.messageId, err instanceof Error ? err.message : String(err));
|
|
12277
|
+
});
|
|
11864
12278
|
},
|
|
11865
12279
|
});
|
|
11866
12280
|
// ── Secret-sync inbound handler (cross-machine secret distribution, spec Phase 4) ──
|
|
@@ -12688,6 +13102,15 @@ export async function startServer(options) {
|
|
|
12688
13102
|
// signal per episode.
|
|
12689
13103
|
const ownerSuspectBreaker = new (await import('../core/OwnerSuspectBreaker.js')).OwnerSuspectBreaker({
|
|
12690
13104
|
logger: (m) => console.log(pc.dim(m)),
|
|
13105
|
+
// Durable Inbound Message Queue §3.2: breaker close delivers held
|
|
13106
|
+
// rows instantly. Engine constructed later in this block — the
|
|
13107
|
+
// closure reads at fire time.
|
|
13108
|
+
onClose: () => { void _inboundQueue?.onBreakerClose(); },
|
|
13109
|
+
flapThresholdPerHour: config.multiMachine?.sessionPool?.holdForStability?.flapThresholdPerHour ?? 6,
|
|
13110
|
+
reportFlapEpisode: ({ machineId, episodesLastHour }) => {
|
|
13111
|
+
const nickname = machinePoolRegistry?.getCapacity(machineId)?.nickname ?? machineId;
|
|
13112
|
+
notify('SUMMARY', 'inbound-queue', `"${nickname}" is flapping (${episodesLastHour} suspect episodes in the last hour) — holds are disabled for it until it calms; its conversations move on the usual failover path.`);
|
|
13113
|
+
},
|
|
12691
13114
|
reportSustained: ({ machineId, suspectForMs, marks }) => {
|
|
12692
13115
|
DegradationReporter.getInstance().report({
|
|
12693
13116
|
feature: 'SessionPool.ownerDelivery',
|
|
@@ -12753,7 +13176,10 @@ export async function startServer(options) {
|
|
|
12753
13176
|
const url = peerUrl(target);
|
|
12754
13177
|
if (!url)
|
|
12755
13178
|
throw new Error(`no peer url for ${target}`);
|
|
12756
|
-
|
|
13179
|
+
// senderEnvelope (Durable Inbound Message Queue §2.2): a drained
|
|
13180
|
+
// forward carries the STORED sender frame; an old peer ignores
|
|
13181
|
+
// the extra field (version skew named in the spec).
|
|
13182
|
+
const res = await meshClient.send({ machineId: target, url }, { type: 'deliverMessage', session: env.sessionKey, messageId: env.messageId, payload: env.payload, ownershipEpoch: env.ownershipEpoch, ...(env.senderEnvelope ? { senderEnvelope: env.senderEnvelope } : {}) }, env.ownershipEpoch);
|
|
12757
13183
|
if (res.ok && res.result && typeof res.result === 'object' && 'accepted' in res.result) {
|
|
12758
13184
|
return res.result;
|
|
12759
13185
|
}
|
|
@@ -12766,11 +13192,192 @@ export async function startServer(options) {
|
|
|
12766
13192
|
await meshClient.send({ machineId, url }, { type: 'deliverMessage', session: msg.sessionKey, messageId: msg.messageId, payload: msg.payload, ownershipEpoch: 0 }, 0);
|
|
12767
13193
|
},
|
|
12768
13194
|
handleLocally: async () => { },
|
|
12769
|
-
|
|
13195
|
+
// Durable Inbound Message Queue §2.2: tri-state custody taking.
|
|
13196
|
+
// Null engine (dark / gate failed / invariants violated) → 'refused'
|
|
13197
|
+
// → the router leaves acked:false → today's fall-through. enqueueLive
|
|
13198
|
+
// never throws (storage failure maps to 'refused' — fail-safe).
|
|
13199
|
+
queueMessage: (msg, reason) => {
|
|
13200
|
+
if (!_inboundQueue)
|
|
13201
|
+
return 'refused';
|
|
13202
|
+
const out = _inboundQueue.enqueueLive({
|
|
13203
|
+
sessionKey: msg.sessionKey,
|
|
13204
|
+
messageId: msg.messageId,
|
|
13205
|
+
payload: typeof msg.payload === 'string' ? msg.payload : JSON.stringify(msg.payload ?? ''),
|
|
13206
|
+
senderEnvelope: msg.senderEnvelope ?? null,
|
|
13207
|
+
topicMetadata: msg.topicMetadata,
|
|
13208
|
+
}, reason);
|
|
13209
|
+
return out.result;
|
|
13210
|
+
},
|
|
12770
13211
|
raiseAttention: (title, body) => console.log(pc.dim(` [session-router] attention: ${title} — ${body}`)),
|
|
12771
13212
|
sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
12772
13213
|
log: (line) => console.log(pc.dim(` [session-router] ${line}`)),
|
|
12773
13214
|
});
|
|
13215
|
+
// ── Durable Inbound Message Queue: engine construction (gated) ──
|
|
13216
|
+
// Gate: pool active (this block) + mesh identity (this block) +
|
|
13217
|
+
// inboundQueue.enabled + !dryRun + the six config-seam invariants.
|
|
13218
|
+
// A violated invariant keeps the queue OFF for the boot — one loud
|
|
13219
|
+
// config-error per violated inequality + one attention item, never a
|
|
13220
|
+
// half-configured queue (spec §Config).
|
|
13221
|
+
try {
|
|
13222
|
+
const iqcMod = await import('../core/inboundQueueConfig.js');
|
|
13223
|
+
const qcfg = { ...iqcMod.DEFAULT_INBOUND_QUEUE_CONFIG, ...(config.multiMachine?.sessionPool?.inboundQueue ?? {}) };
|
|
13224
|
+
const hcfg = { ...iqcMod.DEFAULT_HOLD_FOR_STABILITY_CONFIG, ...(config.multiMachine?.sessionPool?.holdForStability ?? {}) };
|
|
13225
|
+
// Dry-run constructs the engine too (second-pass concern 2): the
|
|
13226
|
+
// §2.4 dry-run branch never takes custody, but its durable
|
|
13227
|
+
// wouldEnqueue/wouldHold/wouldRefuse counters ARE the promotion
|
|
13228
|
+
// evidence and /pool/queue must serve them. The boot sweep already
|
|
13229
|
+
// gate-expired any live→dry-run residual rows.
|
|
13230
|
+
if (qcfg.enabled && _sessionPoolStage() !== 'dark') {
|
|
13231
|
+
const inv = iqcMod.validateInboundQueueInvariants(qcfg, hcfg);
|
|
13232
|
+
if (!inv.ok) {
|
|
13233
|
+
for (const v of inv.violations) {
|
|
13234
|
+
console.error(`[inbound-queue] CONFIG ERROR — invariant ${v.invariant} (${v.name}): ${v.message}`);
|
|
13235
|
+
}
|
|
13236
|
+
notify('IMMEDIATE', 'inbound-queue', `Inbound queue NOT started — ${inv.violations.length} config invariant(s) violated (${inv.violations.map((v) => v.name).join(', ')}). The queue stays OFF; messages use today's delivery path.`);
|
|
13237
|
+
_sweptInboundStore?.close();
|
|
13238
|
+
_sweptInboundStore = null;
|
|
13239
|
+
}
|
|
13240
|
+
else {
|
|
13241
|
+
const qdlMod = await import('../core/QueueDrainLoop.js');
|
|
13242
|
+
const pisStoreMod = await import('../core/PendingInboundStore.js');
|
|
13243
|
+
const pisMod2 = await import('../core/PendingInjectStore.js');
|
|
13244
|
+
const queuePis = new pisMod2.PendingInjectStore(path.join(config.stateDir, 'state'));
|
|
13245
|
+
const store = _sweptInboundStore ?? pisStoreMod.PendingInboundStore.open(config.projectName ?? 'agent', config.stateDir);
|
|
13246
|
+
const bootSessionId = `${meshSelfId}:${Date.now()}`;
|
|
13247
|
+
// §4.2 hold verdict — effective state honesty: always-'failover'
|
|
13248
|
+
// when the policy is off or the queue is dry-run (unreachable
|
|
13249
|
+
// here: dryRun gates construction) — config-coupled both ways.
|
|
13250
|
+
const holdOn = hcfg.enabled === true;
|
|
13251
|
+
const holdVerdict = (sessionKey) => {
|
|
13252
|
+
if (!holdOn)
|
|
13253
|
+
return 'failover';
|
|
13254
|
+
const own = ownReg.read(sessionKey);
|
|
13255
|
+
const owner = own ? ownReg.ownerOf(sessionKey) : null;
|
|
13256
|
+
if (!owner || owner === meshSelfId)
|
|
13257
|
+
return 'deliver';
|
|
13258
|
+
const cap = machinePoolRegistry?.getCapacity(owner);
|
|
13259
|
+
if (!cap?.online)
|
|
13260
|
+
return 'failover'; // heartbeat offline — dead is dead
|
|
13261
|
+
if (ownerSuspectBreaker.isFlapping(owner))
|
|
13262
|
+
return 'failover'; // §4.4
|
|
13263
|
+
if (ownerSuspectBreaker.isSuspect(owner))
|
|
13264
|
+
return 'hold'; // suspect + online
|
|
13265
|
+
return 'deliver'; // not suspect (exhaustion site) — enqueue-and-drain
|
|
13266
|
+
};
|
|
13267
|
+
const stoppedTopics = new Set();
|
|
13268
|
+
_inboundQueue = new qdlMod.QueueDrainLoop({
|
|
13269
|
+
store,
|
|
13270
|
+
qcfg,
|
|
13271
|
+
hcfg,
|
|
13272
|
+
selfMachineId: meshSelfId,
|
|
13273
|
+
// The real serving-lease signal (second-pass concern 3): when
|
|
13274
|
+
// a lease coordinator exists, custody is gated on actually
|
|
13275
|
+
// HOLDING the lease (§2.2 — custody only where it can be
|
|
13276
|
+
// drained); a single-machine install with no coordinator
|
|
13277
|
+
// defaults true (the single-router topology).
|
|
13278
|
+
holdsLease: () => (leaseCoordinatorRef ? leaseCoordinatorRef.holdsLease() : true),
|
|
13279
|
+
isStopped: (sk) => stoppedTopics.has(sk),
|
|
13280
|
+
dispatchInbound: async (msg, handover) => {
|
|
13281
|
+
if (!_drainLocalDeliver)
|
|
13282
|
+
return { kind: 'un-routable', reason: 'local-tail-not-wired' };
|
|
13283
|
+
return _drainLocalDeliver(msg, handover);
|
|
13284
|
+
},
|
|
13285
|
+
forceReplace: async (msg) => {
|
|
13286
|
+
if (!_sessionRouter)
|
|
13287
|
+
return false;
|
|
13288
|
+
return _sessionRouter.forceReplace({
|
|
13289
|
+
sessionKey: msg.sessionKey,
|
|
13290
|
+
messageId: msg.messageId,
|
|
13291
|
+
payload: msg.payload,
|
|
13292
|
+
senderEnvelope: msg.senderEnvelope,
|
|
13293
|
+
topicMetadata: msg.topicMetadata,
|
|
13294
|
+
});
|
|
13295
|
+
},
|
|
13296
|
+
holdVerdict,
|
|
13297
|
+
clearPisRecord: (sk) => {
|
|
13298
|
+
for (const r of queuePis.list().records.filter((x) => String(x.telegramTopicId ?? '') === sk)) {
|
|
13299
|
+
queuePis.clear(r.tmuxSession);
|
|
13300
|
+
}
|
|
13301
|
+
},
|
|
13302
|
+
reportLoss: (items, reason) => {
|
|
13303
|
+
const topics = [...new Set(items.map((i) => i.sessionKey))].join(', ');
|
|
13304
|
+
notify('SUMMARY', 'inbound-queue', `I didn't get to ${items.length} queued message(s) (${reason}; topics: ${topics}) — resend anything still needed.`);
|
|
13305
|
+
},
|
|
13306
|
+
reportPossiblyNotInjected: (items) => {
|
|
13307
|
+
const topics = [...new Set(items.map((i) => i.sessionKey))].join(', ');
|
|
13308
|
+
notify('SUMMARY', 'inbound-queue', `${items.length} message(s) may not have been injected (topics: ${topics}) — if a message went unanswered, resend it.`);
|
|
13309
|
+
},
|
|
13310
|
+
log: (line) => console.log(pc.dim(` ${line}`)),
|
|
13311
|
+
reportDegradation: (reason) => {
|
|
13312
|
+
DegradationReporter.getInstance().report({
|
|
13313
|
+
feature: 'InboundQueue.drain',
|
|
13314
|
+
primary: 'Durable inbound custody drain',
|
|
13315
|
+
fallback: 'Rows remain durable; the Eternal-Sentinel tick keeps retrying',
|
|
13316
|
+
reason,
|
|
13317
|
+
impact: 'Queued inbound messages may deliver late until the tick recovers.',
|
|
13318
|
+
});
|
|
13319
|
+
},
|
|
13320
|
+
now: () => Date.now(),
|
|
13321
|
+
mono: () => performance.now(),
|
|
13322
|
+
bootSessionId,
|
|
13323
|
+
});
|
|
13324
|
+
_inboundQueue.onLeaseAcquired(null);
|
|
13325
|
+
// Expose the stop hook for the emergency-stop integration and
|
|
13326
|
+
// the engine for the /pool/queue route via module state.
|
|
13327
|
+
_inboundQueueStop = (sk) => { stoppedTopics.add(sk); _inboundQueue?.onOperatorStop(sk); };
|
|
13328
|
+
// Backstop tick (Eternal Sentinel — declared in QueueDrainLoop).
|
|
13329
|
+
const tickHandle = setInterval(() => { void _inboundQueue?.tick(); }, qcfg.drainTickMs);
|
|
13330
|
+
tickHandle.unref?.();
|
|
13331
|
+
console.log(pc.dim(` [inbound-queue] engine live — tick every ${qcfg.drainTickMs}ms, tenure ${_inboundQueue.currentTenure()}`));
|
|
13332
|
+
// Survivor arm (spec §5.1, loss window 1): the machine that
|
|
13333
|
+
// holds the routing role checks — once heartbeats have had a
|
|
13334
|
+
// cycle to repopulate — for OFFLINE peers whose last capacity
|
|
13335
|
+
// heartbeat reported nonzero queue depth. ONE loss-SUSPECTED
|
|
13336
|
+
// item per (machine + tenure) episode, then capped synthetic
|
|
13337
|
+
// re-placement for the top-K sessions THROUGH the router path
|
|
13338
|
+
// (PlacementExecutor honors pins; CAS; spawn on the chosen
|
|
13339
|
+
// machine) — SESSION RECOVERY WITHOUT MESSAGE REPLAY, framed
|
|
13340
|
+
// exactly so in the copy. A mesh-less/old peer carries no
|
|
13341
|
+
// depth field — honestly unknown, no item.
|
|
13342
|
+
const survivorEpisodesSeen = new Set();
|
|
13343
|
+
const survivorCheck = () => {
|
|
13344
|
+
try {
|
|
13345
|
+
for (const cap of machinePoolRegistry?.getCapacities() ?? []) {
|
|
13346
|
+
const iq = cap.inboundQueue;
|
|
13347
|
+
if (cap.machineId === meshSelfId || cap.online || !iq || iq.queueDepth <= 0)
|
|
13348
|
+
continue;
|
|
13349
|
+
const episodeKey = `${cap.machineId}|${iq.tenure ?? 'unknown'}`;
|
|
13350
|
+
if (survivorEpisodesSeen.has(episodeKey))
|
|
13351
|
+
continue;
|
|
13352
|
+
survivorEpisodesSeen.add(episodeKey);
|
|
13353
|
+
notify('IMMEDIATE', 'inbound-queue', `"${cap.nickname ?? cap.machineId}" went dark holding ~${iq.queueDepth} queued message(s) — SUSPECTED loss (it may have delivered some before going dark; last heartbeat ${cap.selfReportedLastSeen ?? 'unknown'}). I'm re-opening its top conversations on a healthy machine — session recovery WITHOUT message replay: the queued messages themselves are not recovered, so resend anything still needed.`);
|
|
13354
|
+
const respawnCap = Math.min(qcfg.maxFailoverRespawns, iq.topK.length);
|
|
13355
|
+
for (let i = 0; i < respawnCap; i++) {
|
|
13356
|
+
const sk = iq.topK[i].sessionKey;
|
|
13357
|
+
setTimeout(() => {
|
|
13358
|
+
void _sessionRouter?.forceReplace({ sessionKey: sk, messageId: `survivor-replace:${episodeKey}:${sk}`, payload: '', senderEnvelope: null })
|
|
13359
|
+
.catch(() => { });
|
|
13360
|
+
}, 2000 * i).unref?.(); // staggered
|
|
13361
|
+
}
|
|
13362
|
+
}
|
|
13363
|
+
}
|
|
13364
|
+
catch { /* survivor check is best-effort observability+recovery */ }
|
|
13365
|
+
};
|
|
13366
|
+
const survivorTimer = setInterval(survivorCheck, 60_000);
|
|
13367
|
+
survivorTimer.unref?.();
|
|
13368
|
+
}
|
|
13369
|
+
}
|
|
13370
|
+
else if (_sweptInboundStore) {
|
|
13371
|
+
// Sweep opened the store expecting a live engine but the stage/
|
|
13372
|
+
// dry-run gate says otherwise (mid-boot config nuance) — close it;
|
|
13373
|
+
// the sweep already settled rows per its own gate verdict.
|
|
13374
|
+
_sweptInboundStore.close();
|
|
13375
|
+
_sweptInboundStore = null;
|
|
13376
|
+
}
|
|
13377
|
+
}
|
|
13378
|
+
catch (err) {
|
|
13379
|
+
console.error(`[inbound-queue] engine construction failed (queue stays OFF): ${err instanceof Error ? err.message : String(err)}`);
|
|
13380
|
+
}
|
|
12774
13381
|
// ── B (HTTP presence transport): pull each peer's self-capacity over
|
|
12775
13382
|
// the signed /mesh/rpc channel and record it into the pool registry.
|
|
12776
13383
|
// This is the credential-less presence path — a standby that cannot
|
|
@@ -13164,7 +13771,7 @@ export async function startServer(options) {
|
|
|
13164
13771
|
carrier: _topicProfileCarrier,
|
|
13165
13772
|
}
|
|
13166
13773
|
: null;
|
|
13167
|
-
const server = new AgentServer({ config, sessionManager, state, scheduler, telegram, relationships, feedback, feedbackAnomalyDetector, dispatches, updateChecker, autoUpdater, autoDispatcher, quotaTracker, quotaManager, publisher, viewer, tunnel, evolution, watchdog, topicMemory, triageNurse, projectMapper, cartographer: cartographer ?? undefined, coherenceGate: scopeVerifier, contextHierarchy, canonicalState, operationGate, sentinel, adaptiveTrust, memoryMonitor, orphanReaper, coherenceMonitor, commitmentTracker, subscriptionPool, quotaPoller, quotaAwareScheduler: _quotaAwareScheduler ?? undefined, proactiveSwapMonitor: _proactiveSwapMonitor ?? undefined, inUseAccountResolver, enrollmentWizard, semanticMemory, activitySentinel, rateLimitSentinel, releaseReadinessSentinel: releaseReadinessSentinel ?? undefined, messageRouter, summarySentinel, spawnManager, systemReviewer, capabilityMapper, selfKnowledgeTree, coverageAuditor, topicResumeMap: _topicResumeMap ?? undefined, topicProfile: _topicProfileCtx ?? undefined, sessionRefresh: _sessionRefresh ?? undefined, autonomyManager, trustElevationTracker, autonomousEvolution, coordinator: coordinator.enabled ? coordinator : undefined, localSigningKeyPem, leaseTransport, onLeasePullRequest: () => leaseCoordinatorRef?.currentLease() ?? null, liveTailReceiver, handoffWireTransport, onHandoffBegin, onHandoffInitiate: handoffInitiate, handoffInProgress: handoffSentinelInProgress, messageLedger, currentInboundByTopic, replyMarkerTransport, onReplyMarker: messageLedger ? (marker) => { const m = marker; messageLedger.applyRemoteReplyMarker(m.dedupeKey, { platform: m.platform, replyIdempotencyKey: m.replyIdempotencyKey, epoch: m.epoch, topic: m.topic ?? null }); } : undefined, whatsapp: whatsappAdapter, slack: slackAdapter, imessage: imessageAdapter, whatsappBusinessBackend, messageBridge, hookEventReceiver, worktreeMonitor, subagentTracker, instructionsVerifier, handshakeManager: threadlineHandshake, threadlineRouter, conversationStore, warrantsReplyGate, collaborationSurfacer, threadResumeMap, topicLinkageHandler: topicLinkageHandler ?? undefined, threadlineRelayClient, threadlineReplyWaiters, listenerManager: listenerManager ?? undefined, a2aDeliveryTracker: a2aDeliveryTracker ?? undefined, responseReviewGate, messagingToneGate, outboundDedupGate, telemetryHeartbeat, pasteManager, featureRegistry, discoveryEvaluator, completionEvaluator, unifiedTrust, liveConfig, sharedStateLedger, ledgerSessionRegistry, worktreeManager, oidcEnrolledRepos: parallelDevConfig?.oidcEnrolledRepos, initiativeTracker, projectRoundRunner, projectDriftChecker, machineHeartbeat, machinePoolRegistry, meshRpcDispatcher, workingSetPullCoordinator, commitmentReplicaStore, forwardCommitmentMutate, sessionOwnershipRegistry, topicPinStore: _topicPinStore ?? undefined, streamTicketStore: _streamTicketStore ?? undefined, poolStreamAllowRemoteInput: config.dashboard?.poolStream?.allowRemoteInput ?? false, poolStreamConnector: _poolStreamConnector ?? undefined, secretSync: _secretSyncHandle ?? undefined, meshSelfId: _meshSelfId ?? undefined, resolveRouterUrl: _resolveRouterUrl ?? undefined, resolvePeerUrls: _resolvePeerUrls ?? undefined, guardRegistry, listPoolMachines: _listPoolMachines ?? undefined, sessionPoolE2EResultStore, proxyCoordinator, topicIntentStore, topicIntentArcCheck, usherSignalStore, intelligence: sharedIntelligence ?? undefined, telegramBridgeConfig, telegramBridge: telegramBridge ?? undefined, threadlineObservability, briefDeps, workingMemory, taskFlowRegistry, threadlineFlowBridge, sessionReaper, agentWorktreeReaper, mcpProcessReaper, geminiLoopRunner, sleepController, agentActivityState, reapLog, resumeQueue, resumeDrainer, operatorStopRecorder: recordOperatorStop, sleepWakeDetector, unjustifiedStopGate, stopGateDb, stopNotifier });
|
|
13774
|
+
const server = new AgentServer({ config, sessionManager, state, scheduler, telegram, relationships, feedback, feedbackAnomalyDetector, dispatches, updateChecker, autoUpdater, autoDispatcher, quotaTracker, quotaManager, publisher, viewer, tunnel, evolution, watchdog, topicMemory, triageNurse, projectMapper, cartographer: cartographer ?? undefined, coherenceGate: scopeVerifier, contextHierarchy, canonicalState, operationGate, sentinel, adaptiveTrust, memoryMonitor, orphanReaper, coherenceMonitor, commitmentTracker, subscriptionPool, quotaPoller, quotaAwareScheduler: _quotaAwareScheduler ?? undefined, proactiveSwapMonitor: _proactiveSwapMonitor ?? undefined, inUseAccountResolver, enrollmentWizard, semanticMemory, activitySentinel, rateLimitSentinel, releaseReadinessSentinel: releaseReadinessSentinel ?? undefined, messageRouter, summarySentinel, spawnManager, systemReviewer, capabilityMapper, selfKnowledgeTree, coverageAuditor, topicResumeMap: _topicResumeMap ?? undefined, topicProfile: _topicProfileCtx ?? undefined, sessionRefresh: _sessionRefresh ?? undefined, autonomyManager, trustElevationTracker, autonomousEvolution, coordinator: coordinator.enabled ? coordinator : undefined, localSigningKeyPem, leaseTransport, onLeasePullRequest: () => leaseCoordinatorRef?.currentLease() ?? null, liveTailReceiver, handoffWireTransport, onHandoffBegin, onHandoffInitiate: handoffInitiate, handoffInProgress: handoffSentinelInProgress, messageLedger, currentInboundByTopic, replyMarkerTransport, onReplyMarker: messageLedger ? (marker) => { const m = marker; messageLedger.applyRemoteReplyMarker(m.dedupeKey, { platform: m.platform, replyIdempotencyKey: m.replyIdempotencyKey, epoch: m.epoch, topic: m.topic ?? null }); } : undefined, whatsapp: whatsappAdapter, slack: slackAdapter, imessage: imessageAdapter, whatsappBusinessBackend, messageBridge, hookEventReceiver, worktreeMonitor, subagentTracker, instructionsVerifier, handshakeManager: threadlineHandshake, threadlineRouter, conversationStore, warrantsReplyGate, collaborationSurfacer, threadResumeMap, topicLinkageHandler: topicLinkageHandler ?? undefined, threadlineRelayClient, threadlineReplyWaiters, listenerManager: listenerManager ?? undefined, a2aDeliveryTracker: a2aDeliveryTracker ?? undefined, responseReviewGate, messagingToneGate, outboundDedupGate, telemetryHeartbeat, pasteManager, featureRegistry, discoveryEvaluator, completionEvaluator, unifiedTrust, liveConfig, sharedStateLedger, ledgerSessionRegistry, worktreeManager, oidcEnrolledRepos: parallelDevConfig?.oidcEnrolledRepos, initiativeTracker, projectRoundRunner, projectDriftChecker, machineHeartbeat, machinePoolRegistry, getInboundQueue: () => _inboundQueue, meshRpcDispatcher, workingSetPullCoordinator, commitmentReplicaStore, forwardCommitmentMutate, sessionOwnershipRegistry, topicPinStore: _topicPinStore ?? undefined, streamTicketStore: _streamTicketStore ?? undefined, poolStreamAllowRemoteInput: config.dashboard?.poolStream?.allowRemoteInput ?? false, poolStreamConnector: _poolStreamConnector ?? undefined, secretSync: _secretSyncHandle ?? undefined, meshSelfId: _meshSelfId ?? undefined, resolveRouterUrl: _resolveRouterUrl ?? undefined, resolvePeerUrls: _resolvePeerUrls ?? undefined, guardRegistry, listPoolMachines: _listPoolMachines ?? undefined, sessionPoolE2EResultStore, proxyCoordinator, topicIntentStore, topicIntentArcCheck, usherSignalStore, intelligence: sharedIntelligence ?? undefined, telegramBridgeConfig, telegramBridge: telegramBridge ?? undefined, threadlineObservability, briefDeps, workingMemory, taskFlowRegistry, threadlineFlowBridge, sessionReaper, agentWorktreeReaper, mcpProcessReaper, geminiLoopRunner, sleepController, agentActivityState, reapLog, resumeQueue, resumeDrainer, operatorStopRecorder: recordOperatorStop, sleepWakeDetector, unjustifiedStopGate, stopGateDb, stopNotifier });
|
|
13168
13775
|
// Resolve the late-bound topic-operator getter (increment 2e): routing was
|
|
13169
13776
|
// wired before the server existed; from here on inbound binds use the
|
|
13170
13777
|
// server's own store instance.
|