@cello-protocol/daemon 0.0.167 → 0.0.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +115 -254
- package/dist/daemon.js.map +1 -1
- package/dist/document-amendment-store.d.ts +94 -46
- package/dist/document-amendment-store.d.ts.map +1 -1
- package/dist/document-amendment-store.js +217 -102
- package/dist/document-amendment-store.js.map +1 -1
- package/dist/document-delivery-transport.d.ts +41 -52
- package/dist/document-delivery-transport.d.ts.map +1 -1
- package/dist/document-delivery-transport.js +6 -156
- package/dist/document-delivery-transport.js.map +1 -1
- package/dist/document-frame-router.d.ts +13 -17
- package/dist/document-frame-router.d.ts.map +1 -1
- package/dist/document-frame-router.js +21 -93
- package/dist/document-frame-router.js.map +1 -1
- package/dist/document-handlers.d.ts +1 -1
- package/dist/document-handlers.d.ts.map +1 -1
- package/dist/document-handlers.js +471 -466
- package/dist/document-handlers.js.map +1 -1
- package/dist/document-handshake.d.ts +4 -1
- package/dist/document-handshake.d.ts.map +1 -1
- package/dist/document-handshake.js +3 -1
- package/dist/document-handshake.js.map +1 -1
- package/dist/document-inbound.d.ts +18 -5
- package/dist/document-inbound.d.ts.map +1 -1
- package/dist/document-inbound.js +94 -110
- package/dist/document-inbound.js.map +1 -1
- package/dist/document-layer.d.ts +28 -67
- package/dist/document-layer.d.ts.map +1 -1
- package/dist/document-layer.js +670 -565
- package/dist/document-layer.js.map +1 -1
- package/dist/document-lifecycle.d.ts +11 -157
- package/dist/document-lifecycle.d.ts.map +1 -1
- package/dist/document-lifecycle.js +26 -561
- package/dist/document-lifecycle.js.map +1 -1
- package/dist/document-publish.d.ts +13 -0
- package/dist/document-publish.d.ts.map +1 -1
- package/dist/document-publish.js +19 -11
- package/dist/document-publish.js.map +1 -1
- package/dist/document-reconcile-engine.d.ts +66 -0
- package/dist/document-reconcile-engine.d.ts.map +1 -0
- package/dist/document-reconcile-engine.js +225 -0
- package/dist/document-reconcile-engine.js.map +1 -0
- package/dist/document-reconcile-scheduler.d.ts +91 -0
- package/dist/document-reconcile-scheduler.d.ts.map +1 -0
- package/dist/document-reconcile-scheduler.js +153 -0
- package/dist/document-reconcile-scheduler.js.map +1 -0
- package/dist/document-rejection.d.ts +2 -0
- package/dist/document-rejection.d.ts.map +1 -1
- package/dist/document-rejection.js +5 -1
- package/dist/document-rejection.js.map +1 -1
- package/dist/document-store.d.ts +18 -220
- package/dist/document-store.d.ts.map +1 -1
- package/dist/document-store.js +95 -717
- package/dist/document-store.js.map +1 -1
- package/dist/initiate-session-handler.d.ts +5 -0
- package/dist/initiate-session-handler.d.ts.map +1 -1
- package/dist/initiate-session-handler.js +1 -0
- package/dist/initiate-session-handler.js.map +1 -1
- package/package.json +5 -5
- package/dist/delivery-sweep-bound.d.ts +0 -79
- package/dist/delivery-sweep-bound.d.ts.map +0 -1
- package/dist/delivery-sweep-bound.js +0 -109
- package/dist/delivery-sweep-bound.js.map +0 -1
- package/dist/document-ack-inbound.d.ts +0 -71
- package/dist/document-ack-inbound.d.ts.map +0 -1
- package/dist/document-ack-inbound.js +0 -210
- package/dist/document-ack-inbound.js.map +0 -1
- package/dist/document-control-notifier.d.ts +0 -107
- package/dist/document-control-notifier.d.ts.map +0 -1
- package/dist/document-control-notifier.js +0 -137
- package/dist/document-control-notifier.js.map +0 -1
- package/dist/document-delivery.d.ts +0 -229
- package/dist/document-delivery.d.ts.map +0 -1
- package/dist/document-delivery.js +0 -449
- package/dist/document-delivery.js.map +0 -1
- package/dist/document-join-store.d.ts +0 -80
- package/dist/document-join-store.d.ts.map +0 -1
- package/dist/document-join-store.js +0 -197
- package/dist/document-join-store.js.map +0 -1
package/dist/daemon.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../src/daemon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAKH,OAAO,KAAK,EACV,YAAY,EACZ,oBAAoB,EAMrB,MAAM,YAAY,CAAC;AAIpB,OAAO,EAAmC,KAAK,UAAU,EAAsB,MAAM,iBAAiB,CAAC;AAEvG,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAO/D,OAAO,EAAoD,KAAK,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAiB7G,OAAO,KAAK,EAAE,mBAAmB,EAAE,iBAAiB,EAAsB,MAAM,2BAA2B,CAAC;AAS5G,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAIlE,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../src/daemon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAKH,OAAO,KAAK,EACV,YAAY,EACZ,oBAAoB,EAMrB,MAAM,YAAY,CAAC;AAIpB,OAAO,EAAmC,KAAK,UAAU,EAAsB,MAAM,iBAAiB,CAAC;AAEvG,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAO/D,OAAO,EAAoD,KAAK,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAiB7G,OAAO,KAAK,EAAE,mBAAmB,EAAE,iBAAiB,EAAsB,MAAM,2BAA2B,CAAC;AAS5G,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAIlE,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,2BAA2B,CAAC;AA2BnF,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAWlD,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,SAAS,IAAI,oBAAoB,CAAC;IAClC;;;;OAIG;IACH,qBAAqB,IAAI,kBAAkB,CAAC;IAC5C;;;;OAIG;IACH,oBAAoB,IAAI,kBAAkB,CAAC;IAC3C;;;OAGG;IACH,iBAAiB,IAAI,eAAe,CAAC;IACrC;;OAEG;IACH,eAAe,IAAI,YAAY,CAAC;IAChC;;;;;;;;OAQG;IACH,WAAW,IAAI,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CACxC;AAYD,qBAAa,4BAA6B,YAAW,mBAAmB;IAChE,UAAU,CAAC,MAAM,EAAE,iBAAiB;CA8B3C;AAID,eAAO,MAAM,sBAAsB,QAAsB,CAAC;AAE1D;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAS7E"}
|
package/dist/daemon.js
CHANGED
|
@@ -67,13 +67,10 @@ import { createTelegramDoorbell } from "./telegram-doorbell.js";
|
|
|
67
67
|
import { registerSessionContentHandlers } from "./session-content-handlers.js";
|
|
68
68
|
import { createDocumentLayer, agentPublicKeyFromId } from "./document-layer.js";
|
|
69
69
|
import { registerDocumentHandlers } from "./document-handlers.js";
|
|
70
|
-
import { createDocumentControlNotifier } from "./document-control-notifier.js";
|
|
71
70
|
import { wireContentHash } from "./wire-content-hash.js";
|
|
72
71
|
import { DocumentPublish } from "./document-publish.js";
|
|
73
72
|
import { createDocumentDeliveryTransport } from "./document-delivery-transport.js";
|
|
74
|
-
import {
|
|
75
|
-
import { sweepAgentEvicting } from "./delivery-sweep-bound.js";
|
|
76
|
-
import { encodeDocumentUpdateEnvelope, DOCUMENT_UPDATE_ENCODING_V1 } from "@cello-protocol/protocol-types";
|
|
73
|
+
import { ReconcileScheduler } from "./document-reconcile-scheduler.js";
|
|
77
74
|
import { createSealFlows } from "./seal-flows.js";
|
|
78
75
|
import { registerCloseSessionHandler } from "./close-session-handler.js";
|
|
79
76
|
import { createInboundSessions } from "./inbound-sessions.js";
|
|
@@ -1213,10 +1210,29 @@ async function startDaemonHoldingLock(config, singletonLock) {
|
|
|
1213
1210
|
});
|
|
1214
1211
|
return { who: resolved.who, whoKnown: resolved.whoKnown };
|
|
1215
1212
|
}
|
|
1213
|
+
// SYNC-P5: assigned when the document layer is wired (below); the state-change hook runs for
|
|
1214
|
+
// sessions, which exist only after startup completes — the guard covers the boot window where
|
|
1215
|
+
// agent restoration can dispatch state events before the scheduler exists.
|
|
1216
|
+
let reconcileScheduler;
|
|
1217
|
+
let documentOwnerKeyForHook;
|
|
1216
1218
|
function dispatchSessionStateChangedWithTelegram(agentName, sessionId, state, counterpartyPubkey) {
|
|
1217
1219
|
// MONIKER-4 AC2: stamp who/whoKnown on the counterparty-bearing frame. Resolved BEFORE the
|
|
1218
1220
|
// offered-name drop below so a terminal state's own doorbell still shows the name.
|
|
1219
1221
|
const who = counterpartyPubkey ? resolveWho(agentName, counterpartyPubkey, sessionId) : undefined;
|
|
1222
|
+
// SYNC-P5 (R39): a session coming up IS the party-became-reachable signal — every shared
|
|
1223
|
+
// document gets a reconcile attempt, and the scheduler's backoff resets (they just answered).
|
|
1224
|
+
if (reconcileScheduler && documentOwnerKeyForHook && counterpartyPubkey && state === "created") {
|
|
1225
|
+
const ownerAgentId = documentOwnerKeyForHook(agentName);
|
|
1226
|
+
if (ownerAgentId !== null) {
|
|
1227
|
+
void reconcileScheduler
|
|
1228
|
+
.onReachable(ownerAgentId, counterpartyPubkey.toLowerCase())
|
|
1229
|
+
.catch((err) => {
|
|
1230
|
+
logger.warn("document.reconcile.reachable_trigger_failed", {
|
|
1231
|
+
agentName, reason: err instanceof Error ? err.message : String(err),
|
|
1232
|
+
});
|
|
1233
|
+
});
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1220
1236
|
notificationDispatcher.dispatchSessionStateChanged(agentName, sessionId, state, counterpartyPubkey, who);
|
|
1221
1237
|
void sendTelegramDoorbell(agentName, sessionId, "state_change", `Session ${state}`);
|
|
1222
1238
|
// Reviewer HIGH fix (a60d68ed): telegramRungUnread had NO cleanup at all — a session that
|
|
@@ -2927,6 +2943,23 @@ async function startDaemonHoldingLock(config, singletonLock) {
|
|
|
2927
2943
|
handlers,
|
|
2928
2944
|
logger,
|
|
2929
2945
|
sessionNodeManager,
|
|
2946
|
+
// SYNC-P5 (R39, review F5): the INITIATOR-side reachable trigger — the inbound side fires
|
|
2947
|
+
// from the session-state dispatch; without this half, only the answering daemon reconciled
|
|
2948
|
+
// on session-up and the initiator waited out a sweep interval.
|
|
2949
|
+
onSessionOpened: (agentName, counterpartyPubkey) => {
|
|
2950
|
+
if (!reconcileScheduler || !documentOwnerKeyForHook)
|
|
2951
|
+
return;
|
|
2952
|
+
const ownerAgentId = documentOwnerKeyForHook(agentName);
|
|
2953
|
+
if (ownerAgentId === null)
|
|
2954
|
+
return;
|
|
2955
|
+
void reconcileScheduler
|
|
2956
|
+
.onReachable(ownerAgentId, counterpartyPubkey.toLowerCase())
|
|
2957
|
+
.catch((err) => {
|
|
2958
|
+
logger.warn("document.reconcile.reachable_trigger_failed", {
|
|
2959
|
+
agentName, reason: err instanceof Error ? err.message : String(err),
|
|
2960
|
+
});
|
|
2961
|
+
});
|
|
2962
|
+
},
|
|
2930
2963
|
getConnState: (connectionId) => perConnectionState.get(connectionId),
|
|
2931
2964
|
resolveCurrentAgent,
|
|
2932
2965
|
NO_CURRENT_AGENT_RESPONSE,
|
|
@@ -3399,38 +3432,6 @@ async function startDaemonHoldingLock(config, singletonLock) {
|
|
|
3399
3432
|
return;
|
|
3400
3433
|
notificationDispatcher.dispatchDocumentWatch(agentName, documentId, paths);
|
|
3401
3434
|
},
|
|
3402
|
-
notifyPeer: createDocumentControlNotifier({
|
|
3403
|
-
get store() {
|
|
3404
|
-
// Lazily, because `documentLayer` is what is being constructed. The notifier is only ever
|
|
3405
|
-
// called from lifecycle, long after construction returns.
|
|
3406
|
-
return documentLayer.store;
|
|
3407
|
-
},
|
|
3408
|
-
owners: () => loadedAgents
|
|
3409
|
-
.filter((a) => a.pubkey)
|
|
3410
|
-
.map((a) => ({ agentName: a.name, ownerAgentId: a.pubkey.toLowerCase() })),
|
|
3411
|
-
// DOD-MP-CONTROL-N-1 — the ONE implementation, on the layer, shared with every test fixture.
|
|
3412
|
-
// A closure here instead would be untested by construction and hand-copied into fixtures
|
|
3413
|
-
// that cannot disagree with it, which is exactly how the original defect survived.
|
|
3414
|
-
holders: (ownerAgentId, documentId) => documentLayer.controlHolders(ownerAgentId, documentId),
|
|
3415
|
-
sign: async (agentName, tbs) => {
|
|
3416
|
-
const provider = keyProviders.get(agentName);
|
|
3417
|
-
if (!provider)
|
|
3418
|
-
throw new Error(`document_control_unsigned: no key provider for ${agentName}`);
|
|
3419
|
-
return provider.sign(tbs);
|
|
3420
|
-
},
|
|
3421
|
-
send: (agentName, input) => documentTransportFor(agentName).sendBytes(input),
|
|
3422
|
-
// Given a logger so a signing failure is REPORTED rather than swallowed — the reason a
|
|
3423
|
-
// local key fault used to reach the operator dressed as an absent peer.
|
|
3424
|
-
logger,
|
|
3425
|
-
now: () => Date.now(),
|
|
3426
|
-
}),
|
|
3427
|
-
rollback: () => ({
|
|
3428
|
-
// Likewise: withdraw REFUSES rather than claiming a rollback that did not happen. Reporting
|
|
3429
|
-
// success having reverted nothing tells an operator their update was retracted while their
|
|
3430
|
-
// file still contains it.
|
|
3431
|
-
ok: false,
|
|
3432
|
-
reason: "document_rollback_not_wired",
|
|
3433
|
-
}),
|
|
3434
3435
|
sign: async (ownerAgentId, tbs) => {
|
|
3435
3436
|
// Signs as the OWNING agent, over the rejection's canonical preimage (DOD-DOC-REJECT-2).
|
|
3436
3437
|
//
|
|
@@ -3467,14 +3468,8 @@ async function startDaemonHoldingLock(config, singletonLock) {
|
|
|
3467
3468
|
// agent, sessions belong to one agent, and `openSessionFor` signs as one agent. A single worker
|
|
3468
3469
|
// with an agent-shaped hole in it is how a delivery ends up sent under the wrong identity.
|
|
3469
3470
|
//
|
|
3470
|
-
//
|
|
3471
|
-
//
|
|
3472
|
-
// The sweep's serial loop happens to cover it today; a cached worker means it is covered because
|
|
3473
|
-
// the guard exists, not by accident of loop shape.
|
|
3474
|
-
const documentDeliveryWorkers = new Map();
|
|
3475
|
-
// Cached separately from the worker because the OPERATOR SURFACE needs it too: a proposal is not
|
|
3476
|
-
// in the envelope log — there is no document yet — so it has no delivery record to schedule, and
|
|
3477
|
-
// it still has to travel the same open-or-reuse-then-seal path an update does.
|
|
3471
|
+
// The frame carrier, cached per agent (SYNC-P4: the delivery worker and its sweep are deleted —
|
|
3472
|
+
// scheduling returns as the P5 reconcile triggers).
|
|
3478
3473
|
const documentTransports = new Map();
|
|
3479
3474
|
const documentTransportFor = (agentName) => {
|
|
3480
3475
|
const cached = documentTransports.get(agentName);
|
|
@@ -3544,83 +3539,17 @@ async function startDaemonHoldingLock(config, singletonLock) {
|
|
|
3544
3539
|
appendLeaf: (agent, sessionId, contentHash, correlationId) => {
|
|
3545
3540
|
sessionNodeManager.appendSessionLeaf(agent, sessionId, "doc", Buffer.from(contentHash).toString("hex"), correlationId);
|
|
3546
3541
|
},
|
|
3547
|
-
encodeEnvelope: (envelope) => {
|
|
3548
|
-
const bytes = encodeDocumentUpdateEnvelope({
|
|
3549
|
-
type: "document_update",
|
|
3550
|
-
document_id: envelope.documentId,
|
|
3551
|
-
epoch_id: envelope.epochId,
|
|
3552
|
-
doc_prev_hash: envelope.docPrevHash,
|
|
3553
|
-
sender_agent_id: envelope.senderAgentId,
|
|
3554
|
-
sender_client_id: envelope.senderClientId ?? 0,
|
|
3555
|
-
update_encoding: DOCUMENT_UPDATE_ENCODING_V1,
|
|
3556
|
-
state_vector: envelope.stateVector,
|
|
3557
|
-
update: envelope.payload ?? new Uint8Array(0),
|
|
3558
|
-
signature: envelope.signature,
|
|
3559
|
-
});
|
|
3560
|
-
// Same defect as `sendBytes` had, on the UPDATE path: the receiver recomputes with the
|
|
3561
|
-
// `0x00` domain prefix, so a bare hash is discarded at the authenticity check.
|
|
3562
|
-
return { bytes, hash: wireContentHash(bytes) };
|
|
3563
|
-
},
|
|
3564
|
-
// The owner KEY, not the agent name — every document row is scoped by our own pubkey hex
|
|
3565
|
-
// (M14-D5), and the store query behind this reads that column. Passing the name here
|
|
3566
|
-
// returns no rows and the wait times out on every delivery.
|
|
3567
|
-
awaitAck: async (envelopeHash, expectedAckerAgentId, timeoutMs) => {
|
|
3568
|
-
const ownerKey = documentOwnerKeyFor(agentName);
|
|
3569
|
-
if (ownerKey === null) {
|
|
3570
|
-
// NOT "the peer stayed silent". `false` means the grace expired with no answer, and the
|
|
3571
|
-
// caller logs exactly that — `graceMs: 10000`, a claim that ten seconds elapsed when
|
|
3572
|
-
// none did, about a peer that was never asked. A delivery worker running for an agent
|
|
3573
|
-
// whose own pubkey cannot be resolved is a wiring bug, and it says so here rather than
|
|
3574
|
-
// arriving downstream dressed as a fact about the counterparty. Same distinction
|
|
3575
|
-
// `abandoned_at` exists to keep: giving up is our decision, not evidence about them.
|
|
3576
|
-
logger.error("document.delivery.ack_wait_unwired", { agentName, envelopeHash });
|
|
3577
|
-
return null;
|
|
3578
|
-
}
|
|
3579
|
-
return documentLayer.awaitAck(ownerKey, envelopeHash, expectedAckerAgentId, timeoutMs);
|
|
3580
|
-
},
|
|
3581
|
-
drainHeld: async (sessionId, correlationId) => {
|
|
3582
|
-
// ONLY WHEN THERE IS A GAP. `sealReadiness` already answers exactly this question, and
|
|
3583
|
-
// gating on it keeps the ordinary delivery — the overwhelming majority — free of a relay
|
|
3584
|
-
// round trip it does not need.
|
|
3585
|
-
const readiness = sessionNodeManager.sealReadiness(agentName, sessionId);
|
|
3586
|
-
if (readiness.heldCount === 0 && readiness.missingLeaves === 0)
|
|
3587
|
-
return;
|
|
3588
|
-
logger.info("document.delivery.drain_held", {
|
|
3589
|
-
agentName,
|
|
3590
|
-
sessionId,
|
|
3591
|
-
heldCount: readiness.heldCount,
|
|
3592
|
-
missingLeaves: readiness.missingLeaves,
|
|
3593
|
-
correlationId,
|
|
3594
|
-
});
|
|
3595
|
-
await autoRecoverForAgent(agentName, "delivery_ack_grace").catch((err) => {
|
|
3596
|
-
// CONTAINED. A recovery that fails leaves the ack held and the grace expires — the same
|
|
3597
|
-
// outcome as before, and never a reason to fail a delivery whose content already left.
|
|
3598
|
-
logger.warn("document.delivery.drain_held_failed", {
|
|
3599
|
-
agentName,
|
|
3600
|
-
sessionId,
|
|
3601
|
-
correlationId,
|
|
3602
|
-
error: err instanceof Error ? err.message : String(err),
|
|
3603
|
-
});
|
|
3604
|
-
});
|
|
3605
|
-
},
|
|
3606
3542
|
});
|
|
3607
3543
|
documentTransports.set(agentName, transport);
|
|
3608
3544
|
return transport;
|
|
3609
3545
|
};
|
|
3610
|
-
const documentDeliveryFor = (agentName) => {
|
|
3611
|
-
const existing = documentDeliveryWorkers.get(agentName);
|
|
3612
|
-
if (existing)
|
|
3613
|
-
return existing;
|
|
3614
|
-
const worker = new DocumentDelivery(documentLayer.store, documentTransportFor(agentName), logger);
|
|
3615
|
-
documentDeliveryWorkers.set(agentName, worker);
|
|
3616
|
-
return worker;
|
|
3617
|
-
};
|
|
3618
3546
|
// M14 / DOD-DOC-TOOLS-1 — the OPERATOR SURFACE. Registered last of the document wiring, because
|
|
3619
3547
|
// it is the only part that can create a document, and everything it creates has to have somewhere
|
|
3620
3548
|
// to go: without the inbound path a peer's answer is unroutable, and without the delivery worker a
|
|
3621
3549
|
// published update never leaves.
|
|
3622
3550
|
const documentPublish = new DocumentPublish({
|
|
3623
3551
|
holdersFor: (ownerAgentId, documentId) => documentLayer.holdersFor(ownerAgentId, documentId),
|
|
3552
|
+
governanceFrontierFor: (ownerAgentId, documentId) => documentLayer.governanceFrontierFor(ownerAgentId, documentId),
|
|
3624
3553
|
store: documentLayer.store,
|
|
3625
3554
|
engine: documentLayer.engine,
|
|
3626
3555
|
logger,
|
|
@@ -3640,6 +3569,30 @@ async function startDaemonHoldingLock(config, singletonLock) {
|
|
|
3640
3569
|
// different facts that happen to coincide — see DocumentStore.pendingDeliveries.
|
|
3641
3570
|
senderIdFor: (ownerAgentId) => ownerAgentId,
|
|
3642
3571
|
canPublish: (ownerAgentId, documentId) => documentLayer.lifecycle.canPublish(ownerAgentId, documentId),
|
|
3572
|
+
// SYNC-P4 (R39's first trigger): each seat gets an initiated reconcile exchange — the
|
|
3573
|
+
// envelope rides the exchange's own difference computation. Fire-and-forget by contract.
|
|
3574
|
+
nudgeSeats: (ownerAgentId, documentId, seats) => {
|
|
3575
|
+
for (const seat of seats) {
|
|
3576
|
+
void documentLayer
|
|
3577
|
+
.initiateReconcile(ownerAgentId, seat, [documentId])
|
|
3578
|
+
.then((sent) => {
|
|
3579
|
+
if (!sent.ok) {
|
|
3580
|
+
// NOT SILENT (review F3): a lost nudge is legal for correctness (any later
|
|
3581
|
+
// exchange recomputes the difference — R40), but until P5's sweeps exist this
|
|
3582
|
+
// line is the only trace an operator has for "my edit never arrived".
|
|
3583
|
+
logger.warn("document.reconcile.nudge_failed", {
|
|
3584
|
+
documentId, peerAgentId: seat, reason: sent.reason,
|
|
3585
|
+
});
|
|
3586
|
+
}
|
|
3587
|
+
})
|
|
3588
|
+
.catch((err) => {
|
|
3589
|
+
logger.warn("document.reconcile.nudge_failed", {
|
|
3590
|
+
documentId, peerAgentId: seat,
|
|
3591
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
3592
|
+
});
|
|
3593
|
+
});
|
|
3594
|
+
}
|
|
3595
|
+
},
|
|
3643
3596
|
});
|
|
3644
3597
|
registerDocumentHandlers({
|
|
3645
3598
|
handlers,
|
|
@@ -3657,128 +3610,69 @@ async function startDaemonHoldingLock(config, singletonLock) {
|
|
|
3657
3610
|
},
|
|
3658
3611
|
now: () => Date.now(),
|
|
3659
3612
|
});
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
3670
|
-
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
|
|
3677
|
-
//
|
|
3678
|
-
|
|
3679
|
-
|
|
3680
|
-
|
|
3681
|
-
|
|
3682
|
-
//
|
|
3683
|
-
//
|
|
3684
|
-
//
|
|
3685
|
-
//
|
|
3686
|
-
// bounded
|
|
3687
|
-
|
|
3688
|
-
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
|
|
3700
|
-
documentDeliveryRunning = true;
|
|
3701
|
-
documentDeliveryInFlight = (async () => {
|
|
3613
|
+
// SYNC-P5 (R39–R43) — WHEN reconciling is attempted: the periodic sweep and the party-became-
|
|
3614
|
+
// reachable trigger. All state volatile (R41); a lost tick costs latency, never correctness.
|
|
3615
|
+
reconcileScheduler = new ReconcileScheduler({
|
|
3616
|
+
now: () => Date.now(),
|
|
3617
|
+
logger,
|
|
3618
|
+
sweepTargets: (ownerAgentId) => documentLayer.sweepTargets(ownerAgentId),
|
|
3619
|
+
partySync: (ownerAgentId, documentId, partyAgentId) => documentLayer.partySync(ownerAgentId, documentId, partyAgentId),
|
|
3620
|
+
initiateReconcile: (ownerAgentId, peerAgentId, documentIds) => documentLayer.initiateReconcile(ownerAgentId, peerAgentId, documentIds),
|
|
3621
|
+
...(Number.isFinite(Number(process.env["CELLO_DOCUMENT_RECONCILE_BACKOFF_MS"])) &&
|
|
3622
|
+
Number(process.env["CELLO_DOCUMENT_RECONCILE_BACKOFF_MS"]) >= 250
|
|
3623
|
+
? { backoffBaseMs: Number(process.env["CELLO_DOCUMENT_RECONCILE_BACKOFF_MS"]) }
|
|
3624
|
+
: {}),
|
|
3625
|
+
...(Number.isFinite(Number(process.env["CELLO_DOCUMENT_RECONCILE_CURRENT_MS"])) &&
|
|
3626
|
+
Number(process.env["CELLO_DOCUMENT_RECONCILE_CURRENT_MS"]) >= 250
|
|
3627
|
+
? { believedCurrentMs: Number(process.env["CELLO_DOCUMENT_RECONCILE_CURRENT_MS"]) }
|
|
3628
|
+
: {}),
|
|
3629
|
+
});
|
|
3630
|
+
// Overridable for tests (the delivery tick precedent); floored so a misread env cannot busy-loop.
|
|
3631
|
+
const reconcileTickOverride = Number(process.env["CELLO_DOCUMENT_RECONCILE_SWEEP_MS"]);
|
|
3632
|
+
const RECONCILE_SWEEP_MS = Number.isFinite(reconcileTickOverride) && reconcileTickOverride >= 250
|
|
3633
|
+
? reconcileTickOverride
|
|
3634
|
+
: 120_000;
|
|
3635
|
+
// Review F2: the guard is TIME-STAMPED, not a bare boolean — a pass that never settles (a
|
|
3636
|
+
// dial with no timeout) would otherwise end the sweep silently for the daemon's lifetime,
|
|
3637
|
+
// the exact R42 stall one level above the module that fixed it. Past the bound the wedge is
|
|
3638
|
+
// WARNED and the guard force-released; the abandoned pass's own scheduler marks are
|
|
3639
|
+
// themselves time-bounded, so double-attempts are latency, never correctness.
|
|
3640
|
+
const RECONCILE_PASS_BOUND_MS = 5 * 60_000;
|
|
3641
|
+
let reconcileSweepStartedAt = null;
|
|
3642
|
+
const reconcileSweepTimer = setInterval(() => {
|
|
3643
|
+
if (reconcileSweepStartedAt !== null) {
|
|
3644
|
+
if (Date.now() - reconcileSweepStartedAt < RECONCILE_PASS_BOUND_MS)
|
|
3645
|
+
return;
|
|
3646
|
+
logger.warn("document.reconcile.sweep_wedged", {
|
|
3647
|
+
heldMs: Date.now() - reconcileSweepStartedAt,
|
|
3648
|
+
impact: "a sweep pass never settled; the guard is force-released so sweeping continues",
|
|
3649
|
+
});
|
|
3650
|
+
}
|
|
3651
|
+
reconcileSweepStartedAt = Date.now();
|
|
3652
|
+
void (async () => {
|
|
3702
3653
|
try {
|
|
3703
|
-
let agentsSwept = 0;
|
|
3704
|
-
let stuck = 0;
|
|
3705
|
-
let attempted = 0;
|
|
3706
|
-
let delivered = 0;
|
|
3707
|
-
let failed = 0;
|
|
3708
3654
|
for (const agentName of perAgentSignaling.keys()) {
|
|
3709
|
-
// M2: a tick already running when the daemon stops must not keep dialling into a
|
|
3710
|
-
// half-torn-down transport. `clearInterval` stops the NEXT tick, not this one.
|
|
3711
|
-
if (documentDeliveryStopping)
|
|
3712
|
-
break;
|
|
3713
3655
|
const ownerAgentId = documentOwnerKeyFor(agentName);
|
|
3714
|
-
// M14-D5: the owner key is our own pubkey hex. It is what BOTH halves scope by — the
|
|
3715
|
-
// inbound router resolves the same function — and it is what the wire sender id is.
|
|
3716
|
-
// Scoping the two halves differently returns nothing pending, reports nothing attempted,
|
|
3717
|
-
// and leaves a fully synced document invisible with no error on any path.
|
|
3718
3656
|
if (ownerAgentId === null)
|
|
3719
3657
|
continue;
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
// config, never the genesis peer column (§7-1's silent-divergence hazard).
|
|
3724
|
-
const holdersFor = (documentId) => {
|
|
3725
|
-
const holders = documentLayer.holdersFor(ownerAgentId, documentId);
|
|
3726
|
-
return holders === null ? null : holders.filter((h) => h !== ownerAgentId);
|
|
3727
|
-
};
|
|
3728
|
-
// DOD-MP-SWEEP-ALIVE-1 — BOUNDED. The `finally` below releases the overlap guard only
|
|
3729
|
-
// when this body settles, so an await that never settles ends document delivery for the
|
|
3730
|
-
// life of the process, silently: the interval keeps firing, returns at the guard, and
|
|
3731
|
-
// logs nothing. Observed live 2026-08-13. One wedged agent now degrades alone and loudly
|
|
3732
|
-
// instead of taking every other agent's documents down with it.
|
|
3733
|
-
const bounded = await sweepAgentEvicting({
|
|
3734
|
-
logger,
|
|
3735
|
-
timeoutMs: DOCUMENT_DELIVERY_PASS_TIMEOUT_MS,
|
|
3736
|
-
agentName,
|
|
3737
|
-
// Evicting the wedged worker is what makes the recovery real: `DocumentDelivery.tick`
|
|
3738
|
-
// caches its pass and would otherwise hand back the same hung promise forever, so the
|
|
3739
|
-
// agent would never deliver again AND every later pass would pay the full bound
|
|
3740
|
-
// waiting for it — a 2x sweep-interval regression for every healthy agent.
|
|
3741
|
-
workers: documentDeliveryWorkers,
|
|
3742
|
-
run: () => documentDeliveryFor(agentName).tick(ownerAgentId, holdersFor, Date.now(), {
|
|
3743
|
-
senderAgentId: ownerAgentId,
|
|
3744
|
-
}),
|
|
3745
|
-
});
|
|
3746
|
-
if (!bounded.completed) {
|
|
3747
|
-
stuck += 1;
|
|
3748
|
-
continue;
|
|
3658
|
+
const result = await reconcileScheduler.sweep(ownerAgentId);
|
|
3659
|
+
if (result.attempted > 0 || result.failed > 0) {
|
|
3660
|
+
logger.debug("document.reconcile.sweep", { agentName, ...result });
|
|
3749
3661
|
}
|
|
3750
|
-
agentsSwept++;
|
|
3751
|
-
const result = bounded.value;
|
|
3752
|
-
attempted += result.attempted;
|
|
3753
|
-
delivered += result.delivered;
|
|
3754
|
-
failed += result.failed;
|
|
3755
|
-
}
|
|
3756
|
-
// H3: a sweep that delivers nothing was byte-for-byte identical to a healthy idle one, which
|
|
3757
|
-
// is what made every scoping bug above invisible. One line per tick, always.
|
|
3758
|
-
// `stuck` IS PART OF THE LINE. Counting a wedged agent as swept reported `agents: 3,
|
|
3759
|
-
// attempted: 0` — byte-for-byte a healthy idle sweep, which is the exact shape the comment
|
|
3760
|
-
// below exists to prevent, on the one occasion it matters most.
|
|
3761
|
-
logger.debug("document.delivery.sweep", { agents: agentsSwept, stuck, attempted, delivered, failed });
|
|
3762
|
-
if (agentsSwept === 0 && documentLayer.store.anyDocumentExists()) {
|
|
3763
|
-
logger.warn("document.delivery.sweep_empty", {
|
|
3764
|
-
reason: "documents exist but no agent was swept — nothing will ever be delivered",
|
|
3765
|
-
});
|
|
3766
3662
|
}
|
|
3767
3663
|
}
|
|
3768
3664
|
catch (err) {
|
|
3769
|
-
|
|
3770
|
-
|
|
3771
|
-
|
|
3665
|
+
logger.warn("document.reconcile.sweep_threw", {
|
|
3666
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
3667
|
+
});
|
|
3772
3668
|
}
|
|
3773
3669
|
finally {
|
|
3774
|
-
|
|
3775
|
-
documentDeliveryInFlight = null;
|
|
3670
|
+
reconcileSweepStartedAt = null;
|
|
3776
3671
|
}
|
|
3777
3672
|
})();
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
3781
|
-
documentDeliveryTimer.unref?.();
|
|
3673
|
+
}, RECONCILE_SWEEP_MS);
|
|
3674
|
+
reconcileSweepTimer.unref?.();
|
|
3675
|
+
documentOwnerKeyForHook = documentOwnerKeyFor;
|
|
3782
3676
|
// MCP-001: Clean up per-connection state when a connection disconnects
|
|
3783
3677
|
// MCP-002: Also unregister from notification dispatcher
|
|
3784
3678
|
ipcServer.onDisconnect((connectionId) => {
|
|
@@ -3836,40 +3730,7 @@ async function startDaemonHoldingLock(config, singletonLock) {
|
|
|
3836
3730
|
// per daemon no matter how many times stop() is called.
|
|
3837
3731
|
let stoppedHookFired = false;
|
|
3838
3732
|
async function stop(reason) {
|
|
3839
|
-
|
|
3840
|
-
// teardown would dial into a daemon that is going away.
|
|
3841
|
-
clearInterval(documentDeliveryTimer);
|
|
3842
|
-
// `clearInterval` stops the NEXT tick; the flag stops the running one at its next agent.
|
|
3843
|
-
documentDeliveryStopping = true;
|
|
3844
|
-
// BOUNDED. This was a bare `await documentDeliveryInFlight`, and shutdown is the one place that
|
|
3845
|
-
// must not block on the network: a sweep mid-dial against an unreachable peer held `stop()`
|
|
3846
|
-
// open, so the interrupt-marking writes that follow did not complete before the process died —
|
|
3847
|
-
// and sessions came back `active` after a restart, which is the exact defect AC-009 exists to
|
|
3848
|
-
// catch. It failed on CI, where a directory node times out at 5s, and passed locally, where it
|
|
3849
|
-
// does not.
|
|
3850
|
-
//
|
|
3851
|
-
// A tick still gets a moment to finish so the common case is orderly; after that we proceed
|
|
3852
|
-
// without it. What the sweep leaves half-done is safe by construction — delivery state is
|
|
3853
|
-
// derived from the log, so an interrupted pass is re-derived on the next start.
|
|
3854
|
-
// NOT AWAITED AT ALL. Two attempts at waiting for it were both wrong, and the second was worse
|
|
3855
|
-
// than the first:
|
|
3856
|
-
//
|
|
3857
|
-
// 1. a bare await blocked shutdown on the network — a sweep mid-dial against an unreachable
|
|
3858
|
-
// peer held `stop()` open past the point where the interrupt-marking writes had to happen;
|
|
3859
|
-
// 2. bounding it with `setTimeout(...).unref()` looked like the fix and could HANG FOREVER: an
|
|
3860
|
-
// unref'd timer does not hold the event loop open, so while the loop is draining during
|
|
3861
|
-
// shutdown it may never fire, the race never settles, and `stop()` never reaches the line
|
|
3862
|
-
// that logs `daemon.stopped` — which is exactly what CI showed, a daemon whose log ends at
|
|
3863
|
-
// `daemon.started` with no shutdown events at all.
|
|
3864
|
-
//
|
|
3865
|
-
// There is nothing to wait for. `documentDeliveryStopping` stops the loop at its next agent, and
|
|
3866
|
-
// whatever a half-finished sweep leaves behind is safe by construction: delivery state is
|
|
3867
|
-
// DERIVED from the envelope log, so an interrupted pass is simply re-derived on the next start —
|
|
3868
|
-
// the property the offline enforcer proves against two real daemons.
|
|
3869
|
-
//
|
|
3870
|
-
// The rule this cost two CI round trips to learn: shutdown may not await anything that can
|
|
3871
|
-
// block on I/O, and a timeout that can outlive the event loop is not a bound.
|
|
3872
|
-
void documentDeliveryInFlight;
|
|
3733
|
+
clearInterval(reconcileSweepTimer);
|
|
3873
3734
|
// M8C-TGDOOR-1: stop the single long-lived getUpdates poller (no-op if never started) — bump
|
|
3874
3735
|
// the generation so the running loop's while-condition fails on its next check.
|
|
3875
3736
|
stopTelegramPoller(); // M8C-TGDOOR-1: invalidate the poll loop; it exits on its next generation check
|