@cello-protocol/daemon 0.0.181 → 0.0.182
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-id-migration.d.ts +9 -0
- package/dist/agent-id-migration.d.ts.map +1 -1
- package/dist/agent-id-migration.js +73 -1
- package/dist/agent-id-migration.js.map +1 -1
- package/dist/agent-selection.d.ts +73 -0
- package/dist/agent-selection.d.ts.map +1 -0
- package/dist/agent-selection.js +80 -0
- package/dist/agent-selection.js.map +1 -0
- package/dist/assignment-verify.d.ts +62 -0
- package/dist/assignment-verify.d.ts.map +1 -0
- package/dist/assignment-verify.js +185 -0
- package/dist/assignment-verify.js.map +1 -0
- package/dist/backup-restore.d.ts +102 -0
- package/dist/backup-restore.d.ts.map +1 -0
- package/dist/backup-restore.js +340 -0
- package/dist/backup-restore.js.map +1 -0
- package/dist/bin/cello-daemon.js +91 -49
- package/dist/bin/cello-daemon.js.map +1 -1
- package/dist/close-commitment.d.ts +67 -0
- package/dist/close-commitment.d.ts.map +1 -0
- package/dist/close-commitment.js +78 -0
- package/dist/close-commitment.js.map +1 -0
- package/dist/close-session-handler.d.ts +20 -0
- package/dist/close-session-handler.d.ts.map +1 -1
- package/dist/close-session-handler.js +351 -58
- package/dist/close-session-handler.js.map +1 -1
- package/dist/consortium-bootstrap.d.ts +2 -0
- package/dist/consortium-bootstrap.d.ts.map +1 -1
- package/dist/consortium-bootstrap.js +65 -7
- package/dist/consortium-bootstrap.js.map +1 -1
- package/dist/content-park.d.ts.map +1 -1
- package/dist/content-park.js +141 -10
- package/dist/content-park.js.map +1 -1
- package/dist/daemon.d.ts +13 -1
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +866 -122
- package/dist/daemon.js.map +1 -1
- package/dist/delivery-session-suspects.d.ts.map +1 -1
- package/dist/delivery-session-suspects.js +5 -0
- package/dist/delivery-session-suspects.js.map +1 -1
- package/dist/directory-auth-posture.d.ts +87 -0
- package/dist/directory-auth-posture.d.ts.map +1 -0
- package/dist/directory-auth-posture.js +134 -0
- package/dist/directory-auth-posture.js.map +1 -0
- package/dist/directory-bootstrap.d.ts +66 -3
- package/dist/directory-bootstrap.d.ts.map +1 -1
- package/dist/directory-bootstrap.js +133 -7
- package/dist/directory-bootstrap.js.map +1 -1
- package/dist/document-delivery-transport.d.ts +22 -2
- package/dist/document-delivery-transport.d.ts.map +1 -1
- package/dist/document-delivery-transport.js +11 -3
- package/dist/document-delivery-transport.js.map +1 -1
- package/dist/inbound-sessions.d.ts +31 -1
- package/dist/inbound-sessions.d.ts.map +1 -1
- package/dist/inbound-sessions.js +276 -16
- package/dist/inbound-sessions.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/ipc-server.d.ts +5 -1
- package/dist/ipc-server.d.ts.map +1 -1
- package/dist/ipc-server.js +19 -3
- package/dist/ipc-server.js.map +1 -1
- package/dist/manifest-validity.d.ts +151 -0
- package/dist/manifest-validity.d.ts.map +1 -0
- package/dist/manifest-validity.js +268 -0
- package/dist/manifest-validity.js.map +1 -0
- package/dist/notification-handlers.d.ts.map +1 -1
- package/dist/notification-handlers.js +15 -0
- package/dist/notification-handlers.js.map +1 -1
- package/dist/outbound-sessions.d.ts +31 -0
- package/dist/outbound-sessions.d.ts.map +1 -1
- package/dist/outbound-sessions.js +198 -17
- package/dist/outbound-sessions.js.map +1 -1
- package/dist/park-envelope.d.ts +72 -1
- package/dist/park-envelope.d.ts.map +1 -1
- package/dist/park-envelope.js +119 -4
- package/dist/park-envelope.js.map +1 -1
- package/dist/refusal-reasons.d.ts +72 -0
- package/dist/refusal-reasons.d.ts.map +1 -0
- package/dist/refusal-reasons.js +69 -0
- package/dist/refusal-reasons.js.map +1 -0
- package/dist/registration-manager.d.ts.map +1 -1
- package/dist/registration-manager.js +20 -1
- package/dist/registration-manager.js.map +1 -1
- package/dist/restart-seal-resolver.d.ts.map +1 -1
- package/dist/restart-seal-resolver.js +6 -0
- package/dist/restart-seal-resolver.js.map +1 -1
- package/dist/retry-queue.d.ts +23 -1
- package/dist/retry-queue.d.ts.map +1 -1
- package/dist/retry-queue.js +37 -9
- package/dist/retry-queue.js.map +1 -1
- package/dist/roster-freshness.d.ts +169 -0
- package/dist/roster-freshness.d.ts.map +1 -0
- package/dist/roster-freshness.js +249 -0
- package/dist/roster-freshness.js.map +1 -0
- package/dist/seal-certificate-pull.d.ts +17 -1
- package/dist/seal-certificate-pull.d.ts.map +1 -1
- package/dist/seal-certificate-pull.js +37 -3
- package/dist/seal-certificate-pull.js.map +1 -1
- package/dist/seal-coordinator.d.ts +17 -1
- package/dist/seal-coordinator.d.ts.map +1 -1
- package/dist/seal-coordinator.js +71 -0
- package/dist/seal-coordinator.js.map +1 -1
- package/dist/seal-failure-store.d.ts +104 -0
- package/dist/seal-failure-store.d.ts.map +1 -0
- package/dist/seal-failure-store.js +126 -0
- package/dist/seal-failure-store.js.map +1 -0
- package/dist/seal-flows.d.ts.map +1 -1
- package/dist/seal-flows.js +35 -6
- package/dist/seal-flows.js.map +1 -1
- package/dist/seal-upgrade.d.ts +20 -5
- package/dist/seal-upgrade.d.ts.map +1 -1
- package/dist/seal-upgrade.js +28 -5
- package/dist/seal-upgrade.js.map +1 -1
- package/dist/session-assignment-parser.d.ts +41 -5
- package/dist/session-assignment-parser.d.ts.map +1 -1
- package/dist/session-assignment-parser.js +77 -14
- package/dist/session-assignment-parser.js.map +1 -1
- package/dist/session-ceremony.d.ts +29 -0
- package/dist/session-ceremony.d.ts.map +1 -1
- package/dist/session-ceremony.js +71 -0
- package/dist/session-ceremony.js.map +1 -1
- package/dist/session-connection-gater.d.ts +46 -7
- package/dist/session-connection-gater.d.ts.map +1 -1
- package/dist/session-connection-gater.js +163 -12
- package/dist/session-connection-gater.js.map +1 -1
- package/dist/session-content-handlers.d.ts.map +1 -1
- package/dist/session-content-handlers.js +110 -4
- package/dist/session-content-handlers.js.map +1 -1
- package/dist/session-node-manager.d.ts +406 -36
- package/dist/session-node-manager.d.ts.map +1 -1
- package/dist/session-node-manager.js +3025 -245
- package/dist/session-node-manager.js.map +1 -1
- package/dist/session-read-handlers.d.ts +27 -0
- package/dist/session-read-handlers.d.ts.map +1 -1
- package/dist/session-read-handlers.js +112 -6
- package/dist/session-read-handlers.js.map +1 -1
- package/dist/session-relay-client.d.ts +61 -2
- package/dist/session-relay-client.d.ts.map +1 -1
- package/dist/session-relay-client.js +170 -11
- package/dist/session-relay-client.js.map +1 -1
- package/dist/session-salt-agreement.d.ts +284 -0
- package/dist/session-salt-agreement.d.ts.map +1 -0
- package/dist/session-salt-agreement.js +371 -0
- package/dist/session-salt-agreement.js.map +1 -0
- package/dist/signal-submission.d.ts.map +1 -1
- package/dist/signal-submission.js +10 -1
- package/dist/signal-submission.js.map +1 -1
- package/dist/signaling-connect.d.ts.map +1 -1
- package/dist/signaling-connect.js +42 -5
- package/dist/signaling-connect.js.map +1 -1
- package/dist/types.d.ts +54 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/vocabulary.d.ts.map +1 -1
- package/dist/vocabulary.js +10 -0
- package/dist/vocabulary.js.map +1 -1
- package/dist/wire-content-hash.d.ts +56 -0
- package/dist/wire-content-hash.d.ts.map +1 -1
- package/dist/wire-content-hash.js +71 -0
- package/dist/wire-content-hash.js.map +1 -1
- package/package.json +5 -5
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* cello_close_session — the bilateral close, and every way it can fail.
|
|
3
3
|
*
|
|
4
4
|
* M7 error discipline: each distinct failure cause produces a DISTINCT error code
|
|
5
|
-
* (session_already_sealed,
|
|
5
|
+
* (session_already_sealed, seal_in_progress, seal_interrupted_counterparty_unavailable,
|
|
6
6
|
* seal_interrupted_rejected_by_counterparty, signaling_reconnecting). A close that fails must tell
|
|
7
7
|
* the operator WHY, not merely that it failed — this handler exists as much for its error paths as
|
|
8
8
|
* for its happy path.
|
|
@@ -42,8 +42,9 @@ const ABANDON_NOTICE_DEADLINE_MS = 3_000;
|
|
|
42
42
|
import { classifyOnlineResult } from "./cross-node-negotiation.js";
|
|
43
43
|
import { validateSessionName } from "./session-name.js";
|
|
44
44
|
import { escalateToUnilateralSeal as runUnilateralEscalation, UNILATERAL_SEAL_TIMEOUT_MS } from "./seal-escalation.js";
|
|
45
|
+
import { describeSealCommitted } from "./close-commitment.js";
|
|
45
46
|
export function registerCloseSessionHandler(deps) {
|
|
46
|
-
const { resolveConsortiumRoster, handlers, logger, sessionNodeManager, getConnState, resolveCurrentAgent, NO_CURRENT_AGENT_RESPONSE, getKeyProvider, signalingFor, sendOver, waitForSignalingConnected, openVisitingConnection, runDiscoveryLookup, crossNodeBrokerBySession, sealKey, sealInterruptedInProgress, pendingSealWaiters, pendingUnilateralWaiters, handleSealInterruptedFlow, handleActiveSealFlow, recoverParkedContent, } = deps;
|
|
47
|
+
const { resolveConsortiumRoster, handlers, logger, sessionNodeManager, getConnState, resolveCurrentAgent, NO_CURRENT_AGENT_RESPONSE, getKeyProvider, signalingFor, sendOver, waitForSignalingConnected, openVisitingConnection, runDiscoveryLookup, crossNodeBrokerBySession, sealKey, sealInterruptedInProgress, registerBackgroundSeal, sealFailures, pendingSealWaiters, pendingUnilateralWaiters, handleSealInterruptedFlow, handleActiveSealFlow, recoverParkedContent, } = deps;
|
|
47
48
|
const UNILATERAL_TIMEOUT_MS = deps.unilateralTimeoutMs ?? UNILATERAL_SEAL_TIMEOUT_MS;
|
|
48
49
|
/**
|
|
49
50
|
* How long the seal path waits to learn where the counterparty is homed.
|
|
@@ -137,7 +138,7 @@ export function registerCloseSessionHandler(deps) {
|
|
|
137
138
|
// ─── M7-SESSION-001: cello_close_session ────────────────────────────────────
|
|
138
139
|
// M7 error discipline: each distinct failure cause produces a distinct error code.
|
|
139
140
|
// AC-010: session_already_sealed
|
|
140
|
-
// AC-011:
|
|
141
|
+
// AC-011: seal_in_progress
|
|
141
142
|
// AC-012: seal_interrupted_counterparty_unavailable
|
|
142
143
|
// AC-013: seal_interrupted_rejected_by_counterparty
|
|
143
144
|
// DB-001: signaling_reconnecting
|
|
@@ -164,6 +165,13 @@ export function registerCloseSessionHandler(deps) {
|
|
|
164
165
|
"seal_unilateral_timeout",
|
|
165
166
|
"session_seal_already_pending",
|
|
166
167
|
"session_sealed",
|
|
168
|
+
/**
|
|
169
|
+
* DOD-M15-TERMINAL-REASON-1. Without this, a close on a session the relay REFUSED skips
|
|
170
|
+
* `pullSealCertificate` entirely — the one probe that distinguishes "never sealed" from "sealed
|
|
171
|
+
* and you were never told". Losing it here is how a close ends with no receipt and no
|
|
172
|
+
* explanation of why.
|
|
173
|
+
*/
|
|
174
|
+
"seal_refused",
|
|
167
175
|
"relay_session_gone",
|
|
168
176
|
]);
|
|
169
177
|
/**
|
|
@@ -428,6 +436,17 @@ export function registerCloseSessionHandler(deps) {
|
|
|
428
436
|
}
|
|
429
437
|
const told = notice.told;
|
|
430
438
|
await sessionNodeManager.abandonSession(record.agent_name, sessionId);
|
|
439
|
+
/**
|
|
440
|
+
* FORGET ANY SEAL FAILURE — review MEDIUM-5.
|
|
441
|
+
*
|
|
442
|
+
* An abandoned session is TERMINAL: there is no ceremony to retry and no receipt to obtain.
|
|
443
|
+
* Leaving the marker made two surfaces contradict each other — `cello_sealed_receipt` reported
|
|
444
|
+
* `seal_failed` and told the agent to *"call cello_close_session again"*, while
|
|
445
|
+
* `cello_close_session` refused with `session_abandoned` and *"it is terminal… it cannot be
|
|
446
|
+
* closed."* A refused loop, and exactly the contradicting-surfaces pair the previous unit's
|
|
447
|
+
* review flagged, reintroduced one unit later.
|
|
448
|
+
*/
|
|
449
|
+
sealFailures.clear(record.agent_name, sessionId);
|
|
431
450
|
logger.info("session.force_abandoned", {
|
|
432
451
|
agentName: record.agent_name,
|
|
433
452
|
sessionId,
|
|
@@ -475,14 +494,20 @@ export function registerCloseSessionHandler(deps) {
|
|
|
475
494
|
const sealable = record.status === "active" || record.status === "interrupted";
|
|
476
495
|
let readiness = sealable
|
|
477
496
|
? sessionNodeManager.sealReadiness(record.agent_name, sessionId)
|
|
478
|
-
|
|
497
|
+
// DOD-M15-DIVERGE-1: `diverged: false` for the same reason every other field here is a
|
|
498
|
+
// no-op value — this branch is the NOT-sealable statuses, where readiness is never consulted
|
|
499
|
+
// and both gates below are skipped. It asserts nothing about the record; it keeps the shape.
|
|
500
|
+
: { ready: true, treeSize: 0, highWaterSeq: -1, heldCount: 0, missingLeaves: 0, heldOwn: 0, heldReceived: 0, diverged: false };
|
|
479
501
|
// Review HIGH-1: the guidance used to promise "the daemon pulls missing content automatically"
|
|
480
502
|
// while nothing on this path pulled anything — autoRecoverForAgent fires on signaling reconnect,
|
|
481
503
|
// seal-upgrade and agent start, none of which a close triggers. So the operator waited for an
|
|
482
504
|
// event that would never happen, retried, got the same refusal, and reached for force:true — the
|
|
483
505
|
// terminal receipt-less outcome this gate exists to prevent. Drain FIRST, then judge, which is
|
|
484
506
|
// the sequence seal-upgrade.ts already documents: "Recover content -> consult the gate -> REFUSE".
|
|
485
|
-
|
|
507
|
+
// Review LOW-10: skip the drain for a diverged record. `diverged` is now a term in `ready`, so
|
|
508
|
+
// without this a diverged session pays a relay round trip that cannot change the answer — the
|
|
509
|
+
// gap the drain fills is not the condition that refused it.
|
|
510
|
+
if (sealable && !readiness.ready && !readiness.diverged && recoverParkedContent) {
|
|
486
511
|
try {
|
|
487
512
|
await recoverParkedContent(record.agent_name, "close_readiness_gate");
|
|
488
513
|
readiness = sessionNodeManager.sealReadiness(record.agent_name, sessionId);
|
|
@@ -496,6 +521,62 @@ export function registerCloseSessionHandler(deps) {
|
|
|
496
521
|
});
|
|
497
522
|
}
|
|
498
523
|
}
|
|
524
|
+
/**
|
|
525
|
+
* DOD-M15-DIVERGE-1 — a PERMANENT parting gets its own answer, before the transient one.
|
|
526
|
+
*
|
|
527
|
+
* `#diverged` is set when an ack came back behind this side's frontier, which proves the tree
|
|
528
|
+
* and the relay's counter can never agree on a root again. It was detected correctly, logged at
|
|
529
|
+
* ERROR, and reached exactly one consumer: the text `cello status` prints. This gate — the one
|
|
530
|
+
* place the fact changes an outcome — could not see it, so the operator learned only when the
|
|
531
|
+
* counterparty answered `leaf_count_mismatch`, by which point the receipt was gone.
|
|
532
|
+
*
|
|
533
|
+
* BRANCHED, NOT FOLDED INTO `session_incomplete`, and that is the whole point of the ordering.
|
|
534
|
+
* The refusal below tells the operator to "wait a moment and close again" and says the daemon
|
|
535
|
+
* just pulled from the relay — true and useful for a session waiting on arrival, and false for
|
|
536
|
+
* this one. Nothing backfills a position the relay already assigned to something else, so a
|
|
537
|
+
* retry loop on that guidance ends where every one of them ends: `force: true`, terminal, no
|
|
538
|
+
* receipt. Substituting a transient explanation for a permanent condition is the error class
|
|
539
|
+
* this whole milestone exists to remove; doing it inside its own fix would be the worst place.
|
|
540
|
+
*
|
|
541
|
+
* The ERROR log at the detection site STAYS. This is the second half of it, not a relocation:
|
|
542
|
+
* the log is the forensic record and this is the control. (M15-PROCEDURE §2b, Invariant 2.)
|
|
543
|
+
*/
|
|
544
|
+
if (sealable && readiness.diverged) {
|
|
545
|
+
logger.warn("session.seal.blocked_diverged", {
|
|
546
|
+
agentName: record.agent_name, sessionId,
|
|
547
|
+
treeSize: readiness.treeSize, highWaterSeq: readiness.highWaterSeq,
|
|
548
|
+
// Review MEDIUM-5: the same four counters the sibling refusal carries. Without them a
|
|
549
|
+
// session that is BOTH diverged and gapped reported its gap on no surface at all — this
|
|
550
|
+
// branch preempts the incomplete one, and `sealReadinessView` short-circuits to `unknown`.
|
|
551
|
+
heldCount: readiness.heldCount, missingLeaves: readiness.missingLeaves,
|
|
552
|
+
heldOwn: readiness.heldOwn, heldReceived: readiness.heldReceived,
|
|
553
|
+
impact: "this side's tree holds a leaf at a position the relay assigned to something else; the relay's ordering is what gets notarized, so a bilateral seal may be refused and this side cannot tell in advance",
|
|
554
|
+
});
|
|
555
|
+
return {
|
|
556
|
+
ok: false,
|
|
557
|
+
reason: "session_record_diverged",
|
|
558
|
+
tree_size: readiness.treeSize,
|
|
559
|
+
relay_high_water: readiness.highWaterSeq,
|
|
560
|
+
held_messages: readiness.heldCount,
|
|
561
|
+
missing_leaves: readiness.missingLeaves,
|
|
562
|
+
held_own: readiness.heldOwn,
|
|
563
|
+
held_received: readiness.heldReceived,
|
|
564
|
+
// Review HIGH-1: SAY WHAT WAS MEASURED, NOT WHAT IS PREDICTED. The earlier wording asserted
|
|
565
|
+
// the counterparty would refuse with `leaf_count_mismatch` and that force was the only exit.
|
|
566
|
+
// Neither is established here. What is measured is that THIS tree parted from THE RELAY's
|
|
567
|
+
// counter. The bilateral check compares LEAF COUNT, not the root (`seal-flows.ts`:
|
|
568
|
+
// "Merkle-root agreement is NOT verified at this leaf-exchange layer"), and both sides
|
|
569
|
+
// append a behind-frontier leaf at the tail — so if the counterparty skewed the same way
|
|
570
|
+
// the counts still agree and the seal can succeed. Predicting their refusal was an
|
|
571
|
+
// over-claim, and offering force-abandon as "the only exit" pointed at the one irreversible
|
|
572
|
+
// action while the codebase's own mismatch handling leaves the session retryable.
|
|
573
|
+
guidance: `This side's record no longer agrees with the relay's ordering — your tree holds ${readiness.treeSize} message(s) and the relay's counter is at ${readiness.highWaterSeq + 1}. ` +
|
|
574
|
+
`The relay's ordering is what gets notarized, so sealing now may be refused. It may also succeed: the bilateral check compares message COUNTS, not contents, so if the counterparty's record skewed the same way the counts still match. This side cannot tell which from here. ` +
|
|
575
|
+
`What is certain is that this will not resolve on its own — nothing backfills or re-numbers a leaf, so waiting does not change it. ` +
|
|
576
|
+
`Compare message counts with the counterparty before deciding; cello_transcript ${sessionId} shows your full record and it stays readable whatever you choose. ` +
|
|
577
|
+
`cello_close_session ${sessionId} { force: true } ends the session with NO notarized receipt and cannot be undone — the last resort, not the next step.`,
|
|
578
|
+
};
|
|
579
|
+
}
|
|
499
580
|
if (sealable && !readiness.ready) {
|
|
500
581
|
logger.warn("session.seal.blocked_incomplete", {
|
|
501
582
|
agentName: record.agent_name, sessionId,
|
|
@@ -523,12 +604,34 @@ export function registerCloseSessionHandler(deps) {
|
|
|
523
604
|
`. Everything here is waiting on an earlier message from the counterparty that has not arrived; the daemon just pulled from the relay and the gap is still there, so wait a moment and close again. If it does not resolve, cello_transcript ${sessionId} shows what did arrive, and cello_close_session ${sessionId} { force: true } abandons it terminally (no receipt).`,
|
|
524
605
|
};
|
|
525
606
|
}
|
|
526
|
-
|
|
607
|
+
/**
|
|
608
|
+
* AC-011: a seal attempt is already running for this session.
|
|
609
|
+
*
|
|
610
|
+
* DOD-M15-CLOSEWAIT-1 review HIGH-3 rewrote this, because the change made it COMMON and the old
|
|
611
|
+
* wording did not survive contact:
|
|
612
|
+
*
|
|
613
|
+
* - it said *"wait for `session.interrupted.sealed` to appear in the daemon logs"*. That event
|
|
614
|
+
* is emitted NOWHERE in the tree — grep finds it only inside this string. An operator would
|
|
615
|
+
* tail a log for a line that cannot arrive.
|
|
616
|
+
* - it named the seal-INTERRUPTED subsystem, but the common case now is an ACTIVE session
|
|
617
|
+
* whose background ceremony is mid-flight. Wrong subsystem, on the path an operator most
|
|
618
|
+
* often reaches.
|
|
619
|
+
* - it was hard to reach before, because the first close held the caller for the whole
|
|
620
|
+
* ceremony. Now the operator has their terminal back for up to eleven minutes, and
|
|
621
|
+
* re-closing is the obvious move.
|
|
622
|
+
*/
|
|
527
623
|
if (sealInterruptedInProgress.has(sealKey(record.agent_name, sessionId))) {
|
|
528
624
|
return {
|
|
529
625
|
ok: false,
|
|
530
|
-
reason: "
|
|
531
|
-
|
|
626
|
+
reason: "seal_in_progress",
|
|
627
|
+
seal_status: "committed",
|
|
628
|
+
guidance: "A seal ceremony is ALREADY RUNNING for this session — your commitment is recorded and " +
|
|
629
|
+
"there is nothing to retry. This is the normal state after a close: the close answers as " +
|
|
630
|
+
"soon as the commitment is durable and notarizes in the background, which waits for the " +
|
|
631
|
+
"counterparty and can take several minutes. Fetch the result with cello_sealed_receipt " +
|
|
632
|
+
`(session_id ${sessionId}); a seal_in_progress answer there means the same thing — still ` +
|
|
633
|
+
"running, not failed. Do NOT re-close with { force: true } to hurry it: forcing ABANDONS " +
|
|
634
|
+
"the session and permanently forfeits the receipt this ceremony is about to produce.",
|
|
532
635
|
};
|
|
533
636
|
}
|
|
534
637
|
// DB-001: signaling stream reconnecting
|
|
@@ -706,6 +809,16 @@ export function registerCloseSessionHandler(deps) {
|
|
|
706
809
|
// seal handlers) for the duration of the seal, then release it in finally. Same-node sessions have
|
|
707
810
|
// no entry here and use the home stream unchanged.
|
|
708
811
|
let sealBrokerConn = null;
|
|
812
|
+
// DOD-M15-CLOSEWAIT-1: set when the seal tail is handed to a background task, so the enclosing
|
|
813
|
+
// finally does not release a broker connection that ceremony still needs.
|
|
814
|
+
let handedOff = false;
|
|
815
|
+
/**
|
|
816
|
+
* The caller may still ASK to block. Default is the new contract (answer on commitment);
|
|
817
|
+
* `wait_for_seal: true` restores the inline behaviour for a script that genuinely wants the
|
|
818
|
+
* receipt in one call and will wait up to eleven minutes for it. Opt-IN rather than opt-out,
|
|
819
|
+
* because the default has to be the one that does not freeze an interactive operator.
|
|
820
|
+
*/
|
|
821
|
+
const waitForSeal = params?.["wait_for_seal"] === true;
|
|
709
822
|
try {
|
|
710
823
|
// Memory only, deliberately. An ACTIVE session has not been through the restart that
|
|
711
824
|
// empties the broker map, so the entry is there when it is needed — and a lookup here would
|
|
@@ -812,60 +925,216 @@ export function registerCloseSessionHandler(deps) {
|
|
|
812
925
|
// long this can legitimately take, and what forcing costs.
|
|
813
926
|
logger.warn("session.seal.awaiting_counterparty", {
|
|
814
927
|
sessionId, agentName: record.agent_name, deadlineMs: bilateralTimeoutMs, correlationId,
|
|
815
|
-
impact: `
|
|
928
|
+
impact: `the seal is waiting for the counterparty for up to ${Math.round(bilateralTimeoutMs / 60_000)} minutes, then escalates to a unilateral seal and produces a real receipt. It is working. Do NOT force-abandon it — that forfeits the receipt this wait is earning.`,
|
|
816
929
|
});
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
930
|
+
/**
|
|
931
|
+
* DOD-M15-CLOSEWAIT-1 — THE TAIL, EXTRACTED SO IT CAN OUTLIVE THE RESPONSE.
|
|
932
|
+
*
|
|
933
|
+
* Everything from here to the escalation is "wait for the counterparty, then escalate if
|
|
934
|
+
* they never came". It ran inline, which is why the caller sat on a frozen command for up to
|
|
935
|
+
* eleven minutes while it worked correctly — and why one operator force-abandoned seventeen
|
|
936
|
+
* sessions, forfeiting the exact receipts the wait was earning.
|
|
937
|
+
*
|
|
938
|
+
* Extracting it changes nothing about WHAT is signed or in what order: same race, same
|
|
939
|
+
* escalation, same leaf. It changes only who waits for it. (Decisions Carried #4.)
|
|
940
|
+
*
|
|
941
|
+
* `rec`/`sid` are captured because TypeScript discards the narrowing of `record` and
|
|
942
|
+
* `sessionId` inside a nested function.
|
|
943
|
+
*/
|
|
944
|
+
const rec = record;
|
|
945
|
+
const sid = sessionId;
|
|
946
|
+
const awaitSealAndEscalate = async () => {
|
|
947
|
+
let timer;
|
|
948
|
+
const timeoutP = new Promise((r) => { timer = setTimeout(() => r(null), bilateralTimeoutMs); });
|
|
949
|
+
const sealedCompletion = await Promise.race([sealedP, timeoutP]);
|
|
950
|
+
clearTimeout(timer);
|
|
951
|
+
pendingSealWaiters.delete(sealKey(record.agent_name, sessionId));
|
|
952
|
+
/**
|
|
953
|
+
* THE CERTIFICATE WAS REFUSED — `DOD-M15-SEALWIRE-1` bullet 2 (review F4).
|
|
954
|
+
*
|
|
955
|
+
* The seal arrived, its signature verified, and its root did not describe this conversation.
|
|
956
|
+
* Before this the coordinator logged and dropped the waiter, so control fell through to the
|
|
957
|
+
* eleven-minute timeout and the operator was told the counterparty had not closed — about a
|
|
958
|
+
* counterparty who had, and whose certificate had just been refused here.
|
|
959
|
+
*
|
|
960
|
+
* Answered immediately instead, with what was actually detected. NOT retryable, and it says
|
|
961
|
+
* so: `session_sealed` is delivered once, so a retry re-enters the same wait and then finds
|
|
962
|
+
* the relay session gone.
|
|
963
|
+
*/
|
|
964
|
+
if (sealedCompletion !== null && "refused" in sealedCompletion) {
|
|
965
|
+
logger.error("session.seal.refused", {
|
|
966
|
+
sessionId, agentName: record.agent_name, correlationId,
|
|
967
|
+
reason: sealedCompletion.reason, detail: sealedCompletion.detail,
|
|
968
|
+
});
|
|
969
|
+
return {
|
|
970
|
+
ok: false,
|
|
971
|
+
reason: sealedCompletion.reason,
|
|
972
|
+
seal_status: "refused",
|
|
973
|
+
...(sealedCompletion.ownRootHex ? { own_root: sealedCompletion.ownRootHex } : {}),
|
|
974
|
+
guidance: "The directory returned a VALIDLY SIGNED seal whose root does not describe this " +
|
|
975
|
+
`conversation (${sealedCompletion.detail}). This daemon refused it: the session is NOT ` +
|
|
976
|
+
"marked sealed, your transcript is untouched, and nothing was signed with your key. " +
|
|
977
|
+
"Do NOT retry the close — the seal notification is delivered once, so a retry waits the " +
|
|
978
|
+
"full window and then finds the relay session gone. Do NOT force-abandon either: that " +
|
|
979
|
+
"permanently forfeits any receipt. Compare the leaf count and the message list with your " +
|
|
980
|
+
"counterparty, and report the session id — this is a directory or relay fault, not " +
|
|
981
|
+
"something you did.",
|
|
982
|
+
};
|
|
983
|
+
}
|
|
984
|
+
if (sealedCompletion !== null) {
|
|
985
|
+
logger.info("session.seal.completed", { sessionId, sealedRoot: sealedCompletion.rootHex, role: "bilateral", correlationId });
|
|
986
|
+
crossNodeBrokerBySession.delete(`${record.agent_name}:${sessionId}`); // Fix #1 review: evict on terminal seal success (a FAILED close keeps the entry so a retry can still reconnect).
|
|
987
|
+
// M7-SESSION-004 (AC-006): return the legibility certificate on the seal completion so
|
|
988
|
+
// a reader gets it on the same surface that proves the seal — receipt-not-assent,
|
|
989
|
+
// per-party frontiers, attestation modes, and final_message.answered.
|
|
990
|
+
return { ok: true, sealed_root: sealedCompletion.rootHex, legibility: sealedCompletion.legibility };
|
|
991
|
+
}
|
|
992
|
+
// M8B FINDING-1: `responder_seal_already_submitted` has two producers — (a) the auto-ack
|
|
993
|
+
// path submitted our leaf because the COUNTERPARTY's SEAL arrived, or (b) our OWN earlier
|
|
994
|
+
// close submitted it and the counterparty never co-closed. The result now carries the
|
|
995
|
+
// first submit's reportedRootHex/sequenceNumber, so a retry close can still escalate to a
|
|
996
|
+
// unilateral seal (case b — the live-deadlock path). We deliberately do NOT distinguish
|
|
997
|
+
// (a) from (b) client-side: the bilateral wait above already gave case (a) its window, and
|
|
998
|
+
// the directory's grace/already-sealed gates arbitrate a redundant seal_unilateral — that
|
|
999
|
+
// is the sovereign-node-correct shape. Only when the not-ok result carries NO root (the
|
|
1000
|
+
// first submit is still in flight) is "pending" the honest terminal answer here.
|
|
1001
|
+
const escalation = submit.ok
|
|
844
1002
|
? { reportedRootHex: submit.reportedRootHex, sequenceNumber: submit.sequenceNumber }
|
|
845
|
-
:
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
1003
|
+
: submit.reason === "responder_seal_already_submitted" &&
|
|
1004
|
+
typeof submit.reportedRootHex === "string" &&
|
|
1005
|
+
typeof submit.sequenceNumber === "number"
|
|
1006
|
+
? { reportedRootHex: submit.reportedRootHex, sequenceNumber: submit.sequenceNumber }
|
|
1007
|
+
: null;
|
|
1008
|
+
if (!escalation) {
|
|
1009
|
+
return {
|
|
1010
|
+
ok: false,
|
|
1011
|
+
reason: "seal_pending_bilateral",
|
|
1012
|
+
guidance: "Your SEAL leaf is recorded (auto-acknowledged) and the bilateral seal is completing, but it did not finalize within the wait window. Check cello status and the daemon logs; retry cello_close_session if the session remains unsealed.",
|
|
1013
|
+
};
|
|
1014
|
+
}
|
|
1015
|
+
// SESSION-002 (DOD-SEAL): the counterparty did not co-close. Escalate to a UNILATERAL
|
|
1016
|
+
// seal. The body now lives in escalateToUnilateralSeal so the INTERRUPTED branch can reach
|
|
1017
|
+
// it too — it never could, and that is why an interrupted session could not get a receipt.
|
|
1018
|
+
return await escalateToUnilateralSeal(rec, sid, escalation, correlationId);
|
|
1019
|
+
};
|
|
1020
|
+
/**
|
|
1021
|
+
* OWNERSHIP OF THE BROKER CONNECTION MOVES WITH THE TAIL.
|
|
1022
|
+
*
|
|
1023
|
+
* This is the one genuinely dangerous part of the change. The enclosing `finally` releases
|
|
1024
|
+
* `sealBrokerConn`; if it ran while the tail was still going, the background seal would lose
|
|
1025
|
+
* the connection it is waiting on. So the tail releases it itself, and the enclosing
|
|
1026
|
+
* `finally` stands down via `handedOff`.
|
|
1027
|
+
*
|
|
1028
|
+
* PRECISELY WHAT IS LOST, corrected after review: not a corrupted seal. The escalation goes
|
|
1029
|
+
* over the HOME stream, so it still completes. What this connection carries is the
|
|
1030
|
+
* `seal_verified` / `session_sealed` push for a CROSS-NODE bilateral seal — so releasing it
|
|
1031
|
+
* early silently downgrades every cross-node close from a bilateral receipt to a unilateral
|
|
1032
|
+
* one, eleven minutes later. Quieter than corruption, and still worth the guard.
|
|
1033
|
+
*/
|
|
1034
|
+
const finishSeal = async () => {
|
|
1035
|
+
try {
|
|
1036
|
+
return await awaitSealAndEscalate();
|
|
1037
|
+
}
|
|
1038
|
+
finally {
|
|
1039
|
+
if (sealBrokerConn) {
|
|
1040
|
+
try {
|
|
1041
|
+
await sealBrokerConn.stop("seal-complete");
|
|
1042
|
+
}
|
|
1043
|
+
catch (err) {
|
|
1044
|
+
logger.warn("session.seal.broker.release_failed", { sessionId: sid, reason: err instanceof Error ? err.message : String(err) });
|
|
1045
|
+
}
|
|
1046
|
+
sealBrokerConn = null;
|
|
1047
|
+
}
|
|
1048
|
+
sealInterruptedInProgress.delete(sealKey(rec.agent_name, sid));
|
|
1049
|
+
}
|
|
1050
|
+
};
|
|
1051
|
+
/**
|
|
1052
|
+
* A NEW CEREMONY IS STARTING, so any previous failure verdict is now false.
|
|
1053
|
+
*
|
|
1054
|
+
* ABOVE the `waitForSeal` branch — review MEDIUM-4. It used to sit below, so the clear ran
|
|
1055
|
+
* only on the handed-off path: a blocking re-close (which the MCP tool exposes and the
|
|
1056
|
+
* document-delivery seal uses) left the old marker in place, and a stale `seal_failed` from
|
|
1057
|
+
* half an hour earlier was then reported as the current state. That is `STALEROSTER-1`'s
|
|
1058
|
+
* defect, in the store whose own docstring says it avoids it.
|
|
1059
|
+
*/
|
|
1060
|
+
sealFailures.clear(rec.agent_name, sid);
|
|
1061
|
+
if (waitForSeal)
|
|
1062
|
+
return await finishSeal();
|
|
1063
|
+
handedOff = true;
|
|
1064
|
+
/**
|
|
1065
|
+
* Detached deliberately, and never awaited. A rejection must not become an unhandled
|
|
1066
|
+
* rejection — and must not be silent either: the operator has already been told the seal is
|
|
1067
|
+
* running, so a failure they never hear about is the worst of both worlds.
|
|
1068
|
+
*/
|
|
1069
|
+
const tail = finishSeal();
|
|
1070
|
+
/**
|
|
1071
|
+
* TRACKED FOR SHUTDOWN — review MEDIUM-6.
|
|
1072
|
+
*
|
|
1073
|
+
* `stop()` cancels or awaits every other background worker: the reconcile scheduler, the
|
|
1074
|
+
* telegram poller, the manifest poll, the roster sweep. `RestartSealResolver.stop()` goes
|
|
1075
|
+
* further and AWAITS its in-flight seal, with a comment saying exactly why — *"severing
|
|
1076
|
+
* signaling under a half-finished seal-interrupted exchange leaves the counterparty holding
|
|
1077
|
+
* a commitment we never acknowledged… permanently divergent."*
|
|
1078
|
+
*
|
|
1079
|
+
* A detached, untracked ceremony was the one background task that could be cut at an
|
|
1080
|
+
* arbitrary point by `cello logout`. Recoverable on the next boot, but recoverable is not
|
|
1081
|
+
* the same as not breaking it.
|
|
1082
|
+
*/
|
|
1083
|
+
registerBackgroundSeal?.(tail);
|
|
1084
|
+
void tail.then((result) => {
|
|
1085
|
+
const r = result;
|
|
1086
|
+
if (r?.ok) {
|
|
1087
|
+
sealFailures.clear(rec.agent_name, sid);
|
|
1088
|
+
logger.info("session.seal.background.completed", { sessionId: sid, agentName: rec.agent_name, correlationId });
|
|
1089
|
+
}
|
|
1090
|
+
else {
|
|
1091
|
+
/**
|
|
1092
|
+
* THE BRANCH PRODUCTION ACTUALLY TAKES — review HIGH-1.
|
|
1093
|
+
*
|
|
1094
|
+
* `escalateToUnilateralSeal` contains zero `throw`s: all nine of its failure paths
|
|
1095
|
+
* RESOLVE with `{ ok: false, reason }`. So recording only in the `.catch` below meant
|
|
1096
|
+
* every ordinary dead ceremony went unrecorded and `cello_sealed_receipt` kept
|
|
1097
|
+
* answering `not_sealed_yet` — the very answer this unit exists to replace. The log
|
|
1098
|
+
* line three lines down said so out loud the whole time.
|
|
1099
|
+
*/
|
|
1100
|
+
sealFailures.record(rec.agent_name, sid, r?.reason ?? "seal_unresolved", new Date().toISOString(), "unresolved");
|
|
1101
|
+
logger.warn("session.seal.background.unresolved", {
|
|
1102
|
+
sessionId: sid, agentName: rec.agent_name, correlationId, reason: r?.reason,
|
|
1103
|
+
impact: "the close already answered; this session holds a durable commitment but has no receipt yet.",
|
|
1104
|
+
guidance: "cello_sealed_receipt now reports seal_failed with this reason; a daemon restart also retries it.",
|
|
1105
|
+
});
|
|
1106
|
+
}
|
|
1107
|
+
}, (err) => {
|
|
1108
|
+
// RECORDED FOR THE RESPONSE, not only the log. The caller already holds `ok: true`, so a
|
|
1109
|
+
// failure that lives only in daemon.log is one the agent has no way to discover.
|
|
1110
|
+
sealFailures.record(rec.agent_name, sid, err instanceof Error ? err.message : String(err), new Date().toISOString(), "threw");
|
|
1111
|
+
logger.error("session.seal.background.failed", {
|
|
1112
|
+
sessionId: sid, agentName: rec.agent_name, correlationId,
|
|
1113
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1114
|
+
impact: "the close already answered ok; the notarization did NOT complete and no receipt exists.",
|
|
1115
|
+
guidance: "The commitment is durable — a daemon restart resolves it via the restart seal resolver.",
|
|
1116
|
+
});
|
|
1117
|
+
});
|
|
1118
|
+
return describeSealCommitted({ sessionId: sid, deadlineMs: bilateralTimeoutMs });
|
|
857
1119
|
}
|
|
858
1120
|
finally {
|
|
859
1121
|
// Fix #1: release the transient broker seal-connection (best-effort; the seal result stands).
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
1122
|
+
//
|
|
1123
|
+
// DOD-M15-CLOSEWAIT-1: skipped when the tail was handed to a background task, which owns
|
|
1124
|
+
// this cleanup itself. Releasing here would pull the transport out from under a ceremony
|
|
1125
|
+
// still in flight — the one way this change could corrupt a seal rather than merely report
|
|
1126
|
+
// it differently.
|
|
1127
|
+
if (!handedOff) {
|
|
1128
|
+
if (sealBrokerConn) {
|
|
1129
|
+
try {
|
|
1130
|
+
await sealBrokerConn.stop("seal-complete");
|
|
1131
|
+
}
|
|
1132
|
+
catch (err) {
|
|
1133
|
+
logger.warn("session.seal.broker.release_failed", { sessionId, reason: err instanceof Error ? err.message : String(err) });
|
|
1134
|
+
}
|
|
866
1135
|
}
|
|
1136
|
+
sealInterruptedInProgress.delete(sealKey(record.agent_name, sessionId));
|
|
867
1137
|
}
|
|
868
|
-
sealInterruptedInProgress.delete(sealKey(record.agent_name, sessionId));
|
|
869
1138
|
}
|
|
870
1139
|
}
|
|
871
1140
|
// DOD-M12B-PENDING-EXIT-1 — `seal_interrupted_pending` had NO exit, and the refusal below told
|
|
@@ -897,12 +1166,36 @@ export function registerCloseSessionHandler(deps) {
|
|
|
897
1166
|
// started yet", which a freshly booted daemon reports for every session. The genuinely
|
|
898
1167
|
// permanent case (the relay released the session) has its own reason,
|
|
899
1168
|
// `seal_carry_empty`, raised inside the escalation where it is actually known.
|
|
1169
|
+
/**
|
|
1170
|
+
* ⚠️ "USUALLY LOCAL AND TEMPORARY" IS NOT TRUE OF EVERY REASON, AND THE ONES IT IS FALSE
|
|
1171
|
+
* FOR ARE THE ONES DESIGNED TO FAIL LOUD — review pass 2, MEDIUM-4.
|
|
1172
|
+
*
|
|
1173
|
+
* The sentence below was fixed text wrapped around whatever reason arrived. That is right
|
|
1174
|
+
* for `standing_receiver_unavailable` and a relay this daemon cannot reach. It is wrong
|
|
1175
|
+
* for the client-side seal-payload guards, which fire only on a deterministic defect that
|
|
1176
|
+
* will never clear — and the operator was being sent to `cello_start_agent`, `cello_status`
|
|
1177
|
+
* and an endless retry for a bug in this daemon.
|
|
1178
|
+
*
|
|
1179
|
+
* A guard built to fail loud, converted back into "try again later" by the sentence that
|
|
1180
|
+
* reports it. Enumerated, never pattern-matched: a substring rule would absorb the next
|
|
1181
|
+
* reason nobody has considered, which is the same collapse in a new coat.
|
|
1182
|
+
*/
|
|
1183
|
+
const NON_RETRYABLE_SEAL_SUBMIT_REASONS = new Set([
|
|
1184
|
+
"seal_payload_not_carried",
|
|
1185
|
+
"seal_payload_unbound",
|
|
1186
|
+
"seal_payload_invalid",
|
|
1187
|
+
"content_not_permitted_for_leaf_kind",
|
|
1188
|
+
]);
|
|
900
1189
|
return {
|
|
901
1190
|
ok: false,
|
|
902
1191
|
reason: uni.reason,
|
|
903
|
-
guidance:
|
|
904
|
-
`
|
|
905
|
-
|
|
1192
|
+
guidance: NON_RETRYABLE_SEAL_SUBMIT_REASONS.has(uni.reason)
|
|
1193
|
+
? `This session holds a bilateral commitment, but no notarization could be requested for it: ${uni.reason}. ` +
|
|
1194
|
+
`This one is NOT transient and retrying will not clear it — it is a defect in this daemon's seal path, refused locally before anything was sent. ` +
|
|
1195
|
+
`The conversation is intact and nothing was disclosed. Report the reason code above; the daemon log carries the detail under session.relay.submit.*.`
|
|
1196
|
+
: `This session holds a bilateral commitment, but no notarization could be requested for it: ${uni.reason}. ` +
|
|
1197
|
+
`That is usually local and temporary — an agent that is not started yet (cello_start_agent), or a relay this daemon cannot currently reach (cello_status). ` +
|
|
1198
|
+
`The conversation is intact either way; retry cello_close_session once the daemon reports healthy.`,
|
|
906
1199
|
};
|
|
907
1200
|
}
|
|
908
1201
|
if (uni.ok)
|