@cello-protocol/daemon 0.0.28 → 0.0.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cross-node-negotiation.d.ts +44 -0
- package/dist/cross-node-negotiation.d.ts.map +1 -0
- package/dist/cross-node-negotiation.js +34 -0
- package/dist/cross-node-negotiation.js.map +1 -0
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +357 -52
- package/dist/daemon.js.map +1 -1
- package/dist/session-assignment-parser.d.ts +17 -0
- package/dist/session-assignment-parser.d.ts.map +1 -1
- package/dist/session-assignment-parser.js +20 -2
- package/dist/session-assignment-parser.js.map +1 -1
- package/dist/signaling-connect.d.ts +9 -0
- package/dist/signaling-connect.d.ts.map +1 -1
- package/dist/signaling-connect.js +2 -0
- package/dist/signaling-connect.js.map +1 -1
- package/package.json +4 -4
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-node session establishment — discovery outcome types + the pure online-result classifier
|
|
3
|
+
* (Story B, item 2).
|
|
4
|
+
*
|
|
5
|
+
* The daemon's discovery step produces a DiscoveryOutcome; the retry loop handles the transport /
|
|
6
|
+
* directory-fault / no-reply variants attempt-aware (each with its own log + terminal reason), and
|
|
7
|
+
* defers the actual 3-state routing decision to classifyOnlineResult — kept pure so the branch and
|
|
8
|
+
* error-code mapping are unit-testable in isolation (mirrors the directory-side resolveDiscoveryState).
|
|
9
|
+
*
|
|
10
|
+
* Design: docs/planning/discussion_logs/2026-07-04_1730_cross-node-session-topology.md §"Item 2".
|
|
11
|
+
*/
|
|
12
|
+
export type DiscoveryOutcome = {
|
|
13
|
+
kind: "result";
|
|
14
|
+
state: "online" | "offline" | "unknown_agent";
|
|
15
|
+
owningNodeIds: string[];
|
|
16
|
+
} | {
|
|
17
|
+
kind: "error";
|
|
18
|
+
reason: string;
|
|
19
|
+
} | {
|
|
20
|
+
kind: "malformed";
|
|
21
|
+
} | {
|
|
22
|
+
kind: "timeout";
|
|
23
|
+
} | {
|
|
24
|
+
kind: "send_failed";
|
|
25
|
+
reason: string;
|
|
26
|
+
};
|
|
27
|
+
export type ResultAction = {
|
|
28
|
+
kind: "same_node";
|
|
29
|
+
} | {
|
|
30
|
+
kind: "cross_node";
|
|
31
|
+
owningNodeId: string;
|
|
32
|
+
} | {
|
|
33
|
+
kind: "unknown_agent";
|
|
34
|
+
} | {
|
|
35
|
+
kind: "offline";
|
|
36
|
+
} | {
|
|
37
|
+
kind: "retry";
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* Route a 3-state discovery RESULT into the next action. Pure. Retry counting, backoff, transport /
|
|
41
|
+
* directory-fault / timeout handling, and the actual I/O live in the daemon loop.
|
|
42
|
+
*/
|
|
43
|
+
export declare function classifyOnlineResult(state: "online" | "offline" | "unknown_agent", owningNodeIds: string[], homeNodeId: string | null): ResultAction;
|
|
44
|
+
//# sourceMappingURL=cross-node-negotiation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cross-node-negotiation.d.ts","sourceRoot":"","sources":["../src/cross-node-negotiation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,MAAM,gBAAgB,GAExB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,QAAQ,GAAG,SAAS,GAAG,eAAe,CAAC;IAAC,aAAa,EAAE,MAAM,EAAE,CAAA;CAAE,GAG1F;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAEjC;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GAGrB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GAGnB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5C,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,eAAe,CAAA;CAAE,GACzB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,CAAC;AAEtB;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,QAAQ,GAAG,SAAS,GAAG,eAAe,EAC7C,aAAa,EAAE,MAAM,EAAE,EACvB,UAAU,EAAE,MAAM,GAAG,IAAI,GACxB,YAAY,CAcd"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-node session establishment — discovery outcome types + the pure online-result classifier
|
|
3
|
+
* (Story B, item 2).
|
|
4
|
+
*
|
|
5
|
+
* The daemon's discovery step produces a DiscoveryOutcome; the retry loop handles the transport /
|
|
6
|
+
* directory-fault / no-reply variants attempt-aware (each with its own log + terminal reason), and
|
|
7
|
+
* defers the actual 3-state routing decision to classifyOnlineResult — kept pure so the branch and
|
|
8
|
+
* error-code mapping are unit-testable in isolation (mirrors the directory-side resolveDiscoveryState).
|
|
9
|
+
*
|
|
10
|
+
* Design: docs/planning/discussion_logs/2026-07-04_1730_cross-node-session-topology.md §"Item 2".
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Route a 3-state discovery RESULT into the next action. Pure. Retry counting, backoff, transport /
|
|
14
|
+
* directory-fault / timeout handling, and the actual I/O live in the daemon loop.
|
|
15
|
+
*/
|
|
16
|
+
export function classifyOnlineResult(state, owningNodeIds, homeNodeId) {
|
|
17
|
+
if (state === "unknown_agent")
|
|
18
|
+
return { kind: "unknown_agent" };
|
|
19
|
+
if (state === "offline")
|
|
20
|
+
return { kind: "offline" };
|
|
21
|
+
// state === "online": pick the owning node (list-valued; length 1 until the k>1 homing knob).
|
|
22
|
+
const owningNodeId = owningNodeIds[0];
|
|
23
|
+
// Online but no owner named is malformed — treat as transiently unresolved, retry (never dial a
|
|
24
|
+
// fabricated node).
|
|
25
|
+
if (!owningNodeId)
|
|
26
|
+
return { kind: "retry" };
|
|
27
|
+
// Same-node shortcut requires KNOWING our own node id (step-6 identity). Without it (homeNodeId
|
|
28
|
+
// null), we cannot prove co-location, so we take the cross-node path (which validates the owning
|
|
29
|
+
// node through the signed manifest before dialing).
|
|
30
|
+
if (homeNodeId !== null && owningNodeId === homeNodeId)
|
|
31
|
+
return { kind: "same_node" };
|
|
32
|
+
return { kind: "cross_node", owningNodeId };
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=cross-node-negotiation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cross-node-negotiation.js","sourceRoot":"","sources":["../src/cross-node-negotiation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAwBH;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAA6C,EAC7C,aAAuB,EACvB,UAAyB;IAEzB,IAAI,KAAK,KAAK,eAAe;QAAE,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;IAChE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAEpD,8FAA8F;IAC9F,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;IACtC,gGAAgG;IAChG,oBAAoB;IACpB,IAAI,CAAC,YAAY;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC5C,gGAAgG;IAChG,iGAAiG;IACjG,oDAAoD;IACpD,IAAI,UAAU,KAAK,IAAI,IAAI,YAAY,KAAK,UAAU;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;IACrF,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;AAC9C,CAAC"}
|
package/dist/daemon.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../src/daemon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAKH,OAAO,KAAK,EACV,YAAY,EACZ,oBAAoB,EASrB,MAAM,YAAY,CAAC;AAIpB,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAM/D,OAAO,EAAoD,KAAK,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAsB7G,OAAO,KAAK,EAAE,kBAAkB,EAA+C,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../src/daemon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAKH,OAAO,KAAK,EACV,YAAY,EACZ,oBAAoB,EASrB,MAAM,YAAY,CAAC;AAIpB,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAM/D,OAAO,EAAoD,KAAK,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAsB7G,OAAO,KAAK,EAAE,kBAAkB,EAA+C,MAAM,yBAAyB,CAAC;AAO/G,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,2BAA2B,CAAC;AA0InF,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,SAAS,IAAI,oBAAoB,CAAC;IAClC;;;;OAIG;IACH,qBAAqB,IAAI,kBAAkB,CAAC;IAC5C;;;;;;;;OAQG;IACH,gBAAgB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrC;;;;OAIG;IACH,oBAAoB,IAAI,kBAAkB,CAAC;IAC3C;;;OAGG;IACH,iBAAiB,IAAI,eAAe,CAAC;CACtC;AAyCD,wBAAsB,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAmkJ7E"}
|
package/dist/daemon.js
CHANGED
|
@@ -51,7 +51,8 @@ import { upgradeAbsentToRecovered, hasAbsentParticipant } from "./seal-receipt-u
|
|
|
51
51
|
import { MAX_CONTENT_BYTES, computeGenesisPrevRoot, buildAgentRevocationTbs } from "@cello-protocol/protocol-types";
|
|
52
52
|
import { resolveCelloEnv, createTransportSelector, isProductionVariant, } from "./transport-composition.js";
|
|
53
53
|
import { selectAdvertisedAddress } from "./transport-selector.js";
|
|
54
|
-
import { parseSessionAssignment, sessionRequestErrorReason } from "./session-assignment-parser.js";
|
|
54
|
+
import { parseSessionAssignment, sessionRequestErrorReason, parseDiscoveryLookupResult, discoveryLookupErrorReason } from "./session-assignment-parser.js";
|
|
55
|
+
import { classifyOnlineResult } from "./cross-node-negotiation.js";
|
|
55
56
|
import { wireSessionCeremonyHandler, wireSessionOfferHandler, wireSealCeremonyHandler, verifyUnilateralCertificate, verifyBilateralSealCertificate, runAgentRefresh } from "./session-ceremony.js";
|
|
56
57
|
import { reDeriveFrontiers, findInflatedFrontier, checkUnilateralFrontier } from "./seal-frontier-verify.js";
|
|
57
58
|
import { LocalAutoNatStub } from "@cello-protocol/transport";
|
|
@@ -749,6 +750,229 @@ export async function startDaemon(config) {
|
|
|
749
750
|
// agent concurrency is unaffected (separate streams). A directory-side echoed request id
|
|
750
751
|
// would allow true concurrency later; this is the correct minimum.
|
|
751
752
|
const negotiationInProgress = new Set();
|
|
753
|
+
// ─── Cross-node session establishment (Story B, item 2) ──────────────────────
|
|
754
|
+
// Helpers the negotiator composes: discover the target's home from replicated presence, then run
|
|
755
|
+
// the session_request over the RIGHT connection — the existing home stream when same-node, or a
|
|
756
|
+
// transient VISITING connection into the target's home node when cross-node. Directories never talk
|
|
757
|
+
// to each other; the client spans nodes on demand.
|
|
758
|
+
const sleepMs = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
759
|
+
/**
|
|
760
|
+
* Issue a discovery_lookup on `signaling` and await the 3-state answer (bounded). Distinguishes,
|
|
761
|
+
* for the caller's retry logic and truthful error surface:
|
|
762
|
+
* - send_failed (home stream down — TRANSPORT, never "old directory"),
|
|
763
|
+
* - timeout (no reply — old directory OR a slow/dropped reply on a new one; retry, fall back last),
|
|
764
|
+
* - error (directory DB fault — retryable, a DIRECTORY fault, not the counterparty being offline),
|
|
765
|
+
* - malformed (a reply that didn't parse — protocol anomaly, retryable, surfaced distinctly),
|
|
766
|
+
* - result (the 3-state answer).
|
|
767
|
+
* Emits the mandated observability events: directory.discovery.lookup on a result,
|
|
768
|
+
* directory.discovery.lookup.failed on a directory error / malformed reply.
|
|
769
|
+
*/
|
|
770
|
+
async function runDiscoveryLookup(signaling, targetHex, timeoutMs, correlationId) {
|
|
771
|
+
let resolveFrame;
|
|
772
|
+
const pending = new Promise((r) => { resolveFrame = r; });
|
|
773
|
+
const unregister = signaling.registerInboundHandler((frame) => {
|
|
774
|
+
const t = frame["type"];
|
|
775
|
+
if (t === "discovery_lookup_result" || t === "discovery_lookup_error")
|
|
776
|
+
resolveFrame(frame);
|
|
777
|
+
});
|
|
778
|
+
try {
|
|
779
|
+
const sent = await signaling.sendRaw({
|
|
780
|
+
type: "discovery_lookup",
|
|
781
|
+
target_pubkey: new Uint8Array(Buffer.from(targetHex, "hex")),
|
|
782
|
+
});
|
|
783
|
+
if (!sent.ok) {
|
|
784
|
+
// The home stream is down — cannot look up (and today's local-only fallback would fail the
|
|
785
|
+
// same way). Surface the real transport reason, not an "old directory" misdiagnosis.
|
|
786
|
+
return { kind: "send_failed", reason: sent.reason ?? "signaling_unavailable" };
|
|
787
|
+
}
|
|
788
|
+
let timer;
|
|
789
|
+
const timeoutP = new Promise((r) => { timer = setTimeout(() => r({ type: "__timeout__" }), timeoutMs); });
|
|
790
|
+
const frame = await Promise.race([pending, timeoutP]);
|
|
791
|
+
clearTimeout(timer);
|
|
792
|
+
if (frame["type"] === "__timeout__")
|
|
793
|
+
return { kind: "timeout" };
|
|
794
|
+
if (frame["type"] === "discovery_lookup_error") {
|
|
795
|
+
const reason = discoveryLookupErrorReason(frame);
|
|
796
|
+
logger.warn("directory.discovery.lookup.failed", { target: targetHex.slice(0, 16), reason, correlationId });
|
|
797
|
+
return { kind: "error", reason };
|
|
798
|
+
}
|
|
799
|
+
const parsed = parseDiscoveryLookupResult(frame);
|
|
800
|
+
if (!parsed) {
|
|
801
|
+
// A reply came back but did not parse — a protocol/version anomaly on a directory that DID
|
|
802
|
+
// respond. Surface it distinctly from a clean directory DB error, and never as availability.
|
|
803
|
+
logger.warn("directory.discovery.lookup.failed", { target: targetHex.slice(0, 16), reason: "malformed_reply", correlationId });
|
|
804
|
+
return { kind: "malformed" };
|
|
805
|
+
}
|
|
806
|
+
logger.info("directory.discovery.lookup", {
|
|
807
|
+
target: targetHex.slice(0, 16),
|
|
808
|
+
state: parsed.state,
|
|
809
|
+
owningNode: parsed.owningNodeIds[0] ?? null,
|
|
810
|
+
correlationId,
|
|
811
|
+
});
|
|
812
|
+
return { kind: "result", state: parsed.state, owningNodeIds: parsed.owningNodeIds };
|
|
813
|
+
}
|
|
814
|
+
finally {
|
|
815
|
+
unregister();
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
/** Send a session_request over `signaling` and await the assignment / error (the extracted core). */
|
|
819
|
+
async function runSessionRequestOverSignaling(signaling, targetHex, sr, correlationId, agentName) {
|
|
820
|
+
let resolveFrame;
|
|
821
|
+
const pending = new Promise((r) => { resolveFrame = r; });
|
|
822
|
+
const unregister = signaling.registerInboundHandler((frame) => {
|
|
823
|
+
const t = frame["type"];
|
|
824
|
+
if (t === "session_assignment" || t === "session_request_error")
|
|
825
|
+
resolveFrame(frame);
|
|
826
|
+
});
|
|
827
|
+
try {
|
|
828
|
+
const sent = await signaling.sendRaw({
|
|
829
|
+
type: "session_request",
|
|
830
|
+
target_pubkey: new Uint8Array(Buffer.from(targetHex, "hex")),
|
|
831
|
+
initiator_session_peer_id: sr.peerId,
|
|
832
|
+
initiator_session_addrs: sr.addrs,
|
|
833
|
+
wants_session_offer: true,
|
|
834
|
+
});
|
|
835
|
+
if (!sent.ok) {
|
|
836
|
+
return { ok: false, reason: sent.reason ?? "directory_unreachable", guidance: sent.guidance ?? "Could not send session_request over the directory signaling stream." };
|
|
837
|
+
}
|
|
838
|
+
let timer;
|
|
839
|
+
const timeoutP = new Promise((r) => { timer = setTimeout(() => r({ type: "__timeout__" }), 30_000); });
|
|
840
|
+
const frame = await Promise.race([pending, timeoutP]);
|
|
841
|
+
clearTimeout(timer);
|
|
842
|
+
if (frame["type"] === "__timeout__") {
|
|
843
|
+
return { ok: false, reason: "timeout", guidance: "The directory did not return a session assignment within 30s. Retry once cello status shows directory_signaling connected." };
|
|
844
|
+
}
|
|
845
|
+
if (frame["type"] === "session_request_error") {
|
|
846
|
+
const reason = sessionRequestErrorReason(frame);
|
|
847
|
+
return { ok: false, reason, guidance: `The directory refused the session request (${reason}). Ensure the counterparty is registered and online.` };
|
|
848
|
+
}
|
|
849
|
+
const raw = frame["assignment"];
|
|
850
|
+
const assignment = raw ? parseSessionAssignment(raw) : null;
|
|
851
|
+
if (!assignment) {
|
|
852
|
+
return { ok: false, reason: "assignment_parse_failed", guidance: "The directory's session_assignment was missing or malformed." };
|
|
853
|
+
}
|
|
854
|
+
logger.info("session.negotiate.assignment.received", { agentName, correlationId, signatureType: assignment.signature_type });
|
|
855
|
+
return { ok: true, assignment };
|
|
856
|
+
}
|
|
857
|
+
finally {
|
|
858
|
+
unregister();
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
/**
|
|
862
|
+
* Open a transient VISITING signaling connection into a specific directory node (the target's home /
|
|
863
|
+
* broker) and wire what a cross-node initiator needs there:
|
|
864
|
+
* - the delegated-signer ceremony handler (the broker asks the initiator to co-sign the assignment
|
|
865
|
+
* over THIS connection); and
|
|
866
|
+
* - Fix #1 (cross-node seal-liveness): the seal ceremony handler + session_sealed/unilateral
|
|
867
|
+
* listeners, because for a cross-node CLOSE the broker ALSO pushes seal_verified then session_sealed
|
|
868
|
+
* over the connection the initiator holds to it. The seal FROST still runs over the agent's OWN
|
|
869
|
+
* roster (getNode: () => nodeRef) — only the control frames traverse this stream.
|
|
870
|
+
* Transient and initiator-only. The caller MUST stop() it after the assignment (setup path) or after
|
|
871
|
+
* the seal reaches a terminal outcome (close path).
|
|
872
|
+
*/
|
|
873
|
+
function openVisitingConnection(agentName, agentKeyProvider, agentPubkeyHex, endpoint, correlationId, nodeId) {
|
|
874
|
+
let nodeRef = null;
|
|
875
|
+
const connect = createSignalingConnect({
|
|
876
|
+
getDirectoryEndpoint: () => endpoint,
|
|
877
|
+
getAuthIdentity: () => ({ keyProvider: agentKeyProvider, pubkeyHex: agentPubkeyHex }),
|
|
878
|
+
logger,
|
|
879
|
+
challengeVerifier,
|
|
880
|
+
getManifestVersion: () => verifiedManifestVersion,
|
|
881
|
+
visiting: true, // cross-node item 3: the directory must NOT write presence for this connection
|
|
882
|
+
publishNode: (n) => { nodeRef = n; },
|
|
883
|
+
});
|
|
884
|
+
// maxReconnectAttempts: 1 — a transient connection should fail fast, not reconnect-loop forever.
|
|
885
|
+
const mgr = new SignalingManager({ connect, logger, maxReconnectAttempts: 1, maxBackoffMs: 3_000 });
|
|
886
|
+
wireSessionCeremonyHandler({
|
|
887
|
+
agentName,
|
|
888
|
+
persistence: getPersistence(agentName),
|
|
889
|
+
agentPubkeyHex,
|
|
890
|
+
getNode: () => nodeRef,
|
|
891
|
+
getDirectoryEndpoint: getFailoverEndpoint,
|
|
892
|
+
getConsortiumEndpoints: resolveConsortiumRoster,
|
|
893
|
+
signaling: mgr,
|
|
894
|
+
logger,
|
|
895
|
+
});
|
|
896
|
+
// Fix #1 (cross-node seal-liveness): a visiting connection must ALSO be able to complete the SEAL.
|
|
897
|
+
// The broker (the node this visiting connection targets) pushes seal_verified then session_sealed to
|
|
898
|
+
// whichever stream the initiator holds to it — for a cross-node close that is THIS visiting stream.
|
|
899
|
+
// Run the seal FROST ceremony over the agent's own node (getNode: () => nodeRef, exactly as the
|
|
900
|
+
// session ceremony above) and reply/resolve on this stream. Harmless on a setup-only visiting
|
|
901
|
+
// connection: these frames never arrive before it is released after handoff.
|
|
902
|
+
wireSealCeremonyHandler({
|
|
903
|
+
agentName,
|
|
904
|
+
persistence: getPersistence(agentName),
|
|
905
|
+
agentPubkeyHex,
|
|
906
|
+
getNode: () => nodeRef,
|
|
907
|
+
getDirectoryEndpoint: getFailoverEndpoint,
|
|
908
|
+
getConsortiumEndpoints: resolveConsortiumRoster,
|
|
909
|
+
signaling: mgr,
|
|
910
|
+
logger,
|
|
911
|
+
});
|
|
912
|
+
registerSessionSealedListener(mgr, agentName, agentPubkeyHex);
|
|
913
|
+
registerUnilateralConfirmedListener(mgr, agentName, agentPubkeyHex);
|
|
914
|
+
logger.info("signaling.visiting.connected", { agentName, node: nodeId, correlationId });
|
|
915
|
+
return {
|
|
916
|
+
mgr,
|
|
917
|
+
stop: async (reason) => {
|
|
918
|
+
logger.info("signaling.visiting.released", { agentName, node: nodeId, reason, correlationId });
|
|
919
|
+
await mgr.stop();
|
|
920
|
+
},
|
|
921
|
+
};
|
|
922
|
+
}
|
|
923
|
+
// Fix #1 (cross-node seal-liveness): the broker node (the counterparty's home) for each cross-node
|
|
924
|
+
// session we initiated, keyed `${agentName}:${sessionIdHex}`. Set when runCrossNodeSetup establishes
|
|
925
|
+
// the session; read by cello_close_session so it can re-open a visiting connection to the broker and
|
|
926
|
+
// complete the seal there. The broker pushes seal_verified/session_sealed, but the initiator RELEASES
|
|
927
|
+
// its visiting connection after setup — so without reconnecting, the close times out
|
|
928
|
+
// (seal_unilateral_timeout) and the seal only completes whenever the daemon next happens to reconnect.
|
|
929
|
+
// Presence of an entry means the session is cross-node (same-node sessions never go through
|
|
930
|
+
// runCrossNodeSetup). In-memory only: a close in the SAME process (the common case) is covered; a close
|
|
931
|
+
// after a daemon restart falls back to the pre-fix behavior (deferred hardening: persist on the row).
|
|
932
|
+
const crossNodeBrokerBySession = new Map();
|
|
933
|
+
/**
|
|
934
|
+
* Resolve `owningNodeId` through the signed manifest and run the session_request over a transient
|
|
935
|
+
* visiting connection there. A node id that doesn't resolve in the roster is a hard error
|
|
936
|
+
* (discovery_node_unresolvable) — never a dial to an unvalidated endpoint.
|
|
937
|
+
*/
|
|
938
|
+
async function runCrossNodeSetup(agentName, agentKeyProvider, agentPubkeyHex, owningNodeId, targetHex, sr, correlationId) {
|
|
939
|
+
const roster = await resolveConsortiumRoster();
|
|
940
|
+
const target = roster?.find((e) => e.nodeId === owningNodeId) ?? null;
|
|
941
|
+
if (!target) {
|
|
942
|
+
// Manifest MISS — the node genuinely isn't in the signed roster. A hard, non-retryable cause
|
|
943
|
+
// (distinct from a reachable-but-unconnectable node below).
|
|
944
|
+
logger.warn("session.crossnode.failed", { agentName, brokerNode: owningNodeId, reason: "discovery_node_unresolvable", correlationId });
|
|
945
|
+
return { ok: false, reason: "discovery_node_unresolvable", guidance: `The counterparty's home node (${owningNodeId}) is not in the signed consortium manifest. It may have left the consortium.` };
|
|
946
|
+
}
|
|
947
|
+
logger.info("session.crossnode.initiated", { agentName, brokerNode: owningNodeId, correlationId });
|
|
948
|
+
const visiting = openVisitingConnection(agentName, agentKeyProvider, agentPubkeyHex, { peerId: target.peerId, multiaddr: target.multiaddr }, correlationId, owningNodeId);
|
|
949
|
+
// Track the release reason across the finally (result is block-scoped in the try).
|
|
950
|
+
let releaseReason = "failure";
|
|
951
|
+
try {
|
|
952
|
+
if (!(await waitForSignalingConnected(visiting.mgr, 10_000))) {
|
|
953
|
+
// The node RESOLVED but the visiting dial/auth didn't complete in time — a TRANSIENT
|
|
954
|
+
// cross-region condition (latency, blip, cold handshake), NOT a manifest miss. Distinct
|
|
955
|
+
// reason, and the caller's retry loop treats it as retryable (re-discover → retry).
|
|
956
|
+
logger.warn("session.crossnode.failed", { agentName, brokerNode: owningNodeId, reason: "visiting_connection_unreachable", correlationId });
|
|
957
|
+
return { ok: false, reason: "visiting_connection_unreachable", guidance: `Could not establish a visiting connection to the counterparty's home node (${owningNodeId}) within 10s. Retry.` };
|
|
958
|
+
}
|
|
959
|
+
const result = await runSessionRequestOverSignaling(visiting.mgr, targetHex, sr, correlationId, agentName);
|
|
960
|
+
if (result.ok) {
|
|
961
|
+
releaseReason = "handoff-complete";
|
|
962
|
+
// Fix #1: remember the broker for this session so cello_close_session can reconnect to complete the seal.
|
|
963
|
+
const brokerSessionIdHex = Buffer.from(result.assignment.session_id).toString("hex");
|
|
964
|
+
crossNodeBrokerBySession.set(`${agentName}:${brokerSessionIdHex}`, owningNodeId);
|
|
965
|
+
logger.info("session.crossnode.established", { agentName, brokerNode: owningNodeId, correlationId });
|
|
966
|
+
}
|
|
967
|
+
else {
|
|
968
|
+
logger.warn("session.crossnode.failed", { agentName, brokerNode: owningNodeId, reason: result.reason, correlationId });
|
|
969
|
+
}
|
|
970
|
+
return result;
|
|
971
|
+
}
|
|
972
|
+
finally {
|
|
973
|
+
await visiting.stop(releaseReason);
|
|
974
|
+
}
|
|
975
|
+
}
|
|
752
976
|
const resolvedSessionNegotiator = sessionNegotiator ?? {
|
|
753
977
|
negotiate: async (ctx) => {
|
|
754
978
|
const kp = keyProviders.get(ctx.agentName);
|
|
@@ -791,62 +1015,96 @@ export async function startDaemon(config) {
|
|
|
791
1015
|
guidance: `Agent '${ctx.agentName}' could not establish its directory signaling stream within 10s. Check CELLO_DIRECTORY_URL and that the directory is reachable, then retry.`,
|
|
792
1016
|
};
|
|
793
1017
|
}
|
|
794
|
-
// Single-flight claimed
|
|
795
|
-
//
|
|
1018
|
+
// Single-flight claimed → the discover-first flow (one discovery + up to 3 session_request
|
|
1019
|
+
// attempts, possibly over a transient visiting connection) runs under ONE slot; released in
|
|
1020
|
+
// finally. A single directory-side echoed request id would allow true concurrency later.
|
|
796
1021
|
negotiationInProgress.add(ctx.agentName);
|
|
797
|
-
let resolveFrame;
|
|
798
|
-
const pending = new Promise((r) => {
|
|
799
|
-
resolveFrame = r;
|
|
800
|
-
});
|
|
801
|
-
const unregister = signaling.registerInboundHandler((frame) => {
|
|
802
|
-
const t = frame["type"];
|
|
803
|
-
if (t === "session_assignment" || t === "session_request_error")
|
|
804
|
-
resolveFrame(frame);
|
|
805
|
-
});
|
|
806
1022
|
try {
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
1023
|
+
// DISCOVER FIRST. Because identity + presence are fully replicated, the agent's OWN home
|
|
1024
|
+
// stream answers "where is the target?" authoritatively-enough (advisory — the target node's
|
|
1025
|
+
// live #streams check stays the authority; a stale answer just triggers the retry below).
|
|
1026
|
+
const homeNodeId = signaling.currentDirectoryNodeId;
|
|
1027
|
+
const backoffs = [1_000, 3_000]; // between attempts — covers re-home + replication lag
|
|
1028
|
+
const MAX_ATTEMPTS = 3;
|
|
1029
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
1030
|
+
const disc = await runDiscoveryLookup(signaling, targetHex, 5_000, ctx.correlationId);
|
|
1031
|
+
// TRANSPORT: the home stream is down — retry (it may reconnect); local-only fallback would
|
|
1032
|
+
// fail the same way. Exhausted → the TRUTHFUL transport reason, never a false "offline".
|
|
1033
|
+
if (disc.kind === "send_failed") {
|
|
1034
|
+
logger.warn("session.discovery.send_failed", { agentName: ctx.agentName, attempt, reason: disc.reason, correlationId: ctx.correlationId });
|
|
1035
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
1036
|
+
await sleepMs(backoffs[attempt - 1]);
|
|
1037
|
+
continue;
|
|
1038
|
+
}
|
|
1039
|
+
return { ok: false, reason: "directory_unreachable", guidance: "The home directory stream is not connected, so the counterparty's location could not be looked up. Check cello status (directory_signaling), then retry." };
|
|
1040
|
+
}
|
|
1041
|
+
// NO REPLY: an old directory (predates discovery) OR a slow/dropped reply on a new one. Retry;
|
|
1042
|
+
// fall back to today's local-only behavior ONLY as a last resort (after the retries), so a
|
|
1043
|
+
// single dropped reply no longer misroutes a reachable cross-node peer to the home node.
|
|
1044
|
+
if (disc.kind === "timeout") {
|
|
1045
|
+
logger.warn("session.discovery.no_reply", { agentName: ctx.agentName, attempt, correlationId: ctx.correlationId });
|
|
1046
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
1047
|
+
await sleepMs(backoffs[attempt - 1]);
|
|
1048
|
+
continue;
|
|
1049
|
+
}
|
|
1050
|
+
logger.info("session.discovery.unsupported_fallback", { agentName: ctx.agentName, correlationId: ctx.correlationId });
|
|
1051
|
+
return await runSessionRequestOverSignaling(signaling, targetHex, sr, ctx.correlationId, ctx.agentName);
|
|
1052
|
+
}
|
|
1053
|
+
// DIRECTORY-SIDE lookup fault (DB error / malformed reply): RETRYABLE — but a DIRECTORY fault,
|
|
1054
|
+
// reported truthfully as directory_unreachable, NEVER as the counterparty being offline.
|
|
1055
|
+
if (disc.kind === "error" || disc.kind === "malformed") {
|
|
1056
|
+
logger.info("directory.discovery.lookup.retry", { agentName: ctx.agentName, attempt, kind: disc.kind, correlationId: ctx.correlationId });
|
|
1057
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
1058
|
+
await sleepMs(backoffs[attempt - 1]);
|
|
1059
|
+
continue;
|
|
1060
|
+
}
|
|
1061
|
+
return { ok: false, reason: "directory_unreachable", guidance: "The directory could not resolve the counterparty's location (a directory-side lookup error) after several attempts. Retry shortly." };
|
|
1062
|
+
}
|
|
1063
|
+
// A 3-state RESULT. Route it (pure classifier).
|
|
1064
|
+
const action = classifyOnlineResult(disc.state, disc.owningNodeIds, homeNodeId);
|
|
1065
|
+
// State 3: no such agent — a bad address, not a transient outage. NO retry (distinct code).
|
|
1066
|
+
if (action.kind === "unknown_agent") {
|
|
1067
|
+
return { ok: false, reason: "unknown_agent", guidance: "No agent is registered under that public key — the counterparty address is unknown. Verify the pubkey with the counterparty's operator." };
|
|
1068
|
+
}
|
|
1069
|
+
// State 2: known but offline. Definitive — NO retry storm.
|
|
1070
|
+
if (action.kind === "offline") {
|
|
1071
|
+
return { ok: false, reason: "counterparty_offline", guidance: "The counterparty exists but is not currently online. Have its operator bring it online (cello_status), then retry." };
|
|
1072
|
+
}
|
|
1073
|
+
// Online but no owner named — transient, re-discover.
|
|
1074
|
+
if (action.kind === "retry") {
|
|
1075
|
+
logger.info("directory.discovery.lookup.retry", { agentName: ctx.agentName, attempt, kind: "online_no_owner", correlationId: ctx.correlationId });
|
|
1076
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
1077
|
+
await sleepMs(backoffs[attempt - 1]);
|
|
1078
|
+
continue;
|
|
1079
|
+
}
|
|
1080
|
+
return { ok: false, reason: "counterparty_offline", guidance: "The directory reported the counterparty online but named no home node. Retry shortly." };
|
|
1081
|
+
}
|
|
1082
|
+
const result = action.kind === "same_node"
|
|
1083
|
+
// SAME-NODE: the target is on the node we're already connected to. The existing path runs
|
|
1084
|
+
// unchanged — ZERO visiting connections, ZERO new frames beyond the one discovery_lookup.
|
|
1085
|
+
? await runSessionRequestOverSignaling(signaling, targetHex, sr, ctx.correlationId, ctx.agentName)
|
|
1086
|
+
// CROSS-NODE: reach into the target's home over a transient visiting connection.
|
|
1087
|
+
: await runCrossNodeSetup(ctx.agentName, kp, agentRec.pubkey, action.owningNodeId, targetHex, sr, ctx.correlationId);
|
|
1088
|
+
// RETRY triggers: (a) the broker reported target_offline (stale replicated presence or a
|
|
1089
|
+
// re-home between discovery and the request); (b) a transient visiting-connection failure
|
|
1090
|
+
// (cross-region blip). Both re-discover → retry, bounded.
|
|
1091
|
+
if (!result.ok && (result.reason === "target_offline" || result.reason === "visiting_connection_unreachable")) {
|
|
1092
|
+
logger.info("session.crossnode.stale_discovery_retry", { agentName: ctx.agentName, attempt, reason: result.reason, correlationId: ctx.correlationId });
|
|
1093
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
1094
|
+
await sleepMs(backoffs[attempt - 1]);
|
|
1095
|
+
continue;
|
|
1096
|
+
}
|
|
1097
|
+
// Exhausted: keep the truthful transient-connection reason; a stale target_offline surfaces
|
|
1098
|
+
// as counterparty_offline (state 2).
|
|
1099
|
+
return result.reason === "visiting_connection_unreachable"
|
|
1100
|
+
? { ok: false, reason: "visiting_connection_unreachable", guidance: "The counterparty's home node was reachable at discovery but its visiting connection could not be established after several attempts. Retry shortly." }
|
|
1101
|
+
: { ok: false, reason: "counterparty_offline", guidance: "The counterparty was online at discovery but not reachable at its home node after several attempts (it may be re-homing). Retry shortly." };
|
|
1102
|
+
}
|
|
1103
|
+
return result;
|
|
840
1104
|
}
|
|
841
|
-
|
|
842
|
-
agentName: ctx.agentName,
|
|
843
|
-
correlationId: ctx.correlationId,
|
|
844
|
-
signatureType: assignment.signature_type,
|
|
845
|
-
});
|
|
846
|
-
return { ok: true, assignment };
|
|
1105
|
+
return { ok: false, reason: "counterparty_offline", guidance: "Session establishment did not succeed after several attempts. Retry shortly." };
|
|
847
1106
|
}
|
|
848
1107
|
finally {
|
|
849
|
-
unregister();
|
|
850
1108
|
negotiationInProgress.delete(ctx.agentName);
|
|
851
1109
|
}
|
|
852
1110
|
},
|
|
@@ -2362,7 +2620,43 @@ export async function startDaemon(config) {
|
|
|
2362
2620
|
if (record.status === "active") {
|
|
2363
2621
|
sealInterruptedInProgress.add(sealKey(record.agent_name, sessionId));
|
|
2364
2622
|
const correlationId = randomUUID();
|
|
2623
|
+
// Fix #1 (cross-node seal-liveness): if this session was brokered by another node, the seal_verified
|
|
2624
|
+
// + session_sealed frames are pushed by that BROKER — but the initiator released its visiting
|
|
2625
|
+
// connection after setup, so on the home stream they never arrive and close times out. Re-open a
|
|
2626
|
+
// transient visiting connection to the broker (seal-capable — openVisitingConnection now wires the
|
|
2627
|
+
// seal handlers) for the duration of the seal, then release it in finally. Same-node sessions have
|
|
2628
|
+
// no entry here and use the home stream unchanged.
|
|
2629
|
+
let sealBrokerConn = null;
|
|
2365
2630
|
try {
|
|
2631
|
+
const brokerNodeForSeal = crossNodeBrokerBySession.get(`${record.agent_name}:${sessionId}`);
|
|
2632
|
+
if (brokerNodeForSeal) {
|
|
2633
|
+
const sealKp = keyProviders.get(record.agent_name);
|
|
2634
|
+
if (!sealKp) {
|
|
2635
|
+
// Fallback-finder #1: never skip the cross-node reconnect silently — without a key provider
|
|
2636
|
+
// we cannot open the broker connection, so the seal will revert to the pre-fix timeout. Log
|
|
2637
|
+
// WHY so it is not indistinguishable from a normal counterparty-didn't-close timeout.
|
|
2638
|
+
logger.warn("session.seal.broker.no_keyprovider", { agentName: record.agent_name, brokerNode: brokerNodeForSeal, correlationId });
|
|
2639
|
+
}
|
|
2640
|
+
else {
|
|
2641
|
+
const sealPubHex = Buffer.from(await sealKp.getPublicKey()).toString("hex");
|
|
2642
|
+
const roster = await resolveConsortiumRoster();
|
|
2643
|
+
const brokerTarget = roster?.find((e) => e.nodeId === brokerNodeForSeal) ?? null;
|
|
2644
|
+
if (!brokerTarget) {
|
|
2645
|
+
logger.warn("session.seal.broker.unresolved", { agentName: record.agent_name, brokerNode: brokerNodeForSeal, correlationId });
|
|
2646
|
+
}
|
|
2647
|
+
else {
|
|
2648
|
+
const conn = openVisitingConnection(record.agent_name, sealKp, sealPubHex, { peerId: brokerTarget.peerId, multiaddr: brokerTarget.multiaddr }, correlationId, brokerNodeForSeal);
|
|
2649
|
+
if (await waitForSignalingConnected(conn.mgr, 10_000)) {
|
|
2650
|
+
sealBrokerConn = conn;
|
|
2651
|
+
logger.info("session.seal.broker.reconnected", { agentName: record.agent_name, brokerNode: brokerNodeForSeal, correlationId });
|
|
2652
|
+
}
|
|
2653
|
+
else {
|
|
2654
|
+
await conn.stop("seal-broker-unreachable");
|
|
2655
|
+
logger.warn("session.seal.broker.unreachable", { agentName: record.agent_name, brokerNode: brokerNodeForSeal, correlationId });
|
|
2656
|
+
}
|
|
2657
|
+
}
|
|
2658
|
+
}
|
|
2659
|
+
}
|
|
2366
2660
|
// M7 DOD-SPINE-7: relay-mediated bilateral seal. Submit our SEAL ctrl leaf to the
|
|
2367
2661
|
// relay witness; when the counterparty ALSO closes, the relay's #maybeProcessSeal
|
|
2368
2662
|
// fires → directory processSeal rebuilds + verifies the signed chain → FROST
|
|
@@ -2401,6 +2695,7 @@ export async function startDaemon(config) {
|
|
|
2401
2695
|
pendingSealWaiters.delete(sealKey(record.agent_name, sessionId));
|
|
2402
2696
|
if (sealedCompletion !== null) {
|
|
2403
2697
|
logger.info("session.seal.completed", { sessionId, sealedRoot: sealedCompletion.rootHex, role: "bilateral", correlationId });
|
|
2698
|
+
crossNodeBrokerBySession.delete(`${record.agent_name}:${sessionId}`); // Fix #1 review: evict on terminal seal success (a FAILED close keeps the entry so a retry can still reconnect).
|
|
2404
2699
|
// M7-SESSION-004 (AC-006): return the legibility certificate on the seal completion so
|
|
2405
2700
|
// a reader gets it on the same surface that proves the seal — receipt-not-assent,
|
|
2406
2701
|
// per-party frontiers, attestation modes, and final_message.answered.
|
|
@@ -2476,6 +2771,7 @@ export async function startDaemon(config) {
|
|
|
2476
2771
|
pendingUnilateralWaiters.delete(sessionId);
|
|
2477
2772
|
if (uniResult.ok) {
|
|
2478
2773
|
logger.info("session.seal.completed", { sessionId, sealedRoot: uniResult.sealedRootHex, role: "unilateral", correlationId });
|
|
2774
|
+
crossNodeBrokerBySession.delete(`${record.agent_name}:${sessionId}`); // Fix #1 review: evict on terminal seal success (a FAILED close keeps the entry so a retry can still reconnect).
|
|
2479
2775
|
// M8B FINDING-3 (cascade-2): return the legibility certificate inline — same shape as the
|
|
2480
2776
|
// bilateral close (AC-006) — so the operator gets the receipt on the surface that proves
|
|
2481
2777
|
// the seal, not just a bare root. It is also persisted (above) for cello_get_sealed_receipt.
|
|
@@ -2504,6 +2800,15 @@ export async function startDaemon(config) {
|
|
|
2504
2800
|
};
|
|
2505
2801
|
}
|
|
2506
2802
|
finally {
|
|
2803
|
+
// Fix #1: release the transient broker seal-connection (best-effort; the seal result stands).
|
|
2804
|
+
if (sealBrokerConn) {
|
|
2805
|
+
try {
|
|
2806
|
+
await sealBrokerConn.stop("seal-complete");
|
|
2807
|
+
}
|
|
2808
|
+
catch (err) {
|
|
2809
|
+
logger.warn("session.seal.broker.release_failed", { sessionId, reason: err instanceof Error ? err.message : String(err) });
|
|
2810
|
+
}
|
|
2811
|
+
}
|
|
2507
2812
|
sealInterruptedInProgress.delete(sealKey(record.agent_name, sessionId));
|
|
2508
2813
|
}
|
|
2509
2814
|
}
|