@cello-protocol/daemon 0.0.27 → 0.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cross-node-negotiation.d.ts +44 -0
- package/dist/cross-node-negotiation.d.ts.map +1 -0
- package/dist/cross-node-negotiation.js +34 -0
- package/dist/cross-node-negotiation.js.map +1 -0
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +274 -52
- package/dist/daemon.js.map +1 -1
- package/dist/db-identity-store.d.ts +2 -0
- package/dist/db-identity-store.d.ts.map +1 -1
- package/dist/db-identity-store.js +24 -1
- package/dist/db-identity-store.js.map +1 -1
- package/dist/registration-manager.d.ts.map +1 -1
- package/dist/registration-manager.js +16 -12
- package/dist/registration-manager.js.map +1 -1
- package/dist/registration-persistence.d.ts +6 -0
- package/dist/registration-persistence.d.ts.map +1 -1
- package/dist/registration-persistence.js +1 -0
- package/dist/registration-persistence.js.map +1 -1
- package/dist/session-assignment-parser.d.ts +17 -0
- package/dist/session-assignment-parser.d.ts.map +1 -1
- package/dist/session-assignment-parser.js +20 -2
- package/dist/session-assignment-parser.js.map +1 -1
- package/dist/session-ceremony.d.ts.map +1 -1
- package/dist/session-ceremony.js +14 -2
- package/dist/session-ceremony.js.map +1 -1
- package/dist/signaling-connect.d.ts +9 -0
- package/dist/signaling-connect.d.ts.map +1 -1
- package/dist/signaling-connect.js +2 -0
- package/dist/signaling-connect.js.map +1 -1
- package/package.json +3 -3
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-node session establishment — discovery outcome types + the pure online-result classifier
|
|
3
|
+
* (Story B, item 2).
|
|
4
|
+
*
|
|
5
|
+
* The daemon's discovery step produces a DiscoveryOutcome; the retry loop handles the transport /
|
|
6
|
+
* directory-fault / no-reply variants attempt-aware (each with its own log + terminal reason), and
|
|
7
|
+
* defers the actual 3-state routing decision to classifyOnlineResult — kept pure so the branch and
|
|
8
|
+
* error-code mapping are unit-testable in isolation (mirrors the directory-side resolveDiscoveryState).
|
|
9
|
+
*
|
|
10
|
+
* Design: docs/planning/discussion_logs/2026-07-04_1730_cross-node-session-topology.md §"Item 2".
|
|
11
|
+
*/
|
|
12
|
+
export type DiscoveryOutcome = {
|
|
13
|
+
kind: "result";
|
|
14
|
+
state: "online" | "offline" | "unknown_agent";
|
|
15
|
+
owningNodeIds: string[];
|
|
16
|
+
} | {
|
|
17
|
+
kind: "error";
|
|
18
|
+
reason: string;
|
|
19
|
+
} | {
|
|
20
|
+
kind: "malformed";
|
|
21
|
+
} | {
|
|
22
|
+
kind: "timeout";
|
|
23
|
+
} | {
|
|
24
|
+
kind: "send_failed";
|
|
25
|
+
reason: string;
|
|
26
|
+
};
|
|
27
|
+
export type ResultAction = {
|
|
28
|
+
kind: "same_node";
|
|
29
|
+
} | {
|
|
30
|
+
kind: "cross_node";
|
|
31
|
+
owningNodeId: string;
|
|
32
|
+
} | {
|
|
33
|
+
kind: "unknown_agent";
|
|
34
|
+
} | {
|
|
35
|
+
kind: "offline";
|
|
36
|
+
} | {
|
|
37
|
+
kind: "retry";
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* Route a 3-state discovery RESULT into the next action. Pure. Retry counting, backoff, transport /
|
|
41
|
+
* directory-fault / timeout handling, and the actual I/O live in the daemon loop.
|
|
42
|
+
*/
|
|
43
|
+
export declare function classifyOnlineResult(state: "online" | "offline" | "unknown_agent", owningNodeIds: string[], homeNodeId: string | null): ResultAction;
|
|
44
|
+
//# sourceMappingURL=cross-node-negotiation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cross-node-negotiation.d.ts","sourceRoot":"","sources":["../src/cross-node-negotiation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,MAAM,gBAAgB,GAExB;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,QAAQ,GAAG,SAAS,GAAG,eAAe,CAAC;IAAC,aAAa,EAAE,MAAM,EAAE,CAAA;CAAE,GAG1F;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAEjC;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GAGrB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GAGnB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5C,MAAM,MAAM,YAAY,GACpB;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GACrB;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GAC5C;IAAE,IAAI,EAAE,eAAe,CAAA;CAAE,GACzB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,CAAC;AAEtB;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,QAAQ,GAAG,SAAS,GAAG,eAAe,EAC7C,aAAa,EAAE,MAAM,EAAE,EACvB,UAAU,EAAE,MAAM,GAAG,IAAI,GACxB,YAAY,CAcd"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-node session establishment — discovery outcome types + the pure online-result classifier
|
|
3
|
+
* (Story B, item 2).
|
|
4
|
+
*
|
|
5
|
+
* The daemon's discovery step produces a DiscoveryOutcome; the retry loop handles the transport /
|
|
6
|
+
* directory-fault / no-reply variants attempt-aware (each with its own log + terminal reason), and
|
|
7
|
+
* defers the actual 3-state routing decision to classifyOnlineResult — kept pure so the branch and
|
|
8
|
+
* error-code mapping are unit-testable in isolation (mirrors the directory-side resolveDiscoveryState).
|
|
9
|
+
*
|
|
10
|
+
* Design: docs/planning/discussion_logs/2026-07-04_1730_cross-node-session-topology.md §"Item 2".
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Route a 3-state discovery RESULT into the next action. Pure. Retry counting, backoff, transport /
|
|
14
|
+
* directory-fault / timeout handling, and the actual I/O live in the daemon loop.
|
|
15
|
+
*/
|
|
16
|
+
export function classifyOnlineResult(state, owningNodeIds, homeNodeId) {
|
|
17
|
+
if (state === "unknown_agent")
|
|
18
|
+
return { kind: "unknown_agent" };
|
|
19
|
+
if (state === "offline")
|
|
20
|
+
return { kind: "offline" };
|
|
21
|
+
// state === "online": pick the owning node (list-valued; length 1 until the k>1 homing knob).
|
|
22
|
+
const owningNodeId = owningNodeIds[0];
|
|
23
|
+
// Online but no owner named is malformed — treat as transiently unresolved, retry (never dial a
|
|
24
|
+
// fabricated node).
|
|
25
|
+
if (!owningNodeId)
|
|
26
|
+
return { kind: "retry" };
|
|
27
|
+
// Same-node shortcut requires KNOWING our own node id (step-6 identity). Without it (homeNodeId
|
|
28
|
+
// null), we cannot prove co-location, so we take the cross-node path (which validates the owning
|
|
29
|
+
// node through the signed manifest before dialing).
|
|
30
|
+
if (homeNodeId !== null && owningNodeId === homeNodeId)
|
|
31
|
+
return { kind: "same_node" };
|
|
32
|
+
return { kind: "cross_node", owningNodeId };
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=cross-node-negotiation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cross-node-negotiation.js","sourceRoot":"","sources":["../src/cross-node-negotiation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAwBH;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAA6C,EAC7C,aAAuB,EACvB,UAAyB;IAEzB,IAAI,KAAK,KAAK,eAAe;QAAE,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;IAChE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;IAEpD,8FAA8F;IAC9F,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;IACtC,gGAAgG;IAChG,oBAAoB;IACpB,IAAI,CAAC,YAAY;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC5C,gGAAgG;IAChG,iGAAiG;IACjG,oDAAoD;IACpD,IAAI,UAAU,KAAK,IAAI,IAAI,YAAY,KAAK,UAAU;QAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;IACrF,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;AAC9C,CAAC"}
|
package/dist/daemon.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../src/daemon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAKH,OAAO,KAAK,EACV,YAAY,EACZ,oBAAoB,EASrB,MAAM,YAAY,CAAC;AAIpB,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAM/D,OAAO,EAAoD,KAAK,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAsB7G,OAAO,KAAK,EAAE,kBAAkB,EAA+C,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../src/daemon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAKH,OAAO,KAAK,EACV,YAAY,EACZ,oBAAoB,EASrB,MAAM,YAAY,CAAC;AAIpB,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAM/D,OAAO,EAAoD,KAAK,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAsB7G,OAAO,KAAK,EAAE,kBAAkB,EAA+C,MAAM,yBAAyB,CAAC;AAO/G,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,2BAA2B,CAAC;AA0InF,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,SAAS,IAAI,oBAAoB,CAAC;IAClC;;;;OAIG;IACH,qBAAqB,IAAI,kBAAkB,CAAC;IAC5C;;;;;;;;OAQG;IACH,gBAAgB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrC;;;;OAIG;IACH,oBAAoB,IAAI,kBAAkB,CAAC;IAC3C;;;OAGG;IACH,iBAAiB,IAAI,eAAe,CAAC;CACtC;AAyCD,wBAAsB,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAs/I7E"}
|
package/dist/daemon.js
CHANGED
|
@@ -51,7 +51,8 @@ import { upgradeAbsentToRecovered, hasAbsentParticipant } from "./seal-receipt-u
|
|
|
51
51
|
import { MAX_CONTENT_BYTES, computeGenesisPrevRoot, buildAgentRevocationTbs } from "@cello-protocol/protocol-types";
|
|
52
52
|
import { resolveCelloEnv, createTransportSelector, isProductionVariant, } from "./transport-composition.js";
|
|
53
53
|
import { selectAdvertisedAddress } from "./transport-selector.js";
|
|
54
|
-
import { parseSessionAssignment, sessionRequestErrorReason } from "./session-assignment-parser.js";
|
|
54
|
+
import { parseSessionAssignment, sessionRequestErrorReason, parseDiscoveryLookupResult, discoveryLookupErrorReason } from "./session-assignment-parser.js";
|
|
55
|
+
import { classifyOnlineResult } from "./cross-node-negotiation.js";
|
|
55
56
|
import { wireSessionCeremonyHandler, wireSessionOfferHandler, wireSealCeremonyHandler, verifyUnilateralCertificate, verifyBilateralSealCertificate, runAgentRefresh } from "./session-ceremony.js";
|
|
56
57
|
import { reDeriveFrontiers, findInflatedFrontier, checkUnilateralFrontier } from "./seal-frontier-verify.js";
|
|
57
58
|
import { LocalAutoNatStub } from "@cello-protocol/transport";
|
|
@@ -749,6 +750,193 @@ export async function startDaemon(config) {
|
|
|
749
750
|
// agent concurrency is unaffected (separate streams). A directory-side echoed request id
|
|
750
751
|
// would allow true concurrency later; this is the correct minimum.
|
|
751
752
|
const negotiationInProgress = new Set();
|
|
753
|
+
// ─── Cross-node session establishment (Story B, item 2) ──────────────────────
|
|
754
|
+
// Helpers the negotiator composes: discover the target's home from replicated presence, then run
|
|
755
|
+
// the session_request over the RIGHT connection — the existing home stream when same-node, or a
|
|
756
|
+
// transient VISITING connection into the target's home node when cross-node. Directories never talk
|
|
757
|
+
// to each other; the client spans nodes on demand.
|
|
758
|
+
const sleepMs = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
759
|
+
/**
|
|
760
|
+
* Issue a discovery_lookup on `signaling` and await the 3-state answer (bounded). Distinguishes,
|
|
761
|
+
* for the caller's retry logic and truthful error surface:
|
|
762
|
+
* - send_failed (home stream down — TRANSPORT, never "old directory"),
|
|
763
|
+
* - timeout (no reply — old directory OR a slow/dropped reply on a new one; retry, fall back last),
|
|
764
|
+
* - error (directory DB fault — retryable, a DIRECTORY fault, not the counterparty being offline),
|
|
765
|
+
* - malformed (a reply that didn't parse — protocol anomaly, retryable, surfaced distinctly),
|
|
766
|
+
* - result (the 3-state answer).
|
|
767
|
+
* Emits the mandated observability events: directory.discovery.lookup on a result,
|
|
768
|
+
* directory.discovery.lookup.failed on a directory error / malformed reply.
|
|
769
|
+
*/
|
|
770
|
+
async function runDiscoveryLookup(signaling, targetHex, timeoutMs, correlationId) {
|
|
771
|
+
let resolveFrame;
|
|
772
|
+
const pending = new Promise((r) => { resolveFrame = r; });
|
|
773
|
+
const unregister = signaling.registerInboundHandler((frame) => {
|
|
774
|
+
const t = frame["type"];
|
|
775
|
+
if (t === "discovery_lookup_result" || t === "discovery_lookup_error")
|
|
776
|
+
resolveFrame(frame);
|
|
777
|
+
});
|
|
778
|
+
try {
|
|
779
|
+
const sent = await signaling.sendRaw({
|
|
780
|
+
type: "discovery_lookup",
|
|
781
|
+
target_pubkey: new Uint8Array(Buffer.from(targetHex, "hex")),
|
|
782
|
+
});
|
|
783
|
+
if (!sent.ok) {
|
|
784
|
+
// The home stream is down — cannot look up (and today's local-only fallback would fail the
|
|
785
|
+
// same way). Surface the real transport reason, not an "old directory" misdiagnosis.
|
|
786
|
+
return { kind: "send_failed", reason: sent.reason ?? "signaling_unavailable" };
|
|
787
|
+
}
|
|
788
|
+
let timer;
|
|
789
|
+
const timeoutP = new Promise((r) => { timer = setTimeout(() => r({ type: "__timeout__" }), timeoutMs); });
|
|
790
|
+
const frame = await Promise.race([pending, timeoutP]);
|
|
791
|
+
clearTimeout(timer);
|
|
792
|
+
if (frame["type"] === "__timeout__")
|
|
793
|
+
return { kind: "timeout" };
|
|
794
|
+
if (frame["type"] === "discovery_lookup_error") {
|
|
795
|
+
const reason = discoveryLookupErrorReason(frame);
|
|
796
|
+
logger.warn("directory.discovery.lookup.failed", { target: targetHex.slice(0, 16), reason, correlationId });
|
|
797
|
+
return { kind: "error", reason };
|
|
798
|
+
}
|
|
799
|
+
const parsed = parseDiscoveryLookupResult(frame);
|
|
800
|
+
if (!parsed) {
|
|
801
|
+
// A reply came back but did not parse — a protocol/version anomaly on a directory that DID
|
|
802
|
+
// respond. Surface it distinctly from a clean directory DB error, and never as availability.
|
|
803
|
+
logger.warn("directory.discovery.lookup.failed", { target: targetHex.slice(0, 16), reason: "malformed_reply", correlationId });
|
|
804
|
+
return { kind: "malformed" };
|
|
805
|
+
}
|
|
806
|
+
logger.info("directory.discovery.lookup", {
|
|
807
|
+
target: targetHex.slice(0, 16),
|
|
808
|
+
state: parsed.state,
|
|
809
|
+
owningNode: parsed.owningNodeIds[0] ?? null,
|
|
810
|
+
correlationId,
|
|
811
|
+
});
|
|
812
|
+
return { kind: "result", state: parsed.state, owningNodeIds: parsed.owningNodeIds };
|
|
813
|
+
}
|
|
814
|
+
finally {
|
|
815
|
+
unregister();
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
/** Send a session_request over `signaling` and await the assignment / error (the extracted core). */
|
|
819
|
+
async function runSessionRequestOverSignaling(signaling, targetHex, sr, correlationId, agentName) {
|
|
820
|
+
let resolveFrame;
|
|
821
|
+
const pending = new Promise((r) => { resolveFrame = r; });
|
|
822
|
+
const unregister = signaling.registerInboundHandler((frame) => {
|
|
823
|
+
const t = frame["type"];
|
|
824
|
+
if (t === "session_assignment" || t === "session_request_error")
|
|
825
|
+
resolveFrame(frame);
|
|
826
|
+
});
|
|
827
|
+
try {
|
|
828
|
+
const sent = await signaling.sendRaw({
|
|
829
|
+
type: "session_request",
|
|
830
|
+
target_pubkey: new Uint8Array(Buffer.from(targetHex, "hex")),
|
|
831
|
+
initiator_session_peer_id: sr.peerId,
|
|
832
|
+
initiator_session_addrs: sr.addrs,
|
|
833
|
+
wants_session_offer: true,
|
|
834
|
+
});
|
|
835
|
+
if (!sent.ok) {
|
|
836
|
+
return { ok: false, reason: sent.reason ?? "directory_unreachable", guidance: sent.guidance ?? "Could not send session_request over the directory signaling stream." };
|
|
837
|
+
}
|
|
838
|
+
let timer;
|
|
839
|
+
const timeoutP = new Promise((r) => { timer = setTimeout(() => r({ type: "__timeout__" }), 30_000); });
|
|
840
|
+
const frame = await Promise.race([pending, timeoutP]);
|
|
841
|
+
clearTimeout(timer);
|
|
842
|
+
if (frame["type"] === "__timeout__") {
|
|
843
|
+
return { ok: false, reason: "timeout", guidance: "The directory did not return a session assignment within 30s. Retry once cello status shows directory_signaling connected." };
|
|
844
|
+
}
|
|
845
|
+
if (frame["type"] === "session_request_error") {
|
|
846
|
+
const reason = sessionRequestErrorReason(frame);
|
|
847
|
+
return { ok: false, reason, guidance: `The directory refused the session request (${reason}). Ensure the counterparty is registered and online.` };
|
|
848
|
+
}
|
|
849
|
+
const raw = frame["assignment"];
|
|
850
|
+
const assignment = raw ? parseSessionAssignment(raw) : null;
|
|
851
|
+
if (!assignment) {
|
|
852
|
+
return { ok: false, reason: "assignment_parse_failed", guidance: "The directory's session_assignment was missing or malformed." };
|
|
853
|
+
}
|
|
854
|
+
logger.info("session.negotiate.assignment.received", { agentName, correlationId, signatureType: assignment.signature_type });
|
|
855
|
+
return { ok: true, assignment };
|
|
856
|
+
}
|
|
857
|
+
finally {
|
|
858
|
+
unregister();
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
/**
|
|
862
|
+
* Open a transient VISITING signaling connection into a specific directory node (the target's home)
|
|
863
|
+
* and wire ONLY what a cross-node initiator needs there: the delegated-signer ceremony handler (the
|
|
864
|
+
* broker asks the initiator to co-sign the assignment over THIS connection). Seal + inbound handlers
|
|
865
|
+
* are NOT wired — the seal is client-coordinated over the agent's own roster, and this connection is
|
|
866
|
+
* initiator-only and transient. Caller MUST stop() it after the assignment arrives.
|
|
867
|
+
*/
|
|
868
|
+
function openVisitingConnection(agentName, agentKeyProvider, agentPubkeyHex, endpoint, correlationId, nodeId) {
|
|
869
|
+
let nodeRef = null;
|
|
870
|
+
const connect = createSignalingConnect({
|
|
871
|
+
getDirectoryEndpoint: () => endpoint,
|
|
872
|
+
getAuthIdentity: () => ({ keyProvider: agentKeyProvider, pubkeyHex: agentPubkeyHex }),
|
|
873
|
+
logger,
|
|
874
|
+
challengeVerifier,
|
|
875
|
+
getManifestVersion: () => verifiedManifestVersion,
|
|
876
|
+
visiting: true, // cross-node item 3: the directory must NOT write presence for this connection
|
|
877
|
+
publishNode: (n) => { nodeRef = n; },
|
|
878
|
+
});
|
|
879
|
+
// maxReconnectAttempts: 1 — a transient connection should fail fast, not reconnect-loop forever.
|
|
880
|
+
const mgr = new SignalingManager({ connect, logger, maxReconnectAttempts: 1, maxBackoffMs: 3_000 });
|
|
881
|
+
wireSessionCeremonyHandler({
|
|
882
|
+
agentName,
|
|
883
|
+
persistence: getPersistence(agentName),
|
|
884
|
+
agentPubkeyHex,
|
|
885
|
+
getNode: () => nodeRef,
|
|
886
|
+
getDirectoryEndpoint: getFailoverEndpoint,
|
|
887
|
+
getConsortiumEndpoints: resolveConsortiumRoster,
|
|
888
|
+
signaling: mgr,
|
|
889
|
+
logger,
|
|
890
|
+
});
|
|
891
|
+
logger.info("signaling.visiting.connected", { agentName, node: nodeId, correlationId });
|
|
892
|
+
return {
|
|
893
|
+
mgr,
|
|
894
|
+
stop: async (reason) => {
|
|
895
|
+
logger.info("signaling.visiting.released", { agentName, node: nodeId, reason, correlationId });
|
|
896
|
+
await mgr.stop();
|
|
897
|
+
},
|
|
898
|
+
};
|
|
899
|
+
}
|
|
900
|
+
/**
|
|
901
|
+
* Resolve `owningNodeId` through the signed manifest and run the session_request over a transient
|
|
902
|
+
* visiting connection there. A node id that doesn't resolve in the roster is a hard error
|
|
903
|
+
* (discovery_node_unresolvable) — never a dial to an unvalidated endpoint.
|
|
904
|
+
*/
|
|
905
|
+
async function runCrossNodeSetup(agentName, agentKeyProvider, agentPubkeyHex, owningNodeId, targetHex, sr, correlationId) {
|
|
906
|
+
const roster = await resolveConsortiumRoster();
|
|
907
|
+
const target = roster?.find((e) => e.nodeId === owningNodeId) ?? null;
|
|
908
|
+
if (!target) {
|
|
909
|
+
// Manifest MISS — the node genuinely isn't in the signed roster. A hard, non-retryable cause
|
|
910
|
+
// (distinct from a reachable-but-unconnectable node below).
|
|
911
|
+
logger.warn("session.crossnode.failed", { agentName, brokerNode: owningNodeId, reason: "discovery_node_unresolvable", correlationId });
|
|
912
|
+
return { ok: false, reason: "discovery_node_unresolvable", guidance: `The counterparty's home node (${owningNodeId}) is not in the signed consortium manifest. It may have left the consortium.` };
|
|
913
|
+
}
|
|
914
|
+
logger.info("session.crossnode.initiated", { agentName, brokerNode: owningNodeId, correlationId });
|
|
915
|
+
const visiting = openVisitingConnection(agentName, agentKeyProvider, agentPubkeyHex, { peerId: target.peerId, multiaddr: target.multiaddr }, correlationId, owningNodeId);
|
|
916
|
+
// Track the release reason across the finally (result is block-scoped in the try).
|
|
917
|
+
let releaseReason = "failure";
|
|
918
|
+
try {
|
|
919
|
+
if (!(await waitForSignalingConnected(visiting.mgr, 10_000))) {
|
|
920
|
+
// The node RESOLVED but the visiting dial/auth didn't complete in time — a TRANSIENT
|
|
921
|
+
// cross-region condition (latency, blip, cold handshake), NOT a manifest miss. Distinct
|
|
922
|
+
// reason, and the caller's retry loop treats it as retryable (re-discover → retry).
|
|
923
|
+
logger.warn("session.crossnode.failed", { agentName, brokerNode: owningNodeId, reason: "visiting_connection_unreachable", correlationId });
|
|
924
|
+
return { ok: false, reason: "visiting_connection_unreachable", guidance: `Could not establish a visiting connection to the counterparty's home node (${owningNodeId}) within 10s. Retry.` };
|
|
925
|
+
}
|
|
926
|
+
const result = await runSessionRequestOverSignaling(visiting.mgr, targetHex, sr, correlationId, agentName);
|
|
927
|
+
if (result.ok) {
|
|
928
|
+
releaseReason = "handoff-complete";
|
|
929
|
+
logger.info("session.crossnode.established", { agentName, brokerNode: owningNodeId, correlationId });
|
|
930
|
+
}
|
|
931
|
+
else {
|
|
932
|
+
logger.warn("session.crossnode.failed", { agentName, brokerNode: owningNodeId, reason: result.reason, correlationId });
|
|
933
|
+
}
|
|
934
|
+
return result;
|
|
935
|
+
}
|
|
936
|
+
finally {
|
|
937
|
+
await visiting.stop(releaseReason);
|
|
938
|
+
}
|
|
939
|
+
}
|
|
752
940
|
const resolvedSessionNegotiator = sessionNegotiator ?? {
|
|
753
941
|
negotiate: async (ctx) => {
|
|
754
942
|
const kp = keyProviders.get(ctx.agentName);
|
|
@@ -791,62 +979,96 @@ export async function startDaemon(config) {
|
|
|
791
979
|
guidance: `Agent '${ctx.agentName}' could not establish its directory signaling stream within 10s. Check CELLO_DIRECTORY_URL and that the directory is reachable, then retry.`,
|
|
792
980
|
};
|
|
793
981
|
}
|
|
794
|
-
// Single-flight claimed
|
|
795
|
-
//
|
|
982
|
+
// Single-flight claimed → the discover-first flow (one discovery + up to 3 session_request
|
|
983
|
+
// attempts, possibly over a transient visiting connection) runs under ONE slot; released in
|
|
984
|
+
// finally. A single directory-side echoed request id would allow true concurrency later.
|
|
796
985
|
negotiationInProgress.add(ctx.agentName);
|
|
797
|
-
let resolveFrame;
|
|
798
|
-
const pending = new Promise((r) => {
|
|
799
|
-
resolveFrame = r;
|
|
800
|
-
});
|
|
801
|
-
const unregister = signaling.registerInboundHandler((frame) => {
|
|
802
|
-
const t = frame["type"];
|
|
803
|
-
if (t === "session_assignment" || t === "session_request_error")
|
|
804
|
-
resolveFrame(frame);
|
|
805
|
-
});
|
|
806
986
|
try {
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
987
|
+
// DISCOVER FIRST. Because identity + presence are fully replicated, the agent's OWN home
|
|
988
|
+
// stream answers "where is the target?" authoritatively-enough (advisory — the target node's
|
|
989
|
+
// live #streams check stays the authority; a stale answer just triggers the retry below).
|
|
990
|
+
const homeNodeId = signaling.currentDirectoryNodeId;
|
|
991
|
+
const backoffs = [1_000, 3_000]; // between attempts — covers re-home + replication lag
|
|
992
|
+
const MAX_ATTEMPTS = 3;
|
|
993
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
994
|
+
const disc = await runDiscoveryLookup(signaling, targetHex, 5_000, ctx.correlationId);
|
|
995
|
+
// TRANSPORT: the home stream is down — retry (it may reconnect); local-only fallback would
|
|
996
|
+
// fail the same way. Exhausted → the TRUTHFUL transport reason, never a false "offline".
|
|
997
|
+
if (disc.kind === "send_failed") {
|
|
998
|
+
logger.warn("session.discovery.send_failed", { agentName: ctx.agentName, attempt, reason: disc.reason, correlationId: ctx.correlationId });
|
|
999
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
1000
|
+
await sleepMs(backoffs[attempt - 1]);
|
|
1001
|
+
continue;
|
|
1002
|
+
}
|
|
1003
|
+
return { ok: false, reason: "directory_unreachable", guidance: "The home directory stream is not connected, so the counterparty's location could not be looked up. Check cello status (directory_signaling), then retry." };
|
|
1004
|
+
}
|
|
1005
|
+
// NO REPLY: an old directory (predates discovery) OR a slow/dropped reply on a new one. Retry;
|
|
1006
|
+
// fall back to today's local-only behavior ONLY as a last resort (after the retries), so a
|
|
1007
|
+
// single dropped reply no longer misroutes a reachable cross-node peer to the home node.
|
|
1008
|
+
if (disc.kind === "timeout") {
|
|
1009
|
+
logger.warn("session.discovery.no_reply", { agentName: ctx.agentName, attempt, correlationId: ctx.correlationId });
|
|
1010
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
1011
|
+
await sleepMs(backoffs[attempt - 1]);
|
|
1012
|
+
continue;
|
|
1013
|
+
}
|
|
1014
|
+
logger.info("session.discovery.unsupported_fallback", { agentName: ctx.agentName, correlationId: ctx.correlationId });
|
|
1015
|
+
return await runSessionRequestOverSignaling(signaling, targetHex, sr, ctx.correlationId, ctx.agentName);
|
|
1016
|
+
}
|
|
1017
|
+
// DIRECTORY-SIDE lookup fault (DB error / malformed reply): RETRYABLE — but a DIRECTORY fault,
|
|
1018
|
+
// reported truthfully as directory_unreachable, NEVER as the counterparty being offline.
|
|
1019
|
+
if (disc.kind === "error" || disc.kind === "malformed") {
|
|
1020
|
+
logger.info("directory.discovery.lookup.retry", { agentName: ctx.agentName, attempt, kind: disc.kind, correlationId: ctx.correlationId });
|
|
1021
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
1022
|
+
await sleepMs(backoffs[attempt - 1]);
|
|
1023
|
+
continue;
|
|
1024
|
+
}
|
|
1025
|
+
return { ok: false, reason: "directory_unreachable", guidance: "The directory could not resolve the counterparty's location (a directory-side lookup error) after several attempts. Retry shortly." };
|
|
1026
|
+
}
|
|
1027
|
+
// A 3-state RESULT. Route it (pure classifier).
|
|
1028
|
+
const action = classifyOnlineResult(disc.state, disc.owningNodeIds, homeNodeId);
|
|
1029
|
+
// State 3: no such agent — a bad address, not a transient outage. NO retry (distinct code).
|
|
1030
|
+
if (action.kind === "unknown_agent") {
|
|
1031
|
+
return { ok: false, reason: "unknown_agent", guidance: "No agent is registered under that public key — the counterparty address is unknown. Verify the pubkey with the counterparty's operator." };
|
|
1032
|
+
}
|
|
1033
|
+
// State 2: known but offline. Definitive — NO retry storm.
|
|
1034
|
+
if (action.kind === "offline") {
|
|
1035
|
+
return { ok: false, reason: "counterparty_offline", guidance: "The counterparty exists but is not currently online. Have its operator bring it online (cello_status), then retry." };
|
|
1036
|
+
}
|
|
1037
|
+
// Online but no owner named — transient, re-discover.
|
|
1038
|
+
if (action.kind === "retry") {
|
|
1039
|
+
logger.info("directory.discovery.lookup.retry", { agentName: ctx.agentName, attempt, kind: "online_no_owner", correlationId: ctx.correlationId });
|
|
1040
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
1041
|
+
await sleepMs(backoffs[attempt - 1]);
|
|
1042
|
+
continue;
|
|
1043
|
+
}
|
|
1044
|
+
return { ok: false, reason: "counterparty_offline", guidance: "The directory reported the counterparty online but named no home node. Retry shortly." };
|
|
1045
|
+
}
|
|
1046
|
+
const result = action.kind === "same_node"
|
|
1047
|
+
// SAME-NODE: the target is on the node we're already connected to. The existing path runs
|
|
1048
|
+
// unchanged — ZERO visiting connections, ZERO new frames beyond the one discovery_lookup.
|
|
1049
|
+
? await runSessionRequestOverSignaling(signaling, targetHex, sr, ctx.correlationId, ctx.agentName)
|
|
1050
|
+
// CROSS-NODE: reach into the target's home over a transient visiting connection.
|
|
1051
|
+
: await runCrossNodeSetup(ctx.agentName, kp, agentRec.pubkey, action.owningNodeId, targetHex, sr, ctx.correlationId);
|
|
1052
|
+
// RETRY triggers: (a) the broker reported target_offline (stale replicated presence or a
|
|
1053
|
+
// re-home between discovery and the request); (b) a transient visiting-connection failure
|
|
1054
|
+
// (cross-region blip). Both re-discover → retry, bounded.
|
|
1055
|
+
if (!result.ok && (result.reason === "target_offline" || result.reason === "visiting_connection_unreachable")) {
|
|
1056
|
+
logger.info("session.crossnode.stale_discovery_retry", { agentName: ctx.agentName, attempt, reason: result.reason, correlationId: ctx.correlationId });
|
|
1057
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
1058
|
+
await sleepMs(backoffs[attempt - 1]);
|
|
1059
|
+
continue;
|
|
1060
|
+
}
|
|
1061
|
+
// Exhausted: keep the truthful transient-connection reason; a stale target_offline surfaces
|
|
1062
|
+
// as counterparty_offline (state 2).
|
|
1063
|
+
return result.reason === "visiting_connection_unreachable"
|
|
1064
|
+
? { ok: false, reason: "visiting_connection_unreachable", guidance: "The counterparty's home node was reachable at discovery but its visiting connection could not be established after several attempts. Retry shortly." }
|
|
1065
|
+
: { ok: false, reason: "counterparty_offline", guidance: "The counterparty was online at discovery but not reachable at its home node after several attempts (it may be re-homing). Retry shortly." };
|
|
1066
|
+
}
|
|
1067
|
+
return result;
|
|
840
1068
|
}
|
|
841
|
-
|
|
842
|
-
agentName: ctx.agentName,
|
|
843
|
-
correlationId: ctx.correlationId,
|
|
844
|
-
signatureType: assignment.signature_type,
|
|
845
|
-
});
|
|
846
|
-
return { ok: true, assignment };
|
|
1069
|
+
return { ok: false, reason: "counterparty_offline", guidance: "Session establishment did not succeed after several attempts. Retry shortly." };
|
|
847
1070
|
}
|
|
848
1071
|
finally {
|
|
849
|
-
unregister();
|
|
850
1072
|
negotiationInProgress.delete(ctx.agentName);
|
|
851
1073
|
}
|
|
852
1074
|
},
|