@kubun/plugin-p2p 0.10.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/context/delegation.js +130 -1
- package/lib/context/group.js +1646 -1
- package/lib/context/hub.js +151 -1
- package/lib/context/join.js +115 -1
- package/lib/context/peer.d.ts +92 -0
- package/lib/context/peer.js +1216 -0
- package/lib/context/require-admin.d.ts +57 -0
- package/lib/context/require-admin.js +57 -0
- package/lib/context/sync.d.ts +12 -0
- package/lib/context/sync.js +252 -1
- package/lib/context/types.d.ts +58 -22
- package/lib/context/types.js +48 -1
- package/lib/groups/access-default-token.d.ts +44 -0
- package/lib/groups/access-default-token.js +103 -0
- package/lib/groups/admin-roster.d.ts +55 -0
- package/lib/groups/admin-roster.js +69 -0
- package/lib/groups/anchor-store.d.ts +15 -0
- package/lib/groups/anchor-store.js +37 -0
- package/lib/groups/app-cursor-store.d.ts +17 -0
- package/lib/groups/app-cursor-store.js +34 -0
- package/lib/groups/broadcast-codec.js +6 -1
- package/lib/groups/broadcast.d.ts +238 -106
- package/lib/groups/broadcast.js +702 -1
- package/lib/groups/catalog-token.d.ts +50 -0
- package/lib/groups/catalog-token.js +96 -0
- package/lib/groups/circle-projection.d.ts +90 -0
- package/lib/groups/circle-projection.js +202 -0
- package/lib/groups/circle-reducers.d.ts +139 -0
- package/lib/groups/circle-reducers.js +128 -0
- package/lib/groups/commit-adoption.d.ts +77 -0
- package/lib/groups/commit-adoption.js +108 -0
- package/lib/groups/commit-journal.d.ts +14 -0
- package/lib/groups/commit-journal.js +65 -0
- package/lib/groups/control-request.d.ts +66 -0
- package/lib/groups/control-request.js +101 -0
- package/lib/groups/events.d.ts +19 -2
- package/lib/groups/events.js +10 -1
- package/lib/groups/group-anchor.d.ts +24 -0
- package/lib/groups/group-anchor.js +22 -0
- package/lib/groups/group-crypto.d.ts +42 -0
- package/lib/groups/group-crypto.js +159 -0
- package/lib/groups/group-handle-registry.d.ts +138 -9
- package/lib/groups/group-handle-registry.js +533 -1
- package/lib/groups/group-handlers.d.ts +30 -0
- package/lib/groups/group-handlers.js +218 -0
- package/lib/groups/group-health-monitor.d.ts +11 -0
- package/lib/groups/group-health-monitor.js +139 -1
- package/lib/groups/group-mls.d.ts +66 -0
- package/lib/groups/group-mls.js +696 -0
- package/lib/groups/group-peer-manager.d.ts +301 -0
- package/lib/groups/group-peer-manager.js +1336 -0
- package/lib/groups/group-protocols.d.ts +754 -0
- package/lib/groups/group-protocols.js +448 -0
- package/lib/groups/invite-payload.d.ts +49 -1
- package/lib/groups/invite-payload.js +26 -1
- package/lib/groups/join-request-identity.d.ts +21 -0
- package/lib/groups/join-request-identity.js +43 -0
- package/lib/groups/join-utils.d.ts +61 -0
- package/lib/groups/join-utils.js +366 -1
- package/lib/groups/ledger-affected-events.d.ts +20 -0
- package/lib/groups/ledger-affected-events.js +85 -0
- package/lib/groups/ledger-commit-fold.d.ts +107 -0
- package/lib/groups/ledger-commit-fold.js +167 -0
- package/lib/groups/ledger-ingest.d.ts +124 -0
- package/lib/groups/ledger-ingest.js +212 -0
- package/lib/groups/ledger.d.ts +44 -0
- package/lib/groups/ledger.js +30 -0
- package/lib/groups/manager.d.ts +217 -85
- package/lib/groups/manager.js +798 -1
- package/lib/groups/mls-codec.d.ts +28 -7
- package/lib/groups/mls-codec.js +33 -1
- package/lib/groups/mls-encryptor.d.ts +2 -2
- package/lib/groups/mls-encryptor.js +33 -1
- package/lib/groups/mls-group-handle.d.ts +37 -2
- package/lib/groups/mls-group-handle.js +18 -1
- package/lib/groups/mls-json.js +17 -1
- package/lib/groups/mls-receive-errors.d.ts +27 -0
- package/lib/groups/mls-receive-errors.js +39 -0
- package/lib/groups/mls-state.d.ts +6 -4
- package/lib/groups/mls-state.js +79 -1
- package/lib/groups/peer-presence.d.ts +90 -0
- package/lib/groups/peer-presence.js +295 -0
- package/lib/groups/peer-selection.d.ts +37 -0
- package/lib/groups/peer-selection.js +72 -0
- package/lib/groups/roster-projection.d.ts +53 -0
- package/lib/groups/roster-projection.js +65 -0
- package/lib/groups/store-received-grant.d.ts +7 -3
- package/lib/groups/store-received-grant.js +161 -1
- package/lib/groups/store-received-revocation.d.ts +19 -16
- package/lib/groups/store-received-revocation.js +83 -1
- package/lib/hub/ack-backoff.d.ts +22 -0
- package/lib/hub/ack-backoff.js +88 -0
- package/lib/hub/http-client.d.ts +18 -10
- package/lib/hub/http-client.js +15 -1
- package/lib/hub/hub-like.d.ts +75 -0
- package/lib/hub/hub-like.js +1022 -0
- package/lib/hub/loopback-log-hub.d.ts +20 -0
- package/lib/hub/loopback-log-hub.js +178 -0
- package/lib/hub/manager.d.ts +25 -40
- package/lib/hub/manager.js +155 -1
- package/lib/hub/peer-scoped-hub-view.d.ts +35 -0
- package/lib/hub/peer-scoped-hub-view.js +92 -0
- package/lib/hub/sender-scoped-hub-view.d.ts +41 -0
- package/lib/hub/sender-scoped-hub-view.js +59 -0
- package/lib/hub/server-did.d.ts +67 -0
- package/lib/hub/server-did.js +131 -0
- package/lib/hub/wiring.d.ts +77 -51
- package/lib/hub/wiring.js +189 -1
- package/lib/index.d.ts +58 -22
- package/lib/index.js +563 -1
- package/lib/peer/connection-registry.d.ts +45 -0
- package/lib/peer/connection-registry.js +52 -0
- package/lib/peer/handlers.d.ts +69 -0
- package/lib/peer/handlers.js +110 -0
- package/lib/protocol.d.ts +214 -1
- package/lib/protocol.js +519 -1
- package/lib/schema.d.ts +3 -1
- package/lib/schema.js +1150 -39
- package/lib/sync/access-default-sender.d.ts +23 -0
- package/lib/sync/access-default-sender.js +80 -0
- package/lib/sync/authorize.d.ts +58 -4
- package/lib/sync/authorize.js +114 -1
- package/lib/sync/broadcast-queue.d.ts +3 -4
- package/lib/sync/broadcast-queue.js +182 -1
- package/lib/sync/broadcast-sender.js +179 -1
- package/lib/sync/catalog-scope.d.ts +8 -7
- package/lib/sync/catalog-scope.js +84 -1
- package/lib/sync/errors.d.ts +12 -2
- package/lib/sync/errors.js +29 -1
- package/lib/sync/forwarder.d.ts +13 -19
- package/lib/sync/forwarder.js +271 -1
- package/lib/sync/handlers.js +200 -1
- package/lib/sync/hub-tunnel-sync-listener.d.ts +19 -5
- package/lib/sync/hub-tunnel-sync-listener.js +261 -1
- package/lib/sync/hub-tunnel-sync-provider.d.ts +34 -14
- package/lib/sync/hub-tunnel-sync-provider.js +113 -1
- package/lib/sync/merkle-apply.d.ts +5 -0
- package/lib/sync/merkle-apply.js +194 -1
- package/lib/sync/merkle-channel.d.ts +11 -3
- package/lib/sync/merkle-channel.js +0 -0
- package/lib/sync/merkle-tree.js +120 -1
- package/lib/sync/peer-registry.d.ts +15 -11
- package/lib/sync/peer-registry.js +82 -1
- package/lib/sync/receive-access-gate.js +24 -1
- package/lib/sync/scope-resolver.js +41 -1
- package/lib/sync/sync-client.d.ts +14 -4
- package/lib/sync/sync-client.js +254 -1
- package/lib/sync/sync-manager.d.ts +24 -6
- package/lib/sync/sync-manager.js +301 -1
- package/lib/sync/tunnel-listeners.d.ts +52 -0
- package/lib/sync/tunnel-listeners.js +175 -0
- package/lib/sync/tunnel-observability.d.ts +25 -0
- package/lib/sync/tunnel-observability.js +63 -0
- package/lib/sync/tunnel-topics.d.ts +20 -0
- package/lib/sync/tunnel-topics.js +28 -0
- package/lib/types.d.ts +704 -22
- package/lib/types.js +1 -1
- package/lib/util/handler-error.d.ts +16 -0
- package/lib/util/handler-error.js +64 -0
- package/lib/util/mutex.d.ts +14 -1
- package/lib/util/mutex.js +44 -1
- package/lib/utils.js +21 -1
- package/package.json +58 -47
- package/lib/groups/broadcast-service.d.ts +0 -263
- package/lib/groups/broadcast-service.js +0 -1
- package/lib/groups/rejoin-codec.d.ts +0 -14
- package/lib/groups/rejoin-codec.js +0 -1
- package/lib/groups/wire-frame.d.ts +0 -34
- package/lib/groups/wire-frame.js +0 -1
- package/lib/hub/circle-catchup-requester.d.ts +0 -58
- package/lib/hub/circle-catchup-requester.js +0 -1
- package/lib/hub/circle-catchup-responder.d.ts +0 -50
- package/lib/hub/circle-catchup-responder.js +0 -1
- package/lib/hub/connection-pool.d.ts +0 -43
- package/lib/hub/connection-pool.js +0 -1
- package/lib/hub/did-observing-transport.d.ts +0 -64
- package/lib/hub/did-observing-transport.js +0 -1
- package/lib/hub/epoch-stale-detector.d.ts +0 -18
- package/lib/hub/epoch-stale-detector.js +0 -1
- package/lib/hub/errors.d.ts +0 -30
- package/lib/hub/errors.js +0 -1
- package/lib/hub/forward-remote-broadcast.d.ts +0 -15
- package/lib/hub/forward-remote-broadcast.js +0 -1
- package/lib/hub/group-channel.d.ts +0 -89
- package/lib/hub/group-channel.js +0 -1
- package/lib/hub/hub-connection.d.ts +0 -96
- package/lib/hub/hub-connection.js +0 -1
- package/lib/hub/receive-handler.d.ts +0 -51
- package/lib/hub/receive-handler.js +0 -1
- package/lib/hub/rejoin-manager.d.ts +0 -78
- package/lib/hub/rejoin-manager.js +0 -1
- package/lib/hub/rejoin-responder.d.ts +0 -32
- package/lib/hub/rejoin-responder.js +0 -1
- package/lib/hub/relay-manager.d.ts +0 -142
- package/lib/hub/relay-manager.js +0 -1
- package/lib/hub/send-handler.d.ts +0 -40
- package/lib/hub/send-handler.js +0 -1
- package/lib/hub/tunnel-inbox.d.ts +0 -20
- package/lib/hub/tunnel-inbox.js +0 -1
- package/lib/hub/wait-for-gate.d.ts +0 -14
- package/lib/hub/wait-for-gate.js +0 -1
|
@@ -1 +1,113 @@
|
|
|
1
|
-
import{createEncryptedHubTunnelTransport
|
|
1
|
+
import { createEncryptedHubTunnelTransport } from '@kumiai/hub-tunnel';
|
|
2
|
+
import { APP_TOPIC_LABEL } from '@kumiai/rpc';
|
|
3
|
+
import { createGroupCrypto } from '../groups/group-crypto.js';
|
|
4
|
+
import { MLSEncryptor } from '../groups/mls-encryptor.js';
|
|
5
|
+
import { createSenderScopedHubView } from '../hub/sender-scoped-hub-view.js';
|
|
6
|
+
import { createTunnelEventLogger } from './tunnel-observability.js';
|
|
7
|
+
import { tunnelTopic } from './tunnel-topics.js';
|
|
8
|
+
export class HubTunnelSyncProvider {
|
|
9
|
+
#hub;
|
|
10
|
+
#registry;
|
|
11
|
+
#groupID;
|
|
12
|
+
#localDID;
|
|
13
|
+
#peerDID;
|
|
14
|
+
#runtime;
|
|
15
|
+
#idleTimeoutMs;
|
|
16
|
+
#reconnectTimeoutMs;
|
|
17
|
+
#inboxCapacity;
|
|
18
|
+
#onEvent;
|
|
19
|
+
#logger;
|
|
20
|
+
constructor(params){
|
|
21
|
+
this.#hub = params.hub;
|
|
22
|
+
this.#registry = params.registry;
|
|
23
|
+
this.#groupID = params.groupID;
|
|
24
|
+
this.#localDID = params.localDID;
|
|
25
|
+
this.#peerDID = params.peerDID;
|
|
26
|
+
this.#runtime = params.runtime;
|
|
27
|
+
this.#idleTimeoutMs = params.idleTimeoutMs;
|
|
28
|
+
this.#reconnectTimeoutMs = params.reconnectTimeoutMs;
|
|
29
|
+
this.#inboxCapacity = params.inboxCapacity;
|
|
30
|
+
this.#onEvent = params.onEvent;
|
|
31
|
+
this.#logger = params.logger;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Build a fresh client transport for one tunnel sync session.
|
|
35
|
+
*
|
|
36
|
+
* The tunnel rides group-scoped, ROLE-scoped topics derived from the group's
|
|
37
|
+
* current MLS epoch secret: outbound frames publish to the peer's responder
|
|
38
|
+
* topic, and the transport subscribes to this device's own dialer topic —
|
|
39
|
+
* which the peer's listener writes its answers to and nothing else does.
|
|
40
|
+
* Topics rotate with the epoch, so a transport built in epoch N stops matching
|
|
41
|
+
* once a commit advances the group — callers create a new transport per
|
|
42
|
+
* session.
|
|
43
|
+
*/ async createSyncTransport(signal) {
|
|
44
|
+
const sessionID = this.#runtime.getRandomID();
|
|
45
|
+
const encryptor = new MLSEncryptor({
|
|
46
|
+
registry: this.#registry,
|
|
47
|
+
groupID: this.#groupID
|
|
48
|
+
});
|
|
49
|
+
// The seed epoch is unread here: this port never classifies commit frames,
|
|
50
|
+
// and `exportSecret()` below records the live epoch before `epoch()` is
|
|
51
|
+
// asked for it. Only a peer that opens a receive drain needs a real seed.
|
|
52
|
+
const crypto = createGroupCrypto({
|
|
53
|
+
registry: this.#registry,
|
|
54
|
+
groupID: this.#groupID,
|
|
55
|
+
initialEpoch: 0,
|
|
56
|
+
runtime: this.#runtime
|
|
57
|
+
});
|
|
58
|
+
const secret = await crypto.exportSecret(APP_TOPIC_LABEL);
|
|
59
|
+
const epoch = crypto.epoch();
|
|
60
|
+
const sendTopicID = tunnelTopic(secret, epoch, 'responder', this.#peerDID);
|
|
61
|
+
const receiveTopicID = tunnelTopic(secret, epoch, 'dialer', this.#localDID);
|
|
62
|
+
// Bound once and carried by every line this session logs, including the
|
|
63
|
+
// frame drops below. The transport's own idle timeout names none of this —
|
|
64
|
+
// it raises a bare `TimeoutInterruption` from inside the dependency — so
|
|
65
|
+
// attributing a starved session to a group, a peer and an epoch is only
|
|
66
|
+
// possible from here.
|
|
67
|
+
const sessionLogger = this.#logger?.with({
|
|
68
|
+
role: 'dialer',
|
|
69
|
+
groupID: this.#groupID,
|
|
70
|
+
peerDID: this.#peerDID,
|
|
71
|
+
sessionID,
|
|
72
|
+
epoch,
|
|
73
|
+
receiveTopicID
|
|
74
|
+
});
|
|
75
|
+
sessionLogger?.debug('tunnel session opening');
|
|
76
|
+
const transport = createEncryptedHubTunnelTransport({
|
|
77
|
+
// The peer-scoped VIEW, never the device hub: two sessions open at once
|
|
78
|
+
// receive on this same inbox topic, and a frame one of them opens is
|
|
79
|
+
// destroyed for the other — MLS consumes a ratchet generation per open.
|
|
80
|
+
// The loser starves to its idle timeout with nothing raised.
|
|
81
|
+
// The idle timeout is reused as the publish bound on purpose: "a session
|
|
82
|
+
// that gets nothing for N gives up" and "a session that cannot even send
|
|
83
|
+
// for N gives up" are one promise to whoever is waiting, and a second knob
|
|
84
|
+
// would only let the two drift apart.
|
|
85
|
+
hub: createSenderScopedHubView({
|
|
86
|
+
hub: this.#hub,
|
|
87
|
+
peerDID: this.#peerDID,
|
|
88
|
+
...this.#idleTimeoutMs == null ? {} : {
|
|
89
|
+
publishTimeoutMs: this.#idleTimeoutMs
|
|
90
|
+
}
|
|
91
|
+
}),
|
|
92
|
+
encryptor,
|
|
93
|
+
groupID: this.#groupID,
|
|
94
|
+
sessionID,
|
|
95
|
+
localDID: this.#localDID,
|
|
96
|
+
sendTopicID,
|
|
97
|
+
receiveTopicID,
|
|
98
|
+
signal,
|
|
99
|
+
idleTimeoutMs: this.#idleTimeoutMs,
|
|
100
|
+
reconnectTimeoutMs: this.#reconnectTimeoutMs,
|
|
101
|
+
inboxCapacity: this.#inboxCapacity,
|
|
102
|
+
onEvent: sessionLogger == null ? this.#onEvent : createTunnelEventLogger(sessionLogger, this.#onEvent)
|
|
103
|
+
});
|
|
104
|
+
// The close is as load-bearing as the open: a session that ends without
|
|
105
|
+
// having received anything is what an unreachable peer looks like, and the
|
|
106
|
+
// pair of lines is what says whether it ran for milliseconds or waited out
|
|
107
|
+
// the full idle window.
|
|
108
|
+
transport.events.on('disposed', ()=>{
|
|
109
|
+
sessionLogger?.debug('tunnel session closed');
|
|
110
|
+
});
|
|
111
|
+
return transport;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
@@ -27,6 +27,11 @@ export type ApplySyncMutationsResult = {
|
|
|
27
27
|
skipped: number;
|
|
28
28
|
/** Count of mutations denied by the receive-time access gate. */
|
|
29
29
|
dropped: number;
|
|
30
|
+
/**
|
|
31
|
+
* Count of mutations whose apply failed transiently (store/db error). No log
|
|
32
|
+
* row is written for these, so the next merkle round re-fetches and retries.
|
|
33
|
+
*/
|
|
34
|
+
deferred: number;
|
|
30
35
|
};
|
|
31
36
|
/**
|
|
32
37
|
* Apply mutation JWTs received from a peer via Merkle sync.
|
package/lib/sync/merkle-apply.js
CHANGED
|
@@ -1 +1,194 @@
|
|
|
1
|
-
import{
|
|
1
|
+
import { verifyToken } from '@kokuin/token';
|
|
2
|
+
import { computeMutationHash } from '@kubun/engine';
|
|
3
|
+
import { DocumentID } from '@kubun/id';
|
|
4
|
+
import { isPermanentApplyError } from '@kubun/mutation';
|
|
5
|
+
import { documentMutation } from '@kubun/protocol';
|
|
6
|
+
import { getGraphStore } from '@kubun/store-graph';
|
|
7
|
+
import { asType, createValidator } from '@sozai/schema';
|
|
8
|
+
import { rethrowIfProgrammerError } from './forwarder.js';
|
|
9
|
+
import { createReceiveAccessGate } from './receive-access-gate.js';
|
|
10
|
+
const validateMutation = createValidator(documentMutation);
|
|
11
|
+
/**
|
|
12
|
+
* Apply mutation JWTs received from a peer via Merkle sync.
|
|
13
|
+
*
|
|
14
|
+
* Routes the successful-apply path through {@link GraphInternals.applyVerifiedMutation}
|
|
15
|
+
* with `origin: 'peer'` so GraphQL subscriptions (and other engine event
|
|
16
|
+
* consumers) fire for peer-received documents. The engine handles JWT
|
|
17
|
+
* verification, validator cache, applying the mutation, and inserting the
|
|
18
|
+
* `status: 'applied'` mutation log entry.
|
|
19
|
+
*
|
|
20
|
+
* Paths that bypass the engine (manual mutation-log insert):
|
|
21
|
+
* - **skipped**: mutation hash already seen — no-op.
|
|
22
|
+
* - **pending**: change mutation for a document that doesn't exist yet;
|
|
23
|
+
* parked in the log with `status: 'pending'` to be resolved when the
|
|
24
|
+
* set arrives.
|
|
25
|
+
* - **rejected**: JWT verify/validate fails, or engine throws during apply;
|
|
26
|
+
* recorded with `status: 'rejected'`.
|
|
27
|
+
*
|
|
28
|
+
* We still verify + validate the JWT up front to peek at `mutation.typ` for
|
|
29
|
+
* the pending-path routing decision. The engine re-verifies on the success
|
|
30
|
+
* path (defense in depth + avoids plumbing verified payloads through the
|
|
31
|
+
* public API).
|
|
32
|
+
*/ export async function applySyncMutations(params) {
|
|
33
|
+
const { db, graph, mutationJWTs } = params;
|
|
34
|
+
const graphStore = await getGraphStore(db);
|
|
35
|
+
const storeUnreadable = params.storeUnreadable ?? 'persist';
|
|
36
|
+
// Build the receive-time access gate as a factory over the apply's
|
|
37
|
+
// transaction-scoped stores so its access reads run inside the apply
|
|
38
|
+
// transaction (a main-connection read there deadlocks on single-connection
|
|
39
|
+
// SQLite). When `'persist'` (default), the gate is omitted —
|
|
40
|
+
// `applyVerifiedMutation` takes its existing zero-overhead no-gate path.
|
|
41
|
+
let accessGate;
|
|
42
|
+
if (storeUnreadable === 'drop') {
|
|
43
|
+
if (params.selfDID == null) {
|
|
44
|
+
throw new Error("applySyncMutations: 'storeUnreadable: drop' requires selfDID");
|
|
45
|
+
}
|
|
46
|
+
const selfDID = params.selfDID;
|
|
47
|
+
const defaultAccessLevel = params.defaultAccessLevel ?? {
|
|
48
|
+
read: 'only_owner',
|
|
49
|
+
write: 'only_owner'
|
|
50
|
+
};
|
|
51
|
+
accessGate = ({ graphStore: txGraphStore, p2pStore: txP2PStore })=>{
|
|
52
|
+
if (txP2PStore == null) {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
return createReceiveAccessGate({
|
|
56
|
+
selfDID,
|
|
57
|
+
db: {
|
|
58
|
+
getUserModelAccessDefault: (ownerDID, modelID, permissionType)=>txGraphStore.getUserModelAccessDefault(ownerDID, modelID, permissionType),
|
|
59
|
+
isMemberOfAnyCircle: (viewerDID, circleIDs)=>txP2PStore.isMemberOfAnyCircle(viewerDID, circleIDs),
|
|
60
|
+
isMemberOfAnyGroup: (viewerDID, groupIDs)=>txP2PStore.isMemberOfAnyGroup(viewerDID, groupIDs),
|
|
61
|
+
getModelInterfaces: (modelID)=>txGraphStore.getModelInterfaces(modelID)
|
|
62
|
+
},
|
|
63
|
+
defaultAccessLevel
|
|
64
|
+
});
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
let applied = 0;
|
|
68
|
+
let rejected = 0;
|
|
69
|
+
let pending = 0;
|
|
70
|
+
let skipped = 0;
|
|
71
|
+
let dropped = 0;
|
|
72
|
+
let deferred = 0;
|
|
73
|
+
for (const jwt of mutationJWTs){
|
|
74
|
+
const mutationHash = computeMutationHash(jwt);
|
|
75
|
+
// Step 1: Dedup
|
|
76
|
+
if (await graphStore.hasMutationHash(mutationHash)) {
|
|
77
|
+
skipped++;
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
// Step 2: Verify JWT signature — needed to peek at mutation.typ for
|
|
81
|
+
// pending-path routing.
|
|
82
|
+
let mutation;
|
|
83
|
+
try {
|
|
84
|
+
const verified = await verifyToken(jwt);
|
|
85
|
+
mutation = asType(validateMutation, verified.payload);
|
|
86
|
+
} catch {
|
|
87
|
+
rejected++;
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
const documentID = mutation.sub;
|
|
91
|
+
const modelID = DocumentID.fromString(documentID).model.toString();
|
|
92
|
+
// Step 3: Check if document exists for change mutations
|
|
93
|
+
const docID = DocumentID.fromString(documentID);
|
|
94
|
+
const doc = await graphStore.getDocument(docID);
|
|
95
|
+
if (doc == null && mutation.typ === 'change') {
|
|
96
|
+
await graphStore.insertMutationLogEntry({
|
|
97
|
+
mutation_hash: mutationHash,
|
|
98
|
+
model_id: modelID,
|
|
99
|
+
document_id: documentID,
|
|
100
|
+
author_did: mutation.iss,
|
|
101
|
+
hlc: mutation.hlc,
|
|
102
|
+
mutation_jwt: jwt,
|
|
103
|
+
status: 'pending'
|
|
104
|
+
});
|
|
105
|
+
pending++;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
// Step 4: Route through engine so GraphQL subscriptions fire
|
|
109
|
+
try {
|
|
110
|
+
const applyResult = await graph.applyVerifiedMutation({
|
|
111
|
+
token: jwt,
|
|
112
|
+
origin: 'peer',
|
|
113
|
+
...accessGate != null ? {
|
|
114
|
+
accessGate
|
|
115
|
+
} : {}
|
|
116
|
+
});
|
|
117
|
+
if (applyResult.dropped) {
|
|
118
|
+
dropped++;
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
applied++;
|
|
122
|
+
// Step 5: Resolve pending mutations after set
|
|
123
|
+
if (mutation.typ === 'set') {
|
|
124
|
+
const pendingMutations = await graphStore.getPendingMutations(documentID);
|
|
125
|
+
for (const pendingEntry of pendingMutations){
|
|
126
|
+
try {
|
|
127
|
+
const pendingResult = await graph.applyVerifiedMutation({
|
|
128
|
+
token: pendingEntry.mutation_jwt,
|
|
129
|
+
origin: 'peer',
|
|
130
|
+
...accessGate != null ? {
|
|
131
|
+
accessGate
|
|
132
|
+
} : {}
|
|
133
|
+
});
|
|
134
|
+
if (pendingResult.dropped) {
|
|
135
|
+
// Pending entry denied by gate. Mark the original pending row
|
|
136
|
+
// resolved as `rejected` so downstream consumers don't keep
|
|
137
|
+
// retrying it. The doc itself is on disk via the parent set
|
|
138
|
+
// (which already passed the gate). Counted as `dropped`,
|
|
139
|
+
// not `rejected`, to keep semantics aligned with the gate
|
|
140
|
+
// definition.
|
|
141
|
+
await graphStore.updateMutationStatus(pendingEntry.mutation_hash, 'rejected');
|
|
142
|
+
dropped++;
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
// Engine inserted a fresh `applied` log entry for the pending
|
|
146
|
+
// mutation. Mark the original `pending` entry resolved too so
|
|
147
|
+
// downstream consumers see consistent state.
|
|
148
|
+
await graphStore.updateMutationStatus(pendingEntry.mutation_hash, 'applied');
|
|
149
|
+
applied++;
|
|
150
|
+
} catch (err) {
|
|
151
|
+
rethrowIfProgrammerError(err);
|
|
152
|
+
// A permanent reject converges the tree (mark resolved). A transient
|
|
153
|
+
// failure leaves the `pending` row untouched: its hash already
|
|
154
|
+
// dedups in the tree, so it is retried not by the next merkle round
|
|
155
|
+
// but when a later set to this document re-walks `getPendingMutations`.
|
|
156
|
+
if (isPermanentApplyError(err)) {
|
|
157
|
+
await graphStore.updateMutationStatus(pendingEntry.mutation_hash, 'rejected');
|
|
158
|
+
rejected++;
|
|
159
|
+
} else {
|
|
160
|
+
deferred++;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
} catch (err) {
|
|
166
|
+
rethrowIfProgrammerError(err);
|
|
167
|
+
// A permanent reject persists a `rejected` row so the tree converges and
|
|
168
|
+
// the sender stops resending. A transient failure writes no row and is
|
|
169
|
+
// counted as deferred, so the next merkle round re-fetches and retries.
|
|
170
|
+
if (isPermanentApplyError(err)) {
|
|
171
|
+
await graphStore.insertMutationLogEntry({
|
|
172
|
+
mutation_hash: mutationHash,
|
|
173
|
+
model_id: modelID,
|
|
174
|
+
document_id: documentID,
|
|
175
|
+
author_did: mutation.iss,
|
|
176
|
+
hlc: mutation.hlc,
|
|
177
|
+
mutation_jwt: jwt,
|
|
178
|
+
status: 'rejected'
|
|
179
|
+
});
|
|
180
|
+
rejected++;
|
|
181
|
+
} else {
|
|
182
|
+
deferred++;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return {
|
|
187
|
+
applied,
|
|
188
|
+
rejected,
|
|
189
|
+
pending,
|
|
190
|
+
skipped,
|
|
191
|
+
dropped,
|
|
192
|
+
deferred
|
|
193
|
+
};
|
|
194
|
+
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import type { VerifyTokenHook } from '@
|
|
2
|
-
import type { SignedPayload } from '@
|
|
1
|
+
import type { VerifyTokenHook } from '@kokuin/capability';
|
|
2
|
+
import type { SignedPayload } from '@kokuin/token';
|
|
3
3
|
import type { KubunDB } from '@kubun/db';
|
|
4
|
-
import type
|
|
4
|
+
import { type DefaultAccessLevel, type GraphInternals } from '@kubun/engine';
|
|
5
5
|
import type { Logger } from '@kubun/logger';
|
|
6
6
|
import type { SyncMerkleSyncParams, SyncMerkleSyncReceive, SyncMerkleSyncResult, SyncMerkleSyncSend } from '../protocol.js';
|
|
7
7
|
import type { StoreUnreadableMode } from '../types.js';
|
|
8
|
+
import { type CircleReadGrantResolver } from './authorize.js';
|
|
8
9
|
import { type PeerRegistry } from './peer-registry.js';
|
|
9
10
|
/**
|
|
10
11
|
* Minimal handler context shape for sync/merkle-sync channel.
|
|
@@ -47,6 +48,13 @@ export type CreateMerkleSyncChannelHandlerParams = {
|
|
|
47
48
|
* state and cannot drift.
|
|
48
49
|
*/
|
|
49
50
|
getRevocationChecker?: () => Promise<VerifyTokenHook>;
|
|
51
|
+
/**
|
|
52
|
+
* Resolves the circle-granted read resolver. Shared with the negotiate handler
|
|
53
|
+
* so both gates accept identical scopes. Enables a circle member to pull an
|
|
54
|
+
* owner's docs for a model the owner's tier-2 read default shares with that
|
|
55
|
+
* circle.
|
|
56
|
+
*/
|
|
57
|
+
getCircleReadGrant?: () => Promise<CircleReadGrantResolver>;
|
|
50
58
|
};
|
|
51
59
|
export declare function createMerkleSyncChannelHandler(params: CreateMerkleSyncChannelHandlerParams): (ctx: MerkleSyncChannelContext) => Promise<SyncMerkleSyncResult>;
|
|
52
60
|
export {};
|
|
Binary file
|
package/lib/sync/merkle-tree.js
CHANGED
|
@@ -1 +1,120 @@
|
|
|
1
|
-
import{blake3
|
|
1
|
+
import { blake3 } from '@noble/hashes/blake3.js';
|
|
2
|
+
/** Maximum number of mutation JWTs to send in a single channel message. */ export const SYNC_BATCH_SIZE = 1000;
|
|
3
|
+
// Bucket key lengths: year=4, month=7, day=10, minute=16
|
|
4
|
+
const BUCKET_LEVELS = [
|
|
5
|
+
4,
|
|
6
|
+
7,
|
|
7
|
+
10,
|
|
8
|
+
16
|
|
9
|
+
];
|
|
10
|
+
const MINUTE_KEY_LENGTH = 16;
|
|
11
|
+
const encoder = new TextEncoder();
|
|
12
|
+
export function getTimeBuckets(hlc) {
|
|
13
|
+
// HLC format: "2026-02-14T10:30:45.123Z:0001:nodeID"
|
|
14
|
+
return {
|
|
15
|
+
year: hlc.slice(0, 4),
|
|
16
|
+
month: hlc.slice(0, 7),
|
|
17
|
+
day: hlc.slice(0, 10),
|
|
18
|
+
minute: hlc.slice(0, 16)
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
function hashHex(input) {
|
|
22
|
+
const digest = blake3(encoder.encode(input));
|
|
23
|
+
return Array.from(digest).map((b)=>b.toString(16).padStart(2, '0')).join('');
|
|
24
|
+
}
|
|
25
|
+
export function buildMerkleTree(entries) {
|
|
26
|
+
if (entries.length === 0) {
|
|
27
|
+
return {
|
|
28
|
+
root: '',
|
|
29
|
+
buckets: {}
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
// Group entries by minute bucket
|
|
33
|
+
const minuteGroups = new Map();
|
|
34
|
+
for (const entry of entries){
|
|
35
|
+
const { minute } = getTimeBuckets(entry.hlc);
|
|
36
|
+
let group = minuteGroups.get(minute);
|
|
37
|
+
if (group == null) {
|
|
38
|
+
group = [];
|
|
39
|
+
minuteGroups.set(minute, group);
|
|
40
|
+
}
|
|
41
|
+
group.push(entry);
|
|
42
|
+
}
|
|
43
|
+
const buckets = {};
|
|
44
|
+
// Hash each minute bucket (leaf nodes)
|
|
45
|
+
for (const [minuteKey, bucketEntries] of minuteGroups){
|
|
46
|
+
const sorted = [
|
|
47
|
+
...bucketEntries
|
|
48
|
+
].sort((a, b)=>a.mutation_hash < b.mutation_hash ? -1 : a.mutation_hash > b.mutation_hash ? 1 : 0);
|
|
49
|
+
const content = sorted.map((e)=>e.mutation_hash).join('\n');
|
|
50
|
+
buckets[minuteKey] = hashHex(content);
|
|
51
|
+
}
|
|
52
|
+
// Build parent buckets bottom-up: day from minutes, month from days, year from months
|
|
53
|
+
const parentLevels = [
|
|
54
|
+
10,
|
|
55
|
+
7,
|
|
56
|
+
4
|
|
57
|
+
]// day, month, year (key lengths)
|
|
58
|
+
;
|
|
59
|
+
for (const parentLength of parentLevels){
|
|
60
|
+
const childLength = parentLength === 10 ? 16 : parentLength === 7 ? 10 : 7;
|
|
61
|
+
const groups = new Map();
|
|
62
|
+
for (const key of Object.keys(buckets)){
|
|
63
|
+
if (key.length !== childLength) continue;
|
|
64
|
+
const parentKey = key.slice(0, parentLength);
|
|
65
|
+
let group = groups.get(parentKey);
|
|
66
|
+
if (group == null) {
|
|
67
|
+
group = [];
|
|
68
|
+
groups.set(parentKey, group);
|
|
69
|
+
}
|
|
70
|
+
group.push(key);
|
|
71
|
+
}
|
|
72
|
+
for (const [parentKey, childKeys] of groups){
|
|
73
|
+
const sorted = childKeys.sort();
|
|
74
|
+
const content = sorted.map((k)=>`${k}\0${buckets[k]}`).join('\n');
|
|
75
|
+
buckets[parentKey] = hashHex(content);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
// Build root hash from year buckets
|
|
79
|
+
const yearKeys = Object.keys(buckets).filter((k)=>k.length === 4).sort();
|
|
80
|
+
const rootContent = yearKeys.map((k)=>`${k}\0${buckets[k]}`).join('\n');
|
|
81
|
+
const root = hashHex(rootContent);
|
|
82
|
+
buckets.root = root;
|
|
83
|
+
return {
|
|
84
|
+
root,
|
|
85
|
+
buckets
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
export function findDivergentBuckets(local, remote) {
|
|
89
|
+
if (local.root === remote.root) return [];
|
|
90
|
+
// If either tree is empty, all of the other's minute buckets diverge
|
|
91
|
+
if (local.root === '') {
|
|
92
|
+
return Object.keys(remote.buckets).filter((k)=>k.length === MINUTE_KEY_LENGTH).sort();
|
|
93
|
+
}
|
|
94
|
+
if (remote.root === '') {
|
|
95
|
+
return Object.keys(local.buckets).filter((k)=>k.length === MINUTE_KEY_LENGTH).sort();
|
|
96
|
+
}
|
|
97
|
+
const divergent = [];
|
|
98
|
+
const allKeys = new Set([
|
|
99
|
+
...Object.keys(local.buckets),
|
|
100
|
+
...Object.keys(remote.buckets)
|
|
101
|
+
]);
|
|
102
|
+
function walk(prefix, levelIndex) {
|
|
103
|
+
const keyLength = BUCKET_LEVELS[levelIndex];
|
|
104
|
+
const keysAtLevel = [
|
|
105
|
+
...allKeys
|
|
106
|
+
].filter((k)=>k.length === keyLength && (prefix === '' || k.startsWith(prefix))).sort();
|
|
107
|
+
for (const key of keysAtLevel){
|
|
108
|
+
if (local.buckets[key] === remote.buckets[key]) continue;
|
|
109
|
+
if (levelIndex === BUCKET_LEVELS.length - 1) {
|
|
110
|
+
// Leaf level (minute) — this bucket diverges
|
|
111
|
+
divergent.push(key);
|
|
112
|
+
} else {
|
|
113
|
+
// Drill into next level
|
|
114
|
+
walk(key, levelIndex + 1);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
walk('', 0);
|
|
119
|
+
return divergent.sort();
|
|
120
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { Runtime } from '@enkaku/runtime';
|
|
2
1
|
import type { StoreProvider } from '@kubun/db';
|
|
2
|
+
import type { Runtime } from '@sozai/runtime';
|
|
3
3
|
import type { SyncDirection } from '../protocol.js';
|
|
4
4
|
/**
|
|
5
5
|
* Resolve the effective sync direction given what the initiator requested
|
|
@@ -8,15 +8,23 @@ import type { SyncDirection } from '../protocol.js';
|
|
|
8
8
|
* Otherwise the responder's policy wins.
|
|
9
9
|
*/
|
|
10
10
|
export declare function negotiateDirection(requested: SyncDirection, allowed: SyncDirection): SyncDirection;
|
|
11
|
+
/**
|
|
12
|
+
* How to REACH a peer, plus the one policy a responder applies to it.
|
|
13
|
+
*
|
|
14
|
+
* Deliberately not a place to say what a peer may see. This row is written
|
|
15
|
+
* locally by whichever side dialled, nothing authenticates it, and it is absent
|
|
16
|
+
* entirely for a co-member reached through a group tunnel — so a gate keyed on
|
|
17
|
+
* it would be missing exactly when peering works as designed. What a peer may
|
|
18
|
+
* read is decided per document at the far end, against its authenticated DID.
|
|
19
|
+
*/
|
|
11
20
|
export type PeerConfig = {
|
|
12
21
|
peerDID: string;
|
|
13
22
|
endpoint: string;
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
trustLevel: 'trusted' | 'restricted';
|
|
23
|
+
/**
|
|
24
|
+
* What this device is willing to do with this peer — a resource decision
|
|
25
|
+
* about its own disk and apply path, not a claim about the peer's rights.
|
|
26
|
+
* The responder's policy wins; see {@link negotiateDirection}.
|
|
27
|
+
*/
|
|
20
28
|
allowedDirection?: SyncDirection;
|
|
21
29
|
};
|
|
22
30
|
export type PeerConfigWithID = PeerConfig & {
|
|
@@ -50,8 +58,4 @@ export declare class PeerRegistry {
|
|
|
50
58
|
peerDID: string;
|
|
51
59
|
stores: StoreProvider;
|
|
52
60
|
}): Promise<void>;
|
|
53
|
-
isPeerAllowed(params: {
|
|
54
|
-
peerDID: string;
|
|
55
|
-
stores: StoreProvider;
|
|
56
|
-
}): Promise<boolean>;
|
|
57
61
|
}
|
|
@@ -1 +1,82 @@
|
|
|
1
|
-
import{getP2PStore
|
|
1
|
+
import { getP2PStore } from '@kubun/store-p2p';
|
|
2
|
+
/**
|
|
3
|
+
* Resolve the effective sync direction given what the initiator requested
|
|
4
|
+
* and what the responder's per-peer policy allows.
|
|
5
|
+
* If allowed is 'both' (or unset), the request passes through.
|
|
6
|
+
* Otherwise the responder's policy wins.
|
|
7
|
+
*/ export function negotiateDirection(requested, allowed) {
|
|
8
|
+
if (allowed === 'both') return requested;
|
|
9
|
+
if (requested === 'both') return allowed;
|
|
10
|
+
if (requested === allowed) return requested;
|
|
11
|
+
// Incompatible — responder policy wins
|
|
12
|
+
return allowed;
|
|
13
|
+
}
|
|
14
|
+
function parseConfig(value) {
|
|
15
|
+
if (typeof value === 'string') {
|
|
16
|
+
return JSON.parse(value);
|
|
17
|
+
}
|
|
18
|
+
return value ?? {};
|
|
19
|
+
}
|
|
20
|
+
function rowToPeerConfig(row) {
|
|
21
|
+
const config = parseConfig(row.config);
|
|
22
|
+
return {
|
|
23
|
+
id: row.id,
|
|
24
|
+
peerDID: row.peer_did,
|
|
25
|
+
endpoint: row.endpoint,
|
|
26
|
+
allowedDirection: config.allowedDirection,
|
|
27
|
+
createdAt: row.created_at,
|
|
28
|
+
updatedAt: row.updated_at
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
export class PeerRegistry {
|
|
32
|
+
#runtime;
|
|
33
|
+
constructor(params){
|
|
34
|
+
this.#runtime = params.runtime;
|
|
35
|
+
}
|
|
36
|
+
async addPeer(params) {
|
|
37
|
+
const store = await getP2PStore(params.stores);
|
|
38
|
+
await store.addSyncPeer({
|
|
39
|
+
id: this.#runtime.getRandomID(),
|
|
40
|
+
peer_did: params.config.peerDID,
|
|
41
|
+
endpoint: params.config.endpoint,
|
|
42
|
+
config: {
|
|
43
|
+
allowedDirection: params.config.allowedDirection
|
|
44
|
+
},
|
|
45
|
+
created_at: Date.now(),
|
|
46
|
+
updated_at: Date.now()
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
async getPeer(params) {
|
|
50
|
+
const store = await getP2PStore(params.stores);
|
|
51
|
+
const row = await store.getSyncPeer(params.peerDID);
|
|
52
|
+
if (!row) return undefined;
|
|
53
|
+
return rowToPeerConfig(row);
|
|
54
|
+
}
|
|
55
|
+
async listPeers(params) {
|
|
56
|
+
const store = await getP2PStore(params.stores);
|
|
57
|
+
const rows = await store.listSyncPeers();
|
|
58
|
+
return rows.map((row)=>rowToPeerConfig(row));
|
|
59
|
+
}
|
|
60
|
+
async updatePeer(params) {
|
|
61
|
+
const current = await this.getPeer({
|
|
62
|
+
peerDID: params.peerDID,
|
|
63
|
+
stores: params.stores
|
|
64
|
+
});
|
|
65
|
+
if (!current) throw new Error(`Peer ${params.peerDID} not found`);
|
|
66
|
+
const updated = {
|
|
67
|
+
...current,
|
|
68
|
+
...params.updates
|
|
69
|
+
};
|
|
70
|
+
const store = await getP2PStore(params.stores);
|
|
71
|
+
await store.updateSyncPeer(params.peerDID, {
|
|
72
|
+
endpoint: updated.endpoint,
|
|
73
|
+
config: {
|
|
74
|
+
allowedDirection: updated.allowedDirection
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
async removePeer(params) {
|
|
79
|
+
const store = await getP2PStore(params.stores);
|
|
80
|
+
await store.removeSyncPeer(params.peerDID);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -1 +1,24 @@
|
|
|
1
|
-
import{createAccessChecker
|
|
1
|
+
import { createAccessChecker } from '@kubun/engine';
|
|
2
|
+
/**
|
|
3
|
+
* Build an {@link AccessGate} that returns `true` iff the local peer can
|
|
4
|
+
* read the post-state document under the resolved access rule.
|
|
5
|
+
*
|
|
6
|
+
* Used by the receive paths (`processBroadcast` mutation:apply branch and
|
|
7
|
+
* `applySyncMutations`) when `storeUnreadable: 'drop'` is configured. The
|
|
8
|
+
* gate is intentionally cheap to construct — the heavy work (rule
|
|
9
|
+
* resolution + membership lookups) happens lazily when the gate is invoked
|
|
10
|
+
* inside the engine's apply pipeline.
|
|
11
|
+
*
|
|
12
|
+
* Drop-mode access checks happen at receive time without delegation
|
|
13
|
+
* token context (the broadcast doesn't carry tokens). A peer who would
|
|
14
|
+
* gain read access only via a delegation token will have the doc
|
|
15
|
+
* dropped under `storeUnreadable: 'drop'`. Use `'persist'` if delegation
|
|
16
|
+
* chains must be honored at receive time.
|
|
17
|
+
*/ export function createReceiveAccessGate(params) {
|
|
18
|
+
const checker = createAccessChecker({
|
|
19
|
+
viewerDID: params.selfDID,
|
|
20
|
+
db: params.db,
|
|
21
|
+
defaultAccessLevel: params.defaultAccessLevel
|
|
22
|
+
});
|
|
23
|
+
return (postState)=>checker(postState, 'read');
|
|
24
|
+
}
|