@starkeep/sync-engine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +120 -0
- package/dist/index.cjs +793 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +373 -0
- package/dist/index.d.ts +373 -0
- package/dist/index.js +762 -0
- package/dist/index.js.map +1 -0
- package/package.json +45 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,793 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/index.ts
|
|
21
|
+
var index_exports = {};
|
|
22
|
+
__export(index_exports, {
|
|
23
|
+
SyncError: () => SyncError,
|
|
24
|
+
advanceWatermark: () => advanceWatermark,
|
|
25
|
+
createChangeNotifier: () => createChangeNotifier,
|
|
26
|
+
createFileSyncEngine: () => createFileSyncEngine,
|
|
27
|
+
createHttpSyncHandler: () => createHttpSyncHandler,
|
|
28
|
+
createHttpSyncTransport: () => createHttpSyncTransport,
|
|
29
|
+
createInProcessSyncTransport: () => createInProcessSyncTransport,
|
|
30
|
+
createSqliteSyncStateStore: () => createSqliteSyncStateStore,
|
|
31
|
+
createSyncEngine: () => createSyncEngine,
|
|
32
|
+
mergeWatermarks: () => mergeWatermarks,
|
|
33
|
+
residencyOf: () => residencyOf,
|
|
34
|
+
selectUnseen: () => selectUnseen,
|
|
35
|
+
watermarkFor: () => watermarkFor
|
|
36
|
+
});
|
|
37
|
+
module.exports = __toCommonJS(index_exports);
|
|
38
|
+
|
|
39
|
+
// src/sync-state-sqlite.ts
|
|
40
|
+
var CREATE_TABLE_SQL = `
|
|
41
|
+
CREATE TABLE IF NOT EXISTS sync_state (
|
|
42
|
+
key TEXT PRIMARY KEY,
|
|
43
|
+
value_json TEXT NOT NULL,
|
|
44
|
+
updated_at INTEGER NOT NULL DEFAULT (strftime('%s','now'))
|
|
45
|
+
)
|
|
46
|
+
`;
|
|
47
|
+
var WATERMARKS = "watermarks";
|
|
48
|
+
var PEER_WATERMARKS = "peer_watermarks";
|
|
49
|
+
var HLC_CLOCK = "hlc_clock";
|
|
50
|
+
function createSqliteSyncStateStore(options) {
|
|
51
|
+
const { db } = options;
|
|
52
|
+
db.exec(CREATE_TABLE_SQL);
|
|
53
|
+
const getStmt = db.prepare(
|
|
54
|
+
"SELECT value_json FROM sync_state WHERE key = ?"
|
|
55
|
+
);
|
|
56
|
+
const setStmt = db.prepare(
|
|
57
|
+
`INSERT INTO sync_state (key, value_json, updated_at)
|
|
58
|
+
VALUES (?, ?, strftime('%s','now'))
|
|
59
|
+
ON CONFLICT(key) DO UPDATE SET
|
|
60
|
+
value_json = excluded.value_json,
|
|
61
|
+
updated_at = excluded.updated_at`
|
|
62
|
+
);
|
|
63
|
+
function getJson(key) {
|
|
64
|
+
const row = getStmt.get(key);
|
|
65
|
+
if (!row) return null;
|
|
66
|
+
return JSON.parse(row.value_json);
|
|
67
|
+
}
|
|
68
|
+
function setJson(key, value) {
|
|
69
|
+
setStmt.run(key, JSON.stringify(value));
|
|
70
|
+
}
|
|
71
|
+
return {
|
|
72
|
+
async getWatermarks() {
|
|
73
|
+
return getJson(WATERMARKS) ?? {};
|
|
74
|
+
},
|
|
75
|
+
async setWatermarks(watermarks) {
|
|
76
|
+
setJson(WATERMARKS, watermarks);
|
|
77
|
+
},
|
|
78
|
+
async getPeerWatermarks() {
|
|
79
|
+
return getJson(PEER_WATERMARKS) ?? {};
|
|
80
|
+
},
|
|
81
|
+
async setPeerWatermarks(watermarks) {
|
|
82
|
+
setJson(PEER_WATERMARKS, watermarks);
|
|
83
|
+
},
|
|
84
|
+
async getHlcClockState() {
|
|
85
|
+
return getJson(HLC_CLOCK);
|
|
86
|
+
},
|
|
87
|
+
async setHlcClockState(state) {
|
|
88
|
+
setJson(HLC_CLOCK, state);
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// src/change-notifier.ts
|
|
94
|
+
function createChangeNotifier() {
|
|
95
|
+
const listeners = /* @__PURE__ */ new Set();
|
|
96
|
+
return {
|
|
97
|
+
subscribe(listener) {
|
|
98
|
+
listeners.add(listener);
|
|
99
|
+
return () => {
|
|
100
|
+
listeners.delete(listener);
|
|
101
|
+
};
|
|
102
|
+
},
|
|
103
|
+
emit(event) {
|
|
104
|
+
for (const listener of listeners) {
|
|
105
|
+
listener(event);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// src/watermarks.ts
|
|
112
|
+
var import_protocol_primitives = require("@starkeep/protocol-primitives");
|
|
113
|
+
function advanceWatermark(watermarks, hlc) {
|
|
114
|
+
const node = hlc.nodeId;
|
|
115
|
+
const existing = watermarks[node];
|
|
116
|
+
if (!existing || (0, import_protocol_primitives.compareHLC)(hlc, existing) > 0) {
|
|
117
|
+
watermarks[node] = hlc;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
function mergeWatermarks(into, incoming) {
|
|
121
|
+
const out = { ...into };
|
|
122
|
+
for (const [node, hlc] of Object.entries(incoming)) {
|
|
123
|
+
const existing = out[node];
|
|
124
|
+
out[node] = existing ? (0, import_protocol_primitives.maxHLC)(existing, hlc) : hlc;
|
|
125
|
+
}
|
|
126
|
+
return out;
|
|
127
|
+
}
|
|
128
|
+
function watermarkFor(watermarks, nodeId) {
|
|
129
|
+
return watermarks[nodeId] ?? import_protocol_primitives.ZERO_HLC;
|
|
130
|
+
}
|
|
131
|
+
function selectUnseen(records, peerWatermarks) {
|
|
132
|
+
return records.filter(
|
|
133
|
+
(r) => (0, import_protocol_primitives.compareHLC)(r.updatedAt, watermarkFor(peerWatermarks, r.updatedAt.nodeId)) > 0
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// src/file-sync-engine.ts
|
|
138
|
+
function createFileSyncEngine() {
|
|
139
|
+
const inFlightKeys = /* @__PURE__ */ new Set();
|
|
140
|
+
return {
|
|
141
|
+
isTransferInFlight(key) {
|
|
142
|
+
return inFlightKeys.has(key);
|
|
143
|
+
},
|
|
144
|
+
async getFilesToPush(localStorage, remoteStorage, entries) {
|
|
145
|
+
const manifests = [];
|
|
146
|
+
for (const entry of entries) {
|
|
147
|
+
const existsRemotely = await remoteStorage.has(entry.key);
|
|
148
|
+
if (!existsRemotely) {
|
|
149
|
+
const localFile = await localStorage.get(entry.key);
|
|
150
|
+
if (localFile) {
|
|
151
|
+
manifests.push({
|
|
152
|
+
fileHash: entry.key,
|
|
153
|
+
objectStorageKey: entry.key,
|
|
154
|
+
sizeBytes: localFile.size,
|
|
155
|
+
mimeType: entry.mimeType
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return manifests;
|
|
161
|
+
},
|
|
162
|
+
async getFilesToPull(localStorage, remoteStorage, entries) {
|
|
163
|
+
const manifests = [];
|
|
164
|
+
for (const entry of entries) {
|
|
165
|
+
const existsLocally = await localStorage.has(entry.key);
|
|
166
|
+
if (!existsLocally) {
|
|
167
|
+
const remoteFile = await remoteStorage.get(entry.key);
|
|
168
|
+
if (remoteFile) {
|
|
169
|
+
manifests.push({
|
|
170
|
+
fileHash: entry.key,
|
|
171
|
+
objectStorageKey: entry.key,
|
|
172
|
+
sizeBytes: remoteFile.size,
|
|
173
|
+
mimeType: entry.mimeType
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return manifests;
|
|
179
|
+
},
|
|
180
|
+
async transferFile(manifest, source, destination) {
|
|
181
|
+
const key = manifest.objectStorageKey;
|
|
182
|
+
if (inFlightKeys.has(key)) {
|
|
183
|
+
return false;
|
|
184
|
+
}
|
|
185
|
+
inFlightKeys.add(key);
|
|
186
|
+
try {
|
|
187
|
+
if (await destination.has(key)) {
|
|
188
|
+
return true;
|
|
189
|
+
}
|
|
190
|
+
const file = await source.get(key);
|
|
191
|
+
if (!file) {
|
|
192
|
+
return false;
|
|
193
|
+
}
|
|
194
|
+
await destination.put(key, file.data, {
|
|
195
|
+
contentType: manifest.mimeType
|
|
196
|
+
});
|
|
197
|
+
return true;
|
|
198
|
+
} finally {
|
|
199
|
+
inFlightKeys.delete(key);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// src/sync-engine.ts
|
|
206
|
+
var import_protocol_primitives2 = require("@starkeep/protocol-primitives");
|
|
207
|
+
var FILE_RECORDS_TABLE = "_starkeep_sync_records";
|
|
208
|
+
function createSyncEngine(options) {
|
|
209
|
+
const {
|
|
210
|
+
localDatabaseAdapter,
|
|
211
|
+
localObjectStorage,
|
|
212
|
+
remoteObjectStorage,
|
|
213
|
+
transport,
|
|
214
|
+
clock,
|
|
215
|
+
syncState,
|
|
216
|
+
appSyncableSource,
|
|
217
|
+
syncSharedRecords = true,
|
|
218
|
+
pageLimit = 1e3,
|
|
219
|
+
scanPageSize = 500
|
|
220
|
+
} = options;
|
|
221
|
+
const changeNotifier = createChangeNotifier();
|
|
222
|
+
const fileSyncEngine = createFileSyncEngine();
|
|
223
|
+
async function loadOwnWatermarks() {
|
|
224
|
+
if (!syncState) return {};
|
|
225
|
+
return syncState.getWatermarks();
|
|
226
|
+
}
|
|
227
|
+
async function loadPeerWatermarks() {
|
|
228
|
+
if (!syncState) return {};
|
|
229
|
+
return syncState.getPeerWatermarks();
|
|
230
|
+
}
|
|
231
|
+
return {
|
|
232
|
+
async exchange() {
|
|
233
|
+
const ownWatermarks = await loadOwnWatermarks();
|
|
234
|
+
const peerWatermarks = await loadPeerWatermarks();
|
|
235
|
+
const recordCandidates = [];
|
|
236
|
+
if (syncSharedRecords) {
|
|
237
|
+
let scanCursor = void 0;
|
|
238
|
+
let scanHasMore = true;
|
|
239
|
+
while (recordCandidates.length < pageLimit && scanHasMore) {
|
|
240
|
+
const page = await localDatabaseAdapter.query({
|
|
241
|
+
limit: scanPageSize,
|
|
242
|
+
...scanCursor !== void 0 ? { cursor: scanCursor } : {}
|
|
243
|
+
});
|
|
244
|
+
if (page.records.length === 0) break;
|
|
245
|
+
for (const r of page.records) {
|
|
246
|
+
const peerHlc = peerWatermarks[r.updatedAt.nodeId];
|
|
247
|
+
if (!peerHlc || (0, import_protocol_primitives2.compareHLC)(r.updatedAt, peerHlc) > 0) {
|
|
248
|
+
recordCandidates.push(r);
|
|
249
|
+
if (recordCandidates.length >= pageLimit) break;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
scanHasMore = page.hasMore;
|
|
253
|
+
scanCursor = page.nextCursor ?? void 0;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
const appRowCandidates = [];
|
|
257
|
+
if (appSyncableSource) {
|
|
258
|
+
const zeroStr = (0, import_protocol_primitives2.serializeHLC)(import_protocol_primitives2.ZERO_HLC);
|
|
259
|
+
outer: for (const ns of appSyncableSource.namespaces.list()) {
|
|
260
|
+
for (const tableInfo of ns.tables) {
|
|
261
|
+
let appScanCursor = void 0;
|
|
262
|
+
let appScanHasMore = true;
|
|
263
|
+
while (recordCandidates.length + appRowCandidates.length < pageLimit && appScanHasMore) {
|
|
264
|
+
let page;
|
|
265
|
+
try {
|
|
266
|
+
page = await appSyncableSource.applier.scanSince(
|
|
267
|
+
ns.appId,
|
|
268
|
+
tableInfo.name,
|
|
269
|
+
zeroStr,
|
|
270
|
+
{
|
|
271
|
+
limit: scanPageSize,
|
|
272
|
+
...appScanCursor !== void 0 ? { cursor: appScanCursor } : {}
|
|
273
|
+
}
|
|
274
|
+
);
|
|
275
|
+
} catch (err) {
|
|
276
|
+
console.warn(
|
|
277
|
+
`[sync] exchange scanSince failed for ${ns.appId}.${tableInfo.name}: ${err.message}`
|
|
278
|
+
);
|
|
279
|
+
break;
|
|
280
|
+
}
|
|
281
|
+
if (page.rows.length === 0) break;
|
|
282
|
+
for (const r of page.rows) {
|
|
283
|
+
const peerHlc = peerWatermarks[r.timestamp.nodeId];
|
|
284
|
+
if (!peerHlc || (0, import_protocol_primitives2.compareHLC)(r.timestamp, peerHlc) > 0) {
|
|
285
|
+
appRowCandidates.push(r);
|
|
286
|
+
if (recordCandidates.length + appRowCandidates.length >= pageLimit) {
|
|
287
|
+
break;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
appScanHasMore = page.hasMore;
|
|
292
|
+
appScanCursor = page.nextCursor ?? void 0;
|
|
293
|
+
}
|
|
294
|
+
if (recordCandidates.length + appRowCandidates.length >= pageLimit) {
|
|
295
|
+
break outer;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
const cappedRecords = [];
|
|
301
|
+
const cappedAppRows = [];
|
|
302
|
+
if (recordCandidates.length + appRowCandidates.length <= pageLimit) {
|
|
303
|
+
cappedRecords.push(...recordCandidates);
|
|
304
|
+
cappedAppRows.push(...appRowCandidates);
|
|
305
|
+
} else {
|
|
306
|
+
const tagged = [
|
|
307
|
+
...recordCandidates.map(
|
|
308
|
+
(r) => ({ kind: "r", rec: r, hlc: r.updatedAt })
|
|
309
|
+
),
|
|
310
|
+
...appRowCandidates.map(
|
|
311
|
+
(e) => ({ kind: "a", row: e, hlc: e.timestamp })
|
|
312
|
+
)
|
|
313
|
+
];
|
|
314
|
+
tagged.sort((a, b) => (0, import_protocol_primitives2.compareHLC)(a.hlc, b.hlc));
|
|
315
|
+
for (const t of tagged.slice(0, pageLimit)) {
|
|
316
|
+
if (t.kind === "r") cappedRecords.push(t.rec);
|
|
317
|
+
else cappedAppRows.push(t.row);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
const outboundByNode = groupOutboundByNodeId(
|
|
321
|
+
cappedRecords,
|
|
322
|
+
cappedAppRows
|
|
323
|
+
);
|
|
324
|
+
const outboundRecords = [];
|
|
325
|
+
const outboundAppRows = [];
|
|
326
|
+
const peerSafeAdvance = /* @__PURE__ */ new Map();
|
|
327
|
+
for (const [nodeId, items] of outboundByNode) {
|
|
328
|
+
for (const item of items) {
|
|
329
|
+
const manifest = outboundManifest(item);
|
|
330
|
+
if (manifest) {
|
|
331
|
+
const ok = await transferBlobSafe(
|
|
332
|
+
manifest,
|
|
333
|
+
localObjectStorage,
|
|
334
|
+
remoteObjectStorage,
|
|
335
|
+
fileSyncEngine,
|
|
336
|
+
"upload",
|
|
337
|
+
outboundItemId(item)
|
|
338
|
+
);
|
|
339
|
+
if (!ok) break;
|
|
340
|
+
}
|
|
341
|
+
if (item.kind === "record") {
|
|
342
|
+
outboundRecords.push(item.record);
|
|
343
|
+
peerSafeAdvance.set(nodeId, item.record.updatedAt);
|
|
344
|
+
} else {
|
|
345
|
+
outboundAppRows.push(item.entry);
|
|
346
|
+
peerSafeAdvance.set(nodeId, item.entry.timestamp);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
const response = await transport.exchange({
|
|
351
|
+
watermarks: ownWatermarks,
|
|
352
|
+
records: outboundRecords.length > 0 ? outboundRecords : void 0,
|
|
353
|
+
appSyncableRows: outboundAppRows.length > 0 ? outboundAppRows : void 0,
|
|
354
|
+
limit: pageLimit
|
|
355
|
+
});
|
|
356
|
+
if (!syncSharedRecords && (response.records?.length ?? 0) > 0) {
|
|
357
|
+
console.warn(
|
|
358
|
+
`[sync] dropped ${response.records?.length ?? 0} shared record(s) received on a per-app channel (syncSharedRecords=false)`
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
const inboundByNode = groupInboundByNodeId(
|
|
362
|
+
syncSharedRecords ? response.records : [],
|
|
363
|
+
response.appSyncableRows
|
|
364
|
+
);
|
|
365
|
+
const appliedIds = [];
|
|
366
|
+
const ownSafeAdvance = /* @__PURE__ */ new Map();
|
|
367
|
+
for (const [nodeId, items] of inboundByNode) {
|
|
368
|
+
let contiguous = true;
|
|
369
|
+
for (const item of items) {
|
|
370
|
+
const itemHlc = inboundItemHlc(item);
|
|
371
|
+
const existing = peerSafeAdvance.get(nodeId);
|
|
372
|
+
if (!existing || (0, import_protocol_primitives2.compareHLC)(itemHlc, existing) > 0) {
|
|
373
|
+
peerSafeAdvance.set(nodeId, itemHlc);
|
|
374
|
+
}
|
|
375
|
+
if (item.kind === "record") {
|
|
376
|
+
const snapshot = item.record;
|
|
377
|
+
const current = await localDatabaseAdapter.get(snapshot.id);
|
|
378
|
+
const metadataAlreadyApplied = current !== null && (0, import_protocol_primitives2.compareHLC)(current.updatedAt, snapshot.updatedAt) >= 0;
|
|
379
|
+
if (!metadataAlreadyApplied) {
|
|
380
|
+
clock.receive(snapshot.updatedAt);
|
|
381
|
+
await localDatabaseAdapter.put(snapshot);
|
|
382
|
+
}
|
|
383
|
+
const manifest = manifestForRecord(snapshot);
|
|
384
|
+
const blobOk = await transferBlobSafe(
|
|
385
|
+
manifest,
|
|
386
|
+
remoteObjectStorage,
|
|
387
|
+
localObjectStorage,
|
|
388
|
+
fileSyncEngine,
|
|
389
|
+
"download",
|
|
390
|
+
snapshot.id
|
|
391
|
+
);
|
|
392
|
+
if (!blobOk) {
|
|
393
|
+
contiguous = false;
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
if (!metadataAlreadyApplied) appliedIds.push(snapshot.id);
|
|
397
|
+
if (contiguous) ownSafeAdvance.set(nodeId, snapshot.updatedAt);
|
|
398
|
+
} else {
|
|
399
|
+
const entry = item.entry;
|
|
400
|
+
if (!appSyncableSource) {
|
|
401
|
+
contiguous = false;
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
clock.receive(entry.timestamp);
|
|
405
|
+
try {
|
|
406
|
+
await appSyncableSource.applier.apply(entry);
|
|
407
|
+
} catch (err) {
|
|
408
|
+
console.warn(
|
|
409
|
+
`[sync] appSyncableRow apply failed (app=${entry.appId} table=${entry.table}): ${err.message}`
|
|
410
|
+
);
|
|
411
|
+
contiguous = false;
|
|
412
|
+
continue;
|
|
413
|
+
}
|
|
414
|
+
const manifest = manifestForAppRow(entry);
|
|
415
|
+
const blobOk = await transferBlobSafe(
|
|
416
|
+
manifest,
|
|
417
|
+
remoteObjectStorage,
|
|
418
|
+
localObjectStorage,
|
|
419
|
+
fileSyncEngine,
|
|
420
|
+
"download",
|
|
421
|
+
`${entry.appId}.${entry.table}`
|
|
422
|
+
);
|
|
423
|
+
if (!blobOk) {
|
|
424
|
+
contiguous = false;
|
|
425
|
+
continue;
|
|
426
|
+
}
|
|
427
|
+
if (contiguous) ownSafeAdvance.set(nodeId, entry.timestamp);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
if (syncState) {
|
|
432
|
+
const nextOwnWatermarks = { ...ownWatermarks };
|
|
433
|
+
for (const hlc of ownSafeAdvance.values()) {
|
|
434
|
+
advanceWatermark(nextOwnWatermarks, hlc);
|
|
435
|
+
}
|
|
436
|
+
await syncState.setWatermarks(nextOwnWatermarks);
|
|
437
|
+
const nextPeerWatermarks = { ...peerWatermarks };
|
|
438
|
+
for (const hlc of peerSafeAdvance.values()) {
|
|
439
|
+
advanceWatermark(nextPeerWatermarks, hlc);
|
|
440
|
+
}
|
|
441
|
+
await syncState.setPeerWatermarks(nextPeerWatermarks);
|
|
442
|
+
}
|
|
443
|
+
if (appliedIds.length > 0) {
|
|
444
|
+
changeNotifier.emit({
|
|
445
|
+
eventType: "local-data-synced",
|
|
446
|
+
recordIds: appliedIds,
|
|
447
|
+
timestamp: clock.now()
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
return {
|
|
451
|
+
applied: appliedIds.length,
|
|
452
|
+
shipped: outboundRecords.length + outboundAppRows.length,
|
|
453
|
+
hasMore: response.hasMore
|
|
454
|
+
};
|
|
455
|
+
},
|
|
456
|
+
changeNotifier
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
function outboundItemHlc(item) {
|
|
460
|
+
return item.kind === "record" ? item.record.updatedAt : item.entry.timestamp;
|
|
461
|
+
}
|
|
462
|
+
function inboundItemHlc(item) {
|
|
463
|
+
return item.kind === "record" ? item.record.updatedAt : item.entry.timestamp;
|
|
464
|
+
}
|
|
465
|
+
function outboundItemId(item) {
|
|
466
|
+
return item.kind === "record" ? item.record.id : `${item.entry.appId}.${item.entry.table}`;
|
|
467
|
+
}
|
|
468
|
+
function groupOutboundByNodeId(records, appRows) {
|
|
469
|
+
const out = /* @__PURE__ */ new Map();
|
|
470
|
+
for (const r of records) {
|
|
471
|
+
pushToBucket(out, r.updatedAt.nodeId, { kind: "record", record: r });
|
|
472
|
+
}
|
|
473
|
+
for (const e of appRows) {
|
|
474
|
+
pushToBucket(out, e.timestamp.nodeId, { kind: "appRow", entry: e });
|
|
475
|
+
}
|
|
476
|
+
for (const arr of out.values()) {
|
|
477
|
+
arr.sort((a, b) => (0, import_protocol_primitives2.compareHLC)(outboundItemHlc(a), outboundItemHlc(b)));
|
|
478
|
+
}
|
|
479
|
+
return out;
|
|
480
|
+
}
|
|
481
|
+
function groupInboundByNodeId(records, appRows) {
|
|
482
|
+
const out = /* @__PURE__ */ new Map();
|
|
483
|
+
for (const r of records) {
|
|
484
|
+
pushToBucket(out, r.updatedAt.nodeId, { kind: "record", record: r });
|
|
485
|
+
}
|
|
486
|
+
for (const e of appRows) {
|
|
487
|
+
pushToBucket(out, e.timestamp.nodeId, { kind: "appRow", entry: e });
|
|
488
|
+
}
|
|
489
|
+
for (const arr of out.values()) {
|
|
490
|
+
arr.sort((a, b) => (0, import_protocol_primitives2.compareHLC)(inboundItemHlc(a), inboundItemHlc(b)));
|
|
491
|
+
}
|
|
492
|
+
return out;
|
|
493
|
+
}
|
|
494
|
+
function pushToBucket(map, key, value) {
|
|
495
|
+
const arr = map.get(key) ?? [];
|
|
496
|
+
arr.push(value);
|
|
497
|
+
map.set(key, arr);
|
|
498
|
+
}
|
|
499
|
+
function outboundManifest(item) {
|
|
500
|
+
return item.kind === "record" ? manifestForRecord(item.record) : manifestForAppRow(item.entry);
|
|
501
|
+
}
|
|
502
|
+
function manifestForRecord(record) {
|
|
503
|
+
if (!record.objectStorageKey || record.deletedAt) return null;
|
|
504
|
+
return {
|
|
505
|
+
fileHash: record.contentHash || record.objectStorageKey,
|
|
506
|
+
objectStorageKey: record.objectStorageKey,
|
|
507
|
+
sizeBytes: record.sizeBytes,
|
|
508
|
+
mimeType: record.mimeType
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
function manifestForAppRow(entry) {
|
|
512
|
+
if (entry.table !== FILE_RECORDS_TABLE) return null;
|
|
513
|
+
if (entry.op === "delete") return null;
|
|
514
|
+
const row = entry.row;
|
|
515
|
+
if (!row) return null;
|
|
516
|
+
const key = row["object_storage_key"];
|
|
517
|
+
if (typeof key !== "string" || key.length === 0) return null;
|
|
518
|
+
const contentHash = row["content_hash"];
|
|
519
|
+
const mimeType = row["mime_type"];
|
|
520
|
+
const sizeBytes = row["size_bytes"];
|
|
521
|
+
return {
|
|
522
|
+
fileHash: typeof contentHash === "string" && contentHash.length > 0 ? contentHash : key,
|
|
523
|
+
objectStorageKey: key,
|
|
524
|
+
sizeBytes: typeof sizeBytes === "number" ? sizeBytes : Number(sizeBytes) || 0,
|
|
525
|
+
mimeType: typeof mimeType === "string" ? mimeType : void 0
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
async function transferBlobSafe(manifest, source, destination, fileSyncEngine, direction, itemId) {
|
|
529
|
+
if (!manifest) return true;
|
|
530
|
+
try {
|
|
531
|
+
return await fileSyncEngine.transferFile(manifest, source, destination);
|
|
532
|
+
} catch (err) {
|
|
533
|
+
console.warn(
|
|
534
|
+
`[sync] blob ${direction} failed for ${itemId} (${manifest.objectStorageKey}): ${err.message}`
|
|
535
|
+
);
|
|
536
|
+
return false;
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
// src/residency.ts
|
|
541
|
+
async function residencyOf(recordRow, localStorage) {
|
|
542
|
+
if (!recordRow) return "absent";
|
|
543
|
+
if (recordRow.deleted_at) return "tombstoned";
|
|
544
|
+
const blobHere = await localStorage.has(recordRow.object_storage_key);
|
|
545
|
+
return blobHere ? "resident" : "staged";
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// src/transports/in-process-transport.ts
|
|
549
|
+
var import_protocol_primitives3 = require("@starkeep/protocol-primitives");
|
|
550
|
+
function createInProcessSyncTransport(options) {
|
|
551
|
+
const { databaseAdapter, clock, appSyncableSource, syncSharedRecords = true } = options;
|
|
552
|
+
return {
|
|
553
|
+
async exchange(request) {
|
|
554
|
+
if (syncSharedRecords) {
|
|
555
|
+
for (const snapshot of request.records ?? []) {
|
|
556
|
+
const current = await databaseAdapter.get(snapshot.id);
|
|
557
|
+
if (current && (0, import_protocol_primitives3.compareHLC)(current.updatedAt, snapshot.updatedAt) >= 0) {
|
|
558
|
+
continue;
|
|
559
|
+
}
|
|
560
|
+
clock.receive(snapshot.updatedAt);
|
|
561
|
+
await databaseAdapter.put(snapshot);
|
|
562
|
+
}
|
|
563
|
+
} else if ((request.records?.length ?? 0) > 0) {
|
|
564
|
+
console.warn(
|
|
565
|
+
`[sync] in-process transport dropped ${request.records?.length ?? 0} shared record(s) on a per-app channel (syncSharedRecords=false)`
|
|
566
|
+
);
|
|
567
|
+
}
|
|
568
|
+
for (const entry of request.appSyncableRows ?? []) {
|
|
569
|
+
if (!appSyncableSource) continue;
|
|
570
|
+
const ns = appSyncableSource.namespaces.get(entry.appId);
|
|
571
|
+
if (!ns) continue;
|
|
572
|
+
clock.receive(entry.timestamp);
|
|
573
|
+
try {
|
|
574
|
+
await appSyncableSource.applier.apply(entry);
|
|
575
|
+
} catch (err) {
|
|
576
|
+
console.warn(
|
|
577
|
+
`[sync] exchange apply appSyncableRow failed (app=${entry.appId} table=${entry.table}): ${err.message}`
|
|
578
|
+
);
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
const limit = request.limit ?? 1e3;
|
|
582
|
+
const SCAN_PAGE = 500;
|
|
583
|
+
const collected = [];
|
|
584
|
+
let cursor = void 0;
|
|
585
|
+
let scanHasMore = syncSharedRecords;
|
|
586
|
+
let overflowed = false;
|
|
587
|
+
while (!overflowed && scanHasMore) {
|
|
588
|
+
const page = await databaseAdapter.query({
|
|
589
|
+
limit: SCAN_PAGE,
|
|
590
|
+
...cursor !== void 0 ? { cursor } : {}
|
|
591
|
+
});
|
|
592
|
+
if (page.records.length === 0) break;
|
|
593
|
+
for (const r of page.records) {
|
|
594
|
+
const peerHlc = request.watermarks[r.updatedAt.nodeId];
|
|
595
|
+
if (!peerHlc || (0, import_protocol_primitives3.compareHLC)(r.updatedAt, peerHlc) > 0) {
|
|
596
|
+
if (collected.length >= limit) {
|
|
597
|
+
overflowed = true;
|
|
598
|
+
break;
|
|
599
|
+
}
|
|
600
|
+
collected.push(r);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
if (overflowed) break;
|
|
604
|
+
scanHasMore = page.hasMore;
|
|
605
|
+
cursor = page.nextCursor ?? void 0;
|
|
606
|
+
}
|
|
607
|
+
const records = collected;
|
|
608
|
+
const appSyncableRows = [];
|
|
609
|
+
if (appSyncableSource && records.length < limit) {
|
|
610
|
+
const scanCapable = appSyncableSource.applier;
|
|
611
|
+
if (typeof scanCapable.scanSince === "function") {
|
|
612
|
+
const zeroStr = (0, import_protocol_primitives3.serializeHLC)(import_protocol_primitives3.ZERO_HLC);
|
|
613
|
+
outer: for (const ns of appSyncableSource.namespaces.list()) {
|
|
614
|
+
for (const tableInfo of ns.tables) {
|
|
615
|
+
let appCursor = void 0;
|
|
616
|
+
let appHasMore = true;
|
|
617
|
+
while (records.length + appSyncableRows.length < limit && appHasMore) {
|
|
618
|
+
let page;
|
|
619
|
+
try {
|
|
620
|
+
page = await scanCapable.scanSince(
|
|
621
|
+
ns.appId,
|
|
622
|
+
tableInfo.name,
|
|
623
|
+
zeroStr,
|
|
624
|
+
{
|
|
625
|
+
limit: SCAN_PAGE,
|
|
626
|
+
...appCursor !== void 0 ? { cursor: appCursor } : {}
|
|
627
|
+
}
|
|
628
|
+
);
|
|
629
|
+
} catch (err) {
|
|
630
|
+
console.warn(
|
|
631
|
+
`[sync] in-process transport scanSince failed for ${ns.appId}.${tableInfo.name}: ${err.message}`
|
|
632
|
+
);
|
|
633
|
+
break;
|
|
634
|
+
}
|
|
635
|
+
if (page.rows.length === 0) break;
|
|
636
|
+
for (const r of page.rows) {
|
|
637
|
+
const peerHlc = request.watermarks[r.timestamp.nodeId];
|
|
638
|
+
if (!peerHlc || (0, import_protocol_primitives3.compareHLC)(r.timestamp, peerHlc) > 0) {
|
|
639
|
+
appSyncableRows.push(r);
|
|
640
|
+
if (records.length + appSyncableRows.length >= limit) break;
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
appHasMore = page.hasMore;
|
|
644
|
+
appCursor = page.nextCursor ?? void 0;
|
|
645
|
+
}
|
|
646
|
+
if (records.length + appSyncableRows.length >= limit) break outer;
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
const hasMore = overflowed || records.length + appSyncableRows.length >= limit;
|
|
652
|
+
return { records, appSyncableRows, hasMore };
|
|
653
|
+
}
|
|
654
|
+
};
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
// src/errors.ts
|
|
658
|
+
var import_protocol_primitives4 = require("@starkeep/protocol-primitives");
|
|
659
|
+
var SyncError = class extends import_protocol_primitives4.StarkeepError {
|
|
660
|
+
constructor(message, cause) {
|
|
661
|
+
super(message, "SYNC_ERROR", cause);
|
|
662
|
+
this.name = "SyncError";
|
|
663
|
+
}
|
|
664
|
+
};
|
|
665
|
+
|
|
666
|
+
// src/transports/http-transport.ts
|
|
667
|
+
function createHttpSyncTransport(options) {
|
|
668
|
+
const { baseUrl, fetch: fetchImpl = globalThis.fetch, getAuthHeader } = options;
|
|
669
|
+
const trimmed = baseUrl.replace(/\/+$/, "");
|
|
670
|
+
async function postJson(path, body) {
|
|
671
|
+
const headers = {
|
|
672
|
+
"Content-Type": "application/json"
|
|
673
|
+
};
|
|
674
|
+
const auth = getAuthHeader?.();
|
|
675
|
+
if (auth) headers["Authorization"] = auth;
|
|
676
|
+
const response = await fetchImpl(`${trimmed}${path}`, {
|
|
677
|
+
method: "POST",
|
|
678
|
+
headers,
|
|
679
|
+
body: JSON.stringify(body)
|
|
680
|
+
});
|
|
681
|
+
if (!response.ok) {
|
|
682
|
+
const text = await response.text().catch(() => "");
|
|
683
|
+
throw new SyncError(
|
|
684
|
+
`${path} failed: ${response.status} ${response.statusText} ${text}`
|
|
685
|
+
);
|
|
686
|
+
}
|
|
687
|
+
return await response.json();
|
|
688
|
+
}
|
|
689
|
+
return {
|
|
690
|
+
async exchange(request) {
|
|
691
|
+
return postJson(
|
|
692
|
+
"/sync/exchange",
|
|
693
|
+
request
|
|
694
|
+
);
|
|
695
|
+
}
|
|
696
|
+
};
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
// src/transports/http-server.ts
|
|
700
|
+
function createHttpSyncHandler(options) {
|
|
701
|
+
const transport = options.transport ?? createInProcessSyncTransport({
|
|
702
|
+
databaseAdapter: options.databaseAdapter,
|
|
703
|
+
clock: options.clock,
|
|
704
|
+
objectStorage: options.objectStorageAdapter
|
|
705
|
+
});
|
|
706
|
+
return async (req, res) => {
|
|
707
|
+
const url = new URL(
|
|
708
|
+
req.url || "/",
|
|
709
|
+
`http://${req.headers.host ?? "localhost"}`
|
|
710
|
+
);
|
|
711
|
+
if (req.method === "POST" && url.pathname === "/sync/exchange") {
|
|
712
|
+
const body = await readJson(req);
|
|
713
|
+
const response = await transport.exchange(body);
|
|
714
|
+
sendJson(res, 200, response);
|
|
715
|
+
return true;
|
|
716
|
+
}
|
|
717
|
+
const fileMatch = url.pathname.match(/^\/files\/(.+)$/);
|
|
718
|
+
if (fileMatch) {
|
|
719
|
+
const key = decodeURIComponent(fileMatch[1]);
|
|
720
|
+
const storage = options.objectStorageAdapter;
|
|
721
|
+
if (req.method === "HEAD") {
|
|
722
|
+
const exists = await storage.has(key);
|
|
723
|
+
res.writeHead(exists ? 200 : 404);
|
|
724
|
+
res.end();
|
|
725
|
+
return true;
|
|
726
|
+
}
|
|
727
|
+
if (req.method === "GET") {
|
|
728
|
+
const file = await storage.get(key);
|
|
729
|
+
if (!file) {
|
|
730
|
+
res.writeHead(404);
|
|
731
|
+
res.end();
|
|
732
|
+
return true;
|
|
733
|
+
}
|
|
734
|
+
res.writeHead(200, {
|
|
735
|
+
"Content-Type": file.contentType ?? "application/octet-stream",
|
|
736
|
+
"Content-Length": String(file.size)
|
|
737
|
+
});
|
|
738
|
+
res.end(Buffer.from(file.data));
|
|
739
|
+
return true;
|
|
740
|
+
}
|
|
741
|
+
if (req.method === "PUT") {
|
|
742
|
+
const bytes = await readBinary(req);
|
|
743
|
+
const contentType = req.headers["content-type"];
|
|
744
|
+
await storage.put(key, bytes, {
|
|
745
|
+
contentType: typeof contentType === "string" ? contentType : void 0
|
|
746
|
+
});
|
|
747
|
+
res.writeHead(200);
|
|
748
|
+
res.end();
|
|
749
|
+
return true;
|
|
750
|
+
}
|
|
751
|
+
if (req.method === "DELETE") {
|
|
752
|
+
await storage.delete(key);
|
|
753
|
+
res.writeHead(204);
|
|
754
|
+
res.end();
|
|
755
|
+
return true;
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
return false;
|
|
759
|
+
};
|
|
760
|
+
}
|
|
761
|
+
function sendJson(res, status, body) {
|
|
762
|
+
res.writeHead(status, { "Content-Type": "application/json" });
|
|
763
|
+
res.end(JSON.stringify(body));
|
|
764
|
+
}
|
|
765
|
+
async function readJson(req) {
|
|
766
|
+
const buf = await readBinary(req);
|
|
767
|
+
return JSON.parse(buf.toString("utf-8"));
|
|
768
|
+
}
|
|
769
|
+
function readBinary(req) {
|
|
770
|
+
return new Promise((resolve, reject) => {
|
|
771
|
+
const chunks = [];
|
|
772
|
+
req.on("data", (chunk) => chunks.push(chunk));
|
|
773
|
+
req.on("end", () => resolve(Buffer.concat(chunks)));
|
|
774
|
+
req.on("error", reject);
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
778
|
+
0 && (module.exports = {
|
|
779
|
+
SyncError,
|
|
780
|
+
advanceWatermark,
|
|
781
|
+
createChangeNotifier,
|
|
782
|
+
createFileSyncEngine,
|
|
783
|
+
createHttpSyncHandler,
|
|
784
|
+
createHttpSyncTransport,
|
|
785
|
+
createInProcessSyncTransport,
|
|
786
|
+
createSqliteSyncStateStore,
|
|
787
|
+
createSyncEngine,
|
|
788
|
+
mergeWatermarks,
|
|
789
|
+
residencyOf,
|
|
790
|
+
selectUnseen,
|
|
791
|
+
watermarkFor
|
|
792
|
+
});
|
|
793
|
+
//# sourceMappingURL=index.cjs.map
|