@syncular/server 0.15.46 → 0.15.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -4
- package/dist/admin.d.ts +1 -5
- package/dist/admin.js +2 -7
- package/dist/blob-handlers.js +4 -1
- package/dist/context.d.ts +5 -2
- package/dist/context.js +4 -0
- package/dist/d1-storage.d.ts +4 -1
- package/dist/d1-storage.js +75 -11
- package/dist/errors.js +6 -0
- package/dist/events.d.ts +2 -1
- package/dist/frame-bytes.d.ts +2 -2
- package/dist/frame-bytes.js +33 -14
- package/dist/handler.js +58 -14
- package/dist/index-bun.d.ts +2 -0
- package/dist/index-bun.js +2 -0
- package/dist/index-node.d.ts +2 -0
- package/dist/index-node.js +2 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +4 -6
- package/dist/operations.js +2 -1
- package/dist/postgres-storage.d.ts +5 -2
- package/dist/postgres-storage.js +71 -10
- package/dist/pull.d.ts +1 -1
- package/dist/pull.js +12 -6
- package/dist/realtime.d.ts +4 -1
- package/dist/realtime.js +33 -9
- package/dist/restore.d.ts +13 -0
- package/dist/restore.js +13 -0
- package/dist/s3-segment-store.js +10 -1
- package/dist/seed.js +36 -4
- package/dist/segment-download.js +5 -2
- package/dist/segment-store.d.ts +3 -0
- package/dist/segment-store.js +1 -0
- package/dist/sqlite-blob-store.d.ts +4 -9
- package/dist/sqlite-blob-store.js +5 -10
- package/dist/sqlite-bun-driver.d.ts +12 -0
- package/dist/sqlite-bun-driver.js +30 -0
- package/dist/sqlite-bun.d.ts +24 -0
- package/dist/sqlite-bun.js +40 -0
- package/dist/sqlite-dialect.d.ts +8 -8
- package/dist/sqlite-dialect.js +9 -2
- package/dist/sqlite-driver.d.ts +26 -0
- package/dist/sqlite-driver.js +8 -0
- package/dist/sqlite-image.d.ts +7 -9
- package/dist/sqlite-image.js +26 -28
- package/dist/sqlite-lease-store.d.ts +4 -9
- package/dist/sqlite-lease-store.js +5 -10
- package/dist/sqlite-node-driver.d.ts +10 -0
- package/dist/sqlite-node-driver.js +30 -0
- package/dist/sqlite-node.d.ts +24 -0
- package/dist/sqlite-node.js +50 -0
- package/dist/sqlite-segment-store.d.ts +4 -10
- package/dist/sqlite-segment-store.js +20 -14
- package/dist/sqlite-storage.d.ts +7 -12
- package/dist/sqlite-storage.js +89 -16
- package/dist/storage-errors.js +4 -1
- package/dist/storage.d.ts +18 -5
- package/package.json +18 -3
- package/src/admin.ts +3 -9
- package/src/blob-handlers.ts +8 -1
- package/src/context.ts +18 -2
- package/src/d1-storage.ts +107 -12
- package/src/errors.ts +6 -0
- package/src/events.ts +2 -1
- package/src/frame-bytes.ts +40 -14
- package/src/handler.ts +102 -29
- package/src/index-bun.ts +9 -0
- package/src/index-node.ts +9 -0
- package/src/index.ts +9 -6
- package/src/operations.ts +6 -1
- package/src/postgres-storage.ts +102 -13
- package/src/pull.ts +12 -1
- package/src/realtime.ts +46 -7
- package/src/restore.ts +28 -0
- package/src/s3-segment-store.ts +10 -1
- package/src/seed.ts +46 -4
- package/src/segment-download.ts +11 -2
- package/src/segment-store.ts +4 -0
- package/src/sqlite-blob-store.ts +11 -10
- package/src/sqlite-bun-driver.ts +46 -0
- package/src/sqlite-bun.ts +53 -0
- package/src/sqlite-dialect.ts +14 -7
- package/src/sqlite-driver.ts +44 -0
- package/src/sqlite-image.ts +44 -49
- package/src/sqlite-lease-store.ts +11 -10
- package/src/sqlite-node-driver.ts +46 -0
- package/src/sqlite-node.ts +62 -0
- package/src/sqlite-segment-store.ts +29 -15
- package/src/sqlite-storage.ts +131 -19
- package/src/storage-errors.ts +4 -1
- package/src/storage.ts +28 -5
package/src/postgres-storage.ts
CHANGED
|
@@ -85,6 +85,7 @@ import type {
|
|
|
85
85
|
IndexRowScanQuery,
|
|
86
86
|
NewCommit,
|
|
87
87
|
NewReaction,
|
|
88
|
+
PartitionRegistryEntry,
|
|
88
89
|
PrunedReactionCounts,
|
|
89
90
|
ReactionClaimQuery,
|
|
90
91
|
ReactionFailure,
|
|
@@ -136,6 +137,12 @@ CREATE TABLE IF NOT EXISTS sync_partitions(
|
|
|
136
137
|
max_commit_seq BIGINT NOT NULL DEFAULT 0,
|
|
137
138
|
horizon_seq BIGINT NOT NULL DEFAULT 0
|
|
138
139
|
);
|
|
140
|
+
CREATE TABLE IF NOT EXISTS sync_partition_registry(
|
|
141
|
+
partition TEXT PRIMARY KEY,
|
|
142
|
+
log_epoch TEXT NOT NULL,
|
|
143
|
+
epoch_required BOOLEAN NOT NULL DEFAULT FALSE,
|
|
144
|
+
last_authenticated_at_ms BIGINT NOT NULL
|
|
145
|
+
);
|
|
139
146
|
CREATE TABLE IF NOT EXISTS sync_row_scopes(
|
|
140
147
|
partition TEXT NOT NULL, tbl TEXT NOT NULL,
|
|
141
148
|
var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
@@ -189,10 +196,13 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
|
|
|
189
196
|
ON sync_reactions(partition, status, available_at_ms, idempotency_key);
|
|
190
197
|
CREATE TABLE IF NOT EXISTS sync_clients(
|
|
191
198
|
partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
|
|
199
|
+
wire_version INTEGER NOT NULL DEFAULT 1,
|
|
192
200
|
cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,
|
|
193
201
|
updated_at_ms BIGINT NOT NULL,
|
|
194
202
|
PRIMARY KEY(partition, client_id)
|
|
195
203
|
);
|
|
204
|
+
ALTER TABLE sync_clients
|
|
205
|
+
ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;
|
|
196
206
|
CREATE TABLE IF NOT EXISTS sync_blob_refs(
|
|
197
207
|
partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
198
208
|
blob_id TEXT NOT NULL,
|
|
@@ -998,6 +1008,86 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
998
1008
|
this.#schemaVersion = schema.version;
|
|
999
1009
|
}
|
|
1000
1010
|
|
|
1011
|
+
async touchPartition(
|
|
1012
|
+
partition: string,
|
|
1013
|
+
authenticatedAtMs: number,
|
|
1014
|
+
initialLogEpoch: string,
|
|
1015
|
+
): Promise<PartitionRegistryEntry> {
|
|
1016
|
+
if (initialLogEpoch.length === 0) {
|
|
1017
|
+
throw new Error('initial log epoch must be non-empty');
|
|
1018
|
+
}
|
|
1019
|
+
const { rows } = await this.#exec.query<{
|
|
1020
|
+
log_epoch: string;
|
|
1021
|
+
epoch_required: boolean;
|
|
1022
|
+
last_authenticated_at_ms: unknown;
|
|
1023
|
+
}>(
|
|
1024
|
+
`INSERT INTO sync_partition_registry(
|
|
1025
|
+
partition, log_epoch, last_authenticated_at_ms
|
|
1026
|
+
) VALUES ($1,$2,$3)
|
|
1027
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
1028
|
+
last_authenticated_at_ms=EXCLUDED.last_authenticated_at_ms
|
|
1029
|
+
RETURNING log_epoch, epoch_required, last_authenticated_at_ms`,
|
|
1030
|
+
[partition, initialLogEpoch, authenticatedAtMs],
|
|
1031
|
+
);
|
|
1032
|
+
const row = rows[0];
|
|
1033
|
+
if (row === undefined)
|
|
1034
|
+
throw new Error('partition registry write did not persist');
|
|
1035
|
+
return {
|
|
1036
|
+
partition,
|
|
1037
|
+
logEpoch: row.log_epoch,
|
|
1038
|
+
epochRequired: row.epoch_required,
|
|
1039
|
+
lastAuthenticatedAtMs: asNumber(row.last_authenticated_at_ms),
|
|
1040
|
+
};
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
async rotatePartitionLogEpoch(
|
|
1044
|
+
partition: string,
|
|
1045
|
+
logEpoch: string,
|
|
1046
|
+
authenticatedAtMs: number,
|
|
1047
|
+
): Promise<PartitionRegistryEntry> {
|
|
1048
|
+
if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
|
|
1049
|
+
await this.#exec.transaction(async (client) => {
|
|
1050
|
+
await client.query(
|
|
1051
|
+
`INSERT INTO sync_partition_registry(
|
|
1052
|
+
partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
1053
|
+
) VALUES ($1,$2,TRUE,$3)
|
|
1054
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
1055
|
+
log_epoch=EXCLUDED.log_epoch,
|
|
1056
|
+
epoch_required=TRUE,
|
|
1057
|
+
last_authenticated_at_ms=EXCLUDED.last_authenticated_at_ms`,
|
|
1058
|
+
[partition, logEpoch, authenticatedAtMs],
|
|
1059
|
+
);
|
|
1060
|
+
await client.query('DELETE FROM sync_clients WHERE partition=$1', [
|
|
1061
|
+
partition,
|
|
1062
|
+
]);
|
|
1063
|
+
});
|
|
1064
|
+
return {
|
|
1065
|
+
partition,
|
|
1066
|
+
logEpoch,
|
|
1067
|
+
epochRequired: true,
|
|
1068
|
+
lastAuthenticatedAtMs: authenticatedAtMs,
|
|
1069
|
+
};
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
async listPartitionRegistry(): Promise<PartitionRegistryEntry[]> {
|
|
1073
|
+
const { rows } = await this.#exec.query<{
|
|
1074
|
+
partition: string;
|
|
1075
|
+
log_epoch: string;
|
|
1076
|
+
epoch_required: boolean;
|
|
1077
|
+
last_authenticated_at_ms: unknown;
|
|
1078
|
+
}>(
|
|
1079
|
+
`SELECT partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
1080
|
+
FROM sync_partition_registry ORDER BY partition`,
|
|
1081
|
+
[],
|
|
1082
|
+
);
|
|
1083
|
+
return rows.map((row) => ({
|
|
1084
|
+
partition: row.partition,
|
|
1085
|
+
logEpoch: row.log_epoch,
|
|
1086
|
+
epochRequired: row.epoch_required,
|
|
1087
|
+
lastAuthenticatedAtMs: asNumber(row.last_authenticated_at_ms),
|
|
1088
|
+
}));
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1001
1091
|
/**
|
|
1002
1092
|
* Open a real Postgres transaction. The push handler drives the returned
|
|
1003
1093
|
* `StorageTransaction` imperatively (getRow/upsert/…/commit), but the
|
|
@@ -1496,11 +1586,12 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1496
1586
|
const { rows } = await this.#exec.query<{
|
|
1497
1587
|
client_id: string;
|
|
1498
1588
|
actor_id: string;
|
|
1589
|
+
wire_version: unknown;
|
|
1499
1590
|
cursor: unknown;
|
|
1500
1591
|
subscriptions: unknown;
|
|
1501
1592
|
updated_at_ms: unknown;
|
|
1502
1593
|
}>(
|
|
1503
|
-
'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 AND client_id=$2',
|
|
1594
|
+
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 AND client_id=$2',
|
|
1504
1595
|
[partition, clientId],
|
|
1505
1596
|
);
|
|
1506
1597
|
const record = rows[0];
|
|
@@ -1508,6 +1599,7 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1508
1599
|
return {
|
|
1509
1600
|
clientId: record.client_id,
|
|
1510
1601
|
actorId: record.actor_id,
|
|
1602
|
+
wireVersion: asNumber(record.wire_version),
|
|
1511
1603
|
cursor: asNumber(record.cursor),
|
|
1512
1604
|
updatedAtMs: asNumber(record.updated_at_ms),
|
|
1513
1605
|
subscriptions: asJson<ClientSubscription[]>(record.subscriptions),
|
|
@@ -1519,16 +1611,18 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1519
1611
|
record: ClientRecord,
|
|
1520
1612
|
): Promise<void> {
|
|
1521
1613
|
await this.#exec.query(
|
|
1522
|
-
`INSERT INTO sync_clients(partition, client_id, actor_id, cursor, subscriptions, updated_at_ms)
|
|
1523
|
-
VALUES ($1,$2,$3,$4,$5,$6)
|
|
1614
|
+
`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms)
|
|
1615
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
|
1524
1616
|
ON CONFLICT (partition, client_id) DO UPDATE
|
|
1525
|
-
SET actor_id=EXCLUDED.actor_id,
|
|
1617
|
+
SET actor_id=EXCLUDED.actor_id, wire_version=EXCLUDED.wire_version,
|
|
1618
|
+
cursor=EXCLUDED.cursor,
|
|
1526
1619
|
subscriptions=EXCLUDED.subscriptions,
|
|
1527
1620
|
updated_at_ms=EXCLUDED.updated_at_ms`,
|
|
1528
1621
|
[
|
|
1529
1622
|
partition,
|
|
1530
1623
|
record.clientId,
|
|
1531
1624
|
record.actorId,
|
|
1625
|
+
record.wireVersion,
|
|
1532
1626
|
record.cursor,
|
|
1533
1627
|
JSON.stringify(record.subscriptions),
|
|
1534
1628
|
record.updatedAtMs,
|
|
@@ -1608,16 +1702,18 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1608
1702
|
const { rows } = await this.#exec.query<{
|
|
1609
1703
|
client_id: string;
|
|
1610
1704
|
actor_id: string;
|
|
1705
|
+
wire_version: unknown;
|
|
1611
1706
|
cursor: unknown;
|
|
1612
1707
|
subscriptions: unknown;
|
|
1613
1708
|
updated_at_ms: unknown;
|
|
1614
1709
|
}>(
|
|
1615
|
-
'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC',
|
|
1710
|
+
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC',
|
|
1616
1711
|
[partition],
|
|
1617
1712
|
);
|
|
1618
1713
|
return rows.map((r) => ({
|
|
1619
1714
|
clientId: r.client_id,
|
|
1620
1715
|
actorId: r.actor_id,
|
|
1716
|
+
wireVersion: asNumber(r.wire_version),
|
|
1621
1717
|
cursor: asNumber(r.cursor),
|
|
1622
1718
|
updatedAtMs: asNumber(r.updated_at_ms),
|
|
1623
1719
|
subscriptions: (typeof r.subscriptions === 'string'
|
|
@@ -1742,13 +1838,6 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1742
1838
|
}
|
|
1743
1839
|
|
|
1744
1840
|
async listPartitions(): Promise<string[]> {
|
|
1745
|
-
|
|
1746
|
-
// first pull — a partition with only one of the two still shows up.
|
|
1747
|
-
const { rows } = await this.#exec.query<{ partition: string }>(
|
|
1748
|
-
`SELECT partition FROM sync_partitions
|
|
1749
|
-
UNION SELECT partition FROM sync_clients ORDER BY partition`,
|
|
1750
|
-
[],
|
|
1751
|
-
);
|
|
1752
|
-
return rows.map((r) => r.partition);
|
|
1841
|
+
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|
|
1753
1842
|
}
|
|
1754
1843
|
}
|
package/src/pull.ts
CHANGED
|
@@ -36,7 +36,7 @@ async function resolveImageBuilder(
|
|
|
36
36
|
if (cachedDefaultBuilder === undefined) {
|
|
37
37
|
const hasBun = (globalThis as { Bun?: unknown }).Bun !== undefined;
|
|
38
38
|
cachedDefaultBuilder = hasBun
|
|
39
|
-
? (await import('./sqlite-
|
|
39
|
+
? (await import('./sqlite-bun')).buildSqliteImage
|
|
40
40
|
: null;
|
|
41
41
|
}
|
|
42
42
|
return cachedDefaultBuilder ?? undefined;
|
|
@@ -212,12 +212,14 @@ async function* sqliteImageSegment(
|
|
|
212
212
|
asOf: number,
|
|
213
213
|
digest: string,
|
|
214
214
|
trace: PullSectionTrace | undefined,
|
|
215
|
+
logEpoch: string,
|
|
215
216
|
): AsyncGenerator<ResponseFrame, boolean> {
|
|
216
217
|
const { storage, segments, partition } = ctx;
|
|
217
218
|
const now = clockOf(ctx)();
|
|
218
219
|
const existing = await segments.find(
|
|
219
220
|
{
|
|
220
221
|
partition,
|
|
222
|
+
logEpoch,
|
|
221
223
|
table: plan.table.name,
|
|
222
224
|
schemaVersion: schema.version,
|
|
223
225
|
mediaType: 'sqlite',
|
|
@@ -286,6 +288,7 @@ async function* sqliteImageSegment(
|
|
|
286
288
|
const record = await segments.put(
|
|
287
289
|
{
|
|
288
290
|
partition,
|
|
291
|
+
logEpoch,
|
|
289
292
|
table: plan.table.name,
|
|
290
293
|
schemaVersion: schema.version,
|
|
291
294
|
mediaType: 'sqlite',
|
|
@@ -320,6 +323,7 @@ async function* bootstrapSegments(
|
|
|
320
323
|
asOf: number,
|
|
321
324
|
startRowCursor: string | null,
|
|
322
325
|
trace: PullSectionTrace | undefined,
|
|
326
|
+
logEpoch: string,
|
|
323
327
|
): AsyncGenerator<
|
|
324
328
|
ResponseFrame,
|
|
325
329
|
{ complete: boolean; rowCursor: string | null }
|
|
@@ -347,6 +351,7 @@ async function* bootstrapSegments(
|
|
|
347
351
|
asOf,
|
|
348
352
|
digest,
|
|
349
353
|
trace,
|
|
354
|
+
logEpoch,
|
|
350
355
|
);
|
|
351
356
|
if (imaged) return { complete: true, rowCursor: null };
|
|
352
357
|
}
|
|
@@ -393,6 +398,7 @@ async function* bootstrapSegments(
|
|
|
393
398
|
const record = await segments.put(
|
|
394
399
|
{
|
|
395
400
|
partition,
|
|
401
|
+
logEpoch,
|
|
396
402
|
table: plan.table.name,
|
|
397
403
|
schemaVersion: schema.version,
|
|
398
404
|
mediaType: 'rows',
|
|
@@ -435,8 +441,12 @@ export async function* subscriptionSection(
|
|
|
435
441
|
maxSeq: number,
|
|
436
442
|
horizonSeq: number,
|
|
437
443
|
trace?: PullSectionTrace,
|
|
444
|
+
logEpoch?: string,
|
|
438
445
|
): AsyncGenerator<ResponseFrame, SubscriptionResult> {
|
|
439
446
|
const sub = plan.frame;
|
|
447
|
+
if (logEpoch === undefined || logEpoch.length === 0) {
|
|
448
|
+
throw new Error('subscriptionSection requires a non-empty log epoch');
|
|
449
|
+
}
|
|
440
450
|
|
|
441
451
|
if (plan.status === 'revoked') {
|
|
442
452
|
yield {
|
|
@@ -496,6 +506,7 @@ export async function* subscriptionSection(
|
|
|
496
506
|
asOf,
|
|
497
507
|
startCursor,
|
|
498
508
|
trace,
|
|
509
|
+
logEpoch,
|
|
499
510
|
);
|
|
500
511
|
if (outcome.complete) {
|
|
501
512
|
yield { type: 'SUB_END', nextCursor: asOf };
|
package/src/realtime.ts
CHANGED
|
@@ -31,7 +31,11 @@ import {
|
|
|
31
31
|
type WakeReason,
|
|
32
32
|
} from '@syncular/core';
|
|
33
33
|
import type { SyncRequestContext, SyncServerConfig } from './context';
|
|
34
|
-
import {
|
|
34
|
+
import {
|
|
35
|
+
REMOTE_COMMAND_CLIENT_ID_PREFIX,
|
|
36
|
+
RESOLVER_OUTAGE,
|
|
37
|
+
touchAuthenticatedPartition,
|
|
38
|
+
} from './context';
|
|
35
39
|
import { SyncError, syncError } from './errors';
|
|
36
40
|
import { emitEvent, type SyncularServerEvents } from './events';
|
|
37
41
|
import { createSyncResponseStream } from './handler';
|
|
@@ -215,6 +219,9 @@ export class RealtimeSession {
|
|
|
215
219
|
readonly partition: string;
|
|
216
220
|
readonly actorId: string;
|
|
217
221
|
readonly clientId: string;
|
|
222
|
+
readonly logEpoch: string;
|
|
223
|
+
/** Response/delta layout selected by the most recent socket round. */
|
|
224
|
+
wireVersion: number;
|
|
218
225
|
/** Highest contiguously applied commitSeq acknowledged by the client. */
|
|
219
226
|
cursor: number;
|
|
220
227
|
/** Suppress deltas until the client catches up via pull + ack (§8.2). */
|
|
@@ -256,12 +263,16 @@ export class RealtimeSession {
|
|
|
256
263
|
clock: () => number,
|
|
257
264
|
maxDeltaBytes: number,
|
|
258
265
|
storage: ServerStorage,
|
|
266
|
+
logEpoch: string,
|
|
267
|
+
wireVersion: number,
|
|
259
268
|
events: SyncularServerEvents | undefined,
|
|
260
269
|
) {
|
|
261
270
|
this.sessionId = crypto.randomUUID();
|
|
262
271
|
this.partition = options.partition;
|
|
263
272
|
this.actorId = options.actorId;
|
|
264
273
|
this.clientId = options.clientId;
|
|
274
|
+
this.logEpoch = logEpoch;
|
|
275
|
+
this.wireVersion = wireVersion;
|
|
265
276
|
this.cursor = cursor;
|
|
266
277
|
this.lastKnownSeq = latestSeq;
|
|
267
278
|
this.wakePending = cursor < latestSeq;
|
|
@@ -515,6 +526,11 @@ export class RealtimeSession {
|
|
|
515
526
|
if (this.#activeRound === token) this.#activeRound = undefined;
|
|
516
527
|
};
|
|
517
528
|
try {
|
|
529
|
+
const requestedWireVersion =
|
|
530
|
+
(requestBytes[4] ?? 0) | ((requestBytes[5] ?? 0) << 8);
|
|
531
|
+
if (requestedWireVersion === 1 || requestedWireVersion === 2) {
|
|
532
|
+
this.wireVersion = requestedWireVersion;
|
|
533
|
+
}
|
|
518
534
|
let stream: AsyncIterable<Uint8Array> | undefined;
|
|
519
535
|
try {
|
|
520
536
|
// §8.7: the round's clientId must match the connection's —
|
|
@@ -543,7 +559,9 @@ export class RealtimeSession {
|
|
|
543
559
|
? error
|
|
544
560
|
: syncError(error.code, error.message);
|
|
545
561
|
finishRound(); // END is in this one chunk
|
|
546
|
-
await this.#sendRoundChunk(
|
|
562
|
+
await this.#sendRoundChunk(
|
|
563
|
+
errorResponseBytes(sync, this.wireVersion, this.logEpoch),
|
|
564
|
+
);
|
|
547
565
|
return;
|
|
548
566
|
}
|
|
549
567
|
throw error;
|
|
@@ -712,7 +730,14 @@ export class RealtimeSession {
|
|
|
712
730
|
this.sendWake('catchup-required');
|
|
713
731
|
return;
|
|
714
732
|
}
|
|
715
|
-
const frames: ResponseFrame[] = [
|
|
733
|
+
const frames: ResponseFrame[] = [
|
|
734
|
+
{
|
|
735
|
+
type: 'RESP_HEADER',
|
|
736
|
+
...(this.wireVersion >= 2
|
|
737
|
+
? { logEpoch: this.logEpoch, resetRequired: false }
|
|
738
|
+
: {}),
|
|
739
|
+
},
|
|
740
|
+
];
|
|
716
741
|
for (const section of sections) {
|
|
717
742
|
frames.push({
|
|
718
743
|
type: 'SUB_START',
|
|
@@ -733,7 +758,7 @@ export class RealtimeSession {
|
|
|
733
758
|
frames.push({ type: 'SUB_END', nextCursor: commit.commitSeq });
|
|
734
759
|
}
|
|
735
760
|
const bytes = encodeMessage({
|
|
736
|
-
wireVersion:
|
|
761
|
+
wireVersion: this.wireVersion,
|
|
737
762
|
msgKind: 'response',
|
|
738
763
|
frames,
|
|
739
764
|
});
|
|
@@ -963,6 +988,11 @@ export class RealtimeHub {
|
|
|
963
988
|
async connect(options: RealtimeConnectOptions): Promise<RealtimeSession> {
|
|
964
989
|
const { storage } = this.#config;
|
|
965
990
|
const clock = this.#config.clock ?? Date.now;
|
|
991
|
+
const registry = await touchAuthenticatedPartition({
|
|
992
|
+
...this.#config,
|
|
993
|
+
partition: options.partition,
|
|
994
|
+
actorId: options.actorId,
|
|
995
|
+
});
|
|
966
996
|
if (options.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
|
|
967
997
|
throw syncError(
|
|
968
998
|
'sync.invalid_client_id',
|
|
@@ -995,6 +1025,8 @@ export class RealtimeHub {
|
|
|
995
1025
|
clock,
|
|
996
1026
|
this.#config.maxDeltaBytes ?? DEFAULT_MAX_DELTA_BYTES,
|
|
997
1027
|
storage,
|
|
1028
|
+
registry.logEpoch,
|
|
1029
|
+
record?.wireVersion ?? PROTOCOL_WIRE_VERSION,
|
|
998
1030
|
this.#config.events,
|
|
999
1031
|
);
|
|
1000
1032
|
this.#sessions.add(session);
|
|
@@ -1079,12 +1111,19 @@ export function createRealtimeHub(config: RealtimeHubConfig): RealtimeHub {
|
|
|
1079
1111
|
/** §8.7 failures: the socket has no HTTP status surface, so a
|
|
1080
1112
|
* request-level failure becomes a minimal RESP_HEADER/ERROR/END
|
|
1081
1113
|
* response message delivered as the round's response stream. */
|
|
1082
|
-
function errorResponseBytes(
|
|
1114
|
+
function errorResponseBytes(
|
|
1115
|
+
error: SyncError,
|
|
1116
|
+
wireVersion: number,
|
|
1117
|
+
logEpoch: string,
|
|
1118
|
+
): Uint8Array {
|
|
1083
1119
|
return encodeMessage({
|
|
1084
|
-
wireVersion
|
|
1120
|
+
wireVersion,
|
|
1085
1121
|
msgKind: 'response',
|
|
1086
1122
|
frames: [
|
|
1087
|
-
{
|
|
1123
|
+
{
|
|
1124
|
+
type: 'RESP_HEADER',
|
|
1125
|
+
...(wireVersion >= 2 ? { logEpoch, resetRequired: false } : {}),
|
|
1126
|
+
},
|
|
1088
1127
|
{
|
|
1089
1128
|
type: 'ERROR',
|
|
1090
1129
|
code: error.code,
|
package/src/restore.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { PartitionRegistryEntry, ServerStorage } from './storage';
|
|
2
|
+
|
|
3
|
+
export interface RotatePartitionLogEpochOptions {
|
|
4
|
+
readonly storage: ServerStorage;
|
|
5
|
+
readonly partition: string;
|
|
6
|
+
readonly nowMs?: number;
|
|
7
|
+
/** Deterministic operator/test override. Defaults to a random UUID. */
|
|
8
|
+
readonly logEpoch?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Fence clients from a prior authoritative database timeline after restore.
|
|
13
|
+
* Call while traffic and realtime delivery remain stopped (§2.1).
|
|
14
|
+
*/
|
|
15
|
+
export function rotatePartitionLogEpoch(
|
|
16
|
+
options: RotatePartitionLogEpochOptions,
|
|
17
|
+
): Promise<PartitionRegistryEntry> {
|
|
18
|
+
if (options.partition.length === 0) {
|
|
19
|
+
throw new TypeError('partition must be non-empty');
|
|
20
|
+
}
|
|
21
|
+
const logEpoch = options.logEpoch ?? crypto.randomUUID();
|
|
22
|
+
if (logEpoch.length === 0) throw new TypeError('logEpoch must be non-empty');
|
|
23
|
+
return options.storage.rotatePartitionLogEpoch(
|
|
24
|
+
options.partition,
|
|
25
|
+
logEpoch,
|
|
26
|
+
options.nowMs ?? Date.now(),
|
|
27
|
+
);
|
|
28
|
+
}
|
package/src/s3-segment-store.ts
CHANGED
|
@@ -152,7 +152,13 @@ function parseRecordJson(json: string, source: string): SegmentRecord {
|
|
|
152
152
|
throw new Error(`S3SegmentStore: corrupt record in ${source}`);
|
|
153
153
|
}
|
|
154
154
|
const r = parsed as Record<string, unknown>;
|
|
155
|
-
const strings = [
|
|
155
|
+
const strings = [
|
|
156
|
+
'segmentId',
|
|
157
|
+
'partition',
|
|
158
|
+
'logEpoch',
|
|
159
|
+
'table',
|
|
160
|
+
'scopeDigest',
|
|
161
|
+
] as const;
|
|
156
162
|
const numbers = [
|
|
157
163
|
'schemaVersion',
|
|
158
164
|
'asOfCommitSeq',
|
|
@@ -182,6 +188,7 @@ function parseRecordJson(json: string, source: string): SegmentRecord {
|
|
|
182
188
|
return {
|
|
183
189
|
segmentId: r.segmentId as string,
|
|
184
190
|
partition: r.partition as string,
|
|
191
|
+
logEpoch: r.logEpoch as string,
|
|
185
192
|
table: r.table as string,
|
|
186
193
|
schemaVersion: r.schemaVersion as number,
|
|
187
194
|
mediaType: r.mediaType,
|
|
@@ -223,6 +230,7 @@ export class S3SegmentStore implements SegmentStore {
|
|
|
223
230
|
async #findKeyFor(key: SegmentFindKey): Promise<string> {
|
|
224
231
|
const canonical = JSON.stringify([
|
|
225
232
|
key.partition,
|
|
233
|
+
key.logEpoch,
|
|
226
234
|
key.table,
|
|
227
235
|
key.schemaVersion,
|
|
228
236
|
key.mediaType,
|
|
@@ -431,6 +439,7 @@ export class S3SegmentStore implements SegmentStore {
|
|
|
431
439
|
const record = parseRecordJson(await response.text(), 'reuse pointer');
|
|
432
440
|
if (
|
|
433
441
|
record.partition !== key.partition ||
|
|
442
|
+
record.logEpoch !== key.logEpoch ||
|
|
434
443
|
record.table !== key.table ||
|
|
435
444
|
record.schemaVersion !== key.schemaVersion ||
|
|
436
445
|
record.mediaType !== key.mediaType ||
|
package/src/seed.ts
CHANGED
|
@@ -186,8 +186,52 @@ export async function seedMutations(
|
|
|
186
186
|
};
|
|
187
187
|
});
|
|
188
188
|
|
|
189
|
+
const requestContext = {
|
|
190
|
+
...config,
|
|
191
|
+
partition: target.partition,
|
|
192
|
+
actorId: target.actorId,
|
|
193
|
+
};
|
|
194
|
+
const acquisition = decodeMessage(
|
|
195
|
+
await handleSyncRequest(
|
|
196
|
+
encodeMessage({
|
|
197
|
+
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
198
|
+
msgKind: 'request',
|
|
199
|
+
frames: [
|
|
200
|
+
{
|
|
201
|
+
type: 'REQ_HEADER',
|
|
202
|
+
clientId,
|
|
203
|
+
schemaVersion: config.schema.version,
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
type: 'PULL_HEADER',
|
|
207
|
+
limitCommits: 0,
|
|
208
|
+
limitSnapshotRows: 0,
|
|
209
|
+
maxSnapshotPages: 0,
|
|
210
|
+
accept: 0b0011,
|
|
211
|
+
},
|
|
212
|
+
],
|
|
213
|
+
}),
|
|
214
|
+
requestContext,
|
|
215
|
+
),
|
|
216
|
+
);
|
|
217
|
+
const acquisitionHeader = acquisition.frames[0];
|
|
218
|
+
if (
|
|
219
|
+
acquisition.msgKind !== 'response' ||
|
|
220
|
+
acquisitionHeader?.type !== 'RESP_HEADER' ||
|
|
221
|
+
acquisitionHeader.logEpoch === undefined
|
|
222
|
+
) {
|
|
223
|
+
throw new SyncError(
|
|
224
|
+
'sync.invalid_request',
|
|
225
|
+
'seedMutations: epoch acquisition returned an invalid response',
|
|
226
|
+
);
|
|
227
|
+
}
|
|
189
228
|
const frames: RequestFrame[] = [
|
|
190
|
-
{
|
|
229
|
+
{
|
|
230
|
+
type: 'REQ_HEADER',
|
|
231
|
+
clientId,
|
|
232
|
+
schemaVersion: config.schema.version,
|
|
233
|
+
logEpoch: acquisitionHeader.logEpoch,
|
|
234
|
+
},
|
|
191
235
|
{ type: 'PUSH_COMMIT', clientCommitId, operations },
|
|
192
236
|
// A pull that asks for nothing: the round exists for its push half.
|
|
193
237
|
{
|
|
@@ -221,9 +265,7 @@ export async function seedMutations(
|
|
|
221
265
|
frames,
|
|
222
266
|
}),
|
|
223
267
|
{
|
|
224
|
-
...
|
|
225
|
-
partition: target.partition,
|
|
226
|
-
actorId: target.actorId,
|
|
268
|
+
...requestContext,
|
|
227
269
|
events:
|
|
228
270
|
config.events === undefined
|
|
229
271
|
? capture
|
package/src/segment-download.ts
CHANGED
|
@@ -9,7 +9,11 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import type { ScopeMap } from '@syncular/core';
|
|
11
11
|
import type { SyncRequestContext } from './context';
|
|
12
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
clockOf,
|
|
14
|
+
RESOLVER_OUTAGE,
|
|
15
|
+
touchAuthenticatedPartition,
|
|
16
|
+
} from './context';
|
|
13
17
|
import { SyncError, syncError } from './errors';
|
|
14
18
|
import { emitEvent } from './events';
|
|
15
19
|
import { compileSchema } from './schema';
|
|
@@ -96,8 +100,13 @@ async function downloadSegment(
|
|
|
96
100
|
ctx: SyncRequestContext,
|
|
97
101
|
request: SegmentDownloadRequest,
|
|
98
102
|
): Promise<SegmentDownloadResult> {
|
|
103
|
+
const registry = await touchAuthenticatedPartition(ctx);
|
|
99
104
|
const entry = await ctx.segments.get(request.segmentId);
|
|
100
|
-
if (
|
|
105
|
+
if (
|
|
106
|
+
entry === undefined ||
|
|
107
|
+
entry.record.partition !== ctx.partition ||
|
|
108
|
+
entry.record.logEpoch !== registry.logEpoch
|
|
109
|
+
) {
|
|
101
110
|
throw syncError('sync.not_found', 'unknown segment (§5.5)');
|
|
102
111
|
}
|
|
103
112
|
if (entry.record.expiresAtMs <= clockOf(ctx)()) {
|
package/src/segment-store.ts
CHANGED
|
@@ -12,6 +12,8 @@ export const DEFAULT_SEGMENT_TTL_MS = 24 * 60 * 60 * 1000;
|
|
|
12
12
|
|
|
13
13
|
export interface SegmentMetadata {
|
|
14
14
|
readonly partition: string;
|
|
15
|
+
/** Partition log continuity that produced these bytes (§2.1). */
|
|
16
|
+
readonly logEpoch: string;
|
|
15
17
|
readonly table: string;
|
|
16
18
|
readonly schemaVersion: number;
|
|
17
19
|
readonly mediaType: 'rows' | 'sqlite';
|
|
@@ -36,6 +38,7 @@ export interface SegmentRecord extends SegmentMetadata {
|
|
|
36
38
|
*/
|
|
37
39
|
export interface SegmentFindKey {
|
|
38
40
|
readonly partition: string;
|
|
41
|
+
readonly logEpoch: string;
|
|
39
42
|
readonly table: string;
|
|
40
43
|
readonly schemaVersion: number;
|
|
41
44
|
readonly mediaType: 'rows' | 'sqlite';
|
|
@@ -126,6 +129,7 @@ export class MemorySegmentStore implements SegmentStore {
|
|
|
126
129
|
for (const { record } of this.#entries.values()) {
|
|
127
130
|
if (
|
|
128
131
|
record.partition === key.partition &&
|
|
132
|
+
record.logEpoch === key.logEpoch &&
|
|
129
133
|
record.table === key.table &&
|
|
130
134
|
record.schemaVersion === key.schemaVersion &&
|
|
131
135
|
record.mediaType === key.mediaType &&
|
package/src/sqlite-blob-store.ts
CHANGED
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* SQLite-backed blob store
|
|
3
|
-
* dependency-free). Bun-specific by design (top-level `bun:sqlite` import),
|
|
4
|
-
* so it lives in its own module — the runtime-neutral `BlobStore` interface,
|
|
5
|
-
* `MemoryBlobStore`, `blobIdFor`, and `isBlobId` stay in `blob-store.ts` for
|
|
6
|
-
* the Workers/edge core (runtime neutrality is enforced by
|
|
7
|
-
* `test/runtime-neutrality.test.ts`).
|
|
2
|
+
* SQLite-backed blob store over the shared synchronous driver.
|
|
8
3
|
*/
|
|
9
|
-
import { Database } from 'bun:sqlite';
|
|
10
4
|
import type { BlobRecord, BlobStore, BlobStoreStats } from './blob-store';
|
|
5
|
+
import {
|
|
6
|
+
SqliteAdapterRequiredError,
|
|
7
|
+
type SqliteDatabase,
|
|
8
|
+
} from './sqlite-driver';
|
|
11
9
|
|
|
12
10
|
export class SqliteBlobStore implements BlobStore {
|
|
13
|
-
readonly db:
|
|
11
|
+
readonly db: SqliteDatabase;
|
|
14
12
|
|
|
15
|
-
constructor(db:
|
|
16
|
-
|
|
13
|
+
constructor(db: SqliteDatabase | string = ':memory:') {
|
|
14
|
+
if (typeof db === 'string') {
|
|
15
|
+
throw new SqliteAdapterRequiredError();
|
|
16
|
+
}
|
|
17
|
+
this.db = db;
|
|
17
18
|
this.db.exec(`
|
|
18
19
|
CREATE TABLE IF NOT EXISTS sync_blobs(
|
|
19
20
|
partition TEXT NOT NULL, blob_id TEXT NOT NULL,
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { Database, type SQLQueryBindings } from 'bun:sqlite';
|
|
2
|
+
import type {
|
|
3
|
+
SqliteDatabase,
|
|
4
|
+
SqliteRunResult,
|
|
5
|
+
SqliteStatement,
|
|
6
|
+
SqliteValue,
|
|
7
|
+
} from './sqlite-driver';
|
|
8
|
+
|
|
9
|
+
export class BunSqliteDatabase implements SqliteDatabase {
|
|
10
|
+
readonly native: Database;
|
|
11
|
+
|
|
12
|
+
constructor(value: string | Database = ':memory:') {
|
|
13
|
+
this.native = typeof value === 'string' ? new Database(value) : value;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
static deserialize(bytes: Uint8Array): BunSqliteDatabase {
|
|
17
|
+
return new BunSqliteDatabase(Database.deserialize(bytes));
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
exec(sql: string): void {
|
|
21
|
+
this.native.exec(sql);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
run(sql: string, bindings: readonly SqliteValue[] = []): SqliteRunResult {
|
|
25
|
+
return this.native.run(sql, [...bindings]);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
query<Row, Params extends readonly SqliteValue[]>(
|
|
29
|
+
sql: string,
|
|
30
|
+
): SqliteStatement<Row, Params> {
|
|
31
|
+
const statement = this.native.query<Row, SQLQueryBindings[]>(sql);
|
|
32
|
+
return {
|
|
33
|
+
run: (...params) => statement.run(...params),
|
|
34
|
+
get: (...params) => statement.get(...params),
|
|
35
|
+
all: (...params) => statement.all(...params),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
serialize(): Uint8Array {
|
|
40
|
+
return new Uint8Array(this.native.serialize());
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
close(): void {
|
|
44
|
+
this.native.close();
|
|
45
|
+
}
|
|
46
|
+
}
|