@syncular/server 0.15.46 → 0.15.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -4
- package/dist/admin.d.ts +1 -5
- package/dist/admin.js +2 -7
- package/dist/blob-handlers.js +4 -1
- package/dist/context.d.ts +5 -2
- package/dist/context.js +4 -0
- package/dist/d1-storage.d.ts +4 -1
- package/dist/d1-storage.js +75 -11
- package/dist/errors.js +6 -0
- package/dist/events.d.ts +2 -1
- package/dist/frame-bytes.d.ts +2 -2
- package/dist/frame-bytes.js +33 -14
- package/dist/handler.js +58 -14
- package/dist/index-bun.d.ts +2 -0
- package/dist/index-bun.js +2 -0
- package/dist/index-node.d.ts +2 -0
- package/dist/index-node.js +2 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +4 -6
- package/dist/operations.js +2 -1
- package/dist/postgres-storage.d.ts +5 -2
- package/dist/postgres-storage.js +71 -10
- package/dist/pull.d.ts +1 -1
- package/dist/pull.js +12 -6
- package/dist/realtime.d.ts +4 -1
- package/dist/realtime.js +33 -9
- package/dist/restore.d.ts +13 -0
- package/dist/restore.js +13 -0
- package/dist/s3-segment-store.js +10 -1
- package/dist/seed.js +36 -4
- package/dist/segment-download.js +5 -2
- package/dist/segment-store.d.ts +3 -0
- package/dist/segment-store.js +1 -0
- package/dist/sqlite-blob-store.d.ts +4 -9
- package/dist/sqlite-blob-store.js +5 -10
- package/dist/sqlite-bun-driver.d.ts +12 -0
- package/dist/sqlite-bun-driver.js +30 -0
- package/dist/sqlite-bun.d.ts +24 -0
- package/dist/sqlite-bun.js +40 -0
- package/dist/sqlite-dialect.d.ts +8 -8
- package/dist/sqlite-dialect.js +9 -2
- package/dist/sqlite-driver.d.ts +26 -0
- package/dist/sqlite-driver.js +8 -0
- package/dist/sqlite-image.d.ts +7 -9
- package/dist/sqlite-image.js +26 -28
- package/dist/sqlite-lease-store.d.ts +4 -9
- package/dist/sqlite-lease-store.js +5 -10
- package/dist/sqlite-node-driver.d.ts +10 -0
- package/dist/sqlite-node-driver.js +30 -0
- package/dist/sqlite-node.d.ts +24 -0
- package/dist/sqlite-node.js +50 -0
- package/dist/sqlite-segment-store.d.ts +4 -10
- package/dist/sqlite-segment-store.js +20 -14
- package/dist/sqlite-storage.d.ts +7 -12
- package/dist/sqlite-storage.js +89 -16
- package/dist/storage-errors.js +4 -1
- package/dist/storage.d.ts +18 -5
- package/package.json +18 -3
- package/src/admin.ts +3 -9
- package/src/blob-handlers.ts +8 -1
- package/src/context.ts +18 -2
- package/src/d1-storage.ts +107 -12
- package/src/errors.ts +6 -0
- package/src/events.ts +2 -1
- package/src/frame-bytes.ts +40 -14
- package/src/handler.ts +102 -29
- package/src/index-bun.ts +9 -0
- package/src/index-node.ts +9 -0
- package/src/index.ts +9 -6
- package/src/operations.ts +6 -1
- package/src/postgres-storage.ts +102 -13
- package/src/pull.ts +12 -1
- package/src/realtime.ts +46 -7
- package/src/restore.ts +28 -0
- package/src/s3-segment-store.ts +10 -1
- package/src/seed.ts +46 -4
- package/src/segment-download.ts +11 -2
- package/src/segment-store.ts +4 -0
- package/src/sqlite-blob-store.ts +11 -10
- package/src/sqlite-bun-driver.ts +46 -0
- package/src/sqlite-bun.ts +53 -0
- package/src/sqlite-dialect.ts +14 -7
- package/src/sqlite-driver.ts +44 -0
- package/src/sqlite-image.ts +44 -49
- package/src/sqlite-lease-store.ts +11 -10
- package/src/sqlite-node-driver.ts +46 -0
- package/src/sqlite-node.ts +62 -0
- package/src/sqlite-segment-store.ts +29 -15
- package/src/sqlite-storage.ts +131 -19
- package/src/storage-errors.ts +4 -1
- package/src/storage.ts +28 -5
package/src/sqlite-storage.ts
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* SQLite storage
|
|
2
|
+
* SQLite server storage over the shared synchronous driver.
|
|
3
3
|
*
|
|
4
4
|
* Scope fanout is index-first: both the commit log and the
|
|
5
5
|
* current-row table carry a (table, variable, value) inverted index; reads
|
|
6
6
|
* select candidates from the index and verify the full multi-variable
|
|
7
7
|
* match against the stored scope map — never a log scan.
|
|
8
8
|
*/
|
|
9
|
-
import { Database } from 'bun:sqlite';
|
|
10
9
|
import {
|
|
11
10
|
bindAuthoritativePartition,
|
|
12
11
|
prepareAuthoritativeQuery,
|
|
@@ -45,6 +44,10 @@ import {
|
|
|
45
44
|
serializePushResult,
|
|
46
45
|
toStoredRow,
|
|
47
46
|
} from './sqlite-dialect';
|
|
47
|
+
import {
|
|
48
|
+
SqliteAdapterRequiredError,
|
|
49
|
+
type SqliteDatabase,
|
|
50
|
+
} from './sqlite-driver';
|
|
48
51
|
import type {
|
|
49
52
|
AuthoritativeQueryRequest,
|
|
50
53
|
AuthoritativeQueryResult,
|
|
@@ -59,6 +62,7 @@ import type {
|
|
|
59
62
|
IndexRowScanQuery,
|
|
60
63
|
NewCommit,
|
|
61
64
|
NewReaction,
|
|
65
|
+
PartitionRegistryEntry,
|
|
62
66
|
PrunedReactionCounts,
|
|
63
67
|
ReactionClaimQuery,
|
|
64
68
|
ReactionFailure,
|
|
@@ -160,7 +164,7 @@ class SqliteTransaction implements StorageTransaction {
|
|
|
160
164
|
clientCommitId: string,
|
|
161
165
|
): Promise<StoredPushResult | undefined> {
|
|
162
166
|
this.#assertOpen();
|
|
163
|
-
// One shared
|
|
167
|
+
// One shared SQLite connection: this read runs inside this
|
|
164
168
|
// transaction's BEGIN IMMEDIATE.
|
|
165
169
|
return this.#storage.getPushResult(
|
|
166
170
|
this.#partition,
|
|
@@ -379,8 +383,8 @@ class SqliteTransaction implements StorageTransaction {
|
|
|
379
383
|
}
|
|
380
384
|
|
|
381
385
|
export class SqliteServerStorage implements ServerStorage {
|
|
382
|
-
readonly db:
|
|
383
|
-
/** One
|
|
386
|
+
readonly db: SqliteDatabase;
|
|
387
|
+
/** One SQLite connection can own only one transaction at a time. */
|
|
384
388
|
#transactionTail: Promise<void> = Promise.resolve();
|
|
385
389
|
/** Set by `ensureSchema`: app-table lookup for the relational row store. */
|
|
386
390
|
#tables: ReadonlyMap<string, CompiledTable> | undefined;
|
|
@@ -400,9 +404,20 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
400
404
|
}
|
|
401
405
|
}
|
|
402
406
|
|
|
403
|
-
constructor(db:
|
|
404
|
-
|
|
407
|
+
constructor(db: SqliteDatabase | string = ':memory:') {
|
|
408
|
+
if (typeof db === 'string') {
|
|
409
|
+
throw new SqliteAdapterRequiredError();
|
|
410
|
+
}
|
|
411
|
+
this.db = db;
|
|
405
412
|
this.db.exec(SQLITE_DDL);
|
|
413
|
+
const clientColumns = this.db
|
|
414
|
+
.query<{ name: string }, []>('PRAGMA table_info("sync_clients")')
|
|
415
|
+
.all();
|
|
416
|
+
if (!clientColumns.some((column) => column.name === 'wire_version')) {
|
|
417
|
+
this.db.exec(
|
|
418
|
+
'ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1',
|
|
419
|
+
);
|
|
420
|
+
}
|
|
406
421
|
}
|
|
407
422
|
|
|
408
423
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
@@ -498,6 +513,106 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
498
513
|
this.#schemaVersion = schema.version;
|
|
499
514
|
}
|
|
500
515
|
|
|
516
|
+
async touchPartition(
|
|
517
|
+
partition: string,
|
|
518
|
+
authenticatedAtMs: number,
|
|
519
|
+
initialLogEpoch: string,
|
|
520
|
+
): Promise<PartitionRegistryEntry> {
|
|
521
|
+
if (initialLogEpoch.length === 0) {
|
|
522
|
+
throw new Error('initial log epoch must be non-empty');
|
|
523
|
+
}
|
|
524
|
+
this.db
|
|
525
|
+
.query(
|
|
526
|
+
`INSERT INTO sync_partition_registry(
|
|
527
|
+
partition, log_epoch, last_authenticated_at_ms
|
|
528
|
+
) VALUES (?,?,?)
|
|
529
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
530
|
+
last_authenticated_at_ms=excluded.last_authenticated_at_ms`,
|
|
531
|
+
)
|
|
532
|
+
.run(partition, initialLogEpoch, authenticatedAtMs);
|
|
533
|
+
const row = this.db
|
|
534
|
+
.query<
|
|
535
|
+
{
|
|
536
|
+
log_epoch: string;
|
|
537
|
+
epoch_required: number;
|
|
538
|
+
last_authenticated_at_ms: number;
|
|
539
|
+
},
|
|
540
|
+
[string]
|
|
541
|
+
>(
|
|
542
|
+
`SELECT log_epoch, epoch_required, last_authenticated_at_ms
|
|
543
|
+
FROM sync_partition_registry WHERE partition=?`,
|
|
544
|
+
)
|
|
545
|
+
.get(partition);
|
|
546
|
+
if (row === null)
|
|
547
|
+
throw new Error('partition registry write did not persist');
|
|
548
|
+
return {
|
|
549
|
+
partition,
|
|
550
|
+
logEpoch: row.log_epoch,
|
|
551
|
+
epochRequired: row.epoch_required === 1,
|
|
552
|
+
lastAuthenticatedAtMs: row.last_authenticated_at_ms,
|
|
553
|
+
};
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
async rotatePartitionLogEpoch(
|
|
557
|
+
partition: string,
|
|
558
|
+
logEpoch: string,
|
|
559
|
+
authenticatedAtMs: number,
|
|
560
|
+
): Promise<PartitionRegistryEntry> {
|
|
561
|
+
if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
|
|
562
|
+
return this.#serializeReactionWrite(() => {
|
|
563
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
564
|
+
try {
|
|
565
|
+
this.db
|
|
566
|
+
.query(
|
|
567
|
+
`INSERT INTO sync_partition_registry(
|
|
568
|
+
partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
569
|
+
) VALUES (?,?,1,?)
|
|
570
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
571
|
+
log_epoch=excluded.log_epoch,
|
|
572
|
+
epoch_required=1,
|
|
573
|
+
last_authenticated_at_ms=excluded.last_authenticated_at_ms`,
|
|
574
|
+
)
|
|
575
|
+
.run(partition, logEpoch, authenticatedAtMs);
|
|
576
|
+
this.db
|
|
577
|
+
.query('DELETE FROM sync_clients WHERE partition=?')
|
|
578
|
+
.run(partition);
|
|
579
|
+
this.db.exec('COMMIT');
|
|
580
|
+
} catch (error) {
|
|
581
|
+
this.db.exec('ROLLBACK');
|
|
582
|
+
throw error;
|
|
583
|
+
}
|
|
584
|
+
return {
|
|
585
|
+
partition,
|
|
586
|
+
logEpoch,
|
|
587
|
+
epochRequired: true,
|
|
588
|
+
lastAuthenticatedAtMs: authenticatedAtMs,
|
|
589
|
+
};
|
|
590
|
+
});
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
async listPartitionRegistry(): Promise<PartitionRegistryEntry[]> {
|
|
594
|
+
return this.db
|
|
595
|
+
.query<
|
|
596
|
+
{
|
|
597
|
+
partition: string;
|
|
598
|
+
log_epoch: string;
|
|
599
|
+
epoch_required: number;
|
|
600
|
+
last_authenticated_at_ms: number;
|
|
601
|
+
},
|
|
602
|
+
[]
|
|
603
|
+
>(
|
|
604
|
+
`SELECT partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
605
|
+
FROM sync_partition_registry ORDER BY partition`,
|
|
606
|
+
)
|
|
607
|
+
.all()
|
|
608
|
+
.map((row) => ({
|
|
609
|
+
partition: row.partition,
|
|
610
|
+
logEpoch: row.log_epoch,
|
|
611
|
+
epochRequired: row.epoch_required === 1,
|
|
612
|
+
lastAuthenticatedAtMs: row.last_authenticated_at_ms,
|
|
613
|
+
}));
|
|
614
|
+
}
|
|
615
|
+
|
|
501
616
|
/**
|
|
502
617
|
* Migration rewrite: keyset-paged walk of a row table. When `oldLayout` is
|
|
503
618
|
* given every payload re-encodes under
|
|
@@ -1071,19 +1186,21 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
1071
1186
|
{
|
|
1072
1187
|
client_id: string;
|
|
1073
1188
|
actor_id: string;
|
|
1189
|
+
wire_version: number;
|
|
1074
1190
|
cursor: number;
|
|
1075
1191
|
subscriptions: string;
|
|
1076
1192
|
updated_at_ms: number;
|
|
1077
1193
|
},
|
|
1078
1194
|
[string, string]
|
|
1079
1195
|
>(
|
|
1080
|
-
'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?',
|
|
1196
|
+
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?',
|
|
1081
1197
|
)
|
|
1082
1198
|
.get(partition, clientId);
|
|
1083
1199
|
if (record === null) return undefined;
|
|
1084
1200
|
return {
|
|
1085
1201
|
clientId: record.client_id,
|
|
1086
1202
|
actorId: record.actor_id,
|
|
1203
|
+
wireVersion: record.wire_version,
|
|
1087
1204
|
cursor: record.cursor,
|
|
1088
1205
|
updatedAtMs: record.updated_at_ms,
|
|
1089
1206
|
subscriptions: JSON.parse(record.subscriptions) as ClientSubscription[],
|
|
@@ -1096,12 +1213,13 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
1096
1213
|
): Promise<void> {
|
|
1097
1214
|
this.db
|
|
1098
1215
|
.query(
|
|
1099
|
-
'INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, cursor, subscriptions, updated_at_ms) VALUES (
|
|
1216
|
+
'INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms) VALUES (?,?,?,?,?,?,?)',
|
|
1100
1217
|
)
|
|
1101
1218
|
.run(
|
|
1102
1219
|
partition,
|
|
1103
1220
|
record.clientId,
|
|
1104
1221
|
record.actorId,
|
|
1222
|
+
record.wireVersion,
|
|
1105
1223
|
record.cursor,
|
|
1106
1224
|
JSON.stringify(record.subscriptions),
|
|
1107
1225
|
record.updatedAtMs,
|
|
@@ -1181,18 +1299,20 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
1181
1299
|
{
|
|
1182
1300
|
client_id: string;
|
|
1183
1301
|
actor_id: string;
|
|
1302
|
+
wire_version: number;
|
|
1184
1303
|
cursor: number;
|
|
1185
1304
|
subscriptions: string;
|
|
1186
1305
|
updated_at_ms: number;
|
|
1187
1306
|
},
|
|
1188
1307
|
[string]
|
|
1189
1308
|
>(
|
|
1190
|
-
'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC',
|
|
1309
|
+
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC',
|
|
1191
1310
|
)
|
|
1192
1311
|
.all(partition);
|
|
1193
1312
|
return records.map((record) => ({
|
|
1194
1313
|
clientId: record.client_id,
|
|
1195
1314
|
actorId: record.actor_id,
|
|
1315
|
+
wireVersion: record.wire_version,
|
|
1196
1316
|
cursor: record.cursor,
|
|
1197
1317
|
updatedAtMs: record.updated_at_ms,
|
|
1198
1318
|
subscriptions: JSON.parse(record.subscriptions) as ClientSubscription[],
|
|
@@ -1318,14 +1438,6 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
1318
1438
|
}
|
|
1319
1439
|
|
|
1320
1440
|
async listPartitions(): Promise<string[]> {
|
|
1321
|
-
|
|
1322
|
-
// first pull — a partition with only one of the two still shows up.
|
|
1323
|
-
const rows = this.db
|
|
1324
|
-
.query<{ partition: string }, []>(
|
|
1325
|
-
`SELECT partition FROM sync_partitions
|
|
1326
|
-
UNION SELECT partition FROM sync_clients ORDER BY partition`,
|
|
1327
|
-
)
|
|
1328
|
-
.all();
|
|
1329
|
-
return rows.map((r) => r.partition);
|
|
1441
|
+
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|
|
1330
1442
|
}
|
|
1331
1443
|
}
|
package/src/storage-errors.ts
CHANGED
|
@@ -52,6 +52,7 @@ export class StorageQueryError extends Error {
|
|
|
52
52
|
interface DriverError {
|
|
53
53
|
readonly code?: unknown;
|
|
54
54
|
readonly errno?: unknown;
|
|
55
|
+
readonly errcode?: unknown;
|
|
55
56
|
readonly message?: unknown;
|
|
56
57
|
}
|
|
57
58
|
|
|
@@ -72,7 +73,9 @@ export function isSqliteConstraintError(error: unknown): boolean {
|
|
|
72
73
|
return true;
|
|
73
74
|
}
|
|
74
75
|
const errno = candidate?.errno;
|
|
75
|
-
|
|
76
|
+
if (typeof errno === 'number' && (errno & 0xff) === 19) return true;
|
|
77
|
+
const errcode = candidate?.errcode;
|
|
78
|
+
return typeof errcode === 'number' && (errcode & 0xff) === 19;
|
|
76
79
|
}
|
|
77
80
|
|
|
78
81
|
/** PostgreSQL SQLSTATE class 23: integrity constraint violation. */
|
package/src/storage.ts
CHANGED
|
@@ -164,6 +164,8 @@ export interface ClientSubscription {
|
|
|
164
164
|
export interface ClientRecord {
|
|
165
165
|
readonly clientId: string;
|
|
166
166
|
readonly actorId: string;
|
|
167
|
+
/** SSP2 version last accepted from this client; selects realtime deltas. */
|
|
168
|
+
readonly wireVersion: number;
|
|
167
169
|
/** Minimum `nextCursor` across the last pull's active subscriptions. */
|
|
168
170
|
readonly cursor: number;
|
|
169
171
|
readonly updatedAtMs: number;
|
|
@@ -220,6 +222,14 @@ export interface ClientCursorInfo {
|
|
|
220
222
|
readonly updatedAtMs: number;
|
|
221
223
|
}
|
|
222
224
|
|
|
225
|
+
/** Durable partition identity refreshed after host authentication (§2.1). */
|
|
226
|
+
export interface PartitionRegistryEntry {
|
|
227
|
+
readonly partition: string;
|
|
228
|
+
readonly logEpoch: string;
|
|
229
|
+
readonly epochRequired: boolean;
|
|
230
|
+
readonly lastAuthenticatedAtMs: number;
|
|
231
|
+
}
|
|
232
|
+
|
|
223
233
|
/**
|
|
224
234
|
* Commit-log metadata (no change payloads) for the admin/console read
|
|
225
235
|
* surface. `changeCount` is the number of changes the commit carries;
|
|
@@ -397,6 +407,21 @@ export interface ServerStorage {
|
|
|
397
407
|
*/
|
|
398
408
|
ensureSchema(schema: CompiledSchema): Promise<void>;
|
|
399
409
|
|
|
410
|
+
/** Create or refresh the authenticated partition registry row (§2.1). */
|
|
411
|
+
touchPartition(
|
|
412
|
+
partition: string,
|
|
413
|
+
authenticatedAtMs: number,
|
|
414
|
+
initialLogEpoch: string,
|
|
415
|
+
): Promise<PartitionRegistryEntry>;
|
|
416
|
+
/** Rotate log continuity after restore and discard stale cursor records. */
|
|
417
|
+
rotatePartitionLogEpoch(
|
|
418
|
+
partition: string,
|
|
419
|
+
logEpoch: string,
|
|
420
|
+
authenticatedAtMs: number,
|
|
421
|
+
): Promise<PartitionRegistryEntry>;
|
|
422
|
+
/** Registry entries ordered by partition for maintenance loops. */
|
|
423
|
+
listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
|
|
424
|
+
|
|
400
425
|
begin(partition: string): Promise<StorageTransaction>;
|
|
401
426
|
|
|
402
427
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
@@ -551,10 +576,8 @@ export interface ServerStorage {
|
|
|
551
576
|
* change scope index (never a log scan).
|
|
552
577
|
* `getRowScopes`: the (table, rowId) row's current server_version and
|
|
553
578
|
* stored scopes without decoding its payload — the row inspector.
|
|
554
|
-
* `listPartitions`:
|
|
555
|
-
*
|
|
556
|
-
* records, sorted. Powers the console's fleet view / partition picker;
|
|
557
|
-
* deliberately NOT partition-scoped (the one cross-partition read).
|
|
579
|
+
* `listPartitions`: the partition-only compatibility view of
|
|
580
|
+
* `listPartitionRegistry`, sorted.
|
|
558
581
|
*/
|
|
559
582
|
listClientRecords?(partition: string): Promise<ClientRecord[]>;
|
|
560
583
|
listCommitMetadata?(
|
|
@@ -572,7 +595,7 @@ export interface ServerStorage {
|
|
|
572
595
|
): Promise<
|
|
573
596
|
{ serverVersion: number; scopes: Record<string, string> } | undefined
|
|
574
597
|
>;
|
|
575
|
-
listPartitions
|
|
598
|
+
listPartitions(): Promise<string[]>;
|
|
576
599
|
}
|
|
577
600
|
|
|
578
601
|
/** A row referencing a blob, with the scopes needed to authorize download. */
|