@syncular/server 0.15.47 → 0.15.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/admin.d.ts +1 -5
- package/dist/admin.js +2 -7
- package/dist/blob-handlers.js +4 -1
- package/dist/context.d.ts +3 -1
- package/dist/context.js +4 -0
- package/dist/d1-storage.d.ts +4 -1
- package/dist/d1-storage.js +75 -11
- package/dist/errors.js +6 -0
- package/dist/events.d.ts +2 -1
- package/dist/frame-bytes.d.ts +2 -2
- package/dist/frame-bytes.js +33 -14
- package/dist/handler.js +58 -14
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/operations.js +2 -1
- package/dist/postgres-storage.d.ts +5 -2
- package/dist/postgres-storage.js +71 -10
- package/dist/pull.d.ts +1 -1
- package/dist/pull.js +11 -5
- package/dist/realtime.d.ts +4 -1
- package/dist/realtime.js +33 -9
- package/dist/restore.d.ts +13 -0
- package/dist/restore.js +13 -0
- package/dist/s3-segment-store.js +10 -1
- package/dist/seed.js +36 -4
- package/dist/segment-download.js +5 -2
- package/dist/segment-store.d.ts +3 -0
- package/dist/segment-store.js +1 -0
- package/dist/sqlite-bun-driver.d.ts +2 -1
- package/dist/sqlite-bun-driver.js +5 -2
- package/dist/sqlite-dialect.d.ts +1 -1
- package/dist/sqlite-dialect.js +7 -0
- package/dist/sqlite-segment-store.js +14 -5
- package/dist/sqlite-storage.d.ts +4 -1
- package/dist/sqlite-storage.js +81 -11
- package/dist/storage.d.ts +18 -5
- package/package.json +2 -2
- package/src/admin.ts +3 -9
- package/src/blob-handlers.ts +8 -1
- package/src/context.ts +16 -1
- package/src/d1-storage.ts +107 -12
- package/src/errors.ts +6 -0
- package/src/events.ts +2 -1
- package/src/frame-bytes.ts +40 -14
- package/src/handler.ts +102 -29
- package/src/index.ts +1 -0
- package/src/operations.ts +6 -1
- package/src/postgres-storage.ts +102 -13
- package/src/pull.ts +11 -0
- package/src/realtime.ts +46 -7
- package/src/restore.ts +28 -0
- package/src/s3-segment-store.ts +10 -1
- package/src/seed.ts +46 -4
- package/src/segment-download.ts +11 -2
- package/src/segment-store.ts +4 -0
- package/src/sqlite-bun-driver.ts +6 -2
- package/src/sqlite-dialect.ts +7 -0
- package/src/sqlite-segment-store.ts +18 -4
- package/src/sqlite-storage.ts +118 -12
- package/src/storage.ts +28 -5
|
@@ -15,6 +15,7 @@ export class SqliteSegmentStore {
|
|
|
15
15
|
this.db.exec(`
|
|
16
16
|
CREATE TABLE IF NOT EXISTS sync_segments(
|
|
17
17
|
segment_id TEXT PRIMARY KEY, partition TEXT NOT NULL,
|
|
18
|
+
log_epoch TEXT NOT NULL,
|
|
18
19
|
tbl TEXT NOT NULL, schema_version INTEGER NOT NULL,
|
|
19
20
|
media_type TEXT NOT NULL, scope_digest TEXT NOT NULL,
|
|
20
21
|
as_of_commit_seq INTEGER NOT NULL, row_count INTEGER NOT NULL,
|
|
@@ -23,6 +24,12 @@ export class SqliteSegmentStore {
|
|
|
23
24
|
expires_at_ms INTEGER NOT NULL, bytes BLOB NOT NULL
|
|
24
25
|
);
|
|
25
26
|
`);
|
|
27
|
+
const columns = this.db
|
|
28
|
+
.query('PRAGMA table_info(sync_segments)')
|
|
29
|
+
.all();
|
|
30
|
+
if (!columns.some((column) => column.name === 'log_epoch')) {
|
|
31
|
+
this.db.exec("ALTER TABLE sync_segments ADD COLUMN log_epoch TEXT NOT NULL DEFAULT ''");
|
|
32
|
+
}
|
|
26
33
|
}
|
|
27
34
|
async put(metadata, bytes, nowMs) {
|
|
28
35
|
const segmentId = await segmentIdFor(bytes);
|
|
@@ -35,11 +42,11 @@ export class SqliteSegmentStore {
|
|
|
35
42
|
};
|
|
36
43
|
this.db
|
|
37
44
|
.query(`INSERT OR REPLACE INTO sync_segments(
|
|
38
|
-
segment_id, partition, tbl, schema_version, media_type,
|
|
45
|
+
segment_id, partition, log_epoch, tbl, schema_version, media_type,
|
|
39
46
|
scope_digest, as_of_commit_seq, row_count, row_cursor,
|
|
40
47
|
next_row_cursor, byte_length, created_at_ms, expires_at_ms, bytes
|
|
41
|
-
) VALUES (
|
|
42
|
-
.run(record.segmentId, record.partition, record.table, record.schemaVersion, record.mediaType, record.scopeDigest, record.asOfCommitSeq, record.rowCount, record.rowCursor, record.nextRowCursor, record.byteLength, record.createdAtMs, record.expiresAtMs, bytes);
|
|
48
|
+
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
|
|
49
|
+
.run(record.segmentId, record.partition, record.logEpoch, record.table, record.schemaVersion, record.mediaType, record.scopeDigest, record.asOfCommitSeq, record.rowCount, record.rowCursor, record.nextRowCursor, record.byteLength, record.createdAtMs, record.expiresAtMs, bytes);
|
|
43
50
|
return record;
|
|
44
51
|
}
|
|
45
52
|
async get(segmentId) {
|
|
@@ -52,6 +59,7 @@ export class SqliteSegmentStore {
|
|
|
52
59
|
record: {
|
|
53
60
|
segmentId: row.segment_id,
|
|
54
61
|
partition: row.partition,
|
|
62
|
+
logEpoch: row.log_epoch,
|
|
55
63
|
table: row.tbl,
|
|
56
64
|
schemaVersion: row.schema_version,
|
|
57
65
|
mediaType: row.media_type === 'sqlite' ? 'sqlite' : 'rows',
|
|
@@ -72,16 +80,17 @@ export class SqliteSegmentStore {
|
|
|
72
80
|
.query(`SELECT segment_id, row_count, next_row_cursor, byte_length,
|
|
73
81
|
created_at_ms, expires_at_ms
|
|
74
82
|
FROM sync_segments
|
|
75
|
-
WHERE partition=? AND tbl=? AND schema_version=? AND media_type=?
|
|
83
|
+
WHERE partition=? AND log_epoch=? AND tbl=? AND schema_version=? AND media_type=?
|
|
76
84
|
AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS NULL
|
|
77
85
|
AND expires_at_ms > ?
|
|
78
86
|
LIMIT 1`)
|
|
79
|
-
.get(key.partition, key.table, key.schemaVersion, key.mediaType, key.scopeDigest, key.asOfCommitSeq, nowMs);
|
|
87
|
+
.get(key.partition, key.logEpoch, key.table, key.schemaVersion, key.mediaType, key.scopeDigest, key.asOfCommitSeq, nowMs);
|
|
80
88
|
if (row === null)
|
|
81
89
|
return undefined;
|
|
82
90
|
return {
|
|
83
91
|
segmentId: row.segment_id,
|
|
84
92
|
partition: key.partition,
|
|
93
|
+
logEpoch: key.logEpoch,
|
|
85
94
|
table: key.table,
|
|
86
95
|
schemaVersion: key.schemaVersion,
|
|
87
96
|
mediaType: key.mediaType,
|
package/dist/sqlite-storage.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
2
2
|
import { type SqliteDatabase } from './sqlite-driver.js';
|
|
3
|
-
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, StorageTransaction, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
3
|
+
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, StorageTransaction, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
4
4
|
export declare class SqliteServerStorage implements ServerStorage {
|
|
5
5
|
#private;
|
|
6
6
|
readonly db: SqliteDatabase;
|
|
@@ -8,6 +8,9 @@ export declare class SqliteServerStorage implements ServerStorage {
|
|
|
8
8
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
9
9
|
table(name: string): CompiledTable;
|
|
10
10
|
ensureSchema(schema: CompiledSchema): Promise<void>;
|
|
11
|
+
touchPartition(partition: string, authenticatedAtMs: number, initialLogEpoch: string): Promise<PartitionRegistryEntry>;
|
|
12
|
+
rotatePartitionLogEpoch(partition: string, logEpoch: string, authenticatedAtMs: number): Promise<PartitionRegistryEntry>;
|
|
13
|
+
listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
|
|
11
14
|
begin(partition: string): Promise<StorageTransaction>;
|
|
12
15
|
/** Internal: write a row + refresh its scope-index entries. */
|
|
13
16
|
writeRow(partition: string, table: string, row: StoredRow): void;
|
package/dist/sqlite-storage.js
CHANGED
|
@@ -219,6 +219,12 @@ export class SqliteServerStorage {
|
|
|
219
219
|
}
|
|
220
220
|
this.db = db;
|
|
221
221
|
this.db.exec(SQLITE_DDL);
|
|
222
|
+
const clientColumns = this.db
|
|
223
|
+
.query('PRAGMA table_info("sync_clients")')
|
|
224
|
+
.all();
|
|
225
|
+
if (!clientColumns.some((column) => column.name === 'wire_version')) {
|
|
226
|
+
this.db.exec('ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1');
|
|
227
|
+
}
|
|
222
228
|
}
|
|
223
229
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
224
230
|
table(name) {
|
|
@@ -294,6 +300,74 @@ export class SqliteServerStorage {
|
|
|
294
300
|
this.#tables = schema.tables;
|
|
295
301
|
this.#schemaVersion = schema.version;
|
|
296
302
|
}
|
|
303
|
+
async touchPartition(partition, authenticatedAtMs, initialLogEpoch) {
|
|
304
|
+
if (initialLogEpoch.length === 0) {
|
|
305
|
+
throw new Error('initial log epoch must be non-empty');
|
|
306
|
+
}
|
|
307
|
+
this.db
|
|
308
|
+
.query(`INSERT INTO sync_partition_registry(
|
|
309
|
+
partition, log_epoch, last_authenticated_at_ms
|
|
310
|
+
) VALUES (?,?,?)
|
|
311
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
312
|
+
last_authenticated_at_ms=excluded.last_authenticated_at_ms`)
|
|
313
|
+
.run(partition, initialLogEpoch, authenticatedAtMs);
|
|
314
|
+
const row = this.db
|
|
315
|
+
.query(`SELECT log_epoch, epoch_required, last_authenticated_at_ms
|
|
316
|
+
FROM sync_partition_registry WHERE partition=?`)
|
|
317
|
+
.get(partition);
|
|
318
|
+
if (row === null)
|
|
319
|
+
throw new Error('partition registry write did not persist');
|
|
320
|
+
return {
|
|
321
|
+
partition,
|
|
322
|
+
logEpoch: row.log_epoch,
|
|
323
|
+
epochRequired: row.epoch_required === 1,
|
|
324
|
+
lastAuthenticatedAtMs: row.last_authenticated_at_ms,
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
async rotatePartitionLogEpoch(partition, logEpoch, authenticatedAtMs) {
|
|
328
|
+
if (logEpoch.length === 0)
|
|
329
|
+
throw new Error('log epoch must be non-empty');
|
|
330
|
+
return this.#serializeReactionWrite(() => {
|
|
331
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
332
|
+
try {
|
|
333
|
+
this.db
|
|
334
|
+
.query(`INSERT INTO sync_partition_registry(
|
|
335
|
+
partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
336
|
+
) VALUES (?,?,1,?)
|
|
337
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
338
|
+
log_epoch=excluded.log_epoch,
|
|
339
|
+
epoch_required=1,
|
|
340
|
+
last_authenticated_at_ms=excluded.last_authenticated_at_ms`)
|
|
341
|
+
.run(partition, logEpoch, authenticatedAtMs);
|
|
342
|
+
this.db
|
|
343
|
+
.query('DELETE FROM sync_clients WHERE partition=?')
|
|
344
|
+
.run(partition);
|
|
345
|
+
this.db.exec('COMMIT');
|
|
346
|
+
}
|
|
347
|
+
catch (error) {
|
|
348
|
+
this.db.exec('ROLLBACK');
|
|
349
|
+
throw error;
|
|
350
|
+
}
|
|
351
|
+
return {
|
|
352
|
+
partition,
|
|
353
|
+
logEpoch,
|
|
354
|
+
epochRequired: true,
|
|
355
|
+
lastAuthenticatedAtMs: authenticatedAtMs,
|
|
356
|
+
};
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
async listPartitionRegistry() {
|
|
360
|
+
return this.db
|
|
361
|
+
.query(`SELECT partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
362
|
+
FROM sync_partition_registry ORDER BY partition`)
|
|
363
|
+
.all()
|
|
364
|
+
.map((row) => ({
|
|
365
|
+
partition: row.partition,
|
|
366
|
+
logEpoch: row.log_epoch,
|
|
367
|
+
epochRequired: row.epoch_required === 1,
|
|
368
|
+
lastAuthenticatedAtMs: row.last_authenticated_at_ms,
|
|
369
|
+
}));
|
|
370
|
+
}
|
|
297
371
|
/**
|
|
298
372
|
* Migration rewrite: keyset-paged walk of a row table. When `oldLayout` is
|
|
299
373
|
* given every payload re-encodes under
|
|
@@ -659,13 +733,14 @@ export class SqliteServerStorage {
|
|
|
659
733
|
}
|
|
660
734
|
async getClientRecord(partition, clientId) {
|
|
661
735
|
const record = this.db
|
|
662
|
-
.query('SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?')
|
|
736
|
+
.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?')
|
|
663
737
|
.get(partition, clientId);
|
|
664
738
|
if (record === null)
|
|
665
739
|
return undefined;
|
|
666
740
|
return {
|
|
667
741
|
clientId: record.client_id,
|
|
668
742
|
actorId: record.actor_id,
|
|
743
|
+
wireVersion: record.wire_version,
|
|
669
744
|
cursor: record.cursor,
|
|
670
745
|
updatedAtMs: record.updated_at_ms,
|
|
671
746
|
subscriptions: JSON.parse(record.subscriptions),
|
|
@@ -673,8 +748,8 @@ export class SqliteServerStorage {
|
|
|
673
748
|
}
|
|
674
749
|
async putClientRecord(partition, record) {
|
|
675
750
|
this.db
|
|
676
|
-
.query('INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, cursor, subscriptions, updated_at_ms) VALUES (
|
|
677
|
-
.run(partition, record.clientId, record.actorId, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs);
|
|
751
|
+
.query('INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms) VALUES (?,?,?,?,?,?,?)')
|
|
752
|
+
.run(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs);
|
|
678
753
|
}
|
|
679
754
|
async listClientCursors(partition) {
|
|
680
755
|
const records = this.db
|
|
@@ -719,11 +794,12 @@ export class SqliteServerStorage {
|
|
|
719
794
|
// -- admin/console read surface --------------------------------------------
|
|
720
795
|
async listClientRecords(partition) {
|
|
721
796
|
const records = this.db
|
|
722
|
-
.query('SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
|
|
797
|
+
.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
|
|
723
798
|
.all(partition);
|
|
724
799
|
return records.map((record) => ({
|
|
725
800
|
clientId: record.client_id,
|
|
726
801
|
actorId: record.actor_id,
|
|
802
|
+
wireVersion: record.wire_version,
|
|
727
803
|
cursor: record.cursor,
|
|
728
804
|
updatedAtMs: record.updated_at_ms,
|
|
729
805
|
subscriptions: JSON.parse(record.subscriptions),
|
|
@@ -800,12 +876,6 @@ export class SqliteServerStorage {
|
|
|
800
876
|
};
|
|
801
877
|
}
|
|
802
878
|
async listPartitions() {
|
|
803
|
-
|
|
804
|
-
// first pull — a partition with only one of the two still shows up.
|
|
805
|
-
const rows = this.db
|
|
806
|
-
.query(`SELECT partition FROM sync_partitions
|
|
807
|
-
UNION SELECT partition FROM sync_clients ORDER BY partition`)
|
|
808
|
-
.all();
|
|
809
|
-
return rows.map((r) => r.partition);
|
|
879
|
+
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|
|
810
880
|
}
|
|
811
881
|
}
|
package/dist/storage.d.ts
CHANGED
|
@@ -144,6 +144,8 @@ export interface ClientSubscription {
|
|
|
144
144
|
export interface ClientRecord {
|
|
145
145
|
readonly clientId: string;
|
|
146
146
|
readonly actorId: string;
|
|
147
|
+
/** SSP2 version last accepted from this client; selects realtime deltas. */
|
|
148
|
+
readonly wireVersion: number;
|
|
147
149
|
/** Minimum `nextCursor` across the last pull's active subscriptions. */
|
|
148
150
|
readonly cursor: number;
|
|
149
151
|
readonly updatedAtMs: number;
|
|
@@ -195,6 +197,13 @@ export interface ClientCursorInfo {
|
|
|
195
197
|
readonly cursor: number;
|
|
196
198
|
readonly updatedAtMs: number;
|
|
197
199
|
}
|
|
200
|
+
/** Durable partition identity refreshed after host authentication (§2.1). */
|
|
201
|
+
export interface PartitionRegistryEntry {
|
|
202
|
+
readonly partition: string;
|
|
203
|
+
readonly logEpoch: string;
|
|
204
|
+
readonly epochRequired: boolean;
|
|
205
|
+
readonly lastAuthenticatedAtMs: number;
|
|
206
|
+
}
|
|
198
207
|
/**
|
|
199
208
|
* Commit-log metadata (no change payloads) for the admin/console read
|
|
200
209
|
* surface. `changeCount` is the number of changes the commit carries;
|
|
@@ -340,6 +349,12 @@ export interface ServerStorage {
|
|
|
340
349
|
* lazily as a defensive backstop.
|
|
341
350
|
*/
|
|
342
351
|
ensureSchema(schema: CompiledSchema): Promise<void>;
|
|
352
|
+
/** Create or refresh the authenticated partition registry row (§2.1). */
|
|
353
|
+
touchPartition(partition: string, authenticatedAtMs: number, initialLogEpoch: string): Promise<PartitionRegistryEntry>;
|
|
354
|
+
/** Rotate log continuity after restore and discard stale cursor records. */
|
|
355
|
+
rotatePartitionLogEpoch(partition: string, logEpoch: string, authenticatedAtMs: number): Promise<PartitionRegistryEntry>;
|
|
356
|
+
/** Registry entries ordered by partition for maintenance loops. */
|
|
357
|
+
listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
|
|
343
358
|
begin(partition: string): Promise<StorageTransaction>;
|
|
344
359
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
345
360
|
getHorizonSeq(partition: string): Promise<number>;
|
|
@@ -427,10 +442,8 @@ export interface ServerStorage {
|
|
|
427
442
|
* change scope index (never a log scan).
|
|
428
443
|
* `getRowScopes`: the (table, rowId) row's current server_version and
|
|
429
444
|
* stored scopes without decoding its payload — the row inspector.
|
|
430
|
-
* `listPartitions`:
|
|
431
|
-
*
|
|
432
|
-
* records, sorted. Powers the console's fleet view / partition picker;
|
|
433
|
-
* deliberately NOT partition-scoped (the one cross-partition read).
|
|
445
|
+
* `listPartitions`: the partition-only compatibility view of
|
|
446
|
+
* `listPartitionRegistry`, sorted.
|
|
434
447
|
*/
|
|
435
448
|
listClientRecords?(partition: string): Promise<ClientRecord[]>;
|
|
436
449
|
listCommitMetadata?(partition: string, query: CommitMetadataQuery): Promise<CommitMetadata[]>;
|
|
@@ -439,7 +452,7 @@ export interface ServerStorage {
|
|
|
439
452
|
serverVersion: number;
|
|
440
453
|
scopes: Record<string, string>;
|
|
441
454
|
} | undefined>;
|
|
442
|
-
listPartitions
|
|
455
|
+
listPartitions(): Promise<string[]>;
|
|
443
456
|
}
|
|
444
457
|
/** A row referencing a blob, with the scopes needed to authorize download. */
|
|
445
458
|
export interface BlobReferencingRow {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.15.
|
|
3
|
+
"version": "0.15.48",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.15.
|
|
71
|
+
"@syncular/core": "0.15.48"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/admin.ts
CHANGED
|
@@ -520,17 +520,11 @@ export class SyncularAdmin {
|
|
|
520
520
|
};
|
|
521
521
|
}
|
|
522
522
|
|
|
523
|
-
/**
|
|
524
|
-
* Every partition the storage knows (commit log + client records) — the
|
|
525
|
-
* fleet-view backing and the console's partition picker. Fails loud when
|
|
526
|
-
* the backend omits the optional `listPartitions`.
|
|
527
|
-
*/
|
|
523
|
+
/** Every authenticated partition in the storage-backed registry. */
|
|
528
524
|
async listPartitions(): Promise<string[]> {
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
'storage',
|
|
525
|
+
return (await this.#storage.listPartitionRegistry()).map(
|
|
526
|
+
(entry) => entry.partition,
|
|
532
527
|
);
|
|
533
|
-
return list();
|
|
534
528
|
}
|
|
535
529
|
|
|
536
530
|
/**
|
package/src/blob-handlers.ts
CHANGED
|
@@ -8,7 +8,11 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { blobIdFor, isBlobId } from './blob-store';
|
|
10
10
|
import type { SyncRequestContext } from './context';
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
clockOf,
|
|
13
|
+
RESOLVER_OUTAGE,
|
|
14
|
+
touchAuthenticatedPartition,
|
|
15
|
+
} from './context';
|
|
12
16
|
import { SyncError, syncError } from './errors';
|
|
13
17
|
import { emitEvent } from './events';
|
|
14
18
|
import { compileSchema } from './schema';
|
|
@@ -72,6 +76,7 @@ export async function handleBlobUpload(
|
|
|
72
76
|
ctx: SyncRequestContext,
|
|
73
77
|
request: BlobUploadRequest,
|
|
74
78
|
): Promise<void> {
|
|
79
|
+
await touchAuthenticatedPartition(ctx);
|
|
75
80
|
const store = ctx.blobs;
|
|
76
81
|
if (store === undefined) {
|
|
77
82
|
throw syncError('blob.not_found', 'this server has no blob store (§5.9)');
|
|
@@ -125,6 +130,7 @@ export async function handleBlobDownload(
|
|
|
125
130
|
ctx: SyncRequestContext,
|
|
126
131
|
blobId: string,
|
|
127
132
|
): Promise<BlobDownloadResult> {
|
|
133
|
+
await touchAuthenticatedPartition(ctx);
|
|
128
134
|
const events = ctx.events;
|
|
129
135
|
if (events === undefined) return downloadBlob(ctx, blobId);
|
|
130
136
|
const clock = clockOf(ctx);
|
|
@@ -301,6 +307,7 @@ export async function handleBlobUploadGrant(
|
|
|
301
307
|
ctx: SyncRequestContext,
|
|
302
308
|
request: BlobUploadGrantRequest,
|
|
303
309
|
): Promise<BlobUploadGrantResult> {
|
|
310
|
+
await touchAuthenticatedPartition(ctx);
|
|
304
311
|
const store = ctx.blobs;
|
|
305
312
|
if (store === undefined) {
|
|
306
313
|
throw syncError('blob.not_found', 'this server has no blob store (§5.9)');
|
package/src/context.ts
CHANGED
|
@@ -18,7 +18,11 @@ import type {
|
|
|
18
18
|
SegmentUrlConfig,
|
|
19
19
|
} from './signed-url';
|
|
20
20
|
import type { SqliteImageBuilder } from './sqlite-image';
|
|
21
|
-
import type {
|
|
21
|
+
import type {
|
|
22
|
+
PartitionRegistryEntry,
|
|
23
|
+
ServerStorage,
|
|
24
|
+
StoredCommit,
|
|
25
|
+
} from './storage';
|
|
22
26
|
import type { CommitValidator, ValidatorRegistry } from './validate';
|
|
23
27
|
|
|
24
28
|
/** SSP2 body content type (§1.1). */
|
|
@@ -192,3 +196,14 @@ export function clockOf(ctx: SyncServerConfig): () => number {
|
|
|
192
196
|
export function limitsOf(ctx: SyncServerConfig): ServerLimits {
|
|
193
197
|
return { ...DEFAULT_LIMITS, ...ctx.limits };
|
|
194
198
|
}
|
|
199
|
+
|
|
200
|
+
/** Refresh the registry after host authentication and return log continuity. */
|
|
201
|
+
export function touchAuthenticatedPartition(
|
|
202
|
+
ctx: SyncRequestContext,
|
|
203
|
+
): Promise<PartitionRegistryEntry> {
|
|
204
|
+
return ctx.storage.touchPartition(
|
|
205
|
+
ctx.partition,
|
|
206
|
+
clockOf(ctx)(),
|
|
207
|
+
crypto.randomUUID(),
|
|
208
|
+
);
|
|
209
|
+
}
|
package/src/d1-storage.ts
CHANGED
|
@@ -95,6 +95,7 @@ import type {
|
|
|
95
95
|
IndexRowScanQuery,
|
|
96
96
|
NewCommit,
|
|
97
97
|
NewReaction,
|
|
98
|
+
PartitionRegistryEntry,
|
|
98
99
|
PrunedReactionCounts,
|
|
99
100
|
ReactionClaimQuery,
|
|
100
101
|
ReactionFailure,
|
|
@@ -752,6 +753,14 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
752
753
|
for (const statement of sqliteDdlStatements()) {
|
|
753
754
|
await this.#db.exec(`${statement.replace(/\s+/g, ' ')};`);
|
|
754
755
|
}
|
|
756
|
+
const { results } = await this.#db
|
|
757
|
+
.prepare('PRAGMA table_info("sync_clients")')
|
|
758
|
+
.all<{ name: string }>();
|
|
759
|
+
if (!results.some((column) => column.name === 'wire_version')) {
|
|
760
|
+
await this.#db.exec(
|
|
761
|
+
'ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1;',
|
|
762
|
+
);
|
|
763
|
+
}
|
|
755
764
|
}
|
|
756
765
|
|
|
757
766
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
@@ -859,6 +868,95 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
859
868
|
this.#schemaVersion = schema.version;
|
|
860
869
|
}
|
|
861
870
|
|
|
871
|
+
async touchPartition(
|
|
872
|
+
partition: string,
|
|
873
|
+
authenticatedAtMs: number,
|
|
874
|
+
initialLogEpoch: string,
|
|
875
|
+
): Promise<PartitionRegistryEntry> {
|
|
876
|
+
if (initialLogEpoch.length === 0) {
|
|
877
|
+
throw new Error('initial log epoch must be non-empty');
|
|
878
|
+
}
|
|
879
|
+
await this.#db
|
|
880
|
+
.prepare(
|
|
881
|
+
`INSERT INTO sync_partition_registry(
|
|
882
|
+
partition, log_epoch, last_authenticated_at_ms
|
|
883
|
+
) VALUES (?,?,?)
|
|
884
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
885
|
+
last_authenticated_at_ms=excluded.last_authenticated_at_ms`,
|
|
886
|
+
)
|
|
887
|
+
.bind(partition, initialLogEpoch, authenticatedAtMs)
|
|
888
|
+
.run();
|
|
889
|
+
const row = await this.#db
|
|
890
|
+
.prepare(
|
|
891
|
+
`SELECT log_epoch, epoch_required, last_authenticated_at_ms
|
|
892
|
+
FROM sync_partition_registry WHERE partition=?`,
|
|
893
|
+
)
|
|
894
|
+
.bind(partition)
|
|
895
|
+
.first<{
|
|
896
|
+
log_epoch: string;
|
|
897
|
+
epoch_required: number;
|
|
898
|
+
last_authenticated_at_ms: number;
|
|
899
|
+
}>();
|
|
900
|
+
if (row === null)
|
|
901
|
+
throw new Error('partition registry write did not persist');
|
|
902
|
+
return {
|
|
903
|
+
partition,
|
|
904
|
+
logEpoch: row.log_epoch,
|
|
905
|
+
epochRequired: row.epoch_required === 1,
|
|
906
|
+
lastAuthenticatedAtMs: row.last_authenticated_at_ms,
|
|
907
|
+
};
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
async rotatePartitionLogEpoch(
|
|
911
|
+
partition: string,
|
|
912
|
+
logEpoch: string,
|
|
913
|
+
authenticatedAtMs: number,
|
|
914
|
+
): Promise<PartitionRegistryEntry> {
|
|
915
|
+
if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
|
|
916
|
+
await this.#db.batch([
|
|
917
|
+
this.#db
|
|
918
|
+
.prepare(
|
|
919
|
+
`INSERT INTO sync_partition_registry(
|
|
920
|
+
partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
921
|
+
) VALUES (?,?,1,?)
|
|
922
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
923
|
+
log_epoch=excluded.log_epoch,
|
|
924
|
+
epoch_required=1,
|
|
925
|
+
last_authenticated_at_ms=excluded.last_authenticated_at_ms`,
|
|
926
|
+
)
|
|
927
|
+
.bind(partition, logEpoch, authenticatedAtMs),
|
|
928
|
+
this.#db
|
|
929
|
+
.prepare('DELETE FROM sync_clients WHERE partition=?')
|
|
930
|
+
.bind(partition),
|
|
931
|
+
]);
|
|
932
|
+
return {
|
|
933
|
+
partition,
|
|
934
|
+
logEpoch,
|
|
935
|
+
epochRequired: true,
|
|
936
|
+
lastAuthenticatedAtMs: authenticatedAtMs,
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
async listPartitionRegistry(): Promise<PartitionRegistryEntry[]> {
|
|
941
|
+
const { results } = await this.#db
|
|
942
|
+
.prepare(
|
|
943
|
+
`SELECT partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
944
|
+
FROM sync_partition_registry ORDER BY partition`,
|
|
945
|
+
)
|
|
946
|
+
.all<{
|
|
947
|
+
partition: string;
|
|
948
|
+
log_epoch: string;
|
|
949
|
+
epoch_required: number;
|
|
950
|
+
last_authenticated_at_ms: number;
|
|
951
|
+
}>();
|
|
952
|
+
return results.map((row) => ({
|
|
953
|
+
partition: row.partition,
|
|
954
|
+
logEpoch: row.log_epoch,
|
|
955
|
+
epochRequired: row.epoch_required === 1,
|
|
956
|
+
lastAuthenticatedAtMs: row.last_authenticated_at_ms,
|
|
957
|
+
}));
|
|
958
|
+
}
|
|
959
|
+
|
|
862
960
|
/** Keyset-paged migration rewrite (see the sqlite storage's counterpart). */
|
|
863
961
|
async #rewriteRows(
|
|
864
962
|
table: CompiledTable,
|
|
@@ -1390,12 +1488,13 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
1390
1488
|
): Promise<ClientRecord | undefined> {
|
|
1391
1489
|
const record = await this.#db
|
|
1392
1490
|
.prepare(
|
|
1393
|
-
'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?',
|
|
1491
|
+
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?',
|
|
1394
1492
|
)
|
|
1395
1493
|
.bind(partition, clientId)
|
|
1396
1494
|
.first<{
|
|
1397
1495
|
client_id: string;
|
|
1398
1496
|
actor_id: string;
|
|
1497
|
+
wire_version: number;
|
|
1399
1498
|
cursor: number;
|
|
1400
1499
|
subscriptions: string;
|
|
1401
1500
|
updated_at_ms: number;
|
|
@@ -1404,6 +1503,7 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
1404
1503
|
return {
|
|
1405
1504
|
clientId: record.client_id,
|
|
1406
1505
|
actorId: record.actor_id,
|
|
1506
|
+
wireVersion: record.wire_version,
|
|
1407
1507
|
cursor: record.cursor,
|
|
1408
1508
|
updatedAtMs: record.updated_at_ms,
|
|
1409
1509
|
subscriptions: JSON.parse(record.subscriptions) as ClientSubscription[],
|
|
@@ -1416,12 +1516,13 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
1416
1516
|
): Promise<void> {
|
|
1417
1517
|
await this.#db
|
|
1418
1518
|
.prepare(
|
|
1419
|
-
'INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, cursor, subscriptions, updated_at_ms) VALUES (
|
|
1519
|
+
'INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms) VALUES (?,?,?,?,?,?,?)',
|
|
1420
1520
|
)
|
|
1421
1521
|
.bind(
|
|
1422
1522
|
partition,
|
|
1423
1523
|
record.clientId,
|
|
1424
1524
|
record.actorId,
|
|
1525
|
+
record.wireVersion,
|
|
1425
1526
|
record.cursor,
|
|
1426
1527
|
JSON.stringify(record.subscriptions),
|
|
1427
1528
|
record.updatedAtMs,
|
|
@@ -1494,12 +1595,13 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
1494
1595
|
async listClientRecords(partition: string): Promise<ClientRecord[]> {
|
|
1495
1596
|
const { results } = await this.#db
|
|
1496
1597
|
.prepare(
|
|
1497
|
-
'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC',
|
|
1598
|
+
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC',
|
|
1498
1599
|
)
|
|
1499
1600
|
.bind(partition)
|
|
1500
1601
|
.all<{
|
|
1501
1602
|
client_id: string;
|
|
1502
1603
|
actor_id: string;
|
|
1604
|
+
wire_version: number;
|
|
1503
1605
|
cursor: number;
|
|
1504
1606
|
subscriptions: string;
|
|
1505
1607
|
updated_at_ms: number;
|
|
@@ -1507,6 +1609,7 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
1507
1609
|
return results.map((record) => ({
|
|
1508
1610
|
clientId: record.client_id,
|
|
1509
1611
|
actorId: record.actor_id,
|
|
1612
|
+
wireVersion: record.wire_version,
|
|
1510
1613
|
cursor: record.cursor,
|
|
1511
1614
|
updatedAtMs: record.updated_at_ms,
|
|
1512
1615
|
subscriptions: JSON.parse(record.subscriptions) as ClientSubscription[],
|
|
@@ -1628,14 +1731,6 @@ export class D1ServerStorage implements ServerStorage {
|
|
|
1628
1731
|
}
|
|
1629
1732
|
|
|
1630
1733
|
async listPartitions(): Promise<string[]> {
|
|
1631
|
-
|
|
1632
|
-
// first pull — a partition with only one of the two still shows up.
|
|
1633
|
-
const { results } = await this.#db
|
|
1634
|
-
.prepare(
|
|
1635
|
-
`SELECT partition FROM sync_partitions
|
|
1636
|
-
UNION SELECT partition FROM sync_clients ORDER BY partition`,
|
|
1637
|
-
)
|
|
1638
|
-
.all<{ partition: string }>();
|
|
1639
|
-
return results.map((r) => r.partition);
|
|
1734
|
+
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|
|
1640
1735
|
}
|
|
1641
1736
|
}
|
package/src/errors.ts
CHANGED
|
@@ -199,6 +199,12 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
|
|
|
199
199
|
recommendedAction: 'upgradeClient',
|
|
200
200
|
httpStatus: 400,
|
|
201
201
|
},
|
|
202
|
+
'sync.client_wire_unsupported': {
|
|
203
|
+
category: 'schema-mismatch',
|
|
204
|
+
retryable: false,
|
|
205
|
+
recommendedAction: 'upgradeClient',
|
|
206
|
+
httpStatus: 400,
|
|
207
|
+
},
|
|
202
208
|
'sync.websocket_connection_limit': {
|
|
203
209
|
category: 'rate-limited',
|
|
204
210
|
retryable: true,
|
package/src/events.ts
CHANGED
|
@@ -29,10 +29,11 @@ export interface RequestHandledEvent {
|
|
|
29
29
|
/**
|
|
30
30
|
* `ok` — response streamed to END;
|
|
31
31
|
* `schema_floor` — §2.4 required-schema answer;
|
|
32
|
+
* `reset` — §2.1 log-epoch reset answer;
|
|
32
33
|
* `rejected` — request validation failed before any bytes (§1.7);
|
|
33
34
|
* `error` — in-band ERROR frame (§1.6) or a thrown host failure.
|
|
34
35
|
*/
|
|
35
|
-
readonly outcome: 'ok' | 'schema_floor' | 'rejected' | 'error';
|
|
36
|
+
readonly outcome: 'ok' | 'schema_floor' | 'reset' | 'rejected' | 'error';
|
|
36
37
|
/** §10.2 code for `rejected`/`error`; `"internal"` for non-SyncErrors. */
|
|
37
38
|
readonly errorCode?: string;
|
|
38
39
|
readonly pushCommits: number;
|