@syncular/server 0.15.48 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -9
- package/dist/admin.js +1 -5
- package/dist/authoritative-query.d.ts +14 -6
- package/dist/authoritative-query.js +60 -82
- package/dist/d1-storage.d.ts +13 -1
- package/dist/d1-storage.js +468 -121
- package/dist/operations.d.ts +2 -0
- package/dist/operations.js +23 -4
- package/dist/postgres-storage.d.ts +6 -1
- package/dist/postgres-storage.js +80 -24
- package/dist/prune.d.ts +3 -1
- package/dist/prune.js +18 -14
- package/dist/pull.js +79 -39
- package/dist/push.js +5 -5
- package/dist/realtime.d.ts +1 -1
- package/dist/realtime.js +16 -10
- package/dist/relational-rows.d.ts +7 -1
- package/dist/relational-rows.js +17 -2
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-image.d.ts +3 -3
- package/dist/sqlite-image.js +10 -6
- package/dist/sqlite-node.js +2 -2
- package/dist/sqlite-storage.d.ts +6 -1
- package/dist/sqlite-storage.js +93 -32
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +7 -0
- package/dist/storage.d.ts +29 -5
- package/package.json +2 -2
- package/src/admin.ts +4 -6
- package/src/authoritative-query.ts +89 -94
- package/src/d1-storage.ts +641 -159
- package/src/operations.ts +31 -3
- package/src/postgres-storage.ts +146 -46
- package/src/prune.ts +29 -15
- package/src/pull.ts +102 -49
- package/src/push.ts +5 -5
- package/src/realtime.ts +20 -9
- package/src/relational-rows.ts +18 -2
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-image.ts +26 -15
- package/src/sqlite-node.ts +2 -2
- package/src/sqlite-storage.ts +131 -37
- package/src/storage-errors.ts +22 -1
- package/src/storage.ts +50 -5
package/dist/relational-rows.js
CHANGED
|
@@ -177,7 +177,7 @@ export function physicalIndexName(declaredName) {
|
|
|
177
177
|
}
|
|
178
178
|
/**
|
|
179
179
|
* CREATE INDEX IF NOT EXISTS for the table's user-declared indexes. These use
|
|
180
|
-
* the
|
|
180
|
+
* the declared columns with a leading server partition column. Cross-table
|
|
181
181
|
* index-name uniqueness is the user's schema concern, as it is client-side.
|
|
182
182
|
* Server-side the physical name carries the {@link SYNC_INDEX_PREFIX}
|
|
183
183
|
* ownership marker.
|
|
@@ -188,7 +188,7 @@ export function createIndexDdl(table) {
|
|
|
188
188
|
return [];
|
|
189
189
|
return table.indexes.map((index) => {
|
|
190
190
|
const unique = index.unique ? 'UNIQUE ' : '';
|
|
191
|
-
const columns = index.columns
|
|
191
|
+
const columns = [SYNC_PARTITION_COLUMN, ...index.columns]
|
|
192
192
|
.map((column) => quoteIdent(column))
|
|
193
193
|
.join(', ');
|
|
194
194
|
return `CREATE ${unique}INDEX IF NOT EXISTS ${quoteIdent(physicalIndexName(index.name))} ON ${quoteIdent(table.name)} (${columns})`;
|
|
@@ -401,6 +401,21 @@ export function deleteRowSql(table, dialect) {
|
|
|
401
401
|
const p = dialect === 'sqlite' ? ['?', '?'] : ['$1', '$2'];
|
|
402
402
|
return `DELETE FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${p[0]} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=${p[1]}`;
|
|
403
403
|
}
|
|
404
|
+
/**
|
|
405
|
+
* SQLite: remove the old row's exact scope keys before replacing or deleting the row.
|
|
406
|
+
* Params: [partition, table, rowId, partition, rowId]. The caller keeps this
|
|
407
|
+
* statement and the row write in the same transaction (or D1 atomic batch).
|
|
408
|
+
*/
|
|
409
|
+
export function deleteSqliteRowScopesSql(table) {
|
|
410
|
+
return `DELETE FROM sync_row_scopes
|
|
411
|
+
WHERE partition=? AND tbl=? AND row_id=?
|
|
412
|
+
AND (var, value) IN (
|
|
413
|
+
SELECT key, value FROM json_each((
|
|
414
|
+
SELECT ${quoteIdent(SYNC_SCOPES_COLUMN)} FROM ${quoteIdent(table.name)}
|
|
415
|
+
WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=?
|
|
416
|
+
))
|
|
417
|
+
)`;
|
|
418
|
+
}
|
|
404
419
|
/**
|
|
405
420
|
* The schema-version marker table gates DDL work. `ensureSchema` compares the
|
|
406
421
|
* stored version and skips
|
package/dist/sqlite-bun.js
CHANGED
|
@@ -27,10 +27,10 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
|
|
|
27
27
|
super(database(value), options);
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
|
-
export const buildSqliteImage = (input) => {
|
|
30
|
+
export const buildSqliteImage = async (input) => {
|
|
31
31
|
const db = new BunSqliteDatabase();
|
|
32
32
|
try {
|
|
33
|
-
writeSqliteImage(db, input);
|
|
33
|
+
await writeSqliteImage(db, input);
|
|
34
34
|
return db.serialize();
|
|
35
35
|
}
|
|
36
36
|
finally {
|
package/dist/sqlite-image.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ export interface SqliteImageInput {
|
|
|
10
10
|
readonly schemaVersion: number;
|
|
11
11
|
readonly asOfCommitSeq: number;
|
|
12
12
|
readonly scopeDigest: string;
|
|
13
|
-
readonly
|
|
13
|
+
readonly rowBatches: AsyncIterable<readonly StoredRow[]> | Iterable<readonly StoredRow[]>;
|
|
14
14
|
}
|
|
15
15
|
/**
|
|
16
16
|
* The §5.3 image-builder capability, injected through
|
|
@@ -20,6 +20,6 @@ export interface SqliteImageInput {
|
|
|
20
20
|
* on the pull path. A Bun or Node host passes `buildSqliteImage`; a Workers
|
|
21
21
|
* host omits it and serves the rows lane.
|
|
22
22
|
*/
|
|
23
|
-
export type SqliteImageBuilder = (input: SqliteImageInput) => Uint8Array
|
|
23
|
+
export type SqliteImageBuilder = (input: SqliteImageInput) => Promise<Uint8Array>;
|
|
24
24
|
/** Populate a §5.3 image database for a whole-table snapshot. */
|
|
25
|
-
export declare function writeSqliteImage(db: SqliteDatabase, input: SqliteImageInput): void
|
|
25
|
+
export declare function writeSqliteImage(db: SqliteDatabase, input: SqliteImageInput): Promise<void>;
|
package/dist/sqlite-image.js
CHANGED
|
@@ -45,8 +45,8 @@ function toSql(value) {
|
|
|
45
45
|
return value;
|
|
46
46
|
}
|
|
47
47
|
/** Populate a §5.3 image database for a whole-table snapshot. */
|
|
48
|
-
export function writeSqliteImage(db, input) {
|
|
49
|
-
const { table,
|
|
48
|
+
export async function writeSqliteImage(db, input) {
|
|
49
|
+
const { table, rowBatches } = input;
|
|
50
50
|
const primaryKey = table.columns[table.primaryKeyIndex]?.name;
|
|
51
51
|
const columnDefs = table.columns.map((column) => {
|
|
52
52
|
const notNull = column.nullable ? '' : ' NOT NULL';
|
|
@@ -59,7 +59,6 @@ export function writeSqliteImage(db, input) {
|
|
|
59
59
|
format INTEGER NOT NULL, "table" TEXT NOT NULL,
|
|
60
60
|
"schemaVersion" INTEGER NOT NULL, "asOfCommitSeq" INTEGER NOT NULL,
|
|
61
61
|
"scopeDigest" TEXT NOT NULL, "rowCount" INTEGER NOT NULL)`);
|
|
62
|
-
db.query(`INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`).run(table.name, input.schemaVersion, input.asOfCommitSeq, input.scopeDigest, rows.length);
|
|
63
62
|
const names = [
|
|
64
63
|
...table.columns.map((column) => quoteIdent(column.name)),
|
|
65
64
|
quoteIdent(IMAGE_VERSION_COLUMN),
|
|
@@ -68,10 +67,15 @@ export function writeSqliteImage(db, input) {
|
|
|
68
67
|
VALUES (${names.map(() => '?').join(', ')})`);
|
|
69
68
|
db.exec('BEGIN');
|
|
70
69
|
try {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
70
|
+
let rowCount = 0;
|
|
71
|
+
for await (const rows of rowBatches) {
|
|
72
|
+
for (const row of rows) {
|
|
73
|
+
const values = decodeRow(table.columns, row.payload);
|
|
74
|
+
insert.run(...values.map(toSql), row.serverVersion);
|
|
75
|
+
}
|
|
76
|
+
rowCount += rows.length;
|
|
74
77
|
}
|
|
78
|
+
db.query(`INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`).run(table.name, input.schemaVersion, input.asOfCommitSeq, input.scopeDigest, rowCount);
|
|
75
79
|
db.exec('COMMIT');
|
|
76
80
|
}
|
|
77
81
|
catch (error) {
|
package/dist/sqlite-node.js
CHANGED
|
@@ -30,13 +30,13 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
|
|
|
30
30
|
super(database(value), options);
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
|
-
export const buildSqliteImage = (input) => {
|
|
33
|
+
export const buildSqliteImage = async (input) => {
|
|
34
34
|
const directory = mkdtempSync(join(tmpdir(), 'syncular-server-image-'));
|
|
35
35
|
const path = join(directory, 'segment.db');
|
|
36
36
|
const db = new NodeSqliteDatabase(path);
|
|
37
37
|
try {
|
|
38
38
|
try {
|
|
39
|
-
writeSqliteImage(db, input);
|
|
39
|
+
await writeSqliteImage(db, input);
|
|
40
40
|
}
|
|
41
41
|
finally {
|
|
42
42
|
db.close();
|
package/dist/sqlite-storage.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
1
2
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
2
3
|
import { type SqliteDatabase } from './sqlite-driver.js';
|
|
3
4
|
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, StorageTransaction, StoredCommit, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
|
@@ -16,9 +17,10 @@ export declare class SqliteServerStorage implements ServerStorage {
|
|
|
16
17
|
writeRow(partition: string, table: string, row: StoredRow): void;
|
|
17
18
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
18
19
|
queryAuthoritative(partition: string, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
|
|
20
|
+
getPartitionLogEpoch(partition: string): Promise<string | undefined>;
|
|
19
21
|
getHorizonSeq(partition: string): Promise<number>;
|
|
20
22
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
21
|
-
pruneCommitsThrough(partition: string,
|
|
23
|
+
pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
|
|
22
24
|
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|
|
23
25
|
getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
24
26
|
getPushResult(partition: string, clientId: string, clientCommitId: string): Promise<StoredPushResult | undefined>;
|
|
@@ -35,6 +37,9 @@ export declare class SqliteServerStorage implements ServerStorage {
|
|
|
35
37
|
scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
|
36
38
|
getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
|
|
37
39
|
putClientRecord(partition: string, record: ClientRecord): Promise<void>;
|
|
40
|
+
advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
41
|
+
updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
42
|
+
getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
|
|
38
43
|
listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
|
|
39
44
|
listRowsReferencingBlob(partition: string, blobId: string): Promise<{
|
|
40
45
|
readonly table: string;
|
package/dist/sqlite-storage.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { validateCommitPruneQuery } from './prune.js';
|
|
2
|
+
import { StorageQueryError } from './storage-errors.js';
|
|
1
3
|
/**
|
|
2
4
|
* SQLite server storage over the shared synchronous driver.
|
|
3
5
|
*
|
|
@@ -8,7 +10,7 @@
|
|
|
8
10
|
*/
|
|
9
11
|
import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
10
12
|
import { syncError } from './errors.js';
|
|
11
|
-
import { commitWindowPageSql, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
13
|
+
import { commitWindowPageSql, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
12
14
|
import { matchesEffective } from './scopes.js';
|
|
13
15
|
import { collectCommitWindowPage, deserializePushResult, SQLITE_DDL, serializePushResult, toStoredRow, } from './sqlite-dialect.js';
|
|
14
16
|
import { SqliteAdapterRequiredError, } from './sqlite-driver.js';
|
|
@@ -106,8 +108,9 @@ class SqliteTransaction {
|
|
|
106
108
|
async deleteRow(table, rowId) {
|
|
107
109
|
this.#assertOpen();
|
|
108
110
|
const db = this.#storage.db;
|
|
109
|
-
|
|
110
|
-
db.query(
|
|
111
|
+
const compiled = this.#storage.table(table);
|
|
112
|
+
db.query(deleteSqliteRowScopesSql(compiled)).run(this.#partition, table, rowId, this.#partition, rowId);
|
|
113
|
+
db.query(deleteRowSql(compiled, 'sqlite')).run(this.#partition, rowId);
|
|
111
114
|
// §5.9.4: a deleted row references no blobs.
|
|
112
115
|
db.query('DELETE FROM sync_blob_refs WHERE partition=? AND tbl=? AND row_id=?').run(this.#partition, table, rowId);
|
|
113
116
|
}
|
|
@@ -199,7 +202,7 @@ export class SqliteServerStorage {
|
|
|
199
202
|
/** Set by `ensureSchema`: app-table lookup for the relational row store. */
|
|
200
203
|
#tables;
|
|
201
204
|
#schemaVersion;
|
|
202
|
-
async #
|
|
205
|
+
async #serializeWrite(operation) {
|
|
203
206
|
const previous = this.#transactionTail;
|
|
204
207
|
let release;
|
|
205
208
|
this.#transactionTail = new Promise((resolve) => {
|
|
@@ -275,6 +278,9 @@ export class SqliteServerStorage {
|
|
|
275
278
|
this.db
|
|
276
279
|
.query('DELETE FROM sync_row_scopes WHERE tbl=?')
|
|
277
280
|
.run(tableName);
|
|
281
|
+
this.db
|
|
282
|
+
.query('DELETE FROM sync_blob_refs WHERE tbl=?')
|
|
283
|
+
.run(tableName);
|
|
278
284
|
this.db.exec(dropTableDdl(tableName));
|
|
279
285
|
}
|
|
280
286
|
for (const statement of schemaDdl(schema, existing, 'sqlite', existingIndexes)) {
|
|
@@ -327,7 +333,7 @@ export class SqliteServerStorage {
|
|
|
327
333
|
async rotatePartitionLogEpoch(partition, logEpoch, authenticatedAtMs) {
|
|
328
334
|
if (logEpoch.length === 0)
|
|
329
335
|
throw new Error('log epoch must be non-empty');
|
|
330
|
-
return this.#
|
|
336
|
+
return this.#serializeWrite(() => {
|
|
331
337
|
this.db.exec('BEGIN IMMEDIATE');
|
|
332
338
|
try {
|
|
333
339
|
this.db
|
|
@@ -421,12 +427,12 @@ export class SqliteServerStorage {
|
|
|
421
427
|
/** Internal: write a row + refresh its scope-index entries. */
|
|
422
428
|
writeRow(partition, table, row) {
|
|
423
429
|
const compiled = this.table(table);
|
|
430
|
+
this.db
|
|
431
|
+
.query(deleteSqliteRowScopesSql(compiled))
|
|
432
|
+
.run(partition, table, row.rowId, partition, row.rowId);
|
|
424
433
|
this.db
|
|
425
434
|
.query(upsertSql(compiled, 'sqlite'))
|
|
426
435
|
.run(...upsertValues(compiled, partition, row, 'sqlite'));
|
|
427
|
-
this.db
|
|
428
|
-
.query('DELETE FROM sync_row_scopes WHERE partition=? AND tbl=? AND row_id=?')
|
|
429
|
-
.run(partition, table, row.rowId);
|
|
430
436
|
for (const [variable, value] of Object.entries(row.scopes)) {
|
|
431
437
|
this.db
|
|
432
438
|
.query('INSERT OR IGNORE INTO sync_row_scopes(partition, tbl, var, value, row_id) VALUES (?,?,?,?,?)')
|
|
@@ -443,7 +449,7 @@ export class SqliteServerStorage {
|
|
|
443
449
|
if (this.#tables === undefined) {
|
|
444
450
|
throw new Error('ensureSchema(schema) must run before registered queries');
|
|
445
451
|
}
|
|
446
|
-
const prepared = bindAuthoritativePartition(prepareAuthoritativeQuery(query.
|
|
452
|
+
const prepared = bindAuthoritativePartition(prepareAuthoritativeQuery(query.plan, query.params, query.tables, this.#tables), partition);
|
|
447
453
|
const previous = this.#transactionTail;
|
|
448
454
|
let release;
|
|
449
455
|
this.#transactionTail = new Promise((resolve) => {
|
|
@@ -473,6 +479,11 @@ export class SqliteServerStorage {
|
|
|
473
479
|
release();
|
|
474
480
|
}
|
|
475
481
|
}
|
|
482
|
+
async getPartitionLogEpoch(partition) {
|
|
483
|
+
return this.db
|
|
484
|
+
.query('SELECT log_epoch FROM sync_partition_registry WHERE partition=?')
|
|
485
|
+
.get(partition)?.log_epoch;
|
|
486
|
+
}
|
|
476
487
|
async getHorizonSeq(partition) {
|
|
477
488
|
const row = this.db
|
|
478
489
|
.query('SELECT horizon_seq FROM sync_partitions WHERE partition=?')
|
|
@@ -480,24 +491,52 @@ export class SqliteServerStorage {
|
|
|
480
491
|
return row?.horizon_seq ?? 0;
|
|
481
492
|
}
|
|
482
493
|
async setHorizonSeq(partition, seq) {
|
|
483
|
-
this
|
|
484
|
-
.
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
494
|
+
await this.#serializeWrite(() => {
|
|
495
|
+
this.db
|
|
496
|
+
.query(`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
|
|
497
|
+
ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`)
|
|
498
|
+
.run(partition, seq);
|
|
499
|
+
});
|
|
489
500
|
}
|
|
490
|
-
async pruneCommitsThrough(partition,
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
.
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
+
async pruneCommitsThrough(partition, query) {
|
|
502
|
+
validateCommitPruneQuery(query);
|
|
503
|
+
return this.#serializeWrite(() => {
|
|
504
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
505
|
+
try {
|
|
506
|
+
const epoch = this.db
|
|
507
|
+
.query('SELECT log_epoch FROM sync_partition_registry WHERE partition=?')
|
|
508
|
+
.get(partition)?.log_epoch;
|
|
509
|
+
if (epoch !== query.logEpoch)
|
|
510
|
+
throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
|
|
511
|
+
const previousHorizonSeq = this.db
|
|
512
|
+
.query('SELECT horizon_seq FROM sync_partitions WHERE partition=?')
|
|
513
|
+
.get(partition)?.horizon_seq ?? 0;
|
|
514
|
+
const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
|
|
515
|
+
this.db
|
|
516
|
+
.query(`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
|
|
517
|
+
ON CONFLICT(partition) DO UPDATE SET horizon_seq=excluded.horizon_seq`)
|
|
518
|
+
.run(partition, horizonSeq);
|
|
519
|
+
const removed = this.db
|
|
520
|
+
.query('DELETE FROM sync_commits WHERE partition=? AND commit_seq<=?')
|
|
521
|
+
.run(partition, horizonSeq);
|
|
522
|
+
this.db
|
|
523
|
+
.query('DELETE FROM sync_changes WHERE partition=? AND commit_seq<=?')
|
|
524
|
+
.run(partition, horizonSeq);
|
|
525
|
+
this.db
|
|
526
|
+
.query('DELETE FROM sync_change_scopes WHERE partition=? AND commit_seq<=?')
|
|
527
|
+
.run(partition, horizonSeq);
|
|
528
|
+
this.db.exec('COMMIT');
|
|
529
|
+
return {
|
|
530
|
+
previousHorizonSeq,
|
|
531
|
+
horizonSeq,
|
|
532
|
+
removedCommits: Number(removed.changes),
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
catch (error) {
|
|
536
|
+
this.db.exec('ROLLBACK');
|
|
537
|
+
throw error;
|
|
538
|
+
}
|
|
539
|
+
});
|
|
501
540
|
}
|
|
502
541
|
async getCommitSeqBefore(partition, createdBeforeMs) {
|
|
503
542
|
const row = this.db
|
|
@@ -527,7 +566,7 @@ export class SqliteServerStorage {
|
|
|
527
566
|
async claimReactions(partition, query) {
|
|
528
567
|
if (query.types.length === 0 || query.limit <= 0)
|
|
529
568
|
return [];
|
|
530
|
-
return this.#
|
|
569
|
+
return this.#serializeWrite(() => {
|
|
531
570
|
const typeParams = query.types.map(() => '?').join(',');
|
|
532
571
|
const records = this.db
|
|
533
572
|
.query(`UPDATE sync_reactions
|
|
@@ -553,7 +592,7 @@ export class SqliteServerStorage {
|
|
|
553
592
|
});
|
|
554
593
|
}
|
|
555
594
|
async completeReaction(partition, idempotencyKey, leaseOwner, completedAtMs) {
|
|
556
|
-
return this.#
|
|
595
|
+
return this.#serializeWrite(() => {
|
|
557
596
|
const result = this.db
|
|
558
597
|
.query(`UPDATE sync_reactions
|
|
559
598
|
SET status='completed', completed_at_ms=?,
|
|
@@ -565,7 +604,7 @@ export class SqliteServerStorage {
|
|
|
565
604
|
});
|
|
566
605
|
}
|
|
567
606
|
async extendReactionLease(partition, idempotencyKey, leaseOwner, leaseExpiresAtMs) {
|
|
568
|
-
return this.#
|
|
607
|
+
return this.#serializeWrite(() => {
|
|
569
608
|
const result = this.db
|
|
570
609
|
.query(`UPDATE sync_reactions SET lease_expires_at_ms=?
|
|
571
610
|
WHERE partition=? AND idempotency_key=?
|
|
@@ -576,7 +615,7 @@ export class SqliteServerStorage {
|
|
|
576
615
|
}
|
|
577
616
|
async failReaction(partition, idempotencyKey, update) {
|
|
578
617
|
const retry = update.retryAtMs !== undefined;
|
|
579
|
-
return this.#
|
|
618
|
+
return this.#serializeWrite(() => {
|
|
580
619
|
const result = this.db
|
|
581
620
|
.query(`UPDATE sync_reactions
|
|
582
621
|
SET status=?, available_at_ms=?, last_failure=?,
|
|
@@ -588,7 +627,7 @@ export class SqliteServerStorage {
|
|
|
588
627
|
});
|
|
589
628
|
}
|
|
590
629
|
async retryReaction(partition, idempotencyKey, nowMs) {
|
|
591
|
-
return this.#
|
|
630
|
+
return this.#serializeWrite(() => {
|
|
592
631
|
const result = this.db
|
|
593
632
|
.query(`UPDATE sync_reactions
|
|
594
633
|
SET status='pending', attempts=0, available_at_ms=?,
|
|
@@ -626,7 +665,7 @@ export class SqliteServerStorage {
|
|
|
626
665
|
async pruneReactions(partition, query) {
|
|
627
666
|
if (query.limit <= 0)
|
|
628
667
|
return { completed: 0, deadLetter: 0 };
|
|
629
|
-
return this.#
|
|
668
|
+
return this.#serializeWrite(() => {
|
|
630
669
|
const records = this.db
|
|
631
670
|
.query(`DELETE FROM sync_reactions
|
|
632
671
|
WHERE partition=? AND idempotency_key IN (
|
|
@@ -751,6 +790,28 @@ export class SqliteServerStorage {
|
|
|
751
790
|
.query('INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms) VALUES (?,?,?,?,?,?,?)')
|
|
752
791
|
.run(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs);
|
|
753
792
|
}
|
|
793
|
+
async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
|
|
794
|
+
this.db
|
|
795
|
+
.query(`UPDATE sync_clients
|
|
796
|
+
SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
|
|
797
|
+
WHERE partition=? AND client_id=? AND actor_id=?
|
|
798
|
+
AND EXISTS (SELECT 1 FROM sync_partition_registry
|
|
799
|
+
WHERE partition=sync_clients.partition AND log_epoch=?)`)
|
|
800
|
+
.run(cursor, updatedAtMs, partition, clientId, actorId, logEpoch);
|
|
801
|
+
}
|
|
802
|
+
async updateClientCursor(partition, clientId, cursor, updatedAtMs) {
|
|
803
|
+
this.db
|
|
804
|
+
.query(`UPDATE sync_clients
|
|
805
|
+
SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
|
|
806
|
+
WHERE partition=? AND client_id=?`)
|
|
807
|
+
.run(cursor, updatedAtMs, partition, clientId);
|
|
808
|
+
}
|
|
809
|
+
async getActiveClientCursorFloor(partition, cutoffMs) {
|
|
810
|
+
const row = this.db
|
|
811
|
+
.query('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=? AND updated_at_ms>=?')
|
|
812
|
+
.get(partition, cutoffMs);
|
|
813
|
+
return row.cursor;
|
|
814
|
+
}
|
|
754
815
|
async listClientCursors(partition) {
|
|
755
816
|
const records = this.db
|
|
756
817
|
.query('SELECT client_id, cursor, updated_at_ms FROM sync_clients WHERE partition=?')
|
package/dist/storage-errors.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export declare class StorageConstraintError extends Error {
|
|
|
9
9
|
constructor(cause: unknown, opIndex?: number);
|
|
10
10
|
}
|
|
11
11
|
/** Stable, privacy-safe failures for trusted server storage queries. */
|
|
12
|
-
export type StorageQueryErrorCode = 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit';
|
|
12
|
+
export type StorageQueryErrorCode = 'sync.storage.schema_migration_pending' | 'sync.storage.schema_migration_conflict' | 'sync.storage.schema_changed' | 'sync.storage.invalid_migration_budget' | 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit' | 'sync.storage.prune_epoch_mismatch' | 'sync.storage.partition_unregistered' | 'sync.storage.invalid_prune_cursor';
|
|
13
13
|
/**
|
|
14
14
|
* Host-only query error. Messages never include identifiers, values, SQL,
|
|
15
15
|
* paths, or row data; callers branch on `code`, never message text.
|
package/dist/storage-errors.js
CHANGED
|
@@ -12,6 +12,13 @@ export class StorageConstraintError extends Error {
|
|
|
12
12
|
}
|
|
13
13
|
}
|
|
14
14
|
const STORAGE_QUERY_MESSAGES = {
|
|
15
|
+
'sync.storage.schema_migration_pending': 'schema migration requires another invocation',
|
|
16
|
+
'sync.storage.schema_migration_conflict': 'another schema migration target is already pending',
|
|
17
|
+
'sync.storage.schema_changed': 'storage schema is not ready for this operation',
|
|
18
|
+
'sync.storage.invalid_migration_budget': 'migration statement budget must be an integer from 10 through 1000',
|
|
19
|
+
'sync.storage.prune_epoch_mismatch': 'partition log epoch changed; recompute retention inputs',
|
|
20
|
+
'sync.storage.partition_unregistered': 'pruning requires a registered partition',
|
|
21
|
+
'sync.storage.invalid_prune_cursor': 'pruning requires a non-negative safe integer cursor and a non-empty log epoch',
|
|
15
22
|
'sync.storage.scan_requires_scope': 'scope-indexed row scans require at least one scope variable',
|
|
16
23
|
'sync.storage.index_not_found': 'trusted row lookup requires a declared relational index',
|
|
17
24
|
'sync.storage.index_not_materialized': 'trusted row lookup requires a materialized relational table',
|
package/dist/storage.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { AuthoritativeRelationPlan } from './authoritative-query.js';
|
|
1
2
|
/**
|
|
2
3
|
* Storage interface (defined by the SPEC's needs, implementation-agnostic).
|
|
3
4
|
*
|
|
@@ -18,6 +19,15 @@
|
|
|
18
19
|
*/
|
|
19
20
|
import type { PushOperationResult, RowValue, ScopeMap } from '@syncular/core';
|
|
20
21
|
import type { CompiledSchema } from './schema.js';
|
|
22
|
+
export interface CommitPruneQuery {
|
|
23
|
+
readonly logEpoch: string;
|
|
24
|
+
readonly throughSeq: number;
|
|
25
|
+
}
|
|
26
|
+
export interface CommitPruneResult {
|
|
27
|
+
readonly previousHorizonSeq: number;
|
|
28
|
+
readonly horizonSeq: number;
|
|
29
|
+
readonly removedCommits: number;
|
|
30
|
+
}
|
|
21
31
|
/** The current stored state of a synced row. */
|
|
22
32
|
export interface StoredRow {
|
|
23
33
|
readonly rowId: string;
|
|
@@ -244,7 +254,7 @@ export interface ScopeActivityQuery {
|
|
|
244
254
|
export type AuthoritativeQueryValue = string | number | bigint | boolean | Uint8Array | null;
|
|
245
255
|
export interface AuthoritativeQueryRequest {
|
|
246
256
|
/** Generated, positional SQLite-family SQL. It never comes from the request. */
|
|
247
|
-
readonly
|
|
257
|
+
readonly plan: AuthoritativeRelationPlan;
|
|
248
258
|
readonly params: readonly AuthoritativeQueryValue[];
|
|
249
259
|
/** Generated dependency set, used to validate and partition every relation. */
|
|
250
260
|
readonly tables: readonly string[];
|
|
@@ -355,15 +365,18 @@ export interface ServerStorage {
|
|
|
355
365
|
rotatePartitionLogEpoch(partition: string, logEpoch: string, authenticatedAtMs: number): Promise<PartitionRegistryEntry>;
|
|
356
366
|
/** Registry entries ordered by partition for maintenance loops. */
|
|
357
367
|
listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
|
|
368
|
+
/** Read continuity without refreshing authenticated activity. */
|
|
369
|
+
getPartitionLogEpoch(partition: string): Promise<string | undefined>;
|
|
358
370
|
begin(partition: string): Promise<StorageTransaction>;
|
|
359
371
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
360
372
|
getHorizonSeq(partition: string): Promise<number>;
|
|
373
|
+
/** Monotonic within the current epoch; use atomic pruning for maintenance. */
|
|
361
374
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
362
375
|
/**
|
|
363
|
-
*
|
|
364
|
-
*
|
|
376
|
+
* Atomically verifies the log epoch, advances the horizon monotonically,
|
|
377
|
+
* and removes log/change/scope records through the effective horizon.
|
|
365
378
|
*/
|
|
366
|
-
pruneCommitsThrough(partition: string,
|
|
379
|
+
pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
|
|
367
380
|
/** Newest commitSeq created strictly before the timestamp; 0 if none. */
|
|
368
381
|
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|
|
369
382
|
getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
@@ -412,7 +425,18 @@ export interface ServerStorage {
|
|
|
412
425
|
scanRowsByIndex?(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
|
|
413
426
|
getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
|
|
414
427
|
putClientRecord(partition: string, record: ClientRecord): Promise<void>;
|
|
415
|
-
/**
|
|
428
|
+
/**
|
|
429
|
+
* Atomically advance an existing record's cursor and activity timestamp.
|
|
430
|
+
* Preserve registration fields; require matching actor and current log epoch.
|
|
431
|
+
* Missing records and mismatched identities are unchanged (§8.2).
|
|
432
|
+
*/
|
|
433
|
+
advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
434
|
+
/** Advance an existing client's ACK cursor and timestamp atomically,
|
|
435
|
+
* preserving actor, wire version, and subscriptions. Missing records stay absent. */
|
|
436
|
+
updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
|
|
437
|
+
/** Minimum cursor with updatedAtMs >= cutoff; null when none are active. */
|
|
438
|
+
getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;
|
|
439
|
+
/** Cursor records for client listings and administrative counts. */
|
|
416
440
|
listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
|
|
417
441
|
/**
|
|
418
442
|
* Blob reference index reads (§5.9.4) — ADDITIVE, optional (mirrors the
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.17.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/admin.ts
CHANGED
|
@@ -396,12 +396,10 @@ export class SyncularAdmin {
|
|
|
396
396
|
const nowMs = this.#clock();
|
|
397
397
|
const maxCommitSeq = await this.#storage.getMaxCommitSeq(partition);
|
|
398
398
|
const horizonSeq = await this.#storage.getHorizonSeq(partition);
|
|
399
|
-
const
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
const activeCursorFloor =
|
|
404
|
-
activeCursors.length > 0 ? Math.min(...activeCursors) : null;
|
|
399
|
+
const activeCursorFloor = await this.#storage.getActiveClientCursorFloor(
|
|
400
|
+
partition,
|
|
401
|
+
nowMs - this.#retention.activeWindowMs,
|
|
402
|
+
);
|
|
405
403
|
const cursorFloor = activeCursorFloor ?? Number.MAX_SAFE_INTEGER;
|
|
406
404
|
const forcedSeq = await this.#storage.getCommitSeqBefore(
|
|
407
405
|
partition,
|