@syncular/server 0.15.48 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -9
- package/dist/admin.js +1 -5
- package/dist/authoritative-query.d.ts +14 -6
- package/dist/authoritative-query.js +60 -82
- package/dist/d1-storage.d.ts +13 -1
- package/dist/d1-storage.js +468 -121
- package/dist/operations.d.ts +2 -0
- package/dist/operations.js +23 -4
- package/dist/postgres-storage.d.ts +6 -1
- package/dist/postgres-storage.js +80 -24
- package/dist/prune.d.ts +3 -1
- package/dist/prune.js +18 -14
- package/dist/pull.js +79 -39
- package/dist/push.js +5 -5
- package/dist/realtime.d.ts +1 -1
- package/dist/realtime.js +16 -10
- package/dist/relational-rows.d.ts +7 -1
- package/dist/relational-rows.js +17 -2
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-image.d.ts +3 -3
- package/dist/sqlite-image.js +10 -6
- package/dist/sqlite-node.js +2 -2
- package/dist/sqlite-storage.d.ts +6 -1
- package/dist/sqlite-storage.js +93 -32
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +7 -0
- package/dist/storage.d.ts +29 -5
- package/package.json +2 -2
- package/src/admin.ts +4 -6
- package/src/authoritative-query.ts +89 -94
- package/src/d1-storage.ts +641 -159
- package/src/operations.ts +31 -3
- package/src/postgres-storage.ts +146 -46
- package/src/prune.ts +29 -15
- package/src/pull.ts +102 -49
- package/src/push.ts +5 -5
- package/src/realtime.ts +20 -9
- package/src/relational-rows.ts +18 -2
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-image.ts +26 -15
- package/src/sqlite-node.ts +2 -2
- package/src/sqlite-storage.ts +131 -37
- package/src/storage-errors.ts +22 -1
- package/src/storage.ts +50 -5
package/src/realtime.ts
CHANGED
|
@@ -307,6 +307,10 @@ export class RealtimeSession {
|
|
|
307
307
|
const cursor = (parsed as { cursor?: unknown }).cursor;
|
|
308
308
|
if (typeof cursor !== 'number' || !Number.isSafeInteger(cursor)) return;
|
|
309
309
|
this.cursor = Math.max(this.cursor, cursor);
|
|
310
|
+
// The client may have caught up through another binding (§8.4). Its ack
|
|
311
|
+
// proves those commits no longer need socket notifications, so the next
|
|
312
|
+
// notification must be adjacent to the acknowledged cursor.
|
|
313
|
+
this.lastKnownSeq = Math.max(this.lastKnownSeq, this.cursor);
|
|
310
314
|
if (this.cursor >= this.lastKnownSeq) this.wakePending = false;
|
|
311
315
|
// §8.2: acks update the client cursor record without an HTTP pull.
|
|
312
316
|
void this.#persistCursor();
|
|
@@ -626,16 +630,14 @@ export class RealtimeSession {
|
|
|
626
630
|
|
|
627
631
|
async #persistCursor(): Promise<void> {
|
|
628
632
|
try {
|
|
629
|
-
|
|
633
|
+
await this.#storage.advanceClientCursor(
|
|
630
634
|
this.partition,
|
|
631
635
|
this.clientId,
|
|
636
|
+
this.actorId,
|
|
637
|
+
this.logEpoch,
|
|
638
|
+
this.cursor,
|
|
639
|
+
this.#clock(),
|
|
632
640
|
);
|
|
633
|
-
if (record === undefined) return;
|
|
634
|
-
await this.#storage.putClientRecord(this.partition, {
|
|
635
|
-
...record,
|
|
636
|
-
cursor: Math.max(record.cursor, this.cursor),
|
|
637
|
-
updatedAtMs: this.#clock(),
|
|
638
|
-
});
|
|
639
641
|
} catch {
|
|
640
642
|
// Cursor persistence is best-effort; the next pull repairs it.
|
|
641
643
|
}
|
|
@@ -688,8 +690,9 @@ export class RealtimeSession {
|
|
|
688
690
|
}
|
|
689
691
|
}
|
|
690
692
|
|
|
691
|
-
/** Called by the hub for every applied commit
|
|
693
|
+
/** Called by the hub for every applied commit notification. */
|
|
692
694
|
deliverCommit(commit: StoredCommit): void {
|
|
695
|
+
const contiguous = commit.commitSeq === this.lastKnownSeq + 1;
|
|
693
696
|
this.lastKnownSeq = Math.max(this.lastKnownSeq, commit.commitSeq);
|
|
694
697
|
const sections: Array<{
|
|
695
698
|
registration: Registration;
|
|
@@ -716,6 +719,14 @@ export class RealtimeSession {
|
|
|
716
719
|
);
|
|
717
720
|
if (changes.length > 0) sections.push({ registration, changes });
|
|
718
721
|
}
|
|
722
|
+
if (!contiguous) {
|
|
723
|
+
// Pushes allocate commitSeq under the partition lock but notify after
|
|
724
|
+
// commit, so racing notifications may arrive out of order. A gap,
|
|
725
|
+
// duplicate, or regression cannot be a delta: the client would apply
|
|
726
|
+
// and acknowledge a cursor that does not prove the intervening log.
|
|
727
|
+
this.sendWake('catchup-required');
|
|
728
|
+
return;
|
|
729
|
+
}
|
|
719
730
|
if (sections.length === 0) return;
|
|
720
731
|
if (this.#activeRound !== undefined) {
|
|
721
732
|
// §8.7 interleaving: no 0x00 messages while a response stream is
|
|
@@ -771,7 +782,7 @@ export class RealtimeSession {
|
|
|
771
782
|
tagged[0] = REALTIME_TAG_DELTA;
|
|
772
783
|
tagged.set(bytes, 1);
|
|
773
784
|
this.#sendSafe(tagged);
|
|
774
|
-
this.cursor = commit.commitSeq;
|
|
785
|
+
this.cursor = Math.max(this.cursor, commit.commitSeq);
|
|
775
786
|
const events = this.#events;
|
|
776
787
|
if (events !== undefined) {
|
|
777
788
|
emitEvent(events, {
|
package/src/relational-rows.ts
CHANGED
|
@@ -205,7 +205,7 @@ export function physicalIndexName(declaredName: string): string {
|
|
|
205
205
|
|
|
206
206
|
/**
|
|
207
207
|
* CREATE INDEX IF NOT EXISTS for the table's user-declared indexes. These use
|
|
208
|
-
* the
|
|
208
|
+
* the declared columns with a leading server partition column. Cross-table
|
|
209
209
|
* index-name uniqueness is the user's schema concern, as it is client-side.
|
|
210
210
|
* Server-side the physical name carries the {@link SYNC_INDEX_PREFIX}
|
|
211
211
|
* ownership marker.
|
|
@@ -215,7 +215,7 @@ export function createIndexDdl(table: CompiledTable): string[] {
|
|
|
215
215
|
if (!table.materialize) return [];
|
|
216
216
|
return table.indexes.map((index) => {
|
|
217
217
|
const unique = index.unique ? 'UNIQUE ' : '';
|
|
218
|
-
const columns = index.columns
|
|
218
|
+
const columns = [SYNC_PARTITION_COLUMN, ...index.columns]
|
|
219
219
|
.map((column) => quoteIdent(column))
|
|
220
220
|
.join(', ');
|
|
221
221
|
return `CREATE ${unique}INDEX IF NOT EXISTS ${quoteIdent(physicalIndexName(index.name))} ON ${quoteIdent(table.name)} (${columns})`;
|
|
@@ -477,6 +477,22 @@ export function deleteRowSql(
|
|
|
477
477
|
return `DELETE FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${p[0]} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=${p[1]}`;
|
|
478
478
|
}
|
|
479
479
|
|
|
480
|
+
/**
|
|
481
|
+
* SQLite: remove the old row's exact scope keys before replacing or deleting the row.
|
|
482
|
+
* Params: [partition, table, rowId, partition, rowId]. The caller keeps this
|
|
483
|
+
* statement and the row write in the same transaction (or D1 atomic batch).
|
|
484
|
+
*/
|
|
485
|
+
export function deleteSqliteRowScopesSql(table: CompiledTable): string {
|
|
486
|
+
return `DELETE FROM sync_row_scopes
|
|
487
|
+
WHERE partition=? AND tbl=? AND row_id=?
|
|
488
|
+
AND (var, value) IN (
|
|
489
|
+
SELECT key, value FROM json_each((
|
|
490
|
+
SELECT ${quoteIdent(SYNC_SCOPES_COLUMN)} FROM ${quoteIdent(table.name)}
|
|
491
|
+
WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=?
|
|
492
|
+
))
|
|
493
|
+
)`;
|
|
494
|
+
}
|
|
495
|
+
|
|
480
496
|
/**
|
|
481
497
|
* The schema-version marker table gates DDL work. `ensureSchema` compares the
|
|
482
498
|
* stored version and skips
|
package/src/sqlite-bun.ts
CHANGED
|
@@ -40,10 +40,10 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
|
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
|
|
43
|
-
export const buildSqliteImage: SqliteImageBuilder = (input) => {
|
|
43
|
+
export const buildSqliteImage: SqliteImageBuilder = async (input) => {
|
|
44
44
|
const db = new BunSqliteDatabase();
|
|
45
45
|
try {
|
|
46
|
-
writeSqliteImage(db, input);
|
|
46
|
+
await writeSqliteImage(db, input);
|
|
47
47
|
return db.serialize();
|
|
48
48
|
} finally {
|
|
49
49
|
db.close();
|
package/src/sqlite-image.ts
CHANGED
|
@@ -57,7 +57,9 @@ export interface SqliteImageInput {
|
|
|
57
57
|
readonly schemaVersion: number;
|
|
58
58
|
readonly asOfCommitSeq: number;
|
|
59
59
|
readonly scopeDigest: string;
|
|
60
|
-
readonly
|
|
60
|
+
readonly rowBatches:
|
|
61
|
+
| AsyncIterable<readonly StoredRow[]>
|
|
62
|
+
| Iterable<readonly StoredRow[]>;
|
|
61
63
|
}
|
|
62
64
|
|
|
63
65
|
/**
|
|
@@ -68,14 +70,16 @@ export interface SqliteImageInput {
|
|
|
68
70
|
* on the pull path. A Bun or Node host passes `buildSqliteImage`; a Workers
|
|
69
71
|
* host omits it and serves the rows lane.
|
|
70
72
|
*/
|
|
71
|
-
export type SqliteImageBuilder = (
|
|
73
|
+
export type SqliteImageBuilder = (
|
|
74
|
+
input: SqliteImageInput,
|
|
75
|
+
) => Promise<Uint8Array>;
|
|
72
76
|
|
|
73
77
|
/** Populate a §5.3 image database for a whole-table snapshot. */
|
|
74
|
-
export function writeSqliteImage(
|
|
78
|
+
export async function writeSqliteImage(
|
|
75
79
|
db: SqliteDatabase,
|
|
76
80
|
input: SqliteImageInput,
|
|
77
|
-
): void {
|
|
78
|
-
const { table,
|
|
81
|
+
): Promise<void> {
|
|
82
|
+
const { table, rowBatches } = input;
|
|
79
83
|
const primaryKey = table.columns[table.primaryKeyIndex]?.name;
|
|
80
84
|
const columnDefs = table.columns.map((column) => {
|
|
81
85
|
const notNull = column.nullable ? '' : ' NOT NULL';
|
|
@@ -90,13 +94,6 @@ export function writeSqliteImage(
|
|
|
90
94
|
"schemaVersion" INTEGER NOT NULL, "asOfCommitSeq" INTEGER NOT NULL,
|
|
91
95
|
"scopeDigest" TEXT NOT NULL, "rowCount" INTEGER NOT NULL)`,
|
|
92
96
|
);
|
|
93
|
-
db.query(`INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`).run(
|
|
94
|
-
table.name,
|
|
95
|
-
input.schemaVersion,
|
|
96
|
-
input.asOfCommitSeq,
|
|
97
|
-
input.scopeDigest,
|
|
98
|
-
rows.length,
|
|
99
|
-
);
|
|
100
97
|
const names = [
|
|
101
98
|
...table.columns.map((column) => quoteIdent(column.name)),
|
|
102
99
|
quoteIdent(IMAGE_VERSION_COLUMN),
|
|
@@ -107,10 +104,24 @@ export function writeSqliteImage(
|
|
|
107
104
|
);
|
|
108
105
|
db.exec('BEGIN');
|
|
109
106
|
try {
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
107
|
+
let rowCount = 0;
|
|
108
|
+
for await (const rows of rowBatches) {
|
|
109
|
+
for (const row of rows) {
|
|
110
|
+
const values = decodeRow(table.columns, row.payload);
|
|
111
|
+
insert.run(...values.map(toSql), row.serverVersion);
|
|
112
|
+
}
|
|
113
|
+
rowCount += rows.length;
|
|
113
114
|
}
|
|
115
|
+
db.query(
|
|
116
|
+
`INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`,
|
|
117
|
+
).run(
|
|
118
|
+
table.name,
|
|
119
|
+
input.schemaVersion,
|
|
120
|
+
input.asOfCommitSeq,
|
|
121
|
+
input.scopeDigest,
|
|
122
|
+
rowCount,
|
|
123
|
+
);
|
|
124
|
+
|
|
114
125
|
db.exec('COMMIT');
|
|
115
126
|
} catch (error) {
|
|
116
127
|
db.exec('ROLLBACK');
|
package/src/sqlite-node.ts
CHANGED
|
@@ -43,13 +43,13 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
|
|
|
43
43
|
}
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
-
export const buildSqliteImage: SqliteImageBuilder = (input) => {
|
|
46
|
+
export const buildSqliteImage: SqliteImageBuilder = async (input) => {
|
|
47
47
|
const directory = mkdtempSync(join(tmpdir(), 'syncular-server-image-'));
|
|
48
48
|
const path = join(directory, 'segment.db');
|
|
49
49
|
const db = new NodeSqliteDatabase(path);
|
|
50
50
|
try {
|
|
51
51
|
try {
|
|
52
|
-
writeSqliteImage(db, input);
|
|
52
|
+
await writeSqliteImage(db, input);
|
|
53
53
|
} finally {
|
|
54
54
|
db.close();
|
|
55
55
|
}
|
package/src/sqlite-storage.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { validateCommitPruneQuery } from './prune';
|
|
2
|
+
import { StorageQueryError } from './storage-errors';
|
|
3
|
+
import type { CommitPruneQuery, CommitPruneResult } from './storage';
|
|
1
4
|
/**
|
|
2
5
|
* SQLite server storage over the shared synchronous driver.
|
|
3
6
|
*
|
|
@@ -14,6 +17,7 @@ import { syncError } from './errors';
|
|
|
14
17
|
import {
|
|
15
18
|
commitWindowPageSql,
|
|
16
19
|
deleteRowSql,
|
|
20
|
+
deleteSqliteRowScopesSql,
|
|
17
21
|
dropTableDdl,
|
|
18
22
|
indexRowPageStatement,
|
|
19
23
|
layoutsOf,
|
|
@@ -225,13 +229,15 @@ class SqliteTransaction implements StorageTransaction {
|
|
|
225
229
|
async deleteRow(table: string, rowId: string): Promise<void> {
|
|
226
230
|
this.#assertOpen();
|
|
227
231
|
const db = this.#storage.db;
|
|
228
|
-
|
|
232
|
+
const compiled = this.#storage.table(table);
|
|
233
|
+
db.query(deleteSqliteRowScopesSql(compiled)).run(
|
|
234
|
+
this.#partition,
|
|
235
|
+
table,
|
|
236
|
+
rowId,
|
|
229
237
|
this.#partition,
|
|
230
238
|
rowId,
|
|
231
239
|
);
|
|
232
|
-
db.query(
|
|
233
|
-
'DELETE FROM sync_row_scopes WHERE partition=? AND tbl=? AND row_id=?',
|
|
234
|
-
).run(this.#partition, table, rowId);
|
|
240
|
+
db.query(deleteRowSql(compiled, 'sqlite')).run(this.#partition, rowId);
|
|
235
241
|
// §5.9.4: a deleted row references no blobs.
|
|
236
242
|
db.query(
|
|
237
243
|
'DELETE FROM sync_blob_refs WHERE partition=? AND tbl=? AND row_id=?',
|
|
@@ -390,7 +396,7 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
390
396
|
#tables: ReadonlyMap<string, CompiledTable> | undefined;
|
|
391
397
|
#schemaVersion: number | undefined;
|
|
392
398
|
|
|
393
|
-
async #
|
|
399
|
+
async #serializeWrite<T>(operation: () => T): Promise<T> {
|
|
394
400
|
const previous = this.#transactionTail;
|
|
395
401
|
let release!: () => void;
|
|
396
402
|
this.#transactionTail = new Promise<void>((resolve) => {
|
|
@@ -482,6 +488,9 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
482
488
|
this.db
|
|
483
489
|
.query('DELETE FROM sync_row_scopes WHERE tbl=?')
|
|
484
490
|
.run(tableName);
|
|
491
|
+
this.db
|
|
492
|
+
.query('DELETE FROM sync_blob_refs WHERE tbl=?')
|
|
493
|
+
.run(tableName);
|
|
485
494
|
this.db.exec(dropTableDdl(tableName));
|
|
486
495
|
}
|
|
487
496
|
for (const statement of schemaDdl(
|
|
@@ -559,7 +568,7 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
559
568
|
authenticatedAtMs: number,
|
|
560
569
|
): Promise<PartitionRegistryEntry> {
|
|
561
570
|
if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
|
|
562
|
-
return this.#
|
|
571
|
+
return this.#serializeWrite(() => {
|
|
563
572
|
this.db.exec('BEGIN IMMEDIATE');
|
|
564
573
|
try {
|
|
565
574
|
this.db
|
|
@@ -680,6 +689,9 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
680
689
|
/** Internal: write a row + refresh its scope-index entries. */
|
|
681
690
|
writeRow(partition: string, table: string, row: StoredRow): void {
|
|
682
691
|
const compiled = this.table(table);
|
|
692
|
+
this.db
|
|
693
|
+
.query(deleteSqliteRowScopesSql(compiled))
|
|
694
|
+
.run(partition, table, row.rowId, partition, row.rowId);
|
|
683
695
|
this.db
|
|
684
696
|
.query(upsertSql(compiled, 'sqlite'))
|
|
685
697
|
.run(
|
|
@@ -691,11 +703,6 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
691
703
|
| null
|
|
692
704
|
)[]),
|
|
693
705
|
);
|
|
694
|
-
this.db
|
|
695
|
-
.query(
|
|
696
|
-
'DELETE FROM sync_row_scopes WHERE partition=? AND tbl=? AND row_id=?',
|
|
697
|
-
)
|
|
698
|
-
.run(partition, table, row.rowId);
|
|
699
706
|
for (const [variable, value] of Object.entries(row.scopes)) {
|
|
700
707
|
this.db
|
|
701
708
|
.query(
|
|
@@ -725,7 +732,7 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
725
732
|
}
|
|
726
733
|
const prepared = bindAuthoritativePartition(
|
|
727
734
|
prepareAuthoritativeQuery(
|
|
728
|
-
query.
|
|
735
|
+
query.plan,
|
|
729
736
|
query.params,
|
|
730
737
|
query.tables,
|
|
731
738
|
this.#tables,
|
|
@@ -763,6 +770,14 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
763
770
|
}
|
|
764
771
|
}
|
|
765
772
|
|
|
773
|
+
async getPartitionLogEpoch(partition: string): Promise<string | undefined> {
|
|
774
|
+
return this.db
|
|
775
|
+
.query<{ log_epoch: string }, [string]>(
|
|
776
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=?',
|
|
777
|
+
)
|
|
778
|
+
.get(partition)?.log_epoch;
|
|
779
|
+
}
|
|
780
|
+
|
|
766
781
|
async getHorizonSeq(partition: string): Promise<number> {
|
|
767
782
|
const row = this.db
|
|
768
783
|
.query<{ horizon_seq: number }, [string]>(
|
|
@@ -773,27 +788,62 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
773
788
|
}
|
|
774
789
|
|
|
775
790
|
async setHorizonSeq(partition: string, seq: number): Promise<void> {
|
|
776
|
-
this
|
|
777
|
-
.
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
791
|
+
await this.#serializeWrite(() => {
|
|
792
|
+
this.db
|
|
793
|
+
.query(`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
|
|
794
|
+
ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`)
|
|
795
|
+
.run(partition, seq);
|
|
796
|
+
});
|
|
782
797
|
}
|
|
783
798
|
|
|
784
|
-
async pruneCommitsThrough(
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
.
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
799
|
+
async pruneCommitsThrough(
|
|
800
|
+
partition: string,
|
|
801
|
+
query: CommitPruneQuery,
|
|
802
|
+
): Promise<CommitPruneResult> {
|
|
803
|
+
validateCommitPruneQuery(query);
|
|
804
|
+
return this.#serializeWrite(() => {
|
|
805
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
806
|
+
try {
|
|
807
|
+
const epoch = this.db
|
|
808
|
+
.query<{ log_epoch: string }, [string]>(
|
|
809
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=?',
|
|
810
|
+
)
|
|
811
|
+
.get(partition)?.log_epoch;
|
|
812
|
+
if (epoch !== query.logEpoch)
|
|
813
|
+
throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
|
|
814
|
+
const previousHorizonSeq =
|
|
815
|
+
this.db
|
|
816
|
+
.query<{ horizon_seq: number }, [string]>(
|
|
817
|
+
'SELECT horizon_seq FROM sync_partitions WHERE partition=?',
|
|
818
|
+
)
|
|
819
|
+
.get(partition)?.horizon_seq ?? 0;
|
|
820
|
+
const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
|
|
821
|
+
this.db
|
|
822
|
+
.query(`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
|
|
823
|
+
ON CONFLICT(partition) DO UPDATE SET horizon_seq=excluded.horizon_seq`)
|
|
824
|
+
.run(partition, horizonSeq);
|
|
825
|
+
const removed = this.db
|
|
826
|
+
.query('DELETE FROM sync_commits WHERE partition=? AND commit_seq<=?')
|
|
827
|
+
.run(partition, horizonSeq);
|
|
828
|
+
this.db
|
|
829
|
+
.query('DELETE FROM sync_changes WHERE partition=? AND commit_seq<=?')
|
|
830
|
+
.run(partition, horizonSeq);
|
|
831
|
+
this.db
|
|
832
|
+
.query(
|
|
833
|
+
'DELETE FROM sync_change_scopes WHERE partition=? AND commit_seq<=?',
|
|
834
|
+
)
|
|
835
|
+
.run(partition, horizonSeq);
|
|
836
|
+
this.db.exec('COMMIT');
|
|
837
|
+
return {
|
|
838
|
+
previousHorizonSeq,
|
|
839
|
+
horizonSeq,
|
|
840
|
+
removedCommits: Number(removed.changes),
|
|
841
|
+
};
|
|
842
|
+
} catch (error) {
|
|
843
|
+
this.db.exec('ROLLBACK');
|
|
844
|
+
throw error;
|
|
845
|
+
}
|
|
846
|
+
});
|
|
797
847
|
}
|
|
798
848
|
|
|
799
849
|
async getCommitSeqBefore(
|
|
@@ -847,7 +897,7 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
847
897
|
query: ReactionClaimQuery,
|
|
848
898
|
): Promise<StoredReaction[]> {
|
|
849
899
|
if (query.types.length === 0 || query.limit <= 0) return [];
|
|
850
|
-
return this.#
|
|
900
|
+
return this.#serializeWrite(() => {
|
|
851
901
|
const typeParams = query.types.map(() => '?').join(',');
|
|
852
902
|
const records = this.db
|
|
853
903
|
.query<SqliteReactionRecord, (string | number)[]>(
|
|
@@ -895,7 +945,7 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
895
945
|
leaseOwner: string,
|
|
896
946
|
completedAtMs: number,
|
|
897
947
|
): Promise<boolean> {
|
|
898
|
-
return this.#
|
|
948
|
+
return this.#serializeWrite(() => {
|
|
899
949
|
const result = this.db
|
|
900
950
|
.query(
|
|
901
951
|
`UPDATE sync_reactions
|
|
@@ -915,7 +965,7 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
915
965
|
leaseOwner: string,
|
|
916
966
|
leaseExpiresAtMs: number,
|
|
917
967
|
): Promise<boolean> {
|
|
918
|
-
return this.#
|
|
968
|
+
return this.#serializeWrite(() => {
|
|
919
969
|
const result = this.db
|
|
920
970
|
.query(
|
|
921
971
|
`UPDATE sync_reactions SET lease_expires_at_ms=?
|
|
@@ -933,7 +983,7 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
933
983
|
update: ReactionFailureUpdate,
|
|
934
984
|
): Promise<boolean> {
|
|
935
985
|
const retry = update.retryAtMs !== undefined;
|
|
936
|
-
return this.#
|
|
986
|
+
return this.#serializeWrite(() => {
|
|
937
987
|
const result = this.db
|
|
938
988
|
.query(
|
|
939
989
|
`UPDATE sync_reactions
|
|
@@ -959,7 +1009,7 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
959
1009
|
idempotencyKey: string,
|
|
960
1010
|
nowMs: number,
|
|
961
1011
|
): Promise<boolean> {
|
|
962
|
-
return this.#
|
|
1012
|
+
return this.#serializeWrite(() => {
|
|
963
1013
|
const result = this.db
|
|
964
1014
|
.query(
|
|
965
1015
|
`UPDATE sync_reactions
|
|
@@ -1014,7 +1064,7 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
1014
1064
|
query: ReactionPruneQuery,
|
|
1015
1065
|
): Promise<PrunedReactionCounts> {
|
|
1016
1066
|
if (query.limit <= 0) return { completed: 0, deadLetter: 0 };
|
|
1017
|
-
return this.#
|
|
1067
|
+
return this.#serializeWrite(() => {
|
|
1018
1068
|
const records = this.db
|
|
1019
1069
|
.query<
|
|
1020
1070
|
{ status: 'completed' | 'dead-letter' },
|
|
@@ -1226,6 +1276,50 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
1226
1276
|
);
|
|
1227
1277
|
}
|
|
1228
1278
|
|
|
1279
|
+
async advanceClientCursor(
|
|
1280
|
+
partition: string,
|
|
1281
|
+
clientId: string,
|
|
1282
|
+
actorId: string,
|
|
1283
|
+
logEpoch: string,
|
|
1284
|
+
cursor: number,
|
|
1285
|
+
updatedAtMs: number,
|
|
1286
|
+
): Promise<void> {
|
|
1287
|
+
this.db
|
|
1288
|
+
.query(`UPDATE sync_clients
|
|
1289
|
+
SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
|
|
1290
|
+
WHERE partition=? AND client_id=? AND actor_id=?
|
|
1291
|
+
AND EXISTS (SELECT 1 FROM sync_partition_registry
|
|
1292
|
+
WHERE partition=sync_clients.partition AND log_epoch=?)`)
|
|
1293
|
+
.run(cursor, updatedAtMs, partition, clientId, actorId, logEpoch);
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
async updateClientCursor(
|
|
1297
|
+
partition: string,
|
|
1298
|
+
clientId: string,
|
|
1299
|
+
cursor: number,
|
|
1300
|
+
updatedAtMs: number,
|
|
1301
|
+
): Promise<void> {
|
|
1302
|
+
this.db
|
|
1303
|
+
.query(
|
|
1304
|
+
`UPDATE sync_clients
|
|
1305
|
+
SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
|
|
1306
|
+
WHERE partition=? AND client_id=?`,
|
|
1307
|
+
)
|
|
1308
|
+
.run(cursor, updatedAtMs, partition, clientId);
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
async getActiveClientCursorFloor(
|
|
1312
|
+
partition: string,
|
|
1313
|
+
cutoffMs: number,
|
|
1314
|
+
): Promise<number | null> {
|
|
1315
|
+
const row = this.db
|
|
1316
|
+
.query<{ cursor: number | null }, [string, number]>(
|
|
1317
|
+
'SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=? AND updated_at_ms>=?',
|
|
1318
|
+
)
|
|
1319
|
+
.get(partition, cutoffMs);
|
|
1320
|
+
return row!.cursor;
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1229
1323
|
async listClientCursors(partition: string): Promise<ClientCursorInfo[]> {
|
|
1230
1324
|
const records = this.db
|
|
1231
1325
|
.query<
|
package/src/storage-errors.ts
CHANGED
|
@@ -15,14 +15,35 @@ export class StorageConstraintError extends Error {
|
|
|
15
15
|
|
|
16
16
|
/** Stable, privacy-safe failures for trusted server storage queries. */
|
|
17
17
|
export type StorageQueryErrorCode =
|
|
18
|
+
| 'sync.storage.schema_migration_pending'
|
|
19
|
+
| 'sync.storage.schema_migration_conflict'
|
|
20
|
+
| 'sync.storage.schema_changed'
|
|
21
|
+
| 'sync.storage.invalid_migration_budget'
|
|
18
22
|
| 'sync.storage.scan_requires_scope'
|
|
19
23
|
| 'sync.storage.index_not_found'
|
|
20
24
|
| 'sync.storage.index_not_materialized'
|
|
21
25
|
| 'sync.storage.index_value_count_mismatch'
|
|
22
|
-
| 'sync.storage.invalid_limit'
|
|
26
|
+
| 'sync.storage.invalid_limit'
|
|
27
|
+
| 'sync.storage.prune_epoch_mismatch'
|
|
28
|
+
| 'sync.storage.partition_unregistered'
|
|
29
|
+
| 'sync.storage.invalid_prune_cursor';
|
|
23
30
|
|
|
24
31
|
const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
|
|
25
32
|
{
|
|
33
|
+
'sync.storage.schema_migration_pending':
|
|
34
|
+
'schema migration requires another invocation',
|
|
35
|
+
'sync.storage.schema_migration_conflict':
|
|
36
|
+
'another schema migration target is already pending',
|
|
37
|
+
'sync.storage.schema_changed':
|
|
38
|
+
'storage schema is not ready for this operation',
|
|
39
|
+
'sync.storage.invalid_migration_budget':
|
|
40
|
+
'migration statement budget must be an integer from 10 through 1000',
|
|
41
|
+
'sync.storage.prune_epoch_mismatch':
|
|
42
|
+
'partition log epoch changed; recompute retention inputs',
|
|
43
|
+
'sync.storage.partition_unregistered':
|
|
44
|
+
'pruning requires a registered partition',
|
|
45
|
+
'sync.storage.invalid_prune_cursor':
|
|
46
|
+
'pruning requires a non-negative safe integer cursor and a non-empty log epoch',
|
|
26
47
|
'sync.storage.scan_requires_scope':
|
|
27
48
|
'scope-indexed row scans require at least one scope variable',
|
|
28
49
|
'sync.storage.index_not_found':
|
package/src/storage.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { AuthoritativeRelationPlan } from './authoritative-query';
|
|
1
2
|
/**
|
|
2
3
|
* Storage interface (defined by the SPEC's needs, implementation-agnostic).
|
|
3
4
|
*
|
|
@@ -19,6 +20,17 @@
|
|
|
19
20
|
import type { PushOperationResult, RowValue, ScopeMap } from '@syncular/core';
|
|
20
21
|
import type { CompiledSchema } from './schema';
|
|
21
22
|
|
|
23
|
+
export interface CommitPruneQuery {
|
|
24
|
+
readonly logEpoch: string;
|
|
25
|
+
readonly throughSeq: number;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface CommitPruneResult {
|
|
29
|
+
readonly previousHorizonSeq: number;
|
|
30
|
+
readonly horizonSeq: number;
|
|
31
|
+
readonly removedCommits: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
22
34
|
/** The current stored state of a synced row. */
|
|
23
35
|
export interface StoredRow {
|
|
24
36
|
readonly rowId: string;
|
|
@@ -281,7 +293,7 @@ export type AuthoritativeQueryValue =
|
|
|
281
293
|
|
|
282
294
|
export interface AuthoritativeQueryRequest {
|
|
283
295
|
/** Generated, positional SQLite-family SQL. It never comes from the request. */
|
|
284
|
-
readonly
|
|
296
|
+
readonly plan: AuthoritativeRelationPlan;
|
|
285
297
|
readonly params: readonly AuthoritativeQueryValue[];
|
|
286
298
|
/** Generated dependency set, used to validate and partition every relation. */
|
|
287
299
|
readonly tables: readonly string[];
|
|
@@ -422,16 +434,23 @@ export interface ServerStorage {
|
|
|
422
434
|
/** Registry entries ordered by partition for maintenance loops. */
|
|
423
435
|
listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
|
|
424
436
|
|
|
437
|
+
/** Read continuity without refreshing authenticated activity. */
|
|
438
|
+
getPartitionLogEpoch(partition: string): Promise<string | undefined>;
|
|
439
|
+
|
|
425
440
|
begin(partition: string): Promise<StorageTransaction>;
|
|
426
441
|
|
|
427
442
|
getMaxCommitSeq(partition: string): Promise<number>;
|
|
428
443
|
getHorizonSeq(partition: string): Promise<number>;
|
|
444
|
+
/** Monotonic within the current epoch; use atomic pruning for maintenance. */
|
|
429
445
|
setHorizonSeq(partition: string, seq: number): Promise<void>;
|
|
430
446
|
/**
|
|
431
|
-
*
|
|
432
|
-
*
|
|
447
|
+
* Atomically verifies the log epoch, advances the horizon monotonically,
|
|
448
|
+
* and removes log/change/scope records through the effective horizon.
|
|
433
449
|
*/
|
|
434
|
-
pruneCommitsThrough(
|
|
450
|
+
pruneCommitsThrough(
|
|
451
|
+
partition: string,
|
|
452
|
+
query: CommitPruneQuery,
|
|
453
|
+
): Promise<CommitPruneResult>;
|
|
435
454
|
/** Newest commitSeq created strictly before the timestamp; 0 if none. */
|
|
436
455
|
getCommitSeqBefore(
|
|
437
456
|
partition: string,
|
|
@@ -541,7 +560,33 @@ export interface ServerStorage {
|
|
|
541
560
|
clientId: string,
|
|
542
561
|
): Promise<ClientRecord | undefined>;
|
|
543
562
|
putClientRecord(partition: string, record: ClientRecord): Promise<void>;
|
|
544
|
-
/**
|
|
563
|
+
/**
|
|
564
|
+
* Atomically advance an existing record's cursor and activity timestamp.
|
|
565
|
+
* Preserve registration fields; require matching actor and current log epoch.
|
|
566
|
+
* Missing records and mismatched identities are unchanged (§8.2).
|
|
567
|
+
*/
|
|
568
|
+
advanceClientCursor(
|
|
569
|
+
partition: string,
|
|
570
|
+
clientId: string,
|
|
571
|
+
actorId: string,
|
|
572
|
+
logEpoch: string,
|
|
573
|
+
cursor: number,
|
|
574
|
+
updatedAtMs: number,
|
|
575
|
+
): Promise<void>;
|
|
576
|
+
/** Advance an existing client's ACK cursor and timestamp atomically,
|
|
577
|
+
* preserving actor, wire version, and subscriptions. Missing records stay absent. */
|
|
578
|
+
updateClientCursor(
|
|
579
|
+
partition: string,
|
|
580
|
+
clientId: string,
|
|
581
|
+
cursor: number,
|
|
582
|
+
updatedAtMs: number,
|
|
583
|
+
): Promise<void>;
|
|
584
|
+
/** Minimum cursor with updatedAtMs >= cutoff; null when none are active. */
|
|
585
|
+
getActiveClientCursorFloor(
|
|
586
|
+
partition: string,
|
|
587
|
+
cutoffMs: number,
|
|
588
|
+
): Promise<number | null>;
|
|
589
|
+
/** Cursor records for client listings and administrative counts. */
|
|
545
590
|
listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
|
|
546
591
|
|
|
547
592
|
/**
|