@syncular/server-dialect-sqlite 0.0.1 → 0.0.2-127
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -0
- package/dist/index.d.ts +12 -61
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +110 -212
- package/dist/index.js.map +1 -1
- package/package.json +25 -5
- package/src/index.test.ts +149 -0
- package/src/index.ts +165 -332
package/README.md
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# @syncular/server-dialect-sqlite
|
|
2
|
+
|
|
3
|
+
SQLite dialect for the Syncular server sync schema and query patterns (commit log, change log, scopes, snapshot chunks, console tables).
|
|
4
|
+
|
|
5
|
+
Commonly used for dev/test setups and for edge deployments that use a SQLite-compatible backend.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install @syncular/server-dialect-sqlite
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
import { ensureSyncSchema } from '@syncular/server';
|
|
17
|
+
import { createSqliteServerDialect } from '@syncular/server-dialect-sqlite';
|
|
18
|
+
|
|
19
|
+
const dialect = createSqliteServerDialect();
|
|
20
|
+
await ensureSyncSchema(db, dialect);
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Documentation
|
|
24
|
+
|
|
25
|
+
- Dialects: https://syncular.dev/docs/server/dialects
|
|
26
|
+
- Server setup: https://syncular.dev/docs/build/server-setup
|
|
27
|
+
|
|
28
|
+
## Links
|
|
29
|
+
|
|
30
|
+
- GitHub: https://github.com/syncular/syncular
|
|
31
|
+
- Issues: https://github.com/syncular/syncular/issues
|
|
32
|
+
|
|
33
|
+
> Status: Alpha. APIs and storage layouts may change between releases.
|
package/dist/index.d.ts
CHANGED
|
@@ -12,11 +12,12 @@
|
|
|
12
12
|
* - No GIN index → regular index + manual filtering
|
|
13
13
|
* - REPEATABLE READ → no-op (SQLite uses serializable by default)
|
|
14
14
|
*/
|
|
15
|
-
import type { ScopeValues, StoredScopes
|
|
16
|
-
import type { DbExecutor
|
|
17
|
-
import
|
|
18
|
-
import type {
|
|
19
|
-
|
|
15
|
+
import type { ScopeValues, StoredScopes } from '@syncular/core';
|
|
16
|
+
import type { DbExecutor } from '@syncular/server';
|
|
17
|
+
import { BaseServerSyncDialect, type IncrementalPullRow, type IncrementalPullRowsArgs } from '@syncular/server';
|
|
18
|
+
import type { SyncChangeRow, SyncCoreDb } from '@syncular/server/schema';
|
|
19
|
+
import type { Kysely, RawBuilder } from 'kysely';
|
|
20
|
+
export declare class SqliteServerSyncDialect extends BaseServerSyncDialect {
|
|
20
21
|
readonly name: "sqlite";
|
|
21
22
|
readonly supportsForUpdate = false;
|
|
22
23
|
readonly supportsSavepoints: boolean;
|
|
@@ -24,74 +25,24 @@ export declare class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
24
25
|
constructor(options?: {
|
|
25
26
|
supportsTransactions?: boolean;
|
|
26
27
|
});
|
|
28
|
+
protected buildNumberListFilter(values: number[]): RawBuilder<unknown>;
|
|
29
|
+
protected buildStringListFilter(values: string[]): RawBuilder<unknown>;
|
|
27
30
|
ensureSyncSchema<DB extends SyncCoreDb>(db: Kysely<DB>): Promise<void>;
|
|
28
31
|
executeInTransaction<DB extends SyncCoreDb, T>(db: Kysely<DB>, fn: (executor: DbExecutor<DB>) => Promise<T>): Promise<T>;
|
|
29
32
|
setRepeatableRead<DB extends SyncCoreDb>(_trx: DbExecutor<DB>): Promise<void>;
|
|
30
|
-
|
|
31
|
-
readMinCommitSeq<DB extends SyncCoreDb>(db: Kysely<DB> | Transaction<DB>): Promise<number>;
|
|
32
|
-
readCommitSeqsForPull<DB extends SyncCoreDb>(db: Kysely<DB> | Transaction<DB>, args: {
|
|
33
|
-
cursor: number;
|
|
34
|
-
limitCommits: number;
|
|
35
|
-
tables: string[];
|
|
36
|
-
}): Promise<number[]>;
|
|
37
|
-
readCommits<DB extends SyncCoreDb>(db: Kysely<DB> | Transaction<DB>, commitSeqs: number[]): Promise<SyncCommitRow[]>;
|
|
38
|
-
readChangesForCommits<DB extends SyncCoreDb>(db: Kysely<DB> | Transaction<DB>, args: {
|
|
33
|
+
readChangesForCommits<DB extends SyncCoreDb>(db: DbExecutor<DB>, args: {
|
|
39
34
|
commitSeqs: number[];
|
|
40
35
|
table: string;
|
|
41
36
|
scopes: ScopeValues;
|
|
37
|
+
partitionId?: string;
|
|
42
38
|
}): Promise<SyncChangeRow[]>;
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
scopes: ScopeValues;
|
|
46
|
-
cursor: number;
|
|
47
|
-
limitCommits: number;
|
|
48
|
-
}): Promise<Array<{
|
|
49
|
-
commit_seq: number;
|
|
50
|
-
actor_id: string;
|
|
51
|
-
created_at: string;
|
|
52
|
-
change_id: number;
|
|
53
|
-
table: string;
|
|
54
|
-
row_id: string;
|
|
55
|
-
op: SyncOp;
|
|
56
|
-
row_json: unknown | null;
|
|
57
|
-
row_version: number | null;
|
|
58
|
-
scopes: StoredScopes;
|
|
59
|
-
}>>;
|
|
60
|
-
/**
|
|
61
|
-
* Streaming version of incremental pull for large result sets.
|
|
62
|
-
* Yields changes one at a time instead of loading all into memory.
|
|
63
|
-
*/
|
|
64
|
-
streamIncrementalPullRows<DB extends SyncCoreDb>(db: Kysely<DB> | Transaction<DB>, args: {
|
|
65
|
-
table: string;
|
|
66
|
-
scopes: ScopeValues;
|
|
67
|
-
cursor: number;
|
|
68
|
-
limitCommits: number;
|
|
69
|
-
}): AsyncGenerator<{
|
|
70
|
-
commit_seq: number;
|
|
71
|
-
actor_id: string;
|
|
72
|
-
created_at: string;
|
|
73
|
-
change_id: number;
|
|
74
|
-
table: string;
|
|
75
|
-
row_id: string;
|
|
76
|
-
op: SyncOp;
|
|
77
|
-
row_json: unknown | null;
|
|
78
|
-
row_version: number | null;
|
|
79
|
-
scopes: StoredScopes;
|
|
80
|
-
}>;
|
|
81
|
-
compactChanges<DB extends SyncCoreDb>(db: Kysely<DB> | Transaction<DB>, args: {
|
|
39
|
+
protected readIncrementalPullRowsBatch<DB extends SyncCoreDb>(db: DbExecutor<DB>, args: Omit<IncrementalPullRowsArgs, 'batchSize'>): Promise<IncrementalPullRow[]>;
|
|
40
|
+
compactChanges<DB extends SyncCoreDb>(db: DbExecutor<DB>, args: {
|
|
82
41
|
fullHistoryHours: number;
|
|
83
42
|
}): Promise<number>;
|
|
84
|
-
recordClientCursor<DB extends SyncCoreDb>(db: Kysely<DB> | Transaction<DB>, args: {
|
|
85
|
-
clientId: string;
|
|
86
|
-
actorId: string;
|
|
87
|
-
cursor: number;
|
|
88
|
-
effectiveScopes: ScopeValues;
|
|
89
|
-
}): Promise<void>;
|
|
90
43
|
scopesToDb(scopes: StoredScopes): string;
|
|
91
|
-
dbToScopes(value: unknown): StoredScopes;
|
|
92
44
|
dbToArray(value: unknown): string[];
|
|
93
45
|
arrayToDb(values: string[]): string;
|
|
94
|
-
readAffectedTablesFromChanges<DB extends SyncCoreDb>(db: Kysely<DB> | Transaction<DB>, commitSeq: number): Promise<string[]>;
|
|
95
46
|
ensureConsoleSchema<DB extends SyncCoreDb>(db: Kysely<DB>): Promise<void>;
|
|
96
47
|
}
|
|
97
48
|
export declare function createSqliteServerDialect(options?: {
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAU,MAAM,gBAAgB,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EACL,qBAAqB,EAGrB,KAAK,kBAAkB,EACvB,KAAK,uBAAuB,EAE7B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAe,MAAM,QAAQ,CAAC;AA4F9D,qBAAa,uBAAwB,SAAQ,qBAAqB;IAChE,QAAQ,CAAC,IAAI,WAAqB;IAClC,QAAQ,CAAC,iBAAiB,SAAS;IACnC,QAAQ,CAAC,kBAAkB,EAAE,OAAO,CAAC;IACrC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAU;IAEhD,YAAY,OAAO,CAAC,EAAE;QAAE,oBAAoB,CAAC,EAAE,OAAO,CAAA;KAAE,EAIvD;IAMD,SAAS,CAAC,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAMrE;IAED,SAAS,CAAC,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAMrE;IAMK,gBAAgB,CAAC,EAAE,SAAS,UAAU,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CA6J3E;IAMK,oBAAoB,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC,EACjD,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,EAAE,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAC3C,OAAO,CAAC,CAAC,CAAC,CAqBZ;IAEK,iBAAiB,CAAC,EAAE,SAAS,UAAU,EAC3C,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC,GACnB,OAAO,CAAC,IAAI,CAAC,CAEf;IAMK,qBAAqB,CAAC,EAAE,SAAS,UAAU,EAC/C,EAAE,EAAE,UAAU,CAAC,EAAE,CAAC,EAClB,IAAI,EAAE;QACJ,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,WAAW,CAAC;QACpB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,GACA,OAAO,CAAC,aAAa,EAAE,CAAC,CA0C1B;IAED,UAAyB,4BAA4B,CAAC,EAAE,SAAS,UAAU,EACzE,EAAE,EAAE,UAAU,CAAC,EAAE,CAAC,EAClB,IAAI,EAAE,IAAI,CAAC,uBAAuB,EAAE,WAAW,CAAC,GAC/C,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAmF/B;IAEK,cAAc,CAAC,EAAE,SAAS,UAAU,EACxC,EAAE,EAAE,UAAU,CAAC,EAAE,CAAC,EAClB,IAAI,EAAE;QAAE,gBAAgB,EAAE,MAAM,CAAA;KAAE,GACjC,OAAO,CAAC,MAAM,CAAC,CA6FjB;IAMD,UAAU,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAEvC;IAED,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CAElC;IAED,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAElC;IAMK,mBAAmB,CAAC,EAAE,SAAS,UAAU,EAC7C,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,GACb,OAAO,CAAC,IAAI,CAAC,CAuDf;CACF;AAED,wBAAgB,yBAAyB,CAAC,OAAO,CAAC,EAAE;IAClD,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAChC,GAAG,uBAAuB,CAE1B"}
|
package/dist/index.js
CHANGED
|
@@ -12,26 +12,14 @@
|
|
|
12
12
|
* - No GIN index → regular index + manual filtering
|
|
13
13
|
* - REPEATABLE READ → no-op (SQLite uses serializable by default)
|
|
14
14
|
*/
|
|
15
|
+
import { BaseServerSyncDialect, coerceIsoString, coerceNumber, parseScopes, } from '@syncular/server';
|
|
15
16
|
import { sql } from 'kysely';
|
|
16
|
-
function
|
|
17
|
-
|
|
18
|
-
return null;
|
|
19
|
-
if (typeof value === 'number')
|
|
20
|
-
return Number.isFinite(value) ? value : null;
|
|
21
|
-
if (typeof value === 'bigint')
|
|
22
|
-
return Number.isFinite(Number(value)) ? Number(value) : null;
|
|
23
|
-
if (typeof value === 'string') {
|
|
24
|
-
const n = Number(value);
|
|
25
|
-
return Number.isFinite(n) ? n : null;
|
|
26
|
-
}
|
|
27
|
-
return null;
|
|
17
|
+
function isActiveTransaction(db) {
|
|
18
|
+
return db.isTransaction === true;
|
|
28
19
|
}
|
|
29
|
-
function
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
if (value instanceof Date)
|
|
33
|
-
return value.toISOString();
|
|
34
|
-
return String(value);
|
|
20
|
+
function createSavepointName() {
|
|
21
|
+
const randomPart = Math.floor(Math.random() * 1_000_000_000).toString(36);
|
|
22
|
+
return `syncular_sp_${Date.now().toString(36)}_${randomPart}`;
|
|
35
23
|
}
|
|
36
24
|
function parseJsonValue(value) {
|
|
37
25
|
if (value === null || value === undefined)
|
|
@@ -48,39 +36,6 @@ function parseJsonValue(value) {
|
|
|
48
36
|
}
|
|
49
37
|
return value;
|
|
50
38
|
}
|
|
51
|
-
function parseScopes(value) {
|
|
52
|
-
if (value === null || value === undefined)
|
|
53
|
-
return {};
|
|
54
|
-
if (typeof value === 'object' && !Array.isArray(value)) {
|
|
55
|
-
const result = {};
|
|
56
|
-
for (const [k, v] of Object.entries(value)) {
|
|
57
|
-
if (typeof v === 'string') {
|
|
58
|
-
result[k] = v;
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
return result;
|
|
62
|
-
}
|
|
63
|
-
if (typeof value === 'string') {
|
|
64
|
-
try {
|
|
65
|
-
const parsed = JSON.parse(value);
|
|
66
|
-
if (typeof parsed === 'object' &&
|
|
67
|
-
parsed !== null &&
|
|
68
|
-
!Array.isArray(parsed)) {
|
|
69
|
-
const result = {};
|
|
70
|
-
for (const [k, v] of Object.entries(parsed)) {
|
|
71
|
-
if (typeof v === 'string') {
|
|
72
|
-
result[k] = v;
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
return result;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
catch {
|
|
79
|
-
// ignore
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
return {};
|
|
83
|
-
}
|
|
84
39
|
function toStringArray(value) {
|
|
85
40
|
if (Array.isArray(value)) {
|
|
86
41
|
return value.filter((k) => typeof k === 'string');
|
|
@@ -118,16 +73,48 @@ function scopesMatch(stored, requested) {
|
|
|
118
73
|
}
|
|
119
74
|
return true;
|
|
120
75
|
}
|
|
121
|
-
|
|
76
|
+
async function ensurePartitionColumn(db, table) {
|
|
77
|
+
try {
|
|
78
|
+
await sql
|
|
79
|
+
.raw(`ALTER TABLE ${table} ADD COLUMN partition_id TEXT NOT NULL DEFAULT 'default'`)
|
|
80
|
+
.execute(db);
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
// Ignore when column already exists (or table is immutable in the current backend).
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
async function ensureTransportPathColumn(db) {
|
|
87
|
+
try {
|
|
88
|
+
await sql
|
|
89
|
+
.raw("ALTER TABLE sync_request_events ADD COLUMN transport_path TEXT NOT NULL DEFAULT 'direct'")
|
|
90
|
+
.execute(db);
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
// Ignore when column already exists (or table is immutable in the current backend).
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
export class SqliteServerSyncDialect extends BaseServerSyncDialect {
|
|
122
97
|
name = 'sqlite';
|
|
123
98
|
supportsForUpdate = false;
|
|
124
99
|
supportsSavepoints;
|
|
125
100
|
_supportsTransactions;
|
|
126
101
|
constructor(options) {
|
|
102
|
+
super();
|
|
127
103
|
this._supportsTransactions = options?.supportsTransactions ?? true;
|
|
128
104
|
this.supportsSavepoints = this._supportsTransactions;
|
|
129
105
|
}
|
|
130
106
|
// ===========================================================================
|
|
107
|
+
// SQL Fragment Hooks
|
|
108
|
+
// ===========================================================================
|
|
109
|
+
buildNumberListFilter(values) {
|
|
110
|
+
const list = sql.join(values.map((v) => sql `${v}`), sql `, `);
|
|
111
|
+
return sql `IN (${list})`;
|
|
112
|
+
}
|
|
113
|
+
buildStringListFilter(values) {
|
|
114
|
+
const list = sql.join(values.map((v) => sql `${v}`), sql `, `);
|
|
115
|
+
return sql `IN (${list})`;
|
|
116
|
+
}
|
|
117
|
+
// ===========================================================================
|
|
131
118
|
// Schema Setup
|
|
132
119
|
// ===========================================================================
|
|
133
120
|
async ensureSyncSchema(db) {
|
|
@@ -138,6 +125,7 @@ export class SqliteServerSyncDialect {
|
|
|
138
125
|
.createTable('sync_commits')
|
|
139
126
|
.ifNotExists()
|
|
140
127
|
.addColumn('commit_seq', 'integer', (col) => col.primaryKey().autoIncrement())
|
|
128
|
+
.addColumn('partition_id', 'text', (col) => col.notNull().defaultTo('default'))
|
|
141
129
|
.addColumn('actor_id', 'text', (col) => col.notNull())
|
|
142
130
|
.addColumn('client_id', 'text', (col) => col.notNull())
|
|
143
131
|
.addColumn('client_commit_id', 'text', (col) => col.notNull())
|
|
@@ -147,23 +135,36 @@ export class SqliteServerSyncDialect {
|
|
|
147
135
|
.addColumn('change_count', 'integer', (col) => col.notNull().defaultTo(0))
|
|
148
136
|
.addColumn('affected_tables', 'text', (col) => col.notNull().defaultTo('[]'))
|
|
149
137
|
.execute();
|
|
138
|
+
await ensurePartitionColumn(db, 'sync_commits');
|
|
139
|
+
await sql `DROP INDEX IF EXISTS idx_sync_commits_client_commit`.execute(db);
|
|
150
140
|
await sql `CREATE UNIQUE INDEX IF NOT EXISTS idx_sync_commits_client_commit
|
|
151
|
-
ON sync_commits(client_id, client_commit_id)`.execute(db);
|
|
141
|
+
ON sync_commits(partition_id, client_id, client_commit_id)`.execute(db);
|
|
152
142
|
// sync_table_commits table (index of which commits affect which tables)
|
|
153
143
|
await db.schema
|
|
154
144
|
.createTable('sync_table_commits')
|
|
155
145
|
.ifNotExists()
|
|
146
|
+
.addColumn('partition_id', 'text', (col) => col.notNull().defaultTo('default'))
|
|
156
147
|
.addColumn('table', 'text', (col) => col.notNull())
|
|
157
148
|
.addColumn('commit_seq', 'integer', (col) => col.notNull().references('sync_commits.commit_seq').onDelete('cascade'))
|
|
158
|
-
.addPrimaryKeyConstraint('sync_table_commits_pk', [
|
|
149
|
+
.addPrimaryKeyConstraint('sync_table_commits_pk', [
|
|
150
|
+
'partition_id',
|
|
151
|
+
'table',
|
|
152
|
+
'commit_seq',
|
|
153
|
+
])
|
|
159
154
|
.execute();
|
|
155
|
+
await ensurePartitionColumn(db, 'sync_table_commits');
|
|
156
|
+
// Ensure unique index matches ON CONFLICT clause in push.ts
|
|
157
|
+
// (needed when migrating from old schema where PK was only (table, commit_seq))
|
|
158
|
+
await sql `CREATE UNIQUE INDEX IF NOT EXISTS idx_sync_table_commits_partition_pk
|
|
159
|
+
ON sync_table_commits(partition_id, "table", commit_seq)`.execute(db);
|
|
160
160
|
await sql `CREATE INDEX IF NOT EXISTS idx_sync_table_commits_commit_seq
|
|
161
|
-
ON sync_table_commits(commit_seq)`.execute(db);
|
|
161
|
+
ON sync_table_commits(partition_id, commit_seq)`.execute(db);
|
|
162
162
|
// sync_changes table - uses JSON for scopes
|
|
163
163
|
await db.schema
|
|
164
164
|
.createTable('sync_changes')
|
|
165
165
|
.ifNotExists()
|
|
166
166
|
.addColumn('change_id', 'integer', (col) => col.primaryKey().autoIncrement())
|
|
167
|
+
.addColumn('partition_id', 'text', (col) => col.notNull().defaultTo('default'))
|
|
167
168
|
.addColumn('commit_seq', 'integer', (col) => col.notNull().references('sync_commits.commit_seq').onDelete('cascade'))
|
|
168
169
|
.addColumn('table', 'text', (col) => col.notNull())
|
|
169
170
|
.addColumn('row_id', 'text', (col) => col.notNull())
|
|
@@ -172,20 +173,31 @@ export class SqliteServerSyncDialect {
|
|
|
172
173
|
.addColumn('row_version', 'integer')
|
|
173
174
|
.addColumn('scopes', 'json', (col) => col.notNull())
|
|
174
175
|
.execute();
|
|
176
|
+
await ensurePartitionColumn(db, 'sync_changes');
|
|
175
177
|
await sql `CREATE INDEX IF NOT EXISTS idx_sync_changes_commit_seq
|
|
176
|
-
ON sync_changes(commit_seq)`.execute(db);
|
|
178
|
+
ON sync_changes(partition_id, commit_seq)`.execute(db);
|
|
177
179
|
await sql `CREATE INDEX IF NOT EXISTS idx_sync_changes_table
|
|
178
|
-
ON sync_changes("table")`.execute(db);
|
|
180
|
+
ON sync_changes(partition_id, "table")`.execute(db);
|
|
179
181
|
// sync_client_cursors table
|
|
180
182
|
await db.schema
|
|
181
183
|
.createTable('sync_client_cursors')
|
|
182
184
|
.ifNotExists()
|
|
183
|
-
.addColumn('
|
|
185
|
+
.addColumn('partition_id', 'text', (col) => col.notNull().defaultTo('default'))
|
|
186
|
+
.addColumn('client_id', 'text', (col) => col.notNull())
|
|
184
187
|
.addColumn('actor_id', 'text', (col) => col.notNull())
|
|
185
188
|
.addColumn('cursor', 'integer', (col) => col.notNull().defaultTo(0))
|
|
186
189
|
.addColumn('effective_scopes', 'json', (col) => col.notNull().defaultTo('{}'))
|
|
187
190
|
.addColumn('updated_at', 'text', (col) => col.notNull().defaultTo(nowIso))
|
|
191
|
+
.addPrimaryKeyConstraint('sync_client_cursors_pk', [
|
|
192
|
+
'partition_id',
|
|
193
|
+
'client_id',
|
|
194
|
+
])
|
|
188
195
|
.execute();
|
|
196
|
+
await ensurePartitionColumn(db, 'sync_client_cursors');
|
|
197
|
+
// Ensure unique index matches ON CONFLICT clause in recordClientCursor
|
|
198
|
+
// (needed when migrating from old schema where PK was only (client_id))
|
|
199
|
+
await sql `CREATE UNIQUE INDEX IF NOT EXISTS idx_sync_client_cursors_partition_pk
|
|
200
|
+
ON sync_client_cursors(partition_id, client_id)`.execute(db);
|
|
189
201
|
await sql `CREATE INDEX IF NOT EXISTS idx_sync_client_cursors_updated_at
|
|
190
202
|
ON sync_client_cursors(updated_at)`.execute(db);
|
|
191
203
|
// sync_snapshot_chunks table
|
|
@@ -193,6 +205,7 @@ export class SqliteServerSyncDialect {
|
|
|
193
205
|
.createTable('sync_snapshot_chunks')
|
|
194
206
|
.ifNotExists()
|
|
195
207
|
.addColumn('chunk_id', 'text', (col) => col.primaryKey())
|
|
208
|
+
.addColumn('partition_id', 'text', (col) => col.notNull().defaultTo('default'))
|
|
196
209
|
.addColumn('scope_key', 'text', (col) => col.notNull())
|
|
197
210
|
.addColumn('scope', 'text', (col) => col.notNull())
|
|
198
211
|
.addColumn('as_of_commit_seq', 'integer', (col) => col.notNull())
|
|
@@ -207,10 +220,11 @@ export class SqliteServerSyncDialect {
|
|
|
207
220
|
.addColumn('created_at', 'text', (col) => col.notNull().defaultTo(nowIso))
|
|
208
221
|
.addColumn('expires_at', 'text', (col) => col.notNull())
|
|
209
222
|
.execute();
|
|
223
|
+
await ensurePartitionColumn(db, 'sync_snapshot_chunks');
|
|
210
224
|
await sql `CREATE INDEX IF NOT EXISTS idx_sync_snapshot_chunks_expires_at
|
|
211
225
|
ON sync_snapshot_chunks(expires_at)`.execute(db);
|
|
212
226
|
await sql `CREATE UNIQUE INDEX IF NOT EXISTS idx_sync_snapshot_chunks_page_key
|
|
213
|
-
ON sync_snapshot_chunks(scope_key, scope, as_of_commit_seq, row_cursor, row_limit, encoding, compression)`.execute(db);
|
|
227
|
+
ON sync_snapshot_chunks(partition_id, scope_key, scope, as_of_commit_seq, row_cursor, row_limit, encoding, compression)`.execute(db);
|
|
214
228
|
// Cleanup orphaned rows
|
|
215
229
|
await sql `
|
|
216
230
|
DELETE FROM sync_table_commits
|
|
@@ -225,6 +239,23 @@ export class SqliteServerSyncDialect {
|
|
|
225
239
|
// Transaction Control
|
|
226
240
|
// ===========================================================================
|
|
227
241
|
async executeInTransaction(db, fn) {
|
|
242
|
+
if (isActiveTransaction(db)) {
|
|
243
|
+
if (!this._supportsTransactions) {
|
|
244
|
+
return fn(db);
|
|
245
|
+
}
|
|
246
|
+
const savepoint = createSavepointName();
|
|
247
|
+
await sql.raw(`SAVEPOINT ${savepoint}`).execute(db);
|
|
248
|
+
try {
|
|
249
|
+
const result = await fn(db);
|
|
250
|
+
await sql.raw(`RELEASE SAVEPOINT ${savepoint}`).execute(db);
|
|
251
|
+
return result;
|
|
252
|
+
}
|
|
253
|
+
catch (error) {
|
|
254
|
+
await sql.raw(`ROLLBACK TO SAVEPOINT ${savepoint}`).execute(db);
|
|
255
|
+
await sql.raw(`RELEASE SAVEPOINT ${savepoint}`).execute(db);
|
|
256
|
+
throw error;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
228
259
|
if (this._supportsTransactions) {
|
|
229
260
|
return db.transaction().execute(fn);
|
|
230
261
|
}
|
|
@@ -234,56 +265,10 @@ export class SqliteServerSyncDialect {
|
|
|
234
265
|
// SQLite uses serializable isolation by default in WAL mode.
|
|
235
266
|
}
|
|
236
267
|
// ===========================================================================
|
|
237
|
-
// Commit/Change Log Queries
|
|
268
|
+
// Commit/Change Log Queries (dialect-specific)
|
|
238
269
|
// ===========================================================================
|
|
239
|
-
async readMaxCommitSeq(db) {
|
|
240
|
-
const res = await sql `
|
|
241
|
-
SELECT max(commit_seq) as max_seq
|
|
242
|
-
FROM sync_commits
|
|
243
|
-
`.execute(db);
|
|
244
|
-
return coerceNumber(res.rows[0]?.max_seq) ?? 0;
|
|
245
|
-
}
|
|
246
|
-
async readMinCommitSeq(db) {
|
|
247
|
-
const res = await sql `
|
|
248
|
-
SELECT min(commit_seq) as min_seq
|
|
249
|
-
FROM sync_commits
|
|
250
|
-
`.execute(db);
|
|
251
|
-
return coerceNumber(res.rows[0]?.min_seq) ?? 0;
|
|
252
|
-
}
|
|
253
|
-
async readCommitSeqsForPull(db, args) {
|
|
254
|
-
if (args.tables.length === 0)
|
|
255
|
-
return [];
|
|
256
|
-
const tablesIn = sql.join(args.tables.map((t) => sql `${t}`), sql `, `);
|
|
257
|
-
const res = await sql `
|
|
258
|
-
SELECT DISTINCT commit_seq
|
|
259
|
-
FROM sync_table_commits
|
|
260
|
-
WHERE "table" IN (${tablesIn})
|
|
261
|
-
AND commit_seq > ${args.cursor}
|
|
262
|
-
ORDER BY commit_seq ASC
|
|
263
|
-
LIMIT ${args.limitCommits}
|
|
264
|
-
`.execute(db);
|
|
265
|
-
return res.rows
|
|
266
|
-
.map((r) => coerceNumber(r.commit_seq))
|
|
267
|
-
.filter((n) => typeof n === 'number' && Number.isFinite(n) && n > args.cursor);
|
|
268
|
-
}
|
|
269
|
-
async readCommits(db, commitSeqs) {
|
|
270
|
-
if (commitSeqs.length === 0)
|
|
271
|
-
return [];
|
|
272
|
-
const commitSeqsIn = sql.join(commitSeqs.map((seq) => sql `${seq}`), sql `, `);
|
|
273
|
-
const res = await sql `
|
|
274
|
-
SELECT commit_seq, actor_id, created_at, result_json
|
|
275
|
-
FROM sync_commits
|
|
276
|
-
WHERE commit_seq IN (${commitSeqsIn})
|
|
277
|
-
ORDER BY commit_seq ASC
|
|
278
|
-
`.execute(db);
|
|
279
|
-
return res.rows.map((row) => ({
|
|
280
|
-
commit_seq: coerceNumber(row.commit_seq) ?? 0,
|
|
281
|
-
actor_id: row.actor_id,
|
|
282
|
-
created_at: coerceIsoString(row.created_at),
|
|
283
|
-
result_json: row.result_json ?? null,
|
|
284
|
-
}));
|
|
285
|
-
}
|
|
286
270
|
async readChangesForCommits(db, args) {
|
|
271
|
+
const partitionId = args.partitionId ?? 'default';
|
|
287
272
|
if (args.commitSeqs.length === 0)
|
|
288
273
|
return [];
|
|
289
274
|
const commitSeqsIn = sql.join(args.commitSeqs.map((seq) => sql `${seq}`), sql `, `);
|
|
@@ -292,6 +277,7 @@ export class SqliteServerSyncDialect {
|
|
|
292
277
|
SELECT commit_seq, "table", row_id, op, row_json, row_version, scopes
|
|
293
278
|
FROM sync_changes
|
|
294
279
|
WHERE commit_seq IN (${commitSeqsIn})
|
|
280
|
+
AND partition_id = ${partitionId}
|
|
295
281
|
AND "table" = ${args.table}
|
|
296
282
|
ORDER BY commit_seq ASC, change_id ASC
|
|
297
283
|
`.execute(db);
|
|
@@ -311,18 +297,21 @@ export class SqliteServerSyncDialect {
|
|
|
311
297
|
scopes: parseScopes(row.scopes),
|
|
312
298
|
}));
|
|
313
299
|
}
|
|
314
|
-
async
|
|
300
|
+
async readIncrementalPullRowsBatch(db, args) {
|
|
301
|
+
const partitionId = args.partitionId ?? 'default';
|
|
315
302
|
const limitCommits = Math.max(1, Math.min(500, args.limitCommits));
|
|
316
303
|
// Get commit_seqs for this table
|
|
317
304
|
const commitSeqsRes = await sql `
|
|
318
305
|
SELECT commit_seq
|
|
319
306
|
FROM sync_table_commits
|
|
320
|
-
WHERE
|
|
307
|
+
WHERE partition_id = ${partitionId}
|
|
308
|
+
AND "table" = ${args.table}
|
|
321
309
|
AND commit_seq > ${args.cursor}
|
|
322
310
|
AND EXISTS (
|
|
323
311
|
SELECT 1
|
|
324
312
|
FROM sync_commits cm
|
|
325
313
|
WHERE cm.commit_seq = sync_table_commits.commit_seq
|
|
314
|
+
AND cm.partition_id = ${partitionId}
|
|
326
315
|
)
|
|
327
316
|
ORDER BY commit_seq ASC
|
|
328
317
|
LIMIT ${limitCommits}
|
|
@@ -349,6 +338,8 @@ export class SqliteServerSyncDialect {
|
|
|
349
338
|
FROM sync_commits cm
|
|
350
339
|
JOIN sync_changes c ON c.commit_seq = cm.commit_seq
|
|
351
340
|
WHERE cm.commit_seq IN (${commitSeqsIn})
|
|
341
|
+
AND cm.partition_id = ${partitionId}
|
|
342
|
+
AND c.partition_id = ${partitionId}
|
|
352
343
|
AND c."table" = ${args.table}
|
|
353
344
|
ORDER BY cm.commit_seq ASC, c.change_id ASC
|
|
354
345
|
`.execute(db);
|
|
@@ -371,88 +362,20 @@ export class SqliteServerSyncDialect {
|
|
|
371
362
|
scopes: parseScopes(row.scopes),
|
|
372
363
|
}));
|
|
373
364
|
}
|
|
374
|
-
/**
|
|
375
|
-
* Streaming version of incremental pull for large result sets.
|
|
376
|
-
* Yields changes one at a time instead of loading all into memory.
|
|
377
|
-
*/
|
|
378
|
-
async *streamIncrementalPullRows(db, args) {
|
|
379
|
-
const limitCommits = Math.max(1, Math.min(500, args.limitCommits));
|
|
380
|
-
// Get commit_seqs for this table
|
|
381
|
-
const commitSeqsRes = await sql `
|
|
382
|
-
SELECT commit_seq
|
|
383
|
-
FROM sync_table_commits
|
|
384
|
-
WHERE "table" = ${args.table}
|
|
385
|
-
AND commit_seq > ${args.cursor}
|
|
386
|
-
AND EXISTS (
|
|
387
|
-
SELECT 1
|
|
388
|
-
FROM sync_commits cm
|
|
389
|
-
WHERE cm.commit_seq = sync_table_commits.commit_seq
|
|
390
|
-
)
|
|
391
|
-
ORDER BY commit_seq ASC
|
|
392
|
-
LIMIT ${limitCommits}
|
|
393
|
-
`.execute(db);
|
|
394
|
-
const commitSeqs = commitSeqsRes.rows
|
|
395
|
-
.map((r) => coerceNumber(r.commit_seq))
|
|
396
|
-
.filter((n) => n !== null);
|
|
397
|
-
if (commitSeqs.length === 0)
|
|
398
|
-
return;
|
|
399
|
-
// Process in smaller batches to avoid memory issues
|
|
400
|
-
const batchSize = 100;
|
|
401
|
-
for (let i = 0; i < commitSeqs.length; i += batchSize) {
|
|
402
|
-
const batch = commitSeqs.slice(i, i + batchSize);
|
|
403
|
-
const commitSeqsIn = sql.join(batch.map((seq) => sql `${seq}`), sql `, `);
|
|
404
|
-
const changesRes = await sql `
|
|
405
|
-
SELECT
|
|
406
|
-
cm.commit_seq,
|
|
407
|
-
cm.actor_id,
|
|
408
|
-
cm.created_at,
|
|
409
|
-
c.change_id,
|
|
410
|
-
c."table",
|
|
411
|
-
c.row_id,
|
|
412
|
-
c.op,
|
|
413
|
-
c.row_json,
|
|
414
|
-
c.row_version,
|
|
415
|
-
c.scopes
|
|
416
|
-
FROM sync_commits cm
|
|
417
|
-
JOIN sync_changes c ON c.commit_seq = cm.commit_seq
|
|
418
|
-
WHERE cm.commit_seq IN (${commitSeqsIn})
|
|
419
|
-
AND c."table" = ${args.table}
|
|
420
|
-
ORDER BY cm.commit_seq ASC, c.change_id ASC
|
|
421
|
-
`.execute(db);
|
|
422
|
-
// Filter and yield each row
|
|
423
|
-
for (const row of changesRes.rows) {
|
|
424
|
-
const storedScopes = parseScopes(row.scopes);
|
|
425
|
-
if (scopesMatch(storedScopes, args.scopes)) {
|
|
426
|
-
yield {
|
|
427
|
-
commit_seq: coerceNumber(row.commit_seq) ?? 0,
|
|
428
|
-
actor_id: row.actor_id,
|
|
429
|
-
created_at: coerceIsoString(row.created_at),
|
|
430
|
-
change_id: coerceNumber(row.change_id) ?? 0,
|
|
431
|
-
table: row.table,
|
|
432
|
-
row_id: row.row_id,
|
|
433
|
-
op: row.op,
|
|
434
|
-
row_json: parseJsonValue(row.row_json),
|
|
435
|
-
row_version: coerceNumber(row.row_version),
|
|
436
|
-
scopes: storedScopes,
|
|
437
|
-
};
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
}
|
|
441
|
-
}
|
|
442
365
|
async compactChanges(db, args) {
|
|
443
366
|
const cutoffIso = new Date(Date.now() - args.fullHistoryHours * 60 * 60 * 1000).toISOString();
|
|
444
367
|
// Find all old changes
|
|
445
368
|
const oldChanges = await sql `
|
|
446
|
-
SELECT c.change_id, c.commit_seq, c."table", c.row_id, c.scopes
|
|
369
|
+
SELECT c.change_id, c.partition_id, c.commit_seq, c."table", c.row_id, c.scopes
|
|
447
370
|
FROM sync_changes c
|
|
448
371
|
JOIN sync_commits cm ON cm.commit_seq = c.commit_seq
|
|
449
372
|
WHERE cm.created_at < ${cutoffIso}
|
|
450
373
|
`.execute(db);
|
|
451
|
-
// Group by (table, row_id, scopes)
|
|
374
|
+
// Group by (partition_id, table, row_id, scopes)
|
|
452
375
|
const groups = new Map();
|
|
453
376
|
for (const row of oldChanges.rows) {
|
|
454
377
|
const scopesStr = JSON.stringify(parseScopes(row.scopes));
|
|
455
|
-
const key = `${row.table}|${row.row_id}|${scopesStr}`;
|
|
378
|
+
const key = `${row.partition_id}|${row.table}|${row.row_id}|${scopesStr}`;
|
|
456
379
|
if (!groups.has(key)) {
|
|
457
380
|
groups.set(key, []);
|
|
458
381
|
}
|
|
@@ -478,10 +401,10 @@ export class SqliteServerSyncDialect {
|
|
|
478
401
|
if (toDelete.length === 0)
|
|
479
402
|
return 0;
|
|
480
403
|
// Delete in batches
|
|
481
|
-
const
|
|
404
|
+
const deleteBatchSize = 500;
|
|
482
405
|
let deleted = 0;
|
|
483
|
-
for (let i = 0; i < toDelete.length; i +=
|
|
484
|
-
const batch = toDelete.slice(i, i +
|
|
406
|
+
for (let i = 0; i < toDelete.length; i += deleteBatchSize) {
|
|
407
|
+
const batch = toDelete.slice(i, i + deleteBatchSize);
|
|
485
408
|
const batchIn = sql.join(batch.map((id) => sql `${id}`), sql `, `);
|
|
486
409
|
const res = await sql `
|
|
487
410
|
DELETE FROM sync_changes
|
|
@@ -496,57 +419,30 @@ export class SqliteServerSyncDialect {
|
|
|
496
419
|
SELECT commit_seq
|
|
497
420
|
FROM sync_commits
|
|
498
421
|
WHERE created_at < ${cutoffIso}
|
|
422
|
+
AND partition_id = sync_table_commits.partition_id
|
|
499
423
|
)
|
|
500
424
|
AND NOT EXISTS (
|
|
501
425
|
SELECT 1
|
|
502
426
|
FROM sync_changes c
|
|
503
427
|
WHERE c.commit_seq = sync_table_commits.commit_seq
|
|
428
|
+
AND c.partition_id = sync_table_commits.partition_id
|
|
504
429
|
AND c."table" = sync_table_commits."table"
|
|
505
430
|
)
|
|
506
431
|
`.execute(db);
|
|
507
432
|
return deleted;
|
|
508
433
|
}
|
|
509
434
|
// ===========================================================================
|
|
510
|
-
// Client Cursor Recording
|
|
511
|
-
// ===========================================================================
|
|
512
|
-
async recordClientCursor(db, args) {
|
|
513
|
-
const now = new Date().toISOString();
|
|
514
|
-
const scopesJson = JSON.stringify(args.effectiveScopes);
|
|
515
|
-
await sql `
|
|
516
|
-
INSERT INTO sync_client_cursors (client_id, actor_id, cursor, effective_scopes, updated_at)
|
|
517
|
-
VALUES (${args.clientId}, ${args.actorId}, ${args.cursor}, ${scopesJson}, ${now})
|
|
518
|
-
ON CONFLICT(client_id) DO UPDATE SET
|
|
519
|
-
actor_id = ${args.actorId},
|
|
520
|
-
cursor = ${args.cursor},
|
|
521
|
-
effective_scopes = ${scopesJson},
|
|
522
|
-
updated_at = ${now}
|
|
523
|
-
`.execute(db);
|
|
524
|
-
}
|
|
525
|
-
// ===========================================================================
|
|
526
435
|
// Scope Conversion Helpers
|
|
527
436
|
// ===========================================================================
|
|
528
437
|
scopesToDb(scopes) {
|
|
529
438
|
return JSON.stringify(scopes);
|
|
530
439
|
}
|
|
531
|
-
dbToScopes(value) {
|
|
532
|
-
return parseScopes(value);
|
|
533
|
-
}
|
|
534
440
|
dbToArray(value) {
|
|
535
441
|
return toStringArray(value);
|
|
536
442
|
}
|
|
537
443
|
arrayToDb(values) {
|
|
538
444
|
return JSON.stringify(values.filter((v) => v.length > 0));
|
|
539
445
|
}
|
|
540
|
-
async readAffectedTablesFromChanges(db, commitSeq) {
|
|
541
|
-
const res = await sql `
|
|
542
|
-
SELECT DISTINCT "table"
|
|
543
|
-
FROM sync_changes
|
|
544
|
-
WHERE commit_seq = ${commitSeq}
|
|
545
|
-
`.execute(db);
|
|
546
|
-
return res.rows
|
|
547
|
-
.map((r) => r.table)
|
|
548
|
-
.filter((t) => typeof t === 'string' && t.length > 0);
|
|
549
|
-
}
|
|
550
446
|
// ===========================================================================
|
|
551
447
|
// Console Schema (Request Events)
|
|
552
448
|
// ===========================================================================
|
|
@@ -567,8 +463,10 @@ export class SqliteServerSyncDialect {
|
|
|
567
463
|
.addColumn('row_count', 'integer')
|
|
568
464
|
.addColumn('tables', 'text', (col) => col.notNull().defaultTo('[]'))
|
|
569
465
|
.addColumn('error_message', 'text')
|
|
466
|
+
.addColumn('transport_path', 'text', (col) => col.notNull().defaultTo('direct'))
|
|
570
467
|
.addColumn('created_at', 'text', (col) => col.notNull().defaultTo(nowIso))
|
|
571
468
|
.execute();
|
|
469
|
+
await ensureTransportPathColumn(db);
|
|
572
470
|
await sql `CREATE INDEX IF NOT EXISTS idx_sync_request_events_created_at
|
|
573
471
|
ON sync_request_events(created_at DESC)`.execute(db);
|
|
574
472
|
await sql `CREATE INDEX IF NOT EXISTS idx_sync_request_events_event_type
|