@syncular/server-dialect-sqlite 0.0.1 → 0.0.2-126
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -0
- package/dist/index.d.ts +12 -61
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +110 -212
- package/dist/index.js.map +1 -1
- package/package.json +25 -5
- package/src/index.test.ts +149 -0
- package/src/index.ts +165 -332
package/src/index.ts
CHANGED
|
@@ -14,31 +14,28 @@
|
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
import type { ScopeValues, StoredScopes, SyncOp } from '@syncular/core';
|
|
17
|
-
import type { DbExecutor
|
|
18
|
-
import
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
17
|
+
import type { DbExecutor } from '@syncular/server';
|
|
18
|
+
import {
|
|
19
|
+
BaseServerSyncDialect,
|
|
20
|
+
coerceIsoString,
|
|
21
|
+
coerceNumber,
|
|
22
|
+
type IncrementalPullRow,
|
|
23
|
+
type IncrementalPullRowsArgs,
|
|
24
|
+
parseScopes,
|
|
25
|
+
} from '@syncular/server';
|
|
26
|
+
import type { SyncChangeRow, SyncCoreDb } from '@syncular/server/schema';
|
|
27
|
+
import type { Kysely, RawBuilder, Transaction } from 'kysely';
|
|
24
28
|
import { sql } from 'kysely';
|
|
25
29
|
|
|
26
|
-
function
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
return Number.isFinite(Number(value)) ? Number(value) : null;
|
|
31
|
-
if (typeof value === 'string') {
|
|
32
|
-
const n = Number(value);
|
|
33
|
-
return Number.isFinite(n) ? n : null;
|
|
34
|
-
}
|
|
35
|
-
return null;
|
|
30
|
+
function isActiveTransaction<DB extends SyncCoreDb>(
|
|
31
|
+
db: Kysely<DB>
|
|
32
|
+
): db is Kysely<DB> & Transaction<DB> {
|
|
33
|
+
return (db as { isTransaction?: boolean }).isTransaction === true;
|
|
36
34
|
}
|
|
37
35
|
|
|
38
|
-
function
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
return String(value);
|
|
36
|
+
function createSavepointName(): string {
|
|
37
|
+
const randomPart = Math.floor(Math.random() * 1_000_000_000).toString(36);
|
|
38
|
+
return `syncular_sp_${Date.now().toString(36)}_${randomPart}`;
|
|
42
39
|
}
|
|
43
40
|
|
|
44
41
|
function parseJsonValue(value: unknown): unknown {
|
|
@@ -54,42 +51,6 @@ function parseJsonValue(value: unknown): unknown {
|
|
|
54
51
|
return value;
|
|
55
52
|
}
|
|
56
53
|
|
|
57
|
-
function parseScopes(value: unknown): StoredScopes {
|
|
58
|
-
if (value === null || value === undefined) return {};
|
|
59
|
-
if (typeof value === 'object' && !Array.isArray(value)) {
|
|
60
|
-
const result: StoredScopes = {};
|
|
61
|
-
for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
|
|
62
|
-
if (typeof v === 'string') {
|
|
63
|
-
result[k] = v;
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
return result;
|
|
67
|
-
}
|
|
68
|
-
if (typeof value === 'string') {
|
|
69
|
-
try {
|
|
70
|
-
const parsed = JSON.parse(value);
|
|
71
|
-
if (
|
|
72
|
-
typeof parsed === 'object' &&
|
|
73
|
-
parsed !== null &&
|
|
74
|
-
!Array.isArray(parsed)
|
|
75
|
-
) {
|
|
76
|
-
const result: StoredScopes = {};
|
|
77
|
-
for (const [k, v] of Object.entries(
|
|
78
|
-
parsed as Record<string, unknown>
|
|
79
|
-
)) {
|
|
80
|
-
if (typeof v === 'string') {
|
|
81
|
-
result[k] = v;
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
return result;
|
|
85
|
-
}
|
|
86
|
-
} catch {
|
|
87
|
-
// ignore
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
return {};
|
|
91
|
-
}
|
|
92
|
-
|
|
93
54
|
function toStringArray(value: unknown): string[] {
|
|
94
55
|
if (Array.isArray(value)) {
|
|
95
56
|
return value.filter((k: unknown): k is string => typeof k === 'string');
|
|
@@ -126,17 +87,67 @@ function scopesMatch(stored: StoredScopes, requested: ScopeValues): boolean {
|
|
|
126
87
|
return true;
|
|
127
88
|
}
|
|
128
89
|
|
|
129
|
-
|
|
90
|
+
async function ensurePartitionColumn<DB extends SyncCoreDb>(
|
|
91
|
+
db: Kysely<DB>,
|
|
92
|
+
table: string
|
|
93
|
+
): Promise<void> {
|
|
94
|
+
try {
|
|
95
|
+
await sql
|
|
96
|
+
.raw(
|
|
97
|
+
`ALTER TABLE ${table} ADD COLUMN partition_id TEXT NOT NULL DEFAULT 'default'`
|
|
98
|
+
)
|
|
99
|
+
.execute(db);
|
|
100
|
+
} catch {
|
|
101
|
+
// Ignore when column already exists (or table is immutable in the current backend).
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function ensureTransportPathColumn<DB extends SyncCoreDb>(
|
|
106
|
+
db: Kysely<DB>
|
|
107
|
+
): Promise<void> {
|
|
108
|
+
try {
|
|
109
|
+
await sql
|
|
110
|
+
.raw(
|
|
111
|
+
"ALTER TABLE sync_request_events ADD COLUMN transport_path TEXT NOT NULL DEFAULT 'direct'"
|
|
112
|
+
)
|
|
113
|
+
.execute(db);
|
|
114
|
+
} catch {
|
|
115
|
+
// Ignore when column already exists (or table is immutable in the current backend).
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export class SqliteServerSyncDialect extends BaseServerSyncDialect {
|
|
130
120
|
readonly name = 'sqlite' as const;
|
|
131
121
|
readonly supportsForUpdate = false;
|
|
132
122
|
readonly supportsSavepoints: boolean;
|
|
133
123
|
private readonly _supportsTransactions: boolean;
|
|
134
124
|
|
|
135
125
|
constructor(options?: { supportsTransactions?: boolean }) {
|
|
126
|
+
super();
|
|
136
127
|
this._supportsTransactions = options?.supportsTransactions ?? true;
|
|
137
128
|
this.supportsSavepoints = this._supportsTransactions;
|
|
138
129
|
}
|
|
139
130
|
|
|
131
|
+
// ===========================================================================
|
|
132
|
+
// SQL Fragment Hooks
|
|
133
|
+
// ===========================================================================
|
|
134
|
+
|
|
135
|
+
protected buildNumberListFilter(values: number[]): RawBuilder<unknown> {
|
|
136
|
+
const list = sql.join(
|
|
137
|
+
values.map((v) => sql`${v}`),
|
|
138
|
+
sql`, `
|
|
139
|
+
);
|
|
140
|
+
return sql`IN (${list})`;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
protected buildStringListFilter(values: string[]): RawBuilder<unknown> {
|
|
144
|
+
const list = sql.join(
|
|
145
|
+
values.map((v) => sql`${v}`),
|
|
146
|
+
sql`, `
|
|
147
|
+
);
|
|
148
|
+
return sql`IN (${list})`;
|
|
149
|
+
}
|
|
150
|
+
|
|
140
151
|
// ===========================================================================
|
|
141
152
|
// Schema Setup
|
|
142
153
|
// ===========================================================================
|
|
@@ -153,6 +164,9 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
153
164
|
.addColumn('commit_seq', 'integer', (col) =>
|
|
154
165
|
col.primaryKey().autoIncrement()
|
|
155
166
|
)
|
|
167
|
+
.addColumn('partition_id', 'text', (col) =>
|
|
168
|
+
col.notNull().defaultTo('default')
|
|
169
|
+
)
|
|
156
170
|
.addColumn('actor_id', 'text', (col) => col.notNull())
|
|
157
171
|
.addColumn('client_id', 'text', (col) => col.notNull())
|
|
158
172
|
.addColumn('client_commit_id', 'text', (col) => col.notNull())
|
|
@@ -164,23 +178,38 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
164
178
|
col.notNull().defaultTo('[]')
|
|
165
179
|
)
|
|
166
180
|
.execute();
|
|
181
|
+
await ensurePartitionColumn(db, 'sync_commits');
|
|
167
182
|
|
|
183
|
+
await sql`DROP INDEX IF EXISTS idx_sync_commits_client_commit`.execute(db);
|
|
168
184
|
await sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_sync_commits_client_commit
|
|
169
|
-
ON sync_commits(client_id, client_commit_id)`.execute(db);
|
|
185
|
+
ON sync_commits(partition_id, client_id, client_commit_id)`.execute(db);
|
|
170
186
|
|
|
171
187
|
// sync_table_commits table (index of which commits affect which tables)
|
|
172
188
|
await db.schema
|
|
173
189
|
.createTable('sync_table_commits')
|
|
174
190
|
.ifNotExists()
|
|
191
|
+
.addColumn('partition_id', 'text', (col) =>
|
|
192
|
+
col.notNull().defaultTo('default')
|
|
193
|
+
)
|
|
175
194
|
.addColumn('table', 'text', (col) => col.notNull())
|
|
176
195
|
.addColumn('commit_seq', 'integer', (col) =>
|
|
177
196
|
col.notNull().references('sync_commits.commit_seq').onDelete('cascade')
|
|
178
197
|
)
|
|
179
|
-
.addPrimaryKeyConstraint('sync_table_commits_pk', [
|
|
198
|
+
.addPrimaryKeyConstraint('sync_table_commits_pk', [
|
|
199
|
+
'partition_id',
|
|
200
|
+
'table',
|
|
201
|
+
'commit_seq',
|
|
202
|
+
])
|
|
180
203
|
.execute();
|
|
204
|
+
await ensurePartitionColumn(db, 'sync_table_commits');
|
|
205
|
+
|
|
206
|
+
// Ensure unique index matches ON CONFLICT clause in push.ts
|
|
207
|
+
// (needed when migrating from old schema where PK was only (table, commit_seq))
|
|
208
|
+
await sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_sync_table_commits_partition_pk
|
|
209
|
+
ON sync_table_commits(partition_id, "table", commit_seq)`.execute(db);
|
|
181
210
|
|
|
182
211
|
await sql`CREATE INDEX IF NOT EXISTS idx_sync_table_commits_commit_seq
|
|
183
|
-
ON sync_table_commits(commit_seq)`.execute(db);
|
|
212
|
+
ON sync_table_commits(partition_id, commit_seq)`.execute(db);
|
|
184
213
|
|
|
185
214
|
// sync_changes table - uses JSON for scopes
|
|
186
215
|
await db.schema
|
|
@@ -189,6 +218,9 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
189
218
|
.addColumn('change_id', 'integer', (col) =>
|
|
190
219
|
col.primaryKey().autoIncrement()
|
|
191
220
|
)
|
|
221
|
+
.addColumn('partition_id', 'text', (col) =>
|
|
222
|
+
col.notNull().defaultTo('default')
|
|
223
|
+
)
|
|
192
224
|
.addColumn('commit_seq', 'integer', (col) =>
|
|
193
225
|
col.notNull().references('sync_commits.commit_seq').onDelete('cascade')
|
|
194
226
|
)
|
|
@@ -199,25 +231,39 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
199
231
|
.addColumn('row_version', 'integer')
|
|
200
232
|
.addColumn('scopes', 'json', (col) => col.notNull())
|
|
201
233
|
.execute();
|
|
234
|
+
await ensurePartitionColumn(db, 'sync_changes');
|
|
202
235
|
|
|
203
236
|
await sql`CREATE INDEX IF NOT EXISTS idx_sync_changes_commit_seq
|
|
204
|
-
ON sync_changes(commit_seq)`.execute(db);
|
|
237
|
+
ON sync_changes(partition_id, commit_seq)`.execute(db);
|
|
205
238
|
|
|
206
239
|
await sql`CREATE INDEX IF NOT EXISTS idx_sync_changes_table
|
|
207
|
-
ON sync_changes("table")`.execute(db);
|
|
240
|
+
ON sync_changes(partition_id, "table")`.execute(db);
|
|
208
241
|
|
|
209
242
|
// sync_client_cursors table
|
|
210
243
|
await db.schema
|
|
211
244
|
.createTable('sync_client_cursors')
|
|
212
245
|
.ifNotExists()
|
|
213
|
-
.addColumn('
|
|
246
|
+
.addColumn('partition_id', 'text', (col) =>
|
|
247
|
+
col.notNull().defaultTo('default')
|
|
248
|
+
)
|
|
249
|
+
.addColumn('client_id', 'text', (col) => col.notNull())
|
|
214
250
|
.addColumn('actor_id', 'text', (col) => col.notNull())
|
|
215
251
|
.addColumn('cursor', 'integer', (col) => col.notNull().defaultTo(0))
|
|
216
252
|
.addColumn('effective_scopes', 'json', (col) =>
|
|
217
253
|
col.notNull().defaultTo('{}')
|
|
218
254
|
)
|
|
219
255
|
.addColumn('updated_at', 'text', (col) => col.notNull().defaultTo(nowIso))
|
|
256
|
+
.addPrimaryKeyConstraint('sync_client_cursors_pk', [
|
|
257
|
+
'partition_id',
|
|
258
|
+
'client_id',
|
|
259
|
+
])
|
|
220
260
|
.execute();
|
|
261
|
+
await ensurePartitionColumn(db, 'sync_client_cursors');
|
|
262
|
+
|
|
263
|
+
// Ensure unique index matches ON CONFLICT clause in recordClientCursor
|
|
264
|
+
// (needed when migrating from old schema where PK was only (client_id))
|
|
265
|
+
await sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_sync_client_cursors_partition_pk
|
|
266
|
+
ON sync_client_cursors(partition_id, client_id)`.execute(db);
|
|
221
267
|
|
|
222
268
|
await sql`CREATE INDEX IF NOT EXISTS idx_sync_client_cursors_updated_at
|
|
223
269
|
ON sync_client_cursors(updated_at)`.execute(db);
|
|
@@ -227,6 +273,9 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
227
273
|
.createTable('sync_snapshot_chunks')
|
|
228
274
|
.ifNotExists()
|
|
229
275
|
.addColumn('chunk_id', 'text', (col) => col.primaryKey())
|
|
276
|
+
.addColumn('partition_id', 'text', (col) =>
|
|
277
|
+
col.notNull().defaultTo('default')
|
|
278
|
+
)
|
|
230
279
|
.addColumn('scope_key', 'text', (col) => col.notNull())
|
|
231
280
|
.addColumn('scope', 'text', (col) => col.notNull())
|
|
232
281
|
.addColumn('as_of_commit_seq', 'integer', (col) => col.notNull())
|
|
@@ -241,12 +290,13 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
241
290
|
.addColumn('created_at', 'text', (col) => col.notNull().defaultTo(nowIso))
|
|
242
291
|
.addColumn('expires_at', 'text', (col) => col.notNull())
|
|
243
292
|
.execute();
|
|
293
|
+
await ensurePartitionColumn(db, 'sync_snapshot_chunks');
|
|
244
294
|
|
|
245
295
|
await sql`CREATE INDEX IF NOT EXISTS idx_sync_snapshot_chunks_expires_at
|
|
246
296
|
ON sync_snapshot_chunks(expires_at)`.execute(db);
|
|
247
297
|
|
|
248
298
|
await sql`CREATE UNIQUE INDEX IF NOT EXISTS idx_sync_snapshot_chunks_page_key
|
|
249
|
-
ON sync_snapshot_chunks(scope_key, scope, as_of_commit_seq, row_cursor, row_limit, encoding, compression)`.execute(
|
|
299
|
+
ON sync_snapshot_chunks(partition_id, scope_key, scope, as_of_commit_seq, row_cursor, row_limit, encoding, compression)`.execute(
|
|
250
300
|
db
|
|
251
301
|
);
|
|
252
302
|
|
|
@@ -269,6 +319,22 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
269
319
|
db: Kysely<DB>,
|
|
270
320
|
fn: (executor: DbExecutor<DB>) => Promise<T>
|
|
271
321
|
): Promise<T> {
|
|
322
|
+
if (isActiveTransaction(db)) {
|
|
323
|
+
if (!this._supportsTransactions) {
|
|
324
|
+
return fn(db);
|
|
325
|
+
}
|
|
326
|
+
const savepoint = createSavepointName();
|
|
327
|
+
await sql.raw(`SAVEPOINT ${savepoint}`).execute(db);
|
|
328
|
+
try {
|
|
329
|
+
const result = await fn(db);
|
|
330
|
+
await sql.raw(`RELEASE SAVEPOINT ${savepoint}`).execute(db);
|
|
331
|
+
return result;
|
|
332
|
+
} catch (error) {
|
|
333
|
+
await sql.raw(`ROLLBACK TO SAVEPOINT ${savepoint}`).execute(db);
|
|
334
|
+
await sql.raw(`RELEASE SAVEPOINT ${savepoint}`).execute(db);
|
|
335
|
+
throw error;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
272
338
|
if (this._supportsTransactions) {
|
|
273
339
|
return db.transaction().execute(fn);
|
|
274
340
|
}
|
|
@@ -282,94 +348,19 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
282
348
|
}
|
|
283
349
|
|
|
284
350
|
// ===========================================================================
|
|
285
|
-
// Commit/Change Log Queries
|
|
351
|
+
// Commit/Change Log Queries (dialect-specific)
|
|
286
352
|
// ===========================================================================
|
|
287
353
|
|
|
288
|
-
async readMaxCommitSeq<DB extends SyncCoreDb>(
|
|
289
|
-
db: Kysely<DB> | Transaction<DB>
|
|
290
|
-
): Promise<number> {
|
|
291
|
-
const res = await sql<{ max_seq: unknown }>`
|
|
292
|
-
SELECT max(commit_seq) as max_seq
|
|
293
|
-
FROM sync_commits
|
|
294
|
-
`.execute(db);
|
|
295
|
-
|
|
296
|
-
return coerceNumber(res.rows[0]?.max_seq) ?? 0;
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
async readMinCommitSeq<DB extends SyncCoreDb>(
|
|
300
|
-
db: Kysely<DB> | Transaction<DB>
|
|
301
|
-
): Promise<number> {
|
|
302
|
-
const res = await sql<{ min_seq: unknown }>`
|
|
303
|
-
SELECT min(commit_seq) as min_seq
|
|
304
|
-
FROM sync_commits
|
|
305
|
-
`.execute(db);
|
|
306
|
-
|
|
307
|
-
return coerceNumber(res.rows[0]?.min_seq) ?? 0;
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
async readCommitSeqsForPull<DB extends SyncCoreDb>(
|
|
311
|
-
db: Kysely<DB> | Transaction<DB>,
|
|
312
|
-
args: { cursor: number; limitCommits: number; tables: string[] }
|
|
313
|
-
): Promise<number[]> {
|
|
314
|
-
if (args.tables.length === 0) return [];
|
|
315
|
-
|
|
316
|
-
const tablesIn = sql.join(
|
|
317
|
-
args.tables.map((t) => sql`${t}`),
|
|
318
|
-
sql`, `
|
|
319
|
-
);
|
|
320
|
-
|
|
321
|
-
const res = await sql<{ commit_seq: unknown }>`
|
|
322
|
-
SELECT DISTINCT commit_seq
|
|
323
|
-
FROM sync_table_commits
|
|
324
|
-
WHERE "table" IN (${tablesIn})
|
|
325
|
-
AND commit_seq > ${args.cursor}
|
|
326
|
-
ORDER BY commit_seq ASC
|
|
327
|
-
LIMIT ${args.limitCommits}
|
|
328
|
-
`.execute(db);
|
|
329
|
-
|
|
330
|
-
return res.rows
|
|
331
|
-
.map((r) => coerceNumber(r.commit_seq))
|
|
332
|
-
.filter(
|
|
333
|
-
(n): n is number =>
|
|
334
|
-
typeof n === 'number' && Number.isFinite(n) && n > args.cursor
|
|
335
|
-
);
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
async readCommits<DB extends SyncCoreDb>(
|
|
339
|
-
db: Kysely<DB> | Transaction<DB>,
|
|
340
|
-
commitSeqs: number[]
|
|
341
|
-
): Promise<SyncCommitRow[]> {
|
|
342
|
-
if (commitSeqs.length === 0) return [];
|
|
343
|
-
|
|
344
|
-
const commitSeqsIn = sql.join(
|
|
345
|
-
commitSeqs.map((seq) => sql`${seq}`),
|
|
346
|
-
sql`, `
|
|
347
|
-
);
|
|
348
|
-
|
|
349
|
-
const res = await sql<{
|
|
350
|
-
commit_seq: unknown;
|
|
351
|
-
actor_id: string;
|
|
352
|
-
created_at: unknown;
|
|
353
|
-
result_json: unknown | null;
|
|
354
|
-
}>`
|
|
355
|
-
SELECT commit_seq, actor_id, created_at, result_json
|
|
356
|
-
FROM sync_commits
|
|
357
|
-
WHERE commit_seq IN (${commitSeqsIn})
|
|
358
|
-
ORDER BY commit_seq ASC
|
|
359
|
-
`.execute(db);
|
|
360
|
-
|
|
361
|
-
return res.rows.map((row) => ({
|
|
362
|
-
commit_seq: coerceNumber(row.commit_seq) ?? 0,
|
|
363
|
-
actor_id: row.actor_id,
|
|
364
|
-
created_at: coerceIsoString(row.created_at),
|
|
365
|
-
result_json: row.result_json ?? null,
|
|
366
|
-
}));
|
|
367
|
-
}
|
|
368
|
-
|
|
369
354
|
async readChangesForCommits<DB extends SyncCoreDb>(
|
|
370
|
-
db:
|
|
371
|
-
args: {
|
|
355
|
+
db: DbExecutor<DB>,
|
|
356
|
+
args: {
|
|
357
|
+
commitSeqs: number[];
|
|
358
|
+
table: string;
|
|
359
|
+
scopes: ScopeValues;
|
|
360
|
+
partitionId?: string;
|
|
361
|
+
}
|
|
372
362
|
): Promise<SyncChangeRow[]> {
|
|
363
|
+
const partitionId = args.partitionId ?? 'default';
|
|
373
364
|
if (args.commitSeqs.length === 0) return [];
|
|
374
365
|
|
|
375
366
|
const commitSeqsIn = sql.join(
|
|
@@ -390,6 +381,7 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
390
381
|
SELECT commit_seq, "table", row_id, op, row_json, row_version, scopes
|
|
391
382
|
FROM sync_changes
|
|
392
383
|
WHERE commit_seq IN (${commitSeqsIn})
|
|
384
|
+
AND partition_id = ${partitionId}
|
|
393
385
|
AND "table" = ${args.table}
|
|
394
386
|
ORDER BY commit_seq ASC, change_id ASC
|
|
395
387
|
`.execute(db);
|
|
@@ -411,40 +403,25 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
411
403
|
}));
|
|
412
404
|
}
|
|
413
405
|
|
|
414
|
-
async
|
|
415
|
-
db:
|
|
416
|
-
args:
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
cursor: number;
|
|
420
|
-
limitCommits: number;
|
|
421
|
-
}
|
|
422
|
-
): Promise<
|
|
423
|
-
Array<{
|
|
424
|
-
commit_seq: number;
|
|
425
|
-
actor_id: string;
|
|
426
|
-
created_at: string;
|
|
427
|
-
change_id: number;
|
|
428
|
-
table: string;
|
|
429
|
-
row_id: string;
|
|
430
|
-
op: SyncOp;
|
|
431
|
-
row_json: unknown | null;
|
|
432
|
-
row_version: number | null;
|
|
433
|
-
scopes: StoredScopes;
|
|
434
|
-
}>
|
|
435
|
-
> {
|
|
406
|
+
protected override async readIncrementalPullRowsBatch<DB extends SyncCoreDb>(
|
|
407
|
+
db: DbExecutor<DB>,
|
|
408
|
+
args: Omit<IncrementalPullRowsArgs, 'batchSize'>
|
|
409
|
+
): Promise<IncrementalPullRow[]> {
|
|
410
|
+
const partitionId = args.partitionId ?? 'default';
|
|
436
411
|
const limitCommits = Math.max(1, Math.min(500, args.limitCommits));
|
|
437
412
|
|
|
438
413
|
// Get commit_seqs for this table
|
|
439
414
|
const commitSeqsRes = await sql<{ commit_seq: unknown }>`
|
|
440
415
|
SELECT commit_seq
|
|
441
416
|
FROM sync_table_commits
|
|
442
|
-
WHERE
|
|
417
|
+
WHERE partition_id = ${partitionId}
|
|
418
|
+
AND "table" = ${args.table}
|
|
443
419
|
AND commit_seq > ${args.cursor}
|
|
444
420
|
AND EXISTS (
|
|
445
421
|
SELECT 1
|
|
446
422
|
FROM sync_commits cm
|
|
447
423
|
WHERE cm.commit_seq = sync_table_commits.commit_seq
|
|
424
|
+
AND cm.partition_id = ${partitionId}
|
|
448
425
|
)
|
|
449
426
|
ORDER BY commit_seq ASC
|
|
450
427
|
LIMIT ${limitCommits}
|
|
@@ -488,6 +465,8 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
488
465
|
FROM sync_commits cm
|
|
489
466
|
JOIN sync_changes c ON c.commit_seq = cm.commit_seq
|
|
490
467
|
WHERE cm.commit_seq IN (${commitSeqsIn})
|
|
468
|
+
AND cm.partition_id = ${partitionId}
|
|
469
|
+
AND c.partition_id = ${partitionId}
|
|
491
470
|
AND c."table" = ${args.table}
|
|
492
471
|
ORDER BY cm.commit_seq ASC, c.change_id ASC
|
|
493
472
|
`.execute(db);
|
|
@@ -512,115 +491,8 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
512
491
|
}));
|
|
513
492
|
}
|
|
514
493
|
|
|
515
|
-
/**
|
|
516
|
-
* Streaming version of incremental pull for large result sets.
|
|
517
|
-
* Yields changes one at a time instead of loading all into memory.
|
|
518
|
-
*/
|
|
519
|
-
async *streamIncrementalPullRows<DB extends SyncCoreDb>(
|
|
520
|
-
db: Kysely<DB> | Transaction<DB>,
|
|
521
|
-
args: {
|
|
522
|
-
table: string;
|
|
523
|
-
scopes: ScopeValues;
|
|
524
|
-
cursor: number;
|
|
525
|
-
limitCommits: number;
|
|
526
|
-
}
|
|
527
|
-
): AsyncGenerator<{
|
|
528
|
-
commit_seq: number;
|
|
529
|
-
actor_id: string;
|
|
530
|
-
created_at: string;
|
|
531
|
-
change_id: number;
|
|
532
|
-
table: string;
|
|
533
|
-
row_id: string;
|
|
534
|
-
op: SyncOp;
|
|
535
|
-
row_json: unknown | null;
|
|
536
|
-
row_version: number | null;
|
|
537
|
-
scopes: StoredScopes;
|
|
538
|
-
}> {
|
|
539
|
-
const limitCommits = Math.max(1, Math.min(500, args.limitCommits));
|
|
540
|
-
|
|
541
|
-
// Get commit_seqs for this table
|
|
542
|
-
const commitSeqsRes = await sql<{ commit_seq: unknown }>`
|
|
543
|
-
SELECT commit_seq
|
|
544
|
-
FROM sync_table_commits
|
|
545
|
-
WHERE "table" = ${args.table}
|
|
546
|
-
AND commit_seq > ${args.cursor}
|
|
547
|
-
AND EXISTS (
|
|
548
|
-
SELECT 1
|
|
549
|
-
FROM sync_commits cm
|
|
550
|
-
WHERE cm.commit_seq = sync_table_commits.commit_seq
|
|
551
|
-
)
|
|
552
|
-
ORDER BY commit_seq ASC
|
|
553
|
-
LIMIT ${limitCommits}
|
|
554
|
-
`.execute(db);
|
|
555
|
-
|
|
556
|
-
const commitSeqs = commitSeqsRes.rows
|
|
557
|
-
.map((r) => coerceNumber(r.commit_seq))
|
|
558
|
-
.filter((n): n is number => n !== null);
|
|
559
|
-
|
|
560
|
-
if (commitSeqs.length === 0) return;
|
|
561
|
-
|
|
562
|
-
// Process in smaller batches to avoid memory issues
|
|
563
|
-
const batchSize = 100;
|
|
564
|
-
for (let i = 0; i < commitSeqs.length; i += batchSize) {
|
|
565
|
-
const batch = commitSeqs.slice(i, i + batchSize);
|
|
566
|
-
const commitSeqsIn = sql.join(
|
|
567
|
-
batch.map((seq) => sql`${seq}`),
|
|
568
|
-
sql`, `
|
|
569
|
-
);
|
|
570
|
-
|
|
571
|
-
const changesRes = await sql<{
|
|
572
|
-
commit_seq: unknown;
|
|
573
|
-
actor_id: string;
|
|
574
|
-
created_at: unknown;
|
|
575
|
-
change_id: unknown;
|
|
576
|
-
table: string;
|
|
577
|
-
row_id: string;
|
|
578
|
-
op: string;
|
|
579
|
-
row_json: unknown | null;
|
|
580
|
-
row_version: unknown | null;
|
|
581
|
-
scopes: unknown;
|
|
582
|
-
}>`
|
|
583
|
-
SELECT
|
|
584
|
-
cm.commit_seq,
|
|
585
|
-
cm.actor_id,
|
|
586
|
-
cm.created_at,
|
|
587
|
-
c.change_id,
|
|
588
|
-
c."table",
|
|
589
|
-
c.row_id,
|
|
590
|
-
c.op,
|
|
591
|
-
c.row_json,
|
|
592
|
-
c.row_version,
|
|
593
|
-
c.scopes
|
|
594
|
-
FROM sync_commits cm
|
|
595
|
-
JOIN sync_changes c ON c.commit_seq = cm.commit_seq
|
|
596
|
-
WHERE cm.commit_seq IN (${commitSeqsIn})
|
|
597
|
-
AND c."table" = ${args.table}
|
|
598
|
-
ORDER BY cm.commit_seq ASC, c.change_id ASC
|
|
599
|
-
`.execute(db);
|
|
600
|
-
|
|
601
|
-
// Filter and yield each row
|
|
602
|
-
for (const row of changesRes.rows) {
|
|
603
|
-
const storedScopes = parseScopes(row.scopes);
|
|
604
|
-
if (scopesMatch(storedScopes, args.scopes)) {
|
|
605
|
-
yield {
|
|
606
|
-
commit_seq: coerceNumber(row.commit_seq) ?? 0,
|
|
607
|
-
actor_id: row.actor_id,
|
|
608
|
-
created_at: coerceIsoString(row.created_at),
|
|
609
|
-
change_id: coerceNumber(row.change_id) ?? 0,
|
|
610
|
-
table: row.table,
|
|
611
|
-
row_id: row.row_id,
|
|
612
|
-
op: row.op as SyncOp,
|
|
613
|
-
row_json: parseJsonValue(row.row_json),
|
|
614
|
-
row_version: coerceNumber(row.row_version),
|
|
615
|
-
scopes: storedScopes,
|
|
616
|
-
};
|
|
617
|
-
}
|
|
618
|
-
}
|
|
619
|
-
}
|
|
620
|
-
}
|
|
621
|
-
|
|
622
494
|
async compactChanges<DB extends SyncCoreDb>(
|
|
623
|
-
db:
|
|
495
|
+
db: DbExecutor<DB>,
|
|
624
496
|
args: { fullHistoryHours: number }
|
|
625
497
|
): Promise<number> {
|
|
626
498
|
const cutoffIso = new Date(
|
|
@@ -630,18 +502,19 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
630
502
|
// Find all old changes
|
|
631
503
|
const oldChanges = await sql<{
|
|
632
504
|
change_id: unknown;
|
|
505
|
+
partition_id: string;
|
|
633
506
|
commit_seq: unknown;
|
|
634
507
|
table: string;
|
|
635
508
|
row_id: string;
|
|
636
509
|
scopes: unknown;
|
|
637
510
|
}>`
|
|
638
|
-
SELECT c.change_id, c.commit_seq, c."table", c.row_id, c.scopes
|
|
511
|
+
SELECT c.change_id, c.partition_id, c.commit_seq, c."table", c.row_id, c.scopes
|
|
639
512
|
FROM sync_changes c
|
|
640
513
|
JOIN sync_commits cm ON cm.commit_seq = c.commit_seq
|
|
641
514
|
WHERE cm.created_at < ${cutoffIso}
|
|
642
515
|
`.execute(db);
|
|
643
516
|
|
|
644
|
-
// Group by (table, row_id, scopes)
|
|
517
|
+
// Group by (partition_id, table, row_id, scopes)
|
|
645
518
|
const groups = new Map<
|
|
646
519
|
string,
|
|
647
520
|
Array<{ change_id: number; commit_seq: number }>
|
|
@@ -649,7 +522,7 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
649
522
|
|
|
650
523
|
for (const row of oldChanges.rows) {
|
|
651
524
|
const scopesStr = JSON.stringify(parseScopes(row.scopes));
|
|
652
|
-
const key = `${row.table}|${row.row_id}|${scopesStr}`;
|
|
525
|
+
const key = `${row.partition_id}|${row.table}|${row.row_id}|${scopesStr}`;
|
|
653
526
|
if (!groups.has(key)) {
|
|
654
527
|
groups.set(key, []);
|
|
655
528
|
}
|
|
@@ -677,11 +550,11 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
677
550
|
if (toDelete.length === 0) return 0;
|
|
678
551
|
|
|
679
552
|
// Delete in batches
|
|
680
|
-
const
|
|
553
|
+
const deleteBatchSize = 500;
|
|
681
554
|
let deleted = 0;
|
|
682
555
|
|
|
683
|
-
for (let i = 0; i < toDelete.length; i +=
|
|
684
|
-
const batch = toDelete.slice(i, i +
|
|
556
|
+
for (let i = 0; i < toDelete.length; i += deleteBatchSize) {
|
|
557
|
+
const batch = toDelete.slice(i, i + deleteBatchSize);
|
|
685
558
|
const batchIn = sql.join(
|
|
686
559
|
batch.map((id) => sql`${id}`),
|
|
687
560
|
sql`, `
|
|
@@ -702,11 +575,13 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
702
575
|
SELECT commit_seq
|
|
703
576
|
FROM sync_commits
|
|
704
577
|
WHERE created_at < ${cutoffIso}
|
|
578
|
+
AND partition_id = sync_table_commits.partition_id
|
|
705
579
|
)
|
|
706
580
|
AND NOT EXISTS (
|
|
707
581
|
SELECT 1
|
|
708
582
|
FROM sync_changes c
|
|
709
583
|
WHERE c.commit_seq = sync_table_commits.commit_seq
|
|
584
|
+
AND c.partition_id = sync_table_commits.partition_id
|
|
710
585
|
AND c."table" = sync_table_commits."table"
|
|
711
586
|
)
|
|
712
587
|
`.execute(db);
|
|
@@ -714,33 +589,6 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
714
589
|
return deleted;
|
|
715
590
|
}
|
|
716
591
|
|
|
717
|
-
// ===========================================================================
|
|
718
|
-
// Client Cursor Recording
|
|
719
|
-
// ===========================================================================
|
|
720
|
-
|
|
721
|
-
async recordClientCursor<DB extends SyncCoreDb>(
|
|
722
|
-
db: Kysely<DB> | Transaction<DB>,
|
|
723
|
-
args: {
|
|
724
|
-
clientId: string;
|
|
725
|
-
actorId: string;
|
|
726
|
-
cursor: number;
|
|
727
|
-
effectiveScopes: ScopeValues;
|
|
728
|
-
}
|
|
729
|
-
): Promise<void> {
|
|
730
|
-
const now = new Date().toISOString();
|
|
731
|
-
const scopesJson = JSON.stringify(args.effectiveScopes);
|
|
732
|
-
|
|
733
|
-
await sql`
|
|
734
|
-
INSERT INTO sync_client_cursors (client_id, actor_id, cursor, effective_scopes, updated_at)
|
|
735
|
-
VALUES (${args.clientId}, ${args.actorId}, ${args.cursor}, ${scopesJson}, ${now})
|
|
736
|
-
ON CONFLICT(client_id) DO UPDATE SET
|
|
737
|
-
actor_id = ${args.actorId},
|
|
738
|
-
cursor = ${args.cursor},
|
|
739
|
-
effective_scopes = ${scopesJson},
|
|
740
|
-
updated_at = ${now}
|
|
741
|
-
`.execute(db);
|
|
742
|
-
}
|
|
743
|
-
|
|
744
592
|
// ===========================================================================
|
|
745
593
|
// Scope Conversion Helpers
|
|
746
594
|
// ===========================================================================
|
|
@@ -749,10 +597,6 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
749
597
|
return JSON.stringify(scopes);
|
|
750
598
|
}
|
|
751
599
|
|
|
752
|
-
dbToScopes(value: unknown): StoredScopes {
|
|
753
|
-
return parseScopes(value);
|
|
754
|
-
}
|
|
755
|
-
|
|
756
600
|
dbToArray(value: unknown): string[] {
|
|
757
601
|
return toStringArray(value);
|
|
758
602
|
}
|
|
@@ -761,21 +605,6 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
761
605
|
return JSON.stringify(values.filter((v) => v.length > 0));
|
|
762
606
|
}
|
|
763
607
|
|
|
764
|
-
async readAffectedTablesFromChanges<DB extends SyncCoreDb>(
|
|
765
|
-
db: Kysely<DB> | Transaction<DB>,
|
|
766
|
-
commitSeq: number
|
|
767
|
-
): Promise<string[]> {
|
|
768
|
-
const res = await sql<{ table: string }>`
|
|
769
|
-
SELECT DISTINCT "table"
|
|
770
|
-
FROM sync_changes
|
|
771
|
-
WHERE commit_seq = ${commitSeq}
|
|
772
|
-
`.execute(db);
|
|
773
|
-
|
|
774
|
-
return res.rows
|
|
775
|
-
.map((r) => r.table)
|
|
776
|
-
.filter((t): t is string => typeof t === 'string' && t.length > 0);
|
|
777
|
-
}
|
|
778
|
-
|
|
779
608
|
// ===========================================================================
|
|
780
609
|
// Console Schema (Request Events)
|
|
781
610
|
// ===========================================================================
|
|
@@ -802,8 +631,12 @@ export class SqliteServerSyncDialect implements ServerSyncDialect {
|
|
|
802
631
|
.addColumn('row_count', 'integer')
|
|
803
632
|
.addColumn('tables', 'text', (col) => col.notNull().defaultTo('[]'))
|
|
804
633
|
.addColumn('error_message', 'text')
|
|
634
|
+
.addColumn('transport_path', 'text', (col) =>
|
|
635
|
+
col.notNull().defaultTo('direct')
|
|
636
|
+
)
|
|
805
637
|
.addColumn('created_at', 'text', (col) => col.notNull().defaultTo(nowIso))
|
|
806
638
|
.execute();
|
|
639
|
+
await ensureTransportPathColumn(db);
|
|
807
640
|
|
|
808
641
|
await sql`CREATE INDEX IF NOT EXISTS idx_sync_request_events_created_at
|
|
809
642
|
ON sync_request_events(created_at DESC)`.execute(db);
|