@syncular/client 0.0.4-26 → 0.0.6-100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts +3 -3
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +7 -1
- package/dist/client.js.map +1 -1
- package/dist/create-client.d.ts +3 -4
- package/dist/create-client.d.ts.map +1 -1
- package/dist/create-client.js +16 -12
- package/dist/create-client.js.map +1 -1
- package/dist/engine/SyncEngine.d.ts.map +1 -1
- package/dist/engine/SyncEngine.js +49 -29
- package/dist/engine/SyncEngine.js.map +1 -1
- package/dist/engine/types.d.ts +3 -3
- package/dist/engine/types.d.ts.map +1 -1
- package/dist/handlers/collection.d.ts +6 -0
- package/dist/handlers/collection.d.ts.map +1 -0
- package/dist/handlers/collection.js +21 -0
- package/dist/handlers/collection.js.map +1 -0
- package/dist/handlers/create-handler.d.ts +1 -1
- package/dist/handlers/create-handler.d.ts.map +1 -1
- package/dist/handlers/create-handler.js +3 -3
- package/dist/handlers/create-handler.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/migrate.d.ts.map +1 -1
- package/dist/migrate.js +12 -0
- package/dist/migrate.js.map +1 -1
- package/dist/mutations.d.ts +1 -1
- package/dist/mutations.d.ts.map +1 -1
- package/dist/mutations.js +3 -3
- package/dist/mutations.js.map +1 -1
- package/dist/pull-engine.d.ts +11 -14
- package/dist/pull-engine.d.ts.map +1 -1
- package/dist/pull-engine.js +68 -6
- package/dist/pull-engine.js.map +1 -1
- package/dist/push-engine.d.ts.map +1 -1
- package/dist/push-engine.js +12 -0
- package/dist/push-engine.js.map +1 -1
- package/dist/sync-loop.d.ts +2 -2
- package/dist/sync-loop.d.ts.map +1 -1
- package/dist/sync-loop.js +5 -2
- package/dist/sync-loop.js.map +1 -1
- package/dist/sync.d.ts +32 -0
- package/dist/sync.d.ts.map +1 -0
- package/dist/sync.js +55 -0
- package/dist/sync.js.map +1 -0
- package/package.json +4 -4
- package/src/client.test.ts +18 -9
- package/src/client.ts +11 -4
- package/src/create-client.test.ts +83 -0
- package/src/create-client.ts +21 -16
- package/src/engine/SyncEngine.test.ts +90 -32
- package/src/engine/SyncEngine.ts +53 -33
- package/src/engine/types.ts +3 -3
- package/src/handlers/collection.ts +36 -0
- package/src/handlers/create-handler.ts +4 -4
- package/src/index.ts +2 -1
- package/src/migrate.ts +14 -0
- package/src/mutations.ts +4 -4
- package/src/pull-engine.test.ts +10 -6
- package/src/pull-engine.ts +93 -21
- package/src/push-engine.ts +15 -0
- package/src/sync-loop.ts +13 -5
- package/src/sync.ts +170 -0
- package/dist/handlers/registry.d.ts +0 -15
- package/dist/handlers/registry.d.ts.map +0 -1
- package/dist/handlers/registry.js +0 -29
- package/dist/handlers/registry.js.map +0 -1
- package/src/handlers/registry.ts +0 -36
package/src/engine/SyncEngine.ts
CHANGED
|
@@ -18,6 +18,8 @@ import {
|
|
|
18
18
|
startSyncSpan,
|
|
19
19
|
} from '@syncular/core';
|
|
20
20
|
import { type Kysely, sql, type Transaction } from 'kysely';
|
|
21
|
+
import { getClientHandler } from '../handlers/collection';
|
|
22
|
+
import { ensureClientSyncSchema } from '../migrate';
|
|
21
23
|
import { syncPushOnce } from '../push-engine';
|
|
22
24
|
import type {
|
|
23
25
|
ConflictResultStatus,
|
|
@@ -1005,7 +1007,7 @@ export class SyncEngine<DB extends SyncClientDb = SyncClientDb> {
|
|
|
1005
1007
|
}
|
|
1006
1008
|
|
|
1007
1009
|
if (options.scope === 'all') {
|
|
1008
|
-
for (const handler of this.config.handlers
|
|
1010
|
+
for (const handler of this.config.handlers) {
|
|
1009
1011
|
await handler.clearAll({ trx, scopes: {} });
|
|
1010
1012
|
clearedTables.push(handler.table);
|
|
1011
1013
|
}
|
|
@@ -1014,7 +1016,7 @@ export class SyncEngine<DB extends SyncClientDb = SyncClientDb> {
|
|
|
1014
1016
|
|
|
1015
1017
|
const seen = new Set<string>();
|
|
1016
1018
|
for (const target of targets) {
|
|
1017
|
-
const handler = this.config.handlers
|
|
1019
|
+
const handler = getClientHandler(this.config.handlers, target.table);
|
|
1018
1020
|
if (!handler) continue;
|
|
1019
1021
|
|
|
1020
1022
|
const key = `${target.table}:${JSON.stringify(target.scopes)}`;
|
|
@@ -1220,40 +1222,44 @@ export class SyncEngine<DB extends SyncClientDb = SyncClientDb> {
|
|
|
1220
1222
|
|
|
1221
1223
|
this.updateState({ enabled: true });
|
|
1222
1224
|
|
|
1223
|
-
// Run
|
|
1224
|
-
if (
|
|
1225
|
-
// Best-effort: push
|
|
1226
|
-
//
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1225
|
+
// Run migrations before first sync.
|
|
1226
|
+
if (!this.migrated) {
|
|
1227
|
+
// Best-effort: push pending commits before user migration, because
|
|
1228
|
+
// app migrations may reset tables and discard unsynced local writes.
|
|
1229
|
+
if (this.config.migrate) {
|
|
1230
|
+
try {
|
|
1231
|
+
const hasOutbox = await sql`
|
|
1232
|
+
select 1 from ${sql.table('sync_outbox_commits')} limit 1
|
|
1233
|
+
`
|
|
1234
|
+
.execute(this.config.db)
|
|
1235
|
+
.then((r) => r.rows.length > 0)
|
|
1236
|
+
.catch(() => false);
|
|
1237
|
+
|
|
1238
|
+
if (hasOutbox) {
|
|
1239
|
+
let pushed = true;
|
|
1240
|
+
while (pushed) {
|
|
1241
|
+
const result = await syncPushOnce(
|
|
1242
|
+
this.config.db,
|
|
1243
|
+
this.config.transport,
|
|
1244
|
+
{
|
|
1245
|
+
clientId: this.config.clientId!,
|
|
1246
|
+
actorId: this.config.actorId ?? undefined,
|
|
1247
|
+
plugins: this.config.plugins,
|
|
1248
|
+
}
|
|
1249
|
+
);
|
|
1250
|
+
pushed = result.pushed;
|
|
1251
|
+
}
|
|
1249
1252
|
}
|
|
1253
|
+
} catch {
|
|
1254
|
+
// Best-effort: continue even if pre-migration push fails.
|
|
1250
1255
|
}
|
|
1251
|
-
} catch {
|
|
1252
|
-
// Best-effort: if push fails (network down, table missing), continue
|
|
1253
1256
|
}
|
|
1254
1257
|
|
|
1255
1258
|
try {
|
|
1256
|
-
|
|
1259
|
+
if (this.config.migrate) {
|
|
1260
|
+
await this.config.migrate(this.config.db);
|
|
1261
|
+
}
|
|
1262
|
+
await ensureClientSyncSchema(this.config.db);
|
|
1257
1263
|
this.migrated = true;
|
|
1258
1264
|
} catch (err) {
|
|
1259
1265
|
const migrationError =
|
|
@@ -1597,7 +1603,7 @@ export class SyncEngine<DB extends SyncClientDb = SyncClientDb> {
|
|
|
1597
1603
|
try {
|
|
1598
1604
|
await this.config.db.transaction().execute(async (trx) => {
|
|
1599
1605
|
for (const change of changes) {
|
|
1600
|
-
const handler = this.config.handlers
|
|
1606
|
+
const handler = getClientHandler(this.config.handlers, change.table);
|
|
1601
1607
|
if (!handler) {
|
|
1602
1608
|
throw new Error(
|
|
1603
1609
|
`Missing client table handler for WS change table "${change.table}"`
|
|
@@ -1852,6 +1858,13 @@ export class SyncEngine<DB extends SyncClientDb = SyncClientDb> {
|
|
|
1852
1858
|
}
|
|
1853
1859
|
|
|
1854
1860
|
const delay = calculateRetryDelay(this.state.retryCount);
|
|
1861
|
+
if (this.state.pendingCount > 0) {
|
|
1862
|
+
countSyncMetric('sync.outbox.retry_count', 1, {
|
|
1863
|
+
attributes: {
|
|
1864
|
+
retryCount: this.state.retryCount,
|
|
1865
|
+
},
|
|
1866
|
+
});
|
|
1867
|
+
}
|
|
1855
1868
|
this.updateState({ isRetrying: true });
|
|
1856
1869
|
|
|
1857
1870
|
this.retryTimeoutId = setTimeout(() => {
|
|
@@ -1975,6 +1988,13 @@ export class SyncEngine<DB extends SyncClientDb = SyncClientDb> {
|
|
|
1975
1988
|
case 'connected': {
|
|
1976
1989
|
const wasConnectedBefore = this.hasRealtimeConnectedOnce;
|
|
1977
1990
|
this.hasRealtimeConnectedOnce = true;
|
|
1991
|
+
if (wasConnectedBefore) {
|
|
1992
|
+
countSyncMetric('sync.transport.reconnects', 1, {
|
|
1993
|
+
attributes: {
|
|
1994
|
+
source: 'client',
|
|
1995
|
+
},
|
|
1996
|
+
});
|
|
1997
|
+
}
|
|
1978
1998
|
this.setConnectionState('connected');
|
|
1979
1999
|
this.updateTransportHealth({
|
|
1980
2000
|
mode: 'realtime',
|
|
@@ -2287,7 +2307,7 @@ export class SyncEngine<DB extends SyncClientDb = SyncClientDb> {
|
|
|
2287
2307
|
|
|
2288
2308
|
await db.transaction().execute(async (trx) => {
|
|
2289
2309
|
for (const input of inputs) {
|
|
2290
|
-
const handler = handlers
|
|
2310
|
+
const handler = getClientHandler(handlers, input.table);
|
|
2291
2311
|
if (!handler) continue;
|
|
2292
2312
|
|
|
2293
2313
|
affectedTables.add(input.table);
|
package/src/engine/types.ts
CHANGED
|
@@ -12,7 +12,7 @@ import type {
|
|
|
12
12
|
SyncTransport,
|
|
13
13
|
} from '@syncular/core';
|
|
14
14
|
import type { Kysely } from 'kysely';
|
|
15
|
-
import type {
|
|
15
|
+
import type { ClientHandlerCollection } from '../handlers/collection';
|
|
16
16
|
import type { SyncClientPlugin } from '../plugins/types';
|
|
17
17
|
import type { SyncClientDb } from '../schema';
|
|
18
18
|
import type { SubscriptionState } from '../subscription-state';
|
|
@@ -225,7 +225,7 @@ export interface SyncEngineConfig<DB extends SyncClientDb = SyncClientDb> {
|
|
|
225
225
|
/** Sync transport */
|
|
226
226
|
transport: SyncTransport;
|
|
227
227
|
/** Client table handler registry */
|
|
228
|
-
handlers:
|
|
228
|
+
handlers: ClientHandlerCollection<DB>;
|
|
229
229
|
/** Actor id for sync scoping (null/undefined disables sync) */
|
|
230
230
|
actorId: string | null | undefined;
|
|
231
231
|
/** Stable device/app installation id */
|
|
@@ -244,7 +244,7 @@ export interface SyncEngineConfig<DB extends SyncClientDb = SyncClientDb> {
|
|
|
244
244
|
pollIntervalMs?: number;
|
|
245
245
|
/** Max retries before giving up */
|
|
246
246
|
maxRetries?: number;
|
|
247
|
-
/**
|
|
247
|
+
/** Optional app migration to run before sync schema migration. */
|
|
248
248
|
migrate?: (db: Kysely<DB>) => Promise<void>;
|
|
249
249
|
/** Called when migration fails. Receives the error. */
|
|
250
250
|
onMigrationError?: (error: Error) => void;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { ClientTableHandler } from './types';
|
|
2
|
+
|
|
3
|
+
export type ClientHandlerCollection<DB> = ClientTableHandler<DB>[];
|
|
4
|
+
|
|
5
|
+
export function createClientHandlerCollection<DB>(
|
|
6
|
+
handlers: ClientTableHandler<DB>[]
|
|
7
|
+
): ClientHandlerCollection<DB> {
|
|
8
|
+
const tables = new Set<string>();
|
|
9
|
+
for (const handler of handlers) {
|
|
10
|
+
if (tables.has(handler.table)) {
|
|
11
|
+
throw new Error(
|
|
12
|
+
`Client table handler already registered: ${handler.table}`
|
|
13
|
+
);
|
|
14
|
+
}
|
|
15
|
+
tables.add(handler.table);
|
|
16
|
+
}
|
|
17
|
+
return handlers;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function getClientHandler<DB>(
|
|
21
|
+
handlers: ClientHandlerCollection<DB>,
|
|
22
|
+
table: string
|
|
23
|
+
): ClientTableHandler<DB> | undefined {
|
|
24
|
+
return handlers.find((handler) => handler.table === table);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function getClientHandlerOrThrow<DB>(
|
|
28
|
+
handlers: ClientHandlerCollection<DB>,
|
|
29
|
+
table: string
|
|
30
|
+
): ClientTableHandler<DB> {
|
|
31
|
+
const handler = getClientHandler(handlers, table);
|
|
32
|
+
if (!handler) {
|
|
33
|
+
throw new Error(`Missing client table handler for table: ${table}`);
|
|
34
|
+
}
|
|
35
|
+
return handler;
|
|
36
|
+
}
|
|
@@ -82,7 +82,7 @@ export interface CreateClientHandlerOptions<
|
|
|
82
82
|
* Optional column codec resolver.
|
|
83
83
|
* Receives `{ table, column, sqlType?, dialect? }` and returns a codec.
|
|
84
84
|
*/
|
|
85
|
-
|
|
85
|
+
codecs?: ColumnCodecSource;
|
|
86
86
|
|
|
87
87
|
/**
|
|
88
88
|
* Dialect used for codec dialect overrides.
|
|
@@ -178,14 +178,14 @@ export function createClientHandler<
|
|
|
178
178
|
const codecDialect = options.codecDialect ?? 'sqlite';
|
|
179
179
|
const codecCache = new Map<string, ReturnType<typeof toTableColumnCodecs>>();
|
|
180
180
|
const resolveTableCodecs = (row: Record<string, unknown>) => {
|
|
181
|
-
const
|
|
182
|
-
if (!
|
|
181
|
+
const codecs = options.codecs;
|
|
182
|
+
if (!codecs) return {};
|
|
183
183
|
const columns = Object.keys(row);
|
|
184
184
|
if (columns.length === 0) return {};
|
|
185
185
|
const cacheKey = columns.slice().sort().join('\u0000');
|
|
186
186
|
const cached = codecCache.get(cacheKey);
|
|
187
187
|
if (cached) return cached;
|
|
188
|
-
const resolved = toTableColumnCodecs(table,
|
|
188
|
+
const resolved = toTableColumnCodecs(table, codecs, columns, {
|
|
189
189
|
dialect: codecDialect,
|
|
190
190
|
});
|
|
191
191
|
codecCache.set(cacheKey, resolved);
|
package/src/index.ts
CHANGED
|
@@ -9,8 +9,8 @@ export * from './client';
|
|
|
9
9
|
export * from './conflicts';
|
|
10
10
|
export * from './create-client';
|
|
11
11
|
export * from './engine';
|
|
12
|
+
export * from './handlers/collection';
|
|
12
13
|
export * from './handlers/create-handler';
|
|
13
|
-
export * from './handlers/registry';
|
|
14
14
|
export * from './handlers/types';
|
|
15
15
|
export * from './migrate';
|
|
16
16
|
export * from './mutations';
|
|
@@ -22,5 +22,6 @@ export * from './push-engine';
|
|
|
22
22
|
export * from './query';
|
|
23
23
|
export * from './schema';
|
|
24
24
|
export * from './subscription-state';
|
|
25
|
+
export * from './sync';
|
|
25
26
|
export * from './sync-loop';
|
|
26
27
|
export * from './utils/id';
|
package/src/migrate.ts
CHANGED
|
@@ -182,6 +182,13 @@ async function ensureClientSyncSchemaCompat<DB extends SyncClientDb>(
|
|
|
182
182
|
.addColumn('resolution', 'text')
|
|
183
183
|
.execute();
|
|
184
184
|
});
|
|
185
|
+
|
|
186
|
+
await db.schema
|
|
187
|
+
.createIndex('idx_sync_outbox_commits_status_updated_at')
|
|
188
|
+
.ifNotExists()
|
|
189
|
+
.on('sync_outbox_commits')
|
|
190
|
+
.columns(['status', 'updated_at', 'created_at'])
|
|
191
|
+
.execute();
|
|
185
192
|
}
|
|
186
193
|
|
|
187
194
|
/**
|
|
@@ -275,6 +282,13 @@ export async function ensureClientSyncSchema<DB extends SyncClientDb>(
|
|
|
275
282
|
.columns(['status', 'created_at'])
|
|
276
283
|
.execute();
|
|
277
284
|
|
|
285
|
+
await db.schema
|
|
286
|
+
.createIndex('idx_sync_outbox_commits_status_updated_at')
|
|
287
|
+
.ifNotExists()
|
|
288
|
+
.on('sync_outbox_commits')
|
|
289
|
+
.columns(['status', 'updated_at', 'created_at'])
|
|
290
|
+
.execute();
|
|
291
|
+
|
|
278
292
|
await db.schema
|
|
279
293
|
.createIndex('idx_sync_conflicts_outbox_commit')
|
|
280
294
|
.ifNotExists()
|
package/src/mutations.ts
CHANGED
|
@@ -400,7 +400,7 @@ export interface OutboxCommitConfig<DB extends SyncClientDb> {
|
|
|
400
400
|
idColumn?: string;
|
|
401
401
|
versionColumn?: string | null;
|
|
402
402
|
omitColumns?: string[];
|
|
403
|
-
|
|
403
|
+
codecs?: ColumnCodecSource;
|
|
404
404
|
codecDialect?: ColumnCodecDialect;
|
|
405
405
|
}
|
|
406
406
|
|
|
@@ -432,8 +432,8 @@ export function createOutboxCommit<DB extends SyncClientDb>(
|
|
|
432
432
|
table: string,
|
|
433
433
|
row: Record<string, unknown>
|
|
434
434
|
) => {
|
|
435
|
-
const
|
|
436
|
-
if (!
|
|
435
|
+
const codecs = config.codecs;
|
|
436
|
+
if (!codecs) return {};
|
|
437
437
|
const columns = Object.keys(row);
|
|
438
438
|
if (columns.length === 0) return {};
|
|
439
439
|
|
|
@@ -450,7 +450,7 @@ export function createOutboxCommit<DB extends SyncClientDb>(
|
|
|
450
450
|
const cached = tableCache.get(cacheKey);
|
|
451
451
|
if (cached) return cached;
|
|
452
452
|
|
|
453
|
-
const resolved = toTableColumnCodecs(table,
|
|
453
|
+
const resolved = toTableColumnCodecs(table, codecs, columns, {
|
|
454
454
|
dialect: codecDialect,
|
|
455
455
|
});
|
|
456
456
|
tableCache.set(cacheKey, resolved);
|
package/src/pull-engine.test.ts
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
2
|
import { gzipSync } from 'node:zlib';
|
|
3
3
|
import {
|
|
4
|
+
createDatabase,
|
|
4
5
|
encodeSnapshotRows,
|
|
5
6
|
type SyncPullResponse,
|
|
6
7
|
type SyncTransport,
|
|
7
8
|
} from '@syncular/core';
|
|
8
9
|
import { type Kysely, sql } from 'kysely';
|
|
9
|
-
import {
|
|
10
|
+
import { createBunSqliteDialect } from '../../dialect-bun-sqlite/src';
|
|
11
|
+
import type { ClientHandlerCollection } from './handlers/collection';
|
|
10
12
|
import { createClientHandler } from './handlers/create-handler';
|
|
11
|
-
import { ClientTableRegistry } from './handlers/registry';
|
|
12
13
|
import { ensureClientSyncSchema } from './migrate';
|
|
13
14
|
import { applyPullResponse, buildPullRequest } from './pull-engine';
|
|
14
15
|
import type { SyncClientDb } from './schema';
|
|
@@ -40,7 +41,10 @@ describe('applyPullResponse chunk streaming', () => {
|
|
|
40
41
|
let db: Kysely<TestDb>;
|
|
41
42
|
|
|
42
43
|
beforeEach(async () => {
|
|
43
|
-
db =
|
|
44
|
+
db = createDatabase<TestDb>({
|
|
45
|
+
dialect: createBunSqliteDialect({ path: ':memory:' }),
|
|
46
|
+
family: 'sqlite',
|
|
47
|
+
});
|
|
44
48
|
await ensureClientSyncSchema(db);
|
|
45
49
|
await db.schema
|
|
46
50
|
.createTable('items')
|
|
@@ -75,12 +79,12 @@ describe('applyPullResponse chunk streaming', () => {
|
|
|
75
79
|
},
|
|
76
80
|
};
|
|
77
81
|
|
|
78
|
-
const handlers
|
|
82
|
+
const handlers: ClientHandlerCollection<TestDb> = [
|
|
79
83
|
createClientHandler({
|
|
80
84
|
table: 'items',
|
|
81
85
|
scopes: ['items:{id}'],
|
|
82
|
-
})
|
|
83
|
-
|
|
86
|
+
}),
|
|
87
|
+
];
|
|
84
88
|
|
|
85
89
|
const options = {
|
|
86
90
|
clientId: 'client-1',
|
package/src/pull-engine.ts
CHANGED
|
@@ -6,13 +6,17 @@ import type {
|
|
|
6
6
|
SyncBootstrapState,
|
|
7
7
|
SyncPullRequest,
|
|
8
8
|
SyncPullResponse,
|
|
9
|
+
SyncPullSubscriptionResponse,
|
|
9
10
|
SyncSnapshot,
|
|
10
11
|
SyncSubscriptionRequest,
|
|
11
12
|
SyncTransport,
|
|
12
13
|
} from '@syncular/core';
|
|
13
14
|
import { decodeSnapshotRows } from '@syncular/core';
|
|
14
15
|
import { type Kysely, sql, type Transaction } from 'kysely';
|
|
15
|
-
import
|
|
16
|
+
import {
|
|
17
|
+
type ClientHandlerCollection,
|
|
18
|
+
getClientHandlerOrThrow,
|
|
19
|
+
} from './handlers/collection';
|
|
16
20
|
import type { ClientTableHandler } from './handlers/types';
|
|
17
21
|
import type {
|
|
18
22
|
SyncClientPlugin,
|
|
@@ -513,6 +517,13 @@ export interface SyncPullOnceOptions {
|
|
|
513
517
|
sha256?: (bytes: Uint8Array) => Promise<string>;
|
|
514
518
|
}
|
|
515
519
|
|
|
520
|
+
export interface SyncPullRequestState {
|
|
521
|
+
request: SyncPullRequest;
|
|
522
|
+
existing: SyncSubscriptionStateTable[];
|
|
523
|
+
existingById: Map<string, SyncSubscriptionStateTable>;
|
|
524
|
+
stateId: string;
|
|
525
|
+
}
|
|
526
|
+
|
|
516
527
|
/**
|
|
517
528
|
* Build a pull request from subscription state. Exported for use
|
|
518
529
|
* by the combined sync path in sync-loop.ts.
|
|
@@ -520,12 +531,7 @@ export interface SyncPullOnceOptions {
|
|
|
520
531
|
export async function buildPullRequest<DB extends SyncClientDb>(
|
|
521
532
|
db: Kysely<DB>,
|
|
522
533
|
options: SyncPullOnceOptions
|
|
523
|
-
): Promise<{
|
|
524
|
-
request: SyncPullRequest;
|
|
525
|
-
existing: SyncSubscriptionStateTable[];
|
|
526
|
-
existingById: Map<string, SyncSubscriptionStateTable>;
|
|
527
|
-
stateId: string;
|
|
528
|
-
}> {
|
|
534
|
+
): Promise<SyncPullRequestState> {
|
|
529
535
|
const stateId = options.stateId ?? 'default';
|
|
530
536
|
|
|
531
537
|
const existingResult = await sql<SyncSubscriptionStateTable>`
|
|
@@ -566,6 +572,70 @@ export async function buildPullRequest<DB extends SyncClientDb>(
|
|
|
566
572
|
return { request, existing, existingById, stateId };
|
|
567
573
|
}
|
|
568
574
|
|
|
575
|
+
export function createFollowupPullState(
|
|
576
|
+
pullState: SyncPullRequestState,
|
|
577
|
+
response: SyncPullResponse
|
|
578
|
+
): SyncPullRequestState {
|
|
579
|
+
const responseById = new Map<string, SyncPullSubscriptionResponse>();
|
|
580
|
+
for (const sub of response.subscriptions ?? []) {
|
|
581
|
+
responseById.set(sub.id, sub);
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
const now = Date.now();
|
|
585
|
+
const nextExisting: SyncSubscriptionStateTable[] = [];
|
|
586
|
+
const nextExistingById = new Map<string, SyncSubscriptionStateTable>();
|
|
587
|
+
|
|
588
|
+
for (const sub of pullState.request.subscriptions ?? []) {
|
|
589
|
+
const res = responseById.get(sub.id);
|
|
590
|
+
if (res?.status === 'revoked') {
|
|
591
|
+
continue;
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
const nextCursor = res ? Math.max(-1, res.nextCursor) : (sub.cursor ?? -1);
|
|
595
|
+
const nextBootstrapState = res
|
|
596
|
+
? res.bootstrap
|
|
597
|
+
? (res.bootstrapState ?? null)
|
|
598
|
+
: null
|
|
599
|
+
: (sub.bootstrapState ?? null);
|
|
600
|
+
const prev = pullState.existingById.get(sub.id);
|
|
601
|
+
const nextRow: SyncSubscriptionStateTable = {
|
|
602
|
+
state_id: pullState.stateId,
|
|
603
|
+
subscription_id: sub.id,
|
|
604
|
+
table: sub.table,
|
|
605
|
+
scopes_json: serializeJsonCached(sub.scopes ?? {}),
|
|
606
|
+
params_json: serializeJsonCached(sub.params ?? {}),
|
|
607
|
+
cursor: nextCursor,
|
|
608
|
+
bootstrap_state_json: nextBootstrapState
|
|
609
|
+
? serializeJsonCached(nextBootstrapState)
|
|
610
|
+
: null,
|
|
611
|
+
status: 'active',
|
|
612
|
+
created_at: prev?.created_at ?? now,
|
|
613
|
+
updated_at: now,
|
|
614
|
+
};
|
|
615
|
+
nextExisting.push(nextRow);
|
|
616
|
+
nextExistingById.set(nextRow.subscription_id, nextRow);
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
const nextRequest: SyncPullRequest = {
|
|
620
|
+
...pullState.request,
|
|
621
|
+
subscriptions: (pullState.request.subscriptions ?? []).map((sub) => {
|
|
622
|
+
const row = nextExistingById.get(sub.id);
|
|
623
|
+
return {
|
|
624
|
+
...sub,
|
|
625
|
+
cursor: Math.max(-1, row?.cursor ?? -1),
|
|
626
|
+
bootstrapState: parseBootstrapState(row?.bootstrap_state_json),
|
|
627
|
+
};
|
|
628
|
+
}),
|
|
629
|
+
};
|
|
630
|
+
|
|
631
|
+
return {
|
|
632
|
+
request: nextRequest,
|
|
633
|
+
existing: nextExisting,
|
|
634
|
+
existingById: nextExistingById,
|
|
635
|
+
stateId: pullState.stateId,
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
|
|
569
639
|
/**
|
|
570
640
|
* Apply a pull response (run plugins + write to local DB).
|
|
571
641
|
* Exported for use by the combined sync path in sync-loop.ts.
|
|
@@ -573,14 +643,9 @@ export async function buildPullRequest<DB extends SyncClientDb>(
|
|
|
573
643
|
export async function applyPullResponse<DB extends SyncClientDb>(
|
|
574
644
|
db: Kysely<DB>,
|
|
575
645
|
transport: SyncTransport,
|
|
576
|
-
handlers:
|
|
646
|
+
handlers: ClientHandlerCollection<DB>,
|
|
577
647
|
options: SyncPullOnceOptions,
|
|
578
|
-
pullState:
|
|
579
|
-
request: SyncPullRequest;
|
|
580
|
-
existing: SyncSubscriptionStateTable[];
|
|
581
|
-
existingById: Map<string, SyncSubscriptionStateTable>;
|
|
582
|
-
stateId: string;
|
|
583
|
-
},
|
|
648
|
+
pullState: SyncPullRequestState,
|
|
584
649
|
rawResponse: SyncPullResponse
|
|
585
650
|
): Promise<SyncPullResponse> {
|
|
586
651
|
const { request, existing, existingById, stateId } = pullState;
|
|
@@ -620,7 +685,10 @@ export async function applyPullResponse<DB extends SyncClientDb>(
|
|
|
620
685
|
? JSON.parse(row.scopes_json)
|
|
621
686
|
: row.scopes_json
|
|
622
687
|
: {};
|
|
623
|
-
await handlers
|
|
688
|
+
await getClientHandlerOrThrow(handlers, row.table).clearAll({
|
|
689
|
+
trx,
|
|
690
|
+
scopes,
|
|
691
|
+
});
|
|
624
692
|
} catch {
|
|
625
693
|
// ignore missing table handler
|
|
626
694
|
}
|
|
@@ -649,7 +717,10 @@ export async function applyPullResponse<DB extends SyncClientDb>(
|
|
|
649
717
|
? JSON.parse(prev.scopes_json)
|
|
650
718
|
: prev.scopes_json
|
|
651
719
|
: {};
|
|
652
|
-
await handlers
|
|
720
|
+
await getClientHandlerOrThrow(handlers, prev.table).clearAll({
|
|
721
|
+
trx,
|
|
722
|
+
scopes,
|
|
723
|
+
});
|
|
653
724
|
} catch {
|
|
654
725
|
// ignore missing handler
|
|
655
726
|
}
|
|
@@ -666,7 +737,7 @@ export async function applyPullResponse<DB extends SyncClientDb>(
|
|
|
666
737
|
// Apply snapshots (bootstrap mode)
|
|
667
738
|
if (sub.bootstrap) {
|
|
668
739
|
for (const snapshot of sub.snapshots ?? []) {
|
|
669
|
-
const handler = handlers
|
|
740
|
+
const handler = getClientHandlerOrThrow(handlers, snapshot.table);
|
|
670
741
|
const hasChunkRefs =
|
|
671
742
|
Array.isArray(snapshot.chunks) && snapshot.chunks.length > 0;
|
|
672
743
|
|
|
@@ -698,7 +769,7 @@ export async function applyPullResponse<DB extends SyncClientDb>(
|
|
|
698
769
|
// Apply incremental changes
|
|
699
770
|
for (const commit of sub.commits) {
|
|
700
771
|
for (const change of commit.changes) {
|
|
701
|
-
const handler = handlers
|
|
772
|
+
const handler = getClientHandlerOrThrow(handlers, change.table);
|
|
702
773
|
await handler.applyChange({ trx }, change);
|
|
703
774
|
}
|
|
704
775
|
}
|
|
@@ -763,10 +834,11 @@ export async function applyPullResponse<DB extends SyncClientDb>(
|
|
|
763
834
|
export async function syncPullOnce<DB extends SyncClientDb>(
|
|
764
835
|
db: Kysely<DB>,
|
|
765
836
|
transport: SyncTransport,
|
|
766
|
-
handlers:
|
|
767
|
-
options: SyncPullOnceOptions
|
|
837
|
+
handlers: ClientHandlerCollection<DB>,
|
|
838
|
+
options: SyncPullOnceOptions,
|
|
839
|
+
pullStateOverride?: SyncPullRequestState
|
|
768
840
|
): Promise<SyncPullResponse> {
|
|
769
|
-
const pullState = await buildPullRequest(db, options);
|
|
841
|
+
const pullState = pullStateOverride ?? (await buildPullRequest(db, options));
|
|
770
842
|
const { clientId, ...pullBody } = pullState.request;
|
|
771
843
|
const combined = await transport.sync({ clientId, pull: pullBody });
|
|
772
844
|
if (!combined.pull) {
|
package/src/push-engine.ts
CHANGED
|
@@ -7,6 +7,7 @@ import type {
|
|
|
7
7
|
SyncPushResponse,
|
|
8
8
|
SyncTransport,
|
|
9
9
|
} from '@syncular/core';
|
|
10
|
+
import { countSyncMetric } from '@syncular/core';
|
|
10
11
|
import type { Kysely } from 'kysely';
|
|
11
12
|
import { upsertConflictsForRejectedCommit } from './conflicts';
|
|
12
13
|
import {
|
|
@@ -85,6 +86,7 @@ export async function syncPushOnce<DB extends SyncClientDb>(
|
|
|
85
86
|
}
|
|
86
87
|
|
|
87
88
|
let res: SyncPushResponse;
|
|
89
|
+
let usedWsPush = false;
|
|
88
90
|
try {
|
|
89
91
|
// Try WS push first if the transport supports it
|
|
90
92
|
let wsResponse: SyncPushResponse | null = null;
|
|
@@ -94,6 +96,7 @@ export async function syncPushOnce<DB extends SyncClientDb>(
|
|
|
94
96
|
|
|
95
97
|
if (wsResponse) {
|
|
96
98
|
res = wsResponse;
|
|
99
|
+
usedWsPush = true;
|
|
97
100
|
} else {
|
|
98
101
|
// Fall back to HTTP
|
|
99
102
|
const combined = await transport.sync({
|
|
@@ -156,6 +159,18 @@ export async function syncPushOnce<DB extends SyncClientDb>(
|
|
|
156
159
|
}
|
|
157
160
|
|
|
158
161
|
const responseJson = JSON.stringify(responseToUse);
|
|
162
|
+
const detectedConflicts = responseToUse.results.reduce(
|
|
163
|
+
(count, result) => count + (result.status === 'conflict' ? 1 : 0),
|
|
164
|
+
0
|
|
165
|
+
);
|
|
166
|
+
if (detectedConflicts > 0 && !usedWsPush) {
|
|
167
|
+
countSyncMetric('sync.conflicts.detected', detectedConflicts, {
|
|
168
|
+
attributes: {
|
|
169
|
+
source: 'client',
|
|
170
|
+
transport: 'http',
|
|
171
|
+
},
|
|
172
|
+
});
|
|
173
|
+
}
|
|
159
174
|
|
|
160
175
|
if (responseToUse.status === 'applied' || responseToUse.status === 'cached') {
|
|
161
176
|
await markOutboxCommitAcked(db, {
|
package/src/sync-loop.ts
CHANGED
|
@@ -14,7 +14,7 @@ import type {
|
|
|
14
14
|
} from '@syncular/core';
|
|
15
15
|
import type { Kysely } from 'kysely';
|
|
16
16
|
import { upsertConflictsForRejectedCommit } from './conflicts';
|
|
17
|
-
import type {
|
|
17
|
+
import type { ClientHandlerCollection } from './handlers/collection';
|
|
18
18
|
import {
|
|
19
19
|
getNextSendableOutboxCommit,
|
|
20
20
|
markOutboxCommitAcked,
|
|
@@ -25,7 +25,9 @@ import type { SyncClientPluginContext } from './plugins/types';
|
|
|
25
25
|
import {
|
|
26
26
|
applyPullResponse,
|
|
27
27
|
buildPullRequest,
|
|
28
|
+
createFollowupPullState,
|
|
28
29
|
type SyncPullOnceOptions,
|
|
30
|
+
type SyncPullRequestState,
|
|
29
31
|
syncPullOnce,
|
|
30
32
|
} from './pull-engine';
|
|
31
33
|
import { type SyncPushOnceOptions, syncPushOnce } from './push-engine';
|
|
@@ -64,6 +66,8 @@ async function syncPushUntilSettled<DB extends SyncClientDb>(
|
|
|
64
66
|
interface SyncPullUntilSettledOptions extends SyncPullOnceOptions {
|
|
65
67
|
/** Max pull rounds per call. Default: 20 */
|
|
66
68
|
maxRounds?: number;
|
|
69
|
+
/** Optional prebuilt state from a prior pull round in the same sync cycle. */
|
|
70
|
+
initialPullState?: SyncPullRequestState;
|
|
67
71
|
}
|
|
68
72
|
|
|
69
73
|
interface SyncPullUntilSettledResult {
|
|
@@ -119,20 +123,23 @@ function mergePullResponse(
|
|
|
119
123
|
async function syncPullUntilSettled<DB extends SyncClientDb>(
|
|
120
124
|
db: Kysely<DB>,
|
|
121
125
|
transport: SyncTransport,
|
|
122
|
-
handlers:
|
|
126
|
+
handlers: ClientHandlerCollection<DB>,
|
|
123
127
|
options: SyncPullUntilSettledOptions
|
|
124
128
|
): Promise<SyncPullUntilSettledResult> {
|
|
125
129
|
const maxRounds = Math.max(1, Math.min(1000, options.maxRounds ?? 20));
|
|
126
130
|
|
|
127
131
|
const aggregatedBySubId = new Map<string, SyncPullSubscriptionResponse>();
|
|
132
|
+
let pullState =
|
|
133
|
+
options.initialPullState ?? (await buildPullRequest(db, options));
|
|
128
134
|
let rounds = 0;
|
|
129
135
|
|
|
130
136
|
for (let i = 0; i < maxRounds; i++) {
|
|
131
137
|
rounds += 1;
|
|
132
|
-
const res = await syncPullOnce(db, transport, handlers, options);
|
|
138
|
+
const res = await syncPullOnce(db, transport, handlers, options, pullState);
|
|
133
139
|
mergePullResponse(aggregatedBySubId, res);
|
|
134
140
|
|
|
135
141
|
if (!needsAnotherPull(res)) break;
|
|
142
|
+
pullState = createFollowupPullState(pullState, res);
|
|
136
143
|
}
|
|
137
144
|
|
|
138
145
|
return {
|
|
@@ -182,7 +189,7 @@ export interface SyncOnceResult {
|
|
|
182
189
|
async function syncOnceCombined<DB extends SyncClientDb>(
|
|
183
190
|
db: Kysely<DB>,
|
|
184
191
|
transport: SyncTransport,
|
|
185
|
-
handlers:
|
|
192
|
+
handlers: ClientHandlerCollection<DB>,
|
|
186
193
|
options: SyncOnceOptions
|
|
187
194
|
): Promise<SyncOnceResult> {
|
|
188
195
|
const pullOpts: SyncPullOnceOptions = {
|
|
@@ -348,6 +355,7 @@ async function syncOnceCombined<DB extends SyncClientDb>(
|
|
|
348
355
|
const more = await syncPullUntilSettled(db, transport, handlers, {
|
|
349
356
|
...pullOpts,
|
|
350
357
|
maxRounds: (options.maxPullRounds ?? 20) - 1,
|
|
358
|
+
initialPullState: createFollowupPullState(pullState, pullResponse),
|
|
351
359
|
});
|
|
352
360
|
pullRounds += more.rounds;
|
|
353
361
|
mergePullResponse(aggregatedBySubId, more.response);
|
|
@@ -364,7 +372,7 @@ async function syncOnceCombined<DB extends SyncClientDb>(
|
|
|
364
372
|
export async function syncOnce<DB extends SyncClientDb>(
|
|
365
373
|
db: Kysely<DB>,
|
|
366
374
|
transport: SyncTransport,
|
|
367
|
-
handlers:
|
|
375
|
+
handlers: ClientHandlerCollection<DB>,
|
|
368
376
|
options: SyncOnceOptions
|
|
369
377
|
): Promise<SyncOnceResult> {
|
|
370
378
|
return syncOnceCombined(db, transport, handlers, options);
|