@syncular/client 0.0.5-42 → 0.0.6-101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts +3 -3
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +7 -1
- package/dist/client.js.map +1 -1
- package/dist/create-client.d.ts +3 -4
- package/dist/create-client.d.ts.map +1 -1
- package/dist/create-client.js +16 -12
- package/dist/create-client.js.map +1 -1
- package/dist/engine/SyncEngine.d.ts.map +1 -1
- package/dist/engine/SyncEngine.js +49 -29
- package/dist/engine/SyncEngine.js.map +1 -1
- package/dist/engine/types.d.ts +3 -3
- package/dist/engine/types.d.ts.map +1 -1
- package/dist/handlers/collection.d.ts +6 -0
- package/dist/handlers/collection.d.ts.map +1 -0
- package/dist/handlers/collection.js +21 -0
- package/dist/handlers/collection.js.map +1 -0
- package/dist/handlers/create-handler.d.ts +1 -1
- package/dist/handlers/create-handler.d.ts.map +1 -1
- package/dist/handlers/create-handler.js +3 -3
- package/dist/handlers/create-handler.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/migrate.d.ts.map +1 -1
- package/dist/migrate.js +12 -0
- package/dist/migrate.js.map +1 -1
- package/dist/mutations.d.ts +1 -1
- package/dist/mutations.d.ts.map +1 -1
- package/dist/mutations.js +3 -3
- package/dist/mutations.js.map +1 -1
- package/dist/pull-engine.d.ts +11 -14
- package/dist/pull-engine.d.ts.map +1 -1
- package/dist/pull-engine.js +68 -6
- package/dist/pull-engine.js.map +1 -1
- package/dist/push-engine.d.ts.map +1 -1
- package/dist/push-engine.js +12 -0
- package/dist/push-engine.js.map +1 -1
- package/dist/sync-loop.d.ts +2 -2
- package/dist/sync-loop.d.ts.map +1 -1
- package/dist/sync-loop.js +5 -2
- package/dist/sync-loop.js.map +1 -1
- package/dist/sync.d.ts +32 -0
- package/dist/sync.d.ts.map +1 -0
- package/dist/sync.js +55 -0
- package/dist/sync.js.map +1 -0
- package/package.json +4 -4
- package/src/client.test.ts +18 -9
- package/src/client.ts +11 -4
- package/src/create-client.test.ts +83 -0
- package/src/create-client.ts +21 -16
- package/src/engine/SyncEngine.test.ts +241 -32
- package/src/engine/SyncEngine.ts +53 -33
- package/src/engine/types.ts +3 -3
- package/src/handlers/collection.ts +36 -0
- package/src/handlers/create-handler.ts +4 -4
- package/src/index.ts +2 -1
- package/src/migrate.ts +14 -0
- package/src/mutations.ts +4 -4
- package/src/pull-engine.test.ts +151 -6
- package/src/pull-engine.ts +93 -21
- package/src/push-engine.ts +15 -0
- package/src/sync-loop.ts +13 -5
- package/src/sync.ts +170 -0
- package/dist/handlers/registry.d.ts +0 -15
- package/dist/handlers/registry.d.ts.map +0 -1
- package/dist/handlers/registry.js +0 -29
- package/dist/handlers/registry.js.map +0 -1
- package/src/handlers/registry.ts +0 -36
package/src/pull-engine.test.ts
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
2
|
import { gzipSync } from 'node:zlib';
|
|
3
3
|
import {
|
|
4
|
+
createDatabase,
|
|
4
5
|
encodeSnapshotRows,
|
|
5
6
|
type SyncPullResponse,
|
|
6
7
|
type SyncTransport,
|
|
7
8
|
} from '@syncular/core';
|
|
8
9
|
import { type Kysely, sql } from 'kysely';
|
|
9
|
-
import {
|
|
10
|
+
import { createBunSqliteDialect } from '../../dialect-bun-sqlite/src';
|
|
11
|
+
import type { ClientHandlerCollection } from './handlers/collection';
|
|
10
12
|
import { createClientHandler } from './handlers/create-handler';
|
|
11
|
-
import { ClientTableRegistry } from './handlers/registry';
|
|
12
13
|
import { ensureClientSyncSchema } from './migrate';
|
|
13
14
|
import { applyPullResponse, buildPullRequest } from './pull-engine';
|
|
14
15
|
import type { SyncClientDb } from './schema';
|
|
@@ -40,7 +41,10 @@ describe('applyPullResponse chunk streaming', () => {
|
|
|
40
41
|
let db: Kysely<TestDb>;
|
|
41
42
|
|
|
42
43
|
beforeEach(async () => {
|
|
43
|
-
db =
|
|
44
|
+
db = createDatabase<TestDb>({
|
|
45
|
+
dialect: createBunSqliteDialect({ path: ':memory:' }),
|
|
46
|
+
family: 'sqlite',
|
|
47
|
+
});
|
|
44
48
|
await ensureClientSyncSchema(db);
|
|
45
49
|
await db.schema
|
|
46
50
|
.createTable('items')
|
|
@@ -75,12 +79,12 @@ describe('applyPullResponse chunk streaming', () => {
|
|
|
75
79
|
},
|
|
76
80
|
};
|
|
77
81
|
|
|
78
|
-
const handlers
|
|
82
|
+
const handlers: ClientHandlerCollection<TestDb> = [
|
|
79
83
|
createClientHandler({
|
|
80
84
|
table: 'items',
|
|
81
85
|
scopes: ['items:{id}'],
|
|
82
|
-
})
|
|
83
|
-
|
|
86
|
+
}),
|
|
87
|
+
];
|
|
84
88
|
|
|
85
89
|
const options = {
|
|
86
90
|
clientId: 'client-1',
|
|
@@ -144,4 +148,145 @@ describe('applyPullResponse chunk streaming', () => {
|
|
|
144
148
|
expect(Number(countResult.rows[0]?.count ?? 0)).toBe(rows.length);
|
|
145
149
|
expect(streamFetchCount).toBe(1);
|
|
146
150
|
});
|
|
151
|
+
|
|
152
|
+
it('rolls back partial chunked bootstrap when a later chunk fails', async () => {
|
|
153
|
+
const firstRows = Array.from({ length: 1500 }, (_, index) => ({
|
|
154
|
+
id: `${index + 1}`,
|
|
155
|
+
name: `Item ${index + 1}`,
|
|
156
|
+
}));
|
|
157
|
+
const secondRows = Array.from({ length: 1500 }, (_, index) => ({
|
|
158
|
+
id: `${index + 1501}`,
|
|
159
|
+
name: `Item ${index + 1501}`,
|
|
160
|
+
}));
|
|
161
|
+
|
|
162
|
+
const firstChunk = new Uint8Array(gzipSync(encodeSnapshotRows(firstRows)));
|
|
163
|
+
const secondChunk = new Uint8Array(
|
|
164
|
+
gzipSync(encodeSnapshotRows(secondRows))
|
|
165
|
+
);
|
|
166
|
+
|
|
167
|
+
let failSecondChunk = true;
|
|
168
|
+
const transport: SyncTransport = {
|
|
169
|
+
async sync() {
|
|
170
|
+
return {};
|
|
171
|
+
},
|
|
172
|
+
async fetchSnapshotChunk() {
|
|
173
|
+
throw new Error('fetchSnapshotChunk should not be used');
|
|
174
|
+
},
|
|
175
|
+
async fetchSnapshotChunkStream({ chunkId }) {
|
|
176
|
+
if (chunkId === 'chunk-2' && failSecondChunk) {
|
|
177
|
+
throw new Error('chunk-2 missing');
|
|
178
|
+
}
|
|
179
|
+
if (chunkId === 'chunk-1') {
|
|
180
|
+
return createStreamFromBytes(firstChunk, 317);
|
|
181
|
+
}
|
|
182
|
+
if (chunkId === 'chunk-2') {
|
|
183
|
+
return createStreamFromBytes(secondChunk, 503);
|
|
184
|
+
}
|
|
185
|
+
throw new Error(`Unexpected chunk id: ${chunkId}`);
|
|
186
|
+
},
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
const handlers: ClientHandlerCollection<TestDb> = [
|
|
190
|
+
createClientHandler({
|
|
191
|
+
table: 'items',
|
|
192
|
+
scopes: ['items:{id}'],
|
|
193
|
+
}),
|
|
194
|
+
];
|
|
195
|
+
|
|
196
|
+
const options = {
|
|
197
|
+
clientId: 'client-1',
|
|
198
|
+
subscriptions: [
|
|
199
|
+
{
|
|
200
|
+
id: 'items-sub',
|
|
201
|
+
table: 'items',
|
|
202
|
+
scopes: {},
|
|
203
|
+
},
|
|
204
|
+
],
|
|
205
|
+
stateId: 'default',
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
const response: SyncPullResponse = {
|
|
209
|
+
ok: true,
|
|
210
|
+
subscriptions: [
|
|
211
|
+
{
|
|
212
|
+
id: 'items-sub',
|
|
213
|
+
status: 'active',
|
|
214
|
+
scopes: {},
|
|
215
|
+
bootstrap: true,
|
|
216
|
+
bootstrapState: null,
|
|
217
|
+
nextCursor: 12,
|
|
218
|
+
commits: [],
|
|
219
|
+
snapshots: [
|
|
220
|
+
{
|
|
221
|
+
table: 'items',
|
|
222
|
+
rows: [],
|
|
223
|
+
chunks: [
|
|
224
|
+
{
|
|
225
|
+
id: 'chunk-1',
|
|
226
|
+
byteLength: firstChunk.length,
|
|
227
|
+
sha256: '',
|
|
228
|
+
encoding: 'json-row-frame-v1',
|
|
229
|
+
compression: 'gzip',
|
|
230
|
+
},
|
|
231
|
+
{
|
|
232
|
+
id: 'chunk-2',
|
|
233
|
+
byteLength: secondChunk.length,
|
|
234
|
+
sha256: '',
|
|
235
|
+
encoding: 'json-row-frame-v1',
|
|
236
|
+
compression: 'gzip',
|
|
237
|
+
},
|
|
238
|
+
],
|
|
239
|
+
isFirstPage: true,
|
|
240
|
+
isLastPage: true,
|
|
241
|
+
},
|
|
242
|
+
],
|
|
243
|
+
},
|
|
244
|
+
],
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
const firstPullState = await buildPullRequest(db, options);
|
|
248
|
+
await expect(
|
|
249
|
+
applyPullResponse(
|
|
250
|
+
db,
|
|
251
|
+
transport,
|
|
252
|
+
handlers,
|
|
253
|
+
options,
|
|
254
|
+
firstPullState,
|
|
255
|
+
response
|
|
256
|
+
)
|
|
257
|
+
).rejects.toThrow('chunk-2 missing');
|
|
258
|
+
|
|
259
|
+
const countAfterFailure = await sql<{ count: number }>`
|
|
260
|
+
select count(*) as count
|
|
261
|
+
from ${sql.table('items')}
|
|
262
|
+
`.execute(db);
|
|
263
|
+
expect(Number(countAfterFailure.rows[0]?.count ?? 0)).toBe(0);
|
|
264
|
+
|
|
265
|
+
const stateAfterFailure = await db
|
|
266
|
+
.selectFrom('sync_subscription_state')
|
|
267
|
+
.select(({ fn }) => fn.countAll().as('total'))
|
|
268
|
+
.where('state_id', '=', 'default')
|
|
269
|
+
.where('subscription_id', '=', 'items-sub')
|
|
270
|
+
.executeTakeFirst();
|
|
271
|
+
expect(Number(stateAfterFailure?.total ?? 0)).toBe(0);
|
|
272
|
+
|
|
273
|
+
failSecondChunk = false;
|
|
274
|
+
const retryPullState = await buildPullRequest(db, options);
|
|
275
|
+
await applyPullResponse(
|
|
276
|
+
db,
|
|
277
|
+
transport,
|
|
278
|
+
handlers,
|
|
279
|
+
options,
|
|
280
|
+
retryPullState,
|
|
281
|
+
response
|
|
282
|
+
);
|
|
283
|
+
|
|
284
|
+
const countAfterRetry = await sql<{ count: number }>`
|
|
285
|
+
select count(*) as count
|
|
286
|
+
from ${sql.table('items')}
|
|
287
|
+
`.execute(db);
|
|
288
|
+
expect(Number(countAfterRetry.rows[0]?.count ?? 0)).toBe(
|
|
289
|
+
firstRows.length + secondRows.length
|
|
290
|
+
);
|
|
291
|
+
});
|
|
147
292
|
});
|
package/src/pull-engine.ts
CHANGED
|
@@ -6,13 +6,17 @@ import type {
|
|
|
6
6
|
SyncBootstrapState,
|
|
7
7
|
SyncPullRequest,
|
|
8
8
|
SyncPullResponse,
|
|
9
|
+
SyncPullSubscriptionResponse,
|
|
9
10
|
SyncSnapshot,
|
|
10
11
|
SyncSubscriptionRequest,
|
|
11
12
|
SyncTransport,
|
|
12
13
|
} from '@syncular/core';
|
|
13
14
|
import { decodeSnapshotRows } from '@syncular/core';
|
|
14
15
|
import { type Kysely, sql, type Transaction } from 'kysely';
|
|
15
|
-
import
|
|
16
|
+
import {
|
|
17
|
+
type ClientHandlerCollection,
|
|
18
|
+
getClientHandlerOrThrow,
|
|
19
|
+
} from './handlers/collection';
|
|
16
20
|
import type { ClientTableHandler } from './handlers/types';
|
|
17
21
|
import type {
|
|
18
22
|
SyncClientPlugin,
|
|
@@ -513,6 +517,13 @@ export interface SyncPullOnceOptions {
|
|
|
513
517
|
sha256?: (bytes: Uint8Array) => Promise<string>;
|
|
514
518
|
}
|
|
515
519
|
|
|
520
|
+
export interface SyncPullRequestState {
|
|
521
|
+
request: SyncPullRequest;
|
|
522
|
+
existing: SyncSubscriptionStateTable[];
|
|
523
|
+
existingById: Map<string, SyncSubscriptionStateTable>;
|
|
524
|
+
stateId: string;
|
|
525
|
+
}
|
|
526
|
+
|
|
516
527
|
/**
|
|
517
528
|
* Build a pull request from subscription state. Exported for use
|
|
518
529
|
* by the combined sync path in sync-loop.ts.
|
|
@@ -520,12 +531,7 @@ export interface SyncPullOnceOptions {
|
|
|
520
531
|
export async function buildPullRequest<DB extends SyncClientDb>(
|
|
521
532
|
db: Kysely<DB>,
|
|
522
533
|
options: SyncPullOnceOptions
|
|
523
|
-
): Promise<{
|
|
524
|
-
request: SyncPullRequest;
|
|
525
|
-
existing: SyncSubscriptionStateTable[];
|
|
526
|
-
existingById: Map<string, SyncSubscriptionStateTable>;
|
|
527
|
-
stateId: string;
|
|
528
|
-
}> {
|
|
534
|
+
): Promise<SyncPullRequestState> {
|
|
529
535
|
const stateId = options.stateId ?? 'default';
|
|
530
536
|
|
|
531
537
|
const existingResult = await sql<SyncSubscriptionStateTable>`
|
|
@@ -566,6 +572,70 @@ export async function buildPullRequest<DB extends SyncClientDb>(
|
|
|
566
572
|
return { request, existing, existingById, stateId };
|
|
567
573
|
}
|
|
568
574
|
|
|
575
|
+
export function createFollowupPullState(
|
|
576
|
+
pullState: SyncPullRequestState,
|
|
577
|
+
response: SyncPullResponse
|
|
578
|
+
): SyncPullRequestState {
|
|
579
|
+
const responseById = new Map<string, SyncPullSubscriptionResponse>();
|
|
580
|
+
for (const sub of response.subscriptions ?? []) {
|
|
581
|
+
responseById.set(sub.id, sub);
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
const now = Date.now();
|
|
585
|
+
const nextExisting: SyncSubscriptionStateTable[] = [];
|
|
586
|
+
const nextExistingById = new Map<string, SyncSubscriptionStateTable>();
|
|
587
|
+
|
|
588
|
+
for (const sub of pullState.request.subscriptions ?? []) {
|
|
589
|
+
const res = responseById.get(sub.id);
|
|
590
|
+
if (res?.status === 'revoked') {
|
|
591
|
+
continue;
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
const nextCursor = res ? Math.max(-1, res.nextCursor) : (sub.cursor ?? -1);
|
|
595
|
+
const nextBootstrapState = res
|
|
596
|
+
? res.bootstrap
|
|
597
|
+
? (res.bootstrapState ?? null)
|
|
598
|
+
: null
|
|
599
|
+
: (sub.bootstrapState ?? null);
|
|
600
|
+
const prev = pullState.existingById.get(sub.id);
|
|
601
|
+
const nextRow: SyncSubscriptionStateTable = {
|
|
602
|
+
state_id: pullState.stateId,
|
|
603
|
+
subscription_id: sub.id,
|
|
604
|
+
table: sub.table,
|
|
605
|
+
scopes_json: serializeJsonCached(sub.scopes ?? {}),
|
|
606
|
+
params_json: serializeJsonCached(sub.params ?? {}),
|
|
607
|
+
cursor: nextCursor,
|
|
608
|
+
bootstrap_state_json: nextBootstrapState
|
|
609
|
+
? serializeJsonCached(nextBootstrapState)
|
|
610
|
+
: null,
|
|
611
|
+
status: 'active',
|
|
612
|
+
created_at: prev?.created_at ?? now,
|
|
613
|
+
updated_at: now,
|
|
614
|
+
};
|
|
615
|
+
nextExisting.push(nextRow);
|
|
616
|
+
nextExistingById.set(nextRow.subscription_id, nextRow);
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
const nextRequest: SyncPullRequest = {
|
|
620
|
+
...pullState.request,
|
|
621
|
+
subscriptions: (pullState.request.subscriptions ?? []).map((sub) => {
|
|
622
|
+
const row = nextExistingById.get(sub.id);
|
|
623
|
+
return {
|
|
624
|
+
...sub,
|
|
625
|
+
cursor: Math.max(-1, row?.cursor ?? -1),
|
|
626
|
+
bootstrapState: parseBootstrapState(row?.bootstrap_state_json),
|
|
627
|
+
};
|
|
628
|
+
}),
|
|
629
|
+
};
|
|
630
|
+
|
|
631
|
+
return {
|
|
632
|
+
request: nextRequest,
|
|
633
|
+
existing: nextExisting,
|
|
634
|
+
existingById: nextExistingById,
|
|
635
|
+
stateId: pullState.stateId,
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
|
|
569
639
|
/**
|
|
570
640
|
* Apply a pull response (run plugins + write to local DB).
|
|
571
641
|
* Exported for use by the combined sync path in sync-loop.ts.
|
|
@@ -573,14 +643,9 @@ export async function buildPullRequest<DB extends SyncClientDb>(
|
|
|
573
643
|
export async function applyPullResponse<DB extends SyncClientDb>(
|
|
574
644
|
db: Kysely<DB>,
|
|
575
645
|
transport: SyncTransport,
|
|
576
|
-
handlers:
|
|
646
|
+
handlers: ClientHandlerCollection<DB>,
|
|
577
647
|
options: SyncPullOnceOptions,
|
|
578
|
-
pullState:
|
|
579
|
-
request: SyncPullRequest;
|
|
580
|
-
existing: SyncSubscriptionStateTable[];
|
|
581
|
-
existingById: Map<string, SyncSubscriptionStateTable>;
|
|
582
|
-
stateId: string;
|
|
583
|
-
},
|
|
648
|
+
pullState: SyncPullRequestState,
|
|
584
649
|
rawResponse: SyncPullResponse
|
|
585
650
|
): Promise<SyncPullResponse> {
|
|
586
651
|
const { request, existing, existingById, stateId } = pullState;
|
|
@@ -620,7 +685,10 @@ export async function applyPullResponse<DB extends SyncClientDb>(
|
|
|
620
685
|
? JSON.parse(row.scopes_json)
|
|
621
686
|
: row.scopes_json
|
|
622
687
|
: {};
|
|
623
|
-
await handlers
|
|
688
|
+
await getClientHandlerOrThrow(handlers, row.table).clearAll({
|
|
689
|
+
trx,
|
|
690
|
+
scopes,
|
|
691
|
+
});
|
|
624
692
|
} catch {
|
|
625
693
|
// ignore missing table handler
|
|
626
694
|
}
|
|
@@ -649,7 +717,10 @@ export async function applyPullResponse<DB extends SyncClientDb>(
|
|
|
649
717
|
? JSON.parse(prev.scopes_json)
|
|
650
718
|
: prev.scopes_json
|
|
651
719
|
: {};
|
|
652
|
-
await handlers
|
|
720
|
+
await getClientHandlerOrThrow(handlers, prev.table).clearAll({
|
|
721
|
+
trx,
|
|
722
|
+
scopes,
|
|
723
|
+
});
|
|
653
724
|
} catch {
|
|
654
725
|
// ignore missing handler
|
|
655
726
|
}
|
|
@@ -666,7 +737,7 @@ export async function applyPullResponse<DB extends SyncClientDb>(
|
|
|
666
737
|
// Apply snapshots (bootstrap mode)
|
|
667
738
|
if (sub.bootstrap) {
|
|
668
739
|
for (const snapshot of sub.snapshots ?? []) {
|
|
669
|
-
const handler = handlers
|
|
740
|
+
const handler = getClientHandlerOrThrow(handlers, snapshot.table);
|
|
670
741
|
const hasChunkRefs =
|
|
671
742
|
Array.isArray(snapshot.chunks) && snapshot.chunks.length > 0;
|
|
672
743
|
|
|
@@ -698,7 +769,7 @@ export async function applyPullResponse<DB extends SyncClientDb>(
|
|
|
698
769
|
// Apply incremental changes
|
|
699
770
|
for (const commit of sub.commits) {
|
|
700
771
|
for (const change of commit.changes) {
|
|
701
|
-
const handler = handlers
|
|
772
|
+
const handler = getClientHandlerOrThrow(handlers, change.table);
|
|
702
773
|
await handler.applyChange({ trx }, change);
|
|
703
774
|
}
|
|
704
775
|
}
|
|
@@ -763,10 +834,11 @@ export async function applyPullResponse<DB extends SyncClientDb>(
|
|
|
763
834
|
export async function syncPullOnce<DB extends SyncClientDb>(
|
|
764
835
|
db: Kysely<DB>,
|
|
765
836
|
transport: SyncTransport,
|
|
766
|
-
handlers:
|
|
767
|
-
options: SyncPullOnceOptions
|
|
837
|
+
handlers: ClientHandlerCollection<DB>,
|
|
838
|
+
options: SyncPullOnceOptions,
|
|
839
|
+
pullStateOverride?: SyncPullRequestState
|
|
768
840
|
): Promise<SyncPullResponse> {
|
|
769
|
-
const pullState = await buildPullRequest(db, options);
|
|
841
|
+
const pullState = pullStateOverride ?? (await buildPullRequest(db, options));
|
|
770
842
|
const { clientId, ...pullBody } = pullState.request;
|
|
771
843
|
const combined = await transport.sync({ clientId, pull: pullBody });
|
|
772
844
|
if (!combined.pull) {
|
package/src/push-engine.ts
CHANGED
|
@@ -7,6 +7,7 @@ import type {
|
|
|
7
7
|
SyncPushResponse,
|
|
8
8
|
SyncTransport,
|
|
9
9
|
} from '@syncular/core';
|
|
10
|
+
import { countSyncMetric } from '@syncular/core';
|
|
10
11
|
import type { Kysely } from 'kysely';
|
|
11
12
|
import { upsertConflictsForRejectedCommit } from './conflicts';
|
|
12
13
|
import {
|
|
@@ -85,6 +86,7 @@ export async function syncPushOnce<DB extends SyncClientDb>(
|
|
|
85
86
|
}
|
|
86
87
|
|
|
87
88
|
let res: SyncPushResponse;
|
|
89
|
+
let usedWsPush = false;
|
|
88
90
|
try {
|
|
89
91
|
// Try WS push first if the transport supports it
|
|
90
92
|
let wsResponse: SyncPushResponse | null = null;
|
|
@@ -94,6 +96,7 @@ export async function syncPushOnce<DB extends SyncClientDb>(
|
|
|
94
96
|
|
|
95
97
|
if (wsResponse) {
|
|
96
98
|
res = wsResponse;
|
|
99
|
+
usedWsPush = true;
|
|
97
100
|
} else {
|
|
98
101
|
// Fall back to HTTP
|
|
99
102
|
const combined = await transport.sync({
|
|
@@ -156,6 +159,18 @@ export async function syncPushOnce<DB extends SyncClientDb>(
|
|
|
156
159
|
}
|
|
157
160
|
|
|
158
161
|
const responseJson = JSON.stringify(responseToUse);
|
|
162
|
+
const detectedConflicts = responseToUse.results.reduce(
|
|
163
|
+
(count, result) => count + (result.status === 'conflict' ? 1 : 0),
|
|
164
|
+
0
|
|
165
|
+
);
|
|
166
|
+
if (detectedConflicts > 0 && !usedWsPush) {
|
|
167
|
+
countSyncMetric('sync.conflicts.detected', detectedConflicts, {
|
|
168
|
+
attributes: {
|
|
169
|
+
source: 'client',
|
|
170
|
+
transport: 'http',
|
|
171
|
+
},
|
|
172
|
+
});
|
|
173
|
+
}
|
|
159
174
|
|
|
160
175
|
if (responseToUse.status === 'applied' || responseToUse.status === 'cached') {
|
|
161
176
|
await markOutboxCommitAcked(db, {
|
package/src/sync-loop.ts
CHANGED
|
@@ -14,7 +14,7 @@ import type {
|
|
|
14
14
|
} from '@syncular/core';
|
|
15
15
|
import type { Kysely } from 'kysely';
|
|
16
16
|
import { upsertConflictsForRejectedCommit } from './conflicts';
|
|
17
|
-
import type {
|
|
17
|
+
import type { ClientHandlerCollection } from './handlers/collection';
|
|
18
18
|
import {
|
|
19
19
|
getNextSendableOutboxCommit,
|
|
20
20
|
markOutboxCommitAcked,
|
|
@@ -25,7 +25,9 @@ import type { SyncClientPluginContext } from './plugins/types';
|
|
|
25
25
|
import {
|
|
26
26
|
applyPullResponse,
|
|
27
27
|
buildPullRequest,
|
|
28
|
+
createFollowupPullState,
|
|
28
29
|
type SyncPullOnceOptions,
|
|
30
|
+
type SyncPullRequestState,
|
|
29
31
|
syncPullOnce,
|
|
30
32
|
} from './pull-engine';
|
|
31
33
|
import { type SyncPushOnceOptions, syncPushOnce } from './push-engine';
|
|
@@ -64,6 +66,8 @@ async function syncPushUntilSettled<DB extends SyncClientDb>(
|
|
|
64
66
|
interface SyncPullUntilSettledOptions extends SyncPullOnceOptions {
|
|
65
67
|
/** Max pull rounds per call. Default: 20 */
|
|
66
68
|
maxRounds?: number;
|
|
69
|
+
/** Optional prebuilt state from a prior pull round in the same sync cycle. */
|
|
70
|
+
initialPullState?: SyncPullRequestState;
|
|
67
71
|
}
|
|
68
72
|
|
|
69
73
|
interface SyncPullUntilSettledResult {
|
|
@@ -119,20 +123,23 @@ function mergePullResponse(
|
|
|
119
123
|
async function syncPullUntilSettled<DB extends SyncClientDb>(
|
|
120
124
|
db: Kysely<DB>,
|
|
121
125
|
transport: SyncTransport,
|
|
122
|
-
handlers:
|
|
126
|
+
handlers: ClientHandlerCollection<DB>,
|
|
123
127
|
options: SyncPullUntilSettledOptions
|
|
124
128
|
): Promise<SyncPullUntilSettledResult> {
|
|
125
129
|
const maxRounds = Math.max(1, Math.min(1000, options.maxRounds ?? 20));
|
|
126
130
|
|
|
127
131
|
const aggregatedBySubId = new Map<string, SyncPullSubscriptionResponse>();
|
|
132
|
+
let pullState =
|
|
133
|
+
options.initialPullState ?? (await buildPullRequest(db, options));
|
|
128
134
|
let rounds = 0;
|
|
129
135
|
|
|
130
136
|
for (let i = 0; i < maxRounds; i++) {
|
|
131
137
|
rounds += 1;
|
|
132
|
-
const res = await syncPullOnce(db, transport, handlers, options);
|
|
138
|
+
const res = await syncPullOnce(db, transport, handlers, options, pullState);
|
|
133
139
|
mergePullResponse(aggregatedBySubId, res);
|
|
134
140
|
|
|
135
141
|
if (!needsAnotherPull(res)) break;
|
|
142
|
+
pullState = createFollowupPullState(pullState, res);
|
|
136
143
|
}
|
|
137
144
|
|
|
138
145
|
return {
|
|
@@ -182,7 +189,7 @@ export interface SyncOnceResult {
|
|
|
182
189
|
async function syncOnceCombined<DB extends SyncClientDb>(
|
|
183
190
|
db: Kysely<DB>,
|
|
184
191
|
transport: SyncTransport,
|
|
185
|
-
handlers:
|
|
192
|
+
handlers: ClientHandlerCollection<DB>,
|
|
186
193
|
options: SyncOnceOptions
|
|
187
194
|
): Promise<SyncOnceResult> {
|
|
188
195
|
const pullOpts: SyncPullOnceOptions = {
|
|
@@ -348,6 +355,7 @@ async function syncOnceCombined<DB extends SyncClientDb>(
|
|
|
348
355
|
const more = await syncPullUntilSettled(db, transport, handlers, {
|
|
349
356
|
...pullOpts,
|
|
350
357
|
maxRounds: (options.maxPullRounds ?? 20) - 1,
|
|
358
|
+
initialPullState: createFollowupPullState(pullState, pullResponse),
|
|
351
359
|
});
|
|
352
360
|
pullRounds += more.rounds;
|
|
353
361
|
mergePullResponse(aggregatedBySubId, more.response);
|
|
@@ -364,7 +372,7 @@ async function syncOnceCombined<DB extends SyncClientDb>(
|
|
|
364
372
|
export async function syncOnce<DB extends SyncClientDb>(
|
|
365
373
|
db: Kysely<DB>,
|
|
366
374
|
transport: SyncTransport,
|
|
367
|
-
handlers:
|
|
375
|
+
handlers: ClientHandlerCollection<DB>,
|
|
368
376
|
options: SyncOnceOptions
|
|
369
377
|
): Promise<SyncOnceResult> {
|
|
370
378
|
return syncOnceCombined(db, transport, handlers, options);
|
package/src/sync.ts
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
ColumnCodecDialect,
|
|
3
|
+
ColumnCodecSource,
|
|
4
|
+
ScopeDefinition,
|
|
5
|
+
ScopeValue,
|
|
6
|
+
ScopeValuesFromPatterns,
|
|
7
|
+
SyncSubscriptionRequest,
|
|
8
|
+
} from '@syncular/core';
|
|
9
|
+
import {
|
|
10
|
+
type CreateClientHandlerOptions,
|
|
11
|
+
createClientHandler,
|
|
12
|
+
} from './handlers/create-handler';
|
|
13
|
+
import type { ClientTableHandler } from './handlers/types';
|
|
14
|
+
import type { SyncClientDb } from './schema';
|
|
15
|
+
|
|
16
|
+
type ClientSyncSubscription<ScopeDefs extends readonly ScopeDefinition[]> =
|
|
17
|
+
Omit<SyncSubscriptionRequest, 'cursor' | 'table' | 'scopes'> & {
|
|
18
|
+
table: string;
|
|
19
|
+
scopes?: ScopeValuesFromPatterns<ScopeDefs>;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
type SharedTableName<DB extends SyncClientDb> = keyof DB & string;
|
|
23
|
+
|
|
24
|
+
export type ClientSyncHandlerOptionsForTable<
|
|
25
|
+
DB extends SyncClientDb,
|
|
26
|
+
TableName extends SharedTableName<DB>,
|
|
27
|
+
ScopeDefs extends readonly ScopeDefinition[],
|
|
28
|
+
Identity,
|
|
29
|
+
> = Omit<
|
|
30
|
+
CreateClientHandlerOptions<DB, TableName, ScopeDefs>,
|
|
31
|
+
'codecs' | 'codecDialect' | 'subscribe'
|
|
32
|
+
> & {
|
|
33
|
+
codecs?: ColumnCodecSource;
|
|
34
|
+
codecDialect?: ColumnCodecDialect;
|
|
35
|
+
subscribe?:
|
|
36
|
+
| ClientSyncSubscription<ScopeDefs>
|
|
37
|
+
| ClientSyncSubscription<ScopeDefs>[]
|
|
38
|
+
| null
|
|
39
|
+
| ((args: {
|
|
40
|
+
identity: Identity;
|
|
41
|
+
}) =>
|
|
42
|
+
| ClientSyncSubscription<ScopeDefs>
|
|
43
|
+
| ClientSyncSubscription<ScopeDefs>[]
|
|
44
|
+
| null);
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
export interface ClientSyncConfig<
|
|
48
|
+
DB extends SyncClientDb = SyncClientDb,
|
|
49
|
+
Identity = { actorId: string },
|
|
50
|
+
> {
|
|
51
|
+
handlers: ClientTableHandler<DB>[];
|
|
52
|
+
subscriptions(
|
|
53
|
+
identity: Identity
|
|
54
|
+
): Array<Omit<SyncSubscriptionRequest, 'cursor'>>;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface DefineClientSyncOptions {
|
|
58
|
+
codecs?: ColumnCodecSource;
|
|
59
|
+
codecDialect?: ColumnCodecDialect;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface ClientSyncBuilder<
|
|
63
|
+
DB extends SyncClientDb,
|
|
64
|
+
ScopeDefs extends readonly ScopeDefinition[],
|
|
65
|
+
Identity,
|
|
66
|
+
> extends ClientSyncConfig<DB, Identity> {
|
|
67
|
+
addHandler<TableName extends SharedTableName<DB>>(
|
|
68
|
+
options: ClientSyncHandlerOptionsForTable<
|
|
69
|
+
DB,
|
|
70
|
+
TableName,
|
|
71
|
+
ScopeDefs,
|
|
72
|
+
Identity
|
|
73
|
+
>
|
|
74
|
+
): this;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function defineClientSync<
|
|
78
|
+
DB extends SyncClientDb,
|
|
79
|
+
ScopeDefs extends readonly ScopeDefinition[],
|
|
80
|
+
Identity,
|
|
81
|
+
>(
|
|
82
|
+
options: DefineClientSyncOptions
|
|
83
|
+
): ClientSyncBuilder<DB, ScopeDefs, Identity> {
|
|
84
|
+
const handlers: ClientTableHandler<DB>[] = [];
|
|
85
|
+
const registeredTables = new Set<string>();
|
|
86
|
+
const subscriptionsByTable = new Map<
|
|
87
|
+
string,
|
|
88
|
+
ClientSyncHandlerOptionsForTable<
|
|
89
|
+
DB,
|
|
90
|
+
SharedTableName<DB>,
|
|
91
|
+
ScopeDefs,
|
|
92
|
+
Identity
|
|
93
|
+
>['subscribe']
|
|
94
|
+
>();
|
|
95
|
+
|
|
96
|
+
const toScopeValues = (
|
|
97
|
+
value: ScopeValuesFromPatterns<ScopeDefs> | undefined
|
|
98
|
+
): Record<string, ScopeValue> => {
|
|
99
|
+
const result: Record<string, ScopeValue> = {};
|
|
100
|
+
for (const [key, scopeValue] of Object.entries(
|
|
101
|
+
(value ?? {}) as Record<string, ScopeValue | undefined>
|
|
102
|
+
)) {
|
|
103
|
+
if (scopeValue === undefined) continue;
|
|
104
|
+
result[key] = scopeValue;
|
|
105
|
+
}
|
|
106
|
+
return result;
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
const sync: ClientSyncBuilder<DB, ScopeDefs, Identity> = {
|
|
110
|
+
handlers,
|
|
111
|
+
addHandler<TableName extends SharedTableName<DB>>(
|
|
112
|
+
handlerOptions: ClientSyncHandlerOptionsForTable<
|
|
113
|
+
DB,
|
|
114
|
+
TableName,
|
|
115
|
+
ScopeDefs,
|
|
116
|
+
Identity
|
|
117
|
+
>
|
|
118
|
+
) {
|
|
119
|
+
if (registeredTables.has(handlerOptions.table)) {
|
|
120
|
+
throw new Error(
|
|
121
|
+
`Client table handler already registered: ${handlerOptions.table}`
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
handlers.push(
|
|
126
|
+
createClientHandler({
|
|
127
|
+
...handlerOptions,
|
|
128
|
+
subscribe: false,
|
|
129
|
+
codecs: options.codecs,
|
|
130
|
+
codecDialect: options.codecDialect,
|
|
131
|
+
})
|
|
132
|
+
);
|
|
133
|
+
subscriptionsByTable.set(
|
|
134
|
+
handlerOptions.table,
|
|
135
|
+
handlerOptions.subscribe as ClientSyncHandlerOptionsForTable<
|
|
136
|
+
DB,
|
|
137
|
+
SharedTableName<DB>,
|
|
138
|
+
ScopeDefs,
|
|
139
|
+
Identity
|
|
140
|
+
>['subscribe']
|
|
141
|
+
);
|
|
142
|
+
registeredTables.add(handlerOptions.table);
|
|
143
|
+
return sync;
|
|
144
|
+
},
|
|
145
|
+
subscriptions(
|
|
146
|
+
identity: Identity
|
|
147
|
+
): Array<Omit<SyncSubscriptionRequest, 'cursor'>> {
|
|
148
|
+
const resolved: Array<Omit<SyncSubscriptionRequest, 'cursor'>> = [];
|
|
149
|
+
for (const [table, subscribe] of subscriptionsByTable.entries()) {
|
|
150
|
+
if (!subscribe) continue;
|
|
151
|
+
const value =
|
|
152
|
+
typeof subscribe === 'function' ? subscribe({ identity }) : subscribe;
|
|
153
|
+
if (!value) continue;
|
|
154
|
+
const entries = Array.isArray(value) ? value : [value];
|
|
155
|
+
for (const entry of entries) {
|
|
156
|
+
resolved.push({
|
|
157
|
+
id: entry.id,
|
|
158
|
+
table: entry.table ?? table,
|
|
159
|
+
scopes: toScopeValues(entry.scopes),
|
|
160
|
+
params: entry.params,
|
|
161
|
+
bootstrapState: entry.bootstrapState,
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return resolved;
|
|
166
|
+
},
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
return sync;
|
|
170
|
+
}
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @syncular/client - Sync client table registry
|
|
3
|
-
*/
|
|
4
|
-
import type { ClientTableHandler } from './types';
|
|
5
|
-
/**
|
|
6
|
-
* Registry for client-side table handlers.
|
|
7
|
-
*/
|
|
8
|
-
export declare class ClientTableRegistry<DB> {
|
|
9
|
-
private handlers;
|
|
10
|
-
register(handler: ClientTableHandler<DB>): this;
|
|
11
|
-
get(table: string): ClientTableHandler<DB> | undefined;
|
|
12
|
-
getOrThrow(table: string): ClientTableHandler<DB>;
|
|
13
|
-
getAll(): ClientTableHandler<DB>[];
|
|
14
|
-
}
|
|
15
|
-
//# sourceMappingURL=registry.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/handlers/registry.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAElD;;GAEG;AACH,qBAAa,mBAAmB,CAAC,EAAE;IACjC,OAAO,CAAC,QAAQ,CAA6C;IAE7D,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC,EAAE,CAAC,GAAG,IAAI,CAQ9C;IAED,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAAC,EAAE,CAAC,GAAG,SAAS,CAErD;IAED,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAAC,EAAE,CAAC,CAIhD;IAED,MAAM,IAAI,kBAAkB,CAAC,EAAE,CAAC,EAAE,CAEjC;CACF"}
|