@syncular/server 0.0.5-44 → 0.0.6-101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dialect/base.d.ts +3 -3
- package/dist/dialect/base.d.ts.map +1 -1
- package/dist/dialect/base.js.map +1 -1
- package/dist/dialect/types.d.ts +5 -7
- package/dist/dialect/types.d.ts.map +1 -1
- package/dist/handlers/collection.d.ts +12 -0
- package/dist/handlers/collection.d.ts.map +1 -0
- package/dist/handlers/collection.js +64 -0
- package/dist/handlers/collection.js.map +1 -0
- package/dist/handlers/create-handler.d.ts +10 -10
- package/dist/handlers/create-handler.d.ts.map +1 -1
- package/dist/handlers/create-handler.js +101 -69
- package/dist/handlers/create-handler.js.map +1 -1
- package/dist/handlers/index.d.ts +1 -1
- package/dist/handlers/index.d.ts.map +1 -1
- package/dist/handlers/index.js +1 -1
- package/dist/handlers/index.js.map +1 -1
- package/dist/handlers/types.d.ts +18 -12
- package/dist/handlers/types.d.ts.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/notify.js +1 -1
- package/dist/notify.js.map +1 -1
- package/dist/proxy/collection.d.ts +9 -0
- package/dist/proxy/collection.d.ts.map +1 -0
- package/dist/proxy/collection.js +21 -0
- package/dist/proxy/collection.js.map +1 -0
- package/dist/proxy/handler.d.ts +3 -3
- package/dist/proxy/handler.d.ts.map +1 -1
- package/dist/proxy/handler.js +2 -1
- package/dist/proxy/handler.js.map +1 -1
- package/dist/proxy/index.d.ts +1 -1
- package/dist/proxy/index.d.ts.map +1 -1
- package/dist/proxy/index.js +3 -3
- package/dist/proxy/index.js.map +1 -1
- package/dist/proxy/oplog.js +1 -1
- package/dist/proxy/oplog.js.map +1 -1
- package/dist/pull.d.ts +12 -5
- package/dist/pull.d.ts.map +1 -1
- package/dist/pull.js +101 -55
- package/dist/pull.js.map +1 -1
- package/dist/push.d.ts +5 -5
- package/dist/push.d.ts.map +1 -1
- package/dist/push.js +6 -4
- package/dist/push.js.map +1 -1
- package/dist/subscriptions/cache.d.ts +55 -0
- package/dist/subscriptions/cache.d.ts.map +1 -0
- package/dist/subscriptions/cache.js +206 -0
- package/dist/subscriptions/cache.js.map +1 -0
- package/dist/subscriptions/index.d.ts +1 -0
- package/dist/subscriptions/index.d.ts.map +1 -1
- package/dist/subscriptions/index.js +1 -0
- package/dist/subscriptions/index.js.map +1 -1
- package/dist/subscriptions/resolve.d.ts +7 -4
- package/dist/subscriptions/resolve.d.ts.map +1 -1
- package/dist/subscriptions/resolve.js +74 -11
- package/dist/subscriptions/resolve.js.map +1 -1
- package/dist/sync.d.ts +21 -0
- package/dist/sync.d.ts.map +1 -0
- package/dist/sync.js +23 -0
- package/dist/sync.js.map +1 -0
- package/package.json +3 -3
- package/src/dialect/base.ts +5 -3
- package/src/dialect/types.ts +11 -8
- package/src/handlers/collection.ts +121 -0
- package/src/handlers/create-handler.ts +163 -109
- package/src/handlers/index.ts +1 -1
- package/src/handlers/types.ts +29 -12
- package/src/index.ts +1 -0
- package/src/notify.test.ts +25 -21
- package/src/notify.ts +1 -1
- package/src/proxy/collection.ts +39 -0
- package/src/proxy/handler.test.ts +15 -9
- package/src/proxy/handler.ts +4 -4
- package/src/proxy/index.ts +8 -3
- package/src/proxy/oplog.ts +1 -1
- package/src/pull.ts +155 -73
- package/src/push.ts +16 -9
- package/src/snapshot-chunks/db-metadata.test.ts +6 -3
- package/src/subscriptions/cache.ts +318 -0
- package/src/subscriptions/index.ts +1 -0
- package/src/subscriptions/resolve.test.ts +180 -0
- package/src/subscriptions/resolve.ts +94 -18
- package/src/sync.ts +101 -0
- package/dist/handlers/registry.d.ts +0 -20
- package/dist/handlers/registry.d.ts.map +0 -1
- package/dist/handlers/registry.js +0 -88
- package/dist/handlers/registry.js.map +0 -1
- package/dist/proxy/registry.d.ts +0 -35
- package/dist/proxy/registry.d.ts.map +0 -1
- package/dist/proxy/registry.js +0 -49
- package/dist/proxy/registry.js.map +0 -1
- package/src/handlers/registry.ts +0 -109
- package/src/proxy/registry.ts +0 -56
package/src/index.ts
CHANGED
package/src/notify.test.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
|
-
import {
|
|
2
|
+
import { createDatabase } from '@syncular/core';
|
|
3
|
+
import { createBunSqliteDialect } from '../../dialect-bun-sqlite/src';
|
|
3
4
|
import { createSqliteServerDialect } from '../../server-dialect-sqlite/src';
|
|
4
|
-
import { createServerHandler } from './handlers';
|
|
5
|
-
import { TableRegistry } from './handlers/registry';
|
|
5
|
+
import { createServerHandler, createServerHandlerCollection } from './handlers';
|
|
6
6
|
import { ensureSyncSchema } from './migrate';
|
|
7
7
|
import { EXTERNAL_CLIENT_ID, notifyExternalDataChange } from './notify';
|
|
8
8
|
import { pull } from './pull';
|
|
@@ -36,7 +36,10 @@ interface ClientDb {
|
|
|
36
36
|
const dialect = createSqliteServerDialect();
|
|
37
37
|
|
|
38
38
|
async function setupDb() {
|
|
39
|
-
const db =
|
|
39
|
+
const db = createDatabase<TestDb>({
|
|
40
|
+
dialect: createBunSqliteDialect({ path: ':memory:' }),
|
|
41
|
+
family: 'sqlite',
|
|
42
|
+
});
|
|
40
43
|
await ensureSyncSchema(db, dialect);
|
|
41
44
|
|
|
42
45
|
await db.schema
|
|
@@ -60,7 +63,7 @@ async function setupDb() {
|
|
|
60
63
|
}
|
|
61
64
|
|
|
62
65
|
describe('notifyExternalDataChange', () => {
|
|
63
|
-
let db: ReturnType<typeof
|
|
66
|
+
let db: ReturnType<typeof createBunSqliteDialect<TestDb>>;
|
|
64
67
|
|
|
65
68
|
beforeEach(async () => {
|
|
66
69
|
db = await setupDb();
|
|
@@ -198,7 +201,7 @@ describe('notifyExternalDataChange', () => {
|
|
|
198
201
|
});
|
|
199
202
|
|
|
200
203
|
describe('pull re-bootstrap after external data change', () => {
|
|
201
|
-
let db: ReturnType<typeof
|
|
204
|
+
let db: ReturnType<typeof createBunSqliteDialect<TestDb>>;
|
|
202
205
|
|
|
203
206
|
beforeEach(async () => {
|
|
204
207
|
db = await setupDb();
|
|
@@ -227,15 +230,14 @@ describe('pull re-bootstrap after external data change', () => {
|
|
|
227
230
|
resolveScopes: async () => ({ catalog_id: '*' }),
|
|
228
231
|
});
|
|
229
232
|
|
|
230
|
-
const handlers =
|
|
231
|
-
handlers.register(codesHandler);
|
|
233
|
+
const handlers = createServerHandlerCollection<TestDb>([codesHandler]);
|
|
232
234
|
|
|
233
235
|
// 1. Initial bootstrap pull
|
|
234
236
|
const firstPull = await pull({
|
|
235
237
|
db,
|
|
236
238
|
dialect,
|
|
237
239
|
handlers,
|
|
238
|
-
actorId: 'u1',
|
|
240
|
+
auth: { actorId: 'u1' },
|
|
239
241
|
request: {
|
|
240
242
|
clientId: 'client-1',
|
|
241
243
|
limitCommits: 10,
|
|
@@ -262,7 +264,7 @@ describe('pull re-bootstrap after external data change', () => {
|
|
|
262
264
|
db,
|
|
263
265
|
dialect,
|
|
264
266
|
handlers,
|
|
265
|
-
actorId: 'u1',
|
|
267
|
+
auth: { actorId: 'u1' },
|
|
266
268
|
request: {
|
|
267
269
|
clientId: 'client-1',
|
|
268
270
|
limitCommits: 10,
|
|
@@ -296,7 +298,7 @@ describe('pull re-bootstrap after external data change', () => {
|
|
|
296
298
|
db,
|
|
297
299
|
dialect,
|
|
298
300
|
handlers,
|
|
299
|
-
actorId: 'u1',
|
|
301
|
+
auth: { actorId: 'u1' },
|
|
300
302
|
request: {
|
|
301
303
|
clientId: 'client-1',
|
|
302
304
|
limitCommits: 10,
|
|
@@ -341,16 +343,17 @@ describe('pull re-bootstrap after external data change', () => {
|
|
|
341
343
|
resolveScopes: async () => ({ catalog_id: '*' }),
|
|
342
344
|
});
|
|
343
345
|
|
|
344
|
-
const handlers =
|
|
345
|
-
|
|
346
|
-
|
|
346
|
+
const handlers = createServerHandlerCollection<TestDb>([
|
|
347
|
+
tasksHandler,
|
|
348
|
+
codesHandler,
|
|
349
|
+
]);
|
|
347
350
|
|
|
348
351
|
// 1. Bootstrap pull for tasks
|
|
349
352
|
const firstPull = await pull({
|
|
350
353
|
db,
|
|
351
354
|
dialect,
|
|
352
355
|
handlers,
|
|
353
|
-
actorId: 'u1',
|
|
356
|
+
auth: { actorId: 'u1' },
|
|
354
357
|
request: {
|
|
355
358
|
clientId: 'client-1',
|
|
356
359
|
limitCommits: 10,
|
|
@@ -382,7 +385,7 @@ describe('pull re-bootstrap after external data change', () => {
|
|
|
382
385
|
db,
|
|
383
386
|
dialect,
|
|
384
387
|
handlers,
|
|
385
|
-
actorId: 'u1',
|
|
388
|
+
auth: { actorId: 'u1' },
|
|
386
389
|
request: {
|
|
387
390
|
clientId: 'client-1',
|
|
388
391
|
limitCommits: 10,
|
|
@@ -437,16 +440,17 @@ describe('pull re-bootstrap after external data change', () => {
|
|
|
437
440
|
resolveScopes: async () => ({ catalog_id: '*' }),
|
|
438
441
|
});
|
|
439
442
|
|
|
440
|
-
const handlers =
|
|
441
|
-
|
|
442
|
-
|
|
443
|
+
const handlers = createServerHandlerCollection<TestDb>([
|
|
444
|
+
tasksHandler,
|
|
445
|
+
codesHandler,
|
|
446
|
+
]);
|
|
443
447
|
|
|
444
448
|
// 1. Bootstrap both subscriptions
|
|
445
449
|
const firstPull = await pull({
|
|
446
450
|
db,
|
|
447
451
|
dialect,
|
|
448
452
|
handlers,
|
|
449
|
-
actorId: 'u1',
|
|
453
|
+
auth: { actorId: 'u1' },
|
|
450
454
|
request: {
|
|
451
455
|
clientId: 'client-1',
|
|
452
456
|
limitCommits: 10,
|
|
@@ -480,7 +484,7 @@ describe('pull re-bootstrap after external data change', () => {
|
|
|
480
484
|
db,
|
|
481
485
|
dialect,
|
|
482
486
|
handlers,
|
|
483
|
-
actorId: 'u1',
|
|
487
|
+
auth: { actorId: 'u1' },
|
|
484
488
|
request: {
|
|
485
489
|
clientId: 'client-1',
|
|
486
490
|
limitCommits: 10,
|
package/src/notify.ts
CHANGED
|
@@ -24,7 +24,7 @@ function toDialectJsonValue(
|
|
|
24
24
|
value: unknown
|
|
25
25
|
): unknown {
|
|
26
26
|
if (value === null || value === undefined) return null;
|
|
27
|
-
if (dialect.
|
|
27
|
+
if (dialect.family === 'sqlite') return JSON.stringify(value);
|
|
28
28
|
return value;
|
|
29
29
|
}
|
|
30
30
|
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { ProxyTableHandler } from './types';
|
|
2
|
+
|
|
3
|
+
export interface ProxyHandlerCollection {
|
|
4
|
+
handlers: ProxyTableHandler[];
|
|
5
|
+
byTable: ReadonlyMap<string, ProxyTableHandler>;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export function createProxyHandlerCollection(
|
|
9
|
+
handlers: ProxyTableHandler[]
|
|
10
|
+
): ProxyHandlerCollection {
|
|
11
|
+
const byTable = new Map<string, ProxyTableHandler>();
|
|
12
|
+
for (const handler of handlers) {
|
|
13
|
+
if (byTable.has(handler.table)) {
|
|
14
|
+
throw new Error(
|
|
15
|
+
`Proxy table handler already registered: ${handler.table}`
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
byTable.set(handler.table, handler);
|
|
19
|
+
}
|
|
20
|
+
return { handlers, byTable };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function getProxyHandler(
|
|
24
|
+
collection: ProxyHandlerCollection,
|
|
25
|
+
tableName: string
|
|
26
|
+
): ProxyTableHandler | undefined {
|
|
27
|
+
return collection.byTable.get(tableName);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function getProxyHandlerOrThrow(
|
|
31
|
+
collection: ProxyHandlerCollection,
|
|
32
|
+
tableName: string
|
|
33
|
+
): ProxyTableHandler {
|
|
34
|
+
const handler = collection.byTable.get(tableName);
|
|
35
|
+
if (!handler) {
|
|
36
|
+
throw new Error(`No proxy table handler for table: ${tableName}`);
|
|
37
|
+
}
|
|
38
|
+
return handler;
|
|
39
|
+
}
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
|
+
import { createDatabase } from '@syncular/core';
|
|
2
3
|
import type { Kysely } from 'kysely';
|
|
3
|
-
import {
|
|
4
|
+
import { createBunSqliteDialect } from '../../../dialect-bun-sqlite/src';
|
|
4
5
|
import { createSqliteServerDialect } from '../../../server-dialect-sqlite/src';
|
|
5
6
|
import { ensureSyncSchema } from '../migrate';
|
|
6
7
|
import type { SyncCoreDb } from '../schema';
|
|
8
|
+
import { createProxyHandlerCollection } from './collection';
|
|
7
9
|
import { executeProxyQuery } from './handler';
|
|
8
|
-
import { ProxyTableRegistry } from './registry';
|
|
9
10
|
|
|
10
11
|
interface TasksTable {
|
|
11
12
|
id: string;
|
|
@@ -21,15 +22,20 @@ interface ProxyTestDb extends SyncCoreDb {
|
|
|
21
22
|
describe('executeProxyQuery', () => {
|
|
22
23
|
let db: Kysely<ProxyTestDb>;
|
|
23
24
|
const dialect = createSqliteServerDialect();
|
|
24
|
-
const handlers =
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
25
|
+
const handlers = createProxyHandlerCollection([
|
|
26
|
+
{
|
|
27
|
+
table: 'tasks',
|
|
28
|
+
computeScopes: (row) => ({
|
|
29
|
+
user_id: String(row.user_id),
|
|
30
|
+
}),
|
|
31
|
+
},
|
|
32
|
+
]);
|
|
30
33
|
|
|
31
34
|
beforeEach(async () => {
|
|
32
|
-
db =
|
|
35
|
+
db = createDatabase<ProxyTestDb>({
|
|
36
|
+
dialect: createBunSqliteDialect({ path: ':memory:' }),
|
|
37
|
+
family: 'sqlite',
|
|
38
|
+
});
|
|
33
39
|
await ensureSyncSchema(db, dialect);
|
|
34
40
|
|
|
35
41
|
await db.schema
|
package/src/proxy/handler.ts
CHANGED
|
@@ -8,6 +8,7 @@ import type { Kysely, RawBuilder } from 'kysely';
|
|
|
8
8
|
import { sql } from 'kysely';
|
|
9
9
|
import type { ServerSyncDialect } from '../dialect/types';
|
|
10
10
|
import type { SyncCoreDb } from '../schema';
|
|
11
|
+
import { getProxyHandler, type ProxyHandlerCollection } from './collection';
|
|
11
12
|
import {
|
|
12
13
|
appendReturning,
|
|
13
14
|
detectMutation,
|
|
@@ -15,7 +16,6 @@ import {
|
|
|
15
16
|
hasReturningWildcard,
|
|
16
17
|
} from './mutation-detector';
|
|
17
18
|
import { createOplogEntries } from './oplog';
|
|
18
|
-
import type { ProxyTableRegistry } from './registry';
|
|
19
19
|
import type { ProxyQueryContext } from './types';
|
|
20
20
|
|
|
21
21
|
export interface ExecuteProxyQueryArgs<DB extends SyncCoreDb = SyncCoreDb> {
|
|
@@ -23,8 +23,8 @@ export interface ExecuteProxyQueryArgs<DB extends SyncCoreDb = SyncCoreDb> {
|
|
|
23
23
|
db: Kysely<DB>;
|
|
24
24
|
/** Server sync dialect */
|
|
25
25
|
dialect: ServerSyncDialect;
|
|
26
|
-
/** Proxy table
|
|
27
|
-
handlers:
|
|
26
|
+
/** Proxy table handlers for oplog generation */
|
|
27
|
+
handlers: ProxyHandlerCollection;
|
|
28
28
|
/** Query context (actor/client IDs) */
|
|
29
29
|
ctx: ProxyQueryContext;
|
|
30
30
|
/** SQL query string */
|
|
@@ -111,7 +111,7 @@ export async function executeProxyQuery<DB extends SyncCoreDb>(
|
|
|
111
111
|
}
|
|
112
112
|
|
|
113
113
|
// Check if this table has a registered handler
|
|
114
|
-
const handler = handlers
|
|
114
|
+
const handler = getProxyHandler(handlers, mutation.tableName);
|
|
115
115
|
if (!handler) {
|
|
116
116
|
// No handler registered - execute without oplog
|
|
117
117
|
// This allows proxy operations on non-synced tables
|
package/src/proxy/index.ts
CHANGED
|
@@ -4,6 +4,14 @@
|
|
|
4
4
|
* Server-side proxy functionality for database access.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
+
// Oplog creation
|
|
8
|
+
// Collections
|
|
9
|
+
export {
|
|
10
|
+
createProxyHandlerCollection,
|
|
11
|
+
getProxyHandler,
|
|
12
|
+
getProxyHandlerOrThrow,
|
|
13
|
+
type ProxyHandlerCollection,
|
|
14
|
+
} from './collection';
|
|
7
15
|
// Query execution
|
|
8
16
|
export {
|
|
9
17
|
type ExecuteProxyQueryArgs,
|
|
@@ -12,7 +20,4 @@ export {
|
|
|
12
20
|
} from './handler';
|
|
13
21
|
// Mutation detection
|
|
14
22
|
export { type DetectedMutation, detectMutation } from './mutation-detector';
|
|
15
|
-
// Oplog creation
|
|
16
|
-
// Registry
|
|
17
|
-
export { ProxyTableRegistry } from './registry';
|
|
18
23
|
// Types
|
package/src/proxy/oplog.ts
CHANGED
|
@@ -15,7 +15,7 @@ function toDialectJsonValue(
|
|
|
15
15
|
value: unknown
|
|
16
16
|
): unknown {
|
|
17
17
|
if (value === null || value === undefined) return null;
|
|
18
|
-
if (dialect.
|
|
18
|
+
if (dialect.family === 'sqlite') return JSON.stringify(value);
|
|
19
19
|
return value;
|
|
20
20
|
}
|
|
21
21
|
|
package/src/pull.ts
CHANGED
|
@@ -22,7 +22,12 @@ import {
|
|
|
22
22
|
} from '@syncular/core';
|
|
23
23
|
import type { Kysely } from 'kysely';
|
|
24
24
|
import type { DbExecutor, ServerSyncDialect } from './dialect/types';
|
|
25
|
-
import
|
|
25
|
+
import {
|
|
26
|
+
getServerBootstrapOrderFor,
|
|
27
|
+
getServerHandlerOrThrow,
|
|
28
|
+
type ServerHandlerCollection,
|
|
29
|
+
} from './handlers/collection';
|
|
30
|
+
import type { ServerTableHandler, SyncServerAuth } from './handlers/types';
|
|
26
31
|
import { EXTERNAL_CLIENT_ID } from './notify';
|
|
27
32
|
import type { SyncCoreDb } from './schema';
|
|
28
33
|
import {
|
|
@@ -30,8 +35,14 @@ import {
|
|
|
30
35
|
readSnapshotChunkRefByPageKey,
|
|
31
36
|
} from './snapshot-chunks';
|
|
32
37
|
import type { SnapshotChunkStorage } from './snapshot-chunks/types';
|
|
38
|
+
import {
|
|
39
|
+
createMemoryScopeCache,
|
|
40
|
+
type ScopeCacheBackend,
|
|
41
|
+
} from './subscriptions/cache';
|
|
33
42
|
import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve';
|
|
34
43
|
|
|
44
|
+
const defaultScopeCache = createMemoryScopeCache();
|
|
45
|
+
|
|
35
46
|
function concatByteChunks(chunks: readonly Uint8Array[]): Uint8Array {
|
|
36
47
|
if (chunks.length === 1) {
|
|
37
48
|
return chunks[0] ?? new Uint8Array();
|
|
@@ -62,6 +73,20 @@ export interface PullResult {
|
|
|
62
73
|
clientCursor: number;
|
|
63
74
|
}
|
|
64
75
|
|
|
76
|
+
interface PendingExternalChunkWrite {
|
|
77
|
+
snapshot: SyncSnapshot;
|
|
78
|
+
cacheLookup: {
|
|
79
|
+
partitionId: string;
|
|
80
|
+
scopeKey: string;
|
|
81
|
+
scope: string;
|
|
82
|
+
asOfCommitSeq: number;
|
|
83
|
+
rowCursor: string | null;
|
|
84
|
+
rowLimit: number;
|
|
85
|
+
};
|
|
86
|
+
rowFramePayload: Uint8Array;
|
|
87
|
+
expiresAt: string;
|
|
88
|
+
}
|
|
89
|
+
|
|
65
90
|
/**
|
|
66
91
|
* Generate a stable cache key for snapshot chunks.
|
|
67
92
|
*/
|
|
@@ -241,12 +266,14 @@ async function readExternalDataChanges<DB extends SyncCoreDb>(
|
|
|
241
266
|
}));
|
|
242
267
|
}
|
|
243
268
|
|
|
244
|
-
export async function pull<
|
|
269
|
+
export async function pull<
|
|
270
|
+
DB extends SyncCoreDb,
|
|
271
|
+
Auth extends SyncServerAuth,
|
|
272
|
+
>(args: {
|
|
245
273
|
db: Kysely<DB>;
|
|
246
274
|
dialect: ServerSyncDialect;
|
|
247
|
-
handlers:
|
|
248
|
-
|
|
249
|
-
partitionId?: string;
|
|
275
|
+
handlers: ServerHandlerCollection<DB, Auth>;
|
|
276
|
+
auth: Auth;
|
|
250
277
|
request: SyncPullRequest;
|
|
251
278
|
/**
|
|
252
279
|
* Optional snapshot chunk storage adapter.
|
|
@@ -254,10 +281,16 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
254
281
|
* instead of inline in the database.
|
|
255
282
|
*/
|
|
256
283
|
chunkStorage?: SnapshotChunkStorage;
|
|
284
|
+
/**
|
|
285
|
+
* Optional shared scope cache backend.
|
|
286
|
+
* Request-local memoization is always applied, even with custom backends.
|
|
287
|
+
* Defaults to process-local memory cache.
|
|
288
|
+
*/
|
|
289
|
+
scopeCache?: ScopeCacheBackend;
|
|
257
290
|
}): Promise<PullResult> {
|
|
258
291
|
const { request, dialect } = args;
|
|
259
292
|
const db = args.db;
|
|
260
|
-
const partitionId = args.partitionId ?? 'default';
|
|
293
|
+
const partitionId = args.auth.partitionId ?? 'default';
|
|
261
294
|
const requestedSubscriptionCount = Array.isArray(request.subscriptions)
|
|
262
295
|
? request.subscriptions.length
|
|
263
296
|
: 0;
|
|
@@ -289,13 +322,15 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
289
322
|
50
|
|
290
323
|
);
|
|
291
324
|
const dedupeRows = request.dedupeRows === true;
|
|
325
|
+
const pendingExternalChunkWrites: PendingExternalChunkWrite[] = [];
|
|
292
326
|
|
|
293
327
|
// Resolve effective scopes for each subscription
|
|
294
328
|
const resolved = await resolveEffectiveScopesForSubscriptions({
|
|
295
329
|
db,
|
|
296
|
-
|
|
330
|
+
auth: args.auth,
|
|
297
331
|
subscriptions: request.subscriptions ?? [],
|
|
298
332
|
handlers: args.handlers,
|
|
333
|
+
scopeCache: args.scopeCache ?? defaultScopeCache,
|
|
299
334
|
});
|
|
300
335
|
|
|
301
336
|
const result = await dialect.executeInTransaction(db, async (trx) => {
|
|
@@ -347,7 +382,7 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
347
382
|
for (const sub of resolved) {
|
|
348
383
|
const cursor = Math.max(-1, sub.cursor ?? -1);
|
|
349
384
|
// Validate table handler exists (throws if not registered)
|
|
350
|
-
args.handlers
|
|
385
|
+
getServerHandlerOrThrow(args.handlers, sub.table);
|
|
351
386
|
|
|
352
387
|
if (
|
|
353
388
|
sub.status === 'revoked' ||
|
|
@@ -379,9 +414,10 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
379
414
|
latestExternalCommitForTable > cursor);
|
|
380
415
|
|
|
381
416
|
if (needsBootstrap) {
|
|
382
|
-
const tables =
|
|
383
|
-
.
|
|
384
|
-
.
|
|
417
|
+
const tables = getServerBootstrapOrderFor(
|
|
418
|
+
args.handlers,
|
|
419
|
+
sub.table
|
|
420
|
+
).map((handler) => handler.table);
|
|
385
421
|
|
|
386
422
|
const initState: SyncBootstrapState = {
|
|
387
423
|
asOfCommitSeq: maxCommitSeq,
|
|
@@ -462,63 +498,51 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
462
498
|
const rowFramePayload = concatByteChunks(
|
|
463
499
|
bundle.rowFrameParts
|
|
464
500
|
);
|
|
465
|
-
const sha256 = await sha256Hex(rowFramePayload);
|
|
466
501
|
const expiresAt = new Date(
|
|
467
502
|
Date.now() + Math.max(1000, bundle.ttlMs)
|
|
468
503
|
).toISOString();
|
|
469
504
|
|
|
470
505
|
if (args.chunkStorage) {
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
506
|
+
const snapshot: SyncSnapshot = {
|
|
507
|
+
table: bundle.table,
|
|
508
|
+
rows: [],
|
|
509
|
+
chunks: [],
|
|
510
|
+
isFirstPage: bundle.isFirstPage,
|
|
511
|
+
isLastPage: bundle.isLastPage,
|
|
512
|
+
};
|
|
513
|
+
snapshots.push(snapshot);
|
|
514
|
+
pendingExternalChunkWrites.push({
|
|
515
|
+
snapshot,
|
|
516
|
+
cacheLookup: {
|
|
475
517
|
partitionId,
|
|
476
518
|
scopeKey: cacheKey,
|
|
477
519
|
scope: bundle.table,
|
|
478
520
|
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
479
521
|
rowCursor: bundle.startCursor,
|
|
480
522
|
rowLimit: bundleRowLimit,
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
sha256,
|
|
484
|
-
byteLength,
|
|
485
|
-
bodyStream,
|
|
486
|
-
expiresAt,
|
|
487
|
-
});
|
|
488
|
-
} else {
|
|
489
|
-
const compressedBody = await gzipBytes(rowFramePayload);
|
|
490
|
-
chunkRef = await args.chunkStorage.storeChunk({
|
|
491
|
-
partitionId,
|
|
492
|
-
scopeKey: cacheKey,
|
|
493
|
-
scope: bundle.table,
|
|
494
|
-
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
495
|
-
rowCursor: bundle.startCursor,
|
|
496
|
-
rowLimit: bundleRowLimit,
|
|
497
|
-
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
498
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
499
|
-
sha256,
|
|
500
|
-
body: compressedBody,
|
|
501
|
-
expiresAt,
|
|
502
|
-
});
|
|
503
|
-
}
|
|
504
|
-
} else {
|
|
505
|
-
const compressedBody = await gzipBytes(rowFramePayload);
|
|
506
|
-
const chunkId = randomId();
|
|
507
|
-
chunkRef = await insertSnapshotChunk(trx, {
|
|
508
|
-
chunkId,
|
|
509
|
-
partitionId,
|
|
510
|
-
scopeKey: cacheKey,
|
|
511
|
-
scope: bundle.table,
|
|
512
|
-
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
513
|
-
rowCursor: bundle.startCursor,
|
|
514
|
-
rowLimit: bundleRowLimit,
|
|
515
|
-
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
516
|
-
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
517
|
-
sha256,
|
|
518
|
-
body: compressedBody,
|
|
523
|
+
},
|
|
524
|
+
rowFramePayload,
|
|
519
525
|
expiresAt,
|
|
520
526
|
});
|
|
527
|
+
return;
|
|
521
528
|
}
|
|
529
|
+
const sha256 = await sha256Hex(rowFramePayload);
|
|
530
|
+
const compressedBody = await gzipBytes(rowFramePayload);
|
|
531
|
+
const chunkId = randomId();
|
|
532
|
+
chunkRef = await insertSnapshotChunk(trx, {
|
|
533
|
+
chunkId,
|
|
534
|
+
partitionId,
|
|
535
|
+
scopeKey: cacheKey,
|
|
536
|
+
scope: bundle.table,
|
|
537
|
+
asOfCommitSeq: effectiveState.asOfCommitSeq,
|
|
538
|
+
rowCursor: bundle.startCursor,
|
|
539
|
+
rowLimit: bundleRowLimit,
|
|
540
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
541
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
542
|
+
sha256,
|
|
543
|
+
body: compressedBody,
|
|
544
|
+
expiresAt,
|
|
545
|
+
});
|
|
522
546
|
}
|
|
523
547
|
|
|
524
548
|
snapshots.push({
|
|
@@ -539,7 +563,8 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
539
563
|
) {
|
|
540
564
|
if (!nextState) break;
|
|
541
565
|
|
|
542
|
-
const nextTableName =
|
|
566
|
+
const nextTableName: string | undefined =
|
|
567
|
+
nextState.tables[nextState.tableIndex];
|
|
543
568
|
if (!nextTableName) {
|
|
544
569
|
if (activeBundle) {
|
|
545
570
|
activeBundle.isLastPage = true;
|
|
@@ -550,7 +575,8 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
550
575
|
break;
|
|
551
576
|
}
|
|
552
577
|
|
|
553
|
-
const tableHandler =
|
|
578
|
+
const tableHandler: ServerTableHandler<DB, Auth> =
|
|
579
|
+
getServerHandlerOrThrow(args.handlers, nextTableName);
|
|
554
580
|
if (!activeBundle || activeBundle.table !== nextTableName) {
|
|
555
581
|
if (activeBundle) {
|
|
556
582
|
await flushSnapshotBundle(activeBundle);
|
|
@@ -568,16 +594,18 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
568
594
|
};
|
|
569
595
|
}
|
|
570
596
|
|
|
571
|
-
const page
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
597
|
+
const page: { rows: unknown[]; nextCursor: string | null } =
|
|
598
|
+
await tableHandler.snapshot(
|
|
599
|
+
{
|
|
600
|
+
db: trx,
|
|
601
|
+
actorId: args.auth.actorId,
|
|
602
|
+
auth: args.auth,
|
|
603
|
+
scopeValues: effectiveScopes,
|
|
604
|
+
cursor: nextState.rowCursor,
|
|
605
|
+
limit: limitSnapshotRows,
|
|
606
|
+
},
|
|
607
|
+
sub.params
|
|
608
|
+
);
|
|
581
609
|
|
|
582
610
|
const rowFrames = encodeSnapshotRowFrames(page.rows ?? []);
|
|
583
611
|
activeBundle.rowFrameParts.push(rowFrames);
|
|
@@ -659,7 +687,6 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
659
687
|
commitSeq: number;
|
|
660
688
|
createdAt: string;
|
|
661
689
|
actorId: string;
|
|
662
|
-
changeId: number;
|
|
663
690
|
change: SyncChange;
|
|
664
691
|
}
|
|
665
692
|
>();
|
|
@@ -682,11 +709,15 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
682
709
|
scopes: r.scopes,
|
|
683
710
|
};
|
|
684
711
|
|
|
712
|
+
// Move row keys to insertion tail so Map iteration yields
|
|
713
|
+
// "latest change wins" order without a full array sort.
|
|
714
|
+
if (latestByRowKey.has(rowKey)) {
|
|
715
|
+
latestByRowKey.delete(rowKey);
|
|
716
|
+
}
|
|
685
717
|
latestByRowKey.set(rowKey, {
|
|
686
718
|
commitSeq: r.commit_seq,
|
|
687
719
|
createdAt: r.created_at,
|
|
688
720
|
actorId: r.actor_id,
|
|
689
|
-
changeId: r.change_id,
|
|
690
721
|
change,
|
|
691
722
|
});
|
|
692
723
|
}
|
|
@@ -706,12 +737,8 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
706
737
|
continue;
|
|
707
738
|
}
|
|
708
739
|
|
|
709
|
-
const latest = Array.from(latestByRowKey.values()).sort(
|
|
710
|
-
(a, b) => a.commitSeq - b.commitSeq || a.changeId - b.changeId
|
|
711
|
-
);
|
|
712
|
-
|
|
713
740
|
const commits: SyncCommit[] = [];
|
|
714
|
-
for (const item of
|
|
741
|
+
for (const item of latestByRowKey.values()) {
|
|
715
742
|
const lastCommit = commits[commits.length - 1];
|
|
716
743
|
if (!lastCommit || lastCommit.commitSeq !== item.commitSeq) {
|
|
717
744
|
commits.push({
|
|
@@ -817,6 +844,61 @@ export async function pull<DB extends SyncCoreDb>(args: {
|
|
|
817
844
|
};
|
|
818
845
|
});
|
|
819
846
|
|
|
847
|
+
const chunkStorage = args.chunkStorage;
|
|
848
|
+
if (chunkStorage && pendingExternalChunkWrites.length > 0) {
|
|
849
|
+
for (const pending of pendingExternalChunkWrites) {
|
|
850
|
+
let chunkRef = await readSnapshotChunkRefByPageKey(db, {
|
|
851
|
+
partitionId: pending.cacheLookup.partitionId,
|
|
852
|
+
scopeKey: pending.cacheLookup.scopeKey,
|
|
853
|
+
scope: pending.cacheLookup.scope,
|
|
854
|
+
asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
|
|
855
|
+
rowCursor: pending.cacheLookup.rowCursor,
|
|
856
|
+
rowLimit: pending.cacheLookup.rowLimit,
|
|
857
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
858
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
859
|
+
});
|
|
860
|
+
|
|
861
|
+
if (!chunkRef) {
|
|
862
|
+
const sha256 = await sha256Hex(pending.rowFramePayload);
|
|
863
|
+
if (chunkStorage.storeChunkStream) {
|
|
864
|
+
const { stream: bodyStream, byteLength } =
|
|
865
|
+
await gzipBytesToStream(pending.rowFramePayload);
|
|
866
|
+
chunkRef = await chunkStorage.storeChunkStream({
|
|
867
|
+
partitionId: pending.cacheLookup.partitionId,
|
|
868
|
+
scopeKey: pending.cacheLookup.scopeKey,
|
|
869
|
+
scope: pending.cacheLookup.scope,
|
|
870
|
+
asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
|
|
871
|
+
rowCursor: pending.cacheLookup.rowCursor,
|
|
872
|
+
rowLimit: pending.cacheLookup.rowLimit,
|
|
873
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
874
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
875
|
+
sha256,
|
|
876
|
+
byteLength,
|
|
877
|
+
bodyStream,
|
|
878
|
+
expiresAt: pending.expiresAt,
|
|
879
|
+
});
|
|
880
|
+
} else {
|
|
881
|
+
const compressedBody = await gzipBytes(pending.rowFramePayload);
|
|
882
|
+
chunkRef = await chunkStorage.storeChunk({
|
|
883
|
+
partitionId: pending.cacheLookup.partitionId,
|
|
884
|
+
scopeKey: pending.cacheLookup.scopeKey,
|
|
885
|
+
scope: pending.cacheLookup.scope,
|
|
886
|
+
asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
|
|
887
|
+
rowCursor: pending.cacheLookup.rowCursor,
|
|
888
|
+
rowLimit: pending.cacheLookup.rowLimit,
|
|
889
|
+
encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
|
|
890
|
+
compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
|
|
891
|
+
sha256,
|
|
892
|
+
body: compressedBody,
|
|
893
|
+
expiresAt: pending.expiresAt,
|
|
894
|
+
});
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
pending.snapshot.chunks = [chunkRef];
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
|
|
820
902
|
const durationMs = Math.max(0, Date.now() - startedAtMs);
|
|
821
903
|
const stats = summarizePullResponse(result.response);
|
|
822
904
|
|