@syncular/server 0.48.0 → 0.49.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/errors.js +6 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/local-authority.d.ts +78 -0
- package/dist/local-authority.js +470 -0
- package/package.json +2 -2
- package/src/errors.ts +6 -0
- package/src/index.ts +1 -0
- package/src/local-authority.ts +654 -0
package/README.md
CHANGED
|
@@ -322,6 +322,18 @@ and the ID check does not authenticate the client. Within one partition the ID
|
|
|
322
322
|
contract is a correctness property: it stops a replayed ID from applying
|
|
323
323
|
different operations.
|
|
324
324
|
|
|
325
|
+
## Local authority
|
|
326
|
+
|
|
327
|
+
`openLocalAuthority({ database, schema })` runs this server inside a client's
|
|
328
|
+
process over its own SQLite database (SPEC §12). Its `sync` and
|
|
329
|
+
`downloadSegment` functions are the client's `transport` and `segments`.
|
|
330
|
+
`attach(transport)` uploads every live row to a remote server as inserts under
|
|
331
|
+
stable commit IDs and then retires the authority (§12.3);
|
|
332
|
+
`abandonAttach(transport)` deletes what a stopped attach uploaded. In a browser
|
|
333
|
+
worker, `clientSqliteDatabase` adapts the worker's sibling database. The
|
|
334
|
+
[local authority guide](https://syncular.dev/guide-local-authority/) covers the
|
|
335
|
+
client setup and the attach sequence.
|
|
336
|
+
|
|
325
337
|
## Seed idempotency and safe revisioning
|
|
326
338
|
|
|
327
339
|
`seedMutations` uses the real push path and a stable `clientId`/`commitId`.
|
package/dist/errors.js
CHANGED
|
@@ -186,6 +186,12 @@ export const ERROR_CATALOG = {
|
|
|
186
186
|
recommendedAction: 'inspectServer',
|
|
187
187
|
httpStatus: 503,
|
|
188
188
|
},
|
|
189
|
+
'sync.authority_unavailable': {
|
|
190
|
+
category: 'internal',
|
|
191
|
+
retryable: false,
|
|
192
|
+
recommendedAction: 'inspectServer',
|
|
193
|
+
httpStatus: 503,
|
|
194
|
+
},
|
|
189
195
|
'sync.schema_not_ready': {
|
|
190
196
|
category: 'internal',
|
|
191
197
|
retryable: true,
|
package/dist/index.d.ts
CHANGED
|
@@ -24,6 +24,7 @@ export * from './events-ring.js';
|
|
|
24
24
|
export * from './frame-bytes.js';
|
|
25
25
|
export * from './handler.js';
|
|
26
26
|
export * from './lease-store.js';
|
|
27
|
+
export * from './local-authority.js';
|
|
27
28
|
export * from './operations.js';
|
|
28
29
|
export * from './operations-realtime.js';
|
|
29
30
|
export * from './pg-executor.js';
|
package/dist/index.js
CHANGED
|
@@ -24,6 +24,7 @@ export * from './events-ring.js';
|
|
|
24
24
|
export * from './frame-bytes.js';
|
|
25
25
|
export * from './handler.js';
|
|
26
26
|
export * from './lease-store.js';
|
|
27
|
+
export * from './local-authority.js';
|
|
27
28
|
export * from './operations.js';
|
|
28
29
|
export * from './operations-realtime.js';
|
|
29
30
|
// The `PgExecutor` seam + Postgres storage/fanout are driver-agnostic (zero
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { CrdtMergerRegistry } from './crdt-merger.js';
|
|
2
|
+
import { type ServerSchema } from './schema.js';
|
|
3
|
+
import type { SqliteDatabase, SqliteValue } from './sqlite-driver.js';
|
|
4
|
+
import type { SqliteImageBuilder } from './sqlite-image.js';
|
|
5
|
+
export type LocalAuthorityState = 'local' | 'attaching' | 'attached';
|
|
6
|
+
export interface LocalAuthorityOptions {
|
|
7
|
+
/** The authority's own database. Never the client replica (§12.1). */
|
|
8
|
+
readonly database: SqliteDatabase;
|
|
9
|
+
readonly schema: ServerSchema;
|
|
10
|
+
/** §5.10 mergers for `crdt` columns, as a remote server would take them. */
|
|
11
|
+
readonly crdtMergers?: CrdtMergerRegistry;
|
|
12
|
+
/** §5.3 image builder from `@syncular/server/sqlite`; absent ⇒ rows segments. */
|
|
13
|
+
readonly sqliteImageBuilder?: SqliteImageBuilder;
|
|
14
|
+
readonly clock?: () => number;
|
|
15
|
+
}
|
|
16
|
+
export interface LocalAuthority {
|
|
17
|
+
/** Current §12.3 state, persisted in the authority database. */
|
|
18
|
+
readonly state: LocalAuthorityState;
|
|
19
|
+
/** The client's sync transport: one combined push+pull round (§1.5). */
|
|
20
|
+
readonly sync: (request: Uint8Array) => Promise<Uint8Array>;
|
|
21
|
+
/** The client's segment downloader (§5.5). */
|
|
22
|
+
readonly downloadSegment: (request: {
|
|
23
|
+
readonly segmentId: string;
|
|
24
|
+
readonly requestedScopesJson: string;
|
|
25
|
+
}) => Promise<Uint8Array>;
|
|
26
|
+
/**
|
|
27
|
+
* §12.3: upload every live row to the remote server behind `transport`
|
|
28
|
+
* (an authenticated `POST <mount>/sync` round trip). Resolves once every
|
|
29
|
+
* commit is applied; a second call after a failure resumes the same plan.
|
|
30
|
+
*/
|
|
31
|
+
readonly attach: (transport: (request: Uint8Array) => Promise<Uint8Array>) => Promise<void>;
|
|
32
|
+
/**
|
|
33
|
+
* §12.3: delete the rows the remote server applied from this attach, then
|
|
34
|
+
* serve the client again. `transport` reaches the same remote server and is
|
|
35
|
+
* required once an upload request may have reached it.
|
|
36
|
+
*/
|
|
37
|
+
readonly abandonAttach: (transport?: (request: Uint8Array) => Promise<Uint8Array>) => Promise<void>;
|
|
38
|
+
}
|
|
39
|
+
export type LocalAuthorityAttachDetails = {
|
|
40
|
+
readonly reason: 'commit_rejected';
|
|
41
|
+
readonly clientCommitId: string;
|
|
42
|
+
readonly opIndex: number;
|
|
43
|
+
readonly code: string;
|
|
44
|
+
} | {
|
|
45
|
+
readonly reason: 'cycle_too_large';
|
|
46
|
+
readonly table: string;
|
|
47
|
+
readonly size: number;
|
|
48
|
+
} | {
|
|
49
|
+
readonly reason: 'request_failed';
|
|
50
|
+
readonly code: string;
|
|
51
|
+
readonly retryable: boolean;
|
|
52
|
+
} | {
|
|
53
|
+
readonly reason: 'log_epoch_changed';
|
|
54
|
+
} | {
|
|
55
|
+
readonly reason: 'schema_floor';
|
|
56
|
+
readonly requiredSchemaVersion: number;
|
|
57
|
+
readonly latestSchemaVersion?: number;
|
|
58
|
+
};
|
|
59
|
+
/** §10.3 client-local `sync.attach_rejected`: the attach upload stopped. */
|
|
60
|
+
export declare class LocalAuthorityAttachError extends Error {
|
|
61
|
+
readonly name = "LocalAuthorityAttachError";
|
|
62
|
+
readonly code = "sync.attach_rejected";
|
|
63
|
+
readonly details: LocalAuthorityAttachDetails;
|
|
64
|
+
constructor(details: LocalAuthorityAttachDetails);
|
|
65
|
+
}
|
|
66
|
+
/** The `ClientDatabase` surface of `@syncular/client`, as a browser worker opens it. */
|
|
67
|
+
export interface LocalAuthorityClientDatabase {
|
|
68
|
+
exec(sql: string, params?: readonly SqliteValue[]): void;
|
|
69
|
+
query(sql: string, params?: readonly SqliteValue[]): Record<string, SqliteValue>[];
|
|
70
|
+
close(): void;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Serve a local authority from a client-side database, such as the sibling
|
|
74
|
+
* file a browser worker opens beside its replica. `exec` without parameters
|
|
75
|
+
* must run several statements, as the sqlite-wasm database does.
|
|
76
|
+
*/
|
|
77
|
+
export declare function clientSqliteDatabase(database: LocalAuthorityClientDatabase): SqliteDatabase;
|
|
78
|
+
export declare function openLocalAuthority(options: LocalAuthorityOptions): Promise<LocalAuthority>;
|
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local authority (SPEC.md §12): the reference server running in the client's
|
|
3
|
+
* process over its own SQLite database. An app syncs its client against it
|
|
4
|
+
* with no remote server, and later moves the authority's rows to a remote
|
|
5
|
+
* server with `attach` (§12.3).
|
|
6
|
+
*/
|
|
7
|
+
import { decodeMessage, decodeRow, encodeMessage, encodeSparseRow, PROTOCOL_WIRE_VERSION, } from '@syncular/core';
|
|
8
|
+
import { SyncError } from './errors.js';
|
|
9
|
+
import { handleSyncRequest } from './handler.js';
|
|
10
|
+
import { pruneCommitLog } from './prune.js';
|
|
11
|
+
import { compileSchema, } from './schema.js';
|
|
12
|
+
import { renderScopeValue } from './scopes.js';
|
|
13
|
+
import { handleSegmentDownload } from './segment-download.js';
|
|
14
|
+
import { MemorySegmentStore } from './segment-store.js';
|
|
15
|
+
import { SqliteServerStorage } from './sqlite-storage.js';
|
|
16
|
+
/** The one partition and actor a local authority serves (§12.2). */
|
|
17
|
+
const LOCAL = 'local';
|
|
18
|
+
/** Operations per attach commit: the reference server's request cap (§6.1). */
|
|
19
|
+
const ATTACH_OPERATION_BUDGET = 500;
|
|
20
|
+
/** §10.3 client-local `sync.attach_rejected`: the attach upload stopped. */
|
|
21
|
+
export class LocalAuthorityAttachError extends Error {
|
|
22
|
+
name = 'LocalAuthorityAttachError';
|
|
23
|
+
code = 'sync.attach_rejected';
|
|
24
|
+
details;
|
|
25
|
+
constructor(details) {
|
|
26
|
+
super('the remote server did not accept the attach upload');
|
|
27
|
+
this.details = details;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Serve a local authority from a client-side database, such as the sibling
|
|
32
|
+
* file a browser worker opens beside its replica. `exec` without parameters
|
|
33
|
+
* must run several statements, as the sqlite-wasm database does.
|
|
34
|
+
*/
|
|
35
|
+
export function clientSqliteDatabase(database) {
|
|
36
|
+
const run = (sql, params) => {
|
|
37
|
+
database.exec(sql, params);
|
|
38
|
+
const result = database.query('SELECT changes() AS changes, last_insert_rowid() AS id')[0];
|
|
39
|
+
return {
|
|
40
|
+
changes: Number(result?.changes ?? 0),
|
|
41
|
+
lastInsertRowid: Number(result?.id ?? 0),
|
|
42
|
+
};
|
|
43
|
+
};
|
|
44
|
+
return {
|
|
45
|
+
exec: (sql) => database.exec(sql),
|
|
46
|
+
run: (sql, bindings = []) => run(sql, bindings),
|
|
47
|
+
query: (sql) => ({
|
|
48
|
+
run: (...params) => run(sql, params),
|
|
49
|
+
get: (...params) => (database.query(sql, params)[0] ?? null),
|
|
50
|
+
all: (...params) => database.query(sql, params),
|
|
51
|
+
}),
|
|
52
|
+
close: () => database.close(),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
export async function openLocalAuthority(options) {
|
|
56
|
+
const { database, schema } = options;
|
|
57
|
+
const clock = options.clock ?? Date.now;
|
|
58
|
+
database.exec('CREATE TABLE IF NOT EXISTS sync_local_authority(key TEXT PRIMARY KEY, value TEXT NOT NULL)');
|
|
59
|
+
const readMeta = (key) => database
|
|
60
|
+
.query('SELECT value FROM sync_local_authority WHERE key=?')
|
|
61
|
+
.get(key)?.value;
|
|
62
|
+
const stored = readMeta('state') ?? 'local';
|
|
63
|
+
if (stored !== 'local' && stored !== 'attaching' && stored !== 'attached') {
|
|
64
|
+
throw new SyncError('sync.internal_error', 'the local authority state is unreadable', JSON.stringify({ state: stored }));
|
|
65
|
+
}
|
|
66
|
+
let state = stored;
|
|
67
|
+
const compiled = compileSchema(schema);
|
|
68
|
+
// §12.3: an attached authority keeps only its state across reopens, so it
|
|
69
|
+
// opens no server storage; the storage constructor creates the server tables.
|
|
70
|
+
const storage = stored === 'attached' ? undefined : new SqliteServerStorage(database);
|
|
71
|
+
if (storage !== undefined) {
|
|
72
|
+
// §12.2: the authority reaches the running schema before it serves.
|
|
73
|
+
await storage.ensureSchema(compiled);
|
|
74
|
+
if ((await storage.getPartitionLogEpoch(LOCAL)) !== undefined) {
|
|
75
|
+
await pruneCommitLog({ storage, partition: LOCAL, nowMs: clock() });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const allowed = Object.fromEntries([...compiled.declaredVariables].map((variable) => [variable, ['*']]));
|
|
79
|
+
const ctx = storage === undefined
|
|
80
|
+
? undefined
|
|
81
|
+
: {
|
|
82
|
+
partition: LOCAL,
|
|
83
|
+
actorId: LOCAL,
|
|
84
|
+
schema,
|
|
85
|
+
storage,
|
|
86
|
+
segments: new MemorySegmentStore(),
|
|
87
|
+
resolveScopes: () => allowed,
|
|
88
|
+
clock,
|
|
89
|
+
...(options.crdtMergers !== undefined
|
|
90
|
+
? { crdtMergers: options.crdtMergers }
|
|
91
|
+
: {}),
|
|
92
|
+
...(options.sqliteImageBuilder !== undefined
|
|
93
|
+
? { sqliteImageBuilder: options.sqliteImageBuilder }
|
|
94
|
+
: {}),
|
|
95
|
+
};
|
|
96
|
+
let uploading = false;
|
|
97
|
+
const refuseDuringUpload = () => {
|
|
98
|
+
if (uploading) {
|
|
99
|
+
throw new SyncError('sync.invalid_request', 'an attach upload is already running');
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
const inflight = new Set();
|
|
103
|
+
const admit = (work) => {
|
|
104
|
+
if (state !== 'local' || ctx === undefined) {
|
|
105
|
+
return Promise.reject(new SyncError('sync.authority_unavailable', 'the local authority serves no requests', JSON.stringify({ state })));
|
|
106
|
+
}
|
|
107
|
+
const promise = work(ctx);
|
|
108
|
+
const settle = () => inflight.delete(promise);
|
|
109
|
+
inflight.add(promise);
|
|
110
|
+
promise.then(settle, settle);
|
|
111
|
+
return promise;
|
|
112
|
+
};
|
|
113
|
+
/** One transaction; a `null` value removes the key. */
|
|
114
|
+
const writeMeta = (entries) => {
|
|
115
|
+
database.exec('BEGIN IMMEDIATE');
|
|
116
|
+
try {
|
|
117
|
+
for (const [key, value] of Object.entries(entries)) {
|
|
118
|
+
if (value === null) {
|
|
119
|
+
database.run('DELETE FROM sync_local_authority WHERE key=?', [key]);
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
database.run('INSERT OR REPLACE INTO sync_local_authority(key,value) VALUES (?,?)', [key, value]);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
database.exec('COMMIT');
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
database.exec('ROLLBACK');
|
|
129
|
+
throw error;
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
const ordinal = (key) => Number(readMeta(key) ?? -1);
|
|
133
|
+
const attachIdOf = () => {
|
|
134
|
+
const attachId = readMeta('attach_id');
|
|
135
|
+
if (attachId === undefined) {
|
|
136
|
+
throw new SyncError('sync.internal_error', 'the attaching local authority has no attach id');
|
|
137
|
+
}
|
|
138
|
+
return attachId;
|
|
139
|
+
};
|
|
140
|
+
const exchange = async (transport, frames) => {
|
|
141
|
+
const message = decodeMessage(await transport(encodeMessage({
|
|
142
|
+
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
143
|
+
msgKind: 'request',
|
|
144
|
+
frames,
|
|
145
|
+
})));
|
|
146
|
+
if (message.msgKind !== 'response') {
|
|
147
|
+
throw new LocalAuthorityAttachError({
|
|
148
|
+
reason: 'request_failed',
|
|
149
|
+
code: 'sync.invalid_request',
|
|
150
|
+
retryable: false,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
const header = message.frames[0];
|
|
154
|
+
if (header?.type === 'RESP_HEADER' && header.requiredSchemaVersion) {
|
|
155
|
+
throw new LocalAuthorityAttachError({
|
|
156
|
+
reason: 'schema_floor',
|
|
157
|
+
requiredSchemaVersion: header.requiredSchemaVersion,
|
|
158
|
+
...(header.latestSchemaVersion !== undefined
|
|
159
|
+
? { latestSchemaVersion: header.latestSchemaVersion }
|
|
160
|
+
: {}),
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
const error = message.frames.find((frame) => frame.type === 'ERROR');
|
|
164
|
+
if (error !== undefined) {
|
|
165
|
+
throw new LocalAuthorityAttachError({
|
|
166
|
+
reason: 'request_failed',
|
|
167
|
+
code: error.code,
|
|
168
|
+
retryable: error.retryable,
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
return message.frames;
|
|
172
|
+
};
|
|
173
|
+
/** Acquire the remote `logEpoch` (§2.1), then send one commit per request. */
|
|
174
|
+
const connect = async (transport, clientId) => {
|
|
175
|
+
const acquired = (await exchange(transport, [
|
|
176
|
+
{ type: 'REQ_HEADER', clientId, schemaVersion: schema.version },
|
|
177
|
+
{
|
|
178
|
+
type: 'PULL_HEADER',
|
|
179
|
+
limitCommits: 0,
|
|
180
|
+
limitSnapshotRows: 0,
|
|
181
|
+
maxSnapshotPages: 0,
|
|
182
|
+
accept: 0b0011,
|
|
183
|
+
},
|
|
184
|
+
]))[0];
|
|
185
|
+
const logEpoch = acquired?.type === 'RESP_HEADER' ? acquired.logEpoch : undefined;
|
|
186
|
+
if (logEpoch === undefined) {
|
|
187
|
+
throw new LocalAuthorityAttachError({
|
|
188
|
+
reason: 'request_failed',
|
|
189
|
+
code: 'sync.invalid_request',
|
|
190
|
+
retryable: false,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
return async (commit) => {
|
|
194
|
+
const frames = await exchange(transport, [
|
|
195
|
+
{
|
|
196
|
+
type: 'REQ_HEADER',
|
|
197
|
+
clientId,
|
|
198
|
+
schemaVersion: schema.version,
|
|
199
|
+
logEpoch,
|
|
200
|
+
},
|
|
201
|
+
commit,
|
|
202
|
+
]);
|
|
203
|
+
const header = frames[0];
|
|
204
|
+
if (header?.type === 'RESP_HEADER' && header.resetRequired === true) {
|
|
205
|
+
throw new LocalAuthorityAttachError({ reason: 'log_epoch_changed' });
|
|
206
|
+
}
|
|
207
|
+
const result = frames.find((frame) => frame.type === 'PUSH_RESULT');
|
|
208
|
+
if (result === undefined) {
|
|
209
|
+
throw new LocalAuthorityAttachError({
|
|
210
|
+
reason: 'request_failed',
|
|
211
|
+
code: 'sync.invalid_request',
|
|
212
|
+
retryable: false,
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
return result;
|
|
216
|
+
};
|
|
217
|
+
};
|
|
218
|
+
const applied = (result) => result.status === 'applied' || result.status === 'cached';
|
|
219
|
+
const rejected = (commit, result) => {
|
|
220
|
+
const failed = result.results.find((r) => r.status !== 'applied');
|
|
221
|
+
return new LocalAuthorityAttachError({
|
|
222
|
+
reason: 'commit_rejected',
|
|
223
|
+
clientCommitId: commit.clientCommitId,
|
|
224
|
+
opIndex: failed?.opIndex ?? 0,
|
|
225
|
+
code: failed?.code ?? 'sync.invalid_request',
|
|
226
|
+
});
|
|
227
|
+
};
|
|
228
|
+
const authority = {
|
|
229
|
+
get state() {
|
|
230
|
+
return state;
|
|
231
|
+
},
|
|
232
|
+
sync: (request) => admit((ctx) => handleSyncRequest(request, ctx)),
|
|
233
|
+
downloadSegment: (request) => admit(async (ctx) => {
|
|
234
|
+
const result = await handleSegmentDownload(ctx, {
|
|
235
|
+
segmentId: request.segmentId,
|
|
236
|
+
scopesHeader: request.requestedScopesJson,
|
|
237
|
+
});
|
|
238
|
+
return result.bytes;
|
|
239
|
+
}),
|
|
240
|
+
async attach(transport) {
|
|
241
|
+
refuseDuringUpload();
|
|
242
|
+
if (state === 'attached')
|
|
243
|
+
return;
|
|
244
|
+
uploading = true;
|
|
245
|
+
try {
|
|
246
|
+
await upload(transport);
|
|
247
|
+
}
|
|
248
|
+
finally {
|
|
249
|
+
uploading = false;
|
|
250
|
+
}
|
|
251
|
+
},
|
|
252
|
+
async abandonAttach(transport) {
|
|
253
|
+
refuseDuringUpload();
|
|
254
|
+
if (state === 'attached') {
|
|
255
|
+
throw new SyncError('sync.authority_unavailable', 'the local authority serves no requests', JSON.stringify({ state }));
|
|
256
|
+
}
|
|
257
|
+
if (state === 'local')
|
|
258
|
+
return;
|
|
259
|
+
if (ordinal('sent') >= 0) {
|
|
260
|
+
if (transport === undefined) {
|
|
261
|
+
throw new SyncError('sync.invalid_request', 'abandoning an attach that reached the remote server needs its transport');
|
|
262
|
+
}
|
|
263
|
+
uploading = true;
|
|
264
|
+
try {
|
|
265
|
+
await rollback(transport);
|
|
266
|
+
}
|
|
267
|
+
finally {
|
|
268
|
+
uploading = false;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
writeMeta({ state: 'local', attach_id: null, sent: null, applied: null });
|
|
272
|
+
state = 'local';
|
|
273
|
+
},
|
|
274
|
+
};
|
|
275
|
+
return authority;
|
|
276
|
+
async function upload(transport) {
|
|
277
|
+
if (state === 'local') {
|
|
278
|
+
// Refuse new rounds first, then let admitted rounds finish: the plan
|
|
279
|
+
// reads the authority database on the connection a round writes on.
|
|
280
|
+
state = 'attaching';
|
|
281
|
+
try {
|
|
282
|
+
await Promise.allSettled([...inflight]);
|
|
283
|
+
writeMeta({ state: 'attaching', attach_id: crypto.randomUUID() });
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
state = 'local';
|
|
287
|
+
throw error;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
const attachId = attachIdOf();
|
|
291
|
+
const commits = planAttach(database, compiled, attachId);
|
|
292
|
+
const send = await connect(transport, `authority-${attachId}`);
|
|
293
|
+
for (const [index, commit] of commits.entries()) {
|
|
294
|
+
// §12.3: `sent` before the request, `applied` after its result, so an
|
|
295
|
+
// abandon knows which commits may have reached the remote server.
|
|
296
|
+
writeMeta({ sent: String(Math.max(index, ordinal('sent'))) });
|
|
297
|
+
const result = await send(commit);
|
|
298
|
+
if (!applied(result))
|
|
299
|
+
throw rejected(commit, result);
|
|
300
|
+
writeMeta({ applied: String(Math.max(index, ordinal('applied'))) });
|
|
301
|
+
}
|
|
302
|
+
writeMeta({ state: 'attached', sent: null, applied: null });
|
|
303
|
+
state = 'attached';
|
|
304
|
+
// §12.3: the retired authority keeps only its state.
|
|
305
|
+
const tables = database
|
|
306
|
+
.query("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name <> 'sync_local_authority'")
|
|
307
|
+
.all();
|
|
308
|
+
for (const { name } of tables) {
|
|
309
|
+
database.exec(`DROP TABLE IF EXISTS "${name.replaceAll('"', '""')}"`);
|
|
310
|
+
}
|
|
311
|
+
database.exec('VACUUM');
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* §12.3 abandon: settle the one commit whose outcome may be unknown, then
|
|
315
|
+
* delete the rows of every applied commit, children before parents.
|
|
316
|
+
*/
|
|
317
|
+
async function rollback(transport) {
|
|
318
|
+
const attachId = attachIdOf();
|
|
319
|
+
const commits = planAttach(database, compiled, attachId);
|
|
320
|
+
const send = await connect(transport, `authority-${attachId}`);
|
|
321
|
+
let through = ordinal('applied');
|
|
322
|
+
const sent = ordinal('sent');
|
|
323
|
+
if (sent > through) {
|
|
324
|
+
const commit = commits[sent];
|
|
325
|
+
if (applied(await send(commit)))
|
|
326
|
+
through = sent;
|
|
327
|
+
writeMeta({ applied: String(through) });
|
|
328
|
+
}
|
|
329
|
+
const deletes = commits
|
|
330
|
+
.slice(0, through + 1)
|
|
331
|
+
.flatMap((commit) => commit.operations)
|
|
332
|
+
.reverse()
|
|
333
|
+
.map((operation) => ({
|
|
334
|
+
table: operation.table,
|
|
335
|
+
rowId: operation.rowId,
|
|
336
|
+
op: 'delete',
|
|
337
|
+
}));
|
|
338
|
+
for (let start = 0; start < deletes.length; start += ATTACH_OPERATION_BUDGET) {
|
|
339
|
+
const commit = {
|
|
340
|
+
type: 'PUSH_COMMIT',
|
|
341
|
+
clientCommitId: `${attachId}:undo:${start / ATTACH_OPERATION_BUDGET}`,
|
|
342
|
+
operations: deletes.slice(start, start + ATTACH_OPERATION_BUDGET),
|
|
343
|
+
};
|
|
344
|
+
const result = await send(commit);
|
|
345
|
+
if (!applied(result))
|
|
346
|
+
throw rejected(commit, result);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* §12.3 upload plan: every live row as one insert, rows visited in
|
|
352
|
+
* (table name, row id) byte order, strongly connected components of the
|
|
353
|
+
* row-level reference graph emitted in Tarjan completion order (every
|
|
354
|
+
* parent before its children) and packed whole into commits.
|
|
355
|
+
*/
|
|
356
|
+
function planAttach(database, schema, attachId) {
|
|
357
|
+
const tables = [...schema.tables.values()].sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
|
|
358
|
+
const nodeTable = [];
|
|
359
|
+
const nodeRowId = [];
|
|
360
|
+
const nodeValues = [];
|
|
361
|
+
const index = new Map();
|
|
362
|
+
for (const table of tables) {
|
|
363
|
+
const rows = database
|
|
364
|
+
.query(`SELECT _sync_row_id AS row_id, _sync_payload AS payload
|
|
365
|
+
FROM "${table.name.replaceAll('"', '""')}"
|
|
366
|
+
WHERE _sync_partition=? ORDER BY _sync_row_id`)
|
|
367
|
+
.all(LOCAL);
|
|
368
|
+
for (const row of rows) {
|
|
369
|
+
index.set(`${table.name}\u0000${row.row_id}`, nodeRowId.length);
|
|
370
|
+
nodeTable.push(table);
|
|
371
|
+
nodeRowId.push(row.row_id);
|
|
372
|
+
nodeValues.push(decodeRow(table.columns, new Uint8Array(row.payload)));
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
const parents = nodeValues.map((values, node) => {
|
|
376
|
+
const edges = [];
|
|
377
|
+
for (const reference of nodeTable[node].references) {
|
|
378
|
+
const parentId = renderScopeValue(values[reference.columnIndex]);
|
|
379
|
+
const parent = parentId === undefined
|
|
380
|
+
? undefined
|
|
381
|
+
: index.get(`${reference.parentTable}\u0000${parentId}`);
|
|
382
|
+
if (parent !== undefined)
|
|
383
|
+
edges.push(parent);
|
|
384
|
+
}
|
|
385
|
+
return edges;
|
|
386
|
+
});
|
|
387
|
+
// Iterative Tarjan: a deep parent chain must not exhaust the call stack.
|
|
388
|
+
const order = new Array(nodeRowId.length).fill(-1);
|
|
389
|
+
const low = new Array(nodeRowId.length).fill(0);
|
|
390
|
+
const onStack = new Array(nodeRowId.length).fill(false);
|
|
391
|
+
const stack = [];
|
|
392
|
+
const components = [];
|
|
393
|
+
let counter = 0;
|
|
394
|
+
for (let root = 0; root < nodeRowId.length; root++) {
|
|
395
|
+
if (order[root] !== -1)
|
|
396
|
+
continue;
|
|
397
|
+
const frames = [{ node: root, edge: 0 }];
|
|
398
|
+
order[root] = low[root] = counter++;
|
|
399
|
+
stack.push(root);
|
|
400
|
+
onStack[root] = true;
|
|
401
|
+
while (frames.length > 0) {
|
|
402
|
+
const frame = frames[frames.length - 1];
|
|
403
|
+
const edges = parents[frame.node];
|
|
404
|
+
if (frame.edge < edges.length) {
|
|
405
|
+
const next = edges[frame.edge++];
|
|
406
|
+
if (order[next] === -1) {
|
|
407
|
+
order[next] = low[next] = counter++;
|
|
408
|
+
stack.push(next);
|
|
409
|
+
onStack[next] = true;
|
|
410
|
+
frames.push({ node: next, edge: 0 });
|
|
411
|
+
}
|
|
412
|
+
else if (onStack[next]) {
|
|
413
|
+
low[frame.node] = Math.min(low[frame.node], order[next]);
|
|
414
|
+
}
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
frames.pop();
|
|
418
|
+
const caller = frames[frames.length - 1];
|
|
419
|
+
if (caller !== undefined) {
|
|
420
|
+
low[caller.node] = Math.min(low[caller.node], low[frame.node]);
|
|
421
|
+
}
|
|
422
|
+
if (low[frame.node] !== order[frame.node])
|
|
423
|
+
continue;
|
|
424
|
+
const component = [];
|
|
425
|
+
let member;
|
|
426
|
+
do {
|
|
427
|
+
member = stack.pop();
|
|
428
|
+
onStack[member] = false;
|
|
429
|
+
component.push(member);
|
|
430
|
+
} while (member !== frame.node);
|
|
431
|
+
components.push(component.sort((a, b) => a - b));
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
const commits = [];
|
|
435
|
+
let operations = [];
|
|
436
|
+
const flush = () => {
|
|
437
|
+
if (operations.length === 0)
|
|
438
|
+
return;
|
|
439
|
+
commits.push({
|
|
440
|
+
type: 'PUSH_COMMIT',
|
|
441
|
+
clientCommitId: `${attachId}:${commits.length}`,
|
|
442
|
+
operations,
|
|
443
|
+
});
|
|
444
|
+
operations = [];
|
|
445
|
+
};
|
|
446
|
+
for (const component of components) {
|
|
447
|
+
if (component.length > ATTACH_OPERATION_BUDGET) {
|
|
448
|
+
throw new LocalAuthorityAttachError({
|
|
449
|
+
reason: 'cycle_too_large',
|
|
450
|
+
table: nodeTable[component[0]].name,
|
|
451
|
+
size: component.length,
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
if (operations.length + component.length > ATTACH_OPERATION_BUDGET) {
|
|
455
|
+
flush();
|
|
456
|
+
}
|
|
457
|
+
for (const node of component) {
|
|
458
|
+
const table = nodeTable[node];
|
|
459
|
+
operations.push({
|
|
460
|
+
table: table.name,
|
|
461
|
+
rowId: nodeRowId[node],
|
|
462
|
+
op: 'upsert',
|
|
463
|
+
baseVersion: 0,
|
|
464
|
+
payload: encodeSparseRow(table.columns, table.primaryKeyIndex, nodeValues[node]),
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
flush();
|
|
469
|
+
return commits;
|
|
470
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.49.1",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.49.1"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/errors.ts
CHANGED
|
@@ -194,6 +194,12 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
|
|
|
194
194
|
recommendedAction: 'inspectServer',
|
|
195
195
|
httpStatus: 503,
|
|
196
196
|
},
|
|
197
|
+
'sync.authority_unavailable': {
|
|
198
|
+
category: 'internal',
|
|
199
|
+
retryable: false,
|
|
200
|
+
recommendedAction: 'inspectServer',
|
|
201
|
+
httpStatus: 503,
|
|
202
|
+
},
|
|
197
203
|
'sync.schema_not_ready': {
|
|
198
204
|
category: 'internal',
|
|
199
205
|
retryable: true,
|
package/src/index.ts
CHANGED
|
@@ -24,6 +24,7 @@ export * from './events-ring';
|
|
|
24
24
|
export * from './frame-bytes';
|
|
25
25
|
export * from './handler';
|
|
26
26
|
export * from './lease-store';
|
|
27
|
+
export * from './local-authority';
|
|
27
28
|
export * from './operations';
|
|
28
29
|
export * from './operations-realtime';
|
|
29
30
|
// The `PgExecutor` seam + Postgres storage/fanout are driver-agnostic (zero
|
|
@@ -0,0 +1,654 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local authority (SPEC.md §12): the reference server running in the client's
|
|
3
|
+
* process over its own SQLite database. An app syncs its client against it
|
|
4
|
+
* with no remote server, and later moves the authority's rows to a remote
|
|
5
|
+
* server with `attach` (§12.3).
|
|
6
|
+
*/
|
|
7
|
+
import {
|
|
8
|
+
decodeMessage,
|
|
9
|
+
decodeRow,
|
|
10
|
+
encodeMessage,
|
|
11
|
+
encodeSparseRow,
|
|
12
|
+
PROTOCOL_WIRE_VERSION,
|
|
13
|
+
type PushCommitFrame,
|
|
14
|
+
type PushOperation,
|
|
15
|
+
type PushResultFrame,
|
|
16
|
+
type RequestFrame,
|
|
17
|
+
type ResponseFrame,
|
|
18
|
+
type RowValue,
|
|
19
|
+
} from '@syncular/core';
|
|
20
|
+
import type { SyncRequestContext } from './context';
|
|
21
|
+
import type { CrdtMergerRegistry } from './crdt-merger';
|
|
22
|
+
import { SyncError } from './errors';
|
|
23
|
+
import { handleSyncRequest } from './handler';
|
|
24
|
+
import { pruneCommitLog } from './prune';
|
|
25
|
+
import {
|
|
26
|
+
type CompiledSchema,
|
|
27
|
+
compileSchema,
|
|
28
|
+
type ServerSchema,
|
|
29
|
+
} from './schema';
|
|
30
|
+
import { renderScopeValue } from './scopes';
|
|
31
|
+
import { handleSegmentDownload } from './segment-download';
|
|
32
|
+
import { MemorySegmentStore } from './segment-store';
|
|
33
|
+
import type {
|
|
34
|
+
SqliteDatabase,
|
|
35
|
+
SqliteRunResult,
|
|
36
|
+
SqliteValue,
|
|
37
|
+
} from './sqlite-driver';
|
|
38
|
+
import type { SqliteImageBuilder } from './sqlite-image';
|
|
39
|
+
import { SqliteServerStorage } from './sqlite-storage';
|
|
40
|
+
|
|
41
|
+
/** The one partition and actor a local authority serves (§12.2). */
|
|
42
|
+
const LOCAL = 'local';
|
|
43
|
+
/** Operations per attach commit: the reference server's request cap (§6.1). */
|
|
44
|
+
const ATTACH_OPERATION_BUDGET = 500;
|
|
45
|
+
|
|
46
|
+
export type LocalAuthorityState = 'local' | 'attaching' | 'attached';
|
|
47
|
+
|
|
48
|
+
export interface LocalAuthorityOptions {
|
|
49
|
+
/** The authority's own database. Never the client replica (§12.1). */
|
|
50
|
+
readonly database: SqliteDatabase;
|
|
51
|
+
readonly schema: ServerSchema;
|
|
52
|
+
/** §5.10 mergers for `crdt` columns, as a remote server would take them. */
|
|
53
|
+
readonly crdtMergers?: CrdtMergerRegistry;
|
|
54
|
+
/** §5.3 image builder from `@syncular/server/sqlite`; absent ⇒ rows segments. */
|
|
55
|
+
readonly sqliteImageBuilder?: SqliteImageBuilder;
|
|
56
|
+
readonly clock?: () => number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export interface LocalAuthority {
|
|
60
|
+
/** Current §12.3 state, persisted in the authority database. */
|
|
61
|
+
readonly state: LocalAuthorityState;
|
|
62
|
+
/** The client's sync transport: one combined push+pull round (§1.5). */
|
|
63
|
+
readonly sync: (request: Uint8Array) => Promise<Uint8Array>;
|
|
64
|
+
/** The client's segment downloader (§5.5). */
|
|
65
|
+
readonly downloadSegment: (request: {
|
|
66
|
+
readonly segmentId: string;
|
|
67
|
+
readonly requestedScopesJson: string;
|
|
68
|
+
}) => Promise<Uint8Array>;
|
|
69
|
+
/**
|
|
70
|
+
* §12.3: upload every live row to the remote server behind `transport`
|
|
71
|
+
* (an authenticated `POST <mount>/sync` round trip). Resolves once every
|
|
72
|
+
* commit is applied; a second call after a failure resumes the same plan.
|
|
73
|
+
*/
|
|
74
|
+
readonly attach: (
|
|
75
|
+
transport: (request: Uint8Array) => Promise<Uint8Array>,
|
|
76
|
+
) => Promise<void>;
|
|
77
|
+
/**
|
|
78
|
+
* §12.3: delete the rows the remote server applied from this attach, then
|
|
79
|
+
* serve the client again. `transport` reaches the same remote server and is
|
|
80
|
+
* required once an upload request may have reached it.
|
|
81
|
+
*/
|
|
82
|
+
readonly abandonAttach: (
|
|
83
|
+
transport?: (request: Uint8Array) => Promise<Uint8Array>,
|
|
84
|
+
) => Promise<void>;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export type LocalAuthorityAttachDetails =
|
|
88
|
+
| {
|
|
89
|
+
readonly reason: 'commit_rejected';
|
|
90
|
+
readonly clientCommitId: string;
|
|
91
|
+
readonly opIndex: number;
|
|
92
|
+
readonly code: string;
|
|
93
|
+
}
|
|
94
|
+
| {
|
|
95
|
+
readonly reason: 'cycle_too_large';
|
|
96
|
+
readonly table: string;
|
|
97
|
+
readonly size: number;
|
|
98
|
+
}
|
|
99
|
+
| {
|
|
100
|
+
readonly reason: 'request_failed';
|
|
101
|
+
readonly code: string;
|
|
102
|
+
readonly retryable: boolean;
|
|
103
|
+
}
|
|
104
|
+
| { readonly reason: 'log_epoch_changed' }
|
|
105
|
+
| {
|
|
106
|
+
readonly reason: 'schema_floor';
|
|
107
|
+
readonly requiredSchemaVersion: number;
|
|
108
|
+
readonly latestSchemaVersion?: number;
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
/** §10.3 client-local `sync.attach_rejected`: the attach upload stopped. */
|
|
112
|
+
export class LocalAuthorityAttachError extends Error {
|
|
113
|
+
override readonly name = 'LocalAuthorityAttachError';
|
|
114
|
+
readonly code = 'sync.attach_rejected';
|
|
115
|
+
readonly details: LocalAuthorityAttachDetails;
|
|
116
|
+
|
|
117
|
+
constructor(details: LocalAuthorityAttachDetails) {
|
|
118
|
+
super('the remote server did not accept the attach upload');
|
|
119
|
+
this.details = details;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** The `ClientDatabase` surface of `@syncular/client`, as a browser worker opens it. */
|
|
124
|
+
export interface LocalAuthorityClientDatabase {
|
|
125
|
+
exec(sql: string, params?: readonly SqliteValue[]): void;
|
|
126
|
+
query(
|
|
127
|
+
sql: string,
|
|
128
|
+
params?: readonly SqliteValue[],
|
|
129
|
+
): Record<string, SqliteValue>[];
|
|
130
|
+
close(): void;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Serve a local authority from a client-side database, such as the sibling
|
|
135
|
+
* file a browser worker opens beside its replica. `exec` without parameters
|
|
136
|
+
* must run several statements, as the sqlite-wasm database does.
|
|
137
|
+
*/
|
|
138
|
+
export function clientSqliteDatabase(
|
|
139
|
+
database: LocalAuthorityClientDatabase,
|
|
140
|
+
): SqliteDatabase {
|
|
141
|
+
const run = (
|
|
142
|
+
sql: string,
|
|
143
|
+
params: readonly SqliteValue[],
|
|
144
|
+
): SqliteRunResult => {
|
|
145
|
+
database.exec(sql, params);
|
|
146
|
+
const result = database.query(
|
|
147
|
+
'SELECT changes() AS changes, last_insert_rowid() AS id',
|
|
148
|
+
)[0];
|
|
149
|
+
return {
|
|
150
|
+
changes: Number(result?.changes ?? 0),
|
|
151
|
+
lastInsertRowid: Number(result?.id ?? 0),
|
|
152
|
+
};
|
|
153
|
+
};
|
|
154
|
+
return {
|
|
155
|
+
exec: (sql) => database.exec(sql),
|
|
156
|
+
run: (sql, bindings = []) => run(sql, bindings),
|
|
157
|
+
query: <Row, Params extends readonly SqliteValue[]>(sql: string) => ({
|
|
158
|
+
run: (...params: Params) => run(sql, params),
|
|
159
|
+
get: (...params: Params) =>
|
|
160
|
+
(database.query(sql, params)[0] ?? null) as Row | null,
|
|
161
|
+
all: (...params: Params) => database.query(sql, params) as Row[],
|
|
162
|
+
}),
|
|
163
|
+
close: () => database.close(),
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export async function openLocalAuthority(
|
|
168
|
+
options: LocalAuthorityOptions,
|
|
169
|
+
): Promise<LocalAuthority> {
|
|
170
|
+
const { database, schema } = options;
|
|
171
|
+
const clock = options.clock ?? Date.now;
|
|
172
|
+
database.exec(
|
|
173
|
+
'CREATE TABLE IF NOT EXISTS sync_local_authority(key TEXT PRIMARY KEY, value TEXT NOT NULL)',
|
|
174
|
+
);
|
|
175
|
+
const readMeta = (key: string): string | undefined =>
|
|
176
|
+
database
|
|
177
|
+
.query<{ value: string }, [string]>(
|
|
178
|
+
'SELECT value FROM sync_local_authority WHERE key=?',
|
|
179
|
+
)
|
|
180
|
+
.get(key)?.value;
|
|
181
|
+
const stored = readMeta('state') ?? 'local';
|
|
182
|
+
if (stored !== 'local' && stored !== 'attaching' && stored !== 'attached') {
|
|
183
|
+
throw new SyncError(
|
|
184
|
+
'sync.internal_error',
|
|
185
|
+
'the local authority state is unreadable',
|
|
186
|
+
JSON.stringify({ state: stored }),
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
let state: LocalAuthorityState = stored;
|
|
190
|
+
const compiled = compileSchema(schema);
|
|
191
|
+
// §12.3: an attached authority keeps only its state across reopens, so it
|
|
192
|
+
// opens no server storage; the storage constructor creates the server tables.
|
|
193
|
+
const storage =
|
|
194
|
+
stored === 'attached' ? undefined : new SqliteServerStorage(database);
|
|
195
|
+
if (storage !== undefined) {
|
|
196
|
+
// §12.2: the authority reaches the running schema before it serves.
|
|
197
|
+
await storage.ensureSchema(compiled);
|
|
198
|
+
if ((await storage.getPartitionLogEpoch(LOCAL)) !== undefined) {
|
|
199
|
+
await pruneCommitLog({ storage, partition: LOCAL, nowMs: clock() });
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
const allowed = Object.fromEntries(
|
|
203
|
+
[...compiled.declaredVariables].map((variable) => [variable, ['*']]),
|
|
204
|
+
);
|
|
205
|
+
const ctx: SyncRequestContext | undefined =
|
|
206
|
+
storage === undefined
|
|
207
|
+
? undefined
|
|
208
|
+
: {
|
|
209
|
+
partition: LOCAL,
|
|
210
|
+
actorId: LOCAL,
|
|
211
|
+
schema,
|
|
212
|
+
storage,
|
|
213
|
+
segments: new MemorySegmentStore(),
|
|
214
|
+
resolveScopes: () => allowed,
|
|
215
|
+
clock,
|
|
216
|
+
...(options.crdtMergers !== undefined
|
|
217
|
+
? { crdtMergers: options.crdtMergers }
|
|
218
|
+
: {}),
|
|
219
|
+
...(options.sqliteImageBuilder !== undefined
|
|
220
|
+
? { sqliteImageBuilder: options.sqliteImageBuilder }
|
|
221
|
+
: {}),
|
|
222
|
+
};
|
|
223
|
+
let uploading = false;
|
|
224
|
+
const refuseDuringUpload = () => {
|
|
225
|
+
if (uploading) {
|
|
226
|
+
throw new SyncError(
|
|
227
|
+
'sync.invalid_request',
|
|
228
|
+
'an attach upload is already running',
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
};
|
|
232
|
+
const inflight = new Set<Promise<unknown>>();
|
|
233
|
+
const admit = <T>(
|
|
234
|
+
work: (ctx: SyncRequestContext) => Promise<T>,
|
|
235
|
+
): Promise<T> => {
|
|
236
|
+
if (state !== 'local' || ctx === undefined) {
|
|
237
|
+
return Promise.reject(
|
|
238
|
+
new SyncError(
|
|
239
|
+
'sync.authority_unavailable',
|
|
240
|
+
'the local authority serves no requests',
|
|
241
|
+
JSON.stringify({ state }),
|
|
242
|
+
),
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
const promise = work(ctx);
|
|
246
|
+
const settle = () => inflight.delete(promise);
|
|
247
|
+
inflight.add(promise);
|
|
248
|
+
promise.then(settle, settle);
|
|
249
|
+
return promise;
|
|
250
|
+
};
|
|
251
|
+
/** One transaction; a `null` value removes the key. */
|
|
252
|
+
const writeMeta = (entries: Readonly<Record<string, string | null>>) => {
|
|
253
|
+
database.exec('BEGIN IMMEDIATE');
|
|
254
|
+
try {
|
|
255
|
+
for (const [key, value] of Object.entries(entries)) {
|
|
256
|
+
if (value === null) {
|
|
257
|
+
database.run('DELETE FROM sync_local_authority WHERE key=?', [key]);
|
|
258
|
+
} else {
|
|
259
|
+
database.run(
|
|
260
|
+
'INSERT OR REPLACE INTO sync_local_authority(key,value) VALUES (?,?)',
|
|
261
|
+
[key, value],
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
database.exec('COMMIT');
|
|
266
|
+
} catch (error) {
|
|
267
|
+
database.exec('ROLLBACK');
|
|
268
|
+
throw error;
|
|
269
|
+
}
|
|
270
|
+
};
|
|
271
|
+
const ordinal = (key: string): number => Number(readMeta(key) ?? -1);
|
|
272
|
+
const attachIdOf = (): string => {
|
|
273
|
+
const attachId = readMeta('attach_id');
|
|
274
|
+
if (attachId === undefined) {
|
|
275
|
+
throw new SyncError(
|
|
276
|
+
'sync.internal_error',
|
|
277
|
+
'the attaching local authority has no attach id',
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
return attachId;
|
|
281
|
+
};
|
|
282
|
+
|
|
283
|
+
const exchange = async (
|
|
284
|
+
transport: (request: Uint8Array) => Promise<Uint8Array>,
|
|
285
|
+
frames: RequestFrame[],
|
|
286
|
+
): Promise<ResponseFrame[]> => {
|
|
287
|
+
const message = decodeMessage(
|
|
288
|
+
await transport(
|
|
289
|
+
encodeMessage({
|
|
290
|
+
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
291
|
+
msgKind: 'request',
|
|
292
|
+
frames,
|
|
293
|
+
}),
|
|
294
|
+
),
|
|
295
|
+
);
|
|
296
|
+
if (message.msgKind !== 'response') {
|
|
297
|
+
throw new LocalAuthorityAttachError({
|
|
298
|
+
reason: 'request_failed',
|
|
299
|
+
code: 'sync.invalid_request',
|
|
300
|
+
retryable: false,
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
const header = message.frames[0];
|
|
304
|
+
if (header?.type === 'RESP_HEADER' && header.requiredSchemaVersion) {
|
|
305
|
+
throw new LocalAuthorityAttachError({
|
|
306
|
+
reason: 'schema_floor',
|
|
307
|
+
requiredSchemaVersion: header.requiredSchemaVersion,
|
|
308
|
+
...(header.latestSchemaVersion !== undefined
|
|
309
|
+
? { latestSchemaVersion: header.latestSchemaVersion }
|
|
310
|
+
: {}),
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
const error = message.frames.find((frame) => frame.type === 'ERROR');
|
|
314
|
+
if (error !== undefined) {
|
|
315
|
+
throw new LocalAuthorityAttachError({
|
|
316
|
+
reason: 'request_failed',
|
|
317
|
+
code: error.code,
|
|
318
|
+
retryable: error.retryable,
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
return message.frames;
|
|
322
|
+
};
|
|
323
|
+
|
|
324
|
+
/** Acquire the remote `logEpoch` (§2.1), then send one commit per request. */
|
|
325
|
+
const connect = async (
|
|
326
|
+
transport: (request: Uint8Array) => Promise<Uint8Array>,
|
|
327
|
+
clientId: string,
|
|
328
|
+
): Promise<(commit: PushCommitFrame) => Promise<PushResultFrame>> => {
|
|
329
|
+
const acquired = (
|
|
330
|
+
await exchange(transport, [
|
|
331
|
+
{ type: 'REQ_HEADER', clientId, schemaVersion: schema.version },
|
|
332
|
+
{
|
|
333
|
+
type: 'PULL_HEADER',
|
|
334
|
+
limitCommits: 0,
|
|
335
|
+
limitSnapshotRows: 0,
|
|
336
|
+
maxSnapshotPages: 0,
|
|
337
|
+
accept: 0b0011,
|
|
338
|
+
},
|
|
339
|
+
])
|
|
340
|
+
)[0];
|
|
341
|
+
const logEpoch =
|
|
342
|
+
acquired?.type === 'RESP_HEADER' ? acquired.logEpoch : undefined;
|
|
343
|
+
if (logEpoch === undefined) {
|
|
344
|
+
throw new LocalAuthorityAttachError({
|
|
345
|
+
reason: 'request_failed',
|
|
346
|
+
code: 'sync.invalid_request',
|
|
347
|
+
retryable: false,
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
return async (commit) => {
|
|
351
|
+
const frames = await exchange(transport, [
|
|
352
|
+
{
|
|
353
|
+
type: 'REQ_HEADER',
|
|
354
|
+
clientId,
|
|
355
|
+
schemaVersion: schema.version,
|
|
356
|
+
logEpoch,
|
|
357
|
+
},
|
|
358
|
+
commit,
|
|
359
|
+
]);
|
|
360
|
+
const header = frames[0];
|
|
361
|
+
if (header?.type === 'RESP_HEADER' && header.resetRequired === true) {
|
|
362
|
+
throw new LocalAuthorityAttachError({ reason: 'log_epoch_changed' });
|
|
363
|
+
}
|
|
364
|
+
const result = frames.find((frame) => frame.type === 'PUSH_RESULT');
|
|
365
|
+
if (result === undefined) {
|
|
366
|
+
throw new LocalAuthorityAttachError({
|
|
367
|
+
reason: 'request_failed',
|
|
368
|
+
code: 'sync.invalid_request',
|
|
369
|
+
retryable: false,
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
return result;
|
|
373
|
+
};
|
|
374
|
+
};
|
|
375
|
+
const applied = (result: PushResultFrame) =>
|
|
376
|
+
result.status === 'applied' || result.status === 'cached';
|
|
377
|
+
const rejected = (commit: PushCommitFrame, result: PushResultFrame) => {
|
|
378
|
+
const failed = result.results.find((r) => r.status !== 'applied');
|
|
379
|
+
return new LocalAuthorityAttachError({
|
|
380
|
+
reason: 'commit_rejected',
|
|
381
|
+
clientCommitId: commit.clientCommitId,
|
|
382
|
+
opIndex: failed?.opIndex ?? 0,
|
|
383
|
+
code: failed?.code ?? 'sync.invalid_request',
|
|
384
|
+
});
|
|
385
|
+
};
|
|
386
|
+
|
|
387
|
+
const authority: LocalAuthority = {
|
|
388
|
+
get state() {
|
|
389
|
+
return state;
|
|
390
|
+
},
|
|
391
|
+
sync: (request) => admit((ctx) => handleSyncRequest(request, ctx)),
|
|
392
|
+
downloadSegment: (request) =>
|
|
393
|
+
admit(async (ctx) => {
|
|
394
|
+
const result = await handleSegmentDownload(ctx, {
|
|
395
|
+
segmentId: request.segmentId,
|
|
396
|
+
scopesHeader: request.requestedScopesJson,
|
|
397
|
+
});
|
|
398
|
+
return result.bytes;
|
|
399
|
+
}),
|
|
400
|
+
async attach(transport) {
|
|
401
|
+
refuseDuringUpload();
|
|
402
|
+
if (state === 'attached') return;
|
|
403
|
+
uploading = true;
|
|
404
|
+
try {
|
|
405
|
+
await upload(transport);
|
|
406
|
+
} finally {
|
|
407
|
+
uploading = false;
|
|
408
|
+
}
|
|
409
|
+
},
|
|
410
|
+
async abandonAttach(transport) {
|
|
411
|
+
refuseDuringUpload();
|
|
412
|
+
if (state === 'attached') {
|
|
413
|
+
throw new SyncError(
|
|
414
|
+
'sync.authority_unavailable',
|
|
415
|
+
'the local authority serves no requests',
|
|
416
|
+
JSON.stringify({ state }),
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
if (state === 'local') return;
|
|
420
|
+
if (ordinal('sent') >= 0) {
|
|
421
|
+
if (transport === undefined) {
|
|
422
|
+
throw new SyncError(
|
|
423
|
+
'sync.invalid_request',
|
|
424
|
+
'abandoning an attach that reached the remote server needs its transport',
|
|
425
|
+
);
|
|
426
|
+
}
|
|
427
|
+
uploading = true;
|
|
428
|
+
try {
|
|
429
|
+
await rollback(transport);
|
|
430
|
+
} finally {
|
|
431
|
+
uploading = false;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
writeMeta({ state: 'local', attach_id: null, sent: null, applied: null });
|
|
435
|
+
state = 'local';
|
|
436
|
+
},
|
|
437
|
+
};
|
|
438
|
+
return authority;
|
|
439
|
+
|
|
440
|
+
async function upload(
|
|
441
|
+
transport: (request: Uint8Array) => Promise<Uint8Array>,
|
|
442
|
+
): Promise<void> {
|
|
443
|
+
if (state === 'local') {
|
|
444
|
+
// Refuse new rounds first, then let admitted rounds finish: the plan
|
|
445
|
+
// reads the authority database on the connection a round writes on.
|
|
446
|
+
state = 'attaching';
|
|
447
|
+
try {
|
|
448
|
+
await Promise.allSettled([...inflight]);
|
|
449
|
+
writeMeta({ state: 'attaching', attach_id: crypto.randomUUID() });
|
|
450
|
+
} catch (error) {
|
|
451
|
+
state = 'local';
|
|
452
|
+
throw error;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
const attachId = attachIdOf();
|
|
456
|
+
const commits = planAttach(database, compiled, attachId);
|
|
457
|
+
const send = await connect(transport, `authority-${attachId}`);
|
|
458
|
+
for (const [index, commit] of commits.entries()) {
|
|
459
|
+
// §12.3: `sent` before the request, `applied` after its result, so an
|
|
460
|
+
// abandon knows which commits may have reached the remote server.
|
|
461
|
+
writeMeta({ sent: String(Math.max(index, ordinal('sent'))) });
|
|
462
|
+
const result = await send(commit);
|
|
463
|
+
if (!applied(result)) throw rejected(commit, result);
|
|
464
|
+
writeMeta({ applied: String(Math.max(index, ordinal('applied'))) });
|
|
465
|
+
}
|
|
466
|
+
writeMeta({ state: 'attached', sent: null, applied: null });
|
|
467
|
+
state = 'attached';
|
|
468
|
+
// §12.3: the retired authority keeps only its state.
|
|
469
|
+
const tables = database
|
|
470
|
+
.query<{ name: string }, []>(
|
|
471
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name <> 'sync_local_authority'",
|
|
472
|
+
)
|
|
473
|
+
.all();
|
|
474
|
+
for (const { name } of tables) {
|
|
475
|
+
database.exec(`DROP TABLE IF EXISTS "${name.replaceAll('"', '""')}"`);
|
|
476
|
+
}
|
|
477
|
+
database.exec('VACUUM');
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/**
|
|
481
|
+
* §12.3 abandon: settle the one commit whose outcome may be unknown, then
|
|
482
|
+
* delete the rows of every applied commit, children before parents.
|
|
483
|
+
*/
|
|
484
|
+
async function rollback(
|
|
485
|
+
transport: (request: Uint8Array) => Promise<Uint8Array>,
|
|
486
|
+
): Promise<void> {
|
|
487
|
+
const attachId = attachIdOf();
|
|
488
|
+
const commits = planAttach(database, compiled, attachId);
|
|
489
|
+
const send = await connect(transport, `authority-${attachId}`);
|
|
490
|
+
let through = ordinal('applied');
|
|
491
|
+
const sent = ordinal('sent');
|
|
492
|
+
if (sent > through) {
|
|
493
|
+
const commit = commits[sent]!;
|
|
494
|
+
if (applied(await send(commit))) through = sent;
|
|
495
|
+
writeMeta({ applied: String(through) });
|
|
496
|
+
}
|
|
497
|
+
const deletes = commits
|
|
498
|
+
.slice(0, through + 1)
|
|
499
|
+
.flatMap((commit) => commit.operations)
|
|
500
|
+
.reverse()
|
|
501
|
+
.map(
|
|
502
|
+
(operation): PushOperation => ({
|
|
503
|
+
table: operation.table,
|
|
504
|
+
rowId: operation.rowId,
|
|
505
|
+
op: 'delete',
|
|
506
|
+
}),
|
|
507
|
+
);
|
|
508
|
+
for (
|
|
509
|
+
let start = 0;
|
|
510
|
+
start < deletes.length;
|
|
511
|
+
start += ATTACH_OPERATION_BUDGET
|
|
512
|
+
) {
|
|
513
|
+
const commit: PushCommitFrame = {
|
|
514
|
+
type: 'PUSH_COMMIT',
|
|
515
|
+
clientCommitId: `${attachId}:undo:${start / ATTACH_OPERATION_BUDGET}`,
|
|
516
|
+
operations: deletes.slice(start, start + ATTACH_OPERATION_BUDGET),
|
|
517
|
+
};
|
|
518
|
+
const result = await send(commit);
|
|
519
|
+
if (!applied(result)) throw rejected(commit, result);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
/**
|
|
525
|
+
* §12.3 upload plan: every live row as one insert, rows visited in
|
|
526
|
+
* (table name, row id) byte order, strongly connected components of the
|
|
527
|
+
* row-level reference graph emitted in Tarjan completion order (every
|
|
528
|
+
* parent before its children) and packed whole into commits.
|
|
529
|
+
*/
|
|
530
|
+
function planAttach(
|
|
531
|
+
database: SqliteDatabase,
|
|
532
|
+
schema: CompiledSchema,
|
|
533
|
+
attachId: string,
|
|
534
|
+
): PushCommitFrame[] {
|
|
535
|
+
const tables = [...schema.tables.values()].sort((a, b) =>
|
|
536
|
+
a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
|
|
537
|
+
);
|
|
538
|
+
const nodeTable: (typeof tables)[number][] = [];
|
|
539
|
+
const nodeRowId: string[] = [];
|
|
540
|
+
const nodeValues: RowValue[][] = [];
|
|
541
|
+
const index = new Map<string, number>();
|
|
542
|
+
for (const table of tables) {
|
|
543
|
+
const rows = database
|
|
544
|
+
.query<{ row_id: string; payload: Uint8Array }, [string]>(
|
|
545
|
+
`SELECT _sync_row_id AS row_id, _sync_payload AS payload
|
|
546
|
+
FROM "${table.name.replaceAll('"', '""')}"
|
|
547
|
+
WHERE _sync_partition=? ORDER BY _sync_row_id`,
|
|
548
|
+
)
|
|
549
|
+
.all(LOCAL);
|
|
550
|
+
for (const row of rows) {
|
|
551
|
+
index.set(`${table.name}\u0000${row.row_id}`, nodeRowId.length);
|
|
552
|
+
nodeTable.push(table);
|
|
553
|
+
nodeRowId.push(row.row_id);
|
|
554
|
+
nodeValues.push(decodeRow(table.columns, new Uint8Array(row.payload)));
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
const parents = nodeValues.map((values, node) => {
|
|
558
|
+
const edges: number[] = [];
|
|
559
|
+
for (const reference of nodeTable[node]!.references) {
|
|
560
|
+
const parentId = renderScopeValue(values[reference.columnIndex]);
|
|
561
|
+
const parent =
|
|
562
|
+
parentId === undefined
|
|
563
|
+
? undefined
|
|
564
|
+
: index.get(`${reference.parentTable}\u0000${parentId}`);
|
|
565
|
+
if (parent !== undefined) edges.push(parent);
|
|
566
|
+
}
|
|
567
|
+
return edges;
|
|
568
|
+
});
|
|
569
|
+
|
|
570
|
+
// Iterative Tarjan: a deep parent chain must not exhaust the call stack.
|
|
571
|
+
const order = new Array<number>(nodeRowId.length).fill(-1);
|
|
572
|
+
const low = new Array<number>(nodeRowId.length).fill(0);
|
|
573
|
+
const onStack = new Array<boolean>(nodeRowId.length).fill(false);
|
|
574
|
+
const stack: number[] = [];
|
|
575
|
+
const components: number[][] = [];
|
|
576
|
+
let counter = 0;
|
|
577
|
+
for (let root = 0; root < nodeRowId.length; root++) {
|
|
578
|
+
if (order[root] !== -1) continue;
|
|
579
|
+
const frames: { node: number; edge: number }[] = [{ node: root, edge: 0 }];
|
|
580
|
+
order[root] = low[root] = counter++;
|
|
581
|
+
stack.push(root);
|
|
582
|
+
onStack[root] = true;
|
|
583
|
+
while (frames.length > 0) {
|
|
584
|
+
const frame = frames[frames.length - 1]!;
|
|
585
|
+
const edges = parents[frame.node]!;
|
|
586
|
+
if (frame.edge < edges.length) {
|
|
587
|
+
const next = edges[frame.edge++]!;
|
|
588
|
+
if (order[next] === -1) {
|
|
589
|
+
order[next] = low[next] = counter++;
|
|
590
|
+
stack.push(next);
|
|
591
|
+
onStack[next] = true;
|
|
592
|
+
frames.push({ node: next, edge: 0 });
|
|
593
|
+
} else if (onStack[next]) {
|
|
594
|
+
low[frame.node] = Math.min(low[frame.node]!, order[next]!);
|
|
595
|
+
}
|
|
596
|
+
continue;
|
|
597
|
+
}
|
|
598
|
+
frames.pop();
|
|
599
|
+
const caller = frames[frames.length - 1];
|
|
600
|
+
if (caller !== undefined) {
|
|
601
|
+
low[caller.node] = Math.min(low[caller.node]!, low[frame.node]!);
|
|
602
|
+
}
|
|
603
|
+
if (low[frame.node] !== order[frame.node]) continue;
|
|
604
|
+
const component: number[] = [];
|
|
605
|
+
let member: number;
|
|
606
|
+
do {
|
|
607
|
+
member = stack.pop()!;
|
|
608
|
+
onStack[member] = false;
|
|
609
|
+
component.push(member);
|
|
610
|
+
} while (member !== frame.node);
|
|
611
|
+
components.push(component.sort((a, b) => a - b));
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
const commits: PushCommitFrame[] = [];
|
|
616
|
+
let operations: PushOperation[] = [];
|
|
617
|
+
const flush = () => {
|
|
618
|
+
if (operations.length === 0) return;
|
|
619
|
+
commits.push({
|
|
620
|
+
type: 'PUSH_COMMIT',
|
|
621
|
+
clientCommitId: `${attachId}:${commits.length}`,
|
|
622
|
+
operations,
|
|
623
|
+
});
|
|
624
|
+
operations = [];
|
|
625
|
+
};
|
|
626
|
+
for (const component of components) {
|
|
627
|
+
if (component.length > ATTACH_OPERATION_BUDGET) {
|
|
628
|
+
throw new LocalAuthorityAttachError({
|
|
629
|
+
reason: 'cycle_too_large',
|
|
630
|
+
table: nodeTable[component[0]!]!.name,
|
|
631
|
+
size: component.length,
|
|
632
|
+
});
|
|
633
|
+
}
|
|
634
|
+
if (operations.length + component.length > ATTACH_OPERATION_BUDGET) {
|
|
635
|
+
flush();
|
|
636
|
+
}
|
|
637
|
+
for (const node of component) {
|
|
638
|
+
const table = nodeTable[node]!;
|
|
639
|
+
operations.push({
|
|
640
|
+
table: table.name,
|
|
641
|
+
rowId: nodeRowId[node]!,
|
|
642
|
+
op: 'upsert',
|
|
643
|
+
baseVersion: 0,
|
|
644
|
+
payload: encodeSparseRow(
|
|
645
|
+
table.columns,
|
|
646
|
+
table.primaryKeyIndex,
|
|
647
|
+
nodeValues[node]!,
|
|
648
|
+
),
|
|
649
|
+
});
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
flush();
|
|
653
|
+
return commits;
|
|
654
|
+
}
|