@syncular/server 0.47.0 → 0.49.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -2
- package/dist/errors.js +6 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/local-authority.d.ts +78 -0
- package/dist/local-authority.js +470 -0
- package/dist/operations.d.ts +10 -1
- package/dist/operations.js +17 -0
- package/dist/push.js +44 -1
- package/dist/validate.d.ts +25 -0
- package/package.json +2 -2
- package/src/errors.ts +6 -0
- package/src/index.ts +1 -0
- package/src/local-authority.ts +654 -0
- package/src/operations.ts +40 -0
- package/src/push.ts +55 -1
- package/src/validate.ts +26 -0
package/README.md
CHANGED
|
@@ -256,7 +256,8 @@ const config: SyncServerConfig = {
|
|
|
256
256
|
```
|
|
257
257
|
|
|
258
258
|
The callback also receives `read.getRow()` and its batch form
|
|
259
|
-
`read.getRows(table, rowIds)`, bounded `read.scanRows()`,
|
|
259
|
+
`read.getRows(table, rowIds)`, bounded `read.scanRows()`,
|
|
260
|
+
`read.scanRowsByIndex()` for an exact declared-index lookup, and
|
|
260
261
|
`read.queryAuthoritative()` APIs. Those reads see the final candidate state, including staged sibling upserts and
|
|
261
262
|
deletes. Syncular serializes the partition before any operation read so two
|
|
262
263
|
aggregate validators cannot both accept mutually invalid candidates. It also
|
|
@@ -321,6 +322,18 @@ and the ID check does not authenticate the client. Within one partition the ID
|
|
|
321
322
|
contract is a correctness property: it stops a replayed ID from applying
|
|
322
323
|
different operations.
|
|
323
324
|
|
|
325
|
+
## Local authority
|
|
326
|
+
|
|
327
|
+
`openLocalAuthority({ database, schema })` runs this server inside a client's
|
|
328
|
+
process over its own SQLite database (SPEC §12). Its `sync` and
|
|
329
|
+
`downloadSegment` functions are the client's `transport` and `segments`.
|
|
330
|
+
`attach(transport)` uploads every live row to a remote server as inserts under
|
|
331
|
+
stable commit IDs and then retires the authority (§12.3);
|
|
332
|
+
`abandonAttach(transport)` deletes what a stopped attach uploaded. In a browser
|
|
333
|
+
worker, `clientSqliteDatabase` adapts the worker's sibling database. The
|
|
334
|
+
[local authority guide](https://syncular.dev/guide-local-authority/) covers the
|
|
335
|
+
client setup and the attach sequence.
|
|
336
|
+
|
|
324
337
|
## Seed idempotency and safe revisioning
|
|
325
338
|
|
|
326
339
|
`seedMutations` uses the real push path and a stable `clientId`/`commitId`.
|
|
@@ -359,7 +372,9 @@ SQLite, PostgreSQL, and D1.
|
|
|
359
372
|
|
|
360
373
|
An authoritative command that needs an exact alternate lookup can instead use
|
|
361
374
|
the optional `storage.scanRowsByIndex(partition, query)` or transactional
|
|
362
|
-
`tx.scanRowsByIndex(query)` capability
|
|
375
|
+
`tx.scanRowsByIndex(query)` capability, or `command.scanRowsByIndex()` on its
|
|
376
|
+
push transaction. The whole-commit validator reaches the same transaction
|
|
377
|
+
lookup as `read.scanRowsByIndex()`; both readers default `limit` to 100. The query names one declared
|
|
363
378
|
`TableSchema.indexes` entry, supplies one exact value per index column, uses an
|
|
364
379
|
exclusive `afterRowId`, and has a required limit from 1 through 1,000. All
|
|
365
380
|
shipped adapters implement it; transaction reads see staged writes and deletes.
|
package/dist/errors.js
CHANGED
|
@@ -186,6 +186,12 @@ export const ERROR_CATALOG = {
|
|
|
186
186
|
recommendedAction: 'inspectServer',
|
|
187
187
|
httpStatus: 503,
|
|
188
188
|
},
|
|
189
|
+
'sync.authority_unavailable': {
|
|
190
|
+
category: 'internal',
|
|
191
|
+
retryable: false,
|
|
192
|
+
recommendedAction: 'inspectServer',
|
|
193
|
+
httpStatus: 503,
|
|
194
|
+
},
|
|
189
195
|
'sync.schema_not_ready': {
|
|
190
196
|
category: 'internal',
|
|
191
197
|
retryable: true,
|
package/dist/index.d.ts
CHANGED
|
@@ -24,6 +24,7 @@ export * from './events-ring.js';
|
|
|
24
24
|
export * from './frame-bytes.js';
|
|
25
25
|
export * from './handler.js';
|
|
26
26
|
export * from './lease-store.js';
|
|
27
|
+
export * from './local-authority.js';
|
|
27
28
|
export * from './operations.js';
|
|
28
29
|
export * from './operations-realtime.js';
|
|
29
30
|
export * from './pg-executor.js';
|
package/dist/index.js
CHANGED
|
@@ -24,6 +24,7 @@ export * from './events-ring.js';
|
|
|
24
24
|
export * from './frame-bytes.js';
|
|
25
25
|
export * from './handler.js';
|
|
26
26
|
export * from './lease-store.js';
|
|
27
|
+
export * from './local-authority.js';
|
|
27
28
|
export * from './operations.js';
|
|
28
29
|
export * from './operations-realtime.js';
|
|
29
30
|
// The `PgExecutor` seam + Postgres storage/fanout are driver-agnostic (zero
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { CrdtMergerRegistry } from './crdt-merger.js';
|
|
2
|
+
import { type ServerSchema } from './schema.js';
|
|
3
|
+
import type { SqliteDatabase, SqliteValue } from './sqlite-driver.js';
|
|
4
|
+
import type { SqliteImageBuilder } from './sqlite-image.js';
|
|
5
|
+
export type LocalAuthorityState = 'local' | 'attaching' | 'attached';
|
|
6
|
+
export interface LocalAuthorityOptions {
|
|
7
|
+
/** The authority's own database. Never the client replica (§12.1). */
|
|
8
|
+
readonly database: SqliteDatabase;
|
|
9
|
+
readonly schema: ServerSchema;
|
|
10
|
+
/** §5.10 mergers for `crdt` columns, as a remote server would take them. */
|
|
11
|
+
readonly crdtMergers?: CrdtMergerRegistry;
|
|
12
|
+
/** §5.3 image builder from `@syncular/server/sqlite`; absent ⇒ rows segments. */
|
|
13
|
+
readonly sqliteImageBuilder?: SqliteImageBuilder;
|
|
14
|
+
readonly clock?: () => number;
|
|
15
|
+
}
|
|
16
|
+
export interface LocalAuthority {
|
|
17
|
+
/** Current §12.3 state, persisted in the authority database. */
|
|
18
|
+
readonly state: LocalAuthorityState;
|
|
19
|
+
/** The client's sync transport: one combined push+pull round (§1.5). */
|
|
20
|
+
readonly sync: (request: Uint8Array) => Promise<Uint8Array>;
|
|
21
|
+
/** The client's segment downloader (§5.5). */
|
|
22
|
+
readonly downloadSegment: (request: {
|
|
23
|
+
readonly segmentId: string;
|
|
24
|
+
readonly requestedScopesJson: string;
|
|
25
|
+
}) => Promise<Uint8Array>;
|
|
26
|
+
/**
|
|
27
|
+
* §12.3: upload every live row to the remote server behind `transport`
|
|
28
|
+
* (an authenticated `POST <mount>/sync` round trip). Resolves once every
|
|
29
|
+
* commit is applied; a second call after a failure resumes the same plan.
|
|
30
|
+
*/
|
|
31
|
+
readonly attach: (transport: (request: Uint8Array) => Promise<Uint8Array>) => Promise<void>;
|
|
32
|
+
/**
|
|
33
|
+
* §12.3: delete the rows the remote server applied from this attach, then
|
|
34
|
+
* serve the client again. `transport` reaches the same remote server and is
|
|
35
|
+
* required once an upload request may have reached it.
|
|
36
|
+
*/
|
|
37
|
+
readonly abandonAttach: (transport?: (request: Uint8Array) => Promise<Uint8Array>) => Promise<void>;
|
|
38
|
+
}
|
|
39
|
+
export type LocalAuthorityAttachDetails = {
|
|
40
|
+
readonly reason: 'commit_rejected';
|
|
41
|
+
readonly clientCommitId: string;
|
|
42
|
+
readonly opIndex: number;
|
|
43
|
+
readonly code: string;
|
|
44
|
+
} | {
|
|
45
|
+
readonly reason: 'cycle_too_large';
|
|
46
|
+
readonly table: string;
|
|
47
|
+
readonly size: number;
|
|
48
|
+
} | {
|
|
49
|
+
readonly reason: 'request_failed';
|
|
50
|
+
readonly code: string;
|
|
51
|
+
readonly retryable: boolean;
|
|
52
|
+
} | {
|
|
53
|
+
readonly reason: 'log_epoch_changed';
|
|
54
|
+
} | {
|
|
55
|
+
readonly reason: 'schema_floor';
|
|
56
|
+
readonly requiredSchemaVersion: number;
|
|
57
|
+
readonly latestSchemaVersion?: number;
|
|
58
|
+
};
|
|
59
|
+
/** §10.3 client-local `sync.attach_rejected`: the attach upload stopped. */
|
|
60
|
+
export declare class LocalAuthorityAttachError extends Error {
|
|
61
|
+
readonly name = "LocalAuthorityAttachError";
|
|
62
|
+
readonly code = "sync.attach_rejected";
|
|
63
|
+
readonly details: LocalAuthorityAttachDetails;
|
|
64
|
+
constructor(details: LocalAuthorityAttachDetails);
|
|
65
|
+
}
|
|
66
|
+
/** The `ClientDatabase` surface of `@syncular/client`, as a browser worker opens it. */
|
|
67
|
+
export interface LocalAuthorityClientDatabase {
|
|
68
|
+
exec(sql: string, params?: readonly SqliteValue[]): void;
|
|
69
|
+
query(sql: string, params?: readonly SqliteValue[]): Record<string, SqliteValue>[];
|
|
70
|
+
close(): void;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Serve a local authority from a client-side database, such as the sibling
|
|
74
|
+
* file a browser worker opens beside its replica. `exec` without parameters
|
|
75
|
+
* must run several statements, as the sqlite-wasm database does.
|
|
76
|
+
*/
|
|
77
|
+
export declare function clientSqliteDatabase(database: LocalAuthorityClientDatabase): SqliteDatabase;
|
|
78
|
+
export declare function openLocalAuthority(options: LocalAuthorityOptions): Promise<LocalAuthority>;
|
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local authority (SPEC.md §12): the reference server running in the client's
|
|
3
|
+
* process over its own SQLite database. An app syncs its client against it
|
|
4
|
+
* with no remote server, and later moves the authority's rows to a remote
|
|
5
|
+
* server with `attach` (§12.3).
|
|
6
|
+
*/
|
|
7
|
+
import { decodeMessage, decodeRow, encodeMessage, encodeSparseRow, PROTOCOL_WIRE_VERSION, } from '@syncular/core';
|
|
8
|
+
import { SyncError } from './errors.js';
|
|
9
|
+
import { handleSyncRequest } from './handler.js';
|
|
10
|
+
import { pruneCommitLog } from './prune.js';
|
|
11
|
+
import { compileSchema, } from './schema.js';
|
|
12
|
+
import { renderScopeValue } from './scopes.js';
|
|
13
|
+
import { handleSegmentDownload } from './segment-download.js';
|
|
14
|
+
import { MemorySegmentStore } from './segment-store.js';
|
|
15
|
+
import { SqliteServerStorage } from './sqlite-storage.js';
|
|
16
|
+
/** The one partition and actor a local authority serves (§12.2). */
|
|
17
|
+
const LOCAL = 'local';
|
|
18
|
+
/** Operations per attach commit: the reference server's request cap (§6.1). */
|
|
19
|
+
const ATTACH_OPERATION_BUDGET = 500;
|
|
20
|
+
/** §10.3 client-local `sync.attach_rejected`: the attach upload stopped. */
|
|
21
|
+
export class LocalAuthorityAttachError extends Error {
|
|
22
|
+
name = 'LocalAuthorityAttachError';
|
|
23
|
+
code = 'sync.attach_rejected';
|
|
24
|
+
details;
|
|
25
|
+
constructor(details) {
|
|
26
|
+
super('the remote server did not accept the attach upload');
|
|
27
|
+
this.details = details;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Serve a local authority from a client-side database, such as the sibling
|
|
32
|
+
* file a browser worker opens beside its replica. `exec` without parameters
|
|
33
|
+
* must run several statements, as the sqlite-wasm database does.
|
|
34
|
+
*/
|
|
35
|
+
export function clientSqliteDatabase(database) {
|
|
36
|
+
const run = (sql, params) => {
|
|
37
|
+
database.exec(sql, params);
|
|
38
|
+
const result = database.query('SELECT changes() AS changes, last_insert_rowid() AS id')[0];
|
|
39
|
+
return {
|
|
40
|
+
changes: Number(result?.changes ?? 0),
|
|
41
|
+
lastInsertRowid: Number(result?.id ?? 0),
|
|
42
|
+
};
|
|
43
|
+
};
|
|
44
|
+
return {
|
|
45
|
+
exec: (sql) => database.exec(sql),
|
|
46
|
+
run: (sql, bindings = []) => run(sql, bindings),
|
|
47
|
+
query: (sql) => ({
|
|
48
|
+
run: (...params) => run(sql, params),
|
|
49
|
+
get: (...params) => (database.query(sql, params)[0] ?? null),
|
|
50
|
+
all: (...params) => database.query(sql, params),
|
|
51
|
+
}),
|
|
52
|
+
close: () => database.close(),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
export async function openLocalAuthority(options) {
|
|
56
|
+
const { database, schema } = options;
|
|
57
|
+
const clock = options.clock ?? Date.now;
|
|
58
|
+
database.exec('CREATE TABLE IF NOT EXISTS sync_local_authority(key TEXT PRIMARY KEY, value TEXT NOT NULL)');
|
|
59
|
+
const readMeta = (key) => database
|
|
60
|
+
.query('SELECT value FROM sync_local_authority WHERE key=?')
|
|
61
|
+
.get(key)?.value;
|
|
62
|
+
const stored = readMeta('state') ?? 'local';
|
|
63
|
+
if (stored !== 'local' && stored !== 'attaching' && stored !== 'attached') {
|
|
64
|
+
throw new SyncError('sync.internal_error', 'the local authority state is unreadable', JSON.stringify({ state: stored }));
|
|
65
|
+
}
|
|
66
|
+
let state = stored;
|
|
67
|
+
const compiled = compileSchema(schema);
|
|
68
|
+
// §12.3: an attached authority keeps only its state across reopens, so it
|
|
69
|
+
// opens no server storage; the storage constructor creates the server tables.
|
|
70
|
+
const storage = stored === 'attached' ? undefined : new SqliteServerStorage(database);
|
|
71
|
+
if (storage !== undefined) {
|
|
72
|
+
// §12.2: the authority reaches the running schema before it serves.
|
|
73
|
+
await storage.ensureSchema(compiled);
|
|
74
|
+
if ((await storage.getPartitionLogEpoch(LOCAL)) !== undefined) {
|
|
75
|
+
await pruneCommitLog({ storage, partition: LOCAL, nowMs: clock() });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const allowed = Object.fromEntries([...compiled.declaredVariables].map((variable) => [variable, ['*']]));
|
|
79
|
+
const ctx = storage === undefined
|
|
80
|
+
? undefined
|
|
81
|
+
: {
|
|
82
|
+
partition: LOCAL,
|
|
83
|
+
actorId: LOCAL,
|
|
84
|
+
schema,
|
|
85
|
+
storage,
|
|
86
|
+
segments: new MemorySegmentStore(),
|
|
87
|
+
resolveScopes: () => allowed,
|
|
88
|
+
clock,
|
|
89
|
+
...(options.crdtMergers !== undefined
|
|
90
|
+
? { crdtMergers: options.crdtMergers }
|
|
91
|
+
: {}),
|
|
92
|
+
...(options.sqliteImageBuilder !== undefined
|
|
93
|
+
? { sqliteImageBuilder: options.sqliteImageBuilder }
|
|
94
|
+
: {}),
|
|
95
|
+
};
|
|
96
|
+
let uploading = false;
|
|
97
|
+
const refuseDuringUpload = () => {
|
|
98
|
+
if (uploading) {
|
|
99
|
+
throw new SyncError('sync.invalid_request', 'an attach upload is already running');
|
|
100
|
+
}
|
|
101
|
+
};
|
|
102
|
+
const inflight = new Set();
|
|
103
|
+
const admit = (work) => {
|
|
104
|
+
if (state !== 'local' || ctx === undefined) {
|
|
105
|
+
return Promise.reject(new SyncError('sync.authority_unavailable', 'the local authority serves no requests', JSON.stringify({ state })));
|
|
106
|
+
}
|
|
107
|
+
const promise = work(ctx);
|
|
108
|
+
const settle = () => inflight.delete(promise);
|
|
109
|
+
inflight.add(promise);
|
|
110
|
+
promise.then(settle, settle);
|
|
111
|
+
return promise;
|
|
112
|
+
};
|
|
113
|
+
/** One transaction; a `null` value removes the key. */
|
|
114
|
+
const writeMeta = (entries) => {
|
|
115
|
+
database.exec('BEGIN IMMEDIATE');
|
|
116
|
+
try {
|
|
117
|
+
for (const [key, value] of Object.entries(entries)) {
|
|
118
|
+
if (value === null) {
|
|
119
|
+
database.run('DELETE FROM sync_local_authority WHERE key=?', [key]);
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
database.run('INSERT OR REPLACE INTO sync_local_authority(key,value) VALUES (?,?)', [key, value]);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
database.exec('COMMIT');
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
database.exec('ROLLBACK');
|
|
129
|
+
throw error;
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
const ordinal = (key) => Number(readMeta(key) ?? -1);
|
|
133
|
+
const attachIdOf = () => {
|
|
134
|
+
const attachId = readMeta('attach_id');
|
|
135
|
+
if (attachId === undefined) {
|
|
136
|
+
throw new SyncError('sync.internal_error', 'the attaching local authority has no attach id');
|
|
137
|
+
}
|
|
138
|
+
return attachId;
|
|
139
|
+
};
|
|
140
|
+
const exchange = async (transport, frames) => {
|
|
141
|
+
const message = decodeMessage(await transport(encodeMessage({
|
|
142
|
+
wireVersion: PROTOCOL_WIRE_VERSION,
|
|
143
|
+
msgKind: 'request',
|
|
144
|
+
frames,
|
|
145
|
+
})));
|
|
146
|
+
if (message.msgKind !== 'response') {
|
|
147
|
+
throw new LocalAuthorityAttachError({
|
|
148
|
+
reason: 'request_failed',
|
|
149
|
+
code: 'sync.invalid_request',
|
|
150
|
+
retryable: false,
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
const header = message.frames[0];
|
|
154
|
+
if (header?.type === 'RESP_HEADER' && header.requiredSchemaVersion) {
|
|
155
|
+
throw new LocalAuthorityAttachError({
|
|
156
|
+
reason: 'schema_floor',
|
|
157
|
+
requiredSchemaVersion: header.requiredSchemaVersion,
|
|
158
|
+
...(header.latestSchemaVersion !== undefined
|
|
159
|
+
? { latestSchemaVersion: header.latestSchemaVersion }
|
|
160
|
+
: {}),
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
const error = message.frames.find((frame) => frame.type === 'ERROR');
|
|
164
|
+
if (error !== undefined) {
|
|
165
|
+
throw new LocalAuthorityAttachError({
|
|
166
|
+
reason: 'request_failed',
|
|
167
|
+
code: error.code,
|
|
168
|
+
retryable: error.retryable,
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
return message.frames;
|
|
172
|
+
};
|
|
173
|
+
/** Acquire the remote `logEpoch` (§2.1), then send one commit per request. */
|
|
174
|
+
const connect = async (transport, clientId) => {
|
|
175
|
+
const acquired = (await exchange(transport, [
|
|
176
|
+
{ type: 'REQ_HEADER', clientId, schemaVersion: schema.version },
|
|
177
|
+
{
|
|
178
|
+
type: 'PULL_HEADER',
|
|
179
|
+
limitCommits: 0,
|
|
180
|
+
limitSnapshotRows: 0,
|
|
181
|
+
maxSnapshotPages: 0,
|
|
182
|
+
accept: 0b0011,
|
|
183
|
+
},
|
|
184
|
+
]))[0];
|
|
185
|
+
const logEpoch = acquired?.type === 'RESP_HEADER' ? acquired.logEpoch : undefined;
|
|
186
|
+
if (logEpoch === undefined) {
|
|
187
|
+
throw new LocalAuthorityAttachError({
|
|
188
|
+
reason: 'request_failed',
|
|
189
|
+
code: 'sync.invalid_request',
|
|
190
|
+
retryable: false,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
return async (commit) => {
|
|
194
|
+
const frames = await exchange(transport, [
|
|
195
|
+
{
|
|
196
|
+
type: 'REQ_HEADER',
|
|
197
|
+
clientId,
|
|
198
|
+
schemaVersion: schema.version,
|
|
199
|
+
logEpoch,
|
|
200
|
+
},
|
|
201
|
+
commit,
|
|
202
|
+
]);
|
|
203
|
+
const header = frames[0];
|
|
204
|
+
if (header?.type === 'RESP_HEADER' && header.resetRequired === true) {
|
|
205
|
+
throw new LocalAuthorityAttachError({ reason: 'log_epoch_changed' });
|
|
206
|
+
}
|
|
207
|
+
const result = frames.find((frame) => frame.type === 'PUSH_RESULT');
|
|
208
|
+
if (result === undefined) {
|
|
209
|
+
throw new LocalAuthorityAttachError({
|
|
210
|
+
reason: 'request_failed',
|
|
211
|
+
code: 'sync.invalid_request',
|
|
212
|
+
retryable: false,
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
return result;
|
|
216
|
+
};
|
|
217
|
+
};
|
|
218
|
+
const applied = (result) => result.status === 'applied' || result.status === 'cached';
|
|
219
|
+
const rejected = (commit, result) => {
|
|
220
|
+
const failed = result.results.find((r) => r.status !== 'applied');
|
|
221
|
+
return new LocalAuthorityAttachError({
|
|
222
|
+
reason: 'commit_rejected',
|
|
223
|
+
clientCommitId: commit.clientCommitId,
|
|
224
|
+
opIndex: failed?.opIndex ?? 0,
|
|
225
|
+
code: failed?.code ?? 'sync.invalid_request',
|
|
226
|
+
});
|
|
227
|
+
};
|
|
228
|
+
const authority = {
|
|
229
|
+
get state() {
|
|
230
|
+
return state;
|
|
231
|
+
},
|
|
232
|
+
sync: (request) => admit((ctx) => handleSyncRequest(request, ctx)),
|
|
233
|
+
downloadSegment: (request) => admit(async (ctx) => {
|
|
234
|
+
const result = await handleSegmentDownload(ctx, {
|
|
235
|
+
segmentId: request.segmentId,
|
|
236
|
+
scopesHeader: request.requestedScopesJson,
|
|
237
|
+
});
|
|
238
|
+
return result.bytes;
|
|
239
|
+
}),
|
|
240
|
+
async attach(transport) {
|
|
241
|
+
refuseDuringUpload();
|
|
242
|
+
if (state === 'attached')
|
|
243
|
+
return;
|
|
244
|
+
uploading = true;
|
|
245
|
+
try {
|
|
246
|
+
await upload(transport);
|
|
247
|
+
}
|
|
248
|
+
finally {
|
|
249
|
+
uploading = false;
|
|
250
|
+
}
|
|
251
|
+
},
|
|
252
|
+
async abandonAttach(transport) {
|
|
253
|
+
refuseDuringUpload();
|
|
254
|
+
if (state === 'attached') {
|
|
255
|
+
throw new SyncError('sync.authority_unavailable', 'the local authority serves no requests', JSON.stringify({ state }));
|
|
256
|
+
}
|
|
257
|
+
if (state === 'local')
|
|
258
|
+
return;
|
|
259
|
+
if (ordinal('sent') >= 0) {
|
|
260
|
+
if (transport === undefined) {
|
|
261
|
+
throw new SyncError('sync.invalid_request', 'abandoning an attach that reached the remote server needs its transport');
|
|
262
|
+
}
|
|
263
|
+
uploading = true;
|
|
264
|
+
try {
|
|
265
|
+
await rollback(transport);
|
|
266
|
+
}
|
|
267
|
+
finally {
|
|
268
|
+
uploading = false;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
writeMeta({ state: 'local', attach_id: null, sent: null, applied: null });
|
|
272
|
+
state = 'local';
|
|
273
|
+
},
|
|
274
|
+
};
|
|
275
|
+
return authority;
|
|
276
|
+
async function upload(transport) {
|
|
277
|
+
if (state === 'local') {
|
|
278
|
+
// Refuse new rounds first, then let admitted rounds finish: the plan
|
|
279
|
+
// reads the authority database on the connection a round writes on.
|
|
280
|
+
state = 'attaching';
|
|
281
|
+
try {
|
|
282
|
+
await Promise.allSettled([...inflight]);
|
|
283
|
+
writeMeta({ state: 'attaching', attach_id: crypto.randomUUID() });
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
state = 'local';
|
|
287
|
+
throw error;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
const attachId = attachIdOf();
|
|
291
|
+
const commits = planAttach(database, compiled, attachId);
|
|
292
|
+
const send = await connect(transport, `authority-${attachId}`);
|
|
293
|
+
for (const [index, commit] of commits.entries()) {
|
|
294
|
+
// §12.3: `sent` before the request, `applied` after its result, so an
|
|
295
|
+
// abandon knows which commits may have reached the remote server.
|
|
296
|
+
writeMeta({ sent: String(Math.max(index, ordinal('sent'))) });
|
|
297
|
+
const result = await send(commit);
|
|
298
|
+
if (!applied(result))
|
|
299
|
+
throw rejected(commit, result);
|
|
300
|
+
writeMeta({ applied: String(Math.max(index, ordinal('applied'))) });
|
|
301
|
+
}
|
|
302
|
+
writeMeta({ state: 'attached', sent: null, applied: null });
|
|
303
|
+
state = 'attached';
|
|
304
|
+
// §12.3: the retired authority keeps only its state.
|
|
305
|
+
const tables = database
|
|
306
|
+
.query("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name <> 'sync_local_authority'")
|
|
307
|
+
.all();
|
|
308
|
+
for (const { name } of tables) {
|
|
309
|
+
database.exec(`DROP TABLE IF EXISTS "${name.replaceAll('"', '""')}"`);
|
|
310
|
+
}
|
|
311
|
+
database.exec('VACUUM');
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* §12.3 abandon: settle the one commit whose outcome may be unknown, then
|
|
315
|
+
* delete the rows of every applied commit, children before parents.
|
|
316
|
+
*/
|
|
317
|
+
async function rollback(transport) {
|
|
318
|
+
const attachId = attachIdOf();
|
|
319
|
+
const commits = planAttach(database, compiled, attachId);
|
|
320
|
+
const send = await connect(transport, `authority-${attachId}`);
|
|
321
|
+
let through = ordinal('applied');
|
|
322
|
+
const sent = ordinal('sent');
|
|
323
|
+
if (sent > through) {
|
|
324
|
+
const commit = commits[sent];
|
|
325
|
+
if (applied(await send(commit)))
|
|
326
|
+
through = sent;
|
|
327
|
+
writeMeta({ applied: String(through) });
|
|
328
|
+
}
|
|
329
|
+
const deletes = commits
|
|
330
|
+
.slice(0, through + 1)
|
|
331
|
+
.flatMap((commit) => commit.operations)
|
|
332
|
+
.reverse()
|
|
333
|
+
.map((operation) => ({
|
|
334
|
+
table: operation.table,
|
|
335
|
+
rowId: operation.rowId,
|
|
336
|
+
op: 'delete',
|
|
337
|
+
}));
|
|
338
|
+
for (let start = 0; start < deletes.length; start += ATTACH_OPERATION_BUDGET) {
|
|
339
|
+
const commit = {
|
|
340
|
+
type: 'PUSH_COMMIT',
|
|
341
|
+
clientCommitId: `${attachId}:undo:${start / ATTACH_OPERATION_BUDGET}`,
|
|
342
|
+
operations: deletes.slice(start, start + ATTACH_OPERATION_BUDGET),
|
|
343
|
+
};
|
|
344
|
+
const result = await send(commit);
|
|
345
|
+
if (!applied(result))
|
|
346
|
+
throw rejected(commit, result);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* §12.3 upload plan: every live row as one insert, rows visited in
|
|
352
|
+
* (table name, row id) byte order, strongly connected components of the
|
|
353
|
+
* row-level reference graph emitted in Tarjan completion order (every
|
|
354
|
+
* parent before its children) and packed whole into commits.
|
|
355
|
+
*/
|
|
356
|
+
function planAttach(database, schema, attachId) {
|
|
357
|
+
const tables = [...schema.tables.values()].sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
|
|
358
|
+
const nodeTable = [];
|
|
359
|
+
const nodeRowId = [];
|
|
360
|
+
const nodeValues = [];
|
|
361
|
+
const index = new Map();
|
|
362
|
+
for (const table of tables) {
|
|
363
|
+
const rows = database
|
|
364
|
+
.query(`SELECT _sync_row_id AS row_id, _sync_payload AS payload
|
|
365
|
+
FROM "${table.name.replaceAll('"', '""')}"
|
|
366
|
+
WHERE _sync_partition=? ORDER BY _sync_row_id`)
|
|
367
|
+
.all(LOCAL);
|
|
368
|
+
for (const row of rows) {
|
|
369
|
+
index.set(`${table.name}\u0000${row.row_id}`, nodeRowId.length);
|
|
370
|
+
nodeTable.push(table);
|
|
371
|
+
nodeRowId.push(row.row_id);
|
|
372
|
+
nodeValues.push(decodeRow(table.columns, new Uint8Array(row.payload)));
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
const parents = nodeValues.map((values, node) => {
|
|
376
|
+
const edges = [];
|
|
377
|
+
for (const reference of nodeTable[node].references) {
|
|
378
|
+
const parentId = renderScopeValue(values[reference.columnIndex]);
|
|
379
|
+
const parent = parentId === undefined
|
|
380
|
+
? undefined
|
|
381
|
+
: index.get(`${reference.parentTable}\u0000${parentId}`);
|
|
382
|
+
if (parent !== undefined)
|
|
383
|
+
edges.push(parent);
|
|
384
|
+
}
|
|
385
|
+
return edges;
|
|
386
|
+
});
|
|
387
|
+
// Iterative Tarjan: a deep parent chain must not exhaust the call stack.
|
|
388
|
+
const order = new Array(nodeRowId.length).fill(-1);
|
|
389
|
+
const low = new Array(nodeRowId.length).fill(0);
|
|
390
|
+
const onStack = new Array(nodeRowId.length).fill(false);
|
|
391
|
+
const stack = [];
|
|
392
|
+
const components = [];
|
|
393
|
+
let counter = 0;
|
|
394
|
+
for (let root = 0; root < nodeRowId.length; root++) {
|
|
395
|
+
if (order[root] !== -1)
|
|
396
|
+
continue;
|
|
397
|
+
const frames = [{ node: root, edge: 0 }];
|
|
398
|
+
order[root] = low[root] = counter++;
|
|
399
|
+
stack.push(root);
|
|
400
|
+
onStack[root] = true;
|
|
401
|
+
while (frames.length > 0) {
|
|
402
|
+
const frame = frames[frames.length - 1];
|
|
403
|
+
const edges = parents[frame.node];
|
|
404
|
+
if (frame.edge < edges.length) {
|
|
405
|
+
const next = edges[frame.edge++];
|
|
406
|
+
if (order[next] === -1) {
|
|
407
|
+
order[next] = low[next] = counter++;
|
|
408
|
+
stack.push(next);
|
|
409
|
+
onStack[next] = true;
|
|
410
|
+
frames.push({ node: next, edge: 0 });
|
|
411
|
+
}
|
|
412
|
+
else if (onStack[next]) {
|
|
413
|
+
low[frame.node] = Math.min(low[frame.node], order[next]);
|
|
414
|
+
}
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
frames.pop();
|
|
418
|
+
const caller = frames[frames.length - 1];
|
|
419
|
+
if (caller !== undefined) {
|
|
420
|
+
low[caller.node] = Math.min(low[caller.node], low[frame.node]);
|
|
421
|
+
}
|
|
422
|
+
if (low[frame.node] !== order[frame.node])
|
|
423
|
+
continue;
|
|
424
|
+
const component = [];
|
|
425
|
+
let member;
|
|
426
|
+
do {
|
|
427
|
+
member = stack.pop();
|
|
428
|
+
onStack[member] = false;
|
|
429
|
+
component.push(member);
|
|
430
|
+
} while (member !== frame.node);
|
|
431
|
+
components.push(component.sort((a, b) => a - b));
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
const commits = [];
|
|
435
|
+
let operations = [];
|
|
436
|
+
const flush = () => {
|
|
437
|
+
if (operations.length === 0)
|
|
438
|
+
return;
|
|
439
|
+
commits.push({
|
|
440
|
+
type: 'PUSH_COMMIT',
|
|
441
|
+
clientCommitId: `${attachId}:${commits.length}`,
|
|
442
|
+
operations,
|
|
443
|
+
});
|
|
444
|
+
operations = [];
|
|
445
|
+
};
|
|
446
|
+
for (const component of components) {
|
|
447
|
+
if (component.length > ATTACH_OPERATION_BUDGET) {
|
|
448
|
+
throw new LocalAuthorityAttachError({
|
|
449
|
+
reason: 'cycle_too_large',
|
|
450
|
+
table: nodeTable[component[0]].name,
|
|
451
|
+
size: component.length,
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
if (operations.length + component.length > ATTACH_OPERATION_BUDGET) {
|
|
455
|
+
flush();
|
|
456
|
+
}
|
|
457
|
+
for (const node of component) {
|
|
458
|
+
const table = nodeTable[node];
|
|
459
|
+
operations.push({
|
|
460
|
+
table: table.name,
|
|
461
|
+
rowId: nodeRowId[node],
|
|
462
|
+
op: 'upsert',
|
|
463
|
+
baseVersion: 0,
|
|
464
|
+
payload: encodeSparseRow(table.columns, table.primaryKeyIndex, nodeValues[node]),
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
flush();
|
|
469
|
+
return commits;
|
|
470
|
+
}
|
package/dist/operations.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { type AuthoritativeRelationPlan } from './authoritative-query.js';
|
|
|
2
2
|
import { type RemoteOperationRequest, type RemoteOperationResponse, type RowValue } from '@syncular/core';
|
|
3
3
|
import type { ResolveScopes, SyncRequestContext } from './context.js';
|
|
4
4
|
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, AuthoritativeQueryValue } from './storage.js';
|
|
5
|
-
import { type ValidateRow } from './validate.js';
|
|
5
|
+
import { type IndexScanInput, type ValidateRow } from './validate.js';
|
|
6
6
|
export interface RemoteQueryDependency {
|
|
7
7
|
readonly table: string;
|
|
8
8
|
readonly scopeKeys?: readonly string[];
|
|
@@ -76,6 +76,15 @@ export interface RemoteCommandContext {
|
|
|
76
76
|
* D1 issue one lookup per id.
|
|
77
77
|
*/
|
|
78
78
|
getRows(table: string, rowIds: readonly string[]): Promise<ReadonlyMap<string, ValidateRow>>;
|
|
79
|
+
/**
|
|
80
|
+
* Exact lookup through one declared relational index on the command's push
|
|
81
|
+
* transaction. The values form the complete, order-sensitive index tuple
|
|
82
|
+
* and `afterRowId` is an exclusive keyset cursor. Unlike `getRow` and
|
|
83
|
+
* `getRows`, the rows are not filtered by the actor's scopes: `authorize`
|
|
84
|
+
* gates the command, and this is the same trusted server-host lookup
|
|
85
|
+
* `ServerStorage.scanRowsByIndex` performs.
|
|
86
|
+
*/
|
|
87
|
+
scanRowsByIndex(input: IndexScanInput): Promise<ValidateRow[]>;
|
|
79
88
|
/**
|
|
80
89
|
* Run a generated registered query on the command's push transaction
|
|
81
90
|
* (§6.7). The storage binds every relation to the partition; unlike
|
package/dist/operations.js
CHANGED
|
@@ -314,6 +314,23 @@ function commandContext(ctx, tx, resolved, schema, clientId, operationId, reques
|
|
|
314
314
|
}
|
|
315
315
|
return rows;
|
|
316
316
|
},
|
|
317
|
+
scanRowsByIndex: async ({ table: tableName, index, values, afterRowId = null, limit = 100, }) => {
|
|
318
|
+
const table = schema.tables.get(tableName);
|
|
319
|
+
if (table === undefined) {
|
|
320
|
+
throw syncError('operation.invalid_request', 'command read targets an unknown table');
|
|
321
|
+
}
|
|
322
|
+
if (tx.scanRowsByIndex === undefined) {
|
|
323
|
+
throw new Error('storage transaction does not support command index scans');
|
|
324
|
+
}
|
|
325
|
+
const rows = await tx.scanRowsByIndex({
|
|
326
|
+
table: tableName,
|
|
327
|
+
index,
|
|
328
|
+
values,
|
|
329
|
+
afterRowId,
|
|
330
|
+
limit,
|
|
331
|
+
});
|
|
332
|
+
return rows.map((stored) => toValidateRow(table.columns, decodeRow(table.columns, stored.payload)));
|
|
333
|
+
},
|
|
317
334
|
queryAuthoritative: (query) => transactionQuery(tx, query),
|
|
318
335
|
};
|
|
319
336
|
}
|