@syncular/server 0.57.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
type RemoteOperationRealtimeMessage,
|
|
5
5
|
} from '@syncular/core';
|
|
6
6
|
import type { RealtimeNotifier, SyncRequestContext } from './context';
|
|
7
|
-
import { REMOTE_COMMAND_CLIENT_ID_PREFIX } from './context';
|
|
7
|
+
import { limitsOf, REMOTE_COMMAND_CLIENT_ID_PREFIX } from './context';
|
|
8
8
|
import { reportError, SyncError, syncError } from './errors';
|
|
9
9
|
import type {
|
|
10
10
|
RegisteredRemoteQuery,
|
|
@@ -22,16 +22,24 @@ interface WatchState {
|
|
|
22
22
|
readonly clientId: string;
|
|
23
23
|
readonly operation: RegisteredRemoteQuery;
|
|
24
24
|
readonly params: unknown;
|
|
25
|
-
running: boolean;
|
|
26
|
-
dirty: boolean;
|
|
27
25
|
}
|
|
28
26
|
|
|
27
|
+
/**
|
|
28
|
+
* One socket's registered-query watches. A session runs one refresh at a
|
|
29
|
+
* time: `notify` queues each affected watch once, and the drain reruns the
|
|
30
|
+
* queue in order, so a commit costs at most one query per watch and never
|
|
31
|
+
* more than one query at a time.
|
|
32
|
+
*/
|
|
29
33
|
class WatchSession implements RemoteOperationWatchSession {
|
|
30
34
|
readonly #ctx: SyncRequestContext;
|
|
31
35
|
readonly #registry: RemoteOperationRegistry;
|
|
32
36
|
readonly #send: (bytes: Uint8Array) => void;
|
|
33
37
|
readonly #closed: () => void;
|
|
38
|
+
readonly #maxWatches: number;
|
|
34
39
|
readonly #watches = new Map<string, WatchState>();
|
|
40
|
+
/** Watches waiting for a refresh, in arrival order, each at most once. */
|
|
41
|
+
readonly #pending = new Set<WatchState>();
|
|
42
|
+
#draining: Promise<void> | undefined;
|
|
35
43
|
#isClosed = false;
|
|
36
44
|
|
|
37
45
|
get partition(): string {
|
|
@@ -48,6 +56,7 @@ class WatchSession implements RemoteOperationWatchSession {
|
|
|
48
56
|
this.#registry = registry;
|
|
49
57
|
this.#send = send;
|
|
50
58
|
this.#closed = closed;
|
|
59
|
+
this.#maxWatches = limitsOf(ctx).maxWatchesPerSession;
|
|
51
60
|
}
|
|
52
61
|
|
|
53
62
|
async receive(bytes: Uint8Array): Promise<void> {
|
|
@@ -80,6 +89,8 @@ class WatchSession implements RemoteOperationWatchSession {
|
|
|
80
89
|
throw syncError('operation.invalid_request');
|
|
81
90
|
}
|
|
82
91
|
if (message.kind === 'unwatch') {
|
|
92
|
+
const state = this.#watches.get(message.watchId);
|
|
93
|
+
if (state !== undefined) this.#pending.delete(state);
|
|
83
94
|
this.#watches.delete(message.watchId);
|
|
84
95
|
return;
|
|
85
96
|
}
|
|
@@ -122,13 +133,25 @@ class WatchSession implements RemoteOperationWatchSession {
|
|
|
122
133
|
this.#sendError(message.watchId, syncError('operation.unknown'));
|
|
123
134
|
return;
|
|
124
135
|
}
|
|
136
|
+
if (this.#watches.size >= this.#maxWatches) {
|
|
137
|
+
this.#sendError(
|
|
138
|
+
message.watchId,
|
|
139
|
+
syncError(
|
|
140
|
+
'sync.limit_exceeded',
|
|
141
|
+
'watch session holds the maximum number of watches',
|
|
142
|
+
JSON.stringify({
|
|
143
|
+
limit: 'maxWatchesPerSession',
|
|
144
|
+
max: this.#maxWatches,
|
|
145
|
+
}),
|
|
146
|
+
),
|
|
147
|
+
);
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
125
150
|
const state: WatchState = {
|
|
126
151
|
watchId: message.watchId,
|
|
127
152
|
clientId: message.clientId,
|
|
128
153
|
operation,
|
|
129
154
|
params: message.params,
|
|
130
|
-
running: false,
|
|
131
|
-
dirty: false,
|
|
132
155
|
};
|
|
133
156
|
this.#watches.set(message.watchId, state);
|
|
134
157
|
await this.#refresh(state);
|
|
@@ -136,68 +159,68 @@ class WatchSession implements RemoteOperationWatchSession {
|
|
|
136
159
|
|
|
137
160
|
notify(tables: ReadonlySet<string>): void {
|
|
138
161
|
for (const state of this.#watches.values()) {
|
|
139
|
-
if (
|
|
140
|
-
|
|
141
|
-
|
|
162
|
+
if (state.operation.tables.some((table) => tables.has(table))) {
|
|
163
|
+
void this.#refresh(state);
|
|
164
|
+
}
|
|
142
165
|
}
|
|
143
166
|
}
|
|
144
167
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
168
|
+
/** Queue `state` and resolve when the session's queue has drained. */
|
|
169
|
+
#refresh(state: WatchState): Promise<void> {
|
|
170
|
+
this.#pending.add(state);
|
|
171
|
+
this.#draining ??= this.#drain();
|
|
172
|
+
return this.#draining;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
async #drain(): Promise<void> {
|
|
176
|
+
try {
|
|
177
|
+
// A watch queued again while it runs is appended and runs once more.
|
|
178
|
+
for (const state of this.#pending) {
|
|
179
|
+
this.#pending.delete(state);
|
|
180
|
+
await this.#run(state);
|
|
181
|
+
}
|
|
182
|
+
} finally {
|
|
183
|
+
this.#draining = undefined;
|
|
150
184
|
}
|
|
151
|
-
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async #run(state: WatchState): Promise<void> {
|
|
188
|
+
if (this.#isClosed || this.#watches.get(state.watchId) !== state) return;
|
|
152
189
|
try {
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
}
|
|
187
|
-
this.#sendError(
|
|
188
|
-
state.watchId,
|
|
189
|
-
error instanceof SyncError
|
|
190
|
-
? error
|
|
191
|
-
: syncError('operation.query_failed'),
|
|
192
|
-
);
|
|
193
|
-
}
|
|
194
|
-
} while (
|
|
195
|
-
state.dirty &&
|
|
196
|
-
!this.#isClosed &&
|
|
197
|
-
this.#watches.get(state.watchId) === state
|
|
190
|
+
const response = await state.operation.run(
|
|
191
|
+
this.#ctx,
|
|
192
|
+
state.clientId,
|
|
193
|
+
state.params,
|
|
194
|
+
);
|
|
195
|
+
if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
if (response.kind !== 'query') {
|
|
199
|
+
throw syncError('operation.query_failed');
|
|
200
|
+
}
|
|
201
|
+
this.#emit(
|
|
202
|
+
encodeRemoteOperationRealtimeMessage({
|
|
203
|
+
revision: 1,
|
|
204
|
+
kind: 'snapshot',
|
|
205
|
+
watchId: state.watchId,
|
|
206
|
+
operationId: response.operationId,
|
|
207
|
+
rows: response.rows,
|
|
208
|
+
maxCommitSeq: response.maxCommitSeq,
|
|
209
|
+
}),
|
|
210
|
+
);
|
|
211
|
+
} catch (error) {
|
|
212
|
+
if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
if (!(error instanceof SyncError)) {
|
|
216
|
+
reportError(error, this.#ctx.onError, 'operations');
|
|
217
|
+
}
|
|
218
|
+
this.#sendError(
|
|
219
|
+
state.watchId,
|
|
220
|
+
error instanceof SyncError
|
|
221
|
+
? error
|
|
222
|
+
: syncError('operation.query_failed'),
|
|
198
223
|
);
|
|
199
|
-
} finally {
|
|
200
|
-
state.running = false;
|
|
201
224
|
}
|
|
202
225
|
}
|
|
203
226
|
|
|
@@ -229,6 +252,7 @@ class WatchSession implements RemoteOperationWatchSession {
|
|
|
229
252
|
if (this.#isClosed) return;
|
|
230
253
|
this.#isClosed = true;
|
|
231
254
|
this.#watches.clear();
|
|
255
|
+
this.#pending.clear();
|
|
232
256
|
this.#closed();
|
|
233
257
|
}
|
|
234
258
|
}
|
|
@@ -9,21 +9,18 @@ import { retainRollbackFailure } from './storage-errors';
|
|
|
9
9
|
* `PgExecutor` interface (see the server README).
|
|
10
10
|
*
|
|
11
11
|
* pglite is single-connection, so `transaction` runs `BEGIN`/`COMMIT`/
|
|
12
|
-
* `ROLLBACK` on the one connection.
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
12
|
+
* `ROLLBACK` on the one connection. Transactions and pool-level `query` calls
|
|
13
|
+
* share one promise chain (mirroring `SqliteServerStorage`): a nested BEGIN
|
|
14
|
+
* on Postgres is a warning-level no-op, so interleaved scopes would silently
|
|
15
|
+
* collapse into one SQL transaction, and a pool-level statement issued while
|
|
16
|
+
* a transaction is open would read its uncommitted rows or join it (§6.4).
|
|
17
17
|
*/
|
|
18
18
|
import type { PGlite } from '@electric-sql/pglite';
|
|
19
19
|
import type { PgExecutor, PgQueryable, PgRow } from './pg-executor';
|
|
20
20
|
|
|
21
21
|
/** Minimal structural type for a pglite instance (avoids a hard type dep). */
|
|
22
22
|
interface PgliteLike {
|
|
23
|
-
query<Row>(
|
|
24
|
-
query: string,
|
|
25
|
-
params?: unknown[],
|
|
26
|
-
): Promise<{ rows: Row[]; affectedRows?: number }>;
|
|
23
|
+
query<Row>(query: string, params?: unknown[]): Promise<{ rows: Row[] }>;
|
|
27
24
|
exec(query: string): Promise<unknown>;
|
|
28
25
|
close(): Promise<void>;
|
|
29
26
|
}
|
|
@@ -35,10 +32,7 @@ function queryable(db: PgliteLike): PgQueryable {
|
|
|
35
32
|
text,
|
|
36
33
|
params ? [...params] : undefined,
|
|
37
34
|
);
|
|
38
|
-
return {
|
|
39
|
-
rows: result.rows,
|
|
40
|
-
rowCount: result.affectedRows ?? result.rows.length,
|
|
41
|
-
};
|
|
35
|
+
return { rows: result.rows };
|
|
42
36
|
},
|
|
43
37
|
};
|
|
44
38
|
}
|
|
@@ -46,18 +40,28 @@ function queryable(db: PgliteLike): PgQueryable {
|
|
|
46
40
|
export function pgliteExecutor(db: PGlite | PgliteLike): PgExecutor {
|
|
47
41
|
const like = db as PgliteLike;
|
|
48
42
|
const q = queryable(like);
|
|
49
|
-
// One
|
|
50
|
-
|
|
51
|
-
|
|
43
|
+
// One caller at a time on the single connection (see the file header).
|
|
44
|
+
let tail: Promise<void> = Promise.resolve();
|
|
45
|
+
const acquire = async (): Promise<() => void> => {
|
|
46
|
+
const previous = tail;
|
|
47
|
+
let release!: () => void;
|
|
48
|
+
tail = new Promise<void>((resolve) => {
|
|
49
|
+
release = resolve;
|
|
50
|
+
});
|
|
51
|
+
await previous;
|
|
52
|
+
return release;
|
|
53
|
+
};
|
|
52
54
|
return {
|
|
53
|
-
query:
|
|
55
|
+
async query<Row = PgRow>(text: string, params?: readonly unknown[]) {
|
|
56
|
+
const release = await acquire();
|
|
57
|
+
try {
|
|
58
|
+
return await q.query<Row>(text, params);
|
|
59
|
+
} finally {
|
|
60
|
+
release();
|
|
61
|
+
}
|
|
62
|
+
},
|
|
54
63
|
async transaction<T>(fn: (client: PgQueryable) => Promise<T>): Promise<T> {
|
|
55
|
-
const
|
|
56
|
-
let release!: () => void;
|
|
57
|
-
transactionTail = new Promise<void>((resolve) => {
|
|
58
|
-
release = resolve;
|
|
59
|
-
});
|
|
60
|
-
await previous;
|
|
64
|
+
const release = await acquire();
|
|
61
65
|
try {
|
|
62
66
|
await like.exec('BEGIN');
|
|
63
67
|
try {
|
package/src/pg-executor.ts
CHANGED
|
@@ -49,12 +49,16 @@ export type PgRow = Record<string, unknown>;
|
|
|
49
49
|
* a time. An adapter does not need to queue calls itself, and must not
|
|
50
50
|
* accept two in flight: node-postgres warns about that today and rejects it
|
|
51
51
|
* in node-postgres 9.
|
|
52
|
+
*
|
|
53
|
+
* The result carries returned rows only. Drivers disagree on the affected-row
|
|
54
|
+
* count of a statement without result rows (Bun.sql reports none), so every
|
|
55
|
+
* write whose outcome matters ends in `RETURNING` and counts `rows`.
|
|
52
56
|
*/
|
|
53
57
|
export interface PgQueryable {
|
|
54
58
|
query<Row = PgRow>(
|
|
55
59
|
text: string,
|
|
56
60
|
params?: readonly unknown[],
|
|
57
|
-
): Promise<{ rows: Row[]
|
|
61
|
+
): Promise<{ rows: Row[] }>;
|
|
58
62
|
}
|
|
59
63
|
|
|
60
64
|
/**
|