@syncular/server 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -36
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +46 -40
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +19 -5
- package/dist/context.js +19 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +249 -1119
- package/dist/errors.js +16 -0
- package/dist/events.d.ts +36 -18
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +503 -208
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +24 -12
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +420 -262
- package/dist/pull.d.ts +70 -47
- package/dist/pull.js +270 -301
- package/dist/push.d.ts +12 -3
- package/dist/push.js +361 -173
- package/dist/realtime.d.ts +85 -39
- package/dist/realtime.js +355 -219
- package/dist/relational-rows.d.ts +21 -66
- package/dist/relational-rows.js +28 -109
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +10 -16
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +22 -6
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +164 -84
- package/dist/sqlite-dialect.js +1145 -150
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +303 -1123
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +185 -63
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +99 -57
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +33 -6
- package/src/d1-storage.ts +350 -1700
- package/src/errors.ts +16 -0
- package/src/events.ts +37 -18
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +654 -261
- package/src/local-authority.ts +27 -14
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/operations.ts +44 -42
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +557 -354
- package/src/pull.ts +406 -440
- package/src/push.ts +466 -220
- package/src/realtime.ts +447 -290
- package/src/relational-rows.ts +43 -121
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +13 -20
- package/src/segment-download.ts +33 -7
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1851 -199
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +390 -1821
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +215 -67
package/src/sqlite-dialect.ts
CHANGED
|
@@ -1,29 +1,100 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared SQLite dialect for synchronous `SqliteServerStorage` on Bun or Node
|
|
3
3
|
* and asynchronous `D1ServerStorage` on Cloudflare Workers. D1 is SQLite:
|
|
4
|
-
* same DDL, same statement grammar,
|
|
5
|
-
*
|
|
6
|
-
* OR IGNORE` upsert idioms — so the schema and the value (de)serialization are
|
|
7
|
-
* genuinely common ground and live here.
|
|
4
|
+
* same DDL, same statement grammar, same `?` positional placeholders, same
|
|
5
|
+
* `INSERT ... ON CONFLICT` / `INSERT OR IGNORE` upsert idioms.
|
|
8
6
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
7
|
+
* `SqliteFamilyStorage` and `SqliteFamilyTransaction` hold every statement
|
|
8
|
+
* and row mapping the two backends share. A backend supplies statement
|
|
9
|
+
* execution through a small seam (`all`, `guardedAll`, `atomic`; `run` and
|
|
10
|
+
* `all` inside a transaction) and keeps only what differs: connection
|
|
11
|
+
* handling, the request lock in SQLite, transaction execution (interactive
|
|
12
|
+
* `BEGIN IMMEDIATE` in SQLite, one buffered `db.batch` in D1), D1's
|
|
13
|
+
* bound-parameter cap and its published-schema guard. The seam returns
|
|
14
|
+
* promises; the synchronous SQLite driver resolves them with rows it read at
|
|
15
|
+
* call time. Both classes run the identical `test/storage-contract.ts`.
|
|
17
16
|
*/
|
|
18
|
-
import type { PushOperationResult,
|
|
17
|
+
import type { PushOperationResult, UnitScopes } from '@syncular/core';
|
|
18
|
+
import {
|
|
19
|
+
bindAuthoritativePartition,
|
|
20
|
+
type BoundAuthoritativeQuery,
|
|
21
|
+
prepareAuthoritativeQuery,
|
|
22
|
+
} from './authoritative-query';
|
|
23
|
+
import { syncError } from './errors';
|
|
24
|
+
import { validateCommitPruneQuery } from './prune';
|
|
25
|
+
import { StorageQueryError } from './storage-errors';
|
|
26
|
+
import {
|
|
27
|
+
deleteRowSql,
|
|
28
|
+
deleteSqliteRowScopesSql,
|
|
29
|
+
indexRowPageStatement,
|
|
30
|
+
quoteIdent,
|
|
31
|
+
scanRowPageSql,
|
|
32
|
+
rowsReferencingBlobSql,
|
|
33
|
+
selectRowSql,
|
|
34
|
+
SYNC_COLUMN_VERSIONS_COLUMN,
|
|
35
|
+
SYNC_PARTITION_COLUMN,
|
|
36
|
+
SYNC_PAYLOAD_COLUMN,
|
|
37
|
+
SYNC_ROW_ID_COLUMN,
|
|
38
|
+
SYNC_SCOPES_COLUMN,
|
|
39
|
+
SYNC_VERSION_COLUMN,
|
|
40
|
+
upsertSql,
|
|
41
|
+
upsertValues,
|
|
42
|
+
} from './relational-rows';
|
|
43
|
+
import type { CompiledTable } from './schema';
|
|
19
44
|
import { matchesEffective } from './scopes';
|
|
45
|
+
import type { SqliteValue } from './sqlite-driver';
|
|
46
|
+
import {
|
|
47
|
+
clientRoundState,
|
|
48
|
+
PARTITION_TOUCH_INTERVAL_MS,
|
|
49
|
+
rowsLiveAfterDelete,
|
|
50
|
+
} from './storage';
|
|
20
51
|
import type {
|
|
52
|
+
AuthoritativeQueryRequest,
|
|
53
|
+
ClientCursorInfo,
|
|
54
|
+
ClientPushResult,
|
|
55
|
+
ClientRecord,
|
|
56
|
+
ClientRecordWriteOptions,
|
|
57
|
+
CommitChangeSummary,
|
|
58
|
+
CommitDetail,
|
|
59
|
+
CommitMetadata,
|
|
60
|
+
CommitMetadataQuery,
|
|
61
|
+
CommitPruneQuery,
|
|
62
|
+
CommitPruneResult,
|
|
63
|
+
DurableJsonValue,
|
|
64
|
+
IndexRowScanQuery,
|
|
65
|
+
LiveWindow,
|
|
66
|
+
LiveWindowQuery,
|
|
67
|
+
NewCommit,
|
|
68
|
+
OutboxSequence,
|
|
69
|
+
OutboxSettlement,
|
|
70
|
+
NewReaction,
|
|
71
|
+
PartitionRegistryEntry,
|
|
72
|
+
PrunedReactionCounts,
|
|
73
|
+
PushWatermark,
|
|
74
|
+
ReactionClaimQuery,
|
|
75
|
+
ReactionFailure,
|
|
76
|
+
ReactionFailureUpdate,
|
|
77
|
+
ReactionListQuery,
|
|
78
|
+
ReactionPruneQuery,
|
|
79
|
+
RowChangeRecord,
|
|
80
|
+
RowChangesQuery,
|
|
81
|
+
RowListQuery,
|
|
82
|
+
RowScanQuery,
|
|
83
|
+
ScopeActivityQuery,
|
|
84
|
+
ScopeCommitActivity,
|
|
21
85
|
ScopeValueHistoryQuery,
|
|
86
|
+
ScopeValueQuery,
|
|
22
87
|
StoredChange,
|
|
23
|
-
StoredCommit,
|
|
24
88
|
StoredPushResult,
|
|
89
|
+
StoredReaction,
|
|
25
90
|
StoredRow,
|
|
91
|
+
Tombstone,
|
|
26
92
|
} from './storage';
|
|
93
|
+
import {
|
|
94
|
+
assertScopeIndexedScan,
|
|
95
|
+
readLiveWindowPages,
|
|
96
|
+
resolveIndexRowScan,
|
|
97
|
+
} from './storage-query';
|
|
27
98
|
|
|
28
99
|
/**
|
|
29
100
|
* Schema DDL, one statement per `;`-delimited chunk. Native SQLite applies
|
|
@@ -38,17 +109,15 @@ import type {
|
|
|
38
109
|
* the Postgres storage documents (§3.1, performance-by-
|
|
39
110
|
* construction).
|
|
40
111
|
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* is the LAST PRIMARY KEY column, so that predicate cannot narrow the
|
|
44
|
-
* inverted index. Declared here so every SQLite-family storage (and D1,
|
|
45
|
-
* which applies this same DDL) keeps the same plan shape as Postgres.
|
|
112
|
+
* Changes to the core tables of an existing database (added and dropped
|
|
113
|
+
* columns and indexes) live in `sqliteUpgradeDdl`, which runs after this DDL.
|
|
46
114
|
*/
|
|
47
115
|
export const SQLITE_DDL = `
|
|
48
116
|
CREATE TABLE IF NOT EXISTS sync_partitions(
|
|
49
117
|
partition TEXT PRIMARY KEY,
|
|
50
118
|
max_commit_seq INTEGER NOT NULL DEFAULT 0,
|
|
51
|
-
horizon_seq INTEGER NOT NULL DEFAULT 0
|
|
119
|
+
horizon_seq INTEGER NOT NULL DEFAULT 0,
|
|
120
|
+
delete_floor INTEGER NOT NULL DEFAULT 0
|
|
52
121
|
);
|
|
53
122
|
CREATE TABLE IF NOT EXISTS sync_partition_registry(
|
|
54
123
|
partition TEXT PRIMARY KEY,
|
|
@@ -61,8 +130,6 @@ CREATE TABLE IF NOT EXISTS sync_row_scopes(
|
|
|
61
130
|
var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
62
131
|
PRIMARY KEY(partition, tbl, var, value, row_id)
|
|
63
132
|
);
|
|
64
|
-
CREATE INDEX IF NOT EXISTS sync_row_scopes_by_row
|
|
65
|
-
ON sync_row_scopes(partition, tbl, row_id);
|
|
66
133
|
CREATE TABLE IF NOT EXISTS sync_commits(
|
|
67
134
|
partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,
|
|
68
135
|
client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,
|
|
@@ -88,8 +155,14 @@ CREATE TABLE IF NOT EXISTS sync_change_scopes(
|
|
|
88
155
|
CREATE TABLE IF NOT EXISTS sync_push_results(
|
|
89
156
|
partition TEXT NOT NULL, client_id TEXT NOT NULL,
|
|
90
157
|
client_commit_id TEXT NOT NULL, result TEXT NOT NULL,
|
|
158
|
+
outbox_id TEXT, client_seq INTEGER,
|
|
91
159
|
PRIMARY KEY(partition, client_id, client_commit_id)
|
|
92
160
|
);
|
|
161
|
+
CREATE TABLE IF NOT EXISTS sync_push_watermarks(
|
|
162
|
+
partition TEXT NOT NULL, client_id TEXT NOT NULL, outbox_id TEXT NOT NULL,
|
|
163
|
+
through_seq INTEGER NOT NULL,
|
|
164
|
+
PRIMARY KEY(partition, client_id, outbox_id)
|
|
165
|
+
);
|
|
93
166
|
CREATE TABLE IF NOT EXISTS sync_reactions(
|
|
94
167
|
partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,
|
|
95
168
|
type TEXT NOT NULL, version INTEGER NOT NULL, payload TEXT NOT NULL,
|
|
@@ -113,10 +186,12 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
|
|
|
113
186
|
CREATE TABLE IF NOT EXISTS sync_clients(
|
|
114
187
|
partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
|
|
115
188
|
wire_version INTEGER NOT NULL DEFAULT 1,
|
|
116
|
-
cursor INTEGER NOT NULL,
|
|
189
|
+
cursor INTEGER NOT NULL,
|
|
117
190
|
updated_at_ms INTEGER NOT NULL,
|
|
118
191
|
last_round_at_ms INTEGER, last_error_code TEXT,
|
|
119
192
|
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
193
|
+
schema_version INTEGER, units TEXT, registration_digest BLOB,
|
|
194
|
+
catching_up TEXT NOT NULL DEFAULT '[]',
|
|
120
195
|
PRIMARY KEY(partition, client_id)
|
|
121
196
|
);
|
|
122
197
|
CREATE TABLE IF NOT EXISTS sync_blob_refs(
|
|
@@ -128,7 +203,7 @@ CREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob
|
|
|
128
203
|
ON sync_blob_refs(partition, blob_id);
|
|
129
204
|
CREATE TABLE IF NOT EXISTS sync_tombstones(
|
|
130
205
|
partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
131
|
-
commit_seq INTEGER NOT NULL,
|
|
206
|
+
commit_seq INTEGER NOT NULL, server_version INTEGER, scopes TEXT,
|
|
132
207
|
PRIMARY KEY(partition, tbl, row_id)
|
|
133
208
|
);
|
|
134
209
|
CREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(
|
|
@@ -147,62 +222,126 @@ CREATE TABLE IF NOT EXISTS sync_writer_fence(
|
|
|
147
222
|
);
|
|
148
223
|
`;
|
|
149
224
|
|
|
150
|
-
/**
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
225
|
+
/**
|
|
226
|
+
* Columns added to core tables after their first release: table, column,
|
|
227
|
+
* type. A client record from before wire version 4 gains NULL `units` and
|
|
228
|
+
* `registration_digest`, so it holds no registration list (§9), and a push
|
|
229
|
+
* result gains NULL `outbox_id` and `client_seq`: it is a result stored
|
|
230
|
+
* before the upgrade, settled by the client's next push (§9). The
|
|
231
|
+
* watermark delete and the settlement range over the `(partition,
|
|
232
|
+
* client_id)` prefix of the primary key, which the watermark keeps to one
|
|
233
|
+
* request's results per outbox.
|
|
234
|
+
*/
|
|
235
|
+
const SQLITE_ADDED_COLUMNS = [
|
|
236
|
+
['sync_clients', 'wire_version', 'INTEGER NOT NULL DEFAULT 1'],
|
|
237
|
+
['sync_clients', 'last_round_at_ms', 'INTEGER'],
|
|
238
|
+
['sync_clients', 'last_error_code', 'TEXT'],
|
|
239
|
+
['sync_clients', 'consecutive_failures', 'INTEGER NOT NULL DEFAULT 0'],
|
|
240
|
+
['sync_clients', 'schema_version', 'INTEGER'],
|
|
241
|
+
['sync_clients', 'units', 'TEXT'],
|
|
242
|
+
['sync_clients', 'registration_digest', 'BLOB'],
|
|
243
|
+
['sync_clients', 'catching_up', "TEXT NOT NULL DEFAULT '[]'"],
|
|
244
|
+
['sync_tombstones', 'server_version', 'INTEGER'],
|
|
245
|
+
['sync_tombstones', 'scopes', 'TEXT'],
|
|
246
|
+
['sync_push_results', 'outbox_id', 'TEXT'],
|
|
247
|
+
['sync_push_results', 'client_seq', 'INTEGER'],
|
|
248
|
+
['sync_partitions', 'delete_floor', 'INTEGER NOT NULL DEFAULT 0'],
|
|
249
|
+
] as const;
|
|
163
250
|
|
|
164
|
-
/**
|
|
165
|
-
|
|
166
|
-
'SELECT row_id FROM sync_row_scopes WHERE partition=? AND tbl=? AND var=? AND value=? ORDER BY row_id LIMIT ?';
|
|
251
|
+
/** Core-table columns wire version 4 deleted: the version 3 subscription list. */
|
|
252
|
+
const SQLITE_DROPPED_COLUMNS = [['sync_clients', 'subscriptions']] as const;
|
|
167
253
|
|
|
168
254
|
/**
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
* record (D1 batch results carry no reliable change count).
|
|
255
|
+
* Indexes that `sqliteUpgradeDdl` creates, on a fresh database and an
|
|
256
|
+
* upgraded one alike. `sync_change_scopes_by_key` serves the §4.5 live
|
|
257
|
+
* scan: one range per (variable, value) across tables.
|
|
173
258
|
*/
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
AND s.commit_seq<=?5 AND c.op=1 AND json_extract(c.scopes, ?6)=s.value
|
|
180
|
-
LIMIT ?7) RETURNING 1 AS removed`,
|
|
181
|
-
`DELETE FROM sync_change_scopes WHERE rowid IN (
|
|
182
|
-
SELECT s.rowid FROM sync_change_scopes s
|
|
183
|
-
WHERE s.partition=?1 AND s.tbl=?2 AND s.var=?3 AND s.value=?4
|
|
184
|
-
AND s.commit_seq<=?5 AND NOT EXISTS (
|
|
185
|
-
SELECT 1 FROM sync_changes c
|
|
186
|
-
WHERE c.partition=s.partition AND c.commit_seq=s.commit_seq
|
|
187
|
-
AND c.tbl=s.tbl AND json_extract(c.scopes, ?6)=s.value)
|
|
188
|
-
LIMIT ?7) RETURNING 1 AS removed`,
|
|
259
|
+
const SQLITE_ADDED_INDEXES = [
|
|
260
|
+
[
|
|
261
|
+
'sync_change_scopes_by_key',
|
|
262
|
+
'CREATE INDEX IF NOT EXISTS sync_change_scopes_by_key ON sync_change_scopes(partition, var, value, commit_seq)',
|
|
263
|
+
],
|
|
189
264
|
] as const;
|
|
190
265
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
266
|
+
/**
|
|
267
|
+
* Indexes that `sqliteUpgradeDdl` drops. With `sync_row_scopes_by_row`
|
|
268
|
+
* present the planner answered a scope scan of two or more values by walking
|
|
269
|
+
* it in `row_id` order and filtering, instead of seeking the PRIMARY KEY once
|
|
270
|
+
* per value; the per-row scope replacement (`deleteSqliteRowScopesSql`)
|
|
271
|
+
* binds the full key and needs no secondary index.
|
|
272
|
+
*/
|
|
273
|
+
const SQLITE_DROPPED_INDEXES = ['sync_row_scopes_by_row'] as const;
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* One statement that reads what `sqliteUpgradeDdl` needs: the columns of
|
|
277
|
+
* every table with added or dropped columns, and which added indexes exist.
|
|
278
|
+
*/
|
|
279
|
+
export const SQLITE_UPGRADE_PROBE: SqliteStatementSpec = {
|
|
280
|
+
sql: `SELECT t.value AS tbl, c.name AS name
|
|
281
|
+
FROM json_each(?) AS t, pragma_table_info(t.value) AS c
|
|
282
|
+
UNION ALL
|
|
283
|
+
SELECT '' AS tbl, name FROM sqlite_master
|
|
284
|
+
WHERE type='index' AND name IN (SELECT value FROM json_each(?))`,
|
|
285
|
+
params: [
|
|
286
|
+
JSON.stringify([
|
|
287
|
+
...new Set(
|
|
288
|
+
[...SQLITE_ADDED_COLUMNS, ...SQLITE_DROPPED_COLUMNS].map(
|
|
289
|
+
([table]) => table,
|
|
290
|
+
),
|
|
291
|
+
),
|
|
292
|
+
]),
|
|
293
|
+
JSON.stringify([
|
|
294
|
+
...SQLITE_ADDED_INDEXES.map(([name]) => name),
|
|
295
|
+
...SQLITE_DROPPED_INDEXES,
|
|
296
|
+
]),
|
|
297
|
+
],
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* The statements that bring the core tables of a database from an earlier
|
|
302
|
+
* release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
|
|
303
|
+
* columns, dropped columns, dropped indexes, then missing indexes. Empty for
|
|
304
|
+
* a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
|
|
305
|
+
* construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
|
|
306
|
+
* and the first open of a database at the current schema version.
|
|
307
|
+
*/
|
|
308
|
+
export function sqliteUpgradeDdl(
|
|
309
|
+
present: readonly { readonly tbl: string; readonly name: string }[],
|
|
310
|
+
): string[] {
|
|
311
|
+
const has = (table: string, name: string) =>
|
|
312
|
+
present.some((row) => row.tbl === table && row.name === name);
|
|
195
313
|
return [
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
314
|
+
...SQLITE_ADDED_COLUMNS.filter(([table, name]) => !has(table, name)).map(
|
|
315
|
+
([table, name, definition]) =>
|
|
316
|
+
`ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`,
|
|
317
|
+
),
|
|
318
|
+
// A database from before wire version 4 (and the delete floor) starts
|
|
319
|
+
// the delete floor at the highest version its tombstones recorded
|
|
320
|
+
// (§2.2) and the horizon at its last commit (§9).
|
|
321
|
+
...(has('sync_partitions', 'delete_floor')
|
|
322
|
+
? []
|
|
323
|
+
: [
|
|
324
|
+
`UPDATE sync_partitions SET horizon_seq=max_commit_seq, delete_floor=coalesce((SELECT max(server_version) FROM sync_tombstones t WHERE t.partition=sync_partitions.partition),0)`,
|
|
325
|
+
]),
|
|
326
|
+
...SQLITE_DROPPED_COLUMNS.filter(([table, name]) => has(table, name)).map(
|
|
327
|
+
([table, name]) => `ALTER TABLE ${table} DROP COLUMN ${name}`,
|
|
328
|
+
),
|
|
329
|
+
...SQLITE_DROPPED_INDEXES.filter((name) => has('', name)).map(
|
|
330
|
+
(name) => `DROP INDEX IF EXISTS ${name}`,
|
|
331
|
+
),
|
|
332
|
+
...SQLITE_ADDED_INDEXES.filter(([name]) => !has('', name)).map(
|
|
333
|
+
([, ddl]) => ddl,
|
|
334
|
+
),
|
|
203
335
|
];
|
|
204
336
|
}
|
|
205
337
|
|
|
338
|
+
/** Split the DDL into individual statements (D1 applies them one by one). */
|
|
339
|
+
export function sqliteDdlStatements(): string[] {
|
|
340
|
+
return SQLITE_DDL.split(';')
|
|
341
|
+
.map((statement) => statement.trim())
|
|
342
|
+
.filter((statement) => statement.length > 0);
|
|
343
|
+
}
|
|
344
|
+
|
|
206
345
|
/** `?,?,…` for an `IN (…)` clause of `count` positional parameters. */
|
|
207
346
|
export function placeholders(count: number): string {
|
|
208
347
|
return new Array(count).fill('?').join(',');
|
|
@@ -277,53 +416,63 @@ export function serializePushResult(result: StoredPushResult): string {
|
|
|
277
416
|
});
|
|
278
417
|
}
|
|
279
418
|
|
|
419
|
+
/** Parse a stored push result; an unreadable record is a §6.3 cache miss. */
|
|
280
420
|
export function deserializePushResult(text: string): StoredPushResult {
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
421
|
+
try {
|
|
422
|
+
const parsed = JSON.parse(text) as {
|
|
423
|
+
status: 'applied' | 'rejected';
|
|
424
|
+
commitSeq?: number;
|
|
425
|
+
recordedAtMs?: number;
|
|
426
|
+
cacheIdentity?: string;
|
|
427
|
+
conflictAuthorization?: StoredPushResult['conflictAuthorization'];
|
|
428
|
+
results: SerializedResult[];
|
|
429
|
+
};
|
|
430
|
+
const results: PushOperationResult[] = parsed.results.map((record) => {
|
|
431
|
+
if (record.status === 'conflict') {
|
|
432
|
+
return {
|
|
433
|
+
opIndex: record.opIndex,
|
|
434
|
+
status: 'conflict',
|
|
435
|
+
code: record.code ?? '',
|
|
436
|
+
message: record.message ?? '',
|
|
437
|
+
serverVersion: record.serverVersion ?? 0,
|
|
438
|
+
serverRow: base64ToBytes(record.serverRow ?? ''),
|
|
439
|
+
conflictColumns: base64ToBytes(record.conflictColumns ?? ''),
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
if (record.status === 'error') {
|
|
443
|
+
return {
|
|
444
|
+
opIndex: record.opIndex,
|
|
445
|
+
status: 'error',
|
|
446
|
+
code: record.code ?? '',
|
|
447
|
+
message: record.message ?? '',
|
|
448
|
+
retryable: record.retryable ?? false,
|
|
449
|
+
...(record.details !== undefined ? { details: record.details } : {}),
|
|
450
|
+
};
|
|
451
|
+
}
|
|
452
|
+
return { opIndex: record.opIndex, status: 'applied' };
|
|
453
|
+
});
|
|
454
|
+
return {
|
|
455
|
+
status: parsed.status,
|
|
456
|
+
...(parsed.commitSeq !== undefined
|
|
457
|
+
? { commitSeq: parsed.commitSeq }
|
|
458
|
+
: {}),
|
|
459
|
+
...(parsed.recordedAtMs !== undefined
|
|
460
|
+
? { recordedAtMs: parsed.recordedAtMs }
|
|
461
|
+
: {}),
|
|
462
|
+
...(parsed.cacheIdentity !== undefined
|
|
463
|
+
? { cacheIdentity: parsed.cacheIdentity }
|
|
464
|
+
: {}),
|
|
465
|
+
...(parsed.conflictAuthorization !== undefined
|
|
466
|
+
? { conflictAuthorization: parsed.conflictAuthorization }
|
|
467
|
+
: {}),
|
|
468
|
+
results,
|
|
469
|
+
};
|
|
470
|
+
} catch {
|
|
471
|
+
throw syncError(
|
|
472
|
+
'sync.idempotency_cache_miss',
|
|
473
|
+
'persisted push result unreadable (§6.3)',
|
|
474
|
+
);
|
|
475
|
+
}
|
|
327
476
|
}
|
|
328
477
|
|
|
329
478
|
/** Row-record shape the SQLite `SELECT`s in both storages return. */
|
|
@@ -364,6 +513,24 @@ export function toStoredRow(record: SqliteRowRecord): StoredRow {
|
|
|
364
513
|
};
|
|
365
514
|
}
|
|
366
515
|
|
|
516
|
+
export interface SqliteTombstoneRecord {
|
|
517
|
+
commit_seq: number;
|
|
518
|
+
server_version: number | null;
|
|
519
|
+
scopes: string | null;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
export function toTombstone(record: SqliteTombstoneRecord): Tombstone {
|
|
523
|
+
return {
|
|
524
|
+
commitSeq: record.commit_seq,
|
|
525
|
+
...(record.server_version !== null
|
|
526
|
+
? { serverVersion: record.server_version }
|
|
527
|
+
: {}),
|
|
528
|
+
...(record.scopes !== null
|
|
529
|
+
? { scopes: JSON.parse(record.scopes) as Record<string, string> }
|
|
530
|
+
: {}),
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
|
|
367
534
|
export function toStoredChange(record: SqliteChangeRecord): StoredChange {
|
|
368
535
|
return {
|
|
369
536
|
table: record.tbl,
|
|
@@ -377,98 +544,1583 @@ export function toStoredChange(record: SqliteChangeRecord): StoredChange {
|
|
|
377
544
|
};
|
|
378
545
|
}
|
|
379
546
|
|
|
547
|
+
interface SqliteReactionRecord {
|
|
548
|
+
idempotency_key: string;
|
|
549
|
+
type: string;
|
|
550
|
+
version: number;
|
|
551
|
+
payload: string;
|
|
552
|
+
source_client_id: string;
|
|
553
|
+
source_client_commit_id: string;
|
|
554
|
+
source_commit_seq: number;
|
|
555
|
+
created_at_ms: number;
|
|
556
|
+
available_at_ms: number;
|
|
557
|
+
status: StoredReaction['status'];
|
|
558
|
+
attempts: number;
|
|
559
|
+
max_attempts: number;
|
|
560
|
+
lease_owner: string | null;
|
|
561
|
+
lease_expires_at_ms: number | null;
|
|
562
|
+
completed_at_ms: number | null;
|
|
563
|
+
last_failure: string | null;
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
function toStoredReaction(record: SqliteReactionRecord): StoredReaction {
|
|
567
|
+
return {
|
|
568
|
+
idempotencyKey: record.idempotency_key,
|
|
569
|
+
type: record.type,
|
|
570
|
+
version: record.version,
|
|
571
|
+
payload: JSON.parse(record.payload) as DurableJsonValue,
|
|
572
|
+
sourceClientId: record.source_client_id,
|
|
573
|
+
sourceClientCommitId: record.source_client_commit_id,
|
|
574
|
+
sourceCommitSeq: record.source_commit_seq,
|
|
575
|
+
createdAtMs: record.created_at_ms,
|
|
576
|
+
maxAttempts: record.max_attempts,
|
|
577
|
+
status: record.status,
|
|
578
|
+
attempts: record.attempts,
|
|
579
|
+
availableAtMs: record.available_at_ms,
|
|
580
|
+
...(record.lease_owner !== null ? { leaseOwner: record.lease_owner } : {}),
|
|
581
|
+
...(record.lease_expires_at_ms !== null
|
|
582
|
+
? { leaseExpiresAtMs: record.lease_expires_at_ms }
|
|
583
|
+
: {}),
|
|
584
|
+
...(record.completed_at_ms !== null
|
|
585
|
+
? { completedAtMs: record.completed_at_ms }
|
|
586
|
+
: {}),
|
|
587
|
+
...(record.last_failure !== null
|
|
588
|
+
? { lastFailure: JSON.parse(record.last_failure) as ReactionFailure }
|
|
589
|
+
: {}),
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
interface SqliteClientRecord {
|
|
594
|
+
client_id: string;
|
|
595
|
+
actor_id: string;
|
|
596
|
+
wire_version: number;
|
|
597
|
+
cursor: number;
|
|
598
|
+
updated_at_ms: number;
|
|
599
|
+
schema_version: number | null;
|
|
600
|
+
units: string | null;
|
|
601
|
+
registration_digest: Uint8Array | null;
|
|
602
|
+
catching_up: string;
|
|
603
|
+
last_round_at_ms: number | null;
|
|
604
|
+
last_error_code: string | null;
|
|
605
|
+
consecutive_failures: number;
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
const CLIENT_RECORD_COLUMNS =
|
|
609
|
+
'client_id, actor_id, wire_version, cursor, updated_at_ms, schema_version, units, registration_digest, catching_up, last_round_at_ms, last_error_code, consecutive_failures';
|
|
610
|
+
|
|
611
|
+
function toClientRecord(record: SqliteClientRecord): ClientRecord {
|
|
612
|
+
return {
|
|
613
|
+
clientId: record.client_id,
|
|
614
|
+
actorId: record.actor_id,
|
|
615
|
+
wireVersion: record.wire_version,
|
|
616
|
+
cursor: record.cursor,
|
|
617
|
+
updatedAtMs: record.updated_at_ms,
|
|
618
|
+
...(record.schema_version !== null
|
|
619
|
+
? { schemaVersion: record.schema_version }
|
|
620
|
+
: {}),
|
|
621
|
+
...(record.units !== null && record.registration_digest !== null
|
|
622
|
+
? {
|
|
623
|
+
registration: {
|
|
624
|
+
units: JSON.parse(record.units) as UnitScopes[],
|
|
625
|
+
digest: asUint8Array(record.registration_digest),
|
|
626
|
+
},
|
|
627
|
+
}
|
|
628
|
+
: {}),
|
|
629
|
+
catchingUp: JSON.parse(record.catching_up) as ClientRecord['catchingUp'],
|
|
630
|
+
...clientRoundState(
|
|
631
|
+
record.last_round_at_ms,
|
|
632
|
+
record.last_error_code,
|
|
633
|
+
record.consecutive_failures,
|
|
634
|
+
),
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
interface SqlitePartitionRegistryRecord {
|
|
639
|
+
partition: string;
|
|
640
|
+
log_epoch: string;
|
|
641
|
+
epoch_required: number;
|
|
642
|
+
last_authenticated_at_ms: number;
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
function toPartitionRegistryEntry(
|
|
646
|
+
row: SqlitePartitionRegistryRecord,
|
|
647
|
+
): PartitionRegistryEntry {
|
|
648
|
+
return {
|
|
649
|
+
partition: row.partition,
|
|
650
|
+
logEpoch: row.log_epoch,
|
|
651
|
+
epochRequired: row.epoch_required === 1,
|
|
652
|
+
lastAuthenticatedAtMs: row.last_authenticated_at_ms,
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/** One statement of an atomic unit: its SQL text and bound parameters. */
|
|
657
|
+
export interface SqliteStatementSpec {
|
|
658
|
+
readonly sql: string;
|
|
659
|
+
readonly params: readonly SqliteValue[];
|
|
660
|
+
}
|
|
661
|
+
|
|
380
662
|
/**
|
|
381
|
-
*
|
|
382
|
-
*
|
|
383
|
-
*
|
|
384
|
-
* Native SQLite hands back `Uint8Array`, D1 `ArrayBuffer`; `toStoredChange`
|
|
385
|
-
* normalizes via `asUint8Array`.
|
|
663
|
+
* The `ServerStorage` methods SQLite and D1 share. A subclass supplies the
|
|
664
|
+
* execution seam and implements the rest of the interface: schema migration,
|
|
665
|
+
* `begin`, registered queries, the serve gate and checkpoints.
|
|
386
666
|
*/
|
|
387
|
-
export
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
667
|
+
export abstract class SqliteFamilyStorage {
|
|
668
|
+
/** Set by `ensureSchema`: app-table lookup for the relational row store. */
|
|
669
|
+
protected tables: ReadonlyMap<string, CompiledTable> | undefined;
|
|
670
|
+
protected schemaVersion: number | undefined;
|
|
671
|
+
/** Bound parameters one statement accepts. */
|
|
672
|
+
protected abstract readonly maxBindParams: number;
|
|
673
|
+
|
|
674
|
+
/** Run one statement and return its rows. */
|
|
675
|
+
protected abstract all<T>(
|
|
676
|
+
sql: string,
|
|
677
|
+
params: readonly SqliteValue[],
|
|
678
|
+
): Promise<T[]>;
|
|
679
|
+
|
|
680
|
+
/**
|
|
681
|
+
* Run one read of schema-dependent state (row tables, the commit window,
|
|
682
|
+
* push results). D1 runs it behind its published-schema guard and joins the
|
|
683
|
+
* reads of one microtask turn into one batch; SQLite runs it as `all`.
|
|
684
|
+
*/
|
|
685
|
+
protected abstract guardedAll<T>(
|
|
686
|
+
sql: string,
|
|
687
|
+
params: readonly SqliteValue[],
|
|
688
|
+
): Promise<T[]>;
|
|
689
|
+
|
|
690
|
+
/** Run statements as one all-or-nothing unit; returns each one's rows. */
|
|
691
|
+
protected abstract atomic(
|
|
692
|
+
statements: readonly SqliteStatementSpec[],
|
|
693
|
+
): Promise<unknown[][]>;
|
|
694
|
+
|
|
695
|
+
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
696
|
+
table(name: string): CompiledTable {
|
|
697
|
+
const table = this.tables?.get(name);
|
|
698
|
+
if (table === undefined) {
|
|
699
|
+
throw new Error(
|
|
700
|
+
`unknown table ${JSON.stringify(name)} — ensureSchema(schema) must run before row operations`,
|
|
701
|
+
);
|
|
702
|
+
}
|
|
703
|
+
return table;
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
protected prepareAuthoritativeQuery(
|
|
707
|
+
partition: string,
|
|
708
|
+
query: AuthoritativeQueryRequest,
|
|
709
|
+
): BoundAuthoritativeQuery {
|
|
710
|
+
if (this.tables === undefined) {
|
|
711
|
+
throw new Error(
|
|
712
|
+
'ensureSchema(schema) must run before registered queries',
|
|
713
|
+
);
|
|
714
|
+
}
|
|
715
|
+
return bindAuthoritativePartition(
|
|
716
|
+
prepareAuthoritativeQuery(
|
|
717
|
+
query.plan,
|
|
718
|
+
query.params,
|
|
719
|
+
query.tables,
|
|
720
|
+
this.tables,
|
|
721
|
+
),
|
|
722
|
+
partition,
|
|
723
|
+
);
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
async touchPartition(
|
|
727
|
+
partition: string,
|
|
728
|
+
authenticatedAtMs: number,
|
|
729
|
+
initialLogEpoch: string,
|
|
730
|
+
): Promise<PartitionRegistryEntry> {
|
|
731
|
+
if (initialLogEpoch.length === 0) {
|
|
732
|
+
throw new Error('initial log epoch must be non-empty');
|
|
733
|
+
}
|
|
734
|
+
// One atomic unit, so an idle SQLite connection runs both statements at
|
|
735
|
+
// call time, before a `close()` in the same tick. The upsert writes only
|
|
736
|
+
// a new partition or one whose last authentication is a full interval old.
|
|
737
|
+
const [, rows] = await this.atomic([
|
|
738
|
+
{
|
|
739
|
+
sql: `INSERT INTO sync_partition_registry(
|
|
740
|
+
partition, log_epoch, last_authenticated_at_ms
|
|
741
|
+
) VALUES (?,?,?)
|
|
742
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
743
|
+
last_authenticated_at_ms=excluded.last_authenticated_at_ms
|
|
744
|
+
WHERE excluded.last_authenticated_at_ms - last_authenticated_at_ms >= ?`,
|
|
745
|
+
params: [
|
|
746
|
+
partition,
|
|
747
|
+
initialLogEpoch,
|
|
748
|
+
authenticatedAtMs,
|
|
749
|
+
PARTITION_TOUCH_INTERVAL_MS,
|
|
750
|
+
],
|
|
751
|
+
},
|
|
752
|
+
{
|
|
753
|
+
sql: `SELECT partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
754
|
+
FROM sync_partition_registry WHERE partition=?`,
|
|
755
|
+
params: [partition],
|
|
756
|
+
},
|
|
757
|
+
]);
|
|
758
|
+
const row = rows?.[0] as SqlitePartitionRegistryRecord | undefined;
|
|
759
|
+
if (row === undefined)
|
|
760
|
+
throw new Error('partition registry write did not persist');
|
|
761
|
+
return toPartitionRegistryEntry(row);
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
async rotatePartitionLogEpoch(
|
|
765
|
+
partition: string,
|
|
766
|
+
logEpoch: string,
|
|
767
|
+
authenticatedAtMs: number,
|
|
768
|
+
): Promise<PartitionRegistryEntry> {
|
|
769
|
+
if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
|
|
770
|
+
await this.atomic([
|
|
771
|
+
{
|
|
772
|
+
sql: `INSERT INTO sync_partition_registry(
|
|
773
|
+
partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
774
|
+
) VALUES (?,?,1,?)
|
|
775
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
776
|
+
log_epoch=excluded.log_epoch,
|
|
777
|
+
epoch_required=1,
|
|
778
|
+
last_authenticated_at_ms=excluded.last_authenticated_at_ms`,
|
|
779
|
+
params: [partition, logEpoch, authenticatedAtMs],
|
|
780
|
+
},
|
|
781
|
+
{
|
|
782
|
+
sql: 'DELETE FROM sync_clients WHERE partition=?',
|
|
783
|
+
params: [partition],
|
|
784
|
+
},
|
|
785
|
+
]);
|
|
786
|
+
return {
|
|
787
|
+
partition,
|
|
788
|
+
logEpoch,
|
|
789
|
+
epochRequired: true,
|
|
790
|
+
lastAuthenticatedAtMs: authenticatedAtMs,
|
|
791
|
+
};
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
async listPartitionRegistry(): Promise<PartitionRegistryEntry[]> {
|
|
795
|
+
return (
|
|
796
|
+
await this.all<SqlitePartitionRegistryRecord>(
|
|
797
|
+
`SELECT partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
798
|
+
FROM sync_partition_registry ORDER BY partition`,
|
|
799
|
+
[],
|
|
800
|
+
)
|
|
801
|
+
).map(toPartitionRegistryEntry);
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
async getMaxCommitSeq(partition: string): Promise<number> {
|
|
805
|
+
const [row] = await this.all<{ max_commit_seq: number }>(
|
|
806
|
+
'SELECT max_commit_seq FROM sync_partitions WHERE partition=?',
|
|
807
|
+
[partition],
|
|
808
|
+
);
|
|
809
|
+
return row?.max_commit_seq ?? 0;
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
async getPartitionLogEpoch(partition: string): Promise<string | undefined> {
|
|
813
|
+
const [row] = await this.all<{ log_epoch: string }>(
|
|
814
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=?',
|
|
815
|
+
[partition],
|
|
816
|
+
);
|
|
817
|
+
return row?.log_epoch;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
async getHorizonSeq(partition: string): Promise<number> {
|
|
821
|
+
const [row] = await this.all<{ horizon_seq: number }>(
|
|
822
|
+
'SELECT horizon_seq FROM sync_partitions WHERE partition=?',
|
|
823
|
+
[partition],
|
|
824
|
+
);
|
|
825
|
+
return row?.horizon_seq ?? 0;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
async setHorizonSeq(partition: string, seq: number): Promise<void> {
|
|
829
|
+
await this.all(
|
|
830
|
+
`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
|
|
831
|
+
ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`,
|
|
832
|
+
[partition, seq],
|
|
833
|
+
);
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
async pruneCommitsThrough(
|
|
837
|
+
partition: string,
|
|
838
|
+
query: CommitPruneQuery,
|
|
839
|
+
): Promise<CommitPruneResult> {
|
|
840
|
+
validateCommitPruneQuery(query);
|
|
841
|
+
// Every write re-checks the epoch, so the unit needs no read before its
|
|
842
|
+
// writes: on a mismatch it changes nothing and the first row reports it.
|
|
843
|
+
const epochGuard =
|
|
844
|
+
'EXISTS (SELECT 1 FROM sync_partition_registry WHERE partition=? AND log_epoch=?)';
|
|
845
|
+
const horizon =
|
|
846
|
+
'(SELECT horizon_seq FROM sync_partitions WHERE partition=?)';
|
|
847
|
+
const results = await this.atomic([
|
|
848
|
+
{
|
|
849
|
+
sql: `SELECT log_epoch, coalesce(${horizon},0) AS previous_horizon_seq FROM sync_partition_registry WHERE partition=?`,
|
|
850
|
+
params: [partition, partition],
|
|
851
|
+
},
|
|
852
|
+
{
|
|
853
|
+
sql: `INSERT INTO sync_partitions(partition,horizon_seq) SELECT ?,? WHERE ${epochGuard}
|
|
854
|
+
ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`,
|
|
855
|
+
params: [partition, query.throughSeq, partition, query.logEpoch],
|
|
856
|
+
},
|
|
857
|
+
{
|
|
858
|
+
sql: `SELECT horizon_seq, (SELECT count(*) FROM sync_commits WHERE partition=? AND commit_seq<=${horizon}) AS removed_commits FROM sync_partitions WHERE partition=?`,
|
|
859
|
+
params: [partition, partition, partition],
|
|
860
|
+
},
|
|
861
|
+
// Change-scope entries go first, by the keys of the changes being
|
|
862
|
+
// pruned: a predicate on `commit_seq` alone would scan the partition's
|
|
863
|
+
// whole scope index, because `commit_seq` is its last key column.
|
|
864
|
+
{
|
|
865
|
+
sql: `DELETE FROM sync_change_scopes WHERE (partition, tbl, var, value, commit_seq) IN (
|
|
866
|
+
SELECT c.partition, c.tbl, j.key, j.value, c.commit_seq
|
|
867
|
+
FROM sync_changes c, json_each(c.scopes) j
|
|
868
|
+
WHERE c.partition=? AND c.commit_seq<=${horizon})
|
|
869
|
+
AND ${epochGuard}`,
|
|
870
|
+
params: [partition, partition, partition, query.logEpoch],
|
|
871
|
+
},
|
|
872
|
+
// §4.6: tombstones prune in the same pass as the commit log.
|
|
873
|
+
...['sync_commits', 'sync_changes', 'sync_tombstones'].map((table) => ({
|
|
874
|
+
sql: `DELETE FROM ${table} WHERE partition=? AND commit_seq<=${horizon} AND ${epochGuard}`,
|
|
875
|
+
params: [partition, partition, partition, query.logEpoch],
|
|
876
|
+
})),
|
|
877
|
+
]);
|
|
878
|
+
const [before, after] = [results[0], results[2]].map((rows) => {
|
|
879
|
+
const row: unknown = rows?.[0];
|
|
880
|
+
return typeof row === 'object' && row !== null
|
|
881
|
+
? (row as Readonly<Record<string, unknown>>)
|
|
882
|
+
: undefined;
|
|
883
|
+
});
|
|
884
|
+
if (before?.log_epoch !== query.logEpoch)
|
|
885
|
+
throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
|
|
886
|
+
if (
|
|
887
|
+
typeof before.previous_horizon_seq !== 'number' ||
|
|
888
|
+
typeof after?.horizon_seq !== 'number' ||
|
|
889
|
+
typeof after.removed_commits !== 'number'
|
|
890
|
+
) {
|
|
891
|
+
throw new Error('pruning returned invalid horizon metadata');
|
|
892
|
+
}
|
|
893
|
+
return {
|
|
894
|
+
previousHorizonSeq: before.previous_horizon_seq,
|
|
895
|
+
horizonSeq: after.horizon_seq,
|
|
896
|
+
removedCommits: after.removed_commits,
|
|
897
|
+
};
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
async getCommitSeqBefore(
|
|
901
|
+
partition: string,
|
|
902
|
+
createdBeforeMs: number,
|
|
903
|
+
): Promise<number> {
|
|
904
|
+
const [row] = await this.all<{ seq: number | null }>(
|
|
905
|
+
'SELECT max(commit_seq) AS seq FROM sync_commits WHERE partition=? AND created_at_ms<?',
|
|
906
|
+
[partition, createdBeforeMs],
|
|
907
|
+
);
|
|
908
|
+
return row?.seq ?? 0;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
async sourceCoverageSeq(
|
|
912
|
+
partition: string,
|
|
913
|
+
tables: readonly string[],
|
|
914
|
+
): Promise<number> {
|
|
915
|
+
if (tables.length === 0) return 0;
|
|
916
|
+
const [row] = await this.all<{ seq: number | null }>(
|
|
917
|
+
`SELECT max(commit_seq) AS seq FROM sync_changes
|
|
918
|
+
WHERE partition=? AND tbl IN (${placeholders(tables.length)})`,
|
|
919
|
+
[partition, ...tables],
|
|
920
|
+
);
|
|
921
|
+
return row?.seq ?? 0;
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
async hasSourceChangesAbove(
|
|
925
|
+
partition: string,
|
|
926
|
+
tables: readonly string[],
|
|
927
|
+
seq: number,
|
|
928
|
+
): Promise<'clean' | 'changed' | 'unverifiable'> {
|
|
929
|
+
// Read the horizon before the window scan: a horizon past `seq` means the
|
|
930
|
+
// history that would answer the question is gone, so an empty scan proves
|
|
931
|
+
// nothing and must not be reported as `clean`.
|
|
932
|
+
const horizon = await this.getHorizonSeq(partition);
|
|
933
|
+
if (tables.length === 0) return 'clean';
|
|
934
|
+
const hit = await this.all<{ hit: number }>(
|
|
935
|
+
`SELECT 1 AS hit FROM sync_changes
|
|
936
|
+
WHERE partition=? AND tbl IN (${placeholders(tables.length)})
|
|
937
|
+
AND commit_seq>? LIMIT 1`,
|
|
938
|
+
[partition, ...tables, seq],
|
|
939
|
+
);
|
|
940
|
+
if (hit.length > 0) return 'changed';
|
|
941
|
+
return horizon > seq ? 'unverifiable' : 'clean';
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
async getRow(
|
|
945
|
+
partition: string,
|
|
946
|
+
table: string,
|
|
947
|
+
rowId: string,
|
|
948
|
+
): Promise<StoredRow | undefined> {
|
|
949
|
+
const [record] = await this.guardedAll<SqliteRowRecord>(
|
|
950
|
+
selectRowSql(this.table(table), 'sqlite'),
|
|
951
|
+
[partition, rowId],
|
|
952
|
+
);
|
|
953
|
+
return record === undefined ? undefined : toStoredRow(record);
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
async getPushResult(
|
|
957
|
+
partition: string,
|
|
958
|
+
clientId: string,
|
|
959
|
+
clientCommitId: string,
|
|
960
|
+
): Promise<StoredPushResult | undefined> {
|
|
961
|
+
const [record] = await this.guardedAll<{ result: string }>(
|
|
962
|
+
'SELECT result FROM sync_push_results WHERE partition=? AND client_id=? AND client_commit_id=?',
|
|
963
|
+
[partition, clientId, clientCommitId],
|
|
964
|
+
);
|
|
965
|
+
return record === undefined
|
|
966
|
+
? undefined
|
|
967
|
+
: deserializePushResult(record.result);
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
async claimReactions(
|
|
971
|
+
partition: string,
|
|
972
|
+
query: ReactionClaimQuery,
|
|
973
|
+
): Promise<StoredReaction[]> {
|
|
974
|
+
if (query.types.length === 0 || query.limit <= 0) return [];
|
|
975
|
+
if (query.types.length + 7 > this.maxBindParams) {
|
|
976
|
+
throw new Error('D1 reaction claim exceeds the bound-parameter limit');
|
|
977
|
+
}
|
|
978
|
+
const records = await this.all<SqliteReactionRecord>(
|
|
979
|
+
`UPDATE sync_reactions
|
|
980
|
+
SET status='leased', attempts=attempts+1,
|
|
981
|
+
lease_owner=?, lease_expires_at_ms=?, completed_at_ms=NULL
|
|
982
|
+
WHERE (partition, idempotency_key) IN (
|
|
983
|
+
SELECT partition, idempotency_key
|
|
984
|
+
FROM sync_reactions
|
|
985
|
+
WHERE partition=? AND type IN (${placeholders(query.types.length)})
|
|
986
|
+
AND ((status='pending' AND available_at_ms<=?)
|
|
987
|
+
OR (status='leased' AND lease_expires_at_ms<=?))
|
|
988
|
+
ORDER BY CASE WHEN status='leased' THEN lease_expires_at_ms
|
|
989
|
+
ELSE available_at_ms END,
|
|
990
|
+
created_at_ms, idempotency_key
|
|
991
|
+
LIMIT ?
|
|
992
|
+
)
|
|
993
|
+
RETURNING *`,
|
|
994
|
+
[
|
|
995
|
+
query.leaseOwner,
|
|
996
|
+
Math.min(Number.MAX_SAFE_INTEGER, query.nowMs + query.leaseDurationMs),
|
|
997
|
+
partition,
|
|
998
|
+
...query.types,
|
|
999
|
+
query.nowMs,
|
|
1000
|
+
query.nowMs,
|
|
1001
|
+
query.limit,
|
|
1002
|
+
],
|
|
1003
|
+
);
|
|
1004
|
+
return records
|
|
1005
|
+
.map(toStoredReaction)
|
|
1006
|
+
.sort(
|
|
1007
|
+
(a, b) =>
|
|
1008
|
+
a.createdAtMs - b.createdAtMs ||
|
|
1009
|
+
a.idempotencyKey.localeCompare(b.idempotencyKey),
|
|
1010
|
+
);
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
async completeReaction(
|
|
1014
|
+
partition: string,
|
|
1015
|
+
idempotencyKey: string,
|
|
1016
|
+
leaseOwner: string,
|
|
1017
|
+
completedAtMs: number,
|
|
1018
|
+
): Promise<boolean> {
|
|
1019
|
+
const updated = await this.all(
|
|
1020
|
+
`UPDATE sync_reactions
|
|
1021
|
+
SET status='completed', completed_at_ms=?,
|
|
1022
|
+
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
1023
|
+
WHERE partition=? AND idempotency_key=?
|
|
1024
|
+
AND status='leased' AND lease_owner=?
|
|
1025
|
+
RETURNING idempotency_key`,
|
|
1026
|
+
[completedAtMs, partition, idempotencyKey, leaseOwner],
|
|
1027
|
+
);
|
|
1028
|
+
return updated.length === 1;
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
async extendReactionLease(
|
|
1032
|
+
partition: string,
|
|
1033
|
+
idempotencyKey: string,
|
|
1034
|
+
leaseOwner: string,
|
|
1035
|
+
leaseExpiresAtMs: number,
|
|
1036
|
+
): Promise<boolean> {
|
|
1037
|
+
const updated = await this.all(
|
|
1038
|
+
`UPDATE sync_reactions SET lease_expires_at_ms=?
|
|
1039
|
+
WHERE partition=? AND idempotency_key=?
|
|
1040
|
+
AND status='leased' AND lease_owner=?
|
|
1041
|
+
RETURNING idempotency_key`,
|
|
1042
|
+
[leaseExpiresAtMs, partition, idempotencyKey, leaseOwner],
|
|
1043
|
+
);
|
|
1044
|
+
return updated.length === 1;
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
async failReaction(
|
|
1048
|
+
partition: string,
|
|
1049
|
+
idempotencyKey: string,
|
|
1050
|
+
update: ReactionFailureUpdate,
|
|
1051
|
+
): Promise<boolean> {
|
|
1052
|
+
const updated = await this.all(
|
|
1053
|
+
`UPDATE sync_reactions
|
|
1054
|
+
SET status=?, available_at_ms=?, last_failure=?,
|
|
1055
|
+
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
1056
|
+
WHERE partition=? AND idempotency_key=?
|
|
1057
|
+
AND status='leased' AND lease_owner=?
|
|
1058
|
+
RETURNING idempotency_key`,
|
|
1059
|
+
[
|
|
1060
|
+
update.retryAtMs === undefined ? 'dead-letter' : 'pending',
|
|
1061
|
+
update.retryAtMs ?? update.failure.atMs,
|
|
1062
|
+
JSON.stringify(update.failure),
|
|
1063
|
+
partition,
|
|
1064
|
+
idempotencyKey,
|
|
1065
|
+
update.leaseOwner,
|
|
1066
|
+
],
|
|
1067
|
+
);
|
|
1068
|
+
return updated.length === 1;
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
async retryReaction(
|
|
1072
|
+
partition: string,
|
|
1073
|
+
idempotencyKey: string,
|
|
1074
|
+
nowMs: number,
|
|
1075
|
+
): Promise<boolean> {
|
|
1076
|
+
const updated = await this.all(
|
|
1077
|
+
`UPDATE sync_reactions
|
|
1078
|
+
SET status='pending', attempts=0, available_at_ms=?,
|
|
1079
|
+
last_failure=NULL, lease_owner=NULL, lease_expires_at_ms=NULL,
|
|
1080
|
+
completed_at_ms=NULL
|
|
1081
|
+
WHERE partition=? AND idempotency_key=? AND status='dead-letter'
|
|
1082
|
+
RETURNING idempotency_key`,
|
|
1083
|
+
[nowMs, partition, idempotencyKey],
|
|
1084
|
+
);
|
|
1085
|
+
return updated.length === 1;
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
async getReaction(
|
|
1089
|
+
partition: string,
|
|
1090
|
+
idempotencyKey: string,
|
|
1091
|
+
): Promise<StoredReaction | undefined> {
|
|
1092
|
+
const [record] = await this.all<SqliteReactionRecord>(
|
|
1093
|
+
'SELECT * FROM sync_reactions WHERE partition=? AND idempotency_key=?',
|
|
1094
|
+
[partition, idempotencyKey],
|
|
1095
|
+
);
|
|
1096
|
+
return record === undefined ? undefined : toStoredReaction(record);
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
async listReactions(
|
|
1100
|
+
partition: string,
|
|
1101
|
+
query: ReactionListQuery,
|
|
1102
|
+
): Promise<StoredReaction[]> {
|
|
1103
|
+
const where = ['partition=?'];
|
|
1104
|
+
const params: (string | number)[] = [partition];
|
|
1105
|
+
if (query.statuses !== undefined && query.statuses.length > 0) {
|
|
1106
|
+
where.push(`status IN (${placeholders(query.statuses.length)})`);
|
|
1107
|
+
params.push(...query.statuses);
|
|
1108
|
+
}
|
|
1109
|
+
if (query.types !== undefined && query.types.length > 0) {
|
|
1110
|
+
where.push(`type IN (${placeholders(query.types.length)})`);
|
|
1111
|
+
params.push(...query.types);
|
|
1112
|
+
}
|
|
1113
|
+
if (params.length + 1 > this.maxBindParams) {
|
|
1114
|
+
throw new Error('D1 reaction list exceeds the bound-parameter limit');
|
|
1115
|
+
}
|
|
1116
|
+
params.push(query.limit);
|
|
1117
|
+
const records = await this.all<SqliteReactionRecord>(
|
|
1118
|
+
`SELECT * FROM sync_reactions WHERE ${where.join(' AND ')}
|
|
1119
|
+
ORDER BY created_at_ms DESC, idempotency_key DESC LIMIT ?`,
|
|
1120
|
+
params,
|
|
1121
|
+
);
|
|
1122
|
+
return records.map(toStoredReaction);
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
async pruneReactions(
|
|
1126
|
+
partition: string,
|
|
1127
|
+
query: ReactionPruneQuery,
|
|
1128
|
+
): Promise<PrunedReactionCounts> {
|
|
1129
|
+
if (query.limit <= 0) return { completed: 0, deadLetter: 0 };
|
|
1130
|
+
const records = await this.all<{ status: 'completed' | 'dead-letter' }>(
|
|
1131
|
+
`DELETE FROM sync_reactions
|
|
1132
|
+
WHERE partition=? AND idempotency_key IN (
|
|
1133
|
+
SELECT idempotency_key FROM sync_reactions
|
|
1134
|
+
WHERE partition=?
|
|
1135
|
+
AND ((status='completed' AND completed_at_ms IS NOT NULL
|
|
1136
|
+
AND completed_at_ms<?)
|
|
1137
|
+
OR (status='dead-letter' AND available_at_ms<?))
|
|
1138
|
+
ORDER BY CASE WHEN status='completed' THEN completed_at_ms
|
|
1139
|
+
ELSE available_at_ms END,
|
|
1140
|
+
idempotency_key
|
|
1141
|
+
LIMIT ?
|
|
1142
|
+
)
|
|
1143
|
+
AND ((status='completed' AND completed_at_ms IS NOT NULL
|
|
1144
|
+
AND completed_at_ms<?)
|
|
1145
|
+
OR (status='dead-letter' AND available_at_ms<?))
|
|
1146
|
+
RETURNING status`,
|
|
1147
|
+
[
|
|
1148
|
+
partition,
|
|
1149
|
+
partition,
|
|
1150
|
+
query.completedBeforeMs,
|
|
1151
|
+
query.deadLetterBeforeMs,
|
|
1152
|
+
query.limit,
|
|
1153
|
+
query.completedBeforeMs,
|
|
1154
|
+
query.deadLetterBeforeMs,
|
|
1155
|
+
],
|
|
1156
|
+
);
|
|
1157
|
+
return {
|
|
1158
|
+
completed: records.filter((record) => record.status === 'completed')
|
|
1159
|
+
.length,
|
|
1160
|
+
deadLetter: records.filter((record) => record.status === 'dead-letter')
|
|
1161
|
+
.length,
|
|
1162
|
+
};
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
async latestScopedChangeSeq(
|
|
1166
|
+
partition: string,
|
|
1167
|
+
query: { readonly table: string; readonly scopes: UnitScopes },
|
|
1168
|
+
): Promise<number> {
|
|
1169
|
+
const variable = Object.keys(query.scopes).sort()[0];
|
|
1170
|
+
const value = variable === undefined ? undefined : query.scopes[variable];
|
|
1171
|
+
if (variable === undefined || value === undefined) return 0;
|
|
1172
|
+
// One index-only max over the (partition, tbl, var, value) prefix.
|
|
1173
|
+
const [row] = await this.guardedAll<{ seq: number | null }>(
|
|
1174
|
+
'SELECT max(commit_seq) AS seq FROM sync_change_scopes WHERE partition=? AND tbl=? AND var=? AND value=?',
|
|
1175
|
+
[partition, query.table, variable, value],
|
|
1176
|
+
);
|
|
1177
|
+
return row?.seq ?? 0;
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
readLiveWindow(
|
|
1181
|
+
partition: string,
|
|
1182
|
+
query: LiveWindowQuery,
|
|
1183
|
+
): Promise<LiveWindow> {
|
|
1184
|
+
// One statement per page: the unit keys bind as one JSON array, and the
|
|
1185
|
+
// CROSS JOIN keeps them the outer loop, so each key seeks one range of
|
|
1186
|
+
// `sync_change_scopes_by_key` (INDEXED BY fails loudly when the index is
|
|
1187
|
+
// missing). The candidates LEFT JOIN the commit meta and every change of
|
|
1188
|
+
// the commit; a candidate whose commit vanished comes back with NULL
|
|
1189
|
+
// meta and still advances the scan.
|
|
1190
|
+
return readLiveWindowPages(query, async (keys, limit) =>
|
|
1191
|
+
(
|
|
1192
|
+
await this.guardedAll<{
|
|
1193
|
+
commit_seq: number;
|
|
1194
|
+
actor_id: string | null;
|
|
1195
|
+
created_at_ms: number | null;
|
|
1196
|
+
tbl: string | null;
|
|
1197
|
+
row_id: string;
|
|
1198
|
+
op: number;
|
|
1199
|
+
row_version: number | null;
|
|
1200
|
+
scopes: string;
|
|
1201
|
+
payload: Uint8Array | null;
|
|
1202
|
+
}>(
|
|
1203
|
+
`SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
1204
|
+
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
1205
|
+
FROM (SELECT DISTINCT s.commit_seq AS commit_seq
|
|
1206
|
+
FROM json_each(?2) AS k
|
|
1207
|
+
CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
|
|
1208
|
+
ON s.partition=?1 AND s.var=json_extract(k.value, '$.var')
|
|
1209
|
+
AND s.value=json_extract(k.value, '$.value')
|
|
1210
|
+
AND s.commit_seq>json_extract(k.value, '$.after') AND s.commit_seq<=?3
|
|
1211
|
+
ORDER BY s.commit_seq LIMIT ?4) AS c
|
|
1212
|
+
LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
|
|
1213
|
+
LEFT JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
|
|
1214
|
+
ORDER BY c.commit_seq, ch.idx`,
|
|
1215
|
+
[partition, JSON.stringify(keys), query.throughSeq, limit],
|
|
1216
|
+
)
|
|
1217
|
+
).map((record) => ({
|
|
1218
|
+
commitSeq: record.commit_seq,
|
|
1219
|
+
...(record.actor_id !== null && record.created_at_ms !== null
|
|
1220
|
+
? {
|
|
1221
|
+
meta: {
|
|
1222
|
+
actorId: record.actor_id,
|
|
1223
|
+
createdAtMs: record.created_at_ms,
|
|
1224
|
+
},
|
|
1225
|
+
}
|
|
1226
|
+
: {}),
|
|
1227
|
+
...(record.tbl !== null
|
|
1228
|
+
? { change: toStoredChange({ ...record, tbl: record.tbl }) }
|
|
1229
|
+
: {}),
|
|
1230
|
+
})),
|
|
1231
|
+
);
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
|
|
1235
|
+
const firstVariable = assertScopeIndexedScan(query);
|
|
1236
|
+
const firstValues = query.scopeFilter[firstVariable] ?? [];
|
|
1237
|
+
if (firstValues.length === 0) return [];
|
|
1238
|
+
// Candidates via the inverted index LEFT JOINed to the row table — one
|
|
1239
|
+
// statement per page, never one per row (see `scanRowPageSql`). Exact
|
|
1240
|
+
// multi-variable verification against the stored scope map below.
|
|
1241
|
+
const sql = scanRowPageSql(
|
|
1242
|
+
this.table(query.table),
|
|
1243
|
+
firstValues.length,
|
|
1244
|
+
'sqlite',
|
|
1245
|
+
);
|
|
1246
|
+
const rows: StoredRow[] = [];
|
|
1247
|
+
let afterRowId = query.afterRowId ?? '';
|
|
1248
|
+
const batchSize = Math.max(64, query.limit);
|
|
1249
|
+
while (rows.length < query.limit) {
|
|
1250
|
+
const records = await this.guardedAll<
|
|
1251
|
+
SqliteRowRecord & { payload: Uint8Array | null }
|
|
1252
|
+
>(sql, [
|
|
1253
|
+
partition,
|
|
1254
|
+
query.table,
|
|
1255
|
+
firstVariable,
|
|
1256
|
+
JSON.stringify(firstValues),
|
|
1257
|
+
afterRowId,
|
|
1258
|
+
batchSize,
|
|
1259
|
+
partition,
|
|
1260
|
+
]);
|
|
1261
|
+
if (records.length === 0) break;
|
|
1262
|
+
for (const record of records) {
|
|
1263
|
+
afterRowId = record.row_id;
|
|
1264
|
+
// NULL payload: an index candidate whose row vanished — it still
|
|
1265
|
+
// advances the keyset cursor (LEFT JOIN contract) but yields no row.
|
|
1266
|
+
if (record.payload === null) continue;
|
|
1267
|
+
const stored = toStoredRow(record);
|
|
1268
|
+
if (!matchesEffective(stored.scopes, query.scopeFilter)) continue;
|
|
1269
|
+
rows.push(stored);
|
|
1270
|
+
if (rows.length >= query.limit) break;
|
|
1271
|
+
}
|
|
1272
|
+
if (records.length < batchSize) break;
|
|
1273
|
+
}
|
|
1274
|
+
return rows;
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
async scanRowsByIndex(
|
|
1278
|
+
partition: string,
|
|
1279
|
+
query: IndexRowScanQuery,
|
|
1280
|
+
): Promise<StoredRow[]> {
|
|
1281
|
+
const table = this.table(query.table);
|
|
1282
|
+
const statement = indexRowPageStatement(
|
|
1283
|
+
table,
|
|
1284
|
+
resolveIndexRowScan(table, query),
|
|
1285
|
+
query.values,
|
|
1286
|
+
partition,
|
|
1287
|
+
query.afterRowId,
|
|
1288
|
+
query.limit,
|
|
1289
|
+
'sqlite',
|
|
1290
|
+
);
|
|
1291
|
+
return (
|
|
1292
|
+
await this.guardedAll<SqliteRowRecord>(statement.sql, statement.params)
|
|
1293
|
+
).map(toStoredRow);
|
|
1294
|
+
}
|
|
1295
|
+
|
|
1296
|
+
async getClientRecord(
|
|
1297
|
+
partition: string,
|
|
1298
|
+
clientId: string,
|
|
1299
|
+
): Promise<ClientRecord | undefined> {
|
|
1300
|
+
const [record] = await this.all<SqliteClientRecord>(
|
|
1301
|
+
`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=? AND client_id=?`,
|
|
1302
|
+
[partition, clientId],
|
|
1303
|
+
);
|
|
1304
|
+
return record === undefined ? undefined : toClientRecord(record);
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
async putClientRecord(
|
|
1308
|
+
partition: string,
|
|
1309
|
+
record: ClientRecord,
|
|
1310
|
+
options?: ClientRecordWriteOptions,
|
|
1311
|
+
): Promise<void> {
|
|
1312
|
+
// One row per (partition, client_id): every non-key field is written
|
|
1313
|
+
// from the new record, except the registration list and digest, which
|
|
1314
|
+
// only a traveling list replaces (§4.5). `keepRoundOutcome` leaves the
|
|
1315
|
+
// round-state columns alone (§1.5, §4.5).
|
|
1316
|
+
const registration = record.registration;
|
|
1317
|
+
await this.all(
|
|
1318
|
+
`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, updated_at_ms, last_round_at_ms, schema_version, units, registration_digest, catching_up) VALUES (?,?,?,?,?,?,?,?,?,?,?)
|
|
1319
|
+
ON CONFLICT(partition, client_id) DO UPDATE SET
|
|
1320
|
+
actor_id=excluded.actor_id,
|
|
1321
|
+
wire_version=excluded.wire_version,
|
|
1322
|
+
cursor=excluded.cursor,
|
|
1323
|
+
updated_at_ms=excluded.updated_at_ms,
|
|
1324
|
+
last_round_at_ms=excluded.last_round_at_ms,
|
|
1325
|
+
schema_version=excluded.schema_version,${
|
|
1326
|
+
registration === undefined
|
|
1327
|
+
? ''
|
|
1328
|
+
: `
|
|
1329
|
+
units=excluded.units,
|
|
1330
|
+
registration_digest=excluded.registration_digest,`
|
|
1331
|
+
}
|
|
1332
|
+
catching_up=excluded.catching_up${
|
|
1333
|
+
options?.keepRoundOutcome === true
|
|
1334
|
+
? ''
|
|
1335
|
+
: `,
|
|
1336
|
+
last_error_code=NULL,
|
|
1337
|
+
consecutive_failures=0`
|
|
1338
|
+
}`,
|
|
1339
|
+
[
|
|
1340
|
+
partition,
|
|
1341
|
+
record.clientId,
|
|
1342
|
+
record.actorId,
|
|
1343
|
+
record.wireVersion,
|
|
1344
|
+
record.cursor,
|
|
1345
|
+
record.updatedAtMs,
|
|
1346
|
+
record.lastRoundAtMs ?? record.updatedAtMs,
|
|
1347
|
+
record.schemaVersion ?? null,
|
|
1348
|
+
registration === undefined ? null : JSON.stringify(registration.units),
|
|
1349
|
+
registration?.digest ?? null,
|
|
1350
|
+
JSON.stringify(record.catchingUp),
|
|
1351
|
+
],
|
|
1352
|
+
);
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
async recordClientRoundFailure(
|
|
1356
|
+
partition: string,
|
|
1357
|
+
clientId: string,
|
|
1358
|
+
actorId: string,
|
|
1359
|
+
errorCode: string,
|
|
1360
|
+
atMs: number,
|
|
1361
|
+
): Promise<void> {
|
|
1362
|
+
await this.all(
|
|
1363
|
+
`UPDATE sync_clients
|
|
1364
|
+
SET last_round_at_ms=?, last_error_code=?,
|
|
1365
|
+
consecutive_failures=consecutive_failures+1
|
|
1366
|
+
WHERE partition=? AND client_id=? AND actor_id=?`,
|
|
1367
|
+
[atMs, errorCode, partition, clientId, actorId],
|
|
1368
|
+
);
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
async advanceClientCursor(
|
|
1372
|
+
partition: string,
|
|
1373
|
+
clientId: string,
|
|
1374
|
+
actorId: string,
|
|
1375
|
+
logEpoch: string,
|
|
1376
|
+
cursor: number,
|
|
1377
|
+
updatedAtMs: number,
|
|
1378
|
+
): Promise<void> {
|
|
1379
|
+
// §8.2: the ack never raises the record cursor above a catching-up
|
|
1380
|
+
// unit's cursor, so pruning keeps that unit's history (§4.6).
|
|
1381
|
+
await this.all(
|
|
1382
|
+
`UPDATE sync_clients
|
|
1383
|
+
SET cursor=MAX(cursor, MIN(?, coalesce((
|
|
1384
|
+
SELECT min(json_extract(u.value, '$.cursor'))
|
|
1385
|
+
FROM json_each(sync_clients.catching_up) AS u
|
|
1386
|
+
WHERE json_extract(u.value, '$.cursor')>=0), ?))),
|
|
1387
|
+
updated_at_ms=MAX(updated_at_ms, ?)
|
|
1388
|
+
WHERE partition=? AND client_id=? AND actor_id=?
|
|
1389
|
+
AND EXISTS (SELECT 1 FROM sync_partition_registry
|
|
1390
|
+
WHERE partition=sync_clients.partition AND log_epoch=?)`,
|
|
1391
|
+
[cursor, cursor, updatedAtMs, partition, clientId, actorId, logEpoch],
|
|
1392
|
+
);
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
async getActiveClientCursorFloor(
|
|
1396
|
+
partition: string,
|
|
1397
|
+
cutoffMs: number,
|
|
1398
|
+
): Promise<number | null> {
|
|
1399
|
+
const [row] = await this.all<{ cursor: number | null }>(
|
|
1400
|
+
'SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=? AND updated_at_ms>=? AND cursor>=0',
|
|
1401
|
+
[partition, cutoffMs],
|
|
1402
|
+
);
|
|
1403
|
+
return row!.cursor;
|
|
1404
|
+
}
|
|
1405
|
+
|
|
1406
|
+
async listClientCursors(partition: string): Promise<ClientCursorInfo[]> {
|
|
1407
|
+
const records = await this.all<{
|
|
1408
|
+
client_id: string;
|
|
1409
|
+
cursor: number;
|
|
1410
|
+
updated_at_ms: number;
|
|
1411
|
+
}>(
|
|
1412
|
+
'SELECT client_id, cursor, updated_at_ms FROM sync_clients WHERE partition=?',
|
|
1413
|
+
[partition],
|
|
1414
|
+
);
|
|
1415
|
+
return records.map((r) => ({
|
|
1416
|
+
clientId: r.client_id,
|
|
1417
|
+
cursor: r.cursor,
|
|
1418
|
+
updatedAtMs: r.updated_at_ms,
|
|
1419
|
+
}));
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
async listRowsReferencingBlob(
|
|
1423
|
+
partition: string,
|
|
1424
|
+
blobId: string,
|
|
1425
|
+
): Promise<
|
|
1426
|
+
{
|
|
1427
|
+
readonly table: string;
|
|
1428
|
+
readonly rowId: string;
|
|
1429
|
+
readonly scopes: Record<string, string>;
|
|
1430
|
+
}[]
|
|
1431
|
+
> {
|
|
1432
|
+
// One statement: the by-blob index (§5.9.4/§5.9.5) supplies the candidate
|
|
1433
|
+
// rows and each row's stored scopes come from its row table for the §3.4
|
|
1434
|
+
// authorization test (see `rowsReferencingBlobSql`).
|
|
1435
|
+
if (this.tables === undefined || this.tables.size === 0) return [];
|
|
1436
|
+
return (
|
|
1437
|
+
await this.guardedAll<{
|
|
1438
|
+
tbl: string;
|
|
1439
|
+
row_id: string;
|
|
1440
|
+
scopes: string | null;
|
|
1441
|
+
}>(rowsReferencingBlobSql(this.tables.values(), 'sqlite'), [
|
|
1442
|
+
partition,
|
|
1443
|
+
blobId,
|
|
1444
|
+
])
|
|
1445
|
+
).flatMap((row) =>
|
|
1446
|
+
row.scopes === null
|
|
1447
|
+
? []
|
|
1448
|
+
: [
|
|
1449
|
+
{
|
|
1450
|
+
table: row.tbl,
|
|
1451
|
+
rowId: row.row_id,
|
|
1452
|
+
scopes: JSON.parse(row.scopes) as Record<string, string>,
|
|
1453
|
+
},
|
|
1454
|
+
],
|
|
1455
|
+
);
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
async listReferencedBlobIds(partition: string): Promise<string[]> {
|
|
1459
|
+
const rows = await this.all<{ blob_id: string }>(
|
|
1460
|
+
'SELECT DISTINCT blob_id FROM sync_blob_refs WHERE partition=?',
|
|
1461
|
+
[partition],
|
|
1462
|
+
);
|
|
1463
|
+
return rows.map((r) => r.blob_id);
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
async listScopeValueRows(
|
|
1467
|
+
partition: string,
|
|
1468
|
+
query: ScopeValueQuery,
|
|
1469
|
+
): Promise<string[]> {
|
|
1470
|
+
// §4.6.1: rowIds of one table bound to one scope value, budgeted.
|
|
1471
|
+
const rows = await this.all<{ row_id: string }>(
|
|
1472
|
+
'SELECT row_id FROM sync_row_scopes WHERE partition=? AND tbl=? AND var=? AND value=? ORDER BY row_id LIMIT ?',
|
|
1473
|
+
[partition, query.table, query.variable, query.value, query.limit],
|
|
1474
|
+
);
|
|
1475
|
+
return rows.map((row) => row.row_id);
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
async eraseScopeValueHistory(
|
|
1479
|
+
partition: string,
|
|
1480
|
+
query: ScopeValueHistoryQuery,
|
|
1481
|
+
): Promise<number> {
|
|
1482
|
+
const params = [
|
|
1483
|
+
partition,
|
|
1484
|
+
query.table,
|
|
1485
|
+
query.variable,
|
|
1486
|
+
query.value,
|
|
1487
|
+
query.throughSeq,
|
|
1488
|
+
`$.${JSON.stringify(query.variable)}`,
|
|
1489
|
+
query.limit,
|
|
1490
|
+
];
|
|
1491
|
+
// §4.6.1 history scrub: upsert change records of the binding first, then
|
|
1492
|
+
// the change scope entries whose commit no longer holds a change of the
|
|
1493
|
+
// binding, then `serverRow` of stored conflict results whose conflict
|
|
1494
|
+
// authorization scopes carry the binding. Each statement returns one row
|
|
1495
|
+
// per changed record (D1 batch results carry no reliable change count).
|
|
1496
|
+
const results = await this.atomic(
|
|
1497
|
+
[
|
|
1498
|
+
`DELETE FROM sync_changes WHERE rowid IN (
|
|
1499
|
+
SELECT c.rowid FROM sync_change_scopes s JOIN sync_changes c
|
|
1500
|
+
ON c.partition=s.partition AND c.commit_seq=s.commit_seq AND c.tbl=s.tbl
|
|
1501
|
+
WHERE s.partition=?1 AND s.tbl=?2 AND s.var=?3 AND s.value=?4
|
|
1502
|
+
AND s.commit_seq<=?5 AND c.op=1 AND json_extract(c.scopes, ?6)=s.value
|
|
1503
|
+
LIMIT ?7) RETURNING 1 AS removed`,
|
|
1504
|
+
`DELETE FROM sync_change_scopes WHERE rowid IN (
|
|
1505
|
+
SELECT s.rowid FROM sync_change_scopes s
|
|
1506
|
+
WHERE s.partition=?1 AND s.tbl=?2 AND s.var=?3 AND s.value=?4
|
|
1507
|
+
AND s.commit_seq<=?5 AND NOT EXISTS (
|
|
1508
|
+
SELECT 1 FROM sync_changes c
|
|
1509
|
+
WHERE c.partition=s.partition AND c.commit_seq=s.commit_seq
|
|
1510
|
+
AND c.tbl=s.tbl AND json_extract(c.scopes, ?6)=s.value)
|
|
1511
|
+
LIMIT ?7) RETURNING 1 AS removed`,
|
|
1512
|
+
`UPDATE sync_push_results SET result=json_set(result, '$.results', json((
|
|
1513
|
+
SELECT json_group_array(json(json_remove(e.value, '$.serverRow')))
|
|
1514
|
+
FROM (SELECT value FROM json_each(sync_push_results.result, '$.results')
|
|
1515
|
+
ORDER BY key) e)))
|
|
1516
|
+
WHERE rowid IN (
|
|
1517
|
+
SELECT rowid FROM sync_push_results
|
|
1518
|
+
WHERE partition=?1
|
|
1519
|
+
AND json_extract(result, '$.conflictAuthorization.table')=?2
|
|
1520
|
+
AND json_extract(result,
|
|
1521
|
+
'$.conflictAuthorization.scopes' || substr(?6, 2))=?4
|
|
1522
|
+
AND EXISTS (SELECT 1 FROM json_each(result, '$.results') r
|
|
1523
|
+
WHERE json_type(r.value, '$.serverRow') IS NOT NULL)
|
|
1524
|
+
LIMIT ?7) RETURNING 1 AS removed`,
|
|
1525
|
+
].map((sql) => ({ sql, params })),
|
|
1526
|
+
);
|
|
1527
|
+
return results.reduce((removed, rows) => removed + rows.length, 0);
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
// -- admin/console read surface --------------------------------------------
|
|
1531
|
+
|
|
1532
|
+
async listClientRecords(partition: string): Promise<ClientRecord[]> {
|
|
1533
|
+
return (
|
|
1534
|
+
await this.all<SqliteClientRecord>(
|
|
1535
|
+
`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC`,
|
|
1536
|
+
[partition],
|
|
1537
|
+
)
|
|
1538
|
+
).map(toClientRecord);
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
async listCommitMetadata(
|
|
1542
|
+
partition: string,
|
|
1543
|
+
query: CommitMetadataQuery,
|
|
1544
|
+
): Promise<CommitMetadata[]> {
|
|
1545
|
+
// Newest-first window, resumed by `afterSeq` (exclusive lower bound) and
|
|
1546
|
+
// ended by `beforeSeq` (exclusive upper bound) when paging older. An
|
|
1547
|
+
// omitted bound is the largest safe commitSeq, so one statement serves
|
|
1548
|
+
// both the bounded and unbounded read.
|
|
1549
|
+
const beforeSeq = query.beforeSeq ?? Number.MAX_SAFE_INTEGER;
|
|
1550
|
+
const rows = await this.all<{
|
|
1551
|
+
commit_seq: number;
|
|
1552
|
+
client_id: string;
|
|
1553
|
+
client_commit_id: string;
|
|
1554
|
+
actor_id: string;
|
|
1555
|
+
created_at_ms: number;
|
|
1556
|
+
}>(
|
|
1557
|
+
query.table
|
|
1558
|
+
? `SELECT c.commit_seq, c.client_id, c.client_commit_id, c.actor_id, c.created_at_ms
|
|
1559
|
+
FROM sync_commits c
|
|
1560
|
+
WHERE c.partition=? AND c.commit_seq>? AND c.commit_seq<?
|
|
1561
|
+
AND EXISTS (SELECT 1 FROM sync_changes ch
|
|
1562
|
+
WHERE ch.partition=c.partition AND ch.commit_seq=c.commit_seq AND ch.tbl=?)
|
|
1563
|
+
ORDER BY c.commit_seq DESC LIMIT ?`
|
|
1564
|
+
: `SELECT commit_seq, client_id, client_commit_id, actor_id, created_at_ms
|
|
1565
|
+
FROM sync_commits
|
|
1566
|
+
WHERE partition=? AND commit_seq>? AND commit_seq<?
|
|
1567
|
+
ORDER BY commit_seq DESC LIMIT ?`,
|
|
1568
|
+
query.table
|
|
1569
|
+
? [partition, query.afterSeq, beforeSeq, query.table, query.limit]
|
|
1570
|
+
: [partition, query.afterSeq, beforeSeq, query.limit],
|
|
1571
|
+
);
|
|
1572
|
+
// One grouped statement for the page's per-table change counts and, with
|
|
1573
|
+
// `changeSample`, one for the first `sample` changes (change-index order)
|
|
1574
|
+
// of every listed commit. The commit ids bind as one JSON array, so the
|
|
1575
|
+
// statement count does not grow with the page.
|
|
1576
|
+
const sampleCount =
|
|
1577
|
+
query.changeSample !== undefined &&
|
|
1578
|
+
Number.isInteger(query.changeSample) &&
|
|
1579
|
+
query.changeSample > 0
|
|
1580
|
+
? query.changeSample
|
|
1581
|
+
: 0;
|
|
1582
|
+
const seqs = JSON.stringify(rows.map((row) => row.commit_seq));
|
|
1583
|
+
const counts = new Map<number, { tbl: string; n: number }[]>();
|
|
1584
|
+
for (const count of rows.length === 0
|
|
1585
|
+
? []
|
|
1586
|
+
: await this.all<{ commit_seq: number; tbl: string; n: number }>(
|
|
1587
|
+
`SELECT commit_seq, tbl, count(*) AS n FROM sync_changes
|
|
1588
|
+
WHERE partition=? AND commit_seq IN (SELECT value FROM json_each(?))
|
|
1589
|
+
GROUP BY commit_seq, tbl ORDER BY commit_seq, tbl`,
|
|
1590
|
+
[partition, seqs],
|
|
1591
|
+
)) {
|
|
1592
|
+
counts.set(count.commit_seq, [
|
|
1593
|
+
...(counts.get(count.commit_seq) ?? []),
|
|
1594
|
+
count,
|
|
1595
|
+
]);
|
|
1596
|
+
}
|
|
1597
|
+
const samples = new Map<number, CommitChangeSummary[]>();
|
|
1598
|
+
for (const change of sampleCount === 0 || rows.length === 0
|
|
1599
|
+
? []
|
|
1600
|
+
: await this.all<{
|
|
1601
|
+
commit_seq: number;
|
|
1602
|
+
tbl: string;
|
|
1603
|
+
row_id: string;
|
|
1604
|
+
op: number;
|
|
1605
|
+
}>(
|
|
1606
|
+
`SELECT commit_seq, tbl, row_id, op FROM sync_changes
|
|
1607
|
+
WHERE partition=? AND commit_seq IN (SELECT value FROM json_each(?)) AND idx<?
|
|
1608
|
+
ORDER BY commit_seq, idx`,
|
|
1609
|
+
[partition, seqs, sampleCount],
|
|
1610
|
+
)) {
|
|
1611
|
+
samples.set(change.commit_seq, [
|
|
1612
|
+
...(samples.get(change.commit_seq) ?? []),
|
|
1613
|
+
{
|
|
1614
|
+
table: change.tbl,
|
|
1615
|
+
rowId: change.row_id,
|
|
1616
|
+
op: change.op === 1 ? 'upsert' : 'delete',
|
|
1617
|
+
},
|
|
1618
|
+
]);
|
|
1619
|
+
}
|
|
1620
|
+
const out: CommitMetadata[] = [];
|
|
1621
|
+
for (const row of rows) {
|
|
1622
|
+
const changes = counts.get(row.commit_seq) ?? [];
|
|
1623
|
+
out.push({
|
|
1624
|
+
commitSeq: row.commit_seq,
|
|
1625
|
+
clientId: row.client_id,
|
|
1626
|
+
clientCommitId: row.client_commit_id,
|
|
1627
|
+
actorId: row.actor_id,
|
|
1628
|
+
createdAtMs: row.created_at_ms,
|
|
1629
|
+
changeCount: changes.reduce((sum, c) => sum + c.n, 0),
|
|
1630
|
+
tables: changes.map((c) => c.tbl),
|
|
1631
|
+
...(sampleCount > 0
|
|
1632
|
+
? { sample: samples.get(row.commit_seq) ?? [] }
|
|
1633
|
+
: {}),
|
|
1634
|
+
});
|
|
1635
|
+
}
|
|
1636
|
+
return out;
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1639
|
+
async getCommit(
|
|
1640
|
+
partition: string,
|
|
1641
|
+
commitSeq: number,
|
|
1642
|
+
): Promise<CommitDetail | undefined> {
|
|
1643
|
+
const [meta] = await this.all<{
|
|
1644
|
+
client_id: string;
|
|
1645
|
+
client_commit_id: string;
|
|
1646
|
+
actor_id: string;
|
|
1647
|
+
created_at_ms: number;
|
|
1648
|
+
}>(
|
|
1649
|
+
'SELECT client_id, client_commit_id, actor_id, created_at_ms FROM sync_commits WHERE partition=? AND commit_seq=?',
|
|
1650
|
+
[partition, commitSeq],
|
|
1651
|
+
);
|
|
1652
|
+
if (meta === undefined) return undefined;
|
|
1653
|
+
const changes = (
|
|
1654
|
+
await this.all<SqliteChangeRecord>(
|
|
1655
|
+
'SELECT tbl, row_id, op, row_version, scopes, payload FROM sync_changes WHERE partition=? AND commit_seq=? ORDER BY idx',
|
|
1656
|
+
[partition, commitSeq],
|
|
1657
|
+
)
|
|
1658
|
+
).map(toStoredChange);
|
|
1659
|
+
return {
|
|
1660
|
+
commitSeq,
|
|
1661
|
+
clientId: meta.client_id,
|
|
1662
|
+
clientCommitId: meta.client_commit_id,
|
|
1663
|
+
actorId: meta.actor_id,
|
|
1664
|
+
createdAtMs: meta.created_at_ms,
|
|
1665
|
+
changeCount: changes.length,
|
|
1666
|
+
tables: [...new Set(changes.map((change) => change.table))].sort(),
|
|
1667
|
+
changes,
|
|
1668
|
+
};
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
async listRowChanges(
|
|
1672
|
+
partition: string,
|
|
1673
|
+
query: RowChangesQuery,
|
|
1674
|
+
): Promise<RowChangeRecord[]> {
|
|
1675
|
+
// Retained-log read for one row: no dedicated index (the write path pays
|
|
1676
|
+
// nothing for it), bounded by retention (§4.6).
|
|
1677
|
+
const rows = await this.all<{
|
|
1678
|
+
commit_seq: number;
|
|
1679
|
+
op: number;
|
|
1680
|
+
row_version: number | null;
|
|
1681
|
+
actor_id: string;
|
|
1682
|
+
client_id: string;
|
|
1683
|
+
created_at_ms: number;
|
|
1684
|
+
}>(
|
|
1685
|
+
`SELECT ch.commit_seq, ch.op, ch.row_version, c.actor_id, c.client_id, c.created_at_ms
|
|
1686
|
+
FROM sync_changes ch
|
|
1687
|
+
JOIN sync_commits c
|
|
1688
|
+
ON c.partition=ch.partition AND c.commit_seq=ch.commit_seq
|
|
1689
|
+
WHERE ch.partition=? AND ch.tbl=? AND ch.row_id=?
|
|
1690
|
+
ORDER BY ch.commit_seq DESC, ch.idx DESC LIMIT ?`,
|
|
1691
|
+
[partition, query.table, query.rowId, query.limit],
|
|
1692
|
+
);
|
|
1693
|
+
return rows.map((row) => ({
|
|
1694
|
+
commitSeq: row.commit_seq,
|
|
1695
|
+
op: row.op === 1 ? 'upsert' : 'delete',
|
|
1696
|
+
...(row.row_version !== null ? { rowVersion: row.row_version } : {}),
|
|
1697
|
+
actorId: row.actor_id,
|
|
1698
|
+
clientId: row.client_id,
|
|
1699
|
+
createdAtMs: row.created_at_ms,
|
|
1700
|
+
}));
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
async listRows(partition: string, query: RowListQuery): Promise<StoredRow[]> {
|
|
1704
|
+
// Keyset page over the row table's (partition, rowId) primary key: an
|
|
1705
|
+
// index range read, no scope filter, no per-row lookup.
|
|
1706
|
+
return (
|
|
1707
|
+
await this.guardedAll<SqliteRowRecord>(
|
|
1708
|
+
`SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
|
|
1709
|
+
FROM ${quoteIdent(this.table(query.table).name)}
|
|
1710
|
+
WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}>?
|
|
1711
|
+
ORDER BY ${quoteIdent(SYNC_ROW_ID_COLUMN)} LIMIT ?`,
|
|
1712
|
+
[partition, query.afterRowId ?? '', query.limit],
|
|
1713
|
+
)
|
|
1714
|
+
).map(toStoredRow);
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
async countRows(partition: string, table: string): Promise<number> {
|
|
1718
|
+
const [record] = await this.guardedAll<{ n: number }>(
|
|
1719
|
+
`SELECT count(*) AS n FROM ${quoteIdent(this.table(table).name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=?`,
|
|
1720
|
+
[partition],
|
|
1721
|
+
);
|
|
1722
|
+
return record?.n ?? 0;
|
|
1723
|
+
}
|
|
1724
|
+
|
|
1725
|
+
async listPushResults(
|
|
1726
|
+
partition: string,
|
|
1727
|
+
clientCommitId: string,
|
|
1728
|
+
): Promise<ClientPushResult[]> {
|
|
1729
|
+
// Retained outcomes for one clientCommitId: no dedicated index, ordered
|
|
1730
|
+
// by client so the read is deterministic.
|
|
1731
|
+
const records = await this.guardedAll<{
|
|
1732
|
+
client_id: string;
|
|
1733
|
+
result: string;
|
|
1734
|
+
}>(
|
|
1735
|
+
'SELECT client_id, result FROM sync_push_results WHERE partition=? AND client_commit_id=? ORDER BY client_id',
|
|
1736
|
+
[partition, clientCommitId],
|
|
1737
|
+
);
|
|
1738
|
+
return records.map((record) => ({
|
|
1739
|
+
clientId: record.client_id,
|
|
1740
|
+
result: deserializePushResult(record.result),
|
|
1741
|
+
}));
|
|
1742
|
+
}
|
|
1743
|
+
|
|
1744
|
+
async listPushWatermarks(
|
|
1745
|
+
partition: string,
|
|
1746
|
+
clientCommitId: string,
|
|
1747
|
+
): Promise<PushWatermark[]> {
|
|
1748
|
+
// The clients come from the retained commit log, which has no index on
|
|
1749
|
+
// `client_commit_id`; the read is bounded by retention (§4.6).
|
|
1750
|
+
return (
|
|
1751
|
+
await this.all<{
|
|
1752
|
+
client_id: string;
|
|
1753
|
+
outbox_id: string;
|
|
1754
|
+
through_seq: number;
|
|
1755
|
+
}>(
|
|
1756
|
+
`SELECT client_id, outbox_id, through_seq FROM sync_push_watermarks
|
|
1757
|
+
WHERE partition=? AND client_id IN (
|
|
1758
|
+
SELECT client_id FROM sync_commits WHERE partition=? AND client_commit_id=?)
|
|
1759
|
+
ORDER BY client_id, outbox_id`,
|
|
1760
|
+
[partition, partition, clientCommitId],
|
|
1761
|
+
)
|
|
1762
|
+
).map((row) => ({
|
|
1763
|
+
clientId: row.client_id,
|
|
1764
|
+
outboxId: row.outbox_id,
|
|
1765
|
+
throughSeq: row.through_seq,
|
|
1766
|
+
}));
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
async scopeActivity(
|
|
1770
|
+
partition: string,
|
|
1771
|
+
query: ScopeActivityQuery,
|
|
1772
|
+
): Promise<ScopeCommitActivity[]> {
|
|
1773
|
+
// One statement for the page: candidate (commit, table) pairs from the
|
|
1774
|
+
// `sync_change_scopes_by_key` range of the key, newest first, joined to
|
|
1775
|
+
// the commit metadata with the table's change count. A candidate whose
|
|
1776
|
+
// commit was pruned has no metadata and is skipped.
|
|
1777
|
+
return (
|
|
1778
|
+
await this.all<{
|
|
1779
|
+
commit_seq: number;
|
|
1780
|
+
tbl: string;
|
|
1781
|
+
actor_id: string;
|
|
1782
|
+
created_at_ms: number;
|
|
1783
|
+
n: number;
|
|
1784
|
+
}>(
|
|
1785
|
+
`SELECT s.commit_seq, s.tbl, m.actor_id, m.created_at_ms,
|
|
1786
|
+
(SELECT count(*) FROM sync_changes ch
|
|
1787
|
+
WHERE ch.partition=? AND ch.commit_seq=s.commit_seq AND ch.tbl=s.tbl) AS n
|
|
1788
|
+
FROM (SELECT DISTINCT commit_seq, tbl FROM sync_change_scopes
|
|
1789
|
+
WHERE partition=? AND var=? AND value=?
|
|
1790
|
+
ORDER BY commit_seq DESC LIMIT ?) s
|
|
1791
|
+
JOIN sync_commits m ON m.partition=? AND m.commit_seq=s.commit_seq
|
|
1792
|
+
ORDER BY s.commit_seq DESC`,
|
|
1793
|
+
[
|
|
1794
|
+
partition,
|
|
1795
|
+
partition,
|
|
1796
|
+
query.variable,
|
|
1797
|
+
query.value,
|
|
1798
|
+
query.limit,
|
|
1799
|
+
partition,
|
|
1800
|
+
],
|
|
1801
|
+
)
|
|
1802
|
+
).map((row) => ({
|
|
1803
|
+
commitSeq: row.commit_seq,
|
|
1804
|
+
table: row.tbl,
|
|
1805
|
+
createdAtMs: row.created_at_ms,
|
|
1806
|
+
actorId: row.actor_id,
|
|
1807
|
+
changeCount: row.n,
|
|
1808
|
+
}));
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1811
|
+
async listPartitions(): Promise<string[]> {
|
|
1812
|
+
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|
|
1813
|
+
}
|
|
397
1814
|
}
|
|
398
1815
|
|
|
399
1816
|
/**
|
|
400
|
-
*
|
|
401
|
-
*
|
|
402
|
-
*
|
|
403
|
-
* - rows arrive ordered (commit_seq, idx); consecutive rows with the same
|
|
404
|
-
* `commit_seq` are one candidate commit;
|
|
405
|
-
* - every candidate advances the cursor (`lastSeq`) — including vanished
|
|
406
|
-
* commits (NULL meta) and commits whose changes all fail the exact
|
|
407
|
-
* multi-variable scope verification (`matchesEffective`);
|
|
408
|
-
* - a commit is emitted only with its matching changes, in `idx` order;
|
|
409
|
-
* - stops after the commit that accumulates at least `remaining` matching
|
|
410
|
-
* changes (never splitting a commit).
|
|
411
|
-
*
|
|
412
|
-
* `candidateCount` counts the processed candidates so the caller can detect
|
|
413
|
-
* a short page (window exhausted) exactly as it did with the old candidate
|
|
414
|
-
* query.
|
|
1817
|
+
* The `StorageTransaction` writes SQLite and D1 share. `run` executes a write
|
|
1818
|
+
* at call time on SQLite, inside its `BEGIN IMMEDIATE`; D1 buffers it for the
|
|
1819
|
+
* one `db.batch` at `commit()`.
|
|
415
1820
|
*/
|
|
416
|
-
export
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
1821
|
+
export abstract class SqliteFamilyTransaction {
|
|
1822
|
+
protected readonly partition: string;
|
|
1823
|
+
protected open = true;
|
|
1824
|
+
|
|
1825
|
+
constructor(partition: string) {
|
|
1826
|
+
this.partition = partition;
|
|
1827
|
+
}
|
|
1828
|
+
|
|
1829
|
+
/** Execute (SQLite) or buffer (D1) one write of this transaction. */
|
|
1830
|
+
protected abstract run(sql: string, params: readonly SqliteValue[]): void;
|
|
1831
|
+
|
|
1832
|
+
/** Read persisted state from inside this transaction. */
|
|
1833
|
+
protected abstract all<T>(
|
|
1834
|
+
sql: string,
|
|
1835
|
+
params: readonly SqliteValue[],
|
|
1836
|
+
): Promise<T[]>;
|
|
1837
|
+
|
|
1838
|
+
protected abstract table(name: string): CompiledTable;
|
|
1839
|
+
|
|
1840
|
+
/** Allocate the next dense commitSeq (§2.1) and write its `sync_commits` row. */
|
|
1841
|
+
protected abstract appendCommitRow(commit: NewCommit): Promise<number>;
|
|
1842
|
+
|
|
1843
|
+
abstract getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
1844
|
+
|
|
1845
|
+
protected assertOpen(): void {
|
|
1846
|
+
if (!this.open) throw new Error('transaction already finished');
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
async getRows(
|
|
1850
|
+
table: string,
|
|
1851
|
+
rowIds: readonly string[],
|
|
1852
|
+
): Promise<ReadonlyMap<string, StoredRow>> {
|
|
1853
|
+
this.assertOpen();
|
|
1854
|
+
// One indexed lookup per id: SQLite is in-process, and on D1 `getRow`
|
|
1855
|
+
// owns the buffered-write overlay.
|
|
1856
|
+
const rows = new Map<string, StoredRow>();
|
|
1857
|
+
for (const rowId of new Set(rowIds)) {
|
|
1858
|
+
const row = await this.getRow(table, rowId);
|
|
1859
|
+
if (row !== undefined) rows.set(rowId, row);
|
|
1860
|
+
}
|
|
1861
|
+
return rows;
|
|
1862
|
+
}
|
|
1863
|
+
|
|
1864
|
+
async getTombstone(
|
|
1865
|
+
table: string,
|
|
1866
|
+
rowId: string,
|
|
1867
|
+
): Promise<Tombstone | undefined> {
|
|
1868
|
+
this.assertOpen();
|
|
1869
|
+
const [row] = await this.all<SqliteTombstoneRecord>(
|
|
1870
|
+
'SELECT commit_seq, server_version, scopes FROM sync_tombstones WHERE partition=? AND tbl=? AND row_id=?',
|
|
1871
|
+
[this.partition, table, rowId],
|
|
1872
|
+
);
|
|
1873
|
+
return row === undefined ? undefined : toTombstone(row);
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
async clearTombstone(table: string, rowId: string): Promise<void> {
|
|
1877
|
+
this.assertOpen();
|
|
1878
|
+
this.run(
|
|
1879
|
+
'DELETE FROM sync_tombstones WHERE partition=? AND tbl=? AND row_id=?',
|
|
1880
|
+
[this.partition, table, rowId],
|
|
1881
|
+
);
|
|
1882
|
+
}
|
|
1883
|
+
|
|
1884
|
+
/**
|
|
1885
|
+
* Write a row and refresh its scope-index entries. The DELETE drops only
|
|
1886
|
+
* the keys the new scope map no longer carries and runs before the upsert,
|
|
1887
|
+
* so its subquery reads the pre-upsert scope map; keys present in both
|
|
1888
|
+
* maps are left alone and the `INSERT OR IGNORE` writes nothing for them.
|
|
1889
|
+
*/
|
|
1890
|
+
protected writeRow(table: string, row: StoredRow): void {
|
|
1891
|
+
const p = this.partition;
|
|
1892
|
+
const compiled = this.table(table);
|
|
1893
|
+
this.run(
|
|
1894
|
+
`${deleteSqliteRowScopesSql(compiled)}
|
|
1895
|
+
AND (var, value) NOT IN (SELECT key, value FROM json_each(?))`,
|
|
1896
|
+
[p, table, row.rowId, p, row.rowId, JSON.stringify(row.scopes)],
|
|
1897
|
+
);
|
|
1898
|
+
this.run(
|
|
1899
|
+
upsertSql(compiled, 'sqlite'),
|
|
1900
|
+
upsertValues(compiled, p, row, 'sqlite'),
|
|
1901
|
+
);
|
|
1902
|
+
for (const [variable, value] of Object.entries(row.scopes)) {
|
|
1903
|
+
this.run(
|
|
1904
|
+
'INSERT OR IGNORE INTO sync_row_scopes(partition, tbl, var, value, row_id) VALUES (?,?,?,?,?)',
|
|
1905
|
+
[p, table, variable, value, row.rowId],
|
|
1906
|
+
);
|
|
1907
|
+
}
|
|
1908
|
+
}
|
|
1909
|
+
|
|
1910
|
+
async deleteRow(table: string, rowId: string): Promise<void> {
|
|
1911
|
+
this.assertOpen();
|
|
1912
|
+
const p = this.partition;
|
|
1913
|
+
const compiled = this.table(table);
|
|
1914
|
+
this.run(deleteSqliteRowScopesSql(compiled), [p, table, rowId, p, rowId]);
|
|
1915
|
+
this.run(deleteRowSql(compiled, 'sqlite'), [p, rowId]);
|
|
1916
|
+
// §5.9.4: a deleted row references no blobs.
|
|
1917
|
+
this.run(
|
|
1918
|
+
'DELETE FROM sync_blob_refs WHERE partition=? AND tbl=? AND row_id=?',
|
|
1919
|
+
[p, table, rowId],
|
|
1920
|
+
);
|
|
1921
|
+
}
|
|
1922
|
+
|
|
1923
|
+
async setBlobRefs(
|
|
1924
|
+
table: string,
|
|
1925
|
+
rowId: string,
|
|
1926
|
+
blobIds: readonly string[],
|
|
1927
|
+
): Promise<void> {
|
|
1928
|
+
this.assertOpen();
|
|
1929
|
+
const p = this.partition;
|
|
1930
|
+
// Replace the row's reference set atomically (§5.9.4).
|
|
1931
|
+
this.run(
|
|
1932
|
+
'DELETE FROM sync_blob_refs WHERE partition=? AND tbl=? AND row_id=?',
|
|
1933
|
+
[p, table, rowId],
|
|
1934
|
+
);
|
|
1935
|
+
for (const blobId of blobIds) {
|
|
1936
|
+
this.run(
|
|
1937
|
+
'INSERT OR IGNORE INTO sync_blob_refs(partition, tbl, row_id, blob_id) VALUES (?,?,?,?)',
|
|
1938
|
+
[p, table, rowId, blobId],
|
|
1939
|
+
);
|
|
1940
|
+
}
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1943
|
+
async appendCommit(commit: NewCommit): Promise<number> {
|
|
1944
|
+
this.assertOpen();
|
|
1945
|
+
const p = this.partition;
|
|
1946
|
+
const commitSeq = await this.appendCommitRow(commit);
|
|
1947
|
+
const liveAfterDelete = rowsLiveAfterDelete(commit.changes);
|
|
1948
|
+
commit.changes.forEach((change, idx) => {
|
|
1949
|
+
this.run(
|
|
1950
|
+
'INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload) VALUES (?,?,?,?,?,?,?,?,?)',
|
|
1951
|
+
[
|
|
1952
|
+
p,
|
|
1953
|
+
commitSeq,
|
|
1954
|
+
idx,
|
|
1955
|
+
change.table,
|
|
1956
|
+
change.rowId,
|
|
1957
|
+
change.op === 'upsert' ? 1 : 2,
|
|
1958
|
+
change.rowVersion,
|
|
1959
|
+
JSON.stringify(change.scopes),
|
|
1960
|
+
change.payload ?? null,
|
|
1961
|
+
],
|
|
1962
|
+
);
|
|
443
1963
|
if (
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
record.op === null ||
|
|
447
|
-
record.scopes === null
|
|
1964
|
+
change.op === 'delete' &&
|
|
1965
|
+
!liveAfterDelete.has(`${change.table}\u0000${change.rowId}`)
|
|
448
1966
|
) {
|
|
449
|
-
|
|
1967
|
+
// §5 delete precedence: every applied delete leaves a tombstone,
|
|
1968
|
+
// pruned with the commit log (§4.6). A delete that the same commit
|
|
1969
|
+
// follows with an upsert is a §2.2 scope move, not a deletion.
|
|
1970
|
+
this.run(
|
|
1971
|
+
`INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq, server_version, scopes) VALUES (?,?,?,?,?,?)
|
|
1972
|
+
ON CONFLICT(partition, tbl, row_id) DO UPDATE SET commit_seq=excluded.commit_seq, server_version=excluded.server_version, scopes=excluded.scopes`,
|
|
1973
|
+
[
|
|
1974
|
+
p,
|
|
1975
|
+
change.table,
|
|
1976
|
+
change.rowId,
|
|
1977
|
+
commitSeq,
|
|
1978
|
+
change.rowVersion,
|
|
1979
|
+
JSON.stringify(change.scopes),
|
|
1980
|
+
],
|
|
1981
|
+
);
|
|
1982
|
+
}
|
|
1983
|
+
for (const [variable, value] of Object.entries(change.scopes)) {
|
|
1984
|
+
this.run(
|
|
1985
|
+
'INSERT OR IGNORE INTO sync_change_scopes(partition, tbl, var, value, commit_seq) VALUES (?,?,?,?,?)',
|
|
1986
|
+
[p, change.table, variable, value, commitSeq],
|
|
1987
|
+
);
|
|
450
1988
|
}
|
|
451
|
-
const change = toStoredChange({
|
|
452
|
-
tbl: record.tbl,
|
|
453
|
-
row_id: record.row_id,
|
|
454
|
-
op: record.op,
|
|
455
|
-
row_version: record.row_version,
|
|
456
|
-
scopes: record.scopes,
|
|
457
|
-
payload: record.payload,
|
|
458
|
-
});
|
|
459
|
-
if (matchesEffective(change.scopes, scopeFilter)) changes.push(change);
|
|
460
|
-
}
|
|
461
|
-
// NULL meta: the commit vanished — cursor advanced, nothing emitted.
|
|
462
|
-
if (head.actor_id === null || head.created_at_ms === null) continue;
|
|
463
|
-
if (changes.length === 0) continue;
|
|
464
|
-
commits.push({
|
|
465
|
-
commitSeq: seq,
|
|
466
|
-
createdAtMs: head.created_at_ms,
|
|
467
|
-
actorId: head.actor_id,
|
|
468
|
-
changes,
|
|
469
1989
|
});
|
|
470
|
-
|
|
471
|
-
|
|
1990
|
+
return commitSeq;
|
|
1991
|
+
}
|
|
1992
|
+
|
|
1993
|
+
async putPushResult(
|
|
1994
|
+
clientId: string,
|
|
1995
|
+
clientCommitId: string,
|
|
1996
|
+
result: StoredPushResult,
|
|
1997
|
+
sequence?: OutboxSequence,
|
|
1998
|
+
): Promise<void> {
|
|
1999
|
+
this.assertOpen();
|
|
2000
|
+
this.run(
|
|
2001
|
+
'INSERT OR IGNORE INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq) VALUES (?,?,?,?,?,?)',
|
|
2002
|
+
[
|
|
2003
|
+
this.partition,
|
|
2004
|
+
clientId,
|
|
2005
|
+
clientCommitId,
|
|
2006
|
+
serializePushResult(result),
|
|
2007
|
+
sequence?.outboxId ?? null,
|
|
2008
|
+
sequence?.clientSeq ?? null,
|
|
2009
|
+
],
|
|
2010
|
+
);
|
|
2011
|
+
}
|
|
2012
|
+
|
|
2013
|
+
/**
|
|
2014
|
+
* The §9 settlement and §6.3.2 steps 1 and 2 for one request, then the
|
|
2015
|
+
* idempotency read of its first commit. The reads run before the writes,
|
|
2016
|
+
* so D1, which buffers the writes until commit, answers exactly as SQLite:
|
|
2017
|
+
* the first commit's result counts as deleted when the writes delete it.
|
|
2018
|
+
*/
|
|
2019
|
+
protected async settleOutbox(
|
|
2020
|
+
clientId: string,
|
|
2021
|
+
clientCommitId: string,
|
|
2022
|
+
outbox: OutboxSettlement,
|
|
2023
|
+
): Promise<{
|
|
2024
|
+
readonly watermark: number;
|
|
2025
|
+
readonly persisted: StoredPushResult | undefined;
|
|
2026
|
+
}> {
|
|
2027
|
+
const first = outbox.commits[0];
|
|
2028
|
+
const last = outbox.commits.at(-1);
|
|
2029
|
+
if (
|
|
2030
|
+
first === undefined ||
|
|
2031
|
+
last === undefined ||
|
|
2032
|
+
first.clientCommitId !== clientCommitId
|
|
2033
|
+
) {
|
|
2034
|
+
throw new Error("outbox settlement rides the request's first commit");
|
|
2035
|
+
}
|
|
2036
|
+
const p = this.partition;
|
|
2037
|
+
const [state] = await this.all<{
|
|
2038
|
+
stored: number | null;
|
|
2039
|
+
last_sequenceless: number;
|
|
2040
|
+
result: string | null;
|
|
2041
|
+
outbox_id: string | null;
|
|
2042
|
+
client_seq: number | null;
|
|
2043
|
+
}>(
|
|
2044
|
+
`SELECT (SELECT through_seq FROM sync_push_watermarks
|
|
2045
|
+
WHERE partition=?1 AND client_id=?2 AND outbox_id=?3) AS stored,
|
|
2046
|
+
EXISTS (SELECT 1 FROM sync_push_results WHERE partition=?1
|
|
2047
|
+
AND client_id=?2 AND client_commit_id=?5 AND outbox_id IS NULL) AS last_sequenceless,
|
|
2048
|
+
r.result AS result, r.outbox_id AS outbox_id, r.client_seq AS client_seq
|
|
2049
|
+
FROM (SELECT 1) LEFT JOIN sync_push_results AS r
|
|
2050
|
+
ON r.partition=?1 AND r.client_id=?2 AND r.client_commit_id=?4`,
|
|
2051
|
+
[p, clientId, outbox.outboxId, first.clientCommitId, last.clientCommitId],
|
|
2052
|
+
);
|
|
2053
|
+
const watermark = Math.max(state?.stored ?? 0, first.clientSeq);
|
|
2054
|
+
const commits = JSON.stringify(
|
|
2055
|
+
outbox.commits.map((commit) => [commit.clientCommitId, commit.clientSeq]),
|
|
2056
|
+
);
|
|
2057
|
+
// §9 (b), then (a): with no pre-upgrade result on the request's last
|
|
2058
|
+
// commit, every remaining one outside the request left the outbox.
|
|
2059
|
+
if (state?.last_sequenceless !== 1) {
|
|
2060
|
+
this.run(
|
|
2061
|
+
`DELETE FROM sync_push_results
|
|
2062
|
+
WHERE partition=? AND client_id=? AND outbox_id IS NULL
|
|
2063
|
+
AND client_commit_id NOT IN (SELECT json_extract(value, '$[0]') FROM json_each(?))`,
|
|
2064
|
+
[p, clientId, commits],
|
|
2065
|
+
);
|
|
2066
|
+
}
|
|
2067
|
+
this.run(
|
|
2068
|
+
`UPDATE sync_push_results SET outbox_id=?3, client_seq=(
|
|
2069
|
+
SELECT json_extract(c.value, '$[1]') FROM json_each(?4) AS c
|
|
2070
|
+
WHERE json_extract(c.value, '$[0]')=sync_push_results.client_commit_id)
|
|
2071
|
+
WHERE partition=?1 AND client_id=?2 AND outbox_id IS NULL
|
|
2072
|
+
AND client_commit_id IN (SELECT json_extract(value, '$[0]') FROM json_each(?4))`,
|
|
2073
|
+
[p, clientId, outbox.outboxId, commits],
|
|
2074
|
+
);
|
|
2075
|
+
// §6.3.2 steps 1 and 2.
|
|
2076
|
+
this.run(
|
|
2077
|
+
`INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq) VALUES (?,?,?,?)
|
|
2078
|
+
ON CONFLICT(partition, client_id, outbox_id) DO UPDATE SET through_seq=max(through_seq, excluded.through_seq)`,
|
|
2079
|
+
[p, clientId, outbox.outboxId, watermark],
|
|
2080
|
+
);
|
|
2081
|
+
this.run(
|
|
2082
|
+
'DELETE FROM sync_push_results WHERE partition=? AND client_id=? AND outbox_id=? AND client_seq<?',
|
|
2083
|
+
[p, clientId, outbox.outboxId, watermark],
|
|
2084
|
+
);
|
|
2085
|
+
// The sequence step 2 sees: (a) gives a pre-upgrade result the first
|
|
2086
|
+
// commit's sequence; a result of this outbox keeps its stored one.
|
|
2087
|
+
const deleted =
|
|
2088
|
+
state?.outbox_id === null
|
|
2089
|
+
? first.clientSeq < watermark
|
|
2090
|
+
: state?.outbox_id === outbox.outboxId &&
|
|
2091
|
+
(state.client_seq ?? 0) < watermark;
|
|
2092
|
+
return {
|
|
2093
|
+
watermark,
|
|
2094
|
+
persisted:
|
|
2095
|
+
state?.result === null || state?.result === undefined || deleted
|
|
2096
|
+
? undefined
|
|
2097
|
+
: deserializePushResult(state.result),
|
|
2098
|
+
};
|
|
2099
|
+
}
|
|
2100
|
+
|
|
2101
|
+
async enqueueReactions(reactions: readonly NewReaction[]): Promise<void> {
|
|
2102
|
+
this.assertOpen();
|
|
2103
|
+
for (const reaction of reactions) {
|
|
2104
|
+
this.run(
|
|
2105
|
+
`INSERT INTO sync_reactions(
|
|
2106
|
+
partition, idempotency_key, type, version, payload,
|
|
2107
|
+
source_client_id, source_client_commit_id, source_commit_seq,
|
|
2108
|
+
created_at_ms, available_at_ms, status, attempts, max_attempts
|
|
2109
|
+
) VALUES (?,?,?,?,?,?,?,?,?,?,'pending',0,?)`,
|
|
2110
|
+
[
|
|
2111
|
+
this.partition,
|
|
2112
|
+
reaction.idempotencyKey,
|
|
2113
|
+
reaction.type,
|
|
2114
|
+
reaction.version,
|
|
2115
|
+
JSON.stringify(reaction.payload),
|
|
2116
|
+
reaction.sourceClientId,
|
|
2117
|
+
reaction.sourceClientCommitId,
|
|
2118
|
+
reaction.sourceCommitSeq,
|
|
2119
|
+
reaction.createdAtMs,
|
|
2120
|
+
reaction.createdAtMs,
|
|
2121
|
+
reaction.maxAttempts,
|
|
2122
|
+
],
|
|
2123
|
+
);
|
|
2124
|
+
}
|
|
472
2125
|
}
|
|
473
|
-
return { commits, delivered, lastSeq, candidateCount };
|
|
474
2126
|
}
|