@syncular/server 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -36
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +46 -40
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +19 -5
- package/dist/context.js +19 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +249 -1119
- package/dist/errors.js +16 -0
- package/dist/events.d.ts +36 -18
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +503 -208
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +24 -12
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +420 -262
- package/dist/pull.d.ts +70 -47
- package/dist/pull.js +270 -301
- package/dist/push.d.ts +12 -3
- package/dist/push.js +361 -173
- package/dist/realtime.d.ts +85 -39
- package/dist/realtime.js +355 -219
- package/dist/relational-rows.d.ts +21 -66
- package/dist/relational-rows.js +28 -109
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +10 -16
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +22 -6
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +164 -84
- package/dist/sqlite-dialect.js +1145 -150
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +303 -1123
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +185 -63
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +99 -57
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +33 -6
- package/src/d1-storage.ts +350 -1700
- package/src/errors.ts +16 -0
- package/src/events.ts +37 -18
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +654 -261
- package/src/local-authority.ts +27 -14
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/operations.ts +44 -42
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +557 -354
- package/src/pull.ts +406 -440
- package/src/push.ts +466 -220
- package/src/realtime.ts +447 -290
- package/src/relational-rows.ts +43 -121
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +13 -20
- package/src/segment-download.ts +33 -7
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1851 -199
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +390 -1821
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +215 -67
package/dist/d1-storage.js
CHANGED
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
var _a;
|
|
2
|
-
import { validateCommitPruneQuery } from './prune.js';
|
|
3
|
-
import { StorageQueryError } from './storage-errors.js';
|
|
4
|
-
import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, rowsLiveAfterDelete, } from './storage.js';
|
|
5
2
|
/**
|
|
6
3
|
* Cloudflare D1 server storage for Workers deployments.
|
|
7
4
|
*
|
|
8
5
|
* D1 *is* SQLite exposed over an async, statement-at-a-time API
|
|
9
6
|
* (`prepare(sql).bind(...).all()` / `.first()` / `.run()`, plus `batch([…])`
|
|
10
|
-
* for atomic multi-statement writes).
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* interactive transaction).
|
|
7
|
+
* for atomic multi-statement writes). The statements it shares with
|
|
8
|
+
* `SqliteServerStorage` live in `SqliteFamilyStorage` (`sqlite-dialect.ts`);
|
|
9
|
+
* this class adds the execution seam, the published-schema guard, the
|
|
10
|
+
* resumable schema migration and the buffered transaction.
|
|
15
11
|
*
|
|
16
12
|
* ## Transaction model (the one real divergence)
|
|
17
13
|
*
|
|
@@ -44,38 +40,21 @@ import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, rowsLiveAfterDelete, } f
|
|
|
44
40
|
* rather than a lock D1 does not expose.
|
|
45
41
|
*/
|
|
46
42
|
import { decodeRow } from '@syncular/core';
|
|
47
|
-
import {
|
|
48
|
-
import { syncError } from './errors.js';
|
|
49
|
-
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, upsertSql, upsertValues, } from './relational-rows.js';
|
|
43
|
+
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
|
|
50
44
|
import { matchesEffective } from './scopes.js';
|
|
51
|
-
import { asUint8Array,
|
|
52
|
-
import {
|
|
45
|
+
import { asUint8Array, deserializePushResult, placeholders, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteDdlStatements, SQLITE_UPGRADE_PROBE, sqliteUpgradeDdl, toStoredRow, toTombstone, } from './sqlite-dialect.js';
|
|
46
|
+
import { commitDeleteFloor } from './storage.js';
|
|
47
|
+
import { isD1ConstraintError, StorageConstraintError, StorageQueryError, } from './storage-errors.js';
|
|
53
48
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
createdAtMs: record.created_at_ms,
|
|
64
|
-
maxAttempts: record.max_attempts,
|
|
65
|
-
status: record.status,
|
|
66
|
-
attempts: record.attempts,
|
|
67
|
-
availableAtMs: record.available_at_ms,
|
|
68
|
-
...(record.lease_owner !== null ? { leaseOwner: record.lease_owner } : {}),
|
|
69
|
-
...(record.lease_expires_at_ms !== null
|
|
70
|
-
? { leaseExpiresAtMs: record.lease_expires_at_ms }
|
|
71
|
-
: {}),
|
|
72
|
-
...(record.completed_at_ms !== null
|
|
73
|
-
? { completedAtMs: record.completed_at_ms }
|
|
74
|
-
: {}),
|
|
75
|
-
...(record.last_failure !== null
|
|
76
|
-
? { lastFailure: JSON.parse(record.last_failure) }
|
|
77
|
-
: {}),
|
|
78
|
-
};
|
|
49
|
+
/** The rows of one `db.batch` result entry (`{ results }` per statement). */
|
|
50
|
+
function batchRows(result) {
|
|
51
|
+
if (typeof result !== 'object' ||
|
|
52
|
+
result === null ||
|
|
53
|
+
!('results' in result) ||
|
|
54
|
+
!Array.isArray(result.results)) {
|
|
55
|
+
throw new Error('D1 batch returned an invalid result');
|
|
56
|
+
}
|
|
57
|
+
return result.results;
|
|
79
58
|
}
|
|
80
59
|
function relationalValuesEqual(left, right) {
|
|
81
60
|
if (left instanceof Uint8Array && right instanceof Uint8Array) {
|
|
@@ -92,39 +71,24 @@ async function queryAuthoritativeOn(db, partition, prepared) {
|
|
|
92
71
|
.prepare('SELECT max_commit_seq FROM sync_partitions WHERE partition=?')
|
|
93
72
|
.bind(partition),
|
|
94
73
|
]);
|
|
95
|
-
const
|
|
96
|
-
const cursorResult = results[1];
|
|
97
|
-
if (typeof rowsResult !== 'object' ||
|
|
98
|
-
rowsResult === null ||
|
|
99
|
-
!('results' in rowsResult) ||
|
|
100
|
-
!Array.isArray(rowsResult.results) ||
|
|
101
|
-
typeof cursorResult !== 'object' ||
|
|
102
|
-
cursorResult === null ||
|
|
103
|
-
!('results' in cursorResult) ||
|
|
104
|
-
!Array.isArray(cursorResult.results)) {
|
|
105
|
-
throw new Error('D1 registered query returned an invalid batch result');
|
|
106
|
-
}
|
|
107
|
-
const cursor = cursorResult.results[0];
|
|
108
|
-
const maxCommitSeq = typeof cursor === 'object' &&
|
|
109
|
-
cursor !== null &&
|
|
110
|
-
'max_commit_seq' in cursor &&
|
|
111
|
-
typeof cursor.max_commit_seq === 'number'
|
|
112
|
-
? cursor.max_commit_seq
|
|
113
|
-
: 0;
|
|
74
|
+
const cursor = batchRows(results[1])[0];
|
|
114
75
|
return {
|
|
115
|
-
rows:
|
|
116
|
-
maxCommitSeq
|
|
76
|
+
rows: batchRows(results[0]).filter((row) => typeof row === 'object' && row !== null),
|
|
77
|
+
maxCommitSeq: typeof cursor === 'object' &&
|
|
78
|
+
cursor !== null &&
|
|
79
|
+
'max_commit_seq' in cursor &&
|
|
80
|
+
typeof cursor.max_commit_seq === 'number'
|
|
81
|
+
? cursor.max_commit_seq
|
|
82
|
+
: 0,
|
|
117
83
|
};
|
|
118
84
|
}
|
|
119
|
-
class D1Transaction {
|
|
85
|
+
class D1Transaction extends SqliteFamilyTransaction {
|
|
120
86
|
#db;
|
|
121
|
-
#partition;
|
|
122
87
|
#resolveTable;
|
|
123
88
|
/** Storage-owned relation rewrite, bound to this transaction's partition. */
|
|
124
89
|
#prepareQuery;
|
|
125
90
|
#pushApplySerialized;
|
|
126
91
|
#buffer = [];
|
|
127
|
-
#open = true;
|
|
128
92
|
/** Live snapshot of `max_commit_seq`, advanced within this transaction. */
|
|
129
93
|
#maxCommitSeq;
|
|
130
94
|
#pushApplyCheckpoint;
|
|
@@ -146,99 +110,132 @@ class D1Transaction {
|
|
|
146
110
|
*/
|
|
147
111
|
#pending = new Map();
|
|
148
112
|
#clearedTombstones = new Set();
|
|
113
|
+
/**
|
|
114
|
+
* `prefetchRows` snapshot keyed `tbl\u0000rowId`: the stored row and its §5
|
|
115
|
+
* tombstone, `undefined` when absent. Nothing persists before `commit()`,
|
|
116
|
+
* so the snapshot stays exact; the `#pending` overlay still wins for rows
|
|
117
|
+
* this transaction wrote.
|
|
118
|
+
*/
|
|
119
|
+
#prefetched = new Map();
|
|
149
120
|
constructor(db, partition, resolveTable, prepareQuery, pushApplySerialized) {
|
|
121
|
+
super(partition);
|
|
150
122
|
this.#db = db;
|
|
151
|
-
this.#partition = partition;
|
|
152
123
|
this.#resolveTable = resolveTable;
|
|
153
124
|
this.#prepareQuery = prepareQuery;
|
|
154
125
|
this.#pushApplySerialized = pushApplySerialized;
|
|
155
126
|
}
|
|
156
|
-
|
|
157
|
-
if (!this.#open)
|
|
158
|
-
throw new Error('transaction already finished');
|
|
159
|
-
}
|
|
160
|
-
#buffer_(sql, params) {
|
|
127
|
+
run(sql, params) {
|
|
161
128
|
this.#buffer.push({ sql, params });
|
|
162
129
|
}
|
|
130
|
+
async all(sql, params) {
|
|
131
|
+
return (await this.#db
|
|
132
|
+
.prepare(sql)
|
|
133
|
+
.bind(...params)
|
|
134
|
+
.all()).results;
|
|
135
|
+
}
|
|
136
|
+
table(name) {
|
|
137
|
+
return this.#resolveTable(name);
|
|
138
|
+
}
|
|
163
139
|
static #key(table, rowId) {
|
|
164
140
|
return `${table}\u0000${rowId}`;
|
|
165
141
|
}
|
|
166
142
|
async getRow(table, rowId) {
|
|
167
|
-
this
|
|
143
|
+
this.assertOpen();
|
|
168
144
|
const pending = this.#pending.get(_a.#key(table, rowId));
|
|
169
145
|
if (pending !== undefined) {
|
|
170
146
|
return pending.kind === 'row' ? pending.row : undefined;
|
|
171
147
|
}
|
|
148
|
+
const prefetched = this.#prefetched.get(_a.#key(table, rowId));
|
|
149
|
+
if (prefetched !== undefined)
|
|
150
|
+
return prefetched.row;
|
|
172
151
|
const record = await this.#db
|
|
173
|
-
.prepare(selectRowSql(this
|
|
174
|
-
.bind(this
|
|
152
|
+
.prepare(selectRowSql(this.table(table), 'sqlite'))
|
|
153
|
+
.bind(this.partition, rowId)
|
|
175
154
|
.first();
|
|
176
155
|
return record === null ? undefined : toStoredRow(record);
|
|
177
156
|
}
|
|
178
|
-
async
|
|
179
|
-
this
|
|
180
|
-
// One lookup per id: `getRow` owns the buffered-write overlay, and a
|
|
181
|
-
// mutable id list cannot share one bound statement across D1's limit.
|
|
182
|
-
const rows = new Map();
|
|
183
|
-
for (const rowId of new Set(rowIds)) {
|
|
184
|
-
const row = await this.getRow(table, rowId);
|
|
185
|
-
if (row !== undefined)
|
|
186
|
-
rows.set(rowId, row);
|
|
187
|
-
}
|
|
188
|
-
return rows;
|
|
189
|
-
}
|
|
190
|
-
async getTombstoneSeq(table, rowId) {
|
|
191
|
-
this.#assertOpen();
|
|
192
|
-
if (this.#clearedTombstones.has(_a.#key(table, rowId))) {
|
|
193
|
-
return undefined;
|
|
194
|
-
}
|
|
157
|
+
async getTombstone(table, rowId) {
|
|
158
|
+
this.assertOpen();
|
|
195
159
|
// Persisted read, no #pending overlay: a delete buffered by this commit
|
|
196
160
|
// records its tombstone at appendCommit, matching the SQLite/Postgres
|
|
197
161
|
// adapters where the tombstone insert also runs at commit append.
|
|
198
|
-
const
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
return
|
|
162
|
+
const key = _a.#key(table, rowId);
|
|
163
|
+
if (this.#clearedTombstones.has(key))
|
|
164
|
+
return undefined;
|
|
165
|
+
const prefetched = this.#prefetched.get(key);
|
|
166
|
+
return prefetched === undefined
|
|
167
|
+
? super.getTombstone(table, rowId)
|
|
168
|
+
: prefetched.tombstone;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Read every named row and its tombstone in one `db.batch` (two statements
|
|
172
|
+
* per table, ids bound as a JSON array), so a multi-operation push pays one
|
|
173
|
+
* round trip instead of up to two per operation.
|
|
174
|
+
*/
|
|
175
|
+
async prefetchRows(targets) {
|
|
176
|
+
this.assertOpen();
|
|
177
|
+
const specs = targets
|
|
178
|
+
.map(({ table, rowIds }) => ({
|
|
179
|
+
table,
|
|
180
|
+
ids: [...new Set(rowIds)].filter((rowId) => !this.#prefetched.has(_a.#key(table, rowId))),
|
|
181
|
+
}))
|
|
182
|
+
.filter(({ ids }) => ids.length > 0);
|
|
183
|
+
if (specs.length === 0)
|
|
184
|
+
return;
|
|
185
|
+
const results = await this.#db.batch(specs.flatMap(({ table, ids }) => {
|
|
186
|
+
const bound = JSON.stringify(ids);
|
|
187
|
+
return [
|
|
188
|
+
this.#db
|
|
189
|
+
.prepare(selectRowsSql(this.table(table), 'sqlite'))
|
|
190
|
+
.bind(this.partition, bound),
|
|
191
|
+
this.#db
|
|
192
|
+
.prepare('SELECT row_id, commit_seq, server_version, scopes FROM sync_tombstones WHERE partition=? AND tbl=? AND row_id IN (SELECT value FROM json_each(?))')
|
|
193
|
+
.bind(this.partition, table, bound),
|
|
194
|
+
];
|
|
195
|
+
}));
|
|
196
|
+
specs.forEach(({ table, ids }, i) => {
|
|
197
|
+
const rows = new Map(batchRows(results[2 * i]).map((record) => [
|
|
198
|
+
record.row_id,
|
|
199
|
+
toStoredRow(record),
|
|
200
|
+
]));
|
|
201
|
+
const tombstones = new Map(batchRows(results[2 * i + 1]).map((record) => [record.row_id, toTombstone(record)]));
|
|
202
|
+
for (const rowId of ids) {
|
|
203
|
+
this.#prefetched.set(_a.#key(table, rowId), {
|
|
204
|
+
row: rows.get(rowId),
|
|
205
|
+
tombstone: tombstones.get(rowId),
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
});
|
|
203
209
|
}
|
|
204
210
|
async clearTombstone(table, rowId) {
|
|
205
|
-
|
|
211
|
+
await super.clearTombstone(table, rowId);
|
|
206
212
|
this.#clearedTombstones.add(_a.#key(table, rowId));
|
|
207
|
-
this.#buffer_('DELETE FROM sync_tombstones WHERE partition=? AND tbl=? AND row_id=?', [this.#partition, table, rowId]);
|
|
208
213
|
}
|
|
209
214
|
async getPushResult(clientId, clientCommitId) {
|
|
210
|
-
this
|
|
215
|
+
this.assertOpen();
|
|
211
216
|
// D1 reads run in autocommit; the push layer's duplicate re-check happens
|
|
212
217
|
// before this transaction buffers any write, so a direct read is exact.
|
|
213
|
-
const record = await this
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
.
|
|
217
|
-
if (record === null)
|
|
218
|
-
return undefined;
|
|
219
|
-
try {
|
|
220
|
-
return deserializePushResult(record.result);
|
|
221
|
-
}
|
|
222
|
-
catch {
|
|
223
|
-
throw syncError('sync.idempotency_cache_miss', 'persisted push result unreadable (§6.3)');
|
|
224
|
-
}
|
|
218
|
+
const [record] = await this.all('SELECT result FROM sync_push_results WHERE partition=? AND client_id=? AND client_commit_id=?', [this.partition, clientId, clientCommitId]);
|
|
219
|
+
return record === undefined
|
|
220
|
+
? undefined
|
|
221
|
+
: deserializePushResult(record.result);
|
|
225
222
|
}
|
|
226
223
|
async scanRows(query) {
|
|
227
|
-
this
|
|
224
|
+
this.assertOpen();
|
|
228
225
|
const firstVariable = assertScopeIndexedScan(query);
|
|
229
226
|
const firstValues = query.scopeFilter[firstVariable] ?? [];
|
|
230
227
|
if (firstValues.length === 0)
|
|
231
228
|
return [];
|
|
232
229
|
const pendingForTable = [...this.#pending.entries()].filter(([key]) => key.startsWith(`${query.table}\u0000`));
|
|
233
230
|
const persistedLimit = query.limit + pendingForTable.length;
|
|
234
|
-
const sql = scanRowPageSql(this
|
|
231
|
+
const sql = scanRowPageSql(this.table(query.table), firstValues.length, 'sqlite');
|
|
235
232
|
const persisted = [];
|
|
236
233
|
let afterRowId = query.afterRowId ?? '';
|
|
237
234
|
const batchSize = Math.max(64, persistedLimit);
|
|
238
235
|
while (persisted.length < persistedLimit) {
|
|
239
236
|
const { results: records } = await this.#db
|
|
240
237
|
.prepare(sql)
|
|
241
|
-
.bind(this
|
|
238
|
+
.bind(this.partition, query.table, firstVariable, JSON.stringify(firstValues), afterRowId, batchSize, this.partition)
|
|
242
239
|
.all();
|
|
243
240
|
if (records.length === 0)
|
|
244
241
|
break;
|
|
@@ -272,12 +269,12 @@ class D1Transaction {
|
|
|
272
269
|
.slice(0, query.limit);
|
|
273
270
|
}
|
|
274
271
|
async scanRowsByIndex(query) {
|
|
275
|
-
this
|
|
276
|
-
const table = this
|
|
272
|
+
this.assertOpen();
|
|
273
|
+
const table = this.table(query.table);
|
|
277
274
|
const index = resolveIndexRowScan(table, query);
|
|
278
275
|
const pendingForTable = [...this.#pending.entries()].filter(([key]) => key.startsWith(`${query.table}\u0000`));
|
|
279
276
|
const persistedLimit = query.limit + pendingForTable.length;
|
|
280
|
-
const statement = indexRowPageStatement(table, index, query.values, this
|
|
277
|
+
const statement = indexRowPageStatement(table, index, query.values, this.partition, query.afterRowId, persistedLimit, 'sqlite');
|
|
281
278
|
const { results: records } = await this.#db
|
|
282
279
|
.prepare(statement.sql)
|
|
283
280
|
.bind(...statement.params)
|
|
@@ -309,7 +306,7 @@ class D1Transaction {
|
|
|
309
306
|
.slice(0, query.limit);
|
|
310
307
|
}
|
|
311
308
|
async queryAuthoritative(query) {
|
|
312
|
-
this
|
|
309
|
+
this.assertOpen();
|
|
313
310
|
// D1 buffers writes until commit() and cannot overlay them on generated
|
|
314
311
|
// SQL, so a read over a written table would return committed state
|
|
315
312
|
// without this transaction's staged rows. Refuse it instead.
|
|
@@ -318,27 +315,46 @@ class D1Transaction {
|
|
|
318
315
|
throw new StorageQueryError('sync.storage.query_over_staged_writes');
|
|
319
316
|
}
|
|
320
317
|
}
|
|
321
|
-
return queryAuthoritativeOn(this.#db, this
|
|
318
|
+
return queryAuthoritativeOn(this.#db, this.partition, this.#prepareQuery(query));
|
|
322
319
|
}
|
|
323
|
-
async preparePushApply(clientId, clientCommitId, runningSchemaVersion, _targets = []) {
|
|
324
|
-
this
|
|
320
|
+
async preparePushApply(clientId, clientCommitId, runningSchemaVersion, _targets = [], outbox) {
|
|
321
|
+
this.assertOpen();
|
|
325
322
|
if (!this.#pushApplySerialized) {
|
|
326
323
|
throw new Error('D1 push apply requires externally serialized partition writes');
|
|
327
324
|
}
|
|
328
325
|
// D1 has no interactive lock. The caller explicitly asserted that every
|
|
329
326
|
// write for this partition is already serialized (normally by its DO).
|
|
327
|
+
// The settlement's writes sit below the checkpoint, so a rejection keeps
|
|
328
|
+
// them.
|
|
329
|
+
const settled = outbox === undefined
|
|
330
|
+
? undefined
|
|
331
|
+
: await this.settleOutbox(clientId, clientCommitId, outbox);
|
|
330
332
|
this.#pushApplyCheckpoint = this.#buffer.length;
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
333
|
+
// The serve gate and the delete floor (§2.2) in one round trip.
|
|
334
|
+
const read = await this.#db
|
|
335
|
+
.prepare('SELECT (SELECT schema_version FROM sync_schema_meta WHERE id=1) AS schema_version, (SELECT delete_floor FROM sync_partitions WHERE partition=?) AS delete_floor')
|
|
336
|
+
.bind(this.partition)
|
|
337
|
+
.first();
|
|
338
|
+
const gate = {
|
|
339
|
+
storedSchemaVersion: read?.schema_version ?? 0,
|
|
340
|
+
logEpoch: undefined,
|
|
341
|
+
pending: [],
|
|
334
342
|
};
|
|
343
|
+
const deleteFloor = read?.delete_floor ?? 0;
|
|
344
|
+
return settled === undefined
|
|
345
|
+
? {
|
|
346
|
+
gate,
|
|
347
|
+
deleteFloor,
|
|
348
|
+
persisted: await this.getPushResult(clientId, clientCommitId),
|
|
349
|
+
}
|
|
350
|
+
: { gate, deleteFloor, ...settled };
|
|
335
351
|
}
|
|
336
352
|
advanceCheckpoint(_name, _ownerEpoch, _watermark, _observedRows, _nowMs) {
|
|
337
353
|
// D1 installs no fence and declares no checkpoints in this release.
|
|
338
354
|
return Promise.reject(new StorageQueryError('sync.storage.checkpoint_unsupported'));
|
|
339
355
|
}
|
|
340
356
|
async readServeGate(_runningSchemaVersion) {
|
|
341
|
-
this
|
|
357
|
+
this.assertOpen();
|
|
342
358
|
const marker = await this.#db
|
|
343
359
|
.prepare('SELECT schema_version FROM sync_schema_meta WHERE id=1')
|
|
344
360
|
.first();
|
|
@@ -348,8 +364,8 @@ class D1Transaction {
|
|
|
348
364
|
pending: [],
|
|
349
365
|
};
|
|
350
366
|
}
|
|
351
|
-
async commitRejectedPushResult(clientId, clientCommitId, result) {
|
|
352
|
-
this
|
|
367
|
+
async commitRejectedPushResult(clientId, clientCommitId, result, sequence) {
|
|
368
|
+
this.assertOpen();
|
|
353
369
|
const checkpoint = this.#pushApplyCheckpoint;
|
|
354
370
|
if (checkpoint === undefined) {
|
|
355
371
|
throw new Error('push rejection requires its apply checkpoint');
|
|
@@ -359,7 +375,7 @@ class D1Transaction {
|
|
|
359
375
|
this.#clearedTombstones.clear();
|
|
360
376
|
this.#maxCommitSeq = undefined;
|
|
361
377
|
this.#applicationOpIndexes.clear();
|
|
362
|
-
await this.putPushResult(clientId, clientCommitId, result);
|
|
378
|
+
await this.putPushResult(clientId, clientCommitId, result, sequence);
|
|
363
379
|
await this.commit();
|
|
364
380
|
// The idempotency re-check found no record under the serialized apply, so
|
|
365
381
|
// the insert above is the canonical outcome.
|
|
@@ -408,7 +424,7 @@ class D1Transaction {
|
|
|
408
424
|
});
|
|
409
425
|
const persisted = await this.#db
|
|
410
426
|
.prepare(sql)
|
|
411
|
-
.bind(this
|
|
427
|
+
.bind(this.partition, ...bind, row.rowId)
|
|
412
428
|
.first();
|
|
413
429
|
if (persisted === null)
|
|
414
430
|
continue;
|
|
@@ -425,9 +441,8 @@ class D1Transaction {
|
|
|
425
441
|
}
|
|
426
442
|
}
|
|
427
443
|
async upsertRow(table, row, context) {
|
|
428
|
-
this
|
|
429
|
-
|
|
430
|
-
await this.#assertNoUniqueCollision(compiled, row, context?.opIndex);
|
|
444
|
+
this.assertOpen();
|
|
445
|
+
await this.#assertNoUniqueCollision(this.table(table), row, context?.opIndex);
|
|
431
446
|
if (context?.opIndex !== undefined) {
|
|
432
447
|
this.#applicationOpIndexes.add(context.opIndex);
|
|
433
448
|
}
|
|
@@ -435,63 +450,25 @@ class D1Transaction {
|
|
|
435
450
|
kind: 'row',
|
|
436
451
|
row,
|
|
437
452
|
});
|
|
438
|
-
|
|
439
|
-
// Scope index maintenance: delete only the old keys the new scope map no
|
|
440
|
-
// longer carries, then insert the new keys. Keys present in both are left
|
|
441
|
-
// alone, so the `INSERT OR IGNORE` below finds them and writes nothing.
|
|
442
|
-
// This DELETE precedes the row upsert in the same atomic batch, and its
|
|
443
|
-
// subquery reads the row's pre-upsert scope map.
|
|
444
|
-
const scopesJson = JSON.stringify(row.scopes);
|
|
445
|
-
this.#buffer_(`${deleteSqliteRowScopesSql(compiled)}
|
|
446
|
-
AND (var, value) NOT IN (
|
|
447
|
-
SELECT key, value FROM json_each(?)
|
|
448
|
-
)`, [p, table, row.rowId, p, row.rowId, scopesJson]);
|
|
449
|
-
this.#buffer_(upsertSql(compiled, 'sqlite'), upsertValues(compiled, p, row, 'sqlite'));
|
|
450
|
-
for (const [variable, value] of Object.entries(row.scopes)) {
|
|
451
|
-
this.#buffer_('INSERT OR IGNORE INTO sync_row_scopes(partition, tbl, var, value, row_id) VALUES (?,?,?,?,?)', [p, table, variable, value, row.rowId]);
|
|
452
|
-
}
|
|
453
|
+
this.writeRow(table, row);
|
|
453
454
|
}
|
|
454
455
|
async deleteRow(table, rowId) {
|
|
455
|
-
|
|
456
|
+
await super.deleteRow(table, rowId);
|
|
456
457
|
this.#pending.set(_a.#key(table, rowId), { kind: 'deleted' });
|
|
457
|
-
const p = this.#partition;
|
|
458
|
-
const compiled = this.#resolveTable(table);
|
|
459
|
-
this.#buffer_(deleteSqliteRowScopesSql(compiled), [
|
|
460
|
-
p,
|
|
461
|
-
table,
|
|
462
|
-
rowId,
|
|
463
|
-
p,
|
|
464
|
-
rowId,
|
|
465
|
-
]);
|
|
466
|
-
this.#buffer_(deleteRowSql(compiled, 'sqlite'), [p, rowId]);
|
|
467
|
-
// §5.9.4: a deleted row references no blobs.
|
|
468
|
-
this.#buffer_('DELETE FROM sync_blob_refs WHERE partition=? AND tbl=? AND row_id=?', [p, table, rowId]);
|
|
469
458
|
}
|
|
470
|
-
async
|
|
471
|
-
this
|
|
472
|
-
const p = this.#partition;
|
|
473
|
-
this.#buffer_('DELETE FROM sync_blob_refs WHERE partition=? AND tbl=? AND row_id=?', [p, table, rowId]);
|
|
474
|
-
for (const blobId of blobIds) {
|
|
475
|
-
this.#buffer_('INSERT OR IGNORE INTO sync_blob_refs(partition, tbl, row_id, blob_id) VALUES (?,?,?,?)', [p, table, rowId, blobId]);
|
|
476
|
-
}
|
|
477
|
-
}
|
|
478
|
-
async appendCommit(commit) {
|
|
479
|
-
this.#assertOpen();
|
|
480
|
-
const p = this.#partition;
|
|
459
|
+
async appendCommitRow(commit) {
|
|
460
|
+
const p = this.partition;
|
|
481
461
|
// Read the live counter once, then advance it in memory for this tx; the
|
|
482
462
|
// increment is buffered and lands atomically at commit() (see header for
|
|
483
463
|
// the concurrency posture).
|
|
484
464
|
if (this.#maxCommitSeq === undefined) {
|
|
485
|
-
const row = await this
|
|
486
|
-
.prepare('SELECT max_commit_seq FROM sync_partitions WHERE partition=?')
|
|
487
|
-
.bind(p)
|
|
488
|
-
.first();
|
|
465
|
+
const [row] = await this.all('SELECT max_commit_seq FROM sync_partitions WHERE partition=?', [p]);
|
|
489
466
|
this.#maxCommitSeq = row?.max_commit_seq ?? 0;
|
|
490
467
|
}
|
|
491
468
|
const commitSeq = this.#maxCommitSeq + 1;
|
|
492
469
|
this.#maxCommitSeq = commitSeq;
|
|
493
|
-
this
|
|
494
|
-
this
|
|
470
|
+
this.run('INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor) VALUES (?,?,?) ON CONFLICT(partition) DO UPDATE SET max_commit_seq=excluded.max_commit_seq, delete_floor=max(delete_floor, excluded.delete_floor)', [p, commitSeq, commitDeleteFloor(commit.changes)]);
|
|
471
|
+
this.run('INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms) VALUES (?,?,?,?,?,?)', [
|
|
495
472
|
p,
|
|
496
473
|
commitSeq,
|
|
497
474
|
commit.clientId,
|
|
@@ -499,70 +476,19 @@ class D1Transaction {
|
|
|
499
476
|
commit.actorId,
|
|
500
477
|
commit.createdAtMs,
|
|
501
478
|
]);
|
|
502
|
-
const liveAfterDelete = rowsLiveAfterDelete(commit.changes);
|
|
503
|
-
commit.changes.forEach((change, idx) => {
|
|
504
|
-
this.#buffer_('INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload) VALUES (?,?,?,?,?,?,?,?,?)', [
|
|
505
|
-
p,
|
|
506
|
-
commitSeq,
|
|
507
|
-
idx,
|
|
508
|
-
change.table,
|
|
509
|
-
change.rowId,
|
|
510
|
-
change.op === 'upsert' ? 1 : 2,
|
|
511
|
-
change.rowVersion ?? null,
|
|
512
|
-
JSON.stringify(change.scopes),
|
|
513
|
-
change.payload ?? null,
|
|
514
|
-
]);
|
|
515
|
-
if (change.op === 'delete' &&
|
|
516
|
-
!liveAfterDelete.has(`${change.table}\u0000${change.rowId}`)) {
|
|
517
|
-
// §5 delete precedence: every applied delete leaves a tombstone,
|
|
518
|
-
// pruned with the commit log (§4.6). A delete that the same commit
|
|
519
|
-
// follows with an upsert is a §2.2 scope move, not a deletion.
|
|
520
|
-
this.#buffer_(`INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq) VALUES (?,?,?,?)
|
|
521
|
-
ON CONFLICT(partition, tbl, row_id) DO UPDATE SET commit_seq=excluded.commit_seq`, [p, change.table, change.rowId, commitSeq]);
|
|
522
|
-
}
|
|
523
|
-
for (const [variable, value] of Object.entries(change.scopes)) {
|
|
524
|
-
this.#buffer_('INSERT OR IGNORE INTO sync_change_scopes(partition, tbl, var, value, commit_seq) VALUES (?,?,?,?,?)', [p, change.table, variable, value, commitSeq]);
|
|
525
|
-
}
|
|
526
|
-
});
|
|
527
479
|
return commitSeq;
|
|
528
480
|
}
|
|
529
|
-
async putPushResult(clientId, clientCommitId, result) {
|
|
530
|
-
this.#assertOpen();
|
|
531
|
-
this.#buffer_('INSERT OR IGNORE INTO sync_push_results(partition, client_id, client_commit_id, result) VALUES (?,?,?,?)', [this.#partition, clientId, clientCommitId, serializePushResult(result)]);
|
|
532
|
-
}
|
|
533
|
-
async enqueueReactions(reactions) {
|
|
534
|
-
this.#assertOpen();
|
|
535
|
-
for (const reaction of reactions) {
|
|
536
|
-
this.#buffer_(`INSERT INTO sync_reactions(
|
|
537
|
-
partition, idempotency_key, type, version, payload,
|
|
538
|
-
source_client_id, source_client_commit_id, source_commit_seq,
|
|
539
|
-
created_at_ms, available_at_ms, status, attempts, max_attempts
|
|
540
|
-
) VALUES (?,?,?,?,?,?,?,?,?,?,'pending',0,?)`, [
|
|
541
|
-
this.#partition,
|
|
542
|
-
reaction.idempotencyKey,
|
|
543
|
-
reaction.type,
|
|
544
|
-
reaction.version,
|
|
545
|
-
JSON.stringify(reaction.payload),
|
|
546
|
-
reaction.sourceClientId,
|
|
547
|
-
reaction.sourceClientCommitId,
|
|
548
|
-
reaction.sourceCommitSeq,
|
|
549
|
-
reaction.createdAtMs,
|
|
550
|
-
reaction.createdAtMs,
|
|
551
|
-
reaction.maxAttempts,
|
|
552
|
-
]);
|
|
553
|
-
}
|
|
554
|
-
}
|
|
555
481
|
async commit() {
|
|
556
|
-
this
|
|
482
|
+
this.assertOpen();
|
|
557
483
|
if (this.#buffer.length === 0) {
|
|
558
|
-
this
|
|
484
|
+
this.open = false;
|
|
559
485
|
return;
|
|
560
486
|
}
|
|
561
487
|
const statements = this.#buffer.map((entry) => this.#db.prepare(entry.sql).bind(...entry.params));
|
|
562
488
|
// One atomic D1 batch — the §6.4 all-or-nothing commit.
|
|
563
489
|
try {
|
|
564
490
|
await this.#db.batch(statements);
|
|
565
|
-
this
|
|
491
|
+
this.open = false;
|
|
566
492
|
}
|
|
567
493
|
catch (error) {
|
|
568
494
|
if (isD1ConstraintError(error)) {
|
|
@@ -577,18 +503,13 @@ class D1Transaction {
|
|
|
577
503
|
}
|
|
578
504
|
}
|
|
579
505
|
async rollback() {
|
|
580
|
-
if (!this
|
|
506
|
+
if (!this.open)
|
|
581
507
|
return;
|
|
582
|
-
this
|
|
508
|
+
this.open = false;
|
|
583
509
|
this.#buffer.length = 0;
|
|
584
510
|
}
|
|
585
511
|
}
|
|
586
512
|
_a = D1Transaction;
|
|
587
|
-
/**
|
|
588
|
-
* D1 caps bound parameters at 100 per statement; the relational upsert
|
|
589
|
-
* binds one per app column plus the five `_sync_*` meta columns.
|
|
590
|
-
*/
|
|
591
|
-
const D1_MAX_BIND_PARAMS = 100;
|
|
592
513
|
/**
|
|
593
514
|
* The core tables `D1ServerStorage` reads or writes outside schema migration.
|
|
594
515
|
* `migrateSchema` creates them in its core phase, which runs only when the
|
|
@@ -604,54 +525,97 @@ const D1_CORE_TABLES = [
|
|
|
604
525
|
'sync_changes',
|
|
605
526
|
'sync_change_scopes',
|
|
606
527
|
'sync_push_results',
|
|
528
|
+
'sync_push_watermarks',
|
|
607
529
|
'sync_reactions',
|
|
608
530
|
'sync_clients',
|
|
609
531
|
'sync_blob_refs',
|
|
610
532
|
'sync_tombstones',
|
|
611
533
|
];
|
|
612
|
-
export class D1ServerStorage {
|
|
534
|
+
export class D1ServerStorage extends SqliteFamilyStorage {
|
|
613
535
|
#db;
|
|
614
536
|
#pushApplySerialized;
|
|
615
|
-
/**
|
|
616
|
-
|
|
617
|
-
|
|
537
|
+
/**
|
|
538
|
+
* D1 caps bound parameters at 100 per statement; the relational upsert
|
|
539
|
+
* binds one per app column plus the five `_sync_*` meta columns.
|
|
540
|
+
*/
|
|
541
|
+
maxBindParams = 100;
|
|
618
542
|
/**
|
|
619
543
|
* Page reads waiting for the end of the current microtask turn. A pull
|
|
620
|
-
* starts the
|
|
544
|
+
* starts the first snapshot page and scope pin of every bootstrapping unit
|
|
621
545
|
* before awaiting any of them, so they leave as ONE `db.batch` round trip
|
|
622
|
-
* instead of one per
|
|
546
|
+
* instead of one per unit (SYNCULAR-PULL-ROUNDTRIPS-001).
|
|
623
547
|
*/
|
|
624
548
|
#pendingReads;
|
|
625
549
|
constructor(db, options = {}) {
|
|
550
|
+
super();
|
|
626
551
|
this.#db = db;
|
|
627
552
|
this.#pushApplySerialized =
|
|
628
553
|
options.pushApplySerialized === true ||
|
|
629
554
|
options.commitValidationSerialized === true;
|
|
630
555
|
}
|
|
556
|
+
async all(sql, params) {
|
|
557
|
+
return (await this.#db
|
|
558
|
+
.prepare(sql)
|
|
559
|
+
.bind(...params)
|
|
560
|
+
.all()).results;
|
|
561
|
+
}
|
|
562
|
+
/** Queue one read into the current turn's `db.batch` (see `#pendingReads`). */
|
|
563
|
+
guardedAll(sql, params) {
|
|
564
|
+
let pending = this.#pendingReads;
|
|
565
|
+
if (pending === undefined) {
|
|
566
|
+
const created = {
|
|
567
|
+
db: this.#schemaDatabase(),
|
|
568
|
+
entries: [],
|
|
569
|
+
};
|
|
570
|
+
pending = created;
|
|
571
|
+
this.#pendingReads = created;
|
|
572
|
+
queueMicrotask(() => {
|
|
573
|
+
this.#pendingReads = undefined;
|
|
574
|
+
created.db.batch(created.entries.map((entry) => entry.statement)).then((results) => {
|
|
575
|
+
created.entries.forEach((entry, i) => {
|
|
576
|
+
try {
|
|
577
|
+
entry.resolve(batchRows(results[i]));
|
|
578
|
+
}
|
|
579
|
+
catch (error) {
|
|
580
|
+
entry.reject(error);
|
|
581
|
+
}
|
|
582
|
+
});
|
|
583
|
+
}, (error) => {
|
|
584
|
+
for (const entry of created.entries)
|
|
585
|
+
entry.reject(error);
|
|
586
|
+
});
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
const target = pending;
|
|
590
|
+
const statement = target.db.prepare(sql).bind(...params);
|
|
591
|
+
return new Promise((resolve, reject) => {
|
|
592
|
+
target.entries.push({
|
|
593
|
+
statement,
|
|
594
|
+
resolve: (rows) => resolve(rows),
|
|
595
|
+
reject,
|
|
596
|
+
});
|
|
597
|
+
});
|
|
598
|
+
}
|
|
599
|
+
async atomic(statements) {
|
|
600
|
+
const results = await this.#db.batch(statements.map(({ sql, params }) => this.#db.prepare(sql).bind(...params)));
|
|
601
|
+
return results.map(batchRows);
|
|
602
|
+
}
|
|
631
603
|
/** Apply the schema DDL (idempotent). Call once before use. */
|
|
632
604
|
async migrate() {
|
|
633
605
|
for (const statement of sqliteDdlStatements()) {
|
|
634
606
|
await this.#db.exec(`${statement.replace(/\s+/g, ' ')};`);
|
|
635
607
|
}
|
|
636
|
-
const
|
|
637
|
-
.
|
|
638
|
-
.all();
|
|
639
|
-
if (!results.some((column) => column.name === 'wire_version')) {
|
|
640
|
-
await this.#db.exec('ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1;');
|
|
641
|
-
}
|
|
642
|
-
for (const [name, definition] of Object.entries(SYNC_CLIENT_ROUND_COLUMNS)) {
|
|
643
|
-
if (!results.some((column) => column.name === name)) {
|
|
644
|
-
await this.#db.exec(`ALTER TABLE sync_clients ADD COLUMN ${name} ${definition};`);
|
|
645
|
-
}
|
|
608
|
+
for (const sql of await this.#upgradeDdl()) {
|
|
609
|
+
await this.#db.exec(`${sql};`);
|
|
646
610
|
}
|
|
647
611
|
}
|
|
648
|
-
/**
|
|
649
|
-
|
|
650
|
-
const
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
return
|
|
612
|
+
/** `sqliteUpgradeDdl` for this database, read in one statement. */
|
|
613
|
+
async #upgradeDdl() {
|
|
614
|
+
const { results } = await this.#db
|
|
615
|
+
.prepare(SQLITE_UPGRADE_PROBE.sql)
|
|
616
|
+
.bind(...SQLITE_UPGRADE_PROBE.params)
|
|
617
|
+
.all();
|
|
618
|
+
return sqliteUpgradeDdl(results);
|
|
655
619
|
}
|
|
656
620
|
async ensureSchema(schema, checkpoints) {
|
|
657
621
|
if (checkpoints !== undefined && checkpoints.length > 0) {
|
|
@@ -659,7 +623,7 @@ export class D1ServerStorage {
|
|
|
659
623
|
// visibility here; refuse instead of accepting a weaker guarantee.
|
|
660
624
|
throw new StorageQueryError('sync.storage.checkpoint_unsupported');
|
|
661
625
|
}
|
|
662
|
-
if (this
|
|
626
|
+
if (this.schemaVersion === schema.version) {
|
|
663
627
|
await this.#schemaDatabase().batch([]);
|
|
664
628
|
return;
|
|
665
629
|
}
|
|
@@ -677,7 +641,7 @@ export class D1ServerStorage {
|
|
|
677
641
|
}
|
|
678
642
|
const tables = [...schema.tables.values()].sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
|
|
679
643
|
for (const table of tables) {
|
|
680
|
-
if (tableColumnNames(table).length >
|
|
644
|
+
if (tableColumnNames(table).length > this.maxBindParams) {
|
|
681
645
|
throw new Error('D1 caps statements at 100 bound parameters');
|
|
682
646
|
}
|
|
683
647
|
}
|
|
@@ -740,14 +704,13 @@ export class D1ServerStorage {
|
|
|
740
704
|
primaryKeyPosition: column.pk,
|
|
741
705
|
})), 'sqlite');
|
|
742
706
|
}
|
|
743
|
-
// A database from
|
|
707
|
+
// A database from an earlier release gains the core-table columns and
|
|
708
|
+
// indexes it lacks in place, and loses the columns wire version 4
|
|
709
|
+
// deleted.
|
|
744
710
|
statementsExecuted++;
|
|
745
|
-
const
|
|
746
|
-
|
|
747
|
-
.
|
|
748
|
-
const missingRoundColumns = Object.entries(SYNC_CLIENT_ROUND_COLUMNS).filter(([name]) => !clientColumns.results.some((column) => column.name === name));
|
|
749
|
-
if (missingRoundColumns.length > 0) {
|
|
750
|
-
await batch(missingRoundColumns.map(([name, definition]) => this.#db.prepare(`ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`)));
|
|
711
|
+
const added = await this.#upgradeDdl();
|
|
712
|
+
if (added.length > 0) {
|
|
713
|
+
await batch(added.map((sql) => this.#db.prepare(sql)));
|
|
751
714
|
}
|
|
752
715
|
// Server-only indexes reconcile at every open of a database already at
|
|
753
716
|
// this version: they never reach a client, so no version bump is needed.
|
|
@@ -764,8 +727,8 @@ export class D1ServerStorage {
|
|
|
764
727
|
if (reconcile.length > 0) {
|
|
765
728
|
await batch(reconcile.map((sql) => this.#db.prepare(sql)));
|
|
766
729
|
}
|
|
767
|
-
this
|
|
768
|
-
this
|
|
730
|
+
this.tables = schema.tables;
|
|
731
|
+
this.schemaVersion = schema.version;
|
|
769
732
|
return { complete: true, statementsExecuted };
|
|
770
733
|
}
|
|
771
734
|
if (marker !== null && marker.schema_version > schema.version) {
|
|
@@ -861,17 +824,14 @@ export class D1ServerStorage {
|
|
|
861
824
|
break;
|
|
862
825
|
continue;
|
|
863
826
|
}
|
|
827
|
+
// The column read, the checkpoint pair, and every missing column.
|
|
864
828
|
if (remaining < 4)
|
|
865
829
|
break;
|
|
866
830
|
statementsExecuted++;
|
|
867
|
-
const
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
wire_version: 'INTEGER NOT NULL DEFAULT 1',
|
|
872
|
-
...SYNC_CLIENT_ROUND_COLUMNS,
|
|
873
|
-
}).filter(([name]) => !columns.results.some((column) => column.name === name));
|
|
874
|
-
if (!(await save({ ...state, phase: 'prepare', offset: 0 }, missingColumns.map(([name, definition]) => this.#db.prepare(`ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`)))))
|
|
831
|
+
const added = await this.#upgradeDdl();
|
|
832
|
+
if (remaining < 3 + added.length)
|
|
833
|
+
break;
|
|
834
|
+
if (!(await save({ ...state, phase: 'prepare', offset: 0 }, added.map((sql) => this.#db.prepare(sql)))))
|
|
875
835
|
break;
|
|
876
836
|
}
|
|
877
837
|
else if (state.phase === 'prepare') {
|
|
@@ -971,8 +931,8 @@ export class D1ServerStorage {
|
|
|
971
931
|
.bind(schema.version, layoutsOf(schema)),
|
|
972
932
|
], true)))
|
|
973
933
|
break;
|
|
974
|
-
this
|
|
975
|
-
this
|
|
934
|
+
this.tables = schema.tables;
|
|
935
|
+
this.schemaVersion = schema.version;
|
|
976
936
|
return { complete: true, statementsExecuted };
|
|
977
937
|
}
|
|
978
938
|
const table = schema.tables.get(plan.table);
|
|
@@ -1010,52 +970,11 @@ export class D1ServerStorage {
|
|
|
1010
970
|
return { complete: false, statementsExecuted };
|
|
1011
971
|
}
|
|
1012
972
|
/** Every protected read or write shares a transaction with its schema check. */
|
|
1013
|
-
/** Queue one read into the current turn's `db.batch` (see `#pendingReads`). */
|
|
1014
|
-
#batchedAll(sql, params) {
|
|
1015
|
-
let pending = this.#pendingReads;
|
|
1016
|
-
if (pending === undefined) {
|
|
1017
|
-
const created = {
|
|
1018
|
-
db: this.#schemaDatabase(),
|
|
1019
|
-
entries: [],
|
|
1020
|
-
};
|
|
1021
|
-
pending = created;
|
|
1022
|
-
this.#pendingReads = created;
|
|
1023
|
-
queueMicrotask(() => {
|
|
1024
|
-
this.#pendingReads = undefined;
|
|
1025
|
-
created.db.batch(created.entries.map((entry) => entry.statement)).then((results) => {
|
|
1026
|
-
created.entries.forEach((entry, i) => {
|
|
1027
|
-
const result = results[i];
|
|
1028
|
-
if (typeof result !== 'object' ||
|
|
1029
|
-
result === null ||
|
|
1030
|
-
!('results' in result) ||
|
|
1031
|
-
!Array.isArray(result.results)) {
|
|
1032
|
-
entry.reject(new Error('D1 query returned an invalid batch result'));
|
|
1033
|
-
}
|
|
1034
|
-
else {
|
|
1035
|
-
entry.resolve(result.results);
|
|
1036
|
-
}
|
|
1037
|
-
});
|
|
1038
|
-
}, (error) => {
|
|
1039
|
-
for (const entry of created.entries)
|
|
1040
|
-
entry.reject(error);
|
|
1041
|
-
});
|
|
1042
|
-
});
|
|
1043
|
-
}
|
|
1044
|
-
const target = pending;
|
|
1045
|
-
const statement = target.db.prepare(sql).bind(...params);
|
|
1046
|
-
return new Promise((resolve, reject) => {
|
|
1047
|
-
target.entries.push({
|
|
1048
|
-
statement,
|
|
1049
|
-
resolve: (rows) => resolve(rows),
|
|
1050
|
-
reject,
|
|
1051
|
-
});
|
|
1052
|
-
});
|
|
1053
|
-
}
|
|
1054
973
|
#schemaDatabase() {
|
|
1055
|
-
if (this
|
|
974
|
+
if (this.schemaVersion === undefined)
|
|
1056
975
|
throw new StorageQueryError('sync.storage.schema_changed');
|
|
1057
976
|
const db = this.#db;
|
|
1058
|
-
const version = this
|
|
977
|
+
const version = this.schemaVersion;
|
|
1059
978
|
const sources = new WeakMap();
|
|
1060
979
|
const execute = async (statements) => {
|
|
1061
980
|
try {
|
|
@@ -1086,14 +1005,9 @@ export class D1ServerStorage {
|
|
|
1086
1005
|
const prepare = (sql, params = []) => {
|
|
1087
1006
|
const source = db.prepare(sql).bind(...params);
|
|
1088
1007
|
const all = async () => {
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
!('results' in result) ||
|
|
1093
|
-
!Array.isArray(result.results)) {
|
|
1094
|
-
throw new Error('D1 query returned an invalid batch result');
|
|
1095
|
-
}
|
|
1096
|
-
return { results: result.results };
|
|
1008
|
+
return {
|
|
1009
|
+
results: batchRows((await execute([source]))[0]),
|
|
1010
|
+
};
|
|
1097
1011
|
};
|
|
1098
1012
|
const statement = {
|
|
1099
1013
|
bind: (...values) => prepare(sql, values),
|
|
@@ -1114,180 +1028,27 @@ export class D1ServerStorage {
|
|
|
1114
1028
|
})),
|
|
1115
1029
|
};
|
|
1116
1030
|
}
|
|
1117
|
-
async touchPartition(partition, authenticatedAtMs, initialLogEpoch) {
|
|
1118
|
-
if (initialLogEpoch.length === 0) {
|
|
1119
|
-
throw new Error('initial log epoch must be non-empty');
|
|
1120
|
-
}
|
|
1121
|
-
const select = this.#db
|
|
1122
|
-
.prepare(`SELECT log_epoch, epoch_required, last_authenticated_at_ms
|
|
1123
|
-
FROM sync_partition_registry WHERE partition=?`)
|
|
1124
|
-
.bind(partition);
|
|
1125
|
-
let row = await select.first();
|
|
1126
|
-
if (row === null ||
|
|
1127
|
-
authenticatedAtMs - row.last_authenticated_at_ms >=
|
|
1128
|
-
PARTITION_TOUCH_INTERVAL_MS) {
|
|
1129
|
-
await this.#db
|
|
1130
|
-
.prepare(`INSERT INTO sync_partition_registry(
|
|
1131
|
-
partition, log_epoch, last_authenticated_at_ms
|
|
1132
|
-
) VALUES (?,?,?)
|
|
1133
|
-
ON CONFLICT(partition) DO UPDATE SET
|
|
1134
|
-
last_authenticated_at_ms=excluded.last_authenticated_at_ms`)
|
|
1135
|
-
.bind(partition, initialLogEpoch, authenticatedAtMs)
|
|
1136
|
-
.run();
|
|
1137
|
-
row = await select.first();
|
|
1138
|
-
}
|
|
1139
|
-
if (row === null)
|
|
1140
|
-
throw new Error('partition registry write did not persist');
|
|
1141
|
-
return {
|
|
1142
|
-
partition,
|
|
1143
|
-
logEpoch: row.log_epoch,
|
|
1144
|
-
epochRequired: row.epoch_required === 1,
|
|
1145
|
-
lastAuthenticatedAtMs: row.last_authenticated_at_ms,
|
|
1146
|
-
};
|
|
1147
|
-
}
|
|
1148
|
-
async rotatePartitionLogEpoch(partition, logEpoch, authenticatedAtMs) {
|
|
1149
|
-
if (logEpoch.length === 0)
|
|
1150
|
-
throw new Error('log epoch must be non-empty');
|
|
1151
|
-
await this.#db.batch([
|
|
1152
|
-
this.#db
|
|
1153
|
-
.prepare(`INSERT INTO sync_partition_registry(
|
|
1154
|
-
partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
1155
|
-
) VALUES (?,?,1,?)
|
|
1156
|
-
ON CONFLICT(partition) DO UPDATE SET
|
|
1157
|
-
log_epoch=excluded.log_epoch,
|
|
1158
|
-
epoch_required=1,
|
|
1159
|
-
last_authenticated_at_ms=excluded.last_authenticated_at_ms`)
|
|
1160
|
-
.bind(partition, logEpoch, authenticatedAtMs),
|
|
1161
|
-
this.#db
|
|
1162
|
-
.prepare('DELETE FROM sync_clients WHERE partition=?')
|
|
1163
|
-
.bind(partition),
|
|
1164
|
-
]);
|
|
1165
|
-
return {
|
|
1166
|
-
partition,
|
|
1167
|
-
logEpoch,
|
|
1168
|
-
epochRequired: true,
|
|
1169
|
-
lastAuthenticatedAtMs: authenticatedAtMs,
|
|
1170
|
-
};
|
|
1171
|
-
}
|
|
1172
|
-
async listPartitionRegistry() {
|
|
1173
|
-
const { results } = await this.#db
|
|
1174
|
-
.prepare(`SELECT partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
1175
|
-
FROM sync_partition_registry ORDER BY partition`)
|
|
1176
|
-
.all();
|
|
1177
|
-
return results.map((row) => ({
|
|
1178
|
-
partition: row.partition,
|
|
1179
|
-
logEpoch: row.log_epoch,
|
|
1180
|
-
epochRequired: row.epoch_required === 1,
|
|
1181
|
-
lastAuthenticatedAtMs: row.last_authenticated_at_ms,
|
|
1182
|
-
}));
|
|
1183
|
-
}
|
|
1184
1031
|
async begin(partition) {
|
|
1185
|
-
const tables = this
|
|
1032
|
+
const tables = this.tables;
|
|
1186
1033
|
const db = this.#schemaDatabase();
|
|
1187
1034
|
return new D1Transaction(db, partition, (name) => {
|
|
1188
1035
|
const table = tables?.get(name);
|
|
1189
1036
|
if (table === undefined)
|
|
1190
1037
|
throw new StorageQueryError('sync.storage.schema_changed');
|
|
1191
1038
|
return table;
|
|
1192
|
-
}, (query) => this
|
|
1193
|
-
}
|
|
1194
|
-
async getMaxCommitSeq(partition) {
|
|
1195
|
-
const row = await this.#db
|
|
1196
|
-
.prepare('SELECT max_commit_seq FROM sync_partitions WHERE partition=?')
|
|
1197
|
-
.bind(partition)
|
|
1198
|
-
.first();
|
|
1199
|
-
return row?.max_commit_seq ?? 0;
|
|
1039
|
+
}, (query) => this.prepareAuthoritativeQuery(partition, query), this.#pushApplySerialized);
|
|
1200
1040
|
}
|
|
1201
1041
|
async queryAuthoritative(partition, query, _checkpoints) {
|
|
1202
1042
|
// D1 declares no checkpoints. `#schemaDatabase` prepends a guard that
|
|
1203
1043
|
// re-checks the published marker inside this same batch, so condition 2
|
|
1204
1044
|
// is already evaluated with the query (§2.4 D1 schema readiness).
|
|
1205
1045
|
const db = this.#schemaDatabase();
|
|
1206
|
-
return queryAuthoritativeOn(db, partition, this
|
|
1207
|
-
}
|
|
1208
|
-
#prepareAuthoritativeQuery(partition, query) {
|
|
1209
|
-
if (this.#tables === undefined) {
|
|
1210
|
-
throw new Error('ensureSchema(schema) must run before registered queries');
|
|
1211
|
-
}
|
|
1212
|
-
return bindAuthoritativePartition(prepareAuthoritativeQuery(query.plan, query.params, query.tables, this.#tables), partition);
|
|
1213
|
-
}
|
|
1214
|
-
async getPartitionLogEpoch(partition) {
|
|
1215
|
-
const row = await this.#db
|
|
1216
|
-
.prepare('SELECT log_epoch FROM sync_partition_registry WHERE partition=?')
|
|
1217
|
-
.bind(partition)
|
|
1218
|
-
.first();
|
|
1219
|
-
return row?.log_epoch;
|
|
1220
|
-
}
|
|
1221
|
-
async getHorizonSeq(partition) {
|
|
1222
|
-
const row = await this.#db
|
|
1223
|
-
.prepare('SELECT horizon_seq FROM sync_partitions WHERE partition=?')
|
|
1224
|
-
.bind(partition)
|
|
1225
|
-
.first();
|
|
1226
|
-
return row?.horizon_seq ?? 0;
|
|
1227
|
-
}
|
|
1228
|
-
async setHorizonSeq(partition, seq) {
|
|
1229
|
-
await this.#db
|
|
1230
|
-
.prepare('INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?) ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)')
|
|
1231
|
-
.bind(partition, seq)
|
|
1232
|
-
.run();
|
|
1046
|
+
return queryAuthoritativeOn(db, partition, this.prepareAuthoritativeQuery(partition, query));
|
|
1233
1047
|
}
|
|
1234
1048
|
async pruneCommitsThrough(partition, query) {
|
|
1235
|
-
validateCommitPruneQuery(query);
|
|
1236
1049
|
if (!this.#pushApplySerialized)
|
|
1237
1050
|
throw new Error('D1 pruning requires externally serialized partition writes');
|
|
1238
|
-
|
|
1239
|
-
const horizon = '(SELECT horizon_seq FROM sync_partitions WHERE partition=?)';
|
|
1240
|
-
const results = await this.#db.batch([
|
|
1241
|
-
this.#db
|
|
1242
|
-
.prepare(`SELECT log_epoch, coalesce(${horizon},0) AS previous_horizon_seq FROM sync_partition_registry WHERE partition=?`)
|
|
1243
|
-
.bind(partition, partition),
|
|
1244
|
-
this.#db
|
|
1245
|
-
.prepare(`INSERT INTO sync_partitions(partition,horizon_seq) SELECT ?,? WHERE ${epochGuard}
|
|
1246
|
-
ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`)
|
|
1247
|
-
.bind(partition, query.throughSeq, partition, query.logEpoch),
|
|
1248
|
-
this.#db
|
|
1249
|
-
.prepare(`SELECT horizon_seq, (SELECT count(*) FROM sync_commits WHERE partition=? AND commit_seq<=${horizon}) AS removed_commits FROM sync_partitions WHERE partition=?`)
|
|
1250
|
-
.bind(partition, partition, partition),
|
|
1251
|
-
...[
|
|
1252
|
-
'sync_commits',
|
|
1253
|
-
'sync_changes',
|
|
1254
|
-
'sync_change_scopes',
|
|
1255
|
-
'sync_tombstones',
|
|
1256
|
-
].map((table) => this.#db
|
|
1257
|
-
.prepare(`DELETE FROM ${table} WHERE partition=? AND commit_seq<=${horizon} AND ${epochGuard}`)
|
|
1258
|
-
.bind(partition, partition, partition, query.logEpoch)),
|
|
1259
|
-
]);
|
|
1260
|
-
const [before, after] = [results[0], results[2]].map((result) => {
|
|
1261
|
-
if (typeof result !== 'object' ||
|
|
1262
|
-
result === null ||
|
|
1263
|
-
!('results' in result) ||
|
|
1264
|
-
!Array.isArray(result.results)) {
|
|
1265
|
-
throw new Error('D1 pruning returned an invalid batch result');
|
|
1266
|
-
}
|
|
1267
|
-
const row = result.results[0];
|
|
1268
|
-
return typeof row === 'object' && row !== null
|
|
1269
|
-
? row
|
|
1270
|
-
: undefined;
|
|
1271
|
-
});
|
|
1272
|
-
if (before?.log_epoch !== query.logEpoch)
|
|
1273
|
-
throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
|
|
1274
|
-
if (typeof before.previous_horizon_seq !== 'number' ||
|
|
1275
|
-
typeof after?.horizon_seq !== 'number' ||
|
|
1276
|
-
typeof after.removed_commits !== 'number') {
|
|
1277
|
-
throw new Error('D1 pruning returned invalid horizon metadata');
|
|
1278
|
-
}
|
|
1279
|
-
return {
|
|
1280
|
-
previousHorizonSeq: before.previous_horizon_seq,
|
|
1281
|
-
horizonSeq: after.horizon_seq,
|
|
1282
|
-
removedCommits: after.removed_commits,
|
|
1283
|
-
};
|
|
1284
|
-
}
|
|
1285
|
-
async getCommitSeqBefore(partition, createdBeforeMs) {
|
|
1286
|
-
const row = await this.#db
|
|
1287
|
-
.prepare('SELECT max(commit_seq) AS seq FROM sync_commits WHERE partition=? AND created_at_ms<?')
|
|
1288
|
-
.bind(partition, createdBeforeMs)
|
|
1289
|
-
.first();
|
|
1290
|
-
return row?.seq ?? 0;
|
|
1051
|
+
return super.pruneCommitsThrough(partition, query);
|
|
1291
1052
|
}
|
|
1292
1053
|
async readCheckpoints(_partition) {
|
|
1293
1054
|
// D1 supports no declared backfill checkpoints in this release: migrations
|
|
@@ -1319,640 +1080,9 @@ export class D1ServerStorage {
|
|
|
1319
1080
|
async advanceCheckpoint(_partition, _name, _ownerEpoch, _watermark, _observedRows, _nowMs) {
|
|
1320
1081
|
throw new StorageQueryError('sync.storage.checkpoint_unsupported');
|
|
1321
1082
|
}
|
|
1322
|
-
async sourceCoverageSeq(partition, tables) {
|
|
1323
|
-
if (tables.length === 0)
|
|
1324
|
-
return 0;
|
|
1325
|
-
const tableParams = tables.map(() => '?').join(',');
|
|
1326
|
-
const row = await this.#db
|
|
1327
|
-
.prepare(`SELECT max(commit_seq) AS seq FROM sync_changes
|
|
1328
|
-
WHERE partition=? AND tbl IN (${tableParams})`)
|
|
1329
|
-
.bind(partition, ...tables)
|
|
1330
|
-
.first();
|
|
1331
|
-
return row?.seq ?? 0;
|
|
1332
|
-
}
|
|
1333
|
-
async hasSourceChangesAbove(partition, tables, seq) {
|
|
1334
|
-
// Read the horizon before the window scan: a horizon past `seq` means the
|
|
1335
|
-
// history that would answer the question is gone, so an empty scan proves
|
|
1336
|
-
// nothing and must not be reported as `clean`.
|
|
1337
|
-
const horizon = await this.getHorizonSeq(partition);
|
|
1338
|
-
if (tables.length === 0)
|
|
1339
|
-
return 'clean';
|
|
1340
|
-
const tableParams = tables.map(() => '?').join(',');
|
|
1341
|
-
const hit = await this.#db
|
|
1342
|
-
.prepare(`SELECT 1 AS hit FROM sync_changes
|
|
1343
|
-
WHERE partition=? AND tbl IN (${tableParams}) AND commit_seq>?
|
|
1344
|
-
LIMIT 1`)
|
|
1345
|
-
.bind(partition, ...tables, seq)
|
|
1346
|
-
.first();
|
|
1347
|
-
if (hit !== null)
|
|
1348
|
-
return 'changed';
|
|
1349
|
-
return horizon > seq ? 'unverifiable' : 'clean';
|
|
1350
|
-
}
|
|
1351
|
-
async getRow(partition, table, rowId) {
|
|
1352
|
-
const db = this.#schemaDatabase();
|
|
1353
|
-
const record = await db
|
|
1354
|
-
.prepare(selectRowSql(this.table(table), 'sqlite'))
|
|
1355
|
-
.bind(partition, rowId)
|
|
1356
|
-
.first();
|
|
1357
|
-
return record === null ? undefined : toStoredRow(record);
|
|
1358
|
-
}
|
|
1359
|
-
async getPushResult(partition, clientId, clientCommitId) {
|
|
1360
|
-
const db = this.#schemaDatabase();
|
|
1361
|
-
const record = await db
|
|
1362
|
-
.prepare('SELECT result FROM sync_push_results WHERE partition=? AND client_id=? AND client_commit_id=?')
|
|
1363
|
-
.bind(partition, clientId, clientCommitId)
|
|
1364
|
-
.first();
|
|
1365
|
-
if (record === null)
|
|
1366
|
-
return undefined;
|
|
1367
|
-
try {
|
|
1368
|
-
return deserializePushResult(record.result);
|
|
1369
|
-
}
|
|
1370
|
-
catch {
|
|
1371
|
-
throw syncError('sync.idempotency_cache_miss', 'persisted push result unreadable (§6.3)');
|
|
1372
|
-
}
|
|
1373
|
-
}
|
|
1374
|
-
async claimReactions(partition, query) {
|
|
1375
|
-
if (query.types.length === 0 || query.limit <= 0)
|
|
1376
|
-
return [];
|
|
1377
|
-
if (query.types.length + 7 > D1_MAX_BIND_PARAMS) {
|
|
1378
|
-
throw new Error('D1 reaction claim exceeds the bound-parameter limit');
|
|
1379
|
-
}
|
|
1380
|
-
const typeParams = query.types.map(() => '?').join(',');
|
|
1381
|
-
const { results } = await this.#db
|
|
1382
|
-
.prepare(`UPDATE sync_reactions
|
|
1383
|
-
SET status='leased', attempts=attempts+1,
|
|
1384
|
-
lease_owner=?, lease_expires_at_ms=?, completed_at_ms=NULL
|
|
1385
|
-
WHERE (partition, idempotency_key) IN (
|
|
1386
|
-
SELECT partition, idempotency_key
|
|
1387
|
-
FROM sync_reactions
|
|
1388
|
-
WHERE partition=? AND type IN (${typeParams})
|
|
1389
|
-
AND ((status='pending' AND available_at_ms<=?)
|
|
1390
|
-
OR (status='leased' AND lease_expires_at_ms<=?))
|
|
1391
|
-
ORDER BY CASE WHEN status='leased' THEN lease_expires_at_ms
|
|
1392
|
-
ELSE available_at_ms END,
|
|
1393
|
-
created_at_ms, idempotency_key
|
|
1394
|
-
LIMIT ?
|
|
1395
|
-
)
|
|
1396
|
-
RETURNING *`)
|
|
1397
|
-
.bind(query.leaseOwner, Math.min(Number.MAX_SAFE_INTEGER, query.nowMs + query.leaseDurationMs), partition, ...query.types, query.nowMs, query.nowMs, query.limit)
|
|
1398
|
-
.all();
|
|
1399
|
-
return results
|
|
1400
|
-
.map(toStoredReaction)
|
|
1401
|
-
.sort((a, b) => a.createdAtMs - b.createdAtMs ||
|
|
1402
|
-
a.idempotencyKey.localeCompare(b.idempotencyKey));
|
|
1403
|
-
}
|
|
1404
|
-
async completeReaction(partition, idempotencyKey, leaseOwner, completedAtMs) {
|
|
1405
|
-
const record = await this.#db
|
|
1406
|
-
.prepare(`UPDATE sync_reactions
|
|
1407
|
-
SET status='completed', completed_at_ms=?,
|
|
1408
|
-
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
1409
|
-
WHERE partition=? AND idempotency_key=?
|
|
1410
|
-
AND status='leased' AND lease_owner=?
|
|
1411
|
-
RETURNING idempotency_key`)
|
|
1412
|
-
.bind(completedAtMs, partition, idempotencyKey, leaseOwner)
|
|
1413
|
-
.first();
|
|
1414
|
-
return record !== null;
|
|
1415
|
-
}
|
|
1416
|
-
async extendReactionLease(partition, idempotencyKey, leaseOwner, leaseExpiresAtMs) {
|
|
1417
|
-
const record = await this.#db
|
|
1418
|
-
.prepare(`UPDATE sync_reactions SET lease_expires_at_ms=?
|
|
1419
|
-
WHERE partition=? AND idempotency_key=?
|
|
1420
|
-
AND status='leased' AND lease_owner=?
|
|
1421
|
-
RETURNING idempotency_key`)
|
|
1422
|
-
.bind(leaseExpiresAtMs, partition, idempotencyKey, leaseOwner)
|
|
1423
|
-
.first();
|
|
1424
|
-
return record !== null;
|
|
1425
|
-
}
|
|
1426
|
-
async failReaction(partition, idempotencyKey, update) {
|
|
1427
|
-
const record = await this.#db
|
|
1428
|
-
.prepare(`UPDATE sync_reactions
|
|
1429
|
-
SET status=?, available_at_ms=?, last_failure=?,
|
|
1430
|
-
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
1431
|
-
WHERE partition=? AND idempotency_key=?
|
|
1432
|
-
AND status='leased' AND lease_owner=?
|
|
1433
|
-
RETURNING idempotency_key`)
|
|
1434
|
-
.bind(update.retryAtMs === undefined ? 'dead-letter' : 'pending', update.retryAtMs ?? update.failure.atMs, JSON.stringify(update.failure), partition, idempotencyKey, update.leaseOwner)
|
|
1435
|
-
.first();
|
|
1436
|
-
return record !== null;
|
|
1437
|
-
}
|
|
1438
|
-
async retryReaction(partition, idempotencyKey, nowMs) {
|
|
1439
|
-
const record = await this.#db
|
|
1440
|
-
.prepare(`UPDATE sync_reactions
|
|
1441
|
-
SET status='pending', attempts=0, available_at_ms=?,
|
|
1442
|
-
last_failure=NULL, lease_owner=NULL, lease_expires_at_ms=NULL,
|
|
1443
|
-
completed_at_ms=NULL
|
|
1444
|
-
WHERE partition=? AND idempotency_key=? AND status='dead-letter'
|
|
1445
|
-
RETURNING idempotency_key`)
|
|
1446
|
-
.bind(nowMs, partition, idempotencyKey)
|
|
1447
|
-
.first();
|
|
1448
|
-
return record !== null;
|
|
1449
|
-
}
|
|
1450
|
-
async getReaction(partition, idempotencyKey) {
|
|
1451
|
-
const record = await this.#db
|
|
1452
|
-
.prepare('SELECT * FROM sync_reactions WHERE partition=? AND idempotency_key=?')
|
|
1453
|
-
.bind(partition, idempotencyKey)
|
|
1454
|
-
.first();
|
|
1455
|
-
return record === null ? undefined : toStoredReaction(record);
|
|
1456
|
-
}
|
|
1457
|
-
async listReactions(partition, query) {
|
|
1458
|
-
const where = ['partition=?'];
|
|
1459
|
-
const params = [partition];
|
|
1460
|
-
if (query.statuses !== undefined && query.statuses.length > 0) {
|
|
1461
|
-
where.push(`status IN (${query.statuses.map(() => '?').join(',')})`);
|
|
1462
|
-
params.push(...query.statuses);
|
|
1463
|
-
}
|
|
1464
|
-
if (query.types !== undefined && query.types.length > 0) {
|
|
1465
|
-
where.push(`type IN (${query.types.map(() => '?').join(',')})`);
|
|
1466
|
-
params.push(...query.types);
|
|
1467
|
-
}
|
|
1468
|
-
if (params.length + 1 > D1_MAX_BIND_PARAMS) {
|
|
1469
|
-
throw new Error('D1 reaction list exceeds the bound-parameter limit');
|
|
1470
|
-
}
|
|
1471
|
-
params.push(query.limit);
|
|
1472
|
-
const { results } = await this.#db
|
|
1473
|
-
.prepare(`SELECT * FROM sync_reactions WHERE ${where.join(' AND ')}
|
|
1474
|
-
ORDER BY created_at_ms DESC, idempotency_key DESC LIMIT ?`)
|
|
1475
|
-
.bind(...params)
|
|
1476
|
-
.all();
|
|
1477
|
-
return results.map(toStoredReaction);
|
|
1478
|
-
}
|
|
1479
|
-
async pruneReactions(partition, query) {
|
|
1480
|
-
if (query.limit <= 0)
|
|
1481
|
-
return { completed: 0, deadLetter: 0 };
|
|
1482
|
-
const { results } = await this.#db
|
|
1483
|
-
.prepare(`DELETE FROM sync_reactions
|
|
1484
|
-
WHERE partition=? AND idempotency_key IN (
|
|
1485
|
-
SELECT idempotency_key FROM sync_reactions
|
|
1486
|
-
WHERE partition=?
|
|
1487
|
-
AND ((status='completed' AND completed_at_ms IS NOT NULL
|
|
1488
|
-
AND completed_at_ms<?)
|
|
1489
|
-
OR (status='dead-letter' AND available_at_ms<?))
|
|
1490
|
-
ORDER BY CASE WHEN status='completed' THEN completed_at_ms
|
|
1491
|
-
ELSE available_at_ms END,
|
|
1492
|
-
idempotency_key
|
|
1493
|
-
LIMIT ?
|
|
1494
|
-
)
|
|
1495
|
-
AND ((status='completed' AND completed_at_ms IS NOT NULL
|
|
1496
|
-
AND completed_at_ms<?)
|
|
1497
|
-
OR (status='dead-letter' AND available_at_ms<?))
|
|
1498
|
-
RETURNING status`)
|
|
1499
|
-
.bind(partition, partition, query.completedBeforeMs, query.deadLetterBeforeMs, query.limit, query.completedBeforeMs, query.deadLetterBeforeMs)
|
|
1500
|
-
.all();
|
|
1501
|
-
return {
|
|
1502
|
-
completed: results.filter((row) => row.status === 'completed').length,
|
|
1503
|
-
deadLetter: results.filter((row) => row.status === 'dead-letter').length,
|
|
1504
|
-
};
|
|
1505
|
-
}
|
|
1506
|
-
async latestScopedChangeSeq(partition, query) {
|
|
1507
|
-
const firstVariable = Object.keys(query.scopeFilter).sort()[0];
|
|
1508
|
-
const values = firstVariable === undefined
|
|
1509
|
-
? []
|
|
1510
|
-
: (query.scopeFilter[firstVariable] ?? []);
|
|
1511
|
-
if (firstVariable === undefined || values.length === 0)
|
|
1512
|
-
return 0;
|
|
1513
|
-
// Concurrent reads of a pull share one `db.batch` round trip.
|
|
1514
|
-
const [row] = await this.#batchedAll(LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, [partition, query.table, firstVariable, JSON.stringify(values)]);
|
|
1515
|
-
return row?.seq ?? 0;
|
|
1516
|
-
}
|
|
1517
|
-
async readCommitWindow(partition, query) {
|
|
1518
|
-
const variables = Object.keys(query.scopeFilter).sort();
|
|
1519
|
-
const firstVariable = variables[0];
|
|
1520
|
-
if (firstVariable === undefined)
|
|
1521
|
-
return [];
|
|
1522
|
-
const firstValues = query.scopeFilter[firstVariable] ?? [];
|
|
1523
|
-
if (firstValues.length === 0)
|
|
1524
|
-
return [];
|
|
1525
|
-
// Candidates via the inverted index (one variable) LEFT JOINed to the
|
|
1526
|
-
// commit meta + the table's changes — one statement per page, never two
|
|
1527
|
-
// per candidate (see `commitWindowPageSql`), and concurrent pages share
|
|
1528
|
-
// one D1 round trip. Exact multi-variable verification against the
|
|
1529
|
-
// stored scope map in `collectCommitWindowPage`.
|
|
1530
|
-
const sql = commitWindowPageSql(firstValues.length, 'sqlite');
|
|
1531
|
-
const commits = [];
|
|
1532
|
-
let deliveredChanges = 0;
|
|
1533
|
-
let afterSeq = query.afterSeq;
|
|
1534
|
-
const batchSize = Math.max(64, query.limitChanges);
|
|
1535
|
-
while (deliveredChanges < query.limitChanges) {
|
|
1536
|
-
const records = await this.#batchedAll(sql, [
|
|
1537
|
-
partition,
|
|
1538
|
-
query.table,
|
|
1539
|
-
firstVariable,
|
|
1540
|
-
...firstValues,
|
|
1541
|
-
afterSeq,
|
|
1542
|
-
query.throughSeq,
|
|
1543
|
-
batchSize,
|
|
1544
|
-
partition,
|
|
1545
|
-
partition,
|
|
1546
|
-
query.table,
|
|
1547
|
-
]);
|
|
1548
|
-
if (records.length === 0)
|
|
1549
|
-
break;
|
|
1550
|
-
const page = collectCommitWindowPage(records, query.scopeFilter, query.limitChanges - deliveredChanges);
|
|
1551
|
-
commits.push(...page.commits);
|
|
1552
|
-
deliveredChanges += page.delivered;
|
|
1553
|
-
afterSeq = page.lastSeq;
|
|
1554
|
-
if (page.candidateCount < batchSize)
|
|
1555
|
-
break;
|
|
1556
|
-
}
|
|
1557
|
-
return commits;
|
|
1558
|
-
}
|
|
1559
|
-
async scanRows(partition, query) {
|
|
1560
|
-
const firstVariable = assertScopeIndexedScan(query);
|
|
1561
|
-
const firstValues = query.scopeFilter[firstVariable] ?? [];
|
|
1562
|
-
if (firstValues.length === 0)
|
|
1563
|
-
return [];
|
|
1564
|
-
// Candidates via the inverted index LEFT JOINed to the row table — one
|
|
1565
|
-
// statement per page, never one per row (see `scanRowPageSql`), and
|
|
1566
|
-
// concurrent pages share one D1 round trip.
|
|
1567
|
-
// Exact multi-variable verification against the stored scope map below.
|
|
1568
|
-
const sql = scanRowPageSql(this.table(query.table), firstValues.length, 'sqlite');
|
|
1569
|
-
const rows = [];
|
|
1570
|
-
let afterRowId = query.afterRowId ?? '';
|
|
1571
|
-
const batchSize = Math.max(64, query.limit);
|
|
1572
|
-
while (rows.length < query.limit) {
|
|
1573
|
-
const records = await this.#batchedAll(sql, [
|
|
1574
|
-
partition,
|
|
1575
|
-
query.table,
|
|
1576
|
-
firstVariable,
|
|
1577
|
-
...firstValues,
|
|
1578
|
-
afterRowId,
|
|
1579
|
-
batchSize,
|
|
1580
|
-
partition,
|
|
1581
|
-
]);
|
|
1582
|
-
if (records.length === 0)
|
|
1583
|
-
break;
|
|
1584
|
-
for (const record of records) {
|
|
1585
|
-
afterRowId = record.row_id;
|
|
1586
|
-
// NULL payload: an index candidate whose row vanished — it still
|
|
1587
|
-
// advances the keyset cursor (LEFT JOIN contract) but yields no row.
|
|
1588
|
-
if (record.payload === null)
|
|
1589
|
-
continue;
|
|
1590
|
-
const stored = toStoredRow(record);
|
|
1591
|
-
if (!matchesEffective(stored.scopes, query.scopeFilter))
|
|
1592
|
-
continue;
|
|
1593
|
-
rows.push(stored);
|
|
1594
|
-
if (rows.length >= query.limit)
|
|
1595
|
-
break;
|
|
1596
|
-
}
|
|
1597
|
-
if (records.length < batchSize)
|
|
1598
|
-
break;
|
|
1599
|
-
}
|
|
1600
|
-
return rows;
|
|
1601
|
-
}
|
|
1602
|
-
async scanRowsByIndex(partition, query) {
|
|
1603
|
-
const db = this.#schemaDatabase();
|
|
1604
|
-
const table = this.table(query.table);
|
|
1605
|
-
const index = resolveIndexRowScan(table, query);
|
|
1606
|
-
const statement = indexRowPageStatement(table, index, query.values, partition, query.afterRowId, query.limit, 'sqlite');
|
|
1607
|
-
const { results } = await db
|
|
1608
|
-
.prepare(statement.sql)
|
|
1609
|
-
.bind(...statement.params)
|
|
1610
|
-
.all();
|
|
1611
|
-
return results.map(toStoredRow);
|
|
1612
|
-
}
|
|
1613
|
-
async getClientRecord(partition, clientId) {
|
|
1614
|
-
const record = await this.#db
|
|
1615
|
-
.prepare('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? AND client_id=?')
|
|
1616
|
-
.bind(partition, clientId)
|
|
1617
|
-
.first();
|
|
1618
|
-
if (record === null)
|
|
1619
|
-
return undefined;
|
|
1620
|
-
return {
|
|
1621
|
-
clientId: record.client_id,
|
|
1622
|
-
actorId: record.actor_id,
|
|
1623
|
-
wireVersion: record.wire_version,
|
|
1624
|
-
cursor: record.cursor,
|
|
1625
|
-
updatedAtMs: record.updated_at_ms,
|
|
1626
|
-
subscriptions: JSON.parse(record.subscriptions),
|
|
1627
|
-
...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
|
|
1628
|
-
};
|
|
1629
|
-
}
|
|
1630
|
-
async putClientRecord(partition, record, options) {
|
|
1631
|
-
// Update the existing record in place, so the table keeps one row per
|
|
1632
|
-
// (partition, client_id). Every non-key field is written from the new
|
|
1633
|
-
// record; a delete-and-insert would drop and recreate the row instead.
|
|
1634
|
-
// `keepRoundOutcome` leaves the round-state columns alone (§1.5, §4.5).
|
|
1635
|
-
await this.#db
|
|
1636
|
-
.prepare(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms) VALUES (?,?,?,?,?,?,?,?)
|
|
1637
|
-
ON CONFLICT(partition, client_id) DO UPDATE SET
|
|
1638
|
-
actor_id=excluded.actor_id,
|
|
1639
|
-
wire_version=excluded.wire_version,
|
|
1640
|
-
cursor=excluded.cursor,
|
|
1641
|
-
subscriptions=excluded.subscriptions,
|
|
1642
|
-
updated_at_ms=excluded.updated_at_ms,
|
|
1643
|
-
last_round_at_ms=excluded.last_round_at_ms${options?.keepRoundOutcome === true
|
|
1644
|
-
? ''
|
|
1645
|
-
: `,
|
|
1646
|
-
last_error_code=NULL,
|
|
1647
|
-
consecutive_failures=0`}`)
|
|
1648
|
-
.bind(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs, record.updatedAtMs)
|
|
1649
|
-
.run();
|
|
1650
|
-
}
|
|
1651
|
-
async recordClientRoundFailure(partition, clientId, actorId, errorCode, atMs) {
|
|
1652
|
-
await this.#db
|
|
1653
|
-
.prepare(`UPDATE sync_clients
|
|
1654
|
-
SET last_round_at_ms=?, last_error_code=?,
|
|
1655
|
-
consecutive_failures=consecutive_failures+1
|
|
1656
|
-
WHERE partition=? AND client_id=? AND actor_id=?`)
|
|
1657
|
-
.bind(atMs, errorCode, partition, clientId, actorId)
|
|
1658
|
-
.run();
|
|
1659
|
-
}
|
|
1660
|
-
async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
|
|
1661
|
-
await this.#db
|
|
1662
|
-
.prepare(`UPDATE sync_clients
|
|
1663
|
-
SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
|
|
1664
|
-
WHERE partition=? AND client_id=? AND actor_id=?
|
|
1665
|
-
AND EXISTS (SELECT 1 FROM sync_partition_registry
|
|
1666
|
-
WHERE partition=sync_clients.partition AND log_epoch=?)`)
|
|
1667
|
-
.bind(cursor, updatedAtMs, partition, clientId, actorId, logEpoch)
|
|
1668
|
-
.run();
|
|
1669
|
-
}
|
|
1670
|
-
async updateClientCursor(partition, clientId, cursor, updatedAtMs) {
|
|
1671
|
-
await this.#db
|
|
1672
|
-
.prepare(`UPDATE sync_clients
|
|
1673
|
-
SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
|
|
1674
|
-
WHERE partition=? AND client_id=?`)
|
|
1675
|
-
.bind(cursor, updatedAtMs, partition, clientId)
|
|
1676
|
-
.run();
|
|
1677
|
-
}
|
|
1678
|
-
async getActiveClientCursorFloor(partition, cutoffMs) {
|
|
1679
|
-
const row = await this.#db
|
|
1680
|
-
.prepare('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=? AND updated_at_ms>=?')
|
|
1681
|
-
.bind(partition, cutoffMs)
|
|
1682
|
-
.first();
|
|
1683
|
-
return row.cursor;
|
|
1684
|
-
}
|
|
1685
|
-
async listClientCursors(partition) {
|
|
1686
|
-
const { results } = await this.#db
|
|
1687
|
-
.prepare('SELECT client_id, cursor, updated_at_ms FROM sync_clients WHERE partition=?')
|
|
1688
|
-
.bind(partition)
|
|
1689
|
-
.all();
|
|
1690
|
-
return results.map((r) => ({
|
|
1691
|
-
clientId: r.client_id,
|
|
1692
|
-
cursor: r.cursor,
|
|
1693
|
-
updatedAtMs: r.updated_at_ms,
|
|
1694
|
-
}));
|
|
1695
|
-
}
|
|
1696
|
-
async listRowsReferencingBlob(partition, blobId) {
|
|
1697
|
-
const db = this.#schemaDatabase();
|
|
1698
|
-
const { results: refs } = await db
|
|
1699
|
-
.prepare('SELECT tbl, row_id FROM sync_blob_refs WHERE partition=? AND blob_id=?')
|
|
1700
|
-
.bind(partition, blobId)
|
|
1701
|
-
.all();
|
|
1702
|
-
const out = [];
|
|
1703
|
-
for (const ref of refs) {
|
|
1704
|
-
const compiled = this.#tables?.get(ref.tbl);
|
|
1705
|
-
if (compiled === undefined)
|
|
1706
|
-
continue; // table no longer in the schema
|
|
1707
|
-
const row = await db
|
|
1708
|
-
.prepare(selectRowScopesSql(compiled, 'sqlite'))
|
|
1709
|
-
.bind(partition, ref.row_id)
|
|
1710
|
-
.first();
|
|
1711
|
-
if (row === null)
|
|
1712
|
-
continue;
|
|
1713
|
-
out.push({
|
|
1714
|
-
table: ref.tbl,
|
|
1715
|
-
rowId: ref.row_id,
|
|
1716
|
-
scopes: JSON.parse(row.scopes),
|
|
1717
|
-
});
|
|
1718
|
-
}
|
|
1719
|
-
return out;
|
|
1720
|
-
}
|
|
1721
|
-
async listReferencedBlobIds(partition) {
|
|
1722
|
-
const { results } = await this.#db
|
|
1723
|
-
.prepare('SELECT DISTINCT blob_id FROM sync_blob_refs WHERE partition=?')
|
|
1724
|
-
.bind(partition)
|
|
1725
|
-
.all();
|
|
1726
|
-
return results.map((r) => r.blob_id);
|
|
1727
|
-
}
|
|
1728
|
-
async listScopeValueRows(partition, query) {
|
|
1729
|
-
const { results } = await this.#db
|
|
1730
|
-
.prepare(LIST_SCOPE_VALUE_ROWS_SQLITE_SQL)
|
|
1731
|
-
.bind(partition, query.table, query.variable, query.value, query.limit)
|
|
1732
|
-
.all();
|
|
1733
|
-
return results.map((row) => row.row_id);
|
|
1734
|
-
}
|
|
1735
1083
|
async eraseScopeValueHistory(partition, query) {
|
|
1736
1084
|
if (!this.#pushApplySerialized)
|
|
1737
1085
|
throw new Error('D1 erasure requires externally serialized partition writes');
|
|
1738
|
-
|
|
1739
|
-
const results = await this.#db.batch(ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL.map((sql) => this.#db.prepare(sql).bind(...params)));
|
|
1740
|
-
let removed = 0;
|
|
1741
|
-
for (const result of results) {
|
|
1742
|
-
if (typeof result !== 'object' ||
|
|
1743
|
-
result === null ||
|
|
1744
|
-
!('results' in result) ||
|
|
1745
|
-
!Array.isArray(result.results)) {
|
|
1746
|
-
throw new Error('D1 erasure returned an invalid batch result');
|
|
1747
|
-
}
|
|
1748
|
-
removed += result.results.length;
|
|
1749
|
-
}
|
|
1750
|
-
return removed;
|
|
1751
|
-
}
|
|
1752
|
-
// -- admin/console read surface --------------------------------------------
|
|
1753
|
-
async listClientRecords(partition) {
|
|
1754
|
-
const { results } = await this.#db
|
|
1755
|
-
.prepare('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
|
|
1756
|
-
.bind(partition)
|
|
1757
|
-
.all();
|
|
1758
|
-
return results.map((record) => ({
|
|
1759
|
-
clientId: record.client_id,
|
|
1760
|
-
actorId: record.actor_id,
|
|
1761
|
-
wireVersion: record.wire_version,
|
|
1762
|
-
cursor: record.cursor,
|
|
1763
|
-
updatedAtMs: record.updated_at_ms,
|
|
1764
|
-
subscriptions: JSON.parse(record.subscriptions),
|
|
1765
|
-
...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
|
|
1766
|
-
}));
|
|
1767
|
-
}
|
|
1768
|
-
async listCommitMetadata(partition, query) {
|
|
1769
|
-
// Newest-first window, resumed by `afterSeq` (exclusive lower bound) and
|
|
1770
|
-
// ended by `beforeSeq` (exclusive upper bound) when paging older. An
|
|
1771
|
-
// omitted bound is the largest safe commitSeq, so one statement serves
|
|
1772
|
-
// both the bounded and unbounded read.
|
|
1773
|
-
const beforeSeq = query.beforeSeq ?? Number.MAX_SAFE_INTEGER;
|
|
1774
|
-
const { results: rows } = query.table
|
|
1775
|
-
? await this.#db
|
|
1776
|
-
.prepare(`SELECT c.commit_seq, c.client_id, c.client_commit_id, c.actor_id, c.created_at_ms
|
|
1777
|
-
FROM sync_commits c
|
|
1778
|
-
WHERE c.partition=? AND c.commit_seq>? AND c.commit_seq<?
|
|
1779
|
-
AND EXISTS (SELECT 1 FROM sync_changes ch
|
|
1780
|
-
WHERE ch.partition=c.partition AND ch.commit_seq=c.commit_seq AND ch.tbl=?)
|
|
1781
|
-
ORDER BY c.commit_seq DESC LIMIT ?`)
|
|
1782
|
-
.bind(partition, query.afterSeq, beforeSeq, query.table, query.limit)
|
|
1783
|
-
.all()
|
|
1784
|
-
: await this.#db
|
|
1785
|
-
.prepare(`SELECT commit_seq, client_id, client_commit_id, actor_id, created_at_ms
|
|
1786
|
-
FROM sync_commits
|
|
1787
|
-
WHERE partition=? AND commit_seq>? AND commit_seq<?
|
|
1788
|
-
ORDER BY commit_seq DESC LIMIT ?`)
|
|
1789
|
-
.bind(partition, query.afterSeq, beforeSeq, query.limit)
|
|
1790
|
-
.all();
|
|
1791
|
-
const out = [];
|
|
1792
|
-
// `changeSample`: extra statement(s) for the whole page — the first
|
|
1793
|
-
// `sample` changes (change-index order) of every listed commit.
|
|
1794
|
-
const sampleCount = query.changeSample !== undefined &&
|
|
1795
|
-
Number.isInteger(query.changeSample) &&
|
|
1796
|
-
query.changeSample > 0
|
|
1797
|
-
? query.changeSample
|
|
1798
|
-
: 0;
|
|
1799
|
-
const samples = new Map();
|
|
1800
|
-
if (sampleCount > 0 && rows.length > 0) {
|
|
1801
|
-
const seqs = rows.map((row) => row.commit_seq);
|
|
1802
|
-
// D1 caps the bound parameters of one statement, so a page larger than
|
|
1803
|
-
// that cap splits into a few chunk statements, never one per commit.
|
|
1804
|
-
const chunkSize = D1_MAX_BIND_PARAMS - 2;
|
|
1805
|
-
for (let offset = 0; offset < seqs.length; offset += chunkSize) {
|
|
1806
|
-
const chunk = seqs.slice(offset, offset + chunkSize);
|
|
1807
|
-
const { results: changes } = await this.#db
|
|
1808
|
-
.prepare(`SELECT commit_seq, tbl, row_id, op FROM sync_changes
|
|
1809
|
-
WHERE partition=? AND commit_seq IN (${placeholders(chunk.length)}) AND idx<?
|
|
1810
|
-
ORDER BY commit_seq, idx`)
|
|
1811
|
-
.bind(partition, ...chunk, sampleCount)
|
|
1812
|
-
.all();
|
|
1813
|
-
for (const change of changes) {
|
|
1814
|
-
const sample = samples.get(change.commit_seq) ?? [];
|
|
1815
|
-
sample.push({
|
|
1816
|
-
table: change.tbl,
|
|
1817
|
-
rowId: change.row_id,
|
|
1818
|
-
op: change.op === 1 ? 'upsert' : 'delete',
|
|
1819
|
-
});
|
|
1820
|
-
samples.set(change.commit_seq, sample);
|
|
1821
|
-
}
|
|
1822
|
-
}
|
|
1823
|
-
}
|
|
1824
|
-
for (const row of rows) {
|
|
1825
|
-
const { results: changes } = await this.#db
|
|
1826
|
-
.prepare('SELECT tbl, count(*) AS n FROM sync_changes WHERE partition=? AND commit_seq=? GROUP BY tbl')
|
|
1827
|
-
.bind(partition, row.commit_seq)
|
|
1828
|
-
.all();
|
|
1829
|
-
out.push({
|
|
1830
|
-
commitSeq: row.commit_seq,
|
|
1831
|
-
clientId: row.client_id,
|
|
1832
|
-
clientCommitId: row.client_commit_id,
|
|
1833
|
-
actorId: row.actor_id,
|
|
1834
|
-
createdAtMs: row.created_at_ms,
|
|
1835
|
-
changeCount: changes.reduce((sum, c) => sum + c.n, 0),
|
|
1836
|
-
tables: changes.map((c) => c.tbl),
|
|
1837
|
-
...(sampleCount > 0
|
|
1838
|
-
? { sample: samples.get(row.commit_seq) ?? [] }
|
|
1839
|
-
: {}),
|
|
1840
|
-
});
|
|
1841
|
-
}
|
|
1842
|
-
return out;
|
|
1843
|
-
}
|
|
1844
|
-
async getCommit(partition, commitSeq) {
|
|
1845
|
-
const meta = await this.#db
|
|
1846
|
-
.prepare('SELECT client_id, client_commit_id, actor_id, created_at_ms FROM sync_commits WHERE partition=? AND commit_seq=?')
|
|
1847
|
-
.bind(partition, commitSeq)
|
|
1848
|
-
.first();
|
|
1849
|
-
if (meta === null)
|
|
1850
|
-
return undefined;
|
|
1851
|
-
const { results: changeRows } = await this.#db
|
|
1852
|
-
.prepare('SELECT tbl, row_id, op, row_version, scopes, payload FROM sync_changes WHERE partition=? AND commit_seq=? ORDER BY idx')
|
|
1853
|
-
.bind(partition, commitSeq)
|
|
1854
|
-
.all();
|
|
1855
|
-
const changes = changeRows.map(toStoredChange);
|
|
1856
|
-
return {
|
|
1857
|
-
commitSeq,
|
|
1858
|
-
clientId: meta.client_id,
|
|
1859
|
-
clientCommitId: meta.client_commit_id,
|
|
1860
|
-
actorId: meta.actor_id,
|
|
1861
|
-
createdAtMs: meta.created_at_ms,
|
|
1862
|
-
changeCount: changes.length,
|
|
1863
|
-
tables: [...new Set(changes.map((change) => change.table))].sort(),
|
|
1864
|
-
changes,
|
|
1865
|
-
};
|
|
1866
|
-
}
|
|
1867
|
-
async listRowChanges(partition, query) {
|
|
1868
|
-
// Retained-log read for one row: no dedicated index (the write path pays
|
|
1869
|
-
// nothing for it), bounded by retention (§4.6).
|
|
1870
|
-
const { results: rows } = await this.#db
|
|
1871
|
-
.prepare(`SELECT ch.commit_seq, ch.op, ch.row_version, c.actor_id, c.client_id, c.created_at_ms
|
|
1872
|
-
FROM sync_changes ch
|
|
1873
|
-
JOIN sync_commits c
|
|
1874
|
-
ON c.partition=ch.partition AND c.commit_seq=ch.commit_seq
|
|
1875
|
-
WHERE ch.partition=? AND ch.tbl=? AND ch.row_id=?
|
|
1876
|
-
ORDER BY ch.commit_seq DESC, ch.idx DESC LIMIT ?`)
|
|
1877
|
-
.bind(partition, query.table, query.rowId, query.limit)
|
|
1878
|
-
.all();
|
|
1879
|
-
return rows.map((row) => ({
|
|
1880
|
-
commitSeq: row.commit_seq,
|
|
1881
|
-
op: row.op === 1 ? 'upsert' : 'delete',
|
|
1882
|
-
...(row.row_version !== null ? { rowVersion: row.row_version } : {}),
|
|
1883
|
-
actorId: row.actor_id,
|
|
1884
|
-
clientId: row.client_id,
|
|
1885
|
-
createdAtMs: row.created_at_ms,
|
|
1886
|
-
}));
|
|
1887
|
-
}
|
|
1888
|
-
async listRows(partition, query) {
|
|
1889
|
-
// Keyset page over the row table's (partition, rowId) primary key: an
|
|
1890
|
-
// index range read, no scope filter, no per-row lookup.
|
|
1891
|
-
const { results } = await this.#schemaDatabase()
|
|
1892
|
-
.prepare(`SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
|
|
1893
|
-
FROM ${quoteIdent(this.table(query.table).name)}
|
|
1894
|
-
WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}>?
|
|
1895
|
-
ORDER BY ${quoteIdent(SYNC_ROW_ID_COLUMN)} LIMIT ?`)
|
|
1896
|
-
.bind(partition, query.afterRowId ?? '', query.limit)
|
|
1897
|
-
.all();
|
|
1898
|
-
return results.map(toStoredRow);
|
|
1899
|
-
}
|
|
1900
|
-
async countRows(partition, table) {
|
|
1901
|
-
const record = await this.#schemaDatabase()
|
|
1902
|
-
.prepare(`SELECT count(*) AS n FROM ${quoteIdent(this.table(table).name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=?`)
|
|
1903
|
-
.bind(partition)
|
|
1904
|
-
.first();
|
|
1905
|
-
return record?.n ?? 0;
|
|
1906
|
-
}
|
|
1907
|
-
async listPushResults(partition, clientCommitId) {
|
|
1908
|
-
// Retained outcomes for one clientCommitId: no dedicated index, ordered
|
|
1909
|
-
// by client so the read is deterministic.
|
|
1910
|
-
const { results } = await this.#schemaDatabase()
|
|
1911
|
-
.prepare('SELECT client_id, result FROM sync_push_results WHERE partition=? AND client_commit_id=? ORDER BY client_id')
|
|
1912
|
-
.bind(partition, clientCommitId)
|
|
1913
|
-
.all();
|
|
1914
|
-
return results.map((record) => {
|
|
1915
|
-
try {
|
|
1916
|
-
return {
|
|
1917
|
-
clientId: record.client_id,
|
|
1918
|
-
result: deserializePushResult(record.result),
|
|
1919
|
-
};
|
|
1920
|
-
}
|
|
1921
|
-
catch {
|
|
1922
|
-
throw syncError('sync.idempotency_cache_miss', 'persisted push result unreadable (§6.3)');
|
|
1923
|
-
}
|
|
1924
|
-
});
|
|
1925
|
-
}
|
|
1926
|
-
async scopeActivity(partition, query) {
|
|
1927
|
-
const { results: rows } = await this.#db
|
|
1928
|
-
.prepare(`SELECT DISTINCT commit_seq, tbl FROM sync_change_scopes
|
|
1929
|
-
WHERE partition=? AND var=? AND value=?
|
|
1930
|
-
ORDER BY commit_seq DESC LIMIT ?`)
|
|
1931
|
-
.bind(partition, query.variable, query.value, query.limit)
|
|
1932
|
-
.all();
|
|
1933
|
-
const out = [];
|
|
1934
|
-
for (const row of rows) {
|
|
1935
|
-
const meta = await this.#db
|
|
1936
|
-
.prepare('SELECT actor_id, created_at_ms FROM sync_commits WHERE partition=? AND commit_seq=?')
|
|
1937
|
-
.bind(partition, row.commit_seq)
|
|
1938
|
-
.first();
|
|
1939
|
-
if (meta === null)
|
|
1940
|
-
continue;
|
|
1941
|
-
const count = await this.#db
|
|
1942
|
-
.prepare('SELECT count(*) AS n FROM sync_changes WHERE partition=? AND commit_seq=? AND tbl=?')
|
|
1943
|
-
.bind(partition, row.commit_seq, row.tbl)
|
|
1944
|
-
.first();
|
|
1945
|
-
out.push({
|
|
1946
|
-
commitSeq: row.commit_seq,
|
|
1947
|
-
table: row.tbl,
|
|
1948
|
-
createdAtMs: meta.created_at_ms,
|
|
1949
|
-
actorId: meta.actor_id,
|
|
1950
|
-
changeCount: count?.n ?? 0,
|
|
1951
|
-
});
|
|
1952
|
-
}
|
|
1953
|
-
return out;
|
|
1954
|
-
}
|
|
1955
|
-
async listPartitions() {
|
|
1956
|
-
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|
|
1086
|
+
return super.eraseScopeValueHistory(partition, query);
|
|
1957
1087
|
}
|
|
1958
1088
|
}
|