orez-sync-cf-host 0.16.20 → 0.16.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/dist/generated/sync_wasm.d.ts +27 -2
- package/dist/generated/sync_wasm.js +81 -3
- package/dist/generated/sync_wasm_bg.js +81 -3
- package/dist/generated/sync_wasm_bg.wasm +0 -0
- package/dist/host.js +320 -35
- package/dist/transaction-query.d.ts +1 -0
- package/dist/transaction-query.js +1 -1
- package/dist/wasm.d.ts +6 -1
- package/dist/wasm.js +5 -0
- package/package.json +5 -4
package/README.md
CHANGED
|
@@ -254,6 +254,29 @@ Engine metadata, client last-mutation IDs, operator controls, and the
|
|
|
254
254
|
authoritative upstream database are preserved. The legacy single-response
|
|
255
255
|
`/snapshot` endpoint remains for small datasets and older harnesses.
|
|
256
256
|
|
|
257
|
+
## Rolling schema changes
|
|
258
|
+
|
|
259
|
+
The data worker and this host deploy separately, so for a while one side runs
|
|
260
|
+
a schema version the other does not. An additive column, one the Zero schema
|
|
261
|
+
marks `optional` (nullable or defaulted), can deploy on either side first with
|
|
262
|
+
no failed request:
|
|
263
|
+
|
|
264
|
+
- Each `/changes` and `/snapshot` response from `createOrezDataWorker` carries
|
|
265
|
+
`optionalColumns`, the optional columns of every table in that response. A
|
|
266
|
+
host one version behind stores the rest of each row and leaves out a column
|
|
267
|
+
only when the feed declares it optional. It records each column it had to
|
|
268
|
+
drop a value for, and once it models that column it re-snapshots one time to
|
|
269
|
+
recover the values written in the window.
|
|
270
|
+
- A host one version ahead accepts a row without a column it models as
|
|
271
|
+
optional. The write leaves that column untouched.
|
|
272
|
+
|
|
273
|
+
Everything else stays a `409 schema refresh required` on every pull: a column
|
|
274
|
+
the feed does not declare optional, a missing required column, or a row that
|
|
275
|
+
differs in both directions at once, which is the shape of a rename. Change a
|
|
276
|
+
required column, a type or a key in steps that are each additive.
|
|
277
|
+
`bun run test:rolling-schema` replays both deploy orders and both required
|
|
278
|
+
column cases against a real data worker and host.
|
|
279
|
+
|
|
257
280
|
## Delegated service addresses
|
|
258
281
|
|
|
259
282
|
`mutateBinding` selects the service binding that owns the application's mutate
|
|
@@ -295,6 +318,7 @@ bun install
|
|
|
295
318
|
bun run typecheck # builds probe-enabled wasm for all TS declarations
|
|
296
319
|
bun run test:platform # M0 boundary regression: 36 local workerd assertions
|
|
297
320
|
bun run test:integration # production host: pull/push/rollback/wake/eviction
|
|
321
|
+
bun run test:rolling-schema # additive column in both deploy orders
|
|
298
322
|
bun run bundle # Wrangler dry-run bundle report
|
|
299
323
|
bun run measure # cold start/CPU/storage/memory baseline
|
|
300
324
|
cargo test -p sync-core # engine/model/reference/soot suites (from repo root)
|
|
@@ -11,7 +11,7 @@ export function engine_apply_snapshot_changes(db: any, schema: any, generation:
|
|
|
11
11
|
* Apply one bounded source page to staging and commit its opaque cursor with
|
|
12
12
|
* the rows. The host owns one transaction per call.
|
|
13
13
|
*/
|
|
14
|
-
export function engine_apply_snapshot_page(db: any, schema: any, generation: string, table: string, rows: any, next_cursor
|
|
14
|
+
export function engine_apply_snapshot_page(db: any, schema: any, generation: string, table: string, rows: any, next_cursor: string | null | undefined, optional_columns: any): any;
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
17
|
* Apply one ordered page from the upstream ZeroSqlDO change feed. The host
|
|
@@ -54,6 +54,12 @@ export function engine_begin_snapshot_generation(db: any, schema: any, start_wat
|
|
|
54
54
|
*/
|
|
55
55
|
export function engine_compile_query(schema: any, ast: any, format: any): any;
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* True when the replica dropped values for a feed column it now models and
|
|
59
|
+
* must re-snapshot before further ingest.
|
|
60
|
+
*/
|
|
61
|
+
export function engine_dropped_feed_columns_need_snapshot(db: any, schema: any): boolean;
|
|
62
|
+
|
|
57
63
|
export function engine_finalize(db: any, client_group_id: string, client_id: string, mutation_id: string): void;
|
|
58
64
|
|
|
59
65
|
/**
|
|
@@ -62,12 +68,22 @@ export function engine_finalize(db: any, client_group_id: string, client_id: str
|
|
|
62
68
|
*/
|
|
63
69
|
export function engine_finalize_snapshot_generation(db: any, schema: any, generation: string, watermark: string): any;
|
|
64
70
|
|
|
71
|
+
/**
|
|
72
|
+
* publish selected staged tables without resetting unrelated client state.
|
|
73
|
+
*/
|
|
74
|
+
export function engine_finalize_table_bootstrap(db: any, schema: any, selected: any, generation: string, watermark: string): any;
|
|
75
|
+
|
|
65
76
|
/**
|
|
66
77
|
* Production pull entry point. Desired-query ASTs are already resolved and
|
|
67
78
|
* validated by the consumer host before crossing this boundary.
|
|
68
79
|
*/
|
|
69
80
|
export function engine_handle_query_pull(db: any, schema: any, retain_changes: string, body: any, user_id: string): any;
|
|
70
81
|
|
|
82
|
+
/**
|
|
83
|
+
* Install number guards on the database that owns application writes.
|
|
84
|
+
*/
|
|
85
|
+
export function engine_init_number_storage_guards(db: any, schema: any): void;
|
|
86
|
+
|
|
71
87
|
/**
|
|
72
88
|
* Initialize the additive query-aware durable tables. The host owns the
|
|
73
89
|
* transaction boundary, exactly as it does for the baseline schema.
|
|
@@ -80,6 +96,8 @@ export function engine_init_query_schema(db: any): void;
|
|
|
80
96
|
*/
|
|
81
97
|
export function engine_init_schema(db: any, schema: any): void;
|
|
82
98
|
|
|
99
|
+
export function engine_invalid_number_cells(db: any, schema: any): any;
|
|
100
|
+
|
|
83
101
|
export function engine_invalidate(db: any): void;
|
|
84
102
|
|
|
85
103
|
/**
|
|
@@ -113,6 +131,8 @@ export function engine_schema_revision(): string;
|
|
|
113
131
|
|
|
114
132
|
export function engine_state(db: any): any;
|
|
115
133
|
|
|
134
|
+
export function engine_validate_number_storage_changes(db: any, schema: any, before: any): void;
|
|
135
|
+
|
|
116
136
|
export function engine_version(): string;
|
|
117
137
|
|
|
118
138
|
/**
|
|
@@ -128,18 +148,22 @@ export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembl
|
|
|
128
148
|
export interface InitOutput {
|
|
129
149
|
readonly memory: WebAssembly.Memory;
|
|
130
150
|
readonly engine_apply_snapshot_changes: (a: any, b: any, c: number, d: number, e: any) => [number, number, number];
|
|
131
|
-
readonly engine_apply_snapshot_page: (a: any, b: any, c: number, d: number, e: number, f: number, g: any, h: number, i: number) => [number, number, number];
|
|
151
|
+
readonly engine_apply_snapshot_page: (a: any, b: any, c: number, d: number, e: number, f: number, g: any, h: number, i: number, j: any) => [number, number, number];
|
|
132
152
|
readonly engine_apply_upstream: (a: any, b: any, c: any) => [number, number, number];
|
|
133
153
|
readonly engine_apply_upstream_snapshot: (a: any, b: any, c: any) => [number, number, number];
|
|
134
154
|
readonly engine_assemble_push_response: (a: any) => [number, number, number];
|
|
135
155
|
readonly engine_authorize_realtime_subscription: (a: any, b: any, c: number, d: number, e: number, f: number, g: number, h: number, i: any) => [number, number, number];
|
|
136
156
|
readonly engine_begin_snapshot_generation: (a: any, b: any, c: number, d: number) => [number, number, number];
|
|
137
157
|
readonly engine_compile_query: (a: any, b: any, c: any) => [number, number, number];
|
|
158
|
+
readonly engine_dropped_feed_columns_need_snapshot: (a: any, b: any) => [number, number, number];
|
|
138
159
|
readonly engine_finalize: (a: any, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
|
|
139
160
|
readonly engine_finalize_snapshot_generation: (a: any, b: any, c: number, d: number, e: number, f: number) => [number, number, number];
|
|
161
|
+
readonly engine_finalize_table_bootstrap: (a: any, b: any, c: any, d: number, e: number, f: number, g: number) => [number, number, number];
|
|
140
162
|
readonly engine_handle_query_pull: (a: any, b: any, c: number, d: number, e: any, f: number, g: number) => [number, number, number];
|
|
163
|
+
readonly engine_init_number_storage_guards: (a: any, b: any) => [number, number];
|
|
141
164
|
readonly engine_init_query_schema: (a: any) => [number, number];
|
|
142
165
|
readonly engine_init_schema: (a: any, b: any) => [number, number];
|
|
166
|
+
readonly engine_invalid_number_cells: (a: any, b: any) => [number, number, number];
|
|
143
167
|
readonly engine_invalidate: (a: any) => [number, number];
|
|
144
168
|
readonly engine_memory_bytes: () => [number, number, number];
|
|
145
169
|
readonly engine_preflight: (a: any, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number) => [number, number, number];
|
|
@@ -149,6 +173,7 @@ export interface InitOutput {
|
|
|
149
173
|
readonly engine_record_app_error: (a: any, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number) => [number, number];
|
|
150
174
|
readonly engine_schema_revision: () => [number, number];
|
|
151
175
|
readonly engine_state: (a: any) => [number, number, number];
|
|
176
|
+
readonly engine_validate_number_storage_changes: (a: any, b: any, c: any) => [number, number];
|
|
152
177
|
readonly engine_version: () => [number, number];
|
|
153
178
|
readonly engine_wake_targets: (a: any, b: any, c: number, d: number) => [number, number, number];
|
|
154
179
|
readonly __wbindgen_malloc: (a: number, b: number) => number;
|
|
@@ -27,17 +27,18 @@ export function engine_apply_snapshot_changes(db, schema, generation, batch) {
|
|
|
27
27
|
* @param {string} generation
|
|
28
28
|
* @param {string} table
|
|
29
29
|
* @param {any} rows
|
|
30
|
-
* @param {string | null}
|
|
30
|
+
* @param {string | null | undefined} next_cursor
|
|
31
|
+
* @param {any} optional_columns
|
|
31
32
|
* @returns {any}
|
|
32
33
|
*/
|
|
33
|
-
export function engine_apply_snapshot_page(db, schema, generation, table, rows, next_cursor) {
|
|
34
|
+
export function engine_apply_snapshot_page(db, schema, generation, table, rows, next_cursor, optional_columns) {
|
|
34
35
|
const ptr0 = passStringToWasm0(generation, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
35
36
|
const len0 = WASM_VECTOR_LEN;
|
|
36
37
|
const ptr1 = passStringToWasm0(table, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
37
38
|
const len1 = WASM_VECTOR_LEN;
|
|
38
39
|
var ptr2 = isLikeNone(next_cursor) ? 0 : passStringToWasm0(next_cursor, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
39
40
|
var len2 = WASM_VECTOR_LEN;
|
|
40
|
-
const ret = wasm.engine_apply_snapshot_page(db, schema, ptr0, len0, ptr1, len1, rows, ptr2, len2);
|
|
41
|
+
const ret = wasm.engine_apply_snapshot_page(db, schema, ptr0, len0, ptr1, len1, rows, ptr2, len2, optional_columns);
|
|
41
42
|
if (ret[2]) {
|
|
42
43
|
throw takeFromExternrefTable0(ret[1]);
|
|
43
44
|
}
|
|
@@ -156,6 +157,21 @@ export function engine_compile_query(schema, ast, format) {
|
|
|
156
157
|
return takeFromExternrefTable0(ret[0]);
|
|
157
158
|
}
|
|
158
159
|
|
|
160
|
+
/**
|
|
161
|
+
* True when the replica dropped values for a feed column it now models and
|
|
162
|
+
* must re-snapshot before further ingest.
|
|
163
|
+
* @param {any} db
|
|
164
|
+
* @param {any} schema
|
|
165
|
+
* @returns {boolean}
|
|
166
|
+
*/
|
|
167
|
+
export function engine_dropped_feed_columns_need_snapshot(db, schema) {
|
|
168
|
+
const ret = wasm.engine_dropped_feed_columns_need_snapshot(db, schema);
|
|
169
|
+
if (ret[2]) {
|
|
170
|
+
throw takeFromExternrefTable0(ret[1]);
|
|
171
|
+
}
|
|
172
|
+
return ret[0] !== 0;
|
|
173
|
+
}
|
|
174
|
+
|
|
159
175
|
/**
|
|
160
176
|
* @param {any} db
|
|
161
177
|
* @param {string} client_group_id
|
|
@@ -196,6 +212,27 @@ export function engine_finalize_snapshot_generation(db, schema, generation, wate
|
|
|
196
212
|
return takeFromExternrefTable0(ret[0]);
|
|
197
213
|
}
|
|
198
214
|
|
|
215
|
+
/**
|
|
216
|
+
* publish selected staged tables without resetting unrelated client state.
|
|
217
|
+
* @param {any} db
|
|
218
|
+
* @param {any} schema
|
|
219
|
+
* @param {any} selected
|
|
220
|
+
* @param {string} generation
|
|
221
|
+
* @param {string} watermark
|
|
222
|
+
* @returns {any}
|
|
223
|
+
*/
|
|
224
|
+
export function engine_finalize_table_bootstrap(db, schema, selected, generation, watermark) {
|
|
225
|
+
const ptr0 = passStringToWasm0(generation, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
226
|
+
const len0 = WASM_VECTOR_LEN;
|
|
227
|
+
const ptr1 = passStringToWasm0(watermark, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
228
|
+
const len1 = WASM_VECTOR_LEN;
|
|
229
|
+
const ret = wasm.engine_finalize_table_bootstrap(db, schema, selected, ptr0, len0, ptr1, len1);
|
|
230
|
+
if (ret[2]) {
|
|
231
|
+
throw takeFromExternrefTable0(ret[1]);
|
|
232
|
+
}
|
|
233
|
+
return takeFromExternrefTable0(ret[0]);
|
|
234
|
+
}
|
|
235
|
+
|
|
199
236
|
/**
|
|
200
237
|
* Production pull entry point. Desired-query ASTs are already resolved and
|
|
201
238
|
* validated by the consumer host before crossing this boundary.
|
|
@@ -218,6 +255,18 @@ export function engine_handle_query_pull(db, schema, retain_changes, body, user_
|
|
|
218
255
|
return takeFromExternrefTable0(ret[0]);
|
|
219
256
|
}
|
|
220
257
|
|
|
258
|
+
/**
|
|
259
|
+
* Install number guards on the database that owns application writes.
|
|
260
|
+
* @param {any} db
|
|
261
|
+
* @param {any} schema
|
|
262
|
+
*/
|
|
263
|
+
export function engine_init_number_storage_guards(db, schema) {
|
|
264
|
+
const ret = wasm.engine_init_number_storage_guards(db, schema);
|
|
265
|
+
if (ret[1]) {
|
|
266
|
+
throw takeFromExternrefTable0(ret[0]);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
221
270
|
/**
|
|
222
271
|
* Initialize the additive query-aware durable tables. The host owns the
|
|
223
272
|
* transaction boundary, exactly as it does for the baseline schema.
|
|
@@ -243,6 +292,19 @@ export function engine_init_schema(db, schema) {
|
|
|
243
292
|
}
|
|
244
293
|
}
|
|
245
294
|
|
|
295
|
+
/**
|
|
296
|
+
* @param {any} db
|
|
297
|
+
* @param {any} schema
|
|
298
|
+
* @returns {any}
|
|
299
|
+
*/
|
|
300
|
+
export function engine_invalid_number_cells(db, schema) {
|
|
301
|
+
const ret = wasm.engine_invalid_number_cells(db, schema);
|
|
302
|
+
if (ret[2]) {
|
|
303
|
+
throw takeFromExternrefTable0(ret[1]);
|
|
304
|
+
}
|
|
305
|
+
return takeFromExternrefTable0(ret[0]);
|
|
306
|
+
}
|
|
307
|
+
|
|
246
308
|
/**
|
|
247
309
|
* @param {any} db
|
|
248
310
|
*/
|
|
@@ -382,6 +444,18 @@ export function engine_state(db) {
|
|
|
382
444
|
return takeFromExternrefTable0(ret[0]);
|
|
383
445
|
}
|
|
384
446
|
|
|
447
|
+
/**
|
|
448
|
+
* @param {any} db
|
|
449
|
+
* @param {any} schema
|
|
450
|
+
* @param {any} before
|
|
451
|
+
*/
|
|
452
|
+
export function engine_validate_number_storage_changes(db, schema, before) {
|
|
453
|
+
const ret = wasm.engine_validate_number_storage_changes(db, schema, before);
|
|
454
|
+
if (ret[1]) {
|
|
455
|
+
throw takeFromExternrefTable0(ret[0]);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
|
|
385
459
|
/**
|
|
386
460
|
* @returns {string}
|
|
387
461
|
*/
|
|
@@ -465,6 +539,10 @@ function __wbg_get_imports() {
|
|
|
465
539
|
const ret = typeof(arg0) === 'function';
|
|
466
540
|
return ret;
|
|
467
541
|
},
|
|
542
|
+
__wbg___wbindgen_is_null_ea9085d691f535d3: function(arg0) {
|
|
543
|
+
const ret = arg0 === null;
|
|
544
|
+
return ret;
|
|
545
|
+
},
|
|
468
546
|
__wbg___wbindgen_is_object_a27215656b807791: function(arg0) {
|
|
469
547
|
const val = arg0;
|
|
470
548
|
const ret = typeof(val) === 'object' && val !== null;
|
|
@@ -25,17 +25,18 @@ export function engine_apply_snapshot_changes(db, schema, generation, batch) {
|
|
|
25
25
|
* @param {string} generation
|
|
26
26
|
* @param {string} table
|
|
27
27
|
* @param {any} rows
|
|
28
|
-
* @param {string | null}
|
|
28
|
+
* @param {string | null | undefined} next_cursor
|
|
29
|
+
* @param {any} optional_columns
|
|
29
30
|
* @returns {any}
|
|
30
31
|
*/
|
|
31
|
-
export function engine_apply_snapshot_page(db, schema, generation, table, rows, next_cursor) {
|
|
32
|
+
export function engine_apply_snapshot_page(db, schema, generation, table, rows, next_cursor, optional_columns) {
|
|
32
33
|
const ptr0 = passStringToWasm0(generation, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
33
34
|
const len0 = WASM_VECTOR_LEN;
|
|
34
35
|
const ptr1 = passStringToWasm0(table, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
35
36
|
const len1 = WASM_VECTOR_LEN;
|
|
36
37
|
var ptr2 = isLikeNone(next_cursor) ? 0 : passStringToWasm0(next_cursor, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
37
38
|
var len2 = WASM_VECTOR_LEN;
|
|
38
|
-
const ret = wasm.engine_apply_snapshot_page(db, schema, ptr0, len0, ptr1, len1, rows, ptr2, len2);
|
|
39
|
+
const ret = wasm.engine_apply_snapshot_page(db, schema, ptr0, len0, ptr1, len1, rows, ptr2, len2, optional_columns);
|
|
39
40
|
if (ret[2]) {
|
|
40
41
|
throw takeFromExternrefTable0(ret[1]);
|
|
41
42
|
}
|
|
@@ -154,6 +155,21 @@ export function engine_compile_query(schema, ast, format) {
|
|
|
154
155
|
return takeFromExternrefTable0(ret[0]);
|
|
155
156
|
}
|
|
156
157
|
|
|
158
|
+
/**
|
|
159
|
+
* True when the replica dropped values for a feed column it now models and
|
|
160
|
+
* must re-snapshot before further ingest.
|
|
161
|
+
* @param {any} db
|
|
162
|
+
* @param {any} schema
|
|
163
|
+
* @returns {boolean}
|
|
164
|
+
*/
|
|
165
|
+
export function engine_dropped_feed_columns_need_snapshot(db, schema) {
|
|
166
|
+
const ret = wasm.engine_dropped_feed_columns_need_snapshot(db, schema);
|
|
167
|
+
if (ret[2]) {
|
|
168
|
+
throw takeFromExternrefTable0(ret[1]);
|
|
169
|
+
}
|
|
170
|
+
return ret[0] !== 0;
|
|
171
|
+
}
|
|
172
|
+
|
|
157
173
|
/**
|
|
158
174
|
* @param {any} db
|
|
159
175
|
* @param {string} client_group_id
|
|
@@ -194,6 +210,27 @@ export function engine_finalize_snapshot_generation(db, schema, generation, wate
|
|
|
194
210
|
return takeFromExternrefTable0(ret[0]);
|
|
195
211
|
}
|
|
196
212
|
|
|
213
|
+
/**
|
|
214
|
+
* publish selected staged tables without resetting unrelated client state.
|
|
215
|
+
* @param {any} db
|
|
216
|
+
* @param {any} schema
|
|
217
|
+
* @param {any} selected
|
|
218
|
+
* @param {string} generation
|
|
219
|
+
* @param {string} watermark
|
|
220
|
+
* @returns {any}
|
|
221
|
+
*/
|
|
222
|
+
export function engine_finalize_table_bootstrap(db, schema, selected, generation, watermark) {
|
|
223
|
+
const ptr0 = passStringToWasm0(generation, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
224
|
+
const len0 = WASM_VECTOR_LEN;
|
|
225
|
+
const ptr1 = passStringToWasm0(watermark, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
|
|
226
|
+
const len1 = WASM_VECTOR_LEN;
|
|
227
|
+
const ret = wasm.engine_finalize_table_bootstrap(db, schema, selected, ptr0, len0, ptr1, len1);
|
|
228
|
+
if (ret[2]) {
|
|
229
|
+
throw takeFromExternrefTable0(ret[1]);
|
|
230
|
+
}
|
|
231
|
+
return takeFromExternrefTable0(ret[0]);
|
|
232
|
+
}
|
|
233
|
+
|
|
197
234
|
/**
|
|
198
235
|
* Production pull entry point. Desired-query ASTs are already resolved and
|
|
199
236
|
* validated by the consumer host before crossing this boundary.
|
|
@@ -216,6 +253,18 @@ export function engine_handle_query_pull(db, schema, retain_changes, body, user_
|
|
|
216
253
|
return takeFromExternrefTable0(ret[0]);
|
|
217
254
|
}
|
|
218
255
|
|
|
256
|
+
/**
|
|
257
|
+
* Install number guards on the database that owns application writes.
|
|
258
|
+
* @param {any} db
|
|
259
|
+
* @param {any} schema
|
|
260
|
+
*/
|
|
261
|
+
export function engine_init_number_storage_guards(db, schema) {
|
|
262
|
+
const ret = wasm.engine_init_number_storage_guards(db, schema);
|
|
263
|
+
if (ret[1]) {
|
|
264
|
+
throw takeFromExternrefTable0(ret[0]);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
219
268
|
/**
|
|
220
269
|
* Initialize the additive query-aware durable tables. The host owns the
|
|
221
270
|
* transaction boundary, exactly as it does for the baseline schema.
|
|
@@ -241,6 +290,19 @@ export function engine_init_schema(db, schema) {
|
|
|
241
290
|
}
|
|
242
291
|
}
|
|
243
292
|
|
|
293
|
+
/**
|
|
294
|
+
* @param {any} db
|
|
295
|
+
* @param {any} schema
|
|
296
|
+
* @returns {any}
|
|
297
|
+
*/
|
|
298
|
+
export function engine_invalid_number_cells(db, schema) {
|
|
299
|
+
const ret = wasm.engine_invalid_number_cells(db, schema);
|
|
300
|
+
if (ret[2]) {
|
|
301
|
+
throw takeFromExternrefTable0(ret[1]);
|
|
302
|
+
}
|
|
303
|
+
return takeFromExternrefTable0(ret[0]);
|
|
304
|
+
}
|
|
305
|
+
|
|
244
306
|
/**
|
|
245
307
|
* @param {any} db
|
|
246
308
|
*/
|
|
@@ -380,6 +442,18 @@ export function engine_state(db) {
|
|
|
380
442
|
return takeFromExternrefTable0(ret[0]);
|
|
381
443
|
}
|
|
382
444
|
|
|
445
|
+
/**
|
|
446
|
+
* @param {any} db
|
|
447
|
+
* @param {any} schema
|
|
448
|
+
* @param {any} before
|
|
449
|
+
*/
|
|
450
|
+
export function engine_validate_number_storage_changes(db, schema, before) {
|
|
451
|
+
const ret = wasm.engine_validate_number_storage_changes(db, schema, before);
|
|
452
|
+
if (ret[1]) {
|
|
453
|
+
throw takeFromExternrefTable0(ret[0]);
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
383
457
|
/**
|
|
384
458
|
* @returns {string}
|
|
385
459
|
*/
|
|
@@ -460,6 +534,10 @@ export function __wbg___wbindgen_is_function_1ff95bcc5517c252(arg0) {
|
|
|
460
534
|
const ret = typeof(arg0) === 'function';
|
|
461
535
|
return ret;
|
|
462
536
|
}
|
|
537
|
+
export function __wbg___wbindgen_is_null_ea9085d691f535d3(arg0) {
|
|
538
|
+
const ret = arg0 === null;
|
|
539
|
+
return ret;
|
|
540
|
+
}
|
|
463
541
|
export function __wbg___wbindgen_is_object_a27215656b807791(arg0) {
|
|
464
542
|
const val = arg0;
|
|
465
543
|
const ret = typeof(val) === 'object' && val !== null;
|
|
Binary file
|
package/dist/host.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { DurableObject } from 'cloudflare:workers';
|
|
2
2
|
import { createSyncExecutor, isMutationRetryError } from 'orez-sync-executor/core';
|
|
3
3
|
import { createSocketHost } from 'orez-sync-executor/realtime';
|
|
4
|
+
import { createTableIfAbsentTarget } from 'orez-sync-executor/sqlite';
|
|
4
5
|
import { validateSyncHostConfig } from './config.js';
|
|
5
6
|
import { createQueryCompiler } from './query-compiler.js';
|
|
6
7
|
import { resolveQueryPatch } from './query-patch.js';
|
|
7
8
|
import { ServingLagTracker } from './serving-lag.js';
|
|
8
9
|
import { decodeSqlParams, SqlStorageDirect, SqlStorageMutatorTransaction, SqlStorageSyncDb, } from './sql-storage-adapter.js';
|
|
9
10
|
import { DEFAULT_UPSTREAM_MAX_RESPONSE_BYTES, DEFAULT_UPSTREAM_MAX_REQUEST_BYTES, DEFAULT_UPSTREAM_REQUEST_TIMEOUT_MS, fetchBoundedUpstreamJson, readBoundedJsonResponse, readBoundedStream, UpstreamResponseLimitError, } from './upstream-response.js';
|
|
10
|
-
import { engine_authorize_realtime_subscription, engine_apply_snapshot_changes, engine_apply_snapshot_page, engine_apply_upstream, engine_begin_snapshot_generation, engine_finalize, engine_finalize_snapshot_generation, engine_handle_query_pull, engine_init_query_schema, engine_init_schema, engine_invalidate, engine_memory_bytes, engine_preflight, engine_prune, engine_push_validate, engine_read_snapshot_progress, engine_schema_revision, engine_state, engine_version, engine_wake_targets, } from './wasm.js';
|
|
11
|
+
import { engine_authorize_realtime_subscription, engine_apply_snapshot_changes, engine_apply_snapshot_page, engine_apply_upstream, engine_begin_snapshot_generation, engine_dropped_feed_columns_need_snapshot, engine_finalize, engine_finalize_snapshot_generation, engine_finalize_table_bootstrap, engine_handle_query_pull, engine_init_query_schema, engine_init_schema, engine_init_number_storage_guards, engine_invalid_number_cells, engine_validate_number_storage_changes, engine_invalidate, engine_memory_bytes, engine_preflight, engine_prune, engine_push_validate, engine_read_snapshot_progress, engine_schema_revision, engine_state, engine_version, engine_wake_targets, } from './wasm.js';
|
|
11
12
|
import { IngestBreakerError, IngestCircuitBreaker, retryDelayMs, shouldRetryDelegatedPush, } from './write-safeguards.js';
|
|
12
13
|
const NAMESPACE_HEADER = 'x-orez-sync-namespace';
|
|
13
14
|
const UPSTREAM_PATH_HEADER = 'x-orez-sync-upstream-path';
|
|
@@ -20,6 +21,9 @@ const WAKE_SUBSCRIBER_TAG = 'orez:wake-subscriber';
|
|
|
20
21
|
const IDENTITY_HEADER = 'x-orez-sync-identity';
|
|
21
22
|
const WAKE_ISSUED_AT_HEADER = 'x-orez-wake-issued-at';
|
|
22
23
|
const DEFAULT_SNAPSHOT_PAGE_ROWS = 2_000;
|
|
24
|
+
// a fresh replica pages every modeled table; the first page of each is small
|
|
25
|
+
// and fetched concurrently so the snapshot costs one round trip, not one per table.
|
|
26
|
+
const SNAPSHOT_PREFETCH_PAGE_ROWS = 250;
|
|
23
27
|
const MIN_SNAPSHOT_PAGE_ROWS = 100;
|
|
24
28
|
const WORKER_STAGE_TELEMETRY_SAMPLE_RATE = 0.01;
|
|
25
29
|
const WORKER_STAGE_WAITING_MS = 5_000;
|
|
@@ -225,6 +229,15 @@ async function namespaceHash(namespace) {
|
|
|
225
229
|
const bytes = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(namespace));
|
|
226
230
|
return Array.from(new Uint8Array(bytes).slice(0, 8), (byte) => byte.toString(16).padStart(2, '0')).join('');
|
|
227
231
|
}
|
|
232
|
+
function withServerTiming(response, stages) {
|
|
233
|
+
if (response.webSocket)
|
|
234
|
+
return response;
|
|
235
|
+
const wrapped = new Response(response.body, response);
|
|
236
|
+
const value = stages.map(([name, ms]) => `${name};dur=${ms.toFixed(1)}`).join(', ');
|
|
237
|
+
const existing = wrapped.headers.get('server-timing');
|
|
238
|
+
wrapped.headers.set('server-timing', existing ? `${existing}, ${value}` : value);
|
|
239
|
+
return wrapped;
|
|
240
|
+
}
|
|
228
241
|
async function timeWorkerStage(fields, work) {
|
|
229
242
|
const started = performance.now();
|
|
230
243
|
let waited = false;
|
|
@@ -403,6 +416,8 @@ export function createSyncWorker(config) {
|
|
|
403
416
|
headers.delete(WAKE_ISSUED_AT_HEADER);
|
|
404
417
|
headers.delete(NOTIFY_IF_SUBSCRIBED_HEADER);
|
|
405
418
|
let forwardedBody = null;
|
|
419
|
+
const stages = [];
|
|
420
|
+
const stageStarted = performance.now();
|
|
406
421
|
// /wake and /realtime/produce are both websocket upgrades and have no
|
|
407
422
|
// body to put claims in. wake authentication is normalized from its
|
|
408
423
|
// WebSocket subprotocol above; each route has its own authorization check
|
|
@@ -431,6 +446,7 @@ export function createSyncWorker(config) {
|
|
|
431
446
|
catch (error) {
|
|
432
447
|
return errorResponse(error);
|
|
433
448
|
}
|
|
449
|
+
stages.push(['worker_authenticate', performance.now() - stageStarted]);
|
|
434
450
|
if (!claims ||
|
|
435
451
|
typeof claims.userID !== 'string' ||
|
|
436
452
|
claims.userID.length === 0) {
|
|
@@ -458,6 +474,7 @@ export function createSyncWorker(config) {
|
|
|
458
474
|
headers.set(IDENTITY_HEADER, encodeURIComponent(wakeUserID));
|
|
459
475
|
if (wakeIssuedAt !== null)
|
|
460
476
|
headers.set(WAKE_ISSUED_AT_HEADER, String(wakeIssuedAt));
|
|
477
|
+
const prepared = performance.now();
|
|
461
478
|
const hashedNamespace = await namespaceHash(namespace);
|
|
462
479
|
headers.set(NAMESPACE_HEADER, hashedNamespace);
|
|
463
480
|
try {
|
|
@@ -472,13 +489,17 @@ export function createSyncWorker(config) {
|
|
|
472
489
|
? jsonBodyRequest(request, headers, forwardedBody)
|
|
473
490
|
: new Request(request, { headers });
|
|
474
491
|
const id = env.SYNC_DO.idFromName(namespace);
|
|
475
|
-
|
|
492
|
+
stages.push(['worker_prepare', performance.now() - prepared]);
|
|
493
|
+
const forwardStarted = performance.now();
|
|
494
|
+
const doResponse = await timeWorkerStage({
|
|
476
495
|
hostVersion: config.hostVersion,
|
|
477
496
|
requestKind,
|
|
478
497
|
stage: 'sync_do_forward',
|
|
479
498
|
namespaceHash: hashedNamespace,
|
|
480
499
|
sampled,
|
|
481
500
|
}, () => env.SYNC_DO.get(id).fetch(forwarded));
|
|
501
|
+
stages.push(['worker_do_forward', performance.now() - forwardStarted]);
|
|
502
|
+
return withServerTiming(doResponse, stages);
|
|
482
503
|
};
|
|
483
504
|
return withCors(await handle());
|
|
484
505
|
},
|
|
@@ -536,7 +557,15 @@ export function createSyncDurableObject(config) {
|
|
|
536
557
|
#executorBeforeCommitFault = null;
|
|
537
558
|
#bootID = crypto.randomUUID();
|
|
538
559
|
#initialized = false;
|
|
560
|
+
#tableBootstrap = null;
|
|
561
|
+
#snapshotRequired = false;
|
|
539
562
|
#initSkipped = false;
|
|
563
|
+
// whether this instance has asked the engine if a column it dropped while
|
|
564
|
+
// running behind its feed is now modeled. the answer lives in the engine's
|
|
565
|
+
// own table until a snapshot settles it, so once per instance is enough.
|
|
566
|
+
#droppedFeedColumnsChecked = false;
|
|
567
|
+
#snapshotPrefetch = null;
|
|
568
|
+
#upstreamFetches = 0;
|
|
540
569
|
#lastRequestAt = 0;
|
|
541
570
|
#hibernations = 0;
|
|
542
571
|
#dropNextPushResponse = false;
|
|
@@ -560,6 +589,8 @@ export function createSyncDurableObject(config) {
|
|
|
560
589
|
// during the replay wait for it instead of racing past into an empty hub.
|
|
561
590
|
#realtimeReady = null;
|
|
562
591
|
#ingestPromise = null;
|
|
592
|
+
// set by a commit notification that reached a running alarm's ingest late
|
|
593
|
+
#notifyPending = false;
|
|
563
594
|
#recordingIngestBillable = false;
|
|
564
595
|
#ingestBreaker = new IngestCircuitBreaker({
|
|
565
596
|
budgetRows: ingestBudgetRows,
|
|
@@ -591,6 +622,10 @@ export function createSyncDurableObject(config) {
|
|
|
591
622
|
const database = {
|
|
592
623
|
transaction: async (work) => this.ctx.storage.transaction(async () => {
|
|
593
624
|
let applicationWrite = false;
|
|
625
|
+
let schemaFailure;
|
|
626
|
+
const guarded = (sql, run) => this.#guardSchema(sql, run, (error) => {
|
|
627
|
+
schemaFailure = { error };
|
|
628
|
+
});
|
|
594
629
|
const tx = {
|
|
595
630
|
exec: async (sql, params, metadata) => {
|
|
596
631
|
if (metadata !== undefined ||
|
|
@@ -598,9 +633,9 @@ export function createSyncDurableObject(config) {
|
|
|
598
633
|
!/\b_zsync_[A-Za-z0-9_]+\b/.test(sql))) {
|
|
599
634
|
applicationWrite = true;
|
|
600
635
|
}
|
|
601
|
-
return this.#
|
|
636
|
+
return guarded(sql, () => this.#directSql.exec(sql, params, metadata));
|
|
602
637
|
},
|
|
603
|
-
query: (sql, params) => {
|
|
638
|
+
query: async (sql, params) => {
|
|
604
639
|
// exact helper statements use sqlite returning so the executor
|
|
605
640
|
// can prove the physical primary keys that changed. Keep the
|
|
606
641
|
// host's pre-commit fault boundary after those writes too.
|
|
@@ -608,11 +643,13 @@ export function createSyncDurableObject(config) {
|
|
|
608
643
|
/\bRETURNING\b/i.test(sql)) {
|
|
609
644
|
applicationWrite = true;
|
|
610
645
|
}
|
|
611
|
-
return this.#
|
|
646
|
+
return guarded(sql, () => this.#directSql.query(sql, params));
|
|
612
647
|
},
|
|
613
648
|
queryAst: (ast, format, queryName) => this.#mutatorSql.queryAst(ast, format, queryName),
|
|
614
649
|
};
|
|
615
650
|
const value = await work(tx);
|
|
651
|
+
if (schemaFailure)
|
|
652
|
+
throw schemaFailure.error;
|
|
616
653
|
if (applicationWrite && this.#executorBeforeCommitFault) {
|
|
617
654
|
const fault = this.#executorBeforeCommitFault;
|
|
618
655
|
this.#executorBeforeCommitFault = null;
|
|
@@ -620,7 +657,7 @@ export function createSyncDurableObject(config) {
|
|
|
620
657
|
}
|
|
621
658
|
return value;
|
|
622
659
|
}),
|
|
623
|
-
query: (sql, params) => this.#
|
|
660
|
+
query: async (sql, params) => this.#guardStandalone(sql, () => this.#directSql.query(sql, params)),
|
|
624
661
|
};
|
|
625
662
|
this.#executor = config.mutators
|
|
626
663
|
? createSyncExecutor({
|
|
@@ -650,6 +687,40 @@ export function createSyncDurableObject(config) {
|
|
|
650
687
|
ctx.blockConcurrencyWhile(async () => this.#initialize());
|
|
651
688
|
}
|
|
652
689
|
}
|
|
690
|
+
#guardSchema(sql, run, failed) {
|
|
691
|
+
const schema = /\b(?:create|alter|drop|reindex)\b/i.test(sql);
|
|
692
|
+
const replicaWrite = config.upstream && /\b(?:insert|update|delete|replace)\b/i.test(sql);
|
|
693
|
+
if (!schema && !replicaWrite)
|
|
694
|
+
return run();
|
|
695
|
+
if (/_zsync_number_/i.test(sql))
|
|
696
|
+
throw new Error('sync storage guards are host-owned');
|
|
697
|
+
const existing = createTableIfAbsentTarget(sql);
|
|
698
|
+
if (existing &&
|
|
699
|
+
this.#directSql.query(`PRAGMA table_xinfo("${existing.replaceAll('"', '""')}")`)
|
|
700
|
+
.length > 0)
|
|
701
|
+
return run();
|
|
702
|
+
const before = this.#wasm(() => engine_invalid_number_cells(this.#engineDb, config.schema));
|
|
703
|
+
const result = run();
|
|
704
|
+
try {
|
|
705
|
+
this.#wasm(() => engine_validate_number_storage_changes(this.#engineDb, config.schema, before));
|
|
706
|
+
if (!config.upstream) {
|
|
707
|
+
this.#wasm(() => engine_init_number_storage_guards(this.#engineDb, config.schema));
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
catch (error) {
|
|
711
|
+
failed?.(error);
|
|
712
|
+
throw error;
|
|
713
|
+
}
|
|
714
|
+
return result;
|
|
715
|
+
}
|
|
716
|
+
#guardStandalone(sql, run) {
|
|
717
|
+
const mutation = config.upstream
|
|
718
|
+
? /\b(?:create|alter|drop|reindex|insert|update|delete|replace)\b/i.test(sql)
|
|
719
|
+
: /\b(?:create|alter|drop|reindex)\b/i.test(sql);
|
|
720
|
+
if (!mutation)
|
|
721
|
+
return run();
|
|
722
|
+
return this.ctx.storage.transactionSync(() => this.#guardSchema(sql, run));
|
|
723
|
+
}
|
|
653
724
|
#initialize() {
|
|
654
725
|
if (this.#initialized)
|
|
655
726
|
return;
|
|
@@ -670,7 +741,37 @@ export function createSyncDurableObject(config) {
|
|
|
670
741
|
config.schema,
|
|
671
742
|
config.initialize.toString(),
|
|
672
743
|
]);
|
|
673
|
-
|
|
744
|
+
const previousFingerprint = this.#controlGet('initFingerprint');
|
|
745
|
+
this.#initSkipped = previousFingerprint === initFingerprint;
|
|
746
|
+
const pendingBootstrap = this.#controlGet('tableBootstrap');
|
|
747
|
+
this.#snapshotRequired = pendingBootstrap !== null;
|
|
748
|
+
const storedTables = JSON.parse(pendingBootstrap ?? '[]');
|
|
749
|
+
const pendingTables = storedTables.filter((table) => Object.hasOwn(config.schema.tables, table));
|
|
750
|
+
this.#tableBootstrap = pendingTables.length > 0 ? pendingTables : null;
|
|
751
|
+
if (pendingTables.length !== storedTables.length) {
|
|
752
|
+
if (pendingTables.length > 0)
|
|
753
|
+
this.#controlSet('tableBootstrap', JSON.stringify(pendingTables));
|
|
754
|
+
else {
|
|
755
|
+
this.#controlDelete('tableBootstrap');
|
|
756
|
+
this.#snapshotRequired = false;
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
if (config.upstream && previousFingerprint && !this.#initSkipped) {
|
|
760
|
+
const previous = JSON.parse(previousFingerprint);
|
|
761
|
+
const added = Object.keys(config.schema.tables).filter((table) => !Object.hasOwn(previous[2].tables, table));
|
|
762
|
+
if (added.length > 0) {
|
|
763
|
+
// an existing full snapshot still needs its complete replacement.
|
|
764
|
+
// otherwise only the added tables need an initial source baseline.
|
|
765
|
+
if (this.#tableBootstrap ||
|
|
766
|
+
(!this.#snapshotRequired && !this.#snapshotProgress())) {
|
|
767
|
+
this.#tableBootstrap = [
|
|
768
|
+
...new Set([...(this.#tableBootstrap ?? []), ...added]),
|
|
769
|
+
].sort();
|
|
770
|
+
}
|
|
771
|
+
this.#snapshotRequired = true;
|
|
772
|
+
this.#controlSet('tableBootstrap', JSON.stringify(this.#tableBootstrap ?? []));
|
|
773
|
+
}
|
|
774
|
+
}
|
|
674
775
|
if (!this.#initSkipped) {
|
|
675
776
|
this.#directSql.exec(`CREATE TABLE IF NOT EXISTS orez_revocations (
|
|
676
777
|
user_id TEXT NOT NULL,
|
|
@@ -679,9 +780,19 @@ export function createSyncDurableObject(config) {
|
|
|
679
780
|
this.#directSql.exec('CREATE INDEX IF NOT EXISTS idx_orez_revocations_user_id ON orez_revocations(user_id, revoked_at)');
|
|
680
781
|
this.#directSql.exec('CREATE VIEW IF NOT EXISTS revocations AS SELECT user_id, revoked_at FROM orez_revocations');
|
|
681
782
|
this.#directSql.exec('CREATE TRIGGER IF NOT EXISTS trg_revocations_insert INSTEAD OF INSERT ON revocations BEGIN INSERT INTO orez_revocations(user_id, revoked_at) VALUES (NEW.user_id, NEW.revoked_at); END;');
|
|
783
|
+
const invalidBefore = config.upstream
|
|
784
|
+
? null
|
|
785
|
+
: this.#wasm(() => engine_invalid_number_cells(this.#engineDb, config.schema));
|
|
786
|
+
if (!config.upstream)
|
|
787
|
+
this.#wasm(() => engine_init_number_storage_guards(this.#engineDb, config.schema));
|
|
682
788
|
config.initialize(this.#directSql);
|
|
789
|
+
if (!config.upstream)
|
|
790
|
+
this.#wasm(() => engine_validate_number_storage_changes(this.#engineDb, config.schema, invalidBefore));
|
|
683
791
|
this.#directSql.exec("INSERT OR IGNORE INTO _zsync_host_control (key, value) VALUES ('writerEnabled', '1')");
|
|
684
792
|
this.#wasm(() => engine_init_schema(this.#engineDb, config.schema));
|
|
793
|
+
if (!config.upstream) {
|
|
794
|
+
this.#wasm(() => engine_init_number_storage_guards(this.#engineDb, config.schema));
|
|
795
|
+
}
|
|
685
796
|
this.#wasm(() => engine_init_query_schema(this.#engineDb));
|
|
686
797
|
this.#controlSet('initFingerprint', initFingerprint);
|
|
687
798
|
}
|
|
@@ -863,6 +974,7 @@ export function createSyncDurableObject(config) {
|
|
|
863
974
|
this.#activePullGroups.set(clientGroupID, count - 1);
|
|
864
975
|
}
|
|
865
976
|
async #fetchUpstreamJson(endpoint) {
|
|
977
|
+
this.#upstreamFetches++;
|
|
866
978
|
const result = await fetchBoundedUpstreamJson(this.#serviceBinding(), endpoint.toString(), { headers: { host: endpoint.host } }, {
|
|
867
979
|
timeoutMs: upstreamRequestTimeoutMs,
|
|
868
980
|
maxBytes: upstreamMaxResponseBytes,
|
|
@@ -1002,8 +1114,8 @@ export function createSyncDurableObject(config) {
|
|
|
1002
1114
|
this.#notePublishedTables([table]);
|
|
1003
1115
|
return page;
|
|
1004
1116
|
}
|
|
1005
|
-
async #beginSnapshotGeneration(path) {
|
|
1006
|
-
const table = Object.keys(config.schema.tables).sort()[0];
|
|
1117
|
+
async #beginSnapshotGeneration(path, selected) {
|
|
1118
|
+
const table = selected?.[0] ?? Object.keys(config.schema.tables).sort()[0];
|
|
1007
1119
|
if (!table)
|
|
1008
1120
|
throw requestError('paged snapshots require a modeled table', 500);
|
|
1009
1121
|
let pageLimit = DEFAULT_SNAPSHOT_PAGE_ROWS;
|
|
@@ -1021,13 +1133,58 @@ export function createSyncDurableObject(config) {
|
|
|
1021
1133
|
});
|
|
1022
1134
|
}
|
|
1023
1135
|
}
|
|
1136
|
+
// an intentionally private upstream table remains empty until it is
|
|
1137
|
+
// published. retain the bootstrap request and keep live ingest running.
|
|
1138
|
+
if (selected && page.unpublishedTables?.includes(table)) {
|
|
1139
|
+
return { progress: null, page, pageLimit };
|
|
1140
|
+
}
|
|
1024
1141
|
this.#resetSnapshotBillingWindow();
|
|
1025
1142
|
const progress = this.#withIngestBilling({
|
|
1026
1143
|
phase: 'snapshot_begin',
|
|
1027
1144
|
table,
|
|
1028
1145
|
startWatermark: page.watermark,
|
|
1029
1146
|
}, () => this.ctx.storage.transactionSync(() => this.#wasm(() => engine_begin_snapshot_generation(this.#engineDb, config.schema, String(page.watermark)))));
|
|
1030
|
-
return {
|
|
1147
|
+
return {
|
|
1148
|
+
progress,
|
|
1149
|
+
page: progress.table === table ? page : { ...page, rows: [], nextCursor: null },
|
|
1150
|
+
pageLimit,
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1153
|
+
// starts the first page of every other modeled table once the generation's
|
|
1154
|
+
// start watermark is known, so each prefetched page reads at or after it and
|
|
1155
|
+
// the catch-up pass still covers every later change.
|
|
1156
|
+
#prefetchSnapshotFirstPages(path, generation, firstTable) {
|
|
1157
|
+
if (this.#snapshotPrefetch?.generation === generation)
|
|
1158
|
+
return;
|
|
1159
|
+
const pages = new Map();
|
|
1160
|
+
for (const table of Object.keys(config.schema.tables)) {
|
|
1161
|
+
if (table === firstTable)
|
|
1162
|
+
continue;
|
|
1163
|
+
pages.set(table, this.#fetchSnapshotPage(path, table, null, SNAPSHOT_PREFETCH_PAGE_ROWS).catch(() => null));
|
|
1164
|
+
}
|
|
1165
|
+
this.#snapshotPrefetch = { generation, pages };
|
|
1166
|
+
}
|
|
1167
|
+
async #takePrefetchedSnapshotPage(generation, table) {
|
|
1168
|
+
const prefetch = this.#snapshotPrefetch;
|
|
1169
|
+
if (prefetch?.generation !== generation)
|
|
1170
|
+
return null;
|
|
1171
|
+
const page = prefetch.pages.get(table) ?? null;
|
|
1172
|
+
prefetch.pages.delete(table);
|
|
1173
|
+
return page ? await page : null;
|
|
1174
|
+
}
|
|
1175
|
+
#publishTableBootstrap(tables, progress) {
|
|
1176
|
+
this.#resetSnapshotBillingWindow();
|
|
1177
|
+
const result = this.#withIngestBilling({ phase: 'table_bootstrap_publish', generation: progress.generation }, () => this.ctx.storage.transactionSync(() => {
|
|
1178
|
+
const published = this.#wasm(() => engine_finalize_table_bootstrap(this.#engineDb, config.schema, tables, progress.generation, progress.catchupWatermark));
|
|
1179
|
+
if (published.caughtUp)
|
|
1180
|
+
this.#directSql.exec("DELETE FROM _zsync_host_control WHERE key = 'tableBootstrap'");
|
|
1181
|
+
return published;
|
|
1182
|
+
}));
|
|
1183
|
+
if (result.caughtUp) {
|
|
1184
|
+
this.#tableBootstrap = null;
|
|
1185
|
+
this.#snapshotRequired = false;
|
|
1186
|
+
}
|
|
1187
|
+
return result;
|
|
1031
1188
|
}
|
|
1032
1189
|
#ingest(upstreamPath, forceSnapshot = false) {
|
|
1033
1190
|
if (!config.upstream) {
|
|
@@ -1047,7 +1204,19 @@ export function createSyncDurableObject(config) {
|
|
|
1047
1204
|
: Promise.resolve(0);
|
|
1048
1205
|
}
|
|
1049
1206
|
this.#ingestPromise = (async () => {
|
|
1207
|
+
if (!this.#droppedFeedColumnsChecked) {
|
|
1208
|
+
this.#droppedFeedColumnsChecked = true;
|
|
1209
|
+
// rows written while this replica ran a schema version behind its
|
|
1210
|
+
// feed hold no value for the columns it has since started modeling;
|
|
1211
|
+
// only a fresh snapshot recovers them.
|
|
1212
|
+
if (this.#wasm(() => engine_dropped_feed_columns_need_snapshot(this.#engineDb, config.schema))) {
|
|
1213
|
+
forceSnapshot = true;
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1050
1216
|
let progress = this.#snapshotProgress();
|
|
1217
|
+
const bootstrap = forceSnapshot ? undefined : (this.#tableBootstrap ?? undefined);
|
|
1218
|
+
if (this.#snapshotRequired && !progress)
|
|
1219
|
+
forceSnapshot = true;
|
|
1051
1220
|
this.#ingestBreaker.assertReady();
|
|
1052
1221
|
const startingWatermark = this.#engineState().watermark;
|
|
1053
1222
|
let total = 0;
|
|
@@ -1057,6 +1226,17 @@ export function createSyncDurableObject(config) {
|
|
|
1057
1226
|
let oldestLiveCommitTimeMs;
|
|
1058
1227
|
const changedTables = new Set();
|
|
1059
1228
|
for (;;) {
|
|
1229
|
+
if (forceSnapshot) {
|
|
1230
|
+
if (this.#snapshotRequired && !bootstrap) {
|
|
1231
|
+
this.#tableBootstrap = null;
|
|
1232
|
+
this.#controlSet('tableBootstrap', '[]');
|
|
1233
|
+
}
|
|
1234
|
+
forceSnapshot = false;
|
|
1235
|
+
const begun = await this.#beginSnapshotGeneration(path, bootstrap);
|
|
1236
|
+
progress = begun.progress;
|
|
1237
|
+
pendingPage = begun.page;
|
|
1238
|
+
snapshotPageLimit = begun.pageLimit;
|
|
1239
|
+
}
|
|
1060
1240
|
if (progress?.state === 'paging') {
|
|
1061
1241
|
const activeProgress = progress;
|
|
1062
1242
|
const table = activeProgress.table;
|
|
@@ -1065,7 +1245,25 @@ export function createSyncDurableObject(config) {
|
|
|
1065
1245
|
}
|
|
1066
1246
|
let page = pendingPage;
|
|
1067
1247
|
try {
|
|
1068
|
-
|
|
1248
|
+
if (!bootstrap && snapshotPageLimit === DEFAULT_SNAPSHOT_PAGE_ROWS) {
|
|
1249
|
+
this.#prefetchSnapshotFirstPages(path, activeProgress.generation, table);
|
|
1250
|
+
if (activeProgress.cursor === null) {
|
|
1251
|
+
page ??= await this.#takePrefetchedSnapshotPage(activeProgress.generation, table);
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
page ??=
|
|
1255
|
+
bootstrap && !bootstrap.includes(table)
|
|
1256
|
+
? {
|
|
1257
|
+
watermark: Number(activeProgress.startWatermark),
|
|
1258
|
+
rows: [],
|
|
1259
|
+
nextCursor: null,
|
|
1260
|
+
}
|
|
1261
|
+
: await this.#fetchSnapshotPage(path, table, activeProgress.cursor, snapshotPageLimit);
|
|
1262
|
+
if (bootstrap && page.unpublishedTables?.includes(table)) {
|
|
1263
|
+
progress = null;
|
|
1264
|
+
pendingPage = null;
|
|
1265
|
+
continue;
|
|
1266
|
+
}
|
|
1069
1267
|
const pageToApply = page;
|
|
1070
1268
|
this.#resetSnapshotBillingWindow();
|
|
1071
1269
|
const nextProgress = this.#withIngestBilling({
|
|
@@ -1075,7 +1273,7 @@ export function createSyncDurableObject(config) {
|
|
|
1075
1273
|
cursor: activeProgress.cursor,
|
|
1076
1274
|
pageRows: pageToApply.rows.length,
|
|
1077
1275
|
pageLimit: snapshotPageLimit,
|
|
1078
|
-
}, () => this.ctx.storage.transactionSync(() => this.#wasm(() => engine_apply_snapshot_page(this.#engineDb, config.schema, activeProgress.generation, table, pageToApply.rows, pageToApply.nextCursor))));
|
|
1276
|
+
}, () => this.ctx.storage.transactionSync(() => this.#wasm(() => engine_apply_snapshot_page(this.#engineDb, config.schema, activeProgress.generation, table, pageToApply.rows, pageToApply.nextCursor, pageToApply.optionalColumns))));
|
|
1079
1277
|
total += pageToApply.rows.length;
|
|
1080
1278
|
this.#recordIngestLogicalRows(pageToApply.rows.length);
|
|
1081
1279
|
changedTables.add(table);
|
|
@@ -1095,13 +1293,27 @@ export function createSyncDurableObject(config) {
|
|
|
1095
1293
|
}
|
|
1096
1294
|
if (progress?.state === 'catching_up') {
|
|
1097
1295
|
const activeProgress = progress;
|
|
1098
|
-
|
|
1296
|
+
if (bootstrap && activeProgress.cursor !== null) {
|
|
1297
|
+
const result = this.#publishTableBootstrap(bootstrap, activeProgress);
|
|
1298
|
+
total += result.applied;
|
|
1299
|
+
this.#recordIngestLogicalRows(result.applied);
|
|
1300
|
+
progress = this.#snapshotProgress();
|
|
1301
|
+
continue;
|
|
1302
|
+
}
|
|
1303
|
+
const liveCursor = bootstrap
|
|
1304
|
+
? this.#engineState().upstreamWatermark
|
|
1305
|
+
: activeProgress.catchupWatermark;
|
|
1306
|
+
const cursor = BigInt(activeProgress.catchupWatermark) < BigInt(liveCursor)
|
|
1307
|
+
? activeProgress.catchupWatermark
|
|
1308
|
+
: liveCursor;
|
|
1099
1309
|
const endpoint = new URL(`${path}/changes`, 'https://upstream.invalid');
|
|
1100
1310
|
endpoint.searchParams.set('since', cursor);
|
|
1101
1311
|
endpoint.searchParams.set('limit', String(upstreamLimit));
|
|
1102
1312
|
const { response, body } = await this.#fetchUpstreamJson(endpoint);
|
|
1103
1313
|
if (response.status === 410) {
|
|
1104
|
-
|
|
1314
|
+
if (bootstrap)
|
|
1315
|
+
throw requestError('table bootstrap requires a retained live feed cursor', 409);
|
|
1316
|
+
const begun = await this.#beginSnapshotGeneration(path, bootstrap);
|
|
1105
1317
|
progress = begun.progress;
|
|
1106
1318
|
pendingPage = begun.page;
|
|
1107
1319
|
snapshotPageLimit = begun.pageLimit;
|
|
@@ -1136,18 +1348,44 @@ export function createSyncDurableObject(config) {
|
|
|
1136
1348
|
cursor,
|
|
1137
1349
|
batchWatermark: batch.watermark,
|
|
1138
1350
|
changeRows: batch.changes.length,
|
|
1139
|
-
}, () => this.ctx.storage.transactionSync(() =>
|
|
1351
|
+
}, () => this.ctx.storage.transactionSync(() => {
|
|
1352
|
+
const live = bootstrap
|
|
1353
|
+
? this.#wasm(() => engine_apply_upstream(this.#engineDb, config.schema, batch))
|
|
1354
|
+
: null;
|
|
1355
|
+
const staged = this.#wasm(() => engine_apply_snapshot_changes(this.#engineDb, config.schema, activeProgress.generation, batch));
|
|
1356
|
+
return {
|
|
1357
|
+
...staged,
|
|
1358
|
+
applied: staged.applied + (live?.applied ?? 0),
|
|
1359
|
+
caughtUp: staged.caughtUp && (live?.caughtUp ?? true),
|
|
1360
|
+
};
|
|
1361
|
+
}));
|
|
1140
1362
|
total += result.applied;
|
|
1141
1363
|
this.#recordIngestLogicalRows(result.applied);
|
|
1142
1364
|
if (result.caughtUp) {
|
|
1365
|
+
if (bootstrap) {
|
|
1366
|
+
const published = this.#publishTableBootstrap(bootstrap, {
|
|
1367
|
+
...activeProgress,
|
|
1368
|
+
catchupWatermark: String(result.watermark),
|
|
1369
|
+
});
|
|
1370
|
+
total += published.applied;
|
|
1371
|
+
this.#recordIngestLogicalRows(published.applied);
|
|
1372
|
+
progress = this.#snapshotProgress();
|
|
1373
|
+
continue;
|
|
1374
|
+
}
|
|
1143
1375
|
this.#resetSnapshotBillingWindow();
|
|
1144
1376
|
this.#withIngestBilling({
|
|
1145
1377
|
phase: 'snapshot_finalize',
|
|
1146
1378
|
generation: activeProgress.generation,
|
|
1147
1379
|
watermark: result.watermark,
|
|
1148
|
-
}, () => this.ctx.storage.transactionSync(() =>
|
|
1380
|
+
}, () => this.ctx.storage.transactionSync(() => {
|
|
1381
|
+
this.#wasm(() => engine_finalize_snapshot_generation(this.#engineDb, config.schema, activeProgress.generation, String(result.watermark)));
|
|
1382
|
+
if (this.#snapshotRequired)
|
|
1383
|
+
this.#directSql.exec("DELETE FROM _zsync_host_control WHERE key = 'tableBootstrap'");
|
|
1384
|
+
}));
|
|
1385
|
+
this.#tableBootstrap = null;
|
|
1386
|
+
this.#snapshotRequired = false;
|
|
1149
1387
|
progress = null;
|
|
1150
|
-
snapshotCompleted =
|
|
1388
|
+
snapshotCompleted = !bootstrap;
|
|
1151
1389
|
break;
|
|
1152
1390
|
}
|
|
1153
1391
|
if (String(result.watermark) === cursor) {
|
|
@@ -1167,19 +1405,13 @@ export function createSyncDurableObject(config) {
|
|
|
1167
1405
|
continue;
|
|
1168
1406
|
}
|
|
1169
1407
|
const cursor = this.#engineState().upstreamWatermark;
|
|
1170
|
-
if (forceSnapshot) {
|
|
1171
|
-
forceSnapshot = false;
|
|
1172
|
-
const begun = await this.#beginSnapshotGeneration(path);
|
|
1173
|
-
progress = begun.progress;
|
|
1174
|
-
pendingPage = begun.page;
|
|
1175
|
-
snapshotPageLimit = begun.pageLimit;
|
|
1176
|
-
continue;
|
|
1177
|
-
}
|
|
1178
1408
|
const endpoint = new URL(`${path}/changes`, 'https://upstream.invalid');
|
|
1179
1409
|
endpoint.searchParams.set('watermark', cursor);
|
|
1180
1410
|
endpoint.searchParams.set('limit', String(upstreamLimit));
|
|
1181
1411
|
const { response, body } = await this.#fetchUpstreamJson(endpoint);
|
|
1182
1412
|
if (response.status === 410) {
|
|
1413
|
+
if (bootstrap)
|
|
1414
|
+
throw requestError('table bootstrap requires a retained live feed cursor', 409);
|
|
1183
1415
|
const begun = await this.#beginSnapshotGeneration(path);
|
|
1184
1416
|
progress = begun.progress;
|
|
1185
1417
|
pendingPage = begun.page;
|
|
@@ -1271,6 +1503,7 @@ export function createSyncDurableObject(config) {
|
|
|
1271
1503
|
}
|
|
1272
1504
|
return total;
|
|
1273
1505
|
})().finally(() => {
|
|
1506
|
+
this.#snapshotPrefetch = null;
|
|
1274
1507
|
this.#ingestPromise = null;
|
|
1275
1508
|
});
|
|
1276
1509
|
return this.#ingestPromise;
|
|
@@ -1360,6 +1593,18 @@ export function createSyncDurableObject(config) {
|
|
|
1360
1593
|
return null;
|
|
1361
1594
|
}
|
|
1362
1595
|
}
|
|
1596
|
+
#pushWakeTargets(since, clientGroupID) {
|
|
1597
|
+
const reached = since === null ? null : this.#wakeTargets(since);
|
|
1598
|
+
if (reached === null || typeof clientGroupID !== 'string')
|
|
1599
|
+
return reached;
|
|
1600
|
+
const targets = new Set(reached);
|
|
1601
|
+
for (const socket of this.ctx.getWebSockets(`group:${clientGroupID}`)) {
|
|
1602
|
+
const attachment = socketAttachment(socket);
|
|
1603
|
+
if (attachment)
|
|
1604
|
+
targets.add(attachment.clientID);
|
|
1605
|
+
}
|
|
1606
|
+
return targets;
|
|
1607
|
+
}
|
|
1363
1608
|
#enqueueWake(originClientID, tables = [], targets = null) {
|
|
1364
1609
|
this.#wakeOrigins.add(originClientID);
|
|
1365
1610
|
for (const table of tables)
|
|
@@ -1506,7 +1751,7 @@ export function createSyncDurableObject(config) {
|
|
|
1506
1751
|
wasmBoundaryCalls: this.#counters.wasmBoundaryCalls,
|
|
1507
1752
|
sql: this.#engineDb.stats,
|
|
1508
1753
|
});
|
|
1509
|
-
const result = json(response);
|
|
1754
|
+
const result = withServerTiming(json(response), [['do_tx', transactionMs]]);
|
|
1510
1755
|
const clientGroupID = typeof body.clientGroupID === 'string' ? body.clientGroupID : '';
|
|
1511
1756
|
this.#servingLag.onVersionServed(clientGroupID, response.cookie);
|
|
1512
1757
|
return result;
|
|
@@ -1650,6 +1895,9 @@ export function createSyncDurableObject(config) {
|
|
|
1650
1895
|
// cannot restore the one-shot control flag.
|
|
1651
1896
|
this.#executorBeforeCommitFault = this.#takeFault('push_after_write_before_commit');
|
|
1652
1897
|
const txStarted = performance.now();
|
|
1898
|
+
// every change after this watermark is this push's, or a concurrent
|
|
1899
|
+
// commit's; targeting from it wakes a superset of the reached clients.
|
|
1900
|
+
const pushWatermark = this.#engineStateBestEffort()?.watermark ?? null;
|
|
1653
1901
|
let result;
|
|
1654
1902
|
try {
|
|
1655
1903
|
result = await this.#executor.push(body, claims);
|
|
@@ -1659,6 +1907,7 @@ export function createSyncDurableObject(config) {
|
|
|
1659
1907
|
}
|
|
1660
1908
|
transactionMs += performance.now() - txStarted;
|
|
1661
1909
|
const mutationResults = 'mutations' in result.pushResponse ? result.pushResponse.mutations : [];
|
|
1910
|
+
const wakeOrigins = [];
|
|
1662
1911
|
for (const mutation of mutationResults) {
|
|
1663
1912
|
if ('error' in mutation.result &&
|
|
1664
1913
|
mutation.result.error === 'alreadyProcessed') {
|
|
@@ -1669,7 +1918,16 @@ export function createSyncDurableObject(config) {
|
|
|
1669
1918
|
this.#counters.applicationErrors++;
|
|
1670
1919
|
resultClass = 'application_error';
|
|
1671
1920
|
}
|
|
1672
|
-
|
|
1921
|
+
wakeOrigins.push(mutation.id.clientID);
|
|
1922
|
+
}
|
|
1923
|
+
if (wakeOrigins.length > 0) {
|
|
1924
|
+
// wake the clients the committed rows reach, and the pusher's own
|
|
1925
|
+
// group, whose next pull carries the advanced lmid even when the
|
|
1926
|
+
// mutation changed no row.
|
|
1927
|
+
const targets = this.#pushWakeTargets(pushWatermark, body.clientGroupID);
|
|
1928
|
+
for (const origin of wakeOrigins) {
|
|
1929
|
+
this.ctx.waitUntil(this.#enqueueWake(origin, [], targets));
|
|
1930
|
+
}
|
|
1673
1931
|
}
|
|
1674
1932
|
if (mutationResults.length > 0) {
|
|
1675
1933
|
const txStarted = performance.now();
|
|
@@ -1857,7 +2115,11 @@ export function createSyncDurableObject(config) {
|
|
|
1857
2115
|
clientID,
|
|
1858
2116
|
identity,
|
|
1859
2117
|
});
|
|
1860
|
-
this.ctx.acceptWebSocket(server, [
|
|
2118
|
+
this.ctx.acceptWebSocket(server, [
|
|
2119
|
+
WAKE_SUBSCRIBER_TAG,
|
|
2120
|
+
`client:${clientID}`,
|
|
2121
|
+
...(clientGroupID ? [`group:${clientGroupID}`] : []),
|
|
2122
|
+
]);
|
|
1861
2123
|
if (isRealtimeIdentity(identity)) {
|
|
1862
2124
|
const host = this.#realtimeHost();
|
|
1863
2125
|
if (host) {
|
|
@@ -1954,7 +2216,7 @@ export function createSyncDurableObject(config) {
|
|
|
1954
2216
|
return json({ error: 'query is required' }, 400);
|
|
1955
2217
|
try {
|
|
1956
2218
|
return json({
|
|
1957
|
-
rows: this.#directSql.query(query, decodeSqlParams(params)),
|
|
2219
|
+
rows: this.#guardStandalone(query, () => this.#directSql.query(query, decodeSqlParams(params))),
|
|
1958
2220
|
});
|
|
1959
2221
|
}
|
|
1960
2222
|
catch (error) {
|
|
@@ -2093,6 +2355,14 @@ export function createSyncDurableObject(config) {
|
|
|
2093
2355
|
return this.#realtimeProduce(request, this.env);
|
|
2094
2356
|
}
|
|
2095
2357
|
if (route === '/notify' && request.method === 'POST') {
|
|
2358
|
+
// a commit notification arrives at the depth of the request that wrote,
|
|
2359
|
+
// and ingesting inline would add this object's upstream read to that
|
|
2360
|
+
// chain until the runtime refuses it. the alarm ingests from depth zero.
|
|
2361
|
+
if (request.headers.get(NOTIFY_IF_SUBSCRIBED_HEADER) === '1') {
|
|
2362
|
+
this.#notifyPending = true;
|
|
2363
|
+
await this.ctx.storage.setAlarm(Date.now());
|
|
2364
|
+
return json({ ok: true, scheduled: true });
|
|
2365
|
+
}
|
|
2096
2366
|
try {
|
|
2097
2367
|
const applied = await this.#ingestAfterCurrent(upstreamPath);
|
|
2098
2368
|
return json({ ok: true, applied });
|
|
@@ -2116,16 +2386,25 @@ export function createSyncDurableObject(config) {
|
|
|
2116
2386
|
this.#beginActivePull(clientGroupID);
|
|
2117
2387
|
try {
|
|
2118
2388
|
// pull and push both establish the upstream schema and snapshot barrier.
|
|
2389
|
+
const ingestStarted = performance.now();
|
|
2390
|
+
const fetchesBefore = this.#upstreamFetches;
|
|
2119
2391
|
try {
|
|
2120
2392
|
await this.#ingest(upstreamPath);
|
|
2121
2393
|
}
|
|
2122
2394
|
catch (error) {
|
|
2123
2395
|
return errorResponse(error);
|
|
2124
2396
|
}
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2397
|
+
const ingestMs = performance.now() - ingestStarted;
|
|
2398
|
+
const upstreamFetches = this.#upstreamFetches - fetchesBefore;
|
|
2399
|
+
const handlerStarted = performance.now();
|
|
2400
|
+
const response = route === '/pull'
|
|
2401
|
+
? await this.#pull(forwarded.request, forwarded.claims, namespace)
|
|
2402
|
+
: await this.#push(forwarded.request, forwarded.claims, namespace, upstreamPath);
|
|
2403
|
+
return withServerTiming(response, [
|
|
2404
|
+
['do_ingest', ingestMs],
|
|
2405
|
+
['do_upstream_fetches', upstreamFetches],
|
|
2406
|
+
['do_handle', performance.now() - handlerStarted],
|
|
2407
|
+
]);
|
|
2129
2408
|
}
|
|
2130
2409
|
finally {
|
|
2131
2410
|
if (clientGroupID)
|
|
@@ -2138,8 +2417,9 @@ export function createSyncDurableObject(config) {
|
|
|
2138
2417
|
if (this.ctx.getWebSockets(WAKE_SUBSCRIBER_TAG).length === 0)
|
|
2139
2418
|
return;
|
|
2140
2419
|
this.#initialize();
|
|
2420
|
+
this.#notifyPending = false;
|
|
2141
2421
|
try {
|
|
2142
|
-
await this.#
|
|
2422
|
+
await this.#ingestAfterCurrent(null);
|
|
2143
2423
|
}
|
|
2144
2424
|
catch (error) {
|
|
2145
2425
|
console.error(JSON.stringify({
|
|
@@ -2151,7 +2431,12 @@ export function createSyncDurableObject(config) {
|
|
|
2151
2431
|
finally {
|
|
2152
2432
|
if (this.ctx.getWebSockets(WAKE_SUBSCRIBER_TAG).length > 0) {
|
|
2153
2433
|
const retryAfterMs = this.#ingestBreaker.status().retryAfterMs;
|
|
2154
|
-
|
|
2434
|
+
// a notification during this ingest may name a commit it did not
|
|
2435
|
+
// read, and this re-arm replaces the alarm that notification set
|
|
2436
|
+
await this.ctx.storage.setAlarm(Date.now() +
|
|
2437
|
+
(this.#notifyPending
|
|
2438
|
+
? retryAfterMs
|
|
2439
|
+
: Math.max(upstreamIntervalMs, retryAfterMs)));
|
|
2155
2440
|
}
|
|
2156
2441
|
}
|
|
2157
2442
|
}
|
|
@@ -63,6 +63,7 @@ export declare class TransactionQueryBudgetError extends Error {
|
|
|
63
63
|
type MaterializedRow = Record<string, unknown>;
|
|
64
64
|
type ExecuteSelect = (sql: string, params: readonly unknown[]) => readonly MaterializedRow[];
|
|
65
65
|
type ExecuteSelectAsync = (sql: string, params: readonly unknown[]) => readonly MaterializedRow[] | Promise<readonly MaterializedRow[]>;
|
|
66
|
+
export declare function numberFromStorageText(value: string): number | undefined;
|
|
66
67
|
export declare function executeTransactionQueryPlan<Result = unknown>(plan: CompiledTransactionQueryPlan, execute: ExecuteSelect, options?: TransactionQueryExecutionOptions): Result;
|
|
67
68
|
export declare function executeTransactionQueryPlanAsync<Result = unknown>(plan: CompiledTransactionQueryPlan, execute: ExecuteSelectAsync, options?: TransactionQueryExecutionOptions): Promise<Result>;
|
|
68
69
|
export {};
|
|
@@ -178,7 +178,7 @@ function bytes(value) {
|
|
|
178
178
|
}
|
|
179
179
|
const FINITE_NUMBER_TEXT = /^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/;
|
|
180
180
|
const TIMESTAMP_TEXT = /^(\d{4})-(\d{2})-(\d{2})[ T](\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(?:Z|([+-])(\d{2})(?::?(\d{2}))?)?$/;
|
|
181
|
-
function numberFromStorageText(value) {
|
|
181
|
+
export function numberFromStorageText(value) {
|
|
182
182
|
if (FINITE_NUMBER_TEXT.test(value)) {
|
|
183
183
|
const numeric = Number(value);
|
|
184
184
|
if (Number.isFinite(numeric))
|
package/dist/wasm.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export declare function withWasm<Args extends unknown[], Result>(operation: (...args: Args) => Result): (...args: Args) => Result;
|
|
2
2
|
export declare const engine_apply_snapshot_changes: (db: any, schema: any, generation: string, batch: any) => any;
|
|
3
|
-
export declare const engine_apply_snapshot_page: (db: any, schema: any, generation: string, table: string, rows: any, next_cursor
|
|
3
|
+
export declare const engine_apply_snapshot_page: (db: any, schema: any, generation: string, table: string, rows: any, next_cursor: string | null | undefined, optional_columns: any) => any;
|
|
4
|
+
export declare const engine_dropped_feed_columns_need_snapshot: (db: any, schema: any) => boolean;
|
|
4
5
|
export declare const engine_apply_upstream: (db: any, schema: any, batch: any) => any;
|
|
5
6
|
export declare const engine_assemble_push_response: (results: any) => any;
|
|
6
7
|
export declare const engine_authorize_realtime_subscription: (db: any, schema: any, client_group_id: string, user_id: string, table: string, pk: any) => any;
|
|
@@ -8,9 +9,13 @@ export declare const engine_begin_snapshot_generation: (db: any, schema: any, st
|
|
|
8
9
|
export declare const engine_compile_query: (schema: any, ast: any, format: any) => any;
|
|
9
10
|
export declare const engine_finalize: (db: any, client_group_id: string, client_id: string, mutation_id: string) => void;
|
|
10
11
|
export declare const engine_finalize_snapshot_generation: (db: any, schema: any, generation: string, watermark: string) => any;
|
|
12
|
+
export declare const engine_finalize_table_bootstrap: (db: any, schema: any, selected: any, generation: string, watermark: string) => any;
|
|
11
13
|
export declare const engine_handle_query_pull: (db: any, schema: any, retain_changes: string, body: any, user_id: string) => any;
|
|
12
14
|
export declare const engine_init_query_schema: (db: any) => void;
|
|
13
15
|
export declare const engine_init_schema: (db: any, schema: any) => void;
|
|
16
|
+
export declare const engine_init_number_storage_guards: (db: any, schema: any) => void;
|
|
17
|
+
export declare const engine_invalid_number_cells: (db: any, schema: any) => any;
|
|
18
|
+
export declare const engine_validate_number_storage_changes: (db: any, schema: any, before: any) => void;
|
|
14
19
|
export declare const engine_invalidate: (db: any) => void;
|
|
15
20
|
export declare const engine_memory_bytes: () => number;
|
|
16
21
|
export declare const engine_preflight: (db: any, client_group_id: string, client_id: string, mutation_id: string, user_id: string) => any;
|
package/dist/wasm.js
CHANGED
|
@@ -18,6 +18,7 @@ export function withWasm(operation) {
|
|
|
18
18
|
}
|
|
19
19
|
export const engine_apply_snapshot_changes = withWasm(generated.engine_apply_snapshot_changes);
|
|
20
20
|
export const engine_apply_snapshot_page = withWasm(generated.engine_apply_snapshot_page);
|
|
21
|
+
export const engine_dropped_feed_columns_need_snapshot = withWasm(generated.engine_dropped_feed_columns_need_snapshot);
|
|
21
22
|
export const engine_apply_upstream = withWasm(generated.engine_apply_upstream);
|
|
22
23
|
export const engine_assemble_push_response = withWasm(generated.engine_assemble_push_response);
|
|
23
24
|
export const engine_authorize_realtime_subscription = withWasm(generated.engine_authorize_realtime_subscription);
|
|
@@ -25,9 +26,13 @@ export const engine_begin_snapshot_generation = withWasm(generated.engine_begin_
|
|
|
25
26
|
export const engine_compile_query = withWasm(generated.engine_compile_query);
|
|
26
27
|
export const engine_finalize = withWasm(generated.engine_finalize);
|
|
27
28
|
export const engine_finalize_snapshot_generation = withWasm(generated.engine_finalize_snapshot_generation);
|
|
29
|
+
export const engine_finalize_table_bootstrap = withWasm(generated.engine_finalize_table_bootstrap);
|
|
28
30
|
export const engine_handle_query_pull = withWasm(generated.engine_handle_query_pull);
|
|
29
31
|
export const engine_init_query_schema = withWasm(generated.engine_init_query_schema);
|
|
30
32
|
export const engine_init_schema = withWasm(generated.engine_init_schema);
|
|
33
|
+
export const engine_init_number_storage_guards = withWasm(generated.engine_init_number_storage_guards);
|
|
34
|
+
export const engine_invalid_number_cells = withWasm(generated.engine_invalid_number_cells);
|
|
35
|
+
export const engine_validate_number_storage_changes = withWasm(generated.engine_validate_number_storage_changes);
|
|
31
36
|
export const engine_invalidate = withWasm(generated.engine_invalidate);
|
|
32
37
|
export const engine_memory_bytes = withWasm(generated.engine_memory_bytes);
|
|
33
38
|
export const engine_preflight = withWasm(generated.engine_preflight);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "orez-sync-cf-host",
|
|
3
|
-
"version": "0.16.
|
|
3
|
+
"version": "0.16.21",
|
|
4
4
|
"description": "Internal orez Cloudflare sync host. Do not import directly: consumers use orez-lite/cloudflare/sync.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -47,20 +47,21 @@
|
|
|
47
47
|
"build:dist:platform": "bun run build:dependencies && bun run build:wasm:platform && bun run build:runtime",
|
|
48
48
|
"bundle": "bun run build:dist && wrangler deploy --dry-run --outdir .wrangler/bundle",
|
|
49
49
|
"measure": "bun run build:dist && bun measure.mjs",
|
|
50
|
-
"test": "bun run test:config && bun run test:query-compiler && bun run test:package && bun run test:platform && bun run test:integration && bun run test:large-data && bun run test:ingest && bun run test:restart",
|
|
50
|
+
"test": "bun run test:config && bun run test:query-compiler && bun run test:package && bun run test:platform && bun run test:integration && bun run test:large-data && bun run test:ingest && bun run test:rolling-schema && bun run test:restart",
|
|
51
51
|
"test:config": "bun test config.test.mjs query-patch.test.mjs write-safeguards.test.mjs transaction-query.test.mjs upstream-response.test.mjs serving-lag.test.mjs",
|
|
52
52
|
"test:integration": "bun run build:dist && bun integration-test.mjs",
|
|
53
53
|
"test:ingest": "bun run build:dist && bun ingest-test.mjs",
|
|
54
54
|
"test:package": "bun run build:dist && node package-runtime-test.mjs",
|
|
55
55
|
"test:query-compiler": "bun run build:dist && bun test --preload orez-sync-cf-host/bun-wasm-loader query-compiler.test.mjs && node vite-ssr-query-compiler-test.mjs",
|
|
56
56
|
"test:restart": "bun run build:dist && bun restart-test.mjs",
|
|
57
|
+
"test:rolling-schema": "bun run build:dist && bun rolling-schema-test.mjs",
|
|
57
58
|
"test:platform": "bun run build:dist:platform && bun platform-test.mjs",
|
|
58
59
|
"typecheck": "bun run build:dist:platform && tsc --noEmit",
|
|
59
60
|
"test:large-data": "bun run build:dist && bun large-data-test.mjs"
|
|
60
61
|
},
|
|
61
62
|
"dependencies": {
|
|
62
63
|
"@opentelemetry/api": "^1.9.0",
|
|
63
|
-
"orez-sync-executor": "0.16.
|
|
64
|
+
"orez-sync-executor": "0.16.21"
|
|
64
65
|
},
|
|
65
66
|
"devDependencies": {
|
|
66
67
|
"@cloudflare/workers-types": "4.20260617.1",
|
|
@@ -88,5 +89,5 @@
|
|
|
88
89
|
"url": "git+https://github.com/natew/orez.git",
|
|
89
90
|
"directory": "packages/sync-cf-host"
|
|
90
91
|
},
|
|
91
|
-
"releaseSourceCommit": "
|
|
92
|
+
"releaseSourceCommit": "692d233ff502256274f2ff23036a5d17c0936b5a"
|
|
92
93
|
}
|