orez-sync-cf-host 0.5.18 → 0.5.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -7
- package/package.json +15 -4
- package/src/bun-wasm-loader.ts +16 -0
- package/src/config.ts +9 -0
- package/src/generated/sync_wasm.d.ts +2 -2
- package/src/generated/sync_wasm.js +3 -2
- package/src/generated/sync_wasm_bg.wasm +0 -0
- package/src/host.ts +53 -43
- package/src/index.ts +26 -1
- package/src/ingest-harness-worker.ts +97 -2
- package/src/mutation-error.ts +28 -0
- package/src/platform-probe-worker.ts +122 -9
- package/src/post-commit.ts +40 -0
- package/src/query-compiler.ts +16 -0
- package/src/sql-storage-adapter.ts +88 -19
- package/src/transaction-query.ts +494 -0
- package/src/types.ts +31 -17
- package/src/wasm-platform.ts +10 -0
- package/src/wasm.ts +34 -0
package/README.md
CHANGED
|
@@ -17,6 +17,29 @@ Every SQL cursor is materialized before an await. Mutators may await only their
|
|
|
17
17
|
`context.defer`, which runs only after commit. Application failures use the
|
|
18
18
|
required second transaction to advance the LMID marker.
|
|
19
19
|
|
|
20
|
+
The root `orez/cf-do` executor and this host consume the same `post-commit`
|
|
21
|
+
module, so transaction retries discard effects from abandoned attempts in both
|
|
22
|
+
paths.
|
|
23
|
+
|
|
24
|
+
Transaction-query `ILIKE` folding is ASCII-only on Durable Object SQLite;
|
|
25
|
+
non-ASCII case pairs can diverge from PostgreSQL.
|
|
26
|
+
|
|
27
|
+
### Bun query compiler
|
|
28
|
+
|
|
29
|
+
Wrangler loads the engine's `.wasm` import as a precompiled
|
|
30
|
+
`WebAssembly.Module`. Bun consumers of `createQueryCompiler` must register the
|
|
31
|
+
package loader so the same import has the same value without changing the
|
|
32
|
+
compiler code:
|
|
33
|
+
|
|
34
|
+
```toml
|
|
35
|
+
# bunfig.toml
|
|
36
|
+
preload = ["orez-sync-cf-host/bun-wasm-loader"]
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
For a single command, use
|
|
40
|
+
`bun --preload orez-sync-cf-host/bun-wasm-loader <command>`. The compiler throws
|
|
41
|
+
an error naming this preload when Bun resolves the `.wasm` import as a pathname.
|
|
42
|
+
|
|
20
43
|
## Wake channel and eviction
|
|
21
44
|
|
|
22
45
|
`GET /<namespace>/wake?clientID=<id>&wakeToken=<capability>` upgrades to a
|
|
@@ -50,6 +73,8 @@ every socket attempt, including reconnects, so short-lived tokens are never
|
|
|
50
73
|
reused after the wake connection drops:
|
|
51
74
|
|
|
52
75
|
```ts
|
|
76
|
+
import { ensureHttpPullTransport } from 'orez/zero-http'
|
|
77
|
+
|
|
53
78
|
ensureHttpPullTransport({
|
|
54
79
|
origin: syncOrigin,
|
|
55
80
|
pullIntervalMs: 5_000,
|
|
@@ -105,6 +130,24 @@ service or operator capability. Both `authorizeWake` and `authorizeNotify` run
|
|
|
105
130
|
before `idFromName`, so rejected requests cannot instantiate namespace Durable
|
|
106
131
|
Objects.
|
|
107
132
|
|
|
133
|
+
### Consumer routing traps
|
|
134
|
+
|
|
135
|
+
If an outer application router has its own namespace gate, let only `/wake` and
|
|
136
|
+
`/notify` pass that outer gate. The sync worker's required `authorizeWake` and
|
|
137
|
+
`authorizeNotify` callbacks then enforce the real capability checks. Do not
|
|
138
|
+
bypass pull, push, or admin routes, and do not replace either callback with an
|
|
139
|
+
unconditional allow.
|
|
140
|
+
|
|
141
|
+
A delegated push must terminate at the application worker whose registry owns
|
|
142
|
+
the named mutator. Routing it to a sync host, control-plane worker, or another
|
|
143
|
+
application server with a different registry produces an authoritative error
|
|
144
|
+
such as `could not find mutator <name>`. The optimistic client write may appear
|
|
145
|
+
briefly, then disappear on reload and never reach peers. Route client pushes to
|
|
146
|
+
the sync host only when its `mutateBinding` and `mutateUrl` delegate to the
|
|
147
|
+
owning application's mutate endpoint. Preview and browser-local transports must
|
|
148
|
+
stay pointed at their in-process project server unless they provide the same
|
|
149
|
+
delegated route.
|
|
150
|
+
|
|
108
151
|
`POST /admin/writer` with `{ "enabled": false }` durably stops pushes for that
|
|
109
152
|
namespace. A stopped writer consumes and discards the request body, returns 503,
|
|
110
153
|
and performs no engine or application write. Canary rollback drills must stop
|
|
@@ -119,12 +162,36 @@ not mutate a production route. These controls are mechanisms, not authorization
|
|
|
119
162
|
to perform a production cutover.
|
|
120
163
|
|
|
121
164
|
`POST /admin/resnapshot` is available only when the consumer configured an
|
|
122
|
-
upstream data service. It reads
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
165
|
+
upstream data service. It reads each modeled source table through bounded
|
|
166
|
+
keyset pages and commits them to a staged generation. Progress is durable, so a
|
|
167
|
+
restart resumes at the recorded table and cursor. The host then catches up
|
|
168
|
+
concurrent `/changes`, atomically swaps the generation into place, and bumps the
|
|
169
|
+
engine epoch. Every client performs one expected full resync after cutover.
|
|
170
|
+
Engine metadata, client last-mutation IDs, operator controls, and the
|
|
171
|
+
authoritative upstream database are preserved. The legacy single-response
|
|
172
|
+
`/snapshot` endpoint remains for small datasets and older harnesses.
|
|
173
|
+
|
|
174
|
+
## Delegated service addresses
|
|
175
|
+
|
|
176
|
+
`mutateBinding` selects the service binding that owns the application's mutate
|
|
177
|
+
endpoint and defaults to `upstream.binding`. Set `mutateOrigin` to an exact
|
|
178
|
+
absolute HTTP(S) origin when that worker's routing depends on the request
|
|
179
|
+
origin. `upstream.namespacePath` may return `/` for a root-mounted data feed;
|
|
180
|
+
internally that root is an empty path, while `null` alone means no upstream path
|
|
181
|
+
is configured.
|
|
182
|
+
|
|
183
|
+
A successful delegated mutation response has a hard causality contract: its
|
|
184
|
+
committed application effects must already be readable from the configured
|
|
185
|
+
upstream `/changes` feed before the application worker returns. The sync host
|
|
186
|
+
then ingests through those effects, journals the acknowledged LMID after them,
|
|
187
|
+
and only then returns the push response. This keeps every capped change-log
|
|
188
|
+
prefix ordered so an acknowledgement cannot reach a client before its effects.
|
|
189
|
+
Chat and Soot use an application-to-data service path that provides this
|
|
190
|
+
ordering today. A future topology that cannot provide it must extend the
|
|
191
|
+
delegated response with an upstream watermark receipt and make the host ingest
|
|
192
|
+
through that receipt before finalization. Do not add a second best-effort path.
|
|
193
|
+
The ordering does not add another ingest round: delegated pushes already waited
|
|
194
|
+
for post-mutation ingest before returning, and now finalize after that wait.
|
|
128
195
|
|
|
129
196
|
## Counter and HTTP wire representation
|
|
130
197
|
|
|
@@ -212,4 +279,4 @@ absent when the cleanup command was run (Cloudflare returned error 10090), so no
|
|
|
212
279
|
old probe service remains to receive traffic.
|
|
213
280
|
|
|
214
281
|
The rust toolchain is pinned at the workspace root in
|
|
215
|
-
[rust-toolchain.toml](
|
|
282
|
+
[`rust-toolchain.toml`](../../rust-toolchain.toml).
|
package/package.json
CHANGED
|
@@ -1,25 +1,36 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "orez-sync-cf-host",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.19",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
|
-
".": "./src/index.ts"
|
|
6
|
+
".": "./src/index.ts",
|
|
7
|
+
"./post-commit": "./src/post-commit.ts",
|
|
8
|
+
"./mutation-error": {
|
|
9
|
+
"types": "./src/mutation-error.ts",
|
|
10
|
+
"import": "./src/mutation-error.ts"
|
|
11
|
+
},
|
|
12
|
+
"./bun-wasm-loader": "./src/bun-wasm-loader.ts",
|
|
13
|
+
"./query-compiler": "./src/query-compiler.ts",
|
|
14
|
+
"./transaction-query": "./src/transaction-query.ts"
|
|
7
15
|
},
|
|
8
16
|
"scripts": {
|
|
9
17
|
"build:wasm": "wasm-pack build ../../crates/sync-wasm --target web --release --out-dir ../../packages/sync-cf-host/src/generated --out-name sync_wasm && rm -f src/generated/.gitignore && bun scripts/write-wasm-module-type.ts",
|
|
10
18
|
"build:wasm:platform": "wasm-pack build ../../crates/sync-wasm --target web --release --out-dir ../../packages/sync-cf-host/src/generated --out-name sync_wasm -- --features platform-probes && rm -f src/generated/.gitignore && bun scripts/write-wasm-module-type.ts",
|
|
11
19
|
"bundle": "bun run build:wasm && wrangler deploy --dry-run --outdir dist",
|
|
12
20
|
"measure": "bun run build:wasm && bun measure.mjs",
|
|
13
|
-
"test": "bun run test:config && bun run test:platform && bun run test:integration && bun run test:ingest && bun run test:restart",
|
|
14
|
-
"test:config": "bun test config.test.mjs write-safeguards.test.mjs",
|
|
21
|
+
"test": "bun run test:config && bun run test:query-compiler && bun run test:platform && bun run test:integration && bun run test:ingest && bun run test:restart",
|
|
22
|
+
"test:config": "bun test config.test.mjs write-safeguards.test.mjs mutation-error.test.mjs post-commit.test.mjs",
|
|
15
23
|
"test:integration": "bun run build:wasm && bun integration-test.mjs",
|
|
16
24
|
"test:ingest": "bun run build:wasm && bun ingest-test.mjs",
|
|
25
|
+
"test:query-compiler": "bun run build:wasm && bun test --preload orez-sync-cf-host/bun-wasm-loader query-compiler.test.mjs",
|
|
17
26
|
"test:restart": "bun restart-test.mjs",
|
|
18
27
|
"test:platform": "bun run build:wasm:platform && bun platform-test.mjs",
|
|
19
28
|
"typecheck": "bun run build:wasm:platform && tsc --noEmit"
|
|
20
29
|
},
|
|
21
30
|
"devDependencies": {
|
|
22
31
|
"@cloudflare/workers-types": "4.20260617.1",
|
|
32
|
+
"@types/bun": "1.3.14",
|
|
33
|
+
"@types/node": "22.19.10",
|
|
23
34
|
"typescript": "5.9.3",
|
|
24
35
|
"workerd": "1.20260617.1",
|
|
25
36
|
"wrangler": "4.103.0"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer'
|
|
2
|
+
|
|
3
|
+
import { file, plugin } from 'bun'
|
|
4
|
+
|
|
5
|
+
plugin({
|
|
6
|
+
name: 'orez-sync-cf-host-wasm',
|
|
7
|
+
setup(build) {
|
|
8
|
+
build.onLoad({ filter: /sync_wasm_bg\.wasm$/ }, async ({ path }) => {
|
|
9
|
+
const bytes = Buffer.from(await file(path).arrayBuffer()).toString('base64')
|
|
10
|
+
return {
|
|
11
|
+
contents: `export default new WebAssembly.Module(Uint8Array.fromBase64('${bytes}'))`,
|
|
12
|
+
loader: 'js',
|
|
13
|
+
}
|
|
14
|
+
})
|
|
15
|
+
},
|
|
16
|
+
})
|
package/src/config.ts
CHANGED
|
@@ -89,5 +89,14 @@ export function validateSyncHostConfig<Env extends SyncHostEnv>(
|
|
|
89
89
|
throw new TypeError(`delegatedPushRetry.${name} must be a positive safe integer`)
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
|
+
if (config.transactionQueryBudget) {
|
|
93
|
+
for (const [name, value] of Object.entries(config.transactionQueryBudget)) {
|
|
94
|
+
if (!Number.isSafeInteger(value) || Number(value) < 1) {
|
|
95
|
+
throw new TypeError(
|
|
96
|
+
`transactionQueryBudget.${name} must be a positive safe integer`
|
|
97
|
+
)
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
92
101
|
return config
|
|
93
102
|
}
|
|
@@ -37,7 +37,7 @@ export function engine_begin_snapshot_generation(db: any, schema: any, start_wat
|
|
|
37
37
|
* Compile a validated Zero query AST for a consumer mutator's transactional
|
|
38
38
|
* `tx.run(...)`. Execution remains in the host-owned application transaction.
|
|
39
39
|
*/
|
|
40
|
-
export function engine_compile_query(schema: any, ast: any): any;
|
|
40
|
+
export function engine_compile_query(schema: any, ast: any, format: any): any;
|
|
41
41
|
|
|
42
42
|
export function engine_finalize(db: any, client_group_id: string, client_id: string, mutation_id: string): void;
|
|
43
43
|
|
|
@@ -109,7 +109,7 @@ export interface InitOutput {
|
|
|
109
109
|
readonly engine_apply_upstream_snapshot: (a: any, b: any, c: any) => [number, number, number];
|
|
110
110
|
readonly engine_assemble_push_response: (a: any) => [number, number, number];
|
|
111
111
|
readonly engine_begin_snapshot_generation: (a: any, b: any, c: number, d: number) => [number, number, number];
|
|
112
|
-
readonly engine_compile_query: (a: any, b: any) => [number, number, number];
|
|
112
|
+
readonly engine_compile_query: (a: any, b: any, c: any) => [number, number, number];
|
|
113
113
|
readonly engine_finalize: (a: any, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
|
|
114
114
|
readonly engine_finalize_snapshot_generation: (a: any, b: any, c: number, d: number, e: number, f: number) => [number, number, number];
|
|
115
115
|
readonly engine_handle_pull: (a: any, b: any, c: any, d: any, e: number, f: number, g: any, h: number, i: number) => [number, number, number];
|
|
@@ -111,10 +111,11 @@ export function engine_begin_snapshot_generation(db, schema, start_watermark) {
|
|
|
111
111
|
* `tx.run(...)`. Execution remains in the host-owned application transaction.
|
|
112
112
|
* @param {any} schema
|
|
113
113
|
* @param {any} ast
|
|
114
|
+
* @param {any} format
|
|
114
115
|
* @returns {any}
|
|
115
116
|
*/
|
|
116
|
-
export function engine_compile_query(schema, ast) {
|
|
117
|
-
const ret = wasm.engine_compile_query(schema, ast);
|
|
117
|
+
export function engine_compile_query(schema, ast, format) {
|
|
118
|
+
const ret = wasm.engine_compile_query(schema, ast, format);
|
|
118
119
|
if (ret[2]) {
|
|
119
120
|
throw takeFromExternrefTable0(ret[1]);
|
|
120
121
|
}
|
|
Binary file
|
package/src/host.ts
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
1
|
import { DurableObject } from 'cloudflare:workers'
|
|
2
2
|
|
|
3
3
|
import { validatePullCaps, validateSyncHostConfig } from './config.js'
|
|
4
|
+
import { createPostCommitEffects } from './post-commit.js'
|
|
5
|
+
import { createQueryCompiler } from './query-compiler.js'
|
|
6
|
+
import {
|
|
7
|
+
decodeSqlParams,
|
|
8
|
+
SqlStorageDirect,
|
|
9
|
+
SqlStorageMutatorTransaction,
|
|
10
|
+
SqlStorageSyncDb,
|
|
11
|
+
} from './sql-storage-adapter.js'
|
|
12
|
+
import { isMutationApplicationError } from './types.js'
|
|
4
13
|
import {
|
|
5
14
|
engine_apply_snapshot_changes,
|
|
6
15
|
engine_apply_snapshot_page,
|
|
7
16
|
engine_apply_upstream,
|
|
8
17
|
engine_assemble_push_response,
|
|
9
18
|
engine_begin_snapshot_generation,
|
|
10
|
-
engine_compile_query,
|
|
11
19
|
engine_finalize,
|
|
12
20
|
engine_finalize_snapshot_generation,
|
|
13
21
|
engine_handle_pull,
|
|
@@ -23,15 +31,7 @@ import {
|
|
|
23
31
|
engine_record_app_error,
|
|
24
32
|
engine_state,
|
|
25
33
|
engine_version,
|
|
26
|
-
|
|
27
|
-
} from './generated/sync_wasm.js'
|
|
28
|
-
import wasmModule from './generated/sync_wasm_bg.wasm'
|
|
29
|
-
import {
|
|
30
|
-
SqlStorageDirect,
|
|
31
|
-
SqlStorageMutatorTransaction,
|
|
32
|
-
SqlStorageSyncDb,
|
|
33
|
-
} from './sql-storage-adapter.js'
|
|
34
|
-
import { MutationApplicationError } from './types.js'
|
|
34
|
+
} from './wasm.js'
|
|
35
35
|
import {
|
|
36
36
|
IngestBreakerError,
|
|
37
37
|
IngestCircuitBreaker,
|
|
@@ -40,7 +40,6 @@ import {
|
|
|
40
40
|
} from './write-safeguards.js'
|
|
41
41
|
|
|
42
42
|
import type {
|
|
43
|
-
DeferredEffect,
|
|
44
43
|
JsonValue,
|
|
45
44
|
NormalizedClaims,
|
|
46
45
|
PullCaps,
|
|
@@ -48,8 +47,6 @@ import type {
|
|
|
48
47
|
SyncHostEnv,
|
|
49
48
|
} from './types.js'
|
|
50
49
|
|
|
51
|
-
initSync({ module: wasmModule })
|
|
52
|
-
|
|
53
50
|
const CLAIMS_HEADER = 'x-orez-sync-claims'
|
|
54
51
|
const NAMESPACE_HEADER = 'x-orez-sync-namespace'
|
|
55
52
|
const UPSTREAM_PATH_HEADER = 'x-orez-sync-upstream-path'
|
|
@@ -320,6 +317,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
|
|
|
320
317
|
config: SyncHostConfig<Env>
|
|
321
318
|
) {
|
|
322
319
|
validateSyncHostConfig(config)
|
|
320
|
+
const compileQuery = createQueryCompiler(config.schema)
|
|
323
321
|
const defaultRetainChanges = String(config.retainChanges ?? 4_096)
|
|
324
322
|
const caps: PullCaps = validatePullCaps({ ...DEFAULT_CAPS, ...config.caps })
|
|
325
323
|
const idleTeardownMs = config.idleTeardownMs ?? 5_000
|
|
@@ -368,8 +366,10 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
|
|
|
368
366
|
}
|
|
369
367
|
this.#engineDb = new SqlStorageSyncDb(ctx.storage.sql, recordRowsWritten)
|
|
370
368
|
this.#directSql = new SqlStorageDirect(ctx.storage.sql, recordRowsWritten)
|
|
371
|
-
this.#mutatorSql = new SqlStorageMutatorTransaction(
|
|
372
|
-
this.#
|
|
369
|
+
this.#mutatorSql = new SqlStorageMutatorTransaction(
|
|
370
|
+
this.#directSql,
|
|
371
|
+
(ast, format) => this.#wasm(() => compileQuery(ast, format)),
|
|
372
|
+
config.transactionQueryBudget
|
|
373
373
|
)
|
|
374
374
|
ctx.blockConcurrencyWhile(async () => {
|
|
375
375
|
ctx.storage.transactionSync(() => {
|
|
@@ -1030,6 +1030,13 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
|
|
|
1030
1030
|
return this.#ingestPromise
|
|
1031
1031
|
}
|
|
1032
1032
|
|
|
1033
|
+
#ingestAfterCurrent(upstreamPath: string | null): Promise<number> {
|
|
1034
|
+
const current = this.#ingestPromise
|
|
1035
|
+
return current
|
|
1036
|
+
? current.then(() => this.#ingest(upstreamPath))
|
|
1037
|
+
: this.#ingest(upstreamPath)
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1033
1040
|
async #fetchDelegatedPush(
|
|
1034
1041
|
endpoint: URL,
|
|
1035
1042
|
headers: Headers,
|
|
@@ -1096,23 +1103,6 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
|
|
|
1096
1103
|
)
|
|
1097
1104
|
}
|
|
1098
1105
|
|
|
1099
|
-
async #runEffects(effects: DeferredEffect[]): Promise<void> {
|
|
1100
|
-
for (const effect of effects) {
|
|
1101
|
-
try {
|
|
1102
|
-
await effect()
|
|
1103
|
-
} catch (error) {
|
|
1104
|
-
this.#counters.externalEffectFailures++
|
|
1105
|
-
console.error(
|
|
1106
|
-
JSON.stringify({
|
|
1107
|
-
event: 'sync_external_effect_error',
|
|
1108
|
-
hostVersion: config.hostVersion,
|
|
1109
|
-
error: errorMessage(error),
|
|
1110
|
-
})
|
|
1111
|
-
)
|
|
1112
|
-
}
|
|
1113
|
-
}
|
|
1114
|
-
}
|
|
1115
|
-
|
|
1116
1106
|
#enqueueWake(originClientID: string): Promise<void> {
|
|
1117
1107
|
this.#wakeOrigins.add(originClientID)
|
|
1118
1108
|
for (const socket of this.ctx.getWebSockets()) {
|
|
@@ -1437,7 +1427,18 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
|
|
|
1437
1427
|
result.id?.clientID === mutation.clientID &&
|
|
1438
1428
|
String(result.id?.id) === mutation.id
|
|
1439
1429
|
)
|
|
1440
|
-
if (!ack)
|
|
1430
|
+
if (!ack) {
|
|
1431
|
+
throw new Error(
|
|
1432
|
+
`delegated push did not acknowledge ${mutation.clientID}:${mutation.id}`
|
|
1433
|
+
)
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
// the delegated app response is causally visible through DATA by
|
|
1437
|
+
// contract. start an ingest round after that response, even if an
|
|
1438
|
+
// older round is still in flight, then journal lmids. every capped
|
|
1439
|
+
// log prefix therefore preserves effects-before-ack.
|
|
1440
|
+
await this.#ingestAfterCurrent(upstreamPath)
|
|
1441
|
+
for (const mutation of plan.mutations) {
|
|
1441
1442
|
this.ctx.storage.transactionSync(() => {
|
|
1442
1443
|
const decision = this.#wasm(() =>
|
|
1443
1444
|
engine_preflight(
|
|
@@ -1466,7 +1467,6 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
|
|
|
1466
1467
|
this.#wasm(() => engine_prune(this.#engineDb, this.#retainChanges()))
|
|
1467
1468
|
)
|
|
1468
1469
|
}
|
|
1469
|
-
await this.#ingest(upstreamPath)
|
|
1470
1470
|
return json({ pushResponse: delegatedResponse })
|
|
1471
1471
|
} catch (error) {
|
|
1472
1472
|
const status = statusOf(error)
|
|
@@ -1483,7 +1483,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
|
|
|
1483
1483
|
|
|
1484
1484
|
const results: MutationResult[] = []
|
|
1485
1485
|
for (const mutation of plan.mutations) {
|
|
1486
|
-
const deferred
|
|
1486
|
+
const deferred = createPostCommitEffects()
|
|
1487
1487
|
try {
|
|
1488
1488
|
const txStarted = performance.now()
|
|
1489
1489
|
// consume the before-commit fault OUTSIDE the transaction it is
|
|
@@ -1494,7 +1494,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
|
|
|
1494
1494
|
const preflight = await this.ctx.storage.transaction(async () => {
|
|
1495
1495
|
// Storage transactions may retry their closure. Never carry a
|
|
1496
1496
|
// deferred effect from an abandoned attempt into the commit.
|
|
1497
|
-
deferred.
|
|
1497
|
+
deferred.beginAttempt()
|
|
1498
1498
|
const decision = this.#wasm(() =>
|
|
1499
1499
|
engine_preflight(
|
|
1500
1500
|
this.#engineDb,
|
|
@@ -1511,9 +1511,7 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
|
|
|
1511
1511
|
claims,
|
|
1512
1512
|
clientID: mutation.clientID,
|
|
1513
1513
|
mutationID: mutation.id,
|
|
1514
|
-
defer
|
|
1515
|
-
deferred.push(effect)
|
|
1516
|
-
},
|
|
1514
|
+
defer: deferred.defer,
|
|
1517
1515
|
})
|
|
1518
1516
|
if (beforeCommitFault)
|
|
1519
1517
|
throw this.#faultError(
|
|
@@ -1556,9 +1554,18 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
|
|
|
1556
1554
|
// waitUntil anchors the coalescing timer across request completion,
|
|
1557
1555
|
// while the next serialized client push can join the same batch.
|
|
1558
1556
|
this.ctx.waitUntil(this.#enqueueWake(mutation.clientID))
|
|
1559
|
-
await
|
|
1557
|
+
await deferred.runAfterCommit((error) => {
|
|
1558
|
+
this.#counters.externalEffectFailures++
|
|
1559
|
+
console.error(
|
|
1560
|
+
JSON.stringify({
|
|
1561
|
+
event: 'sync_external_effect_error',
|
|
1562
|
+
hostVersion: config.hostVersion,
|
|
1563
|
+
error: errorMessage(error),
|
|
1564
|
+
})
|
|
1565
|
+
)
|
|
1566
|
+
})
|
|
1560
1567
|
} catch (error) {
|
|
1561
|
-
const isAppError = error
|
|
1568
|
+
const isAppError = isMutationApplicationError(error)
|
|
1562
1569
|
if (!isAppError) throw error
|
|
1563
1570
|
this.#counters.applicationErrors++
|
|
1564
1571
|
resultClass = 'application_error'
|
|
@@ -1714,11 +1721,14 @@ export function createSyncDurableObject<Env extends SyncHostEnv>(
|
|
|
1714
1721
|
}
|
|
1715
1722
|
if (route === '/admin/sql') {
|
|
1716
1723
|
return request.json().then((body) => {
|
|
1717
|
-
const { query } = body as { query?: string }
|
|
1724
|
+
const { params, query } = body as { params?: unknown; query?: string }
|
|
1718
1725
|
if (typeof query !== 'string') return json({ error: 'query is required' }, 400)
|
|
1719
1726
|
try {
|
|
1720
|
-
return json({ rows: this.#directSql.query(query) })
|
|
1727
|
+
return json({ rows: this.#directSql.query(query, decodeSqlParams(params)) })
|
|
1721
1728
|
} catch (error) {
|
|
1729
|
+
if (error instanceof TypeError && error.message.startsWith('params')) {
|
|
1730
|
+
return json({ error: `invalid params: ${error.message}` }, 400)
|
|
1731
|
+
}
|
|
1722
1732
|
if (
|
|
1723
1733
|
error instanceof TypeError &&
|
|
1724
1734
|
error.message === 'transaction SQL is host-owned and forbidden'
|
package/src/index.ts
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
export { createSyncDurableObject, createSyncWorker } from './host.js'
|
|
2
|
-
export {
|
|
2
|
+
export { createQueryCompiler } from './query-compiler.js'
|
|
3
|
+
export {
|
|
4
|
+
MutationApplicationError,
|
|
5
|
+
isMutationApplicationError,
|
|
6
|
+
registerMutators,
|
|
7
|
+
} from './types.js'
|
|
8
|
+
export {
|
|
9
|
+
DEFAULT_TRANSACTION_QUERY_BUDGET,
|
|
10
|
+
TransactionQueryBudgetError,
|
|
11
|
+
executeTransactionQueryPlan,
|
|
12
|
+
executeTransactionQueryPlanAsync,
|
|
13
|
+
} from './transaction-query.js'
|
|
14
|
+
|
|
3
15
|
export type {
|
|
4
16
|
DeferredEffect,
|
|
5
17
|
JsonPrimitive,
|
|
@@ -20,3 +32,16 @@ export type {
|
|
|
20
32
|
VisibilityFilter,
|
|
21
33
|
ZeroSchemaConfig,
|
|
22
34
|
} from './types.js'
|
|
35
|
+
export type {
|
|
36
|
+
CompiledTransactionQueryNode,
|
|
37
|
+
CompiledTransactionQueryPlan,
|
|
38
|
+
CompiledTransactionQueryRelationship,
|
|
39
|
+
TransactionQueryBinding,
|
|
40
|
+
TransactionQueryBudget,
|
|
41
|
+
TransactionQueryColumn,
|
|
42
|
+
TransactionQueryColumnType,
|
|
43
|
+
TransactionQueryExecutionOptions,
|
|
44
|
+
TransactionQueryFormat,
|
|
45
|
+
TransactionQueryWireValue,
|
|
46
|
+
} from './transaction-query.js'
|
|
47
|
+
export type { TransactionQueryCompiler } from './query-compiler.js'
|
|
@@ -24,6 +24,7 @@ type Fetcher = { fetch(input: string | Request, init?: RequestInit): Promise<Res
|
|
|
24
24
|
interface Env extends SyncHostEnv {
|
|
25
25
|
DATA: Fetcher
|
|
26
26
|
APP: Fetcher
|
|
27
|
+
CAPPED_SYNC_DO: DurableObjectNamespace
|
|
27
28
|
UPSTREAM_DO: DurableObjectNamespace
|
|
28
29
|
}
|
|
29
30
|
|
|
@@ -35,6 +36,11 @@ const heldSnapshots = new Set<string>()
|
|
|
35
36
|
const holdSnapshotsAfterCursor = new Set<string>()
|
|
36
37
|
const activeSnapshots = new Set<string>()
|
|
37
38
|
const snapshotLimits = new Map<string, number[]>()
|
|
39
|
+
const heldDelegatedPushes = new Set<string>()
|
|
40
|
+
const activeDelegatedPushes = new Set<string>()
|
|
41
|
+
const completedDelegatedPushes = new Set<string>()
|
|
42
|
+
const heldChangeResponses = new Set<string>()
|
|
43
|
+
const activeHeldChangeResponses = new Set<string>()
|
|
38
44
|
let delegatedFailuresRemaining = 0
|
|
39
45
|
let delegatedAttempts = 0
|
|
40
46
|
let delegatedPushFailedRemaining = 0
|
|
@@ -81,6 +87,11 @@ const config: SyncHostConfig<Env> = {
|
|
|
81
87
|
}
|
|
82
88
|
|
|
83
89
|
export const SyncDurableObject = createSyncDurableObject(config)
|
|
90
|
+
const cappedConfig: SyncHostConfig<Env> = {
|
|
91
|
+
...config,
|
|
92
|
+
caps: { maxChangeRows: 10_000, maxChangeBytes: 1 },
|
|
93
|
+
}
|
|
94
|
+
export const CappedSyncDurableObject = createSyncDurableObject(cappedConfig)
|
|
84
95
|
export { ZeroDO }
|
|
85
96
|
|
|
86
97
|
async function upstreamFetch(request: Request, env: Env): Promise<Response> {
|
|
@@ -251,6 +262,14 @@ export class DataService extends WorkerEntrypoint<Env> {
|
|
|
251
262
|
)
|
|
252
263
|
}
|
|
253
264
|
const response = await upstreamFetch(request, this.env)
|
|
265
|
+
if (pathname.endsWith('/changes') && heldChangeResponses.has(namespace)) {
|
|
266
|
+
activeHeldChangeResponses.add(namespace)
|
|
267
|
+
try {
|
|
268
|
+
while (heldChangeResponses.has(namespace)) await scheduler.wait(10)
|
|
269
|
+
} finally {
|
|
270
|
+
activeHeldChangeResponses.delete(namespace)
|
|
271
|
+
}
|
|
272
|
+
}
|
|
254
273
|
if (pathname.endsWith('/snapshot')) {
|
|
255
274
|
const limit = Number(url.searchParams.get('limit'))
|
|
256
275
|
if (Number.isSafeInteger(limit)) {
|
|
@@ -281,7 +300,7 @@ export class DataService extends WorkerEntrypoint<Env> {
|
|
|
281
300
|
}
|
|
282
301
|
|
|
283
302
|
export class AppService extends WorkerEntrypoint<Env> {
|
|
284
|
-
fetch(request: Request): Promise<Response> {
|
|
303
|
+
async fetch(request: Request): Promise<Response> {
|
|
285
304
|
delegatedUrl = request.url
|
|
286
305
|
if (!new URL(request.url).pathname.endsWith('/api/zero/push')) {
|
|
287
306
|
return Promise.resolve(
|
|
@@ -313,14 +332,81 @@ export class AppService extends WorkerEntrypoint<Env> {
|
|
|
313
332
|
Response.json({ error: 'synthetic delegated push failure' }, { status: 503 })
|
|
314
333
|
)
|
|
315
334
|
}
|
|
316
|
-
|
|
335
|
+
if (heldDelegatedPushes.has(namespace)) {
|
|
336
|
+
activeDelegatedPushes.add(namespace)
|
|
337
|
+
try {
|
|
338
|
+
while (heldDelegatedPushes.has(namespace)) await scheduler.wait(10)
|
|
339
|
+
} finally {
|
|
340
|
+
activeDelegatedPushes.delete(namespace)
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
const push = (await request.clone().json()) as {
|
|
344
|
+
mutations?: Array<{ clientID?: string; id?: number; name?: string }>
|
|
345
|
+
}
|
|
346
|
+
const cleanupIDs = new Set(
|
|
347
|
+
(push.mutations ?? [])
|
|
348
|
+
.filter((mutation) => mutation.name === '_zero_cleanupResults')
|
|
349
|
+
.map((mutation) => `${mutation.clientID}:${mutation.id}`)
|
|
350
|
+
)
|
|
351
|
+
const response = await upstreamFetch(request, this.env)
|
|
352
|
+
completedDelegatedPushes.add(namespace)
|
|
353
|
+
if (response.ok && cleanupIDs.size > 0) {
|
|
354
|
+
const body = (await response.json()) as {
|
|
355
|
+
mutations?: Array<{ id?: { clientID?: string; id?: number } }>
|
|
356
|
+
pushResponse?: {
|
|
357
|
+
mutations?: Array<{ id?: { clientID?: string; id?: number } }>
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
const mutations = body.pushResponse?.mutations ?? body.mutations
|
|
361
|
+
if (Array.isArray(mutations)) {
|
|
362
|
+
const filtered = mutations.filter(
|
|
363
|
+
(mutation) => !cleanupIDs.has(`${mutation.id?.clientID}:${mutation.id?.id}`)
|
|
364
|
+
)
|
|
365
|
+
if (body.pushResponse) body.pushResponse.mutations = filtered
|
|
366
|
+
else body.mutations = filtered
|
|
367
|
+
}
|
|
368
|
+
return Response.json(body, { status: response.status })
|
|
369
|
+
}
|
|
370
|
+
return response
|
|
317
371
|
}
|
|
318
372
|
}
|
|
319
373
|
|
|
320
374
|
const syncWorker = createSyncWorker(config)
|
|
375
|
+
const cappedSyncWorker = createSyncWorker(cappedConfig)
|
|
321
376
|
export default {
|
|
322
377
|
fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
|
|
323
378
|
const url = new URL(request.url)
|
|
379
|
+
if (url.pathname.startsWith('/delegated-ingest-control/')) {
|
|
380
|
+
const namespace = url.pathname.slice('/delegated-ingest-control/'.length)
|
|
381
|
+
if (request.method === 'GET') {
|
|
382
|
+
return Promise.resolve(
|
|
383
|
+
Response.json({
|
|
384
|
+
appHeld: heldDelegatedPushes.has(namespace),
|
|
385
|
+
appActive: activeDelegatedPushes.has(namespace),
|
|
386
|
+
appCompleted: completedDelegatedPushes.has(namespace),
|
|
387
|
+
changesHeld: heldChangeResponses.has(namespace),
|
|
388
|
+
changesActive: activeHeldChangeResponses.has(namespace),
|
|
389
|
+
})
|
|
390
|
+
)
|
|
391
|
+
}
|
|
392
|
+
return request
|
|
393
|
+
.json()
|
|
394
|
+
.catch(() => ({}))
|
|
395
|
+
.then((body) => {
|
|
396
|
+
if ((body as { appHeld?: unknown }).appHeld === true) {
|
|
397
|
+
heldDelegatedPushes.add(namespace)
|
|
398
|
+
completedDelegatedPushes.delete(namespace)
|
|
399
|
+
} else if ((body as { appHeld?: unknown }).appHeld === false) {
|
|
400
|
+
heldDelegatedPushes.delete(namespace)
|
|
401
|
+
}
|
|
402
|
+
if ((body as { changesHeld?: unknown }).changesHeld === true) {
|
|
403
|
+
heldChangeResponses.add(namespace)
|
|
404
|
+
} else if ((body as { changesHeld?: unknown }).changesHeld === false) {
|
|
405
|
+
heldChangeResponses.delete(namespace)
|
|
406
|
+
}
|
|
407
|
+
return Response.json({ ok: true, namespace })
|
|
408
|
+
})
|
|
409
|
+
}
|
|
324
410
|
if (url.pathname.startsWith('/snapshot-control/')) {
|
|
325
411
|
const namespace = url.pathname.slice('/snapshot-control/'.length)
|
|
326
412
|
if (request.method === 'GET') {
|
|
@@ -431,6 +517,15 @@ export default {
|
|
|
431
517
|
})
|
|
432
518
|
})
|
|
433
519
|
}
|
|
520
|
+
if (url.pathname.startsWith('/capped/')) {
|
|
521
|
+
url.pathname = url.pathname.slice('/capped'.length)
|
|
522
|
+
const cappedEnv = { ...env, SYNC_DO: env.CAPPED_SYNC_DO }
|
|
523
|
+
return cappedSyncWorker.fetch!(
|
|
524
|
+
new Request(url, request) as never,
|
|
525
|
+
cappedEnv,
|
|
526
|
+
ctx
|
|
527
|
+
) as Promise<Response>
|
|
528
|
+
}
|
|
434
529
|
if (url.pathname.startsWith('/upstream/')) {
|
|
435
530
|
url.pathname = url.pathname.slice('/upstream'.length)
|
|
436
531
|
return upstreamFetch(new Request(url, request), env)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// the app-rejection contract shared by every orez sync host and by
|
|
2
|
+
// protocol-neutral authoring layers (on-zero adapters). classification is
|
|
3
|
+
// STRUCTURAL, never instanceof: adapters must not depend on orez to reject a
|
|
4
|
+
// mutation, and instanceof breaks across package copies (dual-package
|
|
5
|
+
// installs, --into tarballs, workspace duplication). any error whose name is
|
|
6
|
+
// 'MutationApplicationError' with string details/message is claiming this
|
|
7
|
+
// contract deliberately.
|
|
8
|
+
export class MutationApplicationError extends Error {
|
|
9
|
+
constructor(
|
|
10
|
+
readonly details: string,
|
|
11
|
+
message = details
|
|
12
|
+
) {
|
|
13
|
+
super(message)
|
|
14
|
+
this.name = 'MutationApplicationError'
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function isMutationApplicationError(
|
|
19
|
+
error: unknown
|
|
20
|
+
): error is { name: string; message: string; details: string } {
|
|
21
|
+
if (typeof error !== 'object' || error === null) return false
|
|
22
|
+
const shaped = error as { name?: unknown; message?: unknown; details?: unknown }
|
|
23
|
+
return (
|
|
24
|
+
shaped.name === 'MutationApplicationError' &&
|
|
25
|
+
typeof shaped.message === 'string' &&
|
|
26
|
+
typeof shaped.details === 'string'
|
|
27
|
+
)
|
|
28
|
+
}
|