orez 0.5.9 → 0.5.12

package/dist/worker/zero-cache-embed-cf.js

@@ -35,21 +35,25 @@
  * again from durable DO SQLite. see plans/cf-do-idle-hibernation.md and
  * `shouldHibernateIdleZeroCache` in ./zero-cache-do-idle.ts.
  */
-import './shims/zero-process-env.js';
 import { EventEmitter } from 'node:events';
-// static import so wrangler can follow the dependency tree and bundle
-// zero-cache with all its transitive deps + our shim aliases.
-// @ts-expect-error — internal zero-cache module, no type declarations
-import { runWorker as _runWorker } from '@rocicorp/zero/out/zero-cache/src/server/runner/run-worker.js';
-import { setLogLevel } from '../log.js';
 import { createBrowserProxy } from '../pg-proxy-browser.js';
-import { DoBackend } from '../pg-proxy-do-backend.js';
-import { resetReplicationState } from '../replication/handler.js';
+import { DoBackend, releaseDoBackendInstanceCaches } from '../pg-proxy-do-backend.js';
+import { deleteReplicationState, resetReplicationState, signalReplicationChange, } from '../replication/handler.js';
+import { apiURLForCFInstance, dispatcherFastifyForCFInstance, logCFInstance, postgresURLForCFInstance, registerCFInstanceRuntime, releaseCFInstanceRuntime, setCFInstanceEnv, setCFInstanceProxy, sqliteDirectoryForCFInstance, sqlitePathForCFInstance, } from './cf-instance-runtime.js';
 import { DurableObjectWebSocketHandoff, } from './durable-object-websocket-handoff.js';
-import { sweepLeakedSqliteHandles } from './embed-generation.js';
+import { sweepCFInstanceSqliteHandles } from './embed-generation.js';
 import { createLocalSqlBackend } from './local-sql-backend.js';
-import { resetFastifyRegistry } from './shims/fastify.js';
+import { acquireZeroProcessEnv } from './shims/zero-process-env.js';
+// static import so wrangler follows zero-cache's dependency tree and shim aliases.
+import { runWorker as _runWorker } from './zero-cache-run-worker.js';
 const runWorkerFn = _runWorker;
+const WORKER_SHUTDOWN_TIMEOUT_MS = 5_000;
+function releaseGenerationWhenComplete(generation) {
+    if (generation.workerDone && generation.cleanupDone && !generation.cleanupFailed) {
+        deleteReplicationState(generation.runtime.instanceId);
+        releaseCFInstanceRuntime(generation.runtime);
+    }
+}
 // tx-journal owner id for every pg session this embed opens. recovery at
 // embed boot targets exactly this owner, so it can never roll back another
 // client's live transaction on the shared upstream db (e.g. the app worker's
@@ -57,12 +61,12 @@ const runWorkerFn = _runWorker;
 const EMBED_TX_OWNER = 'orez-embed';
 // roll back journaled transactions a dead embed generation left on the
 // remote SQL DO (upstream db sessions killed mid-transaction).
-async function recoverRemoteTransactions(url, backendFetch) {
+async function recoverRemoteTransactions(url, owner, backendFetch) {
     const fetcher = backendFetch ?? fetch;
     const resp = await fetcher(url, {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ owner: EMBED_TX_OWNER }),
+        body: JSON.stringify({ owner }),
     });
     if (!resp.ok) {
         const text = await resp.text().catch(() => '');
@@ -79,27 +83,6 @@ function addProtocolSessionFactory(backend, createProtocolSession) {
  * a DoBackend target for upstream/CVR/change Postgres connections.
  */
 export async function startZeroCacheEmbedCF(opts) {
-    // wire orez's own logger from env, mirroring the node path's setLogLevel
-    // (index.ts). without this, OREZ_LOG_LEVEL=debug on a CF deploy is silently
-    // ignored and the replication poll loop is undebuggable in tails.
-    setLogLevel(opts.env?.OREZ_LOG_LEVEL || 'warn');
-    // generation hermeticity: the isolate (module state) outlives a stop() —
-    // CF reuses it across DO instance recreation after idle-hibernation
-    // teardown. a fresh embed must start from the replication-handler state a
-    // fresh isolate would have (lsn floor, stream watermark, schema caches);
-    // the reconnect reconciliation in handleStartReplication is designed to
-    // resume from durable state, not from a prior generation's module vars.
-    resetReplicationState();
-    // embed restart contract (see ./embed-generation.ts): reclaim what process
-    // death would have — sqlite handles the dead generation never closed
-    // (zero-cache relies on process-per-worker exit for these), and the
-    // fastify/ws shim instance registry (a dead change-streamer otherwise
-    // captures the new generation's replicator subscription and boot hangs).
-    const leakedHandles = sweepLeakedSqliteHandles();
-    if (leakedHandles > 0) {
-        console.warn(`[orez-zero-cache-cf] closed ${leakedHandles} sqlite handles leaked by the previous embed generation`);
-    }
-    resetFastifyRegistry();
     const appId = opts.appId || 'zero';
     const publications = opts.publications?.join(',') || `orez_${appId}_public`;
     const readyTimeout = opts.readyTimeout ?? 30000;
@@ -107,210 +90,344 @@ export async function startZeroCacheEmbedCF(opts) {
     const pgPassword = opts.pgPassword || '';
     const backendUrl = opts.backendUrl || 'https://orez-do-backend.local';
     const backendNamespace = opts.backendNamespace || appId;
-    // zero-cache's CVR and change DBs are embed-private state; they live in
-    // THIS Durable Object's SQLite so their pg sessions never pay a cross-DO
-    // round-trip. only the shared upstream `postgres` db routes to ZeroSqlDO.
-    const localSql = createLocalSqlBackend(opts.doSqlite);
-    const createBackend = (dbName) => new DoBackend(backendUrl, dbName, backendNamespace, {
-        fetch: dbName === 'postgres' ? opts.backendFetch : localSql.fetch,
-        txOwner: EMBED_TX_OWNER,
-    });
-    // crash recovery: this embed boot proves the previous embed generation is
-    // dead, so any journaled transaction it left mid-flight (DO eviction,
-    // deploy upgrade-kill) is rolled back BEFORE any pg session opens. without
-    // this, a partially-persisted tx (e.g. the change-streamer's cdc changeLog
-    // write) wedges replication on every subsequent boot.
-    localSql.recoverOrphanedTransactions();
-    await recoverRemoteTransactions(`${backendUrl.replace(/\/+$/, '')}/recover-txs?db=postgres&ns=${encodeURIComponent(backendNamespace)}`, opts.backendFetch);
-    const backends = {
-        postgres: createBackend('postgres'),
-        cvr: createBackend('zero_cvr'),
-        cdb: createBackend('zero_cdb'),
-    };
-    const proxyBackends = {
-        postgres: addProtocolSessionFactory(backends.postgres, () => createBackend('postgres')),
-        cvr: addProtocolSessionFactory(backends.cvr, () => createBackend('zero_cvr')),
-        cdb: addProtocolSessionFactory(backends.cdb, () => createBackend('zero_cdb')),
-    };
-    await Promise.all([
-        backends.postgres.waitReady,
-        backends.cvr.waitReady,
-        backends.cdb.waitReady,
-    ]);
-    const proxy = await createBrowserProxy({
-        postgres: proxyBackends.postgres,
-        cvr: proxyBackends.cvr,
-        cdb: proxyBackends.cdb,
-        postgresReplicas: [],
-    }, {
-        pgUser,
+    if (!opts.apiFetch && (opts.env?.ZERO_MUTATE_URL || opts.env?.ZERO_QUERY_URL)) {
+        throw new Error('zero-cache CF embed: apiFetch is required with ZERO_MUTATE_URL or ZERO_QUERY_URL');
+    }
+    const runtime = registerCFInstanceRuntime({
+        apiFetch: opts.apiFetch,
+        doSqlite: opts.doSqlite,
+        env: opts.env ?? {},
+        instanceId: opts.instanceId,
+        log: opts.log,
         pgPassword,
-        singleDb: false,
-        logLevel: opts.env?.ZERO_LOG_LEVEL || 'info',
+        pgUser,
     });
-    globalThis.__orez_do_sqlite = opts.doSqlite;
-    globalThis.__orez_proxy_connect = (port) => {
-        proxy.handleConnection(port);
+    const generation = {
+        cleanupDone: false,
+        cleanupFailed: false,
+        runtime,
+        workerDone: true,
     };
-    globalThis.__orez_proxy_user = pgUser;
-    globalThis.__orez_proxy_password = pgPassword;
-    globalThis.process ??= {};
-    globalThis.process.env ??= {};
-    globalThis.process.pid ??= 1;
-    globalThis.process.argv ??= [];
-    globalThis.process.env.SINGLE_PROCESS = '1';
-    globalThis.process.env.NODE_ENV = 'development';
-    globalThis.process.kill ??= () => { };
-    // create fake parent EventEmitter for zero-cache's runWorker()
-    // must be declared before process.exit shim (which references it)
-    const parent = new EventEmitter();
-    const parentEmitter = new EventEmitter();
-    parent.send = (message, sendHandle) => {
-        parentEmitter.emit('message', message, sendHandle);
-        return true;
+    const instanceId = runtime.instanceId;
+    const globalRecord = globalThis;
+    let processRecord;
+    let releaseProcessEnv = null;
+    const backendRoots = [];
+    let proxy = null;
+    let parent = null;
+    let parentEmitter = null;
+    let wrappedParent = null;
+    let isReady = false;
+    let stopping = false;
+    let runWorkerPromise = null;
+    let workerSettledPromise = null;
+    let workerError;
+    let workerFailed = false;
+    let startupFailure;
+    let shutdownPromise = null;
+    let fastifyInstance = null;
+    let readyTimer;
+    let debugEmbed = false;
+    const webSocketHandoff = new DurableObjectWebSocketHandoff(() => fastifyInstance);
+    const releaseProcessEnvWhenWorkerDone = () => {
+        if (!generation.workerDone || !releaseProcessEnv)
+            return;
+        releaseProcessEnv();
+        releaseProcessEnv = null;
     };
-    parent.kill = (signal = 'SIGTERM') => {
-        parent.emit(signal, signal);
+    const shutdown = () => {
+        if (shutdownPromise)
+            return shutdownPromise;
+        stopping = true;
+        isReady = false;
+        if (readyTimer)
+            clearTimeout(readyTimer);
+        shutdownPromise = (async () => {
+            const cleanupErrors = [];
+            let resourceCleanupFailed = false;
+            if (wrappedParent && runWorkerPromise && !generation.workerDone) {
+                try {
+                    ;
+                    wrappedParent.kill('SIGTERM');
+                }
+                catch (err) {
+                    cleanupErrors.push(err);
+                }
+            }
+            if (workerSettledPromise && !generation.workerDone) {
+                let timeout;
+                const workerStopped = await Promise.race([
+                    workerSettledPromise.then(() => true),
+                    new Promise((resolve) => {
+                        timeout = setTimeout(() => resolve(false), WORKER_SHUTDOWN_TIMEOUT_MS);
+                    }),
+                ]);
+                if (timeout)
+                    clearTimeout(timeout);
+                if (!workerStopped) {
+                    resourceCleanupFailed = true;
+                    cleanupErrors.push(new Error(`zero-cache CF embed: worker did not terminate within ${WORKER_SHUTDOWN_TIMEOUT_MS}ms`));
+                }
+            }
+            if (workerFailed && workerError !== startupFailure) {
+                cleanupErrors.push(workerError);
+            }
+            if (proxy) {
+                try {
+                    await proxy.close();
+                }
+                catch (err) {
+                    resourceCleanupFailed = true;
+                    cleanupErrors.push(err);
+                }
+            }
+            const backendResults = await Promise.allSettled(backendRoots.map((backend) => Promise.resolve().then(() => backend.close())));
+            for (const result of backendResults) {
+                if (result.status === 'rejected') {
+                    resourceCleanupFailed = true;
+                    cleanupErrors.push(result.reason);
+                }
+            }
+            releaseDoBackendInstanceCaches(instanceId);
+            try {
+                webSocketHandoff.closeAll();
+            }
+            catch (err) {
+                resourceCleanupFailed = true;
+                cleanupErrors.push(err);
+            }
+            try {
+                sweepCFInstanceSqliteHandles(runtime);
+            }
+            catch (err) {
+                resourceCleanupFailed = true;
+                cleanupErrors.push(err);
+            }
+            releaseProcessEnvWhenWorkerDone();
+            parentEmitter?.removeAllListeners();
+            if (generation.workerDone)
+                parent?.removeAllListeners();
+            generation.cleanupDone = true;
+            generation.cleanupFailed = resourceCleanupFailed;
+            releaseGenerationWhenComplete(generation);
+            if (cleanupErrors.length > 0) {
+                throw new AggregateError(cleanupErrors, 'zero-cache CF embed: teardown failed');
+            }
+        })();
+        return shutdownPromise;
     };
-    parent.pid = globalThis.process.pid ?? 1;
-    // shim process.exit to emit on parent instead of actually exiting
-    const origExit = globalThis.process.exit;
-    const origNodeEnv = globalThis.process.env.NODE_ENV;
-    const origKill = globalThis.process.kill;
-    const origFetch = globalThis.fetch;
-    globalThis.process.exit = (code) => {
-        parent.emit('exit', code ?? 0);
+    const handleUnexpectedWorkerExit = () => {
+        if (!isReady || stopping)
+            return;
+        isReady = false;
+        if (!workerFailed) {
+            workerFailed = true;
+            workerError = new Error('zero-cache CF embed: runWorker exited after becoming ready');
+        }
+        void shutdown().catch((error) => {
+            logCFInstance(runtime, {
+                component: 'embed',
+                error,
+                event: 'unexpected-worker-exit-cleanup-failed',
+            });
+            console.error(`[orez-zero-cache-cf:${runtime.instanceId}] unexpected worker exit cleanup failed`, error);
+        });
     };
-    if (opts.apiFetch) {
-        ;
-        globalThis.fetch = (input, init) => {
-            const request = new Request(input, init);
-            const url = new URL(request.url);
-            if (url.hostname === 'orez-zero-api.local')
-                return opts.apiFetch(request);
-            return origFetch(input, init);
+    try {
+        releaseProcessEnv = acquireZeroProcessEnv();
+        processRecord = globalRecord.process;
+        const localSql = createLocalSqlBackend(opts.doSqlite);
+        const txOwner = `${EMBED_TX_OWNER}:${runtime.encodedId}`;
+        const instantiateBackend = (dbName) => new DoBackend(backendUrl, dbName, backendNamespace, {
+            allowTransactionalDDL: true,
+            fetch: dbName === 'postgres' ? opts.backendFetch : localSql.fetch,
+            instanceId,
+            signalReplication: () => signalReplicationChange(instanceId),
+            txOwner,
+        });
+        const createRootBackend = (dbName) => {
+            const backend = instantiateBackend(dbName);
+            backendRoots.push(backend);
+            return backend;
         };
-    }
-    // build env for zero-cache
-    const env = {
-        ...globalThis.process.env,
-        SINGLE_PROCESS: '1',
-        NODE_ENV: 'development',
-        // postgres-browser intercepts these URLs and routes PG wire over
-        // MessagePort to the DoBackend-backed proxy above.
-        ZERO_UPSTREAM_DB: `postgres://${pgUser}:ignored@127.0.0.1/postgres`,
-        ZERO_CVR_DB: `postgres://${pgUser}:ignored@127.0.0.1/zero_cvr`,
-        ZERO_CHANGE_DB: `postgres://${pgUser}:ignored@127.0.0.1/zero_cdb`,
-        // this path is intercepted by the sqlite shim
-        ZERO_REPLICA_FILE: ':do-sqlite:',
-        // don't bind a port — we route via inject/handoff
-        ZERO_PORT: '0',
-        ZERO_APP_ID: appId,
-        ZERO_APP_PUBLICATIONS: publications,
-        ZERO_ADMIN_PASSWORD: opts.env?.ZERO_ADMIN_PASSWORD || crypto.randomUUID(),
-        ZERO_NUM_SYNC_WORKERS: opts.env?.ZERO_NUM_SYNC_WORKERS || '1',
-        ZERO_ENABLE_QUERY_PLANNER: 'false',
-        // one isolate, one sync worker — zero-cache's default pg pools (upstream 20,
-        // cvr 30, change 5) would let ~50 DoBackend protocol sessions accumulate
-        // inside the single 128MB DO isolate, each carrying its own rewrite cache +
-        // protocol/schema state. cap all three hard: with SINGLE_PROCESS + one sync
-        // worker a couple connections per db is plenty, and the freed heap is what
-        // keeps cold-boot view-syncer hydration from tipping the isolate over its
-        // memory limit. (these are the max=N pools; zero also opens fixed max=1/max=2
-        // pools per db for the replication stream + initial-sync that we can't cap.)
-        ZERO_UPSTREAM_MAX_CONNS: opts.env?.ZERO_UPSTREAM_MAX_CONNS || '2',
-        ZERO_CVR_MAX_CONNS: opts.env?.ZERO_CVR_MAX_CONNS || '2',
-        ZERO_CHANGE_MAX_CONNS: opts.env?.ZERO_CHANGE_MAX_CONNS || '2',
-        // 'info' dumps the full table schema as JSON on every replication-status
-        // event; during cold-boot hydration + the reconnect loop that string churn
-        // is pure heap pressure in the 128MB isolate. 'warn' keeps errors visible.
-        ZERO_LOG_LEVEL: opts.env?.ZERO_LOG_LEVEL || 'warn',
-        ...opts.env,
-        // shadow sync is an optional upstream canary that imports the initial-sync
-        // copy path. keep it disabled in the CF embed to avoid bundling unused
-        // worker code and storage paths into Durable Objects.
-        ZERO_SHADOW_SYNC_ENABLED: 'false',
-    };
-    Object.assign(globalThis.process.env, env);
-    const debugEmbed = env.OREZ_DEBUG_EMBED === '1' || globalThis.__OREZ_DEBUG_EMBED__ === true;
-    // wrap parent with onMessageType/onceMessageType helpers
-    // must forward sendHandle (second arg) for WebSocket handoff
-    const wrappedParent = new Proxy(parent, {
-        get(target, prop, receiver) {
-            if (prop === 'onMessageType') {
-                return (type, handler) => {
-                    target.on('message', (data, sendHandle) => {
-                        if (Array.isArray(data) && data.length === 2 && data[0] === type) {
-                            handler(data[1], sendHandle);
-                        }
-                    });
-                    return receiver;
-                };
+        localSql.recoverOrphanedTransactions();
+        await recoverRemoteTransactions(`${backendUrl.replace(/\/+$/, '')}/recover-txs?db=postgres&ns=${encodeURIComponent(backendNamespace)}`, txOwner, opts.backendFetch);
+        const backends = {
+            postgres: createRootBackend('postgres'),
+            cvr: createRootBackend('zero_cvr'),
+            cdb: createRootBackend('zero_cdb'),
+        };
+        const proxyBackends = {
+            postgres: addProtocolSessionFactory(backends.postgres, () => instantiateBackend('postgres')),
+            cvr: addProtocolSessionFactory(backends.cvr, () => instantiateBackend('zero_cvr')),
+            cdb: addProtocolSessionFactory(backends.cdb, () => instantiateBackend('zero_cdb')),
+        };
+        const backendReadyResults = await Promise.allSettled([
+            backends.postgres.waitReady,
+            backends.cvr.waitReady,
+            backends.cdb.waitReady,
+        ]);
+        const backendReadyErrors = backendReadyResults.flatMap((result) => result.status === 'rejected' ? [result.reason] : []);
+        if (backendReadyErrors.length === 1)
+            throw backendReadyErrors[0];
+        if (backendReadyErrors.length > 1) {
+            throw new AggregateError(backendReadyErrors, 'zero-cache CF embed: backend setup failed');
+        }
+        proxy = await createBrowserProxy({
+            postgres: proxyBackends.postgres,
+            cvr: proxyBackends.cvr,
+            cdb: proxyBackends.cdb,
+            postgresReplicas: [],
+        }, {
+            debugWire: opts.env?.OREZ_DEBUG_WIRE === '1',
+            instanceId,
+            pgUser,
+            pgPassword,
+            singleDb: false,
+            logLevel: opts.env?.ZERO_LOG_LEVEL || 'info',
+        });
+        setCFInstanceProxy(runtime, (port) => proxy?.handleConnection(port));
+        const createdParent = new EventEmitter();
+        parent = createdParent;
+        parentEmitter = new EventEmitter();
+        createdParent.send = (message, sendHandle) => {
+            parentEmitter?.emit('message', message, sendHandle);
+            return true;
+        };
+        createdParent.kill = (signal = 'SIGTERM') => {
+            createdParent.emit(signal, signal);
+        };
+        createdParent.pid = processRecord.pid ?? 1;
+        const env = {
+            SINGLE_PROCESS: '1',
+            NODE_ENV: 'development',
+            ZERO_APP_ID: appId,
+            ZERO_APP_PUBLICATIONS: publications,
+            ZERO_ADMIN_PASSWORD: opts.env?.ZERO_ADMIN_PASSWORD || crypto.randomUUID(),
+            ZERO_NUM_SYNC_WORKERS: opts.env?.ZERO_NUM_SYNC_WORKERS || '1',
+            ZERO_ENABLE_QUERY_PLANNER: 'false',
+            ZERO_UPSTREAM_MAX_CONNS: opts.env?.ZERO_UPSTREAM_MAX_CONNS || '2',
+            ZERO_CVR_MAX_CONNS: opts.env?.ZERO_CVR_MAX_CONNS || '2',
+            ZERO_CHANGE_MAX_CONNS: opts.env?.ZERO_CHANGE_MAX_CONNS || '2',
+            ZERO_LOG_LEVEL: opts.env?.ZERO_LOG_LEVEL || 'warn',
+            ...opts.env,
+            ZERO_UPSTREAM_DB: postgresURLForCFInstance(instanceId, 'postgres', pgUser),
+            ZERO_CVR_DB: postgresURLForCFInstance(instanceId, 'zero_cvr', pgUser),
+            ZERO_CHANGE_DB: postgresURLForCFInstance(instanceId, 'zero_cdb', pgUser),
+            ZERO_REPLICA_FILE: sqlitePathForCFInstance(instanceId),
+            ZERO_STORAGE_DB_TMP_DIR: sqliteDirectoryForCFInstance(instanceId),
+            ZERO_PORT: String(runtime.basePort),
+            ZERO_TASK_ID: `orez-cf-${runtime.encodedId}`,
+            ZERO_SHADOW_SYNC_ENABLED: 'false',
+        };
+        if (opts.apiFetch) {
+            for (const key of ['ZERO_MUTATE_URL', 'ZERO_QUERY_URL']) {
+                if (env[key])
+                    env[key] = apiURLForCFInstance(instanceId, env[key]);
             }
-            if (prop === 'onceMessageType') {
-                return (type, handler) => {
-                    const listener = (data, sendHandle) => {
-                        if (Array.isArray(data) && data.length === 2 && data[0] === type) {
-                            target.off('message', listener);
-                            handler(data[1], sendHandle);
-                        }
+        }
+        setCFInstanceEnv(runtime, env);
+        resetReplicationState(instanceId, env, opts.log);
+        debugEmbed = env.OREZ_DEBUG_EMBED === '1';
+        wrappedParent = new Proxy(createdParent, {
+            get(target, prop, receiver) {
+                if (prop === 'onMessageType') {
+                    return (type, handler) => {
+                        target.on('message', (data, sendHandle) => {
+                            if (Array.isArray(data) && data.length === 2 && data[0] === type) {
+                                handler(data[1], sendHandle);
+                            }
+                        });
+                        return receiver;
+                    };
+                }
+                if (prop === 'onceMessageType') {
+                    return (type, handler) => {
+                        const listener = (data, sendHandle) => {
+                            if (Array.isArray(data) && data.length === 2 && data[0] === type) {
+                                target.off('message', listener);
+                                handler(data[1], sendHandle);
+                            }
+                        };
+                        target.on('message', listener);
+                        return receiver;
                     };
-                    target.on('message', listener);
-                    return receiver;
+                }
+                const value = Reflect.get(target, prop, target);
+                if (typeof value !== 'function')
+                    return value;
+                return (...args) => {
+                    const result = value.apply(target, args);
+                    return result === target ? receiver : result;
                 };
+            },
+        });
+        const readyPromise = new Promise((resolve, reject) => {
+            readyTimer = setTimeout(() => {
+                reject(new Error(`zero-cache CF embed: timed out waiting for ready after ${readyTimeout}ms`));
+            }, readyTimeout);
+            parentEmitter?.on('message', (msg) => {
+                if (debugEmbed) {
+                    logCFInstance(runtime, {
+                        component: 'embed',
+                        event: 'parent-message',
+                        message: msg,
+                    });
+                }
+                if (!stopping && Array.isArray(msg) && msg[0] === 'ready') {
+                    if (readyTimer)
+                        clearTimeout(readyTimer);
+                    isReady = true;
+                    resolve();
+                }
+            });
+        });
+        generation.workerDone = false;
+        runWorkerPromise = Promise.resolve().then(() => runWorkerFn(wrappedParent, env));
+        void runWorkerPromise.catch((err) => {
+            if (debugEmbed) {
+                logCFInstance(runtime, {
+                    component: 'embed',
+                    error: err,
+                    event: 'worker-error',
+                });
             }
-            return Reflect.get(target, prop, receiver);
-        },
-    });
-    // track state
-    let isReady = false;
-    let runWorkerPromise = null;
-    // capture the Fastify shim instance from zero-cache's HttpService.
-    // the fastify shim stores itself on globalThis when created.
-    let fastifyInstance = null;
-    let readyTimer;
-    const webSocketHandoff = new DurableObjectWebSocketHandoff(() => fastifyInstance);
-    // wait for "ready" message
-    const readyPromise = new Promise((resolve, reject) => {
-        readyTimer = setTimeout(() => {
-            reject(new Error(`zero-cache CF embed: timed out waiting for ready after ${readyTimeout}ms`));
-        }, readyTimeout);
-        parentEmitter.on('message', (msg) => {
-            if (debugEmbed)
-                console.debug('[orez-zero-cache-cf] parent message', msg);
-            if (Array.isArray(msg) && msg[0] === 'ready') {
-                if (readyTimer)
-                    clearTimeout(readyTimer);
-                isReady = true;
-                resolve();
+        });
+        workerSettledPromise = runWorkerPromise.then(() => {
+            generation.workerDone = true;
+            releaseProcessEnvWhenWorkerDone();
+            handleUnexpectedWorkerExit();
+            if (generation.cleanupDone)
+                parent?.removeAllListeners();
+            releaseGenerationWhenComplete(generation);
+        }, (error) => {
+            workerError = error;
+            workerFailed = true;
+            generation.workerDone = true;
+            releaseProcessEnvWhenWorkerDone();
+            handleUnexpectedWorkerExit();
+            if (generation.cleanupDone)
+                parent?.removeAllListeners();
+            releaseGenerationWhenComplete(generation);
+        });
+        const workerStartupPromise = runWorkerPromise.then(() => {
+            if (!isReady) {
+                throw new Error('zero-cache CF embed: runWorker exited before ready');
             }
         });
-    });
-    // start zero-cache
-    runWorkerPromise = runWorkerFn(wrappedParent, env).catch((err) => {
-        if (debugEmbed)
-            console.error('[orez-zero-cache-cf] runWorker error', err);
-        if (!isReady) {
-            throw err;
-        }
-        // after ready, errors during shutdown are expected
-    });
-    const workerStartupPromise = runWorkerPromise.then(() => {
-        if (!isReady) {
-            throw new Error('zero-cache CF embed: runWorker exited before ready');
-        }
-    });
-    // wait for ready
-    try {
         await Promise.race([readyPromise, workerStartupPromise]);
+        fastifyInstance = dispatcherFastifyForCFInstance(instanceId);
     }
-    catch (err) {
-        if (readyTimer)
-            clearTimeout(readyTimer);
-        throw err;
+    catch (startupError) {
+        startupFailure = startupError;
+        try {
+            await shutdown();
+        }
+        catch (cleanupError) {
+            throw new AggregateError([startupError, cleanupError], 'zero-cache CF embed: startup failed and teardown also failed', { cause: startupError });
+        }
+        throw startupError;
     }
-    // get the fastify instance (set by our shim during init)
-    fastifyInstance = globalThis.__orez_fastify_instance;
     return {
         get ready() {
             return isReady;
@@ -330,41 +447,7 @@ export async function startZeroCacheEmbedCF(opts) {
             }
             return handleHttpRequest(request, url, fastifyInstance);
         },
-        async stop() {
-            isReady = false;
-            wrappedParent.kill('SIGTERM');
-            if (runWorkerPromise) {
-                await Promise.race([runWorkerPromise, new Promise((r) => setTimeout(r, 5000))]);
-            }
-            await new Promise((r) => setTimeout(r, 200));
-            proxy.close();
-            await Promise.all([
-                backends.postgres.close(),
-                backends.cvr.close(),
-                backends.cdb.close(),
-            ]);
-            // restore all modified globals
-            if (origExit) {
-                ;
-                globalThis.process.exit = origExit;
-            }
-            if (origNodeEnv !== undefined) {
-                ;
-                globalThis.process.env.NODE_ENV = origNodeEnv;
-            }
-            if (origKill) {
-                ;
-                globalThis.process.kill = origKill;
-            }
-            if (opts.apiFetch) {
-                ;
-                globalThis.fetch = origFetch;
-            }
-            delete globalThis.process.env.SINGLE_PROCESS;
-            delete globalThis.__orez_proxy_connect;
-            delete globalThis.__orez_proxy_user;
-            delete globalThis.__orez_proxy_password;
-        },
+        stop: shutdown,
     };
 }
 // -- HTTP request handling --