@ketvietlab/ketjs 0.1.1

package/dist/server/worker.js

@@ -0,0 +1,428 @@
+// The worker process role. It composes the same AppSpec as HTTP, but opens no
+// socket, builds no theme and reads no session. Jobs retain the tenant, actor and
+// company scope captured when they were enqueued.
+import { randomUUID } from 'node:crypto';
+import { createAppRegistry, restrictManifest } from '../kernel/apps.js';
+import { createAdapterPool } from '../data/pool.js';
+import { migrateOne } from '../data/fleet.js';
+import { createContext } from './ctx.js';
+import { sqliteStore } from './config.js';
+import { jobDefinition } from './jobs.js';
+import { createQueue, JOB_CHANNEL } from './queue.js';
+import { bootRuntime } from './runtime.js';
+import { effectStorage, namespacedStorage, storageFromConfig } from './storage/index.js';
+import { effectTransport, unavailableTransport } from './transport/index.js';
+const parseQueues = (configured, env) => {
+    const value = env.KET_WORKER_QUEUES;
+    if (!value)
+        return configured;
+    const queues = {};
+    for (const item of value.split(',')) {
+        const [name, raw] = item.split(':');
+        const concurrency = Number(raw);
+        if (!name || !/^[a-z][a-z0-9_-]*$/.test(name) || !Number.isInteger(concurrency) || concurrency < 1)
+            throw new Error(`invalid KET_WORKER_QUEUES item "${item}" (expected name:positive-integer)`);
+        queues[name] = concurrency;
+    }
+    return queues;
+};
+async function tenantSource(spec, manifest, config) {
+    const serve = spec.serve ?? {};
+    const bootstrap = config.bootstrapApps ?? serve.bootstrap ?? [];
+    const registries = new WeakMap();
+    const prepare = async (adapter) => {
+        let registry = registries.get(adapter);
+        if (!registry) {
+            if (config.migrateOnBoot)
+                await migrateOne(adapter, manifest);
+            registry = await createAppRegistry(manifest, adapter, { autoInstall: config.autoInstall });
+            if (bootstrap.length && (await registry.enabled()).size === 0)
+                for (const name of bootstrap)
+                    await registry.install(name);
+            registries.set(adapter, registry);
+        }
+        return restrictManifest(manifest, await registry.enabled());
+    };
+    if (!serve.tenants) {
+        const adapter = await (serve.openStore ?? sqliteStore)(config);
+        await prepare(adapter);
+        return {
+            keys: async () => [''],
+            with: async (_key, fn) => fn({ key: '', adapter, live: await prepare(adapter) }),
+            singleAdapter: adapter,
+            close: () => adapter.close(),
+        };
+    }
+    const tenantSpec = serve.tenants;
+    const pool = createAdapterPool({
+        create: (key) => tenantSpec.open(key, config),
+        ...(tenantSpec.max === undefined ? {} : { max: tenantSpec.max }),
+        ...(tenantSpec.idleMs === undefined ? {} : { idleMs: tenantSpec.idleMs }),
+    });
+    return {
+        keys: () => tenantSpec.list(),
+        with: (key, fn) => pool.with(key, async (adapter) => fn({ key, adapter, live: await prepare(adapter) })),
+        singleAdapter: null,
+        close: () => pool.close(),
+    };
+}
+export async function bootWorker(spec, options = {}) {
+    if (!spec.worker || !Object.keys(spec.worker.queues).length)
+        throw new Error(`app "${spec.name}" declares no worker queues`);
+    const env = options.env ?? process.env;
+    const { config, manifest } = await bootRuntime(spec, { env });
+    const baseStorage = await (spec.serve?.openStorage ?? storageFromConfig)(config);
+    const baseTransport = await (spec.serve?.openTransport ?? unavailableTransport)(config);
+    const storageFor = (tenant) => namespacedStorage(baseStorage, tenant || spec.name);
+    const source = await tenantSource(spec, manifest, config);
+    const queues = parseQueues(spec.worker.queues, env);
+    const workerId = options.workerId ?? `${spec.name}-${process.pid}-${randomUUID().slice(0, 8)}`;
+    const now = options.now ?? (() => new Date());
+    const random = options.random ?? Math.random;
+    const log = options.log ?? ((entry) => console.log(JSON.stringify(entry)));
+    const pollMin = spec.worker.pollMinMs ?? 100;
+    const pollMax = spec.worker.pollMaxMs ?? 2_000;
+    const refreshMs = spec.worker.tenantRefreshMs ?? 60_000;
+    const leaseMs = spec.worker.leaseMs ?? 60_000;
+    const shutdownGrace = spec.worker.shutdownGraceMs ?? 15_000;
+    const maxConcurrency = spec.serve?.tenants ? (spec.serve.tenants.max ?? 32) : Infinity;
+    let tenantKeys = [];
+    let refreshedAt = 0;
+    let cursor = 0;
+    let running = false;
+    let closing = false;
+    let loop = null;
+    let unsubscribe = null;
+    let wakeResolve = null;
+    const inFlight = new Set();
+    const controllers = new Map();
+    const active = new Map();
+    const rescuedAt = new Map();
+    const totalActive = () => [...active.values()].reduce((sum, count) => sum + count, 0);
+    const refresh = async () => {
+        if (tenantKeys.length && now().getTime() - refreshedAt < refreshMs)
+            return;
+        tenantKeys = [...new Set(await source.keys())].sort();
+        refreshedAt = now().getTime();
+        if (cursor >= tenantKeys.length)
+            cursor = 0;
+    };
+    const wake = () => {
+        wakeResolve?.();
+        wakeResolve = null;
+    };
+    const waitForWake = (ms) => new Promise((resolve) => {
+        const timer = setTimeout(() => {
+            if (wakeResolve === done)
+                wakeResolve = null;
+            resolve();
+        }, ms);
+        const done = () => {
+            clearTimeout(timer);
+            resolve();
+        };
+        wakeResolve = done;
+    });
+    const retryAt = (attempt) => {
+        const ceiling = Math.min(3_600_000, 1_000 * 2 ** Math.max(0, attempt - 1));
+        return new Date(now().getTime() + Math.floor(random() * ceiling));
+    };
+    const execute = async (tenant, queue, job) => {
+        const started = now().getTime();
+        const definition = jobDefinition(job.job);
+        const meta = tenant.live.jobs[job.job];
+        if (!definition || !meta) {
+            await queue.discard(job.id, new Error(`job "${job.job}" is unknown or its module is disabled`), workerId);
+            log({
+                event: 'discarded',
+                workerId,
+                tenant: tenant.key,
+                jobId: job.id,
+                job: job.job,
+                queue: job.queue,
+                attempt: job.attempt,
+                error: 'unknown or disabled job',
+            });
+            return;
+        }
+        const controller = new AbortController();
+        controllers.set(job.id, controller);
+        let timedOut = false;
+        const timeout = setTimeout(() => {
+            timedOut = true;
+            controller.abort(new Error(`job timed out after ${meta.timeoutMs}ms`));
+        }, meta.timeoutMs);
+        const heartbeat = setInterval(() => {
+            void queue.heartbeat(job.id, workerId, leaseMs).then((held) => {
+                if (!held)
+                    controller.abort(new Error('job lease was lost or cancelled'));
+            });
+        }, Math.max(1_000, Math.floor(leaseMs / 3)));
+        log({
+            event: 'started',
+            workerId,
+            tenant: tenant.key,
+            jobId: job.id,
+            job: job.job,
+            queue: job.queue,
+            attempt: job.attempt,
+        });
+        try {
+            const base = createContext({
+                adapter: tenant.adapter,
+                manifest: tenant.live,
+                fnKey: job.job,
+                kind: 'job',
+                actor: job.actor,
+                scope: job.scope,
+                queueNotify: config.queueNotify,
+            });
+            const context = Object.assign(base, {
+                job: {
+                    id: job.id,
+                    key: job.job,
+                    queue: job.queue,
+                    attempt: job.attempt,
+                    maxAttempts: job.maxAttempts,
+                },
+                signal: controller.signal,
+                storage: effectStorage(storageFor(tenant.key), meta.effects, job.job),
+                transport: effectTransport(baseTransport, meta.effects, job.job),
+            });
+            await definition.handler(context, job.args);
+            if (timedOut || controller.signal.aborted) {
+                log({
+                    event: 'handler_ignored_abort',
+                    workerId,
+                    tenant: tenant.key,
+                    jobId: job.id,
+                    job: job.job,
+                    queue: job.queue,
+                    attempt: job.attempt,
+                    durationMs: now().getTime() - started,
+                    error: String(controller.signal.reason ?? 'job aborted'),
+                });
+                throw controller.signal.reason ?? new Error('job aborted');
+            }
+            if (!(await queue.complete(job.id, workerId))) {
+                log({
+                    event: 'cancelled',
+                    workerId,
+                    tenant: tenant.key,
+                    jobId: job.id,
+                    job: job.job,
+                    queue: job.queue,
+                    attempt: job.attempt,
+                    durationMs: now().getTime() - started,
+                });
+                return;
+            }
+            log({
+                event: 'completed',
+                workerId,
+                tenant: tenant.key,
+                jobId: job.id,
+                job: job.job,
+                queue: job.queue,
+                attempt: job.attempt,
+                durationMs: now().getTime() - started,
+            });
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            if (job.attempt >= job.maxAttempts) {
+                const changed = await queue.discard(job.id, error, workerId);
+                if (!changed) {
+                    log({
+                        event: 'cancelled',
+                        workerId,
+                        tenant: tenant.key,
+                        jobId: job.id,
+                        job: job.job,
+                        queue: job.queue,
+                        attempt: job.attempt,
+                        durationMs: now().getTime() - started,
+                    });
+                    return;
+                }
+                log({
+                    event: 'discarded',
+                    workerId,
+                    tenant: tenant.key,
+                    jobId: job.id,
+                    job: job.job,
+                    queue: job.queue,
+                    attempt: job.attempt,
+                    durationMs: now().getTime() - started,
+                    error: message,
+                });
+            }
+            else {
+                const changed = await queue.retry(job.id, error, retryAt(job.attempt), workerId);
+                if (!changed) {
+                    log({
+                        event: 'cancelled',
+                        workerId,
+                        tenant: tenant.key,
+                        jobId: job.id,
+                        job: job.job,
+                        queue: job.queue,
+                        attempt: job.attempt,
+                        durationMs: now().getTime() - started,
+                    });
+                    return;
+                }
+                log({
+                    event: 'retrying',
+                    workerId,
+                    tenant: tenant.key,
+                    jobId: job.id,
+                    job: job.job,
+                    queue: job.queue,
+                    attempt: job.attempt,
+                    durationMs: now().getTime() - started,
+                    error: message,
+                });
+            }
+        }
+        finally {
+            clearTimeout(timeout);
+            clearInterval(heartbeat);
+            controllers.delete(job.id);
+        }
+    };
+    const startClaimed = (tenantKey, queueName, jobs) => {
+        for (const job of jobs) {
+            active.set(queueName, (active.get(queueName) ?? 0) + 1);
+            let task;
+            task = source
+                .with(tenantKey, async (tenant) => execute(tenant, await createQueue(tenant.adapter), job))
+                .catch((error) => {
+                console.error(JSON.stringify({ event: 'worker_error', workerId, tenant: tenantKey, error: String(error) }));
+            })
+                .finally(() => {
+                active.set(queueName, Math.max(0, (active.get(queueName) ?? 1) - 1));
+                inFlight.delete(task);
+                wake();
+            });
+            inFlight.add(task);
+        }
+    };
+    const runOnce = async () => {
+        if (closing)
+            return 0;
+        await refresh();
+        if (!tenantKeys.length)
+            return 0;
+        let claimed = 0;
+        const ordered = [...tenantKeys.slice(cursor), ...tenantKeys.slice(0, cursor)];
+        cursor = (cursor + 1) % tenantKeys.length;
+        for (const key of ordered) {
+            if (closing || totalActive() >= maxConcurrency)
+                break;
+            await source.with(key, async (tenant) => {
+                const queue = await createQueue(tenant.adapter);
+                if (now().getTime() - (rescuedAt.get(key) ?? 0) >= Math.max(1_000, Math.floor(leaseMs / 3))) {
+                    await queue.rescue();
+                    rescuedAt.set(key, now().getTime());
+                }
+                for (const [name, limit] of Object.entries(queues)) {
+                    const room = Math.min(limit - (active.get(name) ?? 0), maxConcurrency - totalActive());
+                    if (room <= 0)
+                        continue;
+                    const jobs = await queue.claimBatch(name, {
+                        workerId,
+                        leaseMs,
+                        limit: Math.min(10, room),
+                    });
+                    claimed += jobs.length;
+                    startClaimed(key, name, jobs);
+                }
+            });
+        }
+        return claimed;
+    };
+    const drain = async () => {
+        let total = 0;
+        for (;;) {
+            const count = await runOnce();
+            total += count;
+            if (inFlight.size)
+                await Promise.allSettled([...inFlight]);
+            if (count === 0)
+                return total;
+        }
+    };
+    const start = () => {
+        if (running || closing)
+            return;
+        running = true;
+        loop = (async () => {
+            let idle = pollMin;
+            while (!closing) {
+                const claimed = await runOnce().catch((error) => {
+                    console.error(JSON.stringify({ event: 'worker_tick_error', workerId, error: String(error) }));
+                    return 0;
+                });
+                idle = claimed ? pollMin : Math.min(pollMax, Math.max(pollMin, idle * 2));
+                if (!closing)
+                    await waitForWake(idle);
+            }
+        })();
+    };
+    if (source.singleAdapter?.notifications?.subscribe && config.queueNotify) {
+        try {
+            unsubscribe = await source.singleAdapter.notifications.subscribe(JOB_CHANNEL, wake, wake);
+        }
+        catch (error) {
+            console.warn(JSON.stringify({
+                event: 'queue_notifier_unavailable',
+                workerId,
+                error: String(error),
+                fallback: 'polling',
+            }));
+        }
+    }
+    return {
+        workerId,
+        manifest,
+        runOnce,
+        drain,
+        start,
+        async close() {
+            if (closing)
+                return;
+            closing = true;
+            wake();
+            if (loop)
+                await loop;
+            const settled = Promise.allSettled([...inFlight]).then(() => undefined);
+            let graceful = false;
+            let graceTimer;
+            await Promise.race([
+                settled.then(() => {
+                    graceful = true;
+                }),
+                new Promise((resolve) => {
+                    graceTimer = setTimeout(resolve, shutdownGrace);
+                }),
+            ]);
+            if (graceTimer)
+                clearTimeout(graceTimer);
+            if (!graceful)
+                for (const controller of controllers.values())
+                    controller.abort(new Error('worker shutting down'));
+            await unsubscribe?.();
+            await source.close();
+            await baseTransport.close?.();
+        },
+    };
+}
+export async function serveWorker(spec, options = {}) {
+    const worker = await bootWorker(spec, options);
+    worker.start();
+    for (const signal of ['SIGINT', 'SIGTERM'])
+        process.on(signal, () => void worker.close().then(() => process.exit(0)));
+    return worker;
+}
+//# sourceMappingURL=worker.js.map

package/dist/server/worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../../../../../packages/ketjs/src/server/worker.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,kFAAkF;AAClF,kDAAkD;AAElD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AACzC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACxF,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAsC5E,MAAM,WAAW,GAAG,CAAC,UAAkC,EAAE,GAAuC,EAAE,EAAE;IAClG,MAAM,KAAK,GAAG,GAAG,CAAC,iBAAiB,CAAA;IACnC,IAAI,CAAC,KAAK;QAAE,OAAO,UAAU,CAAA;IAC7B,MAAM,MAAM,GAA2B,EAAE,CAAA;IACzC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QACnC,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;QAC/B,IAAI,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC;YAChG,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,oCAAoC,CAAC,CAAA;QAC9F,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAA;IAC5B,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AAED,KAAK,UAAU,YAAY,CAAC,IAAa,EAAE,QAAkB,EAAE,MAAqB;IAClF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAA;IAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,KAAK,CAAC,SAAS,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,OAAO,EAA0D,CAAA;IAExF,MAAM,OAAO,GAAG,KAAK,EAAE,OAAgB,EAAqB,EAAE;QAC5D,IAAI,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACtC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,IAAI,MAAM,CAAC,aAAa;gBAAE,MAAM,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;YAC7D,QAAQ,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,CAAA;YAC1F,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC;gBAC3D,KAAK,MAAM,IAAI,IAAI,SAAS;oBAAE,MAAM,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAC5D,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QACnC,CAAC;QACD,OAAO,gBAAgB,CAAC,QAAQ,EAAE,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAA;IAC7D,CAAC,CAAA;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,IAAI,WAAW,CAAC,CAAC,MAAM,CAAC,CAAA;QAC9D,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;QACtB,OAAO;YACL,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;YACtB,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAChF,aAAa,EAAE,OAAO;YACtB,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE;SAC7B,CAAA;IACH,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAA;IAChC,MAAM,IAAI,GAAG,iBAAiB,CAAC;QAC7B,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAY;QACxD,GAAG,CAAC,UAAU,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,CAAC,GAAG,EAAE,CAAC;QAChE,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC;KAC1E,CAAC,CAAA;IACF,OAAO;QACL,IAAI,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE;QAC7B,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACxG,aAAa,EAAE,IAAI;QACnB,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE;KAC1B,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,IAAa,EACb,OAAO,GAMH,EAAE;IAEN,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM;QACzD,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,IAAI,6BAA6B,CAAC,CAAA;IAEjE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAA;IACtC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAA;IAC7D,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,IAAI,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAA;IAChF,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,oBAAoB,CAAC,CAAC,MAAM,CAAC,CAAA;IACvF,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,EAAE,CAAC,iBAAiB,CAAC,WAAW,EAAE,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,CAAA;IAE1F,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAA;IACzD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAA;IAC9F,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;IAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IAC5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,KAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACrF,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,GAAG,CAAA;IAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,KAAK,CAAA;IAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,MAAM,CAAA;IACvD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAA;IAC7C,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,MAAM,CAAA;IAC3D,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAA;IAEtF,IAAI,UAAU,GAAa,EAAE,CAAA;IAC7B,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,IAAI,GAAyB,IAAI,CAAA;IACrC,IAAI,WAAW,GAAiC,IAAI,CAAA;IACpD,IAAI,WAAW,GAAwB,IAAI,CAAA;IAC3C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAiB,CAAA;IACzC,MAAM,WAAW,GAAG,IAAI,GAAG,EAA2B,CAAA;IACtD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IACxC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC3C,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAA;IAErF,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE;QACzB,IAAI,UAAU,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,WAAW,GAAG,SAAS;YAAE,OAAM;QAC1E,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QACrD,WAAW,GAAG,GAAG,EAAE,CAAC,OAAO,EAAE,CAAA;QAC7B,IAAI,MAAM,IAAI,UAAU,CAAC,MAAM;YAAE,MAAM,GAAG,CAAC,CAAA;IAC7C,CAAC,CAAA;IAED,MAAM,IAAI,GAAG,GAAG,EAAE;QAChB,WAAW,EAAE,EAAE,CAAA;QACf,WAAW,GAAG,IAAI,CAAA;IACpB,CAAC,CAAA;IAED,MAAM,WAAW,GAAG,CAAC,EAAU,EAAE,EAAE,CACjC,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAC5B,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,IAAI,WAAW,KAAK,IAAI;gBAAE,WAAW,GAAG,IAAI,CAAA;YAC5C,OAAO,EAAE,CAAA;QACX,CAAC,EAAE,EAAE,CAAC,CAAA;QACN,MAAM,IAAI,GAAG,GAAG,EAAE;YAChB,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,OAAO,EAAE,CAAA;QACX,CAAC,CAAA;QACD,WAAW,GAAG,IAAI,CAAA;IACpB,CAAC,CAAC,CAAA;IAEJ,MAAM,OAAO,GAAG,CAAC,OAAe,EAAQ,EAAE;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;QAC1E,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAA;IACnE,CAAC,CAAA;IAED,MAAM,OAAO,GAAG,KAAK,EAAE,MAAoB,EAAE,KAAY,EAAE,GAAe,EAAiB,EAAE;QAC3F,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,EAAE,CAAA;QAC/B,MAAM,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACtC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,EAAE,CAAC;YACzB,MAAM,KAAK,CAAC,OAAO,CACjB,GAAG,CAAC,EAAE,EACN,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,GAAG,wCAAwC,CAAC,EAClE,QAAQ,CACT,CAAA;YACD,GAAG,CAAC;gBACF,KAAK,EAAE,WAAW;gBAClB,QAAQ;gBACR,MAAM,EAAE,MAAM,CAAC,GAAG;gBAClB,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,KAAK,EAAE,yBAAyB;aACjC,CAAC,CAAA;YACF,OAAM;QACR,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;QACxC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAA;QACnC,IAAI,QAAQ,GAAG,KAAK,CAAA;QACpB,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;YAC9B,QAAQ,GAAG,IAAI,CAAA;YACf,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,CAAA;QACxE,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;QAClB,MAAM,SAAS,GAAG,WAAW,CAC3B,GAAG,EAAE;YACH,KAAK,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC5D,IAAI,CAAC,IAAI;oBAAE,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAA;YAC3E,CAAC,CAAC,CAAA;QACJ,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CACzC,CAAA;QAED,GAAG,CAAC;YACF,KAAK,EAAE,SAAS;YAChB,QAAQ;YACR,MAAM,EAAE,MAAM,CAAC,GAAG;YAClB,KAAK,EAAE,GAAG,CAAC,EAAE;YACb,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,OAAO,EAAE,GAAG,CAAC,OAAO;SACrB,CAAC,CAAA;QAEF,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,aAAa,CAAC;gBACzB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,QAAQ,EAAE,MAAM,CAAC,IAAI;gBACrB,KAAK,EAAE,GAAG,CAAC,GAAG;gBACd,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,WAAW,EAAE,MAAM,CAAC,WAAW;aAChC,CAAC,CAAA;YACF,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;gBAClC,GAAG,EAAE;oBACH,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,GAAG,EAAE,GAAG,CAAC,GAAG;oBACZ,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,WAAW,EAAE,GAAG,CAAC,WAAW;iBAC7B;gBACD,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;gBACrE,SAAS,EAAE,eAAe,CAAC,aAAa,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC;aACjE,CAAe,CAAA;YAChB,MAAM,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CAAA;YAC3C,IAAI,QAAQ,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC1C,GAAG,CAAC;oBACF,KAAK,EAAE,uBAAuB;oBAC9B,QAAQ;oBACR,MAAM,EAAE,MAAM,CAAC,GAAG;oBAClB,KAAK,EAAE,GAAG,CAAC,EAAE;oBACb,GAAG,EAAE,GAAG,CAAC,GAAG;oBACZ,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,OAAO;oBACrC,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,IAAI,aAAa,CAAC;iBACzD,CAAC,CAAA;gBACF,MAAM,UAAU,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,aAAa,CAAC,CAAA;YAC5D,CAAC;YACD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAC9C,GAAG,CAAC;oBACF,KAAK,EAAE,WAAW;oBAClB,QAAQ;oBACR,MAAM,EAAE,MAAM,CAAC,GAAG;oBAClB,KAAK,EAAE,GAAG,CAAC,EAAE;oBACb,GAAG,EAAE,GAAG,CAAC,GAAG;oBACZ,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,OAAO;iBACtC,CAAC,CAAA;gBACF,OAAM;YACR,CAAC;YACD,GAAG,CAAC;gBACF,KAAK,EAAE,WAAW;gBAClB,QAAQ;gBACR,MAAM,EAAE,MAAM,CAAC,GAAG;gBAClB,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,OAAO;aACtC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACtE,IAAI,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;gBACnC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;gBAC5D,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,GAAG,CAAC;wBACF,KAAK,EAAE,WAAW;wBAClB,QAAQ;wBACR,MAAM,EAAE,MAAM,CAAC,GAAG;wBAClB,KAAK,EAAE,GAAG,CAAC,EAAE;wBACb,GAAG,EAAE,GAAG,CAAC,GAAG;wBACZ,KAAK,EAAE,GAAG,CAAC,KAAK;wBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,OAAO;qBACtC,CAAC,CAAA;oBACF,OAAM;gBACR,CAAC;gBACD,GAAG,CAAC;oBACF,KAAK,EAAE,WAAW;oBAClB,QAAQ;oBACR,MAAM,EAAE,MAAM,CAAC,GAAG;oBAClB,KAAK,EAAE,GAAG,CAAC,EAAE;oBACb,GAAG,EAAE,GAAG,CAAC,GAAG;oBACZ,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,OAAO;oBACrC,KAAK,EAAE,OAAO;iBACf,CAAC,CAAA;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAA;gBAChF,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,GAAG,CAAC;wBACF,KAAK,EAAE,WAAW;wBAClB,QAAQ;wBACR,MAAM,EAAE,MAAM,CAAC,GAAG;wBAClB,KAAK,EAAE,GAAG,CAAC,EAAE;wBACb,GAAG,EAAE,GAAG,CAAC,GAAG;wBACZ,KAAK,EAAE,GAAG,CAAC,KAAK;wBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,OAAO;qBACtC,CAAC,CAAA;oBACF,OAAM;gBACR,CAAC;gBACD,GAAG,CAAC;oBACF,KAAK,EAAE,UAAU;oBACjB,QAAQ;oBACR,MAAM,EAAE,MAAM,CAAC,GAAG;oBAClB,KAAK,EAAE,GAAG,CAAC,EAAE;oBACb,GAAG,EAAE,GAAG,CAAC,GAAG;oBACZ,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,OAAO,EAAE,GAAG,CAAC,OAAO;oBACpB,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,OAAO;oBACrC,KAAK,EAAE,OAAO;iBACf,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,OAAO,CAAC,CAAA;YACrB,aAAa,CAAC,SAAS,CAAC,CAAA;YACxB,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC5B,CAAC;IACH,CAAC,CAAA;IAED,MAAM,YAAY,GAAG,CAAC,SAAiB,EAAE,SAAiB,EAAE,IAAkB,EAAE,EAAE;QAChF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACvD,IAAI,IAAmB,CAAA;YACvB,IAAI,GAAG,MAAM;iBACV,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC;iBAC1F,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,OAAO,CAAC,KAAK,CACX,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAC7F,CAAA;YACH,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBACpE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBACrB,IAAI,EAAE,CAAA;YACR,CAAC,CAAC,CAAA;YACJ,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACpB,CAAC;IACH,CAAC,CAAA;IAED,MAAM,OAAO,GAAG,KAAK,IAAqB,EAAE;QAC1C,IAAI,OAAO;YAAE,OAAO,CAAC,CAAA;QACrB,MAAM,OAAO,EAAE,CAAA;QACf,IAAI,CAAC,UAAU,CAAC,MAAM;YAAE,OAAO,CAAC,CAAA;QAChC,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,MAAM,OAAO,GAAG,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAA;QAC7E,MAAM,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAA;QACzC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,OAAO,IAAI,WAAW,EAAE,IAAI,cAAc;gBAAE,MAAK;YACrD,MAAM,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;gBACtC,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;gBAC/C,IAAI,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC5F,MAAM,KAAK,CAAC,MAAM,EAAE,CAAA;oBACpB,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;gBACrC,CAAC;gBACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;oBACnD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,cAAc,GAAG,WAAW,EAAE,CAAC,CAAA;oBACtF,IAAI,IAAI,IAAI,CAAC;wBAAE,SAAQ;oBACvB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE;wBACxC,QAAQ;wBACR,OAAO;wBACP,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;qBAC1B,CAAC,CAAA;oBACF,OAAO,IAAI,IAAI,CAAC,MAAM,CAAA;oBACtB,YAAY,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;gBAC/B,CAAC;YACH,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC,CAAA;IAED,MAAM,KAAK,GAAG,KAAK,IAAqB,EAAE;QACxC,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,SAAS,CAAC;YACR,MAAM,KAAK,GAAG,MAAM,OAAO,EAAE,CAAA;YAC7B,KAAK,IAAI,KAAK,CAAA;YACd,IAAI,QAAQ,CAAC,IAAI;gBAAE,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAA;YAC1D,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;QAC/B,CAAC;IACH,CAAC,CAAA;IAED,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,OAAO,IAAI,OAAO;YAAE,OAAM;QAC9B,OAAO,GAAG,IAAI,CAAA;QACd,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE;YACjB,IAAI,IAAI,GAAG,OAAO,CAAA;YAClB,OAAO,CAAC,OAAO,EAAE,CAAC;gBAChB,MAAM,OAAO,GAAG,MAAM,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC9C,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;oBAC7F,OAAO,CAAC,CAAA;gBACV,CAAC,CAAC,CAAA;gBACF,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAA;gBACzE,IAAI,CAAC,OAAO;oBAAE,MAAM,WAAW,CAAC,IAAI,CAAC,CAAA;YACvC,CAAC;QACH,CAAC,CAAC,EAAE,CAAA;IACN,CAAC,CAAA;IAED,IAAI,MAAM,CAAC,aAAa,EAAE,aAAa,EAAE,SAAS,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACzE,IAAI,CAAC;YACH,WAAW,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;QAC3F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CACV,IAAI,CAAC,SAAS,CAAC;gBACb,KAAK,EAAE,4BAA4B;gBACnC,QAAQ;gBACR,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;gBACpB,QAAQ,EAAE,SAAS;aACpB,CAAC,CACH,CAAA;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ;QACR,QAAQ;QACR,OAAO;QACP,KAAK;QACL,KAAK;QACL,KAAK,CAAC,KAAK;YACT,IAAI,OAAO;gBAAE,OAAM;YACnB,OAAO,GAAG,IAAI,CAAA;YACd,IAAI,EAAE,CAAA;YACN,IAAI,IAAI;gBAAE,MAAM,IAAI,CAAA;YACpB,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;YACvE,IAAI,QAAQ,GAAG,KAAK,CAAA;YACpB,IAAI,UAAqD,CAAA;YACzD,MAAM,OAAO,CAAC,IAAI,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;oBAChB,QAAQ,GAAG,IAAI,CAAA;gBACjB,CAAC,CAAC;gBACF,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;oBAC5B,UAAU,GAAG,UAAU,CAAC,OAAO,EAAE,aAAa,CAAC,CAAA;gBACjD,CAAC,CAAC;aACH,CAAC,CAAA;YACF,IAAI,UAAU;gBAAE,YAAY,CAAC,UAAU,CAAC,CAAA;YACxC,IAAI,CAAC,QAAQ;gBACX,KAAK,MAAM,UAAU,IAAI,WAAW,CAAC,MAAM,EAAE;oBAAE,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAA;YACpG,MAAM,WAAW,EAAE,EAAE,CAAA;YACrB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACpB,MAAM,aAAa,CAAC,KAAK,EAAE,EAAE,CAAA;QAC/B,CAAC;KACF,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAa,EACb,OAAO,GAAqC,EAAE;IAE9C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAC9C,MAAM,CAAC,KAAK,EAAE,CAAA;IACd,KAAK,MAAM,MAAM,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAU;QACjD,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3E,OAAO,MAAM,CAAA;AACf,CAAC"}

package/dist/testing.d.ts

@@ -0,0 +1,146 @@
+import { type BootedApp } from './server/boot.ts';
+import { type BootedWorker } from './server/worker.ts';
+import type { WorkerLog } from './server/worker.ts';
+import { type CallResult } from './server/fn.ts';
+import type { AppSpec } from './kernel/workspace.ts';
+import type { AppRegistry } from './kernel/apps.ts';
+import type { Adapter, Manifest, Scope } from './types.ts';
+export type TestIdentity = {
+    /** The company new rows are written to. Ignored by apps using sessions. */
+    company?: string | null;
+    /** Other companies a read may include. The write company is included automatically. */
+    companies?: readonly string[] | null;
+    /** The operational branch new company+branch rows are written to. */
+    branch?: string | null;
+    /** Operational branches visible to this client. */
+    branches?: readonly string[] | null;
+    /** Conventional test tenant header. Override tenantHeader for another resolver. */
+    tenant?: string | null;
+    tenantHeader?: string;
+    locale?: string | null;
+    headers?: HeadersInit;
+};
+export type TestClientOptions = TestIdentity & {
+    jar?: CookieJar;
+    /** Follow redirects while retaining cookies set by intermediate responses. */
+    followRedirects?: boolean;
+    maxRedirects?: number;
+};
+export type TestCallOptions = {
+    dryRun?: boolean;
+    idempotencyKey?: string;
+    headers?: HeadersInit;
+    signal?: AbortSignal;
+};
+/** A small, deterministic cookie jar for one test user. */
+export declare class CookieJar {
+    #private;
+    constructor(initial?: Record<string, string>);
+    set(name: string, value: string): void;
+    delete(name: string): void;
+    clear(): void;
+    get(name: string): string | undefined;
+    get size(): number;
+    header(): string;
+    toJSON(): Record<string, string>;
+    clone(): CookieJar;
+    /** Capture every Set-Cookie header before a redirect is followed. */
+    capture(headers: Headers): void;
+    private captureOne;
+    save(path: string): Promise<void>;
+    static load(path: string): Promise<CookieJar>;
+}
+export declare class TestHttpError extends Error {
+    readonly status: number;
+    readonly response: Response;
+    readonly body: unknown;
+    constructor(response: Response, body: unknown);
+}
+/** A real HTTP client with isolated identity and a persistent cookie jar. */
+export declare class TestClient {
+    #private;
+    readonly baseUrl: string;
+    readonly jar: CookieJar;
+    constructor(baseUrl: string | URL, options?: TestClientOptions);
+    private applyIdentity;
+    /** A client for the same logged-in user with a narrowed/overridden request identity. */
+    with(options?: TestClientOptions): TestClient;
+    as(identity: TestIdentity): TestClient;
+    /** A client at the same endpoint with no cookies. */
+    anonymous(identity?: TestIdentity): TestClient;
+    request(path: string | URL, init?: RequestInit): Promise<Response>;
+    get(path: string | URL, init?: RequestInit): Promise<Response>;
+    post(path: string | URL, body?: BodyInit | null, init?: RequestInit): Promise<Response>;
+    json<T>(path: string | URL, body?: unknown, init?: RequestInit): Promise<T>;
+    form<T>(path: string | URL, values: Record<string, string>, init?: RequestInit): Promise<T>;
+    call<T = unknown>(name: string, input?: Record<string, unknown>, options?: TestCallOptions): Promise<CallResult & {
+        value: T;
+    }>;
+    login<T = unknown>(credentials: Record<string, unknown>, path?: string): Promise<T>;
+    logout<T = unknown>(path?: string): Promise<T>;
+}
+export type CreateTestAppOptions = {
+    /** Explicit runtime variables. Host process variables are ignored unless inheritEnv is true. */
+    env?: Record<string, string | undefined>;
+    inheritEnv?: boolean;
+    /** A caller-owned directory. It is never removed by close(). */
+    artifactsDir?: string;
+    /** Keep the automatically-created database and storage for debugging. */
+    keepArtifacts?: boolean;
+    /** Boot a worker handle when the app declares queues. Defaults to true. */
+    worker?: boolean;
+    /** Harness progress. Silent by default so test reporter output stays readable. */
+    log?: (line: string) => void;
+    workerLog?: (entry: WorkerLog) => void;
+    client?: TestClientOptions;
+    port?: number;
+};
+export type TestFixtureCallOptions = {
+    tenant?: string;
+    scope?: Scope;
+    actor?: string | null;
+    allow?: readonly string[] | null;
+    dryRun?: boolean;
+    idempotencyKey?: string | null;
+};
+export type TestFixtureTenant = {
+    key: string;
+    adapter: Adapter;
+    manifest: Manifest;
+    apps: AppRegistry;
+};
+export type TestFixtures = {
+    /**
+     * Seed through a declared function without HTTP identity checks.
+     *
+     * Keep business actions under test on TestClient. This channel is deliberately
+     * named fixture so bypassing the public boundary cannot look accidental.
+     */
+    call<T = unknown>(name: string, input?: Record<string, unknown>, options?: TestFixtureCallOptions): Promise<CallResult & {
+        value: T;
+    }>;
+    /** Direct datastore access for fixtures and invariants that have no public function. */
+    withTenant<T>(tenant: string, fn: (tenant: TestFixtureTenant) => Promise<T>): Promise<T>;
+};
+export type TestApp = {
+    app: BootedApp;
+    worker: BootedWorker | null;
+    client: TestClient;
+    baseUrl: string;
+    adapter: Adapter | null;
+    artifactsDir: string;
+    databasePath: string | null;
+    env: Readonly<Record<string, string | undefined>>;
+    fixture: TestFixtures;
+    drainJobs(): Promise<number>;
+    close(): Promise<void>;
+};
+/**
+ * Boot a real app on an ephemeral port with isolated database and storage.
+ *
+ * No browser is involved, but every TestClient call crosses the real HTTP
+ * boundary. The host environment is ignored by default so a developer cannot
+ * accidentally point an end-to-end suite at their normal or production database.
+ */
+export declare function createTestApp(spec: AppSpec, options?: CreateTestAppOptions): Promise<TestApp>;
+//# sourceMappingURL=testing.d.ts.map

package/dist/testing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"testing.d.ts","sourceRoot":"","sources":["../../../../packages/ketjs/src/testing.ts"],"names":[],"mappings":"AAUA,OAAO,EAAW,KAAK,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC1D,OAAO,EAAc,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAU,KAAK,UAAU,EAAE,MAAM,gBAAgB,CAAA;AACxD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAE1D,MAAM,MAAM,YAAY,GAAG;IACzB,2EAA2E;IAC3E,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,uFAAuF;IACvF,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAA;IACpC,qEAAqE;IACrE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,mDAAmD;IACnD,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAA;IACnC,mFAAmF;IACnF,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,OAAO,CAAC,EAAE,WAAW,CAAA;CACtB,CAAA;AAED,MAAM,MAAM,iBAAiB,GAAG,YAAY,GAAG;IAC7C,GAAG,CAAC,EAAE,SAAS,CAAA;IACf,8EAA8E;IAC9E,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB,CAAA;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,CAAC,EAAE,WAAW,CAAA;IACrB,MAAM,CAAC,EAAE,WAAW,CAAA;CACrB,CAAA;AAOD,2DAA2D;AAC3D,qBAAa,SAAS;;IAGpB,YAAY,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM,EAE/C;IAED,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAGrC;IAED,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAEzB;IAED,KAAK,IAAI,IAAI,CAEZ;IAED,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAEpC;IAED,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,MAAM,IAAI,MAAM,CAEf;IAED,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAE/B;IAED,KAAK,IAAI,SAAS,CAEjB;IAED,qEAAqE;IACrE,OAAO,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAK9B;IAED,OAAO,CAAC,UAAU;IAqBZ,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAItC;IAED,OAAa,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAelD;CACF;AAED,qBAAa,aAAc,SAAQ,KAAK;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAA;IAC3B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;IAEtB,YAAY,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAU5C;CACF;AAkBD,6EAA6E;AAC7E,qBAAa,UAAU;;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;IAMvB,YAAY,OAAO,EAAE,MAAM,GAAG,GAAG,EAAE,OAAO,GAAE,iBAAsB,EAmBjE;IAED,OAAO,CAAC,aAAa;IAarB,wFAAwF;IACxF,IAAI,CAAC,OAAO,GAAE,iBAAsB,GAAG,UAAU,CAehD;IAED,EAAE,CAAC,QAAQ,EAAE,YAAY,GAAG,UAAU,CAErC;IAED,qDAAqD;IACrD,SAAS,CAAC,QAAQ,GAAE,YAAiB,GAAG,UAAU,CAEjD;IAEK,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,CA6C3E;IAED,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAEjE;IAED,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,EAAE,QAAQ,GAAG,IAAI,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAE1F;IAEK,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAYpF;IAEK,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAYpG;IAEK,IAAI,CAAC,CAAC,GAAG,OAAO,EACpB,IAAI,EAAE,MAAM,EACZ,KAAK,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,EACnC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,UAAU,GAAG;QAAE,KAAK,EAAE,CAAC,CAAA;KAAE,CAAC,CAepC;IAED,KAAK,CAAC,CAAC,GAAG,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,SAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAEpF;IAED,MAAM,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,SAAY,GAAG,OAAO,CAAC,CAAC,CAAC,CAEhD;CACF;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,gGAAgG;IAChG,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAA;IACxC,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,gEAAgE;IAChE,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,yEAAyE;IACzE,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,2EAA2E;IAC3E,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,kFAAkF;IAClF,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;IAC5B,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,IAAI,CAAA;IACtC,MAAM,CAAC,EAAE,iBAAiB,CAAA;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAA;CACd,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG;IACnC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,KAAK,CAAA;IACb,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAA;IAChC,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAC/B,CAAA;AAED,MAAM,MAAM,iBAAiB,GAAG;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,OAAO,CAAA;IAChB,QAAQ,EAAE,QAAQ,CAAA;IAClB,IAAI,EAAE,WAAW,CAAA;CAClB,CAAA;AAED,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;OAKG;IACH,IAAI,CAAC,CAAC,GAAG,OAAO,EACd,IAAI,EAAE,MAAM,EACZ,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,UAAU,GAAG;QAAE,KAAK,EAAE,CAAC,CAAA;KAAE,CAAC,CAAA;IACrC,wFAAwF;IACxF,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;CACzF,CAAA;AAED,MAAM,MAAM,OAAO,GAAG;IACpB,GAAG,EAAE,SAAS,CAAA;IACd,MAAM,EAAE,YAAY,GAAG,IAAI,CAAA;IAC3B,MAAM,EAAE,UAAU,CAAA;IAClB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,OAAO,GAAG,IAAI,CAAA;IACvB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAA;IACjD,OAAO,EAAE,YAAY,CAAA;IACrB,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;IAC5B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB,CAAA;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,OAAO,CAAC,CAiGvG"}