@antasphere/chassis-server 1.0.0

package/dist/jobs/pgboss.js

@@ -0,0 +1,521 @@
+import PgBoss from 'pg-boss';
+import pg from 'pg';
+import { Counter } from 'prom-client';
+import { sql } from 'drizzle-orm';
+import { parseSuperadminEmails } from '../accounts/superadmin.js';
+import { PGBOSS_SCHEMA, PgBossTimers } from './timers.js';
+/**
+ * pg-boss job runtime. Queue creation and worker registration follow
+ * SERVICE_ROLE:
+ *  - api:    start({ supervise:false, schedule:false, migrate:false }) — can
+ *            send jobs, runs no maintenance, needs no DDL rights.
+ *  - worker: full start, registers handlers.
+ *  - all:    both in one process (single-container default).
+ * pg-boss serializes its own SCHEMA install internally, but NOT createQueue —
+ * so the whole install section runs under the app's advisory lock (below).
+ * First boot must run with role all|worker once.
+ */
+export const USAGE_QUEUE = 'usage-events';
+export const AUDIT_PURGE_QUEUE = 'audit-purge';
+/**
+ * Where a usage batch goes when the retry budget of `USAGE_QUEUE` runs out
+ * (pg-boss's dead letter, PRDCT-2635): a worker logs it at error level,
+ * counts it, and re-sends every event to `USAGE_QUEUE` after
+ * `heldDelaySeconds` — forever. Failed usage is held, never lost.
+ */
+export const USAGE_HELD_QUEUE = 'usage-events-held';
+/**
+ * The singleton key a window hold (PRDCT-2644) carries on the held queue:
+ * pg-boss's dead-letter copy carries none, so the held worker tells a
+ * retry-budget hold from a window hold by it. The held queue's policy is
+ * `standard` and the send names no `singletonSeconds`, so the key enforces
+ * nothing; it is a label.
+ */
+const WINDOW_HOLD_PREFIX = 'occurred_at_window:';
+const windowHoldKey = (id) => WINDOW_HOLD_PREFIX + id;
+const isWindowHoldKey = (key) => typeof key === 'string' && key.startsWith(WINDOW_HOLD_PREFIX);
+export const APIKEY_EXPIRY_QUEUE = 'apikey-expiry-sweep';
+export const IDEMPOTENCY_PURGE_QUEUE = 'idempotency-purge';
+export const ORPHAN_USER_PURGE_QUEUE = 'orphan-user-purge';
+/**
+ * Advisory-lock key serializing pg-boss's install DDL across replicas.
+ *
+ * pg-boss wraps its schema migration in its own advisory lock, but
+ * `createQueue()` calls `pgboss.create_queue()` (queue-row insert + per-queue
+ * partition CREATE TABLE/attach) with NO lock: two or more all|worker
+ * replicas booting a fresh database reliably deadlock there (Postgres
+ * DeadLockReport; scale drill, I2). Same session-scoped pattern as the
+ * migration lock (packages/chassis-db/src/migrate.ts). The key is distinct from the
+ * migration lock (7432001) and the last-owner locks (7432002 trigger xact /
+ * 7432003 session) — see internal/scale-drill-findings.md.
+ */
+const PGBOSS_INSTALL_LOCK_KEY = 7_432_004;
+/**
+ * Run pg-boss's one-time install section (schema + queues + schedules) under
+ * a session-scoped pg_advisory_lock on a dedicated client — never the pool
+ * (a session lock parked on a pooled connection would poison the pool). Held
+ * across boot setup ONLY; steady-state send/work never touches it.
+ */
+async function withInstallLock(connectionString, logger, fn) {
+    const client = new pg.Client({ connectionString });
+    await client.connect();
+    try {
+        await client.query('SELECT pg_advisory_lock($1)', [PGBOSS_INSTALL_LOCK_KEY]);
+        logger.info({ scope: 'pgboss' }, 'advisory lock acquired, installing pg-boss schema and queues');
+        await fn();
+        logger.info({ scope: 'pgboss' }, 'pg-boss install up to date');
+    }
+    finally {
+        try {
+            await client.query('SELECT pg_advisory_unlock($1)', [PGBOSS_INSTALL_LOCK_KEY]);
+        }
+        finally {
+            await client.end();
+        }
+    }
+}
+export async function createJobs(env, db, 
+/** The pool behind `db`: a keyed timer's replace runs on a client of its own (`./timers.ts`). */
+pool, logger, downstreamUsage, 
+/** Orphan purge deletes through Better Auth's own internalAdapter (FK-safe cascade). */
+auth, 
+/** System-actor audit rows for the orphan purge. */
+audit, 
+/** The tool's own jobs, installed and polled between the chassis ones. */
+toolJobs = [], 
+/** The usage queue's retry budget and the hold before a re-drive (a test shrinks both). */
+usageRetry = DEFAULT_USAGE_RETRY) {
+    const isWorker = env.SERVICE_ROLE === 'all' || env.SERVICE_ROLE === 'worker';
+    // registers: [] — boot attaches it to the app registry.
+    const held = new Counter({
+        name: 'usage_events_held_total',
+        help: 'Usage events held for a re-drive, by reason: retry_budget (the retry budget ran out), occurred_at_window (the hub would refuse the date)',
+        labelNames: ['reason'],
+        registers: []
+    });
+    const boss = new PgBoss({
+        connectionString: env.DATABASE_URL,
+        schema: PGBOSS_SCHEMA,
+        // Completed jobs archive after an hour and the archive is dropped after a
+        // week — the pgboss tables stay bounded on a busy instance.
+        archiveCompletedAfterSeconds: 3600,
+        deleteAfterDays: 7,
+        ...(isWorker ? {} : { supervise: false, schedule: false, migrate: false })
+    });
+    boss.on('error', (err) => logger.error({ err }, 'pg-boss error'));
+    const retentionDays = env.AUDIT_RETENTION_DAYS;
+    const orphanRetentionHours = env.ORPHAN_USER_RETENTION_HOURS;
+    if (isWorker) {
+        // The install section — pg-boss schema (start) + queue partitions
+        // (createQueue) + nightly schedules — is one-time DDL and runs under the
+        // install advisory lock so concurrent fresh-DB boots serialize instead
+        // of deadlocking in pgboss.create_queue. Handler registration below runs
+        // AFTER release: work() is an in-process poller, no DDL, and steady-state
+        // must never hold the lock.
+        await withInstallLock(env.DATABASE_URL, logger, async () => {
+            await boss.start();
+            // The held queue first, then the usage queue pointed at it as its dead
+            // letter — ON THE QUEUE, never on a send: an api-role replica sends
+            // without DDL rights and may roll before the worker of this release,
+            // and pg-boss's job row holds a foreign key on the dead letter it
+            // names, so a per-send dead letter would fail every send until the
+            // held queue exists (/code-review, lane D). pg-boss stamps the queue's
+            // dead letter on each job row at insert, so it covers every job sent
+            // after `updateQueue` ran, whoever sent it; on an installation that
+            // predates the held queue, a job an api-role replica sends between its
+            // own boot and this worker's install carries none (verifier round 2,
+            // accepted: one rollout's gap, and only a long outage on top would end
+            // it in a terminal failure). What pg-boss 10 gives a
+            // held job: the dead-letter copy carries the ORIGINAL retry limit (ten)
+            // and no delay, so a re-send that throws (a local insert refused while
+            // fetches still succeed) is retried ten times back to back, then that
+            // held job fails for good; the retry options on the held queue below
+            // apply to jobs sent to it, not to the copies pg-boss makes (verifier
+            // round 1, accepted: the path needs the database to refuse eleven
+            // inserts in a row while serving the fetches).
+            await boss.createQueue(USAGE_HELD_QUEUE, {
+                name: USAGE_HELD_QUEUE,
+                retryLimit: 10,
+                retryDelay: 60,
+                retryBackoff: true
+            });
+            await boss.createQueue(USAGE_QUEUE);
+            await boss.updateQueue(USAGE_QUEUE, { name: USAGE_QUEUE, deadLetter: USAGE_HELD_QUEUE });
+            // Audit retention queue: 0 = keep forever; the queue always exists so
+            // the schedule can be flipped later.
+            await boss.createQueue(AUDIT_PURGE_QUEUE);
+            if (retentionDays > 0) {
+                await boss.schedule(AUDIT_PURGE_QUEUE, '0 3 * * *');
+            }
+            else {
+                await boss.unschedule(AUDIT_PURGE_QUEUE).catch(() => { });
+            }
+            await boss.createQueue(APIKEY_EXPIRY_QUEUE);
+            await boss.schedule(APIKEY_EXPIRY_QUEUE, '0 3 * * *');
+            await boss.createQueue(IDEMPOTENCY_PURGE_QUEUE);
+            await boss.schedule(IDEMPOTENCY_PURGE_QUEUE, '0 3 * * *');
+            // The tool's queues: same install section, same lock, declaration order.
+            for (const job of toolJobs) {
+                await boss.createQueue(job.queue);
+                if (!job.schedule)
+                    continue;
+                if (job.schedule.enabled !== false) {
+                    await boss.schedule(job.queue, job.schedule.cron);
+                }
+                else {
+                    await boss.unschedule(job.queue).catch(() => { });
+                }
+            }
+            // Orphaned-user GC: 0 = disabled; the queue always exists so the
+            // schedule can be flipped later (audit-purge pattern).
+            await boss.createQueue(ORPHAN_USER_PURGE_QUEUE);
+            if (orphanRetentionHours > 0) {
+                await boss.schedule(ORPHAN_USER_PURGE_QUEUE, '0 3 * * *');
+            }
+            else {
+                await boss.unschedule(ORPHAN_USER_PURGE_QUEUE).catch(() => { });
+            }
+        });
+        // Usage events flow through the durable queue (at-least-once); the
+        // downstream sink (no-op locally, the rail in cloud) dedupes by ULID.
+        // A downstream that takes batches (the hub poster) gets the drained
+        // batch whole; a throw fails every job of it back to the queue's retry.
+        //
+        // A batch downstream may answer an outcome naming events it HELD rather
+        // than posted (PRDCT-2644: an `occurredAt` outside the hub's window).
+        // Each is sent to the held queue, which re-drives it here after
+        // `heldDelaySeconds`, forever, as it does a retry-budget hold: a
+        // future-dated event is posted once its time has come, a stale one stays
+        // held until an operator acts. No event the hub would refuse on its date
+        // is ever posted, and none is dropped.
+        await boss.work(USAGE_QUEUE, { batchSize: 50 }, async (jobs) => {
+            if (downstreamUsage.emitBatch) {
+                const outcome = await downstreamUsage.emitBatch(jobs.map((job) => job.data));
+                if (outcome && outcome.held.length > 0) {
+                    const heldIds = new Set(outcome.held.map((h) => h.id));
+                    const heldJobs = jobs.filter((job) => heldIds.has(job.data.id));
+                    for (const job of heldJobs) {
+                        await boss.send(USAGE_HELD_QUEUE, job.data, { singletonKey: windowHoldKey(job.data.id) });
+                    }
+                    logger.warn({
+                        ids: heldJobs.map((job) => job.data.id),
+                        reason: 'occurred_at_window',
+                        retryAfterSeconds: usageRetry.heldDelaySeconds
+                    }, 'usage: events outside the hub’s occurredAt window — held, never posted, re-driven after the hold; check this instance’s clock (usage_events_held_total{reason="occurred_at_window"})');
+                    // Counted once the sends are in, for the reason the held worker
+                    // below gives: a send that throws fails this batch back to its
+                    // retry, which would count the held events again.
+                    held.inc({ reason: 'occurred_at_window' }, heldJobs.length);
+                }
+            }
+            else {
+                for (const job of jobs) {
+                    await downstreamUsage.emit(job.data);
+                }
+            }
+            logger.debug({ count: jobs.length }, 'usage events flushed');
+        });
+        // The budget's end is a HOLD, never a drop (PRDCT-2635): pg-boss moves a
+        // batch that exhausted its retries here; each event is logged where an
+        // operator sees it, counted on /metrics, and re-sent to the usage queue
+        // after the hold — round after round, until the hub accepts it. An
+        // outage of a day, an older hub, a wrong secret: the usage waits.
+        //
+        // Two kinds of job land here, told apart by the singleton key: pg-boss's
+        // dead-letter copy of a batch that ran out of retries carries none, the
+        // usage worker's window hold (PRDCT-2644) carries `windowHoldKey`. A
+        // window hold was logged and counted where it was held, so it is only
+        // re-driven here: logging or counting it again would name it a
+        // retry-budget hold, which it is not.
+        await boss.work(USAGE_HELD_QUEUE, { batchSize: 50, includeMetadata: true }, async (jobs) => {
+            const budget = jobs.filter((job) => !isWindowHoldKey(job.singletonKey));
+            if (budget.length > 0) {
+                logger.error({
+                    count: budget.length,
+                    ids: budget.map((job) => job.data.id),
+                    retryAfterSeconds: usageRetry.heldDelaySeconds
+                }, 'usage: events exhausted the retry budget — held, re-driven after the hold; check the hub, HUB_CLIENT_ID/HUB_CLIENT_SECRET and the instance’s registry entry (usage_events_held_total)');
+            }
+            for (const job of jobs) {
+                await boss.send(USAGE_QUEUE, job.data, {
+                    ...usageSendOptions(job.data, usageRetry),
+                    startAfter: usageRetry.heldDelaySeconds
+                });
+            }
+            // Counted once the re-sends are in: a re-send that throws midway
+            // fails the held batch back to its own retries, which re-send the
+            // whole batch (the hub answers duplicate for what already went) and
+            // would count it again if the count came first.
+            if (budget.length > 0)
+                held.inc({ reason: 'retry_budget' }, budget.length);
+        });
+        // Audit retention: a nightly purge keeps audit_log from growing without
+        // bound (machine reads land a row each — see audit/service.ts).
+        await boss.work(AUDIT_PURGE_QUEUE, async () => {
+            if (retentionDays <= 0)
+                return;
+            // Delete in bounded batches: one huge DELETE would seq-scan and blow the
+            // pool's statement_timeout on a large table (retention then silently
+            // never runs). Each batch touches at most BATCH rows (fast, index-backed
+            // by audit_log_created_idx) and loops until the tail is drained.
+            const BATCH = 5000;
+            let total = 0;
+            for (let i = 0; i < 10_000; i++) {
+                const res = await db.execute(sql `
+          DELETE FROM audit_log
+          WHERE id IN (
+            SELECT id FROM audit_log
+            WHERE created_at < now() - make_interval(days => ${retentionDays})
+            ORDER BY created_at
+            LIMIT ${BATCH}
+          )
+        `);
+                const deleted = res.rowCount ?? 0;
+                total += deleted;
+                if (deleted < BATCH)
+                    break;
+            }
+            logger.info({ retentionDays, deleted: total }, 'audit retention purge ran');
+        });
+        // API-key expiry: resolution already rejects expired keys fail-closed —
+        // this nightly sweep only normalizes them to the visible terminal state
+        // (revoked_at = expires_at). api_keys is a tiny bounded table, so one
+        // unbatched UPDATE is fine (the audit-purge batching lesson applies to
+        // unbounded tables only).
+        await boss.work(APIKEY_EXPIRY_QUEUE, async () => {
+            const res = await db.execute(sql `
+        UPDATE api_keys
+        SET revoked_at = expires_at
+        WHERE expires_at < now() AND revoked_at IS NULL
+      `);
+            logger.info({ retired: res.rowCount ?? 0 }, 'api key expiry sweep ran');
+        });
+        // Idempotency replay rows carry a 24h TTL (a protocol constant, the
+        // Stripe convention — not an env knob), so the table can never accumulate
+        // beyond a day's create traffic: one unbatched, index-backed DELETE is
+        // fine (the audit-purge batching lesson applies to unbounded tables only).
+        await boss.work(IDEMPOTENCY_PURGE_QUEUE, async () => {
+            const res = await db.execute(sql `
+        DELETE FROM idempotency_keys WHERE expires_at < now()
+      `);
+            logger.info({ deleted: res.rowCount ?? 0 }, 'idempotency purge ran');
+        });
+        // The tool's pollers, registered where they always were: after the
+        // chassis purges, before the orphan GC. One job per fetch (pg-boss's
+        // default batch), its payload and its key handed to the handler: a timer
+        // (`./timers.ts`) rides the same poller as a cron.
+        for (const job of toolJobs) {
+            await boss.work(job.queue, { includeMetadata: true }, async (jobs) => {
+                for (const fetched of jobs) {
+                    await job.handler(fetched.data ?? null, {
+                        id: fetched.id,
+                        queue: job.queue,
+                        key: fetched.singletonKey ?? null
+                    });
+                }
+            });
+        }
+        // Orphaned-user GC: setup-race losers, fail-closed SSO login strands
+        // (cloud), and any other path leave Better Auth `user` rows with NO
+        // workspace_members row — they can sign in but 401 everywhere, and they
+        // accumulate forever. Cloud makes some of them CREDENTIAL-BEARING: a
+        // fail-closed hub login strands a user + `antasphere` account row
+        // HOLDING a live encrypted offline grant (Better Auth writes tokens
+        // before the after-hook revokes the session), minted at an
+        // unauthenticated-reachable rate during a hub outage — so a hub link
+        // must NEVER protect a row from collection (that would accumulate
+        // dormant refresh families without bound).
+        //
+        // What distinguishes the LEGIT zero-membership user (the cloud operator
+        // pre-break-glass, a hub user whose last org was removed — both real
+        // dashboard users of the /me zero state) from the strand is the LIVE
+        // SESSION: the strand's was revoked by the fail-closed login; the legit
+        // user holds one. So the sweep collects zero-membership users with NO
+        // unexpired session, REGARDLESS of any provider link, plus two explicit
+        // exclusions: a live pending invitation (mid-onboarding) and the
+        // SUPERADMIN_EMAILS allowlist (the break-glass operator must survive
+        // even after their session lapses — ADR 010's "arm the allowlist"
+        // guidance, now enforced instead of advised).
+        //
+        // SAFETY INVARIANTS:
+        //  - a user with ANY membership row — even a deactivated one — is a
+        //    real member and is NEVER touched;
+        //  - ── HARD CONSTRAINT (user-scoped federation) ──────────────────────
+        //    the purge must NEVER delete an `antasphere` account row while
+        //    leaving an `origin='hub'` membership row behind: that would make
+        //    the reconciler's `no_link` branch — today unreachable for
+        //    hub-origin principals, and the one unconditional fail-open —
+        //    REACHABLE, i.e. a fail-open authorization hole. Held by
+        //    construction: the only deletion here is the WHOLE user through
+        //    Better Auth's internalAdapter.deleteUser (the exact path both GDPR
+        //    delete surfaces use), whose FK cascade removes account rows AND
+        //    membership rows together, and only ever for users the immediately
+        //    preceding re-check proved to have ZERO membership rows. Never
+        //    replace this with a partial cleanup (account rows, tokens) — whole
+        //    user or nothing.
+        await boss.work(ORPHAN_USER_PURGE_QUEUE, async () => {
+            if (orphanRetentionHours <= 0)
+                return; // 0 = disabled (schedule is off too)
+            const authCtx = await auth.$context;
+            // As ONE json parameter (never a JS array param — driver array
+            // serialization is not worth trusting in a deletion query).
+            const superadmins = JSON.stringify([...parseSuperadminEmails(env.SUPERADMIN_EMAILS)]);
+            const BATCH = 50;
+            let total = 0;
+            const sample = [];
+            for (let i = 0; i < 200; i++) {
+                // Candidates: zero memberships, past the grace period, NO live
+                // (unexpired) session, no LIVE pending invitation for their email —
+                // an orphan someone just re-invited is mid-onboarding, not garbage
+                // (the invitation row exists before its accept URL does, so this
+                // closes the scan-then-accept race for the invited case) — and not
+                // a superadmin-allowlisted address. The exclusions live in the
+                // query (not a JS skip): a skipped candidate would otherwise fill
+                // the batch forever and starve the real orphans behind it.
+                const res = await db.execute(sql `
+          SELECT u.id, u.email FROM "user" u
+          WHERE NOT EXISTS (
+              SELECT 1 FROM workspace_members wm WHERE wm.user_id = u.id
+            )
+            AND u.created_at < now() - make_interval(hours => ${orphanRetentionHours})
+            AND NOT EXISTS (
+              SELECT 1 FROM session s WHERE s.user_id = u.id AND s.expires_at > now()
+            )
+            AND NOT EXISTS (
+              SELECT 1 FROM invitations inv
+              WHERE lower(inv.email) = lower(u.email)
+                AND inv.accepted_at IS NULL
+                AND inv.revoked_at IS NULL
+                AND inv.expires_at > now()
+            )
+            -- BG-2: the allowlist protects exactly what break-glass ACCEPTS —
+            -- a VERIFIED allowlisted address. An unverified squat of an
+            -- allowlisted email is no operator and gets no shelter.
+            AND NOT EXISTS (
+              SELECT 1 FROM jsonb_array_elements_text(${superadmins}::jsonb) AS sa(email)
+              WHERE sa.email = lower(u.email) AND u.email_verified
+            )
+            -- CLOUD-3: the setup operator (instance_settings.operator_user_id)
+            -- is NEVER an orphan — on cloud they hold no membership by
+            -- design, and deleting them loses the only local credential and
+            -- the only break-glass identity. Durable, allowlist or not.
+            AND u.id IS DISTINCT FROM (SELECT operator_user_id FROM instance_settings LIMIT 1)
+          LIMIT ${BATCH}
+        `);
+                const rows = res.rows;
+                if (rows.length === 0)
+                    break;
+                let progressed = 0;
+                for (const row of rows) {
+                    // Final re-check immediately before the delete: a membership OR a
+                    // live session (the user signing in mid-sweep) can appear between
+                    // the candidate scan and here — either one disqualifies.
+                    const still = await db.execute(sql `
+            SELECT 1 FROM workspace_members WHERE user_id = ${row.id}
+            UNION ALL
+            SELECT 1 FROM session WHERE user_id = ${row.id} AND expires_at > now()
+            LIMIT 1
+          `);
+                    if ((still.rows?.length ?? 0) > 0)
+                        continue;
+                    try {
+                        await authCtx.internalAdapter.deleteUser(row.id);
+                        await authCtx.internalAdapter.deleteUserSessions(row.id);
+                        total += 1;
+                        progressed += 1;
+                        if (sample.length < 20)
+                            sample.push(row.email);
+                    }
+                    catch (err) {
+                        // Skip and keep sweeping: a failed delete (e.g. the 0009
+                        // last-owner trigger backstopping the pathological race where
+                        // the user gained a sole-owner membership mid-sweep) aborts
+                        // THIS user only.
+                        logger.warn({ err, userId: row.id }, 'orphan purge: skipping user (delete failed)');
+                    }
+                }
+                if (rows.length < BATCH)
+                    break;
+                if (progressed === 0)
+                    break; // every candidate failed/was skipped — no hot loop
+            }
+            if (total > 0) {
+                // INSTANCE-attributed system row (workspace_id NULL, ADR 014):
+                // orphans belong to no workspace, so no workspace's audit trail is
+                // the honest home — the row is an operator-level record.
+                await audit.write({
+                    workspaceId: null,
+                    principal: null, // → actorVia 'system'
+                    action: 'user.orphan_purge',
+                    resourceType: 'user',
+                    metadata: { deleted: total, retentionHours: orphanRetentionHours, sample }
+                });
+            }
+            logger.info({ retentionHours: orphanRetentionHours, deleted: total }, 'orphan user purge ran');
+        });
+    }
+    else {
+        // api role: migrate:false start() runs no DDL and creates no queues — it
+        // only verifies the pgboss schema exists ('pg-boss is not installed' on a
+        // fresh DB until an all|worker boot installs it; internal/scale-drill-findings.md), so it
+        // needs neither the install lock nor DDL rights.
+        await boss.start();
+    }
+    return {
+        boss,
+        timers: new PgBossTimers(boss, pool, logger),
+        stop: async () => {
+            await boss.stop({ graceful: true, wait: true });
+        },
+        promMetrics: [held]
+    };
+}
+/**
+ * The queue's retry budget: ten retries with exponential backoff from thirty
+ * seconds, about eight and a half hours in all — a hub deploy (the hub
+ * deploys first, and answers 404 until it has) or an outage of an afternoon
+ * costs nothing but time. The budget's END is a hold, not a loss
+ * (PRDCT-2635): the batch moves to `USAGE_HELD_QUEUE`, is logged and
+ * counted, and comes back through the queue an hour later, and an hour
+ * after that, until the hub accepts it. At-least-once is the posture; the
+ * hub dedupes by ULID.
+ */
+export const DEFAULT_USAGE_RETRY = { limit: 10, delaySeconds: 30, heldDelaySeconds: 3600 };
+/**
+ * The ONE statement of how a usage event is sent: the sink and the held
+ * queue's re-drive share it. The dead letter (where the batch goes when the
+ * budget runs out: held, never dropped) is the QUEUE's setting, installed
+ * by the worker boot, so a send names none and an api-role replica needs
+ * nothing to exist beyond the usage queue itself.
+ */
+export function usageSendOptions(event, retry) {
+    return {
+        // The event's ULID, for a reader of the queue table; on a standard
+        // pg-boss queue a singleton key enforces nothing (verifier round 1), and
+        // idempotency is the receiver's job anyway (dedupe by ULID, per contract).
+        singletonKey: event.id,
+        retryLimit: retry.limit,
+        retryDelay: retry.delaySeconds,
+        retryBackoff: true
+    };
+}
+export class PgBossUsageSink {
+    boss;
+    logger;
+    retry;
+    constructor(boss, logger, retry = DEFAULT_USAGE_RETRY) {
+        this.boss = boss;
+        this.logger = logger;
+        this.retry = retry;
+    }
+    async emit(event) {
+        try {
+            await this.boss.send(USAGE_QUEUE, event, usageSendOptions(event, this.retry));
+        }
+        catch (err) {
+            // Usage must never break the request path.
+            this.logger.error({ err, meter: event.meter }, 'usage emit failed');
+        }
+    }
+}
+//# sourceMappingURL=pgboss.js.map

package/dist/jobs/pgboss.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pgboss.js","sourceRoot":"","sources":["../../src/jobs/pgboss.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAOlC,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,YAAY,EAA8B,MAAM,aAAa,CAAC;AAEtF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,cAAc,CAAC;AAC1C,MAAM,CAAC,MAAM,iBAAiB,GAAG,aAAa,CAAC;AAC/C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,mBAAmB,CAAC;AACpD;;;;;;GAMG;AACH,MAAM,kBAAkB,GAAG,qBAAqB,CAAC;AACjD,MAAM,aAAa,GAAG,CAAC,EAAU,EAAU,EAAE,CAAC,kBAAkB,GAAG,EAAE,CAAC;AACtE,MAAM,eAAe,GAAG,CAAC,GAA8B,EAAW,EAAE,CAClE,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;AAChE,MAAM,CAAC,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;AACzD,MAAM,CAAC,MAAM,uBAAuB,GAAG,mBAAmB,CAAC;AAC3D,MAAM,CAAC,MAAM,uBAAuB,GAAG,mBAAmB,CAAC;AA4B3D;;;;;;;;;;;GAWG;AACH,MAAM,uBAAuB,GAAG,SAAS,CAAC;AAE1C;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAC5B,gBAAwB,EACxB,MAAc,EACd,EAAuB;IAEvB,MAAM,MAAM,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACnD,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACvB,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,CAAC,uBAAuB,CAAC,CAAC,CAAC;QAC7E,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,8DAA8D,CAAC,CAAC;QACjG,MAAM,EAAE,EAAE,CAAC;QACX,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,4BAA4B,CAAC,CAAC;IACjE,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,CAAC,uBAAuB,CAAC,CAAC,CAAC;QACjF,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;AACH,CAAC;AAWD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,GAOC,EACD,EAAM;AACN,iGAAiG;AACjG,IAAa,EACb,MAAc,EACd,eAAgC;AAChC,wFAAwF;AACxF,IAAU;AACV,oDAAoD;AACpD,KAAmB;AACnB,0EAA0E;AAC1E,WAAsC,EAAE;AACxC,2FAA2F;AAC3F,aAAyB,mBAAmB;IAE5C,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,KAAK,KAAK,IAAI,GAAG,CAAC,YAAY,KAAK,QAAQ,CAAC;IAC7E,wDAAwD;IACxD,MAAM,IAAI,GAAG,IAAI,OAAO,CAAC;QACvB,IAAI,EAAE,yBAAyB;QAC/B,IAAI,EAAE,0IAA0I;QAChJ,UAAU,EAAE,CAAC,QAAQ,CAAU;QAC/B,SAAS,EAAE,EAAE;KACd,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC;QACtB,gBAAgB,EAAE,GAAG,CAAC,YAAY;QAClC,MAAM,EAAE,aAAa;QACrB,0EAA0E;QAC1E,4DAA4D;QAC5D,4BAA4B,EAAE,IAAI;QAClC,eAAe,EAAE,CAAC;QAClB,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;KAC3E,CAAC,CAAC;IACH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,eAAe,CAAC,CAAC,CAAC;IAElE,MAAM,aAAa,GAAG,GAAG,CAAC,oBAAoB,CAAC;IAC/C,MAAM,oBAAoB,GAAG,GAAG,CAAC,2BAA2B,CAAC;IAE7D,IAAI,QAAQ,EAAE,CAAC;QACb,kEAAkE;QAClE,yEAAyE;QACzE,uEAAuE;QACvE,yEAAyE;QACzE,0EAA0E;QAC1E,4BAA4B;QAC5B,MAAM,eAAe,CAAC,GAAG,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,uEAAuE;YACvE,oEAAoE;YACpE,qEAAqE;YACrE,kEAAkE;YAClE,mEAAmE;YACnE,uEAAuE;YACvE,qEAAqE;YACrE,oEAAoE;YACpE,uEAAuE;YACvE,qEAAqE;YACrE,uEAAuE;YACvE,qDAAqD;YACrD,wEAAwE;YACxE,uEAAuE;YACvE,sEAAsE;YACtE,qEAAqE;YACrE,sEAAsE;YACtE,kEAAkE;YAClE,+CAA+C;YAC/C,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE;gBACvC,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,EAAE;gBACd,UAAU,EAAE,EAAE;gBACd,YAAY,EAAE,IAAI;aACnB,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;YACpC,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC,CAAC;YACzF,sEAAsE;YACtE,qCAAqC;YACrC,MAAM,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,CAAC;YAC1C,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;YACtD,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAC3D,CAAC;YACD,MAAM,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;YAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAC;YACtD,MAAM,IAAI,CAAC,WAAW,CAAC,uBAAuB,CAAC,CAAC;YAChD,MAAM,IAAI,CAAC,QAAQ,CAAC,uBAAuB,EAAE,WAAW,CAAC,CAAC;YAC1D,yEAAyE;YACzE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAClC,IAAI,CAAC,GAAG,CAAC,QAAQ;oBAAE,SAAS;gBAC5B,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;oBACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;YACD,iEAAiE;YACjE,uDAAuD;YACvD,MAAM,IAAI,CAAC,WAAW,CAAC,uBAAuB,CAAC,CAAC;YAChD,IAAI,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,QAAQ,CAAC,uBAAuB,EAAE,WAAW,CAAC,CAAC;YAC5D,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,UAAU,CAAC,uBAAuB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACjE,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,mEAAmE;QACnE,sEAAsE;QACtE,oEAAoE;QACpE,wEAAwE;QACxE,EAAE;QACF,wEAAwE;QACxE,sEAAsE;QACtE,gEAAgE;QAChE,iEAAiE;QACjE,yEAAyE;QACzE,yEAAyE;QACzE,uCAAuC;QACvC,MAAM,IAAI,CAAC,IAAI,CAAa,WAAW,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACzE,IAAI,eAAe,CAAC,SAAS,EAAE,CAAC;gBAC9B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC7E,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;oBAChE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;wBAC3B,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,GAAG,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC5F,CAAC;oBACD,MAAM,CAAC,IAAI,CACT;wBACE,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;wBACvC,MAAM,EAAE,oBAAoB;wBAC5B,iBAAiB,EAAE,UAAU,CAAC,gBAAgB;qBAC/C,EACD,sLAAsL,CACvL,CAAC;oBACF,gEAAgE;oBAChE,+DAA+D;oBAC/D,kDAAkD;oBAClD,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAC9D,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,MAAM,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACvC,CAAC;YACH,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,sBAAsB,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,yEAAyE;QACzE,uEAAuE;QACvE,wEAAwE;QACxE,mEAAmE;QACnE,kEAAkE;QAClE,EAAE;QACF,yEAAyE;QACzE,wEAAwE;QACxE,qEAAqE;QACrE,sEAAsE;QACtE,+DAA+D;QAC/D,sCAAsC;QACtC,MAAM,IAAI,CAAC,IAAI,CAAa,gBAAgB,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACrG,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;YACxE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,CAAC,KAAK,CACV;oBACE,KAAK,EAAE,MAAM,CAAC,MAAM;oBACpB,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACrC,iBAAiB,EAAE,UAAU,CAAC,gBAAgB;iBAC/C,EACD,uLAAuL,CACxL,CAAC;YACJ,CAAC;YACD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,IAAI,EAAE;oBACrC,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC;oBACzC,UAAU,EAAE,UAAU,CAAC,gBAAgB;iBACxC,CAAC,CAAC;YACL,CAAC;YACD,iEAAiE;YACjE,kEAAkE;YAClE,oEAAoE;YACpE,gDAAgD;YAChD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBAAE,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,cAAc,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,wEAAwE;QACxE,gEAAgE;QAChE,MAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,KAAK,IAAI,EAAE;YAC5C,IAAI,aAAa,IAAI,CAAC;gBAAE,OAAO;YAC/B,yEAAyE;YACzE,qEAAqE;YACrE,yEAAyE;YACzE,iEAAiE;YACjE,MAAM,KAAK,GAAG,IAAI,CAAC;YACnB,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;;;+DAIuB,aAAa;;oBAExD,KAAK;;SAEhB,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;gBAClC,KAAK,IAAI,OAAO,CAAC;gBACjB,IAAI,OAAO,GAAG,KAAK;oBAAE,MAAM;YAC7B,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,2BAA2B,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;QAEH,wEAAwE;QACxE,wEAAwE;QACxE,sEAAsE;QACtE,uEAAuE;QACvE,0BAA0B;QAC1B,MAAM,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;;;OAI/B,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,IAAI,CAAC,EAAE,EAAE,0BAA0B,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,oEAAoE;QACpE,0EAA0E;QAC1E,uEAAuE;QACvE,2EAA2E;QAC3E,MAAM,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;OAE/B,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,QAAQ,IAAI,CAAC,EAAE,EAAE,uBAAuB,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,mEAAmE;QACnE,qEAAqE;QACrE,yEAAyE;QACzE,mDAAmD;QACnD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,IAAI,CAAS,GAAG,CAAC,KAAK,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;gBAC3E,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;oBAC3B,MAAM,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI,EAAE;wBACtC,EAAE,EAAE,OAAO,CAAC,EAAE;wBACd,KAAK,EAAE,GAAG,CAAC,KAAK;wBAChB,GAAG,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI;qBAClC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,qEAAqE;QACrE,oEAAoE;QACpE,wEAAwE;QACxE,qEAAqE;QACrE,kEAAkE;QAClE,oEAAoE;QACpE,2DAA2D;QAC3D,qEAAqE;QACrE,kEAAkE;QAClE,2CAA2C;QAC3C,EAAE;QACF,wEAAwE;QACxE,qEAAqE;QACrE,qEAAqE;QACrE,wEAAwE;QACxE,sEAAsE;QACtE,wEAAwE;QACxE,iEAAiE;QACjE,qEAAqE;QACrE,kEAAkE;QAClE,8CAA8C;QAC9C,EAAE;QACF,qBAAqB;QACrB,oEAAoE;QACpE,uCAAuC;QACvC,wEAAwE;QACxE,mEAAmE;QACnE,sEAAsE;QACtE,+DAA+D;QAC/D,kEAAkE;QAClE,6DAA6D;QAC7D,oEAAoE;QACpE,wEAAwE;QACxE,qEAAqE;QACrE,uEAAuE;QACvE,mEAAmE;QACnE,wEAAwE;QACxE,sBAAsB;QACtB,MAAM,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;YAClD,IAAI,oBAAoB,IAAI,CAAC;gBAAE,OAAO,CAAC,qCAAqC;YAC5E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC;YACpC,+DAA+D;YAC/D,4DAA4D;YAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,qBAAqB,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;YACtF,MAAM,KAAK,GAAG,EAAE,CAAC;YACjB,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7B,+DAA+D;gBAC/D,oEAAoE;gBACpE,mEAAmE;gBACnE,iEAAiE;gBACjE,mEAAmE;gBACnE,+DAA+D;gBAC/D,kEAAkE;gBAClE,2DAA2D;gBAC3D,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;;;;;gEAKwB,oBAAoB;;;;;;;;;;;;;;;wDAe5B,WAAW;;;;;;;;kBAQjD,KAAK;SACd,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,GAAG,CAAC,IAAuD,CAAC;gBACzE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,MAAM;gBAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;gBACnB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,kEAAkE;oBAClE,kEAAkE;oBAClE,yDAAyD;oBACzD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,CAAA;8DACkB,GAAG,CAAC,EAAE;;oDAEhB,GAAG,CAAC,EAAE;;WAE/C,CAAC,CAAC;oBACH,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;wBAAE,SAAS;oBAC5C,IAAI,CAAC;wBACH,MAAM,OAAO,CAAC,eAAe,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACjD,MAAM,OAAO,CAAC,eAAe,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;wBACzD,KAAK,IAAI,CAAC,CAAC;wBACX,UAAU,IAAI,CAAC,CAAC;wBAChB,IAAI,MAAM,CAAC,MAAM,GAAG,EAAE;4BAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBACjD,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,yDAAyD;wBACzD,8DAA8D;wBAC9D,4DAA4D;wBAC5D,kBAAkB;wBAClB,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,6CAA6C,CAAC,CAAC;oBACtF,CAAC;gBACH,CAAC;gBACD,IAAI,IAAI,CAAC,MAAM,GAAG,KAAK;oBAAE,MAAM;gBAC/B,IAAI,UAAU,KAAK,CAAC;oBAAE,MAAM,CAAC,mDAAmD;YAClF,CAAC;YACD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,+DAA+D;gBAC/D,mEAAmE;gBACnE,yDAAyD;gBACzD,MAAM,KAAK,CAAC,KAAK,CAAC;oBAChB,WAAW,EAAE,IAAI;oBACjB,SAAS,EAAE,IAAI,EAAE,sBAAsB;oBACvC,MAAM,EAAE,mBAAmB;oBAC3B,YAAY,EAAE,MAAM;oBACpB,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,EAAE;iBAC3E,CAAC,CAAC;YACL,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,oBAAoB,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,uBAAuB,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,yEAAyE;QACzE,0EAA0E;QAC1E,0FAA0F;QAC1F,iDAAiD;QACjD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,IAAI,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC;QAC5C,IAAI,EAAE,KAAK,IAAI,EAAE;YACf,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,WAAW,EAAE,CAAC,IAAI,CAAC;KACpB,CAAC;AACJ,CAAC;AAeD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAe,EAAE,KAAK,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;AAEvG;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAiB,EAAE,KAAiB;IACnE,OAAO;QACL,mEAAmE;QACnE,yEAAyE;QACzE,2EAA2E;QAC3E,YAAY,EAAE,KAAK,CAAC,EAAE;QACtB,UAAU,EAAE,KAAK,CAAC,KAAK;QACvB,UAAU,EAAE,KAAK,CAAC,YAAY;QAC9B,YAAY,EAAE,IAAI;KACnB,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,eAAe;IAEP;IACA;IACA;IAHnB,YACmB,IAAY,EACZ,MAAc,EACd,QAAoB,mBAAmB;QAFvC,SAAI,GAAJ,IAAI,CAAQ;QACZ,WAAM,GAAN,MAAM,CAAQ;QACd,UAAK,GAAL,KAAK,CAAkC;IACvD,CAAC;IAEJ,KAAK,CAAC,IAAI,CAAC,KAAiB;QAC1B,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAChF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,2CAA2C;YAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,mBAAmB,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;CACF"}