@rulemetric/cli 0.1.1 → 0.2.0

package/dist/lib/graphile-worker-runner.js

@@ -0,0 +1,526 @@
+// apps/cli/src/lib/graphile-worker-runner.ts
+//
+// Central worker orchestration: registers the four task handlers, wires
+// `runner.events` so each lifecycle event (job:start/success/error/failed)
+// translates into a mirror-row update via `applyMirrorUpdate`, and runs an
+// in-process heartbeat that carries `currentJobId` + `lastError` so the
+// dashboard can detect hung workers.
+//
+// The heartbeat is started BEFORE `await run(...)` resolves — this is the
+// cold-start fix from the design doc: a worker that boots but takes a few
+// seconds to claim its first job should still show up as alive in the
+// dashboard during that window.
+//
+// Cron tasks are intentionally absent here — they're added in Tasks 17/18.
+import { execFile } from 'node:child_process';
+import * as fs from 'node:fs';
+import { promisify } from 'node:util';
+const execFileAsync = promisify(execFile);
+import os from 'node:os';
+import { run } from 'graphile-worker';
+import postgres from 'postgres';
+import { captureException as sentryCaptureException } from '@rulemetric/telemetry';
+import { apiPost } from './api-client.js';
+import { CRON_REGISTRY, wrapCronWithEnvelope } from './cron-registry.js';
+import { processConversation } from './handlers/process-conversation.js';
+import { processEval } from './handlers/process-eval.js';
+import { processInsights } from './handlers/process-insights.js';
+import { processLaunch } from './handlers/process-launch.js';
+import { jobToMirrorUpdate } from './mirror-writer.js';
+import { applyMirrorUpdate, isMirrorTask } from './mirror-writer-db.js';
+import { isPermanentJobError } from './permanent-job-error.js';
+// Watchdog thresholds. Heartbeat runs every 30s, so 5 consecutive failures =
+// ~2.5 min of API unreachable. The pg-listen threshold is 3 min without a
+// successful LISTEN connection, which is well past graphile-worker's own
+// reconnect backoff (caps around 30s) — so by the time we trip, recovery has
+// genuinely failed rather than just being slow.
+const HEARTBEAT_FAILURE_LIMIT = 5;
+const LISTEN_DEAD_THRESHOLD_MS = 180_000;
+const WATCHDOG_INTERVAL_MS = 30_000;
+/**
+ * Pure decision function: given watchdog state and "now", return a fatal
+ * reason string if the worker should exit, or null if it's healthy. Extracted
+ * so the threshold logic can be unit-tested without booting graphile-worker
+ * or stubbing process.exit.
+ */
+export function watchdogVerdict(_state, _now) {
+    // No process-exit paths. Both signals previously routed here (heartbeat
+    // failure and pg LISTEN dead) fire reliably during Supavisor pool pressure
+    // even when the worker itself is healthy — exiting turned transient outages
+    // into launchd respawn loops whose replacements stacked sockets onto an
+    // already-saturated pooler (EMAXCONN cascade; see scripts/repro-pool-cascade.sh
+    // injections C and D). Visibility now comes from three places that don't
+    // require suicide: (a) heartbeat `lastError` surfaces both conditions to the
+    // dashboard, (b) the server-side mirror reconciler flips stuck mirror rows
+    // to `failed` after 60s, (c) `unhandledRejection` swallows known transient
+    // pg errors instead of letting Node terminate. Kept as a function so test
+    // coverage of "does NOT exit under X" stays explicit.
+    return null;
+}
+/**
+ * Derive a short status string from watchdog state for `lastError` reporting.
+ * Returns null when nothing notable; otherwise a human-readable signal the
+ * dashboard can surface (e.g. "pg listen has been failing for 240s"). This is
+ * the logged-signal replacement for the old exit paths.
+ */
+export function watchdogStatus(state, now) {
+    if (state.lastListenErrorAt !== null) {
+        const lastGood = state.lastListenSuccessAt ?? 0;
+        if (state.lastListenErrorAt > lastGood && now - state.lastListenErrorAt > LISTEN_DEAD_THRESHOLD_MS) {
+            return `pg listen has been failing for ${Math.round((now - state.lastListenErrorAt) / 1000)}s`;
+        }
+    }
+    if (state.consecutiveHeartbeatFailures >= HEARTBEAT_FAILURE_LIMIT) {
+        return `heartbeat has failed ${state.consecutiveHeartbeatFailures} consecutive times`;
+    }
+    return null;
+}
+/**
+ * Coerce graphile-worker's `error: unknown` payload into a string for
+ * `lastError` reporting. Walks `.cause` so a wrapped pg/graphile error still
+ * surfaces its underlying message — important for the unhandledRejection
+ * swallow regex, which would otherwise miss errors thrown as
+ * `new Error("...", { cause: original })`.
+ */
+function errorMessage(err, depth = 0) {
+    if (depth > 4)
+        return '';
+    if (err instanceof Error) {
+        const cause = err.cause;
+        return cause !== undefined ? `${err.message} | cause: ${errorMessage(cause, depth + 1)}` : err.message;
+    }
+    if (typeof err === 'string')
+        return err;
+    try {
+        return JSON.stringify(err);
+    }
+    catch {
+        return String(err);
+    }
+}
+/**
+ * Parse `lsof -i TCP -P -n` output into a count of established sockets to the
+ * Supabase pooler, grouped by remote host.
+ *
+ * Matches BOTH pooler ports: 6543 (transaction mode) and 5432 (session mode).
+ * The worker connects through the session pooler — see deriveSessionPoolerUrl
+ * in commands/evals/listen.ts — while the API uses the transaction pooler; a
+ * counter that matched only :6543 would silently report 0 sockets for every
+ * worker. The `->` anchor is load-bearing: it matches the REMOTE port, not a
+ * local source port that happens to share those digits.
+ */
+export function parsePoolerSockets(lsofStdout) {
+    const byNode = {};
+    let total = 0;
+    for (const line of lsofStdout.split('\n')) {
+        const m = line.match(/->([\d.]+|\[[^\]]+\]):(?:6543|5432)\b/);
+        if (m) {
+            const host = m[1].replace(/^\[|\]$/g, '');
+            byNode[host] = (byNode[host] ?? 0) + 1;
+            total += 1;
+        }
+    }
+    return { total, byNode };
+}
+/**
+ * Release stale Graphile job locks owned by this worker's user.
+ *
+ * The graphile_worker._private_jobs table tracks in-flight jobs via
+ * `locked_at`/`locked_by`. When a worker crashes mid-job (SIGKILL, OOM, the
+ * 10s graceful-shutdown timeout in evals/listen.ts forcing exit before
+ * graphile can release locks) those fields stay populated — and graphile's
+ * built-in lock-timeout sweeper only reclaims them after a 4-hour TTL by
+ * default. Until that fires, the job is invisible to every worker: it's not
+ * `available`, but no one is actually running it.
+ *
+ * At worker BOOT we know that no jobs belonging to OUR user are legitimately
+ * in flight (the previous instance is dead — that's why we're starting). Any
+ * lock on a task identifier ending `__<userId>` is therefore stale and safe
+ * to clear. We scope by suffix so multiple users on a shared pooler don't
+ * accidentally release each other's live locks.
+ *
+ * Best-effort: a thrown executor error is logged but swallowed. If reclaim
+ * fails, graphile's 4h timeout still kicks in eventually, and crippling
+ * worker startup over a transient DB hiccup is worse than waiting it out.
+ *
+ * Returns the count of rows released (0 on error or no-op).
+ */
+export async function reclaimStaleLocks(exec, userId, log) {
+    try {
+        const rows = await exec(`UPDATE graphile_worker._private_jobs j
+       SET locked_at = NULL, locked_by = NULL
+       FROM graphile_worker._private_tasks t
+       WHERE j.task_id = t.id
+         AND t.identifier LIKE $1
+         AND j.locked_at IS NOT NULL
+       RETURNING j.id, t.identifier`, [`%__${userId}`]);
+        if (rows.length > 0) {
+            const summary = rows.map((r) => `${r.identifier}#${r.id}`).join(', ');
+            log(`[startup] reclaimed ${rows.length} stale lock(s): ${summary}`);
+        }
+        return rows.length;
+    }
+    catch (err) {
+        log(`[startup] stale-lock reclaim failed (non-fatal, graphile's 4h timeout will eventually free these): ${err instanceof Error ? err.message : String(err)}`);
+        return 0;
+    }
+}
+/**
+ * Wrap a typed handler so it conforms to graphile-worker's `Task` shape AND
+ * intercepts `PermanentJobError` to skip retry waste.
+ *
+ * How the skip works: graphile-worker decides retry-vs-permanent-fail by
+ * comparing `job.attempts >= job.max_attempts` on the in-memory `Job` object
+ * AFTER the handler throws (see graphile-worker dist/worker.js). We mutate
+ * `helpers.job.max_attempts` to equal the current attempt count just before
+ * re-throwing — that flips the inequality so Graphile emits `job:failed`
+ * instead of scheduling another retry. The mutation is purely in JS memory;
+ * Graphile uses the in-memory `Job` object for its decision and never re-reads
+ * `max_attempts` from the DB on this code path, so a SQL update isn't needed.
+ *
+ * Exported for use in integration tests that want the same retry-skip behavior
+ * the runner applies in production.
+ */
+export function makeTask(handler) {
+    return (async (payload, helpers) => {
+        try {
+            await handler(payload, helpers);
+        }
+        catch (err) {
+            if (isPermanentJobError(err)) {
+                // Mutate the in-memory job so Graphile's post-throw retry-vs-fail
+                // check (`job.attempts >= job.max_attempts`) sees us at the cap.
+                const job = helpers.job;
+                job.max_attempts = job.attempts;
+            }
+            throw err;
+        }
+    });
+}
+/**
+ * Boot a graphile-worker runner with the four task handlers wired up.
+ *
+ * Returns a cleanup function that:
+ *   1. clears the heartbeat interval
+ *   2. fires a final `status: 'shutting_down'` heartbeat so the dashboard
+ *      can show a clean shutdown rather than a missed-heartbeat timeout
+ *   3. awaits `runner.stop()` for graceful drain (in-flight jobs finish)
+ */
+export async function startWorker(options) {
+    const { connectionString, log, userId } = options;
+    // onFatal kept on the options interface for back-compat with callers/tests,
+    // but the runner no longer invokes it. All "fatal" signals are reported via
+    // heartbeat lastError and the server-side reconciler instead — see
+    // watchdogStatus and the comment on watchdogVerdict.
+    void options.onFatal;
+    const now = options.now ?? Date.now;
+    // Stable worker identity: hostname only, NOT including process.pid. Each
+    // PID change (restart, --force install, dev iteration) used to create a
+    // fresh row in worker_connections, leaving "ghost" entries forever in the
+    // dashboard. The heartbeat endpoint UPSERTs by (userId, workerType,
+    // clientName) — so collapsing clientName to hostname means one row per
+    // host per workerType, regardless of restart count.
+    const workerId = os.hostname();
+    // Survive graphile-worker's unhandled rejections. The cron scheduler
+    // (graphile-worker/dist/cron.js scheduleCronJobs) and the internal worker's
+    // completeJob path can throw `(EMAXCONN)` or `Connection terminated
+    // unexpectedly` when the Supavisor pool is briefly saturated. Neither path
+    // catches its own throw, so Node's default unhandledRejection behavior
+    // terminates the process — turning a 30-second transient into an indefinite
+    // launchd-restart loop. Reproduction: scripts/repro-pool-cascade.sh D
+    // (150 psql sleeps + API down) produces "Failed to release job ...;
+    // committing seppuku\nConnection terminated unexpectedly" without these
+    // listeners. With them, graphile-worker logs the error and reschedules.
+    const pgErrorPattern = /EMAXCONN|Connection terminated|EAUTHTIMEOUT|ECONNRESET|getaddrinfo/i;
+    const onUnhandled = (reason) => {
+        const msg = errorMessage(reason);
+        if (pgErrorPattern.test(msg)) {
+            log(`[recovered] transient pool error swallowed: ${msg.slice(0, 200)}`);
+            return;
+        }
+        // Re-throw non-pool errors via the default behavior so real bugs aren't
+        // hidden. Setting exitCode lets the runtime decide.
+        log(`unhandledRejection (not pool-related): ${msg}`);
+        process.exitCode = 1;
+    };
+    process.on('unhandledRejection', onUnhandled);
+    process.on('uncaughtException', onUnhandled);
+    const inFlight = new Map();
+    // `lastError` surfaces the most recent error from any job for dashboard
+    // display — distinct from the per-job `lastError` stored on the Map
+    // entry, which is keyed for cleanup but not currently surfaced.
+    let lastError = null;
+    const mostRecentEntry = () => {
+        let last;
+        for (const value of inFlight.values())
+            last = value;
+        return last;
+    };
+    const watchdog = {
+        consecutiveHeartbeatFailures: 0,
+        lastListenSuccessAt: null,
+        lastListenErrorAt: null,
+    };
+    // What this worker can claim. Reported on every heartbeat in
+    // `metadata.capabilities` so the API can warn when the set drifts from
+    // EXPECTED_WORKER_CAPABILITIES['cli-worker'] (apps/api/src/routes/worker-status.ts).
+    // Use bare task kinds, not the user-suffixed task names — the server's
+    // expected set is `['eval','insights','launch','conversation']`.
+    const CAPABILITIES = ['eval', 'insights', 'launch', 'conversation'];
+    // De-dupe advisory log lines: server returns the same advisory on every
+    // 30s heartbeat, so log only when the message changes (and re-log on
+    // recovery from advisory → no advisory).
+    let lastLoggedAdvisoryMessage = null;
+    const lsofBin = ['/usr/sbin/lsof', '/usr/bin/lsof', '/bin/lsof'].find((p) => fs.existsSync(p)) ?? 'lsof';
+    // Fully async: previously this was execFileSync, which can block the event
+    // loop for up to the 3s timeout under fork()/process-table contention —
+    // exactly the conditions present during Supavisor pool saturation. Blocking
+    // the heartbeat tick on lsof would delay the very heartbeat that's supposed
+    // to surface the pool-saturation state. Fire-and-forget: never awaited by
+    // sendHeartbeat, errors swallowed, no impact on job processing or watchdog.
+    const sendPoolerSelfReport = async () => {
+        try {
+            const { stdout } = await execFileAsync(lsofBin, ['-a', '-p', String(process.pid), '-i', 'TCP', '-P', '-n'], { encoding: 'utf8', timeout: 3000 });
+            const { total, byNode } = parsePoolerSockets(stdout);
+            await apiPost('/api/diag/pg-health/self-report', {
+                process: process.env.PG_APPLICATION_NAME ?? 'rulemetric-worker',
+                pid: process.pid,
+                pool_max: Number(process.env.PG_POOL_MAX) || 3,
+                pooler_sockets: total,
+                pooler_sockets_by_node: byNode,
+            }).catch(() => null);
+        }
+        catch {
+            // Swallow: lsof binary missing, timeout, permission denied, etc. all
+            // mean "no telemetry this tick" — never block job processing.
+        }
+    };
+    const sendHeartbeat = (status = 'alive') => {
+        void sendPoolerSelfReport();
+        const entry = mostRecentEntry();
+        // Merge watchdog status into lastError so the dashboard surfaces wedge
+        // conditions (stuck LISTEN, persistent heartbeat failures) without us
+        // having to exit the process. lastError already captures the most recent
+        // job-level failure; watchdog status takes precedence when present
+        // because it indicates a process-level issue the user should see.
+        const wdStatus = watchdogStatus(watchdog, now());
+        const reportedError = wdStatus ?? lastError;
+        return apiPost('/api/workers/heartbeat', {
+            workerType: 'cli-worker',
+            clientName: workerId,
+            workerVersion: '0.1.0',
+            currentJobId: entry?.mirrorId ?? null,
+            inFlightCount: inFlight.size,
+            lastError: reportedError,
+            status,
+            metadata: { capabilities: CAPABILITIES },
+        }).then((result) => {
+            watchdog.consecutiveHeartbeatFailures = 0;
+            const advisory = result?.advisory;
+            if (advisory) {
+                if (advisory.message !== lastLoggedAdvisoryMessage) {
+                    log(`heartbeat advisory [${advisory.code}]: ${advisory.message}`);
+                    lastLoggedAdvisoryMessage = advisory.message;
+                }
+            }
+            else if (lastLoggedAdvisoryMessage !== null) {
+                log('heartbeat advisory cleared');
+                lastLoggedAdvisoryMessage = null;
+            }
+            return result;
+        }, (err) => {
+            watchdog.consecutiveHeartbeatFailures += 1;
+            log(`heartbeat failed: ${errorMessage(err)}`);
+        });
+    };
+    // Cold-start fix: start the heartbeat BEFORE awaiting run(). If `run()`
+    // takes a few seconds to settle (initial schema check, LISTEN setup),
+    // the worker still appears alive in the dashboard during that window.
+    void sendHeartbeat();
+    const heartbeatTimer = setInterval(() => {
+        void sendHeartbeat();
+    }, 30_000);
+    // Tag this pool's connections so they show up distinctly in pg_stat_activity
+    // (separate from the postgres-js pool inside getDb()).
+    const taggedConnectionString = (() => {
+        try {
+            const u = new URL(connectionString);
+            if (!u.searchParams.has('application_name')) {
+                u.searchParams.set('application_name', 'rulemetric-worker-graphile');
+            }
+            return u.toString();
+        }
+        catch {
+            return connectionString;
+        }
+    })();
+    const runnerOptions = {
+        connectionString: taggedConnectionString,
+        // After SIGTERM/SIGINT, graphile-worker waits for in-flight jobs to drain
+        // before resolving runner.stop(). With handlers that spawn `claude`
+        // subprocesses (insights, cron_suggest_instructions), the drain can take
+        // minutes. Cap it at 5s so graphile triggers its forcefulShutdown path
+        // and runner.stop() resolves promptly. The outer SIGTERM handler in
+        // evals/listen.ts wraps this in a 10s process-level timeout too, so even
+        // if graphile's force-abort hangs, the worker dies cleanly.
+        gracefulShutdownAbortTimeout: 5_000,
+        // PgBouncer transaction-mode (Supabase pooler) doesn't share prepared statements across pooled connections — disable to avoid "prepared statement already exists" crashes.
+        noPreparedStatements: true,
+        // Cap concurrent jobs across all task types. This replaces the global
+        // `processing` flag in the legacy worker — Graphile handles fairness
+        // and per-task ordering automatically.
+        concurrency: 4,
+        // Per-user task names — both work tasks (process_*) AND cron tasks. The
+        // Graphile queue is global; the suffix is what makes each laptop only
+        // claim its own user's jobs. Cron tasks need the suffix too because
+        // cron handlers POST to API endpoints that scope by `c.get('userId')` —
+        // an unsuffixed cron would be claimed by whichever worker polls first
+        // and would only regenerate THAT user's data, leaving every other user
+        // without cron-driven updates. With the suffix, each laptop independently
+        // schedules and claims its own user's crons. Cost: a user whose laptop is
+        // offline at 4am misses that day's skills refresh until they boot back up.
+        taskList: {
+            [`process_eval__${userId}`]: makeTask(processEval),
+            [`process_insights__${userId}`]: makeTask(processInsights),
+            [`process_launch__${userId}`]: makeTask(processLaunch),
+            [`process_conversation__${userId}`]: makeTask(processConversation),
+            // Wrap each cron in `wrapCronWithEnvelope` so we get a uniform
+            // `[cron <name>] start/ok/failed` log envelope on every fire. The
+            // per-handler progress lines stay as-is — the envelope just bookends.
+            // Added 2026-05-21 after the eight-fix arc made it obvious that
+            // half the crons logged ad-hoc start lines and none logged duration.
+            ...Object.fromEntries(CRON_REGISTRY.map((c) => [`${c.name}__${userId}`, wrapCronWithEnvelope(c.name, c.handler)])),
+        },
+        // `?jobKey=cron-<task>__<userId>` dedupes cron-scheduled jobs in graphile's
+        // queue. Without it, every worker restart / missed scheduling could leave
+        // duplicates: we observed 3 simultaneous `cron_refresh_skills` rows with
+        // the same run_at, each locked by a different internal-worker slot — they
+        // saturated concurrency=4 and starved user-facing launches. With jobKey
+        // (and the default jobKeyMode='replace'), a subsequent scheduled fire
+        // replaces any existing queued job of the same key instead of accumulating.
+        // Diagnosed 2026-05-20.
+        crontab: CRON_REGISTRY.map((c) => `${c.schedule} ${c.name}__${userId} ?jobKey=cron-${c.name}__${userId}`).join('\n') + '\n',
+    };
+    // Reclaim stale locks BEFORE booting the runner. When the previous worker
+    // instance dies hard (SIGKILL, OOM, the 10s shutdown-grace timeout in
+    // evals/listen.ts forcing exit before graphile released locks) the jobs
+    // it was holding stay marked `is_available = false AND locked_at IS NOT NULL`
+    // — graphile's built-in lock-timeout reclaim only kicks in after a 4h TTL.
+    // We know no jobs of OUR user are legitimately in flight at boot (this
+    // process IS the user's worker; the previous one is dead). Scoping by
+    // `__<userId>` suffix keeps multi-tenant pools safe.
+    // See reclaimStaleLocks() for the SQL + rationale.
+    {
+        const sql = postgres(taggedConnectionString, { max: 1, prepare: false, idle_timeout: 5 });
+        try {
+            await reclaimStaleLocks(async (text, params) => (await sql.unsafe(text, params)), userId, log);
+        }
+        finally {
+            await sql.end({ timeout: 5 }).catch(() => { });
+        }
+    }
+    const runner = await run(runnerOptions);
+    // Each event listener is fire-and-forget on the mirror write. We don't
+    // want a transient API blip to break the worker — `applyMirrorUpdate`
+    // already retries once internally, and Graphile's own job state is the
+    // source of truth regardless.
+    //
+    // The in-flight Map is keyed by the Graphile job id (numeric, unique
+    // per job) so that concurrent jobs don't trample each other's entries.
+    // We re-set on `start` so the most-recent insertion order reflects what
+    // started last, and delete on success/failed (terminal states).
+    // Cron tasks flow through the same runner.events stream as mirror-backed
+    // tasks, but they have no mirror row and no entry in ENDPOINTS — short-
+    // circuit so they don't fire bogus `/api/.../undefined/state` requests.
+    const onStart = ({ job }) => {
+        const payload = job.payload;
+        const mirrorId = payload.mirrorId ?? String(job.id);
+        inFlight.delete(Number(job.id));
+        inFlight.set(Number(job.id), { mirrorId });
+        if (!isMirrorTask(job.task_identifier))
+            return;
+        void applyMirrorUpdate(job.task_identifier, jobToMirrorUpdate('job:start', job), log);
+    };
+    const onSuccess = ({ job }) => {
+        inFlight.delete(Number(job.id));
+        if (inFlight.size === 0)
+            lastError = null;
+        if (!isMirrorTask(job.task_identifier))
+            return;
+        void applyMirrorUpdate(job.task_identifier, jobToMirrorUpdate('job:success', job), log);
+    };
+    const onError = ({ job, error }) => {
+        // Don't remove from inFlight here — the job may still be in-flight
+        // if Graphile decides to retry. We only delete on success/failed.
+        const message = errorMessage(error);
+        lastError = message;
+        const entry = inFlight.get(Number(job.id));
+        if (entry)
+            entry.lastError = message;
+        if (!isMirrorTask(job.task_identifier))
+            return;
+        void applyMirrorUpdate(job.task_identifier, jobToMirrorUpdate('job:error', job), log);
+    };
+    const onFailed = ({ job, error }) => {
+        inFlight.delete(Number(job.id));
+        lastError = errorMessage(error);
+        // Forward to Sentry with tags that make dashboard filtering useful.
+        // `job:failed` fires only after attempts >= max_attempts, so every event
+        // here represents a real terminal failure — exactly the signal we want
+        // to alert on. The mirrorId is the row in eval_jobs / insights_jobs /
+        // launch_jobs that's about to be flipped to `failed`. Crons have no
+        // mirror row; for them `mirrorId` is the graphile job id (best we have).
+        const payload = (job.payload ?? {});
+        sentryCaptureException(error, {
+            tags: {
+                taskName: job.task_identifier,
+                source: 'graphile-worker',
+            },
+            extra: {
+                graphileJobId: String(job.id),
+                mirrorId: payload.mirrorId ?? null,
+                attempts: job.attempts,
+                maxAttempts: job.max_attempts,
+            },
+        });
+        if (!isMirrorTask(job.task_identifier))
+            return;
+        void applyMirrorUpdate(job.task_identifier, jobToMirrorUpdate('job:failed', job), log);
+    };
+    runner.events.on('job:start', onStart);
+    runner.events.on('job:success', onSuccess);
+    runner.events.on('job:error', onError);
+    runner.events.on('job:failed', onFailed);
+    // Watchdog signals — observed during the 2026-05-06 wedge, where the
+    // PG pool listener died ("Connection terminated unexpectedly" repeating)
+    // and the worker stayed alive but stopped polling for ~9 hours.
+    runner.events.on('pool:listen:success', () => {
+        watchdog.lastListenSuccessAt = now();
+    });
+    runner.events.on('pool:listen:error', ({ error }) => {
+        watchdog.lastListenErrorAt = now();
+        log(`pool:listen:error: ${errorMessage(error)}`);
+    });
+    // Watchdog tick: previously this called onFatal(reason) → process.exit(1).
+    // That path was the cascade root cause — see watchdogVerdict comment. The
+    // tick now only logs persistent wedge conditions so they show up in
+    // stderr alongside the heartbeat-reported status. Kept as a tick (rather
+    // than dropped) because logging on a separate cadence from the 30s
+    // heartbeat helps operators see whether a condition is sticky or transient
+    // when correlating timestamps.
+    const watchdogTimer = setInterval(() => {
+        const status = watchdogStatus(watchdog, now());
+        if (status)
+            log(`[watchdog] ${status}`);
+    }, WATCHDOG_INTERVAL_MS);
+    log(`Worker ${workerId} listening for user ${userId} jobs (eval, insights, launch, conversation)`);
+    return async () => {
+        clearInterval(heartbeatTimer);
+        clearInterval(watchdogTimer);
+        process.off('unhandledRejection', onUnhandled);
+        process.off('uncaughtException', onUnhandled);
+        await sendHeartbeat('shutting_down');
+        await runner.stop();
+    };
+}
+//# sourceMappingURL=graphile-worker-runner.js.map

package/dist/lib/graphile-worker-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"graphile-worker-runner.js","sourceRoot":"","sources":["../../src/lib/graphile-worker-runner.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,wEAAwE;AACxE,2EAA2E;AAC3E,2EAA2E;AAC3E,wEAAwE;AACxE,qCAAqC;AACrC,EAAE;AACF,0EAA0E;AAC1E,0EAA0E;AAC1E,sEAAsE;AACtE,gCAAgC;AAChC,EAAE;AACF,2EAA2E;AAC3E,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC1C,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,GAAG,EAA4D,MAAM,iBAAiB,CAAC;AAChG,OAAO,QAAQ,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,gBAAgB,IAAI,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACnF,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAuB/D,6EAA6E;AAC7E,0EAA0E;AAC1E,yEAAyE;AACzE,6EAA6E;AAC7E,gDAAgD;AAChD,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAClC,MAAM,wBAAwB,GAAG,OAAO,CAAC;AACzC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAUpC;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,MAAqB,EAAE,IAAY;IACjE,wEAAwE;IACxE,2EAA2E;IAC3E,4EAA4E;IAC5E,wEAAwE;IACxE,gFAAgF;IAChF,yEAAyE;IACzE,6EAA6E;IAC7E,2EAA2E;IAC3E,2EAA2E;IAC3E,0EAA0E;IAC1E,sDAAsD;IACtD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,KAAoB,EAAE,GAAW;IAC9D,IAAI,KAAK,CAAC,iBAAiB,KAAK,IAAI,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,KAAK,CAAC,mBAAmB,IAAI,CAAC,CAAC;QAChD,IAAI,KAAK,CAAC,iBAAiB,GAAG,QAAQ,IAAI,GAAG,GAAG,KAAK,CAAC,iBAAiB,GAAG,wBAAwB,EAAE,CAAC;YACnG,OAAO,kCAAkC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;QACjG,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,4BAA4B,IAAI,uBAAuB,EAAE,CAAC;QAClE,OAAO,wBAAwB,KAAK,CAAC,4BAA4B,oBAAoB,CAAC;IACxF,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;GAMG;AACH,SAAS,YAAY,CAAC,GAAY,EAAE,KAAK,GAAG,CAAC;IAC3C,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IACzB,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAI,GAA2B,CAAC,KAAK,CAAC;QACjD,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,aAAa,YAAY,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;IACzG,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACxC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,kBAAkB,CAAC,UAAkB;IAInD,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC9D,IAAI,CAAC,EAAE,CAAC;YACN,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACvC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC;AAsBD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAiB,EACjB,MAAc,EACd,GAA0B;IAE1B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,IAAI,CACrB;;;;;;oCAM8B,EAC9B,CAAC,MAAM,MAAM,EAAE,CAAC,CACjB,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtE,GAAG,CAAC,uBAAuB,IAAI,CAAC,MAAM,mBAAmB,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,sGAAsG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9J,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,QAAQ,CACtB,OAA8D;IAE9D,OAAO,CAAC,KAAK,EAAE,OAAgB,EAAE,OAAmB,EAAE,EAAE;QACtD,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,OAAY,EAAE,OAAO,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,mBAAmB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,kEAAkE;gBAClE,iEAAiE;gBACjE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAiD,CAAC;gBACtE,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,QAAQ,CAAC;YAClC,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC,CAAS,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAsB;IACtD,MAAM,EAAE,gBAAgB,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAClD,4EAA4E;IAC5E,4EAA4E;IAC5E,mEAAmE;IACnE,qDAAqD;IACrD,KAAK,OAAO,CAAC,OAAO,CAAC;IACrB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACpC,yEAAyE;IACzE,wEAAwE;IACxE,0EAA0E;IAC1E,oEAAoE;IACpE,uEAAuE;IACvE,oDAAoD;IACpD,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;IAE/B,qEAAqE;IACrE,4EAA4E;IAC5E,oEAAoE;IACpE,2EAA2E;IAC3E,uEAAuE;IACvE,4EAA4E;IAC5E,sEAAsE;IACtE,oEAAoE;IACpE,wEAAwE;IACxE,wEAAwE;IACxE,MAAM,cAAc,GAAG,qEAAqE,CAAC;IAC7F,MAAM,WAAW,GAAG,CAAC,MAAe,EAAE,EAAE;QACtC,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,GAAG,CAAC,+CAA+C,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACxE,OAAO;QACT,CAAC;QACD,wEAAwE;QACxE,oDAAoD;QACpD,GAAG,CAAC,0CAA0C,GAAG,EAAE,CAAC,CAAC;QACrD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;IAC9C,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAC;IAW7C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC7C,wEAAwE;IACxE,oEAAoE;IACpE,gEAAgE;IAChE,IAAI,SAAS,GAAkB,IAAI,CAAC;IAEpC,MAAM,eAAe,GAAG,GAAyB,EAAE;QACjD,IAAI,IAA0B,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE;YAAE,IAAI,GAAG,KAAK,CAAC;QACpD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAkB;QAC9B,4BAA4B,EAAE,CAAC;QAC/B,mBAAmB,EAAE,IAAI;QACzB,iBAAiB,EAAE,IAAI;KACxB,CAAC;IAEF,6DAA6D;IAC7D,uEAAuE;IACvE,qFAAqF;IACrF,uEAAuE;IACvE,iEAAiE;IACjE,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,CAAU,CAAC;IAE7E,wEAAwE;IACxE,qEAAqE;IACrE,yCAAyC;IACzC,IAAI,yBAAyB,GAAkB,IAAI,CAAC;IAOpD,MAAM,OAAO,GAAG,CAAC,gBAAgB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1E,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CACjB,IAAI,MAAM,CAAC;IAEZ,2EAA2E;IAC3E,wEAAwE;IACxE,4EAA4E;IAC5E,4EAA4E;IAC5E,0EAA0E;IAC1E,4EAA4E;IAC5E,MAAM,oBAAoB,GAAG,KAAK,IAAmB,EAAE;QACrD,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CACpC,OAAO,EACP,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,EAC1D,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CACpC,CAAC;YACF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,OAAO,CAAC,iCAAiC,EAAE;gBAC/C,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,mBAAmB;gBAC/D,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC;gBAC9C,cAAc,EAAE,KAAK;gBACrB,sBAAsB,EAAE,MAAM;aAC/B,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,qEAAqE;YACrE,8DAA8D;QAChE,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,aAAa,GAAG,CAAC,SAAoC,OAAO,EAAoB,EAAE;QACtF,KAAK,oBAAoB,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,eAAe,EAAE,CAAC;QAChC,uEAAuE;QACvE,sEAAsE;QACtE,yEAAyE;QACzE,mEAAmE;QACnE,kEAAkE;QAClE,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;QACjD,MAAM,aAAa,GAAG,QAAQ,IAAI,SAAS,CAAC;QAC5C,OAAO,OAAO,CAAoB,wBAAwB,EAAE;YAC1D,UAAU,EAAE,YAAY;YACxB,UAAU,EAAE,QAAQ;YACpB,aAAa,EAAE,OAAO;YACtB,YAAY,EAAE,KAAK,EAAE,QAAQ,IAAI,IAAI;YACrC,aAAa,EAAE,QAAQ,CAAC,IAAI;YAC5B,SAAS,EAAE,aAAa;YACxB,MAAM;YACN,QAAQ,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE;SACzC,CAAC,CAAC,IAAI,CACL,CAAC,MAAM,EAAE,EAAE;YACT,QAAQ,CAAC,4BAA4B,GAAG,CAAC,CAAC;YAC1C,MAAM,QAAQ,GAAG,MAAM,EAAE,QAAQ,CAAC;YAClC,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,QAAQ,CAAC,OAAO,KAAK,yBAAyB,EAAE,CAAC;oBACnD,GAAG,CAAC,uBAAuB,QAAQ,CAAC,IAAI,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;oBAClE,yBAAyB,GAAG,QAAQ,CAAC,OAAO,CAAC;gBAC/C,CAAC;YACH,CAAC;iBAAM,IAAI,yBAAyB,KAAK,IAAI,EAAE,CAAC;gBAC9C,GAAG,CAAC,4BAA4B,CAAC,CAAC;gBAClC,yBAAyB,GAAG,IAAI,CAAC;YACnC,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,EACD,CAAC,GAAY,EAAE,EAAE;YACf,QAAQ,CAAC,4BAA4B,IAAI,CAAC,CAAC;YAC3C,GAAG,CAAC,qBAAqB,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChD,CAAC,CACF,CAAC;IACJ,CAAC,CAAC;IAEF,wEAAwE;IACxE,sEAAsE;IACtE,sEAAsE;IACtE,KAAK,aAAa,EAAE,CAAC;IACrB,MAAM,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE;QACtC,KAAK,aAAa,EAAE,CAAC;IACvB,CAAC,EAAE,MAAM,CAAC,CAAC;IAEX,6EAA6E;IAC7E,uDAAuD;IACvD,MAAM,sBAAsB,GAAG,CAAC,GAAG,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC;YACpC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAC5C,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,kBAAkB,EAAE,4BAA4B,CAAC,CAAC;YACvE,CAAC;YACD,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,gBAAgB,CAAC;QAC1B,CAAC;IACH,CAAC,CAAC,EAAE,CAAC;IAEL,MAAM,aAAa,GAAkB;QACnC,gBAAgB,EAAE,sBAAsB;QACxC,0EAA0E;QAC1E,oEAAoE;QACpE,yEAAyE;QACzE,uEAAuE;QACvE,oEAAoE;QACpE,yEAAyE;QACzE,4DAA4D;QAC5D,4BAA4B,EAAE,KAAK;QACnC,2KAA2K;QAC3K,oBAAoB,EAAE,IAAI;QAC1B,sEAAsE;QACtE,qEAAqE;QACrE,uCAAuC;QACvC,WAAW,EAAE,CAAC;QACd,wEAAwE;QACxE,sEAAsE;QACtE,oEAAoE;QACpE,wEAAwE;QACxE,sEAAsE;QACtE,uEAAuE;QACvE,0EAA0E;QAC1E,0EAA0E;QAC1E,2EAA2E;QAC3E,QAAQ,EAAE;YACR,CAAC,iBAAiB,MAAM,EAAE,CAAC,EAAE,QAAQ,CAAoC,WAAW,CAAC;YACrF,CAAC,qBAAqB,MAAM,EAAE,CAAC,EAAE,QAAQ,CAAwC,eAAe,CAAC;YACjG,CAAC,mBAAmB,MAAM,EAAE,CAAC,EAAE,QAAQ,CAAsC,aAAa,CAAC;YAC3F,CAAC,yBAAyB,MAAM,EAAE,CAAC,EAAE,QAAQ,CAA4C,mBAAmB,CAAC;YAC7G,+DAA+D;YAC/D,kEAAkE;YAClE,sEAAsE;YACtE,gEAAgE;YAChE,qEAAqE;YACrE,GAAG,MAAM,CAAC,WAAW,CACnB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,EAAE,oBAAoB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAC5F;SACF;QACD,4EAA4E;QAC5E,0EAA0E;QAC1E,yEAAyE;QACzE,0EAA0E;QAC1E,wEAAwE;QACxE,sEAAsE;QACtE,4EAA4E;QAC5E,wBAAwB;QACxB,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,iBAAiB,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI;KAC5H,CAAC;IAEF,0EAA0E;IAC1E,sEAAsE;IACtE,wEAAwE;IACxE,8EAA8E;IAC9E,2EAA2E;IAC3E,uEAAuE;IACvE,sEAAsE;IACtE,qDAAqD;IACrD,mDAAmD;IACnD,CAAC;QACC,MAAM,GAAG,GAAG,QAAQ,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1F,IAAI,CAAC;YACH,MAAM,iBAAiB,CACrB,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAiB,CAAC,CAAY,EAC9E,MAAM,EACN,GAAG,CACJ,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,CAAC;IAExC,uEAAuE;IACvE,sEAAsE;IACtE,uEAAuE;IACvE,8BAA8B;IAC9B,EAAE;IACF,qEAAqE;IACrE,uEAAuE;IACvE,wEAAwE;IACxE,gEAAgE;IAChE,yEAAyE;IACzE,wEAAwE;IACxE,wEAAwE;IACxE,MAAM,OAAO,GAAG,CAAC,EAAE,GAAG,EAAgB,EAAE,EAAE;QACxC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAgC,CAAC;QACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpD,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAChC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC;YAAE,OAAO;QAC/C,KAAK,iBAAiB,CAAC,GAAG,CAAC,eAAe,EAAE,iBAAiB,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;IACxF,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,EAAE,GAAG,EAAgB,EAAE,EAAE;QAC1C,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAChC,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC;YAAE,SAAS,GAAG,IAAI,CAAC;QAC1C,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC;YAAE,OAAO;QAC/C,KAAK,iBAAiB,CAAC,GAAG,CAAC,eAAe,EAAE,iBAAiB,CAAC,aAAa,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;IAC1F,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,EAAgC,EAAE,EAAE;QAC/D,mEAAmE;QACnE,kEAAkE;QAClE,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACpC,SAAS,GAAG,OAAO,CAAC;QACpB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3C,IAAI,KAAK;YAAE,KAAK,CAAC,SAAS,GAAG,OAAO,CAAC;QACrC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC;YAAE,OAAO;QAC/C,KAAK,iBAAiB,CAAC,GAAG,CAAC,eAAe,EAAE,iBAAiB,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;IACxF,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,EAAgC,EAAE,EAAE;QAChE,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAChC,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,oEAAoE;QACpE,yEAAyE;QACzE,uEAAuE;QACvE,sEAAsE;QACtE,oEAAoE;QACpE,yEAAyE;QACzE,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAA0B,CAAC;QAC7D,sBAAsB,CAAC,KAAK,EAAE;YAC5B,IAAI,EAAE;gBACJ,QAAQ,EAAE,GAAG,CAAC,eAAe;gBAC7B,MAAM,EAAE,iBAAiB;aAC1B;YACD,KAAK,EAAE;gBACL,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;gBAClC,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,WAAW,EAAE,GAAG,CAAC,YAAY;aAC9B;SACF,CAAC,CAAC;QACH,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC;YAAE,OAAO;QAC/C,KAAK,iBAAiB,CAAC,GAAG,CAAC,eAAe,EAAE,iBAAiB,CAAC,YAAY,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;IACzF,CAAC,CAAC;IAEF,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACvC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IAC3C,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACvC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;IAEzC,qEAAqE;IACrE,yEAAyE;IACzE,gEAAgE;IAChE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;QAC3C,QAAQ,CAAC,mBAAmB,GAAG,GAAG,EAAE,CAAC;IACvC,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,EAAE,KAAK,EAAsB,EAAE,EAAE;QACtE,QAAQ,CAAC,iBAAiB,GAAG,GAAG,EAAE,CAAC;QACnC,GAAG,CAAC,sBAAsB,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,2EAA2E;IAC3E,0EAA0E;IAC1E,oEAAoE;IACpE,yEAAyE;IACzE,mEAAmE;IACnE,2EAA2E;IAC3E,+BAA+B;IAC/B,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACrC,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;QAC/C,IAAI,MAAM;YAAE,GAAG,CAAC,cAAc,MAAM,EAAE,CAAC,CAAC;IAC1C,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAEzB,GAAG,CAAC,UAAU,QAAQ,uBAAuB,MAAM,8CAA8C,CAAC,CAAC;IAEnG,OAAO,KAAK,IAAI,EAAE;QAChB,aAAa,CAAC,cAAc,CAAC,CAAC;QAC9B,aAAa,CAAC,aAAa,CAAC,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,WAAW,CAAC,CAAC;QAC9C,MAAM,aAAa,CAAC,eAAe,CAAC,CAAC;QACrC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC,CAAC;AACJ,CAAC"}

package/dist/lib/graphile-worker-runner.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=graphile-worker-runner.test.d.ts.map

package/dist/lib/graphile-worker-runner.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graphile-worker-runner.test.d.ts","sourceRoot":"","sources":["../../src/lib/graphile-worker-runner.test.ts"],"names":[],"mappings":""}