@hyperfixation/workflows 0.1.0

package/dist/start-worker.js

@@ -0,0 +1,208 @@
+import { DBOS, DBOSClient } from "@dbos-inc/dbos-sdk";
+import { appPaused, createStepPool, runBootChecks, } from "@hyperfixation/db";
+import { createControlPool } from "./control-pool.js";
+import { registerLangfuse } from "./langfuse.js";
+import { setPausedQueueConcurrency } from "./queue-concurrency.js";
+import { reconcile, startReconciler } from "./reconcile.js";
+import { acquireWorkerLock } from "./worker-lock.js";
+import { setWorkerRuntime } from "./worker-runtime.js";
+/** The value of `HF_PROCESS` in the one process shape allowed to launch DBOS. */
+export const WORKER_PROCESS = "worker";
+/** Enough of a commit sha to be a version; `hf dev` sets `dev-<timestamp>`. */
+export const MIN_BUILD_SHA_LENGTH = 7;
+export const SYSTEM_DATABASE_SCHEMA = "dbos";
+export const SYSTEM_DATABASE_POOL_SIZE = 5;
+/**
+ * The reconciler's own client. Deliberately not `getClient()`: that singleton is the web's, and
+ * calling it here would run the boot checks a second time and import this module back.
+ */
+export const RECONCILER_POOL_SIZE = 2;
+/**
+ * The three queues, by name and concurrency. There is no flow registry yet; when there is,
+ * `defineFlow` names one of these and nothing else may.
+ */
+export const QUEUES = [
+    { name: "llm", globalConcurrency: 4 },
+    { name: "actions", globalConcurrency: 2 },
+    { name: "resolve", globalConcurrency: 1 },
+];
+/**
+ * Logged on either side of the one `DBOS.launch()` call in the system. Redeploy case 6
+ * asserts the first never appears in a second worker's output.
+ */
+export const LAUNCHING_MARKER = "hf-worker: calling DBOS.launch";
+export const LAUNCHED_MARKER = "hf-worker: DBOS launched";
+/** One per SIGTERM the handler acts on; redeploy case 11 counts them. */
+export const SHUTDOWN_MARKER = "hf-worker: SIGTERM, calling DBOS.shutdown";
+export const SHUTDOWN_IGNORED_MARKER = "hf-worker: SIGTERM ignored, already draining";
+export const SHUTDOWN_FAILED_MARKER = "hf-worker: DBOS.shutdown rejected";
+/** How long the drain waits for workflows running here before it abandons them. */
+export const DRAIN_TIMEOUT_MS = 60_000;
+/**
+ * The handler's own bound, past the drain. Compose's `stop_grace_period: 90s` SIGKILL is the
+ * line after this one, and the advisory lock is released by neither before the process dies.
+ */
+export const SHUTDOWN_WATCHDOG_MS = 75_000;
+export class NotAWorkerProcess extends Error {
+    hfProcess;
+    constructor(hfProcess) {
+        super(`NotAWorkerProcess: startWorker() needs HF_PROCESS=${WORKER_PROCESS}, got ` +
+            `${hfProcess === undefined ? "an unset HF_PROCESS" : JSON.stringify(hfProcess)}`);
+        this.name = "NotAWorkerProcess";
+        this.hfProcess = hfProcess;
+    }
+}
+export class MissingBuildSha extends Error {
+    constructor(buildSha) {
+        super(`MissingBuildSha: HF_BUILD_SHA must be at least ${MIN_BUILD_SHA_LENGTH} characters, got ` +
+            `${buildSha === undefined ? "an unset HF_BUILD_SHA" : JSON.stringify(buildSha)}`);
+        this.name = "MissingBuildSha";
+    }
+}
+/** Set when the keys are present; the SIGTERM handler flushes through it. */
+let langfuse;
+/**
+ * The only place `DBOS.launch()` runs. Boot checks, then the two pools, then the advisory
+ * lock, then launch — a worker that cannot prove it is alone never reaches the launch.
+ */
+export async function startWorker(options) {
+    if (process.env.HF_PROCESS !== WORKER_PROCESS) {
+        throw new NotAWorkerProcess(process.env.HF_PROCESS);
+    }
+    // Ahead of the boot checks only because it opens no connection — E001-E006 are still the
+    // first statements this process issues. A missing version is a build misconfiguration, and
+    // failing on it before taking a cluster-wide lock keeps the failure cheap.
+    const applicationVersion = requireBuildSha();
+    await runBootChecks({
+        databaseUrl: options.databaseUrl,
+        recordTables: options.recordTables,
+        appMigrationsDir: options.appMigrationsDir,
+    });
+    const steps = createStepPool({ connectionString: options.databaseUrl });
+    const control = createControlPool({ connectionString: options.databaseUrl });
+    // Before the lock and launch: DBOS recovery can run a flow the instant `launch()` returns,
+    // and a recovered flow reaches `workerRuntime()` the same as a freshly dispatched one.
+    setWorkerRuntime({
+        appName: options.appName,
+        applicationVersion,
+        steps,
+        control,
+        approvalNotifier: options.approvalNotifier,
+    });
+    let client;
+    try {
+        // Before `setConfig`, so DBOS finds a registered provider and a working context manager and
+        // keeps ours: both globals are first-one-wins.
+        langfuse = registerLangfuse();
+        const lock = await acquireWorkerLock(options.databaseUrl, options.appName);
+        DBOS.setConfig({
+            name: options.appName,
+            systemDatabaseUrl: options.databaseUrl,
+            systemDatabaseSchemaName: SYSTEM_DATABASE_SCHEMA,
+            systemDatabasePoolSize: SYSTEM_DATABASE_POOL_SIZE,
+            applicationVersion,
+            executorID: WORKER_PROCESS,
+            enablePatching: false,
+            runAdminServer: false,
+            maxConcurrentQueueDispatches: 1,
+            runMigrations: false,
+            // Without a Langfuse destination DBOS's spans are stubs, which is the cheaper default.
+            tracingEnabled: langfuse !== undefined,
+        });
+        // Installed before `launch`, so a SIGTERM arriving during it is this process's to drain.
+        process.on("SIGTERM", handleSigterm);
+        console.info(LAUNCHING_MARKER, applicationVersion);
+        await DBOS.launch();
+        console.info(LAUNCHED_MARKER, applicationVersion);
+        // `registerQueue` writes the queue's row through the system database, so it can only run
+        // once DBOS is launched.
+        const queues = {};
+        for (const queue of QUEUES) {
+            queues[queue.name] = await DBOS.registerQueue(queue.name, {
+                globalConcurrency: queue.globalConcurrency,
+            });
+        }
+        client = await DBOSClient.create({
+            systemDatabaseUrl: options.databaseUrl,
+            systemDatabaseSchemaName: SYSTEM_DATABASE_SCHEMA,
+            systemDatabasePoolSize: RECONCILER_POOL_SIZE,
+            applicationName: options.appName,
+        });
+        // `registerQueue` above re-wrote each queue's row at its registered concurrency, which on a
+        // deploy into a paused app would undo `pause`'s queue half and let the new worker start
+        // dispatching. The pause flag is still what makes it correct — every dispatched step
+        // suspends at the gate — but a paused app should not be dequeuing at all.
+        if (await appPaused(control.pool))
+            await setPausedQueueConcurrency(client, true);
+        // Once before the worker is ready, so a redeploy's backlog is moved onto this version
+        // before anything else is dispatched, and every minute after. A failure here fails the
+        // boot: a worker that cannot reconcile is a worker the previous version's runs are
+        // stranded behind.
+        await reconcile(control.pool, client, { applicationVersion });
+        const reconciler = startReconciler(control.pool, client, { applicationVersion });
+        return {
+            appName: options.appName,
+            applicationVersion,
+            steps,
+            control,
+            lock,
+            queues,
+            client,
+            reconciler,
+        };
+    }
+    catch (error) {
+        // The lock connection is not closed here either: if it was taken, the lock belongs to
+        // this process until it dies, whatever went wrong afterwards.
+        await client?.destroy().catch(() => undefined);
+        await steps.end().catch(() => undefined);
+        await control.end().catch(() => undefined);
+        throw error;
+    }
+}
+/**
+ * Process-wide rather than per-worker because `DBOS.shutdown()` is static and carries no
+ * re-entry guard of its own: a second delivery that reached it ends in node-pg's "Called end
+ * on pool more than once" (round-3 finding 9).
+ */
+let shuttingDown = false;
+/**
+ * Deliberately not `async`. An `await`ed `DBOS.shutdown()` that rejects leaves an unhandled
+ * rejection, which Sentry's default listener swallows — the process then sits out the whole
+ * watchdog instead of exiting (round-3 finding 10). `.then()` with both arms explicit never
+ * produces one.
+ *
+ * The lock connection is untouched here on purpose: the drain abandons unfinished workflows,
+ * so step bodies of this process can still be writing, and process death is the only release
+ * of the advisory lock that cannot let the next worker in underneath them.
+ */
+function handleSigterm() {
+    if (shuttingDown) {
+        console.info(SHUTDOWN_IGNORED_MARKER);
+        return;
+    }
+    shuttingDown = true;
+    // Armed before anything that could yield, so a `shutdown()` that never settles still ends
+    // the process. `unref` so the watchdog is never itself a reason to stay up.
+    setTimeout(() => process.exit(1), SHUTDOWN_WATCHDOG_MS).unref();
+    console.info(SHUTDOWN_MARKER);
+    DBOS.shutdown({ workflowCompletionTimeoutMS: DRAIN_TIMEOUT_MS }).then(() => flushThenExit(0), (error) => {
+        console.error(SHUTDOWN_FAILED_MARKER, error);
+        flushThenExit(1);
+    });
+}
+/**
+ * The last span batch would otherwise die with the process. Both arms exit with the code the
+ * drain earned: a Langfuse flush that fails is not a reason to change it.
+ */
+function flushThenExit(code) {
+    const flushed = langfuse?.shutdown() ?? Promise.resolve();
+    flushed.then(() => process.exit(code), () => process.exit(code));
+}
+function requireBuildSha() {
+    const buildSha = process.env.HF_BUILD_SHA;
+    if (buildSha === undefined || buildSha.length < MIN_BUILD_SHA_LENGTH) {
+        throw new MissingBuildSha(buildSha);
+    }
+    return buildSha;
+}

package/dist/step.d.ts

@@ -0,0 +1,29 @@
+import { type StepDatabase } from "@hyperfixation/db";
+/** The only database handle a step body is given. */
+export interface StepContext {
+    readonly runId: string;
+    readonly attempt: number;
+    readonly workflowId: string;
+    /** The step's own key: what a ledger row or an action log is keyed by. */
+    readonly key: string;
+    tx<T>(work: (db: StepDatabase) => Promise<T>): Promise<T>;
+}
+export interface StepOptions {
+    /** Distinguishes two calls of the same `name`, as a loop over records does. */
+    key?: string;
+}
+/** The pause gate and the fencing token, read together so one round trip answers both. */
+export declare const STEP_GATE_STATEMENT: string;
+/**
+ * A checkpointed function, preceded by a checkpointed read of the pause flag and the run's
+ * fencing token.
+ *
+ * The gate is a step that returns the *reading* and throws nothing. Were the throw inside it,
+ * DBOS would checkpoint the error through `serialize-error` and a replay would revive a plain
+ * `Error`, so `defineFlow`'s `instanceof Suspend` would stop catching it after a recovery.
+ *
+ * Retries are off because a step body here is a database write or a billable provider call:
+ * "at least once" is the guarantee the ledger and the `ctx.tx` fence are built to survive,
+ * and a silent in-process retry would add attempts neither of them can see.
+ */
+export declare function step<T>(name: string, fn: (ctx: StepContext) => Promise<T>, options?: StepOptions): Promise<T>;

package/dist/step.js

@@ -0,0 +1,54 @@
+import { DBOS } from "@dbos-inc/dbos-sdk";
+import { StaleAttempt } from "@hyperfixation/db";
+import { currentRun } from "./run-context.js";
+import { concludeRun } from "./run-status.js";
+import { Suspend } from "./suspend.js";
+import { workerRuntime } from "./worker-runtime.js";
+/** The pause gate and the fencing token, read together so one round trip answers both. */
+export const STEP_GATE_STATEMENT = "SELECT COALESCE((SELECT paused FROM hf_app_state WHERE id = 1), false) AS paused, " +
+    "(SELECT current_workflow_id FROM hf_run WHERE run_id = $1) AS current_workflow_id";
+/**
+ * A checkpointed function, preceded by a checkpointed read of the pause flag and the run's
+ * fencing token.
+ *
+ * The gate is a step that returns the *reading* and throws nothing. Were the throw inside it,
+ * DBOS would checkpoint the error through `serialize-error` and a replay would revive a plain
+ * `Error`, so `defineFlow`'s `instanceof Suspend` would stop catching it after a recovery.
+ *
+ * Retries are off because a step body here is a database write or a billable provider call:
+ * "at least once" is the guarantee the ledger and the `ctx.tx` fence are built to survive,
+ * and a silent in-process retry would add attempts neither of them can see.
+ */
+export async function step(name, fn, options = {}) {
+    const run = currentRun(`step(${name})`);
+    const runtime = workerRuntime(`step(${name})`);
+    const key = options.key ?? name;
+    const gate = await DBOS.runStep(() => readGate(runtime, run.runId), {
+        name: `${name}:gate`,
+        retriesAllowed: false,
+    });
+    if (gate.paused) {
+        await concludeRun(runtime.control.pool, run.runId, run.workflowId, "paused", null);
+        throw new Suspend(run.runId, "paused", `the app is paused, before step ${key}`);
+    }
+    if (gate.currentWorkflowId !== run.workflowId) {
+        throw new StaleAttempt(run.runId, run.workflowId);
+    }
+    const context = {
+        runId: run.runId,
+        attempt: run.attempt,
+        workflowId: run.workflowId,
+        key,
+        tx: (work) => runtime.steps.tx(run.runId, run.workflowId, work),
+    };
+    return await DBOS.runStep(() => fn(context), {
+        name: key === name ? name : `${name}:${key}`,
+        retriesAllowed: false,
+    });
+}
+/** A plain read, so the step pool passes it without a tag. */
+async function readGate(runtime, runId) {
+    const { rows } = await runtime.steps.pool.query(STEP_GATE_STATEMENT, [runId]);
+    const row = rows[0];
+    return { paused: row.paused, currentWorkflowId: row.current_workflow_id };
+}

package/dist/suspend.d.ts

@@ -0,0 +1,17 @@
+/** The two run statuses an attempt can end in without being finished. */
+export declare const SUSPEND_STATUSES: readonly ["waiting", "paused"];
+export type SuspendStatus = (typeof SUSPEND_STATUSES)[number];
+/**
+ * Ends the current attempt without failing it: the flow stops here and the run is left for a
+ * later attempt to pick up. Thrown by the pause gate in `step()`, and by `waitForApproval`
+ * when that lands.
+ *
+ * It must be thrown from the flow body rather than from inside a `DBOS.runStep` body: a step
+ * that throws has its error checkpointed through `serialize-error`, and a replay revives a
+ * plain `Error` that no `instanceof Suspend` would catch.
+ */
+export declare class Suspend extends Error {
+    readonly runId: string;
+    readonly status: SuspendStatus;
+    constructor(runId: string, status: SuspendStatus, reason: string);
+}

package/dist/suspend.js

@@ -0,0 +1,21 @@
+/** The two run statuses an attempt can end in without being finished. */
+export const SUSPEND_STATUSES = ["waiting", "paused"];
+/**
+ * Ends the current attempt without failing it: the flow stops here and the run is left for a
+ * later attempt to pick up. Thrown by the pause gate in `step()`, and by `waitForApproval`
+ * when that lands.
+ *
+ * It must be thrown from the flow body rather than from inside a `DBOS.runStep` body: a step
+ * that throws has its error checkpointed through `serialize-error`, and a replay revives a
+ * plain `Error` that no `instanceof Suspend` would catch.
+ */
+export class Suspend extends Error {
+    runId;
+    status;
+    constructor(runId, status, reason) {
+        super(`Suspend: run ${runId} ends this attempt ${status} — ${reason}`);
+        this.name = "Suspend";
+        this.runId = runId;
+        this.status = status;
+    }
+}

package/dist/telegram.d.ts

@@ -0,0 +1,91 @@
+import { type ApprovalDecisionKind, type DecideOptions, type DecideResult } from "./approvals.js";
+/**
+ * The callback half of Telegram, and only that half: no bot, no polling, nothing sent. A bot
+ * that writes the message carrying these buttons is Phase 6's, along with `hf_telegram_link`
+ * and a `TELEGRAM_BOT_TOKEN`; what Phase 2 owns is what happens when a button is pressed.
+ */
+/** Telegram's own cap on `callback_data`, in bytes — not characters (Bot API "1-64 bytes"). */
+export declare const CALLBACK_DATA_MAX_BYTES = 64;
+/** Leads every `callback_data` this package writes, so a foreign button is ignored, not decoded. */
+export declare const CALLBACK_DATA_VERSION = "hf1";
+/** What a button can carry. Expiry and cancellation are nobody's button — they are sweeps. */
+export type TelegramDecision = Extract<ApprovalDecisionKind, "approved" | "rejected">;
+export interface TelegramCallbackData {
+    approvalId: number;
+    decision: TelegramDecision;
+    /** Per-message, minted by whoever sent the message; the `decisionKey`'s second half. */
+    nonce: string;
+}
+export declare class CallbackDataTooLong extends Error {
+    constructor(data: string);
+}
+/**
+ * `hf1:<a|r>:<approvalId>:<nonce>`. Fixed-width fields would leave less room for the nonce than
+ * the separator does, and the version leads so a button from an older deploy decodes or is
+ * ignored rather than being misread.
+ */
+export declare function encodeCallbackData(data: TelegramCallbackData): string;
+/** The longest nonce that still fits beside this approval id; how a sender picks its width. */
+export declare function maxNonceLength(approvalId: number): number;
+/** `null` for anything that is not one of ours — nothing here throws on a foreign button. */
+export declare function decodeCallbackData(data: string | undefined): TelegramCallbackData | null;
+/** The approval and the message that offered it, which is what makes a redelivery a replay. */
+export declare function decisionKeyFor(data: TelegramCallbackData): string;
+/** As much of Telegram's `User` as a decision needs; the rest of the object rides along. */
+export interface TelegramCallbackFrom {
+    id: number;
+    username?: string;
+}
+export interface TelegramCallbackOptions {
+    /**
+     * `app.approvals.decide` — the one way an approval is decided. Handed in because `workflows`
+     * cannot import `core`, and because nothing here owns a control pool or a `DBOSClient`.
+     */
+    decide(options: DecideOptions): Promise<DecideResult>;
+    /**
+     * The `X-Telegram-Bot-Api-Secret-Token` pair: what the webhook was registered with, and what
+     * this request carried. Both halves together, because a configured secret with nothing to
+     * compare it against would be a webhook anybody can post to.
+     */
+    secret?: {
+        expected: string;
+        received: string | undefined;
+    };
+    /**
+     * The Telegram user to the app user id written as `decided_by`, which Phase 6's
+     * `hf_telegram_link` will answer. Undefined leaves the decision unattributed.
+     */
+    userFor?: (from: TelegramCallbackFrom) => string | null | undefined;
+    /** True when the resolved user holds the admin role — `DecideOptions.admin`'s meaning. */
+    admin?: boolean;
+}
+/**
+ * `decided` covers the replay too: one delivery or three, the outcome is the decision that was
+ * written. `refused` is a callback that named nothing decidable — an unknown approval, a row
+ * someone else already decided, a stale nonce. `ignored` is an update this handler has no
+ * business with at all.
+ */
+export type TelegramCallbackOutcome = "decided" | "refused" | "ignored";
+export interface TelegramCallbackResult {
+    outcome: TelegramCallbackOutcome;
+    /** `callback_query.id`, which Phase 6's bot answers; null when the update carried none. */
+    callbackQueryId: string | null;
+    approvalId: number | null;
+    decision: TelegramDecision | null;
+    decisionKey: string | null;
+    /** True when this exact callback had already been decided: two deliveries, one decision. */
+    replayed: boolean;
+    /** Why nothing was decided; null when something was. */
+    reason: string | null;
+}
+/**
+ * A Telegram `callback_query` update, decoded and turned into one `decide()` call with
+ * `via: 'telegram'` and `decisionKey = <approvalId>:<nonce>`, so a redelivery of the same button
+ * press returns the first decision and writes nothing.
+ *
+ * Every permanent failure comes back as a result: a webhook that answers Telegram with an error
+ * is redelivered, and a callback that can never succeed would be redelivered forever. A
+ * transient failure — a lost commit, a deadlock, a dead connection — still throws, because that
+ * one *should* be retried, and so does a wiring bug.
+ */
+export declare function handleTelegramCallback(update: unknown, options: TelegramCallbackOptions): Promise<TelegramCallbackResult>;

package/dist/telegram.js

@@ -0,0 +1,161 @@
+import { timingSafeEqual } from "node:crypto";
+import * as z from "zod";
+import { ApprovalBatchRefused, } from "./approvals.js";
+/**
+ * The callback half of Telegram, and only that half: no bot, no polling, nothing sent. A bot
+ * that writes the message carrying these buttons is Phase 6's, along with `hf_telegram_link`
+ * and a `TELEGRAM_BOT_TOKEN`; what Phase 2 owns is what happens when a button is pressed.
+ */
+/** Telegram's own cap on `callback_data`, in bytes — not characters (Bot API "1-64 bytes"). */
+export const CALLBACK_DATA_MAX_BYTES = 64;
+/** Leads every `callback_data` this package writes, so a foreign button is ignored, not decoded. */
+export const CALLBACK_DATA_VERSION = "hf1";
+/** The nonce's charset: one byte per character, and no `:` to confuse the separator with. */
+const NONCE_PATTERN = /^[A-Za-z0-9_-]+$/;
+/** No leading zeros and no sign, so one approval id has exactly one encoding. */
+const APPROVAL_ID_PATTERN = /^[1-9][0-9]*$/;
+const DECISION_CODES = { approved: "a", rejected: "r" };
+const DECISIONS_BY_CODE = { a: "approved", r: "rejected" };
+export class CallbackDataTooLong extends Error {
+    constructor(data) {
+        super(`CallbackDataTooLong: ${JSON.stringify(data)} is ${Buffer.byteLength(data)} bytes, over ` +
+            `Telegram's ${CALLBACK_DATA_MAX_BYTES}-byte limit on callback_data — shorten the nonce`);
+        this.name = "CallbackDataTooLong";
+    }
+}
+/**
+ * `hf1:<a|r>:<approvalId>:<nonce>`. Fixed-width fields would leave less room for the nonce than
+ * the separator does, and the version leads so a button from an older deploy decodes or is
+ * ignored rather than being misread.
+ */
+export function encodeCallbackData(data) {
+    if (!APPROVAL_ID_PATTERN.test(String(data.approvalId))) {
+        throw new TypeError(`encodeCallbackData: ${data.approvalId} is not an approval id`);
+    }
+    if (!NONCE_PATTERN.test(data.nonce)) {
+        throw new TypeError(`encodeCallbackData: nonce ${JSON.stringify(data.nonce)} is not [A-Za-z0-9_-]+`);
+    }
+    const encoded = `${CALLBACK_DATA_VERSION}:${DECISION_CODES[data.decision]}:` +
+        `${data.approvalId}:${data.nonce}`;
+    if (Buffer.byteLength(encoded) > CALLBACK_DATA_MAX_BYTES)
+        throw new CallbackDataTooLong(encoded);
+    return encoded;
+}
+/** The longest nonce that still fits beside this approval id; how a sender picks its width. */
+export function maxNonceLength(approvalId) {
+    return CALLBACK_DATA_MAX_BYTES - `${CALLBACK_DATA_VERSION}:a:${approvalId}:`.length;
+}
+/** `null` for anything that is not one of ours — nothing here throws on a foreign button. */
+export function decodeCallbackData(data) {
+    if (data === undefined)
+        return null;
+    if (Buffer.byteLength(data) > CALLBACK_DATA_MAX_BYTES)
+        return null;
+    const parts = data.split(":");
+    if (parts.length !== 4)
+        return null;
+    const [version, code, id, nonce] = parts;
+    if (version !== CALLBACK_DATA_VERSION)
+        return null;
+    if (!Object.hasOwn(DECISIONS_BY_CODE, code))
+        return null;
+    if (!APPROVAL_ID_PATTERN.test(id))
+        return null;
+    const approvalId = Number(id);
+    if (!Number.isSafeInteger(approvalId))
+        return null;
+    if (!NONCE_PATTERN.test(nonce))
+        return null;
+    return {
+        approvalId,
+        decision: DECISIONS_BY_CODE[code],
+        nonce,
+    };
+}
+/** The approval and the message that offered it, which is what makes a redelivery a replay. */
+export function decisionKeyFor(data) {
+    return `${data.approvalId}:${data.nonce}`;
+}
+const CallbackUpdate = z.looseObject({
+    callback_query: z.looseObject({
+        id: z.string(),
+        data: z.string().optional(),
+        from: z.looseObject({ id: z.number(), username: z.string().optional() }),
+    }),
+});
+/**
+ * A Telegram `callback_query` update, decoded and turned into one `decide()` call with
+ * `via: 'telegram'` and `decisionKey = <approvalId>:<nonce>`, so a redelivery of the same button
+ * press returns the first decision and writes nothing.
+ *
+ * Every permanent failure comes back as a result: a webhook that answers Telegram with an error
+ * is redelivered, and a callback that can never succeed would be redelivered forever. A
+ * transient failure — a lost commit, a deadlock, a dead connection — still throws, because that
+ * one *should* be retried, and so does a wiring bug.
+ */
+export async function handleTelegramCallback(update, options) {
+    // Before the payload is looked at, let alone trusted.
+    if (options.secret !== undefined && !secretMatches(options.secret)) {
+        return ignored(null, "the request's secret token did not match the webhook's");
+    }
+    const parsed = CallbackUpdate.safeParse(update);
+    if (!parsed.success)
+        return ignored(null, "the update carries no callback_query");
+    const query = parsed.data.callback_query;
+    const data = decodeCallbackData(query.data);
+    if (data === null) {
+        return ignored(query.id, `callback data ${JSON.stringify(query.data ?? null)} is not ours`);
+    }
+    const decisionKey = decisionKeyFor(data);
+    try {
+        const result = await options.decide({
+            ids: [data.approvalId],
+            decision: data.decision,
+            via: "telegram",
+            decisionKey,
+            userId: options.userFor?.(query.from) ?? null,
+            ...(options.admin === undefined ? {} : { admin: options.admin }),
+        });
+        return {
+            outcome: "decided",
+            callbackQueryId: query.id,
+            approvalId: data.approvalId,
+            decision: data.decision,
+            decisionKey,
+            replayed: result.replayed,
+            reason: null,
+        };
+    }
+    catch (error) {
+        if (!(error instanceof ApprovalBatchRefused))
+            throw error;
+        return {
+            outcome: "refused",
+            callbackQueryId: query.id,
+            approvalId: data.approvalId,
+            decision: data.decision,
+            decisionKey,
+            replayed: false,
+            reason: error.reasons.map((r) => `${r.approvalId} ${r.reason}`).join("; "),
+        };
+    }
+}
+function secretMatches(secret) {
+    if (secret.received === undefined)
+        return false;
+    const expected = Buffer.from(secret.expected);
+    const received = Buffer.from(secret.received);
+    // `timingSafeEqual` throws on a length mismatch, which is not secret anyway.
+    return expected.length === received.length && timingSafeEqual(expected, received);
+}
+function ignored(callbackQueryId, reason) {
+    return {
+        outcome: "ignored",
+        callbackQueryId,
+        approvalId: null,
+        decision: null,
+        decisionKey: null,
+        replayed: false,
+        reason,
+    };
+}

package/dist/worker-lock.d.ts

@@ -0,0 +1,37 @@
+import { Client } from "pg";
+/**
+ * Session-level, so the lock is held by the connection rather than by a transaction, and
+ * released only when that connection goes away. Advisory locks are scoped to the database,
+ * and the key is namespaced by app name on top of that.
+ */
+export declare const WORKER_LOCK_STATEMENT = "SELECT pg_try_advisory_lock(hashtext('hf-worker:' || $1::text)) AS acquired, now() AS acquired_at";
+/**
+ * `<marker> <appName> <iso>`, where the timestamp is the *database's*, read in the same
+ * statement as the lock. Redeploy case 7 orders it against the previous worker's last
+ * committed write, which is a database timestamp too — two process clocks could not be
+ * compared at all.
+ */
+export declare const LOCK_ACQUIRED_MARKER = "hf-worker: advisory lock acquired";
+export declare class WorkerLockUnavailable extends Error {
+    readonly appName: string;
+    constructor(appName: string);
+}
+/**
+ * `Client` with `end` typed away. The lock must not be free while any step body of this
+ * process can still write — `DBOS.shutdown()`'s drain abandons unfinished workflows, so the
+ * only safe release is process death (round-2 finding 1, process half). Omitting `end` makes
+ * "never closed by code" a type error for any caller, not just a doc comment the SIGTERM
+ * handler has to remember to honor.
+ */
+export type HeldLockConnection = Omit<Client, "end">;
+export interface WorkerLock {
+    readonly appName: string;
+    readonly connection: HeldLockConnection;
+    /** The database clock at acquisition, not this process's. */
+    readonly acquiredAt: Date;
+}
+/**
+ * Taken on a connection of its own, not one of either pool's: a pooled client is returned,
+ * recycled and eventually closed, any of which would drop the lock under a running worker.
+ */
+export declare function acquireWorkerLock(databaseUrl: string, appName: string): Promise<WorkerLock>;

package/dist/worker-lock.js

@@ -0,0 +1,48 @@
+import { Client } from "pg";
+/**
+ * Session-level, so the lock is held by the connection rather than by a transaction, and
+ * released only when that connection goes away. Advisory locks are scoped to the database,
+ * and the key is namespaced by app name on top of that.
+ */
+export const WORKER_LOCK_STATEMENT = "SELECT pg_try_advisory_lock(hashtext('hf-worker:' || $1::text)) AS acquired, now() AS acquired_at";
+/**
+ * `<marker> <appName> <iso>`, where the timestamp is the *database's*, read in the same
+ * statement as the lock. Redeploy case 7 orders it against the previous worker's last
+ * committed write, which is a database timestamp too — two process clocks could not be
+ * compared at all.
+ */
+export const LOCK_ACQUIRED_MARKER = "hf-worker: advisory lock acquired";
+export class WorkerLockUnavailable extends Error {
+    appName;
+    constructor(appName) {
+        super(`WorkerLockUnavailable: another worker process holds the advisory lock for ${appName}; ` +
+            "two workers for one app never run at once");
+        this.name = "WorkerLockUnavailable";
+        this.appName = appName;
+    }
+}
+/**
+ * Taken on a connection of its own, not one of either pool's: a pooled client is returned,
+ * recycled and eventually closed, any of which would drop the lock under a running worker.
+ */
+export async function acquireWorkerLock(databaseUrl, appName) {
+    const connection = new Client({ connectionString: databaseUrl });
+    await connection.connect();
+    let row;
+    try {
+        const result = await connection.query(WORKER_LOCK_STATEMENT, [appName]);
+        row = result.rows[0];
+    }
+    catch (error) {
+        await connection.end().catch(() => undefined);
+        throw error;
+    }
+    if (row?.acquired !== true) {
+        // Nothing is held on this connection, so it is not a lock connection and closing it here
+        // costs nothing — the holder is some other process.
+        await connection.end().catch(() => undefined);
+        throw new WorkerLockUnavailable(appName);
+    }
+    console.info(LOCK_ACQUIRED_MARKER, appName, row.acquired_at.toISOString());
+    return { appName, connection, acquiredAt: row.acquired_at };
+}