@volter/twin-upstash 0.1.0

package/dist/qstash/src/semantics/delivery.js

@@ -0,0 +1,274 @@
+import { accountOf } from "./account.js";
+import { acceptToUrlGroup, ensureQueue, storeMessage } from "./messages.js";
+import { nextFire } from "./schedules.js";
+import { failRun, failureCallSettled, rawSteps, timeOut, workflowHeaders } from "./workflows.js";
+import { b64, iso, multiValue, nowMs } from "./shared.js";
+import { post } from "../egress.js";
+import { sign } from "../signing.js";
+/** The World's word on what a destination answers, stored by `POST /_twin/destinations` (../doors.ts). */
+export const DESTINATION = '_qstash_destination';
+/** What a destination received, read at `GET /_twin/deliveries?to=` (../doors.ts). */
+export const DELIVERY = '_qstash_delivery';
+const DAY_MS = 86_400_000;
+const DLQ_RETENTION_MS = 7 * DAY_MS;
+const PENDING = new Set(['CREATED', 'ACTIVE', 'RETRY']);
+/** The answer the World has stated for a delivery: of the statements whose URL prefixes the delivery's and whose `when`
+ *  headers (if any) it carries, one that names headers before one that does not, then the longest prefix. */
+function stated(ctx, url, sent) {
+    const holds = (d) => url.startsWith(String(d.url)) && Object.entries((d.when ?? {})).every(([k, v]) => sent[k.toLowerCase()] === v);
+    const hit = ctx.rowsRaw(DESTINATION).filter(holds)
+        .sort((a, b) => Number(Object.keys(b.when ?? {}).length > 0) - Number(Object.keys(a.when ?? {}).length > 0) || String(b.url).length - String(a.url).length)[0];
+    return hit ? { status: Number(hit.status), body: String(hit.body ?? ''), headers: (hit.headers ?? {}) } : undefined;
+}
+/** QStash's backoff for retry n, in milliseconds. */
+const backoffMs = (n) => Math.round(Math.min(86_400, Math.exp(2.5 * n)) * 1000);
+/** A `Retry-After` answer in milliseconds: seconds, a duration (`6m5s`) or a date, capped at a day. */
+function retryAfterMs(headers, at) {
+    const raw = headers['retry-after']?.trim();
+    return raw ? retryAfterValue(raw, at) : undefined;
+}
+/** A `Retry-After` value in milliseconds (https://upstash.com/docs/qstash/features/retry, Retry-After Headers). */
+function retryAfterValue(raw, at) {
+    const d = /^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?$/.exec(raw);
+    const sec = /^\d+$/.test(raw) ? Number(raw) : d && raw !== '' ? Number(d[1] ?? 0) * 3600 + Number(d[2] ?? 0) * 60 + Number(d[3] ?? 0) : Number.isNaN(Date.parse(raw)) ? undefined : Math.max(0, (Date.parse(raw) - at) / 1000);
+    return sec === undefined ? undefined : Math.round(Math.min(86_400, sec) * 1000);
+}
+/** An `Upstash-Retry-Delay` expression's value in milliseconds: numbers, `retried`, `+ - * /`, parentheses and `pow sqrt
+ *  abs exp floor ceil round min max` (the retry page's Custom Retry Delay); anything else is undefined. */
+export function retryDelayMs(expr, retried) {
+    const src = expr.replace(/\s+/g, '');
+    let i = 0;
+    const fns = { pow: Math.pow, sqrt: Math.sqrt, abs: Math.abs, exp: Math.exp, floor: Math.floor, ceil: Math.ceil, round: Math.round, min: Math.min, max: Math.max };
+    const fail = () => { throw new Error('retryDelay'); };
+    const primary = () => {
+        if (src[i] === '(') {
+            i += 1;
+            const v = sum();
+            if (src[i] !== ')')
+                fail();
+            i += 1;
+            return v;
+        }
+        const num = /^\d+(\.\d+)?/.exec(src.slice(i));
+        if (num) {
+            i += num[0].length;
+            return Number(num[0]);
+        }
+        const id = /^[a-z]+/.exec(src.slice(i));
+        if (!id)
+            return fail();
+        i += id[0].length;
+        if (id[0] === 'retried')
+            return retried;
+        const fn = fns[id[0]];
+        if (!fn || src[i] !== '(')
+            return fail();
+        i += 1;
+        const args = [sum()];
+        while (src[i] === ',') {
+            i += 1;
+            args.push(sum());
+        }
+        if (src[i] !== ')')
+            fail();
+        i += 1;
+        return fn(...args);
+    };
+    const product = () => { let v = primary(); while (src[i] === '*' || src[i] === '/') {
+        const op = src[i];
+        i += 1;
+        const r = primary();
+        v = op === '*' ? v * r : v / r;
+    } return v; };
+    const sum = () => { let v = product(); while (src[i] === '+' || src[i] === '-') {
+        const op = src[i];
+        i += 1;
+        const r = product();
+        v = op === '+' ? v + r : v - r;
+    } return v; };
+    try {
+        const v = sum();
+        return i === src.length && Number.isFinite(v) && v >= 0 ? v : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** A queue delivers in order, as many at a time as its parallelism ("The number of parallel consumers consuming from the
+ *  queue", the spec's Queue), and none while it is paused ("Pausing a queue stops the delivery of enqueued messages", the
+ *  spec's pause). A message whose queue was deleted goes as a queue of one would. */
+function queueLets(ctx, m, all) {
+    const q = ctx.rowsRaw('Queue').find((r) => r.name === m.queueName);
+    if (q?.paused === true)
+        return false;
+    const ahead = all.filter((o) => o.queueName === m.queueName && PENDING.has(String(o.state)) && Number(o._ordinal) < Number(m._ordinal)).length;
+    return ahead < Math.max(1, Number(q?.parallelism ?? 1));
+}
+/** Whether a pending message may go now: its queue lets it (queueLets), and it is the first undelivered of its run. */
+function eligible(ctx, m, all) {
+    if (m.state !== 'CREATED' && m.state !== 'RETRY')
+        return false;
+    if (typeof m.queueName === 'string' && !queueLets(ctx, m, all))
+        return false;
+    if (typeof m._run === 'string') {
+        const run = ctx.rowsRaw('WorkflowRun').find((r) => r.workflowRunId === m._run);
+        if (!run || run.workflowState !== 'RUN_STARTED')
+            return false;
+        if (all.some((o) => o._run === m._run && PENDING.has(String(o.state)) && Number(o._seq) < Number(m._seq)))
+            return false;
+    }
+    return true;
+}
+function nextDue(ctx, until) {
+    const due = [];
+    for (const s of ctx.rowsRaw('Schedule')) {
+        if (typeof s.nextScheduleTime === 'number' && s.nextScheduleTime <= until)
+            due.push({ when: s.nextScheduleTime, order: 0, act: (at) => fire(at, s) });
+    }
+    for (const w of ctx.rowsRaw('Waiter')) {
+        const when = Number(w.deadline) * 1000;
+        if (when <= until)
+            due.push({ when, order: 1, act: (at) => timeOut(at, w) });
+    }
+    const all = ctx.rowsRaw('Message');
+    for (const m of all) {
+        const when = Number(m._due);
+        if (when > until)
+            continue;
+        if (m.state === 'CANCEL_REQUESTED')
+            due.push({ when, order: 2 + Number(m._ordinal) / 1e9, act: (at) => cancelled(at, m) });
+        else if (eligible(ctx, m, all))
+            due.push({ when, order: 3 + Number(m._ordinal) / 1e9, act: (at, report) => attempt(at, m, report) });
+    }
+    for (const resource of ['DLQMessage', 'WorkflowDLQMessage']) {
+        for (const e of ctx.rowsRaw(resource)) {
+            const when = Number(e._filed_at ?? e.createdAt) + DLQ_RETENTION_MS;
+            if (when <= until)
+                due.push({ when, order: 4, act: (at) => at.write(resource, String(e.id), { deleted: true }, 'dlq.expired').then(() => undefined) });
+        }
+    }
+    return due.sort((a, b) => a.when - b.when || a.order - b.order)[0];
+}
+/** Run everything due by the World's instant, in order, each at its own moment. */
+export async function catchUp(ctx) {
+    const until = nowMs(ctx);
+    const report = { delivered: [], retried: [], failed: [] };
+    // a message held behind another (the one before it in its queue or its run) goes once that one has left, so the World
+    // clock never runs back: each move happens at its own due time or at the last move's, whichever is later
+    let last = 0;
+    for (let n = 0; n < 5000; n += 1) {
+        const next = nextDue(ctx, until);
+        if (!next)
+            break;
+        last = Math.max(last, next.when);
+        await next.act(await ctx.at(iso(last)), report);
+    }
+    return report;
+}
+/** A schedule's cron has fired: its message is published at that moment (a paused schedule's is not), and it is next
+ *  due at the cron's following minute. */
+async function fire(ctx, s) {
+    const when = Number(s.nextScheduleTime);
+    if (s.isPaused !== true) {
+        const { 'upstash-cron': _cron, 'upstash-schedule-id': _id, 'upstash-queue-name': _q, ...headers } = (s._published ?? {});
+        const queue = typeof s._queue === 'string' ? s._queue : undefined;
+        if (queue)
+            await ensureQueue(ctx, queue);
+        const destination = String(s.destination);
+        if (/^https?:\/\//.test(destination))
+            await storeMessage(ctx, { url: destination, headers, body: String(s.body ?? ''), owner: String(s._owner), scheduleId: String(s.scheduleId), queue });
+        else
+            await acceptToUrlGroup(ctx, { destination, headers, body: String(s.body ?? ''), queue }, String(s._owner));
+    }
+    const next = nextFire(String(s.cron), when);
+    await ctx.write('Schedule', String(s.id), { lastScheduleTime: when, ...(next !== undefined ? { nextScheduleTime: next } : {}) }, 'schedule.fire');
+}
+/** A cancelled message reaches its next delivery time: it is CANCELLED, never delivered. */
+async function cancelled(ctx, m) {
+    const refused = ctx.legal('Message', 'state', 'delete_v2_messages_messageid', m.state, 'CANCELLED', String(m.id), 'time');
+    if (refused)
+        throw new Error(refused.message);
+    await ctx.write('Message', String(m.id), { state: 'CANCELLED' }, 'message.cancelled');
+}
+/** Move a message, as its machine allows. */
+async function move(ctx, m, to, actor, fields) {
+    const refused = ctx.legal('Message', 'state', 'delete_v2_messages_messageid', m.state, to, String(m.id), actor);
+    if (refused)
+        throw new Error(refused.message);
+    await ctx.write('Message', String(m.id), { ...fields, state: to }, `message.${to.toLowerCase()}`);
+    m.state = to;
+}
+/** One delivery attempt of a message, at the moment it is due. */
+async function attempt(ctx, m, report) {
+    const when = nowMs(ctx);
+    const tried = Number(m._attempts ?? 0);
+    await move(ctx, m, 'ACTIVE', 'time', { _attempts: tried });
+    const published = (m._published ?? {});
+    const step = typeof m._run === 'string';
+    const body = step ? JSON.stringify(rawSteps(ctx, m._run, Number(m._seq))) : String(m.body ?? '');
+    const base = published['upstash-workflow-runid'] !== undefined ? workflowHeaders(published) : {
+        ...(typeof m._content_type === 'string' ? { 'content-type': m._content_type } : {}),
+        ...Object.fromEntries(Object.entries(published).filter(([k]) => k.startsWith('upstash-forward-')).map(([k, v]) => [k.slice('upstash-forward-'.length), v])),
+    };
+    const url = String(m.url);
+    const key = accountOf(ctx, String(m._owner ?? 'world')).current;
+    const headers = {
+        ...base, 'upstash-message-id': String(m.messageId), 'upstash-retried': String(tried),
+        'upstash-signature': sign(body, url, Math.floor(when / 1000), `jwt_${String(m.messageId)}_${tried}`, key),
+    };
+    const method = String(m.method ?? 'POST');
+    const outcome = stated(ctx, url, headers) ?? await post(url, method, headers, body);
+    await ctx.record(DELIVERY, { to: url, message_id: m.messageId, retried: tried, at: ctx.occurredAt, n: ctx.rowsRaw(DELIVERY).length, method, headers, body, status: outcome.status }, `delivery:${String(m.messageId)}:${tried}`);
+    const n = tried + 1;
+    const response = { _response_status: outcome.status, _response_body: outcome.body, _response_headers: outcome.headers };
+    if (outcome.status >= 200 && outcome.status < 300) {
+        await move(ctx, m, 'DELIVERED', 'vendor', { _attempts: n, ...response });
+        report.delivered.push({ messageId: m.messageId, url, status: outcome.status });
+        if (typeof m._callback_of === 'string')
+            await failureCallSettled(ctx, m._callback_of, true);
+        else if (typeof m.callback === 'string')
+            await callback(ctx, m, String(m.callback), outcome, n, undefined);
+        return;
+    }
+    const nonRetryable = outcome.status === 489 && outcome.headers['upstash-nonretryable-error'] === 'true';
+    if (n <= Number(m.maxRetries ?? 3) && !nonRetryable) {
+        const run = step ? ctx.rowsRaw('WorkflowRun').find((r) => r.workflowRunId === m._run) : undefined;
+        const expr = typeof m._retry_delay === 'string' ? m._retry_delay : typeof run?._retry_delay === 'string' ? run._retry_delay : undefined;
+        const wait = retryAfterMs(outcome.headers, when) ?? (expr !== undefined ? retryDelayMs(expr, tried) : undefined) ?? backoffMs(n);
+        await move(ctx, m, 'RETRY', 'vendor', { _attempts: n, _due: when + wait, ...response });
+        report.retried.push({ messageId: m.messageId, url, status: outcome.status, retried: n, nextDeliveryTime: when + wait });
+        return;
+    }
+    await move(ctx, m, 'FAILED', 'vendor', { _attempts: n, ...response });
+    report.failed.push({ messageId: m.messageId, url, status: outcome.status });
+    if (typeof m._callback_of === 'string') {
+        await failureCallSettled(ctx, m._callback_of, false);
+        return;
+    }
+    if (step) {
+        await failRun(ctx, { ...m, _attempts: n }, outcome);
+        return;
+    }
+    const dlqId = `${when}-${ctx.rowsRaw('DLQMessage', { withDeleted: true }).length}`;
+    const own = ctx.own(m);
+    await ctx.write('DLQMessage', dlqId, {
+        ...Object.fromEntries(Object.entries(own).filter(([k]) => !k.startsWith('_') && k !== 'state')),
+        dlqId, responseStatus: outcome.status, responseHeader: multiValue(outcome.headers), responseBody: outcome.body,
+        _filed_at: when, _published: published, _message_id: m.messageId, _owner: m._owner ?? 'world', ...(m._redact ? { _redact: m._redact } : {}),
+    }, 'dlq.create');
+    if (typeof m.failureCallback === 'string')
+        await callback(ctx, m, String(m.failureCallback), outcome, n, dlqId);
+    if (typeof m.callback === 'string')
+        await callback(ctx, m, String(m.callback), outcome, n, undefined);
+}
+/** A callback: "QStash calls your callback URL with the response"; its body is the callbacks page's (`status`, `header`,
+ *  base64 `body`, `retried`, `maxRetries`, `sourceMessageId`, `url`, `method`, `sourceHeader`, base64 `sourceBody`,
+ *  `notBefore`, `createdAt`, `callerIP`), and a failure callback's adds the `dlqId`. It is a message of its own. */
+async function callback(ctx, m, url, o, attempts, dlqId) {
+    const body = JSON.stringify({
+        status: o.status, header: multiValue(o.headers), body: b64(o.body), retried: attempts - 1, maxRetries: Number(m.maxRetries ?? 3),
+        ...(dlqId ? { dlqId } : {}), sourceMessageId: m.messageId, url: m.url, method: m.method ?? 'POST', sourceHeader: m.header ?? {},
+        sourceBody: b64(String(m.body ?? '')), notBefore: m.notBefore, createdAt: m.createdAt, callerIP: '127.0.0.1',
+    });
+    await storeMessage(ctx, { url, headers: { 'content-type': 'application/json' }, body, owner: String(m._owner ?? 'world'), callbackOf: String(m.messageId) });
+}

package/dist/qstash/src/semantics/delivery.ts

@@ -0,0 +1,264 @@
+// The QStash lane's catch-up (docs/contributing/architecture.md, "Moves no API call makes": time): before the lane answers
+// any request on a writable World, everything the World clock has made due happens, in order, each at the moment it fell
+// due: a schedule's cron fires and publishes its message, a wait times out, a message is delivered (or retried, or
+// failed into the DLQ, its callbacks published), a cancelled message is marked CANCELLED, and a DLQ entry past its
+// retention is removed. There is no scheduler: the World clock is read, never watched.
+//
+// Delivery (https://upstash.com/docs/qstash/howto/debug-logs, /features/retry, /features/callbacks, /features/dlq):
+// - A message is delivered when due: its publish instant, plus `Upstash-Delay`, or its `Upstash-Not-Before`. Messages
+//   of one queue go one at a time in order ("The next message will wait for retries of the current one"), and a
+//   workflow run's messages go in the order the run made them.
+// - The destination receives the body and the publish's `Content-Type`, every `Upstash-Forward-*` header with its prefix
+//   stripped, `Upstash-Message-Id`, `Upstash-Retried` ("0" first, then how many retries so far) and an
+//   `Upstash-Signature` signed with the account's current key (signing.ts); a workflow's delivery carries the run's steps
+//   and headers (workflows.ts).
+// - "If your API does not respond with a success status code (2XX), we retry the request": retry n waits
+//   `min(86400, e ** (2.5*n))` seconds, or what the destination's `Retry-After` asks (seconds, a duration or a date, capped
+//   at a day), or the message's `Upstash-Retry-Delay` expression (milliseconds, `retried` from 0). A 489 with
+//   `Upstash-NonRetryable-Error: true` is not retried.
+// - "If all retries are exhausted, the task has FAILED and the message will be moved to the DLQ", and its
+//   `Upstash-Failure-Callback` is called; its `Upstash-Callback` is called with the answer once delivered or failed. A
+//   callback is a message of its own, delivered, signed and retried the same way, and never called back itself.
+// - DLQ retention: "Messages are deleted when their retention period expires" (/features/dlq; the pricing page's Pay as
+//   You Go column: DLQ 7 days; the Workflow DLQ "Pay-as-you-go: 1 week", https://upstash.com/docs/workflow/features/dlq).
+//
+// What a destination answers is the World's: a destination the World has stated an answer for (the destinations door,
+// ../doors.ts) answers that, and any other is asked over HTTP where the World lets the request out (egress.ts); a
+// sealed World refuses it, which is a failed attempt. Where the documentation stops and the lane decides: the account's
+// plan is Pay as You Go; a callback is called once when the message is delivered or fails, not after every retry.
+import type { SemanticsContext } from '@volter/world-core';
+import { accountOf } from './account.ts';
+import { acceptToUrlGroup, ensureQueue, storeMessage } from './messages.ts';
+import { nextFire } from './schedules.ts';
+import { failRun, failureCallSettled, rawSteps, timeOut, workflowHeaders } from './workflows.ts';
+import { b64, iso, multiValue, nowMs, type Row } from './shared.ts';
+import { post, type Outcome } from '../egress.ts';
+import { sign } from '../signing.ts';
+
+/** The World's word on what a destination answers, stored by `POST /_twin/destinations` (../doors.ts). */
+export const DESTINATION = '_qstash_destination';
+/** What a destination received, read at `GET /_twin/deliveries?to=` (../doors.ts). */
+export const DELIVERY = '_qstash_delivery';
+
+const DAY_MS = 86_400_000;
+const DLQ_RETENTION_MS = 7 * DAY_MS;
+const PENDING = new Set(['CREATED', 'ACTIVE', 'RETRY']);
+
+/** The answer the World has stated for a delivery: of the statements whose URL prefixes the delivery's and whose `when`
+ *  headers (if any) it carries, one that names headers before one that does not, then the longest prefix. */
+function stated(ctx: SemanticsContext, url: string, sent: Record<string, string>): Outcome | undefined {
+  const holds = (d: Row): boolean => url.startsWith(String(d.url)) && Object.entries((d.when ?? {}) as Record<string, string>).every(([k, v]) => sent[k.toLowerCase()] === v);
+  const hit = ctx.rowsRaw(DESTINATION).filter(holds)
+    .sort((a, b) => Number(Object.keys(b.when ?? {}).length > 0) - Number(Object.keys(a.when ?? {}).length > 0) || String(b.url).length - String(a.url).length)[0];
+  return hit ? { status: Number(hit.status), body: String(hit.body ?? ''), headers: (hit.headers ?? {}) as Record<string, string> } : undefined;
+}
+
+/** QStash's backoff for retry n, in milliseconds. */
+const backoffMs = (n: number): number => Math.round(Math.min(86_400, Math.exp(2.5 * n)) * 1000);
+
+/** A `Retry-After` answer in milliseconds: seconds, a duration (`6m5s`) or a date, capped at a day. */
+function retryAfterMs(headers: Record<string, string>, at: number): number | undefined {
+  const raw = headers['retry-after']?.trim();
+  return raw ? retryAfterValue(raw, at) : undefined;
+}
+
+/** A `Retry-After` value in milliseconds (https://upstash.com/docs/qstash/features/retry, Retry-After Headers). */
+function retryAfterValue(raw: string, at: number): number | undefined {
+  const d = /^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?$/.exec(raw);
+  const sec = /^\d+$/.test(raw) ? Number(raw) : d && raw !== '' ? Number(d[1] ?? 0) * 3600 + Number(d[2] ?? 0) * 60 + Number(d[3] ?? 0) : Number.isNaN(Date.parse(raw)) ? undefined : Math.max(0, (Date.parse(raw) - at) / 1000);
+  return sec === undefined ? undefined : Math.round(Math.min(86_400, sec) * 1000);
+}
+
+/** An `Upstash-Retry-Delay` expression's value in milliseconds: numbers, `retried`, `+ - * /`, parentheses and `pow sqrt
+ *  abs exp floor ceil round min max` (the retry page's Custom Retry Delay); anything else is undefined. */
+export function retryDelayMs(expr: string, retried: number): number | undefined {
+  const src = expr.replace(/\s+/g, '');
+  let i = 0;
+  const fns: Record<string, (...a: number[]) => number> = { pow: Math.pow, sqrt: Math.sqrt, abs: Math.abs, exp: Math.exp, floor: Math.floor, ceil: Math.ceil, round: Math.round, min: Math.min, max: Math.max };
+  const fail = (): never => { throw new Error('retryDelay'); };
+  const primary = (): number => {
+    if (src[i] === '(') { i += 1; const v = sum(); if (src[i] !== ')') fail(); i += 1; return v; }
+    const num = /^\d+(\.\d+)?/.exec(src.slice(i));
+    if (num) { i += num[0].length; return Number(num[0]); }
+    const id = /^[a-z]+/.exec(src.slice(i));
+    if (!id) return fail();
+    i += id[0].length;
+    if (id[0] === 'retried') return retried;
+    const fn = fns[id[0]];
+    if (!fn || src[i] !== '(') return fail();
+    i += 1;
+    const args = [sum()];
+    while (src[i] === ',') { i += 1; args.push(sum()); }
+    if (src[i] !== ')') fail();
+    i += 1;
+    return fn(...args);
+  };
+  const product = (): number => { let v = primary(); while (src[i] === '*' || src[i] === '/') { const op = src[i]; i += 1; const r = primary(); v = op === '*' ? v * r : v / r; } return v; };
+  const sum = (): number => { let v = product(); while (src[i] === '+' || src[i] === '-') { const op = src[i]; i += 1; const r = product(); v = op === '+' ? v + r : v - r; } return v; };
+  try { const v = sum(); return i === src.length && Number.isFinite(v) && v >= 0 ? v : undefined; } catch { return undefined; }
+}
+
+/** A queue delivers in order, as many at a time as its parallelism ("The number of parallel consumers consuming from the
+ *  queue", the spec's Queue), and none while it is paused ("Pausing a queue stops the delivery of enqueued messages", the
+ *  spec's pause). A message whose queue was deleted goes as a queue of one would. */
+function queueLets(ctx: SemanticsContext, m: Row, all: Row[]): boolean {
+  const q = ctx.rowsRaw('Queue').find((r) => r.name === m.queueName);
+  if (q?.paused === true) return false;
+  const ahead = all.filter((o) => o.queueName === m.queueName && PENDING.has(String(o.state)) && Number(o._ordinal) < Number(m._ordinal)).length;
+  return ahead < Math.max(1, Number(q?.parallelism ?? 1));
+}
+
+/** Whether a pending message may go now: its queue lets it (queueLets), and it is the first undelivered of its run. */
+function eligible(ctx: SemanticsContext, m: Row, all: Row[]): boolean {
+  if (m.state !== 'CREATED' && m.state !== 'RETRY') return false;
+  if (typeof m.queueName === 'string' && !queueLets(ctx, m, all)) return false;
+  if (typeof m._run === 'string') {
+    const run = ctx.rowsRaw('WorkflowRun').find((r) => r.workflowRunId === m._run);
+    if (!run || run.workflowState !== 'RUN_STARTED') return false;
+    if (all.some((o) => o._run === m._run && PENDING.has(String(o.state)) && Number(o._seq) < Number(m._seq))) return false;
+  }
+  return true;
+}
+
+/** What a catch-up did: the deliveries it made, by outcome (the drain door answers it). */
+export type CatchUpReport = { delivered: Row[]; retried: Row[]; failed: Row[] };
+
+type Due = { when: number; order: number; act: (at: SemanticsContext, report: CatchUpReport) => Promise<void> };
+
+function nextDue(ctx: SemanticsContext, until: number): Due | undefined {
+  const due: Due[] = [];
+  for (const s of ctx.rowsRaw('Schedule')) {
+    if (typeof s.nextScheduleTime === 'number' && s.nextScheduleTime <= until) due.push({ when: s.nextScheduleTime, order: 0, act: (at) => fire(at, s) });
+  }
+  for (const w of ctx.rowsRaw('Waiter')) {
+    const when = Number(w.deadline) * 1000;
+    if (when <= until) due.push({ when, order: 1, act: (at) => timeOut(at, w) });
+  }
+  const all = ctx.rowsRaw('Message');
+  for (const m of all) {
+    const when = Number(m._due);
+    if (when > until) continue;
+    if (m.state === 'CANCEL_REQUESTED') due.push({ when, order: 2 + Number(m._ordinal) / 1e9, act: (at) => cancelled(at, m) });
+    else if (eligible(ctx, m, all)) due.push({ when, order: 3 + Number(m._ordinal) / 1e9, act: (at, report) => attempt(at, m, report) });
+  }
+  for (const resource of ['DLQMessage', 'WorkflowDLQMessage']) {
+    for (const e of ctx.rowsRaw(resource)) {
+      const when = Number(e._filed_at ?? e.createdAt) + DLQ_RETENTION_MS;
+      if (when <= until) due.push({ when, order: 4, act: (at) => at.write(resource, String(e.id), { deleted: true }, 'dlq.expired').then(() => undefined) });
+    }
+  }
+  return due.sort((a, b) => a.when - b.when || a.order - b.order)[0];
+}
+
+/** Run everything due by the World's instant, in order, each at its own moment. */
+export async function catchUp(ctx: SemanticsContext): Promise<CatchUpReport> {
+  const until = nowMs(ctx);
+  const report: CatchUpReport = { delivered: [], retried: [], failed: [] };
+  // a message held behind another (the one before it in its queue or its run) goes once that one has left, so the World
+  // clock never runs back: each move happens at its own due time or at the last move's, whichever is later
+  let last = 0;
+  for (let n = 0; n < 5000; n += 1) {
+    const next = nextDue(ctx, until);
+    if (!next) break;
+    last = Math.max(last, next.when);
+    await next.act(await ctx.at(iso(last)), report);
+  }
+  return report;
+}
+
+/** A schedule's cron has fired: its message is published at that moment (a paused schedule's is not), and it is next
+ *  due at the cron's following minute. */
+async function fire(ctx: SemanticsContext, s: Row): Promise<void> {
+  const when = Number(s.nextScheduleTime);
+  if (s.isPaused !== true) {
+    const { 'upstash-cron': _cron, 'upstash-schedule-id': _id, 'upstash-queue-name': _q, ...headers } = (s._published ?? {}) as Record<string, string>;
+    const queue = typeof s._queue === 'string' ? s._queue : undefined;
+    if (queue) await ensureQueue(ctx, queue);
+    const destination = String(s.destination);
+    if (/^https?:\/\//.test(destination)) await storeMessage(ctx, { url: destination, headers, body: String(s.body ?? ''), owner: String(s._owner), scheduleId: String(s.scheduleId), queue });
+    else await acceptToUrlGroup(ctx, { destination, headers, body: String(s.body ?? ''), queue }, String(s._owner));
+  }
+  const next = nextFire(String(s.cron), when);
+  await ctx.write('Schedule', String(s.id), { lastScheduleTime: when, ...(next !== undefined ? { nextScheduleTime: next } : {}) }, 'schedule.fire');
+}
+
+/** A cancelled message reaches its next delivery time: it is CANCELLED, never delivered. */
+async function cancelled(ctx: SemanticsContext, m: Row): Promise<void> {
+  const refused = ctx.legal('Message', 'state', 'delete_v2_messages_messageid', m.state, 'CANCELLED', String(m.id), 'time');
+  if (refused) throw new Error(refused.message);
+  await ctx.write('Message', String(m.id), { state: 'CANCELLED' }, 'message.cancelled');
+}
+
+/** Move a message, as its machine allows. */
+async function move(ctx: SemanticsContext, m: Row, to: string, actor: 'time' | 'vendor', fields: Row): Promise<void> {
+  const refused = ctx.legal('Message', 'state', 'delete_v2_messages_messageid', m.state, to, String(m.id), actor);
+  if (refused) throw new Error(refused.message);
+  await ctx.write('Message', String(m.id), { ...fields, state: to }, `message.${to.toLowerCase()}`);
+  m.state = to;
+}
+
+/** One delivery attempt of a message, at the moment it is due. */
+async function attempt(ctx: SemanticsContext, m: Row, report: CatchUpReport): Promise<void> {
+  const when = nowMs(ctx);
+  const tried = Number(m._attempts ?? 0);
+  await move(ctx, m, 'ACTIVE', 'time', { _attempts: tried });
+  const published = (m._published ?? {}) as Record<string, string>;
+  const step = typeof m._run === 'string';
+  const body = step ? JSON.stringify(rawSteps(ctx, m._run as string, Number(m._seq))) : String(m.body ?? '');
+  const base = published['upstash-workflow-runid'] !== undefined ? workflowHeaders(published) : {
+    ...(typeof m._content_type === 'string' ? { 'content-type': m._content_type } : {}),
+    ...Object.fromEntries(Object.entries(published).filter(([k]) => k.startsWith('upstash-forward-')).map(([k, v]) => [k.slice('upstash-forward-'.length), v])),
+  };
+  const url = String(m.url);
+  const key = accountOf(ctx, String(m._owner ?? 'world')).current;
+  const headers: Record<string, string> = {
+    ...base, 'upstash-message-id': String(m.messageId), 'upstash-retried': String(tried),
+    'upstash-signature': sign(body, url, Math.floor(when / 1000), `jwt_${String(m.messageId)}_${tried}`, key),
+  };
+  const method = String(m.method ?? 'POST');
+  const outcome = stated(ctx, url, headers) ?? await post(url, method, headers, body);
+  await ctx.record(DELIVERY, { to: url, message_id: m.messageId, retried: tried, at: ctx.occurredAt, n: ctx.rowsRaw(DELIVERY).length, method, headers, body, status: outcome.status }, `delivery:${String(m.messageId)}:${tried}`);
+  const n = tried + 1;
+  const response = { _response_status: outcome.status, _response_body: outcome.body, _response_headers: outcome.headers };
+  if (outcome.status >= 200 && outcome.status < 300) {
+    await move(ctx, m, 'DELIVERED', 'vendor', { _attempts: n, ...response });
+    report.delivered.push({ messageId: m.messageId, url, status: outcome.status });
+    if (typeof m._callback_of === 'string') await failureCallSettled(ctx, m._callback_of, true);
+    else if (typeof m.callback === 'string') await callback(ctx, m, String(m.callback), outcome, n, undefined);
+    return;
+  }
+  const nonRetryable = outcome.status === 489 && outcome.headers['upstash-nonretryable-error'] === 'true';
+  if (n <= Number(m.maxRetries ?? 3) && !nonRetryable) {
+    const run = step ? ctx.rowsRaw('WorkflowRun').find((r) => r.workflowRunId === m._run) : undefined;
+    const expr = typeof m._retry_delay === 'string' ? m._retry_delay : typeof run?._retry_delay === 'string' ? run._retry_delay : undefined;
+    const wait = retryAfterMs(outcome.headers, when) ?? (expr !== undefined ? retryDelayMs(expr, tried) : undefined) ?? backoffMs(n);
+    await move(ctx, m, 'RETRY', 'vendor', { _attempts: n, _due: when + wait, ...response });
+    report.retried.push({ messageId: m.messageId, url, status: outcome.status, retried: n, nextDeliveryTime: when + wait });
+    return;
+  }
+  await move(ctx, m, 'FAILED', 'vendor', { _attempts: n, ...response });
+  report.failed.push({ messageId: m.messageId, url, status: outcome.status });
+  if (typeof m._callback_of === 'string') { await failureCallSettled(ctx, m._callback_of, false); return; }
+  if (step) { await failRun(ctx, { ...m, _attempts: n }, outcome); return; }
+  const dlqId = `${when}-${ctx.rowsRaw('DLQMessage', { withDeleted: true }).length}`;
+  const own = ctx.own(m);
+  await ctx.write('DLQMessage', dlqId, {
+    ...Object.fromEntries(Object.entries(own).filter(([k]) => !k.startsWith('_') && k !== 'state')),
+    dlqId, responseStatus: outcome.status, responseHeader: multiValue(outcome.headers), responseBody: outcome.body,
+    _filed_at: when, _published: published, _message_id: m.messageId, _owner: m._owner ?? 'world', ...(m._redact ? { _redact: m._redact } : {}),
+  }, 'dlq.create');
+  if (typeof m.failureCallback === 'string') await callback(ctx, m, String(m.failureCallback), outcome, n, dlqId);
+  if (typeof m.callback === 'string') await callback(ctx, m, String(m.callback), outcome, n, undefined);
+}
+
+/** A callback: "QStash calls your callback URL with the response"; its body is the callbacks page's (`status`, `header`,
+ *  base64 `body`, `retried`, `maxRetries`, `sourceMessageId`, `url`, `method`, `sourceHeader`, base64 `sourceBody`,
+ *  `notBefore`, `createdAt`, `callerIP`), and a failure callback's adds the `dlqId`. It is a message of its own. */
+async function callback(ctx: SemanticsContext, m: Row, url: string, o: Outcome, attempts: number, dlqId: string | undefined): Promise<void> {
+  const body = JSON.stringify({
+    status: o.status, header: multiValue(o.headers), body: b64(o.body), retried: attempts - 1, maxRetries: Number(m.maxRetries ?? 3),
+    ...(dlqId ? { dlqId } : {}), sourceMessageId: m.messageId, url: m.url, method: m.method ?? 'POST', sourceHeader: m.header ?? {},
+    sourceBody: b64(String(m.body ?? '')), notBefore: m.notBefore, createdAt: m.createdAt, callerIP: '127.0.0.1',
+  });
+  await storeMessage(ctx, { url, headers: { 'content-type': 'application/json' }, body, owner: String(m._owner ?? 'world'), callbackOf: String(m.messageId) });
+}

package/dist/qstash/src/semantics/dlq.d.ts

@@ -0,0 +1,4 @@
+import type { Semantics, SemanticsContext } from '@volter/world-core';
+import { type Row } from './shared.js';
+export declare function dlqView(ctx: SemanticsContext, e: Row): Row;
+export declare const dlqSemantics: Record<string, Semantics>;

package/dist/qstash/src/semantics/dlq.js

@@ -0,0 +1,51 @@
+import { ensureQueue, ownerFor, storeMessage } from "./messages.js";
+import { idsOf, workflowDlq } from "./workflows.js";
+import { matchesFilters, MESSAGE_FIELDS } from "./shared.js";
+import { redacted } from "./messages.js";
+const DLQ_FIELDS = [...MESSAGE_FIELDS, 'dlqId', 'responseStatus', 'responseHeader', 'responseBody', 'responseBodyBase64'];
+export function dlqView(ctx, e) {
+    const own = ctx.own(e);
+    return redacted(Object.fromEntries(DLQ_FIELDS.filter((k) => own[k] !== undefined && own[k] !== null).map((k) => [k, own[k]])), e);
+}
+/** QStash's DLQ fields its list and bulk actions filter on, by query parameter (the spec's list and retry parameters). */
+const FILTERS = { url: 'url', topicName: 'topicName', scheduleId: 'scheduleId', queueName: 'queueName', responseStatus: 'responseStatus', label: 'labels', flowControlKey: 'flowControlKey', messageId: 'messageId', callerIp: 'callerIP' };
+/** The account's entries a request names: by `dlqIds`, else by its filters (all of them when it names none), oldest first. */
+function selected(ctx) {
+    const owner = ownerFor(ctx);
+    const mine = ctx.rowsRaw('DLQMessage').filter((e) => e._owner === owner);
+    const ids = idsOf(ctx, 'dlqIds');
+    return (ids.length ? mine.filter((e) => ids.includes(String(e.dlqId))) : mine.filter((e) => matchesFilters(ctx, e, FILTERS, '_filed_at')))
+        .sort((a, b) => Number(a._filed_at) - Number(b._filed_at));
+}
+/** GET /v2/dlq: QStash's failed messages, the latest to fail first, `{messages}` ("If no cursor is returned, you have reached the
+ *  end"); with `source=workflow`, the Workflow DLQ (the spec patch, from @upstash/workflow's `client.dlq.list`). */
+const list = async (ctx) => {
+    if (new URL(ctx.call.request.url).searchParams.get('source') === 'workflow')
+        return workflowDlq(ctx);
+    const count = Number(new URL(ctx.call.request.url).searchParams.get('count') ?? 0);
+    const all = selected(ctx).reverse();
+    return ctx.reply({ messages: (count > 0 ? all.slice(0, count) : all).map((e) => dlqView(ctx, e)) });
+};
+/** POST /v2/dlq/retry: each entry the request names (by `dlqIds`, by its filters, or all of them) is republished as a new
+ *  message (its destination, body and headers, in its queue) and leaves the DLQ; answered 201 `{responses: [{messageId}]}`.
+ *  An id the DLQ does not hold answers 404 (the spec's 404). */
+const retry = async (ctx) => {
+    const ids = idsOf(ctx, 'dlqIds');
+    const entries = selected(ctx);
+    const missing = ids.find((id) => !entries.some((e) => e.dlqId === id));
+    if (missing)
+        return ctx.refuse({ status: 404, message: `DLQ message ${missing} not found` });
+    const responses = [];
+    for (const e of entries) {
+        const published = (e._published ?? {});
+        const queue = typeof e.queueName === 'string' ? e.queueName : undefined;
+        if (queue)
+            await ensureQueue(ctx, queue);
+        const { 'upstash-delay': _delay, 'upstash-not-before': _notBefore, ...headers } = published;
+        const messageId = await storeMessage(ctx, { url: String(e.url), headers, body: String(e.body ?? ''), owner: ownerFor(ctx), queue });
+        await ctx.write('DLQMessage', String(e.id), { deleted: true }, 'dlq.retried');
+        responses.push({ messageId });
+    }
+    return ctx.reply({ responses }, 201);
+};
+export const dlqSemantics = { get_v2_dlq: list, post_v2_dlq_retry: retry };