@volter/twin-upstash 0.1.0

package/dist/qstash/src/semantics/workflows.ts

@@ -0,0 +1,401 @@
+// Upstash Workflow on QStash: a run is a sequence of QStash messages to the app's workflow route, each delivery carrying
+// every step the run has so far ("Each step is executed in its own HTTP call to your application. After a step finishes,
+// its result is stored in durable state inside Upstash Workflow. On the next execution, Workflow skips completed steps and
+// resumes exactly where it left off by restoring the previous step results.", https://upstash.com/docs/workflow/basics/how).
+//
+// The wire is the vendor's own client's, @upstash/workflow 1.3.0 (the version Dub installs; spec/SOURCE.md names what the
+// documents omit): a run starts with a batch entry carrying `Upstash-Workflow-Init: true`, `Upstash-Workflow-RunId` and
+// `Upstash-Workflow-Url` (`triggerFirstInvocation`, `WorkflowHeaders.addBaseHeaders`); each step the route runs comes back
+// as a batch entry with `Upstash-Workflow-Init: false` and the step as its body (`submitSingleStep`); a delivery's body is
+// the run's steps as `[{messageId, body: <base64>, callType}]`, the first being the run's payload (`processRawSteps`); a
+// wait is `POST /v2/wait/{eventId}` (`LazyWaitEventStep`), and the notified or timed-out step comes back with its `out`
+// (the event's data, base64) and `waitTimeout` (`LazyWaitForEventStep.parseWaitForEventOut`); the route reports the run
+// finished with `DELETE /v2/workflows/runs/{id}?cancel=false` (`triggerWorkflowDelete`).
+import type { Semantics, SemanticsContext } from '@volter/world-core';
+import { labelsOf, ownerFor, storeMessage, type Accepted, type Publication, DEFAULT_RETRIES } from './messages.ts';
+import { b64, durationSeconds, everHeld, headersOf, lowerKeys, matchesFilters, mintId, multiValue, nowMs, refuse, singleValue, type Row } from './shared.ts';
+
+const FAILURE_PREFIX = 'upstash-failure-callback-';
+
+export const isWorkflowMessage = (h: Record<string, string>): boolean => h['upstash-workflow-runid'] !== undefined;
+
+const runOf = (ctx: SemanticsContext, id: string): Row | undefined => ctx.rowsRaw('WorkflowRun').find((r) => r.workflowRunId === id);
+
+/** A run's messages, in the order the run made them. */
+export function runMessages(ctx: SemanticsContext, runId: string): Row[] {
+  return ctx.rowsRaw('Message').filter((m) => m._run === runId).sort((a, b) => Number(a._seq) - Number(b._seq));
+}
+
+/** The steps a delivery of the run's message `seq` carries: the steps a resumed run kept, then the run's own messages up
+ *  to `seq`, each as `{messageId, body: <base64 of what was published>, callType}` (`processRawSteps`). */
+export function rawSteps(ctx: SemanticsContext, runId: string, seq: number): Row[] {
+  const run = runOf(ctx, runId);
+  const carried = Array.isArray(run?._carried) ? (run!._carried as Row[]) : [];
+  const own = runMessages(ctx, runId).filter((m) => Number(m._seq) <= seq && m.state !== 'CANCELLED')
+    .map((m) => ({ messageId: String(m.messageId), body: b64(String(m.body ?? '')), callType: String(m._call ?? 'step') }));
+  return [...carried, ...own];
+}
+
+/** What a workflow's delivery carries in its headers: every `Upstash-Forward-*` header with the prefix stripped (the
+ *  run's `Upstash-Workflow-Sdk-Version` among them, which is how `serve()` tells a delivery from its own first
+ *  invocation), and the run's own workflow headers as published. */
+export function workflowHeaders(published: Record<string, string>): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const [k, v] of Object.entries(published)) if (k.startsWith('upstash-forward-')) out[k.slice('upstash-forward-'.length)] = v;
+  for (const name of ['upstash-workflow-runid', 'upstash-workflow-url', 'upstash-workflow-init', 'upstash-workflow-calltype', 'upstash-feature-set', 'content-type']) {
+    if (published[name] !== undefined && out[name] === undefined) out[name] = published[name]!;
+  }
+  return out;
+}
+
+/** A run's delivery configuration, from the entry that starts it: `Upstash-Retries` (absent: the default 3; the client
+ *  sends it only when it differs), `Upstash-Retry-Delay`, the failure URL (`Upstash-Failure-Callback`, which the client
+ *  sets to the workflow URL itself unless told otherwise: "const failureUrl = option.failureUrl ?? option.url") and the
+ *  `Upstash-Failure-Callback-*` headers the failure call is published with. */
+function runConfig(h: Record<string, string>): Row {
+  const failureHeaders: Record<string, string> = {};
+  for (const [k, v] of Object.entries(h)) if (k.startsWith(FAILURE_PREFIX)) failureHeaders[k] = v;
+  return {
+    _retries: h['upstash-retries'] !== undefined && /^\d+$/.test(h['upstash-retries']) ? Number(h['upstash-retries']) : DEFAULT_RETRIES,
+    ...(h['upstash-retry-delay'] ? { _retry_delay: h['upstash-retry-delay'] } : {}),
+    ...(h['upstash-failure-callback'] ? { _failure_url: h['upstash-failure-callback'] } : {}),
+    _failure_headers: failureHeaders,
+  };
+}
+
+/** One batch or publish entry of a workflow: the first of a run starts it, a later one is a step of an active run. A run
+ *  id the World already holds is not started twice: the client reads `deduplicated` and warns "Workflow run … already
+ *  exists" (`triggerFirstInvocation`). A step for a run no longer active is refused 400, which the client reads as "Tried
+ *  to append to a cancelled workflow" (`triggerRouteFunction`). */
+export async function acceptWorkflowMessage(ctx: SemanticsContext, p: Publication, owner: string): Promise<Accepted> {
+  const h = p.headers;
+  const runId = h['upstash-workflow-runid']!;
+  const url = h['upstash-workflow-url'] ?? p.destination;
+  if (h['upstash-workflow-init'] === 'true') {
+    const existing = runOf(ctx, runId);
+    if (existing) return { messageId: String(runMessages(ctx, runId)[0]?.messageId ?? ''), deduplicated: true };
+    const config = runConfig(h);
+    await ctx.write('WorkflowRun', runId, {
+      workflowRunId: runId, workflowUrl: url, workflowState: 'RUN_STARTED', workflowRunCreatedAt: nowMs(ctx),
+      ...labelsOf(h),
+      ...(h['upstash-flow-control-key'] ? { flowControlKey: h['upstash-flow-control-key'] } : {}),
+      _owner: owner, _published: h, ...config,
+    }, 'workflow_run.start');
+    const messageId = await storeMessage(ctx, { url: p.destination, headers: h, body: p.body, owner, run: { id: runId, seq: 0, call: 'step', initial: true }, retries: Number(config._retries) });
+    return { messageId };
+  }
+  const run = runOf(ctx, runId);
+  if (!run) return { error: `workflow run ${runId} not found`, status: 404 };
+  if (run.workflowState !== 'RUN_STARTED') return { error: `workflow run ${runId} is not active`, status: 400 };
+  return { messageId: await appendStep(ctx, run, p.destination, h, p.body) };
+}
+
+/** Append a step to an active run: the next message of its sequence, to its route. A step's own `Upstash-Retries` governs
+ *  its delivery; without one, the run's. */
+async function appendStep(ctx: SemanticsContext, run: Row, url: string, h: Record<string, string>, body: string): Promise<string> {
+  const runId = String(run.workflowRunId);
+  const seq = runMessages(ctx, runId).reduce((max, m) => Math.max(max, Number(m._seq)), -1) + 1;
+  const own = h['upstash-retries'] !== undefined && /^\d+$/.test(h['upstash-retries']) ? Number(h['upstash-retries']) : undefined;
+  return storeMessage(ctx, { url, headers: h, body, owner: String(run._owner), run: { id: runId, seq, call: h['upstash-workflow-calltype'] ?? 'step', initial: false }, retries: own ?? Number(run._retries ?? DEFAULT_RETRIES) });
+}
+
+/** The headers a step the vendor makes for a run (a notified or timed-out wait) is published with: the run's own
+ *  workflow headers, as a step the route sends carries them. */
+function stepHeaders(run: Row, extra: Record<string, string> = {}): Record<string, string> {
+  const published = (run._published ?? {}) as Record<string, string>;
+  const out: Record<string, string> = {};
+  for (const [k, v] of Object.entries(published)) {
+    if (k.startsWith('upstash-forward-') || ['upstash-workflow-runid', 'upstash-workflow-url', 'upstash-feature-set', 'content-type'].includes(k)) out[k] = v;
+  }
+  return { ...out, 'upstash-workflow-init': 'false', 'upstash-workflow-calltype': 'step', ...extra };
+}
+
+/** The wait step coming back to the run: notified, with the event's data as its `out` (base64, as the client decodes it),
+ *  or timed out. */
+async function resumeWait(ctx: SemanticsContext, run: Row, step: Row, eventId: string, data: string | undefined, headers?: Record<string, string>): Promise<string> {
+  const body = JSON.stringify({ ...step, waitEventId: eventId, ...(data !== undefined ? { out: b64(data), waitTimeout: false } : { waitTimeout: true }) });
+  return appendStep(ctx, run, String(run.workflowUrl), headers ?? stepHeaders(run), body);
+}
+
+// ── waits and notifies ─────────────────────────────────────────────────────────────────────────────────────────
+
+const NOTIFICATION = '_workflow_notification';
+const waiterId = (eventId: string, runId: string): string => `${eventId}:${runId}`;
+
+/** A waiter as the workflow document's `Waiter` answers it. */
+function waiterView(ctx: SemanticsContext, w: Row): Row {
+  const own = ctx.own(w);
+  return { url: own.url, headers: own.headers ?? {}, deadline: own.deadline, ...(own.timeoutBody ? { timeoutBody: own.timeoutBody } : {}), timeoutUrl: own.timeoutUrl, timeoutHeaders: own.timeoutHeaders ?? {} };
+}
+
+/** POST /v2/wait/{eventId} (the spec patch, from the client's `LazyWaitEventStep`): the run in the
+ *  `Upstash-Workflow-RunId` header waits for the event until its timeout ("Default timeout value is 7 days",
+ *  https://upstash.com/docs/workflow/basics/context/waitForEvent). A notification the run was sent before it waited is
+ *  consumed at once ("When a waiter becomes active later, it will consume the saved notification", the spec's notify).
+ *  Where the evidence stops: the lane answers 200 with no body (the client does not read it), and a run the World does
+ *  not hold, or one no longer active, 400 in the notify's words. */
+const wait: Semantics = async (ctx) => {
+  const eventId = String(ctx.call.params.eventId);
+  const runId = headersOf(ctx)['upstash-workflow-runid'];
+  const run = runId ? runOf(ctx, runId) : undefined;
+  if (!run || run.workflowState !== 'RUN_STARTED') return refuse(ctx, 400, 'Invalid workflow run ID or event ID');
+  const b = (ctx.body && typeof ctx.body === 'object' ? ctx.body : {}) as Row;
+  const step = (b.step && typeof b.step === 'object' ? b.step : {}) as Row;
+  const saved = ctx.rowsRaw(NOTIFICATION).find((n) => n.id === waiterId(eventId, String(runId)) && n.consumed !== true);
+  if (saved) {
+    await ctx.record(NOTIFICATION, { ...ctx.own(saved), consumed: true }, String(saved.id));
+    await resumeWait(ctx, run, step, eventId, String(saved.data ?? ''));
+    return new Response(null, { status: 200 });
+  }
+  const seconds = durationSeconds(b.timeout ?? '7d') ?? 7 * 86_400;
+  await ctx.write('Waiter', waiterId(eventId, String(runId)), {
+    url: typeof b.url === 'string' ? b.url : String(run.workflowUrl),
+    headers: b.timeoutHeaders ?? {},
+    deadline: Math.floor(nowMs(ctx) / 1000) + seconds,
+    ...(typeof b.timeoutBody === 'string' ? { timeoutBody: b.timeoutBody } : {}),
+    timeoutUrl: typeof b.timeoutUrl === 'string' ? b.timeoutUrl : String(run.workflowUrl),
+    timeoutHeaders: b.timeoutHeaders ?? {},
+    _event: eventId, _run: runId, _step: step,
+  }, 'waiter.create');
+  return new Response(null, { status: 200 });
+};
+
+/** POST /v2/notify/{workflowRunId}/{eventId}: "If a waiter is currently active, it receives the notification
+ *  immediately. If no waiter is active, the notification is saved in the database. When a waiter becomes active later, it
+ *  will consume the saved notification." (the spec). Answered 201 with the `NotifyResponse`: the waiter and the message
+ *  that resumes its run, or the run alone when the notification is saved. */
+const notify: Semantics = async (ctx) => {
+  const runId = String(ctx.call.params.workflowRunId);
+  const eventId = String(ctx.call.params.eventId);
+  const run = runOf(ctx, runId);
+  if (!run || run.workflowState !== 'RUN_STARTED') return refuse(ctx, 400, 'Invalid workflow run ID or event ID');
+  const data = ctx.text ?? '';
+  const who = { workflowRunId: runId, workflowCreatedAt: Number(run.workflowRunCreatedAt) };
+  const w = ctx.rowsRaw('Waiter').find((x) => x.id === waiterId(eventId, runId));
+  if (!w) {
+    await ctx.record(NOTIFICATION, { run: runId, event: eventId, data, at: ctx.occurredAt, consumed: false }, waiterId(eventId, runId));
+    return ctx.reply({ ...who, error: '' }, 201);
+  }
+  const view = waiterView(ctx, w);
+  await ctx.write('Waiter', String(w.id), { deleted: true }, 'waiter.notified');
+  const messageId = await resumeWait(ctx, run, (w._step ?? {}) as Row, eventId, data);
+  return ctx.reply({ waiter: view, messageId, error: '', ...who }, 201);
+};
+
+/** POST /v2/notify/{eventId}: "Notify an event to all waiters listening for that event ID. ... if there are no active
+ *  waiters when the notification is sent, the event is lost and will not be delivered later." (the spec). Answered 201
+ *  with one `NotifyResponse` per waiter notified, an empty list when none waits. */
+const notifyEvent: Semantics = async (ctx) => {
+  const eventId = String(ctx.call.params.eventId);
+  const data = ctx.text ?? '';
+  const out: Row[] = [];
+  for (const w of ctx.rowsRaw('Waiter').filter((x) => x._event === eventId).sort((a, b) => String(a.id).localeCompare(String(b.id)))) {
+    const run = runOf(ctx, String(w._run));
+    if (!run || run.workflowState !== 'RUN_STARTED') continue;
+    const view = waiterView(ctx, w);
+    await ctx.write('Waiter', String(w.id), { deleted: true }, 'waiter.notified');
+    const messageId = await resumeWait(ctx, run, (w._step ?? {}) as Row, eventId, data);
+    out.push({ waiter: view, messageId, error: '', workflowRunId: run.workflowRunId, workflowCreatedAt: run.workflowRunCreatedAt });
+  }
+  return ctx.reply(out, 201);
+};
+
+/** A wait whose timeout has come: the step comes back to its timeout URL timed out, with the headers the run gave for it
+ *  (the catch-up, delivery.ts). */
+export async function timeOut(ctx: SemanticsContext, w: Row): Promise<void> {
+  const run = runOf(ctx, String(w._run));
+  await ctx.write('Waiter', String(w.id), { deleted: true }, 'waiter.timeout');
+  if (!run || run.workflowState !== 'RUN_STARTED') return;
+  const given = lowerKeys(singleValue(w.timeoutHeaders as Row));
+  await resumeWait(ctx, run, (w._step ?? {}) as Row, String(w._event), undefined, stepHeaders(run, given));
+}
+
+// ── ending a run ───────────────────────────────────────────────────────────────────────────────────────────────
+
+/** Cancel a run: it moves to RUN_CANCELED, the steps it still owes are cancelled and its waits end. */
+async function cancelRun(ctx: SemanticsContext, run: Row, operation: string): Promise<void> {
+  const runId = String(run.workflowRunId);
+  const refused = ctx.legal('WorkflowRun', 'workflowState', operation, run.workflowState, 'RUN_CANCELED', runId);
+  if (refused) throw new Error(refused.message);
+  await ctx.write('WorkflowRun', runId, { workflowState: 'RUN_CANCELED', workflowRunCompletedAt: nowMs(ctx) }, 'workflow_run.cancel');
+  for (const m of runMessages(ctx, runId).filter((x) => x.state === 'CREATED' || x.state === 'RETRY')) await cancelStep(ctx, m);
+  for (const w of ctx.rowsRaw('Waiter').filter((x) => x._run === runId)) await ctx.write('Waiter', String(w.id), { deleted: true }, 'waiter.canceled');
+}
+
+/** A step of a cancelled run still waiting for its delivery (a `context.sleep`, a retry): cancelled, never delivered. */
+async function cancelStep(ctx: SemanticsContext, m: Row): Promise<void> {
+  const no = ctx.legal('Message', 'state', 'delete_v2_workflows_runs', m.state, 'CANCEL_REQUESTED', String(m.id));
+  if (no) throw new Error(no.message);
+  await ctx.write('Message', String(m.id), { state: 'CANCEL_REQUESTED' }, 'message.cancel');
+}
+
+/** DELETE /v2/workflows/runs/{workflowRunId}: `?cancel=true` cancels the run ("Cancel an ongoing workflow run."), and
+ *  `?cancel=false` is the route reporting its function returned, the run's result as the body (the spec patch). Answered
+ *  200 with no body; a run the World does not hold, or one already ended, answers 404 in the spec's words. */
+const endRun: Semantics = async (ctx) => {
+  const runId = String(ctx.call.params.workflowRunId);
+  const run = runOf(ctx, runId);
+  if (!run) return refuse(ctx, 404, 'A workflow run is not found with the given id.');
+  const cancel = new URL(ctx.call.request.url).searchParams.get('cancel') === 'true';
+  const to = cancel ? 'RUN_CANCELED' : 'RUN_SUCCESS';
+  const refused = ctx.legal('WorkflowRun', 'workflowState', 'delete_v2_workflows_runs_workflowrunid', run.workflowState, to, runId);
+  if (refused) return ctx.refuse(refused);
+  if (cancel) await cancelRun(ctx, run, 'delete_v2_workflows_runs_workflowrunid');
+  else await ctx.write('WorkflowRun', runId, { workflowState: 'RUN_SUCCESS', workflowRunCompletedAt: nowMs(ctx), ...(ctx.text ? { workflowRunResponse: ctx.text } : {}) }, 'workflow_run.finish');
+  return new Response(null, { status: 200 });
+};
+
+/** The ids a query names, whether repeated or comma-separated ("Multiple values can be passed either by repeating the
+ *  query parameter ... or as a single comma-separated value", the spec's bulk cancel). */
+export function idsOf(ctx: SemanticsContext, name: string): string[] {
+  return new URL(ctx.call.request.url).searchParams.getAll(name).flatMap((v) => v.split(',')).map((s) => s.trim()).filter(Boolean);
+}
+
+/** The runs' fields a bulk cancel filters on, by query parameter (the spec's bulk cancel parameters). */
+const RUN_FILTERS = { workflowUrl: 'workflowUrl', label: 'labels', flowControlKey: 'flowControlKey', callerIp: 'workflowRunCallerIp' };
+
+/** DELETE /v2/workflows/runs: "If you provide a list of workflow run IDs, only those specific workflow runs will be
+ *  canceled. If you include filter parameters, only the workflow runs matching the filters will be canceled. If no filter
+ *  or workflow run IDs are sent, all workflow runs will be canceled." and "Cancelling by `workflowRunIds` is synchronous,
+ *  and `cancelled` is the exact number of workflow runs cancelled." (the spec). A workflow URL filter matches by prefix
+ *  unless `workflowUrlExactMatch` (the client's `workflowUrlStartingWith` and `workflowUrl`). Where the documentation stops
+ *  and the lane decides: a filter-based cancel finishes within the request (the spec's default waits up to a minute for
+ *  it), so it answers the number cancelled and no `actionId`. */
+const cancelRuns: Semantics = async (ctx) => {
+  const ids = idsOf(ctx, 'workflowRunIds');
+  const exact = new URL(ctx.call.request.url).searchParams.get('workflowUrlExactMatch') === 'true';
+  const owner = ownerFor(ctx);
+  const targets = ids.length ? ids.map((id) => runOf(ctx, id)).filter((r): r is Row => !!r)
+    : ctx.rowsRaw('WorkflowRun').filter((r) => r._owner === owner && matchesFilters(ctx, r, RUN_FILTERS, 'workflowRunCreatedAt', exact ? [] : ['workflowUrl']));
+  let cancelled = 0;
+  for (const run of targets.sort((a, b) => Number(a.workflowRunCreatedAt) - Number(b.workflowRunCreatedAt))) {
+    if (run.workflowState !== 'RUN_STARTED') continue;
+    await cancelRun(ctx, run, 'delete_v2_workflows_runs');
+    cancelled += 1;
+  }
+  return ctx.reply({ cancelled });
+};
+
+// ── a run's failure, and its DLQ ───────────────────────────────────────────────────────────────────────────────
+
+/** The `WorkflowDLQMessage` schema's fields. */
+const WORKFLOW_DLQ_FIELDS = ['dlqId', 'workflowUrl', 'workflowRunId', 'workflowCreatedAt', 'url', 'method', 'header', 'body', 'bodyBase64', 'maxRetries', 'createdAt',
+  'failureCallback', 'callerIP', 'label', 'labels', 'flowControlKey', 'failureFunctionState', 'responseStatus', 'responseHeader', 'responseBody', 'responseBodyBase64'];
+
+export function workflowDlqView(ctx: SemanticsContext, e: Row): Row {
+  const own = ctx.own(e);
+  return Object.fromEntries(WORKFLOW_DLQ_FIELDS.filter((k) => own[k] !== undefined && own[k] !== null).map((k) => [k, own[k]]));
+}
+
+/** A step's message has failed after its retries: "When a workflow step fails and exhausts all configured retries,
+ *  Upstash Workflow automatically moves the failed run to the DLQ." (https://upstash.com/docs/workflow/features/dlq); the
+ *  run is RUN_FAILED, and its failure function is called ("a failure function ... executes custom logic when a workflow
+ *  run fails after exhausting all retry attempts", https://upstash.com/docs/workflow/features/failure-callback) as a
+ *  QStash failure callback to the failure URL, with the callback body plus the run's id and creation time, which the
+ *  client's `serve()` reads, and the `Upstash-Failure-Callback-*` headers the run was started with (their prefix made
+ *  `Upstash-`). */
+export async function failRun(ctx: SemanticsContext, m: Row, outcome: { status: number; body: string; headers: Record<string, string> }): Promise<void> {
+  const run = runOf(ctx, String(m._run));
+  if (!run || run.workflowState !== 'RUN_STARTED') return;
+  const runId = String(run.workflowRunId);
+  const dlqId = `${Math.floor(nowMs(ctx))}-${everHeld(ctx, 'WorkflowDLQMessage')}`;
+  const failureUrl = typeof run._failure_url === 'string' ? run._failure_url : undefined;
+  await ctx.write('WorkflowDLQMessage', dlqId, {
+    dlqId, workflowUrl: run.workflowUrl, workflowRunId: runId, workflowCreatedAt: run.workflowRunCreatedAt,
+    url: m.url, method: m.method ?? 'POST', header: multiValue((m._published ?? {}) as Record<string, string>), body: m.body ?? '', maxRetries: m.maxRetries,
+    createdAt: nowMs(ctx), ...(failureUrl ? { failureCallback: failureUrl, failureFunctionState: 'CALLBACK_INPROGRESS' } : {}),
+    ...(run.label ? { label: run.label, labels: run.labels } : {}), ...(run.flowControlKey ? { flowControlKey: run.flowControlKey } : {}),
+    responseStatus: outcome.status, responseHeader: multiValue(outcome.headers), responseBody: outcome.body,
+    _message_id: m.messageId, _seq: m._seq,
+  }, 'workflow_dlq.create');
+  const refused = ctx.legal('WorkflowRun', 'workflowState', 'delete_v2_workflows_runs_workflowrunid', run.workflowState, 'RUN_FAILED', runId, 'vendor');
+  if (refused) throw new Error(refused.message);
+  await ctx.write('WorkflowRun', runId, { workflowState: 'RUN_FAILED', workflowRunCompletedAt: nowMs(ctx), dlqId }, 'workflow_run.fail');
+  if (!failureUrl) return;
+  const headers: Record<string, string> = { 'content-type': 'application/json' };
+  for (const [k, v] of Object.entries((run._failure_headers ?? {}) as Record<string, string>)) headers[`upstash-${k.slice(FAILURE_PREFIX.length)}`] = v;
+  const body = JSON.stringify({
+    status: outcome.status, header: multiValue(outcome.headers), body: b64(outcome.body), retried: Number(m._attempts ?? 1) - 1,
+    maxRetries: Number(m.maxRetries ?? DEFAULT_RETRIES), dlqId, sourceMessageId: m.messageId, url: run.workflowUrl, method: 'POST',
+    sourceHeader: multiValue((m._published ?? {}) as Record<string, string>), sourceBody: b64(String(runMessages(ctx, runId)[0]?.body ?? '')),
+    notBefore: 0, createdAt: nowMs(ctx), callerIP: '127.0.0.1', workflowRunId: runId, workflowCreatedAt: run.workflowRunCreatedAt,
+  });
+  await storeMessage(ctx, { url: failureUrl, headers: { ...headers, 'upstash-retries': headers['upstash-retries'] ?? String(DEFAULT_RETRIES) }, body, owner: String(run._owner), callbackOf: String(m.messageId) });
+  await ctx.record('_failure_call', { dlq: dlqId, run: runId }, `failure:${dlqId}`);
+}
+
+/** The failure call's outcome on its DLQ entry: CALLBACK_SUCCESS once it is answered, CALLBACK_FAIL once its retries are
+ *  spent (the workflow document's `failureFunctionState` filter lists CALLBACK_INPROGRESS, CALLBACK_SUCCESS, CALLBACK_FAIL). */
+export async function failureCallSettled(ctx: SemanticsContext, callbackOf: string, delivered: boolean): Promise<void> {
+  const entry = ctx.rowsRaw('WorkflowDLQMessage').find((e) => e._message_id === callbackOf);
+  if (entry) await ctx.write('WorkflowDLQMessage', String(entry.id), { failureFunctionState: delivered ? 'CALLBACK_SUCCESS' : 'CALLBACK_FAIL' }, 'workflow_dlq.failure_call');
+}
+
+/** When an entry was filed, in the order entries were filed: its time, then its place in the DLQ (the `-<n>` of its id), so
+ *  that runs failing in one catch-up keep the order they failed in. */
+const filedOrder = (e: Row): number => Number(e.createdAt) * 1000 + Number(String(e.dlqId).split('-')[1] ?? 0) % 1000;
+
+/** The Workflow DLQ, newest first, as `{messages}`: "If no cursor is returned, you have reached the end". Answers
+ *  `GET /v2/workflows/dlq` and the client's `GET /v2/dlq?source=workflow` (the spec patch). */
+export function workflowDlq(ctx: SemanticsContext): Response {
+  const owner = ownerFor(ctx);
+  const count = Number(new URL(ctx.call.request.url).searchParams.get('count') ?? 0);
+  const all = ctx.rowsRaw('WorkflowDLQMessage').filter((e) => runOf(ctx, String(e.workflowRunId))?._owner === owner && matchesFilters(ctx, e, DLQ_FILTERS, 'createdAt'))
+    .sort((a, b) => filedOrder(b) - filedOrder(a));
+  const messages = (count > 0 ? all.slice(0, count) : all).map((e) => workflowDlqView(ctx, e));
+  return ctx.reply({ messages });
+}
+
+/** POST /v2/workflows/dlq/resume?dlqIds=: "continue a failed workflow run from the exact point of failure, preserving all
+ *  successfully completed steps and their results" (https://upstash.com/docs/workflow/features/dlq/resume); "A new
+ *  workflow run ID is generated, but the workflow maintains the state and results from previously completed steps."
+ *  (the spec's resume). The new run keeps the steps the failed one had delivered and sends the failed step again, with
+ *  the run's retries, or those the request names in `Upstash-Retries` (the client's `buildResumeRestartHeaders`).
+ *  Where the documentation stops: the DLQ entry stays until its retention ends (delivery.ts). */
+const resume: Semantics = async (ctx) => {
+  const byId = idsOf(ctx, 'dlqIds');
+  const owner = ownerFor(ctx);
+  const ids = byId.length ? byId : ctx.rowsRaw('WorkflowDLQMessage').filter((e) => runOf(ctx, String(e.workflowRunId))?._owner === owner && !e._resumed_as && matchesFilters(ctx, e, DLQ_FILTERS, 'createdAt'))
+    .sort((a, b) => filedOrder(a) - filedOrder(b)).map((e) => String(e.dlqId));
+  const h = headersOf(ctx);
+  const workflowRuns: Row[] = [];
+  for (const dlqId of ids) {
+    const entry = ctx.rowsRaw('WorkflowDLQMessage').find((e) => e.dlqId === dlqId);
+    const source = entry ? runOf(ctx, String(entry.workflowRunId)) : undefined;
+    if (!entry || !source) return refuse(ctx, 404, `DLQ message ${dlqId} not found`);
+    const failedSeq = Number(entry._seq);
+    const kept = rawSteps(ctx, String(source.workflowRunId), failedSeq - 1);
+    const runId = mintId(ctx, 'wfr_', 'workflow_run', everHeld(ctx, 'WorkflowRun'));
+    const created = nowMs(ctx);
+    const retries = h['upstash-retries'] !== undefined && /^\d+$/.test(h['upstash-retries']) ? Number(h['upstash-retries']) : Number(source._retries ?? DEFAULT_RETRIES);
+    const published = { ...((source._published ?? {}) as Record<string, string>), 'upstash-workflow-runid': runId };
+    await ctx.write('WorkflowRun', runId, {
+      workflowRunId: runId, workflowUrl: source.workflowUrl, workflowState: 'RUN_STARTED', workflowRunCreatedAt: created,
+      ...(source.label ? { label: source.label, labels: source.labels } : {}), ...(source.flowControlKey ? { flowControlKey: source.flowControlKey } : {}),
+      _owner: source._owner, _published: published, _retries: retries, ...(source._retry_delay ? { _retry_delay: source._retry_delay } : {}),
+      ...(source._failure_url ? { _failure_url: source._failure_url } : {}), _failure_headers: source._failure_headers ?? {},
+      _carried: kept, _resumed_from: dlqId,
+    }, 'workflow_run.resume');
+    const failed = runMessages(ctx, String(source.workflowRunId)).find((m) => m.messageId === entry._message_id);
+    const stepPublished = { ...((failed?._published ?? {}) as Record<string, string>), 'upstash-workflow-runid': runId };
+    await storeMessage(ctx, { url: String(entry.url), headers: stepPublished, body: String(entry.body ?? ''), owner: String(source._owner), run: { id: runId, seq: kept.length, call: String(failed?._call ?? 'step'), initial: false }, retries });
+    await ctx.write('WorkflowDLQMessage', String(entry.id), { _resumed_as: runId }, 'workflow_dlq.resumed');
+    workflowRuns.push({ workflowRunId: runId, workflowCreatedAt: created });
+  }
+  return ctx.reply({ workflowRuns });
+};
+
+/** The Workflow DLQ's fields its bulk actions and list filter on, by query parameter (the spec's resume parameters); each
+ *  matches exactly: "Filter by exact workflow URL." (https://upstash.com/docs/workflow/basics/client/dlq/list). */
+const DLQ_FILTERS = { workflowUrl: 'workflowUrl', workflowRunId: 'workflowRunId', label: 'labels', failureFunctionState: 'failureFunctionState', callerIp: 'callerIP', flowControlKey: 'flowControlKey' };
+
+export const workflowSemantics: Record<string, Semantics> = {
+  post_v2_notify_eventid: notifyEvent,
+  post_v2_wait_eventid: wait,
+  post_v2_notify_workflowrunid_eventid: notify,
+  delete_v2_workflows_runs_workflowrunid: endRun,
+  delete_v2_workflows_runs: cancelRuns,
+  post_v2_workflows_dlq_resume: resume,
+  get_v2_workflows_dlq: async (ctx) => workflowDlq(ctx),
+};
+

package/dist/qstash/src/signing.d.ts

@@ -0,0 +1,4 @@
+/** base64url(SHA-256(body)): the `body` claim. */
+export declare const bodyHash: (body: string) => string;
+/** The JWT for one delivery of `body` to `sub`, issued at `iat` (Unix seconds), valid five minutes. */
+export declare function sign(body: string, sub: string, iat: number, jti: string, key: string): string;

package/dist/qstash/src/signing.js

@@ -0,0 +1,16 @@
+// The `Upstash-Signature` a delivery carries: a JWT signed with the account's current signing key (HS256), whose claims
+// name the issuer, the destination and the body's SHA-256. "Upstash provides a JWT with each request. This JWT is signed by
+// your individual secret signing keys." (https://upstash.com/docs/qstash/howto/roll-signing-keys); the claims are the ones
+// the vendor's client checks (@upstash/qstash's `Receiver`: `iss` "Upstash", `sub` the URL, `body` the base64url SHA-256
+// of the body, `exp`/`nbf`; https://upstash.com/docs/qstash/howto/signature). Pure: no clock and no randomness of its own;
+// the `jti` is derived from the message and the attempt, so a World signs the same bytes every run.
+import { createHash, createHmac } from 'node:crypto';
+const b64url = (b) => b.toString('base64url');
+const b64urlJson = (o) => b64url(Buffer.from(JSON.stringify(o), 'utf8'));
+/** base64url(SHA-256(body)): the `body` claim. */
+export const bodyHash = (body) => b64url(createHash('sha256').update(body, 'utf8').digest());
+/** The JWT for one delivery of `body` to `sub`, issued at `iat` (Unix seconds), valid five minutes. */
+export function sign(body, sub, iat, jti, key) {
+    const input = `${b64urlJson({ alg: 'HS256', typ: 'JWT' })}.${b64urlJson({ iss: 'Upstash', sub, iat, nbf: iat, exp: iat + 300, jti, body: bodyHash(body) })}`;
+    return `${input}.${b64url(createHmac('sha256', key).update(input).digest())}`;
+}

package/dist/qstash/src/signing.ts

@@ -0,0 +1,19 @@
+// The `Upstash-Signature` a delivery carries: a JWT signed with the account's current signing key (HS256), whose claims
+// name the issuer, the destination and the body's SHA-256. "Upstash provides a JWT with each request. This JWT is signed by
+// your individual secret signing keys." (https://upstash.com/docs/qstash/howto/roll-signing-keys); the claims are the ones
+// the vendor's client checks (@upstash/qstash's `Receiver`: `iss` "Upstash", `sub` the URL, `body` the base64url SHA-256
+// of the body, `exp`/`nbf`; https://upstash.com/docs/qstash/howto/signature). Pure: no clock and no randomness of its own;
+// the `jti` is derived from the message and the attempt, so a World signs the same bytes every run.
+import { createHash, createHmac } from 'node:crypto';
+
+const b64url = (b: Buffer): string => b.toString('base64url');
+const b64urlJson = (o: unknown): string => b64url(Buffer.from(JSON.stringify(o), 'utf8'));
+
+/** base64url(SHA-256(body)): the `body` claim. */
+export const bodyHash = (body: string): string => b64url(createHash('sha256').update(body, 'utf8').digest());
+
+/** The JWT for one delivery of `body` to `sub`, issued at `iat` (Unix seconds), valid five minutes. */
+export function sign(body: string, sub: string, iat: number, jti: string, key: string): string {
+  const input = `${b64urlJson({ alg: 'HS256', typ: 'JWT' })}.${b64urlJson({ iss: 'Upstash', sub, iat, nbf: iat, exp: iat + 300, jti, body: bodyHash(body) })}`;
+  return `${input}.${b64url(createHmac('sha256', key).update(input).digest())}`;
+}

package/dist/src/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/src/cli.js

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+import { keepProcessAlive } from '@volter/world-core/lifecycle';
+// world-upstash CLI: serve the KERNEL-BACKED Upstash Redis REST twin, or run conformance.
+// State lives in the @volter/world-core action log (no in-memory side-store). Conformance is dev-only +
+// lazy-imported so the bin runs without @volter/world-tooling (E2).
+import { hasFlag, optionValue } from '@volter/world-core/args';
+import { createUpstashRedisTwinServer } from "./upstash-server.js";
+const [cmd, ...rest] = process.argv.slice(2);
+const port = Number(optionValue(rest, '--port', String(process.env.PORT ?? '0'))) || undefined;
+const root = optionValue(rest, '--root') || undefined;
+// The token this twin will DEMAND. Omit it and any non-empty credential is accepted (a missing one
+// is still a vendor-shaped 401) — the right default for a local twin that holds no real secret.
+const token = optionValue(rest, '--token') || undefined;
+const readOnly = hasFlag(rest, '--read-only'); // a twin accepts writes unless started read-only
+if (cmd === 'serve' || cmd === undefined) {
+    const s = await createUpstashRedisTwinServer({
+        readOnly,
+        ...(root ? { root } : {}),
+        ...(port ? { port } : {}),
+        ...(token ? { token } : {}),
+    });
+    process.stdout.write(`upstash twin (Redis over Upstash REST: POST /, /pipeline, /multi-exec, path-style, EVAL/EVALSHA Lua)${readOnly ? ' [read-only]' : ''} at http://127.0.0.1:${s.port}\n`
+        + `  point a client at it with UPSTASH_REDIS_REST_URL=http://127.0.0.1:${s.port} and any non-empty UPSTASH_REDIS_REST_TOKEN\n`);
+    await keepProcessAlive();
+}
+else if (cmd === 'conformance') {
+    const { checkUpstashRedisConformance } = await import("./upstash-conformance.js");
+    const report = await checkUpstashRedisConformance();
+    process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+    if (!report.ok)
+        process.exitCode = 1;
+}
+else {
+    process.stdout.write('Usage: world-upstash serve|conformance [--port N] [--root DIR] [--token T] [--read-only]\n');
+}