@jarenjs/db 0.56.0 → 0.67.0

package/src/cursor.js

@@ -0,0 +1,417 @@
+//@ts-check
+/**
+ * @file The ONE item cursor. `next()` / `return()` / `[Symbol.asyncIterator]`
+ * over a source that is either a ROW ITERATOR pulled one row at a time
+ * or a MATERIALISED item array (a barrier). Every engine — the
+ * collection engine's `query()`, the entity engine's, the graph
+ * loader's — builds its cursor here, so the release-at-a-row-boundary
+ * rule exists in one place:
+ *
+ *  - a row source is opened on the first pull and released exactly
+ *    once — on `return()`, on an error raised while a row is mapped or
+ *    pulled, and on abort — through the driver iterator's own
+ *    `return()`; a source pulled to exhaustion has already reset its
+ *    statement and is released without a second call;
+ *  - a buffered source materialises on the first pull and SAYS SO:
+ *    `streaming: 'buffered'` beside the `barrier` that forced it, a
+ *    `{ construct, reason }` pair whose construct is a stable
+ *    identifier a test can assert and `explain()` can repeat;
+ *  - `signal` cancels at a row boundary: an aborted cursor releases its
+ *    statement and every later pull is `JD2072`, so an abandoned
+ *    request neither keeps a statement open nor pulls another row.
+ *
+ * The source never materialises on its own: this module knows neither
+ * `.all()` nor `toArray()` — a gate holds it to that — and an engine
+ * that must buffer hands the buffered items in as `materialize`, named.
+ */
+
+import { utf8ByteLength } from '@jarenjs/core/string';
+
+import { DbRuntimeError } from './errors.js';
+import { chain, isThenable } from './driver.js';
+
+/** Preserve a failure, but acknowledge source cleanup before rejecting it.
+ * @param {() => any} call @param {() => any} release @param {(error:any)=>any} [wrap]
+ */
+function settling(call, release, wrap = (error) => error) {
+  const failed = (error) => {
+    const failure = wrap(error);
+    let cleanup;
+    try { cleanup = release(); } catch { throw failure; }
+    if (isThenable(cleanup)) return cleanup.then(() => { throw failure; }, () => { throw failure; });
+    throw failure;
+  };
+  let result;
+  try { result = call(); } catch (error) { return failed(error); }
+  return isThenable(result) ? result.then(undefined, failed) : result;
+}
+
+/**
+ * The serialised size of a JSON text in UTF-8 bytes — the one measure
+ * every byte bound in this package counts (an include per root, a page,
+ * a change record): the suite's counter, re-exported under the name the
+ * engines read.
+ * @param {string} text
+ * @returns {number}
+ */
+export function utf8Length(text) {
+  return utf8ByteLength(text);
+}
+
+/**
+ * @typedef {{ construct: string, reason: string }} CursorBarrier - what
+ *   forces a cursor to buffer: the construct is the stable identifier
+ *   (a planner construct such as `$orderby`, or `external`, `window`,
+ *   `pushdown`), the reason the sentence for a person
+ */
+
+/**
+ * How a cursor over a source the driver iterates actually behaves on
+ * THIS connection: one row per pull where the binding has a lazy
+ * iterator, and a buffer — declared as such, with the driver named as
+ * the barrier — where the driver composed `iterate` over `all()`. The
+ * capability is probed once at open; a cursor is classified at
+ * construction, before any statement exists, so the fact has to be the
+ * connection's. The one classification every engine and the job queue
+ * read.
+ * @param {any} connection
+ * @returns {{ streaming: 'row' | 'buffered', barrier: CursorBarrier | null }}
+ */
+export function rowClassOf(connection) {
+  return connection.capabilities.lazyIteration === false
+    ? { streaming: 'buffered', barrier: { construct: 'driver',
+      reason: 'the driver binding has no lazy iterator; the first pull materialises the whole result' } }
+    : { streaming: 'row', barrier: null };
+}
+
+/**
+ * @typedef {object} CursorSpec
+ * @property {'row' | 'buffered'} streaming - whether items arrive one
+ *   database row per pull or from a buffer the first pull filled
+ * @property {CursorBarrier | null} [barrier] - what forces buffering;
+ *   `null` when the cursor streams
+ * @property {AbortSignal} [signal] - cancellation, honoured at a row boundary
+ * @property {() => any} [materialize] - a buffered source: value-or-promise
+ *   of the whole item array, called once on the first pull
+ * @property {() => any} [open] - a row source: value-or-promise of the
+ *   driver iterator, called once on the first pull
+ * @property {(row: any) => any[]} [items] - a row source: the items one
+ *   row yields (none, one, or several); a throw releases the source
+ * @property {number} [deadline] - an epoch-millisecond deadline checked at
+ *   every pull: past it, the cursor releases its source and refuses
+ *   `JD2075` — a row-boundary check, never a statement interrupt
+ * @property {() => number} [now] - the clock the deadline is read
+ *   against — the store's runtime record's; required beside a deadline,
+ *   so no cursor reads the platform clock on its own
+ * @property {(error: any) => Error} [wrap] - classifies a failure raised
+ *   while the source is opened, pulled or mapped — the engine's driver
+ *   wrap, so no raw driver error leaves a cursor; a coded error passes
+ *   through it unchanged
+ * @property {(opened: boolean) => void} [onSettle] - called exactly once
+ *   when the cursor settles — exhausted, released, or aborted — with
+ *   whether a pull ever reached the source; what an engine finalises its
+ *   run accounting on
+ */
+
+/**
+ * Build an asynchronous item cursor over one source.
+ * @param {CursorSpec} spec
+ * @returns {any}
+ */
+export function createCursor(spec) {
+  return itemCursor(spec, false);
+}
+
+/**
+ * The same lifecycle over a synchronous source, answering values.
+ * @param {CursorSpec} spec
+ * @returns {any}
+ */
+export function createSyncCursor(spec) {
+  return itemCursor(spec, true);
+}
+
+/** @param {CursorSpec} spec @param {boolean} synchronous */
+function itemCursor(spec, synchronous) {
+  const { streaming, signal } = spec;
+  if (spec.deadline !== undefined && spec.now === undefined)
+    throw new TypeError('a cursor with a deadline is built with the clock it is read against (now)');
+  let buffered = [];
+  let offset = 0;
+  let underlying = null;
+  let opening = null;
+  let cleanup;
+  let sourceReleased = false;
+  let done = false;
+  let opened = false;
+  let tail = synchronous ? undefined : Promise.resolve();
+  const settle = () => {
+    if (done) return;
+    done = true;
+    buffered = [];
+    signal?.removeEventListener('abort', onAbort);
+    spec.onSettle?.(opened);
+  };
+  const releaseSource = () => {
+    if (sourceReleased || underlying === null) return;
+    sourceReleased = true;
+    return underlying.return?.(undefined);
+  };
+  const release = () => {
+    settle();
+    if (cleanup === undefined) cleanup = chain(opening, releaseSource);
+    return cleanup;
+  };
+  const onAbort = () => {
+    // Event listeners cannot await remote acknowledgement. The public
+    // return still awaits it, and a rejection is observed here as well.
+    try { const pending = release(); if (isThenable(pending)) pending.catch(() => {}); }
+    catch { /* the next/return boundary reports the source failure */ }
+  };
+  const aborted = () => new DbRuntimeError('JD2072',
+    'the cursor was aborted: its statement was released at a row boundary and it pulls '
+    + 'no further row', { cause: signal?.reason });
+  const end = () => ({ done: true, value: undefined });
+  const boundary = () => {
+    if (signal?.aborted) throw aborted();
+    if (done) return;
+    if (spec.deadline !== undefined && spec.now() > spec.deadline)
+      throw new DbRuntimeError('JD2075',
+        `the deadline passed before the next row (${new Date(spec.deadline).toISOString()}); `
+        + 'the statement was released at a row boundary');
+  };
+  const accept = (step) => {
+    boundary();
+    if (done) return end();
+    if (step.done === true) {
+      sourceReleased = true; // exhausted native iterators already reset
+      settle();
+      return end();
+    }
+    buffered = spec.items?.(step.value) ?? [];
+    offset = 0;
+    return null;
+  };
+  const rows = () => {
+    for (;;) {
+      boundary();
+      if (done) return end();
+      if (offset < buffered.length) return { done: false, value: buffered[offset++] };
+      const next = underlying.next();
+      if (isThenable(next)) return next.then((step) => accept(step) ?? rows());
+      const result = accept(next);
+      if (result !== null) return result;
+    }
+  };
+  const pull = () => {
+    boundary();
+    if (done) return end();
+    if (spec.materialize !== undefined) {
+      if (!opened) {
+        opened = true;
+        return chain(spec.materialize(), (items) => {
+          boundary();
+          if (done) return end();
+          buffered = items;
+          return pull();
+        });
+      }
+      if (offset < buffered.length) return { done: false, value: buffered[offset++] };
+      settle();
+      return end();
+    }
+    if (!opened) {
+      opened = true;
+      opening = chain(spec.open?.(), (iterator) => { underlying = iterator; });
+      return chain(opening, () => done
+        ? chain(releaseSource(), () => { boundary(); return end(); }) : rows());
+    }
+    return rows();
+  };
+  const guarded = () => settling(pull, release, spec.wrap);
+  if (signal?.aborted) settle();
+  else signal?.addEventListener('abort', onAbort, { once: true });
+  const next = synchronous ? guarded : () => {
+    const result = tail.then(guarded);
+    tail = result.then(() => undefined, () => undefined);
+    return result;
+  };
+  const finish = () => chain(release(), end);
+  const cursor = {
+    streaming,
+    barrier: spec.barrier ?? null,
+    get settled() { return done; },
+    next,
+    return: synchronous ? finish : () => {
+      try { return Promise.resolve(finish()); }
+      catch (error) { return Promise.reject(error); }
+    },
+    ...(synchronous
+      ? { [Symbol.iterator]: () => cursor, [Symbol.dispose]: finish }
+      : { [Symbol.asyncIterator]: () => cursor, [Symbol.asyncDispose]: async () => { await finish(); } }),
+  };
+  return Object.freeze(cursor);
+}
+
+/**
+ * A ROOT cursor's admission: the one decorator every store-level cursor
+ * — a collection's `query()`, an entity set's `cursor()` and
+ * `loadCursor()` — is handed back through, so the rule that a root read
+ * cannot fall inside a transaction it is not part of (MODEL-FORMAT §5.1)
+ * holds for a streaming read exactly as it holds for a finite one, at the
+ * granularity a stream can afford: **per pull**. Construction holds
+ * nothing. Each `next()` borrows the store gate for all the source work
+ * one public item needs (opening the statement on the first pull, the
+ * row steps, a row that yields no item), then releases before the
+ * promise settles, so a consumer paused between pulls blocks no
+ * transaction and a pull made while one is open waits for its commit
+ * and reads committed state only. `return()` is admitted the same way;
+ * a release the gate REFUSES (a contended `transactions: 'strict'`
+ * store, a queue timeout) still runs, off-gate — a statement reset reads
+ * and writes nothing, and a statement left open until the store closes
+ * is the worse outcome — and answers `{ done: true }`. An abort likewise
+ * resets the source at once, off-gate, through the inner cursor's own
+ * listener: an abort asks for the statement to be let go, not for it to
+ * be held until a stranger's transaction commits.
+ *
+ * Refusals keep their granularity: a pull abandoned while QUEUED is the
+ * gate's `JD2064` (the source, never opened for that pull, needs no
+ * release); a pull whose signal is already aborted, or aborts at a row
+ * boundary, is the cursor's own `JD2072` and releases the source once; a
+ * passed deadline is `JD2075`. A cursor that has settled — exhausted,
+ * released, aborted — answers `{ done: true }` without borrowing the
+ * gate at all. The decorator buffers no item and holds no gate between
+ * two public pulls: `streaming`, `barrier` and the asynchronous-iterator
+ * identity are the inner cursor's own.
+ * @param {any} cursor - the engine's `QueryCursor`
+ * @param {(fn: () => any, what?: string, signal?: AbortSignal) => any} admit
+ *   - the store gate: runs `fn` holding the connection, value-or-promise
+ * @param {AbortSignal | undefined} signal - the cursor's own signal, so an
+ *   abort abandons a queued pull
+ * @param {string} what - what is waiting, for the gate's timeout message
+ * @returns {any} the admitted `QueryCursor`
+ */
+export function admitCursor(cursor, admit, signal, what) {
+  /**
+   * @param {'next' | 'return'} member
+   * @param {string} label
+   * @param {boolean} abandonable - whether an abort leaves the queue
+   */
+  const through = (member, label, abandonable) => () => {
+    // an already-aborted cursor refuses on its own (`JD2072`, every later
+    // pull) and releases its source; the gate has nothing to admit
+    if (abandonable && signal?.aborted === true) return cursor[member]();
+    // a settled cursor holds no source: nothing to admit, nothing to wait for
+    if (cursor.settled === true) return Promise.resolve({ done: true, value: undefined });
+    let admitted;
+    try {
+      admitted = Promise.resolve(admit(() => cursor[member](), label, abandonable ? signal : undefined));
+    }
+    catch (error) {
+      admitted = Promise.reject(error);
+    }
+    if (member !== 'return') return admitted;
+    // a refused release still releases: the reset lands off-gate rather
+    // than leaving the statement open
+    return admitted.catch(() => cursor.return());
+  };
+  /** @type {any} */
+  const admitted = {
+    streaming: cursor.streaming,
+    barrier: cursor.barrier,
+    next: through('next', what, true),
+    return: through('return', `${what} (release)`, false),
+    [Symbol.asyncIterator]: () => admitted,
+  };
+  return Object.freeze(admitted);
+}
+
+/** Synchronous admission per pull; cleanup is permitted even after refusal.
+ * @param {any} cursor @param {(fn: () => any) => any} admit
+ * @returns {any}
+ */
+export function admitSyncCursor(cursor, admit) {
+  const wrapped = {
+    streaming: cursor.streaming,
+    barrier: cursor.barrier,
+    next: () => admit(() => cursor.next()),
+    return: () => cursor.return(),
+    [Symbol.iterator]: () => wrapped,
+    [Symbol.dispose]: () => { cursor.return(); },
+  };
+  return Object.freeze(wrapped);
+}
+
+/** The page size a page takes when none is given. */
+export const PAGE_LIMIT_DEFAULT = 100;
+
+/**
+ * Drain a cursor into ONE page: at most `limit` items, at most `maxBytes`
+ * serialised bytes (`null` for no byte bound), stopping at an item
+ * boundary and releasing the cursor. The one implementation every page
+ * in this package is — an entity page, a change page — so the
+ * `item_too_large` rule exists once: an item that alone exceeds
+ * `maxBytes` when nothing has been delivered yet is the refusal
+ * `JD2074`, raised WITHOUT advancing the continuation, so a caller that
+ * retries meets the same refusal instead of a loop or a silent breach.
+ * An item that does not fit beside earlier ones ends the page before
+ * it: `hasMore` is true and the continuation is the last delivered
+ * item's, so the next page starts at the item that did not fit.
+ * `hasMore` is otherwise decided by one peek past `limit`.
+ * @param {any} cursor - a `QueryCursor`
+ * @param {{ limit: number, maxBytes: number | null, after?: any,
+ *   sizeOf: (item: any) => number, continuationOf: (item: any) => any }} options
+ * @returns {any} value-or-promise, matching the cursor
+ */
+export function drainPage(cursor, options) {
+  const { limit, maxBytes, sizeOf, continuationOf } = options;
+  const after = options.after ?? null;
+  const items = [];
+  let bytes = 0;
+  let last = after;
+  let hasMore = false;
+  const finish = () => chain(cursor.return(), () => ({
+    items, continuation: items.length > 0 ? last : (hasMore ? after : null), hasMore,
+  }));
+  const consume = (pulled) => {
+    if (items.length >= limit) {
+      hasMore = pulled.done !== true;
+      return finish();
+    }
+    if (pulled.done === true) return finish();
+    const item = pulled.value;
+    const size = maxBytes === null ? 0 : sizeOf(item);
+    if (maxBytes !== null && bytes + size > maxBytes) {
+      if (items.length === 0) {
+        return chain(cursor.return(), () => {
+          assertItemBytes(size, maxBytes, continuationOf(item));
+        });
+      }
+      hasMore = true;
+      return finish();
+    }
+    items.push(item);
+    bytes += size;
+    last = continuationOf(item);
+    return null;
+  };
+  const step = () => {
+    for (;;) {
+      const pulled = cursor.next();
+      if (isThenable(pulled)) return pulled.then((value) => consume(value) ?? step());
+      const result = consume(pulled);
+      if (result !== null) return result;
+    }
+  };
+  return settling(step, () => cursor.return());
+}
+
+/** The shared refusal for an indivisible item, including a replicated transaction.
+ * @param {number} size @param {number} maxBytes @param {any} [at] */
+export function assertItemBytes(size, maxBytes, at) {
+  if (size > maxBytes) throw new DbRuntimeError('JD2074',
+    `the next item is ${size} serialised bytes, more than the page's maxBytes bound of `
+    + `${maxBytes}; the continuation was not advanced — raise the bound, or bound the `
+    + "item itself (an include's maxBytes, a narrower document)",
+    { errors: [{ bytes: size, maxBytes, at }] });
+}

package/src/dag-job.js

@@ -8,13 +8,41 @@
  * graph name flow nowhere.
  *
  * Each kind's document compiles ONCE against a delegating checkpoint
- * store; per claimed job, the delegate binds the engine's guarded
- * per-job store (`checkpointsFor`) — `save` refuses once the lease is
- * lost and `complete` records the DAG result, marks the job done and
- * prunes the checkpoint rows in ONE transaction, so a failure leaves
- * neither and a crash resumes instead of restarting.
+ * store; the delegate resolves per ATTEMPT, never per job, because two
+ * attempts of one job are two different runs of the same workflow and
+ * the older one may not write on the younger one's behalf. The engine's
+ * store carries the fence: `save` is refused once the lease is lost, and
+ * `complete` records the DAG result, marks the job done and prunes the
+ * checkpoint rows in ONE transaction, so a failure leaves neither and a
+ * crash resumes instead of restarting.
+ *
+ * A resumed run also has to be the SAME run. The workflow document's
+ * revision and a hash of the input are persisted with the checkpoints,
+ * and a resume that disagrees with either is refused by name — reusing
+ * checkpoints written by a different workflow is not a resume, it is a
+ * silently wrong answer.
  */
 
+import { canonicalizeJson } from '@jarenjs/json/canonical';
+import { hashContent } from '@jarenjs/core/string';
+import { setObjectMember } from '@jarenjs/core/object';
+
+import { DbRuntimeError } from './errors.js';
+
+/** The checkpoint row a run's identity lives in. The leading unit
+ * separator keeps it out of any node-id namespace a document could
+ * declare, and the DAG's seeding skips it because no node is called
+ * that. It is pruned with the run it belongs to. */
+export const RUN_IDENTITY_NODE = '\u001Fidentity';
+
+/** What joins a job id to an attempt's token in the run key the DAG
+ * sees. A unit separator, so no job id can carry one by accident. */
+const RUN_KEY_SEPARATOR = '\u001F';
+
+/** A stable fingerprint of any JSON value: the suite's one content hash
+ * over the suite's one canonical form. */
+const fingerprint = (value) => hashContent(canonicalizeJson(value ?? null));
+
 /**
  * Build a worker whose handlers run checkpointed DAG documents.
  * @param {any} store - an open store with `{ jobs: true }`
@@ -22,8 +50,10 @@
  *   documents: Record<string, any>,
  *   tasks?: Record<string, Function>,
  *   concurrency?: number, pollInterval?: number, leaseMs?: number,
- *   owner?: string, backoffBase?: number, backoffCap?: number }} options
- * @returns {{ start: () => any, stop: () => Promise<void>, stats: () => any }}
+ *   owner?: string, renew?: boolean, onOutcome?: (event: any) => void,
+ *   backoffBase?: number, backoffCap?: number,
+ *   stopGraceMs?: number }} options
+ * @returns {{ start: () => any, stop: (options?: any) => Promise<any>, stats: () => any }}
  */
 export function createDagJobRunner(store, options) {
   if (store?.jobs === undefined) {
@@ -42,20 +72,109 @@ export function createDagJobRunner(store, options) {
       'createDagJobRunner: "documents" must map job kinds to dag documents');
   }
 
-  /** The active claim contexts, keyed by run id (= job id): the
-   * delegate resolves the CURRENT lease binding per store call. */
+  /**
+   * The attempts running right now, keyed by the run key the DAG was
+   * given — the job id and this attempt's fence token. Never by owner,
+   * and never by job id alone: a re-claim of the same job is a different
+   * attempt, and a delegate that could not tell them apart would let the
+   * older one write through the younger one's store.
+   * @type {Map<string, any>}
+   */
   const active = new Map();
-  const boundStore = (runId) => {
-    const context = active.get(runId);
+
+  /** The run key the DAG sees, and the job id the STORE sees, are not
+   * the same string: the store keys rows by job, the delegate keys
+   * bindings by attempt. */
+  const runKeyOf = (jobId, token) => `${jobId}${RUN_KEY_SEPARATOR}${token}`;
+  const jobIdOf = (runKey) => runKey.slice(0, runKey.indexOf(RUN_KEY_SEPARATOR));
+
+  const bound = (runKey) => {
+    const context = active.get(runKey);
     if (context === undefined) {
-      throw new Error(`no active job holds run '${runId}'`);
+      throw new DbRuntimeError('JD2069',
+        `no attempt holds run '${jobIdOf(runKey)}' — its lease was lost, or the run `
+        + 'outlived the handler that started it',
+        { docPath: '/jobs' });
     }
-    return context.checkpointsFor(context.job);
+    return context;
   };
   const checkpoint = {
-    load: (runId) => boundStore(runId).load(runId),
-    save: (runId, nodeId, value) => boundStore(runId).save(runId, nodeId, value),
-    complete: (runId, result) => boundStore(runId).complete(runId, result),
+    load: (runKey) => bound(runKey).checkpoints.load(jobIdOf(runKey)),
+    save: (runKey, nodeId, value) =>
+      bound(runKey).checkpoints.save(jobIdOf(runKey), nodeId, value),
+    complete: (runKey, result) =>
+      bound(runKey).checkpoints.complete(jobIdOf(runKey), result),
+  };
+
+  /** Which declared task identities moved between two version maps. */
+  const describeVersionDrift = (before, after) => {
+    const names = [...new Set([...Object.keys(before ?? {}), ...Object.keys(after ?? {})])].sort();
+    const moved = [];
+    for (const name of names) {
+      const was = before?.[name];
+      const now = after?.[name];
+      if (was === now) continue;
+      if (was === undefined) moved.push(`'${name}' is new at version ${now}`);
+      else if (now === undefined) moved.push(`'${name}' is gone (was version ${was})`);
+      else moved.push(`'${name}' moved from version ${was} to ${now}`);
+    }
+    return moved;
+  };
+
+  /**
+   * The identity a resumed run must agree with: the workflow document,
+   * the input, and the DECLARED versions of the task implementations
+   * (FLOW-FORMAT §7.8). The third closes the gap the first two cannot
+   * see — a handler reimplemented while its document stayed byte-equal
+   * produces checkpoints that describe a computation nobody asked for
+   * just as surely as an edited document does.
+   */
+  const requireSameRun = async (context, jobId, revision, inputHash, taskVersions) => {
+    const loaded = await context.checkpoints.load(jobId);
+    const stored = loaded?.values?.[RUN_IDENTITY_NODE];
+    const taskVersionsHash = fingerprint(taskVersions);
+    const identity = { revision, inputHash, taskVersionsHash, taskVersions };
+    if (stored === undefined) {
+      await context.checkpoints.save(jobId, RUN_IDENTITY_NODE, identity);
+      return;
+    }
+    const differs = [];
+    if (stored.revision !== revision) {
+      differs.push(`the workflow (checkpointed under revision ${stored.revision}, `
+        + `this runner compiles revision ${revision})`);
+    }
+    if (stored.inputHash !== inputHash) {
+      differs.push(`the input (checkpointed under ${stored.inputHash}, `
+        + `this attempt was given ${inputHash})`);
+    }
+    if (stored.taskVersionsHash === undefined) {
+      // A row written before task identity was recorded. Unknown is not
+      // equal: the upgrade is allowed only where nothing can be replayed
+      // wrongly — when no node value has been recorded yet, so the run has
+      // nothing to inherit from an implementation nobody can name.
+      const recorded = Object.keys(loaded?.values ?? {})
+        .filter((nodeId) => nodeId !== RUN_IDENTITY_NODE);
+      if (recorded.length === 0) {
+        await context.checkpoints.save(jobId, RUN_IDENTITY_NODE, identity);
+      }
+      else {
+        differs.push(`the task versions (this run recorded ${recorded.length} node value(s) `
+          + 'before task identity was persisted, so the implementation that produced them '
+          + 'cannot be confirmed)');
+      }
+    }
+    else if (stored.taskVersionsHash !== taskVersionsHash) {
+      const moved = describeVersionDrift(stored.taskVersions, taskVersions);
+      differs.push(`the task versions (${moved.length > 0 ? moved.join(', ')
+        : `checkpointed under ${stored.taskVersionsHash}, this runner compiles `
+          + `${taskVersionsHash}`})`);
+    }
+    if (differs.length === 0) return;
+    throw new DbRuntimeError('JD2069',
+      `run '${jobId}' cannot resume: ${differs.join(' and ')} changed since its `
+      + 'checkpoints were written. Enqueue it under a new id, or drop the run — '
+      + 'reusing them would answer for a computation nobody asked for',
+      { docPath: '/jobs', collection: jobId });
   };
 
   /** @type {Record<string, Function>} */
@@ -63,15 +182,24 @@ export function createDagJobRunner(store, options) {
   for (const kind of Object.keys(documents)) {
     const compiled = compileDag(documents[kind],
       { tasks: options.tasks ?? {}, checkpoint });
-    handlers[kind] = async (payload, context) => {
-      active.set(context.job.id, context);
+    // one revision per compiled document, so every attempt of every run
+    // of this kind compares against the same number
+    const revision = fingerprint(documents[kind]);
+    setObjectMember(handlers, kind, async (payload, context) => {
+      const input = payload?.input ?? null;
+      const runKey = runKeyOf(context.job.id, context.job.lease.token);
+      active.set(runKey, context);
       try {
-        return await compiled.run(payload?.input ?? null, { runId: context.job.id });
+        await requireSameRun(context, context.job.id, revision, fingerprint(input),
+          compiled.taskVersions);
+        // the handler's signal reaches every task: a worker winding down
+        // inside its grace period, or a lease this attempt has lost
+        return await compiled.run(input, { runId: runKey, signal: context.signal });
       }
       finally {
-        active.delete(context.job.id);
+        active.delete(runKey);
       }
-    };
+    });
   }
 
   return store.jobs.createWorker({
@@ -80,7 +208,12 @@ export function createDagJobRunner(store, options) {
     pollInterval: options.pollInterval,
     leaseMs: options.leaseMs,
     owner: options.owner,
+    renew: options.renew,
+    onOutcome: options.onOutcome,
     backoffBase: options.backoffBase,
     backoffCap: options.backoffCap,
+    // the runner's declared default for `stop()` with no override; an
+    // explicit `stop({ graceMs })` still wins inside the worker
+    stopGraceMs: options.stopGraceMs,
   });
 }