@bevel-software/ael 0.0.0-stage → 0.1.0

package/src/interpreter/dispatcher.workflow.ts

@@ -0,0 +1,412 @@
+import { ParentClosePolicy, condition, continueAsNew, defineQuery, defineSignal, getExternalWorkflowHandle, log, proxyActivities, setHandler, startChild, workflowInfo } from '@temporalio/workflow';
+import { PLATFORM_TASK_QUEUE, RUN_AT_GATE_SIGNAL, RUN_NEEDS_SLOT_SIGNAL, RUN_SETTLED_SIGNAL, SLOT_GRANTED_SIGNAL, type InterpretInput, type InterpreterActivities } from './contract.js';
+import { interpret } from './interpret.workflow.js';
+
+/**
+ * The dispatcher: one per trigger. It receives the trigger's firings as
+ * signals, orders them by priority and filing time, keeps at most `cap` runs
+ * at work, and allows one active run per key (a kb node) with one follow-up
+ * queued behind it. Temporal task queues are FIFO; this is where priority
+ * lives. It is the one place that admits a firing: a change while a run of
+ * the node is active becomes its follow-up, a change while one waits
+ * replaces it, and a firing that will never run (replaced, or waiting when
+ * the trigger is removed) has its run thread closed on the platform.
+ *
+ * The cap counts runs at work, not runs in flight: a run waiting at a gate
+ * says so and its slot is free for another, while its key stays held. Past
+ * its gate it asks for a slot and goes on when it is told it has one — ahead
+ * of the firings that wait, since work begun is finished before more starts.
+ *
+ * The trigger's row reaches it as a `settings` signal: its cap and
+ * priorities, and whether it is paused — a paused dispatcher keeps its
+ * queue and starts nothing.
+ *
+ * A run's end reaches the dispatcher three ways: the child's result, in the
+ * execution that started it; the `runSettled` signal every run sends its
+ * parent as it finishes; and, for the runs an execution inherited from the
+ * one before it, Temporal's own word, asked after a while without news. The
+ * signal is what lets the dispatcher continue as new while runs are in
+ * flight — a busy trigger never has an idle moment, and a history that only
+ * ever grows is terminated by Temporal with every queued firing in it — and
+ * the new execution, which started none of the runs it inherits, still
+ * learns when each one is over. The check behind it is for the run that
+ * never signals: terminated, or ended without finishing. Without it such a
+ * run would hold its key, and a slot under the cap, for ever.
+ */
+
+export interface Enqueued {
+  /** One active run per key; a `kb` trigger uses the node's path. */
+  key: string;
+  /** The level as the work item writes it; ranked by `priorities`. */
+  priority: string | null;
+  /** ISO time the firing was filed; earlier first within a level. */
+  filedAt: string;
+  run: InterpretInput;
+}
+
+/** The trigger row's part of the dispatcher's state, sent whenever the row changes. */
+export interface DispatchSettings {
+  /** Runs at work at once; a run waiting at a gate is not counted. */
+  cap: number;
+  /** Priority levels, highest first; a level not listed ranks after every listed one. */
+  priorities: string[];
+  /** A paused dispatcher keeps its queue and starts nothing. */
+  paused: boolean;
+}
+
+export interface DispatchInput extends DispatchSettings {
+  triggerId: string;
+  /** Carried across continue-as-new. */
+  queued?: Enqueued[];
+  followUps?: Enqueued[];
+  /** The runs in flight when the previous execution continued as new: key and run id. */
+  active?: Array<[string, string]>;
+  /** The runs in flight that had given their slot back at a gate, by run id. */
+  atGate?: string[];
+  /** Those of them past their gate and waiting for a slot, first asked first. */
+  resuming?: string[];
+  replaced?: number;
+  /** Continue as new once this many runs were started by one execution (tests lower it). */
+  continueAfterStarts?: number;
+  /** Continue as new once this many signals were handled by one execution, starts or none (tests lower it). */
+  continueAfterSignals?: number;
+  /** How long an inherited run goes without news before Temporal is asked whether it still runs (tests lower it). */
+  carriedRunCheckMs?: number;
+  /** When the inherited runs are next asked about (epoch ms), carried across continue-as-new so rotating often never postpones it. */
+  nextCarriedCheckAt?: number;
+}
+
+/**
+ * What the dispatcher holds, for the Triggers screen: the keys in flight,
+ * the keys waiting in priority order, the keys with a follow-up queued
+ * behind their active run, how many firings were dropped because a newer
+ * one for the same key replaced them (a change filed twice while a run
+ * waited, or while it ran) — those are the trigger's skipped fires — and
+ * the settings it runs with.
+ */
+export interface DispatcherState {
+  queued: string[];
+  active: string[];
+  followUps: string[];
+  replaced: number;
+  paused: boolean;
+  cap: number;
+}
+
+export const enqueueSignal = defineSignal<[Enqueued]>('enqueue');
+export const settingsSignal = defineSignal<[DispatchSettings]>('settings');
+export const stopDispatcherSignal = defineSignal<[]>('stop');
+/** A run this dispatcher started is over (its run id); sent by the run itself. */
+export const runSettledSignal = defineSignal<[string]>(RUN_SETTLED_SIGNAL);
+/** A run this dispatcher started waits at a gate (its run id): its slot is free. */
+export const runAtGateSignal = defineSignal<[string]>(RUN_AT_GATE_SIGNAL);
+/** A run past its gate asks for a slot (its run id); answered with `SLOT_GRANTED_SIGNAL`. */
+export const runNeedsSlotSignal = defineSignal<[string]>(RUN_NEEDS_SLOT_SIGNAL);
+export const dispatcherStateQuery = defineQuery<DispatcherState>('dispatcherState');
+/**
+ * The run ids this dispatcher holds — queued, waiting as a follow-up, or in
+ * flight. A run thread of this trigger that is not among them has no
+ * dispatcher behind it (its firing was never admitted), whatever the
+ * dispatcher's own state; the platform's orphan check asks this.
+ */
+export const dispatcherRunsQuery = defineQuery<string[]>('dispatcherRuns');
+
+/** Continue as new once this many runs were started by one execution, to keep the history short. */
+const CONTINUE_AFTER_STARTS = 500;
+/**
+ * Continue as new once this many signals were handled by one execution,
+ * whether or not anything started: a paused dispatcher, or one at its cap,
+ * still takes every firing, replacement and settings change into its
+ * history, and a history that only ever grows is terminated by Temporal
+ * with the queue in it. Well under the server's own suggestion, which is
+ * honoured too when it comes first.
+ */
+const CONTINUE_AFTER_SIGNALS = 1000;
+/**
+ * How long an inherited run goes without news before Temporal is asked
+ * whether it still runs. The run's own signal is the fast path; this is the
+ * net under it, so minutes are fine — a slot held a few minutes past a run
+ * that died is nothing, a slot held for ever is a trigger that stopped.
+ */
+const CARRIED_RUN_CHECK_MS = 5 * 60_000;
+
+/** The workflow id a run is started under, and reached by afterwards. */
+const runWorkflowId = (runId: string) => `run-${runId}`;
+
+/**
+ * A withdrawal closes a firing's run thread on the platform: cleanup that
+ * must land, so it is retried the way a step's end is — for about a day,
+ * the interval capped — rather than given up after a few tries, after which
+ * a replaced or removed firing would read as running for ever.
+ */
+const platform = proxyActivities<Pick<InterpreterActivities, 'withdrawRun'>>({
+  startToCloseTimeout: '1 minute',
+  retry: { maximumAttempts: 300, initialInterval: '2 seconds', backoffCoefficient: 2, maximumInterval: '5 minutes' },
+});
+/** The check of inherited runs: a read, asked again at the next interval when it fails. */
+const temporal = proxyActivities<Pick<InterpreterActivities, 'runsInFlight'>>({
+  startToCloseTimeout: '1 minute',
+  retry: { maximumAttempts: 3, initialInterval: '2 seconds' },
+});
+
+export async function dispatch(input: DispatchInput): Promise<void> {
+  const queued: Enqueued[] = [...(input.queued ?? [])];
+  const followUps = new Map<string, Enqueued>((input.followUps ?? []).map((f) => [f.key, f]));
+  /** key → run id */
+  const active = new Map<string, string>(input.active ?? []);
+  /**
+   * The runs in flight that this execution inherited and did not start: no
+   * child result will ever come for them, so their end is learned from the
+   * run's own signal — or, after a while without one, from Temporal.
+   */
+  const carried = new Set<string>(active.values());
+  /** The runs in flight that gave their slot back at a gate: they hold their key and no slot. */
+  const atGate = new Set<string>(input.atGate ?? []);
+  /** Those of them past their gate and waiting for a slot, first asked first. */
+  const resuming: string[] = [...(input.resuming ?? [])];
+  /** The runs to be told they hold a slot. */
+  const granted: string[] = [];
+  const atWork = () => active.size - atGate.size;
+  const continueAfterStarts = input.continueAfterStarts ?? CONTINUE_AFTER_STARTS;
+  const continueAfterSignals = input.continueAfterSignals ?? CONTINUE_AFTER_SIGNALS;
+  const carriedRunCheckMs = input.carriedRunCheckMs ?? CARRIED_RUN_CHECK_MS;
+  let cap = input.cap;
+  let priorities = input.priorities;
+  let paused = input.paused;
+  let stopped = false;
+  let settled = 0;
+  let started = 0;
+  /** Signals handled by this execution: each is history, whether or not it started anything. */
+  let signalled = 0;
+  let replaced = input.replaced ?? 0;
+  /** Firings that will never run, closed on the platform; awaited before this execution ends. */
+  const withdrawals: Promise<void>[] = [];
+
+  const withdraw = (item: Enqueued, reason: string) => {
+    // Logged only once the day-long retry is spent: the dispatcher goes on
+    // for the firings that can still run rather than failing with them.
+    withdrawals.push(platform.withdrawRun({ runId: item.run.runId, reason }).catch((err) => log.warn(`run ${item.run.runId} not withdrawn after a day of retries: ${String(err)}`)));
+  };
+  /**
+   * Every withdrawal started so far, landed — the ones started while this
+   * waited included: a signal handled during the wait may add one, and a
+   * snapshot of the list would leave it behind.
+   */
+  let drained = 0;
+  const drainWithdrawals = async () => {
+    while (drained < withdrawals.length) {
+      const upTo = withdrawals.length;
+      await Promise.all(withdrawals.slice(drained, upTo));
+      drained = upTo;
+    }
+  };
+
+  const rank = (p: string | null) => {
+    if (p === null) return priorities.length;
+    const i = priorities.findIndex((level) => level.toLowerCase() === p.toLowerCase());
+    return i === -1 ? priorities.length : i;
+  };
+  const sortQueue = () => queued.sort((a, b) => rank(a.priority) - rank(b.priority) || a.filedAt.localeCompare(b.filedAt));
+
+  /**
+   * A run is over: its key is free, its follow-up (if any) joins the queue.
+   * By run id, not by key: the run's own signal and its result both say so,
+   * and by the time the second arrives a follow-up may hold the key — that
+   * follow-up keeps it. A run nobody holds is nothing to settle.
+   */
+  const settleRun = (runId: string) => {
+    carried.delete(runId);
+    atGate.delete(runId);
+    if (resuming.includes(runId)) resuming.splice(resuming.indexOf(runId), 1);
+    for (const [key, id] of active) {
+      if (id !== runId) continue;
+      active.delete(key);
+      settled += 1;
+      const next = followUps.get(key);
+      if (next) {
+        followUps.delete(key);
+        queued.push(next);
+        sortQueue();
+      }
+      return;
+    }
+  };
+
+  setHandler(enqueueSignal, (item) => {
+    signalled += 1;
+    // A firing admitted while the dispatcher stops (the row went between the
+    // listener's read and its signal) will never run: told so now, never
+    // queued behind a stop that has already withdrawn the queue.
+    if (stopped) {
+      withdraw(item, 'the trigger was removed');
+      return;
+    }
+    if (active.has(item.key)) {
+      // A change during a run queues ONE follow-up; a newer one replaces it.
+      const prior = followUps.get(item.key);
+      if (prior) {
+        replaced += 1;
+        if (prior.run.runId !== item.run.runId) withdraw(prior, 'replaced by a newer firing for the same node');
+      }
+      followUps.set(item.key, item);
+      return;
+    }
+    const i = queued.findIndex((q) => q.key === item.key);
+    if (i === -1) queued.push(item);
+    else {
+      replaced += 1;
+      if (queued[i].run.runId !== item.run.runId) withdraw(queued[i], 'replaced by a newer firing for the same node');
+      queued[i] = item;
+    }
+    sortQueue();
+  });
+  setHandler(settingsSignal, (settings) => {
+    signalled += 1;
+    cap = settings.cap;
+    priorities = settings.priorities;
+    paused = settings.paused;
+    sortQueue();
+  });
+  setHandler(stopDispatcherSignal, () => {
+    stopped = true;
+  });
+  setHandler(runSettledSignal, (runId) => {
+    signalled += 1;
+    settleRun(runId);
+  });
+  setHandler(runAtGateSignal, (runId) => {
+    signalled += 1;
+    if ([...active.values()].includes(runId)) atGate.add(runId);
+  });
+  setHandler(runNeedsSlotSignal, (runId) => {
+    signalled += 1;
+    // A run that asks again while it waits keeps its place. One this
+    // dispatcher holds no slot back for (its word from the gate never came,
+    // or it is not a run held here) has nothing to wait for and is told so.
+    if (!atGate.has(runId)) granted.push(runId);
+    else if (!resuming.includes(runId)) resuming.push(runId);
+  });
+  setHandler(dispatcherStateQuery, () => ({ queued: queued.map((q) => q.key), active: [...active.keys()], followUps: [...followUps.keys()], replaced, paused, cap }));
+  setHandler(dispatcherRunsQuery, () => [...queued.map((q) => q.run.runId), ...[...followUps.values()].map((f) => f.run.runId), ...active.values()]);
+  // A queue handed in (a continue-as-new, a seeded start) is ordered the same way a signalled one is.
+  sortQueue();
+
+  // Asked before every start, `stopped` included: a stop that lands while a
+  // child is being started ends the starting there, and the loop's top
+  // withdraws what is left.
+  const mayStart = () => !stopped && !paused && queued.length > 0 && atWork() < cap;
+  // A run past its gate is in flight already, so a pause, which starts
+  // nothing new, does not hold it; a stop ends the cap with the dispatcher.
+  const mayResume = () => resuming.length > 0 && (stopped || atWork() < cap);
+  /**
+   * Time to continue as new: this execution started or handled enough to
+   * keep its history short, or Temporal says so. Asked in the wait as well
+   * as after the starts — a dispatcher that is paused, or at its cap, starts
+   * nothing and would otherwise never look, while every firing it takes
+   * lengthens a history that Temporal ends with the queue in it.
+   */
+  const rotate = () => started >= continueAfterStarts || signalled >= continueAfterSignals || workflowInfo().continueAsNewSuggested;
+  const wakeUp = () => stopped || mayStart() || mayResume() || granted.length > 0 || settled > 0 || rotate();
+
+  /**
+   * The inherited runs without news: those Temporal no longer runs are
+   * settled as if they had signalled. A check that fails is asked again at
+   * the next interval; a signal that lands while it runs has settled its
+   * run already, and the answer about that run is then nobody's.
+   */
+  const checkCarried = async () => {
+    const asked = [...carried];
+    try {
+      const { running } = await temporal.runsInFlight({ runIds: asked });
+      for (const runId of asked) {
+        if (running.includes(runId) || !carried.has(runId)) continue;
+        log.warn(`run ${runId} is no longer running and never said so; its key is freed`);
+        settleRun(runId);
+      }
+    } catch (err) {
+      log.warn(`inherited runs not checked: ${String(err)}`);
+    }
+  };
+
+  /**
+   * When the inherited runs are next asked about: a fixed deadline, not a
+   * quiet spell — on a busy trigger something settles or arrives every few
+   * minutes, and a wait restarted by each would never reach the check, so a
+   * dead run would hold its slot for ever.
+   */
+  let nextCarriedCheck = input.nextCarriedCheckAt ?? Date.now() + carriedRunCheckMs;
+
+  for (;;) {
+    // With inherited runs in flight the wait ends at the check's deadline at
+    // the latest. Without any, nothing but a signal or a start can change the
+    // picture, and the wait is open.
+    if (carried.size > 0) {
+      await condition(wakeUp, Math.max(1, nextCarriedCheck - Date.now()));
+      if (Date.now() >= nextCarriedCheck) {
+        await checkCarried();
+        nextCarriedCheck = Date.now() + carriedRunCheckMs;
+      }
+    } else {
+      await condition(wakeUp);
+    }
+    settled = 0;
+    // Free slots go to the runs past their gate first, then to new firings.
+    while (mayResume()) {
+      const runId = resuming.shift()!;
+      atGate.delete(runId);
+      granted.push(runId);
+    }
+    while (granted.length > 0) {
+      const runId = granted.shift()!;
+      try {
+        await getExternalWorkflowHandle(runWorkflowId(runId)).signal(SLOT_GRANTED_SIGNAL);
+      } catch (err) {
+        // The run ended meanwhile: its own signal, or the check, settles it.
+        log.warn(`run ${runId} could not be told it holds a slot: ${String(err)}`);
+      }
+    }
+    if (stopped) {
+      // The trigger is gone: what waited will never run, and its threads say
+      // so — the firings that arrive while those threads are being told
+      // included (the handler withdraws them itself once `stopped` is set,
+      // and the drain waits for every withdrawal, however late it started).
+      for (const item of [...queued, ...followUps.values()]) withdraw(item, 'the trigger was removed');
+      queued.length = 0;
+      followUps.clear();
+      await drainWithdrawals();
+      log.info(`dispatcher ${input.triggerId} stopped with ${active.size} active`);
+      return;
+    }
+    while (mayStart()) {
+      const item = queued.shift()!;
+      started += 1;
+      const workflowId = runWorkflowId(item.run.runId);
+      active.set(item.key, item.run.runId);
+      const handle = await startChild(interpret, {
+        workflowId,
+        args: [{ ...item.run, capped: true }],
+        taskQueue: PLATFORM_TASK_QUEUE,
+        // A run outlives the dispatcher that started it: a continue-as-new or
+        // a stop must never cancel work in flight.
+        parentClosePolicy: ParentClosePolicy.ABANDON,
+      });
+      void handle
+        .result()
+        .catch((err) => log.warn(`run ${workflowId} ended with ${String(err)}`))
+        .finally(() => settleRun(item.run.runId));
+    }
+    // A stop that landed during the starts is handled at the top, never
+    // carried into a new execution that would not know of it.
+    if (stopped) continue;
+    // Whether or not runs are in flight: the ones in flight are carried, and
+    // their end reaches the next execution as a signal.
+    if (rotate()) {
+      await drainWithdrawals();
+      // A stop that landed while the withdrawals were being waited for is
+      // handled at the top too: a new execution would start un-stopped.
+      // So is a run that asked for a slot meanwhile and is owed its answer.
+      if (stopped || granted.length > 0) continue;
+      await continueAsNew<typeof dispatch>({ ...input, cap, priorities, paused, queued, followUps: [...followUps.values()], active: [...active], atGate: [...atGate], resuming, replaced, carriedRunCheckMs, nextCarriedCheckAt: nextCarriedCheck });
+    }
+  }
+}