@boxxy/dispatch 0.1.1 → 0.1.3

package/src/readings/pick-reading-action.ts

@@ -0,0 +1,315 @@
+import type {
+  AttemptRow,
+  ClosingCause,
+  ForcedFailure,
+  Instant,
+  LiveAttemptState,
+  SessionReading,
+} from '@boxxy/model';
+import invariant from 'tiny-invariant';
+import { timeouts } from '../timeouts.ts';
+import { findWaitingSince } from './find-waiting-since.ts';
+import type { ReadingAction, ReadingInput, ReadingTable, RunningBound } from './types.ts';
+
+const minuteMs = 60_000;
+const secondMs = 1000;
+
+// The reconcile table: one action for each reading of a live attempt's session. Pure.
+export function pickReadingAction(input: ReadingInput): ReadingAction {
+  const state = input.attempt.state;
+
+  invariant(isLiveState(state), `the runner reads only live attempts, not ${state}`);
+
+  return pickCell(input, state);
+}
+
+function isLiveState(state: AttemptRow['state']): state is LiveAttemptState {
+  return state !== 'succeeded' && state !== 'failed';
+}
+
+// Read it as: the reading, then the attempt's state. v1-lifecycle.md has the same table.
+const readingTable: ReadingTable = {
+  running: {
+    starting: moveToRunning,
+    reconciling: moveToRunning,
+    running: pickRunningBound,
+    sleeping: pickSessionRan,
+    waking: () => ({ kind: 'move-to-running', via: 'resume' }),
+    closing: pickClosingStop,
+  },
+  suspended: {
+    starting: sleepUnasked,
+    reconciling: sleepUnasked,
+    running: sleepUnasked,
+    sleeping: nothing,
+    waking: resumeOrLost,
+    closing: settleClosing,
+  },
+  ended: {
+    starting: settleWith(null),
+    reconciling: settleWith(null),
+    running: sleepOrSettle,
+    sleeping: nothing,
+    waking: resumeOrLost,
+    closing: settleClosing,
+  },
+  absent: {
+    starting: respawnOrLost,
+    reconciling: respawnOrLost,
+    running: settleWith('session-lost'),
+    sleeping: settleWith('session-lost'),
+    waking: settleWith('session-lost'),
+    closing: awaitSpawnOrSettle,
+  },
+};
+
+// Each reading kind narrows the input before it reaches its row of the table.
+function pickCell(input: ReadingInput, state: LiveAttemptState): ReadingAction {
+  const reading = input.reading;
+
+  if (reading.kind === 'running') {
+    return readingTable.running[state]({ ...input, reading });
+  }
+
+  if (reading.kind === 'suspended') {
+    return readingTable.suspended[state]({ ...input, reading });
+  }
+
+  if (reading.kind === 'ended') {
+    return readingTable.ended[state]({ ...input, reading });
+  }
+
+  return readingTable.absent[state]({ ...input, reading });
+}
+
+function moveToRunning(): ReadingAction {
+  return { kind: 'move-to-running', via: 'reconcile' };
+}
+
+function pickRunningBound(
+  input: ReadingInput<Extract<SessionReading, { kind: 'running' }>>,
+): ReadingAction {
+  for (const bound of runningBounds) {
+    const action = bound.pick(input);
+
+    if (action !== null) {
+      return action;
+    }
+  }
+
+  return { kind: 'bind-only' };
+}
+
+// The bounds of a running attempt whose session runs, in the order they are checked. A session
+// that waits for the owner sleeps before the stall rule can stop it: both fall due 30 minutes
+// after the last activity, and a stall first would leave the idle sleep no case in agent mode.
+const runningBounds: readonly RunningBound[] = [
+  {
+    name: 'a suspend that did not happen',
+    pick: (input) => (input.attempt.suspendRequestedAt === null ? null : { kind: 'clear-mark' }),
+  },
+  {
+    name: 'the usage limit',
+    pick: (input) =>
+      input.reading.usageLimitResetsAt === null
+        ? null
+        : {
+            kind: 'close',
+            cause: 'usage-limit',
+            resetsAt: input.reading.usageLimitResetsAt,
+          },
+  },
+  {
+    name: 'the step timeout, agent mode',
+    pick: (input) => {
+      const step = input.step;
+      const since = getStretchStart(input.attempt);
+
+      return input.attempt.mode === 'agent' &&
+        step !== null &&
+        input.now - since >= step.timeoutMinutes * minuteMs
+        ? {
+            kind: 'close',
+            cause: 'timed-out',
+            afterMinutes: step.timeoutMinutes,
+          }
+        : null;
+    },
+  },
+  {
+    name: 'a turn that ended with no close-out, agent mode',
+    pick: (input) =>
+      input.attempt.mode === 'agent' && input.reading.attention === 'turn-ended'
+        ? { kind: 'close', cause: 'turn-ended' }
+        : null,
+  },
+  {
+    name: 'a session that waits for the owner',
+    pick: (input) => {
+      const waitingSince = findWaitingSince({
+        attempt: input.attempt,
+        reading: input.reading,
+        quietAfterMinutes: input.sleep.afterQuietMinutes,
+        now: input.now,
+      });
+
+      // atc cannot revive a session stopped before it captured the agent session ID
+      if (waitingSince === null || input.reading.agentSession === null) {
+        return null;
+      }
+
+      return input.now - waitingSince >= input.sleep.afterIdleMinutes * minuteMs
+        ? { kind: 'sleep-if-idle', waitingSince }
+        : { kind: 'bind-only' };
+    },
+  },
+  {
+    name: 'a stall, agent mode',
+    pick: (input) =>
+      input.attempt.mode === 'agent' &&
+      input.now - input.reading.lastActivityAt >= timeouts.stallMinutes * minuteMs
+        ? { kind: 'close', cause: 'stalled' }
+        : null,
+  },
+  {
+    name: 'an owner who has not attached, attended mode',
+    pick: (input) => {
+      const since = getLater(getStretchStart(input.attempt), input.reading.lastOwnerAt);
+
+      return input.attempt.mode === 'attended' &&
+        input.reading.agentSession !== null &&
+        input.now - since >= input.sleep.afterUnattendedMinutes * minuteMs
+        ? { kind: 'sleep-unattended', since }
+        : null;
+    },
+  },
+];
+
+// One stretch of running starts at the start or at the latest wake.
+function getStretchStart(attempt: AttemptRow): Instant {
+  return getLater(attempt.startedAt ?? attempt.claimedAt, attempt.wokenAt);
+}
+
+function getLater(first: Instant, second: Instant | null): Instant {
+  return second !== null && second > first ? second : first;
+}
+
+// The owner revived a sleeping session in atc: ask for the wake, and wake over the cap when no
+// seat came free in time.
+function pickSessionRan(input: ReadingInput): ReadingAction {
+  const requestedAt = input.attempt.wakeRequestedAt;
+
+  if (requestedAt === null) {
+    return { kind: 'request-wake', by: 'session-ran' };
+  }
+
+  return input.now - requestedAt >= timeouts.seatlessSeconds * secondMs
+    ? { kind: 'wake-over-cap' }
+    : { kind: 'nothing' };
+}
+
+function pickClosingStop(
+  input: ReadingInput<Extract<SessionReading, { kind: 'running' }>>,
+): ReadingAction {
+  const closingAt = input.attempt.closingAt;
+
+  invariant(closingAt !== null, 'a closing attempt records when it closed');
+
+  const closingMs = input.now - closingAt;
+
+  if (closingMs >= timeouts.closingGiveUpMinutes * minuteMs) {
+    return {
+      kind: 'settle',
+      forced: findClosingFailure(input.attempt),
+      sessionStopped: false,
+    };
+  }
+
+  const isInsideGrace =
+    input.attempt.closing !== 'item-closed' &&
+    input.reading.attention === 'working' &&
+    closingMs < timeouts.stopGraceSeconds * secondMs;
+
+  return isInsideGrace ? { kind: 'nothing' } : { kind: 'stop' };
+}
+
+// A closed-out attempt settles by its close-out; one that closed for its turn end has none
+const closingFailures: Readonly<Record<ClosingCause, ForcedFailure | null>> = {
+  'closed-out': null,
+  'turn-ended': null,
+  'timed-out': 'timed-out',
+  'item-closed': 'item-closed',
+  stalled: 'stalled',
+  'skill-missing': 'skill-missing',
+  'usage-limit': 'usage-limit',
+};
+
+function findClosingFailure(attempt: AttemptRow): ForcedFailure | null {
+  invariant(attempt.closing !== null, 'a closing attempt has a cause');
+
+  return closingFailures[attempt.closing];
+}
+
+// A session that stopped with no request of boxxy's: the host slept, so it wakes by itself.
+function sleepUnasked(
+  input: ReadingInput<Extract<SessionReading, { kind: 'suspended' }>>,
+): ReadingAction {
+  return {
+    kind: 'sleep',
+    requestWake: !input.reading.asked && input.attempt.suspendRequestedAt === null,
+  };
+}
+
+function nothing(): ReadingAction {
+  return { kind: 'nothing' };
+}
+
+function resumeOrLost(input: ReadingInput): ReadingAction {
+  const wokenAt = input.attempt.wokenAt;
+
+  invariant(wokenAt !== null, 'a waking attempt records when it woke');
+
+  return input.now - wokenAt >= timeouts.resumeLostMinutes * minuteMs
+    ? { kind: 'settle', forced: 'resume-lost', sessionStopped: true }
+    : { kind: 'resume' };
+}
+
+function settleClosing(input: ReadingInput): ReadingAction {
+  return {
+    kind: 'settle',
+    forced: findClosingFailure(input.attempt),
+    sessionStopped: true,
+  };
+}
+
+function settleWith(forced: ForcedFailure | null): () => ReadingAction {
+  return () => ({ kind: 'settle', forced, sessionStopped: true });
+}
+
+function sleepOrSettle(input: ReadingInput): ReadingAction {
+  return input.attempt.suspendRequestedAt === null
+    ? { kind: 'settle', forced: null, sessionStopped: true }
+    : { kind: 'sleep', requestWake: false };
+}
+
+function respawnOrLost(input: ReadingInput): ReadingAction {
+  return input.now - input.attempt.claimedAt >= timeouts.spawnLostMinutes * minuteMs
+    ? { kind: 'settle', forced: 'spawn-lost', sessionStopped: true }
+    : { kind: 'respawn' };
+}
+
+// A closing attempt whose session was never seen: its spawn may still land. After the spawn
+// time it settles as a stray, so the runner keeps looking for the session by its spawn name.
+function awaitSpawnOrSettle(input: ReadingInput): ReadingAction {
+  if (input.attempt.atcSession !== null) {
+    return settleClosing(input);
+  }
+
+  return input.now - input.attempt.claimedAt >= timeouts.spawnLostMinutes * minuteMs
+    ? {
+        kind: 'settle',
+        forced: findClosingFailure(input.attempt),
+        sessionStopped: false,
+      }
+    : { kind: 'await-spawn' };
+}

package/src/readings/types.ts

@@ -0,0 +1,98 @@
+import type {
+  AttemptRow,
+  CapacityBackoff,
+  ClaimStore,
+  ForcedFailure,
+  Instant,
+  LiveAttemptState,
+  LoadedRules,
+  NewId,
+  PipelineStep,
+  SessionReading,
+  SleepRules,
+  WakeSource,
+  WorkerKind,
+} from '@boxxy/model';
+
+// What the runner does with one reading of a live attempt's session.
+export type ReadingAction =
+  | { readonly kind: 'nothing' }
+  | { readonly kind: 'bind-only' }
+  | { readonly kind: 'move-to-running'; readonly via: 'reconcile' | 'resume' }
+
+  // sends the stored spawn request again
+  | { readonly kind: 'respawn' }
+  | { readonly kind: 'clear-mark' }
+  | { readonly kind: 'sleep-if-idle'; readonly waitingSince: Instant }
+
+  // the mark, then a suspend with no idle check, then `slept`
+  | { readonly kind: 'sleep-unattended'; readonly since: Instant }
+
+  // `closing` with no session seen: no respawn and no settle yet
+  | { readonly kind: 'await-spawn' }
+
+  // the session stopped with no sleep of boxxy's in progress, or after the mark
+  | { readonly kind: 'sleep'; readonly requestWake: boolean }
+  | { readonly kind: 'request-wake'; readonly by: WakeSource }
+  | { readonly kind: 'wake-over-cap' }
+
+  // asks the worker to resume the woken session again
+  | { readonly kind: 'resume' }
+  | {
+      readonly kind: 'close';
+      readonly cause: 'timed-out';
+      readonly afterMinutes: number;
+    }
+  | { readonly kind: 'close'; readonly cause: 'turn-ended' | 'stalled' }
+  | {
+      readonly kind: 'close';
+      readonly cause: 'usage-limit';
+      readonly resetsAt: Instant;
+    }
+  | { readonly kind: 'stop' }
+  | {
+      readonly kind: 'settle';
+      readonly forced: ForcedFailure | null;
+      readonly sessionStopped: boolean;
+    };
+
+export interface ReadingInput<Reading extends SessionReading = SessionReading> {
+  readonly attempt: AttemptRow;
+  readonly reading: Reading;
+
+  // the attempt's step in the item's pipeline; null when the rules lost the phase
+  readonly step: PipelineStep | null;
+  readonly sleep: SleepRules;
+  readonly now: Instant;
+}
+
+type ReadingRule<Kind extends SessionReading['kind']> = (
+  input: ReadingInput<Extract<SessionReading, { kind: Kind }>>,
+) => ReadingAction;
+
+// One rule for each reading and each live state, so the type check proves every cell is there.
+export type ReadingTable = {
+  readonly [Kind in SessionReading['kind']]: Readonly<Record<LiveAttemptState, ReadingRule<Kind>>>;
+};
+
+export interface ActOnReadingInput {
+  readonly store: ClaimStore;
+  readonly worker: WorkerKind;
+  readonly rules: LoadedRules;
+  readonly backoff: CapacityBackoff;
+
+  // the attempt as the runner listed it before the reading
+  readonly attempt: AttemptRow;
+  readonly reading: SessionReading;
+  readonly action: ReadingAction;
+  readonly now: Instant;
+  readonly newId: NewId;
+}
+
+// One bound of a running attempt whose session runs. The first bound that gives an action wins.
+export interface RunningBound {
+  readonly name: string;
+  readonly pick: (
+    input: ReadingInput<Extract<SessionReading, { kind: 'running' }>>,
+  ) => ReadingAction | null;
+}

package/src/sessions/build-keep-writes.ts

@@ -0,0 +1,55 @@
+import type { AttemptRow, Instant, ItemView, LoadedRules, RowWrite } from '@boxxy/model';
+import { spawnRequestSchema } from '@boxxy/model';
+
+interface KeepInput {
+  // the attempt that settled `succeeded` with its session stopped
+  readonly attempt: AttemptRow;
+  readonly item: ItemView;
+  readonly rules: LoadedRules;
+  readonly now: Instant;
+}
+
+// The write that records the attempt's session as the item's kept session, when an edge of the
+// item's pipeline comes back to the attempt's phase with `resume: true`; else none. Pure.
+export function buildKeepWrites(input: KeepInput): RowWrite[] {
+  const attempt = input.attempt;
+  const item = input.item;
+
+  if (attempt.atcSession === null) {
+    return [];
+  }
+
+  const steps = input.rules.kinds.get(item.kind)?.pipeline.steps ?? [];
+
+  const isResumeEdge = steps.some((step) =>
+    Object.values(step.on).some(
+      (edge) =>
+        typeof edge === 'object' &&
+        'to' in edge &&
+        edge.to === attempt.phase &&
+        edge.resume === true,
+    ),
+  );
+
+  if (!isResumeEdge) {
+    return [];
+  }
+
+  return [
+    {
+      op: 'update',
+      table: 'items',
+      key: { id: item.id },
+      expect: { keptSession: item.keptSession },
+      set: {
+        keptSession: {
+          atcSession: attempt.atcSession,
+          target: spawnRequestSchema.parse(attempt.spawnRequest).target,
+          attempt: attempt.id,
+          keptAt: input.now,
+          caller: attempt.caller,
+        },
+      },
+    },
+  ];
+}

package/src/sessions/build-kept-clear.ts

@@ -0,0 +1,12 @@
+import type { ItemId, KeptSession, RowWrite } from '@boxxy/model';
+
+// The write that clears the item's kept session, guarded by the value it was read as.
+export function buildKeptClear(item: ItemId, kept: KeptSession): RowWrite {
+  return {
+    op: 'update',
+    table: 'items',
+    key: { id: item },
+    expect: { keptSession: kept },
+    set: { keptSession: null },
+  };
+}

package/src/sessions/forget-sessions.ts

@@ -0,0 +1,106 @@
+import type { AtcSessionId, AttemptRow, ItemId, KeptSession } from '@boxxy/model';
+import { brandSchemas } from '@boxxy/model';
+import { writeAttemptMove } from '../attempts/write-attempt-move.ts';
+import { timeouts } from '../timeouts.ts';
+import { buildKeptClear } from './build-kept-clear.ts';
+import { isKeptDue } from './is-kept-due.ts';
+import type { ForgetReport, ForgetSessionsInput } from './types.ts';
+
+const hourMs = 3_600_000;
+
+// The cleanup of one runner tick: the sessions of settled attempts that are not kept, and the
+// kept sessions that are due. A forget the worker does not confirm changes nothing and is tried
+// again on the next tick.
+export async function forgetSessions(input: ForgetSessionsInput): Promise<ForgetReport> {
+  const cutoff = brandSchemas.instant.parse(
+    Math.max(0, input.now - timeouts.failedSessionHours * hourMs),
+  );
+
+  const rows = await input.store.read(async (tx) => ({
+    kept: await tx.listKeptSessions(),
+    due: await tx.listSessionsToForget(cutoff),
+  }));
+
+  const kept = rows.kept;
+  const due = rows.due;
+  const forgotten: AtcSessionId[] = [];
+  const failed: AtcSessionId[] = [];
+
+  for (const entry of kept) {
+    const isDue = await input.store.read(async (tx) =>
+      isKeptDue({
+        kept: entry.kept,
+        item: await tx.readItem(entry.item, input.now, input.rules),
+        latest: await tx.findAttemptBySession(entry.kept.atcSession),
+        now: input.now,
+      }),
+    );
+
+    if (isDue) {
+      const done = await forgetKept(input, entry.item, entry.kept);
+
+      (done ? forgotten : failed).push(entry.kept.atcSession);
+    }
+  }
+
+  for (const attempt of due) {
+    const done = await forgetAttempt(input, attempt);
+
+    if (attempt.atcSession !== null) {
+      (done ? forgotten : failed).push(attempt.atcSession);
+    }
+  }
+
+  return { forgotten, failed };
+}
+
+async function forgetKept(
+  input: ForgetSessionsInput,
+  item: ItemId,
+  kept: KeptSession,
+): Promise<boolean> {
+  const outcome = await input.worker.forget(kept.atcSession);
+
+  if (outcome.kind !== 'forgotten' && outcome.kind !== 'absent') {
+    return false;
+  }
+
+  await input.store.exclusive(async (tx) => {
+    const moved = await writeAttemptMove(
+      tx,
+      kept.attempt,
+      { kind: 'session-forgotten' },
+      { ...buildBase(input), also: [buildKeptClear(item, kept)] },
+    );
+
+    if (moved.kind !== 'applied') {
+      await tx.applyChange({
+        at: input.now,
+        writes: [buildKeptClear(item, kept)],
+        facts: [],
+      });
+    }
+  });
+
+  return true;
+}
+
+function buildBase(input: ForgetSessionsInput) {
+  return { actor: input.actor, now: input.now, newId: input.newId };
+}
+
+async function forgetAttempt(input: ForgetSessionsInput, attempt: AttemptRow): Promise<boolean> {
+  if (attempt.atcSession !== null) {
+    const outcome = await input.worker.forget(attempt.atcSession);
+
+    if (outcome.kind !== 'forgotten' && outcome.kind !== 'absent') {
+      return false;
+    }
+  }
+
+  await input.store.exclusive((tx) =>
+    writeAttemptMove(tx, attempt.id, { kind: 'session-forgotten' }, buildBase(input)),
+  );
+
+  return true;
+}

package/src/sessions/is-kept-due.ts

@@ -0,0 +1,27 @@
+import { timeouts } from '../timeouts.ts';
+import type { KeptForgetInput } from './types.ts';
+
+const hourMs = 3_600_000;
+
+// Whether the kept session is due to be forgotten: its item is done or closed, or it was kept
+// `keptSessionHours` ago. A session that a live later attempt runs on is never due. Pure.
+export function isKeptDue(input: KeptForgetInput): boolean {
+  if (
+    input.latest !== null &&
+    input.latest.id !== input.kept.attempt &&
+    input.latest.endedAt === null
+  ) {
+    return false;
+  }
+
+  const item = input.item;
+  const expired = input.now - input.kept.keptAt >= timeouts.keptSessionHours * hourMs;
+
+  return (
+    item === null ||
+    item.closedAt !== null ||
+    item.derived.state === 'done' ||
+    item.derived.state === 'archived' ||
+    expired
+  );
+}

package/src/sessions/resume-kept-session.ts

@@ -0,0 +1,25 @@
+import type { KeptSession, SpawnRequest, StartOutcome, WorkerKind } from '@boxxy/model';
+
+export interface ResumedSession {
+  readonly outcome: StartOutcome;
+
+  // true when the wake failed and the outcome is that of a fresh spawn
+  readonly resumeFallback: boolean;
+}
+
+// Wakes the kept session with the new attempt's prompt as a message. A session that is gone or
+// refuses the wake is replaced by a fresh spawn. A refusal for capacity or an unanswered wake is
+// not: a spawn would meet the same capacity, and the session may be running.
+export async function resumeKeptSession(
+  worker: WorkerKind,
+  kept: KeptSession,
+  spawn: SpawnRequest,
+): Promise<ResumedSession> {
+  const outcome = await worker.resumeKept(kept, spawn);
+
+  if (outcome.kind === 'refused' && outcome.cause === 'other') {
+    return { outcome: await worker.start(spawn), resumeFallback: true };
+  }
+
+  return { outcome, resumeFallback: false };
+}

package/src/sessions/types.ts

@@ -0,0 +1,39 @@
+import type {
+  ActorRef,
+  AtcSessionId,
+  AttemptRow,
+  ClaimStore,
+  Instant,
+  ItemView,
+  KeptSession,
+  LoadedRules,
+  NewId,
+  WorkerKind,
+} from '@boxxy/model';
+
+export interface ForgetSessionsInput {
+  readonly store: ClaimStore;
+  readonly worker: WorkerKind;
+  readonly rules: LoadedRules;
+  readonly actor: ActorRef;
+  readonly now: Instant;
+  readonly newId: NewId;
+}
+
+// The sessions this tick forgot, and those whose forget the worker did not confirm. A failed
+// forget is tried again on the next tick.
+export interface ForgetReport {
+  readonly forgotten: readonly AtcSessionId[];
+  readonly failed: readonly AtcSessionId[];
+}
+
+export interface KeptForgetInput {
+  readonly kept: KeptSession;
+
+  // null when the store has no such item
+  readonly item: ItemView | null;
+
+  // the latest attempt that the kept session carries
+  readonly latest: AttemptRow | null;
+  readonly now: Instant;
+}