@boxxy/dispatch 0.1.1 → 0.1.3

package/src/pull-requests/watch-pull-requests.ts

@@ -0,0 +1,308 @@
+import type {
+  ActorRef,
+  AnyNewFact,
+  AttemptId,
+  AttemptRow,
+  PullRequestReading,
+  PullRequestRef,
+  PullRequestRow,
+  RepoName,
+  RowWrite,
+  StoreTx,
+} from '@boxxy/model';
+import { brandSchemas, buildPhase } from '@boxxy/model';
+import { buildLinkChange } from './build-link-change.ts';
+import { buildStateFact } from './build-state-fact.ts';
+import { collectAttemptTrailers } from './collect-attempt-trailers.ts';
+import { collectSeenState } from './collect-seen-state.ts';
+import { watchLimits } from './constants.ts';
+import type { WatchPullRequestsInput, WatchReport } from './types.ts';
+
+interface Tally {
+  linked: number;
+  commitsLinked: number;
+  merged: number;
+  closed: number;
+  unanswered: number;
+}
+
+// One pass of the watch. It links the pull request of each succeeded build that the hook missed,
+// then reads each open pull request to link its commits and record its merge or close. A forge
+// call that throws counts in `unanswered` and writes nothing; the next pass tries again.
+export async function watchPullRequests(input: WatchPullRequestsInput): Promise<WatchReport> {
+  const tally: Tally = { linked: 0, commitsLinked: 0, merged: 0, closed: 0, unanswered: 0 };
+
+  await linkMissed(input, tally);
+
+  const watched = await input.store.read((reader) => reader.listWatchedPullRequests());
+
+  for (const row of watched) {
+    await watchOne(input, row, tally);
+  }
+
+  return tally;
+}
+
+async function linkMissed(input: WatchPullRequestsInput, tally: Tally): Promise<void> {
+  const byRepo = await input.store.read(async (reader) => {
+    const succeeded = await reader.listSucceededAttemptsWithoutPullRequest();
+    const watched = await reader.listWatchedPullRequests();
+
+    const busyItems = new Set<string>();
+
+    for (const row of watched) {
+      const owner = await reader.readAttempt(row.attempt);
+
+      if (owner !== null) {
+        busyItems.add(owner.item);
+      }
+    }
+
+    const candidates = succeeded.filter(
+      (attempt) =>
+        attempt.phase === buildPhase &&
+        !busyItems.has(attempt.item) &&
+        input.now - (attempt.endedAt ?? attempt.claimedAt) <= watchLimits.linkWindowMs,
+    );
+
+    const grouped = new Map<RepoName, AttemptRow[]>();
+
+    for (const attempt of candidates) {
+      const item = await reader.readItem(attempt.item, input.now, input.rules);
+
+      if (item !== null) {
+        grouped.set(item.repo, [...(grouped.get(item.repo) ?? []), attempt]);
+      }
+    }
+
+    return grouped;
+  });
+
+  for (const [repo, attempts] of byRepo) {
+    await linkRepo(input, tally, repo, attempts);
+  }
+}
+
+async function linkRepo(
+  input: WatchPullRequestsInput,
+  tally: Tally,
+  repo: RepoName,
+  attempts: readonly AttemptRow[],
+): Promise<void> {
+  const open = await askForge(tally, () => input.forge.listOpenPullRequests(repo));
+
+  if (open === null) {
+    return;
+  }
+
+  const readings = new Map<number, PullRequestReading | null>();
+  const taken = new Set<number>();
+
+  for (const attempt of attempts) {
+    for (const ref of open.filter((candidate) => !taken.has(candidate.number))) {
+      if (!readings.has(ref.number)) {
+        const fresh = await readOpen(input, tally, ref);
+
+        readings.set(ref.number, fresh);
+      }
+
+      const reading = readings.get(ref.number) ?? null;
+
+      if (reading === null || !isMadeBy(reading, attempt.id)) {
+        continue;
+      }
+
+      taken.add(ref.number);
+
+      const isLinked = await writeLink(input, attempt, ref, reading);
+
+      tally.linked += isLinked ? 1 : 0;
+      break;
+    }
+  }
+}
+
+async function askForge<Answer>(tally: Tally, call: () => Promise<Answer>): Promise<Answer | null> {
+  try {
+    return await call();
+  } catch {
+    tally.unanswered += 1;
+
+    return null;
+  }
+}
+
+async function readOpen(
+  input: WatchPullRequestsInput,
+  tally: Tally,
+  ref: PullRequestRef,
+): Promise<PullRequestReading | null> {
+  const reading = await askForge(tally, () => input.forge.readPullRequest(ref));
+
+  return reading === null || reading === 'not-found' ? null : reading;
+}
+
+function isMadeBy(reading: PullRequestReading, attempt: AttemptId): boolean {
+  return reading.commits.some((commit) => collectAttemptTrailers(commit.message).includes(attempt));
+}
+
+function writeLink(
+  input: WatchPullRequestsInput,
+  attempt: AttemptRow,
+  ref: PullRequestRef,
+  reading: PullRequestReading,
+): Promise<boolean> {
+  return input.store.exclusive(async (tx) => {
+    const existing = await tx.readPullRequestRow(ref);
+
+    if (existing !== null) {
+      return false;
+    }
+
+    const result = await tx.applyChange(
+      buildLinkChange({
+        attempt,
+        ref,
+        reading,
+        source: 'watch',
+        actor: `boxxy:runner:${input.runner}`,
+        now: input.now,
+        newId: input.newId,
+      }),
+    );
+
+    return result.kind === 'applied';
+  });
+}
+
+async function watchOne(
+  input: WatchPullRequestsInput,
+  row: PullRequestRow,
+  tally: Tally,
+): Promise<void> {
+  const ref = { repo: row.repo, number: row.number };
+
+  const reading = await askForge(tally, () => input.forge.readPullRequest(ref));
+
+  if (reading === null) {
+    return;
+  }
+
+  await input.store.exclusive(async (tx) => {
+    if (reading !== 'not-found') {
+      await recordReading({ input, tx, row, reading, tally });
+    }
+
+    await tx.markPullRequestChecked(ref, input.now);
+  });
+}
+
+interface RecordInput {
+  readonly input: WatchPullRequestsInput;
+  readonly tx: StoreTx;
+  readonly row: PullRequestRow;
+  readonly reading: PullRequestReading;
+  readonly tally: Tally;
+}
+
+async function recordReading(record: RecordInput): Promise<void> {
+  const input = record.input;
+  const tx = record.tx;
+  const row = record.row;
+
+  const owner = await tx.readAttempt(row.attempt);
+
+  if (owner === null) {
+    return;
+  }
+
+  const actor: ActorRef = `boxxy:runner:${input.runner}`;
+  const entity = `boxxy:item:${owner.item}` as const;
+  const writes: RowWrite[] = [];
+  const facts: AnyNewFact[] = [];
+
+  const handled = new Set<string>();
+
+  for (const commit of record.reading.commits) {
+    for (const id of collectAttemptTrailers(commit.message)) {
+      const key = `${id} ${commit.sha}`;
+
+      if (handled.has(key)) {
+        continue;
+      }
+
+      handled.add(key);
+
+      const trailerAttempt = await tx.readAttempt(id);
+      const evidence = await tx.listEvidence(id);
+
+      const isNew =
+        trailerAttempt?.item === owner.item &&
+        !evidence.some((entry) => entry.kind === 'commit' && entry.ref === commit.sha);
+
+      if (!isNew) {
+        continue;
+      }
+
+      const factId = brandSchemas.factId.parse(input.newId());
+
+      writes.push({
+        op: 'insert',
+        table: 'evidence',
+        row: {
+          factId,
+          attempt: id,
+          kind: 'commit',
+          ref: commit.sha,
+          observed: null,
+          source: 'trailer',
+          at: input.now,
+        },
+      });
+
+      facts.push({
+        id: factId,
+        type: 'boxxy.commit.linked',
+        entity,
+        actor,
+        payload: { attempt: id, repo: row.repo, sha: commit.sha, pullRequestNumber: row.number },
+      });
+    }
+  }
+
+  const commitCount = facts.length;
+  const seenState = collectSeenState(record.reading);
+
+  if (seenState.state !== 'open') {
+    const stateFact = buildStateFact({
+      row: { ...row, ...seenState },
+      item: owner.item,
+      actor,
+      id: brandSchemas.factId.parse(input.newId()),
+    });
+
+    writes.push({
+      op: 'update',
+      table: 'pull_requests',
+      key: { repo: row.repo, number: row.number },
+      expect: { state: 'open' },
+      set: { ...seenState, checkedAt: input.now },
+    });
+
+    if (stateFact !== null) {
+      facts.push(stateFact);
+    }
+  }
+
+  if (writes.length === 0) {
+    return;
+  }
+
+  const result = await tx.applyChange({ at: input.now, writes, facts });
+
+  if (result.kind === 'applied') {
+    record.tally.commitsLinked += commitCount;
+    record.tally.merged += seenState.state === 'merged' ? 1 : 0;
+    record.tally.closed += seenState.state === 'closed' ? 1 : 0;
+  }
+}

package/src/readings/act-on-reading.ts

@@ -0,0 +1,318 @@
+import type {
+  ActorRef,
+  AttemptRow,
+  Instant,
+  SessionFault,
+  SessionReading,
+  StoreTx,
+} from '@boxxy/model';
+import { brandSchemas, isFilterMatch, spawnRequestSchema } from '@boxxy/model';
+import invariant from 'tiny-invariant';
+import { match } from 'ts-pattern';
+import { isSameGuard } from '../attempts/is-same-guard.ts';
+import type { MoveBase, MoveEvent } from '../attempts/types.ts';
+import { writeAttemptMove } from '../attempts/write-attempt-move.ts';
+import { writeSessionBind } from '../attempts/write-session-bind.ts';
+import { buildSnapshotHash } from '../claims/build-snapshot-hash.ts';
+import { resumeClaim } from '../claims/resume-claim.ts';
+import { startClaim } from '../claims/start-claim.ts';
+import { settleAttempt } from '../closeouts/settle-attempt.ts';
+import { timeouts } from '../timeouts.ts';
+import type { ActOnReadingInput, ReadingAction } from './types.ts';
+
+// Carries out the reconcile table's action for one reading, after it binds a new agent session
+// that the reading shows. An action whose row moved on since the runner read it does nothing.
+// Returns the fault of an agent session that another attempt holds, or null.
+export async function actOnReading(input: ActOnReadingInput): Promise<SessionFault | null> {
+  const base: MoveBase = {
+    actor: `boxxy:runner:${input.attempt.runner}` satisfies ActorRef,
+    now: input.now,
+    newId: input.newId,
+  };
+
+  const fault = await writeReadingBind(input, base);
+
+  await runAction(input, input.action, base);
+
+  return fault;
+}
+
+function writeReadingBind(input: ActOnReadingInput, base: MoveBase): Promise<SessionFault | null> {
+  const reading = input.reading;
+
+  if (reading.kind === 'absent' || reading.agentSession === null) {
+    return Promise.resolve(null);
+  }
+
+  const agentSession = reading.agentSession;
+
+  return input.store.exclusive(async (tx) => {
+    const row = await tx.readAttempt(input.attempt.id);
+
+    invariant(row !== null, `the runner read a stored attempt: ${input.attempt.id}`);
+
+    return writeSessionBind(tx, row, { atcSession: reading.atcSession, agentSession }, base);
+  });
+}
+
+const hourMs = 3_600_000;
+
+function runAction(input: ActOnReadingInput, action: ReadingAction, base: MoveBase) {
+  const id = input.attempt.id;
+  const seen = { ...base, seen: input.attempt };
+
+  const writeMove = (event: MoveEvent) =>
+    input.store.exclusive((tx) => writeAttemptMove(tx, id, event, seen));
+
+  return match(action)
+    .with({ kind: 'nothing' }, { kind: 'bind-only' }, { kind: 'await-spawn' }, async () => {})
+    .with({ kind: 'move-to-running', via: 'reconcile' }, async () => {
+      const running = getRunningReading(input.reading);
+
+      await writeMove({
+        kind: 'started',
+        via: 'reconcile',
+        atcSession: running.atcSession,
+        caller: running.caller,
+        resumeFallback: false,
+      });
+    })
+    .with({ kind: 'move-to-running', via: 'resume' }, async () => {
+      const running = getRunningReading(input.reading);
+
+      await writeMove({ kind: 'resumed', atcSession: running.atcSession, caller: running.caller });
+    })
+    .with({ kind: 'respawn' }, async () => {
+      if (!(await probeSeenRow(input))) {
+        return;
+      }
+
+      await startClaim({
+        ...input,
+        claim: {
+          kind: 'start',
+          attempt: input.attempt,
+          spawn: spawnRequestSchema.parse(input.attempt.spawnRequest),
+          kept: null,
+        },
+      });
+    })
+    .with({ kind: 'clear-mark' }, async () => {
+      await writeMove({ kind: 'suspend-refused' });
+    })
+    .with({ kind: 'sleep-if-idle' }, (sleep) =>
+      runSleepSteps(input, base, {
+        ifIdleSince: sleep.waitingSince,
+        idleSince: sleep.waitingSince,
+      }),
+    )
+    .with({ kind: 'sleep-unattended' }, (sleep) =>
+      runSleepSteps(input, base, { ifIdleSince: null, idleSince: sleep.since }),
+    )
+    .with({ kind: 'sleep' }, (sleep) =>
+      input.store.exclusive(async (tx) => {
+        const moved = await writeAttemptMove(
+          tx,
+          id,
+          {
+            kind: 'slept',
+            idleSince: input.attempt.suspendRequestedAt ?? input.now,
+            asked: !sleep.requestWake,
+          },
+          seen,
+        );
+
+        // the host slept with no request, so the attempt wakes by itself
+        if (moved.kind === 'applied' && sleep.requestWake) {
+          await writeAttemptMove(
+            tx,
+            id,
+            { kind: 'wake-requested', by: 'host-slept-unasked' },
+            base,
+          );
+        }
+      }),
+    )
+    .with({ kind: 'request-wake' }, async (wake) => {
+      await writeMove({ kind: 'wake-requested', by: wake.by });
+    })
+    .with({ kind: 'wake-over-cap' }, () =>
+      input.store.exclusive((tx) => writeWakeOverCap(tx, input, base)),
+    )
+    .with({ kind: 'resume' }, async () => {
+      if (!(await probeSeenRow(input))) {
+        return;
+      }
+
+      await resumeClaim({ ...input, claim: { kind: 'wake', attempt: input.attempt } });
+    })
+    .with({ kind: 'close', cause: 'timed-out' }, async (close) => {
+      await writeMove({ kind: 'timed-out', afterMinutes: close.afterMinutes });
+    })
+    .with({ kind: 'close', cause: 'turn-ended' }, async () => {
+      await writeMove({ kind: 'turn-ended' });
+    })
+    .with({ kind: 'close', cause: 'stalled' }, async () => {
+      await writeMove({ kind: 'stalled' });
+    })
+    .with({ kind: 'close', cause: 'usage-limit' }, async (close) => {
+      const latest = input.now + timeouts.usageLimitBackoffHours * hourMs;
+
+      // the worker adapter sends the time it saw the limit when atc gives no reset time
+      const until = close.resetsAt > input.now ? Math.min(close.resetsAt, latest) : latest;
+
+      const moved = await writeMove({ kind: 'usage-limited' });
+
+      if (moved.kind === 'applied') {
+        input.backoff.hold(input.attempt.pool, brandSchemas.instant.parse(until));
+      }
+    })
+    .with({ kind: 'stop' }, () => runStop(input, base))
+    .with({ kind: 'settle' }, async (settle) => {
+      await settleAttempt({
+        ...base,
+        store: input.store,
+        attempt: id,
+        forced: settle.forced,
+        sessionStopped: settle.sessionStopped,
+        seen: input.attempt,
+        rules: input.rules,
+      });
+    })
+    .exhaustive();
+}
+
+function getRunningReading(reading: SessionReading) {
+  invariant(
+    reading.kind === 'running',
+    'the table moves an attempt to running on a running reading',
+  );
+
+  return reading;
+}
+
+// The worker call of a respawn or a resume runs outside the lock, so it checks the row first: a
+// row that moved since the reading makes the call wait for the next tick.
+async function probeSeenRow(input: ActOnReadingInput): Promise<boolean> {
+  const row = await readRow(input);
+
+  return isSameGuard(input.attempt, row);
+}
+
+function readRow(input: ActOnReadingInput): Promise<AttemptRow> {
+  return input.store.read(async (tx) => {
+    const row = await tx.readAttempt(input.attempt.id);
+
+    invariant(row !== null, `the runner read a stored attempt: ${input.attempt.id}`);
+
+    return row;
+  });
+}
+
+interface SleepTimes {
+  // null: suspend with no idle check
+  readonly ifIdleSince: Instant | null;
+  readonly idleSince: Instant;
+}
+
+// The mark, the suspend outside the lock, then `slept`, which releases the seats. A crash, or an
+// unknown answer, leaves the mark, and the next reading clears it or sleeps the attempt.
+async function runSleepSteps(
+  input: ActOnReadingInput,
+  base: MoveBase,
+  times: SleepTimes,
+): Promise<void> {
+  const id = input.attempt.id;
+
+  const marked = await input.store.exclusive((tx) =>
+    writeAttemptMove(tx, id, { kind: 'suspend-asked' }, { ...base, seen: input.attempt }),
+  );
+
+  if (marked.kind !== 'applied') {
+    return;
+  }
+
+  const row = await readRow(input);
+  const outcome = await input.worker.suspend(row, times.ifIdleSince);
+
+  if (outcome.kind === 'unknown') {
+    return;
+  }
+
+  const event: MoveEvent =
+    outcome.kind === 'suspended'
+      ? { kind: 'slept', idleSince: times.idleSince, asked: true }
+      : { kind: 'suspend-refused' };
+
+  await input.store.exclusive((tx) => writeAttemptMove(tx, id, event, { ...base, seen: row }));
+}
+
+// The owner revived the session and no seat came free in time: the attempt takes its seats past
+// the cap, with no resume, because the session runs already. The decision row records each
+// limit's count and max at that moment.
+async function writeWakeOverCap(tx: StoreTx, input: ActOnReadingInput, base: MoveBase) {
+  const row = await tx.readAttempt(input.attempt.id);
+
+  invariant(row !== null, `the runner read a stored attempt: ${input.attempt.id}`);
+
+  const limits = await tx.listLimits();
+
+  const counting = limits.filter((limit) => isFilterMatch(limit.counts, row.fields));
+
+  const snapshot = await tx.readSnapshot(input.now, input.rules);
+
+  return writeAttemptMove(
+    tx,
+    row.id,
+    {
+      kind: 'woken',
+      pick: { kind: 'wake', attempt: row.id, limits: counting.map((limit) => limit.name) },
+      decision: {
+        item: row.item,
+        kind: 'wake',
+        pool: row.pool,
+        rulesVersion: input.rules.version,
+        snapshotHash: buildSnapshotHash(snapshot, input.backoff.listHeld(input.now)),
+        storeVersion: snapshot.storeVersion,
+        pepysOffsets: {},
+        faults: [],
+        limits: counting.map((limit) => ({ limit: limit.name, held: limit.held, max: limit.max })),
+      },
+      overCap: true,
+    },
+    { ...base, seen: input.attempt },
+  );
+}
+
+// A closing attempt whose spawn landed after its item closed records the session first, so the
+// stop reaches it. The stop's answer moves nothing: the next reading settles a stopped session.
+async function runStop(input: ActOnReadingInput, base: MoveBase): Promise<void> {
+  const reading = input.reading;
+
+  if (input.attempt.atcSession === null && reading.kind === 'running') {
+    const moved = await input.store.exclusive((tx) =>
+      writeAttemptMove(
+        tx,
+        input.attempt.id,
+        {
+          kind: 'started',
+          via: 'start',
+          atcSession: reading.atcSession,
+          caller: reading.caller,
+          resumeFallback: false,
+        },
+        { ...base, seen: input.attempt },
+      ),
+    );
+
+    if (moved.kind !== 'applied') {
+      return;
+    }
+  } else if (!(await probeSeenRow(input))) {
+    return;
+  }
+
+  const row = await readRow(input);
+
+  await input.worker.stop(row);
+}

package/src/readings/find-waiting-since.ts

@@ -0,0 +1,46 @@
+import type { AttemptRow, Instant, SessionReading } from '@boxxy/model';
+
+const minuteMs = 60_000;
+
+interface WaitingInput {
+  readonly attempt: AttemptRow;
+  readonly reading: SessionReading;
+
+  // the rules' `sleep.afterQuietMinutes`
+  readonly quietAfterMinutes: number;
+  readonly now: Instant;
+}
+
+// Since when the session has waited for the owner, from the reading alone, or null. Pure.
+export function findWaitingSince(input: WaitingInput): Instant | null {
+  const reading = input.reading;
+
+  if (reading.kind !== 'running') {
+    return null;
+  }
+
+  if (reading.attention === 'needs-input') {
+    return reading.lastActivityAt;
+  }
+
+  if (reading.attention === 'turn-ended' && input.attempt.mode === 'attended') {
+    return reading.lastActivityAt;
+  }
+
+  // atc lists a revived session as running, so a woken session that did nothing since waits
+  const wokenAt = input.attempt.wokenAt;
+
+  if (wokenAt !== null && reading.lastActivityAt <= wokenAt) {
+    return wokenAt;
+  }
+
+  // the quiet rule: an attach moves atc's state to running, so a prompt the owner read and left
+  // reads as working with no later activity
+  const quietMs = input.now - reading.lastActivityAt;
+
+  if (input.attempt.mode === 'attended' && quietMs >= input.quietAfterMinutes * minuteMs) {
+    return reading.lastActivityAt;
+  }
+
+  return null;
+}