@boxxy/server 0.0.1 → 0.1.4

package/src/tick/run-tick.ts

@@ -0,0 +1,212 @@
+import {
+  actOnReading,
+  claimNext,
+  forgetSessions,
+  pickReadingAction,
+  resumeClaim,
+  sendNotices,
+  startClaim,
+  stopStraySession,
+  timeouts,
+} from '@boxxy/dispatch';
+import type { NoticeReport, SpawnSettings } from '@boxxy/dispatch';
+import type {
+  AttemptId,
+  AttemptRow,
+  CapacityBackoff,
+  ClaimStore,
+  Instant,
+  ItemId,
+  LoadedRules,
+  NewId,
+  Notifier,
+  PipelineStep,
+  RunnerId,
+  SessionFault,
+  SessionReading,
+  WorkerKind,
+} from '@boxxy/model';
+import { brandSchemas } from '@boxxy/model';
+import type { ServerLog } from '../types.ts';
+
+export interface TickDeps {
+  readonly store: ClaimStore;
+  readonly runner: RunnerId;
+  readonly worker: WorkerKind;
+  readonly rules: LoadedRules;
+  readonly backoff: CapacityBackoff;
+  readonly spawn: SpawnSettings;
+  readonly notifier: Notifier;
+
+  // one value for the whole tick
+  readonly now: Instant;
+  readonly newId: NewId;
+
+  // the last reading of each live attempt; the tick replaces it whole
+  readonly readings: Map<AttemptId, SessionReading>;
+  readonly log: ServerLog;
+}
+
+export interface TickReport {
+  readonly claimed: number;
+  readonly read: number;
+  readonly strays: number;
+  readonly forgotten: number;
+  readonly notices: NoticeReport | null;
+}
+
+// One tick of the runner. Each part after the heartbeat runs even when an earlier part threw, so
+// a fault in the claim loop never holds the readings back.
+export async function runTick(deps: TickDeps): Promise<TickReport> {
+  await deps.store.exclusive(async (tx) => {
+    await tx.writeHeartbeat(deps.runner, deps.now);
+    await tx.syncLimits(deps.rules.limits, deps.now);
+  });
+
+  const claimed = await runClaimLoop(deps);
+  const read = await runReadings(deps);
+  const strays = await runStrayStops(deps);
+  const forgotten = await runForget(deps);
+  const notices = await runNotices(deps);
+
+  return { claimed, read, strays, forgotten, notices };
+}
+
+// The claim loop ends on three rules: `claimNext` gives no pick in a pool in backoff, no item is
+// claimed twice in one tick, and an error ends the loop and not the tick.
+async function runClaimLoop(deps: TickDeps): Promise<number> {
+  const claimed = new Set<ItemId>();
+
+  try {
+    for (let claim = await claimNext(deps); claim !== null; claim = await claimNext(deps)) {
+      const fault =
+        claim.kind === 'start'
+          ? await startClaim({ ...deps, claim })
+          : await resumeClaim({ ...deps, claim });
+
+      logFault(deps, fault);
+
+      if (claimed.has(claim.attempt.item)) {
+        deps.log.warn('the claim loop ended: an item was claimed twice in one tick', {
+          item: claim.attempt.item,
+        });
+
+        break;
+      }
+
+      claimed.add(claim.attempt.item);
+    }
+  } catch (error) {
+    deps.log.warn('the claim loop ended', { error: String(error) });
+  }
+
+  return claimed.size;
+}
+
+function logFault(deps: TickDeps, fault: SessionFault | null): void {
+  if (fault !== null) {
+    deps.log.warn('an agent session is bound to another attempt', {
+      attempt: fault.attempt,
+      holder: fault.holder,
+    });
+  }
+}
+
+// One reading per live attempt. A reading that throws skips its attempt; the next tick reads it
+// again.
+async function runReadings(deps: TickDeps): Promise<number> {
+  const live = await deps.store.read((tx) => tx.listLiveAttempts());
+
+  const readings = new Map<AttemptId, SessionReading>();
+
+  for (const attempt of live) {
+    try {
+      const reading = await deps.worker.reconcile(attempt);
+      const step = await readAttemptStep(deps, attempt);
+
+      readings.set(attempt.id, reading);
+
+      const action = pickReadingAction({
+        attempt,
+        reading,
+        step,
+        sleep: deps.rules.sleep,
+        now: deps.now,
+      });
+
+      const fault = await actOnReading({ ...deps, attempt, reading, action });
+
+      logFault(deps, fault);
+    } catch (error) {
+      deps.log.warn('the reading of an attempt failed', {
+        attempt: attempt.id,
+        error: String(error),
+      });
+    }
+  }
+
+  deps.readings.clear();
+
+  for (const [attempt, reading] of readings) {
+    deps.readings.set(attempt, reading);
+  }
+
+  return readings.size;
+}
+
+async function readAttemptStep(deps: TickDeps, attempt: AttemptRow): Promise<PipelineStep | null> {
+  const item = await deps.store.read((tx) => tx.readItem(attempt.item, deps.now, deps.rules));
+
+  const pipeline = item === null ? undefined : deps.rules.kinds.get(item.kind)?.pipeline;
+
+  return pipeline?.steps.find((step) => step.phase === attempt.phase) ?? null;
+}
+
+const hourMs = 3_600_000;
+
+// The stop of each session that the give-up rule left running, sent again on every tick for
+// `timeouts.strayHours`.
+async function runStrayStops(deps: TickDeps): Promise<number> {
+  const since = brandSchemas.instant.parse(Math.max(0, deps.now - timeouts.strayHours * hourMs));
+  let stopped = 0;
+
+  try {
+    const strays = await deps.store.read((tx) => tx.listStrayAttempts(since));
+
+    for (const attempt of strays) {
+      const result = await stopStraySession({ ...deps, attempt, actor: runnerActor(deps) });
+
+      stopped += result === null ? 0 : 1;
+    }
+  } catch (error) {
+    deps.log.warn('the stop of the stray sessions failed', { error: String(error) });
+  }
+
+  return stopped;
+}
+
+function runnerActor(deps: TickDeps) {
+  return `boxxy:runner:${deps.runner}` as const;
+}
+
+async function runForget(deps: TickDeps): Promise<number> {
+  try {
+    const report = await forgetSessions({ ...deps, actor: runnerActor(deps) });
+
+    return report.forgotten.length;
+  } catch (error) {
+    deps.log.warn('the forget of the done sessions failed', { error: String(error) });
+
+    return 0;
+  }
+}
+
+async function runNotices(deps: TickDeps): Promise<NoticeReport | null> {
+  try {
+    return await sendNotices(deps);
+  } catch (error) {
+    deps.log.warn('the send of the hold notices failed', { error: String(error) });
+
+    return null;
+  }
+}

package/src/tick/start-tick-loop.ts

@@ -0,0 +1,46 @@
+import { setTimeout as sleep } from 'node:timers/promises';
+import type { ServerLog } from '../types.ts';
+
+export interface TickLoop {
+  // ends the loop after the tick in flight
+  readonly stop: () => Promise<void>;
+}
+
+interface TickLoopInput {
+  readonly runOnce: () => Promise<void>;
+  readonly intervalMs: number;
+
+  // the log line for a tick that throws
+  readonly failureMessage: string;
+  readonly log: ServerLog;
+}
+
+// Runs one tick, waits `intervalMs`, and runs the next. A tick that throws is logged, and the
+// loop goes on; no two ticks overlap.
+export function startTickLoop(input: TickLoopInput): TickLoop {
+  const run = { isStopped: false, wake: new AbortController() };
+
+  const runLoop = async (): Promise<void> => {
+    while (!run.isStopped) {
+      try {
+        await input.runOnce();
+      } catch (error) {
+        input.log.warn(input.failureMessage, { error: String(error) });
+      }
+
+      await sleep(input.intervalMs, null, { signal: run.wake.signal }).catch(() => null);
+    }
+  };
+
+  const loop = runLoop();
+
+  return {
+    stop: async () => {
+      run.isStopped = true;
+
+      run.wake.abort();
+
+      await loop;
+    },
+  };
+}

package/src/types.ts

@@ -0,0 +1,86 @@
+import type {
+  AtcSessionId,
+  AttemptId,
+  CallerIdentity,
+  CallerProof,
+  ClaimStore,
+  Clock,
+  Forge,
+  LoadRejection,
+  LoadResult,
+  LoadedRules,
+  NewId,
+  OutboxStatus,
+  PepysReader,
+  RunnerId,
+  SessionReading,
+} from '@boxxy/model';
+
+// The rules the server runs, and the last checkout the loader refused, which the previous rules
+// outlive.
+export interface RulesState {
+  readonly rules: LoadedRules;
+  readonly lastRejected: {
+    readonly commit: string;
+    readonly rejections: readonly LoadRejection[];
+  } | null;
+}
+
+export interface OutboxControl {
+  readonly status: () => Promise<OutboxStatus>;
+
+  // asks the sender for a turn now, as after `outbox.retry`
+  readonly nudge: () => void;
+}
+
+// Failures and state changes only. Never a token, a header or a secret.
+export interface ServerLog {
+  readonly warn: (message: string, fields: Readonly<Record<string, unknown>>) => void;
+}
+
+// What every handler reads. The server builds it once; the tests build it on fakes.
+export interface ServerDeps {
+  readonly store: ClaimStore;
+  readonly runner: RunnerId;
+  readonly rulesState: () => RulesState;
+
+  // loads the server's own rules checkout with this text as its rules file, for the dry run
+  readonly loadProposedRules: (text: string) => LoadResult;
+  readonly clock: Clock;
+  readonly newId: NewId;
+  readonly callerProof: CallerProof;
+
+  // the SHA-256 digest of the worker token; the server never keeps the token itself
+  readonly workerTokenDigest: Uint8Array;
+
+  // null: the configuration has no pepys section
+  readonly pepys: PepysReader | null;
+
+  // null: the configuration has no github section, so the pull request link answers `retry`
+  readonly forge: Forge | null;
+  readonly outbox: OutboxControl;
+
+  // the last reading of each live attempt, which the tick writes and `runner.status` reads
+  readonly readings: ReadonlyMap<AttemptId, SessionReading>;
+  readonly log: ServerLog;
+}
+
+// The two listeners. `owner` is on the owner's private network; `worker` is the one the broker
+// reaches, and every request to it needs the worker token.
+export type Listener = 'owner' | 'worker';
+
+// `worker`: the request carries the worker token. `owner`: it carries none, on the owner listener.
+export type RequestScope = 'owner' | 'worker';
+
+export interface RequestContext {
+  readonly deps: ServerDeps;
+  readonly scope: RequestScope;
+  readonly request: Request;
+}
+
+// What the root middleware proved for a session or hook call: the caller, and the session whose
+// latest attempt holds it. null: no session holds the caller.
+export interface ProvenCall {
+  readonly session: AtcSessionId;
+  readonly caller: CallerIdentity;
+}