@boxxy/dispatch 0.1.0 → 0.1.3

package/src/outbox/run-sender-once.ts

@@ -0,0 +1,174 @@
+import { brandSchemas } from '@boxxy/model';
+import type { Instant, OutboxFact, SinkAnswer } from '@boxxy/model';
+import { match } from 'ts-pattern';
+import { buildBackoffMs } from './build-backoff-ms.ts';
+import { pickSenderMove } from './pick-sender-move.ts';
+import { takeBatch } from './take-batch.ts';
+import type { SenderDeps, SenderMove, SenderState, SenderTurn } from './types.ts';
+
+// One turn of the sender: sends one batch and does what the answer says. It takes the state that
+// the last turn returned. The caller waits until `nextTryAt` after `retry` and `blocked`, and
+// calls again at once after `stored`, `parked` and `split`.
+export async function runSenderOnce(deps: SenderDeps, state: SenderState): Promise<SenderTurn> {
+  const queue = state.queue.length > 0 ? state.queue : await readQueue(deps);
+
+  if (queue === 'parked') {
+    return { step: 'parked', state: { ...state, phase: 'idle', nextTryAt: null } };
+  }
+
+  const [head, ...rest] = queue;
+
+  if (head === undefined) {
+    return { step: 'idle', state: { ...state, queue: [], phase: 'idle', nextTryAt: null } };
+  }
+
+  const answer = await deps.sink.send(head);
+
+  const move = pickSenderMove({
+    answer,
+    batchSize: head.length,
+    parkRun: state.parkRun,
+    limits: deps.limits,
+  });
+
+  return runMove(deps, { state, batch: head, rest, answer, move, at: deps.clock.now() });
+}
+
+// The next batch from the store. An oversize first row is parked and no batch is sent this turn.
+async function readQueue(deps: SenderDeps): Promise<readonly (readonly OutboxFact[])[] | 'parked'> {
+  const rows = await deps.store.read((tx) => tx.readUnsentFacts(deps.limits.batchFacts));
+
+  const taken = takeBatch(rows, deps.limits);
+
+  if (taken.kind === 'batch') {
+    return taken.facts.length === 0 ? [] : [taken.facts];
+  }
+
+  const at = deps.clock.now();
+
+  await deps.store.exclusive((tx) => tx.parkFacts([taken.fact.factId], 'too-large', at));
+
+  return 'parked';
+}
+
+interface MoveInput {
+  readonly state: SenderState;
+  readonly batch: readonly OutboxFact[];
+  readonly rest: readonly (readonly OutboxFact[])[];
+  readonly answer: SinkAnswer;
+  readonly move: SenderMove;
+  readonly at: Instant;
+}
+
+function runMove(deps: SenderDeps, input: MoveInput): Promise<SenderTurn> {
+  const state = input.state;
+  const failingSince = state.failingSince ?? input.at;
+  const lastError = buildLastError(input.answer, input.move, input.at);
+
+  return match(input.move)
+    .with({ kind: 'mark-sent' }, async (): Promise<SenderTurn> => {
+      const ids = input.batch.map((fact) => fact.factId);
+
+      await deps.store.exclusive((tx) => tx.markFactsSent(ids, input.at));
+
+      return {
+        step: 'stored',
+        state: {
+          ...state,
+          queue: input.rest,
+          retries: 0,
+          parkRun: 0,
+          phase: 'idle',
+          failingSince: null,
+          nextTryAt: null,
+          lastStoredAt: input.at,
+        },
+      };
+    })
+    .with({ kind: 'retry-same' }, (): Promise<SenderTurn> => {
+      const retries = state.retries + 1;
+      const waitMs = buildBackoffMs(retries, deps.limits, deps.random());
+
+      return Promise.resolve({
+        step: 'retry',
+        state: {
+          ...state,
+          queue: [input.batch, ...input.rest],
+          retries,
+          phase: 'backing-off',
+          failingSince,
+          nextTryAt: brandSchemas.instant.parse(input.at + waitMs),
+          lastError,
+        },
+      });
+    })
+    .with({ kind: 'split' }, (): Promise<SenderTurn> => {
+      const middle = Math.ceil(input.batch.length / 2);
+
+      return Promise.resolve({
+        step: 'split',
+        state: {
+          ...state,
+          queue: [input.batch.slice(0, middle), input.batch.slice(middle), ...input.rest],
+          phase: 'idle',
+          failingSince,
+          nextTryAt: null,
+          lastError,
+        },
+      });
+    })
+    .with({ kind: 'park' }, async (parked): Promise<SenderTurn> => {
+      const ids = input.batch.map((fact) => fact.factId);
+
+      await deps.store.exclusive((tx) => tx.parkFacts(ids, parked.reason, input.at));
+
+      return {
+        step: 'parked',
+        state: {
+          ...state,
+          queue: input.rest,
+          parkRun: state.parkRun + 1,
+          phase: 'idle',
+          failingSince,
+          nextTryAt: null,
+          lastError,
+        },
+      };
+    })
+    .with({ kind: 'block' }, (): Promise<SenderTurn> =>
+      Promise.resolve({
+        step: 'blocked',
+        state: {
+          ...state,
+          queue: [input.batch, ...input.rest],
+          phase: 'blocked',
+          failingSince,
+          nextTryAt: brandSchemas.instant.parse(input.at + deps.limits.blockedRetryMs),
+          lastError,
+        },
+      }),
+    )
+    .exhaustive();
+}
+
+function buildLastError(
+  answer: SinkAnswer,
+  move: SenderMove,
+  at: Instant,
+): SenderState['lastError'] {
+  return match(answer)
+    .with({ kind: 'stored' }, () => null)
+    .with({ kind: 'retry' }, (retry) => ({
+      kind: retry.kind,
+      status: null,
+      message: retry.reason,
+      at,
+    }))
+    .with({ kind: 'refused-batch' }, { kind: 'refused-sink' }, (refused) => ({
+      kind: refused.kind,
+      status: refused.status,
+      message: move.kind === 'block' ? move.reason : refused.reason,
+      at,
+    }))
+    .exhaustive();
+}

package/src/outbox/take-batch.ts

@@ -0,0 +1,35 @@
+import type { OutboxFact } from '@boxxy/model';
+import { measureWireBytes } from './measure-wire-bytes.ts';
+import type { OutboxLimits, TakenBatch } from './types.ts';
+
+// The longest prefix of the rows that holds `batchFacts` facts or fewer and stays inside
+// `batchBytes`. A first row over `factBytes` comes back alone as `oversize`. The batch is empty
+// only when no row is given, because one fact and its allowance fit `batchBytes`.
+export function takeBatch(rows: readonly OutboxFact[], limits: OutboxLimits): TakenBatch {
+  const taken: OutboxFact[] = [];
+  let batchBytes = 0;
+
+  for (const row of rows) {
+    const wireBytes = measureWireBytes(row);
+
+    if (wireBytes > limits.factBytes) {
+      if (taken.length === 0) {
+        return { kind: 'oversize', fact: row };
+      }
+
+      break;
+    }
+
+    const rowBytes = wireBytes + limits.envelopeBytes + 8;
+
+    if (taken.length >= limits.batchFacts || batchBytes + rowBytes > limits.batchBytes) {
+      break;
+    }
+
+    taken.push(row);
+
+    batchBytes += rowBytes;
+  }
+
+  return { kind: 'batch', facts: taken };
+}

package/src/outbox/types.ts

@@ -0,0 +1,76 @@
+import type { ClaimStore, Clock, FactSink, Instant, OutboxFact, OutboxStatus } from '@boxxy/model';
+
+export interface OutboxLimits {
+  // ingest stores 100 facts of one batch at most
+  readonly batchFacts: 100;
+
+  // one fact on the wire: the UTF-8 bytes of its JSON
+  readonly factBytes: 262_144;
+
+  // the sum over a batch of (wire bytes + `envelopeBytes` + 8)
+  readonly batchBytes: 900_000;
+
+  // what the stored envelope adds to a wire item
+  readonly envelopeBytes: 512;
+  readonly requestTimeoutMs: 10_000;
+  readonly backoff: {
+    readonly firstMs: 1000;
+    readonly factor: 2;
+    readonly maxMs: 300_000;
+    readonly jitter: 0.2;
+  };
+  readonly blockedRetryMs: 300_000;
+
+  // this many rows parked in a row, with none stored between, reads as "pepys refuses everything"
+  readonly parkRun: 25;
+}
+
+export type TakenBatch =
+  | { readonly kind: 'batch'; readonly facts: readonly OutboxFact[] }
+  | { readonly kind: 'oversize'; readonly fact: OutboxFact };
+
+export type SenderMove =
+  | { readonly kind: 'mark-sent' }
+  | { readonly kind: 'retry-same' }
+  | { readonly kind: 'split' }
+  | { readonly kind: 'park'; readonly reason: string }
+  | { readonly kind: 'block'; readonly reason: string };
+
+export type SenderStep = 'idle' | 'stored' | 'retry' | 'split' | 'parked' | 'blocked';
+
+type SenderPhase = 'idle' | 'backing-off' | 'blocked';
+
+type LastError = NonNullable<OutboxStatus['lastError']>;
+
+// What the sender remembers between turns. A restart forgets all of it.
+export interface SenderState {
+  // batches to send before the next read of the store, in order: the one that got `retry`, or the
+  // halves of a split
+  readonly queue: readonly (readonly OutboxFact[])[];
+
+  // consecutive `retry` answers
+  readonly retries: number;
+
+  // rows parked since the last `stored`
+  readonly parkRun: number;
+  readonly phase: SenderPhase;
+  readonly failingSince: Instant | null;
+  readonly nextTryAt: Instant | null;
+  readonly lastError: LastError | null;
+  readonly lastStoredAt: Instant | null;
+}
+
+export interface SenderDeps {
+  readonly store: ClaimStore;
+  readonly sink: FactSink;
+  readonly clock: Clock;
+  readonly limits: OutboxLimits;
+
+  // a number in [0, 1), for the jitter of the backoff
+  readonly random: () => number;
+}
+
+export interface SenderTurn {
+  readonly step: SenderStep;
+  readonly state: SenderState;
+}

package/src/pull-requests/build-link-change.ts

@@ -0,0 +1,95 @@
+import type {
+  ActorRef,
+  AnyNewFact,
+  AttemptRow,
+  Instant,
+  NewId,
+  PullRequestReading,
+  PullRequestRef,
+  PullRequestRow,
+  StoreChange,
+} from '@boxxy/model';
+import { brandSchemas } from '@boxxy/model';
+import { buildStateFact } from './build-state-fact.ts';
+import { collectSeenState } from './collect-seen-state.ts';
+
+export interface LinkChangeInput {
+  readonly attempt: AttemptRow;
+  readonly ref: PullRequestRef;
+  readonly reading: PullRequestReading;
+  readonly source: 'hook' | 'watch';
+  readonly actor: ActorRef;
+  readonly now: Instant;
+  readonly newId: NewId;
+}
+
+// The change that links a pull request to an attempt: the pull request row, the evidence row
+// and `boxxy.pull_request.linked` in one change. A pull request the forge already shows merged
+// or closed also gets its merge or close fact, because the watch reads only open rows.
+export function buildLinkChange(input: LinkChangeInput): StoreChange {
+  const attempt = input.attempt;
+  const ref = input.ref;
+  const reading = input.reading;
+  const now = input.now;
+  const seen = collectSeenState(reading);
+  const linkedFactId = brandSchemas.factId.parse(input.newId());
+
+  const row: PullRequestRow = {
+    repo: ref.repo,
+    number: ref.number,
+    attempt: attempt.id,
+    githubId: reading.githubId,
+    url: reading.url,
+    state: seen.state,
+    mergedAt: seen.mergedAt,
+    mergeCommit: seen.mergeCommit,
+    checkedAt: now,
+  };
+
+  const entity = `boxxy:item:${attempt.item}` as const;
+
+  const linked: AnyNewFact = {
+    id: linkedFactId,
+    type: 'boxxy.pull_request.linked',
+    entity,
+    actor: input.actor,
+    payload: {
+      attempt: attempt.id,
+      repo: ref.repo,
+      number: ref.number,
+      url: reading.url,
+      pullRequestId: String(reading.githubId),
+      source: input.source,
+    },
+  };
+
+  const stateFact = buildStateFact({
+    row,
+    item: attempt.item,
+    actor: input.actor,
+    id: brandSchemas.factId.parse(input.newId()),
+  });
+
+  const stateFacts = stateFact === null ? [] : [stateFact];
+
+  return {
+    at: now,
+    writes: [
+      { op: 'insert', table: 'pull_requests', row },
+      {
+        op: 'insert',
+        table: 'evidence',
+        row: {
+          factId: linkedFactId,
+          attempt: attempt.id,
+          kind: 'pull_request',
+          ref: reading.url,
+          observed: null,
+          source: input.source,
+          at: now,
+        },
+      },
+    ],
+    facts: [linked, ...stateFacts],
+  };
+}

package/src/pull-requests/build-state-fact.ts

@@ -0,0 +1,39 @@
+import type { ActorRef, AnyNewFact, FactId, ItemId, PullRequestRow } from '@boxxy/model';
+
+export interface StateFactInput {
+  readonly row: PullRequestRow;
+  readonly item: ItemId;
+  readonly actor: ActorRef;
+  readonly id: FactId;
+}
+
+// The fact for a pull request row that is not open: its merge, or its close with no merge. An
+// open row has none.
+export function buildStateFact(input: StateFactInput): AnyNewFact | null {
+  const row = input.row;
+
+  const base = {
+    id: input.id,
+    entity: `boxxy:item:${input.item}`,
+    actor: input.actor,
+  } as const;
+
+  const shared = {
+    attempt: row.attempt,
+    repo: row.repo,
+    number: row.number,
+    pullRequestId: String(row.githubId),
+  };
+
+  if (row.state === 'merged' && row.mergedAt !== null && row.mergeCommit !== null) {
+    return {
+      ...base,
+      type: 'boxxy.pull_request.merged',
+      payload: { ...shared, mergedAt: row.mergedAt, mergeCommit: row.mergeCommit },
+    };
+  }
+
+  return row.state === 'closed'
+    ? { ...base, type: 'boxxy.pull_request.closed', payload: shared }
+    : null;
+}

package/src/pull-requests/collect-attempt-trailers.ts

@@ -0,0 +1,20 @@
+import type { AttemptId } from '@boxxy/model';
+import { brandSchemas } from '@boxxy/model';
+
+const trailerPattern = /^Boxxy-Attempt:[ \t]*(?<attempt>\S+)[ \t]*$/gimu;
+
+// The attempt IDs that the `Boxxy-Attempt` trailers of a commit message carry, in order. A
+// value that is not an attempt ID is left out.
+export function collectAttemptTrailers(message: string): AttemptId[] {
+  const attempts: AttemptId[] = [];
+
+  for (const match of message.matchAll(trailerPattern)) {
+    const parsed = brandSchemas.attemptId.safeParse(match.groups?.['attempt']);
+
+    if (parsed.success) {
+      attempts.push(parsed.data);
+    }
+  }
+
+  return attempts;
+}

package/src/pull-requests/collect-seen-state.ts

@@ -0,0 +1,19 @@
+import type { CommitSha, Instant, PullRequestReading, PullRequestRow } from '@boxxy/model';
+
+export interface SeenState {
+  readonly state: PullRequestRow['state'];
+  readonly mergedAt: Instant | null;
+  readonly mergeCommit: CommitSha | null;
+}
+
+// The state the watch records for a reading. A merged reading with no merge time or no merge
+// commit reads as open, so the next pass sees the merge once the forge shows it whole.
+export function collectSeenState(reading: PullRequestReading): SeenState {
+  if (reading.state === 'merged') {
+    return reading.mergedAt !== null && reading.mergeCommit !== null
+      ? { state: 'merged', mergedAt: reading.mergedAt, mergeCommit: reading.mergeCommit }
+      : { state: 'open', mergedAt: null, mergeCommit: null };
+  }
+
+  return { state: reading.state, mergedAt: null, mergeCommit: null };
+}

package/src/pull-requests/constants.ts

@@ -0,0 +1,5 @@
+export const watchLimits = {
+  // The watch looks for the pull request of a succeeded build for this long after the build
+  // ended; a build that opened none stops costing forge calls after it.
+  linkWindowMs: 7 * 24 * 60 * 60 * 1000,
+} as const;

package/src/pull-requests/link-pull-request.ts

@@ -0,0 +1,119 @@
+import type { ItemId, PullRequestReading, PullRequestRef, StoreReader } from '@boxxy/model';
+import invariant from 'tiny-invariant';
+import { pickCallAttempt } from '../attempts/pick-call-attempt.ts';
+import { buildLinkChange } from './build-link-change.ts';
+import { parsePullRequestUrl } from './parse-pull-request-url.ts';
+import type { LinkPullRequestInput, LinkPullRequestResult } from './types.ts';
+
+// It checks the call before the forge read and again in the write's lock, because a newer attempt
+// can take the session while the forge answers.
+export async function linkPullRequest(input: LinkPullRequestInput): Promise<LinkPullRequestResult> {
+  const ref = parsePullRequestUrl(input.url);
+
+  if (ref === null) {
+    return { kind: 'refused', reason: 'not-a-pull-request-url' };
+  }
+
+  const latest = await input.store.read((reader) =>
+    reader.findAttemptBySession(input.call.session),
+  );
+
+  const picked = pickCallAttempt(latest, input.call);
+
+  if (picked.kind === 'refused') {
+    return picked;
+  }
+
+  const attempt = picked.attempt;
+
+  const item = await input.store.read((reader) =>
+    reader.readItem(attempt.item, input.now, input.rules),
+  );
+
+  invariant(item !== null, 'the attempt of a call has its item');
+
+  if (item.repo.toLowerCase() !== ref.repo.toLowerCase()) {
+    return { kind: 'refused', reason: 'not-a-pull-request-url' };
+  }
+
+  const known = await input.store.read((reader) => readExisting(reader, ref, item.id));
+
+  if (known !== 'none') {
+    return toAnswer(known, ref);
+  }
+
+  const reading = await readOrNull(input, ref);
+
+  if (reading === null) {
+    return { kind: 'retry' };
+  }
+
+  return input.store.exclusive(async (tx) => {
+    const latestInLock = await tx.findAttemptBySession(input.call.session);
+
+    const current = pickCallAttempt(latestInLock, input.call);
+
+    if (current.kind === 'refused') {
+      return current;
+    }
+
+    const raced = await readExisting(tx, ref, item.id);
+
+    if (raced !== 'none') {
+      return toAnswer(raced, ref);
+    }
+
+    const applied = await tx.applyChange(
+      buildLinkChange({
+        attempt: current.attempt,
+        ref,
+        reading,
+        source: 'hook',
+        actor: `boxxy:hook:${current.attempt.id}`,
+        now: input.now,
+        newId: input.newId,
+      }),
+    );
+
+    return applied.kind === 'applied'
+      ? { kind: 'linked', repo: ref.repo, number: ref.number, repeat: false }
+      : { kind: 'retry' };
+  });
+}
+
+type Existing = 'none' | 'repeat' | 'elsewhere';
+
+async function readExisting(
+  reader: StoreReader,
+  ref: PullRequestRef,
+  item: ItemId,
+): Promise<Existing> {
+  const row = await reader.readPullRequestRow(ref);
+
+  if (row === null) {
+    return 'none';
+  }
+
+  const owner = await reader.readAttempt(row.attempt);
+
+  return owner?.item === item ? 'repeat' : 'elsewhere';
+}
+
+function toAnswer(existing: Exclude<Existing, 'none'>, ref: PullRequestRef): LinkPullRequestResult {
+  return existing === 'repeat'
+    ? { kind: 'linked', repo: ref.repo, number: ref.number, repeat: true }
+    : { kind: 'refused', reason: 'linked-elsewhere' };
+}
+
+async function readOrNull(
+  input: LinkPullRequestInput,
+  ref: PullRequestRef,
+): Promise<PullRequestReading | null> {
+  try {
+    const reading = await input.forge.readPullRequest(ref);
+
+    return reading === 'not-found' ? null : reading;
+  } catch {
+    return null;
+  }
+}

package/src/pull-requests/parse-pull-request-url.ts

@@ -0,0 +1,30 @@
+import type { PullRequestRef } from '@boxxy/model';
+import { brandSchemas } from '@boxxy/model';
+
+const pathPattern = /^\/(?<owner>[^/]+)\/(?<name>[^/]+)\/pull\/(?<number>\d+)(?:\/.*)?$/u;
+
+// The repo and number a pull request URL names, or null for any other URL. The host is not
+// checked: the forge decides which host it serves, and the stored URL comes from the forge.
+export function parsePullRequestUrl(url: string): PullRequestRef | null {
+  const parsed = URL.parse(url);
+
+  if (parsed === null || (parsed.protocol !== 'https:' && parsed.protocol !== 'http:')) {
+    return null;
+  }
+
+  const match = pathPattern.exec(parsed.pathname);
+
+  if (match === null) {
+    return null;
+  }
+
+  const repo = brandSchemas.repoName.safeParse(
+    `${match.groups?.['owner']}/${match.groups?.['name']}`,
+  );
+
+  const number = Number(match.groups?.['number']);
+
+  return repo.success && Number.isSafeInteger(number) && number > 0
+    ? { repo: repo.data, number }
+    : null;
+}

package/src/pull-requests/types.ts

@@ -0,0 +1,63 @@
+import type {
+  ClaimStore,
+  Forge,
+  Instant,
+  ItemRules,
+  NewId,
+  RepoName,
+  RunnerId,
+} from '@boxxy/model';
+import type { CallRef } from '../closeouts/types.ts';
+
+export interface LinkPullRequestInput {
+  readonly store: ClaimStore;
+  readonly forge: Forge;
+  readonly rules: ItemRules;
+
+  // the call from the hook; the link reaches only the current attempt of the call's session
+  readonly call: CallRef;
+
+  // the URL the hook read from the output of `gh pr create`
+  readonly url: string;
+  readonly now: Instant;
+  readonly newId: NewId;
+}
+
+export type LinkPullRequestResult =
+  | {
+      readonly kind: 'linked';
+      readonly repo: RepoName;
+      readonly number: number;
+
+      // the pull request was already linked to this item and nothing was written
+      readonly repeat: boolean;
+    }
+
+  // the forge did not answer, or does not know the pull request yet; nothing was written
+  | { readonly kind: 'retry' }
+
+  // `not-a-pull-request-url`: no pull request URL, or one of a repo other than the item's;
+  // `no-attempt` and `not-current`: the call names no current attempt of its session
+  | {
+      readonly kind: 'refused';
+      readonly reason: 'not-a-pull-request-url' | 'linked-elsewhere' | 'no-attempt' | 'not-current';
+    };
+
+export interface WatchPullRequestsInput {
+  readonly store: ClaimStore;
+  readonly forge: Forge;
+  readonly rules: ItemRules;
+  readonly runner: RunnerId;
+  readonly now: Instant;
+  readonly newId: NewId;
+}
+
+// What one pass of the watch did. `unanswered` counts forge calls that threw; each is tried
+// again on the next pass.
+export interface WatchReport {
+  readonly linked: number;
+  readonly commitsLinked: number;
+  readonly merged: number;
+  readonly closed: number;
+  readonly unanswered: number;
+}