@ianwremmel/dispatch 0.32.1-bootstrap.0

package/src/lib/model/types.mts

@@ -0,0 +1,83 @@
+import type {OutcomeKind, PrOrigin, Status, TargetKind} from './status.mts';
+
+/** Every entity's `id` is its tracker/forge external id (e.g. `CLC-945`). */
+export interface Project {
+  id: string;
+  name: string;
+  /** The tracker this project came from; null until a write names one. */
+  source: string | null;
+}
+
+export interface Milestone {
+  id: string;
+  project: string;
+  name: string;
+}
+
+export interface Ticket {
+  id: string;
+  project: string;
+  url: string;
+  title: string;
+  status: Status;
+  targetKind: TargetKind;
+  requiresHuman: boolean;
+  /** Injected at run time; ranks to the top of the frontier. */
+  injected: boolean;
+  /** Lower is more urgent; `null` sorts last. */
+  priority: number | null;
+  branchHint: string | null;
+  labels: string[];
+  updatedAt: string | null;
+}
+
+export interface Pr {
+  id: string;
+  /** The originating ticket, or null for a bare PR / raw prompt. */
+  ticket: string | null;
+  origin: PrOrigin;
+  repo: string | null;
+  prNumber: number | null;
+  url: string | null;
+  branch: string | null;
+  title: string;
+  injected: boolean;
+  priority: number | null;
+  updatedAt: string | null;
+}
+
+/** `blocker` blocks `blocked` — i.e. `blocked` depends on `blocker`. */
+export interface Edge {
+  blocker: string;
+  blocked: string;
+}
+
+export interface Session {
+  id: string;
+  host: string | null;
+  pid: number | null;
+  /** The Claude session this server serves; the caller correlator. */
+  claudeSessionId: string | null;
+  /** When the session acknowledged the probe; work orders wait on it. */
+  ackedAt: string | null;
+  startedAt: string;
+  heartbeatAt: string;
+}
+
+export interface Claim {
+  node: string;
+  session: string;
+  actor: string | null;
+  worktree: string | null;
+  branch: string | null;
+  claimedAt: string;
+}
+
+export interface Outcome {
+  node: string;
+  outcome: OutcomeKind;
+  /** Meaningful only for `failed`; null otherwise. */
+  retryable: boolean | null;
+  detail: string | null;
+  recordedAt: string;
+}

package/src/lib/refresh/index.mts

@@ -0,0 +1,2 @@
+export * from './placeholders.mts';
+export * from './refresh-service.mts';

package/src/lib/refresh/placeholders.mts

@@ -0,0 +1,43 @@
+import type {Database} from '../db/database.mts';
+
+/**
+ * Every node an edge referenced that nobody has since written, grouped by the
+ * tracker that owns it. A placeholder carries no project of its own, so its
+ * tracker comes from a ticket on the other end of an edge touching it — the
+ * only thing that can have referenced it.
+ *
+ * One query for the whole graph rather than one per placeholder: `reconcile`
+ * runs on every write, and a project scan leaves hundreds of placeholders, each
+ * of which would otherwise pay this four-way join.
+ *
+ * A placeholder that no edge connects to a project with a tracker is absent
+ * from the result — `edge add A B` with neither endpoint written yet reaches no
+ * project, so there is nobody to ask. Those surface through the anomalies
+ * read-model, not from here.
+ *
+ * A placeholder referenced from tickets on two trackers picks the smallest
+ * source, deterministically. Cross-tracker dependencies are illegal anyway;
+ * asking the wrong tracker resolves the request `missing`, which the anomalies
+ * read-model then reports rather than the loop spinning.
+ */
+export function placeholdersBySource(db: Database): Map<string, string[]> {
+  const bySource = new Map<string, string[]>();
+  const rows = db.all(
+    `SELECT n.id AS id, n.external_id AS external_id, MIN(p.source) AS source
+     FROM node n
+     JOIN edge e ON (e.blocker = n.id OR e.blocked = n.id)
+     JOIN ticket t
+       ON t.node_id = (CASE WHEN e.blocker = n.id THEN e.blocked ELSE e.blocker END)
+     JOIN project p ON p.node_id = t.project_id
+     WHERE n.kind = 'unknown' AND p.source IS NOT NULL
+     GROUP BY n.id
+     ORDER BY n.id`
+  );
+  for (const row of rows) {
+    const source = String(row.source);
+    const ids = bySource.get(source);
+    if (ids === undefined) bySource.set(source, [String(row.external_id)]);
+    else ids.push(String(row.external_id));
+  }
+  return bySource;
+}

package/src/lib/refresh/refresh-service.mts

@@ -0,0 +1,203 @@
+import type {Database} from '../db/database.mts';
+import {nowIso} from '../db/time.mts';
+import {DataError, ensure, UsageError} from '../errors/index.mts';
+import {
+  CursorStore,
+  FetchRequestStore,
+  findNode,
+  RefreshStore,
+} from '../stores/index.mts';
+import type {FetchRequest, RefreshRow, RefreshState} from '../stores/index.mts';
+import {placeholdersBySource} from './placeholders.mts';
+
+/**
+ * Every decision about what to fetch next and when a refresh is done. All of it
+ * derives from the database, so `reconcile` is idempotent and a write command
+ * can call it unconditionally without knowing which phase it is in.
+ */
+export class RefreshService {
+  readonly #db: Database;
+  readonly #refreshes: RefreshStore;
+  readonly #requests: FetchRequestStore;
+  readonly #cursors: CursorStore;
+  readonly #now: () => string;
+
+  constructor(db: Database, now: () => string = nowIso) {
+    this.#db = db;
+    this.#refreshes = new RefreshStore(db);
+    this.#requests = new FetchRequestStore(db);
+    this.#cursors = new CursorStore(db);
+    this.#now = now;
+  }
+
+  /** Open a refresh, or report that a live one already owns this source. */
+  async startScan(input: {
+    source: string;
+    projects: readonly string[];
+    sessionId: string | null;
+    rebuild: boolean;
+  }): Promise<{resumed: boolean}> {
+    const row = await this.#refreshes.get(input.source);
+    // A refresh that closed but has not yet had its completion pushed still
+    // belongs to whoever is waiting on that event — reopening it here would
+    // blank the completion fields and the event would never fire.
+    const finishing =
+      row !== null &&
+      row.completedAt !== null &&
+      row.completionEmittedAt === null;
+    const busy = row !== null && (row.state !== 'idle' || finishing);
+    if (busy && (await this.#refreshes.hasLiveSession(input.source))) {
+      // Resuming must put the outstanding instructions back on the wire: the
+      // caller is a session that asked again because it is holding nothing, and
+      // the drain only pushes rows it has not already marked delivered.
+      await this.#requests.redeliver(input.source);
+      return {resumed: true};
+    }
+
+    if (input.rebuild) {
+      await this.#db.transaction(() => this.#db.run('DELETE FROM node'));
+      // The delete is graph-wide, so every source's cursor must reset with it —
+      // otherwise another tracker's next delta sync starts past data that no
+      // longer exists and never re-fetches it.
+      await this.#cursors.clearAllCursors();
+    }
+
+    const at = this.#now();
+    await this.#requests.clear(input.source);
+    await this.#refreshes.open({
+      source: input.source,
+      projects: input.projects,
+      sessionId: input.sessionId,
+      at,
+    });
+    const cursor = await this.#cursors.getCursor(input.source);
+    await this.#requests.enqueueScan({
+      source: input.source,
+      projects: input.projects,
+      cursor,
+      at,
+    });
+    return {resumed: false};
+  }
+
+  /** The agent has written everything its scan found. */
+  async completeScan(input: {
+    source: string;
+    cursor: string | null;
+  }): Promise<{state: RefreshState; pending: string[]}> {
+    const row = await this.#refreshes.get(input.source);
+    ensure(
+      row !== null && row.state === 'scanning',
+      () =>
+        new UsageError(`no scan is in progress for ${input.source}`, {
+          hint: 'start one with `dispatch refresh --tracker <id> --project <id>` before reporting it done.',
+        })
+    );
+
+    if (input.cursor !== null) {
+      await this.#refreshes.setPendingCursor(input.source, input.cursor);
+    }
+    await this.#requests.resolveScan(input.source);
+    await this.#refreshes.setState(input.source, 'resolving');
+    await this.reconcile();
+
+    const after = await this.#refreshes.get(input.source);
+    ensure(
+      after !== null,
+      () =>
+        new DataError(`the ${input.source} refresh vanished mid-completion`, {
+          hint: 'another process deleted the refresh row; rerun `dispatch refresh --tracker <id> --project <id>`.',
+        })
+    );
+    const pending = (await this.#requests.openTickets())
+      .filter((request) => request.source === input.source)
+      .map((request) => request.ticket);
+    return {state: after.state, pending};
+  }
+
+  /** The tracker has no such ticket; stop asking for it. */
+  async markMissing(ticket: string): Promise<void> {
+    const request = await this.#requests.openTicketRequest(ticket);
+    ensure(
+      request !== null,
+      () =>
+        new UsageError(`nothing asked for ticket ${ticket}`, {
+          hint: 'only a ticket the CLI requested can be reported missing — check `dispatch refresh status`.',
+        })
+    );
+    await this.#requests.resolve(request.id, 'missing');
+    await this.reconcile();
+  }
+
+  async status(
+    source: string
+  ): Promise<{refresh: RefreshRow | null; requests: FetchRequest[]}> {
+    return {
+      refresh: await this.#refreshes.get(source),
+      requests: await this.#requests.bySource(source),
+    };
+  }
+
+  /**
+   * Bring every source back in line with the graph. Three passes, in order:
+   * satisfy requests whose ticket now exists, chase placeholders nobody is
+   * fetching, close whatever has nothing outstanding.
+   */
+  async reconcile(): Promise<void> {
+    const at = this.#now();
+
+    for (const request of await this.#requests.openTickets()) {
+      const node = findNode(this.#db, request.ticket);
+      if (node !== null && node.kind !== 'unknown') {
+        await this.#requests.resolve(request.id, 'materialized');
+      }
+    }
+
+    // A scan writes edges before their endpoints, so under one every reference
+    // is briefly dangling; chasing them there would ask for most of the project.
+    // The state is read once per source, before any per-placeholder work, so a
+    // scan pays nothing for the placeholders it is still filling in.
+    for (const [source, externalIds] of placeholdersBySource(this.#db)) {
+      const row = await this.#refreshes.get(source);
+      if (row?.state === 'scanning') continue;
+      // A refresh that closed but still owes its completion push is not fair
+      // game to reopen — blanking the completion fields here would leave the
+      // session that is waiting on the event waiting forever.
+      if (
+        row !== null &&
+        row.completedAt !== null &&
+        row.completionEmittedAt === null
+      ) {
+        continue;
+      }
+      // The state advances only once a request exists to justify it.
+      // `enqueueTicket` returns null for an id that is already queued or
+      // already resolved `missing`, and a `resolving` refresh with nothing
+      // outstanding closes on the next pass and owes another completion — for
+      // a `missing` tombstone, which is permanent, that repeats on every write.
+      let resolving = row !== null && row.state === 'resolving';
+      for (const externalId of externalIds) {
+        const enqueued = await this.#requests.enqueueTicket({
+          source,
+          ticket: externalId,
+          at,
+        });
+        if (enqueued === null) continue;
+        if (!resolving) {
+          await this.#refreshes.openResolving({source, sessionId: null, at});
+          resolving = true;
+        }
+      }
+    }
+
+    for (const row of await this.#refreshes.active()) {
+      if (row.state !== 'resolving') continue;
+      if ((await this.#requests.openCount(row.source)) > 0) continue;
+      if (row.pendingCursor !== null) {
+        await this.#cursors.setCursor(row.source, row.pendingCursor);
+      }
+      await this.#requests.clearExceptMissing(row.source);
+      await this.#refreshes.close(row.source, at);
+    }
+  }
+}

package/src/lib/schedule/CLAUDE.md

@@ -0,0 +1,18 @@
+# schedule
+
+The server's deterministic scheduling half.
+
+- `scheduler.mts` — `Scheduler.tick(now)`: heartbeat and sweep, then (only
+  once the session is acked) one admission budget — `max-parallel` minus
+  live claims — spent on milestone-review orders first, then
+  claim-then-emit over the dispatch queue, plus the once-per-episode condition
+  orders (`notice` table). Returns orders; never touches the channel, so it
+  tests without one.
+- `caps.mts` — `RepoAdmission`: the per-repo caps on open PRs and in-flight
+  builds, which bound resources a PR holds while no agent does. Unlike the
+  budget, a cap refuses one queue entry and the pass continues — a later entry
+  for another repo may still fit.
+- `tick.mts` — `runServerTick`: opens the DB, runs the scheduler, pushes the
+  orders, and keeps the probe/ack handshake alive on a capped backoff.
+- `correlate.mts` — `resolveSession`: an explicit registry id, else the one
+  live server carrying the caller's Claude session id.

package/src/lib/schedule/caps.mts

@@ -0,0 +1,113 @@
+import type {RepoPrLoad} from '../graph/index.mts';
+import {limitFor} from '../model/index.mts';
+import type {RepoCap, RepoCapPolicy} from '../model/index.mts';
+
+/** A cap at its limit, and how much work is waiting behind it. */
+export interface CapHold {
+  repo: string;
+  cap: RepoCap;
+  limit: number;
+  observed: number;
+  /** Queue entries this cap refused over the pass. */
+  waiting: number;
+}
+
+/** What the caps need to know about one queue entry. */
+export interface PrItemRef {
+  id: string;
+  repo: string | null;
+  prNumber: number | null;
+}
+
+/**
+ * The per-repo half of admission, run over one pass of the dispatch queue.
+ *
+ * Counts are node sets rather than integers so a reservation is idempotent:
+ * dispatching a PR that is already building must not spend a second build
+ * slot, and only an item with no PR number yet adds to the open-PR count.
+ * Reserving at all is what stops one pass overshooting a cap — every
+ * `dispatch_pr` is expected to produce a build, and each one that opens a PR
+ * to produce an open PR, neither of which the pass's own load reading can
+ * show. Past this pass the claim each order takes carries the same weight,
+ * so the next tick counts it without help from here.
+ *
+ * An item with no repo is admitted: there is nothing to attribute it to.
+ */
+export class RepoAdmission {
+  readonly #policy: RepoCapPolicy;
+  readonly #open = new Map<string, Set<string>>();
+  readonly #building = new Map<string, Set<string>>();
+  readonly #holds = new Map<string, CapHold>();
+
+  constructor(policy: RepoCapPolicy, load: readonly RepoPrLoad[]) {
+    this.#policy = policy;
+    for (const entry of load) {
+      if (entry.open) RepoAdmission.#add(this.#open, entry.repo, entry.node);
+      if (entry.building)
+        RepoAdmission.#add(this.#building, entry.repo, entry.node);
+    }
+  }
+
+  /**
+   * The cap refusing this item, or null when both admit it. A refusal is
+   * recorded, so `holds` reports what the pass left waiting.
+   *
+   * The open-PR cap gates only an item that would open a *new* PR. An item
+   * whose PR already exists is counted against that cap already, so working
+   * it adds nothing to the pool — and gating it would deadlock the cap,
+   * because nothing could finish to free a slot. The build cap gates every
+   * item; it cannot deadlock, since builds drain without an agent.
+   */
+  admit(item: PrItemRef): CapHold | null {
+    if (item.repo === null) return null;
+    if (item.prNumber === null) {
+      const hold = this.#atLimit(item.repo, 'open-prs');
+      if (hold !== null) return this.#record(hold);
+    }
+    const hold = this.#atLimit(item.repo, 'in-flight-builds');
+    return hold === null ? null : this.#record(hold);
+  }
+
+  /** Spend this item's slots for the rest of the pass. */
+  reserve(item: PrItemRef): void {
+    if (item.repo === null) return;
+    if (item.prNumber === null)
+      RepoAdmission.#add(this.#open, item.repo, item.id);
+    RepoAdmission.#add(this.#building, item.repo, item.id);
+  }
+
+  /** Every cap that refused something, repo then cap. */
+  holds(): CapHold[] {
+    return [...this.#holds.values()].sort(
+      (a, b) => a.repo.localeCompare(b.repo) || a.cap.localeCompare(b.cap)
+    );
+  }
+
+  #atLimit(repo: string, cap: RepoCap): CapHold | null {
+    const observed = (cap === 'open-prs' ? this.#open : this.#building).get(
+      repo
+    )?.size;
+    const limit = limitFor(this.#policy, repo, cap);
+    return (observed ?? 0) >= limit
+      ? {repo, cap, limit, observed: observed ?? 0, waiting: 0}
+      : null;
+  }
+
+  #record(hold: CapHold): CapHold {
+    const key = `${hold.repo} ${hold.cap}`;
+    const held = this.#holds.get(key) ?? hold;
+    held.waiting += 1;
+    this.#holds.set(key, held);
+    return held;
+  }
+
+  static #add(
+    counts: Map<string, Set<string>>,
+    repo: string,
+    node: string
+  ): void {
+    const set = counts.get(repo) ?? new Set<string>();
+    set.add(node);
+    counts.set(repo, set);
+  }
+}

package/src/lib/schedule/correlate.mts

@@ -0,0 +1,69 @@
+import type {Database} from '../db/database.mts';
+import {nowIso} from '../db/time.mts';
+import {DataError, ensure} from '../errors/index.mts';
+import {DEFAULT_STALE_AFTER_SECONDS} from '../graph/index.mts';
+import {withLiveProcesses} from '../liveness/index.mts';
+import {SessionStore} from '../stores/index.mts';
+
+/**
+ * Correlate a caller to its server without demanding one exists.
+ *
+ * `null` means no live server rides this caller's session — an operator at a
+ * terminal, or a session whose server died. A command that only wanted to
+ * release its own claim can proceed: there is no claim of its own to release.
+ *
+ * Ambiguity is emphatically not that case. More than one live server carries
+ * this session id, so which one's claim a release should target is unknowable,
+ * and guessing would either strand capacity or revoke a running worker's grant.
+ * That throws, naming `--session` as the way out.
+ */
+export async function correlateSession(
+  db: Database,
+  env: NodeJS.ProcessEnv,
+  explicit: string | undefined
+): Promise<string | null> {
+  if (explicit !== undefined) return explicit;
+  const caller = env.CLAUDE_CODE_SESSION_ID ?? null;
+  if (caller === null) return null;
+  const live = await withLiveProcesses(
+    await new SessionStore(db).liveForCaller(
+      caller,
+      nowIso(),
+      DEFAULT_STALE_AFTER_SECONDS
+    )
+  );
+  ensure(
+    live.length <= 1,
+    () =>
+      new DataError(
+        `${String(live.length)} live servers carry this session id: ${live
+          .map((session) => session.id)
+          .join(', ')}`,
+        {
+          hint: 'pass --session with the registry id from the probe event; correlation cannot choose between them.',
+        }
+      )
+  );
+  return live[0]?.id ?? null;
+}
+
+/**
+ * The session a caller's writes must ride. Claims recorded under it cascade
+ * away if that server dies. Unlike `correlateSession`, having no server at all
+ * is a failure — the caller needs a session to write under.
+ */
+export async function resolveSession(
+  db: Database,
+  env: NodeJS.ProcessEnv,
+  explicit: string | undefined
+): Promise<string> {
+  const session = await correlateSession(db, env, explicit);
+  ensure(
+    session !== null,
+    () =>
+      new DataError('no live server correlates to this session', {
+        hint: 'pass --session with the registry id from the probe event, or start the dispatch MCP server.',
+      })
+  );
+  return session;
+}

package/src/lib/schedule/index.mts

@@ -0,0 +1,7 @@
+export {RepoAdmission} from './caps.mts';
+export type {CapHold, PrItemRef} from './caps.mts';
+export {DEFAULT_MAX_PARALLEL, Scheduler} from './scheduler.mts';
+export type {TickResult, WorkOrder} from './scheduler.mts';
+export {createTickState, runServerTick} from './tick.mts';
+export type {TickState} from './tick.mts';
+export {correlateSession, resolveSession} from './correlate.mts';