@boxxy/store-sqlite 0.1.1 → 0.1.4

package/src/writes/create-sqlite-tx.ts

@@ -0,0 +1,229 @@
+import { formatCanonicalJson } from '@boxxy/content-hash';
+import { collectSeatSync, toJSONValue } from '@boxxy/model';
+import type {
+  FactId,
+  Instant,
+  Limit,
+  PullRequestRef,
+  RunnerId,
+  StoreChange,
+  StoreReader,
+  StoreTx,
+} from '@boxxy/model';
+import type { Kysely } from 'kysely';
+import { buildSQLiteReader } from '../reads/build-sqlite-reader.ts';
+import type { StoreDatabase } from '../types.ts';
+import { applySQLiteChange } from './apply-sqlite-change.ts';
+
+// A call started beside a change would run its statements inside the change's savepoint, and a
+// refusal would roll them back after the call resolved, so every call waits for the one before it.
+export function createSQLiteTx(db: Kysely<StoreDatabase>): StoreTx {
+  const inTurn = makeTurnRunner();
+  const reader = buildSQLiteReader(db);
+
+  return {
+    readSnapshot: inTurn(reader.readSnapshot),
+    readItem: inTurn(reader.readItem),
+    readItemDetail: inTurn(reader.readItemDetail),
+    listItems: inTurn(reader.listItems),
+    listOpenIssues: inTurn(reader.listOpenIssues),
+    readLastLeaseGen: inTurn(reader.readLastLeaseGen),
+    readAttempt: inTurn(reader.readAttempt),
+    listEvidence: inTurn(reader.listEvidence),
+    listItemAttempts: inTurn(reader.listItemAttempts),
+    listLiveAttempts: inTurn(reader.listLiveAttempts),
+    listStrayAttempts: inTurn(reader.listStrayAttempts),
+    listSucceededAttemptsWithoutPullRequest: inTurn(reader.listSucceededAttemptsWithoutPullRequest),
+    findAttemptBySession: inTurn(reader.findAttemptBySession),
+    findAttemptByAgentSession: inTurn(reader.findAttemptByAgentSession),
+    findSessionByCaller: inTurn(reader.findSessionByCaller),
+    readPullRequestRow: inTurn(reader.readPullRequestRow),
+    listWatchedPullRequests: inTurn(reader.listWatchedPullRequests),
+    listLimits: inTurn(reader.listLimits),
+    listLimitSeats: inTurn(reader.listLimitSeats),
+    listItemNotes: inTurn(reader.listItemNotes),
+    listUnsentNotices: inTurn(reader.listUnsentNotices),
+    listKeptSessions: inTurn(reader.listKeptSessions),
+    listSessionsToForget: inTurn(reader.listSessionsToForget),
+    readRunner: inTurn(reader.readRunner),
+    readUnsentFacts: inTurn(reader.readUnsentFacts),
+    readOutboxCounts: inTurn(reader.readOutboxCounts),
+    readItemBehind: inTurn(reader.readItemBehind),
+    applyChange: inTurn((change: StoreChange) => applySQLiteChange(db, change)),
+    markFactsSent: inTurn((ids: readonly FactId[], at: Instant) => writeFactsSent(db, ids, at)),
+    parkFacts: inTurn((ids: readonly FactId[], reason: string, at: Instant) =>
+      writeFactsParked(db, ids, reason, at),
+    ),
+    unparkFacts: inTurn(() => removeFactParks(db)),
+    writeHeartbeat: inTurn((runner: RunnerId, at: Instant) => writeRunnerBeat(db, runner, at)),
+    syncLimits: inTurn((limits: readonly Limit[], at: Instant) =>
+      writeLimits(db, reader, limits, at),
+    ),
+    markPullRequestChecked: inTurn((ref: PullRequestRef, at: Instant) =>
+      writePullRequestChecked(db, ref, at),
+    ),
+    markNoticeTried: inTurn((id: number, at: Instant, sent: boolean) =>
+      writeNoticeTry(db, id, at, sent),
+    ),
+  };
+}
+
+function makeTurnRunner() {
+  let turn: Promise<unknown> = Promise.resolve();
+
+  return <Args extends unknown[], Result>(run: (...args: Args) => Promise<Result>) =>
+    async (...args: Args): Promise<Result> => {
+      const previous = turn;
+      const next = Promise.withResolvers<void>();
+
+      turn = next.promise;
+
+      await previous;
+
+      try {
+        return await run(...args);
+      } finally {
+        next.resolve();
+      }
+    };
+}
+
+async function writeFactsSent(
+  db: Kysely<StoreDatabase>,
+  ids: readonly FactId[],
+  at: Instant,
+): Promise<void> {
+  if (ids.length === 0) {
+    return;
+  }
+
+  await db
+    .updateTable('outbox')
+    .set({ sent_at: at })
+    .where('fact_id', 'in', ids)
+    .where('sent_at', 'is', null)
+    .where('parked_at', 'is', null)
+    .execute();
+}
+
+async function writeFactsParked(
+  db: Kysely<StoreDatabase>,
+  ids: readonly FactId[],
+  reason: string,
+  at: Instant,
+): Promise<void> {
+  if (ids.length === 0) {
+    return;
+  }
+
+  await db
+    .updateTable('outbox')
+    .set({ parked_at: at, parked_reason: reason })
+    .where('fact_id', 'in', ids)
+    .where('sent_at', 'is', null)
+    .execute();
+}
+
+async function removeFactParks(db: Kysely<StoreDatabase>): Promise<number> {
+  const result = await db
+    .updateTable('outbox')
+    .set({ parked_at: null, parked_reason: null })
+    .where('sent_at', 'is', null)
+    .where('parked_at', 'is not', null)
+    .executeTakeFirst();
+
+  return Number(result.numUpdatedRows);
+}
+
+async function writeRunnerBeat(
+  db: Kysely<StoreDatabase>,
+  runner: RunnerId,
+  at: Instant,
+): Promise<void> {
+  await db
+    .insertInto('runners')
+    .values({ id: runner, heartbeat_at: at, paused: 0 })
+    .onConflict((conflict) => conflict.column('id').doUpdateSet({ heartbeat_at: at }))
+    .execute();
+}
+
+async function writeLimits(
+  db: Kysely<StoreDatabase>,
+  reader: StoreReader,
+  limits: readonly Limit[],
+  at: Instant,
+): Promise<void> {
+  const sync = collectSeatSync({
+    limits,
+    held: await reader.listLimitSeats(),
+    attempts: await reader.listLiveAttempts(),
+    at,
+  });
+
+  await db.deleteFrom('limits').execute();
+
+  for (const limit of limits) {
+    await db
+      .insertInto('limits')
+      .values({
+        name: limit.name,
+        max: limit.max,
+        counts: formatCanonicalJson(toJSONValue(limit.counts)),
+      })
+      .execute();
+  }
+
+  for (const seat of sync.release) {
+    await db
+      .updateTable('limit_seats')
+      .set({ released_at: at })
+      .where('limit', '=', seat.limit)
+      .where('attempt', '=', seat.attempt)
+      .where('hold_gen', '=', seat.holdGen)
+      .where('released_at', 'is', null)
+      .execute();
+  }
+
+  for (const seat of sync.take) {
+    await db
+      .insertInto('limit_seats')
+      .values({
+        limit: seat.limit,
+        attempt: seat.attempt,
+        hold_gen: seat.holdGen,
+        over_cap: seat.overCap ? 1 : 0,
+        taken_at: seat.takenAt,
+        released_at: seat.releasedAt,
+      })
+      .execute();
+  }
+}
+
+async function writePullRequestChecked(
+  db: Kysely<StoreDatabase>,
+  ref: PullRequestRef,
+  at: Instant,
+): Promise<void> {
+  await db
+    .updateTable('pull_requests')
+    .set({ checked_at: at })
+    .where('repo', '=', ref.repo)
+    .where('number', '=', ref.number)
+    .execute();
+}
+
+async function writeNoticeTry(
+  db: Kysely<StoreDatabase>,
+  id: number,
+  at: Instant,
+  sent: boolean,
+): Promise<void> {
+  await db
+    .updateTable('notices')
+    .set((builder) => ({
+      tries: builder('tries', '+', 1),
+      ...(sent ? { sent_at: at } : {}),
+    }))
+    .where('id', '=', id)
+    .execute();
+}

package/src/writes/take-write-lock.ts

@@ -0,0 +1,8 @@
+import { sql } from 'kysely';
+import type { Kysely } from 'kysely';
+import type { StoreDatabase } from '../types.ts';
+import { withBusyRetry } from '../with-busy-retry.ts';
+
+export async function takeWriteLock(db: Kysely<StoreDatabase>, waitLimitMs: number): Promise<void> {
+  await withBusyRetry(waitLimitMs, () => sql`BEGIN IMMEDIATE`.execute(db));
+}

package/src/build-sqlite-reader.ts

@@ -1,321 +0,0 @@
-import { brandSchemas, outboxFactSchema } from '@boxxy/model';
-import type { ItemView, ItemWithLevers, PoolName, StoreReader } from '@boxxy/model';
-import type { Kysely } from 'kysely';
-import { parseStoreRow } from './parse-store-row.ts';
-import { readItemViews } from './read-item-views.ts';
-import type { StoreDatabase } from './types.ts';
-
-export function buildSQLiteReader(db: Kysely<StoreDatabase>): StoreReader {
-  return {
-    readSnapshot: async (now, maxAttemptsByKind) => {
-      const views = await readItemViews(
-        db,
-        (builder) => builder('closed_at', 'is', null),
-        now,
-        maxAttemptsByKind,
-      );
-
-      const open = views.filter((item) => isOpen(item));
-
-      const links = await db
-        .selectFrom('links')
-        .selectAll()
-        .where('removed_at', 'is', null)
-        .orderBy('added_at')
-        .orderBy('source')
-        .orderBy('target')
-        .execute();
-
-      const version = await db
-        .selectFrom('outbox')
-        .select((builder) => builder.fn.max('seq').as('seq'))
-        .executeTakeFirst();
-
-      return {
-        storeVersion: version?.seq ?? 0,
-        items: open.map((item) => toItemView(item)),
-        links: links.map((row) => parseStoreRow('links', row)),
-        levers: open
-          .flatMap((item) => item.levers)
-          .toSorted((first, second) => first.seq - second.seq),
-        freeSlots: await readFreeSlots(db),
-      };
-    },
-    readItem: async (id, now, maxAttemptsByKind) => {
-      const views = await readItemViews(
-        db,
-        (builder) => builder('id', '=', id),
-        now,
-        maxAttemptsByKind,
-      );
-
-      const item = views.at(0);
-
-      return item === undefined ? null : toItemView(item);
-    },
-    readItemDetail: async (id, now, maxAttemptsByKind) => {
-      const views = await readItemViews(
-        db,
-        (builder) => builder('id', '=', id),
-        now,
-        maxAttemptsByKind,
-      );
-
-      const item = views.at(0);
-
-      if (item === undefined) {
-        return null;
-      }
-
-      const attemptsOfItem = db.selectFrom('attempts').select('id').where('item', '=', id);
-
-      const attempts = await db
-        .selectFrom('attempts')
-        .selectAll()
-        .where('item', '=', id)
-        .orderBy('claimed_at')
-        .orderBy('id')
-        .execute();
-
-      const links = await db
-        .selectFrom('links')
-        .selectAll()
-        .where('removed_at', 'is', null)
-        .where((builder) => builder.or([builder('source', '=', id), builder('target', '=', id)]))
-        .orderBy('added_at')
-        .orderBy('source')
-        .orderBy('target')
-        .execute();
-
-      const evidence = await db
-        .selectFrom('evidence')
-        .selectAll()
-        .where('attempt', 'in', attemptsOfItem)
-        .orderBy('at')
-        .orderBy('fact_id')
-        .execute();
-
-      const pullRequests = await db
-        .selectFrom('pull_requests')
-        .selectAll()
-        .where('attempt', 'in', attemptsOfItem)
-        .orderBy('repo')
-        .orderBy('number')
-        .execute();
-
-      return {
-        item,
-        attempts: attempts.map((row) => parseStoreRow('attempts', row)),
-        links: links.map((row) => parseStoreRow('links', row)),
-        evidence: evidence.map((row) => parseStoreRow('evidence', row)),
-        pullRequests: pullRequests.map((row) => parseStoreRow('pull_requests', row)),
-      };
-    },
-    listItems: async (filter, now, maxAttemptsByKind) => {
-      const repo = filter.repo;
-
-      const views = await readItemViews(
-        db,
-        (builder) => (repo === undefined ? builder.lit(true) : builder('repo', '=', repo)),
-        now,
-        maxAttemptsByKind,
-      );
-
-      return views.filter(
-        (item) => filter.state === undefined || item.derived.state === filter.state,
-      );
-    },
-    listOpenIssues: async (repo, now, maxAttemptsByKind) => {
-      const views = await readItemViews(
-        db,
-        (builder) =>
-          builder.and([
-            builder('kind', '=', 'core.issue'),
-            builder('repo', '=', repo),
-            builder('closed_at', 'is', null),
-          ]),
-        now,
-        maxAttemptsByKind,
-      );
-
-      return views.filter((item) => isOpen(item)).map((item) => toItemView(item));
-    },
-    readLastLeaseGen: async (item, phase) => {
-      const row = await db
-        .selectFrom('leases')
-        .select((builder) => builder.fn.max('gen').as('gen'))
-        .where('item', '=', item)
-        .where('phase', '=', phase)
-        .executeTakeFirst();
-
-      return row?.gen ?? 0;
-    },
-    readAttempt: async (id) => {
-      const row = await db
-        .selectFrom('attempts')
-        .selectAll()
-        .where('id', '=', id)
-        .executeTakeFirst();
-
-      return row === undefined ? null : parseStoreRow('attempts', row);
-    },
-    listEvidence: async (attempt) => {
-      const rows = await db
-        .selectFrom('evidence')
-        .selectAll()
-        .where('attempt', '=', attempt)
-        .orderBy('at')
-        .orderBy('fact_id')
-        .execute();
-
-      return rows.map((row) => parseStoreRow('evidence', row));
-    },
-    listLiveAttempts: async () => {
-      const rows = await db
-        .selectFrom('attempts')
-        .selectAll()
-        .where('state', 'in', ['starting', 'running', 'reconciling'])
-        .orderBy('claimed_at')
-        .orderBy('id')
-        .execute();
-
-      return rows.map((row) => parseStoreRow('attempts', row));
-    },
-    listSucceededAttemptsWithoutPr: async () => {
-      const linked = db.selectFrom('pull_requests').select('attempt');
-
-      const rows = await db
-        .selectFrom('attempts')
-        .selectAll()
-        .where('state', '=', 'succeeded')
-        .where('id', 'not in', linked)
-        .orderBy('claimed_at')
-        .orderBy('id')
-        .execute();
-
-      return rows.map((row) => parseStoreRow('attempts', row));
-    },
-    // oxlint-disable-next-line zgeoff/verb-contract -- the StoreReader port in @boxxy/model fixes this name
-    findAttemptBySession: async (session) => {
-      const row = await db
-        .selectFrom('attempts')
-        .selectAll()
-        .where('atc_session', '=', session)
-        .orderBy('claimed_at')
-        .orderBy('id')
-        .executeTakeFirst();
-
-      return row === undefined ? null : parseStoreRow('attempts', row);
-    },
-    // oxlint-disable-next-line zgeoff/verb-contract -- the StoreReader port in @boxxy/model fixes this name
-    findAttemptByAgentSession: async (session) => {
-      const row = await db
-        .selectFrom('attempt_sessions')
-        .select('attempt')
-        .where('agent_session', '=', session)
-        .executeTakeFirst();
-
-      return row === undefined ? null : brandSchemas.attemptId.parse(row.attempt);
-    },
-    listWatchedPullRequests: async () => {
-      const rows = await db
-        .selectFrom('pull_requests')
-        .selectAll()
-        .where('state', '=', 'open')
-        .orderBy('repo')
-        .orderBy('number')
-        .execute();
-
-      return rows.map((row) => parseStoreRow('pull_requests', row));
-    },
-    listPoolSlots: async () => {
-      const rows = await db
-        .selectFrom('pool_slots')
-        .selectAll()
-        .orderBy('pool')
-        .orderBy('slot')
-        .execute();
-
-      return rows.map((row) => parseStoreRow('pool_slots', row));
-    },
-    readUnsentFacts: async (limit) => {
-      const rows = await db
-        .selectFrom('outbox')
-        .selectAll()
-        .where('sent_at', 'is', null)
-        .orderBy('seq')
-        .limit(limit)
-        .execute();
-
-      return rows.map((row) =>
-        outboxFactSchema.parse({
-          seq: row.seq,
-          factId: row.fact_id,
-          type: row.type,
-          entity: row.entity,
-          actor: row.actor,
-          at: row.at,
-          payload: parseJSONText(row.payload),
-          sentAt: row.sent_at,
-        }),
-      );
-    },
-    readOutboxLag: async () => {
-      const row = await db
-        .selectFrom('outbox')
-        .select((builder) => [
-          builder.fn.countAll<number>().as('unsent'),
-          builder.fn.min('at').as('oldest'),
-        ])
-        .where('sent_at', 'is', null)
-        .executeTakeFirstOrThrow();
-
-      const oldest = row.oldest === null ? null : brandSchemas.instant.parse(row.oldest);
-
-      return { unsent: row.unsent, oldestUnsentAt: oldest };
-    },
-    readRunner: async (id) => {
-      const row = await db
-        .selectFrom('runners')
-        .selectAll()
-        .where('id', '=', id)
-        .executeTakeFirst();
-
-      return row === undefined ? null : parseStoreRow('runners', row);
-    },
-  };
-}
-
-function isOpen(item: ItemView): boolean {
-  return item.derived.state !== 'done' && item.derived.state !== 'archived';
-}
-
-function toItemView(item: ItemWithLevers): ItemView {
-  const { levers: _levers, ...view } = item;
-
-  return view;
-}
-
-async function readFreeSlots(db: Kysely<StoreDatabase>): Promise<Record<PoolName, number>> {
-  const rows = await db
-    .selectFrom('pool_slots')
-    .select(['pool', 'holder'])
-    .where('retired', '=', 0)
-    .execute();
-
-  const free: Record<PoolName, number> = {};
-
-  for (const row of rows) {
-    const pool = brandSchemas.poolName.parse(row.pool);
-
-    free[pool] = (free[pool] ?? 0) + (row.holder === null ? 1 : 0);
-  }
-
-  return free;
-}
-
-function parseJSONText(text: string): unknown {
-  const parsed: unknown = JSON.parse(text);
-
-  return parsed;
-}

package/src/create-sqlite-tx.ts

@@ -1,134 +0,0 @@
-import type { FactId, Instant, PullRequestRef, RunnerId, StoreChange, StoreTx } from '@boxxy/model';
-import type { Kysely } from 'kysely';
-import { applySQLiteChange } from './apply-sqlite-change.ts';
-import { buildSQLiteReader } from './build-sqlite-reader.ts';
-import type { StoreDatabase } from './types.ts';
-
-type PoolCounts = Parameters<StoreTx['syncPoolSlots']>[0];
-
-// A call started beside a change would run its statements inside the change's savepoint, and a
-// refusal would roll them back after the call resolved, so every call waits for the one before it.
-export function createSQLiteTx(db: Kysely<StoreDatabase>): StoreTx {
-  const inTurn = makeTurnRunner();
-  const reader = buildSQLiteReader(db);
-
-  return {
-    readSnapshot: inTurn(reader.readSnapshot),
-    readItem: inTurn(reader.readItem),
-    readItemDetail: inTurn(reader.readItemDetail),
-    listItems: inTurn(reader.listItems),
-    listOpenIssues: inTurn(reader.listOpenIssues),
-    readLastLeaseGen: inTurn(reader.readLastLeaseGen),
-    readAttempt: inTurn(reader.readAttempt),
-    listEvidence: inTurn(reader.listEvidence),
-    listLiveAttempts: inTurn(reader.listLiveAttempts),
-    listSucceededAttemptsWithoutPr: inTurn(reader.listSucceededAttemptsWithoutPr),
-    findAttemptBySession: inTurn(reader.findAttemptBySession),
-    findAttemptByAgentSession: inTurn(reader.findAttemptByAgentSession),
-    listWatchedPullRequests: inTurn(reader.listWatchedPullRequests),
-    listPoolSlots: inTurn(reader.listPoolSlots),
-    readUnsentFacts: inTurn(reader.readUnsentFacts),
-    readOutboxLag: inTurn(reader.readOutboxLag),
-    readRunner: inTurn(reader.readRunner),
-    applyChange: inTurn((change: StoreChange) => applySQLiteChange(db, change)),
-    markFactsSent: inTurn((ids: FactId[], at: Instant) => writeFactsSent(db, ids, at)),
-    writeHeartbeat: inTurn((runner: RunnerId, at: Instant) => writeRunnerBeat(db, runner, at)),
-    syncPoolSlots: inTurn((pools: PoolCounts) => writePoolSlots(db, pools)),
-    markPullRequestChecked: inTurn((ref: PullRequestRef, at: Instant) =>
-      writePullRequestChecked(db, ref, at),
-    ),
-  };
-}
-
-function makeTurnRunner() {
-  let turn: Promise<unknown> = Promise.resolve();
-
-  return <Args extends unknown[], Result>(run: (...args: Args) => Promise<Result>) =>
-    async (...args: Args): Promise<Result> => {
-      const previous = turn;
-      const next = Promise.withResolvers<void>();
-
-      turn = next.promise;
-
-      await previous;
-
-      try {
-        return await run(...args);
-      } finally {
-        next.resolve();
-      }
-    };
-}
-
-async function writeFactsSent(
-  db: Kysely<StoreDatabase>,
-  ids: FactId[],
-  at: Instant,
-): Promise<void> {
-  if (ids.length === 0) {
-    return;
-  }
-
-  await db
-    .updateTable('outbox')
-    .set({ sent_at: at })
-    .where('fact_id', 'in', ids)
-    .where('sent_at', 'is', null)
-    .execute();
-}
-
-async function writeRunnerBeat(
-  db: Kysely<StoreDatabase>,
-  runner: RunnerId,
-  at: Instant,
-): Promise<void> {
-  await db
-    .insertInto('runners')
-    .values({ id: runner, heartbeat_at: at, paused: 0 })
-    .onConflict((conflict) => conflict.column('id').doUpdateSet({ heartbeat_at: at }))
-    .execute();
-}
-
-async function writePoolSlots(db: Kysely<StoreDatabase>, pools: PoolCounts): Promise<void> {
-  for (const pool of pools) {
-    const rows = Array.from({ length: pool.slots }, (_, index) => ({
-      pool: pool.name,
-      slot: index + 1,
-      holder: null,
-      gen: 0,
-      retired: 0,
-    }));
-
-    if (rows.length > 0) {
-      await db
-        .insertInto('pool_slots')
-        .values(rows)
-        .onConflict((conflict) => conflict.columns(['pool', 'slot']).doNothing())
-        .execute();
-    }
-  }
-
-  await db.updateTable('pool_slots').set({ retired: 1 }).execute();
-
-  for (const pool of pools) {
-    await db
-      .updateTable('pool_slots')
-      .set({ retired: 0 })
-      .where('pool', '=', pool.name)
-      .where('slot', '<=', pool.slots)
-      .execute();
-  }
-}
-
-async function writePullRequestChecked(
-  db: Kysely<StoreDatabase>,
-  ref: PullRequestRef,
-  at: Instant,
-): Promise<void> {
-  await db
-    .updateTable('pull_requests')
-    .set({ checked_at: at })
-    .where('repo', '=', ref.repo)
-    .where('number', '=', ref.number)
-    .execute();
-}