@boxxy/store-sqlite 0.0.0 → 0.1.1

package/src/build-sqlite-reader.ts

@@ -0,0 +1,321 @@
+import { brandSchemas, outboxFactSchema } from '@boxxy/model';
+import type { ItemView, ItemWithLevers, PoolName, StoreReader } from '@boxxy/model';
+import type { Kysely } from 'kysely';
+import { parseStoreRow } from './parse-store-row.ts';
+import { readItemViews } from './read-item-views.ts';
+import type { StoreDatabase } from './types.ts';
+
+export function buildSQLiteReader(db: Kysely<StoreDatabase>): StoreReader {
+  return {
+    readSnapshot: async (now, maxAttemptsByKind) => {
+      const views = await readItemViews(
+        db,
+        (builder) => builder('closed_at', 'is', null),
+        now,
+        maxAttemptsByKind,
+      );
+
+      const open = views.filter((item) => isOpen(item));
+
+      const links = await db
+        .selectFrom('links')
+        .selectAll()
+        .where('removed_at', 'is', null)
+        .orderBy('added_at')
+        .orderBy('source')
+        .orderBy('target')
+        .execute();
+
+      const version = await db
+        .selectFrom('outbox')
+        .select((builder) => builder.fn.max('seq').as('seq'))
+        .executeTakeFirst();
+
+      return {
+        storeVersion: version?.seq ?? 0,
+        items: open.map((item) => toItemView(item)),
+        links: links.map((row) => parseStoreRow('links', row)),
+        levers: open
+          .flatMap((item) => item.levers)
+          .toSorted((first, second) => first.seq - second.seq),
+        freeSlots: await readFreeSlots(db),
+      };
+    },
+    readItem: async (id, now, maxAttemptsByKind) => {
+      const views = await readItemViews(
+        db,
+        (builder) => builder('id', '=', id),
+        now,
+        maxAttemptsByKind,
+      );
+
+      const item = views.at(0);
+
+      return item === undefined ? null : toItemView(item);
+    },
+    readItemDetail: async (id, now, maxAttemptsByKind) => {
+      const views = await readItemViews(
+        db,
+        (builder) => builder('id', '=', id),
+        now,
+        maxAttemptsByKind,
+      );
+
+      const item = views.at(0);
+
+      if (item === undefined) {
+        return null;
+      }
+
+      const attemptsOfItem = db.selectFrom('attempts').select('id').where('item', '=', id);
+
+      const attempts = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('item', '=', id)
+        .orderBy('claimed_at')
+        .orderBy('id')
+        .execute();
+
+      const links = await db
+        .selectFrom('links')
+        .selectAll()
+        .where('removed_at', 'is', null)
+        .where((builder) => builder.or([builder('source', '=', id), builder('target', '=', id)]))
+        .orderBy('added_at')
+        .orderBy('source')
+        .orderBy('target')
+        .execute();
+
+      const evidence = await db
+        .selectFrom('evidence')
+        .selectAll()
+        .where('attempt', 'in', attemptsOfItem)
+        .orderBy('at')
+        .orderBy('fact_id')
+        .execute();
+
+      const pullRequests = await db
+        .selectFrom('pull_requests')
+        .selectAll()
+        .where('attempt', 'in', attemptsOfItem)
+        .orderBy('repo')
+        .orderBy('number')
+        .execute();
+
+      return {
+        item,
+        attempts: attempts.map((row) => parseStoreRow('attempts', row)),
+        links: links.map((row) => parseStoreRow('links', row)),
+        evidence: evidence.map((row) => parseStoreRow('evidence', row)),
+        pullRequests: pullRequests.map((row) => parseStoreRow('pull_requests', row)),
+      };
+    },
+    listItems: async (filter, now, maxAttemptsByKind) => {
+      const repo = filter.repo;
+
+      const views = await readItemViews(
+        db,
+        (builder) => (repo === undefined ? builder.lit(true) : builder('repo', '=', repo)),
+        now,
+        maxAttemptsByKind,
+      );
+
+      return views.filter(
+        (item) => filter.state === undefined || item.derived.state === filter.state,
+      );
+    },
+    listOpenIssues: async (repo, now, maxAttemptsByKind) => {
+      const views = await readItemViews(
+        db,
+        (builder) =>
+          builder.and([
+            builder('kind', '=', 'core.issue'),
+            builder('repo', '=', repo),
+            builder('closed_at', 'is', null),
+          ]),
+        now,
+        maxAttemptsByKind,
+      );
+
+      return views.filter((item) => isOpen(item)).map((item) => toItemView(item));
+    },
+    readLastLeaseGen: async (item, phase) => {
+      const row = await db
+        .selectFrom('leases')
+        .select((builder) => builder.fn.max('gen').as('gen'))
+        .where('item', '=', item)
+        .where('phase', '=', phase)
+        .executeTakeFirst();
+
+      return row?.gen ?? 0;
+    },
+    readAttempt: async (id) => {
+      const row = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('id', '=', id)
+        .executeTakeFirst();
+
+      return row === undefined ? null : parseStoreRow('attempts', row);
+    },
+    listEvidence: async (attempt) => {
+      const rows = await db
+        .selectFrom('evidence')
+        .selectAll()
+        .where('attempt', '=', attempt)
+        .orderBy('at')
+        .orderBy('fact_id')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('evidence', row));
+    },
+    listLiveAttempts: async () => {
+      const rows = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('state', 'in', ['starting', 'running', 'reconciling'])
+        .orderBy('claimed_at')
+        .orderBy('id')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('attempts', row));
+    },
+    listSucceededAttemptsWithoutPr: async () => {
+      const linked = db.selectFrom('pull_requests').select('attempt');
+
+      const rows = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('state', '=', 'succeeded')
+        .where('id', 'not in', linked)
+        .orderBy('claimed_at')
+        .orderBy('id')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('attempts', row));
+    },
+    // oxlint-disable-next-line zgeoff/verb-contract -- the StoreReader port in @boxxy/model fixes this name
+    findAttemptBySession: async (session) => {
+      const row = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('atc_session', '=', session)
+        .orderBy('claimed_at')
+        .orderBy('id')
+        .executeTakeFirst();
+
+      return row === undefined ? null : parseStoreRow('attempts', row);
+    },
+    // oxlint-disable-next-line zgeoff/verb-contract -- the StoreReader port in @boxxy/model fixes this name
+    findAttemptByAgentSession: async (session) => {
+      const row = await db
+        .selectFrom('attempt_sessions')
+        .select('attempt')
+        .where('agent_session', '=', session)
+        .executeTakeFirst();
+
+      return row === undefined ? null : brandSchemas.attemptId.parse(row.attempt);
+    },
+    listWatchedPullRequests: async () => {
+      const rows = await db
+        .selectFrom('pull_requests')
+        .selectAll()
+        .where('state', '=', 'open')
+        .orderBy('repo')
+        .orderBy('number')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('pull_requests', row));
+    },
+    listPoolSlots: async () => {
+      const rows = await db
+        .selectFrom('pool_slots')
+        .selectAll()
+        .orderBy('pool')
+        .orderBy('slot')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('pool_slots', row));
+    },
+    readUnsentFacts: async (limit) => {
+      const rows = await db
+        .selectFrom('outbox')
+        .selectAll()
+        .where('sent_at', 'is', null)
+        .orderBy('seq')
+        .limit(limit)
+        .execute();
+
+      return rows.map((row) =>
+        outboxFactSchema.parse({
+          seq: row.seq,
+          factId: row.fact_id,
+          type: row.type,
+          entity: row.entity,
+          actor: row.actor,
+          at: row.at,
+          payload: parseJSONText(row.payload),
+          sentAt: row.sent_at,
+        }),
+      );
+    },
+    readOutboxLag: async () => {
+      const row = await db
+        .selectFrom('outbox')
+        .select((builder) => [
+          builder.fn.countAll<number>().as('unsent'),
+          builder.fn.min('at').as('oldest'),
+        ])
+        .where('sent_at', 'is', null)
+        .executeTakeFirstOrThrow();
+
+      const oldest = row.oldest === null ? null : brandSchemas.instant.parse(row.oldest);
+
+      return { unsent: row.unsent, oldestUnsentAt: oldest };
+    },
+    readRunner: async (id) => {
+      const row = await db
+        .selectFrom('runners')
+        .selectAll()
+        .where('id', '=', id)
+        .executeTakeFirst();
+
+      return row === undefined ? null : parseStoreRow('runners', row);
+    },
+  };
+}
+
+function isOpen(item: ItemView): boolean {
+  return item.derived.state !== 'done' && item.derived.state !== 'archived';
+}
+
+function toItemView(item: ItemWithLevers): ItemView {
+  const { levers: _levers, ...view } = item;
+
+  return view;
+}
+
+async function readFreeSlots(db: Kysely<StoreDatabase>): Promise<Record<PoolName, number>> {
+  const rows = await db
+    .selectFrom('pool_slots')
+    .select(['pool', 'holder'])
+    .where('retired', '=', 0)
+    .execute();
+
+  const free: Record<PoolName, number> = {};
+
+  for (const row of rows) {
+    const pool = brandSchemas.poolName.parse(row.pool);
+
+    free[pool] = (free[pool] ?? 0) + (row.holder === null ? 1 : 0);
+  }
+
+  return free;
+}
+
+function parseJSONText(text: string): unknown {
+  const parsed: unknown = JSON.parse(text);
+
+  return parsed;
+}

package/src/create-sqlite-claim-store.ts

@@ -0,0 +1,92 @@
+import { Database } from 'bun:sqlite';
+import type { ClaimStore } from '@boxxy/model';
+import { buildSQLiteDialect } from '@zgeoff/kysely-bun-sqlite';
+import { Kysely, sql } from 'kysely';
+import type { RawBuilder } from 'kysely';
+import { buildSQLiteReader } from './build-sqlite-reader.ts';
+import { createSQLiteTx } from './create-sqlite-tx.ts';
+import { runStoreMigrations } from './run-store-migrations.ts';
+import type { StoreDatabase } from './types.ts';
+
+export interface SQLiteClaimStore extends ClaimStore {
+  close: () => Promise<void>;
+}
+
+export async function createSQLiteClaimStore(path: string): Promise<SQLiteClaimStore> {
+  await using opened = new AsyncDisposableStack();
+
+  const writer = openStoreHandle(path);
+
+  opened.defer(() => {
+    writer.close();
+  });
+
+  const reader = openStoreHandle(path);
+
+  opened.defer(() => {
+    reader.close();
+  });
+
+  const writerDB = new Kysely<StoreDatabase>({ dialect: buildSQLiteDialect(writer) });
+  const readerDB = new Kysely<StoreDatabase>({ dialect: buildSQLiteDialect(reader) });
+
+  opened.defer(() => readerDB.destroy());
+  opened.defer(() => writerDB.destroy());
+
+  await runStoreMigrations(writerDB);
+
+  const handles = opened.move();
+
+  return {
+    // Kysely holds the one connection of a handle from acquire to release, so two calls on this
+    // store run one at a time, and BEGIN IMMEDIATE makes another process's writer wait.
+    exclusive: (work) =>
+      writerDB
+        .connection()
+        .execute((db) => withTransaction(db, sql`BEGIN IMMEDIATE`, () => work(createSQLiteTx(db)))),
+
+    // a second handle, so a read sees the last commit and never a running exclusive call's writes
+    read: (work) =>
+      readerDB
+        .connection()
+        .execute((db) => withTransaction(db, sql`BEGIN`, () => work(buildSQLiteReader(db)))),
+
+    close: () => handles.disposeAsync(),
+  };
+}
+
+function openStoreHandle(path: string): Database {
+  const handle = new Database(path, { create: true });
+
+  handle.run('PRAGMA journal_mode = WAL');
+  handle.run('PRAGMA synchronous = NORMAL');
+  handle.run('PRAGMA busy_timeout = 5000');
+
+  return handle;
+}
+
+async function withTransaction<T>(
+  db: Kysely<StoreDatabase>,
+  begin: RawBuilder<unknown>,
+  work: () => Promise<T>,
+): Promise<T> {
+  await begin.execute(db);
+
+  try {
+    const result = await work();
+
+    await sql`COMMIT`.execute(db);
+
+    return result;
+  } catch (error) {
+    // SQLite ends the transaction by itself after some errors, such as a full disk, and ROLLBACK
+    // then fails; its error joins the first one, never replaces it
+    await sql`ROLLBACK`.execute(db).catch((rollbackError: unknown) => {
+      throw new AggregateError([error, rollbackError], 'the work failed, and so did its rollback', {
+        cause: error,
+      });
+    });
+
+    throw error;
+  }
+}

package/src/create-sqlite-tx.ts

@@ -0,0 +1,134 @@
+import type { FactId, Instant, PullRequestRef, RunnerId, StoreChange, StoreTx } from '@boxxy/model';
+import type { Kysely } from 'kysely';
+import { applySQLiteChange } from './apply-sqlite-change.ts';
+import { buildSQLiteReader } from './build-sqlite-reader.ts';
+import type { StoreDatabase } from './types.ts';
+
+type PoolCounts = Parameters<StoreTx['syncPoolSlots']>[0];
+
+// A call started beside a change would run its statements inside the change's savepoint, and a
+// refusal would roll them back after the call resolved, so every call waits for the one before it.
+export function createSQLiteTx(db: Kysely<StoreDatabase>): StoreTx {
+  const inTurn = makeTurnRunner();
+  const reader = buildSQLiteReader(db);
+
+  return {
+    readSnapshot: inTurn(reader.readSnapshot),
+    readItem: inTurn(reader.readItem),
+    readItemDetail: inTurn(reader.readItemDetail),
+    listItems: inTurn(reader.listItems),
+    listOpenIssues: inTurn(reader.listOpenIssues),
+    readLastLeaseGen: inTurn(reader.readLastLeaseGen),
+    readAttempt: inTurn(reader.readAttempt),
+    listEvidence: inTurn(reader.listEvidence),
+    listLiveAttempts: inTurn(reader.listLiveAttempts),
+    listSucceededAttemptsWithoutPr: inTurn(reader.listSucceededAttemptsWithoutPr),
+    findAttemptBySession: inTurn(reader.findAttemptBySession),
+    findAttemptByAgentSession: inTurn(reader.findAttemptByAgentSession),
+    listWatchedPullRequests: inTurn(reader.listWatchedPullRequests),
+    listPoolSlots: inTurn(reader.listPoolSlots),
+    readUnsentFacts: inTurn(reader.readUnsentFacts),
+    readOutboxLag: inTurn(reader.readOutboxLag),
+    readRunner: inTurn(reader.readRunner),
+    applyChange: inTurn((change: StoreChange) => applySQLiteChange(db, change)),
+    markFactsSent: inTurn((ids: FactId[], at: Instant) => writeFactsSent(db, ids, at)),
+    writeHeartbeat: inTurn((runner: RunnerId, at: Instant) => writeRunnerBeat(db, runner, at)),
+    syncPoolSlots: inTurn((pools: PoolCounts) => writePoolSlots(db, pools)),
+    markPullRequestChecked: inTurn((ref: PullRequestRef, at: Instant) =>
+      writePullRequestChecked(db, ref, at),
+    ),
+  };
+}
+
+function makeTurnRunner() {
+  let turn: Promise<unknown> = Promise.resolve();
+
+  return <Args extends unknown[], Result>(run: (...args: Args) => Promise<Result>) =>
+    async (...args: Args): Promise<Result> => {
+      const previous = turn;
+      const next = Promise.withResolvers<void>();
+
+      turn = next.promise;
+
+      await previous;
+
+      try {
+        return await run(...args);
+      } finally {
+        next.resolve();
+      }
+    };
+}
+
+async function writeFactsSent(
+  db: Kysely<StoreDatabase>,
+  ids: FactId[],
+  at: Instant,
+): Promise<void> {
+  if (ids.length === 0) {
+    return;
+  }
+
+  await db
+    .updateTable('outbox')
+    .set({ sent_at: at })
+    .where('fact_id', 'in', ids)
+    .where('sent_at', 'is', null)
+    .execute();
+}
+
+async function writeRunnerBeat(
+  db: Kysely<StoreDatabase>,
+  runner: RunnerId,
+  at: Instant,
+): Promise<void> {
+  await db
+    .insertInto('runners')
+    .values({ id: runner, heartbeat_at: at, paused: 0 })
+    .onConflict((conflict) => conflict.column('id').doUpdateSet({ heartbeat_at: at }))
+    .execute();
+}
+
+async function writePoolSlots(db: Kysely<StoreDatabase>, pools: PoolCounts): Promise<void> {
+  for (const pool of pools) {
+    const rows = Array.from({ length: pool.slots }, (_, index) => ({
+      pool: pool.name,
+      slot: index + 1,
+      holder: null,
+      gen: 0,
+      retired: 0,
+    }));
+
+    if (rows.length > 0) {
+      await db
+        .insertInto('pool_slots')
+        .values(rows)
+        .onConflict((conflict) => conflict.columns(['pool', 'slot']).doNothing())
+        .execute();
+    }
+  }
+
+  await db.updateTable('pool_slots').set({ retired: 1 }).execute();
+
+  for (const pool of pools) {
+    await db
+      .updateTable('pool_slots')
+      .set({ retired: 0 })
+      .where('pool', '=', pool.name)
+      .where('slot', '<=', pool.slots)
+      .execute();
+  }
+}
+
+async function writePullRequestChecked(
+  db: Kysely<StoreDatabase>,
+  ref: PullRequestRef,
+  at: Instant,
+): Promise<void> {
+  await db
+    .updateTable('pull_requests')
+    .set({ checked_at: at })
+    .where('repo', '=', ref.repo)
+    .where('number', '=', ref.number)
+    .execute();
+}

package/src/create-tables-migration.ts

@@ -0,0 +1,146 @@
+import { sql } from 'kysely';
+import type { Kysely } from 'kysely';
+import type { Migration } from 'kysely/migration';
+
+// a STRICT table refuses a value of the wrong type, so a bad write fails at the store
+const statements = [
+  sql`CREATE TABLE items (
+    id TEXT PRIMARY KEY,
+    kind TEXT NOT NULL,
+    repo TEXT NOT NULL,
+    title TEXT NOT NULL,
+    outcome TEXT NOT NULL,
+    acceptance TEXT NOT NULL,
+    found_in_attempt TEXT,
+    filed_by TEXT NOT NULL,
+    filed_at INTEGER NOT NULL,
+    closed_at INTEGER,
+    close_reason TEXT
+  ) STRICT`,
+  sql`CREATE TABLE links (
+    source TEXT NOT NULL,
+    target TEXT NOT NULL,
+    type TEXT NOT NULL,
+    added_by TEXT NOT NULL,
+    added_at INTEGER NOT NULL,
+    removed_at INTEGER,
+    PRIMARY KEY (source, target, type, added_at)
+  ) STRICT`,
+  sql`CREATE UNIQUE INDEX links_live ON links (source, target, type) WHERE removed_at IS NULL`,
+  sql`CREATE TABLE levers (
+    seq INTEGER PRIMARY KEY AUTOINCREMENT,
+    kind TEXT NOT NULL,
+    item TEXT NOT NULL,
+    until INTEGER,
+    expires_at INTEGER,
+    reason TEXT,
+    actor TEXT NOT NULL,
+    at INTEGER NOT NULL
+  ) STRICT`,
+  sql`CREATE INDEX levers_item_kind ON levers (item, kind)`,
+  sql`CREATE TABLE attempts (
+    id TEXT PRIMARY KEY,
+    item TEXT NOT NULL,
+    phase TEXT NOT NULL,
+    number INTEGER NOT NULL,
+    runner TEXT NOT NULL,
+    recipe_hash TEXT NOT NULL,
+    rules_version TEXT NOT NULL,
+    pool TEXT NOT NULL,
+    slot INTEGER NOT NULL,
+    slot_gen INTEGER NOT NULL,
+    lease_gen INTEGER NOT NULL,
+    spawn_request TEXT NOT NULL,
+    spawn_request_hash TEXT NOT NULL,
+    state TEXT NOT NULL,
+    failure TEXT,
+    atc_session TEXT,
+    close_out TEXT,
+    claimed_at INTEGER NOT NULL,
+    started_at INTEGER,
+    ended_at INTEGER,
+    UNIQUE (item, phase, number)
+  ) STRICT`,
+  sql`CREATE INDEX attempts_atc_session ON attempts (atc_session)`,
+  sql`CREATE TABLE attempt_sessions (
+    attempt TEXT NOT NULL,
+    agent_session TEXT NOT NULL UNIQUE,
+    atc_session TEXT NOT NULL,
+    seen_at INTEGER NOT NULL,
+    PRIMARY KEY (attempt, agent_session)
+  ) STRICT`,
+  sql`CREATE TABLE leases (
+    item TEXT NOT NULL,
+    phase TEXT NOT NULL,
+    attempt TEXT NOT NULL,
+    gen INTEGER NOT NULL,
+    taken_at INTEGER NOT NULL,
+    released_at INTEGER,
+    PRIMARY KEY (item, phase, gen)
+  ) STRICT`,
+  sql`CREATE UNIQUE INDEX leases_live ON leases (item, phase) WHERE released_at IS NULL`,
+  sql`CREATE TABLE pool_slots (
+    pool TEXT NOT NULL,
+    slot INTEGER NOT NULL,
+    holder TEXT,
+    gen INTEGER NOT NULL,
+    retired INTEGER NOT NULL,
+    PRIMARY KEY (pool, slot)
+  ) STRICT`,
+  sql`CREATE TABLE decisions (
+    id TEXT PRIMARY KEY,
+    attempt TEXT NOT NULL,
+    item TEXT NOT NULL,
+    pool TEXT NOT NULL,
+    rules_version TEXT NOT NULL,
+    snapshot_hash TEXT NOT NULL,
+    store_version INTEGER NOT NULL,
+    pepys_offsets TEXT NOT NULL,
+    faults TEXT NOT NULL,
+    at INTEGER NOT NULL
+  ) STRICT`,
+  sql`CREATE TABLE evidence (
+    fact_id TEXT PRIMARY KEY,
+    attempt TEXT NOT NULL,
+    kind TEXT NOT NULL,
+    ref TEXT NOT NULL,
+    observed TEXT,
+    source TEXT NOT NULL,
+    at INTEGER NOT NULL,
+    UNIQUE (attempt, kind, ref)
+  ) STRICT`,
+  sql`CREATE TABLE pull_requests (
+    repo TEXT NOT NULL,
+    number INTEGER NOT NULL,
+    attempt TEXT NOT NULL,
+    state TEXT NOT NULL,
+    merged_at INTEGER,
+    merge_commit TEXT,
+    checked_at INTEGER NOT NULL,
+    PRIMARY KEY (repo, number)
+  ) STRICT`,
+  sql`CREATE TABLE runners (
+    id TEXT PRIMARY KEY,
+    heartbeat_at INTEGER NOT NULL,
+    paused INTEGER NOT NULL
+  ) STRICT`,
+  sql`CREATE TABLE outbox (
+    seq INTEGER PRIMARY KEY AUTOINCREMENT,
+    fact_id TEXT NOT NULL UNIQUE,
+    type TEXT NOT NULL,
+    entity TEXT NOT NULL,
+    actor TEXT NOT NULL,
+    at INTEGER NOT NULL,
+    payload TEXT NOT NULL,
+    sent_at INTEGER
+  ) STRICT`,
+  sql`CREATE INDEX outbox_unsent ON outbox (seq) WHERE sent_at IS NULL`,
+];
+
+export const createTablesMigration: Migration = {
+  up: async (db: Kysely<unknown>) => {
+    for (const statement of statements) {
+      await statement.execute(db);
+    }
+  },
+};

package/src/index.ts

@@ -1 +1 @@
-export { packageName } from './package-name.ts';
+export { createSQLiteClaimStore, type SQLiteClaimStore } from './create-sqlite-claim-store.ts';