@boxxy/store-sqlite 0.1.1 → 0.1.4

package/src/migrations/run-store-migrations.ts

@@ -0,0 +1,42 @@
+import type { Kysely } from 'kysely';
+import { Migrator } from 'kysely/migration';
+import { lockWaitLimitMs } from '../constants.ts';
+import type { StoreDatabase } from '../types.ts';
+import { withTransaction } from '../with-transaction.ts';
+import { takeWriteLock } from '../writes/take-write-lock.ts';
+import { createTablesMigration } from './create-tables-migration.ts';
+import { renameRelationsMigration } from './rename-relations-migration.ts';
+import { v1TablesMigration } from './v1-tables-migration.ts';
+
+// Kysely's SQLite migration lock does nothing, so two openers of a new file would both find nothing
+// run; the write lock makes the second one wait and then find the file migrated.
+export async function runStoreMigrations(db: Kysely<StoreDatabase>): Promise<void> {
+  await db.connection().execute((connection) =>
+    withTransaction(
+      connection,
+      (locked) => takeWriteLock(locked, lockWaitLimitMs),
+      () => runMigrator(connection),
+    ),
+  );
+}
+
+// Kysely runs the migrations in the order of their names, and skips each one the file has run.
+const migrations = {
+  '0001-create-tables': createTablesMigration,
+  '0002-rename-relations': renameRelationsMigration,
+  '0003-v1-tables': v1TablesMigration,
+};
+
+async function runMigrator(db: Kysely<StoreDatabase>): Promise<void> {
+  const migrator = new Migrator({
+    db,
+    provider: { getMigrations: () => Promise.resolve(migrations) },
+    disableTransactions: true,
+  });
+
+  const outcome = await migrator.migrateToLatest();
+
+  if (outcome.error !== undefined) {
+    throw new Error('the store migrations failed', { cause: outcome.error });
+  }
+}

package/src/migrations/v1-tables-migration.ts

@@ -0,0 +1,184 @@
+import { sql } from 'kysely';
+import type { Kysely } from 'kysely';
+import type { Migration } from 'kysely/migration';
+
+// Slice 1 never ran outside tests, so this rebuilds each changed table empty, in place of ALTER
+// statements that would need an invented default for every new NOT NULL column. A file that
+// already holds an item or a fact is refused, so no row is ever dropped.
+const statements = [
+  sql`DROP TABLE items`,
+  sql`CREATE TABLE items (
+    id TEXT PRIMARY KEY,
+    kind TEXT NOT NULL,
+    repo TEXT NOT NULL,
+    title TEXT NOT NULL,
+    outcome TEXT NOT NULL,
+    acceptance TEXT NOT NULL,
+    fields TEXT NOT NULL,
+    labels TEXT NOT NULL,
+    found_in_attempt TEXT,
+    filed_by TEXT NOT NULL,
+    filed_at INTEGER NOT NULL,
+    closed_at INTEGER,
+    close_reason TEXT,
+    kept_session TEXT
+  ) STRICT`,
+  sql`DROP TABLE levers`,
+  sql`CREATE TABLE levers (
+    seq INTEGER PRIMARY KEY AUTOINCREMENT,
+    kind TEXT NOT NULL,
+    item TEXT NOT NULL,
+    until INTEGER,
+    expires_at INTEGER,
+    reason TEXT,
+    phase TEXT,
+    actor TEXT NOT NULL,
+    at INTEGER NOT NULL
+  ) STRICT`,
+  sql`CREATE INDEX levers_item_kind ON levers (item, kind)`,
+  sql`DROP TABLE attempts`,
+  sql`CREATE TABLE attempts (
+    id TEXT PRIMARY KEY,
+    item TEXT NOT NULL,
+    phase TEXT NOT NULL,
+    number INTEGER NOT NULL,
+    runner TEXT NOT NULL,
+    recipe_hash TEXT NOT NULL,
+    rules_version TEXT NOT NULL,
+    pool TEXT NOT NULL,
+    lease_gen INTEGER NOT NULL,
+    spawn_request TEXT NOT NULL,
+    spawn_request_hash TEXT NOT NULL,
+    state TEXT NOT NULL,
+    failure TEXT,
+    atc_session TEXT,
+    caller TEXT,
+    claimed_at INTEGER NOT NULL,
+    started_at INTEGER,
+    ended_at INTEGER,
+    fields TEXT NOT NULL,
+    mode TEXT NOT NULL,
+    round INTEGER NOT NULL,
+    input TEXT,
+    hold_gen INTEGER NOT NULL,
+    close_out TEXT,
+    close_out_hash TEXT,
+    close_out_fact TEXT,
+    closing TEXT,
+    closing_at INTEGER,
+    suspend_requested_at INTEGER,
+    slept_at INTEGER,
+    wake_requested_at INTEGER,
+    woken_at INTEGER,
+    outcome TEXT,
+    returned_outcome TEXT,
+    strayed_at INTEGER,
+    session_stopped_at INTEGER,
+    session_forgotten_at INTEGER,
+    UNIQUE (item, phase, number)
+  ) STRICT`,
+  sql`CREATE INDEX attempts_atc_session ON attempts (atc_session)`,
+  sql`DROP TABLE attempt_sessions`,
+
+  // a kept session binds one agent session to each later attempt, so the agent session is no key
+  sql`CREATE TABLE attempt_sessions (
+    attempt TEXT NOT NULL,
+    agent_session TEXT NOT NULL,
+    atc_session TEXT NOT NULL,
+    product TEXT NOT NULL,
+    source TEXT NOT NULL,
+    seen_at INTEGER NOT NULL,
+    PRIMARY KEY (attempt, agent_session)
+  ) STRICT`,
+  sql`CREATE INDEX attempt_sessions_agent_session ON attempt_sessions (agent_session)`,
+  sql`DROP TABLE pool_slots`,
+  sql`CREATE TABLE limits (
+    name TEXT PRIMARY KEY,
+    max INTEGER NOT NULL,
+    counts TEXT NOT NULL
+  ) STRICT`,
+  sql`CREATE TABLE limit_seats (
+    "limit" TEXT NOT NULL,
+    attempt TEXT NOT NULL,
+    hold_gen INTEGER NOT NULL,
+    over_cap INTEGER NOT NULL,
+    taken_at INTEGER NOT NULL,
+    released_at INTEGER
+  ) STRICT`,
+  sql`CREATE UNIQUE INDEX limit_seats_held ON limit_seats ("limit", attempt, hold_gen)
+    WHERE released_at IS NULL`,
+  sql`DROP TABLE decisions`,
+  sql`CREATE TABLE decisions (
+    id TEXT PRIMARY KEY,
+    attempt TEXT NOT NULL,
+    item TEXT NOT NULL,
+    kind TEXT NOT NULL,
+    pool TEXT NOT NULL,
+    rules_version TEXT NOT NULL,
+    snapshot_hash TEXT NOT NULL,
+    store_version INTEGER NOT NULL,
+    pepys_offsets TEXT NOT NULL,
+    faults TEXT NOT NULL,
+    limits TEXT NOT NULL,
+    at INTEGER NOT NULL
+  ) STRICT`,
+  sql`DROP TABLE pull_requests`,
+  sql`CREATE TABLE pull_requests (
+    repo TEXT NOT NULL,
+    number INTEGER NOT NULL,
+    attempt TEXT NOT NULL,
+    github_id INTEGER NOT NULL,
+    url TEXT NOT NULL,
+    state TEXT NOT NULL,
+    merged_at INTEGER,
+    merge_commit TEXT,
+    checked_at INTEGER NOT NULL,
+    PRIMARY KEY (repo, number)
+  ) STRICT`,
+  sql`CREATE TABLE notes (
+    item TEXT NOT NULL,
+    seq INTEGER NOT NULL,
+    text TEXT NOT NULL,
+    "by" TEXT NOT NULL,
+    at INTEGER NOT NULL,
+    PRIMARY KEY (item, seq)
+  ) STRICT`,
+  sql`CREATE TABLE notices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    item TEXT NOT NULL,
+    reason TEXT NOT NULL,
+    at INTEGER NOT NULL,
+    tries INTEGER NOT NULL,
+    sent_at INTEGER
+  ) STRICT`,
+  sql`CREATE INDEX notices_unsent ON notices (id) WHERE sent_at IS NULL`,
+  sql`DROP TABLE outbox`,
+  sql`CREATE TABLE outbox (
+    seq INTEGER PRIMARY KEY AUTOINCREMENT,
+    fact_id TEXT NOT NULL UNIQUE,
+    type TEXT NOT NULL,
+    entity TEXT NOT NULL,
+    actor TEXT NOT NULL,
+    at INTEGER NOT NULL,
+    payload TEXT NOT NULL,
+    sent_at INTEGER,
+    parked_at INTEGER,
+    parked_reason TEXT
+  ) STRICT`,
+  sql`CREATE INDEX outbox_unsent ON outbox (seq) WHERE sent_at IS NULL AND parked_at IS NULL`,
+  sql`CREATE INDEX outbox_behind ON outbox (entity) WHERE sent_at IS NULL`,
+];
+
+export const v1TablesMigration: Migration = {
+  up: async (db: Kysely<unknown>) => {
+    const found = await sql`SELECT 1 FROM items UNION ALL SELECT 1 FROM outbox LIMIT 1`.execute(db);
+
+    if (found.rows.length > 0) {
+      throw new Error('the store holds slice 1 rows, which the v1 tables cannot take');
+    }
+
+    for (const statement of statements) {
+      await statement.execute(db);
+    }
+  },
+};

package/src/reads/build-sqlite-reader.ts

@@ -0,0 +1,484 @@
+import { formatCanonicalJson } from '@boxxy/content-hash';
+import { brandSchemas, limitSeatRowSchema, outboxFactSchema, toJSONValue } from '@boxxy/model';
+import type { ItemView, ItemWithLevers, StoreReader } from '@boxxy/model';
+import type { Kysely } from 'kysely';
+import { parseJSONText } from '../codecs/parse-json-text.ts';
+import { parseStoreRow } from '../codecs/parse-store-row.ts';
+import type { StoreDatabase } from '../types.ts';
+import { isLatestOfSession } from './is-latest-of-session.ts';
+import { readItemViews } from './read-item-views.ts';
+import { readLimitFills } from './read-limit-fills.ts';
+import { readWakes } from './read-wakes.ts';
+
+export function buildSQLiteReader(db: Kysely<StoreDatabase>): StoreReader {
+  return {
+    readSnapshot: async (now, rules) => {
+      const views = await readItemViews(
+        db,
+        (builder) => builder('closed_at', 'is', null),
+        now,
+        rules,
+      );
+
+      const open = views.filter((item) => isOpen(item));
+
+      const relations = await db
+        .selectFrom('relations')
+        .selectAll()
+        .where('removed_at', 'is', null)
+        .orderBy('added_at')
+        .orderBy('source')
+        .orderBy('target')
+        .execute();
+
+      const version = await db
+        .selectFrom('outbox')
+        .select((builder) => builder.fn.max('seq').as('seq'))
+        .executeTakeFirst();
+
+      return {
+        storeVersion: version?.seq ?? 0,
+        items: open.map((item) => toItemView(item)),
+        relations: relations.map((row) => parseStoreRow('relations', row)),
+        levers: open
+          .flatMap((item) => item.levers)
+          .toSorted((first, second) => first.seq - second.seq),
+        limits: await readLimitFills(db),
+        wakes: await readWakes(db),
+      };
+    },
+    readItem: async (id, now, rules) => {
+      const views = await readItemViews(db, (builder) => builder('id', '=', id), now, rules);
+
+      const item = views.at(0);
+
+      return item === undefined ? null : toItemView(item);
+    },
+    readItemDetail: async (id, now, rules) => {
+      const views = await readItemViews(db, (builder) => builder('id', '=', id), now, rules);
+
+      const item = views.at(0);
+
+      if (item === undefined) {
+        return null;
+      }
+
+      const attemptsOfItem = db.selectFrom('attempts').select('id').where('item', '=', id);
+
+      const attempts = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('item', '=', id)
+        .orderBy('claimed_at')
+        .orderBy('id')
+        .execute();
+
+      const relations = await db
+        .selectFrom('relations')
+        .selectAll()
+        .where('removed_at', 'is', null)
+        .where((builder) => builder.or([builder('source', '=', id), builder('target', '=', id)]))
+        .orderBy('added_at')
+        .orderBy('source')
+        .orderBy('target')
+        .execute();
+
+      const evidence = await db
+        .selectFrom('evidence')
+        .selectAll()
+        .where('attempt', 'in', attemptsOfItem)
+        .orderBy('at')
+        .orderBy('fact_id')
+        .execute();
+
+      const pullRequests = await db
+        .selectFrom('pull_requests')
+        .selectAll()
+        .where('attempt', 'in', attemptsOfItem)
+        .orderBy('repo')
+        .orderBy('number')
+        .execute();
+
+      const notes = await db
+        .selectFrom('notes')
+        .selectAll()
+        .where('item', '=', id)
+        .orderBy('seq')
+        .execute();
+
+      return {
+        item,
+        attempts: attempts.map((row) => parseStoreRow('attempts', row)),
+        relations: relations.map((row) => parseStoreRow('relations', row)),
+        evidence: evidence.map((row) => parseStoreRow('evidence', row)),
+        pullRequests: pullRequests.map((row) => parseStoreRow('pull_requests', row)),
+        notes: notes.map((row) => parseStoreRow('notes', row)),
+      };
+    },
+    listItems: async (filter, now, rules) => {
+      const repo = filter.repo;
+
+      const views = await readItemViews(
+        db,
+        (builder) => (repo === undefined ? builder.lit(true) : builder('repo', '=', repo)),
+        now,
+        rules,
+      );
+
+      return views.filter(
+        (item) => filter.state === undefined || item.derived.state === filter.state,
+      );
+    },
+    listOpenIssues: async (repo, now, rules) => {
+      const views = await readItemViews(
+        db,
+        (builder) =>
+          builder.and([
+            builder('kind', '=', 'core.issue'),
+            builder('repo', '=', repo),
+            builder('closed_at', 'is', null),
+          ]),
+        now,
+        rules,
+      );
+
+      return views.filter((item) => isOpen(item)).map((item) => toItemView(item));
+    },
+    readLastLeaseGen: async (item, phase) => {
+      const row = await db
+        .selectFrom('leases')
+        .select((builder) => builder.fn.max('gen').as('gen'))
+        .where('item', '=', item)
+        .where('phase', '=', phase)
+        .executeTakeFirst();
+
+      return row?.gen ?? 0;
+    },
+    readAttempt: async (id) => {
+      const row = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('id', '=', id)
+        .executeTakeFirst();
+
+      return row === undefined ? null : parseStoreRow('attempts', row);
+    },
+    listEvidence: async (attempt) => {
+      const rows = await db
+        .selectFrom('evidence')
+        .selectAll()
+        .where('attempt', '=', attempt)
+        .orderBy('at')
+        .orderBy('fact_id')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('evidence', row));
+    },
+    listItemAttempts: async (item) => {
+      const rows = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('item', '=', item)
+        .orderBy('claimed_at')
+        .orderBy('id')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('attempts', row));
+    },
+    listLiveAttempts: async () => {
+      const rows = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('state', 'not in', ['succeeded', 'failed'])
+        .orderBy('claimed_at')
+        .orderBy('id')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('attempts', row));
+    },
+    listStrayAttempts: async (since) => {
+      const rows = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('strayed_at', '>=', since)
+        .where('session_stopped_at', 'is', null)
+        .orderBy('claimed_at')
+        .orderBy('id')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('attempts', row));
+    },
+    listSucceededAttemptsWithoutPullRequest: async () => {
+      const linked = db.selectFrom('pull_requests').select('attempt');
+
+      const rows = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('state', '=', 'succeeded')
+        .where('id', 'not in', linked)
+        .orderBy('claimed_at')
+        .orderBy('id')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('attempts', row));
+    },
+    // oxlint-disable-next-line zgeoff/verb-contract -- the StoreReader port in @boxxy/model fixes this name
+    findAttemptBySession: async (session) => {
+      const row = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('atc_session', '=', session)
+        .orderBy('claimed_at', 'desc')
+        .orderBy('id', 'desc')
+        .executeTakeFirst();
+
+      return row === undefined ? null : parseStoreRow('attempts', row);
+    },
+    // oxlint-disable-next-line zgeoff/verb-contract -- the StoreReader port in @boxxy/model fixes this name
+    findAttemptByAgentSession: async (session) => {
+      const row = await db
+        .selectFrom('attempts')
+        .select('id')
+        .where(
+          'id',
+          'in',
+          db.selectFrom('attempt_sessions').select('attempt').where('agent_session', '=', session),
+        )
+        .orderBy('claimed_at', 'desc')
+        .orderBy('id', 'desc')
+        .executeTakeFirst();
+
+      return row === undefined ? null : brandSchemas.attemptId.parse(row.id);
+    },
+    // oxlint-disable-next-line zgeoff/verb-contract -- the StoreReader port in @boxxy/model fixes this name
+    findSessionByCaller: async (caller) => {
+      // the store keeps the caller as canonical JSON, so equal callers are equal text
+      const stored = formatCanonicalJson(toJSONValue(caller));
+
+      const rows = await db
+        .selectFrom('attempts')
+        .select('atc_session')
+        .distinct()
+        .where('caller', '=', stored)
+        .where((builder) => isLatestOfSession(builder))
+        .limit(2)
+        .execute();
+
+      if (rows.length > 1) {
+        return 'ambiguous';
+      }
+
+      const session = rows.at(0)?.atc_session;
+
+      return session === undefined || session === null
+        ? null
+        : brandSchemas.atcSessionId.parse(session);
+    },
+    readPullRequestRow: async (ref) => {
+      const row = await db
+        .selectFrom('pull_requests')
+        .selectAll()
+        .where('repo', '=', ref.repo)
+        .where('number', '=', ref.number)
+        .executeTakeFirst();
+
+      return row === undefined ? null : parseStoreRow('pull_requests', row);
+    },
+    listWatchedPullRequests: async () => {
+      const rows = await db
+        .selectFrom('pull_requests')
+        .selectAll()
+        .where('state', '=', 'open')
+        .orderBy('repo')
+        .orderBy('number')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('pull_requests', row));
+    },
+    listLimits: () => readLimitFills(db),
+    listLimitSeats: async () => {
+      const rows = await db
+        .selectFrom('limit_seats')
+        .selectAll()
+        .where('released_at', 'is', null)
+        .orderBy('limit')
+        .orderBy('attempt')
+        .execute();
+
+      return rows.map((row) =>
+        limitSeatRowSchema.parse({
+          limit: row.limit,
+          attempt: row.attempt,
+          holdGen: row.hold_gen,
+          overCap: row.over_cap === 1,
+          takenAt: row.taken_at,
+          releasedAt: row.released_at,
+        }),
+      );
+    },
+    listItemNotes: async (item) => {
+      const rows = await db
+        .selectFrom('notes')
+        .selectAll()
+        .where('item', '=', item)
+        .orderBy('seq')
+        .execute();
+
+      return rows.map((row) => parseStoreRow('notes', row));
+    },
+    listUnsentNotices: async (limit) => {
+      const rows = await db
+        .selectFrom('notices')
+        .selectAll()
+        .where('sent_at', 'is', null)
+        .orderBy('id')
+        .limit(limit)
+        .execute();
+
+      return rows.map((row) => parseStoreRow('notices', row));
+    },
+    listKeptSessions: async () => {
+      const rows = await db
+        .selectFrom('items')
+        .selectAll()
+        .where('kept_session', 'is not', null)
+        .orderBy('filed_at')
+        .orderBy('id')
+        .execute();
+
+      return rows
+        .flatMap((row) => {
+          const item = parseStoreRow('items', row);
+
+          return item.keptSession === null ? [] : [{ item: item.id, kept: item.keptSession }];
+        })
+        .toSorted((first, second) => first.kept.keptAt - second.kept.keptAt);
+    },
+    listSessionsToForget: async (failedEndedBefore) => {
+      const keptRows = await db
+        .selectFrom('items')
+        .selectAll()
+        .where('kept_session', 'is not', null)
+        .execute();
+
+      const kept = new Set(
+        keptRows.flatMap((row) => {
+          const keptSession = parseStoreRow('items', row).keptSession;
+
+          return keptSession === null ? [] : [keptSession.atcSession];
+        }),
+      );
+
+      const rows = await db
+        .selectFrom('attempts')
+        .selectAll()
+        .where('atc_session', 'is not', null)
+        .where('session_forgotten_at', 'is', null)
+        .where((builder) => isLatestOfSession(builder))
+        .where((builder) =>
+          builder.or([
+            builder('strayed_at', 'is', null),
+            builder('session_stopped_at', 'is not', null),
+          ]),
+        )
+        .where((builder) => {
+          const failedBefore = builder.and([
+            builder('state', '=', 'failed'),
+            builder('ended_at', '<=', failedEndedBefore),
+          ]);
+
+          return builder.or([builder('state', '=', 'succeeded'), failedBefore]);
+        })
+        .orderBy('claimed_at')
+        .orderBy('id')
+        .execute();
+
+      return rows
+        .map((row) => parseStoreRow('attempts', row))
+        .filter((attempt) => attempt.atcSession === null || !kept.has(attempt.atcSession));
+    },
+    readRunner: async (id) => {
+      const row = await db
+        .selectFrom('runners')
+        .selectAll()
+        .where('id', '=', id)
+        .executeTakeFirst();
+
+      return row === undefined ? null : parseStoreRow('runners', row);
+    },
+    readUnsentFacts: async (limit) => {
+      const rows = await db
+        .selectFrom('outbox')
+        .selectAll()
+        .where('sent_at', 'is', null)
+        .where('parked_at', 'is', null)
+        .orderBy('seq')
+        .limit(limit)
+        .execute();
+
+      return rows.map((row) =>
+        outboxFactSchema.parse({
+          seq: row.seq,
+          factId: row.fact_id,
+          type: row.type,
+          entity: row.entity,
+          actor: row.actor,
+          at: row.at,
+          payload: parseJSONText(row.payload),
+          sentAt: row.sent_at,
+          parkedAt: row.parked_at,
+          parkedReason: row.parked_reason,
+        }),
+      );
+    },
+    readOutboxCounts: async () => {
+      const waiting = await db
+        .selectFrom('outbox')
+        .select((builder) => [
+          builder.fn.countAll<number>().as('waiting'),
+          builder.fn.count<number>('parked_at').as('parked'),
+        ])
+        .where('sent_at', 'is', null)
+        .executeTakeFirstOrThrow();
+
+      const unsent = await db
+        .selectFrom('outbox')
+        .select((builder) => builder.fn.min('at').as('oldest'))
+        .where('sent_at', 'is', null)
+        .where('parked_at', 'is', null)
+        .executeTakeFirstOrThrow();
+
+      return {
+        unsent: waiting.waiting - waiting.parked,
+        parked: waiting.parked,
+        oldestUnsentAt: unsent.oldest === null ? null : brandSchemas.instant.parse(unsent.oldest),
+      };
+    },
+    readItemBehind: async (item) => {
+      const row = await db
+        .selectFrom('outbox')
+        .select((builder) => [
+          builder.fn.countAll<number>().as('facts'),
+          builder.fn.min('at').as('oldest'),
+        ])
+        .where('sent_at', 'is', null)
+        .where('entity', '=', `boxxy:item:${item}`)
+        .executeTakeFirstOrThrow();
+
+      return {
+        facts: row.facts,
+        oldestAt: row.oldest === null ? null : brandSchemas.instant.parse(row.oldest),
+      };
+    },
+  };
+}
+
+function isOpen(item: ItemView): boolean {
+  return item.derived.state !== 'done' && item.derived.state !== 'archived';
+}
+
+function toItemView(item: ItemWithLevers): ItemView {
+  const { levers: _levers, ...view } = item;
+
+  return view;
+}