@boxxy/store-sqlite 0.1.0 → 0.1.3

package/src/create-sqlite-claim-store.ts

@@ -0,0 +1,114 @@
+import { Database } from 'bun:sqlite';
+import type { ClaimStore, StoreReader } from '@boxxy/model';
+import { buildSQLiteDialect } from '@zgeoff/kysely-bun-sqlite';
+import { Kysely, sql } from 'kysely';
+import { lockWaitLimitMs } from './constants.ts';
+import { runStoreMigrations } from './migrations/run-store-migrations.ts';
+import { buildSQLiteReader } from './reads/build-sqlite-reader.ts';
+import type { StoreDatabase } from './types.ts';
+import { withBusyRetry } from './with-busy-retry.ts';
+import { withTransaction } from './with-transaction.ts';
+import { createSQLiteTx } from './writes/create-sqlite-tx.ts';
+import { takeWriteLock } from './writes/take-write-lock.ts';
+
+export interface SQLiteClaimStore extends ClaimStore {
+  close: () => Promise<void>;
+}
+
+export async function createSQLiteClaimStore(path: string): Promise<SQLiteClaimStore> {
+  await using opened = new AsyncDisposableStack();
+
+  const writer = new Database(path, { readwrite: true, create: true });
+
+  opened.defer(() => {
+    writer.close();
+  });
+
+  writer.run('PRAGMA busy_timeout = 0');
+
+  // The file keeps its journal mode, so a read handle never sets it. SQLite fails these at once,
+  // whatever the busy timeout, while another handle switches the file to WAL or recovers it.
+  await withBusyRetry(lockWaitLimitMs, () => {
+    writer.run('PRAGMA journal_mode = WAL');
+    writer.run('PRAGMA synchronous = NORMAL');
+  });
+
+  const writerDB = new Kysely<StoreDatabase>({ dialect: buildSQLiteDialect(writer) });
+
+  opened.defer(() => writerDB.destroy());
+
+  await runStoreMigrations(writerDB);
+
+  const handles = opened.move();
+
+  const reads = new Set<Promise<unknown>>();
+
+  let isClosed = false;
+
+  return {
+    // Kysely holds the one connection of a handle from acquire to release, so two calls on this
+    // store run one at a time, and the write lock makes another store's writer wait.
+    exclusive: (work) =>
+      writerDB.connection().execute((db) =>
+        withTransaction(
+          db,
+          (locked) => takeWriteLock(locked, lockWaitLimitMs),
+          () => work(createSQLiteTx(db)),
+        ),
+      ),
+
+    read: async (work) => {
+      if (isClosed) {
+        throw new Error('the store is closed');
+      }
+
+      const reading = withReadHandle(path, work);
+
+      reads.add(reading);
+
+      try {
+        return await reading;
+      } finally {
+        reads.delete(reading);
+      }
+    },
+
+    close: async () => {
+      isClosed = true;
+
+      await Promise.allSettled(reads);
+      await handles.disposeAsync();
+    },
+  };
+}
+
+// A handle of its own per call, so a read sees the last commit and never a running exclusive
+// call's writes, and a read started inside another read never waits for the outer one's handle.
+async function withReadHandle<T>(
+  path: string,
+  work: (reader: StoreReader) => Promise<T>,
+): Promise<T> {
+  await using opened = new AsyncDisposableStack();
+
+  // without create, a missing file fails the open, where the default makes an empty file
+  const handle = new Database(path, { readwrite: true, create: false });
+
+  opened.defer(() => {
+    handle.close();
+  });
+
+  handle.run(`PRAGMA busy_timeout = ${lockWaitLimitMs}`);
+  handle.run('PRAGMA synchronous = NORMAL');
+
+  const readerDB = new Kysely<StoreDatabase>({ dialect: buildSQLiteDialect(handle) });
+
+  opened.defer(() => readerDB.destroy());
+
+  return await readerDB.connection().execute((db) =>
+    withTransaction(
+      db,
+      (connection) => sql`BEGIN`.execute(connection),
+      () => work(buildSQLiteReader(db)),
+    ),
+  );
+}

package/src/index.ts

@@ -1 +1 @@
-export { packageName } from './package-name.ts';
+export { createSQLiteClaimStore, type SQLiteClaimStore } from './create-sqlite-claim-store.ts';

package/src/migrations/create-tables-migration.ts

@@ -0,0 +1,146 @@
+import { sql } from 'kysely';
+import type { Kysely } from 'kysely';
+import type { Migration } from 'kysely/migration';
+
+// a STRICT table refuses a value of the wrong type, so a bad write fails at the store
+const statements = [
+  sql`CREATE TABLE items (
+    id TEXT PRIMARY KEY,
+    kind TEXT NOT NULL,
+    repo TEXT NOT NULL,
+    title TEXT NOT NULL,
+    outcome TEXT NOT NULL,
+    acceptance TEXT NOT NULL,
+    found_in_attempt TEXT,
+    filed_by TEXT NOT NULL,
+    filed_at INTEGER NOT NULL,
+    closed_at INTEGER,
+    close_reason TEXT
+  ) STRICT`,
+  sql`CREATE TABLE links (
+    source TEXT NOT NULL,
+    target TEXT NOT NULL,
+    type TEXT NOT NULL,
+    added_by TEXT NOT NULL,
+    added_at INTEGER NOT NULL,
+    removed_at INTEGER,
+    PRIMARY KEY (source, target, type, added_at)
+  ) STRICT`,
+  sql`CREATE UNIQUE INDEX links_live ON links (source, target, type) WHERE removed_at IS NULL`,
+  sql`CREATE TABLE levers (
+    seq INTEGER PRIMARY KEY AUTOINCREMENT,
+    kind TEXT NOT NULL,
+    item TEXT NOT NULL,
+    until INTEGER,
+    expires_at INTEGER,
+    reason TEXT,
+    actor TEXT NOT NULL,
+    at INTEGER NOT NULL
+  ) STRICT`,
+  sql`CREATE INDEX levers_item_kind ON levers (item, kind)`,
+  sql`CREATE TABLE attempts (
+    id TEXT PRIMARY KEY,
+    item TEXT NOT NULL,
+    phase TEXT NOT NULL,
+    number INTEGER NOT NULL,
+    runner TEXT NOT NULL,
+    recipe_hash TEXT NOT NULL,
+    rules_version TEXT NOT NULL,
+    pool TEXT NOT NULL,
+    slot INTEGER NOT NULL,
+    slot_gen INTEGER NOT NULL,
+    lease_gen INTEGER NOT NULL,
+    spawn_request TEXT NOT NULL,
+    spawn_request_hash TEXT NOT NULL,
+    state TEXT NOT NULL,
+    failure TEXT,
+    atc_session TEXT,
+    close_out TEXT,
+    claimed_at INTEGER NOT NULL,
+    started_at INTEGER,
+    ended_at INTEGER,
+    UNIQUE (item, phase, number)
+  ) STRICT`,
+  sql`CREATE INDEX attempts_atc_session ON attempts (atc_session)`,
+  sql`CREATE TABLE attempt_sessions (
+    attempt TEXT NOT NULL,
+    agent_session TEXT NOT NULL UNIQUE,
+    atc_session TEXT NOT NULL,
+    seen_at INTEGER NOT NULL,
+    PRIMARY KEY (attempt, agent_session)
+  ) STRICT`,
+  sql`CREATE TABLE leases (
+    item TEXT NOT NULL,
+    phase TEXT NOT NULL,
+    attempt TEXT NOT NULL,
+    gen INTEGER NOT NULL,
+    taken_at INTEGER NOT NULL,
+    released_at INTEGER,
+    PRIMARY KEY (item, phase, gen)
+  ) STRICT`,
+  sql`CREATE UNIQUE INDEX leases_live ON leases (item, phase) WHERE released_at IS NULL`,
+  sql`CREATE TABLE pool_slots (
+    pool TEXT NOT NULL,
+    slot INTEGER NOT NULL,
+    holder TEXT,
+    gen INTEGER NOT NULL,
+    retired INTEGER NOT NULL,
+    PRIMARY KEY (pool, slot)
+  ) STRICT`,
+  sql`CREATE TABLE decisions (
+    id TEXT PRIMARY KEY,
+    attempt TEXT NOT NULL,
+    item TEXT NOT NULL,
+    pool TEXT NOT NULL,
+    rules_version TEXT NOT NULL,
+    snapshot_hash TEXT NOT NULL,
+    store_version INTEGER NOT NULL,
+    pepys_offsets TEXT NOT NULL,
+    faults TEXT NOT NULL,
+    at INTEGER NOT NULL
+  ) STRICT`,
+  sql`CREATE TABLE evidence (
+    fact_id TEXT PRIMARY KEY,
+    attempt TEXT NOT NULL,
+    kind TEXT NOT NULL,
+    ref TEXT NOT NULL,
+    observed TEXT,
+    source TEXT NOT NULL,
+    at INTEGER NOT NULL,
+    UNIQUE (attempt, kind, ref)
+  ) STRICT`,
+  sql`CREATE TABLE pull_requests (
+    repo TEXT NOT NULL,
+    number INTEGER NOT NULL,
+    attempt TEXT NOT NULL,
+    state TEXT NOT NULL,
+    merged_at INTEGER,
+    merge_commit TEXT,
+    checked_at INTEGER NOT NULL,
+    PRIMARY KEY (repo, number)
+  ) STRICT`,
+  sql`CREATE TABLE runners (
+    id TEXT PRIMARY KEY,
+    heartbeat_at INTEGER NOT NULL,
+    paused INTEGER NOT NULL
+  ) STRICT`,
+  sql`CREATE TABLE outbox (
+    seq INTEGER PRIMARY KEY AUTOINCREMENT,
+    fact_id TEXT NOT NULL UNIQUE,
+    type TEXT NOT NULL,
+    entity TEXT NOT NULL,
+    actor TEXT NOT NULL,
+    at INTEGER NOT NULL,
+    payload TEXT NOT NULL,
+    sent_at INTEGER
+  ) STRICT`,
+  sql`CREATE INDEX outbox_unsent ON outbox (seq) WHERE sent_at IS NULL`,
+];
+
+export const createTablesMigration: Migration = {
+  up: async (db: Kysely<unknown>) => {
+    for (const statement of statements) {
+      await statement.execute(db);
+    }
+  },
+};

package/src/migrations/rename-relations-migration.ts

@@ -0,0 +1,17 @@
+import { sql } from 'kysely';
+import type { Kysely } from 'kysely';
+import type { Migration } from 'kysely/migration';
+
+const statements = [
+  sql`ALTER TABLE links RENAME TO relations`,
+  sql`DROP INDEX links_live`,
+  sql`CREATE UNIQUE INDEX relations_live ON relations (source, target, type) WHERE removed_at IS NULL`,
+];
+
+export const renameRelationsMigration: Migration = {
+  up: async (db: Kysely<unknown>) => {
+    for (const statement of statements) {
+      await statement.execute(db);
+    }
+  },
+};

package/src/migrations/run-store-migrations.ts

@@ -0,0 +1,42 @@
+import type { Kysely } from 'kysely';
+import { Migrator } from 'kysely/migration';
+import { lockWaitLimitMs } from '../constants.ts';
+import type { StoreDatabase } from '../types.ts';
+import { withTransaction } from '../with-transaction.ts';
+import { takeWriteLock } from '../writes/take-write-lock.ts';
+import { createTablesMigration } from './create-tables-migration.ts';
+import { renameRelationsMigration } from './rename-relations-migration.ts';
+import { v1TablesMigration } from './v1-tables-migration.ts';
+
+// Kysely's SQLite migration lock does nothing, so two openers of a new file would both find nothing
+// run; the write lock makes the second one wait and then find the file migrated.
+export async function runStoreMigrations(db: Kysely<StoreDatabase>): Promise<void> {
+  await db.connection().execute((connection) =>
+    withTransaction(
+      connection,
+      (locked) => takeWriteLock(locked, lockWaitLimitMs),
+      () => runMigrator(connection),
+    ),
+  );
+}
+
+// Kysely runs the migrations in the order of their names, and skips each one the file has run.
+const migrations = {
+  '0001-create-tables': createTablesMigration,
+  '0002-rename-relations': renameRelationsMigration,
+  '0003-v1-tables': v1TablesMigration,
+};
+
+async function runMigrator(db: Kysely<StoreDatabase>): Promise<void> {
+  const migrator = new Migrator({
+    db,
+    provider: { getMigrations: () => Promise.resolve(migrations) },
+    disableTransactions: true,
+  });
+
+  const outcome = await migrator.migrateToLatest();
+
+  if (outcome.error !== undefined) {
+    throw new Error('the store migrations failed', { cause: outcome.error });
+  }
+}

package/src/migrations/v1-tables-migration.ts

@@ -0,0 +1,184 @@
+import { sql } from 'kysely';
+import type { Kysely } from 'kysely';
+import type { Migration } from 'kysely/migration';
+
+// Slice 1 never ran outside tests, so this rebuilds each changed table empty, in place of ALTER
+// statements that would need an invented default for every new NOT NULL column. A file that
+// already holds an item or a fact is refused, so no row is ever dropped.
+const statements = [
+  sql`DROP TABLE items`,
+  sql`CREATE TABLE items (
+    id TEXT PRIMARY KEY,
+    kind TEXT NOT NULL,
+    repo TEXT NOT NULL,
+    title TEXT NOT NULL,
+    outcome TEXT NOT NULL,
+    acceptance TEXT NOT NULL,
+    fields TEXT NOT NULL,
+    labels TEXT NOT NULL,
+    found_in_attempt TEXT,
+    filed_by TEXT NOT NULL,
+    filed_at INTEGER NOT NULL,
+    closed_at INTEGER,
+    close_reason TEXT,
+    kept_session TEXT
+  ) STRICT`,
+  sql`DROP TABLE levers`,
+  sql`CREATE TABLE levers (
+    seq INTEGER PRIMARY KEY AUTOINCREMENT,
+    kind TEXT NOT NULL,
+    item TEXT NOT NULL,
+    until INTEGER,
+    expires_at INTEGER,
+    reason TEXT,
+    phase TEXT,
+    actor TEXT NOT NULL,
+    at INTEGER NOT NULL
+  ) STRICT`,
+  sql`CREATE INDEX levers_item_kind ON levers (item, kind)`,
+  sql`DROP TABLE attempts`,
+  sql`CREATE TABLE attempts (
+    id TEXT PRIMARY KEY,
+    item TEXT NOT NULL,
+    phase TEXT NOT NULL,
+    number INTEGER NOT NULL,
+    runner TEXT NOT NULL,
+    recipe_hash TEXT NOT NULL,
+    rules_version TEXT NOT NULL,
+    pool TEXT NOT NULL,
+    lease_gen INTEGER NOT NULL,
+    spawn_request TEXT NOT NULL,
+    spawn_request_hash TEXT NOT NULL,
+    state TEXT NOT NULL,
+    failure TEXT,
+    atc_session TEXT,
+    caller TEXT,
+    claimed_at INTEGER NOT NULL,
+    started_at INTEGER,
+    ended_at INTEGER,
+    fields TEXT NOT NULL,
+    mode TEXT NOT NULL,
+    round INTEGER NOT NULL,
+    input TEXT,
+    hold_gen INTEGER NOT NULL,
+    close_out TEXT,
+    close_out_hash TEXT,
+    close_out_fact TEXT,
+    closing TEXT,
+    closing_at INTEGER,
+    suspend_requested_at INTEGER,
+    slept_at INTEGER,
+    wake_requested_at INTEGER,
+    woken_at INTEGER,
+    outcome TEXT,
+    returned_outcome TEXT,
+    strayed_at INTEGER,
+    session_stopped_at INTEGER,
+    session_forgotten_at INTEGER,
+    UNIQUE (item, phase, number)
+  ) STRICT`,
+  sql`CREATE INDEX attempts_atc_session ON attempts (atc_session)`,
+  sql`DROP TABLE attempt_sessions`,
+
+  // a kept session binds one agent session to each later attempt, so the agent session is no key
+  sql`CREATE TABLE attempt_sessions (
+    attempt TEXT NOT NULL,
+    agent_session TEXT NOT NULL,
+    atc_session TEXT NOT NULL,
+    product TEXT NOT NULL,
+    source TEXT NOT NULL,
+    seen_at INTEGER NOT NULL,
+    PRIMARY KEY (attempt, agent_session)
+  ) STRICT`,
+  sql`CREATE INDEX attempt_sessions_agent_session ON attempt_sessions (agent_session)`,
+  sql`DROP TABLE pool_slots`,
+  sql`CREATE TABLE limits (
+    name TEXT PRIMARY KEY,
+    max INTEGER NOT NULL,
+    counts TEXT NOT NULL
+  ) STRICT`,
+  sql`CREATE TABLE limit_seats (
+    "limit" TEXT NOT NULL,
+    attempt TEXT NOT NULL,
+    hold_gen INTEGER NOT NULL,
+    over_cap INTEGER NOT NULL,
+    taken_at INTEGER NOT NULL,
+    released_at INTEGER
+  ) STRICT`,
+  sql`CREATE UNIQUE INDEX limit_seats_held ON limit_seats ("limit", attempt, hold_gen)
+    WHERE released_at IS NULL`,
+  sql`DROP TABLE decisions`,
+  sql`CREATE TABLE decisions (
+    id TEXT PRIMARY KEY,
+    attempt TEXT NOT NULL,
+    item TEXT NOT NULL,
+    kind TEXT NOT NULL,
+    pool TEXT NOT NULL,
+    rules_version TEXT NOT NULL,
+    snapshot_hash TEXT NOT NULL,
+    store_version INTEGER NOT NULL,
+    pepys_offsets TEXT NOT NULL,
+    faults TEXT NOT NULL,
+    limits TEXT NOT NULL,
+    at INTEGER NOT NULL
+  ) STRICT`,
+  sql`DROP TABLE pull_requests`,
+  sql`CREATE TABLE pull_requests (
+    repo TEXT NOT NULL,
+    number INTEGER NOT NULL,
+    attempt TEXT NOT NULL,
+    github_id INTEGER NOT NULL,
+    url TEXT NOT NULL,
+    state TEXT NOT NULL,
+    merged_at INTEGER,
+    merge_commit TEXT,
+    checked_at INTEGER NOT NULL,
+    PRIMARY KEY (repo, number)
+  ) STRICT`,
+  sql`CREATE TABLE notes (
+    item TEXT NOT NULL,
+    seq INTEGER NOT NULL,
+    text TEXT NOT NULL,
+    "by" TEXT NOT NULL,
+    at INTEGER NOT NULL,
+    PRIMARY KEY (item, seq)
+  ) STRICT`,
+  sql`CREATE TABLE notices (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    item TEXT NOT NULL,
+    reason TEXT NOT NULL,
+    at INTEGER NOT NULL,
+    tries INTEGER NOT NULL,
+    sent_at INTEGER
+  ) STRICT`,
+  sql`CREATE INDEX notices_unsent ON notices (id) WHERE sent_at IS NULL`,
+  sql`DROP TABLE outbox`,
+  sql`CREATE TABLE outbox (
+    seq INTEGER PRIMARY KEY AUTOINCREMENT,
+    fact_id TEXT NOT NULL UNIQUE,
+    type TEXT NOT NULL,
+    entity TEXT NOT NULL,
+    actor TEXT NOT NULL,
+    at INTEGER NOT NULL,
+    payload TEXT NOT NULL,
+    sent_at INTEGER,
+    parked_at INTEGER,
+    parked_reason TEXT
+  ) STRICT`,
+  sql`CREATE INDEX outbox_unsent ON outbox (seq) WHERE sent_at IS NULL AND parked_at IS NULL`,
+  sql`CREATE INDEX outbox_behind ON outbox (entity) WHERE sent_at IS NULL`,
+];
+
+export const v1TablesMigration: Migration = {
+  up: async (db: Kysely<unknown>) => {
+    const found = await sql`SELECT 1 FROM items UNION ALL SELECT 1 FROM outbox LIMIT 1`.execute(db);
+
+    if (found.rows.length > 0) {
+      throw new Error('the store holds slice 1 rows, which the v1 tables cannot take');
+    }
+
+    for (const statement of statements) {
+      await statement.execute(db);
+    }
+  },
+};