@boxxy/store-sqlite 0.0.0 → 0.1.1

package/README.md

@@ -5,17 +5,38 @@
     <a href="https://www.npmjs.com/package/@boxxy/store-sqlite"><img src="https://img.shields.io/npm/v/@boxxy/store-sqlite" alt="npm version"></a>
     <a href="./LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="MIT license"></a>
   </p>
+
+  <p>
+    <a href="../../docs/design/boxxy/README.md">Documentation</a>
+  </p>
 </div>
 
-boxxy's store on SQLite: the schema, the migrations and the transactions.
+boxxy's store on SQLite: the schema, the migrations and the transactions behind the `ClaimStore`
+port.
 
-This version is a placeholder that exports only its package name, as `packageName`. The
-[boxxy README](https://github.com/zgeoff/boxxy#workspace) lists the workspace's packages.
+`createSQLiteClaimStore(path)` opens one SQLite file, creates it when it's missing, runs the
+migrations the file hasn't run, and returns the store with a `close` function.
 
 ## Install
 
+Install the package from [npm](https://www.npmjs.com/package/@boxxy/store-sqlite) with Bun:
+
 ```bash
 bun add @boxxy/store-sqlite
 ```
 
 The package runs on [Bun](https://bun.sh) 1.3 or newer.
+
+## Features
+
+- Items, levers, attempts, leases, pool slots and the outbox in one file, with each change applied
+  whole or not at all ([one write path](../../docs/design/boxxy/slice-1.md#one-write-path)).
+- `exclusive` calls that run one at a time in a `BEGIN IMMEDIATE` transaction, and `read` calls that
+  see the last commit ([store rules](../../docs/design/boxxy/slice-1.md#store-rules)).
+- Item state derived on each read, from the rows and the loaded rules
+  ([derived item state](../../docs/design/boxxy/slice-1.md#derived-item-state)).
+
+## Documentation
+
+[The store schema](../../docs/design/boxxy/slice-1.md#the-store-schema) lists each table, and
+[`02-ports.ts`](../../docs/design/boxxy/sketches/02-ports.ts) gives the shape of the port.

package/dist/apply-sqlite-change.d.ts

@@ -0,0 +1,4 @@
+import type { ApplyResult, StoreChange } from '@boxxy/model';
+import type { Kysely } from 'kysely';
+import type { StoreDatabase } from './types.ts';
+export declare function applySQLiteChange(db: Kysely<StoreDatabase>, change: StoreChange): Promise<ApplyResult>;

package/dist/build-sqlite-reader.d.ts

@@ -0,0 +1,4 @@
+import type { StoreReader } from '@boxxy/model';
+import type { Kysely } from 'kysely';
+import type { StoreDatabase } from './types.ts';
+export declare function buildSQLiteReader(db: Kysely<StoreDatabase>): StoreReader;

package/dist/create-sqlite-claim-store.d.ts

@@ -0,0 +1,5 @@
+import type { ClaimStore } from '@boxxy/model';
+export interface SQLiteClaimStore extends ClaimStore {
+    close: () => Promise<void>;
+}
+export declare function createSQLiteClaimStore(path: string): Promise<SQLiteClaimStore>;

package/dist/create-sqlite-tx.d.ts

@@ -0,0 +1,4 @@
+import type { StoreTx } from '@boxxy/model';
+import type { Kysely } from 'kysely';
+import type { StoreDatabase } from './types.ts';
+export declare function createSQLiteTx(db: Kysely<StoreDatabase>): StoreTx;

package/dist/create-tables-migration.d.ts

@@ -0,0 +1,2 @@
+import type { Migration } from 'kysely/migration';
+export declare const createTablesMigration: Migration;

package/dist/index.d.ts

@@ -1 +1 @@
-export { packageName } from './package-name.ts';
+export { createSQLiteClaimStore, type SQLiteClaimStore } from './create-sqlite-claim-store.ts';

package/dist/parse-store-row.d.ts

@@ -0,0 +1,2 @@
+import type { TableName, Tables } from '@boxxy/model';
+export declare function parseStoreRow<T extends TableName>(table: T, record: object): Tables[T];

package/dist/read-item-views.d.ts

@@ -0,0 +1,5 @@
+import type { Instant, ItemWithLevers, MaxAttemptsByKind } from '@boxxy/model';
+import type { Expression, ExpressionBuilder, Kysely, SqlBool } from 'kysely';
+import type { StoreDatabase } from './types.ts';
+export type ItemScope = (builder: ExpressionBuilder<StoreDatabase, 'items'>) => Expression<SqlBool>;
+export declare function readItemViews(db: Kysely<StoreDatabase>, scope: ItemScope, now: Instant, maxAttemptsByKind: MaxAttemptsByKind): Promise<ItemWithLevers[]>;

package/dist/run-store-migrations.d.ts

@@ -0,0 +1,3 @@
+import type { Kysely } from 'kysely';
+import type { StoreDatabase } from './types.ts';
+export declare function runStoreMigrations(db: Kysely<StoreDatabase>): Promise<void>;

package/dist/table-codecs.d.ts

@@ -0,0 +1,10 @@
+import type { TableName, Tables } from '@boxxy/model';
+interface TableCodec<Row> {
+    readonly fields: readonly (keyof Row & string)[];
+    readonly json: readonly (keyof Row & string)[];
+    readonly flags: readonly (keyof Row & string)[];
+}
+export declare const tableCodecs: {
+    readonly [T in TableName]: TableCodec<Tables[T]>;
+};
+export {};

package/dist/test-utils/create-test-claim-store.d.ts

@@ -0,0 +1,2 @@
+import type { SQLiteClaimStore } from '../create-sqlite-claim-store.ts';
+export declare function createTestClaimStore(): Promise<SQLiteClaimStore>;

package/dist/to-column-name.d.ts

@@ -0,0 +1 @@
+export declare function toColumnName(field: string): string;

package/dist/to-store-columns.d.ts

@@ -0,0 +1,3 @@
+import type { TableName } from '@boxxy/model';
+export type StoreValue = string | number | null;
+export declare function toStoreColumns(table: TableName, fields: object): [string, StoreValue][];

package/dist/types.d.ts

@@ -0,0 +1,122 @@
+import type { Generated } from 'kysely';
+export interface StoreDatabase {
+    items: {
+        id: string;
+        kind: string;
+        repo: string;
+        title: string;
+        outcome: string;
+        acceptance: string;
+        found_in_attempt: string | null;
+        filed_by: string;
+        filed_at: number;
+        closed_at: number | null;
+        close_reason: string | null;
+    };
+    links: {
+        source: string;
+        target: string;
+        type: string;
+        added_by: string;
+        added_at: number;
+        removed_at: number | null;
+    };
+    levers: {
+        seq: number;
+        kind: string;
+        item: string;
+        until: number | null;
+        expires_at: number | null;
+        reason: string | null;
+        actor: string;
+        at: number;
+    };
+    attempts: {
+        id: string;
+        item: string;
+        phase: string;
+        number: number;
+        runner: string;
+        recipe_hash: string;
+        rules_version: string;
+        pool: string;
+        slot: number;
+        slot_gen: number;
+        lease_gen: number;
+        spawn_request: string;
+        spawn_request_hash: string;
+        state: string;
+        failure: string | null;
+        atc_session: string | null;
+        close_out: string | null;
+        claimed_at: number;
+        started_at: number | null;
+        ended_at: number | null;
+    };
+    attempt_sessions: {
+        attempt: string;
+        agent_session: string;
+        atc_session: string;
+        seen_at: number;
+    };
+    leases: {
+        item: string;
+        phase: string;
+        attempt: string;
+        gen: number;
+        taken_at: number;
+        released_at: number | null;
+    };
+    pool_slots: {
+        pool: string;
+        slot: number;
+        holder: string | null;
+        gen: number;
+        retired: number;
+    };
+    decisions: {
+        id: string;
+        attempt: string;
+        item: string;
+        pool: string;
+        rules_version: string;
+        snapshot_hash: string;
+        store_version: number;
+        pepys_offsets: string;
+        faults: string;
+        at: number;
+    };
+    evidence: {
+        fact_id: string;
+        attempt: string;
+        kind: string;
+        ref: string;
+        observed: string | null;
+        source: string;
+        at: number;
+    };
+    pull_requests: {
+        repo: string;
+        number: number;
+        attempt: string;
+        state: string;
+        merged_at: number | null;
+        merge_commit: string | null;
+        checked_at: number;
+    };
+    runners: {
+        id: string;
+        heartbeat_at: number;
+        paused: number;
+    };
+    outbox: {
+        seq: Generated<number>;
+        fact_id: string;
+        type: string;
+        entity: string;
+        actor: string;
+        at: number;
+        payload: string;
+        sent_at: number | null;
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boxxy/store-sqlite",
-  "version": "0.0.0",
+  "version": "0.1.1",
   "description": "boxxy's store on SQLite",
   "homepage": "https://github.com/zgeoff/boxxy/tree/main/adapters/store-sqlite#readme",
   "bugs": "https://github.com/zgeoff/boxxy/issues",
@@ -29,6 +29,18 @@
     "build": "rm -rf dist && tsrs -p tsconfig.build.json",
     "typecheck": "tsrs --noEmit"
   },
+  "dependencies": {
+    "@boxxy/content-hash": "0.1.0",
+    "@boxxy/model": "0.1.2",
+    "@zgeoff/kysely-bun-sqlite": "0.3.2",
+    "kysely": "0.29.7",
+    "tiny-invariant": "1.3.3",
+    "ts-pattern": "5.9.2",
+    "zod": "4.6.7"
+  },
+  "devDependencies": {
+    "@boxxy/dispatch": "0.1.1"
+  },
   "engines": {
     "bun": ">=1.3",
     "node": ">=24"

package/src/apply-sqlite-change.ts

@@ -0,0 +1,220 @@
+import { buildAppliedChange, findSlotTake, toJSONValue } from '@boxxy/model';
+import type { ApplyResult, RowWrite, SlotTake, StoreChange, TakenSlot } from '@boxxy/model';
+import { sql } from 'kysely';
+import type { Kysely, RawBuilder } from 'kysely';
+import invariant from 'tiny-invariant';
+import { match } from 'ts-pattern';
+import { toStoreColumns } from './to-store-columns.ts';
+import type { StoreValue } from './to-store-columns.ts';
+import type { StoreDatabase } from './types.ts';
+
+// A savepoint, not the transaction, so a refusal keeps the caller's earlier changes in the same
+// exclusive call. The caller runs one change at a time, so one savepoint name is enough.
+export async function applySQLiteChange(
+  db: Kysely<StoreDatabase>,
+  change: StoreChange,
+): Promise<ApplyResult> {
+  invariant(change.facts.length > 0, 'a change writes at least one fact');
+
+  await sql`SAVEPOINT apply_change`.execute(db);
+
+  try {
+    const result = await writeSQLiteChange(db, change);
+
+    if (result.kind !== 'applied') {
+      await sql`ROLLBACK TO apply_change`.execute(db);
+    }
+
+    await sql`RELEASE apply_change`.execute(db);
+
+    return result;
+  } catch (error) {
+    await removeSavepoint(db, error);
+
+    throw error;
+  }
+}
+
+async function writeSQLiteChange(
+  db: Kysely<StoreDatabase>,
+  change: StoreChange,
+): Promise<ApplyResult> {
+  if (!(await probeLiveUnder(db, change.under))) {
+    return { kind: 'stale-generation' };
+  }
+
+  const take = findSlotTake(change.writes);
+  const taken = take === null ? null : await writeSlotTake(db, take);
+
+  if (take !== null && taken === null) {
+    return { kind: 'no-free-slot' };
+  }
+
+  const lastLever = await db
+    .selectFrom('levers')
+    .select((builder) => builder.fn.max('seq').as('seq'))
+    .executeTakeFirst();
+
+  const applied = buildAppliedChange(change, { taken, lastLeverSeq: lastLever?.seq ?? 0 });
+
+  for (const write of applied.writes) {
+    const refusal = await writeSQLiteRow(db, write);
+
+    if (refusal !== null) {
+      return refusal;
+    }
+  }
+
+  for (const fact of applied.facts) {
+    await db
+      .insertInto('outbox')
+      .values({
+        fact_id: fact.id,
+        type: fact.type,
+        entity: fact.entity,
+        actor: fact.actor,
+        at: change.at,
+        payload: JSON.stringify(toJSONValue(fact.payload)),
+        sent_at: null,
+      })
+      .execute();
+  }
+
+  const slot = taken === null ? null : { pool: taken.pool, slot: taken.slot, gen: taken.gen };
+
+  return { kind: 'applied', slot };
+}
+
+async function probeLiveUnder(
+  db: Kysely<StoreDatabase>,
+  under: StoreChange['under'],
+): Promise<boolean> {
+  if (under === undefined) {
+    return true;
+  }
+
+  const lease = await db
+    .selectFrom('leases')
+    .select('gen')
+    .where('item', '=', under.item)
+    .where('phase', '=', under.phase)
+    .where('gen', '=', under.gen)
+    .where('released_at', 'is', null)
+    .executeTakeFirst();
+
+  if (lease === undefined || under.slot === undefined) {
+    return lease !== undefined;
+  }
+
+  const slot = await db
+    .selectFrom('pool_slots')
+    .select('gen')
+    .where('pool', '=', under.slot.pool)
+    .where('slot', '=', under.slot.slot)
+    .where('gen', '=', under.slot.gen)
+    .where('holder', 'is not', null)
+    .executeTakeFirst();
+
+  return slot !== undefined;
+}
+
+async function writeSlotTake(db: Kysely<StoreDatabase>, take: SlotTake): Promise<TakenSlot | null> {
+  const lowest = db
+    .selectFrom('pool_slots')
+    .select((builder) => builder.fn.min('slot').as('slot'))
+    .where('pool', '=', take.pool)
+    .where('holder', 'is', null)
+    .where('retired', '=', 0);
+
+  const row = await db
+    .updateTable('pool_slots')
+    .set((builder) => ({ holder: take.holder, gen: builder('gen', '+', 1) }))
+    .where('pool', '=', take.pool)
+    .where('slot', '=', lowest)
+    .where('holder', 'is', null)
+    .where('retired', '=', 0)
+    .returning(['slot', 'gen'])
+    .executeTakeFirst();
+
+  return row === undefined
+    ? null
+    : { holder: take.holder, pool: take.pool, slot: row.slot, gen: row.gen };
+}
+
+function writeSQLiteRow(db: Kysely<StoreDatabase>, write: RowWrite): Promise<ApplyResult | null> {
+  return match(write)
+    .with({ op: 'take-slot' }, () => Promise.resolve(null))
+    .with({ op: 'release-slot' }, async (release) => {
+      const result = await db
+        .updateTable('pool_slots')
+        .set({ holder: null })
+        .where('pool', '=', release.pool)
+        .where('slot', '=', release.slot)
+        .where('holder', '=', release.holder)
+        .executeTakeFirst();
+
+      return result.numUpdatedRows === 0n ? ({ kind: 'stale-generation' } as const) : null;
+    })
+    .with({ op: 'insert', table: 'leases' }, async (insert) => {
+      const columns = toStoreColumns(insert.table, insert.row);
+
+      // ON CONFLICT names the partial unique index, so SQLite skips a second live lease, not throws
+      const result = await sql`${buildInsert(insert.table, columns)}
+        ON CONFLICT (item, phase) WHERE released_at IS NULL DO NOTHING`.execute(db);
+
+      return result.numAffectedRows === 0n ? ({ kind: 'lease-held' } as const) : null;
+    })
+    .with({ op: 'insert' }, async (insert) => {
+      await buildInsert(insert.table, toStoreColumns(insert.table, insert.row)).execute(db);
+
+      return null;
+    })
+    .with({ op: 'update' }, async (update) => {
+      const set = toStoreColumns(update.table, update.set);
+
+      const guards = [
+        ...toStoreColumns(update.table, update.key),
+        ...toStoreColumns(update.table, update.expect),
+      ];
+
+      invariant(set.length > 0, 'an update sets at least one field');
+
+      const result = await sql`UPDATE ${sql.table(update.table)}
+        SET ${sql.join(set.map(([column, value]) => sql`${sql.ref(column)} = ${value}`))}
+        WHERE ${buildGuards(guards)}`.execute(db);
+
+      return result.numAffectedRows === 0n
+        ? ({ kind: 'guard-mismatch', table: update.table, key: toJSONValue(update.key) } as const)
+        : null;
+    })
+    .exhaustive();
+}
+
+function buildInsert(table: string, columns: [string, StoreValue][]): RawBuilder<unknown> {
+  return sql`INSERT INTO ${sql.table(table)}
+    (${sql.join(columns.map(([column]) => sql.ref(column)))})
+    VALUES (${sql.join(columns.map(([, value]) => value))})`;
+}
+
+// IS compares NULL as a value, so an expect of null matches a NULL column
+function buildGuards(guards: [string, StoreValue][]): RawBuilder<unknown> {
+  invariant(guards.length > 0, 'an update names its key');
+
+  return sql.join(
+    guards.map(([column, value]) => sql`${sql.ref(column)} IS ${value}`),
+    sql` AND `,
+  );
+}
+
+// SQLite rolls the whole transaction back by itself after some errors, such as a full disk, and the
+// savepoint is gone; the rollback's own error then joins the first one, never replaces it.
+async function removeSavepoint(db: Kysely<StoreDatabase>, cause: unknown): Promise<void> {
+  try {
+    await sql`ROLLBACK TO apply_change`.execute(db);
+    await sql`RELEASE apply_change`.execute(db);
+  } catch (rollbackError) {
+    throw new AggregateError([cause, rollbackError], 'the change failed, and so did its rollback', {
+      cause: rollbackError,
+    });
+  }
+}