@boxxy/store-sqlite 0.1.1 → 0.1.4

package/README.md

@@ -29,14 +29,16 @@ The package runs on [Bun](https://bun.sh) 1.3 or newer.
 
 ## Features
 
-- Items, levers, attempts, leases, pool slots and the outbox in one file, with each change applied
-  whole or not at all ([one write path](../../docs/design/boxxy/slice-1.md#one-write-path)).
+- Items, levers, attempts, leases, limit seats, notes, notices and the outbox in one file, with each
+  change applied whole or not at all
+  ([one write path](../../docs/design/boxxy/slice-1.md#one-write-path)).
 - `exclusive` calls that run one at a time in a `BEGIN IMMEDIATE` transaction, and `read` calls that
   see the last commit ([store rules](../../docs/design/boxxy/slice-1.md#store-rules)).
-- Item state derived on each read, from the rows and the loaded rules
-  ([derived item state](../../docs/design/boxxy/slice-1.md#derived-item-state)).
+- Each item's position and state derived on each read, from the rows and the loaded rules
+  ([v1](../../docs/design/boxxy/v1.md)).
 
 ## Documentation
 
-[The store schema](../../docs/design/boxxy/slice-1.md#the-store-schema) lists each table, and
-[`02-ports.ts`](../../docs/design/boxxy/sketches/02-ports.ts) gives the shape of the port.
+The [v1 tables migration](./src/migrations/v1-tables-migration.ts) creates each table, and the store
+section of [`02-types.ts`](../../docs/design/boxxy/sketches/v1/core/02-types.ts) gives the port, the
+seat take and the seat sync.

package/dist/codecs/parse-json-text.d.ts

@@ -0,0 +1 @@
+export declare function parseJSONText(text: string): unknown;

package/dist/constants.d.ts

@@ -0,0 +1 @@
+export declare const lockWaitLimitMs = 5000;

package/dist/migrations/rename-relations-migration.d.ts

@@ -0,0 +1,2 @@
+import type { Migration } from 'kysely/migration';
+export declare const renameRelationsMigration: Migration;

package/dist/{run-store-migrations.d.ts → migrations/run-store-migrations.d.ts}

@@ -1,3 +1,3 @@
 import type { Kysely } from 'kysely';
-import type { StoreDatabase } from './types.ts';
+import type { StoreDatabase } from '../types.ts';
 export declare function runStoreMigrations(db: Kysely<StoreDatabase>): Promise<void>;

package/dist/migrations/v1-tables-migration.d.ts

@@ -0,0 +1,2 @@
+import type { Migration } from 'kysely/migration';
+export declare const v1TablesMigration: Migration;

package/dist/{build-sqlite-reader.d.ts → reads/build-sqlite-reader.d.ts}

@@ -1,4 +1,4 @@
 import type { StoreReader } from '@boxxy/model';
 import type { Kysely } from 'kysely';
-import type { StoreDatabase } from './types.ts';
+import type { StoreDatabase } from '../types.ts';
 export declare function buildSQLiteReader(db: Kysely<StoreDatabase>): StoreReader;

package/dist/reads/is-latest-of-session.d.ts

@@ -0,0 +1,3 @@
+import type { Expression, ExpressionBuilder, SqlBool } from 'kysely';
+import type { StoreDatabase } from '../types.ts';
+export declare function isLatestOfSession(builder: ExpressionBuilder<StoreDatabase, 'attempts'>): Expression<SqlBool>;

package/dist/{read-item-views.d.ts → reads/read-item-views.d.ts}

@@ -1,5 +1,5 @@
-import type { Instant, ItemWithLevers, MaxAttemptsByKind } from '@boxxy/model';
+import type { Instant, ItemRules, ItemWithLevers } from '@boxxy/model';
 import type { Expression, ExpressionBuilder, Kysely, SqlBool } from 'kysely';
-import type { StoreDatabase } from './types.ts';
+import type { StoreDatabase } from '../types.ts';
 export type ItemScope = (builder: ExpressionBuilder<StoreDatabase, 'items'>) => Expression<SqlBool>;
-export declare function readItemViews(db: Kysely<StoreDatabase>, scope: ItemScope, now: Instant, maxAttemptsByKind: MaxAttemptsByKind): Promise<ItemWithLevers[]>;
+export declare function readItemViews(db: Kysely<StoreDatabase>, scope: ItemScope, now: Instant, rules: ItemRules): Promise<ItemWithLevers[]>;

package/dist/reads/read-limit-fills.d.ts

@@ -0,0 +1,4 @@
+import type { LimitFill } from '@boxxy/model';
+import type { Kysely } from 'kysely';
+import type { StoreDatabase } from '../types.ts';
+export declare function readLimitFills(db: Kysely<StoreDatabase>): Promise<LimitFill[]>;

package/dist/reads/read-wakes.d.ts

@@ -0,0 +1,4 @@
+import type { Wake } from '@boxxy/model';
+import type { Kysely } from 'kysely';
+import type { StoreDatabase } from '../types.ts';
+export declare function readWakes(db: Kysely<StoreDatabase>): Promise<Wake[]>;

package/dist/test-utils/create-claim-bench.d.ts

@@ -0,0 +1,8 @@
+import type { ItemId } from '@boxxy/model';
+export interface ClaimBenchOptions {
+    readonly itemCount: number;
+}
+export interface ClaimBench extends AsyncDisposable {
+    readonly claim: () => Promise<ItemId>;
+}
+export declare function createClaimBench(options: ClaimBenchOptions): Promise<ClaimBench>;

package/dist/types.d.ts

@@ -7,13 +7,16 @@ export interface StoreDatabase {
         title: string;
         outcome: string;
         acceptance: string;
+        fields: string;
+        labels: string;
         found_in_attempt: string | null;
         filed_by: string;
         filed_at: number;
         closed_at: number | null;
         close_reason: string | null;
+        kept_session: string | null;
     };
-    links: {
+    relations: {
         source: string;
         target: string;
         type: string;
@@ -28,6 +31,7 @@ export interface StoreDatabase {
         until: number | null;
         expires_at: number | null;
         reason: string | null;
+        phase: string | null;
         actor: string;
         at: number;
     };
@@ -40,23 +44,42 @@ export interface StoreDatabase {
         recipe_hash: string;
         rules_version: string;
         pool: string;
-        slot: number;
-        slot_gen: number;
         lease_gen: number;
         spawn_request: string;
         spawn_request_hash: string;
         state: string;
         failure: string | null;
         atc_session: string | null;
-        close_out: string | null;
+        caller: string | null;
         claimed_at: number;
         started_at: number | null;
         ended_at: number | null;
+        fields: string;
+        mode: string;
+        round: number;
+        input: string | null;
+        hold_gen: number;
+        close_out: string | null;
+        close_out_hash: string | null;
+        close_out_fact: string | null;
+        closing: string | null;
+        closing_at: number | null;
+        suspend_requested_at: number | null;
+        slept_at: number | null;
+        wake_requested_at: number | null;
+        woken_at: number | null;
+        outcome: string | null;
+        returned_outcome: string | null;
+        strayed_at: number | null;
+        session_stopped_at: number | null;
+        session_forgotten_at: number | null;
     };
     attempt_sessions: {
         attempt: string;
         agent_session: string;
         atc_session: string;
+        product: string;
+        source: string;
         seen_at: number;
     };
     leases: {
@@ -67,23 +90,31 @@ export interface StoreDatabase {
         taken_at: number;
         released_at: number | null;
     };
-    pool_slots: {
-        pool: string;
-        slot: number;
-        holder: string | null;
-        gen: number;
-        retired: number;
+    limits: {
+        name: string;
+        max: number;
+        counts: string;
+    };
+    limit_seats: {
+        limit: string;
+        attempt: string;
+        hold_gen: number;
+        over_cap: number;
+        taken_at: number;
+        released_at: number | null;
     };
     decisions: {
         id: string;
         attempt: string;
         item: string;
+        kind: string;
         pool: string;
         rules_version: string;
         snapshot_hash: string;
         store_version: number;
         pepys_offsets: string;
         faults: string;
+        limits: string;
         at: number;
     };
     evidence: {
@@ -99,6 +130,8 @@ export interface StoreDatabase {
         repo: string;
         number: number;
         attempt: string;
+        github_id: number;
+        url: string;
         state: string;
         merged_at: number | null;
         merge_commit: string | null;
@@ -109,6 +142,21 @@ export interface StoreDatabase {
         heartbeat_at: number;
         paused: number;
     };
+    notes: {
+        item: string;
+        seq: number;
+        text: string;
+        by: string;
+        at: number;
+    };
+    notices: {
+        id: Generated<number>;
+        item: string;
+        reason: string;
+        at: number;
+        tries: number;
+        sent_at: number | null;
+    };
     outbox: {
         seq: Generated<number>;
         fact_id: string;
@@ -118,5 +166,7 @@ export interface StoreDatabase {
         at: number;
         payload: string;
         sent_at: number | null;
+        parked_at: number | null;
+        parked_reason: string | null;
     };
 }

package/dist/with-busy-retry.d.ts

@@ -0,0 +1 @@
+export declare function withBusyRetry(waitLimitMs: number, work: () => unknown): Promise<void>;

package/dist/with-transaction.d.ts

@@ -0,0 +1,3 @@
+import type { Kysely } from 'kysely';
+import type { StoreDatabase } from './types.ts';
+export declare function withTransaction<T>(db: Kysely<StoreDatabase>, begin: (db: Kysely<StoreDatabase>) => Promise<unknown>, work: () => Promise<T>): Promise<T>;

package/dist/{apply-sqlite-change.d.ts → writes/apply-sqlite-change.d.ts}

@@ -1,4 +1,4 @@
 import type { ApplyResult, StoreChange } from '@boxxy/model';
 import type { Kysely } from 'kysely';
-import type { StoreDatabase } from './types.ts';
+import type { StoreDatabase } from '../types.ts';
 export declare function applySQLiteChange(db: Kysely<StoreDatabase>, change: StoreChange): Promise<ApplyResult>;

package/dist/{create-sqlite-tx.d.ts → writes/create-sqlite-tx.d.ts}

@@ -1,4 +1,4 @@
 import type { StoreTx } from '@boxxy/model';
 import type { Kysely } from 'kysely';
-import type { StoreDatabase } from './types.ts';
+import type { StoreDatabase } from '../types.ts';
 export declare function createSQLiteTx(db: Kysely<StoreDatabase>): StoreTx;

package/dist/writes/take-write-lock.d.ts

@@ -0,0 +1,3 @@
+import type { Kysely } from 'kysely';
+import type { StoreDatabase } from '../types.ts';
+export declare function takeWriteLock(db: Kysely<StoreDatabase>, waitLimitMs: number): Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@boxxy/store-sqlite",
-  "version": "0.1.1",
+  "version": "0.1.4",
   "description": "boxxy's store on SQLite",
   "homepage": "https://github.com/zgeoff/boxxy/tree/main/adapters/store-sqlite#readme",
   "bugs": "https://github.com/zgeoff/boxxy/issues",
@@ -26,20 +26,30 @@
     "access": "public"
   },
   "scripts": {
+    "bench": "rig bench",
     "build": "rm -rf dist && tsrs -p tsconfig.build.json",
     "typecheck": "tsrs --noEmit"
   },
   "dependencies": {
-    "@boxxy/content-hash": "0.1.0",
-    "@boxxy/model": "0.1.2",
+    "@boxxy/content-hash": "0.1.4",
+    "@boxxy/model": "0.1.4",
     "@zgeoff/kysely-bun-sqlite": "0.3.2",
-    "kysely": "0.29.7",
+    "kysely": "0.29.8",
     "tiny-invariant": "1.3.3",
     "ts-pattern": "5.9.2",
     "zod": "4.6.7"
   },
   "devDependencies": {
-    "@boxxy/dispatch": "0.1.1"
+    "@boxxy/dispatch": "0.1.4",
+    "@zgeoff/rig": "0.2.3"
+  },
+  "peerDependencies": {
+    "@boxxy/dispatch": "0.1.4"
+  },
+  "peerDependenciesMeta": {
+    "@boxxy/dispatch": {
+      "optional": true
+    }
   },
   "engines": {
     "bun": ">=1.3",

package/src/codecs/parse-json-text.ts

@@ -0,0 +1,5 @@
+export function parseJSONText(text: string): unknown {
+  const parsed: unknown = JSON.parse(text);
+
+  return parsed;
+}

package/src/{table-codecs.ts → codecs/table-codecs.ts}

@@ -17,22 +17,25 @@ export const tableCodecs: { readonly [T in TableName]: TableCodec<Tables[T]> } =
       'title',
       'outcome',
       'acceptance',
+      'fields',
+      'labels',
       'foundInAttempt',
       'filedBy',
       'filedAt',
       'closedAt',
       'closeReason',
+      'keptSession',
     ],
-    json: ['acceptance'],
+    json: ['acceptance', 'fields', 'labels', 'keptSession'],
     flags: [],
   },
-  links: {
+  relations: {
     fields: ['source', 'target', 'type', 'addedBy', 'addedAt', 'removedAt'],
     json: [],
     flags: [],
   },
   levers: {
-    fields: ['seq', 'kind', 'item', 'until', 'expiresAt', 'reason', 'actor', 'at'],
+    fields: ['seq', 'kind', 'item', 'until', 'expiresAt', 'reason', 'phase', 'actor', 'at'],
     json: [],
     flags: [],
   },
@@ -46,24 +49,41 @@ export const tableCodecs: { readonly [T in TableName]: TableCodec<Tables[T]> } =
       'recipeHash',
       'rulesVersion',
       'pool',
-      'slot',
-      'slotGen',
       'leaseGen',
       'spawnRequest',
       'spawnRequestHash',
       'state',
       'failure',
       'atcSession',
-      'closeOut',
+      'caller',
       'claimedAt',
       'startedAt',
       'endedAt',
+      'fields',
+      'mode',
+      'round',
+      'input',
+      'holdGen',
+      'closeOut',
+      'closeOutHash',
+      'closeOutFact',
+      'closing',
+      'closingAt',
+      'suspendRequestedAt',
+      'sleptAt',
+      'wakeRequestedAt',
+      'wokenAt',
+      'outcome',
+      'returnedOutcome',
+      'strayedAt',
+      'sessionStoppedAt',
+      'sessionForgottenAt',
     ],
-    json: ['spawnRequest', 'closeOut'],
+    json: ['spawnRequest', 'caller', 'fields', 'input', 'closeOut'],
     flags: [],
   },
   attempt_sessions: {
-    fields: ['attempt', 'agentSession', 'atcSession', 'seenAt'],
+    fields: ['attempt', 'agentSession', 'atcSession', 'product', 'source', 'seenAt'],
     json: [],
     flags: [],
   },
@@ -72,25 +92,22 @@ export const tableCodecs: { readonly [T in TableName]: TableCodec<Tables[T]> } =
     json: [],
     flags: [],
   },
-  pool_slots: {
-    fields: ['pool', 'slot', 'holder', 'gen', 'retired'],
-    json: [],
-    flags: ['retired'],
-  },
   decisions: {
     fields: [
       'id',
       'attempt',
       'item',
+      'kind',
       'pool',
       'rulesVersion',
       'snapshotHash',
       'storeVersion',
       'pepysOffsets',
       'faults',
+      'limits',
       'at',
     ],
-    json: ['pepysOffsets', 'faults'],
+    json: ['pepysOffsets', 'faults', 'limits'],
     flags: [],
   },
   evidence: {
@@ -99,7 +116,17 @@ export const tableCodecs: { readonly [T in TableName]: TableCodec<Tables[T]> } =
     flags: [],
   },
   pull_requests: {
-    fields: ['repo', 'number', 'attempt', 'state', 'mergedAt', 'mergeCommit', 'checkedAt'],
+    fields: [
+      'repo',
+      'number',
+      'attempt',
+      'githubId',
+      'url',
+      'state',
+      'mergedAt',
+      'mergeCommit',
+      'checkedAt',
+    ],
     json: [],
     flags: [],
   },
@@ -108,4 +135,14 @@ export const tableCodecs: { readonly [T in TableName]: TableCodec<Tables[T]> } =
     json: [],
     flags: ['paused'],
   },
+  notes: {
+    fields: ['item', 'seq', 'text', 'by', 'at'],
+    json: [],
+    flags: [],
+  },
+  notices: {
+    fields: ['id', 'item', 'reason', 'at', 'tries', 'sentAt'],
+    json: [],
+    flags: [],
+  },
 };

package/src/constants.ts

@@ -0,0 +1 @@
+export const lockWaitLimitMs = 5000;

package/src/create-sqlite-claim-store.ts

@@ -1,12 +1,15 @@
 import { Database } from 'bun:sqlite';
-import type { ClaimStore } from '@boxxy/model';
+import type { ClaimStore, StoreReader } from '@boxxy/model';
 import { buildSQLiteDialect } from '@zgeoff/kysely-bun-sqlite';
 import { Kysely, sql } from 'kysely';
-import type { RawBuilder } from 'kysely';
-import { buildSQLiteReader } from './build-sqlite-reader.ts';
-import { createSQLiteTx } from './create-sqlite-tx.ts';
-import { runStoreMigrations } from './run-store-migrations.ts';
+import { lockWaitLimitMs } from './constants.ts';
+import { runStoreMigrations } from './migrations/run-store-migrations.ts';
+import { buildSQLiteReader } from './reads/build-sqlite-reader.ts';
 import type { StoreDatabase } from './types.ts';
+import { withBusyRetry } from './with-busy-retry.ts';
+import { withTransaction } from './with-transaction.ts';
+import { createSQLiteTx } from './writes/create-sqlite-tx.ts';
+import { takeWriteLock } from './writes/take-write-lock.ts';
 
 export interface SQLiteClaimStore extends ClaimStore {
   close: () => Promise<void>;
@@ -15,78 +18,97 @@ export interface SQLiteClaimStore extends ClaimStore {
 export async function createSQLiteClaimStore(path: string): Promise<SQLiteClaimStore> {
   await using opened = new AsyncDisposableStack();
 
-  const writer = openStoreHandle(path);
+  const writer = new Database(path, { readwrite: true, create: true });
 
   opened.defer(() => {
     writer.close();
   });
 
-  const reader = openStoreHandle(path);
+  writer.run('PRAGMA busy_timeout = 0');
 
-  opened.defer(() => {
-    reader.close();
+  // The file keeps its journal mode, so a read handle never sets it. SQLite fails these at once,
+  // whatever the busy timeout, while another handle switches the file to WAL or recovers it.
+  await withBusyRetry(lockWaitLimitMs, () => {
+    writer.run('PRAGMA journal_mode = WAL');
+    writer.run('PRAGMA synchronous = NORMAL');
   });
 
   const writerDB = new Kysely<StoreDatabase>({ dialect: buildSQLiteDialect(writer) });
-  const readerDB = new Kysely<StoreDatabase>({ dialect: buildSQLiteDialect(reader) });
 
-  opened.defer(() => readerDB.destroy());
   opened.defer(() => writerDB.destroy());
 
   await runStoreMigrations(writerDB);
 
   const handles = opened.move();
 
+  const reads = new Set<Promise<unknown>>();
+
+  let isClosed = false;
+
   return {
     // Kysely holds the one connection of a handle from acquire to release, so two calls on this
-    // store run one at a time, and BEGIN IMMEDIATE makes another process's writer wait.
+    // store run one at a time, and the write lock makes another store's writer wait.
     exclusive: (work) =>
-      writerDB
-        .connection()
-        .execute((db) => withTransaction(db, sql`BEGIN IMMEDIATE`, () => work(createSQLiteTx(db)))),
-
-    // a second handle, so a read sees the last commit and never a running exclusive call's writes
-    read: (work) =>
-      readerDB
-        .connection()
-        .execute((db) => withTransaction(db, sql`BEGIN`, () => work(buildSQLiteReader(db)))),
-
-    close: () => handles.disposeAsync(),
+      writerDB.connection().execute((db) =>
+        withTransaction(
+          db,
+          (locked) => takeWriteLock(locked, lockWaitLimitMs),
+          () => work(createSQLiteTx(db)),
+        ),
+      ),
+
+    read: async (work) => {
+      if (isClosed) {
+        throw new Error('the store is closed');
+      }
+
+      const reading = withReadHandle(path, work);
+
+      reads.add(reading);
+
+      try {
+        return await reading;
+      } finally {
+        reads.delete(reading);
+      }
+    },
+
+    close: async () => {
+      isClosed = true;
+
+      await Promise.allSettled(reads);
+      await handles.disposeAsync();
+    },
   };
 }
 
-function openStoreHandle(path: string): Database {
-  const handle = new Database(path, { create: true });
-
-  handle.run('PRAGMA journal_mode = WAL');
-  handle.run('PRAGMA synchronous = NORMAL');
-  handle.run('PRAGMA busy_timeout = 5000');
+// A handle of its own per call, so a read sees the last commit and never a running exclusive
+// call's writes, and a read started inside another read never waits for the outer one's handle.
+async function withReadHandle<T>(
+  path: string,
+  work: (reader: StoreReader) => Promise<T>,
+): Promise<T> {
+  await using opened = new AsyncDisposableStack();
 
-  return handle;
-}
+  // without create, a missing file fails the open, where the default makes an empty file
+  const handle = new Database(path, { readwrite: true, create: false });
 
-async function withTransaction<T>(
-  db: Kysely<StoreDatabase>,
-  begin: RawBuilder<unknown>,
-  work: () => Promise<T>,
-): Promise<T> {
-  await begin.execute(db);
+  opened.defer(() => {
+    handle.close();
+  });
 
-  try {
-    const result = await work();
+  handle.run(`PRAGMA busy_timeout = ${lockWaitLimitMs}`);
+  handle.run('PRAGMA synchronous = NORMAL');
 
-    await sql`COMMIT`.execute(db);
+  const readerDB = new Kysely<StoreDatabase>({ dialect: buildSQLiteDialect(handle) });
 
-    return result;
-  } catch (error) {
-    // SQLite ends the transaction by itself after some errors, such as a full disk, and ROLLBACK
-    // then fails; its error joins the first one, never replaces it
-    await sql`ROLLBACK`.execute(db).catch((rollbackError: unknown) => {
-      throw new AggregateError([error, rollbackError], 'the work failed, and so did its rollback', {
-        cause: error,
-      });
-    });
+  opened.defer(() => readerDB.destroy());
 
-    throw error;
-  }
+  return await readerDB.connection().execute((db) =>
+    withTransaction(
+      db,
+      (connection) => sql`BEGIN`.execute(connection),
+      () => work(buildSQLiteReader(db)),
+    ),
+  );
 }

package/src/migrations/rename-relations-migration.ts

@@ -0,0 +1,17 @@
+import { sql } from 'kysely';
+import type { Kysely } from 'kysely';
+import type { Migration } from 'kysely/migration';
+
+const statements = [
+  sql`ALTER TABLE links RENAME TO relations`,
+  sql`DROP INDEX links_live`,
+  sql`CREATE UNIQUE INDEX relations_live ON relations (source, target, type) WHERE removed_at IS NULL`,
+];
+
+export const renameRelationsMigration: Migration = {
+  up: async (db: Kysely<unknown>) => {
+    for (const statement of statements) {
+      await statement.execute(db);
+    }
+  },
+};