@boxxy/store-sqlite 0.1.1 → 0.1.4

package/src/reads/is-latest-of-session.ts

@@ -0,0 +1,21 @@
+import type { Expression, ExpressionBuilder, SqlBool } from 'kysely';
+import type { StoreDatabase } from '../types.ts';
+
+// no later attempt carries the same atc session; claim time, then id, orders the attempts
+export function isLatestOfSession(
+  builder: ExpressionBuilder<StoreDatabase, 'attempts'>,
+): Expression<SqlBool> {
+  const later = builder
+    .selectFrom('attempts as later')
+    .select('later.id')
+    .whereRef('later.atc_session', '=', 'attempts.atc_session')
+    .where((inner) => {
+      const claimedAt = inner.ref('attempts.claimed_at');
+      const sameTime = inner('later.claimed_at', '=', claimedAt);
+      const higherId = inner('later.id', '>', inner.ref('attempts.id'));
+
+      return inner.or([inner('later.claimed_at', '>', claimedAt), inner.and([sameTime, higherId])]);
+    });
+
+  return builder.not(builder.exists(later));
+}

package/src/{read-item-views.ts → reads/read-item-views.ts}

@@ -1,8 +1,8 @@
 import { buildItemViews } from '@boxxy/model';
-import type { Instant, ItemWithLevers, MaxAttemptsByKind } from '@boxxy/model';
+import type { Instant, ItemRules, ItemWithLevers } from '@boxxy/model';
 import type { Expression, ExpressionBuilder, Kysely, SqlBool } from 'kysely';
-import { parseStoreRow } from './parse-store-row.ts';
-import type { StoreDatabase } from './types.ts';
+import { parseStoreRow } from '../codecs/parse-store-row.ts';
+import type { StoreDatabase } from '../types.ts';
 
 export type ItemScope = (builder: ExpressionBuilder<StoreDatabase, 'items'>) => Expression<SqlBool>;
 
@@ -11,12 +11,12 @@ export async function readItemViews(
   db: Kysely<StoreDatabase>,
   scope: ItemScope,
   now: Instant,
-  maxAttemptsByKind: MaxAttemptsByKind,
+  rules: ItemRules,
 ): Promise<ItemWithLevers[]> {
   const scoped = db.selectFrom('items').select('id').where(scope);
 
   const blockers = db
-    .selectFrom('links')
+    .selectFrom('relations')
     .select('source')
     .where('removed_at', 'is', null)
     .where('target', 'in', scoped);
@@ -40,8 +40,8 @@ export async function readItemViews(
 
   const inScope = new Set(scopedRows.map((row) => row.id));
 
-  const links = await db
-    .selectFrom('links')
+  const relations = await db
+    .selectFrom('relations')
     .selectAll()
     .where('removed_at', 'is', null)
     .where('target', 'in', loaded)
@@ -78,14 +78,14 @@ export async function readItemViews(
   const views = buildItemViews(
     {
       items: items.map((row) => parseStoreRow('items', row)),
-      links: links.map((row) => parseStoreRow('links', row)),
+      relations: relations.map((row) => parseStoreRow('relations', row)),
       levers: levers.map((row) => parseStoreRow('levers', row)),
       attempts: attempts.map((row) => parseStoreRow('attempts', row)),
       liveLeases: leases.map((row) => parseStoreRow('leases', row)),
       pullRequests: pullRequests.map((row) => parseStoreRow('pull_requests', row)),
     },
     now,
-    maxAttemptsByKind,
+    rules,
   );
 
   return views.filter((view) => inScope.has(view.id));

package/src/reads/read-limit-fills.ts

@@ -0,0 +1,32 @@
+import { limitSchema } from '@boxxy/model';
+import type { LimitFill } from '@boxxy/model';
+import type { Kysely } from 'kysely';
+import { parseJSONText } from '../codecs/parse-json-text.ts';
+import type { StoreDatabase } from '../types.ts';
+
+// the limits of the last sync, by name, each with its held seats counted
+export async function readLimitFills(db: Kysely<StoreDatabase>): Promise<LimitFill[]> {
+  const rows = await db
+    .selectFrom('limits')
+    .selectAll()
+    .select((builder) =>
+      builder
+        .selectFrom('limit_seats')
+        .select(builder.fn.countAll<number>().as('held'))
+        .whereRef('limit_seats.limit', '=', 'limits.name')
+        .where('limit_seats.released_at', 'is', null)
+        .as('held'),
+    )
+    .orderBy('name')
+    .execute();
+
+  return rows.map((row) => {
+    const limit = limitSchema.parse({
+      name: row.name,
+      max: row.max,
+      counts: parseJSONText(row.counts),
+    });
+
+    return { name: limit.name, max: limit.max, counts: limit.counts, held: row.held ?? 0 };
+  });
+}

package/src/reads/read-wakes.ts

@@ -0,0 +1,34 @@
+import type { Wake } from '@boxxy/model';
+import type { Kysely } from 'kysely';
+import { parseStoreRow } from '../codecs/parse-store-row.ts';
+import type { StoreDatabase } from '../types.ts';
+
+// the sleeping attempts whose wake was asked for, oldest request first
+export async function readWakes(db: Kysely<StoreDatabase>): Promise<Wake[]> {
+  const rows = await db
+    .selectFrom('attempts')
+    .selectAll()
+    .where('state', '=', 'sleeping')
+    .where('wake_requested_at', 'is not', null)
+    .orderBy('wake_requested_at')
+    .orderBy('claimed_at')
+    .orderBy('id')
+    .execute();
+
+  return rows.flatMap((row) => {
+    const attempt = parseStoreRow('attempts', row);
+
+    return attempt.wakeRequestedAt === null
+      ? []
+      : [
+          {
+            attempt: attempt.id,
+            item: attempt.item,
+            phase: attempt.phase,
+            pool: attempt.pool,
+            fields: attempt.fields,
+            requestedAt: attempt.wakeRequestedAt,
+          },
+        ];
+  });
+}

package/src/test-utils/create-claim-bench.ts

@@ -0,0 +1,66 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { claimNext } from '@boxxy/dispatch';
+import { createTestDispatch } from '@boxxy/dispatch/test-utils/create-test-dispatch';
+import type { ItemId } from '@boxxy/model';
+import { createSQLiteClaimStore } from '../create-sqlite-claim-store.ts';
+
+export interface ClaimBenchOptions {
+  readonly itemCount: number;
+}
+
+export interface ClaimBench extends AsyncDisposable {
+  // one `claimNext` call, which holds the store's lock for the whole decision; throws when it
+  // picks no item
+  readonly claim: () => Promise<ItemId>;
+}
+
+// A SQLite store in a temp folder, seeded with open items at their pipeline's first phase, and
+// every limit raised to the item count, so each claim finds a free seat with no close or settle
+// between claims.
+export async function createClaimBench(options: ClaimBenchOptions): Promise<ClaimBench> {
+  const stack = new AsyncDisposableStack();
+
+  const folder = await mkdtemp(join(tmpdir(), 'boxxy-claim-bench-'));
+
+  stack.defer(() => rm(folder, { recursive: true, force: true }));
+
+  const store = await createSQLiteClaimStore(join(folder, 'store.db'));
+
+  stack.defer(() => store.close());
+
+  const dispatch = await createTestDispatch({ store });
+
+  const rules = dispatch.rules();
+  const limits = rules.limits.map((limit) => ({ ...limit, max: options.itemCount }));
+
+  dispatch.setRules({ ...rules, limits });
+
+  await store.exclusive((tx) => tx.syncLimits(limits, dispatch.clock.now()));
+
+  for (let index = 0; index < options.itemCount; index += 1) {
+    await dispatch.seedItem({});
+  }
+
+  return {
+    claim: async () => {
+      const claim = await claimNext({
+        store,
+        runner: dispatch.runner,
+        rules: dispatch.rules(),
+        now: dispatch.clock.now(),
+        newId: dispatch.newId,
+        backoff: dispatch.backoff,
+        spawn: dispatch.spawn,
+      });
+
+      if (claim === null) {
+        throw new Error('the claim picked no item');
+      }
+
+      return claim.attempt.item;
+    },
+    [Symbol.asyncDispose]: () => stack.disposeAsync(),
+  };
+}

package/src/test-utils/create-test-claim-store.ts

@@ -10,7 +10,8 @@ export async function createTestClaimStore(): Promise<SQLiteClaimStore> {
 
   onTestFinished(() => cleanup.disposeAsync());
 
-  const folder = await mkdtemp(join(tmpdir(), 'boxxy-store-'));
+  // the process id keeps the folders of test runs that share a temp folder apart by name
+  const folder = await mkdtemp(join(tmpdir(), `boxxy-store-${process.pid}-`));
 
   cleanup.defer(() => rm(folder, { recursive: true, force: true }));
 

package/src/types.ts

@@ -8,13 +8,16 @@ export interface StoreDatabase {
     title: string;
     outcome: string;
     acceptance: string;
+    fields: string;
+    labels: string;
     found_in_attempt: string | null;
     filed_by: string;
     filed_at: number;
     closed_at: number | null;
     close_reason: string | null;
+    kept_session: string | null;
   };
-  links: {
+  relations: {
     source: string;
     target: string;
     type: string;
@@ -29,6 +32,7 @@ export interface StoreDatabase {
     until: number | null;
     expires_at: number | null;
     reason: string | null;
+    phase: string | null;
     actor: string;
     at: number;
   };
@@ -41,23 +45,42 @@ export interface StoreDatabase {
     recipe_hash: string;
     rules_version: string;
     pool: string;
-    slot: number;
-    slot_gen: number;
     lease_gen: number;
     spawn_request: string;
     spawn_request_hash: string;
     state: string;
     failure: string | null;
     atc_session: string | null;
-    close_out: string | null;
+    caller: string | null;
     claimed_at: number;
     started_at: number | null;
     ended_at: number | null;
+    fields: string;
+    mode: string;
+    round: number;
+    input: string | null;
+    hold_gen: number;
+    close_out: string | null;
+    close_out_hash: string | null;
+    close_out_fact: string | null;
+    closing: string | null;
+    closing_at: number | null;
+    suspend_requested_at: number | null;
+    slept_at: number | null;
+    wake_requested_at: number | null;
+    woken_at: number | null;
+    outcome: string | null;
+    returned_outcome: string | null;
+    strayed_at: number | null;
+    session_stopped_at: number | null;
+    session_forgotten_at: number | null;
   };
   attempt_sessions: {
     attempt: string;
     agent_session: string;
     atc_session: string;
+    product: string;
+    source: string;
     seen_at: number;
   };
   leases: {
@@ -68,23 +91,31 @@ export interface StoreDatabase {
     taken_at: number;
     released_at: number | null;
   };
-  pool_slots: {
-    pool: string;
-    slot: number;
-    holder: string | null;
-    gen: number;
-    retired: number;
+  limits: {
+    name: string;
+    max: number;
+    counts: string;
+  };
+  limit_seats: {
+    limit: string;
+    attempt: string;
+    hold_gen: number;
+    over_cap: number;
+    taken_at: number;
+    released_at: number | null;
   };
   decisions: {
     id: string;
     attempt: string;
     item: string;
+    kind: string;
     pool: string;
     rules_version: string;
     snapshot_hash: string;
     store_version: number;
     pepys_offsets: string;
     faults: string;
+    limits: string;
     at: number;
   };
   evidence: {
@@ -100,6 +131,8 @@ export interface StoreDatabase {
     repo: string;
     number: number;
     attempt: string;
+    github_id: number;
+    url: string;
     state: string;
     merged_at: number | null;
     merge_commit: string | null;
@@ -110,6 +143,21 @@ export interface StoreDatabase {
     heartbeat_at: number;
     paused: number;
   };
+  notes: {
+    item: string;
+    seq: number;
+    text: string;
+    by: string;
+    at: number;
+  };
+  notices: {
+    id: Generated<number>;
+    item: string;
+    reason: string;
+    at: number;
+    tries: number;
+    sent_at: number | null;
+  };
   outbox: {
     seq: Generated<number>;
     fact_id: string;
@@ -119,5 +167,7 @@ export interface StoreDatabase {
     at: number;
     payload: string;
     sent_at: number | null;
+    parked_at: number | null;
+    parked_reason: string | null;
   };
 }

package/src/with-busy-retry.ts

@@ -0,0 +1,32 @@
+import { SQLiteError } from 'bun:sqlite';
+
+const retryDelayMs = 10;
+
+// bun:sqlite waits on a busy lock synchronously, which stalls every other store in the process for
+// the whole wait, so the statement fails at once and this waits between tries without blocking.
+export async function withBusyRetry(waitLimitMs: number, work: () => unknown): Promise<void> {
+  const startMs = performance.now();
+
+  for (;;) {
+    try {
+      await work();
+
+      return;
+    } catch (error) {
+      if (!isBusy(error)) {
+        throw error;
+      }
+
+      await Bun.sleep(retryDelayMs);
+
+      // checked after the sleep, so no try starts once the limit has passed
+      if (performance.now() - startMs >= waitLimitMs) {
+        throw error;
+      }
+    }
+  }
+}
+
+function isBusy(error: unknown): boolean {
+  return error instanceof SQLiteError && error.code?.startsWith('SQLITE_BUSY') === true;
+}

package/src/with-transaction.ts

@@ -0,0 +1,29 @@
+import { sql } from 'kysely';
+import type { Kysely } from 'kysely';
+import type { StoreDatabase } from './types.ts';
+
+export async function withTransaction<T>(
+  db: Kysely<StoreDatabase>,
+  begin: (db: Kysely<StoreDatabase>) => Promise<unknown>,
+  work: () => Promise<T>,
+): Promise<T> {
+  await begin(db);
+
+  try {
+    const result = await work();
+
+    await sql`COMMIT`.execute(db);
+
+    return result;
+  } catch (error) {
+    // SQLite ends the transaction by itself after some errors, such as a full disk, and ROLLBACK
+    // then fails; its error joins the first one, never replaces it
+    await sql`ROLLBACK`.execute(db).catch((rollbackError: unknown) => {
+      throw new AggregateError([error, rollbackError], 'the work failed, and so did its rollback', {
+        cause: error,
+      });
+    });
+
+    throw error;
+  }
+}

package/src/{apply-sqlite-change.ts → writes/apply-sqlite-change.ts}

@@ -1,12 +1,13 @@
-import { buildAppliedChange, findSlotTake, toJSONValue } from '@boxxy/model';
-import type { ApplyResult, RowWrite, SlotTake, StoreChange, TakenSlot } from '@boxxy/model';
+import { buildAppliedChange, parseNewFact, pickSeatTake, toJSONValue } from '@boxxy/model';
+import type { ApplyResult, Instant, RowWrite, SeatTake, StoreChange } from '@boxxy/model';
 import { sql } from 'kysely';
 import type { Kysely, RawBuilder } from 'kysely';
 import invariant from 'tiny-invariant';
 import { match } from 'ts-pattern';
-import { toStoreColumns } from './to-store-columns.ts';
-import type { StoreValue } from './to-store-columns.ts';
-import type { StoreDatabase } from './types.ts';
+import { toStoreColumns } from '../codecs/to-store-columns.ts';
+import type { StoreValue } from '../codecs/to-store-columns.ts';
+import { buildSQLiteReader } from '../reads/build-sqlite-reader.ts';
+import type { StoreDatabase } from '../types.ts';
 
 // A savepoint, not the transaction, so a refusal keeps the caller's earlier changes in the same
 // exclusive call. The caller runs one change at a time, so one savepoint name is enough.
@@ -14,12 +15,12 @@ export async function applySQLiteChange(
   db: Kysely<StoreDatabase>,
   change: StoreChange,
 ): Promise<ApplyResult> {
-  invariant(change.facts.length > 0, 'a change writes at least one fact');
+  const facts = change.facts.map((fact) => parseNewFact(fact));
 
   await sql`SAVEPOINT apply_change`.execute(db);
 
   try {
-    const result = await writeSQLiteChange(db, change);
+    const result = await writeSQLiteChange(db, { ...change, facts });
 
     if (result.kind !== 'applied') {
       await sql`ROLLBACK TO apply_change`.execute(db);
@@ -43,22 +44,23 @@ async function writeSQLiteChange(
     return { kind: 'stale-generation' };
   }
 
-  const take = findSlotTake(change.writes);
-  const taken = take === null ? null : await writeSlotTake(db, take);
-
-  if (take !== null && taken === null) {
-    return { kind: 'no-free-slot' };
-  }
-
   const lastLever = await db
     .selectFrom('levers')
     .select((builder) => builder.fn.max('seq').as('seq'))
     .executeTakeFirst();
 
-  const applied = buildAppliedChange(change, { taken, lastLeverSeq: lastLever?.seq ?? 0 });
+  const lastNotice = await db
+    .selectFrom('notices')
+    .select((builder) => builder.fn.max('id').as('id'))
+    .executeTakeFirst();
+
+  const applied = buildAppliedChange(change, {
+    lastLeverSeq: lastLever?.seq ?? 0,
+    lastNoticeId: lastNotice?.id ?? 0,
+  });
 
   for (const write of applied.writes) {
-    const refusal = await writeSQLiteRow(db, write);
+    const refusal = await writeSQLiteRow(db, write, change.at);
 
     if (refusal !== null) {
       return refusal;
@@ -76,84 +78,53 @@ async function writeSQLiteChange(
         at: change.at,
         payload: JSON.stringify(toJSONValue(fact.payload)),
         sent_at: null,
+        parked_at: null,
+        parked_reason: null,
       })
       .execute();
   }
 
-  const slot = taken === null ? null : { pool: taken.pool, slot: taken.slot, gen: taken.gen };
-
-  return { kind: 'applied', slot };
+  return { kind: 'applied' };
 }
 
 async function probeLiveUnder(
   db: Kysely<StoreDatabase>,
   under: StoreChange['under'],
 ): Promise<boolean> {
-  if (under === undefined) {
+  if (under === undefined || under === null) {
     return true;
   }
 
   const lease = await db
     .selectFrom('leases')
-    .select('gen')
-    .where('item', '=', under.item)
-    .where('phase', '=', under.phase)
-    .where('gen', '=', under.gen)
-    .where('released_at', 'is', null)
+    .innerJoin('attempts', 'attempts.id', 'leases.attempt')
+    .select('attempts.hold_gen')
+    .where('leases.item', '=', under.item)
+    .where('leases.phase', '=', under.phase)
+    .where('leases.gen', '=', under.gen)
+    .where('leases.released_at', 'is', null)
     .executeTakeFirst();
 
-  if (lease === undefined || under.slot === undefined) {
-    return lease !== undefined;
-  }
-
-  const slot = await db
-    .selectFrom('pool_slots')
-    .select('gen')
-    .where('pool', '=', under.slot.pool)
-    .where('slot', '=', under.slot.slot)
-    .where('gen', '=', under.slot.gen)
-    .where('holder', 'is not', null)
-    .executeTakeFirst();
-
-  return slot !== undefined;
+  return lease !== undefined && (under.holdGen === undefined || lease.hold_gen === under.holdGen);
 }
 
-async function writeSlotTake(db: Kysely<StoreDatabase>, take: SlotTake): Promise<TakenSlot | null> {
-  const lowest = db
-    .selectFrom('pool_slots')
-    .select((builder) => builder.fn.min('slot').as('slot'))
-    .where('pool', '=', take.pool)
-    .where('holder', 'is', null)
-    .where('retired', '=', 0);
-
-  const row = await db
-    .updateTable('pool_slots')
-    .set((builder) => ({ holder: take.holder, gen: builder('gen', '+', 1) }))
-    .where('pool', '=', take.pool)
-    .where('slot', '=', lowest)
-    .where('holder', 'is', null)
-    .where('retired', '=', 0)
-    .returning(['slot', 'gen'])
-    .executeTakeFirst();
-
-  return row === undefined
-    ? null
-    : { holder: take.holder, pool: take.pool, slot: row.slot, gen: row.gen };
-}
-
-function writeSQLiteRow(db: Kysely<StoreDatabase>, write: RowWrite): Promise<ApplyResult | null> {
+function writeSQLiteRow(
+  db: Kysely<StoreDatabase>,
+  write: RowWrite,
+  at: Instant,
+): Promise<ApplyResult | null> {
   return match(write)
-    .with({ op: 'take-slot' }, () => Promise.resolve(null))
-    .with({ op: 'release-slot' }, async (release) => {
-      const result = await db
-        .updateTable('pool_slots')
-        .set({ holder: null })
-        .where('pool', '=', release.pool)
-        .where('slot', '=', release.slot)
-        .where('holder', '=', release.holder)
-        .executeTakeFirst();
-
-      return result.numUpdatedRows === 0n ? ({ kind: 'stale-generation' } as const) : null;
+    .with({ op: 'take-seats' }, (seats) => writeSeatTake(db, seats.take, at))
+    .with({ op: 'release-seats' }, async (seats) => {
+      await db
+        .updateTable('limit_seats')
+        .set({ released_at: at })
+        .where('attempt', '=', seats.release.attempt)
+        .where('hold_gen', '=', seats.release.holdGen)
+        .where('released_at', 'is', null)
+        .execute();
+
+      return null;
     })
     .with({ op: 'insert', table: 'leases' }, async (insert) => {
       const columns = toStoreColumns(insert.table, insert.row);
@@ -190,6 +161,38 @@ function writeSQLiteRow(db: Kysely<StoreDatabase>, write: RowWrite): Promise<App
     .exhaustive();
 }
 
+// The write lock is held for the whole transaction, so the counts read here stay true until the
+// seats are inserted.
+async function writeSeatTake(
+  db: Kysely<StoreDatabase>,
+  take: SeatTake,
+  at: Instant,
+): Promise<ApplyResult | null> {
+  const limits = await buildSQLiteReader(db).listLimits();
+
+  const answer = pickSeatTake(take, limits, at);
+
+  if (answer.kind !== 'take') {
+    return { kind: answer.kind, limit: answer.limit };
+  }
+
+  for (const seat of answer.seats) {
+    await db
+      .insertInto('limit_seats')
+      .values({
+        limit: seat.limit,
+        attempt: seat.attempt,
+        hold_gen: seat.holdGen,
+        over_cap: seat.overCap ? 1 : 0,
+        taken_at: seat.takenAt,
+        released_at: seat.releasedAt,
+      })
+      .execute();
+  }
+
+  return null;
+}
+
 function buildInsert(table: string, columns: [string, StoreValue][]): RawBuilder<unknown> {
   return sql`INSERT INTO ${sql.table(table)}
     (${sql.join(columns.map(([column]) => sql.ref(column)))})