@boxxy/store-sqlite 0.1.0 → 0.1.1

package/src/parse-store-row.ts

@@ -0,0 +1,36 @@
+import { rowSchemas } from '@boxxy/model';
+import type { TableName, Tables } from '@boxxy/model';
+import { tableCodecs } from './table-codecs.ts';
+import { toColumnName } from './to-column-name.ts';
+
+export function parseStoreRow<T extends TableName>(table: T, record: object): Tables[T] {
+  const codec = tableCodecs[table];
+
+  const columns = new Map<string, unknown>(Object.entries(record));
+  const json = new Set<string>(codec.json);
+  const flags = new Set<string>(codec.flags);
+
+  const fields = Object.fromEntries(
+    codec.fields.map((field) => {
+      const value = columns.get(toColumnName(field));
+
+      return [field, parseStoreValue(value, json.has(field), flags.has(field))];
+    }),
+  );
+
+  return rowSchemas[table].parse(fields);
+}
+
+function parseStoreValue(value: unknown, isJSON: boolean, isFlag: boolean): unknown {
+  if (isJSON && typeof value === 'string') {
+    const parsed: unknown = JSON.parse(value);
+
+    return parsed;
+  }
+
+  if (isFlag) {
+    return value === 1;
+  }
+
+  return value;
+}

package/src/read-item-views.ts

@@ -0,0 +1,92 @@
+import { buildItemViews } from '@boxxy/model';
+import type { Instant, ItemWithLevers, MaxAttemptsByKind } from '@boxxy/model';
+import type { Expression, ExpressionBuilder, Kysely, SqlBool } from 'kysely';
+import { parseStoreRow } from './parse-store-row.ts';
+import type { StoreDatabase } from './types.ts';
+
+export type ItemScope = (builder: ExpressionBuilder<StoreDatabase, 'items'>) => Expression<SqlBool>;
+
+// a target's state needs each blocker's own state, so this loads the blockers too
+export async function readItemViews(
+  db: Kysely<StoreDatabase>,
+  scope: ItemScope,
+  now: Instant,
+  maxAttemptsByKind: MaxAttemptsByKind,
+): Promise<ItemWithLevers[]> {
+  const scoped = db.selectFrom('items').select('id').where(scope);
+
+  const blockers = db
+    .selectFrom('links')
+    .select('source')
+    .where('removed_at', 'is', null)
+    .where('target', 'in', scoped);
+
+  const loaded = db
+    .selectFrom('items')
+    .select('id')
+    .where((builder) => builder.or([builder('id', 'in', scoped), builder('id', 'in', blockers)]));
+
+  const attemptsOfLoaded = db.selectFrom('attempts').select('id').where('item', 'in', loaded);
+
+  const items = await db
+    .selectFrom('items')
+    .selectAll()
+    .where('id', 'in', loaded)
+    .orderBy('filed_at')
+    .orderBy('id')
+    .execute();
+
+  const scopedRows = await scoped.execute();
+
+  const inScope = new Set(scopedRows.map((row) => row.id));
+
+  const links = await db
+    .selectFrom('links')
+    .selectAll()
+    .where('removed_at', 'is', null)
+    .where('target', 'in', loaded)
+    .execute();
+
+  const levers = await db
+    .selectFrom('levers')
+    .selectAll()
+    .where('item', 'in', loaded)
+    .orderBy('seq')
+    .execute();
+
+  const attempts = await db
+    .selectFrom('attempts')
+    .selectAll()
+    .where('item', 'in', loaded)
+    .orderBy('claimed_at')
+    .orderBy('id')
+    .execute();
+
+  const leases = await db
+    .selectFrom('leases')
+    .selectAll()
+    .where('released_at', 'is', null)
+    .where('item', 'in', loaded)
+    .execute();
+
+  const pullRequests = await db
+    .selectFrom('pull_requests')
+    .selectAll()
+    .where('attempt', 'in', attemptsOfLoaded)
+    .execute();
+
+  const views = buildItemViews(
+    {
+      items: items.map((row) => parseStoreRow('items', row)),
+      links: links.map((row) => parseStoreRow('links', row)),
+      levers: levers.map((row) => parseStoreRow('levers', row)),
+      attempts: attempts.map((row) => parseStoreRow('attempts', row)),
+      liveLeases: leases.map((row) => parseStoreRow('leases', row)),
+      pullRequests: pullRequests.map((row) => parseStoreRow('pull_requests', row)),
+    },
+    now,
+    maxAttemptsByKind,
+  );
+
+  return views.filter((view) => inScope.has(view.id));
+}

package/src/run-store-migrations.ts

@@ -0,0 +1,20 @@
+import type { Kysely } from 'kysely';
+import { Migrator } from 'kysely/migration';
+import { createTablesMigration } from './create-tables-migration.ts';
+import type { StoreDatabase } from './types.ts';
+
+// Kysely runs the migrations in the order of their names, and skips each one the file has run.
+const migrations = { '0001-create-tables': createTablesMigration };
+
+export async function runStoreMigrations(db: Kysely<StoreDatabase>): Promise<void> {
+  const migrator = new Migrator({
+    db,
+    provider: { getMigrations: () => Promise.resolve(migrations) },
+  });
+
+  const outcome = await migrator.migrateToLatest();
+
+  if (outcome.error !== undefined) {
+    throw new Error('the store migrations failed', { cause: outcome.error });
+  }
+}

package/src/table-codecs.ts

@@ -0,0 +1,111 @@
+import type { TableName, Tables } from '@boxxy/model';
+
+interface TableCodec<Row> {
+  readonly fields: readonly (keyof Row & string)[];
+
+  readonly json: readonly (keyof Row & string)[];
+
+  readonly flags: readonly (keyof Row & string)[];
+}
+
+export const tableCodecs: { readonly [T in TableName]: TableCodec<Tables[T]> } = {
+  items: {
+    fields: [
+      'id',
+      'kind',
+      'repo',
+      'title',
+      'outcome',
+      'acceptance',
+      'foundInAttempt',
+      'filedBy',
+      'filedAt',
+      'closedAt',
+      'closeReason',
+    ],
+    json: ['acceptance'],
+    flags: [],
+  },
+  links: {
+    fields: ['source', 'target', 'type', 'addedBy', 'addedAt', 'removedAt'],
+    json: [],
+    flags: [],
+  },
+  levers: {
+    fields: ['seq', 'kind', 'item', 'until', 'expiresAt', 'reason', 'actor', 'at'],
+    json: [],
+    flags: [],
+  },
+  attempts: {
+    fields: [
+      'id',
+      'item',
+      'phase',
+      'number',
+      'runner',
+      'recipeHash',
+      'rulesVersion',
+      'pool',
+      'slot',
+      'slotGen',
+      'leaseGen',
+      'spawnRequest',
+      'spawnRequestHash',
+      'state',
+      'failure',
+      'atcSession',
+      'closeOut',
+      'claimedAt',
+      'startedAt',
+      'endedAt',
+    ],
+    json: ['spawnRequest', 'closeOut'],
+    flags: [],
+  },
+  attempt_sessions: {
+    fields: ['attempt', 'agentSession', 'atcSession', 'seenAt'],
+    json: [],
+    flags: [],
+  },
+  leases: {
+    fields: ['item', 'phase', 'attempt', 'gen', 'takenAt', 'releasedAt'],
+    json: [],
+    flags: [],
+  },
+  pool_slots: {
+    fields: ['pool', 'slot', 'holder', 'gen', 'retired'],
+    json: [],
+    flags: ['retired'],
+  },
+  decisions: {
+    fields: [
+      'id',
+      'attempt',
+      'item',
+      'pool',
+      'rulesVersion',
+      'snapshotHash',
+      'storeVersion',
+      'pepysOffsets',
+      'faults',
+      'at',
+    ],
+    json: ['pepysOffsets', 'faults'],
+    flags: [],
+  },
+  evidence: {
+    fields: ['factId', 'attempt', 'kind', 'ref', 'observed', 'source', 'at'],
+    json: [],
+    flags: [],
+  },
+  pull_requests: {
+    fields: ['repo', 'number', 'attempt', 'state', 'mergedAt', 'mergeCommit', 'checkedAt'],
+    json: [],
+    flags: [],
+  },
+  runners: {
+    fields: ['id', 'heartbeatAt', 'paused'],
+    json: [],
+    flags: ['paused'],
+  },
+};

package/src/test-utils/create-test-claim-store.ts

@@ -0,0 +1,22 @@
+import { onTestFinished } from 'bun:test';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createSQLiteClaimStore } from '../create-sqlite-claim-store.ts';
+import type { SQLiteClaimStore } from '../create-sqlite-claim-store.ts';
+
+export async function createTestClaimStore(): Promise<SQLiteClaimStore> {
+  const cleanup = new AsyncDisposableStack();
+
+  onTestFinished(() => cleanup.disposeAsync());
+
+  const folder = await mkdtemp(join(tmpdir(), 'boxxy-store-'));
+
+  cleanup.defer(() => rm(folder, { recursive: true, force: true }));
+
+  const store = await createSQLiteClaimStore(join(folder, 'store.db'));
+
+  cleanup.defer(() => store.close());
+
+  return store;
+}

package/src/to-column-name.ts

@@ -0,0 +1,3 @@
+export function toColumnName(field: string): string {
+  return field.replaceAll(/[A-Z]/gu, (letter) => `_${letter.toLowerCase()}`);
+}

package/src/to-store-columns.ts

@@ -0,0 +1,44 @@
+import { formatCanonicalJson } from '@boxxy/content-hash';
+import { toJSONValue } from '@boxxy/model';
+import type { TableName } from '@boxxy/model';
+import invariant from 'tiny-invariant';
+import { tableCodecs } from './table-codecs.ts';
+import { toColumnName } from './to-column-name.ts';
+
+export type StoreValue = string | number | null;
+
+// a field outside the table's codec fails, so no caller's key reaches the SQL as a column name
+export function toStoreColumns(table: TableName, fields: object): [string, StoreValue][] {
+  const codec = tableCodecs[table];
+
+  const known = new Set<string>(codec.fields);
+  const json = new Set<string>(codec.json);
+
+  return Object.entries(fields).map(([field, value]: [string, unknown]) => {
+    invariant(known.has(field), `${table} has no field ${field}`);
+
+    return [toColumnName(field), toStoreValue(value, json.has(field))];
+  });
+}
+
+function toStoreValue(value: unknown, isJSON: boolean): StoreValue {
+  if (value === null) {
+    return null;
+  }
+
+  if (isJSON) {
+    // an update compares the stored text, so two objects with their keys in another order match
+    return formatCanonicalJson(toJSONValue(value));
+  }
+
+  if (typeof value === 'boolean') {
+    return value ? 1 : 0;
+  }
+
+  invariant(
+    typeof value === 'string' || typeof value === 'number',
+    `a ${typeof value} needs a JSON column`,
+  );
+
+  return value;
+}

package/src/types.ts

@@ -0,0 +1,123 @@
+import type { Generated } from 'kysely';
+
+export interface StoreDatabase {
+  items: {
+    id: string;
+    kind: string;
+    repo: string;
+    title: string;
+    outcome: string;
+    acceptance: string;
+    found_in_attempt: string | null;
+    filed_by: string;
+    filed_at: number;
+    closed_at: number | null;
+    close_reason: string | null;
+  };
+  links: {
+    source: string;
+    target: string;
+    type: string;
+    added_by: string;
+    added_at: number;
+    removed_at: number | null;
+  };
+  levers: {
+    seq: number;
+    kind: string;
+    item: string;
+    until: number | null;
+    expires_at: number | null;
+    reason: string | null;
+    actor: string;
+    at: number;
+  };
+  attempts: {
+    id: string;
+    item: string;
+    phase: string;
+    number: number;
+    runner: string;
+    recipe_hash: string;
+    rules_version: string;
+    pool: string;
+    slot: number;
+    slot_gen: number;
+    lease_gen: number;
+    spawn_request: string;
+    spawn_request_hash: string;
+    state: string;
+    failure: string | null;
+    atc_session: string | null;
+    close_out: string | null;
+    claimed_at: number;
+    started_at: number | null;
+    ended_at: number | null;
+  };
+  attempt_sessions: {
+    attempt: string;
+    agent_session: string;
+    atc_session: string;
+    seen_at: number;
+  };
+  leases: {
+    item: string;
+    phase: string;
+    attempt: string;
+    gen: number;
+    taken_at: number;
+    released_at: number | null;
+  };
+  pool_slots: {
+    pool: string;
+    slot: number;
+    holder: string | null;
+    gen: number;
+    retired: number;
+  };
+  decisions: {
+    id: string;
+    attempt: string;
+    item: string;
+    pool: string;
+    rules_version: string;
+    snapshot_hash: string;
+    store_version: number;
+    pepys_offsets: string;
+    faults: string;
+    at: number;
+  };
+  evidence: {
+    fact_id: string;
+    attempt: string;
+    kind: string;
+    ref: string;
+    observed: string | null;
+    source: string;
+    at: number;
+  };
+  pull_requests: {
+    repo: string;
+    number: number;
+    attempt: string;
+    state: string;
+    merged_at: number | null;
+    merge_commit: string | null;
+    checked_at: number;
+  };
+  runners: {
+    id: string;
+    heartbeat_at: number;
+    paused: number;
+  };
+  outbox: {
+    seq: Generated<number>;
+    fact_id: string;
+    type: string;
+    entity: string;
+    actor: string;
+    at: number;
+    payload: string;
+    sent_at: number | null;
+  };
+}

package/dist/package-name.d.ts

@@ -1 +0,0 @@
-export declare const packageName = "@boxxy/store-sqlite";

package/src/package-name.ts

@@ -1 +0,0 @@
-export const packageName = '@boxxy/store-sqlite';