@pepys/testing 0.0.0-stage → 0.0.1

package/src/orders/apply-in-both-orders.ts

@@ -0,0 +1,59 @@
+import type { LinkRow, Plugin, SourceDefinition } from '@pepys/kit';
+import { createTestStore } from '../test-store/create-test-store.ts';
+import type { AppliedFact } from '../test-store/types.ts';
+import type { Row } from '../utils/types.ts';
+
+export interface StoreSnapshot {
+  rows: Readonly<Record<string, readonly Row[]>>;
+  links: readonly LinkRow[];
+}
+
+export interface OrderReport {
+  given: StoreSnapshot;
+  reversed: StoreSnapshot;
+
+  // the two snapshots are equal
+  isSame: boolean;
+}
+
+interface OrderCheck<Source extends SourceDefinition> {
+  plugins: readonly Plugin[];
+  source: Source;
+  facts: readonly AppliedFact<Source>[];
+}
+
+// One arrival order cannot show a missing tie-break. The reverse order puts every pair of facts the
+// other way round, and the second pass applies them all again as one batch.
+export async function applyInBothOrders<Source extends SourceDefinition>(
+  check: OrderCheck<Source>,
+): Promise<OrderReport> {
+  const given = await readSnapshot(check, check.facts);
+  const reversed = await readSnapshot(check, check.facts.toReversed());
+
+  return { given, reversed, isSame: Bun.deepEquals(given, reversed, true) };
+}
+
+async function readSnapshot<Source extends SourceDefinition>(
+  check: OrderCheck<Source>,
+  facts: readonly AppliedFact<Source>[],
+): Promise<StoreSnapshot> {
+  await using store = await createTestStore({ plugins: check.plugins });
+
+  for (const fact of facts) {
+    await store.apply(check.source, [fact]);
+  }
+
+  await store.apply(check.source, facts);
+
+  const tables = check.plugins
+    .flatMap((plugin) => plugin.consumers)
+    .flatMap((consumer) => Object.keys(consumer.tables));
+
+  const rows: Record<string, readonly Row[]> = {};
+
+  for (const table of tables) {
+    rows[table] = await store.readRows(table);
+  }
+
+  return { rows, links: await store.readLinks() };
+}

package/src/rows/build-key-filter.ts

@@ -0,0 +1,19 @@
+import type { TableDeclaration } from '@pepys/kit';
+import { sql } from 'kysely';
+import type { RawBuilder } from 'kysely';
+import { toSQLValue } from './to-sql-value.ts';
+
+// The `where` clause of one row, from the JSON text of its key's values.
+export function buildKeyFilter(table: TableDeclaration, key: string): RawBuilder<unknown> {
+  const parsed: unknown = JSON.parse(key);
+  const values: readonly unknown[] = Array.isArray(parsed) ? parsed : [];
+
+  return sql.join(
+    table.primaryKey.map((column, index) => {
+      const declaration = table.columns[column] ?? { kind: 'text' };
+
+      return sql`${sql.ref(column)} = ${toSQLValue(declaration, values[index])}`;
+    }),
+    sql` and `,
+  );
+}

package/src/rows/build-row-key.ts

@@ -0,0 +1,8 @@
+import type { TableDeclaration } from '@pepys/kit';
+import type { Row } from '../utils/types.ts';
+
+// The key of a row as the host writes it in `scope_row` and the change log: the JSON text of the
+// primary key's values, in key order, for a key of one column and of several.
+export function buildRowKey(table: TableDeclaration, row: Row): string {
+  return JSON.stringify(table.primaryKey.map((column) => row[column] ?? null));
+}

package/src/rows/find-row-problem.ts

@@ -0,0 +1,43 @@
+import type { TableDeclaration } from '@pepys/kit';
+import { match } from 'ts-pattern';
+import type { Row } from '../utils/types.ts';
+
+// The host's check of a row: every column present, no other key, each value of its column's kind,
+// and null only where the column is nullable. `inexact` names the columns whose number the module
+// wrote with a fraction or an exponent, such as `1.0`: an integer column refuses those.
+export function findRowProblem(
+  table: TableDeclaration,
+  row: Row,
+  inexact: ReadonlySet<string>,
+): string | undefined {
+  const unknown = Object.keys(row).find((name) => !Object.hasOwn(table.columns, name));
+
+  if (unknown !== undefined) {
+    return `${unknown} is not a column`;
+  }
+
+  for (const [name, column] of Object.entries(table.columns)) {
+    if (!Object.hasOwn(row, name)) {
+      return `the column ${name} is absent`;
+    }
+
+    const value = row[name];
+
+    const fits =
+      value === null
+        ? column.nullable === true
+        : match(column.kind)
+            .with('text', 'entityID', 'timestamp', () => typeof value === 'string')
+            .with('boolean', () => typeof value === 'boolean')
+            .with('json', () => true)
+            .with('real', () => typeof value === 'number')
+            .with('integer', () => Number.isSafeInteger(value) && !inexact.has(name))
+            .exhaustive();
+
+    if (!fits) {
+      return `the column ${name} holds ${JSON.stringify(value)}`;
+    }
+  }
+
+  return undefined;
+}

package/src/rows/read-table-rows.ts

@@ -0,0 +1,35 @@
+import type { TableDeclaration } from '@pepys/kit';
+import { sql } from 'kysely';
+import type { QueryExecutorProvider } from 'kysely';
+import type { Row } from '../utils/types.ts';
+import { buildKeyFilter } from './build-key-filter.ts';
+import { toRowValue } from './to-row-value.ts';
+
+// Every row of a table in primary key order, or with `key` the one row of that key, as the module
+// wrote it.
+export async function readTableRows(
+  db: QueryExecutorProvider,
+  name: string,
+  table: TableDeclaration,
+  key?: string,
+): Promise<Row[]> {
+  const order = sql.join(table.primaryKey.map((column) => sql.ref(column)));
+  const filter = key === undefined ? sql`1` : buildKeyFilter(table, key);
+
+  const stored =
+    await sql`select * from ${sql.table(name)} where ${filter} order by ${order}`.execute(db);
+
+  return stored.rows.map((row) =>
+    Object.fromEntries(
+      Object.entries(table.columns).map(([column, declaration]) => {
+        const value = isRecord(row) ? row[column] : undefined;
+
+        return [column, toRowValue(declaration, value)];
+      }),
+    ),
+  );
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null;
+}

package/src/rows/remove-row.ts

@@ -0,0 +1,13 @@
+import type { TableDeclaration } from '@pepys/kit';
+import { sql } from 'kysely';
+import type { QueryExecutorProvider } from 'kysely';
+import { buildKeyFilter } from './build-key-filter.ts';
+
+export async function removeRow(
+  db: QueryExecutorProvider,
+  name: string,
+  table: TableDeclaration,
+  key: string,
+): Promise<void> {
+  await sql`delete from ${sql.table(name)} where ${buildKeyFilter(table, key)}`.execute(db);
+}

package/src/rows/to-row-value.ts

@@ -0,0 +1,15 @@
+import type { ColumnDeclaration } from '@pepys/kit';
+import { match } from 'ts-pattern';
+
+// A stored value as the module wrote it: 0 or 1 back to a boolean, a JSON column's text parsed.
+export function toRowValue(column: ColumnDeclaration, stored: unknown): unknown {
+  if (stored === null || stored === undefined) {
+    return null;
+  }
+
+  return match(column.kind)
+    .with('boolean', () => stored === 1)
+    .with('json', (): unknown => (typeof stored === 'string' ? JSON.parse(stored) : stored))
+    .with('integer', 'real', 'text', 'entityID', 'timestamp', () => stored)
+    .exhaustive();
+}

package/src/rows/to-sql-value.ts

@@ -0,0 +1,18 @@
+import type { ColumnDeclaration } from '@pepys/kit';
+import { match } from 'ts-pattern';
+
+// A value of a checked row as SQLite stores it: a boolean as 0 or 1, a JSON column as its text.
+export function toSQLValue(column: ColumnDeclaration, value: unknown): string | number | null {
+  if (value === null || value === undefined) {
+    return null;
+  }
+
+  return match(column.kind)
+    .with('boolean', () => (value === true ? 1 : 0))
+    .with('json', () => JSON.stringify(value))
+    .with('integer', 'real', () => Number(value))
+    .with('text', 'entityID', 'timestamp', () =>
+      typeof value === 'string' ? value : JSON.stringify(value),
+    )
+    .exhaustive();
+}

package/src/rows/write-row.ts

@@ -0,0 +1,21 @@
+import type { TableDeclaration } from '@pepys/kit';
+import { sql } from 'kysely';
+import type { QueryExecutorProvider } from 'kysely';
+import type { Row } from '../utils/types.ts';
+import { toSQLValue } from './to-sql-value.ts';
+
+export async function writeRow(
+  db: QueryExecutorProvider,
+  name: string,
+  table: TableDeclaration,
+  row: Row,
+): Promise<void> {
+  const columns = Object.entries(table.columns);
+  const names = sql.join(columns.map(([column]) => sql.ref(column)));
+
+  const values = sql.join(
+    columns.map(([column, declaration]) => sql`${toSQLValue(declaration, row[column])}`),
+  );
+
+  await sql`insert or replace into ${sql.table(name)} (${names}) values (${values})`.execute(db);
+}

package/src/scope-replace/build-link-key.ts

@@ -0,0 +1,6 @@
+import type { WireLink } from '../utils/types.ts';
+
+// The key of a link in the change log: the JSON text of its type and its two ends.
+export function buildLinkKey(link: Pick<WireLink, 'type' | 'from' | 'to'>): string {
+  return JSON.stringify([link.type, link.from, link.to]);
+}

package/src/scope-replace/build-row-id.ts

@@ -0,0 +1,4 @@
+// One text for a table and a row key, for a map that holds the rows of several tables.
+export function buildRowID(table: string, key: string): string {
+  return JSON.stringify([table, key]);
+}

package/src/scope-replace/collect-scope-result.ts

@@ -0,0 +1,87 @@
+import { findLinkProblem } from '@pepys/kit';
+import type { ConsumerDeclaration, Result } from '@pepys/kit';
+import type { Operation } from '../operations/types.ts';
+import { buildRowKey } from '../rows/build-row-key.ts';
+import { findRowProblem } from '../rows/find-row-problem.ts';
+import type { ApplyFailure, WireLink } from '../utils/types.ts';
+import { buildLinkKey } from './build-link-key.ts';
+import { buildRowID } from './build-row-id.ts';
+import type { KeyedRow, ScopeResult } from './types.ts';
+
+// The host's checks of one scope's operations, in its order: no delete or unlink, every link, then
+// each row and its key, then each link key.
+export function collectScopeResult(
+  consumer: ConsumerDeclaration,
+  operations: readonly Operation[],
+): Result<ScopeResult, ApplyFailure> {
+  const name = consumer.name;
+  const upserts: (Operation & { verb: 'upsert' })[] = [];
+  const links: WireLink[] = [];
+
+  for (const operation of operations) {
+    if (operation.verb === 'delete' || operation.verb === 'unlink') {
+      const reason = `a scope consumer returned ${operation.verb === 'delete' ? 'a delete' : 'an unlink'}`;
+
+      return { ok: false, failure: { kind: 'bad-operation', consumer: name, reason } };
+    }
+
+    if (operation.verb === 'link') {
+      links.push(operation.link);
+    } else {
+      upserts.push(operation);
+    }
+  }
+
+  for (const link of links) {
+    const problem = findLinkProblem(consumer, link);
+
+    if (problem !== undefined) {
+      return { ok: false, failure: { kind: 'bad-link', consumer: name, problem } };
+    }
+  }
+
+  const rows = new Map<string, KeyedRow>();
+
+  for (const upsert of upserts) {
+    const table = consumer.tables[upsert.table];
+
+    if (table === undefined) {
+      return { ok: false, failure: { kind: 'not-owned', consumer: name, table: upsert.table } };
+    }
+
+    const reason = findRowProblem(table, upsert.row, upsert.inexact);
+
+    if (reason !== undefined) {
+      return {
+        ok: false,
+        failure: { kind: 'bad-row', consumer: name, table: upsert.table, reason },
+      };
+    }
+
+    const key = buildRowKey(table, upsert.row);
+    const id = buildRowID(upsert.table, key);
+
+    if (rows.has(id)) {
+      return {
+        ok: false,
+        failure: { kind: 'duplicate-row', consumer: name, table: upsert.table, key },
+      };
+    }
+
+    rows.set(id, { table: upsert.table, key, row: upsert.row });
+  }
+
+  const keyed = new Map<string, WireLink>();
+
+  for (const link of links) {
+    const key = buildLinkKey(link);
+
+    if (keyed.has(key)) {
+      return { ok: false, failure: { kind: 'duplicate-link', consumer: name, key } };
+    }
+
+    keyed.set(key, link);
+  }
+
+  return { ok: true, value: { rows: [...rows.values()], links: [...keyed.values()] } };
+}

package/src/scope-replace/pick-scope-changes.ts

@@ -0,0 +1,166 @@
+import type { Result } from '@pepys/kit';
+import type { ApplyFailure, ChangeEntry, WireLink } from '../utils/types.ts';
+import { buildLinkKey } from './build-link-key.ts';
+import { buildRowID } from './build-row-id.ts';
+import type { KeyedRow, ScopeChanges, ScopeResult, ScopeState } from './types.ts';
+
+// The replace of one scope as a diff: write each returned row and link that is new or differs,
+// delete each one the scope owned and did not return. Nothing may belong to another scope. The
+// entries come as rows written, links written, rows deleted, links deleted, each in key order.
+export function pickScopeChanges(
+  consumer: string,
+  scope: string,
+  result: ScopeResult,
+  state: ScopeState,
+): Result<ScopeChanges, ApplyFailure> {
+  const rows = result.rows.toSorted((left, right) =>
+    compareKeys([left.table, left.key], [right.table, right.key]),
+  );
+
+  const links = result.links.toSorted((left, right) =>
+    compareKeys([left.type, left.from, left.to], [right.type, right.from, right.to]),
+  );
+
+  const conflict = findConflict(consumer, scope, { rows, links }, state);
+
+  if (conflict !== undefined) {
+    return { ok: false, failure: conflict };
+  }
+
+  const writtenRows = rows.filter((row) => {
+    const stored = state.storedRows.get(buildRowID(row.table, row.key));
+
+    return stored === undefined || !Bun.deepEquals(stored, row.row);
+  });
+
+  const writtenLinks = links.filter((link) => {
+    const stored = state.storedLinks.get(buildLinkKey(link));
+
+    return stored === undefined || !isSameLink(stored.link, link);
+  });
+
+  const returned = new Set(rows.map((row) => buildRowID(row.table, row.key)));
+
+  const deletedRows = [...state.rowOwners]
+    .filter(([id, owner]) => owner === scope && !returned.has(id))
+    .map(([id]) => toTableKey(id))
+    .toSorted((left, right) => compareKeys([left.table, left.key], [right.table, right.key]));
+
+  const returnedLinks = new Set(links.map((link) => buildLinkKey(link)));
+
+  const deletedLinks = [...state.storedLinks]
+    .filter(([key, stored]) => stored.scope === scope && !returnedLinks.has(key))
+    .map(([, stored]) => stored.link)
+    .toSorted((left, right) =>
+      compareKeys([left.type, left.from, left.to], [right.type, right.from, right.to]),
+    );
+
+  const entries: ChangeEntry[] = [
+    ...writtenRows.map((row) => buildRowEntry(row, state)),
+    ...writtenLinks.map((link) => buildLinkEntry(link, state)),
+    ...deletedRows.map((row) => ({
+      type: row.table,
+      key: row.key,
+      old_value: state.storedRows.get(buildRowID(row.table, row.key)) ?? null,
+      headers: { operation: 'delete' as const },
+    })),
+    ...deletedLinks.map((link) => ({
+      type: 'link',
+      key: buildLinkKey(link),
+      headers: { operation: 'delete' as const },
+    })),
+  ];
+
+  return { ok: true, value: { writtenRows, writtenLinks, deletedRows, deletedLinks, entries } };
+}
+
+// Plain `<` on the parts, as Rust's ordered maps and SQLite compare text: never a locale order.
+function compareKeys(left: readonly string[], right: readonly string[]): number {
+  for (const [index, part] of left.entries()) {
+    const other = right[index] ?? '';
+
+    if (part !== other) {
+      return part < other ? -1 : 1;
+    }
+  }
+
+  return left.length - right.length;
+}
+
+function findConflict(
+  consumer: string,
+  scope: string,
+  result: ScopeResult,
+  state: ScopeState,
+): ApplyFailure | undefined {
+  for (const row of result.rows) {
+    const owner = state.rowOwners.get(buildRowID(row.table, row.key));
+
+    if (owner !== undefined && owner !== scope) {
+      return { kind: 'scope-conflict', consumer, scope, table: row.table, key: row.key, owner };
+    }
+  }
+
+  for (const link of result.links) {
+    const key = buildLinkKey(link);
+    const stored = state.storedLinks.get(key);
+
+    if (stored !== undefined && stored.scope !== scope) {
+      return { kind: 'scope-conflict', consumer, scope, table: 'link', key, owner: stored.scope };
+    }
+  }
+
+  return undefined;
+}
+
+// A confidence is a float on both sides, so `1` and `1.0` are one value.
+function isSameLink(stored: WireLink, link: WireLink): boolean {
+  return stored.confidence === link.confidence && stored.evidence === link.evidence;
+}
+
+function toTableKey(id: string): Omit<KeyedRow, 'row'> {
+  const parsed: unknown = JSON.parse(id);
+  const [table, key] = Array.isArray(parsed) ? parsed.map(String) : [];
+
+  return { table: table ?? '', key: key ?? '' };
+}
+
+function buildRowEntry(row: KeyedRow, state: ScopeState): ChangeEntry {
+  const old = state.storedRows.get(buildRowID(row.table, row.key));
+
+  return old === undefined
+    ? { type: row.table, key: row.key, value: row.row, headers: { operation: 'insert' } }
+    : {
+        type: row.table,
+        key: row.key,
+        value: row.row,
+        old_value: old,
+        headers: { operation: 'update' },
+      };
+}
+
+function buildLinkEntry(link: WireLink, state: ScopeState): ChangeEntry {
+  const key = buildLinkKey(link);
+  const old = state.storedLinks.get(key);
+  const value = toLinkValue(link);
+
+  return old === undefined
+    ? { type: 'link', key, value, headers: { operation: 'insert' } }
+    : {
+        type: 'link',
+        key,
+        value,
+        old_value: toLinkValue(old.link),
+        headers: { operation: 'update' },
+      };
+}
+
+function toLinkValue(link: WireLink): WireLink {
+  return {
+    type: link.type,
+    from: link.from,
+    to: link.to,
+    confidence: link.confidence,
+    evidence: link.evidence,
+  };
+}

package/src/scope-replace/read-scope-state.ts

@@ -0,0 +1,61 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+import { readLinkRows } from '../links/read-link-rows.ts';
+import { readTableRows } from '../rows/read-table-rows.ts';
+import type { StoreDatabase } from '../store-database/types.ts';
+import type { Row } from '../utils/types.ts';
+import { buildLinkKey } from './build-link-key.ts';
+import { buildRowID } from './build-row-id.ts';
+import type { ScopeResult, ScopeState, StoredLink } from './types.ts';
+
+export async function readScopeState(
+  db: StoreDatabase,
+  consumer: ConsumerDeclaration,
+  scope: string,
+  result: ScopeResult,
+): Promise<ScopeState> {
+  const owned = await db
+    .selectFrom('scope_row')
+    .select(['table_name', 'key', 'scope'])
+    .where('consumer', '=', consumer.name)
+    .execute();
+
+  const rowOwners = new Map(owned.map((row) => [buildRowID(row.table_name, row.key), row.scope]));
+
+  const wanted = [
+    ...result.rows.map((row) => ({ table: row.table, key: row.key })),
+    ...owned
+      .filter((row) => row.scope === scope)
+      .map((row) => ({ table: row.table_name, key: row.key })),
+  ];
+
+  const storedRows = new Map<string, Row>();
+
+  for (const row of wanted) {
+    const declaration = consumer.tables[row.table];
+
+    const [stored] =
+      declaration === undefined ? [] : await readTableRows(db, row.table, declaration, row.key);
+
+    if (stored !== undefined) {
+      storedRows.set(buildRowID(row.table, row.key), stored);
+    }
+  }
+
+  const links = await readLinkRows(db, consumer.name);
+
+  const storedLinks = new Map<string, StoredLink>(
+    links.map((row) => {
+      const link = {
+        type: row.type,
+        from: row.from_id,
+        to: row.to_id,
+        confidence: row.confidence,
+        evidence: row.evidence,
+      };
+
+      return [buildLinkKey(link), { scope: row.scope, link }];
+    }),
+  );
+
+  return { rowOwners, storedRows, storedLinks };
+}

package/src/scope-replace/replace-scope.ts

@@ -0,0 +1,42 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+import { parseOperations } from '../operations/parse-operations.ts';
+import type { StoreDatabase } from '../store-database/types.ts';
+import { ApplyError } from '../utils/apply-error.ts';
+import type { ChangeEntry } from '../utils/types.ts';
+import { collectScopeResult } from './collect-scope-result.ts';
+import { pickScopeChanges } from './pick-scope-changes.ts';
+import { readScopeState } from './read-scope-state.ts';
+import { writeScopeChanges } from './write-scope-changes.ts';
+
+// The host's scope replace of one scope, from the module's output text, inside the caller's
+// transaction. It throws an ApplyError on a refusal, and the caller rolls the batch back.
+export async function replaceScope(
+  db: StoreDatabase,
+  consumer: ConsumerDeclaration,
+  scope: string,
+  output: string,
+): Promise<readonly ChangeEntry[]> {
+  const operations = parseOperations(consumer, output);
+
+  if (!operations.ok) {
+    throw new ApplyError(operations.failure);
+  }
+
+  const result = collectScopeResult(consumer, operations.value);
+
+  if (!result.ok) {
+    throw new ApplyError(result.failure);
+  }
+
+  const state = await readScopeState(db, consumer, scope, result.value);
+
+  const changes = pickScopeChanges(consumer.name, scope, result.value, state);
+
+  if (!changes.ok) {
+    throw new ApplyError(changes.failure);
+  }
+
+  await writeScopeChanges(db, consumer, scope, changes.value);
+
+  return changes.value.entries;
+}