@pepys/testing 0.0.0-stage → 0.0.1

package/README.md

@@ -1,3 +1,36 @@
-# Temporary Holding Version
+<div align="center">
+  <h1>@pepys/testing</h1>
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+  <p>
+    <a href="https://www.npmjs.com/package/@pepys/testing"><img src="https://img.shields.io/npm/v/@pepys/testing" alt="npm version"></a>
+    <a href="../../LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="MIT license"></a>
+  </p>
+
+  <p>
+    <a href="../../docs/README.md">Documentation</a>
+  </p>
+</div>
+
+Test a pepys plug-in on Bun, with no host and no network.
+
+The test store runs a plug-in's consumers over one in-memory database, with every table, the link
+table and the shared views, and answers the plug-in's queries. It repeats the host's link check and
+scope replace in TypeScript. It does not prove the scriptc compile or the Rust host.
+
+## Install
+
+```sh
+bun add -d @pepys/testing
+```
+
+Set the compiler options from the [kit's README](../kit/README.md#configuration).
+
+## Features
+
+- `createTestStore`: apply facts or write rows, then run a query, read rows and read links.
+- `applyInBothOrders`: apply facts in two orders and compare the stores.
+- The memory adapter of the store reader, for a test of a core query.
+
+## Documentation
+
+The [pepys docs](../../docs/README.md) hold the design of the test store.

package/package.json

@@ -1,6 +1,33 @@
 {
   "name": "@pepys/testing",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.0.1",
+  "description": "Test a pepys plug-in on Bun: an in-memory store that runs its consumers and queries",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/zgeoff/pepys.git",
+    "directory": "packages/testing"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/test-utils"
+  ],
+  "type": "module",
+  "exports": "./src/index.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@pepys/kit": "0.0.1",
+    "@pepys/kysely-turso": "0.0.1",
+    "@tursodatabase/database": "0.8.3",
+    "kysely": "0.29.8",
+    "tiny-invariant": "1.3.3",
+    "ts-pattern": "5.9.2",
+    "zod": "4.6.7"
+  },
+  "devDependencies": {
+    "@zgeoff/std": "0.4.3"
+  }
+}

package/src/consumer-entries/build-read-functions.ts

@@ -0,0 +1,26 @@
+import type { HostFunctions } from './run-entry.ts';
+
+// The two host functions of one read, `<name>Length` and `<name>Into`, over JSON texts by argument.
+// A length of -1 means no value. A read the texts do not hold throws, as the host's trap does.
+export function buildReadFunctions(
+  name: string,
+  texts: ReadonlyMap<string, string>,
+  missing: 'none' | 'trap',
+): HostFunctions {
+  const encodeText = (argument: string): Uint8Array | undefined => {
+    const text = texts.get(argument);
+
+    if (text === undefined && missing === 'trap') {
+      throw new Error(`the module read ${argument}, which the host does not serve it`);
+    }
+
+    return text === undefined ? undefined : new TextEncoder().encode(text);
+  };
+
+  return {
+    [`${name}Length`]: (argument: string): number => encodeText(argument)?.length ?? -1,
+    [`${name}Into`]: (argument: string, buffer: Uint8Array): void => {
+      buffer.set(encodeText(argument) ?? new Uint8Array());
+    },
+  };
+}

package/src/consumer-entries/run-entry.ts

@@ -0,0 +1,46 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+import { ApplyError } from '../utils/apply-error.ts';
+
+// Host functions as globals, the way the glue's `declare function` lines reach them on Bun.
+export type HostFunctions = Readonly<Record<string, (...parameters: never[]) => unknown>>;
+
+// Calls one export of the consumer's TypeScript entry with the host functions in place, and
+// returns the output text. A throw, a missing export or output that is no text is an ApplyError.
+export async function runEntry(
+  consumer: ConsumerDeclaration,
+  exportName: 'applyFact' | 'applyScope',
+  argument: string,
+  host: HostFunctions,
+): Promise<string> {
+  const failure = (reason: string) =>
+    new ApplyError({ kind: 'entry', consumer: consumer.name, reason });
+
+  const entry: unknown = await import(consumer.entry.href);
+
+  const call: unknown =
+    typeof entry === 'object' && entry !== null && exportName in entry
+      ? Reflect.get(entry, exportName)
+      : undefined;
+
+  if (typeof call !== 'function') {
+    throw failure(`the entry file exports no function ${exportName}`);
+  }
+
+  Object.assign(globalThis, host);
+
+  try {
+    const output: unknown = Reflect.apply(call, undefined, [argument]);
+
+    if (typeof output !== 'string') {
+      throw failure(`${exportName} returned no text`);
+    }
+
+    return output;
+  } catch (error) {
+    throw error instanceof ApplyError ? error : failure(`${exportName} threw: ${String(error)}`);
+  } finally {
+    for (const name of Object.keys(host)) {
+      Reflect.deleteProperty(globalThis, name);
+    }
+  }
+}

package/src/fact-index/build-fact-index.ts

@@ -0,0 +1,39 @@
+import type { FactEnvelope } from '@pepys/kit';
+import type { FactIndex, LoggedFact } from './types.ts';
+
+interface IndexedBatch {
+  index: FactIndex;
+
+  // the batch's records, copies included, in log order
+  appended: readonly LoggedFact[];
+}
+
+// The index after a batch is appended to the log. Each stream numbers its records from 0.
+export function buildFactIndex(
+  previous: FactIndex,
+  batch: readonly { stream: string; envelope: FactEnvelope<unknown> }[],
+): IndexedBatch {
+  const counts = new Map<string, number>();
+
+  for (const record of previous.log) {
+    counts.set(record.stream, (counts.get(record.stream) ?? 0) + 1);
+  }
+
+  const appended = batch.map((fact) => {
+    const seqNum = counts.get(fact.stream) ?? 0;
+
+    counts.set(fact.stream, seqNum + 1);
+
+    return { stream: fact.stream, seqNum, envelope: fact.envelope };
+  });
+
+  const firstCopies = new Map(previous.firstCopies);
+
+  for (const record of appended) {
+    if (!firstCopies.has(record.envelope.id)) {
+      firstCopies.set(record.envelope.id, record);
+    }
+  }
+
+  return { index: { log: [...previous.log, ...appended], firstCopies }, appended };
+}

package/src/fact-index/collect-scope-facts.ts

@@ -0,0 +1,42 @@
+import type { FactEnvelope } from '@pepys/kit';
+import type { FactIndex } from './types.ts';
+
+// The read of one scope, as the host serves it: every fact of the entity on `streams`, one per fact
+// ID, sorted by time, then stream, then sequence number.
+export function collectScopeFacts(
+  index: FactIndex,
+  scope: string,
+  streams: readonly string[],
+): FactEnvelope<unknown>[] {
+  return [...index.firstCopies.values()]
+    .filter((record) => record.envelope.entity === scope && streams.includes(record.stream))
+    .map((record) => ({ record, sortKey: toSortKey(record.envelope.at) }))
+    .toSorted(
+      (left, right) =>
+        compareText(left.sortKey, right.sortKey) ||
+        compareText(left.record.stream, right.record.stream) ||
+        left.record.seqNum - right.record.seqNum,
+    )
+    .map((entry) => entry.record.envelope);
+}
+
+// The host's sort key: the fraction of the seconds set to 9 digits, so text order is time order.
+function toSortKey(at: string): string {
+  const parts = /^(?<seconds>[^.]*T\d\d:\d\d:\d\d)(?:\.(?<fraction>\d+))?(?<zone>.*)$/u.exec(at);
+
+  if (parts?.groups === undefined) {
+    return at;
+  }
+
+  const fraction = (parts.groups['fraction'] ?? '').padEnd(9, '0').slice(0, 9);
+
+  return `${parts.groups['seconds']}.${fraction}${parts.groups['zone']}`;
+}
+
+function compareText(left: string, right: string): number {
+  if (left === right) {
+    return 0;
+  }
+
+  return left < right ? -1 : 1;
+}

package/src/fact-index/types.ts

@@ -0,0 +1,15 @@
+import type { FactEnvelope } from '@pepys/kit';
+
+// One record of the test store's log: a fact in its stream, at its sequence number.
+export interface LoggedFact {
+  stream: string;
+  seqNum: number;
+  envelope: FactEnvelope<unknown>;
+}
+
+// The log in memory, with the first copy of each fact ID: the host's fact index holds one row per
+// fact ID, and a module reads synchronously, so the test store keeps the index out of the database.
+export interface FactIndex {
+  log: readonly LoggedFact[];
+  firstCopies: ReadonlyMap<string, LoggedFact>;
+}

package/src/fact-mode/apply-fact-operations.ts

@@ -0,0 +1,71 @@
+import { findLinkProblem } from '@pepys/kit';
+import type { ConsumerDeclaration, TableDeclaration } from '@pepys/kit';
+import invariant from 'tiny-invariant';
+import { match } from 'ts-pattern';
+import { removeLink } from '../links/remove-link.ts';
+import { writeLink } from '../links/write-link.ts';
+import type { Operation } from '../operations/types.ts';
+import { buildRowKey } from '../rows/build-row-key.ts';
+import { findRowProblem } from '../rows/find-row-problem.ts';
+import { removeRow } from '../rows/remove-row.ts';
+import { writeRow } from '../rows/write-row.ts';
+import type { StoreDatabase } from '../store-database/types.ts';
+import { ApplyError } from '../utils/apply-error.ts';
+
+// The operations of one fact, in fact mode: every check first, then the writes in order. A link
+// has no scope.
+export async function applyFactOperations(
+  db: StoreDatabase,
+  consumer: ConsumerDeclaration,
+  operations: readonly Operation[],
+): Promise<void> {
+  for (const operation of operations) {
+    if (operation.verb === 'link') {
+      const problem = findLinkProblem(consumer, operation.link);
+
+      if (problem !== undefined) {
+        throw new ApplyError({ kind: 'bad-link', consumer: consumer.name, problem });
+      }
+    }
+
+    if (operation.verb === 'upsert') {
+      const reason = findRowProblem(
+        getTable(consumer, operation.table),
+        operation.row,
+        operation.inexact,
+      );
+
+      if (reason !== undefined) {
+        throw new ApplyError({
+          kind: 'bad-row',
+          consumer: consumer.name,
+          table: operation.table,
+          reason,
+        });
+      }
+    }
+  }
+
+  for (const operation of operations) {
+    await match(operation)
+      .with({ verb: 'upsert' }, (upsert) =>
+        writeRow(db, upsert.table, getTable(consumer, upsert.table), upsert.row),
+      )
+      .with({ verb: 'delete' }, (removal) => {
+        const table = getTable(consumer, removal.table);
+
+        return removeRow(db, removal.table, table, buildRowKey(table, removal.key));
+      })
+      .with({ verb: 'link' }, (linking) => writeLink(db, consumer.name, null, linking.link))
+      .with({ verb: 'unlink' }, (unlinking) => removeLink(db, consumer.name, unlinking.link))
+      .exhaustive();
+  }
+}
+
+function getTable(consumer: ConsumerDeclaration, name: string): TableDeclaration {
+  const table = consumer.tables[name];
+
+  invariant(table, 'the parse of the operations refuses a table the consumer does not own');
+
+  return table;
+}

package/src/fact-mode/read-row-functions.ts

@@ -0,0 +1,34 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+import { toPascalCase } from '@pepys/kit/generators';
+import { buildReadFunctions } from '../consumer-entries/build-read-functions.ts';
+import type { HostFunctions } from '../consumer-entries/run-entry.ts';
+import { readTableRows } from '../rows/read-table-rows.ts';
+import type { StoreDatabase } from '../store-database/types.ts';
+
+// The row reads of a fact-mode consumer, over the rows its `reads` tables hold now. A module reads
+// synchronously, so the rows are read before each call. The argument of a read is the key: the
+// value of a key of one column as text, or the JSON text of the values of a key of several.
+export async function readRowFunctions(
+  db: StoreDatabase,
+  consumer: ConsumerDeclaration,
+): Promise<HostFunctions> {
+  const functions: HostFunctions[] = [];
+
+  for (const name of consumer.reads) {
+    const table = consumer.tables[name];
+    const rows = table === undefined ? [] : await readTableRows(db, name, table);
+
+    const texts = new Map(
+      rows.map((row) => {
+        const values = (table?.primaryKey ?? []).map((column) => row[column]);
+        const key = values.length === 1 ? String(values[0]) : JSON.stringify(values);
+
+        return [key, JSON.stringify(row)];
+      }),
+    );
+
+    functions.push(buildReadFunctions(`read${toPascalCase(name)}`, texts, 'none'));
+  }
+
+  return Object.fromEntries(functions.flatMap((pair) => Object.entries(pair)));
+}

package/src/index.ts

@@ -0,0 +1,7 @@
+export { applyInBothOrders } from './orders/apply-in-both-orders.ts';
+export type { OrderReport, StoreSnapshot } from './orders/apply-in-both-orders.ts';
+export type { MemoryStoreReader } from './memory-store-reader/create-memory-store-reader.ts';
+export { createTestStore } from './test-store/create-test-store.ts';
+export type { AppliedFact, ApplyOptions, TestStore } from './test-store/types.ts';
+export { ApplyError } from './utils/apply-error.ts';
+export type { ApplyFailure, ChangeEntry, Row } from './utils/types.ts';

package/src/links/read-link-rows.ts

@@ -0,0 +1,14 @@
+import type { LinkRow } from '@pepys/kit';
+import type { StoreDatabase } from '../store-database/types.ts';
+
+// Every link, or every link of one consumer, in the order of its key.
+export function readLinkRows(db: StoreDatabase, origin?: string): Promise<LinkRow[]> {
+  const links = db.selectFrom('link').selectAll();
+
+  return (origin === undefined ? links : links.where('origin', '=', origin))
+    .orderBy('type')
+    .orderBy('from_id')
+    .orderBy('to_id')
+    .orderBy('origin')
+    .execute();
+}

package/src/links/remove-link.ts

@@ -0,0 +1,16 @@
+import type { StoreDatabase } from '../store-database/types.ts';
+import type { WireLink } from '../utils/types.ts';
+
+export async function removeLink(
+  db: StoreDatabase,
+  origin: string,
+  link: Pick<WireLink, 'type' | 'from' | 'to'>,
+): Promise<void> {
+  await db
+    .deleteFrom('link')
+    .where('origin', '=', origin)
+    .where('type', '=', link.type)
+    .where('from_id', '=', link.from)
+    .where('to_id', '=', link.to)
+    .execute();
+}

package/src/links/write-link.ts

@@ -0,0 +1,24 @@
+import type { StoreDatabase } from '../store-database/types.ts';
+import type { WireLink } from '../utils/types.ts';
+
+// Writes the link of `origin`, the consumer's name, with the scope that owns it, or null in fact
+// mode.
+export async function writeLink(
+  db: StoreDatabase,
+  origin: string,
+  scope: string | null,
+  link: WireLink,
+): Promise<void> {
+  await db
+    .replaceInto('link')
+    .values({
+      type: link.type,
+      from_id: link.from,
+      to_id: link.to,
+      scope,
+      origin,
+      confidence: link.confidence,
+      evidence: link.evidence,
+    })
+    .execute();
+}

package/src/memory-store-reader/create-memory-store-reader.ts

@@ -0,0 +1,61 @@
+import type { CheckedDeployment, ConsumersDB, StoreReader } from '@pepys/kit';
+import { createTursoDialect } from '@pepys/kysely-turso';
+import type { Database } from '@tursodatabase/database';
+import { Kysely } from 'kysely';
+
+// The memory adapter of the store reader port. `markDown` gives a test the path of a consumer with
+// no current version.
+export interface MemoryStoreReader extends StoreReader {
+  markDown: (consumer: string) => void;
+}
+
+// One database holds every consumer, so the two shared views show the rows of every consumer, the
+// ones a call does not attach included. Each call runs in a transaction that rolls back, so a
+// handler keeps nothing it writes. The test store owns the database and closes it.
+export function createMemoryStoreReader(
+  database: Database,
+  deployment: CheckedDeployment,
+): MemoryStoreReader {
+  const down = new Set<string>();
+  const deployed = new Set(deployment.consumers.map((consumer) => consumer.name));
+
+  const isCurrent = (name: string) => deployed.has(name) && !down.has(name);
+
+  return {
+    markDown: (consumer) => {
+      down.add(consumer);
+    },
+    withConsumers: async (consumers, work, options) => {
+      const missing = consumers.map((consumer) => consumer.name).filter((name) => !isCurrent(name));
+
+      if (missing.length > 0) {
+        return { ok: false, failure: { reason: 'no-current-version', consumers: missing } };
+      }
+
+      const optional = (options?.whenCurrent ?? []).map((consumer) => consumer.name);
+
+      const attached = [
+        ...consumers.map((consumer) => consumer.name),
+        ...optional.filter((name) => isCurrent(name)),
+      ];
+
+      const db = new Kysely<ConsumersDB<typeof consumers>>({
+        dialect: createTursoDialect({ database: () => Promise.resolve(database) }),
+      });
+
+      const transaction = await db.startTransaction().execute();
+
+      try {
+        const value = await work(transaction, {
+          attached: Object.fromEntries(attached.map((name) => [name, { version: 'memory' }])),
+          skipped: optional.filter((name) => !isCurrent(name)),
+        });
+
+        return { ok: true, value };
+      } finally {
+        await transaction.rollback().execute();
+      }
+    },
+    close: () => Promise.resolve(),
+  };
+}

package/src/operations/parse-operations.ts

@@ -0,0 +1,168 @@
+import type { ConsumerDeclaration, Result } from '@pepys/kit';
+import type { ApplyFailure, Row, WireLink } from '../utils/types.ts';
+import type { Operation } from './types.ts';
+
+// The wire of a module's output: one JSON list, each item an object with exactly one verb.
+export function parseOperations(
+  consumer: ConsumerDeclaration,
+  output: string,
+): Result<Operation[], ApplyFailure> {
+  const bad = (reason: string): Result<Operation[], ApplyFailure> => ({
+    ok: false,
+    failure: { kind: 'bad-operation', consumer: consumer.name, reason },
+  });
+
+  const parsed = parseOutput(output);
+
+  if (!parsed.ok) {
+    return bad('the output is not JSON');
+  }
+
+  if (!Array.isArray(parsed.value)) {
+    return bad('the output is not a list');
+  }
+
+  const operations: Operation[] = [];
+
+  for (const item of parsed.value) {
+    const operation = parseOperation(consumer, item);
+
+    if (!operation.ok) {
+      return operation;
+    }
+
+    operations.push(operation.value);
+  }
+
+  return { ok: true, value: operations };
+}
+
+function parseOutput(output: string): Result<unknown, undefined> {
+  try {
+    const value: unknown = JSON.parse(output, reviveNumber);
+
+    return { ok: true, value };
+  } catch {
+    return { ok: false, failure: undefined };
+  }
+}
+
+interface ReviverContext {
+  source?: string;
+}
+
+// JSON.parse loses the text of a number, so `1.0` reads as `1`. The reviver's source text keeps
+// it: each object records the keys whose number had a fraction or an exponent.
+const inexactKeys = new WeakMap<object, Set<string>>();
+
+function reviveNumber(
+  this: unknown,
+  key: string,
+  value: unknown,
+  context?: ReviverContext,
+): unknown {
+  if (typeof value === 'number' && isObject(this) && /[.eE]/u.test(context?.source ?? '')) {
+    inexactKeys.set(this, new Set([...(inexactKeys.get(this) ?? []), key]));
+  }
+
+  return value;
+}
+
+function isObject(value: unknown): value is Row {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function parseOperation(
+  consumer: ConsumerDeclaration,
+  item: unknown,
+): Result<Operation, ApplyFailure> {
+  const bad = (reason: string): Result<Operation, ApplyFailure> => ({
+    ok: false,
+    failure: { kind: 'bad-operation', consumer: consumer.name, reason },
+  });
+
+  if (!isObject(item)) {
+    return bad('an operation is not an object');
+  }
+
+  const entries = Object.entries(item);
+  const [verb, body] = entries[0] ?? [];
+
+  if (entries.length !== 1) {
+    return bad('an operation does not hold exactly one verb');
+  }
+
+  if (verb === 'link') {
+    const link = parseLink(body);
+
+    return link === undefined
+      ? bad('a link is not a type, two ends, a confidence and an evidence')
+      : { ok: true, value: { verb: 'link', link } };
+  }
+
+  if (verb === 'unlink') {
+    const link = parseLinkKey(body);
+
+    return link === undefined
+      ? bad('an unlink is not a type and two ends')
+      : { ok: true, value: { verb: 'unlink', link } };
+  }
+
+  if (verb !== 'upsert' && verb !== 'delete') {
+    return bad(`the verb ${String(verb)} is unknown`);
+  }
+
+  const tables = isObject(body) ? Object.entries(body) : [];
+  const [table, row] = tables[0] ?? [];
+
+  if (tables.length !== 1 || table === undefined || !isObject(row)) {
+    return bad(`an ${verb} does not hold exactly one table and one row`);
+  }
+
+  if (!Object.hasOwn(consumer.tables, table)) {
+    return { ok: false, failure: { kind: 'not-owned', consumer: consumer.name, table } };
+  }
+
+  const columns = inexactKeys.get(row) ?? new Set<string>();
+
+  return {
+    ok: true,
+    value:
+      verb === 'upsert'
+        ? { verb, table, row, inexact: columns }
+        : { verb, table, key: row, inexact: columns },
+  };
+}
+
+function parseLink(body: unknown): WireLink | undefined {
+  const key = parseLinkKey(body);
+
+  if (key === undefined || !isObject(body)) {
+    return undefined;
+  }
+
+  const confidence = body['confidence'];
+  const evidence = body['evidence'];
+
+  if (typeof confidence !== 'number' || typeof evidence !== 'string') {
+    return undefined;
+  }
+
+  return { ...key, confidence, evidence };
+}
+
+function parseLinkKey(body: unknown): Pick<WireLink, 'type' | 'from' | 'to'> | undefined {
+  if (!isObject(body)) {
+    return undefined;
+  }
+
+  const type = body['type'];
+  const from = body['from'];
+  const to = body['to'];
+
+  if (typeof type !== 'string' || typeof from !== 'string' || typeof to !== 'string') {
+    return undefined;
+  }
+
+  return { type, from, to };
+}

package/src/operations/types.ts

@@ -0,0 +1,9 @@
+import type { Row, WireLink } from '../utils/types.ts';
+
+// One operation of a module's output. `inexact` names the columns whose number the module wrote
+// with a fraction or an exponent.
+export type Operation =
+  | { verb: 'upsert'; table: string; row: Row; inexact: ReadonlySet<string> }
+  | { verb: 'delete'; table: string; key: Row; inexact: ReadonlySet<string> }
+  | { verb: 'link'; link: WireLink }
+  | { verb: 'unlink'; link: Pick<WireLink, 'type' | 'from' | 'to'> };