@pepys/testing 0.0.0-stage → 0.1.0

package/src/scope-replace/types.ts

@@ -0,0 +1,39 @@
+import type { ChangeEntry, Row, WireLink } from '../utils/types.ts';
+
+export interface KeyedRow {
+  table: string;
+  key: string;
+  row: Row;
+}
+
+// What one scope returned, checked and keyed.
+export interface ScopeResult {
+  rows: readonly KeyedRow[];
+  links: readonly WireLink[];
+}
+
+export interface StoredLink {
+  scope: string | null;
+  link: WireLink;
+}
+
+// What the consumer's store holds that the replace of one scope needs. Rows and owners are keyed by
+// `buildRowID`, links by the JSON text of their key.
+export interface ScopeState {
+  // the scope of each row that a scope owns
+  rowOwners: ReadonlyMap<string, string>;
+
+  // the stored row of each key that the result names or that the scope owns
+  storedRows: ReadonlyMap<string, Row>;
+
+  // every link of the consumer
+  storedLinks: ReadonlyMap<string, StoredLink>;
+}
+
+export interface ScopeChanges {
+  writtenRows: readonly KeyedRow[];
+  writtenLinks: readonly WireLink[];
+  deletedRows: readonly Omit<KeyedRow, 'row'>[];
+  deletedLinks: readonly WireLink[];
+  entries: readonly ChangeEntry[];
+}

package/src/scope-replace/write-scope-changes.ts

@@ -0,0 +1,50 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+import { removeLink } from '../links/remove-link.ts';
+import { writeLink } from '../links/write-link.ts';
+import { removeRow } from '../rows/remove-row.ts';
+import { writeRow } from '../rows/write-row.ts';
+import type { StoreDatabase } from '../store-database/types.ts';
+import type { ScopeChanges } from './types.ts';
+
+export async function writeScopeChanges(
+  db: StoreDatabase,
+  consumer: ConsumerDeclaration,
+  scope: string,
+  changes: ScopeChanges,
+): Promise<void> {
+  for (const row of changes.writtenRows) {
+    const table = consumer.tables[row.table];
+
+    if (table !== undefined) {
+      await writeRow(db, row.table, table, row.row);
+
+      await db
+        .replaceInto('scope_row')
+        .values({ consumer: consumer.name, table_name: row.table, key: row.key, scope })
+        .execute();
+    }
+  }
+
+  for (const link of changes.writtenLinks) {
+    await writeLink(db, consumer.name, scope, link);
+  }
+
+  for (const row of changes.deletedRows) {
+    const table = consumer.tables[row.table];
+
+    if (table !== undefined) {
+      await removeRow(db, row.table, table, row.key);
+    }
+
+    await db
+      .deleteFrom('scope_row')
+      .where('consumer', '=', consumer.name)
+      .where('table_name', '=', row.table)
+      .where('key', '=', row.key)
+      .execute();
+  }
+
+  for (const link of changes.deletedLinks) {
+    await removeLink(db, consumer.name, link);
+  }
+}

package/src/store-database/create-store-database.ts

@@ -0,0 +1,31 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+import { createTursoDialect } from '@pepys/kysely-turso';
+import { connect } from '@tursodatabase/database';
+import type { Database } from '@tursodatabase/database';
+import { Kysely } from 'kysely';
+import { formatStoreSchema } from './format-store-schema.ts';
+import type { StoreDatabase, StoreTables } from './types.ts';
+
+interface StoreConnection {
+  db: StoreDatabase;
+
+  // The same database under the table types of one query. Kysely cannot narrow its own table
+  // types to a generic set of consumers, so a query call gets its own instance.
+  database: Database;
+}
+
+// One in-memory database with every table of `consumers`, the link and scope tables, and the two
+// shared views.
+export async function createStoreDatabase(
+  consumers: readonly ConsumerDeclaration[],
+): Promise<StoreConnection> {
+  const database = await connect(':memory:');
+
+  await database.exec(formatStoreSchema(consumers));
+
+  const db = new Kysely<StoreTables>({
+    dialect: createTursoDialect({ database: () => Promise.resolve(database) }),
+  });
+
+  return { db, database };
+}

package/src/store-database/format-store-schema.ts

@@ -0,0 +1,57 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+import { generateSqliteSchema } from '@pepys/kit/generators';
+
+const hostSchema = `
+CREATE TABLE "link" (
+  "type" TEXT NOT NULL,
+  "from_id" TEXT NOT NULL,
+  "to_id" TEXT NOT NULL,
+  "scope" TEXT,
+  "origin" TEXT NOT NULL,
+  "confidence" REAL NOT NULL,
+  "evidence" TEXT NOT NULL,
+  PRIMARY KEY ("origin", "type", "from_id", "to_id")
+);
+
+CREATE TABLE "scope_row" (
+  "consumer" TEXT NOT NULL,
+  "table_name" TEXT NOT NULL,
+  "key" TEXT NOT NULL,
+  "scope" TEXT NOT NULL,
+  PRIMARY KEY ("consumer", "table_name", "key")
+);
+
+CREATE VIEW "link_effective" AS
+SELECT "type", "from_id", "to_id", "scope", "origin", "confidence", "evidence" FROM "link";
+`;
+
+const emptyEntityView = `
+CREATE VIEW "entity" AS
+SELECT '' AS "id", '' AS "kind", '' AS "title", NULL AS "url", NULL AS "actor", '' AS "updated_at"
+WHERE 0;
+`;
+
+// The kit writes one `entity` view per consumer file, so the one database gets a single view over
+// the entity views of every consumer.
+export function formatStoreSchema(consumers: readonly ConsumerDeclaration[]): string {
+  const tables = consumers.map((consumer) =>
+    generateSqliteSchema({ ...consumer, entityViews: [] }),
+  );
+
+  const views = consumers.flatMap((consumer) => consumer.entityViews);
+
+  const entityView =
+    views.length === 0
+      ? emptyEntityView
+      : generateSqliteSchema({
+          name: 'entity',
+          entry: new URL('file:///entity'),
+          input: { mode: 'fact', streams: [] },
+          tables: {},
+          entityViews: views,
+          reads: [],
+          links: [],
+        });
+
+  return [...tables, hostSchema, entityView].join('\n');
+}

package/src/store-database/types.ts

@@ -0,0 +1,12 @@
+import type { LinkRow, SharedViews } from '@pepys/kit';
+import type { Kysely } from 'kysely';
+
+// The tables that the test store keeps beside the consumers' own. The host keeps `link` and
+// `scope_row` in each consumer's file; one database holds every consumer here, so both carry the
+// consumer's name.
+export interface StoreTables extends SharedViews {
+  link: LinkRow;
+  scope_row: { consumer: string; table_name: string; key: string; scope: string };
+}
+
+export type StoreDatabase = Kysely<StoreTables>;

package/src/test-store/create-test-store.ts

@@ -0,0 +1,94 @@
+import { checkDeployment } from '@pepys/kit';
+import type { Plugin, TableDeclaration } from '@pepys/kit';
+import { buildFactIndex } from '../fact-index/build-fact-index.ts';
+import type { FactIndex } from '../fact-index/types.ts';
+import { readLinkRows } from '../links/read-link-rows.ts';
+import { createMemoryStoreReader } from '../memory-store-reader/create-memory-store-reader.ts';
+import { readTableRows } from '../rows/read-table-rows.ts';
+import { writeRow } from '../rows/write-row.ts';
+import { createStoreDatabase } from '../store-database/create-store-database.ts';
+import { ApplyError } from '../utils/apply-error.ts';
+import { parseAppliedFacts } from './parse-applied-facts.ts';
+import { runConsumers } from './run-consumers.ts';
+import type { AppliedFactIDs } from './run-consumers.ts';
+import type { TestStore } from './types.ts';
+
+interface TestStoreOptions {
+  plugins: readonly Plugin[];
+}
+
+// A store for `bun test`: every consumer of the plug-ins runs its TypeScript entry over one
+// in-memory database. It repeats the host's envelope check, link check and scope replace, and
+// proves neither the scriptc compile nor the Rust host.
+export async function createTestStore(options: TestStoreOptions): Promise<TestStore> {
+  const checked = checkDeployment({ name: 'test-store', plugins: options.plugins });
+
+  if (!checked.ok) {
+    throw new Error(`the deployment is invalid: ${JSON.stringify(checked.failure)}`);
+  }
+
+  const deployment = checked.value;
+
+  const store = await createStoreDatabase(deployment.consumers);
+
+  const db = store.db;
+  const reader = createMemoryStoreReader(store.database, deployment);
+  let index: FactIndex = { log: [], firstCopies: new Map() };
+
+  let applied: AppliedFactIDs = new Map();
+
+  const getTable = (name: string): TableDeclaration => {
+    const table = deployment.consumers.find((consumer) => Object.hasOwn(consumer.tables, name))
+      ?.tables[name];
+
+    if (table === undefined) {
+      throw new Error(`no consumer of the deployment has the table ${name}`);
+    }
+
+    return table;
+  };
+
+  return {
+    writeRows: async (_consumer, table, rows) => {
+      for (const row of rows) {
+        await writeRow(db, table, getTable(table), row);
+      }
+    },
+    apply: async (source, facts, applyOptions) => {
+      const envelopes = parseAppliedFacts(source, facts, applyOptions?.sender);
+
+      if (!envelopes.ok) {
+        throw new ApplyError(envelopes.failure);
+      }
+
+      const next = buildFactIndex(index, envelopes.value);
+
+      applied = await db
+        .transaction()
+        .execute((transaction) =>
+          runConsumers(transaction, deployment.consumers, { ...next, applied }),
+        );
+
+      index = next.index;
+    },
+    query: async (query, input) => {
+      const parsed = query.input.parse(input);
+
+      const result = await reader.withConsumers(query.reads, (queryDB) =>
+        query.handler({ input: parsed, db: queryDB }),
+      );
+
+      if (!result.ok) {
+        throw new Error(
+          `the query ${query.name} reads consumers with no current version: ${result.failure.consumers.join(', ')}`,
+        );
+      }
+
+      return query.output.parse(result.value);
+    },
+    readLinks: () => readLinkRows(db),
+    readRows: (table) => readTableRows(db, table, getTable(table)),
+    reader,
+    [Symbol.asyncDispose]: () => db.destroy(),
+  };
+}

package/src/test-store/parse-applied-facts.ts

@@ -0,0 +1,64 @@
+import { parseFactEnvelope, readFactBatch } from '@pepys/kit';
+import type { FactEnvelope, Result, SourceDefinition } from '@pepys/kit';
+import type { ApplyFailure } from '../utils/types.ts';
+import type { ApplyOptions } from './types.ts';
+
+interface AppliedEnvelope {
+  stream: string;
+  envelope: FactEnvelope<unknown>;
+}
+
+// The facts of one apply as ingest would store them: the fact batch read builds each envelope, and
+// the envelope parser checks it.
+export function parseAppliedFacts(
+  source: SourceDefinition,
+  facts: readonly { payload: unknown }[],
+  sender: ApplyOptions['sender'],
+): Result<AppliedEnvelope[], ApplyFailure> {
+  const withOrigin = facts.map((fact) =>
+    sender === undefined || !isObject(fact.payload)
+      ? fact
+      : {
+          ...fact,
+          payload: { ...fact.payload, origin: { issuer: sender.issuer, subject: sender.subject } },
+        },
+  );
+
+  const received = readFactBatch(
+    {
+      headers: new Headers(),
+      body: new TextEncoder().encode(JSON.stringify({ v: 1, facts: withOrigin })),
+    },
+    { name: source.name, families: source.families },
+  );
+
+  if (received.kind === 'refused') {
+    return refuseFacts(received.reason);
+  }
+
+  if (received.kind === 'ignored') {
+    return { ok: true, value: [] };
+  }
+
+  const envelopes: AppliedEnvelope[] = [];
+
+  for (const pending of received.batch.facts) {
+    const parsed = parseFactEnvelope(pending.fact);
+
+    if (!parsed.ok) {
+      return refuseFacts(parsed.failure.issues.map((issue) => issue.message).join('; '));
+    }
+
+    envelopes.push({ stream: pending.stream, envelope: parsed.value });
+  }
+
+  return { ok: true, value: envelopes };
+}
+
+function isObject(value: unknown): value is Readonly<Record<string, unknown>> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function refuseFacts(reason: string): Result<AppliedEnvelope[], ApplyFailure> {
+  return { ok: false, failure: { kind: 'refused-facts', reason } };
+}

package/src/test-store/run-consumers.ts

@@ -0,0 +1,117 @@
+import { parseEntityID } from '@pepys/kit';
+import type { ConsumerDeclaration } from '@pepys/kit';
+import invariant from 'tiny-invariant';
+import { buildReadFunctions } from '../consumer-entries/build-read-functions.ts';
+import { runEntry } from '../consumer-entries/run-entry.ts';
+import { collectScopeFacts } from '../fact-index/collect-scope-facts.ts';
+import type { FactIndex, LoggedFact } from '../fact-index/types.ts';
+import { applyFactOperations } from '../fact-mode/apply-fact-operations.ts';
+import { readRowFunctions } from '../fact-mode/read-row-functions.ts';
+import { parseOperations } from '../operations/parse-operations.ts';
+import { replaceScope } from '../scope-replace/replace-scope.ts';
+import type { StoreDatabase } from '../store-database/types.ts';
+import { ApplyError } from '../utils/apply-error.ts';
+
+// The fact IDs each fact-mode consumer has applied, by consumer name.
+export type AppliedFactIDs = ReadonlyMap<string, ReadonlySet<string>>;
+
+interface ConsumerRun {
+  index: FactIndex;
+  appended: readonly LoggedFact[];
+  applied: AppliedFactIDs;
+}
+
+// Runs every consumer on one appended batch, in deployment order, inside the caller's transaction,
+// and returns the applied fact IDs after it.
+export async function runConsumers(
+  db: StoreDatabase,
+  consumers: readonly ConsumerDeclaration[],
+  run: ConsumerRun,
+): Promise<AppliedFactIDs> {
+  const applied = new Map(run.applied);
+
+  for (const consumer of consumers) {
+    const facts = run.appended.filter((record) =>
+      consumer.input.streams.some((stream) => stream === record.stream),
+    );
+
+    if (consumer.input.mode === 'fact') {
+      const seen = await runFactConsumer(
+        db,
+        consumer,
+        facts,
+        run.applied.get(consumer.name) ?? new Set(),
+      );
+
+      applied.set(consumer.name, seen);
+    } else {
+      await runScopeConsumer(db, consumer, facts, run.index);
+    }
+  }
+
+  return applied;
+}
+
+async function runFactConsumer(
+  db: StoreDatabase,
+  consumer: ConsumerDeclaration,
+  facts: readonly LoggedFact[],
+  applied: ReadonlySet<string>,
+): Promise<ReadonlySet<string>> {
+  const seen = new Set(applied);
+
+  for (const record of facts) {
+    if (seen.has(record.envelope.id)) {
+      continue;
+    }
+
+    seen.add(record.envelope.id);
+
+    const host = await readRowFunctions(db, consumer);
+    const output = await runEntry(consumer, 'applyFact', JSON.stringify(record.envelope), host);
+
+    const operations = parseOperations(consumer, output);
+
+    if (!operations.ok) {
+      throw new ApplyError(operations.failure);
+    }
+
+    await applyFactOperations(db, consumer, operations.value);
+  }
+
+  return seen;
+}
+
+// The dirty scopes are the entities of the batch's facts that have the scope kind, each once, in
+// sorted order. A copy of a fact marks its scope dirty too, and the replace then finds no change.
+async function runScopeConsumer(
+  db: StoreDatabase,
+  consumer: ConsumerDeclaration,
+  facts: readonly LoggedFact[],
+  index: FactIndex,
+): Promise<void> {
+  invariant(consumer.input.mode === 'scope', 'only a scope consumer has a scope kind');
+
+  const scopeKind = consumer.input.scopeKind;
+
+  const scopes = [
+    ...new Set(
+      facts
+        .map((record) => record.envelope.entity)
+        .filter((entity) => {
+          const parsed = parseEntityID(entity);
+
+          return parsed.ok && `${parsed.value.source}:${parsed.value.kind}` === scopeKind;
+        }),
+    ),
+  ].toSorted((left, right) => (left < right ? -1 : 1));
+
+  for (const scope of scopes) {
+    const scopeFacts = JSON.stringify(collectScopeFacts(index, scope, consumer.input.streams));
+    const host = buildReadFunctions('readScopeFacts', new Map([[scope, scopeFacts]]), 'trap');
+
+    const output = await runEntry(consumer, 'applyScope', scope, host);
+
+    await replaceScope(db, consumer, scope, output);
+  }
+}

package/src/test-store/types.ts

@@ -0,0 +1,68 @@
+import type {
+  ConsumerDeclaration,
+  LinkRow,
+  OutgoingFact,
+  QueryDefinition,
+  Sender,
+  SourceDefinition,
+  TableRow,
+} from '@pepys/kit';
+import type { z } from 'zod';
+import type { MemoryStoreReader } from '../memory-store-reader/create-memory-store-reader.ts';
+import type { Row } from '../utils/types.ts';
+
+type WithOptionalOrigin<Payload> = Payload extends { origin: infer Origin }
+  ? Omit<Payload, 'origin'> & { origin?: Origin }
+  : Payload;
+
+// A fact as a sender writes it in a fact batch. With a `sender` on the apply, the payload's
+// `origin` may be left out: the store writes it, as a source's read does for a proven sender.
+export type AppliedFact<Source extends SourceDefinition> =
+  OutgoingFact<Source> extends infer Fact
+    ? Fact extends { payload: infer Payload }
+      ? Omit<Fact, 'payload'> & { payload: WithOptionalOrigin<Payload> }
+      : never
+    : never;
+
+export interface ApplyOptions {
+  // the proven sender of the batch; the store copies it into each payload as `origin`
+  sender?: Pick<Sender, 'issuer' | 'subject'> | undefined;
+}
+
+export interface TestStore extends AsyncDisposable {
+  // Writes rows as they are, with no scope, for a test of a query.
+  writeRows: <
+    Consumer extends ConsumerDeclaration,
+    Table extends keyof Consumer['tables'] & string,
+  >(
+    consumer: Consumer,
+    table: Table,
+    rows: readonly TableRow<Consumer['tables'][Table]>[],
+  ) => Promise<void>;
+
+  // Builds the envelopes as a fact batch path does, with `event:` IDs, parses each with the
+  // envelope parser, and runs each consumer whose input names the fact's stream. A refusal throws
+  // an ApplyError, and the batch then writes nothing.
+  apply: <Source extends SourceDefinition>(
+    source: Source,
+    facts: readonly AppliedFact<Source>[],
+    options?: ApplyOptions,
+  ) => Promise<void>;
+
+  query: <
+    InputSchema extends z.ZodType,
+    OutputSchema extends z.ZodType,
+    Consumers extends readonly ConsumerDeclaration[],
+  >(
+    query: QueryDefinition<string, InputSchema, OutputSchema, Consumers>,
+    input: z.input<InputSchema>,
+  ) => Promise<z.output<OutputSchema>>;
+
+  // every link, in key order
+  readLinks: () => Promise<readonly LinkRow[]>;
+
+  // every row of one table in primary key order, as the module wrote it
+  readRows: (table: string) => Promise<readonly Row[]>;
+
+  reader: MemoryStoreReader;
+}

package/src/utils/apply-error.ts

@@ -0,0 +1,13 @@
+import type { ApplyFailure } from './types.ts';
+
+// What `apply` throws when it refuses a batch. The batch then wrote nothing.
+export class ApplyError extends Error {
+  readonly failure: ApplyFailure;
+
+  constructor(failure: ApplyFailure) {
+    super(`the test store refused the batch: ${JSON.stringify(failure)}`);
+
+    this.name = 'ApplyError';
+    this.failure = failure;
+  }
+}

package/src/utils/types.ts

@@ -0,0 +1,47 @@
+import type { LinkProblem } from '@pepys/kit';
+
+// Why the test store refused an apply. The kinds from `bad-operation` on are the host's refusals
+// of a module's output, which the scope replace case file names.
+export type ApplyFailure =
+  | { kind: 'refused-facts'; reason: string }
+  | { kind: 'entry'; consumer: string; reason: string }
+
+  // output that is no list of operations, or a delete or an unlink from a scope consumer
+  | { kind: 'bad-operation'; consumer: string; reason: string }
+  | { kind: 'not-owned'; consumer: string; table: string }
+  | { kind: 'bad-row'; consumer: string; table: string; reason: string }
+  | { kind: 'bad-link'; consumer: string; problem: LinkProblem }
+  | { kind: 'duplicate-row'; consumer: string; table: string; key: string }
+  | { kind: 'duplicate-link'; consumer: string; key: string }
+
+  // `owner` is null for a link stored with no scope
+  | {
+      kind: 'scope-conflict';
+      consumer: string;
+      scope: string;
+      table: string;
+      key: string;
+      owner: string | null;
+    };
+
+// A row as the module writes it and as the test store reads it back: booleans as booleans and a
+// JSON column as its value.
+export type Row = Readonly<Record<string, unknown>>;
+
+export interface WireLink {
+  type: string;
+  from: string;
+  to: string;
+  confidence: number;
+  evidence: string;
+}
+
+// One State Protocol message of the change log. `key` is the JSON text of the key's values: a
+// row's primary key columns in order, or a link's type, `from` and `to`.
+export interface ChangeEntry {
+  type: string;
+  key: string;
+  value?: unknown;
+  old_value?: unknown;
+  headers: { operation: 'insert' | 'update' | 'delete' };
+}