@pepys/testing 0.1.0 → 0.2.0

package/README.md

@@ -28,6 +28,9 @@ Set the compiler options from the [kit's README](../kit/README.md#configuration)
 ## Features
 
 - `createTestStore`: apply facts or write rows, then run a query, read rows and read links.
+- `applyFactLog`: apply the facts that `createIngest` of `@pepys/ingest` stored in a memory fact log
+  to a test store, in place of the host. A second call applies only the records the log stored after
+  the first.
 - `applyInBothOrders`: apply facts in two orders and compare the stores.
 - The memory adapter of the store reader, for a test of a core query.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pepys/testing",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Test a pepys plug-in on Bun: an in-memory store that runs its consumers and queries",
   "license": "MIT",
   "repository": {
@@ -19,8 +19,9 @@
     "access": "public"
   },
   "dependencies": {
-    "@pepys/kit": "0.1.0",
-    "@pepys/kysely-turso": "0.1.0",
+    "@pepys/ingest": "0.2.0",
+    "@pepys/kit": "0.2.0",
+    "@pepys/kysely-turso": "0.2.0",
     "@tursodatabase/database": "0.8.3",
     "kysely": "0.29.8",
     "tiny-invariant": "1.3.3",

package/src/fact-index/collect-scope-facts.ts

@@ -1,4 +1,5 @@
 import type { FactEnvelope } from '@pepys/kit';
+import { compareCodePoints } from '../utils/compare-code-points.ts';
 import type { FactIndex } from './types.ts';
 
 // The read of one scope, as the host serves it: every fact of the entity on `streams`, one per fact
@@ -13,8 +14,8 @@ export function collectScopeFacts(
     .map((record) => ({ record, sortKey: toSortKey(record.envelope.at) }))
     .toSorted(
       (left, right) =>
-        compareText(left.sortKey, right.sortKey) ||
-        compareText(left.record.stream, right.record.stream) ||
+        compareCodePoints(left.sortKey, right.sortKey) ||
+        compareCodePoints(left.record.stream, right.record.stream) ||
         left.record.seqNum - right.record.seqNum,
     )
     .map((entry) => entry.record.envelope);
@@ -32,11 +33,3 @@ function toSortKey(at: string): string {
 
   return `${parts.groups['seconds']}.${fraction}${parts.groups['zone']}`;
 }
-
-function compareText(left: string, right: string): number {
-  if (left === right) {
-    return 0;
-  }
-
-  return left < right ? -1 : 1;
-}

package/src/fact-log/apply-fact-log.ts

@@ -0,0 +1,58 @@
+import type { MemoryFactLog } from '@pepys/ingest';
+import { toStreamName, toStreamOffset } from '@pepys/kit';
+import type { CheckedDeployment, FactEnvelope, StreamName } from '@pepys/kit';
+import invariant from 'tiny-invariant';
+import { storeHooks } from '../test-store/store-hooks.ts';
+import type { TestStore } from '../test-store/types.ts';
+
+// The host's part for a test: the records of every stream a consumer reads go to the store as one
+// batch, whole streams in the order the consumers name them. A refusal throws an ApplyError and
+// moves no offset, so the next call reads the same records again.
+export async function applyFactLog(
+  store: TestStore,
+  log: MemoryFactLog,
+  deployment: CheckedDeployment,
+): Promise<void> {
+  const hooks = storeHooks.get(store);
+
+  if (hooks === undefined) {
+    throw new Error('the store is not the object that createTestStore returned');
+  }
+
+  const streams = new Set(deployment.consumers.flatMap((consumer) => consumer.input.streams));
+
+  const records: { stream: StreamName; envelope: FactEnvelope<unknown> }[] = [];
+
+  const tails = new Map<StreamName, number>();
+
+  for (const declared of streams) {
+    const stream = toStreamName(declared);
+    const from = hooks.nextOffsets.get(stream) ?? 0;
+
+    const tail = await log.readTail(stream);
+
+    if (!tail.ok) {
+      throw new Error(`the fact log has no stream ${declared}, which a consumer reads`);
+    }
+
+    if (tail.value === from) {
+      continue;
+    }
+
+    const read = await log.read(stream, toStreamOffset(from), tail.value - from);
+
+    invariant(read.ok, 'a stream with a tail can be read');
+
+    for (const stored of read.value) {
+      records.push({ stream, envelope: stored.fact });
+    }
+
+    tails.set(stream, tail.value);
+  }
+
+  await hooks.applyEnvelopes(records);
+
+  for (const [stream, tail] of tails) {
+    hooks.nextOffsets.set(stream, tail);
+  }
+}

package/src/fact-mode/apply-fact-operations.ts

@@ -6,6 +6,7 @@ import { removeLink } from '../links/remove-link.ts';
 import { writeLink } from '../links/write-link.ts';
 import type { Operation } from '../operations/types.ts';
 import { buildRowKey } from '../rows/build-row-key.ts';
+import { findKeyProblem } from '../rows/find-key-problem.ts';
 import { findRowProblem } from '../rows/find-row-problem.ts';
 import { removeRow } from '../rows/remove-row.ts';
 import { writeRow } from '../rows/write-row.ts';
@@ -28,12 +29,13 @@ export async function applyFactOperations(
       }
     }
 
-    if (operation.verb === 'upsert') {
-      const reason = findRowProblem(
-        getTable(consumer, operation.table),
-        operation.row,
-        operation.inexact,
-      );
+    if (operation.verb === 'upsert' || operation.verb === 'delete') {
+      const table = getTable(consumer, operation.table);
+
+      const reason =
+        operation.verb === 'upsert'
+          ? findRowProblem(table, operation.row, operation.inexact)
+          : findKeyProblem(table, operation.key, operation.inexact);
 
       if (reason !== undefined) {
         throw new ApplyError({

package/src/index.ts

@@ -1,3 +1,4 @@
+export { applyFactLog } from './fact-log/apply-fact-log.ts';
 export { applyInBothOrders } from './orders/apply-in-both-orders.ts';
 export type { OrderReport, StoreSnapshot } from './orders/apply-in-both-orders.ts';
 export type { MemoryStoreReader } from './memory-store-reader/create-memory-store-reader.ts';

package/src/links/read-link-rows.ts

@@ -1,7 +1,6 @@
 import type { LinkRow } from '@pepys/kit';
 import type { StoreDatabase } from '../store-database/types.ts';
 
-// Every link, or every link of one consumer, in the order of its key.
 export function readLinkRows(db: StoreDatabase, origin?: string): Promise<LinkRow[]> {
   const links = db.selectFrom('link').selectAll();
 

package/src/rows/build-key-filter.ts

@@ -3,7 +3,6 @@ import { sql } from 'kysely';
 import type { RawBuilder } from 'kysely';
 import { toSQLValue } from './to-sql-value.ts';
 
-// The `where` clause of one row, from the JSON text of its key's values.
 export function buildKeyFilter(table: TableDeclaration, key: string): RawBuilder<unknown> {
   const parsed: unknown = JSON.parse(key);
   const values: readonly unknown[] = Array.isArray(parsed) ? parsed : [];

package/src/rows/build-row-key.ts

@@ -1,8 +1,8 @@
 import type { TableDeclaration } from '@pepys/kit';
 import type { Row } from '../utils/types.ts';
 
-// The key of a row as the host writes it in `scope_row` and the change log: the JSON text of the
-// primary key's values, in key order, for a key of one column and of several.
+// The key of a row as the host writes it in `scope_row` and the change log, for a key of one column
+// and of several. The host sorts an object's keys, so a key that holds an object can differ.
 export function buildRowKey(table: TableDeclaration, row: Row): string {
   return JSON.stringify(table.primaryKey.map((column) => row[column] ?? null));
 }

package/src/rows/find-key-problem.ts

@@ -0,0 +1,36 @@
+import type { TableDeclaration } from '@pepys/kit';
+import invariant from 'tiny-invariant';
+import type { Row } from '../utils/types.ts';
+import { isColumnValue } from './is-column-value.ts';
+
+// The host's check of a delete's key, in the host's order: no column outside the primary key, each
+// key value present fit for its column, then each key column present.
+export function findKeyProblem(
+  table: TableDeclaration,
+  key: Row,
+  inexact: ReadonlySet<string>,
+): string | undefined {
+  const outside = Object.keys(key).find((name) => !table.primaryKey.includes(name));
+
+  if (outside !== undefined) {
+    return `${outside} is not a key column`;
+  }
+
+  const present = table.primaryKey.filter((name) => Object.hasOwn(key, name));
+
+  const unfit = present.find((name) => {
+    const column = table.columns[name];
+
+    invariant(column, 'the deployment check refuses a key column that the table lacks');
+
+    return !isColumnValue(column, key[name], inexact.has(name));
+  });
+
+  if (unfit !== undefined) {
+    return `the column ${unfit} holds ${JSON.stringify(key[unfit])}`;
+  }
+
+  const absent = table.primaryKey.find((name) => !Object.hasOwn(key, name));
+
+  return absent === undefined ? undefined : `the key column ${absent} is absent`;
+}

package/src/rows/find-row-problem.ts

@@ -1,10 +1,9 @@
 import type { TableDeclaration } from '@pepys/kit';
-import { match } from 'ts-pattern';
 import type { Row } from '../utils/types.ts';
+import { isColumnValue } from './is-column-value.ts';
 
-// The host's check of a row: every column present, no other key, each value of its column's kind,
-// and null only where the column is nullable. `inexact` names the columns whose number the module
-// wrote with a fraction or an exponent, such as `1.0`: an integer column refuses those.
+// The host's check of a row: every column present, no other key, and each value fit for its
+// column. `inexact` names the columns whose number the module wrote with a fraction or an exponent.
 export function findRowProblem(
   table: TableDeclaration,
   row: Row,
@@ -21,21 +20,8 @@ export function findRowProblem(
       return `the column ${name} is absent`;
     }
 
-    const value = row[name];
-
-    const fits =
-      value === null
-        ? column.nullable === true
-        : match(column.kind)
-            .with('text', 'entityID', 'timestamp', () => typeof value === 'string')
-            .with('boolean', () => typeof value === 'boolean')
-            .with('json', () => true)
-            .with('real', () => typeof value === 'number')
-            .with('integer', () => Number.isSafeInteger(value) && !inexact.has(name))
-            .exhaustive();
-
-    if (!fits) {
-      return `the column ${name} holds ${JSON.stringify(value)}`;
+    if (!isColumnValue(column, row[name], inexact.has(name))) {
+      return `the column ${name} holds ${JSON.stringify(row[name])}`;
     }
   }
 

package/src/rows/is-column-value.ts

@@ -0,0 +1,23 @@
+import type { ColumnDeclaration } from '@pepys/kit';
+import { match } from 'ts-pattern';
+
+// The host's check of one value against its column: of the column's kind, and null only where the
+// column is nullable. `isInexact` marks a number the module wrote with a fraction or an exponent,
+// such as `1.0`, which an integer column refuses.
+export function isColumnValue(
+  column: ColumnDeclaration,
+  value: unknown,
+  isInexact: boolean,
+): boolean {
+  if (value === null) {
+    return column.nullable === true;
+  }
+
+  return match(column.kind)
+    .with('text', 'entityID', 'timestamp', () => typeof value === 'string')
+    .with('boolean', () => typeof value === 'boolean')
+    .with('json', () => true)
+    .with('real', () => typeof value === 'number')
+    .with('integer', () => Number.isSafeInteger(value) && !isInexact)
+    .exhaustive();
+}

package/src/rows/to-row-value.ts

@@ -1,7 +1,6 @@
 import type { ColumnDeclaration } from '@pepys/kit';
 import { match } from 'ts-pattern';
 
-// A stored value as the module wrote it: 0 or 1 back to a boolean, a JSON column's text parsed.
 export function toRowValue(column: ColumnDeclaration, stored: unknown): unknown {
   if (stored === null || stored === undefined) {
     return null;

package/src/rows/to-sql-value.ts

@@ -1,7 +1,6 @@
 import type { ColumnDeclaration } from '@pepys/kit';
 import { match } from 'ts-pattern';
 
-// A value of a checked row as SQLite stores it: a boolean as 0 or 1, a JSON column as its text.
 export function toSQLValue(column: ColumnDeclaration, value: unknown): string | number | null {
   if (value === null || value === undefined) {
     return null;

package/src/scope-replace/build-row-id.ts

@@ -1,4 +1,3 @@
-// One text for a table and a row key, for a map that holds the rows of several tables.
 export function buildRowID(table: string, key: string): string {
   return JSON.stringify([table, key]);
 }

package/src/scope-replace/pick-scope-changes.ts

@@ -1,4 +1,5 @@
 import type { Result } from '@pepys/kit';
+import { compareCodePoints } from '../utils/compare-code-points.ts';
 import type { ApplyFailure, ChangeEntry, WireLink } from '../utils/types.ts';
 import { buildLinkKey } from './build-link-key.ts';
 import { buildRowID } from './build-row-id.ts';
@@ -74,13 +75,12 @@ export function pickScopeChanges(
   return { ok: true, value: { writtenRows, writtenLinks, deletedRows, deletedLinks, entries } };
 }
 
-// Plain `<` on the parts, as Rust's ordered maps and SQLite compare text: never a locale order.
 function compareKeys(left: readonly string[], right: readonly string[]): number {
   for (const [index, part] of left.entries()) {
     const other = right[index] ?? '';
 
     if (part !== other) {
-      return part < other ? -1 : 1;
+      return compareCodePoints(part, other);
     }
   }
 

package/src/scope-replace/scope-replace-cases.json

@@ -1740,6 +1740,87 @@
         "ledger_tag": []
       },
       "links": []
+    },
+    {
+      "title": "the change entries come in code point order, so a key above U+FFFF sorts after a key at U+E000",
+      "batches": [
+        {
+          "scopes": [
+            {
+              "scope": "ledger:account:a",
+              "output": "[{\"upsert\":{\"ledger_entry\":{\"id\":\"ledger:entry:\\ud800\\udc00\",\"account_id\":\"ledger:account:a\",\"amount\":5,\"rate\":1.25,\"memo\":\"Rope\",\"details\":{\"till\":\"north\"},\"posted_at\":\"2026-11-01T09:00:00.000Z\"}}},{\"upsert\":{\"ledger_entry\":{\"id\":\"ledger:entry:\\ue000\",\"account_id\":\"ledger:account:a\",\"amount\":5,\"rate\":1.25,\"memo\":\"Rope\",\"details\":{\"till\":\"north\"},\"posted_at\":\"2026-11-01T09:00:00.000Z\"}}}]"
+            }
+          ],
+          "changes": [
+            {
+              "type": "ledger_entry",
+              "key": "[\"ledger:entry:\"]",
+              "value": {
+                "id": "ledger:entry:",
+                "account_id": "ledger:account:a",
+                "amount": 5,
+                "rate": 1.25,
+                "memo": "Rope",
+                "details": {
+                  "till": "north"
+                },
+                "posted_at": "2026-11-01T09:00:00.000Z"
+              },
+              "headers": {
+                "operation": "insert"
+              }
+            },
+            {
+              "type": "ledger_entry",
+              "key": "[\"ledger:entry:𐀀\"]",
+              "value": {
+                "id": "ledger:entry:𐀀",
+                "account_id": "ledger:account:a",
+                "amount": 5,
+                "rate": 1.25,
+                "memo": "Rope",
+                "details": {
+                  "till": "north"
+                },
+                "posted_at": "2026-11-01T09:00:00.000Z"
+              },
+              "headers": {
+                "operation": "insert"
+              }
+            }
+          ]
+        }
+      ],
+      "rows": {
+        "ledger_account": [],
+        "ledger_entry": [
+          {
+            "id": "ledger:entry:",
+            "account_id": "ledger:account:a",
+            "amount": 5,
+            "rate": 1.25,
+            "memo": "Rope",
+            "details": {
+              "till": "north"
+            },
+            "posted_at": "2026-11-01T09:00:00.000Z"
+          },
+          {
+            "id": "ledger:entry:𐀀",
+            "account_id": "ledger:account:a",
+            "amount": 5,
+            "rate": 1.25,
+            "memo": "Rope",
+            "details": {
+              "till": "north"
+            },
+            "posted_at": "2026-11-01T09:00:00.000Z"
+          }
+        ],
+        "ledger_rate": [],
+        "ledger_tag": []
+      },
+      "links": []
     }
   ]
 }

package/src/test-store/create-test-store.ts

@@ -11,6 +11,8 @@ import { ApplyError } from '../utils/apply-error.ts';
 import { parseAppliedFacts } from './parse-applied-facts.ts';
 import { runConsumers } from './run-consumers.ts';
 import type { AppliedFactIDs } from './run-consumers.ts';
+import { storeHooks } from './store-hooks.ts';
+import type { StoreHooks } from './store-hooks.ts';
 import type { TestStore } from './types.ts';
 
 interface TestStoreOptions {
@@ -48,7 +50,37 @@ export async function createTestStore(options: TestStoreOptions): Promise<TestSt
     return table;
   };
 
-  return {
+  const applyInTurn = async (records: Parameters<StoreHooks['applyEnvelopes']>[0]) => {
+    const next = buildFactIndex(index, records);
+
+    applied = await db
+      .transaction()
+      .execute((transaction) =>
+        runConsumers(transaction, deployment.consumers, { ...next, applied }),
+      );
+
+    index = next.index;
+  };
+
+  // Each apply builds on the index and applied IDs that the one before it left, so two applies
+  // that run at once take turns. A refused apply rejects for its caller only.
+  let lastApply: Promise<void> = Promise.resolve();
+
+  const applyEnvelopes: StoreHooks['applyEnvelopes'] = async (records) => {
+    const before = lastApply;
+    const turn = Promise.withResolvers<void>();
+
+    lastApply = turn.promise;
+
+    try {
+      await before;
+      await applyInTurn(records);
+    } finally {
+      turn.resolve();
+    }
+  };
+
+  const testStore: TestStore = {
     writeRows: async (_consumer, table, rows) => {
       for (const row of rows) {
         await writeRow(db, table, getTable(table), row);
@@ -61,15 +93,7 @@ export async function createTestStore(options: TestStoreOptions): Promise<TestSt
         throw new ApplyError(envelopes.failure);
       }
 
-      const next = buildFactIndex(index, envelopes.value);
-
-      applied = await db
-        .transaction()
-        .execute((transaction) =>
-          runConsumers(transaction, deployment.consumers, { ...next, applied }),
-        );
-
-      index = next.index;
+      await applyEnvelopes(envelopes.value);
     },
     query: async (query, input) => {
       const parsed = query.input.parse(input);
@@ -91,4 +115,8 @@ export async function createTestStore(options: TestStoreOptions): Promise<TestSt
     reader,
     [Symbol.asyncDispose]: () => db.destroy(),
   };
+
+  storeHooks.set(testStore, { applyEnvelopes, nextOffsets: new Map() });
+
+  return testStore;
 }

package/src/test-store/run-consumers.ts

@@ -11,6 +11,7 @@ import { parseOperations } from '../operations/parse-operations.ts';
 import { replaceScope } from '../scope-replace/replace-scope.ts';
 import type { StoreDatabase } from '../store-database/types.ts';
 import { ApplyError } from '../utils/apply-error.ts';
+import { compareCodePoints } from '../utils/compare-code-points.ts';
 
 // The fact IDs each fact-mode consumer has applied, by consumer name.
 export type AppliedFactIDs = ReadonlyMap<string, ReadonlySet<string>>;
@@ -104,7 +105,7 @@ async function runScopeConsumer(
           return parsed.ok && `${parsed.value.source}:${parsed.value.kind}` === scopeKind;
         }),
     ),
-  ].toSorted((left, right) => (left < right ? -1 : 1));
+  ].toSorted(compareCodePoints);
 
   for (const scope of scopes) {
     const scopeFacts = JSON.stringify(collectScopeFacts(index, scope, consumer.input.streams));

package/src/test-store/store-hooks.ts

@@ -0,0 +1,16 @@
+import type { FactEnvelope, StreamName } from '@pepys/kit';
+import type { TestStore } from './types.ts';
+
+export interface StoreHooks {
+  // Runs the consumers on the records as one batch in one transaction. A refusal throws an
+  // ApplyError, and the batch then writes nothing.
+  applyEnvelopes: (
+    records: readonly { stream: string; envelope: FactEnvelope<unknown> }[],
+  ) => Promise<void>;
+
+  // the offset of the next record to apply from a fact log, by stream
+  nextOffsets: Map<StreamName, number>;
+}
+
+// What a test store gives the package's own functions and keeps out of its public type.
+export const storeHooks = new WeakMap<TestStore, StoreHooks>();

package/src/utils/compare-code-points.ts

@@ -0,0 +1,19 @@
+// The host's Rust strings and SQLite's binary collation order text by code point, but `<` on
+// JavaScript strings compares UTF-16 code units, which puts text above U+FFFF before U+E000.
+export function compareCodePoints(left: string, right: string): number {
+  const length = Math.min(left.length, right.length);
+  let index = 0;
+
+  while (index < length) {
+    const leftPoint = left.codePointAt(index) ?? 0;
+    const rightPoint = right.codePointAt(index) ?? 0;
+
+    if (leftPoint !== rightPoint) {
+      return leftPoint - rightPoint;
+    }
+
+    index += leftPoint > 0xff_ff ? 2 : 1;
+  }
+
+  return left.length - right.length;
+}