@pepys/schema 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,33 @@
-# Temporary Holding Version
+<div align="center">
+  <h1>@pepys/schema</h1>
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+  <p>
+    <a href="https://www.npmjs.com/package/@pepys/schema"><img src="https://img.shields.io/npm/v/@pepys/schema" alt="npm version"></a>
+    <a href="../../LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="MIT license"></a>
+  </p>
+
+  <p>
+    <a href="../../docs/README.md">Documentation</a>
+  </p>
+</div>
+
+The pepys consumer manifest, for Bun: the types and the strict schema of `consumers.json`.
+
+`pepys build` writes `consumers.json`, and the pepys services read it. A plug-in needs only
+[`@pepys/kit`](../kit/README.md).
+
+## Install
+
+```sh
+bun add @pepys/schema
+```
+
+## Features
+
+- `consumerBuildManifestSchema`: the strict schema of `consumers.json`, with its types.
+- The types of a consumer's execution and of the link registry.
+- The Kysely and Rust type generators for the store schema.
+
+## Documentation
+
+The [pepys docs](../../docs/README.md) hold the design of the manifest and the store.

package/package.json

@@ -1,6 +1,27 @@
 {
   "name": "@pepys/schema",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "The pepys consumer manifest: the types and the strict schema of consumers.json, with its shared case file",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/zgeoff/pepys.git",
+    "directory": "packages/schema"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/test-utils"
+  ],
+  "type": "module",
+  "exports": "./src/index.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@pepys/kit": "0.1.0",
+    "@zgeoff/std": "0.4.3",
+    "ts-pattern": "5.9.2",
+    "zod": "4.6.7"
+  }
+}

package/src/consumer-execution/types.ts

@@ -0,0 +1,62 @@
+import type {
+  BlobHash,
+  EntityID,
+  FactEnvelope,
+  FactID,
+  ScopeID,
+  StreamOffset,
+  VersionHash,
+} from '@pepys/kit';
+import type { LinkEnds, LinkRegistry } from '../link-registry/link-registry.ts';
+
+export type ConsumerCallInput = FactInput | ScopeInput;
+
+export interface FactInput {
+  mode: 'fact';
+  offset: StreamOffset;
+  fact: FactEnvelope<unknown>;
+}
+
+export interface ScopeInput {
+  mode: 'scope';
+  scope: ScopeID;
+}
+
+export type ReadRequest =
+  | { row: { table: string; key: string } }
+  | { scan: { table: string; index: string; prefix: readonly string[] } }
+  | { facts: EntityID }
+  | { blob: BlobHash };
+
+export type ReadLimit = { mode: 'fact'; before: StreamOffset } | { mode: 'scope'; scope: ScopeID };
+
+export interface ConsumerReads<Row> {
+  getRow: (table: string, key: string) => Row | undefined;
+  scanTable: (table: string, index: string, prefix: readonly string[]) => readonly Row[];
+  readEntityFacts: (entity: EntityID) => readonly FactEnvelope<unknown>[];
+  readBlob: (hash: BlobHash) => Uint8Array | undefined;
+}
+
+export type ConsumerLink<Registry extends LinkRegistry> = LinkEnds<Registry> & {
+  confidence: number;
+  evidence: string;
+};
+
+export type FactOperation<Row, Registry extends LinkRegistry> =
+  | { upsert: { table: string; row: Row } }
+  | { delete: { table: string; key: string } }
+  | { link: ConsumerLink<Registry> }
+  | { unlink: LinkEnds<Registry> };
+
+export type ScopeOperation<Row, Registry extends LinkRegistry> =
+  | { upsert: { table: string; row: Row } }
+  | { link: ConsumerLink<Registry> };
+
+export type ConsumerOutcome<Operation> = { applied: readonly Operation[] } | { rejected: string };
+
+export type Halt =
+  | { kind: 'rejected-fact'; offset: StreamOffset; fact: FactID; reason: string }
+  | { kind: 'rejected-scope'; scope: ScopeID; reason: string }
+  | { kind: 'refused-operation'; at: StreamOffset | ScopeID; reason: string };
+
+export type Pins = Readonly<Record<string, VersionHash>>;

package/src/consumer-manifest/consumer-build-manifest-schema.ts

@@ -0,0 +1,198 @@
+import type { VersionHash } from '@pepys/kit';
+import { match } from 'ts-pattern';
+import { z } from 'zod';
+import type {
+  ConsumerBuildManifest,
+  ConsumerManifest,
+  ListedConsumer,
+  ManifestTable,
+} from './types.ts';
+
+// The host keeps these tables in every consumer's file.
+const reservedTableNames = new Set([
+  'link',
+  'entity',
+  'offsets',
+  'change_log',
+  'applied_facts',
+  'scope_row',
+  'scope_fact',
+]);
+
+const hashSchema = z.custom<VersionHash>(
+  (value) => typeof value === 'string' && /^[0-9a-f]{64}$/u.test(value),
+  'expected a SHA-256 as 64 lowercase hex digits',
+);
+
+const nameSchema = z.string().min(1);
+const symbolSchema = z.string().min(1);
+const streamSchema = z.templateLiteral(['facts.', z.string(), '.', z.string()]);
+const entityKindSchema = z.templateLiteral([z.string(), ':', z.string()]);
+
+const inputSchema = z.discriminatedUnion('mode', [
+  z.strictObject({ mode: z.literal('fact'), streams: z.array(streamSchema).min(1) }),
+  z.strictObject({
+    mode: z.literal('scope'),
+    scopeKind: entityKindSchema,
+    streams: z.array(streamSchema).min(1),
+  }),
+]);
+
+const readSchema = z.discriminatedUnion('kind', [
+  z.strictObject({
+    kind: z.literal('row'),
+    table: nameSchema,
+    length: symbolSchema,
+    into: symbolSchema,
+  }),
+  z.strictObject({ kind: z.literal('scope_facts'), length: symbolSchema, into: symbolSchema }),
+]);
+
+const columnSchema = z.strictObject({
+  kind: z.enum(['text', 'integer', 'real', 'boolean', 'json', 'entityID', 'timestamp']),
+  nullable: z.boolean(),
+});
+
+const tableSchema = z.strictObject({
+  columns: z.record(nameSchema, columnSchema),
+  primaryKey: z.array(nameSchema).min(1),
+  indexes: z.array(z.array(nameSchema).min(1)),
+});
+
+const linkTypeSchema = z.strictObject({
+  name: nameSchema,
+  from: z.array(entityKindSchema).min(1),
+  to: z.array(entityKindSchema).min(1),
+  hub: z.boolean(),
+});
+
+const manifestSchema = z
+  .strictObject({
+    name: nameSchema,
+    input: inputSchema,
+    exports: z.strictObject({ init: symbolSchema, apply: symbolSchema }),
+    imports: z.array(readSchema),
+    tables: z.record(nameSchema, tableSchema),
+    links: z.array(linkTypeSchema),
+    schema: z.string().min(1),
+  })
+  .superRefine((manifest, ctx) => {
+    for (const message of collectManifestProblems(manifest)) {
+      ctx.addIssue({ code: 'custom', message });
+    }
+  });
+
+const listedConsumerSchema = z
+  .strictObject({
+    name: nameSchema,
+    module: hashSchema,
+    path: z
+      .string()
+      .min(1)
+      .refine(
+        (path) => !path.startsWith('/') && !path.split('/').includes('..'),
+        'expected a path inside the folder of consumers.json',
+      ),
+    pins: z.record(nameSchema, hashSchema),
+    version: hashSchema,
+    manifest: manifestSchema,
+  })
+  .refine(
+    (consumer) => consumer.name === consumer.manifest.name,
+    'expected the name of the entry to equal the name of its manifest',
+  );
+
+// Strict at every level: a field that this schema does not know fails the parse, so a file from a
+// newer build fails at once.
+export const consumerBuildManifestSchema = z
+  .strictObject({
+    kit: z.literal(1),
+    deployment: nameSchema.nullable(),
+    consumers: z.array(listedConsumerSchema),
+  })
+  .superRefine((file, ctx) => {
+    for (const message of collectFileProblems(file.consumers)) {
+      ctx.addIssue({ code: 'custom', message });
+    }
+  }) satisfies z.ZodType<ConsumerBuildManifest>;
+
+function collectManifestProblems(manifest: ConsumerManifest): string[] {
+  const symbols = [
+    manifest.exports.init,
+    manifest.exports.apply,
+    ...manifest.imports.flatMap((read) => [read.length, read.into]),
+  ];
+
+  return [
+    ...collectReadProblems(manifest),
+    ...collectRepeats(manifest.links.map((link) => link.name)).map(
+      (name) => `the link type ${name} is declared twice`,
+    ),
+    ...collectRepeats(symbols).map((symbol) => `the symbol ${symbol} is used twice`),
+    ...Object.entries(manifest.tables).flatMap(([name, table]) =>
+      collectTableProblems(name, table),
+    ),
+  ];
+}
+
+function collectReadProblems(manifest: ConsumerManifest): string[] {
+  return match(manifest.input.mode)
+    .with('scope', () => {
+      const [read, ...others] = manifest.imports;
+
+      return read?.kind === 'scope_facts' && others.length === 0
+        ? []
+        : ['a scope consumer has exactly one read, of kind scope_facts'];
+    })
+    .with('fact', () =>
+      manifest.imports.flatMap((read) =>
+        match(read)
+          .with({ kind: 'scope_facts' }, () => ['a fact consumer reads rows only'])
+          .with({ kind: 'row' }, (row) =>
+            Object.hasOwn(manifest.tables, row.table)
+              ? []
+              : [`the row read names ${row.table}, which is no table of the consumer`],
+          )
+          .exhaustive(),
+      ),
+    )
+    .exhaustive();
+}
+
+function collectRepeats(values: readonly string[]): string[] {
+  const seen = new Set<string>();
+  const repeats = new Set<string>();
+
+  for (const value of values) {
+    if (seen.has(value)) {
+      repeats.add(value);
+    }
+
+    seen.add(value);
+  }
+
+  return [...repeats];
+}
+
+function collectTableProblems(name: string, table: ManifestTable): string[] {
+  const columns = [...table.primaryKey, ...table.indexes.flat()];
+
+  const missing = columns
+    .filter((column) => !Object.hasOwn(table.columns, column))
+    .map((column) => `the table ${name} has no column ${column}`);
+
+  return reservedTableNames.has(name)
+    ? [`the table name ${name} is reserved for the host`, ...missing]
+    : missing;
+}
+
+function collectFileProblems(consumers: readonly ListedConsumer[]): string[] {
+  const tables = consumers.flatMap((consumer) => Object.keys(consumer.manifest.tables));
+
+  return [
+    ...collectRepeats(consumers.map((consumer) => consumer.name)).map(
+      (name) => `the consumer ${name} is listed twice`,
+    ),
+    ...collectRepeats(tables).map((table) => `the table ${table} is in two consumers`),
+  ];
+}