@pepys/kit 0.0.0-stage → 0.0.1

package/README.md

@@ -1,3 +1,64 @@
-# Temporary Holding Version
+<div align="center">
+  <h1>@pepys/kit</h1>
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+  <p>
+    <a href="https://www.npmjs.com/package/@pepys/kit"><img src="https://img.shields.io/npm/v/@pepys/kit" alt="npm version"></a>
+    <a href="../../LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="MIT license"></a>
+  </p>
+
+  <p>
+    <a href="../../docs/README.md">Documentation</a> •
+    <a href="../../docs/design/v1.md">pepys v1</a>
+  </p>
+</div>
+
+Declare a pepys plug-in or deployment as plain data.
+
+A plug-in names its fact families, sources, consumers, tables, links and queries with the kit's
+`define*` functions. The kit checks a deployment, generates each consumer's glue and SQL schema, and
+signs and sends a fact batch. The package ships its TypeScript source and runs on Bun.
+
+## Install
+
+```sh
+bun add @pepys/kit
+bun add zod@"$(npm view @pepys/kit dependencies.zod)"
+bun add -d @types/bun typescript
+```
+
+Install the zod version that the kit pins, as the second line does: a schema from another zod
+version fails the type check of a `define*` call.
+
+Test the plug-in with [`@pepys/testing`](../testing/README.md), and build it with
+[`@pepys/cli`](../cli/README.md).
+
+## Features
+
+- `@pepys/kit`: the `define*` declarations and the deployment check.
+- `@pepys/kit/generators`: the SQL schema and glue generators.
+- `@pepys/kit/send`: the client that signs and sends a fact batch.
+
+## Configuration
+
+Your compiler checks the kit's source with your `tsconfig.json`, so set at least these options:
+
+```json
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "lib": ["ESNext"],
+    "module": "Preserve",
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "types": ["bun"]
+  }
+}
+```
+
+## Documentation
+
+The [pepys docs](../../docs/README.md) hold the design of the kit and of each part of pepys.

package/package.json

@@ -1,6 +1,36 @@
 {
   "name": "@pepys/kit",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.0.1",
+  "description": "Declare a pepys plug-in or deployment: fact families, sources, consumers, tables, links and queries",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/zgeoff/pepys.git",
+    "directory": "packages/kit"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/test-utils"
+  ],
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts",
+    "./generators": "./src/generators.ts",
+    "./send": "./src/send.ts"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@zgeoff/std": "0.4.3",
+    "kysely": "0.29.8",
+    "ts-pattern": "5.9.2",
+    "type-fest": "5.10.0",
+    "zod": "4.6.7"
+  },
+  "devDependencies": {
+    "@octokit/webhooks": "14.2.0",
+    "tiny-invariant": "1.3.3"
+  }
+}

package/src/constants.ts

@@ -0,0 +1 @@
+export const kitVersion = 1;

package/src/consumers/define-consumer.ts

@@ -0,0 +1,20 @@
+import type { LinkTypeDeclaration } from '../link-types/types.ts';
+import type { TableDeclaration } from '../types.ts';
+import type { ConsumerDeclaration, EntityViewDeclaration } from './types.ts';
+
+type ConsumerInputOf<
+  Name extends string,
+  Tables extends Readonly<Record<string, TableDeclaration>>,
+  LinkType extends LinkTypeDeclaration,
+> = Omit<ConsumerDeclaration<Name, Tables, LinkType>, 'entityViews' | 'reads'> & {
+  readonly entityViews: readonly EntityViewDeclaration<NoInfer<keyof Tables & string>>[];
+  readonly reads: readonly NoInfer<keyof Tables & string>[];
+};
+
+export function defineConsumer<
+  const Name extends string,
+  const Tables extends Readonly<Record<string, TableDeclaration>>,
+  const LinkType extends LinkTypeDeclaration = never,
+>(consumer: ConsumerInputOf<Name, Tables, LinkType>): ConsumerDeclaration<Name, Tables, LinkType> {
+  return consumer;
+}

package/src/consumers/types.ts

@@ -0,0 +1,48 @@
+import type { EntityKind, LinkTypeDeclaration } from '../link-types/types.ts';
+import type { DeclaredStreamName, TableDeclaration } from '../types.ts';
+
+// The same shape in the declaration and in the consumer manifest, so the build copies it whole.
+export type ConsumerInput =
+  | { readonly mode: 'fact'; readonly streams: readonly DeclaredStreamName[] }
+
+  // A fact on one of `streams` whose entity is of `scopeKind` marks that entity's scope dirty.
+  | {
+      readonly mode: 'scope';
+      readonly scopeKind: EntityKind;
+      readonly streams: readonly DeclaredStreamName[];
+    };
+
+export interface EntityViewDeclaration<TableName extends string = string> {
+  readonly table: TableName;
+  readonly kind: EntityKind;
+  readonly id: string;
+  readonly title: string;
+  readonly url: string | null;
+  readonly actor: string | null;
+  readonly updatedAt: string;
+}
+
+// The entry file is a scriptc library with the model contract: it exports a model type, a message
+// type, `init` and `update`, beside the consumer's own export. scriptc attests no other shape.
+export interface ConsumerDeclaration<
+  Name extends string = string,
+  Tables extends Readonly<Record<string, TableDeclaration>> = Readonly<
+    Record<string, TableDeclaration>
+  >,
+  LinkType extends LinkTypeDeclaration = LinkTypeDeclaration,
+> {
+  readonly name: Name;
+  readonly entry: URL;
+  readonly input: ConsumerInput;
+  readonly tables: Tables;
+
+  // One view per table whose rows are entities. `defineConsumer` keeps the table names to the
+  // consumer's own; a key type here would make a declaration fit no wider consumer type.
+  readonly entityViews: readonly EntityViewDeclaration[];
+
+  // own tables whose rows the module reads by key; fact mode only
+  readonly reads: readonly string[];
+
+  // the link types the consumer may write
+  readonly links: readonly LinkType[];
+}

package/src/delivery-checks/build-delivery-proof.ts

@@ -0,0 +1,12 @@
+import { createHmac } from 'node:crypto';
+import { match } from 'ts-pattern';
+import type { DeliveryCheck } from '../sources/types.ts';
+
+// The value of the check's header for one body: `sha256=<hex>` of the HMAC for a signed path, and
+// `Bearer <secret>` for a token path.
+export function buildDeliveryProof(check: DeliveryCheck, secret: string, body: Uint8Array): string {
+  return match(check.kind)
+    .with('hmac-sha256', () => `sha256=${createHmac('sha256', secret).update(body).digest('hex')}`)
+    .with('token', () => `Bearer ${secret}`)
+    .exhaustive();
+}

package/src/delivery-checks/is-delivery-authentic.ts

@@ -0,0 +1,49 @@
+import { createHash, createHmac, timingSafeEqual } from 'node:crypto';
+import { match } from 'ts-pattern';
+import type { DeliveryCheck } from '../sources/types.ts';
+
+// The check of one delivery, on the body as sent. Ingest calls it for every receive path, and the
+// sender's proof must pass it. An empty secret passes nothing, so a missing variable never opens a
+// route.
+export function isDeliveryAuthentic(
+  check: DeliveryCheck,
+  secret: string,
+  headers: Headers,
+  body: Uint8Array,
+): boolean {
+  const proof = headers.get(check.header);
+
+  if (secret === '' || proof === null) {
+    return false;
+  }
+
+  return match(check.kind)
+    .with('hmac-sha256', () => isSignatureAuthentic(secret, proof, body))
+    .with('token', () => isTokenAuthentic(secret, proof))
+    .exhaustive();
+}
+
+const signaturePattern = /^sha256=(?<hex>[0-9a-fA-F]{64})$/u;
+
+function isSignatureAuthentic(secret: string, signature: string, body: Uint8Array): boolean {
+  const hex = signaturePattern.exec(signature)?.groups?.['hex'];
+
+  if (hex === undefined) {
+    return false;
+  }
+
+  const expected = createHmac('sha256', secret).update(body).digest();
+
+  return timingSafeEqual(Buffer.from(hex, 'hex'), expected);
+}
+
+const bearerPrefix = 'Bearer ';
+
+// Both sides are hashed first, so the compare takes the same time for a token of any length.
+function isTokenAuthentic(secret: string, proof: string): boolean {
+  const token = proof.startsWith(bearerPrefix) ? proof.slice(bearerPrefix.length) : proof;
+  const tokenDigest = createHash('sha256').update(token).digest();
+  const secretDigest = createHash('sha256').update(secret).digest();
+
+  return timingSafeEqual(tokenDigest, secretDigest);
+}

package/src/deployments/check-deployment.ts

@@ -0,0 +1,444 @@
+import { kitVersion } from '../constants.ts';
+import type { ConsumerDeclaration } from '../consumers/types.ts';
+import { factKinds } from '../entity-ids/fact-kinds.ts';
+import type { EntityKind, LinkTypeDeclaration } from '../link-types/types.ts';
+import type { FactFamily, SourceDefinition } from '../sources/types.ts';
+import type { Result } from '../types.ts';
+import type {
+  CheckedDeployment,
+  Deployment,
+  DeploymentFailure,
+  DuplicateOf,
+  Plugin,
+  SettingValue,
+} from './types.ts';
+
+export function checkDeployment(
+  deployment: Deployment,
+): Result<CheckedDeployment, readonly DeploymentFailure[]> {
+  const parts = collectParts(deployment.plugins);
+  const settings = collectSettings(deployment);
+
+  const failures = [
+    ...collectKitFailures(deployment.plugins),
+    ...collectDuplicates(deployment.plugins, parts),
+    ...collectReservedNames(deployment.plugins, parts),
+    ...collectConsumerFailures(deployment.plugins, parts),
+    ...collectMissingConsumers(deployment.plugins, parts),
+    ...settings.failures,
+  ];
+
+  if (failures.length > 0) {
+    return { ok: false, failure: failures };
+  }
+
+  return {
+    ok: true,
+    value: {
+      name: deployment.name,
+      sources: parts.sources.map((owned) => owned.item),
+      consumers: parts.consumers.map((owned) => owned.item),
+      queries: deployment.plugins.flatMap((plugin) => plugin.queries),
+      entityKinds: [...new Set(deployment.plugins.flatMap((plugin) => plugin.entityKinds))],
+      linkTypes: collectUniqueLinkTypes(deployment.plugins),
+      settings: settings.values,
+    },
+  };
+}
+
+// An item of the deployment with the name of the plug-in that brings it.
+interface Owned<Item> {
+  plugin: string;
+  item: Item;
+}
+
+interface DeploymentParts {
+  sources: Owned<SourceDefinition>[];
+  families: Owned<FactFamily>[];
+  consumers: Owned<ConsumerDeclaration>[];
+}
+
+function collectParts(plugins: readonly Plugin[]): DeploymentParts {
+  const parts: DeploymentParts = { sources: [], families: [], consumers: [] };
+
+  for (const plugin of plugins) {
+    for (const source of plugin.sources) {
+      parts.sources.push({ plugin: plugin.name, item: source });
+
+      for (const family of source.families) {
+        parts.families.push({ plugin: plugin.name, item: family });
+      }
+    }
+
+    for (const consumer of plugin.consumers) {
+      parts.consumers.push({ plugin: plugin.name, item: consumer });
+    }
+  }
+
+  return parts;
+}
+
+type SettingValues = Record<string, Record<string, SettingValue>>;
+
+function collectSettings(deployment: Deployment): {
+  values: SettingValues;
+  failures: DeploymentFailure[];
+} {
+  const given = deployment.settings ?? {};
+  const values: SettingValues = {};
+  const failures: DeploymentFailure[] = [];
+
+  for (const name of Object.keys(given)) {
+    const plugin = deployment.plugins.find((candidate) => candidate.name === name);
+
+    if (plugin === undefined) {
+      failures.push(buildSettingsFailure(name, 'the deployment has no plug-in of this name'));
+    } else if (plugin.settings === undefined) {
+      failures.push(buildSettingsFailure(name, 'the plug-in declares no settings'));
+    }
+  }
+
+  for (const plugin of deployment.plugins) {
+    if (plugin.settings === undefined) {
+      continue;
+    }
+
+    const parsed = plugin.settings.schema.safeParse({
+      ...plugin.settings.defaults,
+      ...given[plugin.name],
+    });
+
+    if (!parsed.success) {
+      const issues = parsed.error.issues.map(
+        (issue) => `${issue.path.join('.')}: ${issue.message}`,
+      );
+
+      failures.push({ kind: 'invalid-settings', plugin: plugin.name, issues });
+      continue;
+    }
+
+    const entries = Object.entries(parsed.data);
+
+    const flat = entries.filter((entry): entry is [string, SettingValue] =>
+      isSettingValue(entry[1]),
+    );
+
+    if (flat.length < entries.length) {
+      failures.push(
+        buildSettingsFailure(plugin.name, 'a setting must be a string, a number or a boolean'),
+      );
+
+      continue;
+    }
+
+    values[plugin.name] = Object.fromEntries(flat);
+  }
+
+  return { values, failures };
+}
+
+function buildSettingsFailure(plugin: string, issue: string): DeploymentFailure {
+  return { kind: 'invalid-settings', plugin, issues: [issue] };
+}
+
+function isSettingValue(value: unknown): value is SettingValue {
+  return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean';
+}
+
+function collectKitFailures(plugins: readonly Plugin[]): DeploymentFailure[] {
+  return plugins
+    .filter((plugin) => plugin.kit !== kitVersion)
+    .map((plugin) => ({ kind: 'kit-version', plugin: plugin.name, kit: plugin.kit }));
+}
+
+interface Named {
+  name: string;
+  plugin: string;
+}
+
+function collectDuplicates(
+  plugins: readonly Plugin[],
+  parts: DeploymentParts,
+): DeploymentFailure[] {
+  const names: Record<DuplicateOf, Named[]> = {
+    plugin: plugins.map((plugin) => ({ name: plugin.name, plugin: plugin.name })),
+    source: parts.sources.map((owned) => ({ name: owned.item.name, plugin: owned.plugin })),
+    stream: parts.families.map((owned) => ({ name: owned.item.stream, plugin: owned.plugin })),
+    route: [],
+    'fact-type': [],
+    consumer: parts.consumers.map((owned) => ({ name: owned.item.name, plugin: owned.plugin })),
+    table: [],
+    query: [],
+    'link-type': collectLinkTypeShapes(parts.consumers),
+    'entity-view-kind': [],
+  };
+
+  for (const owned of parts.sources) {
+    const path = owned.item.live;
+
+    if (path?.kind === 'receive') {
+      for (const route of [path.route, ...(path.ignoredRoutes ?? [])]) {
+        names.route.push({ name: route, plugin: owned.plugin });
+      }
+    }
+  }
+
+  for (const owned of parts.families) {
+    for (const type of Object.keys(owned.item.types)) {
+      names['fact-type'].push({ name: type, plugin: owned.plugin });
+    }
+  }
+
+  for (const owned of parts.consumers) {
+    for (const table of Object.keys(owned.item.tables)) {
+      names.table.push({ name: table, plugin: owned.plugin });
+    }
+
+    for (const view of owned.item.entityViews) {
+      names['entity-view-kind'].push({ name: view.kind, plugin: owned.plugin });
+    }
+  }
+
+  for (const plugin of plugins) {
+    for (const query of plugin.queries) {
+      names.query.push({ name: query.name, plugin: plugin.name });
+    }
+  }
+
+  const failures: DeploymentFailure[] = [];
+
+  for (const [of, entries] of Object.entries(names)) {
+    failures.push(...collectRepeatedNames(of, entries));
+  }
+
+  return failures;
+}
+
+// One entry per distinct shape of a name, so a link type that two consumers share counts once.
+function collectLinkTypeShapes(consumers: readonly Owned<ConsumerDeclaration>[]): Named[] {
+  const shapes = new Map<string, Named>();
+
+  for (const owned of consumers) {
+    for (const linkType of owned.item.links) {
+      const shape = JSON.stringify([linkType.name, linkType.from, linkType.to, linkType.hub]);
+
+      if (!shapes.has(shape)) {
+        shapes.set(shape, { name: linkType.name, plugin: owned.plugin });
+      }
+    }
+  }
+
+  return [...shapes.values()];
+}
+
+function collectRepeatedNames(of: string, entries: readonly Named[]): DeploymentFailure[] {
+  const failures: DeploymentFailure[] = [];
+
+  for (const [name, group] of Map.groupBy(entries, (entry) => entry.name)) {
+    if (group.length > 1 && isDuplicateOf(of)) {
+      failures.push({ kind: 'duplicate', of, name, plugins: group.map((entry) => entry.plugin) });
+    }
+  }
+
+  return failures;
+}
+
+const duplicateOfValues: ReadonlySet<string> = new Set<DuplicateOf>([
+  'plugin',
+  'source',
+  'stream',
+  'route',
+  'fact-type',
+  'consumer',
+  'table',
+  'query',
+  'link-type',
+  'entity-view-kind',
+]);
+
+function isDuplicateOf(text: string): text is DuplicateOf {
+  return duplicateOfValues.has(text);
+}
+
+const coreQueryNames: ReadonlySet<string> = new Set(['get_entity', 'neighbourhood']);
+
+// the host's own tables and views in every consumer file
+const hostTableNames: ReadonlySet<string> = new Set([
+  'link',
+  'link_effective',
+  'entity',
+  'offsets',
+  'change_log',
+  'applied_facts',
+  'scope_row',
+]);
+
+function collectReservedNames(
+  plugins: readonly Plugin[],
+  parts: DeploymentParts,
+): DeploymentFailure[] {
+  const failures: DeploymentFailure[] = [];
+
+  const reservedSources: ReadonlySet<string> = new Set(factKinds);
+
+  for (const owned of parts.sources) {
+    if (reservedSources.has(owned.item.name)) {
+      failures.push({ kind: 'reserved-source-name', source: owned.item.name });
+    }
+  }
+
+  for (const query of plugins.flatMap((plugin) => plugin.queries)) {
+    if (coreQueryNames.has(query.name)) {
+      failures.push({ kind: 'reserved-query-name', query: query.name });
+    }
+  }
+
+  for (const owned of parts.consumers) {
+    for (const table of Object.keys(owned.item.tables)) {
+      if (hostTableNames.has(table)) {
+        failures.push({ kind: 'reserved-table-name', table, consumer: owned.item.name });
+      }
+    }
+  }
+
+  return failures;
+}
+
+function collectConsumerFailures(
+  plugins: readonly Plugin[],
+  parts: DeploymentParts,
+): DeploymentFailure[] {
+  const streams: ReadonlySet<string> = new Set(parts.families.map((owned) => owned.item.stream));
+  const kinds: ReadonlySet<string> = new Set(plugins.flatMap((plugin) => plugin.entityKinds));
+
+  const failures: DeploymentFailure[] = [];
+
+  for (const owned of parts.consumers) {
+    const consumer = owned.item;
+
+    if (consumer.input.mode === 'scope' && consumer.reads.length > 0) {
+      failures.push({ kind: 'scope-reads', consumer: consumer.name });
+    }
+
+    for (const stream of consumer.input.streams.filter((name) => !streams.has(name))) {
+      failures.push({ kind: 'missing-stream', stream, consumer: consumer.name });
+    }
+
+    for (const use of collectKindUses(consumer)) {
+      if (!kinds.has(use.entityKind)) {
+        failures.push({ kind: 'unknown-entity-kind', ...use, plugin: owned.plugin });
+      }
+    }
+
+    failures.push(...collectTableFailures(consumer));
+  }
+
+  return failures;
+}
+
+interface KindUse {
+  entityKind: EntityKind;
+  usedBy: string;
+}
+
+function collectKindUses(consumer: ConsumerDeclaration): KindUse[] {
+  const uses = new Map<string, KindUse>();
+
+  const addUse = (entityKind: EntityKind, usedBy: string): void => {
+    uses.set(`${entityKind}\n${usedBy}`, { entityKind, usedBy });
+  };
+
+  if (consumer.input.mode === 'scope') {
+    addUse(consumer.input.scopeKind, consumer.name);
+  }
+
+  for (const linkType of consumer.links) {
+    for (const entityKind of [...linkType.from, ...linkType.to]) {
+      addUse(entityKind, linkType.name);
+    }
+  }
+
+  return [...uses.values()];
+}
+
+function collectTableFailures(consumer: ConsumerDeclaration): DeploymentFailure[] {
+  const failures: DeploymentFailure[] = [];
+
+  const addUnknownColumns = (
+    table: string,
+    columns: readonly string[],
+    usedBy: 'primary-key' | 'index' | 'entity-view',
+  ): void => {
+    const declared = consumer.tables[table]?.columns ?? {};
+
+    for (const column of columns.filter((name) => !Object.hasOwn(declared, name))) {
+      failures.push({ kind: 'unknown-column', consumer: consumer.name, table, column, usedBy });
+    }
+  };
+
+  for (const [table, declaration] of Object.entries(consumer.tables)) {
+    addUnknownColumns(table, declaration.primaryKey, 'primary-key');
+
+    addUnknownColumns(
+      table,
+      declaration.indexes.flatMap((index) => index.columns),
+      'index',
+    );
+  }
+
+  for (const table of consumer.reads.filter((name) => !Object.hasOwn(consumer.tables, name))) {
+    failures.push({ kind: 'unknown-table', consumer: consumer.name, table, usedBy: 'reads' });
+  }
+
+  for (const view of consumer.entityViews) {
+    if (Object.hasOwn(consumer.tables, view.table)) {
+      const columns = [view.id, view.title, view.url, view.actor, view.updatedAt];
+
+      addUnknownColumns(
+        view.table,
+        columns.filter((column) => column !== null),
+        'entity-view',
+      );
+    } else {
+      failures.push({
+        kind: 'unknown-table',
+        consumer: consumer.name,
+        table: view.table,
+        usedBy: 'entity-view',
+      });
+    }
+  }
+
+  return failures;
+}
+
+function collectMissingConsumers(
+  plugins: readonly Plugin[],
+  parts: DeploymentParts,
+): DeploymentFailure[] {
+  const names: ReadonlySet<string> = new Set(parts.consumers.map((owned) => owned.item.name));
+
+  const failures: DeploymentFailure[] = [];
+
+  for (const plugin of plugins) {
+    for (const query of plugin.queries) {
+      for (const consumer of query.reads.filter((read) => !names.has(read.name))) {
+        failures.push({ kind: 'missing-consumer', consumer: consumer.name, query: query.name });
+      }
+    }
+  }
+
+  return failures;
+}
+
+function collectUniqueLinkTypes(plugins: readonly Plugin[]): LinkTypeDeclaration[] {
+  const byName = new Map<string, LinkTypeDeclaration>();
+
+  for (const linkType of plugins.flatMap((plugin) => plugin.linkTypes)) {
+    if (!byName.has(linkType.name)) {
+      byName.set(linkType.name, linkType);
+    }
+  }
+
+  return [...byName.values()];
+}

package/src/deployments/define-deployment.ts

@@ -0,0 +1,7 @@
+import type { Deployment, Plugin } from './types.ts';
+
+export function defineDeployment<const Plugins extends readonly Plugin[]>(
+  deployment: Deployment<Plugins>,
+): Deployment<Plugins> {
+  return deployment;
+}