@pepys/cli 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,36 @@
-# Temporary Holding Version
+<div align="center">
+  <h1>@pepys/cli</h1>
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+  <p>
+    <a href="https://www.npmjs.com/package/@pepys/cli"><img src="https://img.shields.io/npm/v/@pepys/cli" alt="npm version"></a>
+    <a href="../../LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="MIT license"></a>
+  </p>
+
+  <p>
+    <a href="../../docs/README.md">Documentation</a>
+  </p>
+</div>
+
+The `pepys` command line, for Bun.
+
+`pepys build` checks a deployment or a plug-in, and writes the glue of each consumer: its boundary
+types and its SQL schema. With `--out`, it also compiles each consumer with scriptc and writes
+`consumers.json`.
+
+## Install
+
+```sh
+bun add -d @pepys/cli
+bunx pepys build src/plugin.ts
+```
+
+## Features
+
+- `pepys build <file>`: check the deployment or plug-in that the file exports, and write its glue.
+- `pepys build <file> --out <directory>`: also compile each consumer, which needs zig and
+  wasm-tools.
+- `pepys build <file> --check`: exit 1 when a glue file is stale.
+
+## Documentation
+
+The [pepys docs](../../docs/README.md) hold the design of `pepys build`.

package/package.json

@@ -1,6 +1,36 @@
 {
   "name": "@pepys/cli",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "The pepys command line: pepys build checks a deployment or a plug-in, writes the glue of each consumer, and compiles each consumer with scriptc",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/zgeoff/pepys.git",
+    "directory": "packages/cli"
+  },
+  "bin": {
+    "pepys": "./src/bin.ts"
+  },
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/test-utils"
+  ],
+  "type": "module",
+  "exports": "./src/index.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@pepys/kit": "0.1.0",
+    "@pepys/schema": "0.1.0",
+    "@scriptc/runtime-wasm32-wasi": "0.4.5",
+    "@zgeoff/std": "0.4.3",
+    "scriptc": "0.4.5",
+    "ts-pattern": "5.9.2"
+  },
+  "devDependencies": {
+    "tiny-invariant": "1.3.3",
+    "zod": "4.6.7"
+  }
+}

package/src/bin.ts

@@ -0,0 +1,4 @@
+#!/usr/bin/env bun
+import { runCLI } from './run-cli.ts';
+
+process.exitCode = await runCLI(process.argv.slice(2));

package/src/build/build-consumer-build-manifest.ts

@@ -0,0 +1,50 @@
+import type { VersionHash } from '@pepys/kit';
+import type { ConsumerBuildManifest } from '@pepys/schema';
+import invariant from 'tiny-invariant';
+import { buildConsumerManifest } from './build-consumer-manifest.ts';
+import { buildVersionHash } from './build-version-hash.ts';
+import type { CheckedTarget } from './check-build-target.ts';
+
+export interface BuiltModule {
+  name: string;
+
+  // the SHA-256 of the module's bytes
+  module: VersionHash;
+
+  // the module file, relative to the folder of consumers.json
+  path: string;
+}
+
+export interface BuildManifestParameters {
+  target: CheckedTarget;
+  modules: readonly BuiltModule[];
+}
+
+// `consumers.json`: one whole entry per consumer of the target, in the target's order.
+export function buildConsumerBuildManifest(
+  parameters: BuildManifestParameters,
+): ConsumerBuildManifest {
+  const modules = new Map(parameters.modules.map((built) => [built.name, built]));
+
+  return {
+    kit: 1,
+    deployment: parameters.target.deployment,
+    consumers: parameters.target.checked.consumers.map((consumer) => {
+      const built = modules.get(consumer.name);
+
+      invariant(built !== undefined, `the build compiled no module for ${consumer.name}`);
+
+      const manifest = buildConsumerManifest(consumer);
+      const pins = {};
+
+      return {
+        name: consumer.name,
+        module: built.module,
+        path: built.path,
+        pins,
+        version: buildVersionHash({ module: built.module, pins, manifest }),
+        manifest,
+      };
+    }),
+  };
+}

package/src/build/build-consumer-glue.ts

@@ -0,0 +1,50 @@
+import { fileURLToPath } from 'node:url';
+import type { ConsumerDeclaration, FactFamily, SettingValue } from '@pepys/kit';
+import { generateBoundaryGlue, generateSqliteSchema } from '@pepys/kit/generators';
+
+export interface GlueFile {
+  // an absolute path
+  path: string;
+  content: string;
+}
+
+export interface ConsumerGlueParameters {
+  consumer: ConsumerDeclaration;
+
+  // the families of the deployment
+  families: readonly FactFamily[];
+
+  // the checked settings of the consumer's plug-in, or undefined when it declares none
+  settings: Readonly<Record<string, SettingValue>> | undefined;
+}
+
+// The glue of a consumer sits in the `generated` folder beside the folder of its entry file:
+// `<consumer>-boundary.ts` and `<consumer>.sql`.
+export function buildConsumerGlue(parameters: ConsumerGlueParameters): GlueFile[] {
+  const consumer = parameters.consumer;
+  const stem = consumer.name.replaceAll('_', '-');
+
+  const folder = new URL('../generated/', consumer.entry);
+
+  return [
+    {
+      path: fileURLToPath(new URL(`${stem}-boundary.ts`, folder)),
+      content: withHeader(
+        '//',
+        generateBoundaryGlue({
+          consumer,
+          families: parameters.families,
+          settings: parameters.settings,
+        }),
+      ),
+    },
+    {
+      path: fileURLToPath(new URL(`${stem}.sql`, folder)),
+      content: withHeader('--', generateSqliteSchema(consumer)),
+    },
+  ];
+}
+
+function withHeader(comment: '//' | '--', body: string): string {
+  return `${comment} Generated by \`pepys build\`. Do not edit.\n\n${body}`;
+}

package/src/build/build-consumer-manifest.ts

@@ -0,0 +1,52 @@
+import type { ConsumerDeclaration, ConsumerInput, TableDeclaration } from '@pepys/kit';
+import { generateSqliteSchema } from '@pepys/kit/generators';
+import type { ConsumerManifest, ManifestTable } from '@pepys/schema';
+import { match } from 'ts-pattern';
+import { buildModuleSymbols } from './build-module-symbols.ts';
+
+// What the host knows of one consumer. Each field is copied by name, so a field that a later kit
+// adds to a declaration never reaches the file unseen.
+export function buildConsumerManifest(consumer: ConsumerDeclaration): ConsumerManifest {
+  const symbols = buildModuleSymbols(consumer);
+
+  return {
+    name: consumer.name,
+    input: toManifestInput(consumer.input),
+    exports: symbols.exports,
+    imports: symbols.imports,
+    tables: Object.fromEntries(
+      Object.entries(consumer.tables).map(([name, table]) => [name, toManifestTable(table)]),
+    ),
+    links: consumer.links.map((link) => ({
+      name: link.name,
+      from: [...link.from],
+      to: [...link.to],
+      hub: link.hub,
+    })),
+    schema: generateSqliteSchema(consumer),
+  };
+}
+
+function toManifestInput(input: ConsumerInput): ConsumerInput {
+  return match(input)
+    .with({ mode: 'fact' }, (fact) => ({ mode: fact.mode, streams: [...fact.streams] }))
+    .with({ mode: 'scope' }, (scope) => ({
+      mode: scope.mode,
+      scopeKind: scope.scopeKind,
+      streams: [...scope.streams],
+    }))
+    .exhaustive();
+}
+
+function toManifestTable(table: TableDeclaration): ManifestTable {
+  return {
+    columns: Object.fromEntries(
+      Object.entries(table.columns).map(([name, column]) => [
+        name,
+        { kind: column.kind, nullable: column.nullable === true },
+      ]),
+    ),
+    primaryKey: [...table.primaryKey],
+    indexes: table.indexes.map((index) => [...index.columns]),
+  };
+}

package/src/build/build-module-symbols.ts

@@ -0,0 +1,41 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+import { toPascalCase } from '@pepys/kit/generators';
+import type { ConsumerManifest, ManifestRead } from '@pepys/schema';
+import { match } from 'ts-pattern';
+
+export interface ModuleSymbols {
+  // the function that the entry file exports for the host to call
+  entryExport: 'applyFact' | 'applyScope';
+  exports: ConsumerManifest['exports'];
+  imports: ManifestRead[];
+}
+
+// The one source of the module's symbols: the scriptc profile compiles them, and the manifest tells
+// the host to call and serve them. The read symbols are the names the generated glue declares.
+export function buildModuleSymbols(consumer: ConsumerDeclaration): ModuleSymbols {
+  const prefix = `${consumer.name}_`;
+
+  return match(consumer.input.mode)
+    .with('fact', () => ({
+      entryExport: 'applyFact' as const,
+      exports: { init: `${prefix}init`, apply: `${prefix}apply_fact` },
+      imports: consumer.reads.map((table) => ({
+        kind: 'row' as const,
+        table,
+        length: `read${toPascalCase(table)}Length`,
+        into: `read${toPascalCase(table)}Into`,
+      })),
+    }))
+    .with('scope', () => ({
+      entryExport: 'applyScope' as const,
+      exports: { init: `${prefix}init`, apply: `${prefix}apply_scope` },
+      imports: [
+        {
+          kind: 'scope_facts' as const,
+          length: 'readScopeFactsLength',
+          into: 'readScopeFactsInto',
+        },
+      ],
+    }))
+    .exhaustive();
+}

package/src/build/build-scriptc-profile.ts

@@ -0,0 +1,62 @@
+import type { ConsumerDeclaration } from '@pepys/kit';
+import { toPascalCase } from '@pepys/kit/generators';
+import { buildModuleSymbols } from './build-module-symbols.ts';
+import { consumerFences } from './consumer-fences.ts';
+
+interface ScriptcProfileParameters {
+  consumer: ConsumerDeclaration;
+
+  // scriptc reads the entry relative to the folder of the profile file
+  entry: string;
+}
+
+// The scriptc library profile of one consumer: the entry, the symbols of its exports and of the
+// host functions it imports, and the determinism fences.
+export function buildScriptcProfile(parameters: ScriptcProfileParameters): object {
+  const name = parameters.consumer.name;
+  const prefix = `${name}_`;
+  const pascalName = toPascalCase(name);
+  const symbols = buildModuleSymbols(parameters.consumer);
+
+  const callbacks = symbols.imports.flatMap((read) => [
+    { name: read.length, params: ['string'], returns: 'f64' },
+    { name: read.into, params: ['string', 'bytes'], returns: 'void' },
+  ]);
+
+  return {
+    profile_format: 1,
+    name,
+    entry: parameters.entry,
+    emission: 'llvm',
+    abi: {
+      prefix,
+      init_symbol: symbols.exports.init,
+      sink_register_symbol: `${prefix}sink`,
+
+      // scriptc refuses a register symbol with no callbacks to register (SC4001)
+      ...(callbacks.length === 0 ? {} : { callback_register_symbol: `${prefix}callback` }),
+    },
+    callbacks,
+    exports: [
+      {
+        export: symbols.entryExport,
+        symbol: symbols.exports.apply,
+        params: ['string'],
+        returns: 'string',
+      },
+    ],
+    sidecar: {
+      wire_version: 1,
+      abi_version: 1,
+      snapshot_format: 1,
+      build_id_symbol: `${prefix}build_id`,
+      abi_version_symbol: `${prefix}abi_version`,
+      model: `${pascalName}Model`,
+
+      // scriptc names the sidecar's message type field msg
+      // oxlint-disable-next-line zgeoff/no-abbreviated-names
+      msg: `${pascalName}Message`,
+    },
+    determinism: { fences: consumerFences },
+  };
+}

package/src/build/build-version-hash.ts

@@ -0,0 +1,29 @@
+import type { VersionHash } from '@pepys/kit';
+import type { ConsumerVersion } from '@pepys/schema';
+import { toTagged } from '@zgeoff/std';
+
+// The SHA-256 of the module's hash, the pins and the manifest, as JSON with sorted keys. A changed
+// index or link type with the same module bytes is therefore a new version.
+export function buildVersionHash(
+  parts: Pick<ConsumerVersion, 'module' | 'pins' | 'manifest'>,
+): VersionHash {
+  const text = JSON.stringify(toSortedValue(parts));
+
+  return toTagged<VersionHash>(new Bun.CryptoHasher('sha256').update(text).digest('hex'));
+}
+
+function toSortedValue(value: unknown): unknown {
+  if (Array.isArray(value)) {
+    return value.map((item: unknown) => toSortedValue(item));
+  }
+
+  if (typeof value === 'object' && value !== null) {
+    return Object.fromEntries(
+      Object.entries(value)
+        .toSorted(([first], [second]) => (first < second ? -1 : 1))
+        .map(([key, item]: [string, unknown]) => [key, toSortedValue(item)]),
+    );
+  }
+
+  return value;
+}

package/src/build/check-build-target.ts

@@ -0,0 +1,76 @@
+import { checkDeployment } from '@pepys/kit';
+import type { CheckedDeployment, DeploymentFailure, Plugin, Result } from '@pepys/kit';
+import type { BuildTarget } from './pick-build-target.ts';
+
+export interface CheckedTarget {
+  // the name of the deployment, or null for a plug-in built alone
+  deployment: string | null;
+  checked: CheckedDeployment;
+  plugins: readonly Plugin[];
+}
+
+// These checks name a part that another plug-in can bring, so a plug-in built alone cannot fail
+// them.
+const needsOtherPlugins: ReadonlySet<DeploymentFailure['kind']> = new Set([
+  'unknown-entity-kind',
+  'missing-consumer',
+  'missing-stream',
+]);
+
+export function checkBuildTarget(
+  target: BuildTarget,
+): Result<CheckedTarget, readonly DeploymentFailure[]> {
+  if (target.kind === 'deployment') {
+    const checked = checkDeployment(target.deployment);
+
+    return checked.ok
+      ? {
+          ok: true,
+          value: {
+            deployment: target.deployment.name,
+            checked: checked.value,
+            plugins: target.deployment.plugins,
+          },
+        }
+      : checked;
+  }
+
+  const plugin = target.plugin;
+  const checked = checkDeployment({ name: plugin.name, plugins: [plugin] });
+
+  if (checked.ok) {
+    return { ok: true, value: { deployment: null, checked: checked.value, plugins: [plugin] } };
+  }
+
+  const failures = checked.failure.filter((failure) => !needsOtherPlugins.has(failure.kind));
+
+  if (failures.length > 0) {
+    return { ok: false, failure: failures };
+  }
+
+  // The plug-in is sound alone. Checked with no parts, it still gives its settings.
+  const settings = checkDeployment({
+    name: plugin.name,
+    plugins: [
+      { ...plugin, sources: [], consumers: [], queries: [], linkTypes: [], entityKinds: [] },
+    ],
+  });
+
+  return settings.ok
+    ? {
+        ok: true,
+        value: {
+          deployment: null,
+          checked: {
+            ...settings.value,
+            sources: plugin.sources,
+            consumers: plugin.consumers,
+            queries: plugin.queries,
+            linkTypes: plugin.linkTypes,
+            entityKinds: plugin.entityKinds,
+          },
+          plugins: [plugin],
+        },
+      }
+    : settings;
+}

package/src/build/collect-forbidden-imports.ts

@@ -0,0 +1,7 @@
+const forbiddenImport = /\(import "[^"]+" "(?:random_get|poll_oneoff|sock_[a-z_]+|path_open)"/gu;
+
+// A consumer reads the clock the host binds to a constant and nothing else outside its own memory:
+// no random source, no polling, no socket and no file.
+export function collectForbiddenImports(text: string): string[] {
+  return [...text.matchAll(forbiddenImport)].map((match) => match[0].replace('(import ', ''));
+}

package/src/build/collect-glue-files.ts

@@ -0,0 +1,36 @@
+import { fileURLToPath } from 'node:url';
+import { buildConsumerGlue } from './build-consumer-glue.ts';
+import type { GlueFile } from './build-consumer-glue.ts';
+import type { CheckedTarget } from './check-build-target.ts';
+
+export interface GlueParameters {
+  target: CheckedTarget;
+
+  // The folder of the package that the command runs in. The glue of a consumer whose entry file is
+  // outside it, or under a node_modules folder, is not written here.
+  packageRoot: string;
+}
+
+// The glue that the package commits: that of each consumer whose entry file is the package's own.
+export function collectGlueFiles(parameters: GlueParameters): GlueFile[] {
+  const families = parameters.target.checked.sources.flatMap((source) => source.families);
+  const root = `${parameters.packageRoot.replace(/\/+$/u, '')}/`;
+
+  return parameters.target.plugins.flatMap((plugin) =>
+    plugin.consumers
+      .filter((consumer) => isOwnConsumer(consumer.entry, root))
+      .flatMap((consumer) =>
+        buildConsumerGlue({
+          consumer,
+          families,
+          settings: parameters.target.checked.settings[plugin.name],
+        }),
+      ),
+  );
+}
+
+function isOwnConsumer(entry: URL, root: string): boolean {
+  const path = fileURLToPath(entry);
+
+  return path.startsWith(root) && !path.includes('/node_modules/');
+}

package/src/build/compile-consumer.ts

@@ -0,0 +1,150 @@
+import { cp, mkdir, rm } from 'node:fs/promises';
+import { dirname, join, relative, sep } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import type {
+  ConsumerDeclaration,
+  FactFamily,
+  Result,
+  SettingValue,
+  VersionHash,
+} from '@pepys/kit';
+import { toTagged } from '@zgeoff/std';
+import type { BuiltModule } from './build-consumer-build-manifest.ts';
+import { buildConsumerGlue } from './build-consumer-glue.ts';
+import { buildScriptcProfile } from './build-scriptc-profile.ts';
+import { collectForbiddenImports } from './collect-forbidden-imports.ts';
+import { readPackageRoot } from './read-package-root.ts';
+
+export interface BuildTools {
+  scriptc: string;
+  zig: string;
+  wasmTools: string;
+}
+
+export interface CompileParameters {
+  consumer: ConsumerDeclaration;
+  families: readonly FactFamily[];
+  settings: Readonly<Record<string, SettingValue>> | undefined;
+  out: string;
+  tools: BuildTools;
+}
+
+// Compiles a copy of the package's `src` at `<out>/.work/<consumer>`, with this build's glue in it.
+// scriptc builds the entry's absolute path into the module, so a fixed folder gives the same module
+// hash on every build of the same source.
+export async function compileConsumer(
+  parameters: CompileParameters,
+): Promise<Result<BuiltModule, string>> {
+  const consumer = parameters.consumer;
+  const entryPath = fileURLToPath(consumer.entry);
+
+  const packageRoot = await readPackageRoot(dirname(entryPath));
+
+  if (packageRoot === undefined) {
+    return { ok: false, failure: `the entry ${entryPath} is in no package` };
+  }
+
+  const entryInSource = relative(join(packageRoot, 'src'), entryPath);
+
+  if (entryInSource.startsWith(`..${sep}`)) {
+    return {
+      ok: false,
+      failure: `the entry ${entryPath} is outside the src folder of its package`,
+    };
+  }
+
+  const work = join(parameters.out, '.work', consumer.name);
+
+  await rm(work, { recursive: true, force: true });
+  await cp(join(packageRoot, 'src'), join(work, 'src'), { recursive: true });
+
+  const glue = buildConsumerGlue({
+    consumer: { ...consumer, entry: pathToFileURL(join(work, 'src', entryInSource)) },
+    families: parameters.families,
+    settings: parameters.settings,
+  });
+
+  for (const file of glue) {
+    await mkdir(dirname(file.path), { recursive: true });
+
+    await Bun.write(file.path, file.content);
+  }
+
+  const profile = buildScriptcProfile({ consumer, entry: `src/${entryInSource}` });
+
+  await Bun.write(join(work, 'profile.json'), JSON.stringify(profile));
+
+  const modulePath = join(work, 'module.wasm');
+
+  const compiled = await runScriptc({ work, modulePath, tools: parameters.tools });
+
+  if (!compiled.ok) {
+    return { ok: false, failure: `scriptc refused ${consumer.name}:\n${compiled.failure}` };
+  }
+
+  const contract: unknown = await Bun.file(`${modulePath}.contract.json`).json();
+
+  if (!isAttestedDeterministic(contract)) {
+    return { ok: false, failure: `scriptc does not attest ${consumer.name} deterministic` };
+  }
+
+  const text = await Bun.$`${parameters.tools.wasmTools} print ${modulePath}`.quiet().text();
+
+  const forbidden = collectForbiddenImports(text);
+
+  if (forbidden.length > 0) {
+    return { ok: false, failure: `${consumer.name} imports ${forbidden.join(', ')}` };
+  }
+
+  const bytes = await Bun.file(modulePath).bytes();
+
+  const module = toTagged<VersionHash>(new Bun.CryptoHasher('sha256').update(bytes).digest('hex'));
+  const path = `${consumer.name}/${module}.wasm`;
+
+  await mkdir(join(parameters.out, consumer.name), { recursive: true });
+
+  await Bun.write(join(parameters.out, path), bytes);
+
+  return { ok: true, value: { name: consumer.name, module, path } };
+}
+
+interface ScriptcRun {
+  work: string;
+  modulePath: string;
+  tools: BuildTools;
+}
+
+async function runScriptc(run: ScriptcRun): Promise<Result<undefined, string>> {
+  const child = Bun.spawn(
+    [run.tools.scriptc, 'build', '--lib', '--profile', 'profile.json', '-o', run.modulePath],
+    {
+      cwd: run.work,
+      env: {
+        ...process.env,
+        SCRIPTC_TARGET: 'wasm32-wasi',
+        PATH: `${dirname(run.tools.zig)}:${process.env['PATH'] ?? ''}`,
+      },
+      stdout: 'pipe',
+      stderr: 'pipe',
+    },
+  );
+
+  const [exitCode, stdout, stderr] = await Promise.all([
+    child.exited,
+    new Response(child.stdout).text(),
+    new Response(child.stderr).text(),
+  ]);
+
+  return exitCode === 0
+    ? { ok: true, value: undefined }
+    : { ok: false, failure: `${stdout}${stderr}` };
+}
+
+function isAttestedDeterministic(contract: unknown): boolean {
+  return (
+    typeof contract === 'object' &&
+    contract !== null &&
+    'deterministic' in contract &&
+    contract.deterministic === true
+  );
+}

package/src/build/consumer-fences.ts

@@ -0,0 +1,34 @@
+// The determinism fences of the scriptc library profile: the Date, performance, process, random,
+// filesystem and OS APIs that a consumer may never call.
+export const consumerFences = [
+  { prefix: 'stdlib.date.' },
+  { prefix: 'node-builtin.perf_hooks.performance.' },
+  { prefix: 'node-builtin.process.' },
+  { prefix: 'node-builtin.crypto.random' },
+  { prefix: 'node-builtin.fs.' },
+  { id: 'stdlib.math.random' },
+  { id: 'stdlib.file' },
+  { id: 'node-builtin.tls.getCACertificates' },
+  { id: 'node-builtin.tls.rootCertificates' },
+  { id: 'node-builtin.tls.setDefaultCACertificates' },
+  { id: 'node-builtin.os.arch' },
+  { id: 'node-builtin.os.availableParallelism' },
+  { id: 'node-builtin.os.constants' },
+  { id: 'node-builtin.os.cpus' },
+  { id: 'node-builtin.os.endianness' },
+  { id: 'node-builtin.os.freemem' },
+  { id: 'node-builtin.os.getPriority' },
+  { id: 'node-builtin.os.homedir' },
+  { id: 'node-builtin.os.hostname' },
+  { id: 'node-builtin.os.loadavg' },
+  { id: 'node-builtin.os.machine' },
+  { id: 'node-builtin.os.networkInterfaces' },
+  { id: 'node-builtin.os.platform' },
+  { id: 'node-builtin.os.release' },
+  { id: 'node-builtin.os.tmpdir' },
+  { id: 'node-builtin.os.totalmem' },
+  { id: 'node-builtin.os.type' },
+  { id: 'node-builtin.os.uptime' },
+  { id: 'node-builtin.os.userInfo' },
+  { id: 'node-builtin.os.version' },
+];

package/src/build/format-build-outcome.ts

@@ -0,0 +1,81 @@
+import type { DeploymentFailure } from '@pepys/kit';
+import { match } from 'ts-pattern';
+import type { BuildOutcome } from './run-build.ts';
+
+export interface FormattedOutcome {
+  exitCode: 0 | 1;
+  text: string;
+}
+
+const toolVariables = { zig: 'PEPYS_ZIG_BIN', 'wasm-tools': 'PEPYS_WASM_TOOLS_BIN' } as const;
+
+export function formatBuildOutcome(outcome: BuildOutcome): FormattedOutcome {
+  return match(outcome)
+    .with({ kind: 'built' }, (built) => ({
+      exitCode: 0 as const,
+      text: [
+        `wrote ${built.written.length} glue files, ${built.unchanged.length} unchanged`,
+        ...built.written.map((path) => `  ${path}`),
+        '',
+      ].join('\n'),
+    }))
+    .with({ kind: 'compiled' }, (compiled) => ({
+      exitCode: 0 as const,
+      text: [
+        `wrote ${compiled.written.length} glue files, ${compiled.unchanged.length} unchanged`,
+        ...compiled.written.map((path) => `  ${path}`),
+        `compiled ${compiled.modules.length} consumers`,
+        ...compiled.modules.map((built) => `  ${built.name}: ${built.path}`),
+        `wrote ${compiled.manifest}`,
+        '',
+      ].join('\n'),
+    }))
+    .with({ kind: 'missing-tool' }, (missing) => ({
+      exitCode: 1 as const,
+      text: `the glue is written; compiling needs ${missing.tool}, from ${toolVariables[missing.tool]} or PATH\n`,
+    }))
+    .with({ kind: 'compile-failed' }, (failed) => ({
+      exitCode: 1 as const,
+      text: `${failed.consumer} did not compile:\n${failed.reason}\n`,
+    }))
+    .with({ kind: 'refused-manifest' }, (refused) => ({
+      exitCode: 1 as const,
+      text: [
+        'the host would refuse this consumers.json, so the build wrote none:',
+        ...refused.issues.map((issue) => `  ${issue}`),
+        '',
+      ].join('\n'),
+    }))
+    .with({ kind: 'fresh' }, (fresh) => ({
+      exitCode: 0 as const,
+      text: `${fresh.count} glue files are up to date\n`,
+    }))
+    .with({ kind: 'stale' }, (stale) => ({
+      exitCode: 1 as const,
+      text: [
+        'glue files differ from the declarations; run `pepys build`:',
+        ...stale.stale.map((path) => `  ${path}`),
+        '',
+      ].join('\n'),
+    }))
+    .with({ kind: 'no-target' }, (missing) => ({
+      exitCode: 1 as const,
+      text:
+        missing.reason === 'no-target'
+          ? 'the file exports no Deployment and no Plugin\n'
+          : 'the file exports more than one Deployment or Plugin\n',
+    }))
+    .with({ kind: 'invalid' }, (invalid) => ({
+      exitCode: 1 as const,
+      text: [
+        'the declarations are invalid:',
+        ...invalid.failures.map((failure) => `  ${formatFailure(failure)}`),
+        '',
+      ].join('\n'),
+    }))
+    .exhaustive();
+}
+
+function formatFailure(failure: DeploymentFailure): string {
+  return JSON.stringify(failure);
+}

package/src/build/parse-build-arguments.ts

@@ -0,0 +1,45 @@
+import { parseArgs } from 'node:util';
+import type { Result } from '@pepys/kit';
+
+export interface BuildArguments {
+  file: string;
+  isCheck: boolean;
+  out: string | undefined;
+}
+
+export const usage = 'usage: pepys build <file> [--out <directory>] [--check]';
+
+export function parseBuildArguments(args: readonly string[]): Result<BuildArguments, string> {
+  const [command, ...rest] = args;
+
+  if (command !== 'build') {
+    return { ok: false, failure: usage };
+  }
+
+  try {
+    const parsed = parseArgs({
+      args: rest,
+      allowPositionals: true,
+      options: { check: { type: 'boolean' }, out: { type: 'string' } },
+    });
+
+    const [file, ...extra] = parsed.positionals;
+
+    if (file === undefined || extra.length > 0) {
+      return { ok: false, failure: usage };
+    }
+
+    const isCheck = parsed.values.check === true;
+
+    // --check writes nothing, so it takes no folder to write to
+    if (isCheck && parsed.values.out !== undefined) {
+      return { ok: false, failure: `--check takes no --out\n${usage}` };
+    }
+
+    return { ok: true, value: { file, isCheck, out: parsed.values.out } };
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error);
+
+    return { ok: false, failure: `${reason}\n${usage}` };
+  }
+}

package/src/build/pick-build-target.ts

@@ -0,0 +1,55 @@
+import type { Deployment, Plugin, Result } from '@pepys/kit';
+import { isRecord } from '@zgeoff/std';
+
+export type BuildTarget =
+  | { kind: 'deployment'; deployment: Deployment }
+  | { kind: 'plugin'; plugin: Plugin };
+
+export type TargetFailure = 'no-target' | 'several-targets';
+
+// The file exports one Deployment or Plugin, under any name or as the default export. A value
+// exported twice counts once.
+export function pickBuildTarget(
+  exports: Readonly<Record<string, unknown>>,
+): Result<BuildTarget, TargetFailure> {
+  const targets = [...new Set(Object.values(exports))]
+    .map((value) => toBuildTarget(value))
+    .filter((target) => target !== undefined);
+
+  const [first, ...others] = targets;
+
+  if (first === undefined) {
+    return { ok: false, failure: 'no-target' };
+  }
+
+  if (others.length > 0) {
+    return { ok: false, failure: 'several-targets' };
+  }
+
+  return { ok: true, value: first };
+}
+
+function toBuildTarget(value: unknown): BuildTarget | undefined {
+  if (isDeployment(value)) {
+    return { kind: 'deployment', deployment: value };
+  }
+
+  if (isPlugin(value)) {
+    return { kind: 'plugin', plugin: value };
+  }
+
+  return undefined;
+}
+
+function isDeployment(value: unknown): value is Deployment {
+  return isRecord(value) && typeof value['name'] === 'string' && Array.isArray(value['plugins']);
+}
+
+function isPlugin(value: unknown): value is Plugin {
+  return (
+    isRecord(value) &&
+    typeof value['name'] === 'string' &&
+    typeof value['kit'] === 'number' &&
+    Array.isArray(value['consumers'])
+  );
+}

package/src/build/read-package-root.ts

@@ -0,0 +1,18 @@
+import { dirname, resolve } from 'node:path';
+
+// The nearest folder at or above the start that holds a package.json, or undefined when none does.
+export async function readPackageRoot(start: string): Promise<string | undefined> {
+  let folder = start;
+
+  while (!(await Bun.file(resolve(folder, 'package.json')).exists())) {
+    const parent = dirname(folder);
+
+    if (parent === folder) {
+      return undefined;
+    }
+
+    folder = parent;
+  }
+
+  return folder;
+}

package/src/build/run-build.ts

@@ -0,0 +1,199 @@
+import { mkdir, rm } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import type { DeploymentFailure } from '@pepys/kit';
+import { isRecord, readOptionalText } from '@zgeoff/std';
+import { buildConsumerBuildManifest } from './build-consumer-build-manifest.ts';
+import type { BuiltModule } from './build-consumer-build-manifest.ts';
+import type { GlueFile } from './build-consumer-glue.ts';
+import { checkBuildTarget } from './check-build-target.ts';
+import type { CheckedTarget } from './check-build-target.ts';
+import { collectGlueFiles } from './collect-glue-files.ts';
+import { compileConsumer } from './compile-consumer.ts';
+import type { BuildTools } from './compile-consumer.ts';
+import { pickBuildTarget } from './pick-build-target.ts';
+import type { TargetFailure } from './pick-build-target.ts';
+import { readPackageRoot } from './read-package-root.ts';
+import { writeBuildManifest } from './write-build-manifest.ts';
+
+// scriptc comes with the CLI; a tool that the machine lacks is undefined
+export interface BuildToolPaths {
+  scriptc: string;
+  zig: string | undefined;
+  wasmTools: string | undefined;
+}
+
+export interface BuildOptions {
+  file: string;
+
+  // compare the glue with the files on disk and write nothing
+  isCheck: boolean;
+
+  // The folder for the modules and consumers.json. With none, the build stops after the glue, so a
+  // plug-in author writes and tests a consumer with no zig and no wasm-tools.
+  out: string | undefined;
+
+  tools: BuildToolPaths;
+}
+
+export type BuildOutcome =
+  | { kind: 'built'; written: readonly string[]; unchanged: readonly string[] }
+  | {
+      kind: 'compiled';
+      written: readonly string[];
+      unchanged: readonly string[];
+      modules: readonly BuiltModule[];
+      manifest: string;
+    }
+  | { kind: 'stale'; stale: readonly string[] }
+  | { kind: 'fresh'; count: number }
+  | { kind: 'no-target'; reason: TargetFailure }
+  | { kind: 'invalid'; failures: readonly DeploymentFailure[] }
+  | { kind: 'missing-tool'; tool: 'zig' | 'wasm-tools' }
+  | { kind: 'compile-failed'; consumer: string; reason: string }
+  | { kind: 'refused-manifest'; issues: readonly string[] };
+
+// The four steps of the build: check the target, write the glue of each consumer, compile each
+// entry with scriptc, and write each module and consumers.json.
+export async function runBuild(options: BuildOptions): Promise<BuildOutcome> {
+  const file = resolve(options.file);
+
+  const loaded: unknown = await import(pathToFileURL(file).href);
+
+  const exports = isRecord(loaded) ? loaded : {};
+  const picked = pickBuildTarget(exports);
+
+  if (!picked.ok) {
+    return { kind: 'no-target', reason: picked.failure };
+  }
+
+  const checked = checkBuildTarget(picked.value);
+
+  if (!checked.ok) {
+    return { kind: 'invalid', failures: checked.failure };
+  }
+
+  const foundRoot = await readPackageRoot(dirname(file));
+
+  const glue = collectGlueFiles({
+    target: checked.value,
+    packageRoot: foundRoot ?? dirname(file),
+  });
+
+  const stale = await readStalePaths(glue);
+
+  if (options.isCheck) {
+    return buildCheckOutcome(stale, glue.length);
+  }
+
+  for (const entry of glue.filter((candidate) => stale.includes(candidate.path))) {
+    await mkdir(dirname(entry.path), { recursive: true });
+
+    await Bun.write(entry.path, entry.content);
+  }
+
+  const written = stale;
+  const unchanged = glue.map((entry) => entry.path).filter((path) => !stale.includes(path));
+
+  if (options.out === undefined) {
+    return { kind: 'built', written, unchanged };
+  }
+
+  const zig = options.tools.zig;
+  const wasmTools = options.tools.wasmTools;
+
+  if (zig === undefined) {
+    return { kind: 'missing-tool', tool: 'zig' };
+  }
+
+  if (wasmTools === undefined) {
+    return { kind: 'missing-tool', tool: 'wasm-tools' };
+  }
+
+  const out = resolve(options.out);
+
+  const compiled = await compileTarget({
+    target: checked.value,
+    out,
+    tools: { scriptc: options.tools.scriptc, zig, wasmTools },
+  });
+
+  if (compiled.kind !== 'compiled') {
+    return compiled;
+  }
+
+  const manifest = await writeBuildManifest(
+    out,
+    buildConsumerBuildManifest({ target: checked.value, modules: compiled.modules }),
+  );
+
+  if (!manifest.ok) {
+    return { kind: 'refused-manifest', issues: manifest.failure };
+  }
+
+  return {
+    kind: 'compiled',
+    written,
+    unchanged,
+    modules: compiled.modules,
+    manifest: manifest.value,
+  };
+}
+
+async function readStalePaths(glue: readonly GlueFile[]): Promise<string[]> {
+  const results = await Promise.all(
+    glue.map(async (entry) => ({
+      path: entry.path,
+      isStale: (await readOptionalText(entry.path)) !== entry.content,
+    })),
+  );
+
+  return results.filter((result) => result.isStale).map((result) => result.path);
+}
+
+function buildCheckOutcome(stale: readonly string[], count: number): BuildOutcome {
+  if (stale.length > 0) {
+    return { kind: 'stale', stale };
+  }
+
+  return { kind: 'fresh', count };
+}
+
+interface CompileTargetParameters {
+  target: CheckedTarget;
+  out: string;
+  tools: BuildTools;
+}
+
+type CompileTargetOutcome =
+  | { kind: 'compiled'; modules: readonly BuiltModule[] }
+  | { kind: 'compile-failed'; consumer: string; reason: string };
+
+async function compileTarget(parameters: CompileTargetParameters): Promise<CompileTargetOutcome> {
+  const families = parameters.target.checked.sources.flatMap((source) => source.families);
+  const modules: BuiltModule[] = [];
+
+  try {
+    for (const plugin of parameters.target.plugins) {
+      for (const consumer of plugin.consumers) {
+        const built = await compileConsumer({
+          consumer,
+          families,
+          settings: parameters.target.checked.settings[plugin.name],
+          out: parameters.out,
+          tools: parameters.tools,
+        });
+
+        if (!built.ok) {
+          return { kind: 'compile-failed', consumer: consumer.name, reason: built.failure };
+        }
+
+        modules.push(built.value);
+      }
+    }
+  } finally {
+    await rm(join(parameters.out, '.work'), { recursive: true, force: true });
+  }
+
+  return { kind: 'compiled', modules };
+}

package/src/build/write-build-manifest.ts

@@ -0,0 +1,30 @@
+import { mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { Result } from '@pepys/kit';
+import type { ConsumerBuildManifest } from '@pepys/schema';
+import { consumerBuildManifestSchema } from '@pepys/schema';
+
+// Writes `<out>/consumers.json` and returns its path, or the schema's issues and writes nothing. The
+// file passes the strict schema first, so the build never writes a file that the host refuses, such
+// as one with a table named after a table of the host.
+export async function writeBuildManifest(
+  out: string,
+  manifest: ConsumerBuildManifest,
+): Promise<Result<string, readonly string[]>> {
+  const parsed = consumerBuildManifestSchema.safeParse(manifest);
+
+  if (!parsed.success) {
+    return {
+      ok: false,
+      failure: parsed.error.issues.map((issue) => `${issue.path.join('.')}: ${issue.message}`),
+    };
+  }
+
+  const path = join(out, 'consumers.json');
+
+  await mkdir(out, { recursive: true });
+
+  await Bun.write(path, `${JSON.stringify(manifest, null, 2)}\n`);
+
+  return { ok: true, value: path };
+}

package/src/index.ts

@@ -0,0 +1,6 @@
+export { buildConsumerBuildManifest } from './build/build-consumer-build-manifest.ts';
+export type { BuiltModule } from './build/build-consumer-build-manifest.ts';
+export type { GlueFile } from './build/build-consumer-glue.ts';
+export { checkBuildTarget } from './build/check-build-target.ts';
+export { collectGlueFiles } from './build/collect-glue-files.ts';
+export { writeBuildManifest } from './build/write-build-manifest.ts';

package/src/run-cli.ts

@@ -0,0 +1,42 @@
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { writeStdout } from '@zgeoff/std';
+import { formatBuildOutcome } from './build/format-build-outcome.ts';
+import { parseBuildArguments } from './build/parse-build-arguments.ts';
+import { runBuild } from './build/run-build.ts';
+import type { BuildToolPaths } from './build/run-build.ts';
+
+// Returns the exit code.
+export async function runCLI(args: readonly string[]): Promise<number> {
+  const parsed = parseBuildArguments(args);
+
+  if (!parsed.ok) {
+    process.stderr.write(`${parsed.failure}\n`);
+
+    return 1;
+  }
+
+  const outcome = await runBuild({ ...parsed.value, tools: readBuildTools() });
+
+  const formatted = formatBuildOutcome(outcome);
+
+  if (formatted.exitCode === 0) {
+    writeStdout(formatted.text);
+  } else {
+    process.stderr.write(formatted.text);
+  }
+
+  return formatted.exitCode;
+}
+
+// scriptc comes with the CLI. zig and wasm-tools come from the two variables that pin a version, or
+// from PATH.
+function readBuildTools(): BuildToolPaths {
+  const scriptcPackage = fileURLToPath(import.meta.resolve('scriptc/package.json'));
+
+  return {
+    scriptc: join(dirname(scriptcPackage), 'bin/scriptc.exe'),
+    zig: process.env['PEPYS_ZIG_BIN'] ?? Bun.which('zig') ?? undefined,
+    wasmTools: process.env['PEPYS_WASM_TOOLS_BIN'] ?? Bun.which('wasm-tools') ?? undefined,
+  };
+}