@origingame/engine 0.0.0-stage → 0.1.0

package/knowledge/scripts/lib.ts

@@ -0,0 +1,203 @@
+// The knowledge package as the platform receives it: `payload()` maps the
+// package (and, in the engine repository, templates/game) onto the
+// platform's production layout (`game/…`, `game-scaffold/…`) through
+// knowledge.json, and the helpers read what the platform's own tooling
+// reads (SKILL.md front matter, pack names and hashes).
+
+import { createHash } from "node:crypto";
+import { existsSync, lstatSync, readdirSync, readFileSync } from "node:fs";
+import { join, relative } from "node:path";
+
+export const PACKAGE_ROOT = join(import.meta.dir, "..");
+
+export interface Roots {
+  /** packages/knowledge (or a copy of it). */
+  readonly pkg: string;
+  /** The project template the scaffold overlay carries: templates/game in the repository, scaffold/template/ in the published package. */
+  readonly template: string;
+  /** packages/engine, whose version the scaffold pins. */
+  readonly engine: string;
+}
+
+/** Whether `dir` is the published @origingame/engine package (the knowledge ships inside it as knowledge/). */
+function isEnginePackage(dir: string): boolean {
+  try {
+    return (JSON.parse(readFileSync(join(dir, "package.json"), "utf8")) as { name?: string }).name === "@origingame/engine";
+  } catch {
+    return false;
+  }
+}
+
+/** The roots in the engine repository (packages/knowledge beside packages/engine), or in the published @origingame/engine, whose knowledge/ this is. */
+export function defaultRoots(pkg = PACKAGE_ROOT): Roots {
+  const repoTemplate = join(pkg, "..", "..", "templates", "game");
+  const parent = join(pkg, "..");
+  return {
+    pkg,
+    template: existsSync(join(repoTemplate, "project.ogt")) ? repoTemplate : join(pkg, "scaffold", "template"),
+    engine: isEnginePackage(parent) ? parent : join(parent, "engine"),
+  };
+}
+
+export interface Mount {
+  /** A package path (`skills/`, `wiki/`), or `@template/` for the project template. */
+  readonly from: string;
+  /** Its place in the platform's production tree. */
+  readonly to: string;
+}
+
+export interface Manifest {
+  readonly schemaVersion: 1;
+  readonly overlay: string;
+  readonly mounts: readonly Mount[];
+}
+
+export function readManifest(pkg: string): Manifest {
+  return JSON.parse(readFileSync(join(pkg, "knowledge.json"), "utf8")) as Manifest;
+}
+
+/** One file of the payload: its production path, where it came from, its bytes. */
+export interface PayloadFile {
+  readonly path: string;
+  readonly source: string;
+  readonly bytes: Uint8Array;
+  readonly symlink: boolean;
+}
+
+/** Files under `dir`, sorted, skipping build output a template never carries. */
+export function walk(dir: string, skip: ReadonlySet<string> = new Set()): string[] {
+  const out: string[] = [];
+  const visit = (d: string) => {
+    for (const name of readdirSync(d).sort()) {
+      const full = join(d, name);
+      const rel = relative(dir, full);
+      if (skip.has(name) || skip.has(rel)) continue;
+      const st = lstatSync(full);
+      if (st.isDirectory()) visit(full);
+      else out.push(full);
+    }
+  };
+  if (existsSync(dir)) visit(dir);
+  return out;
+}
+
+/** What never travels with a template: build and run output (the template's .gitignore) and installs. */
+export const TEMPLATE_SKIP: ReadonlySet<string> = new Set(["cooked", "dist", "og-out", ".og", "node_modules", ".DS_Store"]);
+
+/**
+ * Template files stored under another name than the scaffold gives them
+ * (packages/create's RENAMED): the template's `gitattributes` lands in the
+ * overlay as `.gitattributes`, which in the engine repository would route
+ * the template's plain-blob media through LFS.
+ */
+export const TEMPLATE_RENAMED: ReadonlyMap<string, string> = new Map([["gitattributes", ".gitattributes"]]);
+
+const fatal = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+
+/** Whether bytes are text the platform's payload can carry (UTF-8, no NUL). */
+export function isText(bytes: Uint8Array): boolean {
+  if (bytes.includes(0)) return false;
+  try {
+    fatal.decode(bytes);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * The production-tree payload: every mounted file. Template binaries are
+ * included here (so the validator sees them) and flagged by `isText`; the
+ * published bundle carries text only and names binaries in distribution.json.
+ */
+export function payload(roots: Roots): PayloadFile[] {
+  const manifest = readManifest(roots.pkg);
+  const files: PayloadFile[] = [];
+  for (const m of manifest.mounts) {
+    const base = m.from === "@template/" ? roots.template : join(roots.pkg, m.from);
+    const skip = m.from === "@template/" ? TEMPLATE_SKIP : new Set<string>();
+    const sources = m.from.endsWith("/") ? walk(base, skip) : [base];
+    for (const source of sources) {
+      const found = m.from.endsWith("/") ? relative(base, source).split("\\").join("/") : "";
+      const rel = m.from === "@template/" ? (TEMPLATE_RENAMED.get(found) ?? found) : found;
+      const path = m.to.endsWith("/") ? m.to + rel : m.to;
+      const symlink = existsSync(source) ? lstatSync(source).isSymbolicLink() : false;
+      files.push({ path, source, bytes: existsSync(source) && !symlink ? new Uint8Array(readFileSync(source)) : new Uint8Array(), symlink });
+    }
+  }
+  return files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+}
+
+export function sha256(bytes: Uint8Array | string): string {
+  return createHash("sha256").update(bytes).digest("hex");
+}
+
+/** Pack names as agent-server/src/core/knowledge.ts `packsFromFiles` groups production paths. */
+export function packName(path: string): string {
+  const parts = path.split("/");
+  if (path === "game/instructions.md") return "instructions";
+  if (parts[0] === "game" && parts[1] === "skills" && parts[2]) return `skill:${parts[2]}`;
+  if (parts[0] === "game-scaffold" && parts[1]) return `scaffold:${parts[1]}`;
+  return "game";
+}
+
+/** Each pack's content hash, as `packsFromFiles` computes it over the text files. */
+export function packs(files: readonly PayloadFile[]): Record<string, string> {
+  const groups = new Map<string, Record<string, string>>();
+  for (const f of files) {
+    if (!isText(f.bytes)) continue;
+    const name = packName(f.path);
+    const g = groups.get(name) ?? {};
+    g[f.path] = fatal.decode(f.bytes);
+    groups.set(name, g);
+  }
+  return Object.fromEntries(
+    [...groups.entries()]
+      .sort(([a], [b]) => a.localeCompare(b))
+      .map(([name, g]) => [name, sha256(Object.keys(g).sort().map((p) => `${p}\0${sha256(g[p]!)}`).join("\n")).slice(0, 16)]),
+  );
+}
+
+/** A SKILL.md's front matter as `key: value` pairs, or null without the `---` fences. */
+export function frontMatter(text: string): Record<string, string> | null {
+  const lines = text.replace(/^\uFEFF/, "").split(/\r?\n/);
+  if (lines[0]?.trim() !== "---") return null;
+  const end = lines.findIndex((l, i) => i > 0 && l.trim() === "---");
+  if (end < 0) return null;
+  const out: Record<string, string> = {};
+  let key = "";
+  for (const line of lines.slice(1, end)) {
+    const m = /^([A-Za-z][\w-]*):(.*)$/.exec(line);
+    if (m) {
+      key = m[1]!;
+      const rest = m[2]!.trim();
+      out[key] = ["", ">", "|", ">-", "|-"].includes(rest) ? "" : rest.replace(/^["']|["']$/g, "");
+    } else if (key && /^[ \t]/.test(line)) {
+      out[key] = out[key] ? `${out[key]} ${line.trim()}` : line.trim();
+    }
+  }
+  return out;
+}
+
+/** The top-level `key value` entries of an `.ogt` document (values as written, quotes removed). */
+export function ogtTopLevel(text: string): { type: string; entries: Map<string, string> } {
+  const lines = text.split(/\r?\n/);
+  const sep = lines.findIndex((l) => l.trim() === "---");
+  const header = lines.slice(0, sep < 0 ? 0 : sep).map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
+  const type = header.find((l) => l.startsWith("type "))?.slice(5).trim() ?? "";
+  const entries = new Map<string, string>();
+  let depth = 0;
+  for (const raw of lines.slice(sep + 1)) {
+    const line = raw.replace(/(^|\s)#.*$/, "").trim();
+    if (!line) continue;
+    if (line === "}") {
+      depth--;
+      continue;
+    }
+    const [key, ...rest] = line.split(/\s+/);
+    const opens = line.endsWith("{");
+    if (depth === 0) entries.set(key!, (opens ? rest.slice(0, -1) : rest).join(" ").replace(/^"|"$/g, ""));
+    if (opens) depth++;
+  }
+  return { type, entries };
+}

package/knowledge/scripts/mount.ts

@@ -0,0 +1,210 @@
+// Materialises the mounts knowledge.json declares into an OriginGame
+// platform checkout's production tree (packages/game-knowledge/production):
+// skills as game/skills/<name>/, the wiki as og-game's reference/, the
+// project template plus origin-engine.json and distribution.json as the
+// scaffold overlay game-scaffold/<overlay>/.
+//
+//   bun scripts/mount.ts <origingame checkout> [--check] [--commit <rev> | --package <dir>]
+//
+// The package owns, in the platform tree, exactly the directories it mounts:
+// game/skills/<each of its skills>/ and game-scaffold/<overlay>/. Within
+// them the result is byte-for-byte the package's text payload: files are
+// written only when their bytes differ, files the package no longer has are
+// removed, and nothing outside them is touched, so a second run changes
+// nothing. Binary template files (models, textures, audio, fonts) are never
+// written: the platform payload is UTF-8 text only and distribution.json
+// names them with their digests. The package is validated against the
+// platform rules first; an invalid package mounts nothing.
+//
+// --check writes nothing and exits 1 when the platform tree differs, naming
+// each added, changed and removed path. --commit mounts the package as
+// committed at <rev> (a `git archive` of packages/knowledge, templates/game
+// and packages/engine/package.json) instead of the working tree, so work in
+// progress in this checkout never reaches the platform. --package mounts the
+// knowledge of an installed @origingame/engine (the directory `bun install`
+// put it in; the package carries this directory as knowledge/): what the
+// platform does at a pinned version (docs/release.md §4 L2).
+
+import { execFileSync } from "node:child_process";
+import { chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmdirSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { defaultRoots, isText, PACKAGE_ROOT, payload, walk, type Roots } from "./lib.ts";
+import { validate } from "./validate.ts";
+
+export const PRODUCTION = "packages/game-knowledge/production";
+
+export interface Change {
+  readonly kind: "added" | "changed" | "removed";
+  readonly path: string;
+}
+
+export interface MountResult {
+  /** The platform's production root. */
+  readonly production: string;
+  /** The directories the package owns there, production-relative, each ending in `/`. */
+  readonly owned: readonly string[];
+  /** Text files mounted (written or already identical). */
+  readonly files: number;
+  /** Template binaries left to distribution.json. */
+  readonly binaries: readonly string[];
+  readonly changes: readonly Change[];
+}
+
+/** The production-relative directories the payload occupies: game/skills/<name>/ and game-scaffold/<overlay>/. */
+export function ownedRoots(paths: readonly string[]): string[] {
+  const owned = new Set<string>();
+  for (const p of paths) {
+    const parts = p.split("/");
+    if (parts[0] === "game" && parts[1] === "skills" && parts.length > 3) owned.add(`game/skills/${parts[2]}/`);
+    else if (parts[0] === "game-scaffold" && parts.length > 2) owned.add(`game-scaffold/${parts[1]}/`);
+    else throw new Error(`${p}: a mount outside game/skills/<name>/ and game-scaffold/<overlay>/; fix knowledge.json`);
+  }
+  return [...owned].sort();
+}
+
+function executable(path: string): boolean {
+  return (statSync(path).mode & 0o111) !== 0;
+}
+
+/** Compares (and with `write`, materialises) the package's payload in `production`. */
+export function mount(roots: Roots, production: string, write: boolean): MountResult {
+  for (const top of ["game/skills", "game-scaffold"]) {
+    if (!existsSync(join(production, top)) || !statSync(join(production, top)).isDirectory()) {
+      throw new Error(`${production}: no ${top}/; pass the root of an OriginGame checkout (it holds ${PRODUCTION})`);
+    }
+  }
+  const problems = validate(roots);
+  if (problems.length) throw new Error(`the knowledge package is invalid against the platform rules; nothing mounted:\n${problems.join("\n")}`);
+
+  const all = payload(roots);
+  const text = all.filter((f) => isText(f.bytes));
+  const binaries = all.filter((f) => !isText(f.bytes)).map((f) => f.path);
+  const owned = ownedRoots(all.map((f) => f.path));
+  const wanted = new Map(text.map((f) => [f.path, f]));
+  const changes: Change[] = [];
+
+  for (const f of text) {
+    const target = join(production, f.path);
+    const exists = existsSync(target);
+    if (exists && lstatSync(target).isDirectory()) throw new Error(`${f.path}: a directory in the platform tree where the package has a file`);
+    const same = exists && Buffer.compare(readFileSync(target), Buffer.from(f.bytes)) === 0;
+    const mode = executable(f.source) ? 0o755 : 0o644;
+    const modeSame = exists && (statSync(target).mode & 0o777) === mode;
+    if (same && modeSame) continue;
+    changes.push({ kind: exists ? "changed" : "added", path: f.path });
+    if (write) {
+      mkdirSync(dirname(target), { recursive: true });
+      if (!same) writeFileSync(target, f.bytes);
+      chmodSync(target, mode);
+    }
+  }
+
+  for (const root of owned) {
+    const dir = join(production, root);
+    for (const full of walk(dir)) {
+      const path = `${root}${full.slice(dir.length).split("\\").join("/")}`;
+      if (wanted.has(path)) continue;
+      changes.push({ kind: "removed", path });
+      if (write) rmSync(full, { force: true });
+    }
+    if (write) pruneEmpty(dir);
+  }
+
+  changes.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+  return { production, owned, files: text.length, binaries, changes };
+}
+
+/** Removes empty directories under `dir` (not `dir` itself). */
+function pruneEmpty(dir: string): void {
+  if (!existsSync(dir)) return;
+  for (const name of readdirSync(dir)) {
+    const full = join(dir, name);
+    if (!lstatSync(full).isDirectory()) continue;
+    pruneEmpty(full);
+    if (readdirSync(full).length === 0) rmdirSync(full);
+  }
+}
+
+/** What a mount reads from the engine repository. */
+export const SOURCES = ["packages/knowledge", "templates/game", "packages/engine/package.json"] as const;
+
+const git = (repo: string, ...args: string[]) =>
+  execFileSync("git", ["-C", repo, ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim();
+
+/** The working tree's short revision, `+dirty` when the mounted sources differ from it, or `unversioned`. */
+export function revision(repo: string): string {
+  try {
+    const head = git(repo, "rev-parse", "--short", "HEAD");
+    return git(repo, "status", "--porcelain", "--", ...SOURCES) ? `${head}+dirty` : head;
+  } catch {
+    return "unversioned";
+  }
+}
+
+/** The package as committed at `rev`, exported into a temporary directory: its roots, short revision and a cleanup. */
+export function committed(repo: string, rev: string): { roots: Roots; revision: string; cleanup: () => void } {
+  const short = git(repo, "rev-parse", "--short", `${rev}^{commit}`);
+  const dir = mkdtempSync(join(tmpdir(), "og-knowledge-mount-"));
+  execFileSync("sh", ["-c", 'git -C "$1" archive "$2" -- "$3" "$4" "$5" | tar -x -C "$6"', "sh", repo, short, ...SOURCES, dir], {
+    stdio: ["ignore", "ignore", "pipe"],
+  });
+  return { roots: defaultRoots(join(dir, "packages/knowledge")), revision: short, cleanup: () => rmSync(dir, { recursive: true, force: true }) };
+}
+
+/** An installed @origingame/engine (the directory `bun install` put it in, or its knowledge/): the knowledge's roots and `@origingame/engine@<version>`. */
+export function installed(dir: string): { roots: Roots; revision: string; cleanup: () => void } {
+  const manifest = join(dir, "package.json");
+  if (!existsSync(manifest)) throw new Error(`${dir}: no package.json; --package takes the installed @origingame/engine directory`);
+  const pkg = JSON.parse(readFileSync(manifest, "utf8")) as { name?: string; version?: string };
+  let knowledge = dir;
+  if (pkg.name === "@origingame/engine") knowledge = join(dir, "knowledge");
+  else if (pkg.name !== "@origingame/engine-knowledge") throw new Error(`${dir}: ${pkg.name ?? "no name"}, not @origingame/engine`);
+  if (!existsSync(join(knowledge, "knowledge.json"))) throw new Error(`${knowledge}: no knowledge.json; the published @origingame/engine carries the knowledge in knowledge/`);
+  const roots = defaultRoots(knowledge);
+  if (!existsSync(join(roots.engine, "package.json"))) throw new Error(`${roots.engine}: no @origingame/engine package.json`);
+  const engine = JSON.parse(readFileSync(join(roots.engine, "package.json"), "utf8")) as { name?: string; version?: string };
+  return { roots, revision: `${engine.name}@${engine.version} (knowledge)`, cleanup: () => {} };
+}
+
+export function summary(r: MountResult, rev: string, check: boolean): string {
+  const count = (k: Change["kind"]) => r.changes.filter((c) => c.kind === k).length;
+  const lines = [
+    `${check ? "check" : "mount"}: engine knowledge ${rev} -> ${r.production}`,
+    `owned: ${r.owned.join(", ")}`,
+    `text files: ${r.files}; binaries left to distribution.json: ${r.binaries.length}`,
+    `${check ? "differences" : "changes"}: ${count("added")} added, ${count("changed")} changed, ${count("removed")} removed`,
+    ...r.changes.map((c) => `  ${c.kind.padEnd(7)} ${c.path}`),
+  ];
+  return `${lines.join("\n")}\n`;
+}
+
+if (import.meta.main) {
+  const args = process.argv.slice(2);
+  const check = args.includes("--check");
+  const ci = args.indexOf("--commit");
+  const commit = ci >= 0 ? args[ci + 1] : undefined;
+  const pi = args.indexOf("--package");
+  const pkgDir = pi >= 0 ? args[pi + 1] : undefined;
+  const positional = args.filter((a, i) => !a.startsWith("--") && !(ci >= 0 && i === ci + 1) && !(pi >= 0 && i === pi + 1));
+  if ((ci >= 0 && !commit) || (pi >= 0 && !pkgDir) || (commit && pkgDir) || positional.length !== 1 || args.some((a) => a.startsWith("--") && a !== "--check" && a !== "--commit" && a !== "--package")) {
+    process.stderr.write("usage: bun scripts/mount.ts <origingame checkout> [--check] [--commit <rev> | --package <dir>]\n");
+    process.exit(2);
+  }
+  const repo = join(PACKAGE_ROOT, "..", "..");
+  const production = join(resolve(positional[0]!), PRODUCTION);
+  let code = 0;
+  let source: { roots: Roots; revision: string; cleanup: () => void } | undefined;
+  try {
+    source = commit ? committed(repo, commit) : pkgDir ? installed(resolve(pkgDir)) : { roots: defaultRoots(), revision: revision(repo), cleanup: () => {} };
+    const result = mount(source.roots, production, !check);
+    process.stdout.write(summary(result, source.revision, check));
+    if (check && result.changes.length) code = 1;
+  } catch (e) {
+    process.stderr.write(`${e instanceof Error ? e.message : String(e)}\n`);
+    code = 1;
+  } finally {
+    source?.cleanup();
+  }
+  process.exit(code);
+}

package/knowledge/scripts/pack.ts

@@ -0,0 +1,24 @@
+// prepack: copies the project template's text files into scaffold/template/
+// so the published package carries the scaffold overlay without the engine
+// repository (binaries stay out: distribution.json names them). Ignored by Git.
+
+import { cpSync, mkdirSync, readFileSync, rmSync } from "node:fs";
+import { dirname, join, relative } from "node:path";
+import { defaultRoots, isText, PACKAGE_ROOT, TEMPLATE_SKIP, walk } from "./lib.ts";
+
+const source = join(PACKAGE_ROOT, "..", "..", "templates", "game");
+const target = join(PACKAGE_ROOT, "scaffold", "template");
+if (defaultRoots().template !== source) {
+  process.stderr.write(`no ${source}: pack inside the engine repository\n`);
+  process.exit(1);
+}
+rmSync(target, { recursive: true, force: true });
+let n = 0;
+for (const full of walk(source, TEMPLATE_SKIP)) {
+  if (!isText(new Uint8Array(readFileSync(full)))) continue;
+  const to = join(target, relative(source, full));
+  mkdirSync(dirname(to), { recursive: true });
+  cpSync(full, to);
+  n++;
+}
+process.stdout.write(`copied ${n} template text files into scaffold/template/\n`);

package/knowledge/scripts/scaffold.ts

@@ -0,0 +1,72 @@
+// Writes or checks scaffold/distribution.json: the project template's binary
+// files (models, textures, audio, fonts) with their sizes, digests and
+// licences. The platform's scaffold payload is UTF-8 text only, so binaries
+// travel from the engine distribution (the published @origingame/create-game
+// template) and are verified against these digests, as the platform's
+// game-scaffold/origin-engine/distribution.json does for its overlay.
+//
+//   bun scripts/scaffold.ts --write | --check
+
+import { readFileSync, writeFileSync } from "node:fs";
+import { join, relative } from "node:path";
+import { defaultRoots, isText, sha256, TEMPLATE_SKIP, walk } from "./lib.ts";
+
+/** Licence per template file, from the template's LICENSES.md table. */
+function licences(template: string): Map<string, string> {
+  const out = new Map<string, string>();
+  let text = "";
+  try {
+    text = readFileSync(join(template, "LICENSES.md"), "utf8");
+  } catch {
+    return out;
+  }
+  for (const line of text.split("\n")) {
+    if (!line.startsWith("| `")) continue;
+    const cells = line.split("|").map((c) => c.trim());
+    for (const m of cells[1]!.matchAll(/`([^`]+)`/g)) out.set(m[1]!, `${cells[2]} (LICENSES.md)`);
+  }
+  return out;
+}
+
+export function distribution(template: string): string {
+  const lic = licences(template);
+  const files = walk(template, TEMPLATE_SKIP)
+    .map((full) => ({ full, rel: relative(template, full).split("\\").join("/"), bytes: new Uint8Array(readFileSync(full)) }))
+    .filter((f) => !isText(f.bytes))
+    .map((f) => ({
+      path: f.rel,
+      bytes: f.bytes.length,
+      sha256: sha256(f.bytes),
+      licence: lic.get(f.rel) ?? "unrecorded: add it to the template's LICENSES.md",
+      source: `templates/game/${f.rel} of origingame/origin-engine; shipped in @origingame/create-game's template/`,
+    }));
+  return `${JSON.stringify(
+    {
+      schema: 1,
+      about: "Binary files of the Origin Engine project template that a scaffolded Game needs but that are not knowledge text. The scaffolder copies them from the published template (@origingame/create-game) and verifies each digest.",
+      files,
+    },
+    null,
+    2,
+  )}\n`;
+}
+
+if (import.meta.main) {
+  const roots = defaultRoots();
+  const path = join(roots.pkg, "scaffold/distribution.json");
+  const want = distribution(roots.template);
+  if (process.argv.includes("--write")) {
+    writeFileSync(path, want);
+    process.stdout.write(`wrote scaffold/distribution.json from ${relative(process.cwd(), roots.template) || "."}\n`);
+  } else {
+    let have = "";
+    try {
+      have = readFileSync(path, "utf8");
+    } catch {}
+    if (have !== want) {
+      process.stderr.write("scaffold/distribution.json differs from the template's binaries; run bun scripts/scaffold.ts --write\n");
+      process.exit(1);
+    }
+    process.stdout.write("scaffold/distribution.json matches the template.\n");
+  }
+}