@aethervtt/shard-procgen 0.0.1

package/dist/outputs.js

@@ -0,0 +1,250 @@
+import {
+  findComponent,
+  isPlainObject,
+  pointer,
+  ShardError
+} from "@aethervtt/shard-core";
+import { encodeMesh, Mesh } from "@aethervtt/shard-mesh";
+import { PREFAB_VERSION } from "@aethervtt/shard-scene";
+import { buildMips, writeKtx2 } from "@aethervtt/shard-texture";
+function mismatch(gen, message, path, details) {
+  return new ShardError("procgen/output-mismatch", `${gen.name}: ${message}`, {
+    path,
+    hint: gen.output === "mesh" ? "Return ctx.mesh.finish(mesh) or mesh data ({ positions, normals?, indices? })." : gen.output === "texture" ? 'Return { width, height, mips: [rgba8 bytes] } (Float32Array levels for usage "hdr").' : gen.output === "entities" ? "Return an entity { name, components, children } or a list of them; components by name with JSON values." : `Return a value of ${gen.outputType}.`,
+    details
+  });
+}
+function round(n) {
+  return Math.round(n * 1e4) / 1e4;
+}
+function meshInfo(mesh) {
+  const b = mesh.bounds;
+  return {
+    vertices: mesh.vertexCount,
+    triangles: Math.floor(mesh.drawCount / 3),
+    bounds: {
+      min: [round(b[0]), round(b[1]), round(b[2])],
+      max: [round(b[3]), round(b[4]), round(b[5])]
+    }
+  };
+}
+function checkedMesh(gen, data, what) {
+  try {
+    return Mesh.create(data);
+  } catch (err) {
+    throw mismatch(gen, `${what}: ${err.message}`);
+  }
+}
+function encodeMeshOutput(gen, value) {
+  const result = value;
+  const main = result?.kind === "mesh" ? result.mesh : result;
+  if (!main || !(main.positions instanceof Float32Array)) {
+    throw mismatch(gen, `run returned ${describe(value)}, not a mesh`);
+  }
+  const mesh = checkedMesh(gen, main, "the mesh");
+  const lods = result?.kind === "mesh" ? result.lods ?? [] : [];
+  const info = meshInfo(mesh);
+  const out = [{ label: "", type: "Mesh", bytes: encodeMesh(mesh), info }];
+  const lodInfo = [];
+  lods.forEach((lod, i) => {
+    const m = checkedMesh(gen, lod, `LOD ${i + 1}`);
+    const li = meshInfo(m);
+    lodInfo.push(li.triangles);
+    out.push({ label: `LOD${i + 1}`, type: "Mesh", bytes: encodeMesh(m), info: li });
+  });
+  if (lodInfo.length) info.lods = lodInfo;
+  return out;
+}
+function encodeTextureOutput(gen, value) {
+  const tex = value;
+  const w = tex?.width;
+  const h = tex?.height;
+  if (!tex || !Number.isInteger(w) || !Number.isInteger(h) || !Array.isArray(tex.mips)) {
+    throw mismatch(gen, `run returned ${describe(value)}, not a texture`);
+  }
+  const usage = tex.usage ?? "color";
+  const hdr = usage === "hdr";
+  const level0 = tex.mips[0];
+  const expected = w * h * 4;
+  if (!level0 || level0.length !== expected || hdr !== level0 instanceof Float32Array) {
+    throw mismatch(
+      gen,
+      `level 0 must be ${expected} ${hdr ? "floats (Float32Array)" : "bytes (Uint8Array)"} for ${w}\xD7${h} RGBA`
+    );
+  }
+  const chain = tex.mipmaps && tex.mips.length === 1 ? buildMips(
+    { width: w, height: h, kind: hdr ? "f32" : "u8", data: level0 },
+    { usage, mipmaps: true, maxSize: 16384, flipY: false, premultiplyAlpha: false }
+  ) : { width: w, height: h, levels: tex.mips };
+  return [
+    {
+      label: "",
+      type: "Texture",
+      bytes: writeKtx2(chain, usage),
+      info: { width: w, height: h, usage, mips: chain.levels.length }
+    }
+  ];
+}
+function encodeDataOutput(gen, value) {
+  if (gen.dataType) {
+    const json = toJson(value);
+    const errors = gen.dataType.validate(json);
+    if (errors.length > 0) {
+      throw mismatch(
+        gen,
+        `the value isn't a ${gen.outputType}: ${errors[0].message}`,
+        errors[0].path,
+        errors
+      );
+    }
+    return [
+      {
+        label: "",
+        type: gen.outputType,
+        json: gen.dataType.serialize(gen.dataType.deserialize(json))
+      }
+    ];
+  }
+  if (value === void 0) throw mismatch(gen, "run returned undefined");
+  return [{ label: "", type: "Data", json: toJson(value) }];
+}
+function toJson(value) {
+  if (value === null || typeof value !== "object") {
+    return typeof value === "number" && !Number.isFinite(value) ? null : value;
+  }
+  if (ArrayBuffer.isView(value)) return Array.from(value);
+  if (Array.isArray(value)) return value.map(toJson);
+  const obj = value;
+  const keys = Object.keys(obj);
+  const isRef = typeof obj.type === "string" && (typeof obj.guid === "string" || typeof obj.path === "string") && keys.every((k) => k === "type" || k === "guid" || k === "path");
+  const out = {};
+  for (const k of keys) {
+    if (isRef && k === "type") continue;
+    if (obj[k] !== void 0) out[k] = toJson(obj[k]);
+  }
+  return out;
+}
+function describe(value) {
+  if (value === null) return "null";
+  if (Array.isArray(value)) return "an array";
+  if (ArrayBuffer.isView(value)) return `a ${value.constructor.name}`;
+  return typeof value === "object" ? "an object" : `a ${typeof value}`;
+}
+function encodeEntities(gen, value, check) {
+  const fragment = value;
+  const root = Array.isArray(fragment) ? { name: gen.name.slice(gen.name.indexOf("/") + 1), children: fragment } : fragment;
+  if (!isPlainObject(root)) throw mismatch(gen, `run returned ${describe(value)}, not entities`);
+  const errors = [];
+  const counts = {};
+  let entities = 0;
+  const walk = (e, at, isRoot) => {
+    const out = {};
+    if (typeof e.name !== "string" || e.name === "" || e.name.includes("/")) {
+      errors.push(
+        new ShardError(
+          "procgen/output-mismatch",
+          'Every entity needs a "name" (non-empty, no "/")',
+          {
+            path: `${at}/name`
+          }
+        )
+      );
+    }
+    out.name = e.name;
+    if (!isRoot) entities++;
+    if (e.components !== void 0) {
+      const comps = {};
+      for (const [name, raw] of Object.entries(e.components)) {
+        const where = pointer(`${at}/components`, name);
+        if (!check) {
+          comps[name] = toJson(raw);
+          counts[name] = (counts[name] ?? 0) + 1;
+          continue;
+        }
+        let def;
+        try {
+          def = findComponent(name);
+        } catch {
+          def = void 0;
+        }
+        if (!def?.serializable) {
+          errors.push(
+            new ShardError("procgen/output-mismatch", `Unknown component "${name}"`, {
+              path: where,
+              hint: 'Components are by registered name, e.g. "core/Transform" (see .agents/components.md).'
+            })
+          );
+          continue;
+        }
+        const json2 = toJson(raw);
+        for (const err of def.validate(json2)) {
+          errors.push(
+            new ShardError("procgen/output-mismatch", err.message, {
+              path: `${where}${err.path ?? ""}`,
+              hint: err.hint
+            })
+          );
+        }
+        comps[name] = json2;
+        counts[name] = (counts[name] ?? 0) + 1;
+      }
+      out.components = comps;
+    }
+    if (e.children !== void 0) {
+      if (!Array.isArray(e.children)) {
+        errors.push(
+          new ShardError("procgen/output-mismatch", '"children" must be a list', {
+            path: `${at}/children`
+          })
+        );
+      } else {
+        const names = /* @__PURE__ */ new Set();
+        out.children = e.children.map((c, i) => {
+          if (names.has(c?.name)) {
+            errors.push(
+              new ShardError("procgen/output-mismatch", `Two children are named "${c.name}"`, {
+                path: `${at}/children/${i}/name`,
+                hint: "Sibling names must be unique: they form the paths overrides and references use."
+              })
+            );
+          }
+          names.add(c?.name);
+          return walk(c, `${at}/children/${i}`, false);
+        });
+      }
+    }
+    return out;
+  };
+  const json = walk(root, "/root", true);
+  if (errors.length > 0) {
+    throw mismatch(gen, `${errors[0].path}: ${errors[0].message}`, errors[0].path, errors);
+  }
+  return [
+    {
+      label: "",
+      type: "Prefab",
+      json: { version: PREFAB_VERSION, root: json },
+      info: {
+        entities,
+        components: Object.fromEntries(
+          Object.entries(counts).sort(([a], [b]) => a.localeCompare(b))
+        )
+      }
+    }
+  ];
+}
+function encodeOutput(gen, value, options = {}) {
+  if (gen.output === "mesh") return encodeMeshOutput(gen, value);
+  if (gen.output === "texture") return encodeTextureOutput(gen, value);
+  if (gen.output === "entities") return encodeEntities(gen, value, options.checkComponents ?? true);
+  return encodeDataOutput(gen, value);
+}
+function checkFragment(gen, asset) {
+  const root = asset.json?.root;
+  encodeEntities(gen, root, true);
+}
+export {
+  checkFragment,
+  encodeOutput,
+  toJson
+};

package/dist/plugin.d.ts

@@ -0,0 +1,11 @@
+import type { KeyValueStorage, PlatformFileSystem } from '@aethervtt/shard-platform';
+export interface ProcgenPluginOptions {
+    /** Project files: outputs are cached in `.shard/cache/generated/` when writable. */
+    fs?: PlatformFileSystem;
+    /** Where outputs are cached on hosts that can't write files (IndexedDB in browsers). */
+    storage?: KeyValueStorage;
+    /** Bytes of outputs kept in memory (and in storage). Default 256 MB (`shard.json` procgen.cacheSize). */
+    cacheSize?: number;
+}
+/** Generator outputs for an app: the runtime and its caches, GeneratorInstance, procgen.* methods. */
+export declare function procgenPlugin(options?: ProcgenPluginOptions): import("@aethervtt/shard-runtime").Plugin;

package/dist/plugin.js

@@ -0,0 +1,30 @@
+import { assetServer } from "@aethervtt/shard-assets";
+import { definePlugin } from "@aethervtt/shard-runtime";
+import { updateInstances } from "@aethervtt/shard-scene";
+import * as importerModule from "./importer.js";
+import * as instanceModule from "./instance.js";
+import { procgenMethods } from "./methods.js";
+import * as refsModule from "./refs.js";
+import * as runtimeModule from "./runtime.js";
+import { procgen } from "./runtime.js";
+function procgenPlugin(options = {}) {
+  return definePlugin({
+    name: "procgen",
+    provides: [importerModule, instanceModule, refsModule, runtimeModule],
+    build(app) {
+      const runtime = procgen(app.world).configure({
+        ...options.fs ? { fs: options.fs } : {},
+        ...options.storage ? { storage: options.storage } : {},
+        ...options.cacheSize ? { cacheSize: options.cacheSize } : {}
+      });
+      runtime.server.onEvent((e) => {
+        if (e.kind !== "loaded" && e.kind !== "modified") return;
+        if (assetServer(app.world).entry(e.guid)?.type === "Generator") updateInstances(app.world);
+      });
+      app.addMethod(...procgenMethods);
+    }
+  });
+}
+export {
+  procgenPlugin
+};

package/dist/record.d.ts

@@ -0,0 +1,18 @@
+import type { GenRequest } from './generator.js';
+import type { GenOutputAsset } from './outputs.js';
+/** A generator run's output, as cached: content-addressed by its key. */
+export interface OutputRecord {
+    key: string;
+    assets: GenOutputAsset[];
+    children: GenRequest[];
+    warnings: {
+        message: string;
+        path?: string;
+    }[];
+    ms: number;
+    bytes: number;
+}
+export declare function recordBytes(assets: readonly GenOutputAsset[]): number;
+/** One file per record: magic, header length, JSON header, then each asset's bytes (4-aligned). */
+export declare function encodeRecord(record: OutputRecord): Uint8Array;
+export declare function decodeRecord(data: Uint8Array): OutputRecord | undefined;

package/dist/record.js

@@ -0,0 +1,66 @@
+const MAGIC = 1313294419;
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+function recordBytes(assets) {
+  let n = 256;
+  for (const a of assets) {
+    n += a.bytes?.byteLength ?? 0;
+    if (a.json !== void 0) n += JSON.stringify(a.json).length;
+  }
+  return n;
+}
+function encodeRecord(record) {
+  const blobs = [];
+  let offset = 0;
+  const assets = record.assets.map((a) => {
+    const { bytes, ...rest } = a;
+    if (!bytes) return rest;
+    const at2 = offset;
+    blobs.push(bytes);
+    offset += bytes.byteLength + 3 & ~3;
+    return { ...rest, bytes: [at2, bytes.byteLength] };
+  });
+  const header = encoder.encode(
+    JSON.stringify({
+      key: record.key,
+      assets,
+      children: record.children,
+      warnings: record.warnings,
+      ms: record.ms
+    })
+  );
+  const head = 8 + header.byteLength + 3 & ~3;
+  const out = new Uint8Array(head + offset);
+  const view = new DataView(out.buffer);
+  view.setUint32(0, MAGIC, true);
+  view.setUint32(4, header.byteLength, true);
+  out.set(header, 8);
+  let at = head;
+  for (const b of blobs) {
+    out.set(b, at);
+    at += b.byteLength + 3 & ~3;
+  }
+  return out;
+}
+function decodeRecord(data) {
+  if (data.byteLength < 8) return void 0;
+  const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+  if (view.getUint32(0, true) !== MAGIC) return void 0;
+  const length = view.getUint32(4, true);
+  const head = 8 + length + 3 & ~3;
+  const json = JSON.parse(decoder.decode(data.subarray(8, 8 + length)));
+  const assets = json.assets.map((a) => {
+    if (!a.bytes) return a;
+    const [at, n] = a.bytes;
+    return {
+      ...a,
+      bytes: new Uint8Array(data.buffer, data.byteOffset + head + at, n)
+    };
+  });
+  return { ...json, assets, bytes: recordBytes(assets) };
+}
+export {
+  decodeRecord,
+  encodeRecord,
+  recordBytes
+};

package/dist/refs.d.ts

@@ -0,0 +1,11 @@
+export declare const refParser: (spec: string) => import("./generator.js").GenRequest;
+/**
+ * `procedural:<generator>?params&seed=N` in scenes and prefabs: an output of a project or engine
+ * generator. Equal canonical params (and seed) are one asset.
+ */
+export declare const generatorSource: import("@aethervtt/shard-scene").ProceduralSource;
+/**
+ * The catalog knows generators by name (`star-explorer/Rock`, a `Generator` asset) and outputs by
+ * `procedural:` path, without anything imported.
+ */
+export declare const generatorResolver: import("@aethervtt/shard-assets").AssetResolver;

package/dist/refs.js

@@ -0,0 +1,42 @@
+import { defineAssetResolver } from "@aethervtt/shard-assets";
+import { defineProceduralSource } from "@aethervtt/shard-scene";
+import { findGenerator, parseProceduralRef } from "./generator.js";
+import { procgen, setRefParser } from "./runtime.js";
+const GENERATOR_NAME = /^[a-z][a-z0-9-]*\/[A-Za-z][A-Za-z0-9]*$/;
+function label(spec) {
+  const hash = spec.indexOf("#");
+  return hash === -1 ? "" : spec.slice(hash + 1);
+}
+const refParser = setRefParser((spec) => parseProceduralRef(spec));
+const generatorSource = defineProceduralSource({
+  handles: (name) => name.includes("/"),
+  check(spec, path) {
+    const request = parseProceduralRef(spec, path);
+    const sub = label(spec);
+    return { type: sub === "" ? findGenerator(request.generator).outputType : "Mesh" };
+  },
+  resolve(world, spec) {
+    const entry = procgen(world).entryFor(parseProceduralRef(spec), label(spec));
+    return { guid: entry.guid, path: entry.path, type: entry.type };
+  }
+});
+const generatorResolver = defineAssetResolver((server, ref) => {
+  if (ref.startsWith("procedural:")) {
+    const spec = ref.slice("procedural:".length);
+    const name2 = spec.split(/[?#]/)[0];
+    if (!findGenerator(name2)) return void 0;
+    try {
+      return procgen(server.world).entryFor(parseProceduralRef(spec), label(spec));
+    } catch {
+      return void 0;
+    }
+  }
+  const name = ref.startsWith("generator:") ? ref.slice("generator:".length) : ref;
+  if (!GENERATOR_NAME.test(name) || !findGenerator(name)) return void 0;
+  return server.virtual(`generator:${name}`, name, "Generator", () => ({ generator: name }));
+});
+export {
+  generatorResolver,
+  generatorSource,
+  refParser
+};

package/dist/runtime.d.ts

@@ -0,0 +1,159 @@
+import { type AssetEntry, type AssetServer, AssetStore } from '@aethervtt/shard-assets';
+import { type AssetRef, type Fields, type JsonValue, type World } from '@aethervtt/shard-core';
+import type { KeyValueStorage, PlatformFileSystem, Workers } from '@aethervtt/shard-platform';
+import { type Generator, type GenParams, type GenRequest, identityOf, type OutputAsset, type OutputSpec } from './generator.js';
+import { type GenJob, type GenResult } from './job.js';
+import { type OutputRecord } from './record.js';
+/**
+ * What a `Generator` asset names: a generator, plus defaults for its seed and params (a
+ * `*.gen.json` file's `#Generator` sub-asset). Generators by name (`star-explorer/Rock`) are
+ * `Generator` assets with just the name.
+ */
+export interface GeneratorBinding {
+    generator: string;
+    seed?: number;
+    params?: Record<string, JsonValue>;
+}
+export declare const GeneratorBindings: import("@aethervtt/shard-core").ResourceDef<AssetStore<GeneratorBinding, "Generator">>;
+export declare const GeneratorAssetType: import("@aethervtt/shard-assets").AssetTypeDef<GeneratorBinding>;
+/** Untyped `data` outputs, by guid. */
+export declare const GeneratedData: import("@aethervtt/shard-core").ResourceDef<AssetStore<JsonValue, "Data">>;
+export declare const DataAssetType: import("@aethervtt/shard-assets").AssetTypeDef<JsonValue>;
+export interface ProcgenHost {
+    /** The worker pool generators run on. */
+    workers?: Workers;
+    /**
+     * URL of a module exporting `runGeneratorJob` with every generator defined (the project bundle
+     * plus `@aethervtt/shard-procgen/worker`, built self-contained by the host). Without it, jobs run inline.
+     */
+    workerModule?: string | Promise<string>;
+}
+/** Tells procgen where jobs run (workers and the worker bundle). Hosts call it at startup. */
+export declare function configureProcgenHost(options: ProcgenHost): void;
+export declare function procgenHost(): Readonly<ProcgenHost>;
+/**
+ * Starts every worker and loads the worker module in it, so the first jobs don't pay for thread
+ * startup. Resolves once all are ready; hosts can call it without waiting.
+ */
+export declare function warmGeneratorWorkers(): Promise<void>;
+/**
+ * Milliseconds of main-thread procgen work so far (the steps `mainThreadTurn` ran): sample it
+ * each frame to see what generation costs the frame.
+ */
+export declare function procgenMainThreadMs(): number;
+/**
+ * Waits for a turn on the main thread: steps run back to back until about a quarter of a millisecond has
+ * passed, then yield to the event loop, so a burst of finished jobs spreads over frames.
+ */
+export declare function mainThreadTurn(): Promise<void>;
+/**
+ * Runs a job: on the worker pool when the host gave a worker module and every dependency travels
+ * as an artifact, otherwise inline (after an await, so callers never block on it). At most two
+ * jobs per worker are posted at once; the rest wait their turn.
+ */
+export declare function runJob(job: GenJob): Promise<GenResult & {
+    packed?: Uint8Array;
+}>;
+export interface ProcgenOptions {
+    /** Project files: records are written to `<cacheDir>/generated/` when it's writable. */
+    fs?: PlatformFileSystem;
+    /** Where records go on hosts that can't write files (browsers: IndexedDB). */
+    storage?: KeyValueStorage;
+    /** Default `.shard/cache`. */
+    cacheDir?: string;
+    /** Bytes of records kept in memory, and in `storage`. Default 256 MB. */
+    cacheSize?: number;
+}
+/** What happened to one output the last time it was made. */
+export type CacheResult = 'memory' | 'disk' | 'run' | 'shared';
+export interface ProcgenStats {
+    runs: number;
+    memoryHits: number;
+    diskHits: number;
+    failures: number;
+    /** Main-thread ms spent turning finished jobs into assets (bench: per frame). */
+    mainMs: number;
+}
+/** Handles in a request's params, in field order. */
+export declare function paramHandles(gen: Generator, params: JsonValue): {
+    guid?: string;
+    path?: string;
+}[];
+/**
+ * Generator outputs for one world: each request is a virtual asset `gen:<identity>` (with sub-assets
+ * like `gen:<identity>/LOD1`), made from a content-addressed record that's looked up in memory,
+ * then on disk (or platform storage), and generated on the pool when it's in neither.
+ */
+export declare class ProcgenRuntime {
+    readonly world: World;
+    readonly server: AssetServer;
+    options: Required<Pick<ProcgenOptions, 'cacheDir' | 'cacheSize'>> & ProcgenOptions;
+    readonly stats: ProcgenStats;
+    private readonly memory;
+    private memoryBytes;
+    private readonly stored;
+    private storedBytes;
+    private readonly inflight;
+    private readonly live;
+    private readonly requests;
+    /** Imported dependencies' artifacts and hashes, by guid (valid while the version matches). */
+    private readonly artifacts;
+    private pendingDeps;
+    constructor(world: World, options?: ProcgenOptions);
+    configure(options: ProcgenOptions): this;
+    /** The output entry of a request (label '' or a sub-asset like 'LOD1'), registered lazily. */
+    entryFor(request: GenRequest, label?: string): AssetEntry;
+    /** Generates (or finds cached) an output and resolves with its ref once it's loaded. */
+    generate<const P extends Fields, const O extends OutputSpec>(gen: Generator<P, O>, params?: GenParams<P>, seed?: number): Promise<AssetRef<OutputAsset<O>>>;
+    /** What the last make of an output did: cache result, key, and time. */
+    lastResult(guid: string): {
+        hit: CacheResult | undefined;
+        key: string | undefined;
+        ms: number;
+    } | undefined;
+    /** The cached record behind a loaded output, if it's still in memory. */
+    recordOf(guid: string): OutputRecord | undefined;
+    /** `produce`, with failures logged (a GeneratorInstance or scene waiting on it shows why). */
+    private produceLogged;
+    /** Makes the object of `gen:<identity>[/label]`: the virtual asset's `create`. */
+    private produce;
+    private requestOfPath;
+    /** The dependencies of a request: loaded, with their artifacts (for workers) and hashes. */
+    private dependencies;
+    private recordFor;
+    private remember;
+    private storedPath;
+    private readStored;
+    private writeStored;
+    private regenerate;
+    private regenerateForDeps;
+    /**
+     * Regenerates loaded outputs whose generator's code hash changed (or whose generator was
+     * redefined) and those whose dependencies reloaded. They swap in place when ready; returns their
+     * paths.
+     */
+    regenerateStale(): string[];
+    /** Jobs running now, for procgen.describe. */
+    inFlight(): {
+        generator: string;
+        key: string;
+        ms: number;
+    }[];
+    cacheStats(): {
+        records: number;
+        bytes: number;
+        limit: number;
+        live: number;
+    };
+}
+/** Set by the package index (keeps a module cycle out of the runtime). */
+export declare function setRefParser(parse: (spec: string) => GenRequest): (spec: string) => GenRequest;
+export declare const ProcgenResource: import("@aethervtt/shard-core").ResourceDef<ProcgenRuntime>;
+/** The world's procgen runtime, created (memory cache only) on first use. */
+export declare function procgen(world: World): ProcgenRuntime;
+/**
+ * Makes a generator's output at runtime (one planet per star) and resolves with a ref to it once
+ * it's loaded: cached in memory and on disk by the hash of its inputs, generated on the worker pool.
+ */
+export declare function generate<const P extends Fields, const O extends OutputSpec>(world: World, gen: Generator<P, O>, params?: GenParams<P>, seed?: number): Promise<AssetRef<OutputAsset<O>>>;
+export { identityOf };