@forgeax/engine-import 0.1.28 → 0.1.30

package/src/build-production.ts

@@ -109,16 +109,11 @@ function runtimePublicationFor(
     ...(input.digest === undefined ? {} : { digest: input.digest }),
     ...(input.inputFingerprint === undefined ? {} : { inputFingerprint: input.inputFingerprint }),
     ...(input.outputs === undefined ? {} : { outputs: input.outputs }),
+    ...(input.sourceKeys === undefined ? {} : { sourceKeys: input.sourceKeys }),
     ...(input.externalEvidence === undefined ? {} : { externalEvidence: input.externalEvidence }),
     generation: context.runtimeBinding?.generation ?? context.generation,
   };
-  if (input.sourceKeys === undefined) return createRuntimePackPublication(publicationInput);
-  const derived = createRuntimePackPublication(publicationInput);
-  const outputs = derived.publication.outputs.map((output) => ({
-    ...output,
-    sourceKey: input.sourceKeys?.get(output.guid.toLowerCase()) ?? output.sourceKey,
-  }));
-  return createRuntimePackPublication({ ...publicationInput, outputs });
+  return createRuntimePackPublication(publicationInput);
 }
 
 function sourceKeysFor(
@@ -622,6 +617,7 @@ async function emitAuthoredPack(
   };
   const runtimePublication = runtimePublicationFor(work.context, {
     pack: authoredPack,
+    sourceKeys: sourceKeysFor(legacy.assets),
     sourcePath: entry.sourcePath,
     sourceRevision: declaration.sourceRevision,
     packageUrl:

package/src/scriptable-pack-build.ts

@@ -9,6 +9,8 @@ import {
   PackageId,
   type PackBuildContextWithoutParameters,
   type PackBuildContextWithParameters,
+  type PackCookSource,
+  type PackOutputMap,
   type PackParameterDefinition,
   type PackParameterValue,
   resolvePackParameterValues,
@@ -18,6 +20,7 @@ import type {
   Asset,
   AssetGuid as AssetGuidType,
   AssetPublicationEnvelope,
+  AssetRef,
   ImportedAsset,
   Result,
 } from '@forgeax/engine-types';
@@ -37,6 +40,8 @@ import type {
   ScriptablePackSourceClosureEntry,
   ScriptablePackStagedOutput,
 } from './scriptable-pack.js';
+import { scriptablePackFingerprint as digest } from './scriptable-pack-fingerprint.js';
+import { materialAssetOutputProducer } from './scriptable-pack-output-producers.js';
 
 export interface ScriptablePackBuildOptions {
   readonly definition: AnyScriptablePackDefinition;
@@ -80,7 +85,7 @@ export type ScriptablePackBuildResult = Result<
 
 interface ObservedRead {
   readonly guid: string;
-  readonly asset: Asset;
+  readonly asset: Asset | PackCookSource;
   readonly generation: number;
   readonly digest: string;
 }
@@ -101,35 +106,17 @@ function clone<T>(value: T): T {
   return structuredClone(value);
 }
 
-function stable(value: unknown): string {
-  if (value instanceof Uint8Array) return JSON.stringify(Array.from(value));
-  if (Array.isArray(value)) return `[${value.map(stable).join(',')}]`;
-  if (value !== null && typeof value === 'object') {
-    const object = value as Record<string, unknown>;
-    return `{${Object.keys(object)
-      .sort()
-      .map((key) => `${JSON.stringify(key)}:${stable(object[key])}`)
-      .join(',')}}`;
-  }
-  return JSON.stringify(value) ?? 'null';
-}
-
-async function digest(value: unknown): Promise<string> {
-  const crypto = globalThis.crypto?.subtle;
-  if (crypto === undefined) throw new Error('Web Crypto API is required for Pack fingerprints');
-  const bytes = await crypto.digest('SHA-256', new TextEncoder().encode(stable(value)));
-  return `sha256:${Array.from(new Uint8Array(bytes), (byte) => byte.toString(16).padStart(2, '0')).join('')}`;
-}
-
 function observedReader(source: ScriptablePackAssetSnapshotSource | undefined): {
   readonly reader: {
-    readByGuid<TAsset extends Asset = Asset>(guid: AssetGuidType): Promise<Result<TAsset, unknown>>;
+    readByGuid<TAsset extends Asset | PackCookSource = Asset>(
+      guid: AssetGuidType,
+    ): Promise<Result<TAsset, unknown>>;
   };
   readonly reads: ReadonlyMap<string, ObservedRead>;
 } {
   const reads = new Map<string, ObservedRead>();
   const reader = {
-    async readByGuid<TAsset extends Asset = Asset>(
+    async readByGuid<TAsset extends Asset | PackCookSource = Asset>(
       guid: AssetGuidType,
     ): Promise<Result<TAsset, unknown>> {
       const key = AssetGuid.format(guid).toLowerCase();
@@ -218,7 +205,7 @@ function referenceError(
 function productError(
   sourcePath: string,
   sourceKey: string,
-  producer: AssetOutputProducer,
+  producer: Pick<AssetOutputProducer, 'kind'>,
   product: unknown,
 ): PackAuthoringError | undefined {
   if (!record(product)) {
@@ -338,9 +325,7 @@ export async function buildScriptablePack(
     });
   }
 
-  let built:
-    | Result<Record<string, Asset>, unknown>
-    | Promise<Result<Record<string, Asset>, unknown>>;
+  let built: Result<PackOutputMap, unknown> | Promise<Result<PackOutputMap, unknown>>;
   try {
     const context = buildContext(subjectPackageId, effectiveValues, observed.reader);
     built = options.definition.build(context as never);
@@ -392,14 +377,32 @@ export async function buildScriptablePack(
   const imported: ImportedAsset<unknown>[] = [];
   const stagedOutputs: ScriptablePackStagedOutput[] = [];
   const localGuids = new Set<string>();
-  const materialCookerRegistry = new NativeCookerRegistry();
-  for (const cooker of options.cookers ?? []) materialCookerRegistry.register(cooker);
+  const cookerRegistry = new NativeCookerRegistry();
+  for (const cooker of options.cookers ?? []) cookerRegistry.register(cooker);
   const nativeFingerprints = new Map<string, string>();
   for (const sourceKey of Object.keys(built.value).sort()) {
     if (!isValidPackSourceKey(sourceKey))
       return err(sourceKeyFailure(options.sourcePath, sourceKey));
     const asset = built.value[sourceKey];
-    if (!isAsset(asset)) {
+    const customSource = record(asset) && asset.execution === 'cooked';
+    if (
+      customSource &&
+      (typeof asset.kind !== 'string' ||
+        asset.kind.trim().length === 0 ||
+        isScriptablePackAssetKind(asset.kind) ||
+        !('source' in asset) ||
+        Object.keys(asset).some((key) => !['kind', 'execution', 'source'].includes(key)))
+    ) {
+      return err(
+        outputValueError(
+          options.sourcePath,
+          sourceKey,
+          'a custom kind with only execution: cooked and source; ordinary Assets use their standard producers',
+          asset,
+        ),
+      );
+    }
+    if (!customSource && !isAsset(asset)) {
       return err(
         outputValueError(
           options.sourcePath,
@@ -420,6 +423,47 @@ export async function buildScriptablePack(
       });
     }
     localGuids.add(normalizedGuid);
+    if (customSource) {
+      const cooked = await cookerRegistry.runDraft(asset.kind, {
+        guid,
+        sourceKey,
+        sourcePath: options.sourcePath,
+        source: asset.source,
+        refs: [],
+      });
+      if (!cooked.ok) return err(cooked.error);
+      if (cooked.value.guid.toLowerCase() !== normalizedGuid)
+        return err({
+          code: 'pack-source-output-invalid',
+          expected: 'the custom cooker to preserve the derived AssetGuid',
+          hint: 'remove cooker-owned identity generation and retain the host GUID',
+          detail: {
+            sourcePath: options.sourcePath,
+            sourceKey,
+            expectedGuid: guid,
+            actualGuid: cooked.value.guid,
+          },
+        });
+      const product = {
+        payload: cooked.value.payload,
+        refs: cooked.value.refs.map((guid) => ({ guid })),
+        artifacts: cooked.value.artifacts,
+      };
+      const invalid = productError(options.sourcePath, sourceKey, { kind: asset.kind }, product);
+      if (invalid !== undefined) return err(invalid);
+      imported.push({ guid, kind: asset.kind, ...product });
+      nativeFingerprints.set(sourceKey, cooked.value.inputFingerprint);
+      stagedOutputs.push({
+        guid: AssetGuid.derive(subjectPackageId, sourceKey),
+        sourceKey,
+        asset: clone(asset),
+        digest: await digest(asset),
+      });
+      continue;
+    }
+    // A custom source has been handled above; ordinary producers retain the closed Asset union.
+    if (!isAsset(asset))
+      return err(outputValueError(options.sourcePath, sourceKey, 'an ordinary Asset', asset));
     const producer = options.outputs.get(asset.kind);
     if (producer === undefined) {
       return err(
@@ -472,8 +516,9 @@ export async function buildScriptablePack(
     const product = produced.value as ImportAssetProduct<unknown>;
     let payload = product.payload;
     let artifacts = product.artifacts;
-    if (needsMaterialCook(asset) && materialCookerRegistry.get('material') !== undefined) {
-      const cooked = await materialCookerRegistry.runDraft('material', {
+    let refs = product.refs;
+    if (needsMaterialCook(asset) && cookerRegistry.get('material') !== undefined) {
+      const cooked = await cookerRegistry.runDraft('material', {
         guid,
         source: asset,
         sourceKey,
@@ -494,15 +539,64 @@ export async function buildScriptablePack(
           },
         });
       }
-      payload = cooked.value.payload;
-      artifacts = cooked.value.artifacts;
+      // Native cooking owns shader/program composition, but the Pack output
+      // producer owns the serialized wire shape. Re-project the cooked
+      // material once so texture/sampler/parent values become refsIndex
+      // entries again instead of leaking authored GUID strings into JSON.
+      const cookedAsset = cooked.value.payload;
+      if (!isAsset(cookedAsset) || cookedAsset.kind !== 'material') {
+        return err({
+          code: 'pack-source-output-invalid',
+          expected: 'the authored material cooker to return a material payload',
+          hint: 'repair the native material cooker payload and rebuild the Pack',
+          detail: { sourcePath: options.sourcePath, sourceKey },
+        });
+      }
+      const projected = await materialAssetOutputProducer.produce({
+        guid,
+        sourceKey,
+        asset: cookedAsset,
+      });
+      if (!projected.ok) return err(projected.error);
+      const projectedInvalid = productError(
+        options.sourcePath,
+        sourceKey,
+        materialAssetOutputProducer,
+        projected.value,
+      );
+      if (projectedInvalid !== undefined) return err(projectedInvalid);
+      payload = projected.value.payload;
+      artifacts = {
+        ...product.artifacts,
+        ...cooked.value.artifacts,
+        ...projected.value.artifacts,
+      };
+      const mergedRefs: AssetRef[] = [...projected.value.refs];
+      const seenRefs = new Set(mergedRefs.map((reference) => reference.guid.toLowerCase()));
+      for (const cookerRef of cooked.value.refs) {
+        const parsed = AssetGuid.parse(cookerRef);
+        if (!parsed.ok) {
+          return err({
+            code: 'pack-source-output-invalid',
+            expected: 'the authored material cooker refs to contain valid AssetGuid values',
+            hint: 'repair the native material cooker refs and rebuild the Pack',
+            detail: { sourcePath: options.sourcePath, sourceKey, ref: cookerRef },
+          });
+        }
+        const formatted = AssetGuid.format(parsed.value);
+        const normalizedRef = formatted.toLowerCase();
+        if (seenRefs.has(normalizedRef)) continue;
+        seenRefs.add(normalizedRef);
+        mergedRefs.push({ guid: formatted });
+      }
+      refs = mergedRefs;
       nativeFingerprints.set(sourceKey, cooked.value.inputFingerprint);
     }
     imported.push({
       guid,
       kind: asset.kind,
       payload,
-      refs: product.refs,
+      refs,
       artifacts,
     });
     stagedOutputs.push({

package/src/scriptable-pack-file-snapshot.ts

@@ -1,4 +1,3 @@
-import { createHash } from 'node:crypto';
 import { readdir, readFile, stat } from 'node:fs/promises';
 import { resolve } from 'node:path';
 import { isScriptablePackAssetKind, validatePack } from '@forgeax/engine-pack';
@@ -9,6 +8,7 @@ import type {
   ScriptablePackAssetSnapshot,
   ScriptablePackAssetSnapshotSource,
 } from './scriptable-pack.js';
+import { scriptablePackFingerprint as digest } from './scriptable-pack-fingerprint.js';
 
 export interface ScriptablePackFileAssetSnapshotSourceOptions {
   readonly assetRoots: readonly string[];
@@ -90,30 +90,6 @@ function isRecord(value: unknown): value is Record<string, unknown> {
   return value !== null && typeof value === 'object' && !Array.isArray(value);
 }
 
-function stable(value: unknown): string {
-  if (value instanceof ArrayBuffer) {
-    return `ArrayBuffer:${JSON.stringify(Array.from(new Uint8Array(value)))}`;
-  }
-  if (ArrayBuffer.isView(value)) {
-    return `${value.constructor.name}:${JSON.stringify(
-      Array.from(new Uint8Array(value.buffer, value.byteOffset, value.byteLength)),
-    )}`;
-  }
-  if (Array.isArray(value)) return `[${value.map(stable).join(',')}]`;
-  if (value !== null && typeof value === 'object') {
-    const record = value as Record<string, unknown>;
-    return `{${Object.keys(record)
-      .sort()
-      .map((key) => `${JSON.stringify(key)}:${stable(record[key])}`)
-      .join(',')}}`;
-  }
-  return JSON.stringify(value) ?? 'null';
-}
-
-function digest(value: unknown): string {
-  return `sha256:${createHash('sha256').update(stable(value)).digest('hex')}`;
-}
-
 function generationFromDigest(value: string): number {
   const parsed = Number.parseInt(value.slice('sha256:'.length, 'sha256:'.length + 8), 16);
   return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : 1;
@@ -302,9 +278,7 @@ async function indexPackFiles(
     }
   }
 
-  const sourceDigest = digest(
-    [...documents].sort((left, right) => stable(left).localeCompare(stable(right))),
-  );
+  const sourceDigest = digest(documents.map(digest).sort());
   return ok({
     generation: generationFromDigest(sourceDigest),
     assets: byGuid,

package/src/scriptable-pack-fingerprint.ts

@@ -0,0 +1,48 @@
+import { sha256 } from '@noble/hashes/sha2.js';
+import { bytesToHex } from '@noble/hashes/utils.js';
+
+/** One bounded fingerprint encoding for authored, staged and file-backed assets. */
+export function scriptablePackFingerprint(value: unknown): string {
+  const hash = sha256.create();
+  const encoder = new TextEncoder();
+  const text = (part: string) => {
+    hash.update(encoder.encode(part));
+  };
+  const binary = (type: string, bytes: Uint8Array) => {
+    // Raw NUL cannot occur in JSON tokens. The type and byte length delimit the
+    // binary lane without expanding every float into a sorted JSON property.
+    text(`\0${type}:${bytes.byteLength}:`);
+    hash.update(bytes);
+    text('\0');
+  };
+  const visit = (item: unknown): void => {
+    if (ArrayBuffer.isView(item)) {
+      binary(item.constructor.name, new Uint8Array(item.buffer, item.byteOffset, item.byteLength));
+    } else if (item instanceof ArrayBuffer) {
+      binary('ArrayBuffer', new Uint8Array(item));
+    } else if (Array.isArray(item)) {
+      text('[');
+      for (let i = 0; i < item.length; i++) {
+        if (i) text(',');
+        visit(item[i]);
+      }
+      text(']');
+    } else if (item !== null && typeof item === 'object') {
+      const object = item as Record<string, unknown>;
+      text('{');
+      let first = true;
+      for (const key of Object.keys(object).sort()) {
+        if (!first) text(',');
+        first = false;
+        text(`${JSON.stringify(key)}:`);
+        visit(object[key]);
+      }
+      text('}');
+    } else {
+      text(JSON.stringify(item) ?? 'null');
+    }
+  };
+  text('scriptable-pack-fingerprint/2:');
+  visit(value);
+  return `sha256:${bytesToHex(hash.digest())}`;
+}

package/src/scriptable-pack-output-producers.ts

@@ -466,9 +466,24 @@ function ordinaryPodProduct(input: AssetOutputInput): AssetOutputProduct {
     case 'particle-effect': {
       const effect = asset as ParticleEffectAsset;
       const cooked = particleProgramArtifact(effect);
+      const refs = new Set<string>();
+      for (const emitter of cooked.program.emitters) {
+        for (const renderer of emitter.renderers) {
+          if (!('material' in renderer) || typeof renderer.material !== 'string') {
+            throw new Error(`Particle emitter '${emitter.id}' requires a renderer Material GUID`);
+          }
+          refs.add(formatGuid(renderer.material));
+          if ('kind' in renderer && renderer.kind === 'mesh') {
+            if (!('mesh' in renderer) || typeof renderer.mesh !== 'string') {
+              throw new Error(`Particle emitter '${emitter.id}' requires a renderer Mesh GUID`);
+            }
+            refs.add(formatGuid(renderer.mesh));
+          }
+        }
+      }
       return {
         payload: { ...effect, programFingerprint: cooked.fingerprint, program: cooked.program },
-        refs: [],
+        refs: [...refs].sort().map((guid) => ({ guid })),
         artifacts: {
           'particle-effect/program.json': {
             mediaType: 'application/json',
@@ -504,6 +519,7 @@ export const materialAssetOutputProducer = createSafeProducer(
   'material-pack/2',
   materialProduct,
 );
+
 export const meshAssetOutputProducer = createSafeProducer('mesh', 'mesh-binary/4', meshProduct);
 export const textureAssetOutputProducer = createSafeProducer(
   'texture',
@@ -547,7 +563,13 @@ export function createStandardAssetOutputProducerRegistry(
     'particle-effect',
     'ies-profile',
   ] as const) {
-    registry.register(createSafeProducer(kind, 'ordinary-pod/1', ordinaryPodProduct));
+    registry.register(
+      createSafeProducer(
+        kind,
+        kind === 'particle-effect' ? 'particle-effect/2' : 'ordinary-pod/1',
+        ordinaryPodProduct,
+      ),
+    );
   }
   return registry;
 }

package/src/scriptable-pack-staged-snapshot.ts

@@ -1,6 +1,6 @@
 import { AssetGuid } from '@forgeax/engine-pack/guid';
 import type { PackAuthoringError } from '@forgeax/engine-pack/source';
-import type { Asset, AssetGuid as AssetGuidType, ImportError, Result } from '@forgeax/engine-types';
+import type { AssetGuid as AssetGuidType, ImportError, Result } from '@forgeax/engine-types';
 import { AssetError, err, ok } from '@forgeax/engine-types';
 import type {
   ScriptablePackAssetSnapshot,
@@ -8,6 +8,7 @@ import type {
   ScriptablePackDomainError,
   ScriptablePackStagedOutput,
 } from './scriptable-pack.js';
+import { scriptablePackFingerprint as assetDigest } from './scriptable-pack-fingerprint.js';
 
 export type ScriptablePackSnapshotError =
   | AssetError
@@ -29,27 +30,6 @@ export interface ScriptablePackStagedSnapshotOptions {
   readonly declaredExternalOutputs?: readonly ScriptablePackStagedOutput[];
 }
 
-function stable(value: unknown): string {
-  if (ArrayBuffer.isView(value)) {
-    return `${value.constructor.name}:${JSON.stringify(Array.from(new Uint8Array(value.buffer, value.byteOffset, value.byteLength)))}`;
-  }
-  if (Array.isArray(value)) return `[${value.map(stable).join(',')}]`;
-  if (value !== null && typeof value === 'object') {
-    const record = value as Record<string, unknown>;
-    return `{${Object.keys(record)
-      .sort()
-      .map((key) => `${JSON.stringify(key)}:${stable(record[key])}`)
-      .join(',')}}`;
-  }
-  return JSON.stringify(value) ?? 'null';
-}
-
-async function assetDigest(asset: Asset): Promise<string> {
-  const bytes = new TextEncoder().encode(stable(asset));
-  const digest = await globalThis.crypto.subtle.digest('SHA-256', bytes);
-  return `sha256:${Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, '0')).join('')}`;
-}
-
 function cycleError(
   stack: readonly string[],
   owner: string,

package/src/scriptable-pack.ts

@@ -2,6 +2,7 @@ import type { ScriptablePackAssetKind } from '@forgeax/engine-pack';
 import type {
   AssetReader,
   PackAuthoringError,
+  PackCookSource,
   ScriptablePackReadError,
 } from '@forgeax/engine-pack/source';
 import type {
@@ -22,7 +23,7 @@ import type { ImportAssetProduct, TerminalImportProduct } from './import-product
 export type { ScriptablePackSourceClosureEntry } from '@forgeax/engine-pack/source';
 
 export interface ScriptablePackAssetSnapshot {
-  readonly asset: Asset;
+  readonly asset: Asset | PackCookSource;
   readonly generation: number;
   readonly digest: string;
 }
@@ -30,7 +31,7 @@ export interface ScriptablePackAssetSnapshot {
 export interface ScriptablePackStagedOutput {
   readonly guid: AssetGuidType;
   readonly sourceKey: string;
-  readonly asset: Asset;
+  readonly asset: Asset | PackCookSource;
   readonly digest?: string;
 }
 
@@ -44,6 +45,8 @@ export interface AssetOutputInput {
   readonly guid: string;
   readonly sourceKey: string;
   readonly asset: Asset;
+  /** Filesystem-backed authoring host; absent for pure payload projections. */
+  readonly sourcePath?: string;
 }
 
 type MaterialPackPayload = Omit<MaterialAsset, 'parent' | 'values'> & {
@@ -91,7 +94,10 @@ export type AssetOutputPayloadByKind = {
 };
 
 export type AssetOutputPayload = AssetOutputPayloadByKind[ScriptablePackAssetKind];
-export type AssetOutputProduct = ImportAssetProduct<AssetOutputPayload>;
+export interface AssetOutputProduct extends ImportAssetProduct<AssetOutputPayload> {
+  readonly inputFingerprint?: string;
+  readonly sourceDependencies?: readonly string[];
+}
 
 export interface AssetOutputProducer {
   readonly kind: string;