@telorun/kernel 0.6.0 → 0.7.0

package/src/controller-loaders/napi-loader.ts

@@ -39,7 +39,66 @@ export class ControllerEnvMissingError extends Error {
  * crate path (not on the caller's baseUri) ensures two distinct paths that
  * point at the same crate share one cache entry.
  */
-const _napiModuleCache = new Map<string, ControllerInstance>();
+/**
+ * Holds the *raw* `require()`'d module object — i.e. the flat export bag
+ * returned by the napi addon — keyed by canonical crate path. The PURL
+ * fragment (`#entry`) projects out a sub-export at call time; caching the
+ * raw module here means two PURLs that differ only by fragment share one
+ * cargo build / one mmap of the dylib, instead of paying for either twice.
+ */
+const _napiModuleCache = new Map<string, any>();
+
+interface BuildAndLoadResult {
+  rawModule: any;
+  nodePath: string;
+}
+
+/**
+ * Single-flight dedupe for concurrent loads of the same crate. Without this,
+ * two callers (e.g. parallel test kernels) both miss the module cache, both
+ * run cargo + `fs.copyFile` over the same `.node` file, and the second
+ * `copyFile` overwrites a dylib Node has already mmapped — leading to a
+ * segfault when napi finalize callbacks run against torn pages. Keeping the
+ * in-flight promise here lets late arrivals await the in-progress load and
+ * read from the populated module cache when it resolves.
+ */
+const _napiInFlight = new Map<string, Promise<BuildAndLoadResult>>();
+
+/**
+ * @internal Slow-path entry counter — the regression test for concurrent
+ * loads asserts this stays at 1 across N parallel callers, proving the
+ * single-flight gate held. Production code never reads it.
+ */
+let _napiBuildAttempts = 0;
+export function __getNapiBuildAttempts(): number {
+  return _napiBuildAttempts;
+}
+export function __resetNapiLoaderForTest(): void {
+  _napiBuildAttempts = 0;
+  _napiModuleCache.clear();
+  _napiInFlight.clear();
+}
+
+/**
+ * Which branch of the resolver served this load.
+ *
+ * - `cache`      — process-lifetime in-memory hit on `_napiModuleCache`.
+ * - `local`      — `local_path` qualifier; cargo was invoked but the work
+ *                  is conceptually equivalent to "found source on disk and
+ *                  used it" (parallel to the npm `local_path` branch). The
+ *                  CLI silences these by default; the first cold build on
+ *                  a fresh checkout is silent for the same reason.
+ * - `cargo-build`— reserved for distribution-mode resolution (fetch from a
+ *                  registry + compile). Not produced today; the dispatcher
+ *                  errors with `ControllerEnvMissingError` for non-local
+ *                  PURLs (see [napi-loader.ts:62-66] in this file).
+ */
+export type NapiResolveSource = "cache" | "local" | "cargo-build";
+
+export interface NapiLoadResult {
+  instance: ControllerInstance;
+  source: NapiResolveSource;
+}
 
 export class NapiControllerLoader {
   /**
@@ -52,9 +111,19 @@ export class NapiControllerLoader {
    *
    * Distribution mode (no local_path): out of scope for the PoC; the hook is
    * left in place so the dispatcher reports env-missing and falls through.
+   *
+   * Fragment (`#entry`) is optional. When absent, the whole `require()`'d
+   * module is returned as the controller — the legacy single-export
+   * convention. When present, the fragment is treated as a property name on
+   * the loaded module: e.g. `pkg:cargo/foo?local_path=...#bar` returns
+   * `module.bar` as the controller. This mirrors the npm `#entry` semantics
+   * but indexes into the flat napi export bag instead of opening a different
+   * file. The convention is "one source file per controller, top-level
+   * export name matches the file" — files-as-controllers in spirit, even
+   * though all exports come from one linked dylib.
    */
-  async load(purl: string, baseUri: string): Promise<ControllerInstance> {
-    const [, , name, , qualifiers] = PackageURL.parseString(purl);
+  async load(purl: string, baseUri: string): Promise<NapiLoadResult> {
+    const [, , name, , qualifiers, entry] = PackageURL.parseString(purl);
     const localPath = (qualifiers as any)?.get("local_path");
 
     const isLocalManifest =
@@ -82,9 +151,44 @@ export class NapiControllerLoader {
     const cacheKey = canonicalCratePath;
     const cached = _napiModuleCache.get(cacheKey);
     if (cached) {
-      return cached;
+      return { instance: project(cached, entry, cratePath), source: "cache" };
+    }
+
+    // Concurrent callers for the same crate await one shared build. They
+    // report `local` (not `cache`): they paid the same wall-clock cost as
+    // the originator, just by sharing one cargo invocation rather than
+    // running their own. Reporting `cache` here would mislead metrics/event
+    // consumers into thinking it was a sub-ms hit.
+    const existingInFlight = _napiInFlight.get(cacheKey);
+    if (existingInFlight) {
+      const { rawModule } = await existingInFlight;
+      return { instance: project(rawModule, entry, cratePath), source: "local" };
     }
 
+    const buildPromise = this.buildAndLoad(cratePath, name ?? "", cacheKey);
+    _napiInFlight.set(cacheKey, buildPromise);
+    let rawModule: any;
+    let nodePath: string;
+    try {
+      ({ rawModule, nodePath } = await buildPromise);
+    } finally {
+      _napiInFlight.delete(cacheKey);
+    }
+    // `local` rather than `cargo-build` because the only mode currently
+    // wired up is `local_path` dev-mode — cargo's incremental cache means
+    // every run after the first is ~50ms of cargo-startup with no real
+    // compilation, conceptually the same as the npm `local_path` branch
+    // that just imports source already on disk. Distribution mode (when
+    // implemented) will return `cargo-build` from its own branch.
+    return { instance: project(rawModule, entry, nodePath), source: "local" };
+  }
+
+  private async buildAndLoad(
+    cratePath: string,
+    fallbackName: string,
+    cacheKey: string,
+  ): Promise<BuildAndLoadResult> {
+    _napiBuildAttempts++;
     try {
       await execFileAsync("rustc", ["--version"]);
     } catch {
@@ -109,7 +213,7 @@ export class NapiControllerLoader {
       );
     }
 
-    const { targetDir, libName } = await resolveCrateMetadata(cratePath, name ?? "");
+    const { targetDir, libName } = await resolveCrateMetadata(cratePath, fallbackName);
 
     const dylibPath = await findDylib(targetDir, libName);
     if (!dylibPath) {
@@ -122,9 +226,9 @@ export class NapiControllerLoader {
     const nodePath = path.join(path.dirname(dylibPath), `${libName}.node`);
     await fs.copyFile(dylibPath, nodePath);
 
-    let module: any;
+    let rawModule: any;
     try {
-      module = requireFromHere(nodePath);
+      rawModule = requireFromHere(nodePath);
     } catch (err: any) {
       throw new RuntimeError(
         "ERR_CONTROLLER_INVALID",
@@ -132,15 +236,44 @@ export class NapiControllerLoader {
       );
     }
 
-    if (!module || (!module.create && !module.register)) {
+    _napiModuleCache.set(cacheKey, rawModule);
+    return { rawModule, nodePath };
+  }
+}
+
+/**
+ * Pick the controller out of the raw napi module. With no fragment, the
+ * whole module *is* the controller (legacy single-export shape). With a
+ * fragment, look up `module[entry]` — convention: one source file per
+ * controller, top-level export name matches the file.
+ */
+function project(module: any, entry: string | undefined, where: string): ControllerInstance {
+  if (!module) {
+    throw new RuntimeError("ERR_CONTROLLER_INVALID", `napi module from ${where} is empty`);
+  }
+  if (!entry) {
+    if (!module.create && !module.register) {
       throw new RuntimeError(
         "ERR_CONTROLLER_INVALID",
-        `pkg:cargo controller at ${nodePath} exports neither create nor register`,
+        `pkg:cargo controller at ${where} exports neither create nor register`,
       );
     }
-    _napiModuleCache.set(cacheKey, module);
     return module;
   }
+  const sub = module[entry];
+  if (!sub) {
+    throw new RuntimeError(
+      "ERR_CONTROLLER_INVALID",
+      `pkg:cargo controller at ${where}#${entry}: module has no export named "${entry}"`,
+    );
+  }
+  if (!sub.create && !sub.register) {
+    throw new RuntimeError(
+      "ERR_CONTROLLER_INVALID",
+      `pkg:cargo controller at ${where}#${entry} exports neither create nor register`,
+    );
+  }
+  return sub;
 }
 
 async function resolveCrateMetadata(

package/src/controller-loaders/npm-loader.ts

@@ -14,13 +14,26 @@ const cacheRoot = process.env.TELO_CACHE_DIR
 const npmCacheRoot = path.join(cacheRoot, "npm");
 const isBun = typeof (globalThis as any).Bun !== "undefined";
 
+/**
+ * Tells the dispatcher (and any UI consumer downstream) which branch the
+ * resolver actually took. `npm-install` is the only one that hits the network;
+ * the rest are sub-second cache/local lookups, so the CLI uses this to decide
+ * whether a "downloading…" line was honest or should be silently dropped.
+ */
+export type NpmResolveSource = "local" | "node_modules" | "npm-install" | "cache";
+
+export interface NpmLoadResult {
+  instance: ControllerInstance;
+  source: NpmResolveSource;
+}
+
 export class NpmControllerLoader {
   /**
    * Resolve a `pkg:npm/...` PURL to a controller module instance. Tries, in order:
    * a relative `local_path` qualifier, the workspace's `node_modules`, and finally
    * an isolated install under `~/.cache/telo/npm/<hash>`.
    */
-  async load(purl: string, baseUri: string): Promise<ControllerInstance> {
+  async load(purl: string, baseUri: string): Promise<NpmLoadResult> {
     const [, namespace, name, versionSpec, qualifiers, entry] = PackageURL.parseString(purl);
 
     const localPath = (qualifiers as any)?.get("local_path");
@@ -28,6 +41,7 @@ export class NpmControllerLoader {
     const installDir = path.join(npmCacheRoot, cacheKey);
 
     let packageRoot: string;
+    let source: NpmResolveSource;
     const isLocalManifest =
       baseUri && !baseUri.startsWith("http://") && !baseUri.startsWith("https://");
     if (localPath && isLocalManifest) {
@@ -36,12 +50,14 @@ export class NpmControllerLoader {
       const resolvedLocalPath = path.resolve(manifestDir, localPath);
       if (await this.pathExists(resolvedLocalPath)) {
         packageRoot = resolvedLocalPath;
+        source = "local";
       } else {
         const nodeModulesPath = await this.findInNodeModules(`${namespace}/${name}`);
         if (nodeModulesPath) {
           packageRoot = nodeModulesPath;
+          source = "node_modules";
         } else {
-          await this.ensureNpmPackageInstalled(installDir, `${namespace}/${name}@${versionSpec}`);
+          source = await this.ensureNpmPackageInstalled(installDir, `${namespace}/${name}@${versionSpec}`);
           packageRoot = this.getInstalledPackageRoot(installDir, `${namespace}/${name}`);
         }
       }
@@ -49,8 +65,9 @@ export class NpmControllerLoader {
       const nodeModulesPath = await this.findInNodeModules(`${namespace}/${name}`);
       if (nodeModulesPath) {
         packageRoot = nodeModulesPath;
+        source = "node_modules";
       } else {
-        await this.ensureNpmPackageInstalled(installDir, `${namespace}/${name}@${versionSpec}`);
+        source = await this.ensureNpmPackageInstalled(installDir, `${namespace}/${name}@${versionSpec}`);
         packageRoot = this.getInstalledPackageRoot(installDir, `${namespace}/${name}`);
       }
     }
@@ -62,10 +79,13 @@ export class NpmControllerLoader {
         `Invalid controller loaded from "${purl}": missing create or register function`,
       );
     }
-    return instance;
+    return { instance, source };
   }
 
-  private async ensureNpmPackageInstalled(installDir: string, packageSpec: string): Promise<void> {
+  private async ensureNpmPackageInstalled(
+    installDir: string,
+    packageSpec: string,
+  ): Promise<"cache" | "npm-install"> {
     const packageName = this.getPackageName(
       packageSpec.startsWith(".") || path.isAbsolute(packageSpec)
         ? await this.getLocalPackageName(packageSpec)
@@ -74,7 +94,7 @@ export class NpmControllerLoader {
     const packageRoot = this.getInstalledPackageRoot(installDir, packageName);
     const packageJsonPath = path.join(packageRoot, "package.json");
     if (await this.pathExists(packageJsonPath)) {
-      return;
+      return "cache";
     }
 
     await fs.mkdir(installDir, { recursive: true });
@@ -98,6 +118,7 @@ export class NpmControllerLoader {
     ];
 
     await execFileAsync("npm", args);
+    return "npm-install";
   }
 
   private getPackageName(packageSpec: string): string {

package/src/controllers/resource-definition/resource-definition-controller.ts

@@ -27,10 +27,7 @@ type ResourceDefinitionResource = RuntimeResource & {
 class ResourceDefinition implements ResourceInstance {
   readonly kind: "ResourceDefinition" = "ResourceDefinition";
 
-  constructor(
-    readonly resource: ResourceDefinitionResource,
-    private controllerLoader: ControllerLoader,
-  ) {}
+  constructor(readonly resource: ResourceDefinitionResource) {}
 
   async init(ctx: ResourceContext) {
     if (!this.resource.controllers?.length) {
@@ -43,24 +40,25 @@ class ResourceDefinition implements ResourceInstance {
       );
       return;
     }
-    ctx.emit("ControllerLoading", { controllers: this.resource.controllers });
-    try {
-      const controllerInstance = await this.controllerLoader.load(
-        this.resource.controllers,
-        this.resource.metadata.source,
-        ctx.getControllerPolicy(),
-      );
-      ctx.emit("ControllerLoaded", { schema: controllerInstance.schema });
-      ctx.registerDefinition(this.resource);
-      await ctx.registerController(
-        this.resource.metadata.module,
-        this.resource.metadata.name,
-        controllerInstance,
-      );
-    } catch (err) {
-      ctx.emit("ControllerLoadFailed", { error: (err as Error).message });
-      throw err;
-    }
+    // The loader owns ControllerLoading / ControllerLoaded / ControllerLoadFailed
+    // emission so it can fire one event per attempted candidate (env-missing
+    // fallback chains), and so the payload can include the actually-picked PURL,
+    // which branch resolved it (`source`), and timing — none of which are known
+    // here at the call site.
+    const loader = new ControllerLoader({
+      emit: (e) => ctx.emit(e.name, e.payload),
+    });
+    const controllerInstance = await loader.load(
+      this.resource.controllers,
+      this.resource.metadata.source,
+      ctx.getControllerPolicy(),
+    );
+    ctx.registerDefinition(this.resource);
+    await ctx.registerController(
+      this.resource.metadata.module,
+      this.resource.metadata.name,
+      controllerInstance,
+    );
   }
 }
 
@@ -78,7 +76,7 @@ export async function create(resource: any, ctx: ResourceContext): Promise<Resou
 
   // Return a fully-formed ResourceDefinition instance
   const definition = resource as unknown as ResourceDefinitionResource;
-  return new ResourceDefinition(definition, new ControllerLoader());
+  return new ResourceDefinition(definition);
 }
 
 export const schema = {

package/src/evaluation-context.ts

@@ -18,6 +18,52 @@ import { RuntimeError } from "@telorun/sdk";
 
 export { resourceKey };
 
+type Walker = (ctx: Record<string, unknown>) => unknown;
+
+/** Compile a manifest subtree into a tightly-bound walker closure. The returned
+ *  function takes an activation object and rebuilds a fresh container of the
+ *  same shape with all `${{ }}` CompiledValues evaluated against the activation.
+ *  Per-call overhead is one closure invocation per node — no isCompiledValue /
+ *  Array.isArray / typeof / Object.entries checks at runtime. */
+function compileWalker(value: unknown): Walker {
+  if (isCompiledValue(value)) {
+    const compiled = value;
+    return (ctx) => {
+      try {
+        return compiled.call(ctx);
+      } catch (error) {
+        const expr = compiled.source ? `\${{ ${compiled.source} }}` : "unknown expression";
+        const msg = error instanceof Error ? error.message : String(error);
+        throw new Error(`Expression ${expr} failed: ${msg}`);
+      }
+    };
+  }
+  if (Array.isArray(value)) {
+    const childWalkers = value.map(compileWalker);
+    const n = childWalkers.length;
+    return (ctx) => {
+      const out = new Array(n);
+      for (let i = 0; i < n; i++) out[i] = childWalkers[i]!(ctx);
+      return out;
+    };
+  }
+  if (value !== null && typeof value === "object") {
+    const entries = Object.entries(value as Record<string, unknown>).map(
+      ([k, v]) => [k, compileWalker(v)] as const,
+    );
+    const n = entries.length;
+    return (ctx) => {
+      const out: Record<string, unknown> = {};
+      for (let i = 0; i < n; i++) {
+        const [k, fn] = entries[i]!;
+        out[k] = fn(ctx);
+      }
+      return out;
+    };
+  }
+  return () => value;
+}
+
 /**
  * Base class for all evaluation contexts. Owns template
  * expansion, secrets redaction, and the generic resource lifecycle tree.
@@ -453,7 +499,9 @@ export class EvaluationContext implements IEvaluationContext {
         if (declaredCodes && !declaredCodes.has(err.code)) {
           await this.emit(`${kind}.${name}.InvokeRejected.Undeclared`, payload);
         }
-      } else if (err instanceof Error) {
+        throw err;
+      }
+      if (err instanceof Error) {
         await this.emit(`${kind}.${name}.InvokeFailed`, {
           name: err.name,
           message: err.message,
@@ -464,7 +512,22 @@ export class EvaluationContext implements IEvaluationContext {
           message: String(err),
         });
       }
-      throw err;
+      // Already enriched at an inner invoke: keep the innermost (most
+      // specific) resource as the failure location.
+      if (err instanceof RuntimeError && err.diagnostics?.length) throw err;
+      const message = err instanceof Error ? err.message : String(err);
+      const code = err instanceof Error ? err.name : undefined;
+      // Keep `message` raw so callers (Run.Sequence catch blocks, assertions)
+      // see the original error text unchanged. Resource context lives only on
+      // the attached diagnostic, which the CLI's formatter renders as the
+      // location prefix. Attach the original error as `cause` so
+      // formatErrorForDiagnostic walks the chain and surfaces the underlying
+      // stack and well-known error fields (AWS, pg, Node system errors).
+      const wrapped = new RuntimeError("ERR_EXECUTION_FAILED", message, [
+        { kind, resource: name, message, code },
+      ]);
+      (wrapped as { cause?: unknown }).cause = err;
+      throw wrapped;
     }
   }
 
@@ -497,48 +560,62 @@ export class EvaluationContext implements IEvaluationContext {
 
   /**
    * Expand a value that may contain precompiled ${{ }} templates.
-   * Works recursively over CompiledValues, arrays, and objects.
+   *
+   * Hot path: each unique manifest subtree is compiled once into a tightly-bound
+   * walker closure (no per-call `isCompiledValue` / `Array.isArray` / `typeof` /
+   * `Object.entries` overhead, no recursive method dispatch). The walker tree is
+   * cached by the input value's identity in `walkerCache`, so subsequent calls
+   * with the same manifest data reuse it. The walker reads from `this._context`
+   * — which `expandWith` mutates in place — and emits a fresh container per call
+   * to preserve the original recursive `expand`'s semantics.
    */
   expand(value: unknown): unknown {
-    if (isCompiledValue(value)) {
-      try {
-        return value.call(this._context);
-      } catch (error) {
-        const expr = value.source ? `\${{ ${value.source} }}` : "unknown expression";
-        const msg = error instanceof Error ? error.message : String(error);
-        throw new Error(`Expression ${expr} failed: ${msg}`);
-      }
-    }
-    if (Array.isArray(value)) {
-      return value.map((entry) => this.expand(entry));
-    }
-    if (value !== null && typeof value === "object") {
-      const resolved: Record<string, unknown> = {};
-      for (const [key, entry] of Object.entries(value as Record<string, unknown>)) {
-        resolved[key] = this.expand(entry);
-      }
-      return resolved;
-    }
-    return value;
+    if (value === null || typeof value !== "object") return value;
+    const cached = this.walkerCache.get(value as object);
+    if (cached) return cached(this._context);
+    const walker = compileWalker(value);
+    this.walkerCache.set(value as object, walker);
+    return walker(this._context);
   }
 
   /**
    * Expand a value using this context merged with additional properties.
-   * Equivalent to merge(extraContext).expand(value) without allocating a context object.
+   *
+   * Hot path optimisation: rather than allocate a fresh prototype-less object
+   * per call (one allocation + N property copies for N keys in the saved
+   * context), we mutate `_context` in place — adding or overwriting only the
+   * `extraContext` keys — and restore the previous values on exit. Safe because
+   * `expand` is synchronous; cel-vm closures only read from the activation.
    */
   expandWith(value: unknown, extraContext: Record<string, unknown>): unknown {
-    const saved = this._context;
-    this._context = Object.assign(Object.create(null), saved, extraContext) as Record<
-      string,
-      unknown
-    >;
+    const ctx = this._context as Record<string, unknown>;
+    const keys = Object.keys(extraContext);
+    const savedValues: unknown[] = new Array(keys.length);
+    const hadKey: boolean[] = new Array(keys.length);
+    for (let i = 0; i < keys.length; i++) {
+      const k = keys[i]!;
+      hadKey[i] = k in ctx;
+      if (hadKey[i]) savedValues[i] = ctx[k];
+      ctx[k] = extraContext[k];
+    }
     try {
       return this.expand(value);
     } finally {
-      this._context = saved;
+      for (let i = 0; i < keys.length; i++) {
+        const k = keys[i]!;
+        if (hadKey[i]) ctx[k] = savedValues[i];
+        else delete ctx[k];
+      }
     }
   }
 
+  /** Cache of compiled walker closures keyed on the manifest subtree they walk.
+   *  WeakMap so entries are GC'd if the manifest is reloaded. */
+  private readonly walkerCache = new WeakMap<
+    object,
+    (ctx: Record<string, unknown>) => unknown
+  >();
+
 /**
    * Expand specific dot-paths within an object. '**' expands the entire object.
    * Paths listed in excludePaths are left untouched (runtime takes precedence).

package/src/kernel.ts

@@ -766,6 +766,27 @@ function injectAtPath(
   function traverse(obj: unknown, partsLeft: string[]): void {
     if (!obj || typeof obj !== "object" || partsLeft.length === 0) return;
     const [head, ...rest] = partsLeft;
+
+    // Map iteration: descend into every value of the current object (used for
+    // schema fields with `additionalProperties` like `content[mime]`).
+    if (head === "{}") {
+      const container = obj as Record<string, unknown>;
+      for (const mapKey of Object.keys(container)) {
+        const elem = container[mapKey];
+        if (!elem || typeof elem !== "object") continue;
+        if (rest.length === 0) {
+          const ref = elem as Record<string, unknown>;
+          if (typeof ref.kind === "string" && typeof ref.name === "string") {
+            const instance = getInstance(ref.name);
+            if (instance) container[mapKey] = instance;
+          }
+        } else {
+          traverse(elem, rest);
+        }
+      }
+      return;
+    }
+
     const isArr = head.endsWith("[]");
     const key = isArr ? head.slice(0, -2) : head;
     const container = obj as Record<string, unknown>;