@telorun/kernel 0.12.0 → 0.13.0

package/src/manifest-sources/analysis-stamp.ts

@@ -0,0 +1,169 @@
+import type { LoadedGraph } from "@telorun/analyzer";
+import { createHash } from "crypto";
+import { readFileSync } from "fs";
+import * as fs from "fs/promises";
+import { createRequire } from "module";
+import * as path from "path";
+import { fileURLToPath } from "url";
+
+/**
+ * Hash-keyed analysis cache: a tiny JSON sidecar in `.telo/manifests/`
+ * recording that an exact set of manifest bytes — under specific
+ * `@telorun/kernel` and `@telorun/analyzer` package versions — passed
+ * `analyzer.analyzeErrors`. The next `kernel.load` reads the sidecar
+ * and, if signatures match, skips the per-resource validation walk.
+ *
+ * Lives next to the manifest cache (`LocalManifestCacheSource`) but is
+ * independent of it — splitting both for grep-ability and because the
+ * concerns (URL → file content vs. content → analyzer verdict) are
+ * orthogonal.
+ */
+
+const CACHE_SUBDIR = ".telo/manifests";
+
+/** File-format version of the analysis stamp envelope. Only bumped when
+ *  the on-disk *layout* changes (new fields, restructured payload). The
+ *  *semantic* invalidation — "did the analyzer's logic change?" — is
+ *  handled by baking the resolved `@telorun/analyzer` / `@telorun/kernel`
+ *  package versions into the signature itself, so any pnpm/npm install
+ *  that bumps either package automatically invalidates every stamp on
+ *  disk. A hand-maintained integer for that purpose would silently mask
+ *  newly-stricter validation until the next manifest edit. */
+const ANALYSIS_STAMP_FORMAT_VERSION = 1;
+const ANALYSIS_STAMP_FILE = `${CACHE_SUBDIR}/.validated.json`;
+
+const localRequire = createRequire(import.meta.url);
+
+/** Read the kernel's own `package.json` — `createRequire` can't resolve
+ *  `@telorun/kernel/package.json` from inside the kernel package itself
+ *  (the self-reference loops in some node_modules layouts). The file
+ *  sits two levels up from `dist/manifest-sources/`. */
+function readKernelVersion(): string {
+  try {
+    const url = new URL("../../package.json", import.meta.url);
+    const pkg = JSON.parse(readFileSync(fileURLToPath(url), "utf-8"));
+    return typeof pkg.version === "string" ? pkg.version : "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
+function readDepVersion(spec: string): string {
+  // Fast path: direct `require("<pkg>/package.json")`. Fails (with
+  // ERR_PACKAGE_PATH_NOT_EXPORTED) when the dependency declares a strict
+  // `exports` map without listing `./package.json` — common for packages
+  // that consider package.json an implementation detail. Don't return
+  // "unknown" in that case; fall back to resolving the package's main
+  // entry and walking the filesystem up to its package.json.
+  const pkgJsonSpec = spec.endsWith("/package.json")
+    ? spec
+    : `${spec}/package.json`;
+  try {
+    const pkg = localRequire(pkgJsonSpec);
+    if (typeof pkg.version === "string") return pkg.version;
+  } catch {
+    // fall through to filesystem walk
+  }
+  try {
+    const mainSpec = spec.endsWith("/package.json") ? spec.slice(0, -13) : spec;
+    const entry = localRequire.resolve(mainSpec);
+    let dir = path.dirname(entry);
+    while (dir !== path.dirname(dir)) {
+      const candidate = path.join(dir, "package.json");
+      try {
+        const pkg = JSON.parse(readFileSync(candidate, "utf-8"));
+        // Guard against scoped-package interior package.json files (some
+        // packages stamp one in dist/) — match by name when the spec
+        // names a package.
+        const expectedName = mainSpec
+          .split("/")
+          .slice(0, mainSpec.startsWith("@") ? 2 : 1)
+          .join("/");
+        if (typeof pkg.name === "string" && pkg.name === expectedName) {
+          return typeof pkg.version === "string" ? pkg.version : "unknown";
+        }
+      } catch {
+        // not at the package root yet — keep walking
+      }
+      dir = path.dirname(dir);
+    }
+  } catch {
+    // resolution failed — package not installed at all
+  }
+  return "unknown";
+}
+
+const KERNEL_VERSION = readKernelVersion();
+const ANALYZER_VERSION = readDepVersion("@telorun/analyzer");
+
+export interface AnalysisStamp {
+  version: number;
+  signature: string;
+}
+
+/** Hash every owner + partial file in `graph` together with the resolved
+ *  `@telorun/kernel` and `@telorun/analyzer` versions into one content
+ *  signature. Two loads of the same manifest set under the same package
+ *  versions produce the same signature; any edit to any reachable file —
+ *  or any pnpm/npm install that bumps the kernel or analyzer — flips it.
+ *  This is what the kernel uses to decide whether the previous analyzer
+ *  run's verdict still applies. */
+export function computeAnalysisSignature(graph: LoadedGraph): string {
+  const entries: Array<[string, string]> = [];
+  for (const [, mod] of graph.modules) {
+    for (const file of [mod.owner, ...mod.partials]) {
+      const digest = createHash("sha256").update(file.text).digest("hex");
+      entries.push([file.source, digest]);
+    }
+  }
+  entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  return createHash("sha256")
+    .update(
+      JSON.stringify({
+        kernel: KERNEL_VERSION,
+        analyzer: ANALYZER_VERSION,
+        files: entries,
+      }),
+    )
+    .digest("hex");
+}
+
+/** Read the stamped analysis verdict for the entry at `entryDir`, or
+ *  `undefined` when missing / unreadable / format-mismatched. The
+ *  `version` field is the on-disk *format* version; semantic
+ *  invalidation flows through the signature (which embeds package
+ *  versions). A future format change bumps `version` so older kernels
+ *  reading a newer stamp (or vice versa) discard rather than misparse. */
+export async function readAnalysisStamp(
+  entryDir: string,
+): Promise<AnalysisStamp | undefined> {
+  try {
+    const text = await fs.readFile(path.join(entryDir, ANALYSIS_STAMP_FILE), "utf-8");
+    const parsed = JSON.parse(text) as Partial<AnalysisStamp>;
+    if (
+      parsed?.version === ANALYSIS_STAMP_FORMAT_VERSION &&
+      typeof parsed?.signature === "string"
+    ) {
+      return parsed as AnalysisStamp;
+    }
+  } catch {
+    // missing / unreadable / unparseable — treat as cache miss
+  }
+  return undefined;
+}
+
+/** Persist the analysis verdict so the next `kernel.load` can skip the
+ *  per-resource validation walk when the manifest set is unchanged.
+ *  Idempotent; safe to call after every successful load. */
+export async function writeAnalysisStamp(
+  entryDir: string,
+  signature: string,
+): Promise<void> {
+  const stamp: AnalysisStamp = {
+    version: ANALYSIS_STAMP_FORMAT_VERSION,
+    signature,
+  };
+  const target = path.join(entryDir, ANALYSIS_STAMP_FILE);
+  await fs.mkdir(path.dirname(target), { recursive: true });
+  await fs.writeFile(target, JSON.stringify(stamp), "utf-8");
+}

package/src/resource-context.ts

@@ -13,6 +13,7 @@ import {
   type ParsedArgs,
   type TypeRule,
 } from "@telorun/sdk";
+import { isRefSentinel } from "@telorun/templating";
 import AjvModule from "ajv";
 import addFormats from "ajv-formats";
 import { EvaluationContext } from "./evaluation-context.js";
@@ -176,6 +177,14 @@ export class ResourceContextImpl implements ResourceContext {
     return this.kernel.loadManifests(url);
   }
 
+  isImportValidatedAtLoad(url: string): boolean {
+    return this.kernel.isImportValidatedAtLoad(url);
+  }
+
+  resolveImportUrl(fromSource: string, importSource: string): string {
+    return this.kernel.resolveImportUrl(fromSource, importSource);
+  }
+
   /**
    * Resolves a resource into a normalized {kind, name} reference.
    * If the resource contains a definition (kind + properties), registers it as a manifest.
@@ -194,6 +203,31 @@ export class ResourceContextImpl implements ResourceContext {
       );
     }
 
+    // Stopgap: `!ref <name>` sentinels can reach the controller directly
+    // when the slot is hidden behind a local `$ref: "#/$defs/..."` — the
+    // analyzer's field-map walker descends `oneOf`/`anyOf` variant
+    // properties but intentionally early-returns on `$ref` (see
+    // `analyzer/nodejs/src/reference-field-map.ts`). Enabling the `$ref`
+    // descent regresses the kernel's `<Kind>.<Name>.Invoked` event
+    // emission for kinds (notably `Run.Sequence`) whose controllers
+    // call `instance.invoke()` directly on Phase-5-injected instances;
+    // the walker fix needs to land together with routing those callers
+    // through `EvaluationContext.invokeResolved`. Until then, the Node
+    // kernel resolves the sentinel here. Polyglot controllers don't get
+    // this rescue — schemas exercising those hidden slots must use the
+    // legacy string or `{kind, name}` forms for now.
+    if (isRefSentinel(resource)) {
+      const refName = resource.source;
+      const entry = this.moduleContext.resourceInstances.get(refName);
+      if (!entry) {
+        throw new RuntimeError(
+          "ERR_RESOURCE_NOT_FOUND",
+          `[${this.metadata.name}] !ref '${refName}' did not resolve to a registered resource.`,
+        );
+      }
+      return { kind: entry.resource.kind as string, name: refName };
+    }
+
     if (!resource.kind) {
       throw new RuntimeError(
         "ERR_INVALID_VALUE",

package/src/schema-validator.ts

@@ -1,22 +1,104 @@
 import { evaluate } from "@marcbachmann/cel-js";
 import { DataValidator, RuntimeError, TypeRule } from "@telorun/sdk";
-import AjvModule from "ajv";
+import AjvModule, { type ValidateFunction } from "ajv";
+import standaloneCodeMod from "ajv/dist/standalone/index.js";
 import addFormats from "ajv-formats";
+import { createHash } from "node:crypto";
+import * as fs from "node:fs";
+import { createRequire } from "node:module";
+import * as path from "node:path";
 import { formatAjvErrors } from "./manifest-schemas.js";
+import { ManifestRootSchema } from "@telorun/templating";
 
 const Ajv = AjvModule.default ?? AjvModule;
+// AJV's standalone subpath is CJS — the default export shows up as either
+// the function itself or `.default` depending on how the bundler/loader
+// rewrites it. Normalise once.
+const standaloneCode: (...args: any[]) => string =
+  (standaloneCodeMod as any).default ?? (standaloneCodeMod as any);
+
+/** `require` resolved from this file's URL — used to satisfy `ajv/dist/...`
+ *  / `ajv-formats/...` imports embedded in standalone-compiled validators
+ *  loaded back off disk. Anchored here so it always resolves through the
+ *  kernel package's node_modules, regardless of where the cache file
+ *  lives on disk. */
+const kernelRequire = createRequire(import.meta.url);
+
+/** Resolved AJV + ajv-formats versions, baked into every cache key so a
+ *  pnpm/npm install that upgrades either package invalidates all stale
+ *  `<hash>.cjs` files automatically. Standalone-compiled validators
+ *  embed `require("ajv/dist/runtime/...")` — running a validator built
+ *  against an older AJV against the current runtime is undefined
+ *  behaviour, so the version pin must be part of the hash, not a manual
+ *  bump. Falls back to walking up from the package's main entry when
+ *  the dependency restricts subpath access via `exports`. */
+function readDepVersion(spec: string): string {
+  try {
+    const pkg = kernelRequire(`${spec}/package.json`);
+    if (typeof pkg.version === "string") return pkg.version;
+  } catch {
+    // restricted exports — try the filesystem walk below
+  }
+  try {
+    const entry = kernelRequire.resolve(spec);
+    let dir = path.dirname(entry);
+    while (dir !== path.dirname(dir)) {
+      const candidate = path.join(dir, "package.json");
+      try {
+        const pkg = JSON.parse(fs.readFileSync(candidate, "utf-8"));
+        const expectedName = spec.split("/").slice(0, spec.startsWith("@") ? 2 : 1).join("/");
+        if (typeof pkg.name === "string" && pkg.name === expectedName) {
+          return typeof pkg.version === "string" ? pkg.version : "unknown";
+        }
+      } catch {
+        // keep walking — not at the package root yet
+      }
+      dir = path.dirname(dir);
+    }
+  } catch {
+    // package not installed
+  }
+  return "unknown";
+}
+const AJV_VERSION = readDepVersion("ajv");
+const AJV_FORMATS_VERSION = readDepVersion("ajv-formats");
+const VALIDATOR_RUNTIME_TAG = `ajv@${AJV_VERSION}+ajv-formats@${AJV_FORMATS_VERSION}`;
+
+const SHA256_HEADER_PATTERN = /^\/\/ sha256:([0-9a-f]{64})\n/;
+
+/** Verify a cached validator file's SHA-256 integrity header and return
+ *  the body when the digest matches. Returns `null` on any mismatch /
+ *  malformed header — the caller treats that as a cache miss and
+ *  recompiles + overwrites the file. */
+function verifyAndExtractBody(text: string): string | null {
+  const match = text.match(SHA256_HEADER_PATTERN);
+  if (!match) return null;
+  const body = text.slice(match[0].length);
+  const actual = createHash("sha256").update(body).digest("hex");
+  return actual === match[1] ? body : null;
+}
 
 export class SchemaValidator {
   private ajv: InstanceType<typeof Ajv>;
   private typeRules = new Map<string, TypeRule[]>();
   private rawSchemas = new Map<string, object>();
   private compiledValidators = new WeakMap<object, DataValidator>();
+  private cacheDir: string | undefined;
+  /** Tracks (schema-hash → in-memory compiled validator) so two distinct
+   *  but content-equal schema objects share one compile across the kernel
+   *  process — `compiledValidators` is keyed by object identity and would
+   *  miss those cases. */
+  private hashCache = new Map<string, DataValidator>();
 
   constructor() {
     this.ajv = new Ajv({
       strict: false,
       removeAdditional: false,
       useDefaults: true,
+      // Required for `standaloneCode` extraction — tells AJV to keep the
+      // generated validator's source available rather than wrapping it
+      // through `new Function`. The cost at compile time is negligible.
+      code: { source: true },
     });
     addFormats.default(this.ajv);
     for (const kw of [
@@ -33,6 +115,10 @@ export class SchemaValidator {
     ]) {
       this.ajv.addKeyword(kw);
     }
+    // Register the shared manifest root so module schemas can
+    // `$ref: "telo://manifest#/$defs/ResourceRef"` without each manifest
+    // bundling its own copy. Mirrors the analyzer's createAjv().
+    this.ajv.addSchema(ManifestRootSchema);
   }
 
   addSchema(name: string, schema: object): void {
@@ -54,6 +140,19 @@ export class SchemaValidator {
     return this.typeRules.get(name);
   }
 
+  /** Enable the on-disk validator cache rooted at `dir`. Compiled AJV
+   *  validators are written as standalone CJS modules keyed by content
+   *  hash, so subsequent process invocations skip the ≈2–10 ms AJV
+   *  codegen for each unseen schema. Safe to call before or after
+   *  `compile()` — already-compiled in-memory entries are unaffected.
+   *  The caller is responsible for choosing a writable directory; the
+   *  kernel anchors this under `<entry-dir>/.telo/manifests/__validators/`
+   *  so it lives next to the manifest cache and rides along in
+   *  `COPY --from=build /srv /srv` Docker images. */
+  setCacheDir(dir: string | undefined): void {
+    this.cacheDir = dir;
+  }
+
   compile(schema: any): DataValidator {
     if (schema && typeof schema === "object") {
       const cached = this.compiledValidators.get(schema as object);
@@ -85,7 +184,20 @@ export class SchemaValidator {
             },
           }
         : normalized;
-    const validate = this.ajv.compile(injected);
+
+    const hash = createHash("sha256")
+      .update(JSON.stringify({ runtime: VALIDATOR_RUNTIME_TAG, schema: injected }))
+      .digest("hex")
+      .slice(0, 32);
+    const cachedByHash = this.hashCache.get(hash);
+    if (cachedByHash) {
+      if (schema && typeof schema === "object") {
+        this.compiledValidators.set(schema as object, cachedByHash);
+      }
+      return cachedByHash;
+    }
+
+    const validate = this.compileAjvOrLoadCached(injected, hash);
 
     const validator = {
       validate: (data: any) => {
@@ -102,6 +214,7 @@ export class SchemaValidator {
       },
     };
 
+    this.hashCache.set(hash, validator);
     if (schema && typeof schema === "object") {
       this.compiledValidators.set(schema as object, validator);
     }
@@ -109,6 +222,69 @@ export class SchemaValidator {
     return validator;
   }
 
+  /** Load `<cacheDir>/<hash>.cjs` if present, else compile via AJV and
+   *  persist as standalone CJS. Cached files start with a
+   *  `// sha256:<hex>\n` header covering the rest of the file; a
+   *  mismatch (truncated write, FS corruption, tampering inside a baked
+   *  Docker image) is treated as a cache miss and the validator is
+   *  recompiled — and overwritten — so the cache self-heals. The cached
+   *  body is wrapped so its embedded `require("ajv/...")` /
+   *  `require("ajv-formats/...")` calls resolve against the kernel
+   *  package; the cache file lives outside any `node_modules` tree, so a
+   *  bare `require()` from its own path would fail. Read/write failures
+   *  surface to stderr but never abort compilation. */
+  private compileAjvOrLoadCached(
+    schema: any,
+    hash: string,
+  ): ValidateFunction {
+    const cacheDir = this.cacheDir;
+    if (cacheDir) {
+      const cachePath = path.join(cacheDir, `${hash}.cjs`);
+      try {
+        const text = fs.readFileSync(cachePath, "utf-8");
+        const body = verifyAndExtractBody(text);
+        if (body !== null) {
+          const factory = new Function(
+            "require",
+            "module",
+            "exports",
+            `${body}\nreturn module.exports;`,
+          );
+          const mod: { exports: any } = { exports: {} };
+          const loaded = factory(kernelRequire, mod, mod.exports);
+          if (typeof loaded === "function") {
+            return loaded as ValidateFunction;
+          }
+        }
+        // Header missing / mismatched / non-function export — fall
+        // through and recompile. The write step below overwrites the
+        // stale file with a fresh hash header.
+      } catch (err) {
+        if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") {
+          process.stderr.write(
+            `[telo:kernel] validator cache load failed (${hash}): ${err instanceof Error ? err.message : String(err)}\n`,
+          );
+        }
+      }
+    }
+
+    const validate = this.ajv.compile(schema) as ValidateFunction;
+    if (cacheDir) {
+      try {
+        const body = standaloneCode(this.ajv, validate);
+        const integrity = createHash("sha256").update(body).digest("hex");
+        const payload = `// sha256:${integrity}\n${body}`;
+        fs.mkdirSync(cacheDir, { recursive: true });
+        fs.writeFileSync(path.join(cacheDir, `${hash}.cjs`), payload, "utf-8");
+      } catch (err) {
+        process.stderr.write(
+          `[telo:kernel] validator cache write failed (${hash}): ${err instanceof Error ? err.message : String(err)}\n`,
+        );
+      }
+    }
+    return validate;
+  }
+
   composeWithRules(base: DataValidator, typeName: string, rules: TypeRule[]): DataValidator {
     return {
       validate: (data: any) => {

package/dist/generated/runtime-deps.json

@@ -1,6 +0,0 @@
-{
-  "generated": "scripts/generate-runtime-deps.mjs",
-  "names": [
-    "@telorun/sdk"
-  ]
-}