@telorun/kernel 0.12.0 → 0.13.2

package/src/kernel.ts

@@ -5,6 +5,7 @@ import {
   isModuleKind,
   Loader,
   StaticAnalyzer,
+  type LoadedGraph,
   type ManifestSource,
 } from "@telorun/analyzer";
 import {
@@ -30,6 +31,13 @@ import { EventStream } from "./event-stream.js";
 import { EventBus } from "./events.js";
 import { ModuleContext } from "./module-context.js";
 import { ResourceContextImpl } from "./resource-context.js";
+import {
+  computeAnalysisSignature,
+  readAnalysisStamp,
+  writeAnalysisStamp,
+} from "./manifest-sources/analysis-stamp.js";
+import { resolveEntryDir } from "./manifest-sources/local-manifest-cache-source.js";
+import { resolveApplicationEnv } from "./application-env.js";
 import { policyFingerprint } from "./runtime-registry.js";
 import { SchemaValidator } from "./schema-validator.js";
 
@@ -117,6 +125,7 @@ export class Kernel implements IKernel {
   private rootContext!: ModuleContext;
   private staticManifests: ResourceManifest[] = [];
   private _entryUrl?: string;
+  private _loadedGraph?: LoadedGraph;
   // Lifecycle state — guards boot/runTargets/teardown/invoke transitions.
   // teardown() is the only idempotent method; everything else throws on misuse.
   private _bootCalled = false;
@@ -185,6 +194,37 @@ export class Kernel implements IKernel {
     return this.registry;
   }
 
+  /** The full LoadedGraph captured during `load()`. Used by the CLI to
+   *  feed `writeManifestCache` so a successful `telo run` populates
+   *  `<entry-dir>/.telo/manifests/` for subsequent runs — the same on-disk
+   *  layout `telo install` writes. Undefined before `load()` has been
+   *  called or if it threw before the graph was captured. */
+  getLoadedGraph(): LoadedGraph | undefined {
+    return this._loadedGraph;
+  }
+
+  /** True when `url` resolves (via the loader's URL → canonical-source map)
+   *  to a module that was already part of the entry graph successfully
+   *  analyzed during `load()`. The import-controller uses it to skip its
+   *  per-import `analyze()` pass when the kernel's load-time validation
+   *  already covered the same subtree. */
+  isImportValidatedAtLoad(url: string): boolean {
+    if (!this._loadedGraph) return false;
+    const canonical = this.loader.canonicalize(url);
+    if (!canonical) return false;
+    return this._loadedGraph.modules.has(canonical);
+  }
+
+  /** Resolve a `Telo.Import.source` against the importing file's URL
+   *  through the same source-chain `resolveRelative` the loader used at
+   *  graph-walk time. The import-controller routes through here so its
+   *  `resolveImportSource` no longer second-guesses the loader for
+   *  custom `ManifestSource`s — `isImportValidatedAtLoad` etc. only hit
+   *  when both sides agree on the canonical URL. */
+  resolveImportUrl(fromSource: string, importSource: string): string {
+    return this.loader.resolveImportUrl(fromSource, importSource);
+  }
+
   /**
    * Load built-in Runtime definitions (e.g., Telo.Application, Telo.Library).
    * Also declares all known module namespaces upfront so that resources can be
@@ -225,6 +265,16 @@ export class Kernel implements IKernel {
   async load(url: string): Promise<void> {
     const sourceUrl = await this.loader.resolveEntryPoint(url);
     this._entryUrl = sourceUrl;
+    // Point the shared schema validator at the entry-anchored cache so
+    // compiled AJV validators are persisted (standalone CJS) under
+    // `<entry-dir>/.telo/manifests/__validators/`. Memory- or HTTP-rooted
+    // entries skip the cache; their schema compiles stay in-process only.
+    const validatorCacheDir = resolveEntryDir(sourceUrl);
+    this.sharedSchemaValidator.setCacheDir(
+      validatorCacheDir
+        ? `${validatorCacheDir}/.telo/manifests/__validators`
+        : undefined,
+    );
     this.rootContext = new ModuleContext(
       sourceUrl,
       {},
@@ -253,6 +303,7 @@ export class Kernel implements IKernel {
     if (analysisGraph.errors.length > 0) {
       throw analysisGraph.errors[0].error;
     }
+    this._loadedGraph = analysisGraph;
     const staticManifests = flattenForAnalyzer(analysisGraph);
     this.staticManifests = staticManifests;
 
@@ -275,7 +326,22 @@ export class Kernel implements IKernel {
       }
     }
 
-    const errors = this.analyzer.analyzeErrors(staticManifests, {}, this.registry);
+    // Hash-keyed analysis cache: when the entry's full LoadedGraph matches
+    // a previously-stamped successful run (same file bytes, same stamp
+    // protocol version), skip the per-resource validation walk inside
+    // `analyzeErrors`. Registration of identities / aliases / definitions
+    // and inline-resource normalisation still runs — only the diagnostic
+    // passes are elided. Memory- / HTTP-rooted entries have no
+    // local stamp store and always re-validate.
+    const entryDir = resolveEntryDir(sourceUrl);
+    const analysisSignature = computeAnalysisSignature(analysisGraph);
+    const stamp = entryDir ? await readAnalysisStamp(entryDir) : undefined;
+    const skipValidation = stamp?.signature === analysisSignature;
+    const errors = this.analyzer.analyzeErrors(
+      staticManifests,
+      { skipValidation },
+      this.registry,
+    );
     if (errors.length > 0) {
       throw new RuntimeError(
         "ERR_MANIFEST_VALIDATION_FAILED",
@@ -290,6 +356,19 @@ export class Kernel implements IKernel {
         })),
       );
     }
+    if (entryDir && !skipValidation) {
+      // Best-effort: stamp the verdict so subsequent loads hit the fast
+      // path. A read-only filesystem (baked Docker image) reports the
+      // failure on stderr and keeps running — the lookup above will
+      // simply miss next time.
+      try {
+        await writeAnalysisStamp(entryDir, analysisSignature);
+      } catch (err) {
+        this.stderr.write(
+          `[telo:kernel] analysis stamp write failed: ${err instanceof Error ? err.message : String(err)}\n`,
+        );
+      }
+    }
 
     // Load runtime configuration — root module gets access to host env.
     // Imports are loaded separately via the import-controller; this load is
@@ -304,15 +383,54 @@ export class Kernel implements IKernel {
     const normalizedManifests = this.analyzer.normalize(allManifests, this.registry);
     this.staticManifests = normalizedManifests;
 
+    let rootApplicationManifest: ResourceManifest | undefined;
     for (const manifest of normalizedManifests) {
       if (isModuleKind(manifest.kind)) {
-        // Root is always Telo.Application (Library root rejected above). Applications
-        // have no variables/secrets fields — those are a Library-only contract, populated
-        // by importers, not by the root manifest itself.
-        this.rootContext.setTargets(manifest.targets ?? []);
+        // Root is always Telo.Application (Library root rejected above).
+        // Application-level `variables` / `secrets` declarations carry an `env:`
+        // mapping per field; the kernel populates the root scope from
+        // `process.env` after the manifest loop so imports can read
+        // `${{ variables.X }}` during their own init.
+        //
+        // Targets normalize down to bare names regardless of source surface.
+        // The analyzer's `resolveRefSentinels` pass already substituted any
+        // `!ref <name>` to `{kind, name}`; bare-string forms pass through.
+        // Anything else (e.g. an unresolved sentinel because the analyzer
+        // couldn't see it, or a malformed manifest) is a hard error —
+        // silently dropping the entry would leave the user staring at a
+        // "no targets ran" outcome with no signal where it went wrong.
+        const rawTargets = (manifest.targets ?? []) as unknown[];
+        const targetNames = rawTargets.map((t, index) => {
+          if (typeof t === "string") return t;
+          if (t && typeof t === "object" && typeof (t as { name?: unknown }).name === "string") {
+            return (t as { name: string }).name;
+          }
+          throw new RuntimeError(
+            "ERR_INVALID_VALUE",
+            `Telo.Application '${(manifest.metadata as { name?: string } | undefined)?.name ?? "(unnamed)"}' targets[${index}] could not be normalized to a resource name. Got: ${JSON.stringify(t)}`,
+          );
+        });
+        this.rootContext.setTargets(targetNames);
+        if (manifest.kind === "Telo.Application") {
+          rootApplicationManifest = manifest;
+        }
       }
       this.rootContext.registerManifest(manifest);
     }
+
+    if (rootApplicationManifest) {
+      const { variables, secrets } = resolveApplicationEnv(
+        rootApplicationManifest as Record<string, any>,
+        this.env,
+        this.sharedSchemaValidator,
+      );
+      if (Object.keys(variables).length > 0) {
+        this.rootContext.setVariables(variables);
+      }
+      if (Object.keys(secrets).length > 0) {
+        this.rootContext.setSecrets(secrets);
+      }
+    }
   }
 
   /**
@@ -419,6 +537,13 @@ export class Kernel implements IKernel {
     if (this.rootContext) {
       await this.rootContext.teardownResources();
     }
+    // Drop the load-time graph so a teardown'd kernel doesn't pin every
+    // manifest file's text in memory (LoadedFile retains the parsed
+    // documents + the original YAML bytes). Reusing the kernel after
+    // teardown is a hard error elsewhere, so this is purely a memory
+    // hygiene step.
+    this._loadedGraph = undefined;
+    this.staticManifests = [];
     await this.eventBus.emit("Kernel.Stopped", { exitCode: this._exitCode });
   }
 

package/src/manifest-schemas.ts

@@ -1,15 +1,16 @@
-import { Type } from "@sinclair/typebox";
 import AjvModule from "ajv";
 import addFormats from "ajv-formats";
 const Ajv = AjvModule.default ?? AjvModule;
 
-export const RuntimeResourceSchema = Type.Object(
-  {
-    kind: Type.String(),
-    metadata: Type.Object({ name: Type.String() }, { additionalProperties: true }),
-  },
-  { additionalProperties: true },
-);
+// Re-export the shared ResourceRef fragment from the templating package
+// so consumers that pull it from the kernel manifest-schemas surface keep
+// working. The canonical home is `@telorun/templating` because the
+// fragment describes the parsed shape of the `!ref` tag's TaggedSentinel.
+export {
+  MANIFEST_SCHEMA_URI,
+  ManifestRootSchema,
+  ResourceRefSchema,
+} from "@telorun/templating";
 
 const metadataSchema = {
   type: "object",
@@ -149,9 +150,28 @@ export const ResourceAbstractSchema = {
 const ajv = new Ajv({ allErrors: true, strict: false });
 addFormats.default(ajv);
 
-export const validateRuntimeResource = ajv.compile(RuntimeResourceSchema);
-export const validateResourceDefinition = ajv.compile(ResourceDefinitionSchema);
-export const validateResourceAbstract = ajv.compile(ResourceAbstractSchema);
+// Lazy-compile validator: the AJV codegen cost (≈10–15 ms for these
+// schemas) is only paid when a definition / abstract actually needs
+// validating. A hello-world that loads no Telo.Definition or
+// Telo.Abstract documents never triggers either compile; apps that
+// do see them only pay once per process.
+interface LazyValidator {
+  (data: unknown): boolean | Promise<unknown>;
+  errors?: any[] | null;
+}
+function lazyValidator(schema: object): LazyValidator {
+  let compiled: ReturnType<typeof ajv.compile> | undefined;
+  const fn: LazyValidator = (data: unknown) => {
+    if (!compiled) compiled = ajv.compile(schema);
+    const ok = compiled(data);
+    fn.errors = compiled.errors as any[] | null | undefined;
+    return ok;
+  };
+  return fn;
+}
+
+export const validateResourceDefinition = lazyValidator(ResourceDefinitionSchema);
+export const validateResourceAbstract = lazyValidator(ResourceAbstractSchema);
 
 export function formatAjvErrors(errors: any[] | null | undefined): string {
   if (!errors || errors.length === 0) return "Unknown schema error";

package/src/manifest-sources/analysis-stamp.ts

@@ -0,0 +1,169 @@
+import type { LoadedGraph } from "@telorun/analyzer";
+import { createHash } from "crypto";
+import { readFileSync } from "fs";
+import * as fs from "fs/promises";
+import { createRequire } from "module";
+import * as path from "path";
+import { fileURLToPath } from "url";
+
+/**
+ * Hash-keyed analysis cache: a tiny JSON sidecar in `.telo/manifests/`
+ * recording that an exact set of manifest bytes — under specific
+ * `@telorun/kernel` and `@telorun/analyzer` package versions — passed
+ * `analyzer.analyzeErrors`. The next `kernel.load` reads the sidecar
+ * and, if signatures match, skips the per-resource validation walk.
+ *
+ * Lives next to the manifest cache (`LocalManifestCacheSource`) but is
+ * independent of it — splitting both for grep-ability and because the
+ * concerns (URL → file content vs. content → analyzer verdict) are
+ * orthogonal.
+ */
+
+const CACHE_SUBDIR = ".telo/manifests";
+
+/** File-format version of the analysis stamp envelope. Only bumped when
+ *  the on-disk *layout* changes (new fields, restructured payload). The
+ *  *semantic* invalidation — "did the analyzer's logic change?" — is
+ *  handled by baking the resolved `@telorun/analyzer` / `@telorun/kernel`
+ *  package versions into the signature itself, so any pnpm/npm install
+ *  that bumps either package automatically invalidates every stamp on
+ *  disk. A hand-maintained integer for that purpose would silently mask
+ *  newly-stricter validation until the next manifest edit. */
+const ANALYSIS_STAMP_FORMAT_VERSION = 1;
+const ANALYSIS_STAMP_FILE = `${CACHE_SUBDIR}/.validated.json`;
+
+const localRequire = createRequire(import.meta.url);
+
+/** Read the kernel's own `package.json` — `createRequire` can't resolve
+ *  `@telorun/kernel/package.json` from inside the kernel package itself
+ *  (the self-reference loops in some node_modules layouts). The file
+ *  sits two levels up from `dist/manifest-sources/`. */
+function readKernelVersion(): string {
+  try {
+    const url = new URL("../../package.json", import.meta.url);
+    const pkg = JSON.parse(readFileSync(fileURLToPath(url), "utf-8"));
+    return typeof pkg.version === "string" ? pkg.version : "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
+function readDepVersion(spec: string): string {
+  // Fast path: direct `require("<pkg>/package.json")`. Fails (with
+  // ERR_PACKAGE_PATH_NOT_EXPORTED) when the dependency declares a strict
+  // `exports` map without listing `./package.json` — common for packages
+  // that consider package.json an implementation detail. Don't return
+  // "unknown" in that case; fall back to resolving the package's main
+  // entry and walking the filesystem up to its package.json.
+  const pkgJsonSpec = spec.endsWith("/package.json")
+    ? spec
+    : `${spec}/package.json`;
+  try {
+    const pkg = localRequire(pkgJsonSpec);
+    if (typeof pkg.version === "string") return pkg.version;
+  } catch {
+    // fall through to filesystem walk
+  }
+  try {
+    const mainSpec = spec.endsWith("/package.json") ? spec.slice(0, -13) : spec;
+    const entry = localRequire.resolve(mainSpec);
+    let dir = path.dirname(entry);
+    while (dir !== path.dirname(dir)) {
+      const candidate = path.join(dir, "package.json");
+      try {
+        const pkg = JSON.parse(readFileSync(candidate, "utf-8"));
+        // Guard against scoped-package interior package.json files (some
+        // packages stamp one in dist/) — match by name when the spec
+        // names a package.
+        const expectedName = mainSpec
+          .split("/")
+          .slice(0, mainSpec.startsWith("@") ? 2 : 1)
+          .join("/");
+        if (typeof pkg.name === "string" && pkg.name === expectedName) {
+          return typeof pkg.version === "string" ? pkg.version : "unknown";
+        }
+      } catch {
+        // not at the package root yet — keep walking
+      }
+      dir = path.dirname(dir);
+    }
+  } catch {
+    // resolution failed — package not installed at all
+  }
+  return "unknown";
+}
+
+const KERNEL_VERSION = readKernelVersion();
+const ANALYZER_VERSION = readDepVersion("@telorun/analyzer");
+
+export interface AnalysisStamp {
+  version: number;
+  signature: string;
+}
+
+/** Hash every owner + partial file in `graph` together with the resolved
+ *  `@telorun/kernel` and `@telorun/analyzer` versions into one content
+ *  signature. Two loads of the same manifest set under the same package
+ *  versions produce the same signature; any edit to any reachable file —
+ *  or any pnpm/npm install that bumps the kernel or analyzer — flips it.
+ *  This is what the kernel uses to decide whether the previous analyzer
+ *  run's verdict still applies. */
+export function computeAnalysisSignature(graph: LoadedGraph): string {
+  const entries: Array<[string, string]> = [];
+  for (const [, mod] of graph.modules) {
+    for (const file of [mod.owner, ...mod.partials]) {
+      const digest = createHash("sha256").update(file.text).digest("hex");
+      entries.push([file.source, digest]);
+    }
+  }
+  entries.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+  return createHash("sha256")
+    .update(
+      JSON.stringify({
+        kernel: KERNEL_VERSION,
+        analyzer: ANALYZER_VERSION,
+        files: entries,
+      }),
+    )
+    .digest("hex");
+}
+
+/** Read the stamped analysis verdict for the entry at `entryDir`, or
+ *  `undefined` when missing / unreadable / format-mismatched. The
+ *  `version` field is the on-disk *format* version; semantic
+ *  invalidation flows through the signature (which embeds package
+ *  versions). A future format change bumps `version` so older kernels
+ *  reading a newer stamp (or vice versa) discard rather than misparse. */
+export async function readAnalysisStamp(
+  entryDir: string,
+): Promise<AnalysisStamp | undefined> {
+  try {
+    const text = await fs.readFile(path.join(entryDir, ANALYSIS_STAMP_FILE), "utf-8");
+    const parsed = JSON.parse(text) as Partial<AnalysisStamp>;
+    if (
+      parsed?.version === ANALYSIS_STAMP_FORMAT_VERSION &&
+      typeof parsed?.signature === "string"
+    ) {
+      return parsed as AnalysisStamp;
+    }
+  } catch {
+    // missing / unreadable / unparseable — treat as cache miss
+  }
+  return undefined;
+}
+
+/** Persist the analysis verdict so the next `kernel.load` can skip the
+ *  per-resource validation walk when the manifest set is unchanged.
+ *  Idempotent; safe to call after every successful load. */
+export async function writeAnalysisStamp(
+  entryDir: string,
+  signature: string,
+): Promise<void> {
+  const stamp: AnalysisStamp = {
+    version: ANALYSIS_STAMP_FORMAT_VERSION,
+    signature,
+  };
+  const target = path.join(entryDir, ANALYSIS_STAMP_FILE);
+  await fs.mkdir(path.dirname(target), { recursive: true });
+  await fs.writeFile(target, JSON.stringify(stamp), "utf-8");
+}

package/src/resource-context.ts

@@ -13,6 +13,7 @@ import {
   type ParsedArgs,
   type TypeRule,
 } from "@telorun/sdk";
+import { isRefSentinel } from "@telorun/templating";
 import AjvModule from "ajv";
 import addFormats from "ajv-formats";
 import { EvaluationContext } from "./evaluation-context.js";
@@ -176,6 +177,14 @@ export class ResourceContextImpl implements ResourceContext {
     return this.kernel.loadManifests(url);
   }
 
+  isImportValidatedAtLoad(url: string): boolean {
+    return this.kernel.isImportValidatedAtLoad(url);
+  }
+
+  resolveImportUrl(fromSource: string, importSource: string): string {
+    return this.kernel.resolveImportUrl(fromSource, importSource);
+  }
+
   /**
    * Resolves a resource into a normalized {kind, name} reference.
    * If the resource contains a definition (kind + properties), registers it as a manifest.
@@ -194,6 +203,31 @@ export class ResourceContextImpl implements ResourceContext {
       );
     }
 
+    // Stopgap: `!ref <name>` sentinels can reach the controller directly
+    // when the slot is hidden behind a local `$ref: "#/$defs/..."` — the
+    // analyzer's field-map walker descends `oneOf`/`anyOf` variant
+    // properties but intentionally early-returns on `$ref` (see
+    // `analyzer/nodejs/src/reference-field-map.ts`). Enabling the `$ref`
+    // descent regresses the kernel's `<Kind>.<Name>.Invoked` event
+    // emission for kinds (notably `Run.Sequence`) whose controllers
+    // call `instance.invoke()` directly on Phase-5-injected instances;
+    // the walker fix needs to land together with routing those callers
+    // through `EvaluationContext.invokeResolved`. Until then, the Node
+    // kernel resolves the sentinel here. Polyglot controllers don't get
+    // this rescue — schemas exercising those hidden slots must use the
+    // legacy string or `{kind, name}` forms for now.
+    if (isRefSentinel(resource)) {
+      const refName = resource.source;
+      const entry = this.moduleContext.resourceInstances.get(refName);
+      if (!entry) {
+        throw new RuntimeError(
+          "ERR_RESOURCE_NOT_FOUND",
+          `[${this.metadata.name}] !ref '${refName}' did not resolve to a registered resource.`,
+        );
+      }
+      return { kind: entry.resource.kind as string, name: refName };
+    }
+
     if (!resource.kind) {
       throw new RuntimeError(
         "ERR_INVALID_VALUE",