@telorun/sdk 0.2.5 → 0.2.7

package/src/evaluation-context.ts

@@ -1,44 +1,55 @@
-import { evaluate } from "cel-js";
-import { ModuleContext } from "./module-context.js";
+import { isCompiledValue } from "./compiled-value.js";
+import type { ModuleContext } from "./module-context.js";
+import type { ScopeContext, ScopeHandle } from "./ref.js";
 import { ResourceInstance } from "./resource-instance.js";
 import { ResourceManifest } from "./resource-manifest.js";
+import { RuntimeDiagnostic } from "./runtime-error.js";
 import { RuntimeError } from "./types.js";
 
 export type EmitEvent = (event: string, payload?: any) => void | Promise<void>;
 
-const TEMPLATE_REGEX = /\$\{\{\s*([^}]+?)\s*\}\}/g;
-const EXACT_TEMPLATE_REGEX = /^\s*\$\{\{\s*([^}]+?)\s*\}\}\s*$/;
-
 /** Four-stage resource lifecycle defined in resource-lifecycle.md */
 export type LifecycleState = "Pending" | "Validated" | "Initialized" | "Draining" | "Teardown";
 
+/**
+ * Result of the create phase: the instance and its bound ResourceContext.
+ * The ctx is typed as any to avoid a circular import with resource-context.ts.
+ */
+export type CreatedResource = { instance: ResourceInstance; ctx: any };
+
 /**
  * Creates a ResourceInstance for the given manifest, or returns null if not yet
  * ready (e.g. a dependency is still initializing). Injected at construction so
  * every EvaluationContext node owns its full resource lifecycle.
+ * Returns a CreatedResource (instance + ctx) so initializeResources can run
+ * init() separately in a second phase.
  */
 export type InstanceFactory = (
   moduleContext: ModuleContext,
   resource: ResourceManifest,
-) => Promise<ResourceInstance | null>;
+) => Promise<CreatedResource | null>;
+
+/**
+ * Hook called after controller.create() and before controller.init() for each resource.
+ * Implementations (e.g. the kernel) use this to inject live instances into reference
+ * fields of the resource config before the controller sees them in init().
+ *
+ * @param resource  The resource manifest whose config fields may be mutated in-place.
+ * @param getInstance  Looks up an already-initialized instance by resource name.
+ *                     Returns undefined when the named resource is not yet initialized.
+ */
+export type PreInitHook = (
+  resource: ResourceManifest,
+  getInstance: (name: string) => ResourceInstance | undefined,
+) => void;
 
 /** Canonical key for a resource instance: "<module>.<kind>.<name>" */
 export function resourceKey(r: ResourceManifest): string {
   return `${r.kind}.${r.metadata.name}`;
 }
 
-function redactSecrets(message: string, secretValues: Set<string>): string {
-  if (secretValues.size === 0) return message;
-  const sorted = Array.from(secretValues).sort((a, b) => b.length - a.length);
-  let result = message;
-  for (const secret of sorted) {
-    result = result.split(secret).join("[REDACTED]");
-  }
-  return result;
-}
-
 /**
- * Base class for all evaluation contexts. Owns CEL evaluation, template
+ * Base class for all evaluation contexts. Owns template
  * expansion, secrets redaction, and the generic resource lifecycle tree.
  *
  * Every EvaluationContext node can:
@@ -68,9 +79,21 @@ export class EvaluationContext {
     { resource: ResourceManifest; instance: ResourceInstance }
   >();
 
+  /** Resources that have been created but not yet initialized (between phases). */
+  protected readonly createdInstances = new Map<
+    string,
+    { resource: ResourceManifest; instance: ResourceInstance; ctx: any }
+  >();
+
   /** Resources queued for initialization on this context node. */
   private pendingResources: ResourceManifest[] = [];
 
+  /**
+   * Optional hook called between create() and init() for each resource.
+   * Set by the kernel to inject live instances into reference fields.
+   */
+  preInitHook?: PreInitHook;
+
   constructor(
     readonly source: string,
     context: Record<string, unknown>,
@@ -88,6 +111,9 @@ export class EvaluationContext {
     return this._createInstance;
   }
 
+  /** Called after init() when a resource snapshot is available. Overridden by ModuleContext. */
+  protected onResourceSnapshotted(_name: string, _snap: Record<string, unknown>): void {}
+
   get context(): Record<string, unknown> {
     return this._context;
   }
@@ -96,13 +122,40 @@ export class EvaluationContext {
     return this._secretValues;
   }
 
+  /**
+   * Reorder pending resources to match the given name sequence (topo order from Phase 4).
+   * Resources not present in `names` are left at the end in their original order.
+   * Call before initializeResources() so the create/init sub-phases run in dependency order,
+   * guaranteeing that Phase 5 injection always finds initialized dependencies.
+   */
+  setInitOrder(names: string[]): void {
+    const rank = new Map(names.map((n, i) => [n, i]));
+    this.pendingResources.sort((a, b) => {
+      const ra = rank.get(a.metadata.name as string) ?? Infinity;
+      const rb = rank.get(b.metadata.name as string) ?? Infinity;
+      return ra - rb;
+    });
+  }
+
   /**
    * Queue a resource manifest for initialization on this context.
    */
+  hasManifest(name: string): boolean {
+    return (
+      this.resourceInstances.has(name) ||
+      this.createdInstances.has(name) ||
+      this.pendingResources.some((r) => r.metadata.name === name)
+    );
+  }
+
   registerManifest(resource: ResourceManifest): void {
     if (!resource.metadata) {
       resource.metadata = { name: `__unnamed_${Math.random().toString(16).slice(2, 8)}` };
     }
+    const name = resource.metadata.name;
+    if (this.hasManifest(name)) {
+      throw new RuntimeError("ERR_DUPLICATE_RESOURCE", `Resource '${name}' is already registered`);
+    }
     this.pendingResources.push(resource);
   }
 
@@ -113,63 +166,107 @@ export class EvaluationContext {
   spawnChild<T extends EvaluationContext>(child: T): T {
     child.parent = this;
     this.children.push(child);
+    // Propagate injection hook so all child contexts (module imports, scopes) participate
+    // in Phase 5 injection. createScopeHandle overrides this with an extended version.
+    if (this.preInitHook && !child.preInitHook) {
+      child.preInitHook = this.preInitHook;
+    }
     return child;
   }
 
   /**
-   * Multi-pass initialization loop. Processes pendingResources by calling the
-   * supplied instantiator for each resource, retrying failures across up to 10
-   * passes (handles dependency ordering without explicit topological sort).
+   * Interleaved create/init loop.
    *
-   * ERR_VISIBILITY_DENIED errors are fatal and re-thrown immediately.
+   * Each pass has two sub-phases run back-to-back:
+   *   1. Create sub-phase: call controller.create() for each pending resource that
+   *      hasn't been created yet. Successful results go into createdInstances.
+   *   2. Init sub-phase: call instance.init(ctx) for each created-but-not-inited
+   *      resource. Successful results go into resourceInstances.
+   *
+   * Interleaving is necessary because some resources' create() depends on effects
+   * produced by other resources' init() (e.g. Kernel.Import.init() runs
+   * child.initializeResources() which registers controllers needed by sibling
+   * resources' create()). Running both sub-phases each pass lets those effects
+   * propagate before the next create attempt.
+   *
+   * Each resource is created at most once and inited at most once.
+   * ERR_VISIBILITY_DENIED is fatal and re-thrown immediately.
    * All other errors are tracked and retried until no progress is made.
    */
   async initializeResources(): Promise<void> {
     const MAX_PASSES = 10;
-    let pass = 1;
     const errors = new Map<string, string>();
 
+    let pass = 1;
     do {
-      const handled: string[] = [];
+      let progress = false;
 
+      // Create sub-phase
       for (const resource of [...this.pendingResources]) {
-        // const rkey = resourceKey(resource);
-        // const displayKey = rkey;
         const name = resource.metadata.name;
-        if (this.resourceInstances.has(name)) continue;
-
+        if (this.createdInstances.has(name)) continue;
         try {
-          const instance = await this._createInstance(this as any, resource);
-          if (instance) {
-            this.resourceInstances.set(name, { resource, instance });
-            handled.push(name);
+          // const expanded = this.expand(resource) as ResourceManifest;
+          // FIXME: Cannot expand it for all resources, needs to be selective
+          const created = await this._createInstance(this as any, resource);
+          if (created) {
+            this.createdInstances.set(name, {
+              resource,
+              instance: created.instance,
+              ctx: created.ctx,
+            });
+            const idx = this.pendingResources.findIndex((m) => m.metadata.name === name);
+            if (idx >= 0) this.pendingResources.splice(idx, 1);
             errors.delete(name);
+            progress = true;
           }
         } catch (error) {
           if (error instanceof RuntimeError && error.code === "ERR_VISIBILITY_DENIED") throw error;
-          errors.set(name, error instanceof Error ? (error.stack ?? error.message) : String(error));
+          errors.set(name, error instanceof Error ? error.message : String(error));
         }
       }
 
-      for (const name of handled) {
-        const resource = this.pendingResources.find((m) => m.metadata.name === name)!;
-        const idx = this.pendingResources.indexOf(resource);
-        if (idx >= 0) this.pendingResources.splice(idx, 1);
+      // Init sub-phase
+      for (const [name, { resource, instance, ctx }] of [...this.createdInstances]) {
+        if (this.resourceInstances.has(name)) continue;
+        try {
+          if (this.preInitHook) {
+            this.preInitHook(resource, (n) => this.resourceInstances.get(n)?.instance);
+          }
+          if (instance.init) await instance.init(ctx);
+          if (instance.snapshot) {
+            const snap = await Promise.resolve(instance.snapshot()).catch(() => ({}));
+            this.onResourceSnapshotted(name, (snap as Record<string, unknown>) ?? {});
+          }
+          this.resourceInstances.set(name, { resource, instance });
+          this.createdInstances.delete(name);
+          errors.delete(name);
+          progress = true;
+        } catch (error) {
+          if (error instanceof RuntimeError && error.code === "ERR_VISIBILITY_DENIED") throw error;
+          errors.set(name, error instanceof Error ? error.message : String(error));
+        }
       }
 
       pass++;
-      if (handled.length === 0) break;
+      if (!progress) break;
     } while (pass <= MAX_PASSES);
 
-    if (this.pendingResources.length > 0) {
-      const unhandledList = this.pendingResources
-        .reverse()
-        .map((r) => `- ${r.metadata.name}: ${errors.get(r.metadata.name) ?? "Unknown error"}`)
-        .join("\n");
-
+    if (this.pendingResources.length > 0 || this.createdInstances.size > 0) {
+      const diagnostics: RuntimeDiagnostic[] = [
+        ...this.pendingResources.map((r) => ({
+          resource: r.metadata.name,
+          message: errors.get(r.metadata.name) ?? "Unknown error",
+        })),
+        ...[...this.createdInstances.keys()].map((name) => ({
+          resource: name,
+          message: errors.get(name) ?? "Unknown error",
+        })),
+      ];
       throw new RuntimeError(
         "ERR_RESOURCE_INITIALIZATION_FAILED",
-        `Unable to process resources:\n\n${unhandledList}`,
+        "Unable to process resources",
+        diagnostics,
       );
     }
 
@@ -199,13 +296,72 @@ export class EvaluationContext {
     }
   }
 
+  /**
+   * Returns a ScopeHandle that initializes `manifests` in a fresh child context each time
+   * `run()` is called, executes the callback with a ScopeContext, and tears down when done.
+   *
+   * The child inherits the parent's preInitHook (if any), extended so that `getInstance`
+   * also checks the parent's already-initialized singleton instances. This lets scoped
+   * resources hold x-telo-ref slots pointing to outer resources — those deps are already
+   * live when the scope opens.
+   */
+  createScopeHandle(manifests: ResourceManifest[]): ScopeHandle {
+    const parent = this;
+    return {
+      async run<T>(fn: (scope: ScopeContext) => Promise<T>): Promise<T> {
+        const child = parent.spawnChild(
+          new EvaluationContext(
+            parent.source,
+            parent._context,
+            parent._createInstance,
+            parent._secretValues,
+            parent.emit,
+          ),
+        );
+
+        // Propagate injection hook: extend getInstance to also resolve parent singleton instances.
+        if (parent.preInitHook) {
+          const parentHook = parent.preInitHook;
+          child.preInitHook = (resource, childGetInstance) => {
+            parentHook(
+              resource,
+              (name) => childGetInstance(name) ?? parent.resourceInstances.get(name)?.instance,
+            );
+          };
+        }
+
+        try {
+          for (const manifest of manifests) {
+            child.registerManifest(manifest);
+          }
+          await child.initializeResources();
+          const scope: ScopeContext = {
+            getInstance(name: string): ResourceInstance {
+              const childEntry = child.resourceInstances.get(name);
+              if (childEntry) return childEntry.instance;
+              const parentEntry = parent.resourceInstances.get(name);
+              if (parentEntry) return parentEntry.instance;
+              throw new RuntimeError(
+                "ERR_SCOPE_RESOURCE_NOT_FOUND",
+                `Resource '${name}' not found in scope or outer context. Available scoped: ${[...child.resourceInstances.keys()].join(", ")}`,
+              );
+            },
+          };
+          return await fn(scope);
+        } finally {
+          await child.teardownResources();
+          const idx = parent.children.indexOf(child);
+          if (idx >= 0) parent.children.splice(idx, 1);
+        }
+      },
+    };
+  }
+
   /**
    * Cascade teardown depth-first through the tree:
    *   1. Tear down child contexts in reverse registration order.
-   *   2. Tear down own resource instances in reverse registration order.
-   *
-   * Note: Kernel-level events (e.g. Teardown events) are NOT emitted here —
-   * they remain the Kernel's responsibility.
+   *   2. Tear down own resource instances in reverse registration order,
+   *      emitting a Teardown event for each via the injected emit callback.
    */
   async teardownResources(): Promise<void> {
     this.state = "Draining";
@@ -213,18 +369,31 @@ export class EvaluationContext {
       await child.teardownResources();
     }
     const entries = [...this.resourceInstances.entries()].reverse();
-    for (const [key, { instance }] of entries) {
+    for (const [key, { resource, instance }] of entries) {
       if (instance.teardown) await instance.teardown();
+      await this.emit(`${resource.kind}.${resource.metadata.name}.Teardown`, {
+        resource: { kind: resource.kind, name: resource.metadata.name },
+      });
       this.resourceInstances.delete(key);
     }
     this.state = "Teardown";
   }
 
+  transientChild(context: Record<string, any>): EvaluationContext {
+    return new EvaluationContext(
+      this.source,
+      { ...this.context, ...context },
+      this._createInstance,
+      this._secretValues,
+      this.emit,
+    );
+  }
+
   /**
    * Invoke a resource by kind and name within this context's resourceInstances.
    * Emits a scoped Invoked event via the injected emit callback after invocation.
    */
-  async invoke(kind: string, name: string, ...args: any[]): Promise<any> {
+  async invoke<TInputs>(kind: string, name: string, inputs: TInputs): Promise<any> {
     const entry = this.resourceInstances.get(name);
 
     if (entry) {
@@ -234,7 +403,7 @@ export class EvaluationContext {
           `Resource ${kind}.${name} does not have an invoke method`,
         );
       }
-      const outputs = await entry.instance.invoke(args[0]);
+      const outputs = await entry.instance.invoke(inputs as any);
       await this.emit(`${kind}.${name}.Invoked`, { outputs });
       return outputs;
     }
@@ -257,29 +426,12 @@ export class EvaluationContext {
   }
 
   /**
-   * Evaluate a single CEL expression string against the context.
-   * Secret values are redacted from any thrown error message.
-   */
-  evaluate(expression: string): unknown {
-    try {
-      return evaluate(expression, this._context);
-    } catch (error) {
-      const raw = error instanceof Error ? error.message : String(error);
-      const safe = redactSecrets(raw, this._secretValues);
-      throw new Error(`CEL evaluation failed: "${expression}": ${safe}`);
-    }
-  }
-
-  /**
-   * Expand a value that may contain ${{ }} templates.
-   * Works recursively over strings, arrays, and objects.
-   * Templates whose identifiers are not present in the context are left
-   * unchanged (deferred) — they will be resolved at execution time when a
-   * richer ExecutionContext is available. All other CEL errors are propagated.
+   * Expand a value that may contain precompiled ${{ }} templates.
+   * Works recursively over CompiledValues, arrays, and objects.
    */
   expand(value: unknown): unknown {
-    if (typeof value === "string") {
-      return this.expandString(value);
+    if (isCompiledValue(value)) {
+      return value.call(this._context);
     }
     if (Array.isArray(value)) {
       return value.map((entry) => this.expand(entry));
@@ -295,43 +447,73 @@ export class EvaluationContext {
   }
 
   /**
-   * Merge another context on top of this one.
-   * Returns a new base EvaluationContext — 'other' wins on key conflict.
+   * Expand a value using this context merged with additional properties.
+   * Equivalent to merge(extraContext).expand(value) without allocating a context object.
    */
-  merge(other: EvaluationContext | Record<string, unknown>): EvaluationContext {
-    const otherCtx = other instanceof EvaluationContext ? other.context : other;
-    const otherSecrets =
-      other instanceof EvaluationContext ? other.secretValues : new Set<string>();
-    const merged = Object.assign(Object.create(null), this._context, otherCtx) as Record<
+  expandWith(value: unknown, extraContext: Record<string, unknown>): unknown {
+    const saved = this._context;
+    this._context = Object.assign(Object.create(null), saved, extraContext) as Record<
       string,
       unknown
     >;
-    const mergedSecrets = new Set<string>([...this._secretValues, ...otherSecrets]);
-    return new EvaluationContext(
-      this.source,
-      merged,
-      this._createInstance,
-      mergedSecrets,
-      this.emit,
-    );
+    try {
+      return this.expand(value);
+    } finally {
+      this._context = saved;
+    }
   }
 
-  private expandString(value: string): unknown {
-    if (!value.includes("${{")) {
-      return value;
+/**
+   * Expand specific dot-paths within an object. '**' expands the entire object.
+   * Paths listed in excludePaths are left untouched (runtime takes precedence).
+   * Always throws if an expression cannot be resolved.
+   */
+  expandPaths(
+    value: Record<string, unknown>,
+    paths: string[],
+    excludePaths: string[] = [],
+  ): Record<string, unknown> {
+    if (paths.includes("**")) {
+      const result: Record<string, unknown> = {};
+      for (const [key, v] of Object.entries(value)) {
+        result[key] = isExcluded(key, excludePaths) ? v : this.expand(v);
+      }
+      return result;
     }
-
-    const exact = value.match(EXACT_TEMPLATE_REGEX);
-    if (exact) {
-      return this.evaluate(exact[1]);
+    const result = { ...value };
+    for (const path of paths) {
+      if (isExcluded(path, excludePaths)) continue;
+      const parts = path.split(".");
+      const current = getNestedValue(result, parts);
+      if (current !== undefined) {
+        setNestedValue(result, parts, this.expand(current));
+      }
     }
+    return result;
+  }
+}
 
-    return value.replace(TEMPLATE_REGEX, (_match, expr: string) => {
-      const resolved = this.evaluate(expr);
-      if (resolved === null || resolved === undefined) {
-        return "";
-      }
-      return String(resolved);
-    });
+function isExcluded(path: string, excludePaths: string[]): boolean {
+  return excludePaths.some(
+    (ep) => ep === path || ep === "**" || path.startsWith(ep + ".") || ep.startsWith(path + "."),
+  );
+}
+
+function getNestedValue(obj: Record<string, unknown>, parts: string[]): unknown {
+  let current: unknown = obj;
+  for (const part of parts) {
+    if (current === null || typeof current !== "object") return undefined;
+    current = (current as Record<string, unknown>)[part];
+  }
+  return current;
+}
+
+function setNestedValue(obj: Record<string, unknown>, parts: string[], value: unknown): void {
+  let current: Record<string, unknown> = obj;
+  for (let i = 0; i < parts.length - 1; i++) {
+    const next = current[parts[i]];
+    if (next === null || typeof next !== "object") return;
+    current = next as Record<string, unknown>;
   }
+  current[parts[parts.length - 1]] = value;
 }

package/src/index.ts

@@ -1,4 +1,6 @@
+export * from "./compiled-value.js";
 export * from "./capabilities/invokable.js";
+export * from "./ref.js";
 export * from "./capabilities/provider.js";
 export * from "./capabilities/runnable.js";
 export * from "./context-provider.js";

package/src/module-context.ts

@@ -1,6 +1,26 @@
-import { Invokable } from "./capabilities/invokable.js";
+import { Invocable } from "./capabilities/invokable.js";
 import { EmitEvent, EvaluationContext, InstanceFactory } from "./evaluation-context.js";
 
+/** Wraps process.env so that missing keys return null instead of throwing in CEL.
+ * cel-js uses Object.hasOwn(obj, key) before accessing obj[key], so we must
+ * intercept getOwnPropertyDescriptor to report every string key as "own". */
+function lenientEnv(env: Record<string, string | undefined>): Record<string, string | null> {
+  return new Proxy(env as Record<string, string | null>, {
+    get(target, key) {
+      if (typeof key !== "string") return (target as any)[key];
+      return key in target ? (target[key] ?? null) : null;
+    },
+    has() {
+      return true;
+    },
+    getOwnPropertyDescriptor(target, key) {
+      if (typeof key !== "string") return Object.getOwnPropertyDescriptor(target, key);
+      const value = key in target ? (target[key] ?? null) : null;
+      return { configurable: true, enumerable: true, writable: true, value };
+    },
+  });
+}
+
 function collectSecretValues(secrets: Record<string, unknown>): Set<string> {
   const values = new Set<string>();
   for (const value of Object.values(secrets)) {
@@ -42,6 +62,7 @@ export class ModuleContext extends EvaluationContext {
     private targets: string[] = [],
     createInstance: InstanceFactory = async () => null,
     emit: EmitEvent,
+    private readonly _hostEnv?: Record<string, string | undefined>,
   ) {
     super(source, {}, createInstance, new Set(), emit);
     this._variables = variables;
@@ -81,6 +102,10 @@ export class ModuleContext extends EvaluationContext {
     this._rebuildContext();
   }
 
+  protected override onResourceSnapshotted(name: string, snap: Record<string, unknown>): void {
+    this.setResource(name, snap);
+  }
+
   /**
    * Register an imported module under the given alias, with the list of kind names
    * it exports. An empty kinds array means no restriction (used for built-ins like Kernel).
@@ -102,7 +127,9 @@ export class ModuleContext extends EvaluationContext {
     return entry?.instance;
   }
 
-  getInvokable(name: string): Invokable {
+  getInvocable<TInput = Record<string, any>, TOutput = any>(
+    name: string,
+  ): Invocable<TInput, TOutput> {
     const instance = this.getInstance(name);
 
     if (
@@ -113,7 +140,7 @@ export class ModuleContext extends EvaluationContext {
     ) {
       throw new Error(`Resource '${name}' does not have an invoke() method.`);
     }
-    return instance as Invokable;
+    return instance as Invocable<TInput, TOutput>;
   }
 
   /**
@@ -151,12 +178,13 @@ export class ModuleContext extends EvaluationContext {
       variables: this._variables,
       secrets: this._secrets,
       resources: this._resources,
+      ...(this._hostEnv ? { env: lenientEnv(this._hostEnv) } : {}),
     };
     this._secretValues = collectSecretValues(this._secrets);
   }
 
-  override async invoke(kind: string, name: string, ...args: any[]): Promise<any> {
-    const result = await super.invoke(kind, name, ...args);
+  override async invoke<TInputs>(kind: string, name: string, inputs: TInputs): Promise<any> {
+    const result = await super.invoke(kind, name, inputs);
     const entry = this.resourceInstances.get(name);
     if (entry && typeof (entry.instance as any).snapshot === "function") {
       const snap = await Promise.resolve((entry.instance as any).snapshot());