@aigne/ash 0.0.1

package/src/compiler.ts

@@ -0,0 +1,1103 @@
+import type { Program, JobDeclaration, PipelineStage, WhereClause, FanoutExpression, OutputExpression, TopLevelStatement, QueryCondition, LetStatement, GroupByExpression, ActionExpression, Expression, MapExpression, RouteExpression, LookupExpression, ParamDeclaration, TriggerDeclaration } from "./ast.js";
+import { AshLexer } from "./lexer.js";
+import { AshParser } from "./parser.js";
+import { checkPipelineTypes, checkProhibitedPatterns, checkAnnotations, checkJobCaps, typeErrorsToDiagnostics, compileErrorsToDiagnostics, annotationErrorsToDiagnostics, parseSyntaxError, parseCaps, hasCapFor } from "./type-checker.js";
+import type { AshDiagnostic, CapEntry } from "./type-checker.js";
+import { resolveActionParams, resolveTemplatePath } from "./template.js";
+
+export interface CompileResult {
+  program?: CompiledProgram;
+  diagnostics: AshDiagnostic[];
+  sourceHash?: string;
+}
+
+function fnv1aHash(str: string): string {
+  let hash = 2166136261;
+  for (let i = 0; i < str.length; i++) {
+    hash ^= str.charCodeAt(i);
+    hash = Math.imul(hash, 16777619) >>> 0;
+  }
+  return hash.toString(16).padStart(8, "0");
+}
+
+function collectActions(stages: PipelineStage[]): ActionExpression[] {
+  const actions: ActionExpression[] = [];
+  for (const stage of stages) {
+    if (stage.kind === "action") actions.push(stage);
+    else if (stage.kind === "fanout") {
+      for (const branch of stage.branches) actions.push(...collectActions(branch));
+    }
+  }
+  return actions;
+}
+
+export function compileSource(source: string): CompileResult {
+  const diagnostics: AshDiagnostic[] = [];
+  const sourceHash = fnv1aHash(source);
+  const lexer = new AshLexer();
+  const parser = new AshParser();
+  const compiler = new AshCompiler();
+
+  // Lex
+  let tokens;
+  try {
+    tokens = lexer.tokenize(source);
+  } catch (e: any) {
+    return { sourceHash, diagnostics: [parseSyntaxError(e.message ?? String(e))] };
+  }
+
+  // Parse
+  let ast;
+  try {
+    ast = parser.parse(tokens);
+  } catch (e: any) {
+    return { sourceHash, diagnostics: [parseSyntaxError(e.message ?? String(e))] };
+  }
+
+  // Type check
+  const prohibErrors = checkProhibitedPatterns(ast);
+  diagnostics.push(...compileErrorsToDiagnostics(prohibErrors));
+
+  for (const job of ast.jobs) {
+    const typeErrors = checkPipelineTypes(job.pipeline);
+    diagnostics.push(...typeErrorsToDiagnostics(typeErrors));
+
+    const annErrors = checkAnnotations(job);
+    diagnostics.push(...annotationErrorsToDiagnostics(annErrors));
+
+    // @caps: static path security check
+    diagnostics.push(...checkJobCaps(job));
+
+    // @readonly check: pipeline must not contain save, publish, or tee
+    const isReadonly = job.annotations.some(a => a.name === "readonly");
+    if (isReadonly) {
+      for (const stage of job.pipeline) {
+        if (stage.kind === "save" || stage.kind === "publish" || stage.kind === "tee" || stage.kind === "action") {
+          diagnostics.push({
+            code: "ASH_READONLY_VIOLATION",
+            message: `@readonly job '${job.name}' contains '${stage.kind}' which writes data`,
+          });
+        }
+      }
+      // Check route targets: @readonly must propagate to route-target jobs
+      for (const stage of job.pipeline) {
+        if (stage.kind === "route") {
+          const targetNames = [...stage.branches.map(b => b.targetJob), ...(stage.fallback ? [stage.fallback] : [])];
+          for (const targetName of targetNames) {
+            const targetJob = ast.jobs.find(j => j.name === targetName);
+            if (!targetJob) continue;
+            for (const tStage of targetJob.pipeline) {
+              if (tStage.kind === "save" || tStage.kind === "publish" || tStage.kind === "tee" || tStage.kind === "action") {
+                diagnostics.push({
+                  code: "ASH_READONLY_VIOLATION",
+                  message: `@readonly job '${job.name}' routes to '${targetName}' which contains '${tStage.kind}' (write operation)`,
+                });
+                break; // one error per target is enough
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  // Let pipeline restrictions: writes in let bindings are never legitimate
+  const WRITE_STAGES = new Set(["save", "publish", "tee", "action"]);
+  const letStatements = ast.statements.filter((s): s is LetStatement => s.kind === "let" && !!s.pipeline);
+  for (const letStmt of letStatements) {
+    if (letStmt.pipeline) {
+      for (const stage of letStmt.pipeline) {
+        if (WRITE_STAGES.has(stage.kind)) {
+          diagnostics.push({
+            code: "ASH_LET_WRITE",
+            message: `let '${letStmt.name}' pipeline contains '${stage.kind}' — write operations in let bindings are not permitted`,
+          });
+        }
+      }
+    }
+  }
+
+  // Let pipeline pre-approval bypass: let pipelines with find/action execute before @approval checks
+  const hasApproval = ast.jobs.some(j => j.annotations.some(a => a.name === "approval"));
+  if (hasApproval && letStatements.length > 0) {
+    const sideEffectLets = letStatements.filter(s =>
+      s.pipeline?.some(stage => stage.kind === "find" || stage.kind === "action"),
+    );
+    if (sideEffectLets.length > 0) {
+      diagnostics.push({
+        code: "ASH_LET_PRE_APPROVAL",
+        severity: "warning",
+        message: `let bindings (${sideEffectLets.map(s => s.name).join(", ")}) execute before @approval checks — side effects in let pipelines bypass approval`,
+      });
+    }
+  }
+
+  // Param write-gate: $param in where clause before write operation = logic bomb risk
+  const paramNames = new Set(ast.statements.filter((s): s is ParamDeclaration => s.kind === "param").map(s => s.name));
+  if (paramNames.size > 0) {
+    for (const job of ast.jobs) {
+      const hasWrite = job.pipeline.some(s => WRITE_STAGES.has(s.kind));
+      if (!hasWrite) continue;
+      for (const stage of job.pipeline) {
+        if (stage.kind === "where" && typeof stage.right === "string" && stage.right.startsWith("$")) {
+          const varName = stage.right.slice(1);
+          if (paramNames.has(varName)) {
+            diagnostics.push({
+              code: "ASH_PARAM_WRITE_GATE",
+              severity: "warning",
+              message: `Job '${job.name}': param '$${varName}' gates write operations — callers can override param to change behavior`,
+            });
+            break; // one warning per job
+          }
+        }
+      }
+    }
+  }
+
+  // Mixed annotation deception check: @readonly jobs alongside unrestricted write jobs
+  const readonlyJobs = ast.jobs.filter(j => j.annotations.some(a => a.name === "readonly"));
+  if (readonlyJobs.length > 0 && readonlyJobs.length < ast.jobs.length) {
+    const nonReadonly = ast.jobs.filter(j => !j.annotations.some(a => a.name === "readonly"));
+    const writingJobs = nonReadonly.filter(j =>
+      j.pipeline.some(s => s.kind === "save" || s.kind === "publish" || s.kind === "tee" || s.kind === "action"),
+    );
+    if (writingJobs.length > 0) {
+      diagnostics.push({
+        code: "ASH_MIXED_SECURITY",
+        severity: "warning",
+        message: `Program has mixed security posture: ${readonlyJobs.length} @readonly job(s) alongside ${writingJobs.length} unrestricted writing job(s) (${writingJobs.map(j => j.name).join(", ")}) — this can mask malicious intent`,
+      });
+    }
+  }
+
+  // Action hardening: uncapped, amplification (caps+budget gated), cross-provider (caps+budget gated)
+  for (const job of ast.jobs) {
+    const actions = collectActions(job.pipeline);
+    if (actions.length === 0) continue;
+
+    const hasCaps = job.annotations.some(a => a.name === "caps");
+    const hasBudget = job.annotations.some(a => a.name === "budget");
+
+    // No @caps + action → error (actions require explicit capability declaration)
+    if (!hasCaps) {
+      diagnostics.push({
+        code: "ASH_UNCAPPED_ACTION",
+        message: `Job '${job.name}' contains action stages but has no @caps — actions require explicit capability declaration`,
+      });
+    }
+
+    // Multiple actions → caps+budget gated
+    if (actions.length > 1) {
+      if (hasCaps && hasBudget) {
+        diagnostics.push({
+          code: "ASH_ACTION_AMPLIFICATION",
+          severity: "warning",
+          message: `Job '${job.name}' contains ${actions.length} action stages — budget-gated multi-action execution permitted`,
+        });
+      } else {
+        diagnostics.push({
+          code: "ASH_ACTION_AMPLIFICATION",
+          message: `Job '${job.name}' contains ${actions.length} action stages — multi-action jobs require @caps and @budget to prevent amplification attacks`,
+        });
+      }
+    }
+
+    // Actions targeting multiple providers → caps+budget gated
+    const providers = new Set(actions.filter(a => !a.relative).map(a => a.path.split("/")[1]).filter(Boolean));
+    if (providers.size > 1) {
+      if (hasCaps && hasBudget) {
+        diagnostics.push({
+          code: "ASH_CROSS_PROVIDER_ACTION",
+          severity: "warning",
+          message: `Job '${job.name}' has actions targeting ${providers.size} providers (${[...providers].join(", ")}) — budget-gated cross-provider execution permitted`,
+        });
+      } else {
+        diagnostics.push({
+          code: "ASH_CROSS_PROVIDER_ACTION",
+          message: `Job '${job.name}' has actions targeting ${providers.size} providers (${[...providers].join(", ")}) — cross-provider actions require @caps and @budget`,
+        });
+      }
+    }
+
+    // Relative action checks
+    const relativeActions = actions.filter(a => a.relative);
+    if (relativeActions.length > 0) {
+      // Relative action without upstream find → error
+      const hasFindUpstream = job.pipeline.some((s, idx) =>
+        s.kind === "find" && job.pipeline.findIndex(p => p === relativeActions[0]) > idx
+      );
+      if (!hasFindUpstream) {
+        diagnostics.push({
+          code: "ASH_RELATIVE_ACTION_NO_FIND",
+          message: `Job '${job.name}': relative action '${relativeActions[0].path}' has no upstream 'find' — relative actions require a find stage to provide records with paths`,
+        });
+      }
+
+      // Relative action = inherent amplification (1 stage, N execs)
+      if (hasCaps && hasBudget) {
+        diagnostics.push({
+          code: "ASH_ACTION_AMPLIFICATION",
+          severity: "warning",
+          message: `Job '${job.name}': relative action '${relativeActions[0].path}' executes per-record — budget-gated amplification permitted`,
+        });
+      } else if (!hasCaps || !hasBudget) {
+        diagnostics.push({
+          code: "ASH_ACTION_AMPLIFICATION",
+          message: `Job '${job.name}': relative action '${relativeActions[0].path}' executes per-record — requires @caps and @budget to prevent amplification`,
+        });
+      }
+    }
+  }
+
+  // Write operation hardening: publish and tee require @caps
+  for (const job of ast.jobs) {
+    const hasCaps = job.annotations.some(a => a.name === "caps");
+    const hasFind = job.pipeline.some(s => s.kind === "find" || s.kind === "lookup");
+    const writeStages = job.pipeline.filter(s => s.kind === "publish" || s.kind === "tee");
+
+    if (writeStages.length > 0 && hasFind && !hasCaps) {
+      for (const ws of writeStages) {
+        diagnostics.push({
+          code: "ASH_UNCAPPED_WRITE",
+          message: `Job '${job.name}': '${ws.kind}' combined with data reads requires @caps — write operations on external data must declare capabilities`,
+        });
+      }
+    }
+  }
+
+  // Budget ceiling: reject unreasonably high budget values
+  const BUDGET_CEILINGS: Record<string, number> = { actions: 100, writes: 100, records: 10000 };
+  for (const job of ast.jobs) {
+    const budgetAnn = job.annotations.find(a => a.name === "budget");
+    if (!budgetAnn) continue;
+    for (let i = 0; i < budgetAnn.args.length; i += 2) {
+      const dim = budgetAnn.args[i];
+      const val = Number(budgetAnn.args[i + 1]);
+      if (dim in BUDGET_CEILINGS && val > BUDGET_CEILINGS[dim]) {
+        diagnostics.push({
+          code: "ASH_BUDGET_EXCESSIVE",
+          message: `Job '${job.name}': @budget(${dim} ${val}) exceeds ceiling of ${BUDGET_CEILINGS[dim]} — reduce budget or justify with explicit approval`,
+        });
+      }
+    }
+  }
+
+  // Compile
+  const program = compiler.compile(ast);
+  return { program, diagnostics, sourceHash };
+}
+
+export interface OutputEvent {
+  kind: "text";
+  content: string;
+  context?: LogContext;
+}
+
+export interface OutputHandler {
+  output(event: OutputEvent): void;
+}
+
+export interface JobContext {
+  world: WorldInterface;
+  caps: Set<string>;
+  logger: JobLogger;
+  output?: OutputHandler;
+}
+
+export interface WorldInterface {
+  read(path: string, query?: QueryCondition): unknown[] | Promise<unknown[]>;
+  write(path: string, data: unknown[]): void;
+  publish(topic: string, data: unknown[]): void;
+  exec?(path: string, input: unknown[], params?: Record<string, unknown>): Promise<unknown[]>;
+  input?(prompt: string): string | Promise<string>;
+}
+
+export interface LogContext {
+  programId?: string;
+  procId?: string;
+  agentId?: string;
+  sessionId?: string;
+  jobName?: string;
+  stageIndex?: number;
+}
+
+export interface JobLogger {
+  log(stage: string, action: string, detail?: unknown): void;
+  emit?(level: "debug" | "info" | "warn" | "error", message: string, context?: LogContext): void;
+}
+
+export interface JobResult {
+  status: "ok" | "error" | "partial";
+  recordCount: number;
+  errors: string[];
+}
+
+export interface StageMetrics {
+  name: string;
+  index: number;
+  inputCount: number;
+  outputCount: number;
+  durationMs: number;
+  error?: string;
+}
+
+export interface JobReport extends JobResult {
+  jobName: string;
+  totalDurationMs: number;
+  stages: StageMetrics[];
+}
+
+export interface ProgramReport {
+  jobs: JobReport[];
+  totalDurationMs: number;
+}
+
+export interface CompiledJob {
+  kind: "job";
+  name: string;
+  trigger?: TriggerDeclaration;
+  execute(ctx: JobContext, initialStream?: unknown[]): Promise<JobResult>;
+}
+
+export interface CompiledOutput {
+  kind: "output";
+  message: string;
+  execute(ctx: JobContext): Promise<void>;
+}
+
+export type CompiledUnit = CompiledJob | CompiledOutput;
+
+export interface CompiledProgram {
+  jobs: CompiledJob[];
+  units: CompiledUnit[];
+  jobMap: Map<string, CompiledJob>;
+  params: Map<string, string | number>; // param defaults (can be overridden before execution)
+  routeTargets?: Set<string>; // jobs that are ONLY route targets (should not auto-execute as top-level)
+}
+
+function safeOutput(ctx: JobContext, content: string, logContext?: LogContext): void {
+  if (ctx.output) {
+    try {
+      ctx.output.output({ kind: "text", content, context: logContext });
+    } catch {
+      // Output fault must not crash pipeline
+    }
+  }
+  safeEmit(ctx.logger, "info", content, logContext);
+}
+
+function safeEmit(logger: JobLogger, level: "debug" | "info" | "warn" | "error", message: string, context?: LogContext): void {
+  if (logger.emit) {
+    try {
+      logger.emit(level, message, context);
+    } catch {
+      // Log fault must not crash pipeline (INVARIANT: emit failure is silent)
+    }
+  }
+}
+
+/**
+ * Strict equality with boolean coercion only.
+ * No string fallback (prevents type confusion like 0 == "0").
+ * Boolean coercion: true/"true" and false/"false" are equal.
+ */
+function strictEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  // Boolean coercion: "true" matches true, "false" matches false
+  if (typeof a === "boolean" && typeof b === "string") return a === (b === "true");
+  if (typeof b === "boolean" && typeof a === "string") return b === (a === "true");
+  return false;
+}
+
+export class AshCompiler {
+  private _currentJobName?: string;
+  private _currentStageIndex?: number;
+  private _jobMap = new Map<string, CompiledJob>();
+
+  compile(ast: Program): CompiledProgram {
+    // Collect param defaults
+    const params = new Map<string, string | number>();
+    for (const stmt of ast.statements) {
+      if (stmt.kind === "param") {
+        params.set(stmt.name, stmt.defaultValue);
+      }
+    }
+
+    // Collect static let bindings (params first, then lets override)
+    const variables = new Map<string, string | number>(params);
+    const runtimeLets: LetStatement[] = [];
+    for (const stmt of ast.statements) {
+      if (stmt.kind === "let") {
+        if (stmt.pipeline) {
+          runtimeLets.push(stmt);
+        } else {
+          variables.set(stmt.name, stmt.value);
+        }
+      }
+    }
+
+    const units: CompiledUnit[] = ast.statements
+      .filter((s): s is Exclude<TopLevelStatement, LetStatement | ParamDeclaration> => s.kind !== "let" && s.kind !== "param")
+      .map((stmt) => this.compileStatement(stmt, variables, runtimeLets));
+    const jobs = units.filter((u): u is CompiledJob => u.kind === "job");
+    const jobMap = new Map<string, CompiledJob>();
+    for (const job of jobs) jobMap.set(job.name, job);
+    this._jobMap = jobMap;
+
+    // Identify route-target-only jobs: jobs referenced by route stages but with no trigger
+    const routeTargetNames = new Set<string>();
+    const triggeredJobs = new Set<string>();
+    for (const job of ast.jobs) {
+      if (job.trigger) triggeredJobs.add(job.name);
+      for (const stage of job.pipeline) {
+        if (stage.kind === "route") {
+          for (const branch of stage.branches) {
+            routeTargetNames.add(branch.targetJob);
+          }
+          if (stage.fallback) routeTargetNames.add(stage.fallback);
+        }
+      }
+    }
+    // Exclude triggered jobs — they have independent execution reasons
+    for (const name of triggeredJobs) routeTargetNames.delete(name);
+    const routeTargets = routeTargetNames.size > 0 ? routeTargetNames : undefined;
+
+    return { jobs, units, jobMap, params, routeTargets };
+  }
+
+  private compileStatement(stmt: Exclude<TopLevelStatement, LetStatement | ParamDeclaration>, variables: Map<string, string | number>, runtimeLets: LetStatement[] = []): CompiledUnit {
+    if (stmt.kind === "output") {
+      return this.compileTopLevelOutput(stmt);
+    }
+    return this.compileJob(stmt, variables, runtimeLets);
+  }
+
+  private compileTopLevelOutput(output: OutputExpression): CompiledOutput {
+    return {
+      kind: "output",
+      message: output.message,
+      execute: async (ctx: JobContext): Promise<void> => {
+        ctx.logger.log("output", "output", { message: output.message });
+        safeOutput(ctx, output.message);
+      },
+    };
+  }
+
+  private compileJob(job: JobDeclaration, variables: Map<string, string | number> = new Map(), runtimeLets: LetStatement[] = []): CompiledJob {
+    const stages = job.pipeline;
+    const annotations = job.annotations;
+
+    // Extract annotation configs
+    const retryAnn = annotations.find(a => a.name === "retry");
+    const timeoutAnn = annotations.find(a => a.name === "timeout");
+    const onErrorAnn = annotations.find(a => a.name === "on_error");
+    const budgetAnn = annotations.find(a => a.name === "budget");
+    const retryCount = retryAnn ? (retryAnn.args.length > 0 ? Number(retryAnn.args[0]) : 3) : 0;
+    const timeoutMs = timeoutAnn ? (timeoutAnn.args.length > 0 ? Number(timeoutAnn.args[0]) : 0) : 0;
+    const onErrorStrategy = onErrorAnn?.args[0] as "skip" | "save" | "fail" | undefined;
+    const onErrorPath = onErrorStrategy === "save" ? onErrorAnn?.args[1] : undefined;
+
+    // Parse budget limits
+    const budgetLimits: Record<string, number> = {};
+    if (budgetAnn) {
+      for (let i = 0; i < budgetAnn.args.length; i += 2) {
+        budgetLimits[budgetAnn.args[i]] = Number(budgetAnn.args[i + 1]);
+      }
+    }
+
+    // Parse @caps for runtime enforcement (script-declared caps are authoritative)
+    const capsAnns = annotations.filter(a => a.name === "caps");
+    const scriptCaps: CapEntry[] = [];
+    for (const ann of capsAnns) {
+      if (ann.args.length > 0 && ann.args.length % 2 === 0) {
+        scriptCaps.push(...parseCaps(ann));
+      }
+    }
+
+    const self = this;
+
+    const runOnce = async (ctx: JobContext, initialStream?: unknown[]): Promise<JobReport> => {
+      const errors: string[] = [];
+      const jobInitialStream = initialStream ?? [];
+      let stream: unknown[] = [...jobInitialStream];
+      const stageMetrics: StageMetrics[] = [];
+      const jobStart = performance.now();
+      const budgetUsed = { actions: 0, writes: 0, records: 0, tokens: 0, cost: 0 };
+      const checkBudget = (dim: string, increment: number) => {
+        if (!(dim in budgetLimits)) return;
+        (budgetUsed as any)[dim] += increment;
+        if ((budgetUsed as any)[dim] > budgetLimits[dim]) {
+          throw new Error(`budget exceeded: ${dim} used ${(budgetUsed as any)[dim]}, limit ${budgetLimits[dim]}`);
+        }
+      };
+
+      // Runtime caps enforcement: if script declares @caps, enforce them with glob matching.
+      // Script-declared caps are the ceiling — ctx.caps (caller-provided) cannot widen them.
+      const checkCap = (op: CapEntry["op"], path: string): void => {
+        if (scriptCaps.length > 0 && !hasCapFor(scriptCaps, op, path)) {
+          throw new Error(`Permission denied: @caps does not allow ${op} '${path}'`);
+        }
+      };
+
+      // Resolve runtime lets
+      const runtimeVars = new Map(variables);
+      for (const rtLet of runtimeLets) {
+        if (rtLet.pipeline) {
+          try {
+            let rtStream: unknown[] = [];
+            for (const stage of rtLet.pipeline) {
+              rtStream = await self.executeStage(stage, rtStream, ctx, errors, runtimeVars);
+            }
+            // Bind the result: if single object with a numeric/string value, extract it
+            if (rtStream.length === 1 && typeof rtStream[0] === "object" && rtStream[0] !== null) {
+              const obj = rtStream[0] as Record<string, unknown>;
+              const keys = Object.keys(obj);
+              if (keys.length === 1) {
+                const val = obj[keys[0]];
+                if (typeof val === "number" || typeof val === "string") {
+                  runtimeVars.set(rtLet.name, val);
+                  continue;
+                }
+              }
+            }
+            // Fallback: use length or first value
+            if (rtStream.length === 1) {
+              const v = rtStream[0];
+              if (typeof v === "number" || typeof v === "string") {
+                runtimeVars.set(rtLet.name, v);
+              } else {
+                runtimeVars.set(rtLet.name, rtStream.length);
+              }
+            } else {
+              runtimeVars.set(rtLet.name, rtStream.length);
+            }
+          } catch (e: any) {
+            errors.push(`Runtime let '${rtLet.name}' failed: ${e.message ?? String(e)}`);
+            return { status: "error", recordCount: 0, errors, jobName: job.name, totalDurationMs: performance.now() - jobStart, stages: stageMetrics };
+          }
+        }
+      }
+
+      for (let i = 0; i < stages.length; i++) {
+        const stage = stages[i];
+        const inputCount = stream.length;
+        const stageStart = performance.now();
+        self._currentJobName = job.name;
+        self._currentStageIndex = i;
+        try {
+          // Skip absolute actions on empty downstream stream (not first stage).
+          // "action as source" (i === 0) still executes — it generates data from nothing.
+          // Relative actions already skip naturally (for-of loop over empty stream).
+          if (stage.kind === "action" && !stage.relative && i > 0 && stream.length === 0) {
+            const durationMs = performance.now() - stageStart;
+            ctx.logger.log(stage.kind, "skip", { reason: "empty-stream" });
+            safeEmit(ctx.logger, "debug", `${stage.kind} skip (empty stream)`, { jobName: job.name, stageIndex: i });
+            stageMetrics.push({ name: stage.kind, index: i, inputCount, outputCount: 0, durationMs });
+            continue;
+          }
+          ctx.logger.log(stage.kind, "enter", { recordCount: stream.length });
+          safeEmit(ctx.logger, "debug", `${stage.kind} enter`, { jobName: job.name, stageIndex: i });
+          stream = await self.executeStage(stage, stream, ctx, errors, runtimeVars, checkBudget, checkCap, jobInitialStream);
+          // Budget checks after stage execution
+          // Relative actions are checked per-record inside executeStage; only check absolute actions here
+          // Absolute actions with inline params also check per-record inside executeStage
+          const hasInlineParams = stage.kind === "action" && !stage.relative &&
+            stage.params && Object.keys(stage.params).length > 0;
+          if (stage.kind === "action" && !stage.relative && !hasInlineParams) checkBudget("actions", 1);
+          if (stage.kind === "save" || stage.kind === "publish" || stage.kind === "tee") checkBudget("writes", 1);
+          checkBudget("records", stream.length);
+          const durationMs = performance.now() - stageStart;
+          ctx.logger.log(stage.kind, "exit", { recordCount: stream.length });
+          safeEmit(ctx.logger, "debug", `${stage.kind} exit`, { jobName: job.name, stageIndex: i });
+          stageMetrics.push({ name: stage.kind, index: i, inputCount, outputCount: stream.length, durationMs });
+        } catch (e: any) {
+          const durationMs = performance.now() - stageStart;
+          const errMsg = e.message ?? String(e);
+          errors.push(errMsg);
+
+          if (onErrorStrategy === "skip") {
+            // Skip: log error, continue with remaining stream
+            stageMetrics.push({ name: stage.kind, index: i, inputCount, outputCount: stream.length, durationMs, error: errMsg });
+            continue;
+          } else if (onErrorStrategy === "save" && onErrorPath) {
+            // Save: write failed items to error path
+            try {
+              ctx.world.write(onErrorPath, stream.map(item => ({ _error: errMsg, _item: item })));
+            } catch {
+              errors.push(`Failed to write errors to ${onErrorPath}`);
+            }
+            stageMetrics.push({ name: stage.kind, index: i, inputCount, outputCount: stream.length, durationMs, error: errMsg });
+            continue;
+          }
+
+          // Default (fail): terminate
+          stageMetrics.push({ name: stage.kind, index: i, inputCount, outputCount: stream.length, durationMs, error: errMsg });
+          return { status: "error", recordCount: stream.length, errors, jobName: job.name, totalDurationMs: performance.now() - jobStart, stages: stageMetrics };
+        }
+      }
+
+      return {
+        status: errors.length > 0 ? "partial" : "ok",
+        recordCount: stream.length,
+        errors,
+        jobName: job.name,
+        totalDurationMs: performance.now() - jobStart,
+        stages: stageMetrics,
+      };
+    };
+
+    const runWithTimeout = async (ctx: JobContext, initialStream?: unknown[]): Promise<JobResult> => {
+      if (timeoutMs <= 0) return runOnce(ctx, initialStream);
+      return Promise.race([
+        runOnce(ctx, initialStream),
+        new Promise<JobResult>((_, reject) =>
+          setTimeout(() => reject(new Error(`Timeout: job exceeded ${timeoutMs}ms`)), timeoutMs)
+        ),
+      ]);
+    };
+
+    return {
+      kind: "job",
+      name: job.name,
+      trigger: job.trigger,
+      execute: async (ctx: JobContext, initialStream?: unknown[]): Promise<JobResult> => {
+        if (retryCount <= 0 && !retryAnn) {
+          // No retry annotation at all
+          return runWithTimeout(ctx, initialStream);
+        }
+
+        const maxAttempts = retryCount > 0 ? retryCount : 1;
+        const allErrors: string[] = [];
+
+        for (let attempt = 0; attempt < maxAttempts; attempt++) {
+          try {
+            const result = await runWithTimeout(ctx, initialStream);
+            if (result.status !== "error") return result;
+            allErrors.push(...result.errors);
+            // If last attempt, return error
+            if (attempt === maxAttempts - 1) return { status: "error", recordCount: result.recordCount, errors: allErrors };
+            // Backoff (in tests we don't actually wait)
+          } catch (e: any) {
+            allErrors.push(e.message ?? String(e));
+            if (attempt === maxAttempts - 1) {
+              return { status: "error", recordCount: 0, errors: allErrors };
+            }
+          }
+        }
+
+        return { status: "error", recordCount: 0, errors: allErrors };
+      },
+    };
+  }
+
+  private async executeStage(stage: PipelineStage, stream: unknown[], ctx: JobContext, errors: string[], variables: Map<string, string | number> = new Map(), budgetCheck?: (dim: string, increment: number) => void, capCheck?: (op: CapEntry["op"], path: string) => void, initialStream?: unknown[]): Promise<unknown[]> {
+    switch (stage.kind) {
+      case "find": {
+        // Resolve template path from stream context (e.g. find /msgs/${data.messageId})
+        let findPath = stage.path;
+        if (findPath.includes("${") && stream.length > 0) {
+          const rec = (typeof stream[0] === "object" && stream[0] !== null ? stream[0] : {}) as Record<string, unknown>;
+          findPath = resolveTemplatePath(findPath, rec);
+        }
+
+        // Runtime @caps enforcement (script-declared caps)
+        if (capCheck) capCheck("read", findPath);
+        const capPath = findPath.split("/").slice(0, 3).join("/");
+        if (ctx.caps.size > 0 && !ctx.caps.has(capPath) && !ctx.caps.has("*")) {
+          throw new Error(`Permission denied: cannot read '${findPath}'`);
+        }
+        try {
+          const query = stage.query ? this.resolveQueryVars(stage.query, variables) : undefined;
+          let result = await ctx.world.read(findPath, query);
+          // Fallback: if world.read ignores query, apply in-memory filter
+          if (query && result.length > 0) {
+            result = result.filter((item) => {
+              try {
+                return this.evaluateWhere(
+                  { kind: "where", left: query!.field, op: query!.op, right: query!.value },
+                  item,
+                );
+              } catch {
+                return false;
+              }
+            });
+          }
+          return result;
+        } catch {
+          return []; // non-existent path → empty stream
+        }
+      }
+
+      case "where": {
+        // Resolve $variable references in where clause
+        const resolvedStage = this.resolveWhereVars(stage, variables);
+        return stream.filter((item) => {
+          try {
+            return this.evaluateWhere(resolvedStage, item);
+          } catch {
+            return false; // missing field → skip
+          }
+        });
+      }
+
+      case "map": {
+        // System fields that map cannot overwrite (identity fields from AFS nodes)
+        const IMMUTABLE_FIELDS = ["path", "kind"];
+
+        const preserveSystemFields = (original: unknown, mapped: Record<string, unknown>): Record<string, unknown> => {
+          if (typeof original === "object" && original !== null) {
+            for (const field of IMMUTABLE_FIELDS) {
+              if (field in (original as Record<string, unknown>)) {
+                mapped[field] = (original as Record<string, unknown>)[field];
+              }
+            }
+          }
+          return mapped;
+        };
+
+        if (stage.exprMappings) {
+          return stream.map((item) => {
+            const result: Record<string, unknown> = {};
+            for (const [key, expr] of Object.entries(stage.exprMappings!)) {
+              result[key] = this.evaluateExpression(expr, item, variables);
+            }
+            return preserveSystemFields(item, result);
+          });
+        }
+        if (stage.expression) {
+          return stream.map((item) => this.evaluateExpression(stage.expression!, item, variables));
+        }
+        if (stage.mappings) {
+          return stream.map((item) => {
+            const result: Record<string, unknown> = {};
+            for (const [key, field] of Object.entries(stage.mappings!)) {
+              result[key] = this.resolveField(item, field);
+            }
+            return preserveSystemFields(item, result);
+          });
+        }
+        return stream.map((item) => this.resolveField(item, stage.field));
+      }
+
+      case "save": {
+        // Runtime @caps enforcement (script-declared caps)
+        if (capCheck) capCheck("write", stage.path);
+        const capPath = stage.path.split("/").slice(0, 3).join("/");
+        if (ctx.caps.size > 0 && !ctx.caps.has(capPath) && !ctx.caps.has("*")) {
+          errors.push(`Permission denied: cannot write '${stage.path}'`);
+          return stream;
+        }
+        ctx.world.write(stage.path, stream);
+        return [];
+      }
+
+      case "publish": {
+        // Runtime @caps enforcement (script-declared caps)
+        if (capCheck) capCheck("write", stage.path);
+        const capPath = stage.path.split("/").slice(0, 3).join("/");
+        if (ctx.caps.size > 0 && !ctx.caps.has(capPath) && !ctx.caps.has("*")) {
+          errors.push(`Permission denied: cannot publish '${stage.path}'`);
+          return stream;
+        }
+        ctx.world.publish(stage.path, stream);
+        return [];
+      }
+
+      case "tee": {
+        // Runtime @caps enforcement (script-declared caps)
+        if (capCheck) capCheck("write", stage.path);
+        try {
+          ctx.world.write(stage.path, [...stream]);
+        } catch (e: any) {
+          errors.push(`tee side-write failed: ${e.message}`);
+          // Main pipeline continues
+        }
+        return stream;
+      }
+
+      case "fanout": {
+        return await this.executeFanout(stage, stream, ctx, errors, variables, budgetCheck, capCheck, initialStream);
+      }
+
+      case "output": {
+        if (stage.expression) {
+          // Expression mode: evaluate per stream item, emit each result as text
+          for (const item of stream) {
+            const val = this.evaluateExpression(stage.expression, item, variables);
+            const text = val == null ? "" : String(val);
+            ctx.logger.log("output", "output", { message: text, streamSize: stream.length });
+            safeOutput(ctx, text, { jobName: this._currentJobName, stageIndex: this._currentStageIndex });
+          }
+        } else {
+          ctx.logger.log("output", "output", { message: stage.message, streamSize: stream.length });
+          safeOutput(ctx, stage.message, { jobName: this._currentJobName, stageIndex: this._currentStageIndex });
+        }
+        return stream; // pass-through
+      }
+
+      case "input": {
+        ctx.logger.log("input", "prompt", { prompt: stage.prompt });
+        const response = ctx.world.input
+          ? await ctx.world.input(stage.prompt)
+          : "";
+        return [{ prompt: stage.prompt, response }];
+      }
+
+      case "count": {
+        return [{ count: stream.length }];
+      }
+
+      case "group-by": {
+        const groups = new Map<unknown, unknown[]>();
+        for (const item of stream) {
+          const key = this.resolveField(item, stage.field);
+          if (!groups.has(key)) groups.set(key, []);
+          groups.get(key)!.push(item);
+        }
+        return Array.from(groups.entries()).map(([key, items]) => ({ key, items }));
+      }
+
+      case "route": {
+        // Dispatch items to target jobs based on field value
+        const buckets = new Map<string, unknown[]>();
+        for (const item of stream) {
+          const val = String(this.resolveField(item, stage.field) ?? "");
+          const branch = stage.branches.find(b => b.value === val);
+          const targetJob = branch ? branch.targetJob : stage.fallback;
+          if (targetJob) {
+            if (!buckets.has(targetJob)) buckets.set(targetJob, []);
+            buckets.get(targetJob)!.push(item);
+          }
+          // If no match and no fallback, item is dropped (silently)
+        }
+        // Execute each target job with its bucket as initial stream
+        for (const [jobName, items] of buckets) {
+          const compiled = this._jobMap.get(jobName);
+          if (!compiled) {
+            errors.push(`Route target job '${jobName}' not found`);
+            continue;
+          }
+          await compiled.execute(ctx, items);
+        }
+        return []; // route is terminal
+      }
+
+      case "lookup": {
+        // Left join: read lookup source, build index by joinKey, merge into stream items
+        const lookupData = await ctx.world.read(stage.path);
+        const index = new Map<string, unknown>();
+        for (const item of lookupData) {
+          const key = String(this.resolveField(item, stage.joinKey) ?? "");
+          if (!index.has(key)) index.set(key, item);
+        }
+        return stream.map((item) => {
+          const key = String(this.resolveField(item, stage.joinKey) ?? "");
+          const match = index.get(key);
+          if (match && typeof match === "object" && match !== null && typeof item === "object" && item !== null) {
+            return { ...match, ...item }; // item fields take precedence
+          }
+          return item; // LEFT JOIN: unmatched items preserved
+        });
+      }
+
+      case "action": {
+        if (!ctx.world.exec) {
+          throw new Error(`WorldInterface.exec not available — cannot execute action '${stage.path}'`);
+        }
+
+        const hasParams = stage.params && Object.keys(stage.params).length > 0;
+        const hasTemplatePath = stage.path.includes("${");
+
+        if (stage.relative) {
+          // Per-record relative action: resolve path from each record
+          const results: unknown[] = [];
+          for (const record of stream) {
+            // Pre-check budget before each execution
+            if (budgetCheck) budgetCheck("actions", 1);
+            if (typeof record !== "object" || record === null || !("path" in record)) {
+              throw new Error(`Relative action '${stage.path}' requires records with a 'path' field`);
+            }
+            const recordPath = String((record as Record<string, unknown>).path);
+            const fullPath = recordPath + "/.actions/" + stage.path;
+            // Runtime caps enforcement: verify resolved path against @caps (glob-aware)
+            if (capCheck) capCheck("exec", fullPath);
+            const capPath = fullPath.split("/").slice(0, 3).join("/");
+            if (ctx.caps.size > 0 && !ctx.caps.has(capPath) && !ctx.caps.has("*")) {
+              throw new Error(`Permission denied: cannot exec '${fullPath}'`);
+            }
+            const rec = record as Record<string, unknown>;
+            // New merge model: if inline params exist, resolve templates and pass ONLY params (no stream merge)
+            const resolvedParams = hasParams ? resolveActionParams(stage.params as Record<string, unknown>, rec) : undefined;
+            const input = hasParams ? [] : [record];
+            const result = await ctx.world.exec(fullPath, input, resolvedParams);
+            if (result != null) {
+              if (Array.isArray(result)) results.push(...result);
+              else results.push(result);
+            }
+          }
+          return results;
+        }
+
+        // Absolute action — with inline params or template path: per-record execution
+        if (hasParams || hasTemplatePath) {
+          const results: unknown[] = [];
+          const records = stream.length > 0 ? stream : [{}];
+          // Path templates resolve from initial stream (trigger event / job context)
+          // Params templates resolve from current stream record
+          const initialRec = (initialStream && initialStream.length > 0
+            && typeof initialStream[0] === "object" && initialStream[0] !== null)
+            ? initialStream[0] as Record<string, unknown>
+            : undefined;
+          for (const record of records) {
+            if (budgetCheck) budgetCheck("actions", 1);
+            const rec = (typeof record === "object" && record !== null ? record : {}) as Record<string, unknown>;
+            // Path: prefer initial stream record (event context), fall back to current stream record
+            const pathRec = initialRec ?? rec;
+            const resolvedPath = hasTemplatePath ? resolveTemplatePath(stage.path, pathRec) : stage.path;
+            // Caps check on resolved path
+            if (capCheck) capCheck("exec", resolvedPath);
+            const capPath = resolvedPath.split("/").slice(0, 3).join("/");
+            if (ctx.caps.size > 0 && !ctx.caps.has(capPath) && !ctx.caps.has("*")) {
+              throw new Error(`Permission denied: cannot exec '${resolvedPath}'`);
+            }
+            const resolvedParams = hasParams ? resolveActionParams(stage.params as Record<string, unknown>, rec) : undefined;
+            const input = hasParams ? [] : [record];
+            const result = await ctx.world.exec(resolvedPath, input, resolvedParams);
+            if (result != null) {
+              if (Array.isArray(result)) results.push(...result);
+              else results.push(result);
+            }
+          }
+          return results;
+        }
+
+        // No params, no template path — passthrough (backward compatible)
+        // Runtime @caps enforcement (script-declared caps)
+        if (capCheck) capCheck("exec", stage.path);
+        const capPath = stage.path.split("/").slice(0, 3).join("/");
+        if (ctx.caps.size > 0 && !ctx.caps.has(capPath) && !ctx.caps.has("*")) {
+          throw new Error(`Permission denied: cannot exec '${stage.path}'`);
+        }
+        const result = await ctx.world.exec(stage.path, stream, undefined);
+        if (result == null) return [];
+        if (!Array.isArray(result)) return [result];
+        return result;
+      }
+
+      default:
+        throw new Error(`Unknown stage kind: ${(stage as any).kind}`);
+    }
+  }
+
+  private resolveQueryVars(query: QueryCondition, variables: Map<string, string | number>): QueryCondition {
+    if (typeof query.value === "string" && /^\$[a-zA-Z_]\w*$/.test(query.value)) {
+      const varName = query.value.slice(1);
+      if (!variables.has(varName)) {
+        throw new Error(`Undefined variable: $${varName}`);
+      }
+      return { ...query, value: variables.get(varName)! };
+    }
+    return query;
+  }
+
+  private resolveWhereVars(clause: WhereClause, variables: Map<string, string | number>): WhereClause {
+    // Only resolve $varName pattern ($ followed by identifier chars only)
+    if (typeof clause.right === "string" && /^\$[a-zA-Z_]\w*$/.test(clause.right)) {
+      const varName = clause.right.slice(1);
+      if (!variables.has(varName)) {
+        throw new Error(`Undefined variable: $${varName}`);
+      }
+      return { ...clause, right: variables.get(varName)! };
+    }
+    return clause;
+  }
+
+  private evaluateWhere(clause: WhereClause, item: unknown): boolean {
+    const left = this.resolveField(item, clause.left);
+    const right = clause.right;
+
+    switch (clause.op) {
+      case "==": return strictEqual(left, right);
+      case "!=": return !strictEqual(left, right);
+      case ">":
+      case "<":
+      case ">=":
+      case "<=": {
+        const l = Number(left);
+        const r = Number(right);
+        // NaN or Infinity in comparisons → reject (safety: non-finite values should not pass guards)
+        if (!Number.isFinite(l) || !Number.isFinite(r)) return false;
+        if (clause.op === ">") return l > r;
+        if (clause.op === "<") return l < r;
+        if (clause.op === ">=") return l >= r;
+        return l <= r;
+      }
+      default: return false;
+    }
+  }
+
+  private resolveField(item: unknown, field: string): unknown {
+    const parts = field.split(".");
+    let current: any = item;
+    for (const part of parts) {
+      if (current == null) return undefined;
+      if (typeof current !== "object" || !Object.hasOwn(current, part)) return undefined;
+      current = current[part];
+    }
+    return current;
+  }
+
+  private evaluateExpression(expr: Expression, item: unknown, variables: Map<string, string | number>): unknown {
+    switch (expr.kind) {
+      case "literal":
+        return expr.value;
+      case "field_access":
+        return this.resolveField(item, expr.path);
+      case "var_ref": {
+        if (!variables.has(expr.name)) {
+          throw new Error(`Undefined variable: $${expr.name}`);
+        }
+        return variables.get(expr.name)!;
+      }
+      case "binary": {
+        const left = this.evaluateExpression(expr.left, item, variables);
+        const right = this.evaluateExpression(expr.right, item, variables);
+        switch (expr.op) {
+          case "+":
+            // String concat if either side is string
+            if (typeof left === "string" || typeof right === "string") {
+              return String(left ?? "") + String(right ?? "");
+            }
+            return Number(left) + Number(right);
+          case "-":
+            return Number(left) - Number(right);
+          case "*":
+            return Number(left) * Number(right);
+          case "/":
+            return Number(left) / Number(right);
+          default:
+            throw new Error(`Unknown operator: ${(expr as any).op}`);
+        }
+      }
+      default:
+        throw new Error(`Unknown expression kind: ${(expr as any).kind}`);
+    }
+  }
+
+  private async executeFanout(stage: FanoutExpression, stream: unknown[], ctx: JobContext, errors: string[], variables: Map<string, string | number> = new Map(), budgetCheck?: (dim: string, increment: number) => void, capCheck?: (op: CapEntry["op"], path: string) => void, initialStream?: unknown[]): Promise<unknown[]> {
+    const results: unknown[][] = [];
+    for (const branch of stage.branches) {
+      let branchStream = [...stream];
+      try {
+        for (const branchStage of branch) {
+          branchStream = await this.executeStage(branchStage, branchStream, ctx, errors, variables, budgetCheck, capCheck, initialStream);
+        }
+        results.push(branchStream);
+      } catch (e: any) {
+        errors.push(`fanout branch failed: ${e.message}`);
+      }
+    }
+    // Return merged results from all successful branches
+    return results.flat();
+  }
+}