@rigkit/engine 0.2.7 → 0.2.9

package/src/engine.ts

@@ -2,6 +2,7 @@ import { existsSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 import { isRigkitConfig, isProviderDefinition, isWorkflowNode } from "./authoring.ts";
+import { runWithStepConsole, type ConsoleLevel, type StepConsoleSink } from "./console-intercept.ts";
 import { loadDotEnv } from "./env-file.ts";
 import { hash } from "./hash.ts";
 import {
@@ -36,8 +37,10 @@ import type {
   ProviderRuntimeMap,
   WorkflowInputFieldDefinition,
   WorkflowDefinition,
+  WorkflowCacheScope,
   WorkflowEvent,
   WorkflowLogStream,
+  WorkflowNodeKind,
   WorkflowNodeDefinition,
   WorkflowOperationDefinition,
   WorkflowPlan,
@@ -59,6 +62,7 @@ export type CreateDevMachineEngineOptions = {
   providers?: BaseProviderPlugin[];
   providerFactory?: ProviderFactory;
   stateFactory?: StateServiceFactory;
+  globalFragmentStateLocator?: GlobalFragmentStateLocator;
   hostStorageDir?: string;
   hostStorageFactory?: ProviderHostStorageFactory;
   interaction?: {
@@ -69,6 +73,18 @@ export type CreateDevMachineEngineOptions = {
 
 export type { InteractionPresenter, InteractionPresentationRequest };
 
+export type GlobalFragmentStateLocator = (fragment: GlobalFragmentStateLocationInput) =>
+  | string
+  | { statePath: string };
+
+export type GlobalFragmentStateLocationInput = {
+  hash: string;
+  workflow: string;
+  nodePath: string;
+  nodeName: string;
+  nodeKind: WorkflowNodeKind;
+};
+
 export type EngineLoadResult = {
   workflow: LoadedWorkflow;
   workflows: LoadedWorkflow[];
@@ -85,6 +101,30 @@ export type EngineProjectInfo = {
   workflow?: WorkflowSummary;
 };
 
+export type EngineCacheScope = WorkflowCacheScope;
+
+export type EngineCacheEntry = {
+  scope: EngineCacheScope;
+  workflow: string;
+  nodePath: string;
+  nodeName: string;
+  nodeKind: string;
+  runId: string;
+  invalidated: boolean;
+  createdAt: string;
+  fragmentHash?: string;
+};
+
+export type EngineCacheList = {
+  entries: EngineCacheEntry[];
+};
+
+export type EngineCacheClearScope = EngineCacheScope | "all";
+
+export type EngineCacheClearResult = {
+  deleted: number;
+};
+
 export type WorkflowSummary = {
   name: string;
   providers: string[];
@@ -165,6 +205,15 @@ type EvaluationState = {
 type EvaluationPreviousTask = {
   name: string;
   path: string;
+  cache: EvaluationCacheTarget;
+};
+
+type EvaluationCacheTarget = {
+  scope: WorkflowCacheScope;
+  workflow: string;
+  nodePath: string;
+  state: StateService;
+  fragmentHash?: string;
 };
 
 type EvaluationResult = EvaluationState & {
@@ -177,10 +226,14 @@ type EvaluateNodeInput = {
   providers: ProviderControllers;
   providerFingerprint: string;
   mode: EvaluationMode;
+  cache: EvaluationCacheTarget;
+  configStack: JsonObject[];
   state: EvaluationState;
   prefix: string[];
+  cachePrefix: string[];
   root: boolean;
   suppressSequenceName?: string;
+  suppressCacheSequenceName?: string;
   planNodes: WorkflowPlanNode[];
   index: { value: number };
 };
@@ -221,6 +274,8 @@ class StepInvalidationRestart extends Error {
 
 let configImportCounter = 0;
 
+// The engine owns the workflow graph, cache, and event emission for one
+// project. The runtime daemon hosts a single long-lived instance per project.
 export class DevMachineEngine {
   private readonly projectDir: string;
   private readonly configPath: string;
@@ -229,6 +284,9 @@ export class DevMachineEngine {
   private providers: BaseProviderPlugin[];
   private readonly providerFactory: ProviderFactory;
   private readonly stateFactory: StateServiceFactory;
+  private readonly globalFragmentStateLocator: GlobalFragmentStateLocator;
+  private readonly globalFragmentStates = new Map<string, { input: GlobalFragmentStateLocationInput; state: StateService }>();
+  private evaluationFragmentHashes: Set<string> | undefined;
   private readonly hostStorageDir: string;
   private readonly hostStorageFactory: ProviderHostStorageFactory;
   private readonly providerHostStorage = new Map<string, ReturnType<ProviderHostStorageFactory>>();
@@ -247,6 +305,9 @@ export class DevMachineEngine {
     this.providers = options.providers ?? [];
     this.providerFactory = options.providerFactory ?? ((input) => this.createProviderFromPlugin(input));
     this.stateFactory = options.stateFactory ?? createStateStore;
+    this.globalFragmentStateLocator = options.globalFragmentStateLocator ?? ((fragment) => ({
+      statePath: join(this.projectDir, ".rigkit", "fragments", fragment.hash, "state.sqlite"),
+    }));
     this.hostStorageDir = options.hostStorageDir ? resolve(options.hostStorageDir) : defaultProviderHostStorageDir();
     this.hostStorageFactory = options.hostStorageFactory ?? createFileProviderHostStorage;
     this.interactionPresenter = options.interaction?.present ?? defaultInteractionPresenter;
@@ -268,7 +329,7 @@ export class DevMachineEngine {
 
     if (!existsSync(this.configPath)) {
       throw new Error(
-        `No Rigkit config found at ${this.configPath}. Create one with "rig init" or pass --config <file>.`,
+        `No Rigkit config found at ${this.configPath}. Create one with "rig init" or pass -config=<file>.`,
       );
     }
 
@@ -534,6 +595,9 @@ export class DevMachineEngine {
                 stringField({ name: "name", required: true }),
               ],
               cli: {
+                positionals: [
+                  { name: "name", index: 0 },
+                ],
                 options: [
                   { name: "workflow", flag: "--workflow" },
                   { name: "name", flag: "--name", required: true },
@@ -590,6 +654,124 @@ export class DevMachineEngine {
     return this.getStateService().listNodeRuns();
   }
 
+  async listCache(input: {
+    workflow?: string;
+    machine?: string;
+    includeUnreachable?: boolean;
+  } = {}): Promise<EngineCacheList> {
+    const workflow = this.getWorkflow(input.workflow ?? input.machine);
+    const providers = await this.createProviders(workflow);
+    const evaluated = await this.evaluate({
+      workflow,
+      providers,
+      mode: "plan",
+    });
+
+    // The plan tells us which row (by runId) would satisfy each cached node
+    // under the *current* code. Those are the only cache rows that matter.
+    const reachableRunIds = new Set<string>();
+    for (const node of evaluated.plan.nodes) {
+      if (node.status === "cached" && node.runId) reachableRunIds.add(node.runId);
+    }
+
+    if (input.includeUnreachable) {
+      const entries: EngineCacheEntry[] = [
+        ...this.getStateService()
+          .listNodeRuns()
+          .filter((run) => run.workflow === workflow.name)
+          .map((run) => cacheEntryForRun(run, "local")),
+      ];
+      for (const fragmentHash of evaluated.fragments) {
+        const fragment = this.globalFragmentStates.get(fragmentHash);
+        if (!fragment) continue;
+        entries.push(
+          ...fragment.state
+            .listNodeRuns()
+            .map((run) => cacheEntryForRun(run, "global", fragmentHash, workflow.name)),
+        );
+      }
+      entries.sort((left, right) => right.createdAt.localeCompare(left.createdAt));
+      return { entries };
+    }
+
+    // Local state is per-project — anything not reachable is dead weight, so
+    // prune as a side effect. Global fragments are shared across projects;
+    // another project might still reach the row we'd consider unreachable
+    // here, so we only filter the display for globals — never delete.
+    const localRuns = this.getStateService().listNodeRuns()
+      .filter((run) => run.workflow === workflow.name);
+    const localStaleIds = localRuns
+      .filter((run) => !reachableRunIds.has(run.id))
+      .map((run) => run.id);
+    if (localStaleIds.length > 0) {
+      this.getStateService().deleteNodeRunsById(localStaleIds);
+    }
+
+    const entries: EngineCacheEntry[] = localRuns
+      .filter((run) => reachableRunIds.has(run.id))
+      .map((run) => cacheEntryForRun(run, "local"));
+
+    for (const fragmentHash of evaluated.fragments) {
+      const fragment = this.globalFragmentStates.get(fragmentHash);
+      if (!fragment) continue;
+      entries.push(
+        ...fragment.state
+          .listNodeRuns()
+          .filter((run) => reachableRunIds.has(run.id))
+          .map((run) => cacheEntryForRun(run, "global", fragmentHash, workflow.name)),
+      );
+    }
+
+    entries.sort((left, right) => right.createdAt.localeCompare(left.createdAt));
+    return { entries };
+  }
+
+  async invalidateCache(input: {
+    workflow?: string;
+    machine?: string;
+    nodePaths?: readonly string[];
+  } = {}): Promise<{ invalidated: number }> {
+    const workflow = this.getWorkflow(input.workflow ?? input.machine);
+    const state = this.getStateService();
+    // If no explicit paths, invalidate every cached node for this workflow.
+    const paths = input.nodePaths && input.nodePaths.length > 0
+      ? [...input.nodePaths]
+      : [...new Set(state.listNodeRuns()
+          .filter((run) => run.workflow === workflow.name && !run.invalidated)
+          .map((run) => run.nodePath))];
+    if (paths.length === 0) return { invalidated: 0 };
+    const ids = state.invalidateNodeRuns({ workflow: workflow.name, nodePaths: paths });
+    return { invalidated: ids.length };
+  }
+
+  async clearCache(input: {
+    workflow?: string;
+    machine?: string;
+    scope?: EngineCacheClearScope;
+  } = {}): Promise<EngineCacheClearResult> {
+    const scope = input.scope ?? "all";
+    const workflow = this.getWorkflow(input.workflow ?? input.machine);
+    const providers = await this.createProviders(workflow);
+    const evaluated = await this.evaluate({
+      workflow,
+      providers,
+      mode: "plan",
+    });
+
+    let deleted = 0;
+    if (scope === "all" || scope === "local") {
+      deleted += this.getStateService().clearNodeRuns({ workflow: workflow.name });
+    }
+    if (scope === "all" || scope === "global") {
+      for (const fragmentHash of evaluated.fragments) {
+        const fragment = this.globalFragmentStates.get(fragmentHash);
+        if (!fragment) continue;
+        deleted += fragment.state.clearNodeRuns();
+      }
+    }
+    return { deleted };
+  }
+
   hasOperation(operationId: string): boolean {
     return this.listWorkflows().some((workflow) => workflow.operations.some((operation) => operation.id === operationId));
   }
@@ -620,14 +802,15 @@ export class DevMachineEngine {
       metadata,
     });
     const operationInput = this.resolveOperationInput(workflow, operation, input.input ?? {});
-    const result = await operation.run({
+    const operationNodePath = `operation.${operation.id}`;
+    const result = await this.withStepConsole(operationNodePath, () => operation.run({
       ...runtime,
       input: Object.freeze(operationInput),
       providers: runtime,
       local: this.local,
       workflow: workflow.name,
-      step: this.createStepRuntime(workflow.name, `operation.${operation.id}`, metadata),
-    });
+      step: this.createStepRuntime(workflow.name, operationNodePath, metadata),
+    }));
     if (result !== undefined) assertJsonValue(result, `Operation ${operation.id} result`);
     return result ?? null;
   }
@@ -694,6 +877,8 @@ export class DevMachineEngine {
   }> {
     const workflow = this.getWorkflow(input.workflow ?? input.machine);
     const providers = await this.createProviders(workflow);
+    const startedAt = Date.now();
+    this.emit({ type: "workflow.apply.started", workflow: workflow.name });
     let result: { context: Record<string, JsonValue>; plan: WorkflowPlan } | undefined;
     const maxRestarts = 8;
     for (let attempt = 0; attempt <= maxRestarts; attempt++) {
@@ -716,6 +901,14 @@ export class DevMachineEngine {
     }
     if (!result) throw new Error(`Workflow ${workflow.name} did not produce an apply result`);
 
+    this.emit({
+      type: "workflow.apply.completed",
+      workflow: workflow.name,
+      nodeCount: result.plan.nodeCount,
+      cachedNodeCount: result.plan.cachedNodeCount,
+      durationMs: Date.now() - startedAt,
+    });
+
     return {
       context: result.context,
       plan: result.plan,
@@ -751,6 +944,7 @@ export class DevMachineEngine {
     };
 
     this.getStateService().saveWorkspace(workspace);
+    this.emit({ type: "workspace.create.started", workspaceName: input.name });
     try {
       await this.runWorkspaceCreate({
         workflow,
@@ -802,7 +996,10 @@ export class DevMachineEngine {
     const draft = cloneWorkspace(workspace);
     const workspaceRuntime = this.createWorkspaceRuntime(draft);
 
-    await workflow.workspace.remove({
+    const removeNodePath = `workspace.${workspace.name}.remove`;
+    const workspaceDef = workflow.workspace;
+    this.emit({ type: "workspace.remove.started", workspaceName: workspace.name });
+    await this.withStepConsole(removeNodePath, () => workspaceDef.remove({
       ...runtime,
       workflow: {
         name: workflow.name,
@@ -811,10 +1008,11 @@ export class DevMachineEngine {
       workspace: workspaceRuntime,
       providers: runtime,
       local: this.local,
-      step: this.createStepRuntime(workflow.name, `workspace.${workspace.name}.remove`, metadata),
-    });
+      step: this.createStepRuntime(workflow.name, removeNodePath, metadata),
+    }));
 
     this.getStateService().deleteWorkspace(input.workspace);
+    this.emit({ type: "workspace.remove.completed", workspaceName: workspace.name });
     return workspace;
   }
 
@@ -822,26 +1020,42 @@ export class DevMachineEngine {
     workflow: LoadedWorkflow;
     providers: ProviderControllers;
     mode: EvaluationMode;
-  }): Promise<{ context: Record<string, JsonValue>; plan: WorkflowPlan }> {
+  }): Promise<{ context: Record<string, JsonValue>; plan: WorkflowPlan; fragments: Set<string> }> {
     const providerFingerprint = providerFingerprintFor(input.workflow);
     const planNodes: WorkflowPlanNode[] = [];
-    const result = await this.evaluateNode({
-      workflow: input.workflow,
-      providers: input.providers,
-      providerFingerprint,
-      mode: input.mode,
-      node: input.workflow.root,
-      state: {
-        context: {},
-        upstreamRunIds: [],
-        previousTasks: [],
-        known: true,
-      },
-      prefix: [],
-      root: true,
-      planNodes,
-      index: { value: 0 },
-    });
+    const previousEvaluationFragmentHashes = this.evaluationFragmentHashes;
+    const fragments = new Set<string>();
+    this.evaluationFragmentHashes = fragments;
+    let result!: EvaluationResult;
+    try {
+      result = await this.evaluateNode({
+        workflow: input.workflow,
+        providers: input.providers,
+        providerFingerprint,
+        mode: input.mode,
+        node: input.workflow.root,
+        cache: {
+          scope: "local",
+          workflow: input.workflow.name,
+          nodePath: "",
+          state: this.getStateService(),
+        },
+        configStack: [],
+        state: {
+          context: {},
+          upstreamRunIds: [],
+          previousTasks: [],
+          known: true,
+        },
+        prefix: [],
+        cachePrefix: [],
+        root: true,
+        planNodes,
+        index: { value: 0 },
+      });
+    } finally {
+      this.evaluationFragmentHashes = previousEvaluationFragmentHashes;
+    }
     const cachedNodeCount = planNodes.filter((node) => node.status === "cached").length;
     const plan: WorkflowPlan = {
       workflow: input.workflow.name,
@@ -855,29 +1069,79 @@ export class DevMachineEngine {
     return {
       context: result.context,
       plan,
+      fragments,
     };
   }
 
   private async evaluateNode(input: EvaluateNodeInput): Promise<EvaluationResult> {
-    if (input.node.nodeKind === "task") {
-      return await this.evaluateTask(input as EvaluateNodeInput & { node: WorkflowTaskNode<any, any, any> });
+    if (input.node.cacheScope === "global" && input.cache.scope !== "global") {
+      const nodePath = nodeDisplayPath(input.node, input.prefix, input.root, input.suppressSequenceName);
+      const fragmentHash = globalFragmentHashFor({
+        node: input.node,
+        providerFingerprint: input.providerFingerprint,
+      });
+      const fragmentState = await this.getGlobalFragmentState({
+        hash: fragmentHash,
+        workflow: input.workflow.name,
+        nodePath,
+        nodeName: input.node.name,
+        nodeKind: input.node.nodeKind,
+      });
+      return await this.evaluateNode({
+        ...input,
+        cache: {
+          scope: "global",
+          workflow: `fragment:${fragmentHash}`,
+          nodePath: "",
+          state: fragmentState,
+          fragmentHash,
+        },
+        cachePrefix: [],
+        suppressCacheSequenceName: undefined,
+      });
     }
 
-    if (input.node.nodeKind === "parallel") {
-      return await this.evaluateParallel(input);
+    if (input.node.cacheScope === "local" && input.cache.scope !== "local") {
+      return await this.evaluateNode({
+        ...input,
+        cache: {
+          scope: "local",
+          workflow: input.workflow.name,
+          nodePath: "",
+          state: this.getStateService(),
+        },
+        cachePrefix: input.prefix,
+        suppressCacheSequenceName: input.suppressSequenceName,
+      });
     }
 
-    const sequencePrefix = input.root || input.suppressSequenceName === input.node.name
-      ? input.prefix
-      : [...input.prefix, input.node.name];
-    let state = input.state;
+    const configuredInput = input.node.config
+      ? { ...input, configStack: [...input.configStack, input.node.config] }
+      : input;
 
-    for (const child of sequenceChildren(input.node)) {
+    if (configuredInput.node.nodeKind === "task") {
+      return await this.evaluateTask(configuredInput as EvaluateNodeInput & { node: WorkflowTaskNode<any, any, any> });
+    }
+
+    if (configuredInput.node.nodeKind === "parallel") {
+      return await this.evaluateParallel(configuredInput);
+    }
+
+    const sequencePrefix = configuredInput.root || configuredInput.suppressSequenceName === configuredInput.node.name
+      ? configuredInput.prefix
+      : [...configuredInput.prefix, configuredInput.node.name];
+    const cacheSequencePrefix = configuredInput.root || configuredInput.suppressCacheSequenceName === configuredInput.node.name
+      ? configuredInput.cachePrefix
+      : [...configuredInput.cachePrefix, configuredInput.node.name];
+    let state = configuredInput.state;
+
+    for (const child of sequenceChildren(configuredInput.node)) {
       const result = await this.evaluateNode({
-        ...input,
+        ...configuredInput,
         node: child,
         state,
         prefix: sequencePrefix,
+        cachePrefix: cacheSequencePrefix,
         root: false,
       });
       state = {
@@ -889,7 +1153,7 @@ export class DevMachineEngine {
       };
     }
 
-    return { ...state, planNodes: input.planNodes };
+    return { ...state, planNodes: configuredInput.planNodes };
   }
 
   private async evaluateParallel(input: EvaluateNodeInput): Promise<EvaluationResult> {
@@ -916,8 +1180,10 @@ export class DevMachineEngine {
           blockedReason: input.state.blockedReason,
         },
         prefix: [...input.prefix, branchName],
+        cachePrefix: [...input.cachePrefix, branchName],
         root: false,
         suppressSequenceName: branchName,
+        suppressCacheSequenceName: branchName,
       });
 
       if (branchState.known) {
@@ -942,13 +1208,14 @@ export class DevMachineEngine {
 
   private async evaluateTask(input: EvaluateNodeInput & { node: WorkflowTaskNode<any, any, any> }): Promise<EvaluationResult> {
     const nodePath = [...input.prefix, input.node.name].join(".");
+    const cacheNodePath = [...input.cachePrefix, input.node.name].join(".");
     const upstreamRunIds = [...input.state.upstreamRunIds];
     const nodeKey = hash({
-      cache: "task-v3",
+      cache: "task-v4",
       kind: "task",
-      path: nodePath,
+      path: cacheNodePath,
       name: input.node.name,
-      version: input.node.options?.version ?? null,
+      config: input.configStack,
       handler: functionFingerprintFor(input.node.handler),
       output: input.node.options?.output ?? null,
     });
@@ -974,8 +1241,10 @@ export class DevMachineEngine {
     }
 
     const cached = await this.findReusableTaskRun({
-      workflow: input.workflow.name,
-      nodePath,
+      state: input.cache.state,
+      workflow: input.cache.workflow,
+      nodePath: cacheNodePath,
+      displayNodePath: nodePath,
       nodeKey,
       providerFingerprint: input.providerFingerprint,
       upstreamRunIds,
@@ -988,6 +1257,10 @@ export class DevMachineEngine {
       const previousTasks = appendPreviousTask(input.state.previousTasks, {
         name: input.node.name,
         path: nodePath,
+        cache: {
+          ...input.cache,
+          nodePath: cacheNodePath,
+        },
       });
       this.emit({ type: "node.cached", nodePath, runId: cached.id });
       input.planNodes.push({
@@ -1035,6 +1308,7 @@ export class DevMachineEngine {
       nodePath,
       metadata,
     });
+    const config = Object.freeze(mergeConfigStack(input.configStack));
     const step = this.createStepRuntime(
       input.workflow.name,
       nodePath,
@@ -1042,16 +1316,26 @@ export class DevMachineEngine {
       input.state.context,
       input.state.previousTasks,
     );
-    const result = await input.node.handler({
+    const result = await this.withStepConsole(nodePath, () => input.node.handler({
       ...runtime,
       providers: runtime,
       step,
-    });
+      config,
+    }));
     if (isStepInvalidation(result)) {
-      const invalidatedRunIds = this.getStateService().invalidateNodeRuns({
-        workflow: input.workflow.name,
-        nodePaths: [result.targetNodePath, nodePath],
-      });
+      const targetTask = input.state.previousTasks.find((task) => task.path === result.targetNodePath);
+      const invalidatedRunIds = targetTask
+        ? this.invalidateTaskCaches([
+          targetTask.cache,
+          {
+            ...input.cache,
+            nodePath: cacheNodePath,
+          },
+        ])
+        : this.invalidateTaskCaches([{
+          ...input.cache,
+          nodePath: cacheNodePath,
+        }]);
       throw new StepInvalidationRestart({
         workflow: input.workflow.name,
         target: result.target,
@@ -1073,8 +1357,8 @@ export class DevMachineEngine {
     const artifacts = collectArtifacts(output);
     const record: WorkflowNodeRunRecord = {
       id: crypto.randomUUID(),
-      workflow: input.workflow.name,
-      nodePath,
+      workflow: input.cache.workflow,
+      nodePath: cacheNodePath,
       nodeName: input.node.name,
       nodeKind: input.node.nodeKind,
       nodeKey,
@@ -1087,7 +1371,7 @@ export class DevMachineEngine {
       metadata,
     };
 
-    this.getStateService().saveNodeRun(record);
+    input.cache.state.saveNodeRun(record);
     for (const artifact of artifacts) {
       const providerId = providerIdOf(artifact);
       this.emit({
@@ -1103,6 +1387,10 @@ export class DevMachineEngine {
     const previousTasks = appendPreviousTask(input.state.previousTasks, {
       name: input.node.name,
       path: nodePath,
+      cache: {
+        ...input.cache,
+        nodePath: cacheNodePath,
+      },
     });
 
     return {
@@ -1115,8 +1403,10 @@ export class DevMachineEngine {
   }
 
   private async findReusableTaskRun(input: {
+    state: StateService;
     workflow: string;
     nodePath: string;
+    displayNodePath: string;
     nodeKey: string;
     providerFingerprint: string;
     upstreamRunIds: readonly string[];
@@ -1124,11 +1414,11 @@ export class DevMachineEngine {
     outputSchema?: OutputSchema;
     cacheTTL?: WorkflowTaskCacheTTL;
   }): Promise<WorkflowNodeRunRecord | undefined> {
-    const cached = this.getStateService().findReusableNodeRun(input);
+    const cached = input.state.findReusableNodeRun(input);
     if (!cached) return undefined;
     if (!isCacheFresh(cached.createdAt, input.cacheTTL)) return undefined;
 
-    const parsed = normalizeTaskOutput(input.nodePath, cached.output, input.outputSchema, "cached");
+    const parsed = normalizeTaskOutput(input.displayNodePath, cached.output, input.outputSchema, "cached");
     if (!parsed) return undefined;
 
     for (const artifact of cached.artifacts) {
@@ -1223,8 +1513,10 @@ export class DevMachineEngine {
       metadata,
     });
 
+    const createNodePath = `workspace.${input.name}.create`;
+    const workspaceDef = input.workflow.workspace;
     try {
-      const data = await input.workflow.workspace.create({
+      const data = await this.withStepConsole(createNodePath, () => workspaceDef.create({
         ...providers,
         workflow: {
           name: input.workflow.name,
@@ -1235,8 +1527,8 @@ export class DevMachineEngine {
         },
         providers,
         local: this.local,
-        step: this.createStepRuntime(input.workflow.name, `workspace.${input.name}.create`, metadata),
-      });
+        step: this.createStepRuntime(input.workflow.name, createNodePath, metadata),
+      }));
       assertJsonValue(data, `Workflow ${input.workflow.name} workspace create result`);
       if (!isPlainObject(data)) {
         throw new Error(`Workflow ${input.workflow.name} workspace create result must be an object`);
@@ -1266,7 +1558,13 @@ export class DevMachineEngine {
     const workspace = this.createWorkspaceRuntime(draft);
     const operationInput = this.resolveOperationInput(input.workflow, input.operation, input.rawInput ?? {});
 
-    const result = await input.operation.run({
+    const workspaceOperationNodePath = `workspace.${input.workspace.name}.${input.operation.id}`;
+    this.emit({
+      type: "workspace.operation.started",
+      workspaceName: input.workspace.name,
+      operationId: input.operation.id,
+    });
+    const result = await this.withStepConsole(workspaceOperationNodePath, () => input.operation.run({
       ...providers,
       workflow: {
         name: input.workflow.name,
@@ -1278,11 +1576,16 @@ export class DevMachineEngine {
       local: this.local,
       step: this.createStepRuntime(
         input.workflow.name,
-        `workspace.${input.workspace.name}.${input.operation.id}`,
+        workspaceOperationNodePath,
         metadata,
       ),
-    });
+    }));
     if (result !== undefined) assertJsonValue(result, `Workspace operation ${input.operation.id} result`);
+    this.emit({
+      type: "workspace.operation.completed",
+      workspaceName: input.workspace.name,
+      operationId: input.operation.id,
+    });
     return result ?? null;
   }
 
@@ -1293,6 +1596,22 @@ export class DevMachineEngine {
     }) as WorkspaceRuntimeRecord<Data>;
   }
 
+  // Wraps a user step handler in an AsyncLocalStorage scope so that any
+  // console.log / debug / warn / error invoked inside (transitively, through
+  // any helper or third-party SDK) is captured and emitted as a log.output
+  // event tied to this step's node path.
+  private withStepConsole<T>(nodePath: string, fn: () => Promise<T> | T): Promise<T> | T {
+    const sink: StepConsoleSink = ({ level, message }) => {
+      this.emit({
+        type: "log.output",
+        nodePath,
+        stream: consoleLevelToLogStream(level),
+        data: message,
+      });
+    };
+    return runWithStepConsole(sink, fn);
+  }
+
   private createStepRuntime<Context extends JsonObject = JsonObject>(
     workflow: string,
     nodePath: string,
@@ -1307,15 +1626,6 @@ export class DevMachineEngine {
       metadata: (value: JsonObject) => {
         Object.assign(metadata, value);
       },
-      log: (data: string, options: { stream?: WorkflowLogStream; label?: string } = {}) => {
-        this.emit({
-          type: "log.output",
-          nodePath,
-          stream: options.stream ?? "info",
-          label: options.label,
-          data,
-        });
-      },
       invalidate: <Target extends string>(target: Target) => {
         const matches = previousTasks.filter((task) => task.name === target || task.path === target);
         if (matches.length === 0) {
@@ -1459,6 +1769,51 @@ export class DevMachineEngine {
     return this.state;
   }
 
+  private async getGlobalFragmentState(input: GlobalFragmentStateLocationInput): Promise<StateService> {
+    this.evaluationFragmentHashes?.add(input.hash);
+    const existing = this.globalFragmentStates.get(input.hash);
+    if (existing) return existing.state;
+
+    const located = this.globalFragmentStateLocator(input);
+    const statePath = typeof located === "string" ? located : located.statePath;
+    const state = this.stateFactory({
+      projectDir: this.projectDir,
+      configPath: this.configPath,
+      statePath,
+      source: {
+        kind: "global-fragment",
+        hash: input.hash,
+        workflow: input.workflow,
+        nodePath: input.nodePath,
+        nodeName: input.nodeName,
+        nodeKind: input.nodeKind,
+      },
+    });
+    await state.syncSchema();
+    this.globalFragmentStates.set(input.hash, { input, state });
+    return state;
+  }
+
+  private invalidateTaskCaches(targets: EvaluationCacheTarget[]): string[] {
+    const grouped = new Map<string, { target: EvaluationCacheTarget; nodePaths: Set<string> }>();
+    for (const target of targets) {
+      const key = `${target.state.path}\0${target.workflow}`;
+      const existing = grouped.get(key);
+      if (existing) {
+        existing.nodePaths.add(target.nodePath);
+      } else {
+        grouped.set(key, { target, nodePaths: new Set([target.nodePath]) });
+      }
+    }
+
+    return [...grouped.values()].flatMap(({ target, nodePaths }) =>
+      target.state.invalidateNodeRuns({
+        workflow: target.workflow,
+        nodePaths: [...nodePaths],
+      })
+    );
+  }
+
   private async createProviders(workflow: LoadedWorkflow): Promise<ProviderControllers> {
     const entries = await Promise.all(
       Object.entries(workflow.providers).map(async ([name, provider]) => {
@@ -1703,6 +2058,88 @@ function providerPluginFingerprint(plugin: unknown): unknown {
   };
 }
 
+function globalFragmentHashFor(input: {
+  node: WorkflowNodeDefinition<any, any, any>;
+  providerFingerprint: string;
+}): string {
+  return `sha256-${hash({
+    cache: "fragment-v1",
+    graph: graphFingerprintFor(input.node),
+    providerFingerprint: input.providerFingerprint,
+  })}`;
+}
+
+function graphFingerprintFor(node: WorkflowNodeDefinition<any, any, any>): unknown {
+  if (node.nodeKind === "task") {
+    const task = node as WorkflowTaskNode<any, any, any>;
+    return {
+      kind: "task",
+      name: task.name,
+      scope: task.cacheScope ?? null,
+      config: task.config ?? null,
+      handler: functionFingerprintFor(task.handler),
+      output: task.options?.output ?? null,
+      cacheTTL: task.options?.cacheTTL ?? null,
+    };
+  }
+
+  if (node.nodeKind === "parallel") {
+    return {
+      kind: "parallel",
+      name: node.name,
+      scope: node.cacheScope ?? null,
+      config: node.config ?? null,
+      branches: Object.fromEntries(
+        Object.entries(parallelBranches(node)).map(([name, branch]) => [name, graphFingerprintFor(branch)]),
+      ),
+    };
+  }
+
+  return {
+    kind: "sequence",
+    name: node.name,
+    scope: node.cacheScope ?? null,
+    config: node.config ?? null,
+    children: sequenceChildren(node).map((child) => graphFingerprintFor(child)),
+  };
+}
+
+function nodeDisplayPath(
+  node: WorkflowNodeDefinition<any, any, any>,
+  prefix: string[],
+  root: boolean,
+  suppressSequenceName?: string,
+): string {
+  if (node.nodeKind === "task") return [...prefix, node.name].join(".");
+  if (node.nodeKind === "sequence") {
+    return (root || suppressSequenceName === node.name ? prefix : [...prefix, node.name]).join(".");
+  }
+  return [...prefix, node.name].join(".");
+}
+
+function cacheEntryForRun(
+  run: WorkflowNodeRunRecord,
+  scope: WorkflowCacheScope,
+  fragmentHash?: string,
+  workflow?: string,
+): EngineCacheEntry {
+  return {
+    scope,
+    workflow: workflow ?? run.workflow,
+    nodePath: run.nodePath,
+    nodeName: run.nodeName,
+    nodeKind: run.nodeKind,
+    runId: run.id,
+    invalidated: run.invalidated,
+    createdAt: run.createdAt,
+    ...(fragmentHash ? { fragmentHash } : {}),
+  };
+}
+
+function mergeConfigStack(configStack: readonly JsonObject[]): JsonObject {
+  return Object.assign({}, ...configStack);
+}
+
 function functionFingerprintFor(fn: Function): { name: string; length: number; source: string } {
   return {
     name: fn.name,
@@ -2154,3 +2591,14 @@ async function defaultInteractionPresenter(request: InteractionPresentationReque
   if (request.instructions) console.error(request.instructions);
   console.error(`Open ${request.url}`);
 }
+
+function consoleLevelToLogStream(level: ConsoleLevel): WorkflowLogStream {
+  switch (level) {
+    case "debug": return "debug";
+    case "warn":  return "warn";
+    case "error": return "stderr";
+    case "info":
+    case "log":
+    default:      return "info";
+  }
+}