@highstate/backend 0.4.3 → 0.4.5

package/dist/index.mjs

@@ -1,7 +1,6 @@
 import { z } from 'zod';
-import { pickBy, mapKeys, mapValues, funnel } from 'remeda';
+import { pickBy, mapKeys, mapValues, funnel, omit, pick } from 'remeda';
 import { BetterLock } from 'better-lock';
-import { consola } from 'consola';
 import { basename, relative, dirname, resolve as resolve$1 } from 'node:path';
 import { findWorkspaceDir, readPackageJSON } from 'pkg-types';
 import { fileURLToPath } from 'node:url';
@@ -11,8 +10,9 @@ import Watcher from 'watcher';
 import { resolve } from 'import-meta-resolve';
 import { readdir, readFile, writeFile, mkdir } from 'node:fs/promises';
 import { getInstanceId, isUnitModel, parseInstanceId } from '@highstate/contract';
-import { b as instanceModelSchema, I as InputHashCalculator, l as createInstanceState, t as terminalFactorySchema, h as instanceRepresentationSchema, f as instanceStatusFieldMapSchema, c as compositeInstanceSchema, s as projectOperationSchema, k as instanceStateSchema, m as applyInstanceStatePatch, o as createInstanceStateFrontendPatch } from './input-hash-C8HEDMjz.mjs';
-import 'crypto-hash';
+import { h as hubModelSchema, i as instanceModelSchema, c as createInputResolver, a as createInputHashResolver, b as createInstanceState, d as instanceTerminalSchema, e as instancePageSchema, f as instanceFileSchema, g as instanceStatusFieldSchema, j as instanceTriggerSchema, k as compositeInstanceSchema, p as projectOperationSchema, l as instanceStateSchema, m as isFinalOperationStatus, t as terminalSessionSchema, n as applyPartialInstanceState, o as createInstanceStateFrontendPatch } from './terminal-C1HuyJ6e.mjs';
+import { sha256 } from 'crypto-hash';
+import 'ajv';
 import { Readable, PassThrough } from 'node:stream';
 import { tmpdir, homedir } from 'node:os';
 import spawn from 'nano-spawn';
@@ -57,6 +57,12 @@ function errorToString(error) {
   }
   return JSON.stringify(error);
 }
+function arrayAccumulator(values, value) {
+  if (!values) {
+    return [value];
+  }
+  return [...values, value];
+}
 
 class LocalPulumiHost {
   constructor(logger) {
@@ -74,7 +80,7 @@ class LocalPulumiHost {
     }
   }
   async runInline(projectId, pulumiProjectName, pulumiStackName, program, fn) {
-    return this.lock.acquire(`${pulumiProjectName}.${pulumiStackName}`, async () => {
+    return await this.lock.acquire(`${pulumiProjectName}.${pulumiStackName}`, async () => {
       const { LocalWorkspace } = await import('@pulumi/pulumi/automation/index.js');
       const stack = await LocalWorkspace.createOrSelectStack(
         {
@@ -87,13 +93,16 @@ class LocalPulumiHost {
             name: pulumiProjectName,
             runtime: "nodejs"
           },
-          envVars: { PULUMI_CONFIG_PASSPHRASE: this.getPassword(projectId) }
+          envVars: {
+            PULUMI_CONFIG_PASSPHRASE: this.getPassword(projectId),
+            PULUMI_K8S_AWAIT_ALL: "true"
+          }
         }
       );
       try {
         return await runWithRetryOnError(
           () => fn(stack),
-          (error) => LocalPulumiHost.tryUnlockStack(stack, error)
+          (error) => this.tryUnlockStack(stack, error)
         );
       } catch (e) {
         if (e instanceof Error && e.message.includes("canceled")) {
@@ -104,12 +113,12 @@ class LocalPulumiHost {
     });
   }
   async runEmpty(projectId, pulumiProjectName, pulumiStackName, fn) {
-    return this.runInline(projectId, pulumiProjectName, pulumiStackName, async () => {
+    return await this.runInline(projectId, pulumiProjectName, pulumiStackName, async () => {
     }, fn);
   }
   // TODO: extract args to options object
   async runLocal(projectId, pulumiProjectName, pulumiStackName, programPathResolver, fn, stackConfig) {
-    return this.lock.acquire(`${pulumiProjectName}.${pulumiStackName}`, async () => {
+    return await this.lock.acquire(`${pulumiProjectName}.${pulumiStackName}`, async () => {
       const { LocalWorkspace } = await import('@pulumi/pulumi/automation/index.js');
       const stack = await LocalWorkspace.createOrSelectStack(
         {
@@ -125,14 +134,17 @@ class LocalPulumiHost {
             [pulumiStackName]: {
               config: stackConfig
             }
-          } : undefined,
-          envVars: { PULUMI_CONFIG_PASSPHRASE: this.getPassword(projectId) }
+          } : void 0,
+          envVars: {
+            PULUMI_CONFIG_PASSPHRASE: this.getPassword(projectId),
+            PULUMI_K8S_AWAIT_ALL: "true"
+          }
         }
       );
       try {
         return await runWithRetryOnError(
           () => fn(stack),
-          (error) => LocalPulumiHost.tryUnlockStack(stack, error)
+          (error) => this.tryUnlockStack(stack, error)
         );
       } catch (e) {
         if (e instanceof Error && e.message.includes("canceled")) {
@@ -153,9 +165,9 @@ class LocalPulumiHost {
   getPassword(projectId) {
     return this.sharedPassword || this.passwords.get(projectId) || "";
   }
-  static async tryUnlockStack(stack, error) {
+  async tryUnlockStack(stack, error) {
     if (error instanceof Error && error.message.includes("the stack is currently locked")) {
-      consola.warn("Unlocking stack", stack.name);
+      this.logger.warn({ stackName: stack.name }, "inlocking stack");
       await stack.cancel();
       return true;
     }
@@ -446,7 +458,8 @@ const localProjectBackendConfig = z.object({
   HIGHSTATE_BACKEND_PROJECT_PROJECTS_DIR: z.string().optional()
 });
 const projectModelSchema = z.object({
-  instances: z.record(instanceModelSchema)
+  instances: z.record(instanceModelSchema),
+  hubs: z.record(hubModelSchema)
 });
 class LocalProjectBackend {
   constructor(projectsDir) {
@@ -460,10 +473,13 @@ class LocalProjectBackend {
       throw new Error("Failed to get project names", { cause: error });
     }
   }
-  async getInstances(projectId) {
+  async getProject(projectId) {
     try {
       const project = await this.loadProject(projectId);
-      return Object.values(project.instances);
+      return {
+        instances: Object.values(project.instances),
+        hubs: Object.values(project.hubs)
+      };
     } catch (error) {
       throw new Error("Failed to get project instances", { cause: error });
     }
@@ -481,34 +497,40 @@ class LocalProjectBackend {
       throw new Error("Failed to create project instance", { cause: error });
     }
   }
-  async moveInstance(projectId, instanceId, position) {
-    try {
-      return await this.withInstance(projectId, instanceId, (instance) => {
-        instance.position = position;
-        return instance;
-      });
-    } catch (error) {
-      throw new Error("Failed to move project instance", { cause: error });
-    }
-  }
-  async updateInstanceArgs(projectId, instanceId, args) {
-    try {
-      return await this.withInstance(projectId, instanceId, (instance) => {
-        instance.args = args;
-        return instance;
-      });
-    } catch (error) {
-      throw new Error("Failed to update project instance arguments", { cause: error });
-    }
-  }
-  async updateInstanceInputs(projectId, instanceId, inputs) {
+  async updateInstance(projectId, instanceId, patch) {
     try {
       return await this.withInstance(projectId, instanceId, (instance) => {
-        instance.inputs = inputs;
+        if (patch.args) {
+          instance.args = patch.args;
+        }
+        if (patch.position) {
+          instance.position = patch.position;
+        }
+        if (patch.inputs) {
+          if (Object.keys(patch.inputs).length > 0) {
+            instance.inputs = patch.inputs;
+          } else {
+            delete instance.inputs;
+          }
+        }
+        if (patch.hubInputs) {
+          if (Object.keys(patch.hubInputs).length > 0) {
+            instance.hubInputs = patch.hubInputs;
+          } else {
+            delete instance.hubInputs;
+          }
+        }
+        if (patch.injectionInputs) {
+          if (patch.injectionInputs.length > 0) {
+            instance.injectionInputs = patch.injectionInputs;
+          } else {
+            delete instance.injectionInputs;
+          }
+        }
         return instance;
       });
     } catch (error) {
-      throw new Error("Failed to update project instance inputs", { cause: error });
+      throw new Error("Failed to update project instance", { cause: error });
     }
   }
   async deleteInstance(projectId, instanceId) {
@@ -519,14 +541,21 @@ class LocalProjectBackend {
         }
         delete project.instances[instanceId];
         for (const otherInstance of Object.values(project.instances)) {
-          for (const [inputKey, input] of Object.entries(otherInstance.inputs)) {
-            if (Array.isArray(input)) {
-              otherInstance.inputs[inputKey] = input.filter(
-                (inputItem) => inputItem.instanceId !== instanceId
-              );
-            } else if (input.instanceId === instanceId) {
-              delete otherInstance.inputs[inputKey];
-            }
+          if (!otherInstance.inputs) {
+            continue;
+          }
+          this.deleteInstanceReferences(otherInstance.inputs, instanceId);
+          if (Object.keys(otherInstance.inputs).length === 0) {
+            delete otherInstance.inputs;
+          }
+        }
+        for (const hub of Object.values(project.hubs)) {
+          if (!hub.inputs) {
+            continue;
+          }
+          hub.inputs = hub.inputs.filter((input) => input.instanceId !== instanceId);
+          if (hub.inputs.length === 0) {
+            delete hub.inputs;
           }
         }
       });
@@ -534,6 +563,14 @@ class LocalProjectBackend {
       throw new Error("Failed to delete project instance", { cause: error });
     }
   }
+  deleteInstanceReferences(inputs, instanceId) {
+    for (const [inputKey, input] of Object.entries(inputs)) {
+      inputs[inputKey] = input.filter((inputItem) => inputItem.instanceId !== instanceId);
+      if (inputs[inputKey].length === 0) {
+        delete inputs[inputKey];
+      }
+    }
+  }
   async renameInstance(projectId, instanceId, newName) {
     try {
       return await this.withProject(projectId, (project) => {
@@ -550,24 +587,114 @@ class LocalProjectBackend {
         instance.name = newName;
         project.instances[newInstanceId] = instance;
         for (const otherInstance of Object.values(project.instances)) {
-          for (const input of Object.values(otherInstance.inputs)) {
-            if (Array.isArray(input)) {
-              for (const inputItem of input) {
-                if (inputItem.instanceId === instanceId) {
-                  inputItem.instanceId = instance.id;
-                }
-              }
-            } else if (input.instanceId === instanceId) {
-              input.instanceId = instance.id;
-            }
+          for (const inputs of Object.values(otherInstance.inputs ?? {})) {
+            this.renameInstanceReferences(inputs, instanceId, instance.id);
           }
         }
+        for (const hub of Object.values(project.hubs)) {
+          this.renameInstanceReferences(hub.inputs ?? [], instanceId, instance.id);
+        }
         return instance;
       });
     } catch (error) {
       throw new Error("Failed to rename project instance", { cause: error });
     }
   }
+  renameInstanceReferences(inputs, instanceId, newInstanceId) {
+    for (const input of inputs) {
+      if (input.instanceId === instanceId) {
+        input.instanceId = newInstanceId;
+      }
+    }
+  }
+  async createHub(projectId, hub) {
+    try {
+      return await this.withProject(projectId, (project) => {
+        if (project.hubs[hub.id]) {
+          throw new Error(`Hub ${hub.id} already exists`);
+        }
+        project.hubs[hub.id] = hub;
+        return hub;
+      });
+    } catch (error) {
+      throw new Error("Failed to create project hub", { cause: error });
+    }
+  }
+  async updateHub(projectId, hubId, patch) {
+    try {
+      return await this.withProject(projectId, (project) => {
+        const hub = project.hubs[hubId];
+        if (!hub) {
+          throw new Error(`Hub ${hubId} not found`);
+        }
+        if (patch.position) {
+          hub.position = patch.position;
+        }
+        if (patch.inputs) {
+          if (patch.inputs.length > 0) {
+            hub.inputs = patch.inputs;
+          } else {
+            delete hub.inputs;
+          }
+        }
+        if (patch.injectionInputs) {
+          if (patch.injectionInputs.length > 0) {
+            hub.injectionInputs = patch.injectionInputs;
+          } else {
+            delete hub.injectionInputs;
+          }
+        }
+        return hub;
+      });
+    } catch (error) {
+      throw new Error("Failed to update project hub", { cause: error });
+    }
+  }
+  async deleteHub(projectId, hubId) {
+    try {
+      await this.withProject(projectId, (project) => {
+        if (!project.hubs[hubId]) {
+          throw new Error(`Hub ${hubId} not found`);
+        }
+        delete project.hubs[hubId];
+        for (const instance of Object.values(project.instances)) {
+          if (instance.hubInputs) {
+            this.deleteHubReferences(instance.hubInputs, hubId);
+            if (Object.keys(instance.hubInputs).length === 0) {
+              delete instance.hubInputs;
+            }
+          }
+          if (instance.injectionInputs) {
+            instance.injectionInputs = instance.injectionInputs.filter(
+              (input) => input.hubId !== hubId
+            );
+            if (instance.injectionInputs.length === 0) {
+              delete instance.injectionInputs;
+            }
+          }
+        }
+        for (const otherHub of Object.values(project.hubs)) {
+          if (!otherHub.injectionInputs) {
+            continue;
+          }
+          otherHub.injectionInputs = otherHub.injectionInputs.filter((input) => input.hubId !== hubId);
+          if (otherHub.injectionInputs.length === 0) {
+            delete otherHub.injectionInputs;
+          }
+        }
+      });
+    } catch (error) {
+      throw new Error("Failed to delete project hub", { cause: error });
+    }
+  }
+  deleteHubReferences(inputs, hubId) {
+    for (const [inputKey, input] of Object.entries(inputs)) {
+      inputs[inputKey] = input.filter((inputItem) => inputItem.hubId !== hubId);
+      if (inputs[inputKey].length === 0) {
+        delete inputs[inputKey];
+      }
+    }
+  }
   getProjectPath(projectId) {
     return `${this.projectsDir}/${projectId}.json`;
   }
@@ -578,14 +705,14 @@ class LocalProjectBackend {
       return projectModelSchema.parse(JSON.parse(content));
     } catch (error) {
       if (error instanceof Error && "code" in error && error.code === "ENOENT") {
-        return { instances: {} };
+        return { instances: {}, hubs: {} };
       }
       throw error;
     }
   }
   async writeProject(projectId, project) {
     const projectPath = this.getProjectPath(projectId);
-    const content = JSON.stringify(project, undefined, 2);
+    const content = JSON.stringify(project, void 0, 2);
     await writeFile(projectPath, content);
   }
   async withInstance(projectId, instanceId, callback) {
@@ -653,19 +780,10 @@ class ProjectManager {
     this.logger = logger;
   }
   async getCompositeInstance(projectId, instanceId) {
-    const instances = await this.projectBackend.getInstances(projectId);
-    const instance = instances.find((instance2) => instance2.id === instanceId);
-    const instanceMap = /* @__PURE__ */ new Map();
-    for (const instance2 of instances) {
-      instanceMap.set(instance2.id, instance2);
-    }
-    if (!instance) {
-      throw new Error(`instance not found: ${instanceId}`);
-    }
-    const calculator = new InputHashCalculator(instanceMap);
+    const { resolveInputHash } = await this.prepareInputHashResolver(projectId, instanceId);
     const [compositeInstance, actualInputHash] = await Promise.all([
       this.stateBackend.getCompositeInstance(projectId, instanceId),
-      calculator.calculate(instance)
+      resolveInputHash(instanceId)
     ]);
     if (compositeInstance && compositeInstance.inputHash === actualInputHash) {
       return compositeInstance;
@@ -678,7 +796,7 @@ class ProjectManager {
       projectId,
       instanceId,
       actualInputHash,
-      instanceMap
+      resolveInputHash
     );
     await this.operationManager.launch({
       type: "evaluate",
@@ -692,18 +810,8 @@ class ProjectManager {
     await this.updateInstanceChildren(projectId, createdInstance);
     return createdInstance;
   }
-  async updateInstanceArgs(projectId, instanceId, args) {
-    const instance = await this.projectBackend.updateInstanceArgs(projectId, instanceId, args);
-    await this.updateInstanceChildren(projectId, instance);
-    return instance;
-  }
-  async updateInstanceInputs(projectId, instanceId, inputs) {
-    const instance = await this.projectBackend.updateInstanceInputs(projectId, instanceId, inputs);
-    await this.updateInstanceChildren(projectId, instance);
-    return instance;
-  }
-  async moveInstance(projectId, instanceId, position) {
-    const instance = await this.projectBackend.moveInstance(projectId, instanceId, position);
+  async updateInstance(projectId, instanceId, patch) {
+    const instance = await this.projectBackend.updateInstance(projectId, instanceId, patch);
     await this.updateInstanceChildren(projectId, instance);
     return instance;
   }
@@ -719,7 +827,10 @@ class ProjectManager {
     ]);
   }
   async updateInstanceChildren(projectId, instance) {
-    const library = await this.library.loadLibrary();
+    const { resolveInputHash, library } = await this.prepareInputHashResolver(
+      projectId,
+      instance.id
+    );
     const component = library.components[instance.type];
     if (!component) {
       return;
@@ -727,14 +838,8 @@ class ProjectManager {
     if (isUnitModel(component)) {
       return;
     }
-    const instances = await this.projectBackend.getInstances(projectId);
-    const instanceMap = /* @__PURE__ */ new Map();
-    for (const instance2 of instances) {
-      instanceMap.set(instance2.id, instance2);
-    }
-    const inputHashCalculator = new InputHashCalculator(instanceMap);
+    const expectedInputHash = await resolveInputHash(instance.id);
     const inputHash = await this.stateBackend.getCompositeInstanceInputHash(projectId, instance.id);
-    const expectedInputHash = await inputHashCalculator.calculate(instance);
     if (inputHash !== expectedInputHash) {
       this.logger.info("re-evaluating instance since input hash has changed", {
         projectId,
@@ -747,13 +852,12 @@ class ProjectManager {
       });
     }
   }
-  async waitForCompositeInstance(projectId, instanceId, expectedInputHash, instanceMap) {
+  async waitForCompositeInstance(projectId, instanceId, expectedInputHash, resolveInputHash) {
     for await (const instance of this.operationManager.watchCompositeInstance(
       projectId,
       instanceId
     )) {
-      const calculator = new InputHashCalculator(instanceMap);
-      const actualInputHash = await calculator.calculate(instance.instance);
+      const actualInputHash = await resolveInputHash(instanceId);
       if (actualInputHash !== expectedInputHash) {
         throw new Error("Composite instance input hash changed while waiting for evaluation");
       }
@@ -761,6 +865,54 @@ class ProjectManager {
     }
     throw new Error("Composite instance stream ended without receiving the expected instance");
   }
+  async prepareInputHashResolver(projectId, instanceId) {
+    const { instances, hubs } = await this.projectBackend.getProject(projectId);
+    const library = await this.library.loadLibrary();
+    const filteredInstances = instances.filter((instance2) => instance2.type in library.components);
+    const states = await this.stateBackend.getInstanceStates(projectId);
+    const stateMap = new Map(states.map((state) => [state.id, state]));
+    const instance = filteredInstances.find((instance2) => instance2.id === instanceId);
+    if (!instance) {
+      throw new Error(`Instance not found: ${instanceId}`);
+    }
+    const inputResolverNodes = /* @__PURE__ */ new Map();
+    for (const instance2 of filteredInstances) {
+      inputResolverNodes.set(`instance:${instance2.id}`, {
+        kind: "instance",
+        instance: instance2,
+        component: library.components[instance2.type]
+      });
+    }
+    for (const hub of hubs) {
+      inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub });
+    }
+    const resolveInputs = createInputResolver(
+      inputResolverNodes,
+      this.logger.child({ resolver: "input-resolver" })
+    );
+    const inputHashNodes = /* @__PURE__ */ new Map();
+    for (const instance2 of filteredInstances) {
+      const output = await resolveInputs(`instance:${instance2.id}`);
+      if (output.kind !== "instance") {
+        throw new Error("Expected instance node");
+      }
+      inputHashNodes.set(instance2.id, {
+        instance: instance2,
+        resolvedInputs: output.resolvedInputs,
+        state: stateMap.get(instance2.id),
+        sourceHash: void 0
+        // implement source hash
+      });
+    }
+    const resolveInputHash = createInputHashResolver(
+      inputHashNodes,
+      this.logger.child({ resolver: "input-hash-resolver" })
+    );
+    return {
+      resolveInputHash,
+      library
+    };
+  }
   static create(projectBackend, stateBackend, operationManager, library, logger) {
     return new ProjectManager(
       projectBackend,
@@ -788,8 +940,13 @@ done
 
 # Create files
 for key in "\${filesKeys[@]}"; do
-  content=$(jq -r ".files[\\"$key\\"]" <<<"$data")
-  echo -n "$content" > "$key"
+  isBinary=$(jq -r ".files[\\"$key\\"].isBinary // false" <<<"$data")
+  content=$(jq -r ".files[\\"$key\\"].content" <<<"$data")
+  if [ "$isBinary" = "true" ]; then
+    echo -n "$content" | base64 -d > "$key"
+  else
+    echo -n "$content" > "$key"
+  fi
 done
 
 # Execute the command, keeping stdin/stdout open
@@ -863,82 +1020,151 @@ function createTerminalBackend(config, logger) {
 
 const notAttachedTerminalLifetime = 30 * 1e3;
 class TerminalManager {
-  constructor(terminalBackend, logger) {
+  constructor(terminalBackend, stateBackend, runnerBackend, logger) {
     this.terminalBackend = terminalBackend;
+    this.stateBackend = stateBackend;
+    this.runnerBackend = runnerBackend;
     this.logger = logger;
   }
-  terminals = /* @__PURE__ */ new Map();
-  create(factory) {
-    const terminal = {
+  managedTerminals = /* @__PURE__ */ new Map();
+  async createSession(projectId, instanceId, terminalName) {
+    const terminalSession = {
       id: randomUUID(),
-      abortController: new AbortController(),
-      attached: false,
-      stdin: new PassThrough(),
-      stdout: new PassThrough(),
-      history: []
+      terminalName
     };
-    terminal.stdout.on("data", (data) => {
-      terminal.history.push(String(data));
-    });
-    this.terminals.set(terminal.id, terminal);
-    void this.terminalBackend.run({
-      factory,
-      stdin: terminal.stdin,
-      stdout: terminal.stdout,
-      signal: terminal.abortController.signal
-    }).catch((error) => {
-      this.logger.error("terminal failed", { id: terminal.id, error });
-      console.error(error);
-    }).finally(() => {
-      this.logger.info("terminal finished", { id: terminal.id });
-      this.terminals.delete(terminal.id);
-    });
-    setTimeout(() => this.closeTerminalIfNotAttached(terminal), notAttachedTerminalLifetime);
-    this.logger.info("terminal created", { id: terminal.id });
-    return terminal;
+    await this.stateBackend.putTerminalSession(projectId, instanceId, terminalSession);
+    this.logger.info({ msg: "terminal session created", id: terminalSession.id });
+    await this.createManagedTerminal(projectId, instanceId, terminalSession);
+    return terminalSession;
+  }
+  async ensureSessionCreated(projectId, instanceId, terminalName) {
+    const terminalSessions = await this.stateBackend.getTerminalSessions(projectId, instanceId);
+    let session = terminalSessions.find((session2) => session2.terminalName === terminalName);
+    if (session) {
+      this.logger.info({ msg: "reusing existing terminal session", id: session.id });
+    } else {
+      this.logger.info("creating new terminal session");
+      session = await this.createSession(projectId, instanceId, terminalName);
+    }
+    if (!this.managedTerminals.has(session.id)) {
+      this.logger.info({ msg: "no managed terminal found, creating new one", id: session.id });
+      await this.createManagedTerminal(projectId, instanceId, session);
+    }
+    return session;
   }
-  close(id) {
-    const terminal = this.terminals.get(id);
-    if (!terminal) {
-      return;
+  close(sessionId) {
+    this.logger.info({ msg: "closing terminal session", id: sessionId });
+    const managedTerminal = this.managedTerminals.get(sessionId);
+    if (managedTerminal) {
+      managedTerminal.abortController.abort();
     }
-    terminal.abortController.abort();
-    this.terminals.delete(id);
-    this.logger.info("terminal closed", { id });
   }
-  attach(id, stdin, stdout, signal) {
-    const terminal = this.terminals.get(id);
+  async attach(projectId, instanceId, sessionId, stdin, stdout, signal) {
+    this.logger.info({ msg: "attaching terminal", projectId, instanceId, sessionId });
+    let terminal = this.managedTerminals.get(sessionId);
     if (!terminal) {
-      throw new Error("Terminal not found");
+      this.logger.info({ msg: "no managed terminal found, creating new one", sessionId });
+      const session = await this.stateBackend.getTerminalSession(projectId, instanceId, sessionId);
+      if (!session) {
+        throw new Error(`Terminal session "${sessionId}" not found`);
+      }
+      terminal = await this.createManagedTerminal(projectId, instanceId, session);
+      terminal.history = await this.stateBackend.getTerminalSessionHistory(sessionId);
+    }
+    if (terminal.attached) {
+      throw new Error(`Terminal session "${sessionId}" is already attached`);
     }
     terminal.attached = true;
     for (const line of terminal.history) {
-      stdout.write(line + "\n");
+      stdout.write(line);
     }
-    this.logger.info("history replayed", { id });
+    this.logger.info({ msg: "history replayed", count: terminal.history.length, sessionId });
     stdin.pipe(terminal.stdin);
     terminal.stdout.pipe(stdout);
-    this.logger.info("terminal attached", { id });
+    this.logger.info({ msg: "terminal attached", id: sessionId });
     signal.addEventListener("abort", () => {
       terminal.attached = false;
-      this.logger.info("terminal detached", { id });
+      this.logger.info({ msg: "terminal detached", id: sessionId });
     });
   }
+  async createManagedTerminal(projectId, instanceId, terminalSession) {
+    const [instanceType, instanceName] = parseInstanceId(instanceId);
+    const factory = await this.runnerBackend.getTerminalFactory(
+      { projectId, instanceType, instanceName },
+      terminalSession.terminalName
+    );
+    if (!factory) {
+      throw new Error(
+        `Terminal factory for instance "${instanceId}" with name "${terminalSession.terminalName}" not found`
+      );
+    }
+    const managedTerminal = {
+      sessionId: terminalSession.id,
+      abortController: new AbortController(),
+      attached: false,
+      stdin: new PassThrough(),
+      stdout: new PassThrough(),
+      history: []
+    };
+    managedTerminal.stdout.on("data", (data) => {
+      const line = String(data);
+      managedTerminal.history.push(String(data));
+      void this.persistHistory.call([terminalSession.id, line]);
+    });
+    this.managedTerminals.set(managedTerminal.sessionId, managedTerminal);
+    void this.terminalBackend.run({
+      factory,
+      stdin: managedTerminal.stdin,
+      stdout: managedTerminal.stdout,
+      signal: managedTerminal.abortController.signal
+    }).catch((error) => {
+      this.logger.error({
+        msg: "managed terminal failed",
+        id: managedTerminal.sessionId,
+        error
+      });
+    }).finally(() => {
+      this.logger.info({ msg: "managed terminal closed", id: managedTerminal.sessionId });
+      this.managedTerminals.delete(managedTerminal.sessionId);
+    });
+    setTimeout(() => this.closeTerminalIfNotAttached(managedTerminal), notAttachedTerminalLifetime);
+    this.logger.info({ msg: "managed terminal created", id: managedTerminal.sessionId });
+    return managedTerminal;
+  }
   closeTerminalIfNotAttached(terminal) {
-    if (!this.terminals.has(terminal.id)) {
+    if (!this.managedTerminals.has(terminal.sessionId)) {
       return;
     }
     if (!terminal.attached) {
-      this.logger.info("terminal not attached for too long, closing", { id: terminal.id });
+      this.logger.info({
+        msg: "terminal not attached for too long, closing",
+        id: terminal.sessionId
+      });
       terminal.abortController.abort();
-      this.terminals.delete(terminal.id);
+      this.managedTerminals.delete(terminal.sessionId);
       return;
     }
     setTimeout(() => this.closeTerminalIfNotAttached(terminal), notAttachedTerminalLifetime);
   }
-  static create(terminalBackend, logger) {
-    return new TerminalManager(terminalBackend, logger.child({ service: "TerminalManager" }));
+  static create(terminalBackend, stateBackend, runnerBackend, logger) {
+    return new TerminalManager(
+      terminalBackend,
+      stateBackend,
+      runnerBackend,
+      logger.child({ service: "TerminalManager" })
+    );
   }
+  persistHistory = funnel(
+    (entries) => {
+      this.logger.trace({ msg: "persisting history lines", count: entries.length });
+      void this.stateBackend.appendTerminalSessionHistory(entries);
+    },
+    {
+      minQuietPeriodMs: 100,
+      maxBurstDurationMs: 200,
+      reducer: arrayAccumulator
+    }
+  );
 }
 
 class InvalidInstanceStatusError extends Error {
@@ -1008,31 +1234,60 @@ class LocalRunnerBackend {
       }
     );
   }
-  getTerminalFactory(options) {
+  getTerminalFactory(options, terminalName) {
     return this.pulumiProjectHost.runEmpty(
       options.projectId,
       options.instanceType,
       options.instanceName,
       async (stack) => {
         const outputs = await stack.outputs();
-        if (!outputs["$terminal"]) {
+        if (!outputs["$terminals"]) {
           return null;
         }
-        return terminalFactorySchema.parse(outputs["$terminal"].value);
+        const terminals = z.array(instanceTerminalSchema).parse(outputs["$terminals"].value);
+        const terminal = terminals.find((t) => t.name === terminalName);
+        if (!terminal) {
+          return null;
+        }
+        return terminal;
       }
     );
   }
-  getRepresentationContent(options) {
+  getPageContent(options, pageName) {
     return this.pulumiProjectHost.runEmpty(
       options.projectId,
       options.instanceType,
       options.instanceName,
       async (stack) => {
         const outputs = await stack.outputs();
-        if (!outputs["$representation"]) {
+        if (!outputs["$pages"]) {
+          return null;
+        }
+        const pages = z.array(instancePageSchema).parse(outputs["$pages"].value);
+        const page = pages.find((p) => p.name === pageName);
+        if (!page) {
           return null;
         }
-        return instanceRepresentationSchema.parse(outputs["$representation"].value).content;
+        return page.content;
+      }
+    );
+  }
+  getFileContent(options, fileName) {
+    return this.pulumiProjectHost.runEmpty(
+      options.projectId,
+      options.instanceType,
+      options.instanceName,
+      async (stack) => {
+        const outputs = await stack.outputs();
+        if (!outputs["$files"]) {
+          return null;
+        }
+        const files = z.array(instanceFileSchema).parse(outputs["$files"].value);
+        const file = files.find((f) => f.name === fileName);
+        if (!file) {
+          return null;
+        }
+        return file.content;
       }
     );
   }
@@ -1050,23 +1305,26 @@ class LocalRunnerBackend {
       ...mapValues(options.config, (value) => ({ value })),
       ...mapValues(options.secrets, (value) => ({ value, secret: true }))
     };
-    void this.pulumiProjectHost.runLocal(
-      options.projectId,
-      options.instanceType,
-      options.instanceName,
-      () => this.resolveProjectPath(options.source),
-      async (stack) => {
-        await stack.setAllConfig(configMap);
-        const instanceId = LocalRunnerBackend.getInstanceId(options);
-        this.updateState({
-          id: instanceId,
-          status: "updating",
-          currentResourceCount: 0,
-          totalResourceCount: 0
-        });
-        let currentResourceCount = 0;
-        let totalResourceCount = 0;
-        try {
+    void this.updateWorker(options, configMap);
+  }
+  async updateWorker(options, configMap) {
+    const instanceId = LocalRunnerBackend.getInstanceId(options);
+    try {
+      await this.pulumiProjectHost.runLocal(
+        options.projectId,
+        options.instanceType,
+        options.instanceName,
+        () => this.resolveProjectPath(options.source),
+        async (stack) => {
+          await stack.setAllConfig(configMap);
+          this.updateState({
+            id: instanceId,
+            status: "updating",
+            currentResourceCount: 0,
+            totalResourceCount: 0
+          });
+          let currentResourceCount = 0;
+          let totalResourceCount = 0;
           await runWithRetryOnError(
             async () => {
               await stack.up({
@@ -1090,7 +1348,7 @@ class LocalRunnerBackend {
                   }
                 },
                 onOutput: (message) => {
-                  this.updateState({ id: instanceId, message });
+                  this.updateState({ id: instanceId, logLine: message });
                   if (this.printOutput) {
                     console.log(message);
                   }
@@ -1101,21 +1359,28 @@ class LocalRunnerBackend {
               this.updateState({
                 id: instanceId,
                 status: "created",
+                totalResourceCount: currentResourceCount,
                 ...extraOutputs
               });
             },
-            (error) => LocalPulumiHost.tryUnlockStack(stack, error)
+            async (error) => {
+              const isUnlocked = await this.pulumiProjectHost.tryUnlockStack(stack, error);
+              if (isUnlocked) return true;
+              const isResolved = await this.tryInstallMissingDependencies(error);
+              if (isResolved) return true;
+              return false;
+            }
           );
-        } catch (e) {
-          this.updateState({
-            id: instanceId,
-            status: "error",
-            error: e instanceof Error ? e.message : String(e)
-          });
-        }
-      },
-      configMap
-    );
+        },
+        configMap
+      );
+    } catch (error) {
+      this.updateState({
+        id: instanceId,
+        status: "error",
+        error: errorToString(error)
+      });
+    }
   }
   async destroy(options) {
     const currentStatus = await this.validateStatus(options, [
@@ -1124,24 +1389,27 @@ class LocalRunnerBackend {
       "created",
       "error"
     ]);
-    if (currentStatus === "destroying" || currentStatus === "not_created") {
+    if (currentStatus === "destroying") {
       return;
     }
-    void this.pulumiProjectHost.runEmpty(
-      options.projectId,
-      options.instanceType,
-      options.instanceName,
-      async (stack) => {
-        const summary = await stack.workspace.stack();
-        let currentResourceCount = summary?.resourceCount ?? 0;
-        const instanceId = LocalRunnerBackend.getInstanceId(options);
-        this.updateState({
-          id: instanceId,
-          status: "destroying",
-          currentResourceCount,
-          totalResourceCount: currentResourceCount
-        });
-        try {
+    void this.destroyWorker(options);
+  }
+  async destroyWorker(options) {
+    const instanceId = LocalRunnerBackend.getInstanceId(options);
+    try {
+      await this.pulumiProjectHost.runEmpty(
+        options.projectId,
+        options.instanceType,
+        options.instanceName,
+        async (stack) => {
+          const summary = await stack.workspace.stack();
+          let currentResourceCount = summary?.resourceCount ?? 0;
+          this.updateState({
+            id: instanceId,
+            status: "destroying",
+            currentResourceCount,
+            totalResourceCount: currentResourceCount
+          });
           await runWithRetryOnError(
             async () => {
               await stack.destroy({
@@ -1157,7 +1425,7 @@ class LocalRunnerBackend {
                   }
                 },
                 onOutput: (message) => {
-                  this.updateState({ id: instanceId, message });
+                  this.updateState({ id: instanceId, logLine: message });
                   if (this.printOutput) {
                     console.log(message);
                   }
@@ -1168,20 +1436,21 @@ class LocalRunnerBackend {
               this.updateState({
                 id: instanceId,
                 status: "not_created",
+                totalResourceCount: currentResourceCount,
                 ...extraOutputs
               });
             },
-            (error) => LocalPulumiHost.tryUnlockStack(stack, error)
+            (error) => this.pulumiProjectHost.tryUnlockStack(stack, error)
           );
-        } catch (e) {
-          this.updateState({
-            id: instanceId,
-            status: "error",
-            error: e instanceof Error ? e.message : String(e)
-          });
         }
-      }
-    );
+      );
+    } catch (error) {
+      this.updateState({
+        id: instanceId,
+        status: "error",
+        error: errorToString(error)
+      });
+    }
   }
   async refresh(options) {
     const currentStatus = await this.validateStatus(options, [
@@ -1193,26 +1462,38 @@ class LocalRunnerBackend {
     if (currentStatus === "refreshing") {
       return;
     }
-    void this.pulumiProjectHost.runEmpty(
-      options.projectId,
-      options.instanceType,
-      options.instanceName,
-      async (stack) => {
-        const summary = await stack.workspace.stack();
-        let currentResourceCount = summary?.resourceCount ?? 0;
-        const instanceId = LocalRunnerBackend.getInstanceId(options);
-        this.updateState({
-          id: instanceId,
-          status: "refreshing",
-          currentResourceCount,
-          totalResourceCount: currentResourceCount
-        });
-        try {
+    void this.refreshWorker(options);
+  }
+  async refreshWorker(options) {
+    const instanceId = LocalRunnerBackend.getInstanceId(options);
+    try {
+      await this.pulumiProjectHost.runEmpty(
+        options.projectId,
+        options.instanceType,
+        options.instanceName,
+        async (stack) => {
+          const summary = await stack.workspace.stack();
+          let currentResourceCount = 0;
+          let totalResourceCount = summary?.resourceCount ?? 0;
+          this.updateState({
+            id: instanceId,
+            status: "refreshing",
+            currentResourceCount,
+            totalResourceCount
+          });
           await runWithRetryOnError(
             async () => {
               await stack.refresh({
                 color: "always",
                 onEvent: (event) => {
+                  if (event.resourcePreEvent) {
+                    totalResourceCount = updateResourceCount(
+                      event.resourcePreEvent.metadata.op,
+                      totalResourceCount
+                    );
+                    this.updateState({ id: instanceId, totalResourceCount });
+                    return;
+                  }
                   if (event.resOutputsEvent) {
                     currentResourceCount = updateResourceCount(
                       event.resOutputsEvent.metadata.op,
@@ -1223,7 +1504,7 @@ class LocalRunnerBackend {
                   }
                 },
                 onOutput: (message) => {
-                  this.updateState({ id: instanceId, message });
+                  this.updateState({ id: instanceId, logLine: message });
                   if (this.printOutput) {
                     console.log(message);
                   }
@@ -1234,52 +1515,62 @@ class LocalRunnerBackend {
               this.updateState({
                 id: instanceId,
                 status: "created",
+                totalResourceCount: currentResourceCount,
                 ...extraOutputs
               });
             },
-            (error) => LocalPulumiHost.tryUnlockStack(stack, error)
+            (error) => this.pulumiProjectHost.tryUnlockStack(stack, error)
           );
-        } catch (e) {
-          this.updateState({
-            id: instanceId,
-            status: "error",
-            error: e instanceof Error ? e.message : String(e)
-          });
         }
-      }
-    );
+      );
+    } catch (error) {
+      this.updateState({
+        id: instanceId,
+        status: "error",
+        error: errorToString(error)
+      });
+    }
   }
   async getExtraOutputsStatePatch(stack) {
     const outputs = await stack.outputs();
     const patch = {};
     if (outputs["$status"]) {
-      patch.statusFields = instanceStatusFieldMapSchema.parse(outputs["$status"].value);
+      patch.statusFields = z.array(instanceStatusFieldSchema).parse(outputs["$status"].value);
     } else {
-      patch.statusFields = null;
+      patch.statusFields = [];
     }
-    if (outputs["$representation"]) {
-      const instanceRepresentation = instanceRepresentationSchema.parse(
-        outputs["$representation"].value
-      );
-      patch.representationMeta = {
-        contentType: instanceRepresentation.contentType,
-        fileName: instanceRepresentation.fileName,
-        showQRCode: instanceRepresentation.showQRCode
-      };
+    if (outputs["$pages"]) {
+      const pages = z.array(instancePageSchema).parse(outputs["$pages"].value);
+      patch.pages = pages.map((page) => omit(page, ["content"]));
+    } else {
+      patch.pages = [];
+    }
+    if (outputs["$files"]) {
+      const files = z.array(instanceFileSchema).parse(outputs["$files"].value);
+      patch.files = files.map((file) => omit(file, ["content"]));
     } else {
-      patch.representationMeta = null;
+      patch.files = [];
     }
-    if (outputs["$terminal"]) {
-      terminalFactorySchema.parse(outputs["$terminal"].value);
-      patch.hasTerminal = true;
+    if (outputs["$terminals"]) {
+      const terminals = z.array(instanceTerminalSchema).parse(outputs["$terminals"].value);
+      patch.terminals = terminals.map((terminal) => pick(terminal, ["name", "title", "description"]));
     } else {
-      patch.hasTerminal = false;
+      patch.terminals = [];
+    }
+    if (outputs["$triggers"]) {
+      patch.triggers = z.array(instanceTriggerSchema).parse(outputs["$triggers"].value);
+    } else {
+      patch.triggers = [];
     }
     if (outputs["$secrets"]) {
-      patch.secrets = z.record(z.string()).parse(outputs["$secrets"].value);
+      patch.secrets = pickBy(
+        z.record(z.string().nullish()).parse(outputs["$secrets"].value),
+        (v) => !!v
+      );
     } else {
       patch.secrets = null;
     }
+    patch.outputHash = await sha256(JSON.stringify(outputs));
     return patch;
   }
   updateState(patch) {
@@ -1306,26 +1597,36 @@ class LocalRunnerBackend {
       if (!source.path) {
         throw new Error("Source path is required for local units");
       }
-      return resolve(this.sourceBasePath, source.type);
+      return resolve$1(this.sourceBasePath, source.path);
     }
     if (!this.skipSourceCheck) {
       const packageName = source.version ? `${source.package}@${source.version}` : source.package;
       await ensureDependencyInstalled(packageName);
     }
-    if (!source.path) {
-      throw new Error("Source path is required for npm units");
-    }
-    const fullPath = `${source.package}/${source.path}`;
+    const fullPath = source.path ? `${source.package}/${source.path}` : source.package;
     const url = resolve(fullPath, import.meta.url);
     const path = fileURLToPath(url);
     const projectPath = dirname(path);
     return projectPath;
   }
+  async tryInstallMissingDependencies(error) {
+    if (!(error instanceof Error)) {
+      return false;
+    }
+    const pattern = /Cannot find module '(.*)'/;
+    const match = error.message.match(pattern);
+    if (!match) {
+      return false;
+    }
+    const packageName = match[1];
+    await ensureDependencyInstalled(packageName);
+    return true;
+  }
   static async create(config, pulumiProjectHost) {
     let sourceBasePath = config.HIGHSTATE_BACKEND_RUNNER_LOCAL_SOURCE_BASE_PATH;
     if (!sourceBasePath) {
       const [projectPath] = await resolveMainLocalProject();
-      sourceBasePath = resolve(projectPath, "units");
+      sourceBasePath = resolve$1(projectPath, "units");
     }
     return new LocalRunnerBackend(
       config.HIGHSTATE_BACKEND_RUNNER_LOCAL_SKIP_SOURCE_CHECK,
@@ -1363,75 +1664,53 @@ class LocalStateBackend {
     this.logger.debug({ msg: "initialized", dbLocation: db.location });
   }
   async getActiveOperations() {
-    const sublevel = this.getJsonSublevel("activeOperations");
-    const result = [];
-    for await (const operation of sublevel.values()) {
-      result.push(projectOperationSchema.parse(operation));
-    }
-    return result;
+    const sublevel = this.getActiveOperationsSublevel();
+    return this.getSublevelItems(sublevel, projectOperationSchema);
   }
   async getOperations(projectId, beforeOperationId) {
-    const sublevel = this.getJsonSublevel(`projects/${projectId}/operations`);
-    const result = [];
+    const sublevel = this.getProjectOperationsSublevel(projectId);
     const pageSize = 10;
-    for await (const operation of sublevel.values({ lt: beforeOperationId, reverse: true })) {
-      result.push(projectOperationSchema.parse(operation));
-      if (result.length >= pageSize) {
-        break;
-      }
-    }
-    return result;
+    return await this.getSublevelItems(sublevel, projectOperationSchema, pageSize, {
+      lt: beforeOperationId,
+      reverse: true
+    });
   }
   async getInstanceStates(projectId) {
-    const sublevel = this.getJsonSublevel(`projects/${projectId}/instances`);
-    const result = [];
-    for await (const state of sublevel.values()) {
-      result.push(instanceStateSchema.parse(state));
-    }
-    return result;
+    const sublevel = this.getProjectInstanceStatesSublevel(projectId);
+    return await this.getSublevelItems(sublevel, instanceStateSchema);
   }
   async getInstanceState(projectId, instanceID) {
-    const sublevel = this.getJsonSublevel(`projects/${projectId}/instances`);
-    const state = await sublevel.get(instanceID);
-    if (!state) {
-      return undefined;
-    }
-    return instanceStateSchema.parse(state);
+    const sublevel = this.getProjectInstanceStatesSublevel(projectId);
+    return await this.getSublevelItem(sublevel, instanceStateSchema, instanceID);
   }
   async getAffectedInstanceStates(operationId) {
-    const sublevel = this.getJsonSublevel(`operations/${operationId}/instances`);
-    const result = [];
-    for await (const state of sublevel.values()) {
-      result.push(instanceStateSchema.parse(state));
-    }
-    return result;
+    const sublevel = this.getOperationInstanceStatesSublevel(operationId);
+    return await this.getSublevelItems(sublevel, instanceStateSchema);
   }
   async getInstanceLogs(operationId, instanceId) {
-    const sublevel = this.db.sublevel(`operations/${operationId}/instanceLogs/${instanceId}`);
-    const result = [];
-    for await (const line of sublevel.values()) {
-      result.push(line);
-    }
-    return result;
+    const sublevel = this.getOperationInstanceLogsSublevel(operationId, instanceId);
+    return await Array.fromAsync(sublevel.values());
   }
   async putOperation(operation) {
-    const sublevel = this.getJsonSublevel(`projects/${operation.projectId}/operations`);
-    const activeOperationsSublevel = this.getJsonSublevel("activeOperations");
-    if (operation.status !== "completed" && operation.status !== "failed") {
+    this.validateItem(projectOperationSchema, operation);
+    const operationsSublevel = this.getProjectOperationsSublevel(operation.projectId);
+    const activeOperationsSublevel = this.getActiveOperationsSublevel();
+    if (!isFinalOperationStatus(operation.status)) {
       await this.db.batch([
-        { type: "put", key: operation.id, value: operation, sublevel },
+        { type: "put", key: operation.id, value: operation, sublevel: operationsSublevel },
         { type: "put", key: operation.id, value: operation, sublevel: activeOperationsSublevel }
       ]);
     } else {
       await this.db.batch([
-        { type: "put", key: operation.id, value: operation, sublevel },
+        { type: "put", key: operation.id, value: operation, sublevel: operationsSublevel },
         { type: "del", key: operation.id, sublevel: activeOperationsSublevel }
       ]);
     }
   }
   async putAffectedInstanceStates(projectId, operationId, states) {
-    const sublevel = this.getJsonSublevel(`operations/${operationId}/instances`);
-    const projectSublevel = this.getJsonSublevel(`projects/${projectId}/instances`);
+    this.validateArray(instanceStateSchema, states);
+    const operationInstanceStatesSublevel = this.getOperationInstanceStatesSublevel(operationId);
+    const projectInstanceStatesSublevel = this.getProjectInstanceStatesSublevel(projectId);
     await this.db.batch(
       // put the states to both the operation and project sublevels
       // denormalization is cool
@@ -1441,19 +1720,20 @@ class LocalStateBackend {
           type: "put",
           key: state.id,
           value: state,
-          sublevel
+          sublevel: operationInstanceStatesSublevel
         },
         {
           type: "put",
           key: state.id,
           value: state,
-          sublevel: projectSublevel
+          sublevel: projectInstanceStatesSublevel
         }
       ])
     );
   }
   async putInstanceStates(projectId, states) {
-    const sublevel = this.getJsonSublevel(`projects/${projectId}/instances`);
+    this.validateArray(instanceStateSchema, states);
+    const sublevel = this.getProjectInstanceStatesSublevel(projectId);
     await sublevel.batch(
       // as i told before, we update the instance states without operations
       // this method is used when upstream instance state changes are detected
@@ -1470,7 +1750,7 @@ class LocalStateBackend {
       if (sublevels.has(instanceId)) {
         continue;
       }
-      const sublevel = this.db.sublevel(`operations/${operationId}/instanceLogs/${instanceId}`);
+      const sublevel = this.getOperationInstanceLogsSublevel(operationId, instanceId);
       sublevels.set(instanceId, sublevel);
     }
     await this.db.batch(
@@ -1483,22 +1763,18 @@ class LocalStateBackend {
     );
   }
   async getCompositeInstance(projectId, instanceId) {
-    const sublevel = this.getJsonSublevel(`projects/${projectId}/compositeInstances`);
-    const instance = await sublevel.get(instanceId);
-    if (!instance) {
-      return null;
-    }
-    return evaluatedCompositeInstanceSchema.parse(instance);
+    const sublevel = this.getProjectCompositeInstancesSublevel(projectId);
+    return this.getSublevelItem(sublevel, evaluatedCompositeInstanceSchema, instanceId);
   }
   async getCompositeInstanceInputHash(projectId, instanceId) {
-    const sublevel = this.db.sublevel(`projects/${projectId}/compositeInstanceInputHashes`);
-    return await sublevel.get(instanceId) ?? null;
+    const sublevel = this.getProjectCompositeInstanceInputHashesSublevel(projectId);
+    const inputHash = await sublevel.get(instanceId);
+    return inputHash ?? null;
   }
   async putCompositeInstances(projectId, instances) {
-    const sublevel = this.getJsonSublevel(`projects/${projectId}/compositeInstances`);
-    const inputHashesSublevel = this.db.sublevel(
-      `projects/${projectId}/compositeInstanceInputHashes`
-    );
+    this.validateArray(evaluatedCompositeInstanceSchema, instances);
+    const sublevel = this.getProjectCompositeInstancesSublevel(projectId);
+    const inputHashesSublevel = this.getProjectCompositeInstanceInputHashesSublevel(projectId);
     await this.db.batch(
       instances.flatMap((instance) => [
         {
@@ -1517,10 +1793,8 @@ class LocalStateBackend {
     );
   }
   async clearCompositeInstances(projectId, instanceIds) {
-    const sublevel = this.getJsonSublevel(`projects/${projectId}/compositeInstances`);
-    const inputHashesSublevel = this.db.sublevel(
-      `projects/${projectId}/compositeInstanceInputHashes`
-    );
+    const sublevel = this.getProjectCompositeInstancesSublevel(projectId);
+    const inputHashesSublevel = this.getProjectCompositeInstanceInputHashesSublevel(projectId);
     await this.db.batch(
       instanceIds.flatMap((instanceId) => [
         { type: "del", key: instanceId, sublevel },
@@ -1528,6 +1802,138 @@ class LocalStateBackend {
       ])
     );
   }
+  async getTerminalSessions(projectId, instanceId) {
+    const sublevel = this.getTerminalSessionsSublevel(projectId, instanceId);
+    return await this.getSublevelItems(sublevel, terminalSessionSchema);
+  }
+  async getTerminalSession(projectId, instanceId, sessionId) {
+    const sublevel = this.getTerminalSessionsSublevel(projectId, instanceId);
+    return await this.getSublevelItem(sublevel, terminalSessionSchema, sessionId);
+  }
+  async putTerminalSession(projectId, instanceId, session) {
+    this.validateItem(terminalSessionSchema, session);
+    const sublevel = this.getTerminalSessionsSublevel(projectId, instanceId);
+    await sublevel.put(session.id, session);
+  }
+  async getTerminalSessionHistory(sessionId) {
+    const sublevel = this.getTerminalSessionHistorySublevel(sessionId);
+    return await Array.fromAsync(sublevel.values());
+  }
+  async appendTerminalSessionHistory(lines) {
+    const sublevels = /* @__PURE__ */ new Map();
+    for (const [sessionId] of lines) {
+      if (sublevels.has(sessionId)) {
+        continue;
+      }
+      const sublevel = this.getTerminalSessionHistorySublevel(sessionId);
+      sublevels.set(sessionId, sublevel);
+    }
+    await this.db.batch(
+      lines.map(([sessionId, line]) => ({
+        type: "put",
+        key: uuidv7(),
+        value: line,
+        sublevel: sublevels.get(sessionId)
+      }))
+    );
+  }
+  async getSublevelItems(sublevel, schema, limit, options) {
+    const result = [];
+    const iterator = options ? sublevel.iterator(options) : sublevel.iterator();
+    const invalidKeys = [];
+    for await (const [key, value] of iterator) {
+      const parseResult = schema.safeParse(value);
+      if (!parseResult.success) {
+        this.logger.warn({
+          msg: "failed to parse item, it will be deleted",
+          error: parseResult.error,
+          sublevel: sublevel.prefix,
+          key
+        });
+        invalidKeys.push(key);
+        continue;
+      }
+      result.push(parseResult.data);
+      if (limit && result.length >= limit) {
+        break;
+      }
+    }
+    if (invalidKeys.length > 0) {
+      this.logger.info({
+        msg: "deleting invalid items",
+        sublevel: sublevel.prefix,
+        keyCount: invalidKeys.length
+      });
+      await sublevel.batch(invalidKeys.map((key) => ({ type: "del", key })));
+    }
+    return result;
+  }
+  async getSublevelItem(sublevel, schema, key) {
+    const value = await sublevel.get(key);
+    if (!value) {
+      return null;
+    }
+    const parseResult = schema.safeParse(value);
+    if (!parseResult.success) {
+      this.logger.warn({
+        msg: "failed to parse item, it will be deleted",
+        error: parseResult.error,
+        sublevel: sublevel.prefix,
+        key
+      });
+      await sublevel.del(key);
+      return null;
+    }
+    return parseResult.data;
+  }
+  validateItem(schema, item) {
+    const parseResult = schema.safeParse(item);
+    if (!parseResult.success) {
+      this.logger.error({
+        msg: "failed to validate item",
+        error: parseResult.error,
+        item
+      });
+      throw new Error(`Failed to validate item: ${parseResult.error.errors[0].message}`, {
+        cause: parseResult.error
+      });
+    }
+  }
+  validateArray(schema, items) {
+    for (const item of items) {
+      this.validateItem(schema, item);
+    }
+  }
+  getActiveOperationsSublevel() {
+    return this.getJsonSublevel("activeOperations");
+  }
+  getProjectOperationsSublevel(projectId) {
+    return this.getJsonSublevel(`projects/${projectId}/operations`);
+  }
+  getProjectCompositeInstancesSublevel(projectId) {
+    return this.getJsonSublevel(`projects/${projectId}/compositeInstances`);
+  }
+  getProjectCompositeInstanceInputHashesSublevel(projectId) {
+    return this.getStringSublevel(`projects/${projectId}/compositeInstanceInputHashes`);
+  }
+  getProjectInstanceStatesSublevel(projectId) {
+    return this.getJsonSublevel(`projects/${projectId}/instanceStates`);
+  }
+  getOperationInstanceStatesSublevel(operationId) {
+    return this.getJsonSublevel(`operations/${operationId}/instanceStates`);
+  }
+  getOperationInstanceLogsSublevel(operationId, instanceId) {
+    return this.getStringSublevel(`operations/${operationId}/instances/${instanceId}/logs`);
+  }
+  getTerminalSessionsSublevel(projectId, instanceId) {
+    return this.getJsonSublevel(`projects/${projectId}/instances/${instanceId}/terminalSessions`);
+  }
+  getTerminalSessionHistorySublevel(sessionId) {
+    return this.getStringSublevel(`terminalSessions/${sessionId}/history`);
+  }
+  getStringSublevel(path) {
+    return this.db.sublevel(path, { valueEncoding: "utf8" });
+  }
   getJsonSublevel(path) {
     return this.db.sublevel(path, { valueEncoding: "json" });
   }
@@ -1655,19 +2061,22 @@ class RuntimeOperation {
   }
   abortController = new AbortController();
   instanceMap = /* @__PURE__ */ new Map();
+  hubMap = /* @__PURE__ */ new Map();
+  resolvedInstanceInputs = /* @__PURE__ */ new Map();
   compositeInstanceLock = new BetterLock();
   compositeInstanceMap = /* @__PURE__ */ new Map();
   initialStatusMap = /* @__PURE__ */ new Map();
   stateMap = /* @__PURE__ */ new Map();
   instancePromiseMap = /* @__PURE__ */ new Map();
   childrenStateMap = /* @__PURE__ */ new Map();
+  dependentStateMap = /* @__PURE__ */ new Map();
   library;
-  inputHashCalculator;
+  resolveInputHash;
   async operateSafe() {
     try {
       await this.operate();
     } catch (error) {
-      if (isAbortError(error)) {
+      if (RuntimeOperation.isAbortError(error)) {
         this.logger.info("the operation was cancelled");
         this.operation.status = "cancelled";
         await this.updateOperation();
@@ -1692,27 +2101,75 @@ class RuntimeOperation {
       if (state.status !== "pending") {
         continue;
       }
-      const initialStatus = this.initialStatusMap.get(state.id);
+      let initialStatus = this.initialStatusMap.get(state.id);
+      let error = state.error;
+      if (initialStatus === "destroying" || initialStatus === "pending" || initialStatus === "refreshing" || initialStatus === "updating") {
+        initialStatus = "error";
+        error ??= "unexpected progressing instance status";
+      }
       if (initialStatus) {
-        this.updateInstanceState({ id: state.id, status: initialStatus });
+        this.updateInstanceState({ id: state.id, status: initialStatus, error });
       }
     }
   }
   async operate() {
     this.logger.info("starting operation");
-    const allInstances = await this.projectBackend.getInstances(this.operation.projectId);
+    const { instances: projectInstances, hubs } = await this.projectBackend.getProject(
+      this.operation.projectId
+    );
     this.abortController.signal.throwIfAborted();
     const states = await this.stateBackend.getInstanceStates(this.operation.projectId);
     this.abortController.signal.throwIfAborted();
+    this.library = await this.libraryBackend.loadLibrary();
+    this.logger.info("library ready");
+    this.abortController.signal.throwIfAborted();
+    const allInstances = projectInstances.filter((instance) => {
+      return instance.type in this.library.components;
+    });
+    const inputResolverNodes = /* @__PURE__ */ new Map();
+    for (const hub of hubs) {
+      this.hubMap.set(hub.id, hub);
+      inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub });
+    }
     for (const instance of allInstances) {
       this.instanceMap.set(instance.id, instance);
+      inputResolverNodes.set(`instance:${instance.id}`, {
+        kind: "instance",
+        instance,
+        component: this.library.components[instance.type]
+      });
+    }
+    const resolveInputs = createInputResolver(
+      inputResolverNodes,
+      this.logger.child({ resolver: "input-resolver" })
+    );
+    for (const instance of allInstances) {
+      const output = await resolveInputs(`instance:${instance.id}`);
+      if (output.kind !== "instance") {
+        throw new Error("Unexpected output kind, expected instance");
+      }
+      this.resolvedInstanceInputs.set(instance.id, output.resolvedInputs);
     }
-    this.inputHashCalculator = new InputHashCalculator(this.instanceMap);
     for (const state of states) {
       this.stateMap.set(state.id, state);
       this.initialStatusMap.set(state.id, state.status);
       this.tryAddStateToParent(state);
+      this.addDependentState(state);
+    }
+    const inputHashNodes = /* @__PURE__ */ new Map();
+    for (const instance of allInstances) {
+      inputHashNodes.set(instance.id, {
+        instance,
+        resolvedInputs: this.resolvedInstanceInputs.get(instance.id),
+        state: this.stateMap.get(instance.id),
+        sourceHash: void 0
+        // TODO: implement source hash
+      });
     }
+    this.resolveInputHash = createInputHashResolver(
+      inputHashNodes,
+      this.logger.child({ resolver: "input-hash-resolver" })
+    );
     if (this.operation.type === "update") {
       await this.extendWithNotCreatedDependencies();
       await this.updateOperation();
@@ -1726,26 +2183,25 @@ class RuntimeOperation {
     });
     if (this.operation.type === "evaluate") {
       this.logger.info("evaluating instances");
-      await this.evaluateCompositeInstances(allInstances);
+      await this.evaluateCompositeInstances();
     } else {
-      this.library = await this.libraryBackend.loadLibrary();
-      this.logger.info("library ready");
-      this.abortController.signal.throwIfAborted();
       const promises = [];
       for (const instanceId of this.operation.instanceIds) {
         promises.push(this.getInstancePromiseForOperation(instanceId));
       }
-      this.logger.info("all units started");
+      this.logger.info("all units operations started");
       this.operation.status = "running";
       await this.updateOperation();
       await Promise.all(promises);
       this.logger.info("all units completed");
     }
     this.operation.status = "completed";
+    this.operation.error = null;
     await this.updateOperation();
     this.logger.info("operation completed");
   }
   cancel() {
+    this.logger.info("cancelling operation");
     this.abortController.abort();
   }
   getInstancePromiseForOperation(instanceId) {
@@ -1771,7 +2227,7 @@ class RuntimeOperation {
         return this.recreateUnit(instance, component);
       }
       case "destroy": {
-        return this.destroyUnit(instance.id);
+        return this.destroyUnit(instance.id, component);
       }
       case "refresh": {
         return this.refreshUnit(instance.id);
@@ -1809,7 +2265,7 @@ class RuntimeOperation {
           status: "created"
         });
       } catch (error) {
-        if (isAbortError(error)) {
+        if (RuntimeOperation.isAbortError(error)) {
           this.updateInstanceState({
             id: instance.id,
             status: initialStatus
@@ -1841,7 +2297,7 @@ class RuntimeOperation {
           continue;
         }
         logger.info("waiting for dependency", { dependencyId: dependency.id });
-        dependencyPromises.push(this.updateUnit(dependency, component));
+        dependencyPromises.push(this.getInstancePromiseForOperation(dependency.id));
       }
       await Promise.all(dependencyPromises);
       this.abortController.signal.throwIfAborted();
@@ -1869,13 +2325,46 @@ class RuntimeOperation {
         finalStatuses: ["created", "error"]
       });
       await this.watchStateStream(stream);
-      const inputHash = await this.inputHashCalculator.calculate(instance);
+      const inputHash = await this.resolveInputHash(instance.id);
       this.updateInstanceState({ id: instance.id, inputHash });
       logger.debug("input hash after update", { inputHash });
       logger.info("unit updated");
     });
   }
-  async destroyUnit(instanceId) {
+  async processBeforeDestroyTriggers(state, component, logger) {
+    const instance = this.instanceMap.get(state.id);
+    if (!instance) {
+      throw new Error(`Instance not found: ${state.id}`);
+    }
+    const triggers = state.triggers.filter((trigger) => trigger.spec.type === "before-destroy");
+    if (triggers.length === 0) {
+      return;
+    }
+    const invokedTriggers = triggers.map((trigger) => ({ name: trigger.name }));
+    logger.info("updating unit to process before-destroy triggers...");
+    const secrets = await this.secretBackend.get(this.operation.projectId, instance.id);
+    this.abortController.signal.throwIfAborted();
+    logger.debug("secrets loaded", { count: Object.keys(secrets).length });
+    await this.runnerBackend.update({
+      projectId: this.operation.projectId,
+      instanceType: instance.type,
+      instanceName: instance.name,
+      config: await this.prepareUnitConfig(instance, invokedTriggers),
+      secrets: mapValues(secrets, (value) => valueToString(value)),
+      source: this.resolveUnitSource(component),
+      signal: this.abortController.signal
+    });
+    logger.debug("unit update requested");
+    const stream = this.runnerBackend.watch({
+      projectId: this.operation.projectId,
+      instanceType: instance.type,
+      instanceName: instance.name,
+      finalStatuses: ["created", "error"]
+    });
+    await this.watchStateStream(stream);
+    logger.debug("before-destroy triggers processed");
+  }
+  async destroyUnit(instanceId, component) {
     const logger = this.logger.child({ instanceId });
     return this.getInstancePromise(instanceId, async () => {
       this.updateInstanceState({
@@ -1891,12 +2380,13 @@ class RuntimeOperation {
         return;
       }
       const dependentPromises = [];
-      for (const dependentKey of state.dependentKeys ?? []) {
-        logger.info("destroying dependent unit", { dependentKey });
-        dependentPromises.push(this.destroyUnit(dependentKey));
+      const dependents = this.dependentStateMap.get(instanceId) ?? [];
+      for (const dependent of dependents) {
+        dependentPromises.push(this.getInstancePromiseForOperation(dependent.id));
       }
       await Promise.all(dependentPromises);
       this.abortController.signal.throwIfAborted();
+      await this.processBeforeDestroyTriggers(state, component, logger);
       logger.info("destroying unit...");
       const [type, name] = parseInstanceId(instanceId);
       await this.runnerBackend.destroy({
@@ -1926,6 +2416,12 @@ class RuntimeOperation {
         currentResourceCount: 0,
         totalResourceCount: 0
       });
+      const state = this.stateMap.get(instance.id);
+      if (!state) {
+        logger.warn("state not found for unit, but recreate was requested");
+        return;
+      }
+      await this.processBeforeDestroyTriggers(state, component, logger);
       logger.info("destroying unit...");
       await this.runnerBackend.destroy({
         projectId: this.operation.projectId,
@@ -2012,26 +2508,28 @@ class RuntimeOperation {
       throw new Error(`The operation on unit "${statePatch.id}" failed: ${statePatch.error}`);
     }
   }
-  async prepareUnitConfig(instance) {
+  async prepareUnitConfig(instance, invokedTriggers = []) {
     const config = {};
-    for (const [key, value] of Object.entries(instance.args)) {
+    for (const [key, value] of Object.entries(instance.args ?? {})) {
       config[key] = valueToString(value);
     }
-    for (const [key, value] of Object.entries(instance.inputs)) {
-      config[`input.${key}`] = JSON.stringify(await this.resolveInstanceInput(value));
+    const instanceInputs = this.resolvedInstanceInputs.get(instance.id) ?? {};
+    for (const [key, value] of Object.entries(instanceInputs)) {
+      const resolvedInput = await this.resolveInstanceInput(value.map((input) => input.input));
+      config[`input.${key}`] = JSON.stringify(resolvedInput);
     }
+    config["$invokedTriggers"] = JSON.stringify(invokedTriggers);
     return config;
   }
   async resolveInstanceInput(input) {
-    if (Array.isArray(input)) {
-      return (await Promise.all(input.map((x) => this.resolveInstanceInput(x)))).flat();
-    }
-    return await this.resolveSingleInstanceInput(input);
+    const promises = input.map((input2) => this.resolveSingleInstanceInput(input2));
+    const results = await Promise.all(promises);
+    return results.flat();
   }
   async resolveSingleInstanceInput(input) {
     const loadedInstance = this.instanceMap.get(input.instanceId);
     if (loadedInstance && !loadedInstance.parentId) {
-      return input;
+      return [input];
     }
     const parentInstance = await this.loadCompositeInstance(input.instanceId);
     const parentOutput = parentInstance.instance.outputs?.[input.output];
@@ -2061,10 +2559,10 @@ class RuntimeOperation {
       return compositeInstance;
     });
   }
-  async evaluateCompositeInstances(allInstances) {
+  async evaluateCompositeInstances() {
     await this.projectLock.lockInstances(this.operation.instanceIds, async () => {
       const compositeInstances = await this.libraryBackend.evaluateCompositeInstances(
-        allInstances,
+        Array.from(this.instanceMap.values()),
         this.operation.instanceIds
       );
       this.abortController.signal.throwIfAborted();
@@ -2079,7 +2577,7 @@ class RuntimeOperation {
         await Promise.all(
           compositeInstances.map(async (instance) => ({
             ...instance,
-            inputHash: await this.inputHashCalculator.calculate(instance.instance)
+            inputHash: await this.resolveInputHash(instance.instance.id)
           }))
         )
       );
@@ -2090,12 +2588,15 @@ class RuntimeOperation {
     await this.stateBackend.putOperation(this.operation);
   }
   updateInstanceState(patch) {
-    if (patch.message) {
-      this.persistLogs.call([patch.id, patch.message]);
-      this.instanceLogsEE.emit(`${this.operation.id}/${patch.id}`, patch.message);
+    if (!patch.id) {
+      throw new Error("The ID of the instance state is required.");
+    }
+    if (patch.logLine) {
+      this.persistLogs.call([patch.id, patch.logLine]);
+      this.instanceLogsEE.emit(`${this.operation.id}/${patch.id}`, patch.logLine);
       return;
     }
-    const state = applyInstanceStatePatch(this.stateMap, patch);
+    const state = applyPartialInstanceState(this.stateMap, patch);
     this.persistStates.call(state);
     if (patch.secrets) {
       this.persistSecrets.call([patch.id, patch.secrets]);
@@ -2110,6 +2611,20 @@ class RuntimeOperation {
     children.push(state);
     this.childrenStateMap.set(state.parentId, children);
   }
+  addDependentState(state) {
+    const instance = this.instanceMap.get(state.id);
+    if (!instance) {
+      return;
+    }
+    const instanceInputs = this.resolvedInstanceInputs.get(state.id) ?? {};
+    for (const inputs of Object.values(instanceInputs)) {
+      for (const input of inputs) {
+        const dependents = this.dependentStateMap.get(input.input.instanceId) ?? [];
+        dependents.push(state);
+        this.dependentStateMap.set(input.input.instanceId, dependents);
+      }
+    }
+  }
   async extendWithNotCreatedDependencies() {
     const instanceIdsSet = /* @__PURE__ */ new Set();
     const traverse = async (instanceId) => {
@@ -2120,17 +2635,14 @@ class RuntimeOperation {
       if (!instance) {
         return;
       }
-      for (const input of Object.values(instance.inputs)) {
-        if (Array.isArray(input)) {
-          for (const { instanceId: instanceId2 } of input) {
-            await traverse(instanceId2);
-          }
-        } else {
-          await traverse(input.instanceId);
+      const instanceInputs = this.resolvedInstanceInputs.get(instance.id) ?? {};
+      for (const inputs of Object.values(instanceInputs)) {
+        for (const input of inputs) {
+          await traverse(input.input.instanceId);
         }
       }
       const state = this.stateMap.get(instance.id);
-      const expectedInputHash = await this.inputHashCalculator.calculate(instance);
+      const expectedInputHash = await this.resolveInputHash(instance.id);
       if (state?.status !== "created" || state.inputHash !== expectedInputHash) {
         instanceIdsSet.add(instanceId);
       }
@@ -2151,8 +2663,9 @@ class RuntimeOperation {
       if (!state || state.status === "not_created") {
         return;
       }
-      for (const dependent of state.dependentKeys ?? []) {
-        traverse(dependent);
+      const dependents = this.dependentStateMap.get(instanceKey) ?? [];
+      for (const dependent of dependents) {
+        traverse(dependent.id);
         instanceIdsSet.add(instanceKey);
       }
     };
@@ -2192,16 +2705,10 @@ class RuntimeOperation {
   }
   getInstanceDependencies(instance) {
     const dependencies = [];
-    for (const input of Object.values(instance.inputs)) {
-      if (Array.isArray(input)) {
-        for (const { instanceId } of input) {
-          const dependency = this.instanceMap.get(instanceId);
-          if (dependency) {
-            dependencies.push(dependency);
-          }
-        }
-      } else {
-        const dependency = this.instanceMap.get(input.instanceId);
+    const instanceInputs = this.resolvedInstanceInputs.get(instance.id) ?? {};
+    for (const inputs of Object.values(instanceInputs)) {
+      for (const input of inputs) {
+        const dependency = this.instanceMap.get(input.input.instanceId);
         if (dependency) {
           dependencies.push(dependency);
         }
@@ -2210,22 +2717,14 @@ class RuntimeOperation {
     return dependencies;
   }
   resolveUnitSource(component) {
-    if (!component.source) {
+    if (!component.source || component.source.type === "local") {
       return {
         type: "local",
-        path: component.type.replaceAll(".", "/")
+        // auto-generate path for local units
+        path: component.source?.path ?? component.type.split(".").join("/")
       };
     }
-    return {
-      ...component.source,
-      path: component.source.path ?? component.type.replaceAll(".", "/")
-    };
-  }
-  static accumulator(values, value) {
-    if (!values) {
-      return [value];
-    }
-    return [...values, value];
+    return component.source;
   }
   persistStates = funnel(
     (states) => {
@@ -2240,18 +2739,18 @@ class RuntimeOperation {
       minQuietPeriodMs: 100,
       maxBurstDurationMs: 1e3,
       triggerAt: "end",
-      reducer: RuntimeOperation.accumulator
+      reducer: arrayAccumulator
     }
   );
   persistLogs = funnel(
     (entries) => {
-      this.logger.debug({ msg: "persisting logs", count: entries.length });
+      this.logger.trace({ msg: "persisting logs", count: entries.length });
       void this.stateBackend.appendInstanceLogs(this.operation.id, entries);
     },
     {
       minQuietPeriodMs: 100,
       maxBurstDurationMs: 200,
-      reducer: RuntimeOperation.accumulator
+      reducer: arrayAccumulator
     }
   );
   persistSecrets = funnel(
@@ -2267,9 +2766,22 @@ class RuntimeOperation {
     {
       minQuietPeriodMs: 100,
       maxBurstDurationMs: 200,
-      reducer: RuntimeOperation.accumulator
+      reducer: arrayAccumulator
     }
   );
+  static abortMessagePatterns = [
+    "Operation aborted",
+    "Command was killed with SIGINT"
+  ];
+  static isAbortError(err) {
+    if (isAbortError(err)) {
+      return true;
+    }
+    if (err instanceof Error) {
+      return RuntimeOperation.abortMessagePatterns.some((pattern) => err.message.includes(pattern));
+    }
+    return false;
+  }
 }
 
 class OperationManager {
@@ -2395,7 +2907,8 @@ class OperationManager {
     const activeOperations = await stateBackend.getActiveOperations();
     for (const operation of activeOperations) {
       logger.info({ msg: "relaunching operation", operationId: operation.id });
-      operator.startOperation(operation);
+      operation.status = "cancelled";
+      await stateBackend.putOperation(operation);
     }
     return operator;
   }
@@ -2420,7 +2933,8 @@ async function createServices({
   config ??= await loadConfig();
   logger ??= pino({
     name: config.HIGHSTATE_BACKEND_LOGGER_NAME,
-    level: config.HIGHSTATE_BACKEND_LOGGER_LEVEL
+    level: config.HIGHSTATE_BACKEND_LOGGER_LEVEL,
+    errorKey: "error"
   });
   const localPulumiHost = LocalPulumiHost.create(logger);
   const projectLockManager = new ProjectLockManager();
@@ -2430,7 +2944,7 @@ async function createServices({
   runnerBackend ??= await createRunnerBackend(config, localPulumiHost);
   projectBackend ??= await createProjectBackend(config);
   terminalBackend ??= createTerminalBackend(config, logger);
-  terminalManager ??= TerminalManager.create(terminalBackend, logger);
+  terminalManager ??= TerminalManager.create(terminalBackend, stateBackend, runnerBackend, logger);
   operationManager ??= await OperationManager.create(
     runnerBackend,
     stateBackend,