@highstate/backend 0.9.18 → 0.9.20

package/src/orchestrator/operation.ts

@@ -1,88 +1,71 @@
-import type { LibraryBackend } from "../library"
-import type { StateManager } from "../state"
-import type { ProjectBackend } from "../project"
-import type {
-  RunnerArtifact,
-  RunnerBackend,
-  TypedUnitStateUpdate,
-  UnitStateUpdate,
-} from "../runner"
 import type { Logger } from "pino"
+import type { ArtifactService } from "../artifact"
 import type {
   InstanceLockService,
+  InstanceStatePatch,
   InstanceStateService,
   OperationService,
+  ProjectModelService,
   SecretService,
-  WorkerService,
+  UnitExtraService,
 } from "../business"
-import type { PubSubManager } from "../pubsub"
-import { randomBytes } from "node:crypto"
-import { v7 as uuidv7 } from "uuid"
+import type { Operation, OperationUpdateInput, Project } from "../database"
+import type { LibraryBackend } from "../library"
+import type { RunnerBackend, TypedUnitStateUpdate, UnitStateUpdate } from "../runner"
 import {
-  isUnitModel,
-  parseInstanceId,
-  type ComponentModel,
+  type InstanceId,
   type InstanceModel,
-  type UnitSecretModel,
+  parseInstanceId,
+  type TriggerInvocation,
+  type UnitConfig,
+  type VersionedName,
 } from "@highstate/contract"
-import { mapValues, unique } from "remeda"
-import { ArtifactService } from "../artifact"
+import { createId } from "@paralleldrive/cuid2"
+import { mapValues } from "remeda"
+import { AbortError, errorToString, isAbortErrorLike, waitForAbort } from "../common"
 import {
   type InstanceState,
-  type UnitPage,
-  type Operation,
-  type Page,
-  type Trigger,
-  type UnitTrigger,
-  type UnitTerminal,
-  type Terminal,
-  type TerminalSpec,
-  type TriggerInvocation,
+  isTransientInstanceOperationStatus,
+  type OperationPhase,
   PromiseTracker,
-  type InstanceStatePatch,
-  type PageBlock,
-  type Project,
+  waitAll,
 } from "../shared"
-import {
-  AbortError,
-  errorToString,
-  isAbortErrorLike,
-  PerformanceLogger,
-  stringToValue,
-  valueToString,
-  waitForAbort,
-} from "../common"
-import { OperationWorkset, type OperationPhase } from "./operation-workset"
+import { OperationContext } from "./operation-context"
+import { createOperationPlan } from "./operation-plan"
+import { OperationWorkset } from "./operation-workset"
 
 export class RuntimeOperation {
-  private readonly abortController = new AbortController()
-  private readonly forceAbortController = new AbortController()
-
-  private readonly instanceAbortControllers = new Map<string, AbortController>()
-  private readonly instanceForceAbortControllers = new Map<string, AbortController>()
-
-  private readonly instancePromiseMap = new Map<string, Promise<void>>()
+  private readonly instancePromiseMap = new Map<InstanceId, Promise<void>>()
   private readonly promiseTracker = new PromiseTracker()
 
   private workset!: OperationWorkset
+  private context!: OperationContext
+
+  private readonly unlockToken = createId()
 
   constructor(
     private readonly project: Project,
     private readonly operation: Operation,
     private readonly runnerBackend: RunnerBackend,
     private readonly libraryBackend: LibraryBackend,
-    private readonly projectBackend: ProjectBackend,
     private readonly artifactService: ArtifactService,
-    private readonly stateManager: StateManager,
     private readonly instanceLockService: InstanceLockService,
     private readonly operationService: OperationService,
     private readonly secretService: SecretService,
     private readonly instanceStateService: InstanceStateService,
-    private readonly pubsubManager: PubSubManager,
-    private readonly workerService: WorkerService,
+    private readonly projectModelService: ProjectModelService,
+    private readonly unitExtraService: UnitExtraService,
     private readonly logger: Logger,
   ) {}
 
+  cancel(): void {
+    this.workset.cancel()
+  }
+
+  cancelInstance(instanceId: InstanceId): void {
+    this.workset.cancelInstance(instanceId)
+  }
+
   async operateSafe(): Promise<void> {
     try {
       await this.operate()
@@ -92,21 +75,20 @@ export class RuntimeOperation {
     } catch (error) {
       if (isAbortErrorLike(error)) {
         this.logger.info("the operation was cancelled")
-        this.operation.status = "cancelled"
-        this.operation.error = null
 
-        await this.operationService.updateOperation(this.project.id, this.operation)
+        await this.updateOperation({ status: "cancelled" })
         return
       }
 
       this.logger.error({ error }, "an error occurred while running the operation")
 
-      this.operation.status = "failed"
-      this.operation.error = errorToString(error)
-
-      await this.operationService.updateOperation(this.project.id, this.operation)
+      await this.updateOperation({ status: "failed" })
+      await this.writeOperationLog(errorToString(error))
     } finally {
       try {
+        this.promiseTracker.track(this.ensureInstancesUnlocked())
+        this.promiseTracker.track(this.ensureOperationStatesFinalized())
+
         // ensure that all promises are resolved even if the operation failed
         await this.promiseTracker.waitForAll()
       } catch (error) {
@@ -121,261 +103,181 @@ export class RuntimeOperation {
   private async operate(): Promise<void> {
     this.logger.info("starting operation")
 
-    // create the workset for the operation
-    this.workset = await OperationWorkset.load(
-      this.project,
-      this.operation,
-      this.projectBackend,
+    // 1. create the context for the operation
+    this.context = await OperationContext.load(
+      this.project.id,
       this.libraryBackend,
-      this.stateManager,
-      this.instanceLockService,
       this.instanceStateService,
+      this.projectModelService,
       this.logger,
-      this.abortController.signal,
     )
 
-    // start the process to lock more instances when they become available
-    const lockAbortSignal = new AbortController()
-    this.promiseTracker.track(this.tryLockMore(lockAbortSignal.signal))
-
-    // try to lock as many instances as possible
-    await this.workset.tryLock()
+    // 2. create the plan for the operation (or use provided one)
+    let plan: OperationPhase[]
+    if (this.operation.phases) {
+      plan = this.operation.phases
+    } else {
+      plan = createOperationPlan(
+        this.context,
+        this.operation.type,
+        this.operation.requestedInstanceIds,
+        this.operation.options,
+      )
 
-    if (this.workset.instanceIdsToLockIds.size === 0) {
-      this.logger.debug("successfully locked all instances for the operation before starting")
-      lockAbortSignal.abort()
+      // persist the generated plan
+      await this.updateOperation({ phases: plan })
     }
 
-    // setup abort controllers for the instances
-    for (const instanceId of this.workset.getAllAffectedInstanceIds()) {
-      this.setupInstanceAbortControllers(instanceId)
-    }
+    // 3. create the workset to track the state of the operation
+    this.workset = new OperationWorkset(
+      this.project,
+      this.operation.id,
+      plan,
+      this.context,
+      this.instanceStateService,
+    )
 
-    try {
-      // run the operation
-      await this.processOperation()
-    } finally {
-      lockAbortSignal.abort()
+    // 4. create operation states in datbase for all affected instances in all phases
+    await this.workset.setupOperationStates()
 
-      if (this.operation.type === "preview") {
-        // stream initial states for preview operations
-        await this.workset.restoreAffectedInitialStates()
-      }
-    }
-  }
+    // 5. setup abort controllers for all affected instances in all phases
+    this.workset.setupAbortControllersForAllInstances()
 
-  private setupInstanceAbortControllers(instanceId: string): void {
-    const abortController = new AbortController()
-    this.abortController.signal.addEventListener("abort", () => abortController.abort())
-    this.instanceAbortControllers.set(instanceId, abortController)
+    // 6. progressively lock instances and launch them as they get locked
+    this.launchLockAcquisitionSequence()
 
-    const forceAbortController = new AbortController()
-    this.forceAbortController.signal.addEventListener("abort", () => forceAbortController.abort())
-    this.instanceForceAbortControllers.set(instanceId, forceAbortController)
+    // 7. run the operation
+    await this.processOperation()
   }
 
-  private async tryLockMore(signal: AbortSignal): Promise<void> {
-    const eventStream = this.pubsubManager.subscribe(["instance-lock", this.project.id], signal)
-
-    for await (const event of eventStream) {
-      if (event.type !== "unlocked") {
-        continue
-      }
-
-      const instanceIdsToLock = event.instanceIds.filter(
-        //
-        instanceId => this.workset.instanceIdsToLockIds.has(instanceId),
-      )
-
-      if (instanceIdsToLock.length === 0) {
-        try {
-          await this.workset.tryLock(instanceIdsToLock)
-        } catch (error) {
-          this.logger.error({ error }, "failed to lock more instances during operation")
-        }
-      }
-
-      if (this.workset.instanceIdsToLockIds.size === 0) {
-        // no more instances to lock, stop listening for events
-        this.logger.debug("successfully locked all instances for the operation")
-        break
-      }
-    }
+  private async updateOperation(patch: OperationUpdateInput): Promise<void> {
+    Object.assign(this.operation, patch)
+    await this.operationService.updateOperation(this.project.id, this.operation.id, patch)
   }
 
-  private async processOperation(): Promise<void> {
-    this.operation.instanceIdsToUpdate = this.workset.operation.instanceIdsToUpdate
-    this.operation.instanceIdsToDestroy = this.workset.operation.instanceIdsToDestroy
+  private async writeOperationLog(message: string): Promise<void> {
+    this.promiseTracker.track(
+      this.operationService.appendLog(this.project.id, this.operation.id, null, message),
+    )
+  }
 
-    this.logger.info(
-      "operation started with %s instances to update and %s to destroy",
-      this.operation.instanceIdsToUpdate.length,
-      this.operation.instanceIdsToDestroy.length,
+  private launchLockAcquisitionSequence(): void {
+    this.promiseTracker.track(
+      this.instanceLockService.lockInstances(
+        this.project.id,
+        Array.from(this.workset.allAffectedStateIds),
+        {
+          title: "Operation Lock",
+          description: `The instance is locked for the ${this.operation.type} operation "${this.operation.id}".`,
+          icon: "mdi:cog-sync",
+        },
+        async (_tx, newlyLockedStateIds) => {
+          // trigger all newly locked instances to start their processing
+          for (const stateId of newlyLockedStateIds) {
+            this.workset.markInstanceLocked(stateId)
+          }
+        },
+        true, // allow partial locks to allow independent free branches to run
+        this.workset.abortController.signal,
+        60_000, // wait up to 60 seconds for unlock events before retrying
+        this.unlockToken,
+      ),
     )
+  }
 
-    const phases = this.getOperationPhases()
+  private async processOperation(): Promise<void> {
     const errors: string[] = []
-    let hasAbortError = false
 
-    for (const phase of phases) {
-      this.workset.currentPhase = phase
+    while (this.workset.hasRemainingPhases()) {
+      this.workset.nextPhase()
+      this.instancePromiseMap.clear()
 
-      const promises: Promise<void>[] = []
-      for (const instanceId of this.workset.getAffectedInstanceIds()) {
-        const parentId = this.workset.getParentId(instanceId)
-        if (parentId && this.workset.isAffected(parentId)) {
-          // do not call the operation for child instances of affected composites,
-          // they will be called by their parent instance
-          continue
-        }
+      // lauch all instances in this phase
+      for (const instanceId of this.workset.phaseAffectedInstanceIds) {
+        const instance = this.context.getInstance(instanceId)
+        const state = this.context.getState(instanceId)
+        const promise = this.getInstancePromiseForOperation(instance, state)
 
-        promises.push(this.getInstancePromiseForOperation(instanceId))
+        this.instancePromiseMap.set(instanceId, promise)
+        this.promiseTracker.track(promise)
       }
 
-      this.logger.info(`all operations for phase "%s" started`, phase)
-      this.operation.status = "running"
-      await this.operationService.updateOperation(this.project.id, this.operation)
-
-      const phaseResults = await Promise.allSettled(promises)
-      for (const result of phaseResults) {
-        if (result.status !== "rejected") {
-          continue
-        }
-
-        if (isAbortErrorLike(result.reason)) {
-          hasAbortError = true
-        } else {
-          errors.push(errorToString(result.reason))
-        }
-      }
+      // wait for all instances in this phase to complete
+      await this.promiseTracker.waitForAll()
 
-      this.logger.info(`all operations for phase "%s" completed`, phase)
+      this.logger.info(`phase "%s" completed`, this.workset.currentPhase)
     }
 
     if (errors.length > 0) {
       this.operation.status = "failed"
-      // TODO: map errors to instances inside operation
-      this.operation.error = errors.join("\n\n")
-    } else if (hasAbortError) {
-      this.operation.status = "cancelled"
-      this.operation.error = null
+
+      this.operationService.appendLog(
+        this.project.id,
+        this.operation.id,
+        null,
+        `Operation failed with the following errors:\n${errors.join("\n")}`,
+      )
     } else {
       this.operation.status = "completed"
-      this.operation.error = null
     }
 
-    await this.operationService.updateOperation(this.project.id, this.operation)
+    await this.operationService.markOperationFinished(
+      this.project.id,
+      this.operation.id,
+      this.operation.status,
+    )
 
     this.logger.info("operation completed")
   }
 
-  cancelInstance(instanceId: string): void {
-    const abortController = this.instanceAbortControllers.get(instanceId)
-    if (!abortController) {
-      throw new Error(`No abort controller found for instance "${instanceId}"`)
-    }
-
-    if (!abortController.signal.aborted) {
-      this.logger.info(`cancelling operation for instance "${instanceId}"`)
-      abortController.abort()
-
-      // just the UX feature to indicate that we are trying to cancel the operation
-      this.promiseTracker.track(
-        this.workset.patchState({
-          id: instanceId,
-          operationStatus: {
-            status: "cancelling",
-          },
-        }),
-      )
-
-      return
-    }
-
-    // then try to force cancel the operation
-    const forceAbortController = this.instanceForceAbortControllers.get(instanceId)
-    if (!forceAbortController) {
-      throw new Error(`No force abort controller found for instance "${instanceId}"`)
-    }
-
-    if (!forceAbortController.signal.aborted) {
-      this.logger.info(`force cancelling operation for instance "${instanceId}"`)
-      forceAbortController.abort()
-      return
-    }
-
-    this.logger.warn(`operation for instance "${instanceId}" is already force cancelled`)
-  }
-
-  cancel(): void {
-    if (!this.abortController.signal.aborted) {
-      this.logger.info("cancelling the operation")
-      this.abortController.abort()
-      return
-    }
-
-    // then try to force cancel the operation
-    if (!this.forceAbortController.signal.aborted) {
-      this.logger.info("force cancelling the operation")
-      this.forceAbortController.abort()
-      return
-    }
-
-    this.logger.warn("the operation is already cancelled or force cancelled")
-  }
-
-  private getInstancePromiseForOperation(instanceId: string): Promise<void> {
-    const [instanceType] = parseInstanceId(instanceId)
-    const component = this.workset.library.components[instanceType]
-
-    if (isUnitModel(component)) {
-      return this.getUnitPromise(instanceId)
+  private getInstancePromiseForOperation(
+    instance: InstanceModel,
+    state: InstanceState,
+  ): Promise<void> {
+    if (instance.kind === "unit") {
+      return this.getUnitPromise(instance, state)
     }
 
-    return this.getCompositePromise(instanceId)
-  }
-
-  private getOperationPhases(): OperationPhase[] {
-    switch (this.operation.type) {
-      case "update": {
-        if (this.operation.instanceIdsToDestroy.length > 0) {
-          return ["destroy", "update"]
-        }
-
-        return ["update"]
-      }
-      case "preview":
-        return ["update"]
-      case "recreate":
-        return ["destroy", "update"]
-      case "destroy":
-        return ["destroy"]
-      case "refresh":
-        return ["refresh"]
-    }
+    return this.getCompositePromise(instance)
   }
 
-  private async getUnitPromise(instanceId: string): Promise<void> {
+  private getUnitPromise(instance: InstanceModel, state: InstanceState): Promise<void> {
     switch (this.workset.currentPhase) {
       case "update": {
-        return this.updateUnit(instanceId)
+        return this.updateUnit(instance, state)
       }
       case "destroy": {
-        return this.destroyUnit(instanceId)
+        return this.destroyUnit(instance, state)
       }
       case "refresh": {
-        return this.refreshUnit(instanceId)
+        return this.refreshUnit(instance, state)
       }
     }
   }
 
-  private getCompositePromise(instanceId: string): Promise<void> {
-    return this.getInstancePromise(instanceId, async logger => {
-      const instance = this.workset.getInstance(instanceId)
+  private getCompositePromise(instance: InstanceModel): Promise<void> {
+    return this.getInstancePromise(instance.id, async logger => {
+      let instanceState: InstanceStatePatch | undefined
 
-      await this.setInitialOperationStatus(instance)
+      // set parent ID on update operation
+      if (this.workset.currentPhase === "update") {
+        if (instance.parentId) {
+          const parentState = this.context.getState(instance.parentId)
+          instanceState = { parentId: parentState.id }
+        } else {
+          instanceState = { parentId: null }
+        }
+      }
+
+      await this.workset.updateState(instance.id, {
+        operationState: {
+          status: this.workset.getTransientStatusByOperationPhase(),
+          startedAt: new Date(),
+        },
+        instanceState,
+      })
 
-      const children = this.workset.getAffectedCompositeChildren(instanceId)
+      const children = this.workset.getAffectedCompositeChildren(instance.id)
       const childPromises: Promise<void>[] = []
 
       if (children.length) {
@@ -385,65 +287,37 @@ export class RuntimeOperation {
       }
 
       for (const child of children) {
-        if (!this.operation.options.forceUpdateChildren && !this.workset.isAffected(child.id)) {
-          // skip children that are not affected by the operation
-          continue
-        }
+        logger.debug(`waiting for child "%s"`, child)
 
-        logger.debug(`waiting for child "%s"`, child.id)
-        childPromises.push(this.getInstancePromiseForOperation(child.id))
-      }
-
-      try {
-        await Promise.all(childPromises)
+        const instance = this.context.getInstance(child)
+        const state = this.context.getState(child)
+        const promise = this.getInstancePromiseForOperation(instance, state)
 
-        if (children.length > 0) {
-          logger.debug("all children completed")
-        }
+        childPromises.push(promise)
+      }
 
-        if (this.operation.type === "destroy") {
-          await this.clearOperationState(instanceId)
-          return
-        }
+      await waitAll(childPromises)
 
-        const { inputHash, dependencyOutputHash } =
-          await this.workset.getUpToDateInputHashOutput(instance)
+      logger.debug("all children completed")
 
-        await this.workset.patchState({
-          id: instanceId,
-          operationStatus: {
-            status: "created",
-            inputHash,
-            dependencyOutputHash,
+      // update the instance and operation state after all children are completed in last phase
+      if (this.workset.isLastPhaseForInstance(instance.id)) {
+        await this.workset.updateState(instance.id, {
+          operationState: {
+            status: this.workset.getStableStatusByOperationPhase(),
+            finishedAt: new Date(),
           },
-        })
-      } catch (error) {
-        if (isAbortErrorLike(error)) {
-          await this.workset.restoreInitialState(instanceId)
-          return
-        }
-
-        await this.workset.patchState({
-          id: instanceId,
-          operationStatus: {
-            status: "error",
-            message: errorToString(error),
+          instanceState: {
+            status: this.workset.getNextStableInstanceStatus(instance.id),
+            message: null,
           },
         })
       }
     })
   }
 
-  private updateUnit(instanceId: string): Promise<void> {
-    return this.getInstancePromise(instanceId, async (logger, signal, forceSignal) => {
-      const instance = this.workset.getInstance(instanceId)
-      const component = this.workset.library.components[instance.type]
-      const perfLogger = new PerformanceLogger(logger)
-      perfLogger.log("starting update promise for instance")
-
-      await this.setInitialOperationStatus(instance)
-      perfLogger.log("initial operation status set")
-
+  private updateUnit(instance: InstanceModel, state: InstanceState): Promise<void> {
+    return this.getInstancePromise(instance.id, async (logger, signal, forceSignal) => {
       await Promise.race([
         this.updateUnitDependencies(instance.id, logger),
 
@@ -451,91 +325,81 @@ export class RuntimeOperation {
         waitForAbort(signal),
       ])
 
-      if (!signal.aborted) {
-        perfLogger.log("dependencies updated")
-      }
-
       signal.throwIfAborted()
 
       logger.info("updating unit")
 
-      await this.workset.patchState({
-        id: instance.id,
-        operationStatus: {
+      await this.workset.updateState(instance.id, {
+        operationState: {
           status: "updating",
-          currentResourceCount: 0,
-          totalResourceCount: 0,
+          startedAt: new Date(),
         },
       })
 
-      perfLogger.log("set 'updating' operation status")
       signal.throwIfAborted()
 
-      const secrets = await this.secretService.getInstanceSecretValues(this.project, instance.id)
+      const secrets = await this.secretService.getInstanceSecretValues(this.project.id, state.id)
 
-      perfLogger.log("secrets loaded")
       signal.throwIfAborted()
 
-      const config = this.prepareUnitConfig(instance, component)
-      perfLogger.log("unit config prepared")
+      const config = this.prepareUnitConfig(instance, secrets)
 
-      // extract artifact hashes from dependencies based on instance inputs
-      const artifactHashes = this.collectArtifactIdsForInstance(instance)
-      const artifacts = await this.stateManager
-        .getArtifactHashIndexRepository(this.project.id)
-        .getManyItems(artifactHashes)
+      // collect artifacts authorized for this instance
+      const artifactIds = this.collectArtifactIdsForInstance(instance)
+      const artifacts = await this.artifactService.getArtifactsByIds(this.project.id, artifactIds)
 
-      logger.debug({ count: artifactHashes.length }, "artifact hashes collected from dependencies")
-      perfLogger.log("artifact hashes collected")
+      logger.debug({ count: artifactIds.length }, "artifact ids collected from dependencies")
 
       await this.runnerBackend[this.operation.type === "preview" ? "preview" : "update"]({
         projectId: this.project.id,
         libraryId: this.project.libraryId,
+        stateId: state.id,
         instanceType: instance.type,
         instanceName: instance.name,
         config,
         refresh: this.operation.options.refresh,
-        secrets: mapValues(secrets, value => valueToString(value)),
+        secrets,
         artifacts,
         signal,
         forceSignal,
+        debug: this.operation.options.debug,
       })
 
-      perfLogger.log("unit update requested")
-
-      await this.watchStateStream(instance.type, instance.name, logger)
-      perfLogger.log("unit update completed")
+      await this.watchStateStream(state, instance.type, instance.name, logger)
       logger.info("unit updated")
     })
   }
 
-  private async updateUnitDependencies(instanceId: string, logger: Logger): Promise<void> {
+  private async updateUnitDependencies(instanceId: InstanceId, logger: Logger): Promise<void> {
     try {
-      const dependencies = this.getInstanceDependencyIds(instanceId)
+      const dependencies = this.context.getDependencies(instanceId)
       const dependencyPromises: Promise<void>[] = []
 
-      for (const dependencyId of dependencies) {
-        if (!this.operation.instanceIdsToUpdate.includes(dependencyId)) {
+      for (const dependency of dependencies) {
+        if (!this.workset.phaseAffectedInstanceIds.has(dependency.id)) {
           // skip dependencies that are not affected by the operation
           continue
         }
 
-        logger.debug(`waiting for dependency "${dependencyId}"`)
-        dependencyPromises.push(this.getInstancePromiseForOperation(dependencyId))
+        const state = this.context.getState(dependency.id)
+
+        logger.debug(`waiting for dependency "%s"`, dependency.id)
+        dependencyPromises.push(this.getInstancePromiseForOperation(dependency, state))
       }
 
-      await Promise.all(dependencyPromises)
+      await waitAll(dependencyPromises)
 
       if (dependencies.length > 0) {
         logger.info("all dependencies completed")
       }
-    } catch {
+    } catch (error) {
       // abort the instance if any dependency fails
-      throw new AbortError()
+      throw new AbortError("One of the dependencies failed", { cause: error })
     }
   }
 
   private async processBeforeDestroyTriggers(
+    instance: InstanceModel,
     state: InstanceState,
     logger: Logger,
     signal: AbortSignal,
@@ -546,15 +410,7 @@ export class RuntimeOperation {
       return
     }
 
-    const instance = this.workset.getInstance(state.id)
-    const component = this.workset.library.components[instance.type]
-
-    // fetch triggers from state backend by their IDs
-    const triggerIds = Object.values(state.extra?.triggerIds ?? {})
-
-    const allTriggers = await this.stateManager
-      .getTriggerRepository(this.project.id)
-      .getManyItems(triggerIds)
+    const allTriggers = await this.unitExtraService.getInstanceTriggers(this.project.id, state.id)
 
     const beforeDestroyTriggers = allTriggers.filter(
       trigger => trigger.spec.type === "before-destroy",
@@ -564,95 +420,73 @@ export class RuntimeOperation {
       return
     }
 
-    // get trigger names from state mapping by finding the key for each trigger ID
-    const invokedTriggers = beforeDestroyTriggers.map(trigger => {
-      const triggerName = Object.keys(state?.extra?.triggerIds ?? {}).find(
-        name => state?.extra?.triggerIds?.[name] === trigger.id,
-      )
-
-      return { name: triggerName || "unknown" }
-    })
+    const invokedTriggers = beforeDestroyTriggers.map(trigger => ({
+      name: trigger.name,
+    }))
 
-    await this.workset.patchState({
-      id: instance.id,
-      operationStatus: {
-        status: "processing-triggers",
-      },
+    await this.workset.updateState(instance.id, {
+      operationState: { status: "processing_triggers" },
     })
 
     logger.info("updating unit to process before-destroy triggers...")
 
-    const secrets = await this.secretService.getInstanceSecretValues(this.project, instance.id)
+    const secrets = await this.secretService.getInstanceSecretValues(this.project.id, state.id)
 
     await this.runnerBackend.update({
       projectId: this.project.id,
+      stateId: state.id,
       libraryId: this.project.libraryId,
       instanceType: instance.type,
       instanceName: instance.name,
-      config: this.prepareUnitConfig(instance, component, invokedTriggers),
+      config: this.prepareUnitConfig(instance, secrets, invokedTriggers),
       refresh: this.operation.options.refresh,
-      secrets: mapValues(secrets, value => valueToString(value)),
+      secrets,
       signal,
       forceSignal,
+      debug: this.operation.options.debug,
     })
 
     logger.debug("unit update requested")
 
-    await this.watchStateStream(instance.type, instance.name, logger)
+    await this.watchStateStream(state, instance.type, instance.name, logger)
     logger.debug("before-destroy triggers processed")
   }
 
-  private async destroyUnit(instanceId: string): Promise<void> {
-    return this.getInstancePromise(instanceId, async (logger, signal, forceSignal) => {
-      await this.workset.patchState({
-        id: instanceId,
-        operationStatus: {
-          status: "pending",
-        },
-      })
-
-      signal.throwIfAborted()
-
-      const state = this.workset.getState(instanceId)
-      if (!state) {
-        logger.warn("state not found for unit, but destroy was requested")
-        return
-      }
-
+  private async destroyUnit(instance: InstanceModel, state: InstanceState): Promise<void> {
+    return this.getInstancePromise(instance.id, async (logger, signal, forceSignal) => {
       const dependentPromises: Promise<void>[] = []
-      const dependents = this.workset.getDependentStates(state.id)
+      const dependents = this.context.getDependentStates(instance.id)
 
       for (const dependent of dependents) {
-        if (
-          !this.operation.options.destroyDependentInstances &&
-          !this.operation.instanceIdsToDestroy.includes(dependent.id)
-        ) {
+        if (!this.workset.phaseAffectedInstanceIds.has(dependent.instanceId)) {
           // skip dependents that are not affected by the operation
           continue
         }
 
-        dependentPromises.push(this.getInstancePromiseForOperation(dependent.id))
+        const instance = this.context.getInstance(dependent.instanceId)
+        dependentPromises.push(this.getInstancePromiseForOperation(instance, dependent))
       }
 
-      await Promise.all(dependentPromises)
+      await waitAll(dependentPromises)
       signal.throwIfAborted()
 
-      await this.processBeforeDestroyTriggers(state, logger, signal, forceSignal)
+      await this.processBeforeDestroyTriggers(instance, state, logger, signal, forceSignal)
       signal.throwIfAborted()
 
       logger.info("destroying unit...")
 
-      await this.workset.patchState({
-        id: instanceId,
-        operationStatus: {
+      await this.workset.updateState(instance.id, {
+        operationState: {
           status: "destroying",
+          startedAt: new Date(),
         },
       })
 
-      const [type, name] = parseInstanceId(instanceId)
+      const [type, name] = parseInstanceId(instance.id)
 
       await this.runnerBackend.destroy({
         projectId: this.project.id,
+        stateId: state.id,
         libraryId: this.project.libraryId,
         instanceType: type,
         instanceName: name,
@@ -661,89 +495,76 @@ export class RuntimeOperation {
         forceSignal,
         deleteUnreachable: this.operation.options.deleteUnreachableResources,
         forceDeleteState: this.operation.options.forceDeleteState,
+        debug: this.operation.options.debug,
       })
 
       logger.debug("destroy request sent")
 
-      await this.watchStateStream(type, name, logger)
+      await this.watchStateStream(state, type, name, logger)
 
       // clean up artifacts after successful destruction
-      try {
-        const artifactIds = unique(Object.values(state.extra?.exportedArtifactIds ?? {}).flat())
-
-        await this.artifactService.removeUsages(
-          //
-          this.project.id,
-          artifactIds,
-          [{ type: "instance", instanceId: state.id }],
-        )
-      } catch (error) {
-        logger.warn({ error }, "failed to cleanup artifacts for destroyed instance")
-      }
+      await this.artifactService.clearInstanceArtifactReferences(this.project.id, instance.id)
 
       logger.info("unit destroyed")
     })
   }
 
-  private async refreshUnit(instanceId: string) {
-    return this.getInstancePromise(instanceId, async (logger, signal, forceSignal) => {
-      await this.workset.patchState({
-        id: instanceId,
-        operationStatus: {
+  private async refreshUnit(instance: InstanceModel, state: InstanceState): Promise<void> {
+    return this.getInstancePromise(instance.id, async (logger, signal, forceSignal) => {
+      await this.workset.updateState(instance.id, {
+        operationState: {
           status: "refreshing",
-          currentResourceCount: 0,
-          totalResourceCount: 0,
+          startedAt: new Date(),
         },
       })
 
       logger.info("refreshing unit...")
 
-      const [type, name] = parseInstanceId(instanceId)
+      const [type, name] = parseInstanceId(instance.id)
 
       await this.runnerBackend.refresh({
         projectId: this.project.id,
+        stateId: state.id,
         libraryId: this.project.libraryId,
         instanceType: type,
         instanceName: name,
         signal,
         forceSignal,
+        debug: this.operation.options.debug,
       })
 
       logger.debug("unit refresh requested")
 
-      await this.watchStateStream(type, name, logger)
+      await this.watchStateStream(state, type, name, logger)
       logger.info("unit refreshed")
     })
   }
 
   private async watchStateStream(
-    instanceType: string,
+    state: InstanceState,
+    instanceType: VersionedName,
     instanceName: string,
     logger: Logger,
   ): Promise<void> {
     const stream = this.runnerBackend.watch({
       projectId: this.project.id,
+      stateId: state.id,
       libraryId: this.project.libraryId,
       instanceType,
       instanceName,
+      debug: this.operation.options.debug,
     })
 
     let update: UnitStateUpdate | undefined
 
     for await (update of stream) {
       try {
-        await this.handleUnitStateUpdate(update)
+        await this.handleUnitStateUpdate(update, state)
       } catch (error) {
         logger.error({ error }, "failed to handle unit state update")
       }
 
       if (update.type === "error") {
-        if (isAbortErrorLike(update.message)) {
-          // abort the unit if the returned error contains some abort-like pattern
-          // generally, this might not be safe, but for now, we assume that
-          throw new AbortError()
-        }
-
         // rethrow the error to stop the execution of dependent units
         throw new Error(`An error occurred while processing the unit "${update.unitId}"`, {
           cause: update.message,
@@ -762,204 +583,155 @@ export class RuntimeOperation {
 
   private prepareUnitConfig(
     instance: InstanceModel,
-    component: ComponentModel,
+    secrets: Record<string, unknown>,
     invokedTriggers: TriggerInvocation[] = [],
-  ): Record<string, string> {
-    const config: Record<string, string> = {}
+  ): UnitConfig {
+    const resolvedInputs = this.context.getResolvedInputs(instance.id)
 
-    for (const key of Object.keys(component.args)) {
-      // explicitly set empty values to remove old values in the config
-      const value = instance.args?.[key]
-      config[key] = value ? valueToString(value) : ""
+    return {
+      instanceId: instance.id,
+      args: instance.args ?? {},
+      inputs: mapValues(resolvedInputs ?? {}, input => input.map(value => value.input)),
+      invokedTriggers,
+      secretNames: Object.keys(secrets),
+      stateIdMap: this.context.getInstanceIdToStateIdMap(),
     }
-
-    const instanceInputs = this.workset.resolvedInstanceInputs.get(instance.id) ?? {}
-
-    for (const key of Object.keys(component.inputs)) {
-      const inputs = instanceInputs[key] ?? []
-
-      config[`input.${key}`] = JSON.stringify(inputs.map(input => input.input))
-    }
-
-    config["$invokedTriggers"] = JSON.stringify(invokedTriggers)
-
-    return config
   }
 
-  private async handleUnitStateUpdate(update: UnitStateUpdate): Promise<void> {
+  private async handleUnitStateUpdate(
+    update: UnitStateUpdate,
+    state: InstanceState,
+  ): Promise<void> {
     switch (update.type) {
       case "message":
-        this.handleUnitMessage(update)
+        this.handleUnitMessage(update, state)
         return
       case "progress":
         await this.handleUnitProgress(update)
         return
       case "error":
-        await this.handleUnitError(update)
+        await this.handleUnitError(update, state)
         return
       case "completion":
-        await this.handleUnitCompletion(update)
+        await this.handleUnitCompletion(update, state)
         return
     }
   }
 
-  private handleUnitMessage(update: TypedUnitStateUpdate<"message">): void {
-    // collect all instances in the hierarchy that should receive this log
-    const instanceIdsInHierarchy: string[] = []
-    let instance: InstanceModel | null = this.workset.getInstance(update.unitId)
-
+  private handleUnitMessage(update: TypedUnitStateUpdate<"message">, state: InstanceState): void {
+    // append logs to the instance and all its parents
     for (;;) {
-      instanceIdsInHierarchy.push(instance.id)
+      this.promiseTracker.track(
+        this.operationService.appendLog(
+          this.project.id,
+          this.operation.id,
+          state.id,
+          update.message,
+        ),
+      )
 
-      if (!instance.parentId) {
+      if (!state.parentInstanceId) {
         break
       }
 
-      instance = this.workset.getInstance(instance.parentId)
+      state = this.context.getState(state.parentInstanceId)
     }
-
-    // persist the log for all instances in the hierarchy
-    // TODO: batch
-    this.promiseTracker.track(
-      this.operationService.appendLogs(this.project.id, this.operation.id, instanceIdsInHierarchy, [
-        update.message,
-      ]),
-    )
-    return
   }
 
   private async handleUnitProgress(update: TypedUnitStateUpdate<"progress">): Promise<void> {
-    const patch: InstanceStatePatch = { id: update.unitId }
-    patch.operationStatus = {}
-
-    if (update.currentResourceCount !== undefined) {
-      patch.operationStatus.currentResourceCount = update.currentResourceCount
-    }
-
-    if (update.totalResourceCount !== undefined) {
-      patch.operationStatus.totalResourceCount = update.totalResourceCount
-    }
-
-    await this.workset.patchState(patch)
+    await this.workset.updateState(update.unitId, {
+      operationState: {
+        currentResourceCount: update.currentResourceCount,
+        totalResourceCount: update.totalResourceCount,
+      },
+    })
   }
 
-  private async handleUnitError(update: TypedUnitStateUpdate<"error">): Promise<void> {
-    if (isAbortErrorLike(update.message)) {
-      return
-    }
-
-    const patch: InstanceStatePatch = {
-      id: update.unitId,
-      operationStatus: {
-        status: "error",
+  private async handleUnitError(
+    update: TypedUnitStateUpdate<"error">,
+    state: InstanceState,
+  ): Promise<void> {
+    await this.workset.updateState(update.unitId, {
+      instanceState: {
+        // keep "deployed" status for initially deployed instances even if the operation was failed or cancelled
+        status: state.status === "deployed" ? "deployed" : "failed",
         message: update.message,
       },
-    }
-
-    await this.workset.patchState(patch)
+      operationState: {
+        status: isAbortErrorLike(update.message) ? "cancelled" : "failed",
+        finishedAt: new Date(),
+      },
+    })
   }
 
-  private async handleUnitCompletion(update: TypedUnitStateUpdate<"completion">): Promise<void> {
-    if (this.workset.currentPhase === "destroy") {
-      // if the destroy operation was completed, clear the state from all operation-related fields
-      await this.clearOperationState(update.unitId)
-      return
-    }
-
-    const state = this.workset.getState(update.unitId)
-    if (!state) {
-      throw new Error(
-        `Cannot handle completion for unit "${update.unitId}" because its state is not found.`,
-      )
-    }
-
-    const instance = this.workset.getInstance(update.unitId)
+  private async handleUnitCompletion(
+    update: TypedUnitStateUpdate<"completion">,
+    state: InstanceState,
+  ): Promise<void> {
+    const instance = this.context.getInstance(update.unitId)
 
-    // patch the state with output hash first to recalculate the input hash
-    if (update.outputHash) {
-      await this.workset.patchState({
-        id: update.unitId,
-        operationStatus: { outputHash: update.outputHash },
-      })
+    const data: InstanceStatePatch = {
+      status: this.workset.getNextStableInstanceStatus(instance.id),
+      message: update.message,
+      statusFields: update.statusFields ?? null,
     }
 
-    const { inputHash, dependencyOutputHash } =
-      await this.workset.getUpToDateInputHashOutput(instance)
-
-    const patch: InstanceStatePatch = {
-      id: update.unitId,
-      operationStatus: {
-        status: "created",
-        message:
-          update.message ?? "The operation on this instance has been completed successfully.",
-        outputHash: update.outputHash ?? undefined,
-        inputHash,
-        dependencyOutputHash,
-      },
-      extra: {
-        statusFields: update.statusFields,
-
-        terminalIds: update.terminals
-          ? this.processTerminals(update.unitId, update.terminals, state.extra?.terminalIds)
-          : null,
+    if (update.operationType !== "destroy") {
+      // давайте еще больше усложним и без того сложную штуку
+      // set output hash before calculating input hash to capture up-to-date output hash for dependencies
+      state.outputHash = update.outputHash ?? null
 
-        pageIds: update.pages
-          ? this.processPages(update.unitId, update.pages, state.extra?.pageIds)
-          : null,
+      // recalculate the input and output hashes for the instance
+      const { inputHash, dependencyOutputHash } =
+        await this.context.getUpToDateInputHashOutput(instance)
 
-        triggerIds: update.triggers
-          ? this.processTriggers(update.unitId, update.triggers, state.extra?.triggerIds)
-          : null,
+      data.inputHash = inputHash
+      data.dependencyOutputHash = dependencyOutputHash
+      data.outputHash = update.outputHash
 
-        workerRegistrationIds: update.workers
-          ? await this.workerService.updateUnitRegistrations(
-              this.project.id,
-              update.unitId,
-              state.extra?.workerRegistrationIds ?? {},
-              update.workers,
-            )
-          : null,
-
-        exportedArtifactIds: update.exportedArtifacts
-          ? this.processArtifacts(
-              update.unitId,
-              update.exportedArtifacts,
-              state.extra?.exportedArtifactIds,
-            )
-          : null,
-
-        exportedSecretIds: update.exportedSecrets
-          ? this.processExportedSecrets(update.unitId, update.exportedSecrets)
-          : null,
+      // also update the parent ID
+      if (instance.parentId) {
+        const parentState = this.context.getState(instance.parentId)
+        data.parentId = parentState.id
+      } else {
+        data.parentId = null
+      }
+    } else {
+      data.message = null
+      data.inputHash = null
+      data.dependencyOutputHash = null
+      data.outputHash = null
+      data.parentId = null
+    }
+
+    // update the operation state
+    await this.workset.updateState(instance.id, {
+      // TODO: honestly, it is not correct
+      // may be we should track operation phases separately
+      // or introduce status like "destroyed-before-recreation" (quite ugly though)
+      operationState: {
+        status: this.workset.getStableStatusByOperationPhase(),
+        finishedAt: new Date(),
       },
-    }
-
-    if (update.secrets) {
-      const instance = this.workset.getInstance(update.unitId)
 
-      this.promiseTracker.track(
-        this.updateInstanceSecrets(
-          instance,
-          mapValues(update.secrets, value => stringToValue(value)),
-        ),
-      )
-    }
-
-    await this.workset.patchState(patch)
-  }
-
-  private async clearOperationState(instanceId: string): Promise<void> {
-    const patch: InstanceStatePatch = {
-      id: instanceId,
-      operationStatus: null,
-      extra: null,
-    }
-
-    await this.workset.patchState(patch)
+      // do not write instance state for non-last phases of the instance
+      instanceState: this.workset.isLastPhaseForInstance(instance.id) ? data : undefined,
+
+      // also do not write unit extra data for non-last phases of the instance
+      unitExtra: this.workset.isLastPhaseForInstance(instance.id)
+        ? {
+            pages: update.pages ?? [],
+            terminals: update.terminals ?? [],
+            triggers: update.triggers ?? [],
+            workers: update.workers ?? [],
+            secrets: update.secrets ?? {},
+          }
+        : undefined,
+    })
   }
 
   private getInstancePromise(
-    instanceId: string,
+    instanceId: InstanceId,
     fn: (logger: Logger, signal: AbortSignal, forceSignal: AbortSignal) => Promise<void>,
   ): Promise<void> {
     let instancePromise = this.instancePromiseMap.get(instanceId)
@@ -967,30 +739,54 @@ export class RuntimeOperation {
       return instancePromise
     }
 
-    const logger = this.logger.child({ instanceId }, { msgPrefix: `[${instanceId}] ` })
-    const abortController = this.instanceAbortControllers.get(instanceId)
-    const forceAbortController = this.instanceForceAbortControllers.get(instanceId)
+    // "pending" -> "running" if at least one instance is running
+    if (this.operation.status === "pending") {
+      this.operation.status = "running"
+      this.promiseTracker.track(this.updateOperation({ status: this.operation.status }))
+    }
+
+    const state = this.context.getState(instanceId)
 
-    if (!abortController || !forceAbortController) {
+    const logger = this.logger.child({ instanceId }, { msgPrefix: `[${instanceId}] ` })
+    const abortControllerPair = this.workset.instanceAbortControllers.get(instanceId)
+    if (!abortControllerPair) {
       throw new Error(`Abort controllers for instance "${instanceId}" are not initialized`)
     }
 
+    const { abortController, forceAbortController } = abortControllerPair
+
     instancePromise = this.workset
-      .ensureInstanceLocked(instanceId, abortController.signal)
+      .waitForInstanceLock(state.id, abortController.signal)
       .then(() => fn(logger, abortController.signal, forceAbortController.signal))
       .catch(error => {
-        if (isAbortErrorLike(error)) {
-          // if the operation was aborted, restore the initial state of the instance
-          this.promiseTracker.track(this.workset.restoreInitialState(instanceId))
-
-          throw error
-        }
-
         if (this.operation.status !== "failing") {
           // report the failing status of the operation
           this.operation.status = "failing"
+          this.promiseTracker.track(this.updateOperation({ status: this.operation.status }))
+        }
+
+        if (isTransientInstanceOperationStatus(state.lastOperationState?.status)) {
+          // if the underlying method did not correctly update the instance status, do it here
           this.promiseTracker.track(
-            this.operationService.updateOperation(this.project.id, this.operation),
+            this.workset.updateState(instanceId, {
+              operationState: {
+                status: isAbortErrorLike(error) ? "cancelled" : "failed",
+                finishedAt: new Date(),
+              },
+              instanceState: {
+                // keep "deployed" status for initially deployed instances even if the operation was failed or cancelled
+                status: state.status === "deployed" ? "deployed" : "failed",
+              },
+            }),
+          )
+
+          this.promiseTracker.track(
+            this.operationService.appendLog(
+              this.project.id,
+              this.operation.id,
+              state.id,
+              errorToString(error),
+            ),
           )
         }
 
@@ -998,48 +794,67 @@ export class RuntimeOperation {
         throw error
       })
       .finally(() => {
+        if (!this.workset.isLastPhaseForInstance(instanceId)) {
+          // do not finalize the instance if it has more phases to run
+          return
+        }
+
         this.instancePromiseMap.delete(instanceId)
 
         // TODO: ideally we should defer unlocking until all direct dependents are completed,
         // to ensure that they are received expected inputs from this instance
         this.promiseTracker.track(
-          this.instanceLockService.unlockInstancesUnconditionally(this.project.id, [instanceId]),
+          this.instanceLockService
+            .unlockInstances(this.project.id, [state.id], this.unlockToken)
+            .then(() => this.workset.markInstanceUnlocked(state.id)),
         )
+
+        this.logger.debug(`promise for instance "%s" completed`, instanceId)
       })
 
     this.instancePromiseMap.set(instanceId, instancePromise)
+    this.logger.trace(`created new promise for instance "%s"`, instanceId)
 
     return instancePromise
   }
 
-  private async setInitialOperationStatus(instance: InstanceModel): Promise<void> {
-    const state = this.workset.getState(instance.id)
-
-    await this.workset.patchState({
-      id: instance.id,
-      parentId: instance.parentId,
-      operationStatus: {
-        status: "pending",
-        message: "",
-        dependencyIds: this.getInstanceDependencyIds(instance.id),
-        latestOperationId: this.operation.id,
-        inputHashNonce: state?.operationStatus?.inputHashNonce ?? randomBytes(4).readInt32LE(),
-        args: instance.args,
-      },
-    })
-  }
+  private async ensureInstancesUnlocked(): Promise<void> {
+    const lockedStateIds = Array.from(this.workset.getLockedStateIds())
+    if (lockedStateIds.length === 0) {
+      return
+    }
 
-  private getInstanceDependencyIds(instanceId: string): string[] {
-    const dependencies = new Set<string>()
-    const instanceInputs = this.workset.resolvedInstanceInputs.get(instanceId) ?? {}
+    this.logger.warn("unlocking %d locked instances before shutting down", lockedStateIds.length)
 
-    for (const inputs of Object.values(instanceInputs)) {
-      for (const input of inputs) {
-        dependencies.add(input.input.instanceId)
-      }
+    await this.instanceLockService.unlockInstances(
+      this.project.id,
+      lockedStateIds,
+      this.unlockToken,
+    )
+  }
+
+  private async ensureOperationStatesFinalized(): Promise<void> {
+    const unfinishedStates = this.context.getUnfinishedOperationStates()
+    if (unfinishedStates.length === 0) {
+      return
     }
 
-    return Array.from(dependencies)
+    this.logger.warn(
+      "finalizing %d unfinished operation states before shutting down",
+      unfinishedStates.length,
+    )
+
+    for (const state of unfinishedStates) {
+      await this.workset.updateState(state.instanceId, {
+        operationState: {
+          status: "failed",
+          finishedAt: new Date(),
+        },
+        instanceState: {
+          status: state.status === "deployed" ? "deployed" : "failed",
+        },
+      })
+    }
   }
 
   /**
@@ -1048,22 +863,22 @@ export class RuntimeOperation {
    */
   private collectArtifactIdsForInstance(instance: InstanceModel): string[] {
     const artifactIds = new Set<string>()
-    const instanceInputs = this.workset.resolvedInstanceInputs.get(instance.id) ?? {}
+    const instanceInputs = this.context.getResolvedInputs(instance.id) ?? {}
 
     for (const inputs of Object.values(instanceInputs)) {
       for (const input of inputs) {
-        const dependencyState = this.workset.getState(input.input.instanceId)
-        if (!dependencyState?.extra?.exportedArtifactIds) {
+        const dependencyState = this.context.getState(input.input.instanceId)
+        if (!dependencyState.exportedArtifactIds) {
           continue
         }
 
         const outputKey = input.input.output
-        const outputArtifacts = dependencyState.extra.exportedArtifactIds[outputKey]
-        if (!outputArtifacts) {
+        const outputArtifactIds = dependencyState.exportedArtifactIds[outputKey]
+        if (!outputArtifactIds) {
           continue
         }
 
-        for (const hash of outputArtifacts) {
+        for (const hash of outputArtifactIds) {
           artifactIds.add(hash)
         }
       }
@@ -1071,327 +886,4 @@ export class RuntimeOperation {
 
     return Array.from(artifactIds)
   }
-
-  /**
-   * Collects secret IDs from dependencies based on the direct connections
-   * from instance inputs to dependency outputs.
-   */
-  private collectSecretIdsForInstance(instance: InstanceModel): Set<string> {
-    const secretIds = new Set<string>()
-    const instanceInputs = this.workset.resolvedInstanceInputs.get(instance.id) ?? {}
-
-    for (const inputs of Object.values(instanceInputs)) {
-      for (const input of inputs) {
-        const dependencyState = this.workset.getState(input.input.instanceId)
-        if (!dependencyState?.extra?.exportedSecretIds) {
-          continue
-        }
-
-        const outputKey = input.input.output
-        const outputSecrets = dependencyState.extra.exportedSecretIds[outputKey]
-        if (!outputSecrets) {
-          continue
-        }
-
-        for (const secretId of outputSecrets) {
-          secretIds.add(secretId)
-        }
-      }
-    }
-
-    return secretIds
-  }
-
-  /**
-   * Updates the secrets for a specific instance.
-   * Processes terminals from instance state update.
-   *
-   * Converts unit terminals to instance terminals and handles cleanup.
-   */
-  private processTerminals(
-    instanceId: string,
-    unitTerminals: UnitTerminal[],
-    currentTerminalIds: Record<string, string> | undefined,
-  ): Record<string, string> {
-    const terminals: Terminal[] = unitTerminals.map(unitTerminal => ({
-      id: currentTerminalIds?.[unitTerminal.name] ?? uuidv7(),
-      instanceId,
-      meta: unitTerminal.meta ?? {},
-    }))
-
-    const terminalSpecEntries: [string, TerminalSpec][] = unitTerminals
-      //
-      .map((unitTerminal, index) => [terminals[index].id, unitTerminal.spec])
-
-    // create mapping from local name to UUID
-    const newTerminalIds: Record<string, string> = {}
-    for (let i = 0; i < unitTerminals.length; i++) {
-      const unitTerminal = unitTerminals[i]
-      const instanceTerminal = terminals[i]
-      newTerminalIds[unitTerminal.name] = instanceTerminal.id
-    }
-
-    // find terminals that need to be deleted (old terminals not in new list)
-    const terminalsToDelete = Object.values(currentTerminalIds ?? {}).filter(
-      id => !Object.values(newTerminalIds).includes(id),
-    )
-
-    if (terminals.length > 0) {
-      this.promiseTracker.track(this.putTerminals(terminals, terminalSpecEntries))
-    }
-
-    if (terminalsToDelete.length > 0) {
-      this.promiseTracker.track(this.deleteTerminals(terminalsToDelete))
-    }
-
-    return newTerminalIds
-  }
-
-  private async putTerminals(
-    terminals: Terminal[],
-    terminalSpecEntries: [string, TerminalSpec][],
-  ): Promise<void> {
-    if (this.operation.type === "preview") {
-      return
-    }
-
-    await using batch = this.stateManager.batch()
-
-    await Promise.all([
-      this.stateManager.getTerminalRepository(this.project.id).putManyItems(terminals, batch),
-
-      this.stateManager
-        .getTerminalSpecRepository(this.project.id)
-        .putMany(terminalSpecEntries, batch),
-    ])
-
-    await batch.write()
-  }
-
-  private async deleteTerminals(terminalIds: string[]): Promise<void> {
-    if (this.operation.type === "preview") {
-      return
-    }
-
-    await using batch = this.stateManager.batch()
-
-    await Promise.all([
-      this.stateManager.getTerminalRepository(this.project.id).deleteMany(terminalIds, batch),
-      this.stateManager.getTerminalSpecRepository(this.project.id).deleteMany(terminalIds, batch),
-    ])
-
-    await batch.write()
-  }
-
-  /**
-   * Processes pages from instance state update.
-   *
-   * Converts unit pages to instance pages and handles cleanup.
-   */
-  private processPages(
-    instanceId: string,
-    unitPages: UnitPage[],
-    currentPageIds: Record<string, string> | undefined,
-  ): Record<string, string> {
-    const pages: Page[] = unitPages.map(unitPage => ({
-      id: currentPageIds?.[unitPage.name] ?? uuidv7(),
-      instanceId,
-      meta: unitPage.meta,
-    }))
-
-    const pageContentEntries: [string, PageBlock[]][] = unitPages
-      //
-      .map((unitPage, index) => [pages[index].id, unitPage.content])
-
-    // create mapping from local name to UUID
-    const newPageIds: Record<string, string> = {}
-    for (let i = 0; i < unitPages.length; i++) {
-      const unitPage = unitPages[i]
-      const instancePage = pages[i]
-      newPageIds[unitPage.name] = instancePage.id
-    }
-
-    // find pages that need to be deleted (old pages not in new list)
-    const pagesToDelete = Object.values(currentPageIds ?? {}).filter(
-      id => !Object.values(newPageIds).includes(id),
-    )
-
-    if (pages.length > 0) {
-      this.promiseTracker.track(this.putPages(pages, pageContentEntries))
-    }
-
-    if (pagesToDelete.length > 0) {
-      this.promiseTracker.track(this.deletePages(pagesToDelete))
-    }
-
-    return newPageIds
-  }
-
-  private async putPages(
-    pages: Page[],
-    pageContentEntries: [string, PageBlock[]][],
-  ): Promise<void> {
-    if (this.operation.type === "preview") {
-      return
-    }
-
-    const batch = this.stateManager.batch()
-
-    await Promise.all([
-      this.stateManager.getPageRepository(this.project.id).putManyItems(pages, batch),
-      this.stateManager
-        .getPageContentRepository(this.project.id)
-        .putMany(pageContentEntries, batch),
-    ])
-
-    await batch.write()
-  }
-
-  private async deletePages(pageIds: string[]): Promise<void> {
-    if (this.operation.type === "preview") {
-      return
-    }
-
-    const batch = this.stateManager.batch()
-
-    await Promise.all([
-      this.stateManager.getPageRepository(this.project.id).deleteMany(pageIds, batch),
-      this.stateManager.getPageContentRepository(this.project.id).deleteMany(pageIds, batch),
-    ])
-
-    await batch.write()
-  }
-
-  /**
-   * Processes triggers from instance state update.
-   *
-   * Converts unit triggers to instance triggers and handles cleanup.
-   */
-  private processTriggers(
-    instanceId: string,
-    unitTriggers: UnitTrigger[],
-    currentTriggerIds: Record<string, string> | undefined,
-  ): Record<string, string> {
-    const triggers: Trigger[] = unitTriggers.map(unitTrigger => ({
-      id: currentTriggerIds?.[unitTrigger.name] ?? uuidv7(),
-      instanceId,
-      meta: unitTrigger.meta ?? {},
-      spec: unitTrigger.spec,
-    }))
-
-    // create mapping from local name to UUID
-    const newTriggerIds: Record<string, string> = {}
-    for (let i = 0; i < unitTriggers.length; i++) {
-      const unitTrigger = unitTriggers[i]
-      newTriggerIds[unitTrigger.name] = triggers[i].id
-    }
-
-    // find triggers that need to be deleted (old triggers not in new list)
-    const triggersToDelete = Object.values(currentTriggerIds ?? {}).filter(
-      id => !Object.values(newTriggerIds).includes(id),
-    )
-
-    if (triggers.length > 0) {
-      this.promiseTracker.track(this.putTriggers(triggers))
-    }
-
-    if (triggersToDelete.length > 0) {
-      this.promiseTracker.track(this.deleteTriggers(triggersToDelete))
-    }
-
-    return newTriggerIds
-  }
-
-  private async putTriggers(triggers: Trigger[]): Promise<void> {
-    if (this.operation.type === "preview") {
-      return
-    }
-
-    await this.stateManager.getTriggerRepository(this.project.id).putManyItems(triggers)
-  }
-
-  private async deleteTriggers(triggerIds: string[]): Promise<void> {
-    if (this.operation.type === "preview") {
-      return
-    }
-
-    await this.stateManager.getTriggerRepository(this.project.id).deleteMany(triggerIds)
-  }
-
-  /**
-   * Processes artifacts from instance state update.
-   */
-  private processArtifacts(
-    instanceId: string,
-    newOwnedArtifacts: Record<string, RunnerArtifact[]>,
-    oldOwnedArtifactIds: Record<string, string[]> | undefined,
-  ): Record<string, string[]> {
-    if (this.operation.type !== "preview") {
-      // persist the new owned artifacts if this is not a preview operation
-      this.promiseTracker.track(
-        this.artifactService.updateUsage(
-          this.project.id,
-          { type: "instance", instanceId },
-          unique(Object.values(oldOwnedArtifactIds ?? {}).flat()),
-          unique(
-            Object.values(newOwnedArtifacts).flatMap(artifacts =>
-              artifacts.map(artifact => artifact.id),
-            ),
-          ),
-        ),
-      )
-    }
-
-    return mapValues(newOwnedArtifacts, artifacts => artifacts.map(artifact => artifact.id))
-  }
-
-  /**
-   * Processes exported secrets from instance state update.
-   *
-   * Validates that all secret IDs are either present in the instance secrets or are in "exportedSecrets" of the direct dependencies
-   * from connected inputs.
-   *
-   * Returns a mapping of output key to array of secret IDs exported via this output.
-   */
-  private processExportedSecrets(
-    instanceId: string,
-    exportedSecrets: Record<string, Omit<UnitSecretModel, "value">[]>,
-  ): Record<string, string[]> {
-    const instance = this.workset.getInstance(instanceId)
-    const allowedSecretIds = this.collectSecretIdsForInstance(instance)
-
-    return mapValues(exportedSecrets, secrets => {
-      const exportedSecretIds = secrets.map(secret => secret.id)
-      for (const exportedSecretId of exportedSecretIds) {
-        if (!allowedSecretIds.has(exportedSecretId)) {
-          throw new Error(
-            `The secret "${exportedSecretId}" is not allowed to be exported since it is not present in the instance secrets or in the exported secrets of connected dependencies.`,
-          )
-        }
-      }
-
-      return exportedSecretIds
-    })
-  }
-
-  /**
-   * Updates the secrets for a specific instance.
-   */
-  private async updateInstanceSecrets(
-    instance: InstanceModel,
-    secrets: Record<string, unknown>,
-  ): Promise<void> {
-    if (this.operation.type === "preview") {
-      // do not update secrets in preview mode
-      return
-    }
-
-    await this.secretService.updateInstanceSecrets(
-      this.project,
-      instance.id,
-      secrets,
-      [], // no secrets to delete in this context
-      false, // do not invalidate state even if the secrets were updated
-    )
-  }
 }