@highstate/backend 0.9.15 → 0.9.18

package/src/orchestrator/operation.ts

@@ -1,114 +1,204 @@
 import type { LibraryBackend } from "../library"
-import type { LogEntry, StateBackend, StateManager } from "../state"
-import type { ProjectBackend, ProjectLock } from "../project"
-import type { SecretBackend } from "../secret"
-import type { RunnerBackend } from "../runner"
-import type { InstanceLogsEvents, OperationEvents } from "./manager"
-import type { EventEmitter } from "node:events"
+import type { StateManager } from "../state"
+import type { ProjectBackend } from "../project"
+import type {
+  RunnerArtifact,
+  RunnerBackend,
+  TypedUnitStateUpdate,
+  UnitStateUpdate,
+} from "../runner"
 import type { Logger } from "pino"
-import { isUnitModel, parseInstanceId, type InstanceModel } from "@highstate/contract"
-import { mapValues } from "remeda"
+import type {
+  InstanceLockService,
+  InstanceStateService,
+  OperationService,
+  SecretService,
+  WorkerService,
+} from "../business"
+import type { PubSubManager } from "../pubsub"
+import { randomBytes } from "node:crypto"
+import { v7 as uuidv7 } from "uuid"
+import {
+  isUnitModel,
+  parseInstanceId,
+  type ComponentModel,
+  type InstanceModel,
+  type UnitSecretModel,
+} from "@highstate/contract"
+import { mapValues, unique } from "remeda"
+import { ArtifactService } from "../artifact"
 import {
   type InstanceState,
-  type InstanceStateUpdate,
-  type ProjectOperation,
-  type InstanceTriggerInvocation,
-  createInstanceState,
-  createAsyncBatcher,
+  type UnitPage,
+  type Operation,
+  type Page,
+  type Trigger,
+  type UnitTrigger,
+  type UnitTerminal,
+  type Terminal,
+  type TerminalSpec,
+  type TriggerInvocation,
+  PromiseTracker,
+  type InstanceStatePatch,
+  type PageBlock,
+  type Project,
 } from "../shared"
 import {
+  AbortError,
   errorToString,
-  isAbortError,
   isAbortErrorLike,
-  tryWrapAbortErrorLike,
+  PerformanceLogger,
+  stringToValue,
   valueToString,
+  waitForAbort,
 } from "../common"
 import { OperationWorkset, type OperationPhase } from "./operation-workset"
 
 export class RuntimeOperation {
   private readonly abortController = new AbortController()
+  private readonly forceAbortController = new AbortController()
+
+  private readonly instanceAbortControllers = new Map<string, AbortController>()
+  private readonly instanceForceAbortControllers = new Map<string, AbortController>()
+
   private readonly instancePromiseMap = new Map<string, Promise<void>>()
-  private workset!: OperationWorkset
+  private readonly promiseTracker = new PromiseTracker()
 
-  private currentPhase!: OperationPhase
+  private workset!: OperationWorkset
 
   constructor(
-    private readonly operation: ProjectOperation,
+    private readonly project: Project,
+    private readonly operation: Operation,
     private readonly runnerBackend: RunnerBackend,
-    private readonly stateBackend: StateBackend,
     private readonly libraryBackend: LibraryBackend,
     private readonly projectBackend: ProjectBackend,
-    private readonly secretBackend: SecretBackend,
-    private readonly projectLock: ProjectLock,
+    private readonly artifactService: ArtifactService,
     private readonly stateManager: StateManager,
-    private readonly operationEE: EventEmitter<OperationEvents>,
-    private readonly instanceLogsEE: EventEmitter<InstanceLogsEvents>,
+    private readonly instanceLockService: InstanceLockService,
+    private readonly operationService: OperationService,
+    private readonly secretService: SecretService,
+    private readonly instanceStateService: InstanceStateService,
+    private readonly pubsubManager: PubSubManager,
+    private readonly workerService: WorkerService,
     private readonly logger: Logger,
   ) {}
 
   async operateSafe(): Promise<void> {
     try {
       await this.operate()
+
+      // ensure that all promises are resolved
+      await this.promiseTracker.waitForAll()
     } catch (error) {
-      if (isAbortError(error)) {
+      if (isAbortErrorLike(error)) {
         this.logger.info("the operation was cancelled")
         this.operation.status = "cancelled"
+        this.operation.error = null
 
-        await this.updateOperation()
+        await this.operationService.updateOperation(this.project.id, this.operation)
         return
       }
 
-      this.logger.error({ msg: "an error occurred while running the operation", error })
+      this.logger.error({ error }, "an error occurred while running the operation")
 
       this.operation.status = "failed"
       this.operation.error = errorToString(error)
 
-      await this.updateOperation()
+      await this.operationService.updateOperation(this.project.id, this.operation)
     } finally {
-      await Promise.all([
-        this.persistStates.flush(),
-        this.persistLogs.flush(),
-        this.persistSecrets.flush(),
-      ])
-
-      this.logger.debug("operation finished, all entries persisted")
+      try {
+        // ensure that all promises are resolved even if the operation failed
+        await this.promiseTracker.waitForAll()
+      } catch (error) {
+        this.logger.error(
+          { error },
+          "one of the tracked promises failed after the operation failed",
+        )
+      }
     }
   }
 
   private async operate(): Promise<void> {
     this.logger.info("starting operation")
-    let lockInstanceIds: string[]
-
-    // keep recalculating the workset until we can acquire the locks and actually start the operation
-    while (true) {
-      this.workset = await OperationWorkset.load(
-        this.operation,
-        this.projectBackend,
-        this.libraryBackend,
-        this.stateBackend,
-        this.stateManager,
-        this.logger,
-        this.abortController.signal,
-      )
 
-      lockInstanceIds = this.workset.getLockInstanceIds()
+    // create the workset for the operation
+    this.workset = await OperationWorkset.load(
+      this.project,
+      this.operation,
+      this.projectBackend,
+      this.libraryBackend,
+      this.stateManager,
+      this.instanceLockService,
+      this.instanceStateService,
+      this.logger,
+      this.abortController.signal,
+    )
 
-      if (this.projectLock.canImmediatelyAcquireLocks(lockInstanceIds)) {
-        break
-      }
+    // start the process to lock more instances when they become available
+    const lockAbortSignal = new AbortController()
+    this.promiseTracker.track(this.tryLockMore(lockAbortSignal.signal))
+
+    // try to lock as many instances as possible
+    await this.workset.tryLock()
 
-      this.logger.info("waiting for locks to be available")
+    if (this.workset.instanceIdsToLockIds.size === 0) {
+      this.logger.debug("successfully locked all instances for the operation before starting")
+      lockAbortSignal.abort()
+    }
 
-      await this.projectLock.lockInstances(lockInstanceIds, () => Promise.resolve())
+    // setup abort controllers for the instances
+    for (const instanceId of this.workset.getAllAffectedInstanceIds()) {
+      this.setupInstanceAbortControllers(instanceId)
     }
 
     try {
-      // actually acquire the locks and start the operation
-      await this.projectLock.lockInstances(lockInstanceIds, () => this.processOperation())
+      // run the operation
+      await this.processOperation()
     } finally {
+      lockAbortSignal.abort()
+
       if (this.operation.type === "preview") {
         // stream initial states for preview operations
-        this.workset.emitAffectedInitialStates()
+        await this.workset.restoreAffectedInitialStates()
+      }
+    }
+  }
+
+  private setupInstanceAbortControllers(instanceId: string): void {
+    const abortController = new AbortController()
+    this.abortController.signal.addEventListener("abort", () => abortController.abort())
+    this.instanceAbortControllers.set(instanceId, abortController)
+
+    const forceAbortController = new AbortController()
+    this.forceAbortController.signal.addEventListener("abort", () => forceAbortController.abort())
+    this.instanceForceAbortControllers.set(instanceId, forceAbortController)
+  }
+
+  private async tryLockMore(signal: AbortSignal): Promise<void> {
+    const eventStream = this.pubsubManager.subscribe(["instance-lock", this.project.id], signal)
+
+    for await (const event of eventStream) {
+      if (event.type !== "unlocked") {
+        continue
+      }
+
+      const instanceIdsToLock = event.instanceIds.filter(
+        //
+        instanceId => this.workset.instanceIdsToLockIds.has(instanceId),
+      )
+
+      if (instanceIdsToLock.length === 0) {
+        try {
+          await this.workset.tryLock(instanceIdsToLock)
+        } catch (error) {
+          this.logger.error({ error }, "failed to lock more instances during operation")
+        }
+      }
+
+      if (this.workset.instanceIdsToLockIds.size === 0) {
+        // no more instances to lock, stop listening for events
+        this.logger.debug("successfully locked all instances for the operation")
+        break
       }
     }
   }
@@ -118,22 +208,22 @@ export class RuntimeOperation {
     this.operation.instanceIdsToDestroy = this.workset.operation.instanceIdsToDestroy
 
     this.logger.info(
-      {
-        instanceIdsToUpdate: this.operation.instanceIdsToUpdate,
-        instanceIdsToDestroy: this.operation.instanceIdsToDestroy,
-      },
-      "operation started",
+      "operation started with %s instances to update and %s to destroy",
+      this.operation.instanceIdsToUpdate.length,
+      this.operation.instanceIdsToDestroy.length,
     )
 
     const phases = this.getOperationPhases()
+    const errors: string[] = []
+    let hasAbortError = false
 
     for (const phase of phases) {
-      this.currentPhase = phase
+      this.workset.currentPhase = phase
 
       const promises: Promise<void>[] = []
-      for (const instanceId of this.workset.getAffectedInstanceIds(phase)) {
-        const parentId = this.workset.getParentId(instanceId, phase)
-        if (parentId && this.workset.isAffected(parentId, phase)) {
+      for (const instanceId of this.workset.getAffectedInstanceIds()) {
+        const parentId = this.workset.getParentId(instanceId)
+        if (parentId && this.workset.isAffected(parentId)) {
           // do not call the operation for child instances of affected composites,
           // they will be called by their parent instance
           continue
@@ -144,23 +234,94 @@ export class RuntimeOperation {
 
       this.logger.info(`all operations for phase "%s" started`, phase)
       this.operation.status = "running"
-      await this.updateOperation()
+      await this.operationService.updateOperation(this.project.id, this.operation)
 
-      await Promise.all(promises)
+      const phaseResults = await Promise.allSettled(promises)
+      for (const result of phaseResults) {
+        if (result.status !== "rejected") {
+          continue
+        }
+
+        if (isAbortErrorLike(result.reason)) {
+          hasAbortError = true
+        } else {
+          errors.push(errorToString(result.reason))
+        }
+      }
 
       this.logger.info(`all operations for phase "%s" completed`, phase)
     }
 
-    this.operation.status = "completed"
-    this.operation.error = null
-    await this.updateOperation()
+    if (errors.length > 0) {
+      this.operation.status = "failed"
+      // TODO: map errors to instances inside operation
+      this.operation.error = errors.join("\n\n")
+    } else if (hasAbortError) {
+      this.operation.status = "cancelled"
+      this.operation.error = null
+    } else {
+      this.operation.status = "completed"
+      this.operation.error = null
+    }
+
+    await this.operationService.updateOperation(this.project.id, this.operation)
 
     this.logger.info("operation completed")
   }
 
-  cancel() {
-    this.logger.info("cancelling operation")
-    this.abortController.abort()
+  cancelInstance(instanceId: string): void {
+    const abortController = this.instanceAbortControllers.get(instanceId)
+    if (!abortController) {
+      throw new Error(`No abort controller found for instance "${instanceId}"`)
+    }
+
+    if (!abortController.signal.aborted) {
+      this.logger.info(`cancelling operation for instance "${instanceId}"`)
+      abortController.abort()
+
+      // just the UX feature to indicate that we are trying to cancel the operation
+      this.promiseTracker.track(
+        this.workset.patchState({
+          id: instanceId,
+          operationStatus: {
+            status: "cancelling",
+          },
+        }),
+      )
+
+      return
+    }
+
+    // then try to force cancel the operation
+    const forceAbortController = this.instanceForceAbortControllers.get(instanceId)
+    if (!forceAbortController) {
+      throw new Error(`No force abort controller found for instance "${instanceId}"`)
+    }
+
+    if (!forceAbortController.signal.aborted) {
+      this.logger.info(`force cancelling operation for instance "${instanceId}"`)
+      forceAbortController.abort()
+      return
+    }
+
+    this.logger.warn(`operation for instance "${instanceId}" is already force cancelled`)
+  }
+
+  cancel(): void {
+    if (!this.abortController.signal.aborted) {
+      this.logger.info("cancelling the operation")
+      this.abortController.abort()
+      return
+    }
+
+    // then try to force cancel the operation
+    if (!this.forceAbortController.signal.aborted) {
+      this.logger.info("force cancelling the operation")
+      this.forceAbortController.abort()
+      return
+    }
+
+    this.logger.warn("the operation is already cancelled or force cancelled")
   }
 
   private getInstancePromiseForOperation(instanceId: string): Promise<void> {
@@ -195,7 +356,7 @@ export class RuntimeOperation {
   }
 
   private async getUnitPromise(instanceId: string): Promise<void> {
-    switch (this.currentPhase) {
+    switch (this.workset.currentPhase) {
       case "update": {
         return this.updateUnit(instanceId)
       }
@@ -208,22 +369,13 @@ export class RuntimeOperation {
     }
   }
 
-  private async getCompositePromise(instanceId: string): Promise<void> {
-    const logger = this.logger.child({ instanceId })
-
-    return this.getInstancePromise(instanceId, async () => {
-      const state = this.workset.getState(instanceId) ?? createInstanceState(instanceId)
-      const instance = this.workset.getInstanceOrUndefined(instanceId)
+  private getCompositePromise(instanceId: string): Promise<void> {
+    return this.getInstancePromise(instanceId, async logger => {
+      const instance = this.workset.getInstance(instanceId)
 
-      this.updateInstanceState({
-        ...state,
-        parentId: instance?.parentId,
-        latestOperationId: this.operation.id,
-        status: "pending",
-        error: null,
-      })
+      await this.setInitialOperationStatus(instance)
 
-      const children = this.workset.getAffectedCompositeChildren(instanceId, this.currentPhase)
+      const children = this.workset.getAffectedCompositeChildren(instanceId)
       const childPromises: Promise<void>[] = []
 
       if (children.length) {
@@ -233,15 +385,12 @@ export class RuntimeOperation {
       }
 
       for (const child of children) {
-        if (
-          !this.operation.options.forceUpdateChildren &&
-          !this.workset.isAffected(child.id, this.currentPhase)
-        ) {
+        if (!this.operation.options.forceUpdateChildren && !this.workset.isAffected(child.id)) {
           // skip children that are not affected by the operation
           continue
         }
 
-        logger.debug(`waiting for child: "%s"`, child.id)
+        logger.debug(`waiting for child "%s"`, child.id)
         childPromises.push(this.getInstancePromiseForOperation(child.id))
       }
 
@@ -249,105 +398,130 @@ export class RuntimeOperation {
         await Promise.all(childPromises)
 
         if (children.length > 0) {
-          logger.info("all children completed")
+          logger.debug("all children completed")
+        }
+
+        if (this.operation.type === "destroy") {
+          await this.clearOperationState(instanceId)
+          return
         }
 
-        this.updateInstanceState({
+        const { inputHash, dependencyOutputHash } =
+          await this.workset.getUpToDateInputHashOutput(instance)
+
+        await this.workset.patchState({
           id: instanceId,
-          status: this.operation.type === "destroy" ? "not_created" : "created",
-          inputHash: instance ? await this.workset.getUpToDateInputHash(instance) : undefined,
+          operationStatus: {
+            status: "created",
+            inputHash,
+            dependencyOutputHash,
+          },
         })
       } catch (error) {
         if (isAbortErrorLike(error)) {
-          this.workset.restoreInitialStatus(instanceId)
+          await this.workset.restoreInitialState(instanceId)
           return
         }
 
-        this.updateInstanceState({
+        await this.workset.patchState({
           id: instanceId,
-          status: "error",
-          error: errorToString(error),
+          operationStatus: {
+            status: "error",
+            message: errorToString(error),
+          },
         })
       }
     })
   }
 
   private updateUnit(instanceId: string): Promise<void> {
-    return this.getInstancePromise(instanceId, async logger => {
+    return this.getInstancePromise(instanceId, async (logger, signal, forceSignal) => {
       const instance = this.workset.getInstance(instanceId)
+      const component = this.workset.library.components[instance.type]
+      const perfLogger = new PerformanceLogger(logger)
+      perfLogger.log("starting update promise for instance")
 
-      this.updateInstanceState({
-        id: instance.id,
-        parentId: instance.parentId,
-        latestOperationId: this.operation.id,
-        status: "pending",
-        error: null,
-        currentResourceCount: 0,
-      })
+      await this.setInitialOperationStatus(instance)
+      perfLogger.log("initial operation status set")
 
-      let dependencyIds: string[] = []
-      try {
-        dependencyIds = await this.updateUnitDependencies(instance, logger)
-      } catch (error) {
-        // restore the initial status of the instance if one of the dependencies failed
-        this.workset.restoreInitialStatus(instance.id)
-        throw error
+      await Promise.race([
+        this.updateUnitDependencies(instance.id, logger),
+
+        // to immediately abort the operation if requested
+        waitForAbort(signal),
+      ])
+
+      if (!signal.aborted) {
+        perfLogger.log("dependencies updated")
       }
 
+      signal.throwIfAborted()
+
       logger.info("updating unit")
 
-      const secrets = await this.secretBackend.get(this.operation.projectId, instance.id)
-      this.abortController.signal.throwIfAborted()
+      await this.workset.patchState({
+        id: instance.id,
+        operationStatus: {
+          status: "updating",
+          currentResourceCount: 0,
+          totalResourceCount: 0,
+        },
+      })
+
+      perfLogger.log("set 'updating' operation status")
+      signal.throwIfAborted()
+
+      const secrets = await this.secretService.getInstanceSecretValues(this.project, instance.id)
 
-      logger.debug("secrets loaded", { count: Object.keys(secrets).length })
+      perfLogger.log("secrets loaded")
+      signal.throwIfAborted()
+
+      const config = this.prepareUnitConfig(instance, component)
+      perfLogger.log("unit config prepared")
+
+      // extract artifact hashes from dependencies based on instance inputs
+      const artifactHashes = this.collectArtifactIdsForInstance(instance)
+      const artifacts = await this.stateManager
+        .getArtifactHashIndexRepository(this.project.id)
+        .getManyItems(artifactHashes)
+
+      logger.debug({ count: artifactHashes.length }, "artifact hashes collected from dependencies")
+      perfLogger.log("artifact hashes collected")
 
       await this.runnerBackend[this.operation.type === "preview" ? "preview" : "update"]({
-        projectId: this.operation.projectId,
+        projectId: this.project.id,
+        libraryId: this.project.libraryId,
         instanceType: instance.type,
         instanceName: instance.name,
-        config: this.prepareUnitConfig(instance),
+        config,
         refresh: this.operation.options.refresh,
         secrets: mapValues(secrets, value => valueToString(value)),
-        signal: this.abortController.signal,
-      })
-
-      logger.debug("unit update requested")
-
-      const stream = this.runnerBackend.watch({
-        projectId: this.operation.projectId,
-        instanceType: instance.type,
-        instanceName: instance.name,
-        finalStatuses: ["created", "error"],
+        artifacts,
+        signal,
+        forceSignal,
       })
 
-      await this.watchStateStream(stream)
+      perfLogger.log("unit update requested")
 
-      const inputHash = await this.workset.getUpToDateInputHash(instance)
-
-      this.updateInstanceState({
-        id: instance.id,
-        inputHash,
-        dependencyIds,
-      })
-
-      logger.debug("input hash after update", { inputHash })
+      await this.watchStateStream(instance.type, instance.name, logger)
+      perfLogger.log("unit update completed")
       logger.info("unit updated")
     })
   }
 
-  private async updateUnitDependencies(instance: InstanceModel, logger: Logger): Promise<string[]> {
+  private async updateUnitDependencies(instanceId: string, logger: Logger): Promise<void> {
     try {
-      const dependencies = this.getInstanceDependencies(instance)
+      const dependencies = this.getInstanceDependencyIds(instanceId)
       const dependencyPromises: Promise<void>[] = []
 
-      for (const dependency of dependencies) {
-        if (!this.operation.instanceIdsToUpdate.includes(dependency.id)) {
+      for (const dependencyId of dependencies) {
+        if (!this.operation.instanceIdsToUpdate.includes(dependencyId)) {
           // skip dependencies that are not affected by the operation
           continue
         }
 
-        logger.debug(`waiting for dependency: ${dependency.id}`)
-        dependencyPromises.push(this.getInstancePromiseForOperation(dependency.id))
+        logger.debug(`waiting for dependency "${dependencyId}"`)
+        dependencyPromises.push(this.getInstancePromiseForOperation(dependencyId))
       }
 
       await Promise.all(dependencyPromises)
@@ -355,71 +529,90 @@ export class RuntimeOperation {
       if (dependencies.length > 0) {
         logger.info("all dependencies completed")
       }
-
-      return dependencies.map(dependency => dependency.id)
-    } catch (error) {
-      // restore the initial status of the instance if one of the dependencies failed
-      this.workset.restoreInitialStatus(instance.id)
-      throw error
+    } catch {
+      // abort the instance if any dependency fails
+      throw new AbortError()
     }
   }
 
-  private async processBeforeDestroyTriggers(state: InstanceState, logger: Logger): Promise<void> {
+  private async processBeforeDestroyTriggers(
+    state: InstanceState,
+    logger: Logger,
+    signal: AbortSignal,
+    forceSignal: AbortSignal,
+  ): Promise<void> {
     if (!this.operation.options.invokeDestroyTriggers) {
       logger.debug("destroy triggers are disabled for the operation")
       return
     }
 
     const instance = this.workset.getInstance(state.id)
+    const component = this.workset.library.components[instance.type]
 
-    const triggers = state.triggers.filter(trigger => trigger.spec.type === "before-destroy")
-    if (triggers.length === 0) {
+    // fetch triggers from state backend by their IDs
+    const triggerIds = Object.values(state.extra?.triggerIds ?? {})
+
+    const allTriggers = await this.stateManager
+      .getTriggerRepository(this.project.id)
+      .getManyItems(triggerIds)
+
+    const beforeDestroyTriggers = allTriggers.filter(
+      trigger => trigger.spec.type === "before-destroy",
+    )
+
+    if (beforeDestroyTriggers.length === 0) {
       return
     }
 
-    const invokedTriggers = triggers.map(trigger => ({ name: trigger.name }))
+    // get trigger names from state mapping by finding the key for each trigger ID
+    const invokedTriggers = beforeDestroyTriggers.map(trigger => {
+      const triggerName = Object.keys(state?.extra?.triggerIds ?? {}).find(
+        name => state?.extra?.triggerIds?.[name] === trigger.id,
+      )
+
+      return { name: triggerName || "unknown" }
+    })
+
+    await this.workset.patchState({
+      id: instance.id,
+      operationStatus: {
+        status: "processing-triggers",
+      },
+    })
 
     logger.info("updating unit to process before-destroy triggers...")
 
-    const secrets = await this.secretBackend.get(
-      this.operation.projectId,
-      instance.id,
-      this.abortController.signal,
-    )
+    const secrets = await this.secretService.getInstanceSecretValues(this.project, instance.id)
 
     await this.runnerBackend.update({
-      projectId: this.operation.projectId,
+      projectId: this.project.id,
+      libraryId: this.project.libraryId,
       instanceType: instance.type,
       instanceName: instance.name,
-      config: this.prepareUnitConfig(instance, invokedTriggers),
+      config: this.prepareUnitConfig(instance, component, invokedTriggers),
       refresh: this.operation.options.refresh,
       secrets: mapValues(secrets, value => valueToString(value)),
-      signal: this.abortController.signal,
+      signal,
+      forceSignal,
     })
 
     logger.debug("unit update requested")
 
-    const stream = this.runnerBackend.watch({
-      projectId: this.operation.projectId,
-      instanceType: instance.type,
-      instanceName: instance.name,
-      finalStatuses: ["created", "error"],
-    })
-
-    await this.watchStateStream(stream)
-
+    await this.watchStateStream(instance.type, instance.name, logger)
     logger.debug("before-destroy triggers processed")
   }
 
   private async destroyUnit(instanceId: string): Promise<void> {
-    return this.getInstancePromise(instanceId, async logger => {
-      this.updateInstanceState({
+    return this.getInstancePromise(instanceId, async (logger, signal, forceSignal) => {
+      await this.workset.patchState({
         id: instanceId,
-        latestOperationId: this.operation.id,
-        status: "pending",
-        error: null,
+        operationStatus: {
+          status: "pending",
+        },
       })
 
+      signal.throwIfAborted()
+
       const state = this.workset.getState(instanceId)
       if (!state) {
         logger.warn("state not found for unit, but destroy was requested")
@@ -442,48 +635,65 @@ export class RuntimeOperation {
       }
 
       await Promise.all(dependentPromises)
-      this.abortController.signal.throwIfAborted()
+      signal.throwIfAborted()
 
-      await this.processBeforeDestroyTriggers(state, logger)
+      await this.processBeforeDestroyTriggers(state, logger, signal, forceSignal)
+      signal.throwIfAborted()
 
       logger.info("destroying unit...")
 
+      await this.workset.patchState({
+        id: instanceId,
+        operationStatus: {
+          status: "destroying",
+        },
+      })
+
       const [type, name] = parseInstanceId(instanceId)
 
       await this.runnerBackend.destroy({
-        projectId: this.operation.projectId,
+        projectId: this.project.id,
+        libraryId: this.project.libraryId,
         instanceType: type,
         instanceName: name,
         refresh: this.operation.options.refresh,
-        signal: this.abortController.signal,
+        signal,
+        forceSignal,
         deleteUnreachable: this.operation.options.deleteUnreachableResources,
         forceDeleteState: this.operation.options.forceDeleteState,
       })
 
-      this.logger.debug("destroy request sent")
+      logger.debug("destroy request sent")
 
-      const stream = this.runnerBackend.watch({
-        projectId: this.operation.projectId,
-        instanceType: type,
-        instanceName: name,
-        finalStatuses: ["not_created", "error"],
-      })
+      await this.watchStateStream(type, name, logger)
+
+      // clean up artifacts after successful destruction
+      try {
+        const artifactIds = unique(Object.values(state.extra?.exportedArtifactIds ?? {}).flat())
+
+        await this.artifactService.removeUsages(
+          //
+          this.project.id,
+          artifactIds,
+          [{ type: "instance", instanceId: state.id }],
+        )
+      } catch (error) {
+        logger.warn({ error }, "failed to cleanup artifacts for destroyed instance")
+      }
 
-      await this.watchStateStream(stream)
-      this.logger.info("unit destroyed")
+      logger.info("unit destroyed")
     })
   }
 
   private async refreshUnit(instanceId: string) {
-    const logger = this.logger.child({ instanceId })
-
-    return this.getInstancePromise(instanceId, async () => {
-      this.updateInstanceState({
+    return this.getInstancePromise(instanceId, async (logger, signal, forceSignal) => {
+      await this.workset.patchState({
         id: instanceId,
-        latestOperationId: this.operation.id,
-        status: "refreshing",
-        currentResourceCount: 0,
-        totalResourceCount: 0,
+        operationStatus: {
+          status: "refreshing",
+          currentResourceCount: 0,
+          totalResourceCount: 0,
+        },
       })
 
       logger.info("refreshing unit...")
@@ -491,62 +701,84 @@ export class RuntimeOperation {
       const [type, name] = parseInstanceId(instanceId)
 
       await this.runnerBackend.refresh({
-        projectId: this.operation.projectId,
+        projectId: this.project.id,
+        libraryId: this.project.libraryId,
         instanceType: type,
         instanceName: name,
-        signal: this.abortController.signal,
+        signal,
+        forceSignal,
       })
 
       logger.debug("unit refresh requested")
 
-      const stream = this.runnerBackend.watch({
-        projectId: this.operation.projectId,
-        instanceType: type,
-        instanceName: name,
-        finalStatuses: ["created", "error"],
-      })
-
-      await this.watchStateStream(stream)
+      await this.watchStateStream(type, name, logger)
       logger.info("unit refreshed")
     })
   }
 
-  private async watchStateStream(stream: AsyncIterable<InstanceStateUpdate>) {
-    let statePatch: InstanceStateUpdate | undefined
-    for await (statePatch of stream) {
-      if (statePatch.status === "not_created" && this.operation.type === "recreate") {
-        // do not stream "not_created" status for recreate operation to improve UX
-        continue
+  private async watchStateStream(
+    instanceType: string,
+    instanceName: string,
+    logger: Logger,
+  ): Promise<void> {
+    const stream = this.runnerBackend.watch({
+      projectId: this.project.id,
+      libraryId: this.project.libraryId,
+      instanceType,
+      instanceName,
+    })
+
+    let update: UnitStateUpdate | undefined
+
+    for await (update of stream) {
+      try {
+        await this.handleUnitStateUpdate(update)
+      } catch (error) {
+        logger.error({ error }, "failed to handle unit state update")
       }
 
-      this.updateInstanceState(statePatch)
-    }
+      if (update.type === "error") {
+        if (isAbortErrorLike(update.message)) {
+          // abort the unit if the returned error contains some abort-like pattern
+          // generally, this might not be safe, but for now, we assume that
+          throw new AbortError()
+        }
 
-    if (!statePatch) {
-      throw new Error("The stream ended without emitting any state.")
-    }
+        // rethrow the error to stop the execution of dependent units
+        throw new Error(`An error occurred while processing the unit "${update.unitId}"`, {
+          cause: update.message,
+        })
+      }
 
-    if (statePatch.status === "error") {
-      throw tryWrapAbortErrorLike(
-        new Error(`The operation on unit "${statePatch.id}" failed: ${statePatch.error}`),
-      )
+      if (update.type === "completion") {
+        return
+      }
     }
+
+    throw new Error(
+      "The unit state stream was closed without a completion update or it was not handled properly.",
+    )
   }
 
   private prepareUnitConfig(
     instance: InstanceModel,
-    invokedTriggers: InstanceTriggerInvocation[] = [],
+    component: ComponentModel,
+    invokedTriggers: TriggerInvocation[] = [],
   ): Record<string, string> {
     const config: Record<string, string> = {}
 
-    for (const [key, value] of Object.entries(instance.args ?? {})) {
-      config[key] = valueToString(value)
+    for (const key of Object.keys(component.args)) {
+      // explicitly set empty values to remove old values in the config
+      const value = instance.args?.[key]
+      config[key] = value ? valueToString(value) : ""
     }
 
     const instanceInputs = this.workset.resolvedInstanceInputs.get(instance.id) ?? {}
 
-    for (const [key, value] of Object.entries(instanceInputs)) {
-      config[`input.${key}`] = JSON.stringify(value.map(input => input.input))
+    for (const key of Object.keys(component.inputs)) {
+      const inputs = instanceInputs[key] ?? []
+
+      config[`input.${key}`] = JSON.stringify(inputs.map(input => input.input))
     }
 
     config["$invokedTriggers"] = JSON.stringify(invokedTriggers)
@@ -554,49 +786,181 @@ export class RuntimeOperation {
     return config
   }
 
-  private async updateOperation(): Promise<void> {
-    this.operationEE.emit(this.operation.projectId, this.operation)
-    await this.stateBackend.putOperation(this.operation)
+  private async handleUnitStateUpdate(update: UnitStateUpdate): Promise<void> {
+    switch (update.type) {
+      case "message":
+        this.handleUnitMessage(update)
+        return
+      case "progress":
+        await this.handleUnitProgress(update)
+        return
+      case "error":
+        await this.handleUnitError(update)
+        return
+      case "completion":
+        await this.handleUnitCompletion(update)
+        return
+    }
+  }
+
+  private handleUnitMessage(update: TypedUnitStateUpdate<"message">): void {
+    // collect all instances in the hierarchy that should receive this log
+    const instanceIdsInHierarchy: string[] = []
+    let instance: InstanceModel | null = this.workset.getInstance(update.unitId)
+
+    for (;;) {
+      instanceIdsInHierarchy.push(instance.id)
+
+      if (!instance.parentId) {
+        break
+      }
+
+      instance = this.workset.getInstance(instance.parentId)
+    }
+
+    // persist the log for all instances in the hierarchy
+    // TODO: batch
+    this.promiseTracker.track(
+      this.operationService.appendLogs(this.project.id, this.operation.id, instanceIdsInHierarchy, [
+        update.message,
+      ]),
+    )
+    return
   }
 
-  private updateInstanceState(patch: InstanceStateUpdate): void {
-    if (!patch.id) {
-      throw new Error("The ID of the instance state is required.")
+  private async handleUnitProgress(update: TypedUnitStateUpdate<"progress">): Promise<void> {
+    const patch: InstanceStatePatch = { id: update.unitId }
+    patch.operationStatus = {}
+
+    if (update.currentResourceCount !== undefined) {
+      patch.operationStatus.currentResourceCount = update.currentResourceCount
     }
 
-    if (patch.logLine) {
-      // recursively persist logs for instance and all parent instances
+    if (update.totalResourceCount !== undefined) {
+      patch.operationStatus.totalResourceCount = update.totalResourceCount
+    }
 
-      let instance: InstanceModel | null = this.workset.getInstance(patch.id)
+    await this.workset.patchState(patch)
+  }
 
-      for (;;) {
-        this.persistLogs.call([instance.id, patch.logLine])
-        this.instanceLogsEE.emit(`${this.operation.id}/${instance.id}`, patch.logLine)
+  private async handleUnitError(update: TypedUnitStateUpdate<"error">): Promise<void> {
+    if (isAbortErrorLike(update.message)) {
+      return
+    }
 
-        if (!instance.parentId) {
-          break
-        }
+    const patch: InstanceStatePatch = {
+      id: update.unitId,
+      operationStatus: {
+        status: "error",
+        message: update.message,
+      },
+    }
 
-        instance = this.workset.getInstance(instance.parentId)
-      }
+    await this.workset.patchState(patch)
+  }
+
+  private async handleUnitCompletion(update: TypedUnitStateUpdate<"completion">): Promise<void> {
+    if (this.workset.currentPhase === "destroy") {
+      // if the destroy operation was completed, clear the state from all operation-related fields
+      await this.clearOperationState(update.unitId)
       return
     }
 
-    const state = this.workset.updateState(patch, this.currentPhase)
+    const state = this.workset.getState(update.unitId)
+    if (!state) {
+      throw new Error(
+        `Cannot handle completion for unit "${update.unitId}" because its state is not found.`,
+      )
+    }
 
-    // do not persist anyting for preview operations
-    if (this.operation.type !== "preview") {
-      this.persistStates.call(state)
+    const instance = this.workset.getInstance(update.unitId)
 
-      if (patch.secrets) {
-        this.persistSecrets.call([patch.id, patch.secrets])
-      }
+    // patch the state with output hash first to recalculate the input hash
+    if (update.outputHash) {
+      await this.workset.patchState({
+        id: update.unitId,
+        operationStatus: { outputHash: update.outputHash },
+      })
     }
+
+    const { inputHash, dependencyOutputHash } =
+      await this.workset.getUpToDateInputHashOutput(instance)
+
+    const patch: InstanceStatePatch = {
+      id: update.unitId,
+      operationStatus: {
+        status: "created",
+        message:
+          update.message ?? "The operation on this instance has been completed successfully.",
+        outputHash: update.outputHash ?? undefined,
+        inputHash,
+        dependencyOutputHash,
+      },
+      extra: {
+        statusFields: update.statusFields,
+
+        terminalIds: update.terminals
+          ? this.processTerminals(update.unitId, update.terminals, state.extra?.terminalIds)
+          : null,
+
+        pageIds: update.pages
+          ? this.processPages(update.unitId, update.pages, state.extra?.pageIds)
+          : null,
+
+        triggerIds: update.triggers
+          ? this.processTriggers(update.unitId, update.triggers, state.extra?.triggerIds)
+          : null,
+
+        workerRegistrationIds: update.workers
+          ? await this.workerService.updateUnitRegistrations(
+              this.project.id,
+              update.unitId,
+              state.extra?.workerRegistrationIds ?? {},
+              update.workers,
+            )
+          : null,
+
+        exportedArtifactIds: update.exportedArtifacts
+          ? this.processArtifacts(
+              update.unitId,
+              update.exportedArtifacts,
+              state.extra?.exportedArtifactIds,
+            )
+          : null,
+
+        exportedSecretIds: update.exportedSecrets
+          ? this.processExportedSecrets(update.unitId, update.exportedSecrets)
+          : null,
+      },
+    }
+
+    if (update.secrets) {
+      const instance = this.workset.getInstance(update.unitId)
+
+      this.promiseTracker.track(
+        this.updateInstanceSecrets(
+          instance,
+          mapValues(update.secrets, value => stringToValue(value)),
+        ),
+      )
+    }
+
+    await this.workset.patchState(patch)
+  }
+
+  private async clearOperationState(instanceId: string): Promise<void> {
+    const patch: InstanceStatePatch = {
+      id: instanceId,
+      operationStatus: null,
+      extra: null,
+    }
+
+    await this.workset.patchState(patch)
   }
 
   private getInstancePromise(
     instanceId: string,
-    fn: (logger: Logger) => Promise<void>,
+    fn: (logger: Logger, signal: AbortSignal, forceSignal: AbortSignal) => Promise<void>,
   ): Promise<void> {
     let instancePromise = this.instancePromiseMap.get(instanceId)
     if (instancePromise) {
@@ -604,56 +968,430 @@ export class RuntimeOperation {
     }
 
     const logger = this.logger.child({ instanceId }, { msgPrefix: `[${instanceId}] ` })
+    const abortController = this.instanceAbortControllers.get(instanceId)
+    const forceAbortController = this.instanceForceAbortControllers.get(instanceId)
+
+    if (!abortController || !forceAbortController) {
+      throw new Error(`Abort controllers for instance "${instanceId}" are not initialized`)
+    }
+
+    instancePromise = this.workset
+      .ensureInstanceLocked(instanceId, abortController.signal)
+      .then(() => fn(logger, abortController.signal, forceAbortController.signal))
+      .catch(error => {
+        if (isAbortErrorLike(error)) {
+          // if the operation was aborted, restore the initial state of the instance
+          this.promiseTracker.track(this.workset.restoreInitialState(instanceId))
+
+          throw error
+        }
+
+        if (this.operation.status !== "failing") {
+          // report the failing status of the operation
+          this.operation.status = "failing"
+          this.promiseTracker.track(
+            this.operationService.updateOperation(this.project.id, this.operation),
+          )
+        }
+
+        // rethrow the error
+        throw error
+      })
+      .finally(() => {
+        this.instancePromiseMap.delete(instanceId)
+
+        // TODO: ideally we should defer unlocking until all direct dependents are completed,
+        // to ensure that they are received expected inputs from this instance
+        this.promiseTracker.track(
+          this.instanceLockService.unlockInstancesUnconditionally(this.project.id, [instanceId]),
+        )
+      })
 
-    instancePromise = fn(logger).finally(() => this.instancePromiseMap.delete(instanceId))
     this.instancePromiseMap.set(instanceId, instancePromise)
 
     return instancePromise
   }
 
-  private getInstanceDependencies(instance: InstanceModel): InstanceModel[] {
-    const dependencies: InstanceModel[] = []
+  private async setInitialOperationStatus(instance: InstanceModel): Promise<void> {
+    const state = this.workset.getState(instance.id)
+
+    await this.workset.patchState({
+      id: instance.id,
+      parentId: instance.parentId,
+      operationStatus: {
+        status: "pending",
+        message: "",
+        dependencyIds: this.getInstanceDependencyIds(instance.id),
+        latestOperationId: this.operation.id,
+        inputHashNonce: state?.operationStatus?.inputHashNonce ?? randomBytes(4).readInt32LE(),
+        args: instance.args,
+      },
+    })
+  }
+
+  private getInstanceDependencyIds(instanceId: string): string[] {
+    const dependencies = new Set<string>()
+    const instanceInputs = this.workset.resolvedInstanceInputs.get(instanceId) ?? {}
+
+    for (const inputs of Object.values(instanceInputs)) {
+      for (const input of inputs) {
+        dependencies.add(input.input.instanceId)
+      }
+    }
+
+    return Array.from(dependencies)
+  }
+
+  /**
+   * Collects artifact IDs from dependencies based on the direct connections
+   * from instance inputs to dependency outputs.
+   */
+  private collectArtifactIdsForInstance(instance: InstanceModel): string[] {
+    const artifactIds = new Set<string>()
     const instanceInputs = this.workset.resolvedInstanceInputs.get(instance.id) ?? {}
 
     for (const inputs of Object.values(instanceInputs)) {
       for (const input of inputs) {
-        const dependency = this.workset.getInstance(input.input.instanceId)
+        const dependencyState = this.workset.getState(input.input.instanceId)
+        if (!dependencyState?.extra?.exportedArtifactIds) {
+          continue
+        }
 
-        dependencies.push(dependency)
+        const outputKey = input.input.output
+        const outputArtifacts = dependencyState.extra.exportedArtifactIds[outputKey]
+        if (!outputArtifacts) {
+          continue
+        }
+
+        for (const hash of outputArtifacts) {
+          artifactIds.add(hash)
+        }
       }
     }
 
-    return dependencies
+    return Array.from(artifactIds)
   }
 
-  private persistStates = createAsyncBatcher(async (states: InstanceState[]) => {
-    this.logger.debug({ msg: "persisting states", count: states.length })
+  /**
+   * Collects secret IDs from dependencies based on the direct connections
+   * from instance inputs to dependency outputs.
+   */
+  private collectSecretIdsForInstance(instance: InstanceModel): Set<string> {
+    const secretIds = new Set<string>()
+    const instanceInputs = this.workset.resolvedInstanceInputs.get(instance.id) ?? {}
+
+    for (const inputs of Object.values(instanceInputs)) {
+      for (const input of inputs) {
+        const dependencyState = this.workset.getState(input.input.instanceId)
+        if (!dependencyState?.extra?.exportedSecretIds) {
+          continue
+        }
+
+        const outputKey = input.input.output
+        const outputSecrets = dependencyState.extra.exportedSecretIds[outputKey]
+        if (!outputSecrets) {
+          continue
+        }
+
+        for (const secretId of outputSecrets) {
+          secretIds.add(secretId)
+        }
+      }
+    }
 
-    await this.stateBackend.putAffectedInstanceStates(
-      this.operation.projectId,
-      this.operation.id,
-      states,
+    return secretIds
+  }
+
+  /**
+   * Updates the secrets for a specific instance.
+   * Processes terminals from instance state update.
+   *
+   * Converts unit terminals to instance terminals and handles cleanup.
+   */
+  private processTerminals(
+    instanceId: string,
+    unitTerminals: UnitTerminal[],
+    currentTerminalIds: Record<string, string> | undefined,
+  ): Record<string, string> {
+    const terminals: Terminal[] = unitTerminals.map(unitTerminal => ({
+      id: currentTerminalIds?.[unitTerminal.name] ?? uuidv7(),
+      instanceId,
+      meta: unitTerminal.meta ?? {},
+    }))
+
+    const terminalSpecEntries: [string, TerminalSpec][] = unitTerminals
+      //
+      .map((unitTerminal, index) => [terminals[index].id, unitTerminal.spec])
+
+    // create mapping from local name to UUID
+    const newTerminalIds: Record<string, string> = {}
+    for (let i = 0; i < unitTerminals.length; i++) {
+      const unitTerminal = unitTerminals[i]
+      const instanceTerminal = terminals[i]
+      newTerminalIds[unitTerminal.name] = instanceTerminal.id
+    }
+
+    // find terminals that need to be deleted (old terminals not in new list)
+    const terminalsToDelete = Object.values(currentTerminalIds ?? {}).filter(
+      id => !Object.values(newTerminalIds).includes(id),
+    )
+
+    if (terminals.length > 0) {
+      this.promiseTracker.track(this.putTerminals(terminals, terminalSpecEntries))
+    }
+
+    if (terminalsToDelete.length > 0) {
+      this.promiseTracker.track(this.deleteTerminals(terminalsToDelete))
+    }
+
+    return newTerminalIds
+  }
+
+  private async putTerminals(
+    terminals: Terminal[],
+    terminalSpecEntries: [string, TerminalSpec][],
+  ): Promise<void> {
+    if (this.operation.type === "preview") {
+      return
+    }
+
+    await using batch = this.stateManager.batch()
+
+    await Promise.all([
+      this.stateManager.getTerminalRepository(this.project.id).putManyItems(terminals, batch),
+
+      this.stateManager
+        .getTerminalSpecRepository(this.project.id)
+        .putMany(terminalSpecEntries, batch),
+    ])
+
+    await batch.write()
+  }
+
+  private async deleteTerminals(terminalIds: string[]): Promise<void> {
+    if (this.operation.type === "preview") {
+      return
+    }
+
+    await using batch = this.stateManager.batch()
+
+    await Promise.all([
+      this.stateManager.getTerminalRepository(this.project.id).deleteMany(terminalIds, batch),
+      this.stateManager.getTerminalSpecRepository(this.project.id).deleteMany(terminalIds, batch),
+    ])
+
+    await batch.write()
+  }
+
+  /**
+   * Processes pages from instance state update.
+   *
+   * Converts unit pages to instance pages and handles cleanup.
+   */
+  private processPages(
+    instanceId: string,
+    unitPages: UnitPage[],
+    currentPageIds: Record<string, string> | undefined,
+  ): Record<string, string> {
+    const pages: Page[] = unitPages.map(unitPage => ({
+      id: currentPageIds?.[unitPage.name] ?? uuidv7(),
+      instanceId,
+      meta: unitPage.meta,
+    }))
+
+    const pageContentEntries: [string, PageBlock[]][] = unitPages
+      //
+      .map((unitPage, index) => [pages[index].id, unitPage.content])
+
+    // create mapping from local name to UUID
+    const newPageIds: Record<string, string> = {}
+    for (let i = 0; i < unitPages.length; i++) {
+      const unitPage = unitPages[i]
+      const instancePage = pages[i]
+      newPageIds[unitPage.name] = instancePage.id
+    }
+
+    // find pages that need to be deleted (old pages not in new list)
+    const pagesToDelete = Object.values(currentPageIds ?? {}).filter(
+      id => !Object.values(newPageIds).includes(id),
+    )
+
+    if (pages.length > 0) {
+      this.promiseTracker.track(this.putPages(pages, pageContentEntries))
+    }
+
+    if (pagesToDelete.length > 0) {
+      this.promiseTracker.track(this.deletePages(pagesToDelete))
+    }
+
+    return newPageIds
+  }
+
+  private async putPages(
+    pages: Page[],
+    pageContentEntries: [string, PageBlock[]][],
+  ): Promise<void> {
+    if (this.operation.type === "preview") {
+      return
+    }
+
+    const batch = this.stateManager.batch()
+
+    await Promise.all([
+      this.stateManager.getPageRepository(this.project.id).putManyItems(pages, batch),
+      this.stateManager
+        .getPageContentRepository(this.project.id)
+        .putMany(pageContentEntries, batch),
+    ])
+
+    await batch.write()
+  }
+
+  private async deletePages(pageIds: string[]): Promise<void> {
+    if (this.operation.type === "preview") {
+      return
+    }
+
+    const batch = this.stateManager.batch()
+
+    await Promise.all([
+      this.stateManager.getPageRepository(this.project.id).deleteMany(pageIds, batch),
+      this.stateManager.getPageContentRepository(this.project.id).deleteMany(pageIds, batch),
+    ])
+
+    await batch.write()
+  }
+
+  /**
+   * Processes triggers from instance state update.
+   *
+   * Converts unit triggers to instance triggers and handles cleanup.
+   */
+  private processTriggers(
+    instanceId: string,
+    unitTriggers: UnitTrigger[],
+    currentTriggerIds: Record<string, string> | undefined,
+  ): Record<string, string> {
+    const triggers: Trigger[] = unitTriggers.map(unitTrigger => ({
+      id: currentTriggerIds?.[unitTrigger.name] ?? uuidv7(),
+      instanceId,
+      meta: unitTrigger.meta ?? {},
+      spec: unitTrigger.spec,
+    }))
+
+    // create mapping from local name to UUID
+    const newTriggerIds: Record<string, string> = {}
+    for (let i = 0; i < unitTriggers.length; i++) {
+      const unitTrigger = unitTriggers[i]
+      newTriggerIds[unitTrigger.name] = triggers[i].id
+    }
+
+    // find triggers that need to be deleted (old triggers not in new list)
+    const triggersToDelete = Object.values(currentTriggerIds ?? {}).filter(
+      id => !Object.values(newTriggerIds).includes(id),
     )
-  })
 
-  private persistLogs = createAsyncBatcher(async (entries: LogEntry[]) => {
-    this.logger.trace({ msg: "persisting logs", count: entries.length })
+    if (triggers.length > 0) {
+      this.promiseTracker.track(this.putTriggers(triggers))
+    }
+
+    if (triggersToDelete.length > 0) {
+      this.promiseTracker.track(this.deleteTriggers(triggersToDelete))
+    }
+
+    return newTriggerIds
+  }
+
+  private async putTriggers(triggers: Trigger[]): Promise<void> {
+    if (this.operation.type === "preview") {
+      return
+    }
+
+    await this.stateManager.getTriggerRepository(this.project.id).putManyItems(triggers)
+  }
 
-    await this.stateBackend.appendInstanceLogs(this.operation.id, entries)
-  })
+  private async deleteTriggers(triggerIds: string[]): Promise<void> {
+    if (this.operation.type === "preview") {
+      return
+    }
 
-  private persistSecrets = createAsyncBatcher(
-    async (entries: [string, Record<string, string>][]) => {
-      this.logger.debug({ msg: "persisting secrets", count: entries.length })
+    await this.stateManager.getTriggerRepository(this.project.id).deleteMany(triggerIds)
+  }
 
-      // TODO: may be batched (and patched without reading)
-      for (const [instanceId, secrets] of entries) {
-        const existingSecrets = await this.secretBackend.get(this.operation.projectId, instanceId)
+  /**
+   * Processes artifacts from instance state update.
+   */
+  private processArtifacts(
+    instanceId: string,
+    newOwnedArtifacts: Record<string, RunnerArtifact[]>,
+    oldOwnedArtifactIds: Record<string, string[]> | undefined,
+  ): Record<string, string[]> {
+    if (this.operation.type !== "preview") {
+      // persist the new owned artifacts if this is not a preview operation
+      this.promiseTracker.track(
+        this.artifactService.updateUsage(
+          this.project.id,
+          { type: "instance", instanceId },
+          unique(Object.values(oldOwnedArtifactIds ?? {}).flat()),
+          unique(
+            Object.values(newOwnedArtifacts).flatMap(artifacts =>
+              artifacts.map(artifact => artifact.id),
+            ),
+          ),
+        ),
+      )
+    }
 
-        Object.assign(existingSecrets, secrets)
+    return mapValues(newOwnedArtifacts, artifacts => artifacts.map(artifact => artifact.id))
+  }
 
-        await this.secretBackend.set(this.operation.projectId, instanceId, existingSecrets)
+  /**
+   * Processes exported secrets from instance state update.
+   *
+   * Validates that all secret IDs are either present in the instance secrets or are in "exportedSecrets" of the direct dependencies
+   * from connected inputs.
+   *
+   * Returns a mapping of output key to array of secret IDs exported via this output.
+   */
+  private processExportedSecrets(
+    instanceId: string,
+    exportedSecrets: Record<string, Omit<UnitSecretModel, "value">[]>,
+  ): Record<string, string[]> {
+    const instance = this.workset.getInstance(instanceId)
+    const allowedSecretIds = this.collectSecretIdsForInstance(instance)
+
+    return mapValues(exportedSecrets, secrets => {
+      const exportedSecretIds = secrets.map(secret => secret.id)
+      for (const exportedSecretId of exportedSecretIds) {
+        if (!allowedSecretIds.has(exportedSecretId)) {
+          throw new Error(
+            `The secret "${exportedSecretId}" is not allowed to be exported since it is not present in the instance secrets or in the exported secrets of connected dependencies.`,
+          )
+        }
       }
-    },
-  )
+
+      return exportedSecretIds
+    })
+  }
+
+  /**
+   * Updates the secrets for a specific instance.
+   */
+  private async updateInstanceSecrets(
+    instance: InstanceModel,
+    secrets: Record<string, unknown>,
+  ): Promise<void> {
+    if (this.operation.type === "preview") {
+      // do not update secrets in preview mode
+      return
+    }
+
+    await this.secretService.updateInstanceSecrets(
+      this.project,
+      instance.id,
+      secrets,
+      [], // no secrets to delete in this context
+      false, // do not invalidate state even if the secrets were updated
+    )
+  }
 }