@highstate/backend 0.9.18 → 0.9.19

package/src/orchestrator/operation-workset.ts

@@ -1,509 +1,236 @@
-import type { LibraryBackend } from "../library"
-import type { ProjectBackend } from "../project"
-import type { StateManager } from "../state"
-import type { Logger } from "pino"
-import type { InstanceLockService, InstanceStateService } from "../business"
+import type { InstanceId } from "@highstate/contract"
+import type { InstanceStateService, UpdateOperationStateOptions } from "../business"
+import type { InstanceOperationStatus } from "../database"
+import type { InstanceStatus, OperationPhase, OperationPhaseType, ProjectOutput } from "../shared"
+import type { OperationContext } from "./operation-context"
 import { EventEmitter, on } from "node:events"
-import {
-  isUnitModel,
-  parseInstanceId,
-  type ComponentModel,
-  type InstanceModel,
-} from "@highstate/contract"
-import { unique } from "remeda"
-import { BetterLock } from "better-lock"
-import {
-  InputHashResolver,
-  InputResolver,
-  type InputHashNode,
-  type InputHashOutput,
-  type InputResolverNode,
-  type InstanceOperataionStatusEnum,
-  type InstanceState,
-  type InstanceStatePatch,
-  type LibraryModel,
-  type Operation,
-  type Project,
-  type ResolvedInstanceInput,
-} from "../shared"
-
-export type OperationPhase = "update" | "destroy" | "refresh"
+import { mapValues } from "remeda"
 
-export class OperationWorkset {
-  private readonly lockedInstanceIds = new Set<string>()
-  public readonly instanceIdsToLockIds = new Set<string>()
-
-  private readonly instanceIdsToUpdate = new Set<string>()
-  private readonly instanceIdsToDestroy = new Set<string>()
-
-  private readonly instanceMap = new Map<string, InstanceModel>()
-  private readonly instanceChildrenMap = new Map<string, InstanceModel[]>()
-
-  private readonly initialStateMap = new Map<string, InstanceState>()
-  private readonly stateMap = new Map<string, InstanceState>()
-  private readonly dependentStateIdMap = new Map<string, Set<string>>()
-  private readonly stateChildIdMap = new Map<string, string[]>()
-
-  private readonly unitSourceHashMap = new Map<string, number>()
-
-  private inputResolver!: InputResolver
-  private readonly inputResolverNodes = new Map<string, InputResolverNode>()
-
-  private inputHashResolver!: InputHashResolver
-  private readonly inputHashNodes = new Map<string, InputHashNode>()
-  private readonly inputHashResolverLock = new BetterLock()
+type AbortControllerPair = {
+  abortController: AbortController
+  forceAbortController: AbortController
+}
 
-  public readonly resolvedInstanceInputs = new Map<
-    string,
-    Record<string, ResolvedInstanceInput[]>
-  >()
+export class OperationWorkset {
+  readonly abortController = new AbortController()
+  private readonly forceAbortController = new AbortController()
 
-  public currentPhase!: OperationPhase
+  readonly instanceAbortControllers = new Map<string, AbortControllerPair>()
 
+  private readonly lockedStateIds = new Set<string>()
   private readonly lockEventEmitter = new EventEmitter()
-
-  private constructor(
-    public readonly project: Project,
-    public readonly operation: Operation,
-    public readonly library: LibraryModel,
-    private readonly instanceLockService: InstanceLockService,
+  private currentPhaseIndex = -1
+
+  currentPhase!: OperationPhaseType
+  allAffectedInstanceIds!: ReadonlySet<InstanceId>
+  allAffectedStateIds!: ReadonlySet<string>
+  phaseAffectedInstanceIds!: ReadonlySet<InstanceId>
+
+  constructor(
+    readonly project: ProjectOutput,
+    readonly operationId: string,
+    readonly phases: OperationPhase[],
+    private readonly context: OperationContext,
     private readonly instanceStateService: InstanceStateService,
-    private readonly logger: Logger,
-  ) {}
+  ) {
+    const affectedInstanceIds = new Set<InstanceId>()
+    const affectedStateIds = new Set<string>()
 
-  public async ensureInstanceLocked(instanceId: string, signal: AbortSignal): Promise<void> {
-    if (this.lockedInstanceIds.has(instanceId)) {
-      return
-    }
+    for (const phase of phases) {
+      for (const instance of phase.instances) {
+        affectedInstanceIds.add(instance.id)
 
-    for await (const _ of on(this.lockEventEmitter, instanceId, { signal })) {
-      return
+        const state = this.context.getState(instance.id)
+        affectedStateIds.add(state.id)
+      }
     }
-  }
 
-  public getInstance(instanceId: string): InstanceModel {
-    const instance = this.instanceMap.get(instanceId)
-    if (!instance) {
-      throw new Error(`Instance with ID ${instanceId} not found in the operation workset`)
-    }
+    this.allAffectedInstanceIds = affectedInstanceIds
+    this.allAffectedStateIds = affectedStateIds
 
-    return instance
+    // we will basically create one listener per affected instance
+    this.lockEventEmitter.setMaxListeners(this.allAffectedInstanceIds.size)
   }
 
-  public getAffectedInstanceIds(): string[] {
-    if (this.currentPhase === "destroy") {
-      return Array.from(this.instanceIdsToDestroy)
-    }
-
-    return Array.from(this.instanceIdsToUpdate)
+  hasRemainingPhases(): boolean {
+    return this.currentPhaseIndex < this.phases.length - 1
   }
 
-  public getAllAffectedInstanceIds(): string[] {
-    return Array.from(this.instanceIdsToUpdate.union(this.instanceIdsToDestroy))
+  getLockedStateIds(): Iterable<string> {
+    return this.lockedStateIds
   }
 
-  public getInstanceOrUndefined(instanceId: string): InstanceModel | undefined {
-    return this.instanceMap.get(instanceId)
-  }
+  isLastPhaseForInstance(instanceId: InstanceId): boolean {
+    if (!this.hasRemainingPhases()) {
+      return true
+    }
 
-  public isAffected(instanceId: string): boolean {
-    if (this.currentPhase === "destroy") {
-      return this.instanceIdsToDestroy.has(instanceId)
+    // TODO: create instance id sets for each phase on initialization to speed this up
+    for (let i = this.currentPhaseIndex + 1; i < this.phases.length; i++) {
+      if (this.phases[i].instances.find(i => i.id === instanceId)) {
+        return false
+      }
     }
 
-    return this.instanceIdsToUpdate.has(instanceId)
+    return true
   }
 
-  public async patchState(patch: InstanceStatePatch): Promise<InstanceState> {
-    const state = await this.instanceStateService.patchOperationInstanceState(
-      this.project.id,
-      this.operation,
-      patch,
-    )
-    this.stateMap.set(state.id, state)
-
-    if (state.parentId) {
-      await this.recalculateCompositeInstanceState(state.parentId)
+  nextPhase(): void {
+    if (!this.hasRemainingPhases()) {
+      throw new Error("No remaining phases")
     }
 
-    return state
-  }
-
-  public setState(state: InstanceState): void {
-    this.stateMap.set(state.id, state)
-    this.initialStateMap.set(state.id, state)
+    this.currentPhaseIndex++
+    this.currentPhase = this.phases[this.currentPhaseIndex].type
 
-    if (state.parentId) {
-      let children = this.stateChildIdMap.get(state.parentId)
-      if (!children) {
-        children = []
-        this.stateChildIdMap.set(state.parentId, children)
-      }
+    this.phaseAffectedInstanceIds = new Set(
+      this.phases[this.currentPhaseIndex].instances.map(i => i.id),
+    )
+  }
 
-      children.push(state.id)
+  async waitForInstanceLock(stateId: string, signal: AbortSignal): Promise<void> {
+    if (this.lockedStateIds.has(stateId)) {
+      return
     }
 
-    for (const dependencyId of state.operationStatus?.dependencyIds ?? []) {
-      this.addDependentState(state.id, dependencyId)
+    for await (const _ of on(this.lockEventEmitter, stateId, { signal })) {
+      return
     }
   }
 
-  private addDependentState(instanceId: string, dependencyId: string): void {
-    let dependentStates = this.dependentStateIdMap.get(dependencyId)
-
-    if (!dependentStates) {
-      dependentStates = new Set<string>()
-      this.dependentStateIdMap.set(dependencyId, dependentStates)
-    }
-
-    dependentStates.add(instanceId)
+  markInstanceLocked(stateId: string): void {
+    this.lockedStateIds.add(stateId)
+    this.lockEventEmitter.emit(stateId)
   }
 
-  public async restoreInitialState(instanceId: string): Promise<void> {
-    const initialState = this.initialStateMap.get(instanceId) ?? { id: instanceId }
-
-    this.logger.debug(
-      { initialState },
-      `resetting state for instance "%s" to initial state`,
-      instanceId,
-    )
+  markInstanceUnlocked(stateId: string): void {
+    this.lockedStateIds.delete(stateId)
+  }
 
-    await this.instanceStateService.updateInstanceStates(
+  async setupOperationStates(): Promise<void> {
+    const patches = await this.instanceStateService.createOperationStates(
       this.project.id,
-      [initialState],
-      true,
-      false,
+      Array.from(this.allAffectedInstanceIds).map(instanceId => {
+        const instance = this.context.getInstance(instanceId)
+        const state = this.context.getState(instanceId)
+
+        const resolvedInputs = mapValues(
+          //
+          this.context.getResolvedInputs(instance.id) ?? {},
+          inputs => inputs.map(input => input.input),
+        )
+
+        return [
+          {
+            stateId: state.id,
+            operationId: this.operationId,
+            status: "pending",
+            model: instance,
+            resolvedInputs,
+          },
+          {
+            status: state.status === "undeployed" ? "attempted" : state.status,
+            model: instance,
+            resolvedInputs,
+          },
+        ]
+      }),
     )
 
-    if (initialState.parentId) {
-      await this.recalculateCompositeInstanceState(initialState.parentId)
+    for (const patch of patches) {
+      // biome-ignore lint/style/noNonNullAssertion: we know it's there (should be (please))
+      const state = this.context.getState(patch.instanceId!)
+      Object.assign(state, patch)
     }
   }
 
-  public async restoreAffectedInitialStates(): Promise<void> {
-    const instanceStates: InstanceState[] = []
+  async updateState(instanceId: InstanceId, options: UpdateOperationStateOptions): Promise<void> {
+    const state = this.context.getState(instanceId)
 
-    for (const instanceId of this.getAffectedInstanceIds()) {
-      const state = this.initialStateMap.get(instanceId)
-      if (state) {
-        instanceStates.push(state)
-      }
-    }
-
-    await this.instanceStateService.updateInstanceStates(
+    const patch = await this.instanceStateService.updateOperationState(
       this.project.id,
-      instanceStates,
-      false,
-      false,
+      state.id,
+      this.operationId,
+      options,
     )
-  }
-
-  public getAffectedCompositeChildren(instanceId: string): InstanceModel[] {
-    const children = this.instanceChildrenMap.get(instanceId)
-    if (!children) {
-      return []
-    }
 
-    if (this.currentPhase === "destroy") {
-      return children.filter(child => this.instanceIdsToDestroy.has(child.id))
+    if (state.parentInstanceId) {
+      // TODO: update all updates in single transaction
+      await this.recalculateCompositeInstanceState(state.parentInstanceId)
     }
 
-    return children.filter(child => this.instanceIdsToUpdate.has(child.id))
-  }
-
-  public getState(instanceId: string): InstanceState | undefined {
-    return this.stateMap.get(instanceId)
+    Object.assign(state, patch)
   }
 
-  public getParentId(instanceId: string): string | undefined {
+  getAffectedCompositeChildren(instanceId: InstanceId): InstanceId[] {
     if (this.currentPhase === "destroy") {
-      const state = this.stateMap.get(instanceId)
-      if (!state) {
-        return undefined
-      }
-
-      return state.parentId
+      // when destroying, only consider children fixed in the state
+      return this.context
+        .getStateChildIds(instanceId)
+        .filter(child => this.phaseAffectedInstanceIds.has(child))
     }
 
-    const instance = this.getInstance(instanceId)
-
-    return instance.parentId
+    // for other phases, consider all children defined in the model
+    return this.context
+      .getInstanceChildren(instanceId)
+      .map(child => child.id)
+      .filter(childId => this.phaseAffectedInstanceIds.has(childId))
   }
 
-  public getDependentStates(instanceId: string): InstanceState[] {
-    const dependentStateIds = this.dependentStateIdMap.get(instanceId)
-    if (!dependentStateIds) {
-      return []
-    }
-
-    return Array.from(dependentStateIds)
-      .map(id => this.stateMap.get(id))
-      .filter((state): state is InstanceState => !!state)
-  }
-
-  private async recalculateCompositeInstanceState(instanceId: string): Promise<void> {
-    const state = this.stateMap.get(instanceId)
-    if (!state) {
-      this.logger.warn(
-        `cannot recalculate composite instance state for "${instanceId}" because it is not in the state map`,
-      )
-      return
-    }
+  private async recalculateCompositeInstanceState(instanceId: InstanceId): Promise<void> {
+    const state = this.context.getState(instanceId)
 
     let currentResourceCount = 0
     let totalResourceCount = 0
+    let knownTotatalResourceCount = 0
 
-    const children = this.stateChildIdMap.get(instanceId) ?? []
+    const children = this.context.getStateChildIds(instanceId)
     for (const childId of children) {
-      const child = this.stateMap.get(childId)
-
-      if (child?.operationStatus?.currentResourceCount) {
-        currentResourceCount += child.operationStatus.currentResourceCount
-      }
+      const child = this.context.getState(childId)
 
-      if (child?.operationStatus?.totalResourceCount) {
-        totalResourceCount += child.operationStatus?.totalResourceCount
+      if (child?.lastOperationState?.currentResourceCount) {
+        currentResourceCount += child.lastOperationState.currentResourceCount
       }
-    }
-
-    const patch: InstanceStatePatch = {
-      id: instanceId,
-      operationStatus: {
-        status: this.getTransientStatusByOperationPhase(),
-        currentResourceCount,
-        totalResourceCount,
-      },
-    }
-
-    await this.patchState(patch)
-  }
-
-  private addInstance(instance: InstanceModel): void {
-    if (this.instanceMap.has(instance.id)) {
-      throw new Error(`Found multiple instances with the same ID: ${instance.id}`)
-    }
-
-    if (!(instance.type in this.library.components)) {
-      this.logger.warn(
-        `ignoring instance "${instance.id}" because its type "${instance.type}" is not in the library`,
-      )
-      return
-    }
-
-    this.instanceMap.set(instance.id, instance)
 
-    if (instance.parentId) {
-      let children = this.instanceChildrenMap.get(instance.parentId)
-      if (!children) {
-        children = []
-        this.instanceChildrenMap.set(instance.parentId, children)
+      if (child?.lastOperationState?.totalResourceCount) {
+        totalResourceCount += child.lastOperationState.totalResourceCount
+        knownTotatalResourceCount += 1
       }
-
-      children.push(instance)
     }
-  }
 
-  private async calculateInstanceIdsToUpdate(): Promise<void> {
-    const traverse = async (instanceId: string) => {
-      if (this.instanceIdsToUpdate.has(instanceId)) {
-        return
-      }
+    // extrapolate total resource count for other resources without total resource count
+    const averageTotalResourceCount =
+      knownTotatalResourceCount > 0 ? Math.round(totalResourceCount / knownTotatalResourceCount) : 0
 
-      const instance = this.instanceMap.get(instanceId)
-      if (!instance) {
-        return
-      }
+    const notKnownTotalResourceCount = children.length - knownTotatalResourceCount
+    totalResourceCount += notKnownTotalResourceCount * averageTotalResourceCount
 
-      const instanceInputs = this.resolvedInstanceInputs.get(instance.id) ?? {}
+    const finalTotalResourceCount =
+      this.currentPhase === "destroy" && state.lastOperationState?.totalResourceCount
+        ? // do not override totalResourceCount with lower values when destroying instances
+          Math.min(totalResourceCount, state.lastOperationState.totalResourceCount)
+        : totalResourceCount
 
-      for (const inputs of Object.values(instanceInputs)) {
-        for (const input of inputs) {
-          await traverse(input.input.instanceId)
-        }
-      }
-
-      const state = this.stateMap.get(instance.id)
-      const { inputHash: expectedInputHash } = this.inputHashResolver.requireOutput(instance.id)
-
-      if (this.operation.options.forceUpdateDependencies) {
-        this.instanceIdsToUpdate.add(instanceId)
-        return
-      }
-
-      if (
-        !state?.operationStatus ||
-        state.operationStatus.status === "error" ||
-        state.operationStatus.inputHash !== expectedInputHash
-      ) {
-        this.instanceIdsToUpdate.add(instanceId)
-      }
-    }
-
-    // 1. extend affected instance IDs with their not-created or not-up-to-date dependencies (only for "update" operations)
-    for (const instanceId of this.operation.requestedInstanceIds) {
-      if (this.operation.type === "update") {
-        await traverse(instanceId)
-      }
-
-      this.instanceIdsToUpdate.add(instanceId)
-    }
-
-    // 2. extend affected instance IDs with the children of the affected composite instances
-    const compositeInstanceQueue = Array.from(this.instanceIdsToUpdate)
-    while (compositeInstanceQueue.length > 0) {
-      const childId = compositeInstanceQueue.pop()!
-      const children = this.instanceChildrenMap.get(childId) ?? []
-
-      for (const child of children) {
-        compositeInstanceQueue.push(child.id)
-
-        if (this.operation.options.forceUpdateChildren) {
-          this.instanceIdsToUpdate.add(child.id)
-          continue
-        }
-
-        const state = this.stateMap.get(child.id)
-        const { inputHash: expectedInputHash } = this.inputHashResolver.requireOutput(child.id)
-
-        if (
-          !state?.operationStatus ||
-          state.operationStatus.status === "error" ||
-          state.operationStatus.inputHash !== expectedInputHash
-        ) {
-          this.instanceIdsToUpdate.add(child.id)
-        }
-      }
-    }
-
-    // 3. detect composite instance children and include their parents (recursively)
-    for (const instanceId of this.instanceIdsToUpdate) {
-      let instance = this.instanceMap.get(instanceId)
-      while (instance?.parentId) {
-        this.instanceIdsToUpdate.add(instance.parentId)
-        instance = this.instanceMap.get(instance.parentId)
-      }
-    }
-
-    // 4. if "allowPartialCompositeInstanceUpdates" is not set, include all children of the composite instances
-    if (!this.operation.options.allowPartialCompositeInstanceUpdates) {
-      for (const instanceId of this.instanceIdsToUpdate) {
-        const children = this.instanceChildrenMap.get(instanceId) ?? []
-        for (const child of children) {
-          this.instanceIdsToUpdate.add(child.id)
-        }
-      }
-    }
-
-    this.operation.instanceIdsToUpdate = Array.from(this.instanceIdsToUpdate)
-  }
-
-  private calculateInstanceIdsToDestroy() {
-    const traverse = (instanceKey: string) => {
-      if (this.instanceIdsToDestroy.has(instanceKey)) {
-        return
-      }
-
-      const state = this.stateMap.get(instanceKey)
-      if (!state) {
-        return
-      }
-
-      const dependentIds = this.dependentStateIdMap.get(instanceKey) ?? []
-
-      for (const dependentId of dependentIds) {
-        traverse(dependentId)
-        this.instanceIdsToDestroy.add(dependentId)
-      }
-    }
-
-    if (this.operation.type === "destroy" || this.operation.type === "recreate") {
-      // 1.a. extend affected instance IDs with their created dependents (if not forbidden by the operation options)
-      for (const instanceId of this.operation.requestedInstanceIds) {
-        const instance = this.instanceMap.get(instanceId)
-        if (!instance) {
-          throw new Error(`Instance not found: ${instanceId}`)
-        }
-
-        if (this.operation.options.destroyDependentInstances) {
-          traverse(instance.id)
-        }
-
-        this.instanceIdsToDestroy.add(instanceId)
-      }
-    } else if (this.operation.type === "update") {
-      // 1.b. find all children instances of the affected to-update instances which are in the state map, but not in the instance map
-      // in other words, this code cleans up child instances which are not produced by composite instances anymore
-
-      for (const instanceId of this.operation.instanceIdsToUpdate) {
-        const [type] = parseInstanceId(instanceId)
-        const component = this.library.components[type]
-
-        if (!component || isUnitModel(component)) {
-          // ignore non-composite instances
-          continue
-        }
-
-        const childrenQueue = [...(this.stateChildIdMap.get(instanceId) ?? [])]
-
-        while (childrenQueue.length > 0) {
-          const childId = childrenQueue.pop()!
-          if (!this.instanceMap.has(childId)) {
-            this.instanceIdsToDestroy.add(childId)
-          }
-
-          childrenQueue.push(...(this.stateChildIdMap.get(childId) ?? []))
-        }
-      }
-    } else {
-      return
-    }
-
-    // 2. extend affected instance IDs with the children of the affected composite instances
-    const compositeInstanceQueue = Array.from(this.instanceIdsToDestroy)
-    while (compositeInstanceQueue.length > 0) {
-      const childId = compositeInstanceQueue.pop()!
-      const children = this.stateChildIdMap.get(childId) ?? []
-
-      for (const child of children) {
-        compositeInstanceQueue.push(child)
-        this.instanceIdsToDestroy.add(child)
-      }
-    }
-
-    // 3. detect composite instance children and include their parents (recursively)
-    for (const instanceId of this.instanceIdsToDestroy) {
-      let state = this.stateMap.get(instanceId)
-      while (state?.parentId) {
-        this.instanceIdsToDestroy.add(state.parentId)
-        state = this.stateMap.get(state.parentId)
-      }
-    }
-
-    // 4. if "allowPartialCompositeInstanceUpdates" is not set, include all children of the composite instances
-    if (!this.operation.options.allowPartialCompositeInstanceUpdates) {
-      for (const instanceId of this.instanceIdsToDestroy) {
-        const children = this.stateChildIdMap.get(instanceId) ?? []
-        for (const childId of children) {
-          this.instanceIdsToDestroy.add(childId)
-        }
-      }
-    }
-
-    this.operation.instanceIdsToDestroy = Array.from(this.instanceIdsToDestroy)
+    await this.updateState(instanceId, {
+      operationState: {
+        status: this.getTransientStatusByOperationPhase(),
+        currentResourceCount,
+        totalResourceCount: finalTotalResourceCount,
+      },
+    })
   }
 
-  private getSourceHashIfApplicable(
-    instance: InstanceModel,
-    component: ComponentModel,
-  ): number | undefined {
-    if (isUnitModel(component)) {
-      return this.unitSourceHashMap.get(instance.type)
+  getPhaseParentId(instanceId: InstanceId): InstanceId | null {
+    if (this.currentPhase === "destroy") {
+      // when destroying, only consider parent fixed in the state
+      return this.context.getState(instanceId).parentInstanceId ?? null
     }
 
-    return undefined
+    // for other phases, consider parent defined in the model
+    return this.context.getInstance(instanceId).parentId ?? null
   }
 
-  private getTransientStatusByOperationPhase(): InstanceOperataionStatusEnum {
+  getTransientStatusByOperationPhase(): InstanceOperationStatus {
     switch (this.currentPhase) {
       case "update":
         return "updating"
@@ -514,180 +241,108 @@ export class OperationWorkset {
     }
   }
 
-  public async getUpToDateInputHashOutput(instance: InstanceModel): Promise<InputHashOutput> {
-    return await this.inputHashResolverLock.acquire(async () => {
-      const component = this.library.components[instance.type]
-
-      this.inputHashNodes.set(instance.id, {
-        instance,
-        component,
-        resolvedInputs: this.resolvedInstanceInputs.get(instance.id)!,
-        state: this.stateMap.get(instance.id),
-        sourceHash: this.getSourceHashIfApplicable(instance, component),
-      })
-
-      this.inputHashResolver.invalidate(instance.id)
-      await this.inputHashResolver.process()
-
-      return this.inputHashResolver.requireOutput(instance.id)
-    })
-  }
-
-  /**
-   * Tries to acquire not-locked instances in the workset.
-   *
-   * Should not be called if the whole workset is already locked.
-   *
-   * If `instancesToLock` is provided, it will try to lock only those instances.
-   */
-  public async tryLock(instancesToLock?: string[]): Promise<void> {
-    const [, lockedInstanceIds] = await this.instanceLockService.tryLockInstances(
-      this.project.id,
-      instancesToLock ?? Array.from(this.instanceIdsToLockIds),
-      {
-        title: "Operation Lock",
-        description: `The instance is locked for the ${this.operation.type} operation with ID "${this.operation.id}".`,
-        icon: "mdi:cog-sync",
-      },
-      { type: "operation", operationId: this.operation.id },
-      true, // allow partial locking
-    )
-
-    // add locked instance IDs to the workset and remove them from the instanceIdsToLockIds
-    for (const instanceId of lockedInstanceIds) {
-      this.lockedInstanceIds.add(instanceId)
-      this.instanceIdsToLockIds.delete(instanceId)
-      this.lockEventEmitter.emit(instanceId, null)
+  getStableStatusByOperationPhase(): InstanceOperationStatus {
+    switch (this.currentPhase) {
+      case "update":
+        return "updated"
+      case "destroy":
+        return "destroyed"
+      case "refresh":
+        return "refreshed"
     }
   }
 
-  public static async load(
-    project: Project,
-    operation: Operation,
-    projectBackend: ProjectBackend,
-    libraryBackend: LibraryBackend,
-    stateManager: StateManager,
-    instanceLockService: InstanceLockService,
-    instanceStateService: InstanceStateService,
-    logger: Logger,
-    signal: AbortSignal,
-  ): Promise<OperationWorkset> {
-    // TODO: use hotstate for virtual instances
-    const [library, { instances, hubs }, virtualInstances, states] = await Promise.all([
-      libraryBackend.loadLibrary(project.libraryId, signal),
-      projectBackend.getProjectModel(project, signal),
-      stateManager.getVirtualInstanceRepository(project.id).getAllItems(),
-      instanceStateService.getInstanceStates(project.id),
-    ])
-
-    const workset = new OperationWorkset(
-      project,
-      operation,
-      library,
-      instanceLockService,
-      instanceStateService,
-      logger.child({ operationId: operation.id, service: "OperationWorkset" }),
-    )
+  getNextStableInstanceStatus(instanceId: InstanceId): InstanceStatus {
+    const state = this.context.getState(instanceId)
 
-    // prepare instances
-    for (const instance of instances) {
-      workset.addInstance(instance)
+    switch (this.currentPhase) {
+      case "update":
+        return "deployed"
+      case "destroy":
+        return "undeployed"
+      case "refresh":
+        return state.status // do not change instance status when refreshing
     }
+  }
 
-    for (const instance of virtualInstances) {
-      const worksetInstance = workset.instanceMap.get(instance.id)
-
-      if (worksetInstance) {
-        worksetInstance.outputs = instance.outputs
-        worksetInstance.resolvedOutputs = instance.resolvedOutputs
-      } else if (instance.parentId) {
-        workset.addInstance(instance)
-      } else {
-        workset.logger.warn(
-          `ignoring virtual instance "${instance.id}" because it is not in the project or is not a part of known composite instance`,
-        )
-        continue
-      }
+  setupAbortControllersForAllInstances(): void {
+    for (const instanceId of this.allAffectedInstanceIds) {
+      this.setupInstanceAbortControllers(instanceId)
     }
+  }
 
-    const unitSources = await libraryBackend.getResolvedUnitSources(
-      project.libraryId,
-      unique(Array.from(workset.instanceMap.values()).map(i => i.type)),
-    )
-
-    for (const unitSource of unitSources) {
-      workset.unitSourceHashMap.set(unitSource.unitType, unitSource.sourceHash)
+  private setupInstanceAbortControllers(instanceId: InstanceId): AbortControllerPair {
+    const existingPair = this.instanceAbortControllers.get(instanceId)
+    if (existingPair) {
+      return existingPair
     }
 
-    for (const state of states) {
-      if (!workset.instanceMap.has(state.id)) {
-        workset.logger.warn(
-          `ignoring instance "${state.id}" from state because it is not in the project or is not a part of a composite instance`,
-        )
-        continue
-      }
+    // create abort controllers and setup them to abort when the operation is aborted
+    const abortController = new AbortController()
+    this.abortController.signal.addEventListener("abort", () => abortController.abort())
 
-      workset.setState(state)
-    }
+    abortController.signal.addEventListener("abort", () => {
+      // notify frontend that the instance is being cancelled
+      this.updateState(instanceId, { operationState: { status: "cancelling" } })
+    })
 
-    // prepare input resolver
-    for (const instance of workset.instanceMap.values()) {
-      workset.inputResolverNodes.set(`instance:${instance.id}`, {
-        kind: "instance",
-        instance,
-        component: library.components[instance.type],
-      })
-    }
+    const forceAbortController = new AbortController()
+    this.forceAbortController.signal.addEventListener("abort", () => forceAbortController.abort())
 
-    for (const hub of hubs) {
-      workset.inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
-    }
+    const pair: AbortControllerPair = { abortController, forceAbortController }
+    this.instanceAbortControllers.set(instanceId, pair)
 
-    workset.inputResolver = new InputResolver(workset.inputResolverNodes, logger)
-    workset.inputResolver.addAllNodesToWorkset()
+    // abort if the parent instance is cancelled
+    const children = this.context.getInstanceChildren(instanceId)
+    for (const child of children) {
+      const childPair = this.setupInstanceAbortControllers(child.id)
 
-    await workset.inputResolver.process()
-
-    // resolve inputs for all instances and pass outputs to input hash resolver
-    for (const instance of workset.instanceMap.values()) {
-      const output = workset.inputResolver.requireOutput(`instance:${instance.id}`)
-      if (output.kind !== "instance") {
-        throw new Error("Unexpected output kind")
-      }
+      abortController.signal.addEventListener("abort", () => childPair.abortController.abort())
 
-      workset.resolvedInstanceInputs.set(instance.id, output.resolvedInputs)
+      forceAbortController.signal.addEventListener("abort", () =>
+        childPair.forceAbortController.abort(),
+      )
+    }
 
-      const component = workset.library.components[instance.type]
+    return pair
+  }
 
-      workset.inputHashNodes.set(instance.id, {
-        instance,
-        component,
-        resolvedInputs: output.resolvedInputs,
-        state: workset.stateMap.get(instance.id),
-        sourceHash: workset.getSourceHashIfApplicable(instance, component),
-      })
+  cancelInstance(instanceId: InstanceId, allowForceAbort = true): void {
+    const abortControllerPair = this.instanceAbortControllers.get(instanceId)
+    if (!abortControllerPair) {
+      throw new Error(`No abort controller found for instance "${instanceId}"`)
     }
 
-    // prepare input hash resolver
-    workset.inputHashResolver = new InputHashResolver(workset.inputHashNodes, logger)
-    workset.inputHashResolver.addAllNodesToWorkset()
+    const { abortController, forceAbortController } = abortControllerPair
 
-    await workset.inputHashResolver.process()
+    // first try to cancel the operation gracefully
 
-    if (operation.type !== "destroy") {
-      await workset.calculateInstanceIdsToUpdate()
+    if (!abortController.signal.aborted) {
+      abortController.abort()
+      return
     }
 
-    workset.calculateInstanceIdsToDestroy()
+    if (!allowForceAbort) {
+      return
+    }
 
-    for (const instanceId of workset.instanceIdsToUpdate) {
-      workset.instanceIdsToLockIds.add(instanceId)
+    // then try to force cancel the operation
+    if (!forceAbortController.signal.aborted) {
+      forceAbortController.abort()
+      return
     }
+  }
 
-    for (const instanceId of workset.instanceIdsToDestroy) {
-      workset.instanceIdsToLockIds.add(instanceId)
+  cancel(): void {
+    if (!this.abortController.signal.aborted) {
+      this.abortController.abort()
+      return
     }
 
-    return workset
+    // then try to force cancel the operation
+    if (!this.forceAbortController.signal.aborted) {
+      this.forceAbortController.abort()
+      return
+    }
   }
 }