@highstate/backend 0.9.15 → 0.9.18

package/src/orchestrator/operation-workset.ts

@@ -1,7 +1,9 @@
 import type { LibraryBackend } from "../library"
 import type { ProjectBackend } from "../project"
-import type { StateBackend, StateManager } from "../state"
+import type { StateManager } from "../state"
 import type { Logger } from "pino"
+import type { InstanceLockService, InstanceStateService } from "../business"
+import { EventEmitter, on } from "node:events"
 import {
   isUnitModel,
   parseInstanceId,
@@ -11,24 +13,26 @@ import {
 import { unique } from "remeda"
 import { BetterLock } from "better-lock"
 import {
-  applyPartialInstanceState,
-  createInstanceState,
-  createInstanceStatePatch,
   InputHashResolver,
   InputResolver,
   type InputHashNode,
+  type InputHashOutput,
   type InputResolverNode,
+  type InstanceOperataionStatusEnum,
   type InstanceState,
-  type InstanceStateUpdate,
-  type InstanceStatus,
+  type InstanceStatePatch,
   type LibraryModel,
-  type ProjectOperation,
+  type Operation,
+  type Project,
   type ResolvedInstanceInput,
 } from "../shared"
 
 export type OperationPhase = "update" | "destroy" | "refresh"
 
 export class OperationWorkset {
+  private readonly lockedInstanceIds = new Set<string>()
+  public readonly instanceIdsToLockIds = new Set<string>()
+
   private readonly instanceIdsToUpdate = new Set<string>()
   private readonly instanceIdsToDestroy = new Set<string>()
 
@@ -40,7 +44,7 @@ export class OperationWorkset {
   private readonly dependentStateIdMap = new Map<string, Set<string>>()
   private readonly stateChildIdMap = new Map<string, string[]>()
 
-  private readonly unitSourceHashMap = new Map<string, string>()
+  private readonly unitSourceHashMap = new Map<string, number>()
 
   private inputResolver!: InputResolver
   private readonly inputResolverNodes = new Map<string, InputResolverNode>()
@@ -54,13 +58,29 @@ export class OperationWorkset {
     Record<string, ResolvedInstanceInput[]>
   >()
 
+  public currentPhase!: OperationPhase
+
+  private readonly lockEventEmitter = new EventEmitter()
+
   private constructor(
-    public readonly operation: ProjectOperation,
+    public readonly project: Project,
+    public readonly operation: Operation,
     public readonly library: LibraryModel,
-    private readonly stateManager: StateManager,
+    private readonly instanceLockService: InstanceLockService,
+    private readonly instanceStateService: InstanceStateService,
     private readonly logger: Logger,
   ) {}
 
+  public async ensureInstanceLocked(instanceId: string, signal: AbortSignal): Promise<void> {
+    if (this.lockedInstanceIds.has(instanceId)) {
+      return
+    }
+
+    for await (const _ of on(this.lockEventEmitter, instanceId, { signal })) {
+      return
+    }
+  }
+
   public getInstance(instanceId: string): InstanceModel {
     const instance = this.instanceMap.get(instanceId)
     if (!instance) {
@@ -70,35 +90,43 @@ export class OperationWorkset {
     return instance
   }
 
-  public getAffectedInstanceIds(phase: OperationPhase): string[] {
-    if (phase === "destroy") {
+  public getAffectedInstanceIds(): string[] {
+    if (this.currentPhase === "destroy") {
       return Array.from(this.instanceIdsToDestroy)
     }
 
     return Array.from(this.instanceIdsToUpdate)
   }
 
+  public getAllAffectedInstanceIds(): string[] {
+    return Array.from(this.instanceIdsToUpdate.union(this.instanceIdsToDestroy))
+  }
+
   public getInstanceOrUndefined(instanceId: string): InstanceModel | undefined {
     return this.instanceMap.get(instanceId)
   }
 
-  public isAffected(instanceId: string, phase: OperationPhase): boolean {
-    if (phase === "destroy") {
+  public isAffected(instanceId: string): boolean {
+    if (this.currentPhase === "destroy") {
       return this.instanceIdsToDestroy.has(instanceId)
     }
 
     return this.instanceIdsToUpdate.has(instanceId)
   }
 
-  public updateState(update: InstanceStateUpdate, phase: OperationPhase): InstanceState {
-    const finalState = applyPartialInstanceState(this.stateMap, update)
-    this.stateManager.emitStatePatch(this.operation.projectId, createInstanceStatePatch(update))
+  public async patchState(patch: InstanceStatePatch): Promise<InstanceState> {
+    const state = await this.instanceStateService.patchOperationInstanceState(
+      this.project.id,
+      this.operation,
+      patch,
+    )
+    this.stateMap.set(state.id, state)
 
-    if (finalState.parentId) {
-      this.recalculateCompositeInstanceState(finalState.parentId, phase)
+    if (state.parentId) {
+      await this.recalculateCompositeInstanceState(state.parentId)
     }
 
-    return finalState
+    return state
   }
 
   public setState(state: InstanceState): void {
@@ -115,7 +143,7 @@ export class OperationWorkset {
       children.push(state.id)
     }
 
-    for (const dependencyId of state.dependencyIds) {
+    for (const dependencyId of state.operationStatus?.dependencyIds ?? []) {
       this.addDependentState(state.id, dependencyId)
     }
   }
@@ -131,35 +159,52 @@ export class OperationWorkset {
     dependentStates.add(instanceId)
   }
 
-  public restoreInitialStatus(instanceId: string): void {
-    const state = this.stateMap.get(instanceId)
-    const initialState = this.initialStateMap.get(instanceId)
+  public async restoreInitialState(instanceId: string): Promise<void> {
+    const initialState = this.initialStateMap.get(instanceId) ?? { id: instanceId }
 
-    if (!state || !initialState) {
-      this.logger.warn(
-        `cannot reset status for instance "${instanceId}" because it is not in the state map`,
-      )
-      return
-    }
+    this.logger.debug(
+      { initialState },
+      `resetting state for instance "%s" to initial state`,
+      instanceId,
+    )
 
-    this.stateMap.set(instanceId, { ...initialState, status: initialState.status })
+    await this.instanceStateService.updateInstanceStates(
+      this.project.id,
+      [initialState],
+      true,
+      false,
+    )
+
+    if (initialState.parentId) {
+      await this.recalculateCompositeInstanceState(initialState.parentId)
+    }
   }
 
-  public emitAffectedInitialStates(): void {
-    for (const state of this.initialStateMap.values()) {
-      if (this.instanceIdsToUpdate.has(state.id)) {
-        this.stateManager.emitStatePatch(this.operation.projectId, state)
+  public async restoreAffectedInitialStates(): Promise<void> {
+    const instanceStates: InstanceState[] = []
+
+    for (const instanceId of this.getAffectedInstanceIds()) {
+      const state = this.initialStateMap.get(instanceId)
+      if (state) {
+        instanceStates.push(state)
       }
     }
+
+    await this.instanceStateService.updateInstanceStates(
+      this.project.id,
+      instanceStates,
+      false,
+      false,
+    )
   }
 
-  public getAffectedCompositeChildren(instanceId: string, phase: OperationPhase): InstanceModel[] {
+  public getAffectedCompositeChildren(instanceId: string): InstanceModel[] {
     const children = this.instanceChildrenMap.get(instanceId)
     if (!children) {
       return []
     }
 
-    if (phase === "destroy") {
+    if (this.currentPhase === "destroy") {
       return children.filter(child => this.instanceIdsToDestroy.has(child.id))
     }
 
@@ -170,11 +215,11 @@ export class OperationWorkset {
     return this.stateMap.get(instanceId)
   }
 
-  public getParentId(instanceId: string, phase: OperationPhase): string | null {
-    if (phase === "destroy") {
+  public getParentId(instanceId: string): string | undefined {
+    if (this.currentPhase === "destroy") {
       const state = this.stateMap.get(instanceId)
       if (!state) {
-        return null
+        return undefined
       }
 
       return state.parentId
@@ -182,7 +227,7 @@ export class OperationWorkset {
 
     const instance = this.getInstance(instanceId)
 
-    return instance.parentId ?? null
+    return instance.parentId
   }
 
   public getDependentStates(instanceId: string): InstanceState[] {
@@ -196,8 +241,15 @@ export class OperationWorkset {
       .filter((state): state is InstanceState => !!state)
   }
 
-  private recalculateCompositeInstanceState(instanceId: string, phase: OperationPhase): void {
-    const state = this.stateMap.get(instanceId) ?? createInstanceState(instanceId)
+  private async recalculateCompositeInstanceState(instanceId: string): Promise<void> {
+    const state = this.stateMap.get(instanceId)
+    if (!state) {
+      this.logger.warn(
+        `cannot recalculate composite instance state for "${instanceId}" because it is not in the state map`,
+      )
+      return
+    }
+
     let currentResourceCount = 0
     let totalResourceCount = 0
 
@@ -205,28 +257,25 @@ export class OperationWorkset {
     for (const childId of children) {
       const child = this.stateMap.get(childId)
 
-      if (child?.currentResourceCount) {
-        currentResourceCount += child.currentResourceCount
+      if (child?.operationStatus?.currentResourceCount) {
+        currentResourceCount += child.operationStatus.currentResourceCount
       }
 
-      if (child?.totalResourceCount) {
-        totalResourceCount += child.totalResourceCount
+      if (child?.operationStatus?.totalResourceCount) {
+        totalResourceCount += child.operationStatus?.totalResourceCount
       }
     }
 
-    const updatedState = {
-      ...state,
-      status: OperationWorkset.getStatusByOperationPhase(phase),
-      currentResourceCount,
-      totalResourceCount,
+    const patch: InstanceStatePatch = {
+      id: instanceId,
+      operationStatus: {
+        status: this.getTransientStatusByOperationPhase(),
+        currentResourceCount,
+        totalResourceCount,
+      },
     }
 
-    this.stateMap.set(instanceId, updatedState)
-    this.stateManager.emitStatePatch(this.operation.projectId, updatedState)
-
-    if (state.parentId) {
-      this.recalculateCompositeInstanceState(state.parentId, phase)
-    }
+    await this.patchState(patch)
   }
 
   private addInstance(instance: InstanceModel): void {
@@ -281,7 +330,11 @@ export class OperationWorkset {
         return
       }
 
-      if (state?.status !== "created" || state.inputHash !== expectedInputHash) {
+      if (
+        !state?.operationStatus ||
+        state.operationStatus.status === "error" ||
+        state.operationStatus.inputHash !== expectedInputHash
+      ) {
         this.instanceIdsToUpdate.add(instanceId)
       }
     }
@@ -312,7 +365,11 @@ export class OperationWorkset {
         const state = this.stateMap.get(child.id)
         const { inputHash: expectedInputHash } = this.inputHashResolver.requireOutput(child.id)
 
-        if (state?.status !== "created" || state.inputHash !== expectedInputHash) {
+        if (
+          !state?.operationStatus ||
+          state.operationStatus.status === "error" ||
+          state.operationStatus.inputHash !== expectedInputHash
+        ) {
           this.instanceIdsToUpdate.add(child.id)
         }
       }
@@ -347,7 +404,7 @@ export class OperationWorkset {
       }
 
       const state = this.stateMap.get(instanceKey)
-      if (!state || state.status === "not_created") {
+      if (!state) {
         return
       }
 
@@ -438,7 +495,7 @@ export class OperationWorkset {
   private getSourceHashIfApplicable(
     instance: InstanceModel,
     component: ComponentModel,
-  ): string | undefined {
+  ): number | undefined {
     if (isUnitModel(component)) {
       return this.unitSourceHashMap.get(instance.type)
     }
@@ -446,8 +503,8 @@ export class OperationWorkset {
     return undefined
   }
 
-  private static getStatusByOperationPhase(phase: OperationPhase): InstanceStatus {
-    switch (phase) {
+  private getTransientStatusByOperationPhase(): InstanceOperataionStatusEnum {
+    switch (this.currentPhase) {
       case "update":
         return "updating"
       case "destroy":
@@ -457,7 +514,7 @@ export class OperationWorkset {
     }
   }
 
-  public async getUpToDateInputHash(instance: InstanceModel): Promise<string> {
+  public async getUpToDateInputHashOutput(instance: InstanceModel): Promise<InputHashOutput> {
     return await this.inputHashResolverLock.acquire(async () => {
       const component = this.library.components[instance.type]
 
@@ -472,67 +529,90 @@ export class OperationWorkset {
       this.inputHashResolver.invalidate(instance.id)
       await this.inputHashResolver.process()
 
-      const { inputHash } = this.inputHashResolver.requireOutput(instance.id)
-      return inputHash
+      return this.inputHashResolver.requireOutput(instance.id)
     })
   }
 
-  public getLockInstanceIds(): string[] {
-    return Array.from(this.instanceIdsToUpdate.union(this.instanceIdsToDestroy))
+  /**
+   * Tries to acquire not-locked instances in the workset.
+   *
+   * Should not be called if the whole workset is already locked.
+   *
+   * If `instancesToLock` is provided, it will try to lock only those instances.
+   */
+  public async tryLock(instancesToLock?: string[]): Promise<void> {
+    const [, lockedInstanceIds] = await this.instanceLockService.tryLockInstances(
+      this.project.id,
+      instancesToLock ?? Array.from(this.instanceIdsToLockIds),
+      {
+        title: "Operation Lock",
+        description: `The instance is locked for the ${this.operation.type} operation with ID "${this.operation.id}".`,
+        icon: "mdi:cog-sync",
+      },
+      { type: "operation", operationId: this.operation.id },
+      true, // allow partial locking
+    )
+
+    // add locked instance IDs to the workset and remove them from the instanceIdsToLockIds
+    for (const instanceId of lockedInstanceIds) {
+      this.lockedInstanceIds.add(instanceId)
+      this.instanceIdsToLockIds.delete(instanceId)
+      this.lockEventEmitter.emit(instanceId, null)
+    }
   }
 
   public static async load(
-    operation: ProjectOperation,
+    project: Project,
+    operation: Operation,
     projectBackend: ProjectBackend,
     libraryBackend: LibraryBackend,
-    stateBackend: StateBackend,
     stateManager: StateManager,
+    instanceLockService: InstanceLockService,
+    instanceStateService: InstanceStateService,
     logger: Logger,
     signal: AbortSignal,
   ): Promise<OperationWorkset> {
-    const [library, project, compositeInstances, states] = await Promise.all([
-      libraryBackend.loadLibrary(signal),
-      projectBackend.getProject(operation.projectId, signal),
-      stateBackend.getCompositeInstances(operation.projectId, signal),
-      stateBackend.getAllInstanceStates(operation.projectId, signal),
+    // TODO: use hotstate for virtual instances
+    const [library, { instances, hubs }, virtualInstances, states] = await Promise.all([
+      libraryBackend.loadLibrary(project.libraryId, signal),
+      projectBackend.getProjectModel(project, signal),
+      stateManager.getVirtualInstanceRepository(project.id).getAllItems(),
+      instanceStateService.getInstanceStates(project.id),
     ])
 
     const workset = new OperationWorkset(
+      project,
       operation,
       library,
-      stateManager,
+      instanceLockService,
+      instanceStateService,
       logger.child({ operationId: operation.id, service: "OperationWorkset" }),
     )
 
     // prepare instances
-    for (const instance of project.instances) {
+    for (const instance of instances) {
       workset.addInstance(instance)
     }
 
-    for (const instance of compositeInstances) {
-      const worksetInstance = workset.instanceMap.get(instance.instance.id)
+    for (const instance of virtualInstances) {
+      const worksetInstance = workset.instanceMap.get(instance.id)
 
       if (worksetInstance) {
-        worksetInstance.outputs = instance.instance.outputs
-        worksetInstance.resolvedOutputs = instance.instance.resolvedOutputs
-      } else if (instance.instance.parentId) {
-        workset.addInstance(instance.instance)
+        worksetInstance.outputs = instance.outputs
+        worksetInstance.resolvedOutputs = instance.resolvedOutputs
+      } else if (instance.parentId) {
+        workset.addInstance(instance)
       } else {
         workset.logger.warn(
-          `ignoring instance "${instance.instance.id}" from composite instance state because it is not in the project or is not a part of known composite instance`,
+          `ignoring virtual instance "${instance.id}" because it is not in the project or is not a part of known composite instance`,
         )
         continue
       }
-
-      for (const child of instance.children) {
-        if (!workset.instanceMap.has(child.id)) {
-          workset.addInstance(child)
-        }
-      }
     }
 
     const unitSources = await libraryBackend.getResolvedUnitSources(
-      unique(Array.from(workset.instanceMap.values().map(i => i.type))),
+      project.libraryId,
+      unique(Array.from(workset.instanceMap.values()).map(i => i.type)),
     )
 
     for (const unitSource of unitSources) {
@@ -559,7 +639,7 @@ export class OperationWorkset {
       })
     }
 
-    for (const hub of project.hubs) {
+    for (const hub of hubs) {
       workset.inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
     }
 
@@ -600,6 +680,14 @@ export class OperationWorkset {
 
     workset.calculateInstanceIdsToDestroy()
 
+    for (const instanceId of workset.instanceIdsToUpdate) {
+      workset.instanceIdsToLockIds.add(instanceId)
+    }
+
+    for (const instanceId of workset.instanceIdsToDestroy) {
+      workset.instanceIdsToLockIds.add(instanceId)
+    }
+
     return workset
   }
 }