@highstate/backend 0.7.2 → 0.7.3

package/src/orchestrator/operation-workset.ts

@@ -0,0 +1,483 @@
+import type { LibraryBackend } from "../library"
+import type { ProjectBackend } from "../project"
+import type { StateBackend, StateManager } from "../state"
+import type { Logger } from "pino"
+import { isUnitModel, type ComponentModel, type InstanceModel } from "@highstate/contract"
+import {
+  applyPartialInstanceState,
+  createInputHashResolver,
+  createInputResolver,
+  createInstanceState,
+  createInstanceStatePatch,
+  type GraphResolver,
+  type InputHashResolverInput,
+  type InputHashResolverOutput,
+  type InputResolverInput,
+  type InputResolverOutput,
+  type InstanceState,
+  type InstanceStateUpdate,
+  type LibraryModel,
+  type ProjectOperation,
+  type ResolvedInstanceInput,
+} from "../shared"
+
+export class OperationWorkset {
+  private readonly affectedInstanceIdSet = new Set<string>()
+  private readonly instanceMap = new Map<string, InstanceModel>()
+  private readonly instanceChildrenMap = new Map<string, InstanceModel[]>()
+
+  private readonly initialStateMap = new Map<string, InstanceState>()
+  private readonly stateMap = new Map<string, InstanceState>()
+  private readonly dependentStateMap = new Map<string, InstanceState[]>()
+  private readonly stateChildIdMap = new Map<string, string[]>()
+
+  private readonly unitSourceHashMap = new Map<string, string>()
+
+  private inputResolver!: GraphResolver<InputResolverOutput>
+  private readonly inputResolverInputs = new Map<string, InputResolverInput>()
+  private readonly inputResolverPromiseCache = new Map<string, Promise<InputResolverOutput>>()
+
+  private inputHashResolver!: GraphResolver<InputHashResolverOutput>
+  private readonly inputHashResolverInputs = new Map<string, InputHashResolverInput>()
+
+  private readonly inputHashResolverPromiseCache = new Map<
+    string,
+    Promise<InputHashResolverOutput>
+  >()
+
+  public readonly resolvedInstanceInputs = new Map<
+    string,
+    Record<string, ResolvedInstanceInput[]>
+  >()
+
+  private constructor(
+    public readonly operation: ProjectOperation,
+    public readonly library: LibraryModel,
+    private readonly stateManager: StateManager,
+    private readonly logger: Logger,
+  ) {}
+
+  public getInstance(instanceId: string): InstanceModel {
+    const instance = this.instanceMap.get(instanceId)
+    if (!instance) {
+      throw new Error(`Instance with ID ${instanceId} not found in the operation workset`)
+    }
+
+    return instance
+  }
+
+  public isAffected(instanceId: string): boolean {
+    return this.affectedInstanceIdSet.has(instanceId)
+  }
+
+  public updateState(update: InstanceStateUpdate): InstanceState {
+    const finalState = applyPartialInstanceState(this.stateMap, update)
+    this.stateManager.emitStatePatch(this.operation.projectId, createInstanceStatePatch(update))
+
+    if (finalState.parentId) {
+      this.recalculateCompositeState(finalState.parentId)
+    }
+
+    return finalState
+  }
+
+  public setState(state: InstanceState): void {
+    this.stateMap.set(state.id, state)
+    this.initialStateMap.set(state.id, state)
+
+    if (state.parentId) {
+      let children = this.stateChildIdMap.get(state.parentId)
+      if (!children) {
+        children = []
+        this.stateChildIdMap.set(state.parentId, children)
+      }
+
+      children.push(state.id)
+    }
+  }
+
+  public restoreInitialStatus(instanceId: string): void {
+    const state = this.stateMap.get(instanceId)
+    const initialState = this.initialStateMap.get(instanceId)
+
+    if (!state || !initialState) {
+      this.logger.warn(
+        `cannot reset status for instance "${instanceId}" because it is not in the state map`,
+      )
+      return
+    }
+
+    this.stateMap.set(instanceId, { ...initialState, status: initialState.status })
+  }
+
+  public emitAffectedInitialStates(): void {
+    for (const state of this.initialStateMap.values()) {
+      if (this.affectedInstanceIdSet.has(state.id)) {
+        this.stateManager.emitStatePatch(this.operation.projectId, state)
+      }
+    }
+  }
+
+  public getAffectedCompositeChildren(instanceId: string): InstanceModel[] {
+    const children = this.instanceChildrenMap.get(instanceId)
+    if (!children) {
+      return []
+    }
+
+    return children.filter(child => this.affectedInstanceIdSet.has(child.id))
+  }
+
+  public getState(instanceId: string): InstanceState | undefined {
+    return this.stateMap.get(instanceId)
+  }
+
+  public getDependentStates(instanceId: string): InstanceState[] {
+    return this.dependentStateMap.get(instanceId) ?? []
+  }
+
+  private recalculateCompositeState(instanceId: string): void {
+    const state = this.stateMap.get(instanceId) ?? createInstanceState(instanceId)
+    let currentResourceCount = 0
+    let totalResourceCount = 0
+
+    const children = this.stateChildIdMap.get(instanceId) ?? []
+    for (const childId of children) {
+      const child = this.stateMap.get(childId)
+
+      if (child?.currentResourceCount) {
+        currentResourceCount += child.currentResourceCount
+      }
+
+      if (child?.totalResourceCount) {
+        totalResourceCount += child.totalResourceCount
+      }
+    }
+
+    const updatedState = {
+      ...state,
+      currentResourceCount,
+      totalResourceCount,
+    }
+
+    this.stateMap.set(instanceId, updatedState)
+    this.stateManager.emitStatePatch(this.operation.projectId, updatedState)
+
+    if (state.parentId) {
+      this.recalculateCompositeState(state.parentId)
+    }
+  }
+
+  private addInstance(instance: InstanceModel): void {
+    if (this.instanceMap.has(instance.id)) {
+      throw new Error(`Found multiple instances with the same ID: ${instance.id}`)
+    }
+
+    if (!(instance.type in this.library.components)) {
+      this.logger.warn(
+        `ignoring instance "${instance.id}" because its type "${instance.type}" is not in the library`,
+      )
+      return
+    }
+
+    this.instanceMap.set(instance.id, instance)
+
+    if (instance.parentId) {
+      let children = this.instanceChildrenMap.get(instance.parentId)
+      if (!children) {
+        children = []
+        this.instanceChildrenMap.set(instance.parentId, children)
+      }
+
+      children.push(instance)
+    }
+  }
+
+  private async extendForUpdate(): Promise<void> {
+    const traverse = async (instanceId: string) => {
+      if (this.affectedInstanceIdSet.has(instanceId)) {
+        return
+      }
+
+      const instance = this.instanceMap.get(instanceId)
+      if (!instance) {
+        return
+      }
+
+      const instanceInputs = this.resolvedInstanceInputs.get(instance.id) ?? {}
+
+      for (const inputs of Object.values(instanceInputs)) {
+        for (const input of inputs) {
+          await traverse(input.input.instanceId)
+        }
+      }
+
+      const state = this.stateMap.get(instance.id)
+      const { inputHash: expectedInputHash } = await this.inputHashResolver(instance.id)
+
+      if (this.operation.options.forceUpdateDependencies) {
+        this.affectedInstanceIdSet.add(instanceId)
+        return
+      }
+
+      if (state?.status !== "created" || state.inputHash !== expectedInputHash) {
+        this.affectedInstanceIdSet.add(instanceId)
+      }
+    }
+
+    // 1. extend affected instance IDs with their not-created or not-up-to-date dependencies (only for "update" operations)
+    for (const instanceId of this.operation.instanceIds) {
+      if (this.operation.type === "update") {
+        await traverse(instanceId)
+      }
+
+      this.affectedInstanceIdSet.add(instanceId)
+    }
+
+    // 2. extend affected instance IDs with the children of the affected composite instances
+    const compositeInstanceQueue = Array.from(this.affectedInstanceIdSet)
+    while (compositeInstanceQueue.length > 0) {
+      const childId = compositeInstanceQueue.pop()!
+      const children = this.instanceChildrenMap.get(childId) ?? []
+
+      for (const child of children) {
+        compositeInstanceQueue.push(child.id)
+        this.affectedInstanceIdSet.add(child.id)
+      }
+    }
+
+    // 3. detect composite instance children and include their parents (recursively)
+    for (const instanceId of this.affectedInstanceIdSet) {
+      let instance = this.instanceMap.get(instanceId)
+      while (instance?.parentId) {
+        this.affectedInstanceIdSet.add(instance.parentId)
+        instance = this.instanceMap.get(instance.parentId)
+      }
+    }
+
+    this.operation.affectedInstanceIds = Array.from(this.affectedInstanceIdSet)
+  }
+
+  private extendForDestroy() {
+    const traverse = (instanceKey: string) => {
+      if (this.affectedInstanceIdSet.has(instanceKey)) {
+        return
+      }
+
+      const state = this.stateMap.get(instanceKey)
+      if (!state || state.status === "not_created") {
+        return
+      }
+
+      const dependents = this.dependentStateMap.get(instanceKey) ?? []
+
+      for (const dependent of dependents) {
+        traverse(dependent.id)
+        this.affectedInstanceIdSet.add(instanceKey)
+      }
+    }
+
+    // 1. extend affected instance IDs with their created dependents (if not forbidden by the operation options)
+    for (const instanceId of this.operation.instanceIds) {
+      const instance = this.instanceMap.get(instanceId)
+      if (!instance) {
+        throw new Error(`Instance not found: ${instanceId}`)
+      }
+
+      if (this.operation.options.destroyDependentInstances) {
+        traverse(instance.id)
+      }
+
+      this.affectedInstanceIdSet.add(instanceId)
+    }
+
+    // 2. extend affected instance IDs with the children of the affected composite instances
+    const compositeInstanceQueue = Array.from(this.affectedInstanceIdSet)
+    while (compositeInstanceQueue.length > 0) {
+      const childId = compositeInstanceQueue.pop()!
+      const children = this.stateChildIdMap.get(childId) ?? []
+
+      for (const child of children) {
+        compositeInstanceQueue.push(child)
+        this.affectedInstanceIdSet.add(child)
+      }
+    }
+
+    // 3. detect composite instance children and include their parents (recursively)
+    for (const instanceId of this.affectedInstanceIdSet) {
+      let state = this.stateMap.get(instanceId)
+      while (state?.parentId) {
+        this.affectedInstanceIdSet.add(state.parentId)
+        state = this.stateMap.get(state.parentId)
+      }
+    }
+
+    this.operation.affectedInstanceIds = Array.from(this.affectedInstanceIdSet)
+  }
+
+  private getSourceHashIfApplicable(
+    instance: InstanceModel,
+    component: ComponentModel,
+  ): string | undefined {
+    if (isUnitModel(component)) {
+      return this.unitSourceHashMap.get(instance.type)
+    }
+
+    return undefined
+  }
+
+  public async getUpToDateInputHash(instance: InstanceModel): Promise<string> {
+    const component = this.library.components[instance.type]
+
+    this.inputHashResolverInputs.set(instance.id, {
+      instance,
+      component,
+      resolvedInputs: this.resolvedInstanceInputs.get(instance.id)!,
+      state: this.stateMap.get(instance.id),
+      sourceHash: this.getSourceHashIfApplicable(instance, component),
+    })
+
+    this.inputHashResolverPromiseCache.delete(instance.id)
+
+    const { inputHash } = await this.inputHashResolver(instance.id)
+    return inputHash
+  }
+
+  public getLockInstanceIds(): string[] {
+    const instanceIds = new Set<string>(this.operation.affectedInstanceIds)
+
+    // enrich with the parent IDs of the affected instances in order to lock and update parent states
+    for (const instanceId of this.operation.affectedInstanceIds) {
+      const instance = this.getInstance(instanceId)
+
+      if (instance.parentId) {
+        instanceIds.add(instance.parentId)
+      }
+    }
+
+    return Array.from(instanceIds)
+  }
+
+  public static async load(
+    operation: ProjectOperation,
+    projectBackend: ProjectBackend,
+    libraryBackend: LibraryBackend,
+    stateBackend: StateBackend,
+    stateManager: StateManager,
+    logger: Logger,
+    signal: AbortSignal,
+  ): Promise<OperationWorkset> {
+    const [library, unitSources, project, compositeInstances, states] = await Promise.all([
+      libraryBackend.loadLibrary(signal),
+      libraryBackend.getResolvedUnitSources(),
+      projectBackend.getProject(operation.projectId, signal),
+      stateBackend.getCompositeInstances(operation.projectId, signal),
+      stateBackend.getAllInstanceStates(operation.projectId, signal),
+    ])
+
+    const workset = new OperationWorkset(
+      operation,
+      library,
+      stateManager,
+      logger.child({ operationId: operation.id, service: "OperationWorkset" }),
+    )
+
+    for (const unitSource of unitSources) {
+      workset.unitSourceHashMap.set(unitSource.unitType, unitSource.sourceHash)
+    }
+
+    // prepare instances
+    for (const instance of project.instances) {
+      workset.addInstance(instance)
+    }
+
+    for (const instance of compositeInstances) {
+      const worksetInstance = workset.instanceMap.get(instance.instance.id)
+
+      if (worksetInstance) {
+        worksetInstance.outputs = instance.instance.outputs
+        worksetInstance.resolvedOutputs = instance.instance.resolvedOutputs
+      } else {
+        workset.addInstance(instance.instance)
+      }
+
+      for (const child of instance.children) {
+        workset.addInstance(child)
+      }
+    }
+
+    for (const state of states) {
+      if (!workset.instanceMap.has(state.id)) {
+        workset.logger.warn(
+          `ignoring instance "${state.id}" from state because it is not in the project or is not a part of a composite instance`,
+        )
+        continue
+      }
+
+      workset.setState(state)
+    }
+
+    // prepare input resolver
+    for (const instance of workset.instanceMap.values()) {
+      workset.inputResolverInputs.set(`instance:${instance.id}`, {
+        kind: "instance",
+        instance,
+        component: library.components[instance.type],
+      })
+    }
+
+    for (const hub of project.hubs) {
+      workset.inputResolverInputs.set(`hub:${hub.id}`, { kind: "hub", hub })
+    }
+
+    workset.inputResolver = createInputResolver(
+      //
+      workset.inputResolverInputs,
+      logger,
+      { promiseCache: workset.inputResolverPromiseCache },
+    )
+
+    // resolve inputs for all instances and pass outputs to input hash resolver
+    for (const instance of workset.instanceMap.values()) {
+      const output = await workset.inputResolver(`instance:${instance.id}`)
+      if (output.kind !== "instance") {
+        throw new Error("Unexpected output kind")
+      }
+
+      workset.resolvedInstanceInputs.set(instance.id, output.resolvedInputs)
+
+      const component = workset.library.components[instance.type]
+
+      workset.inputHashResolverInputs.set(instance.id, {
+        instance,
+        component,
+        resolvedInputs: output.resolvedInputs,
+        state: workset.stateMap.get(instance.id),
+        sourceHash: workset.getSourceHashIfApplicable(instance, component),
+      })
+    }
+
+    // prepare input hash resolver
+    workset.inputHashResolver = createInputHashResolver(
+      //
+      workset.inputHashResolverInputs,
+      logger,
+      { promiseCache: workset.inputHashResolverPromiseCache },
+    )
+
+    switch (operation.type) {
+      case "update":
+      case "preview":
+      case "refresh":
+        await workset.extendForUpdate()
+        break
+      case "recreate":
+        workset.extendForDestroy()
+        break
+      case "destroy":
+        workset.extendForDestroy()
+        break
+    }
+
+    return workset
+  }
+}