@highstate/backend 0.9.18 → 0.9.19

package/src/orchestrator/operation-context.ts

@@ -0,0 +1,357 @@
+import type { Logger } from "pino"
+import type { InstanceStateService, ProjectModelService } from "../business"
+import type { LibraryBackend } from "../library"
+import {
+  type ComponentModel,
+  type InstanceId,
+  type InstanceModel,
+  isUnitModel,
+} from "@highstate/contract"
+import { unique } from "remeda"
+import {
+  type InputHashNode,
+  type InputHashOutput,
+  InputHashResolver,
+  InputResolver,
+  type InputResolverNode,
+  type InstanceState,
+  isTransientInstanceOperationStatus,
+  type LibraryModel,
+  type ProjectOutput,
+  type ResolvedInstanceInput,
+} from "../shared"
+import { BetterLock } from "better-lock"
+
+export class OperationContext {
+  private readonly instanceMap = new Map<InstanceId, InstanceModel>()
+  private readonly instanceChildrenMap = new Map<InstanceId, InstanceModel[]>()
+
+  private readonly stateMap = new Map<InstanceId, InstanceState>()
+  private readonly dependentStateIdMap = new Map<InstanceId, Set<InstanceId>>()
+  private readonly stateChildIdMap = new Map<InstanceId, InstanceId[]>()
+
+  public readonly unitSourceHashMap = new Map<string, number>()
+
+  public inputResolver!: InputResolver
+  public readonly inputResolverNodes = new Map<string, InputResolverNode>()
+
+  public inputHashResolver!: InputHashResolver
+  public readonly inputHashNodes = new Map<string, InputHashNode>()
+  private readonly inputHashResolverLock = new BetterLock()
+
+  private readonly resolvedInstanceInputs = new Map<
+    string,
+    Record<InstanceId, ResolvedInstanceInput[]>
+  >()
+
+  private instanceIdToStateIdMap: Record<InstanceId, string> | null = null
+
+  private constructor(
+    public readonly project: ProjectOutput,
+    public readonly library: LibraryModel,
+    private readonly logger: Logger,
+  ) {}
+
+  public getInstance(instanceId: InstanceId): InstanceModel {
+    const instance = this.instanceMap.get(instanceId)
+    if (!instance) {
+      throw new Error(`Instance with ID ${instanceId} not found in the operation context`)
+    }
+
+    return instance
+  }
+
+  public getInstanceIds(): IterableIterator<InstanceId> {
+    return this.instanceMap.keys()
+  }
+
+  public getInstanceChildren(instanceId: InstanceId): InstanceModel[] {
+    return this.instanceChildrenMap.get(instanceId) ?? []
+  }
+
+  public getStateChildIds(instanceId: InstanceId): InstanceId[] {
+    return this.stateChildIdMap.get(instanceId) ?? []
+  }
+
+  public getResolvedInputs(
+    instanceId: InstanceId,
+  ): Record<string, ResolvedInstanceInput[]> | undefined {
+    return this.resolvedInstanceInputs.get(instanceId)
+  }
+
+  public setState(state: InstanceState): void {
+    this.stateMap.set(state.instanceId, state)
+
+    if (state.parentInstanceId) {
+      let children = this.stateChildIdMap.get(state.parentInstanceId)
+      if (!children) {
+        children = []
+        this.stateChildIdMap.set(state.parentInstanceId, children)
+      }
+
+      children.push(state.instanceId)
+    }
+
+    // traverse resolvedInputs to build dependency relationships
+    if (state.resolvedInputs) {
+      for (const inputGroup of Object.values(state.resolvedInputs)) {
+        for (const input of inputGroup) {
+          if (input.instanceId) {
+            this.addDependentState(state.instanceId, input.instanceId)
+          }
+        }
+      }
+    }
+  }
+
+  private addDependentState(instanceId: InstanceId, dependencyId: InstanceId): void {
+    let dependentStates = this.dependentStateIdMap.get(dependencyId)
+
+    if (!dependentStates) {
+      dependentStates = new Set<InstanceId>()
+      this.dependentStateIdMap.set(dependencyId, dependentStates)
+    }
+
+    dependentStates.add(instanceId)
+  }
+
+  public getState(instanceId: InstanceId): InstanceState {
+    const state = this.stateMap.get(instanceId)
+    if (!state) {
+      throw new Error(`Instance state for "${instanceId}" not found in the operation context`)
+    }
+
+    return state
+  }
+
+  public getDependentStates(instanceId: InstanceId): InstanceState[] {
+    const dependentStateIds = this.dependentStateIdMap.get(instanceId)
+    if (!dependentStateIds) {
+      return []
+    }
+
+    return Array.from(dependentStateIds)
+      .map(id => this.stateMap.get(id))
+      .filter((state): state is InstanceState => !!state)
+  }
+
+  public getDependencies(instanceId: InstanceId): InstanceModel[] {
+    const dependencies: InstanceModel[] = []
+    const resolvedInputs = this.resolvedInstanceInputs.get(instanceId)
+
+    if (resolvedInputs) {
+      for (const inputGroup of Object.values(resolvedInputs)) {
+        for (const resolvedInput of inputGroup) {
+          if (resolvedInput.input.instanceId && resolvedInput.input.instanceId !== instanceId) {
+            dependencies.push(this.getInstance(resolvedInput.input.instanceId))
+          }
+        }
+      }
+    }
+
+    return dependencies
+  }
+
+  private addInstance(instance: InstanceModel): void {
+    if (this.instanceMap.has(instance.id)) {
+      throw new Error(`Found multiple instances with the same ID: ${instance.id}`)
+    }
+
+    if (!(instance.type in this.library.components)) {
+      this.logger.warn(
+        `ignoring instance "${instance.id}" because its type "${instance.type}" is not in the library`,
+      )
+      return
+    }
+
+    this.instanceMap.set(instance.id, instance)
+
+    if (instance.parentId) {
+      let children = this.instanceChildrenMap.get(instance.parentId)
+      if (!children) {
+        children = []
+        this.instanceChildrenMap.set(instance.parentId, children)
+      }
+
+      children.push(instance)
+    }
+  }
+
+  private getSourceHashIfApplicable(
+    instance: InstanceModel,
+    component: ComponentModel,
+  ): number | undefined {
+    if (isUnitModel(component)) {
+      return this.unitSourceHashMap.get(instance.type)
+    }
+
+    return undefined
+  }
+
+  public getUpToDateInputHashOutput(instance: InstanceModel): Promise<InputHashOutput> {
+    return this.inputHashResolverLock.acquire(async () => {
+      const component = this.library.components[instance.type]
+
+      this.inputHashNodes.set(instance.id, {
+        instance,
+        component,
+        // biome-ignore lint/style/noNonNullAssertion: я разрешаю
+        resolvedInputs: this.resolvedInstanceInputs.get(instance.id)!,
+        state: this.stateMap.get(instance.id),
+        sourceHash: this.getSourceHashIfApplicable(instance, component),
+      })
+
+      this.inputHashResolver.invalidateSingle(instance.id)
+      await this.inputHashResolver.process()
+
+      return this.inputHashResolver.requireOutput(instance.id)
+    })
+  }
+
+  public setStates(states: InstanceState[]): void {
+    for (const state of states) {
+      this.setState(state)
+    }
+  }
+
+  public getInstanceIdToStateIdMap(): Record<InstanceId, string> {
+    if (this.instanceIdToStateIdMap) {
+      return this.instanceIdToStateIdMap
+    }
+
+    // build map on first request and cache it
+    const map: Record<InstanceId, string> = {}
+    for (const [instanceId, state] of this.stateMap.entries()) {
+      map[instanceId] = state.id
+    }
+
+    this.instanceIdToStateIdMap = map
+    return map
+  }
+
+  getUnfinishedOperationStates(): InstanceState[] {
+    const unfinishedStates: InstanceState[] = []
+    for (const state of this.stateMap.values()) {
+      if (isTransientInstanceOperationStatus(state.lastOperationState?.status)) {
+        unfinishedStates.push(state)
+      }
+    }
+
+    return unfinishedStates
+  }
+
+  public static async load(
+    projectId: string,
+    libraryBackend: LibraryBackend,
+    instanceStateService: InstanceStateService,
+    projectModelService: ProjectModelService,
+    logger: Logger,
+  ): Promise<OperationContext> {
+    const [{ instances, virtualInstances, hubs, ghostInstances }, project] =
+      await projectModelService.getProjectModel(projectId, {
+        includeVirtualInstances: true,
+        includeGhostInstances: true,
+      })
+
+    const [library, states] = await Promise.all([
+      libraryBackend.loadLibrary(project.libraryId),
+      instanceStateService.getInstanceStates(projectId, {
+        includeEvaluationState: true,
+        includeParentInstanceId: true,
+      }),
+    ])
+
+    const context = new OperationContext(
+      project,
+      library,
+      logger.child({ service: "OperationContext" }),
+    )
+
+    // prepare instances
+    for (const instance of instances) {
+      context.addInstance(instance)
+    }
+
+    for (const instance of ghostInstances) {
+      context.addInstance(instance)
+    }
+
+    for (const instance of virtualInstances) {
+      const contextInstance = context.instanceMap.get(instance.id)
+
+      if (contextInstance) {
+        // use evaluated inputs and outputs for real instances from their virtual counterparts
+        contextInstance.inputs = instance.resolvedInputs
+        contextInstance.outputs = instance.outputs
+        contextInstance.resolvedOutputs = instance.resolvedOutputs
+      } else if (instance.parentId) {
+        // always use resolved inputs for virtual instances
+        instance.inputs = instance.resolvedInputs
+        context.addInstance(instance)
+      } else {
+        context.logger.warn(
+          `ignoring virtual instance "${instance.id}" because it is not in the project or is not a part of known composite instance`,
+        )
+      }
+    }
+
+    const unitSources = await libraryBackend.getResolvedUnitSources(
+      project.libraryId,
+      unique(Array.from(context.instanceMap.values()).map(i => i.type)),
+    )
+
+    for (const unitSource of unitSources) {
+      context.unitSourceHashMap.set(unitSource.unitType, unitSource.sourceHash)
+    }
+
+    context.setStates(states)
+
+    // prepare input resolver
+    for (const instance of context.instanceMap.values()) {
+      context.inputResolverNodes.set(`instance:${instance.id}`, {
+        kind: "instance",
+        instance,
+        component: library.components[instance.type],
+      })
+    }
+
+    for (const hub of hubs) {
+      context.inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
+    }
+
+    context.inputResolver = new InputResolver(context.inputResolverNodes, logger)
+    context.inputResolver.addAllNodesToWorkset()
+
+    await context.inputResolver.process()
+
+    // resolve inputs for all instances and pass outputs to input hash resolver
+    for (const instance of context.instanceMap.values()) {
+      const output = context.inputResolver.requireOutput(`instance:${instance.id}`)
+      if (output.kind !== "instance") {
+        throw new Error("Unexpected output kind")
+      }
+
+      context.resolvedInstanceInputs.set(instance.id, output.resolvedInputs)
+
+      if (instance.kind === "unit") {
+        const component = context.library.components[instance.type]
+
+        context.inputHashNodes.set(instance.id, {
+          instance,
+          component,
+          resolvedInputs: output.resolvedInputs,
+          state: context.stateMap.get(instance.id),
+          sourceHash: context.getSourceHashIfApplicable(instance, component),
+        })
+      }
+    }
+
+    // prepare input hash resolver
+    context.inputHashResolver = new InputHashResolver(context.inputHashNodes, logger)
+    context.inputHashResolver.addAllNodesToWorkset()
+
+    await context.inputHashResolver.process()
+
+    return context
+  }
+}

package/src/orchestrator/operation-plan.destroy.test.md

@@ -0,0 +1,357 @@
+# Destroy Phase Examples
+
+**Legend:**
+
+- `✅` = Deployed instance
+- `🚀` = Explicitly requested instance
+- `👻` = Ghost instance (virtual)
+
+### Example 1: Simple Dependency Chain
+
+**Test**: `should include all dependents in linear chain when destroyDependentInstances enabled`
+
+```mermaid
+graph RL
+    A["A 🚀"]
+    B["B ✅"]
+    C["C ✅"]
+
+    C --> B --> A
+```
+
+**Options:**
+
+- `destroyDependentInstances`: `true` **(default)**
+
+**Decision Steps**:
+
+1. `A` explicitly requested;
+2. `B` depends on `A`, cascade enabled → `B` included (dependent of `A`);
+3. `C` depends on `B`, cascade enabled → `C` included (dependent of `B`).
+
+**Destroy Phase**: `C`, `B`, `A`
+
+### Example 2: Simple Dependency Chain with Cascade Disabled
+
+**Test**: `should not include dependents in linear chain when destroyDependentInstances disabled`
+
+```mermaid
+graph RL
+    A["A 🚀"]
+    B["B ✅"]
+    C["C ✅"]
+
+    C --> B --> A
+```
+
+**Options:**
+
+- `destroyDependentInstances`: `false`
+
+**Decision Steps**:
+
+1. `A` explicitly requested;
+2. `B` depends on `A`, cascade disabled → `B` excluded;
+3. `C` depends on `B`, cascade disabled → `C` excluded.
+
+**Destroy Phase**: `A`
+
+### Example 3: Simple Dependency Chain - Middle Node Requested
+
+**Test**: `should include all dependents when middle node requested with destroyDependentInstances enabled`
+
+```mermaid
+graph RL
+    A["A ✅"]
+    B["B 🚀"]
+    C["C ✅"]
+
+    C --> B --> A
+```
+
+**Decision Steps**:
+
+1. `B` explicitly requested;
+2. `C` depends on `B`, cascade enabled → `C` included (dependent of `B`);
+3. `A` does not depend on `B` → `A` excluded.
+
+**Destroy Phase**: `C`, `B`
+
+### Example 4: Composite Boundary Isolation
+
+**Test**: `should not propagate beyond compositional inclusion`
+
+```mermaid
+graph RL
+    subgraph GrandParent
+        subgraph Parent
+            A["A 🚀"]
+        end
+        B["B ✅"]
+        C["C ✅"]
+    end
+
+    B --> A
+```
+
+**Decision Steps**:
+
+1. `A` explicitly requested;
+2. `B` depends on `A`, cascade enabled → `B` included (dependent of `A`);
+3. `A` is child of `Parent` → `Parent` included (compositional);
+4. `Parent` is child of `GrandParent` → `GrandParent` NOT included (compositional boundary);
+5. `C` is sibling of `Parent` but `GrandParent` is not included → `C` NOT included.
+
+**Destroy Phase**: `B`, `A`, `Parent`
+
+### Example 5: Substantive Composite with Mixed Child States
+
+**Test**: `should include all children of substantive composite`
+
+```mermaid
+graph RL
+    subgraph Parent["Parent 🚀"]
+        Child1["Child1 ✅"]
+        Child2["Child2 ✅"]
+        Child3["Child3 ✅"]
+    end
+```
+
+**Decision Steps**:
+
+1. `Parent` explicitly requested (substantive composite);
+2. `Child1` is child of substantive composite → `Child1` included;
+3. `Child2` is child of substantive composite → `Child2` included;
+4. `Child3` is child of substantive composite → `Child3` included.
+
+**Destroy Phase**: `Child1`, `Child2`, `Child3`, `Parent`
+
+### Example 6: Nested Composites with Dependencies
+
+**Test**: `should handle complex nested hierarchy with dependencies`
+
+```mermaid
+graph RL
+    subgraph GrandParent
+        subgraph Parent1
+            Child1["Child1 🚀"]
+            Child2["Child2 ✅"]
+        end
+        subgraph Parent2
+            Child3["Child3 ✅"]
+        end
+    end
+
+    Child1 --> Child3
+```
+
+**Decision Steps**:
+
+1. `Child1` explicitly requested;
+2. No instances depend on `Child1` (Child1 depends on Child3, not vice versa);
+3. `Child1` is child of `Parent1` → `Parent1` included (compositional);
+4. `Parent1` is child of `GrandParent` → `GrandParent` NOT included (compositional boundary).
+
+**Destroy Phase**: `Child1`, `Parent1`
+
+### Example 7: Request Child with Isolated Destroy
+
+**Test**: `should not include siblings when child explicitly requested`
+
+```mermaid
+graph RL
+    subgraph Parent
+        Child1["Child1 🚀"]
+        Child2["Child2 ✅"]
+        Child3["Child3 ✅"]
+    end
+```
+
+**Decision Steps**:
+
+1. `Child1` explicitly requested;
+2. No instances depend on `Child1`;
+3. `Child1` is child of `Parent` → `Parent` included (compositional);
+4. `Child2` and `Child3` are siblings but not affected → excluded.
+
+**Destroy Phase**: `Child1`, `Parent`
+
+### Example 8: Cross-Composite Dependencies
+
+**Test**: `should handle dependencies crossing composite boundaries`
+
+```mermaid
+graph RL
+    subgraph CompositeA
+        ChildA["ChildA 🚀"]
+    end
+    subgraph CompositeB
+        ChildB["ChildB ✅"]
+    end
+
+    ChildB --> ChildA
+```
+
+**Decision Steps**:
+
+1. `ChildA` explicitly requested;
+2. `ChildB` depends on `ChildA`, cascade enabled → `ChildB` included (dependent of `ChildA`);
+3. `ChildA` is child of `CompositeA` → `CompositeA` included (compositional);
+4. `ChildB` is child of `CompositeB`, included due to external dependency → `CompositeB` included (substantive).
+
+**Destroy Phase**: `ChildB`, `CompositeB`, `ChildA`, `CompositeA`
+
+### Example 9: Unrelated Instance Isolation
+
+**Test**: `should not include unrelated instances that don't depend on destroyed instance`
+
+```mermaid
+graph RL
+    subgraph Parent
+        Child1["Child1 ✅"]
+        Child2["Child2 ✅"]
+    end
+    UnrelatedX["UnrelatedX ✅"]
+    ExternalY["ExternalY 🚀"]
+
+    Child1 --> ExternalY
+    Child2 --> UnrelatedX
+```
+
+**Decision Steps**:
+
+1. `ExternalY` explicitly requested;
+2. `Child1` depends on `ExternalY`, cascade enabled → `Child1` included (dependent of `ExternalY`);
+3. `Child1` is child of `Parent`, included due to external dependency → `Parent` becomes substantive;
+4. `Child2` is child of substantive composite → `Child2` included;
+5. `UnrelatedX` does not depend on any destroyed instance → `UnrelatedX` excluded.
+
+**Destroy Phase**: `Child1`, `Child2`, `Parent`, `ExternalY`
+
+### Example 10: Multiple Explicit Requests
+
+**Test**: `should handle multiple explicit requests with overlapping dependencies`
+
+```mermaid
+graph RL
+    A["A 🚀"]
+    B["B ✅"]
+    C["C 🚀"]
+
+    C --> B --> A
+```
+
+**Decision Steps**:
+
+1. `A` explicitly requested;
+2. `C` explicitly requested;
+3. `B` depends on `A`, cascade enabled → `B` included (dependent of `A`);
+4. `C` depends on `B`, `B` already included → no additional change.
+
+**Destroy Phase**: `C`, `B`, `A`
+
+### Example 11: Deep Nesting with Boundary Isolation
+
+**Test**: `should isolate boundaries in deep composite hierarchies`
+
+```mermaid
+graph RL
+    subgraph GreatGrandParent
+        subgraph GrandParent
+            subgraph Parent
+                Child["Child 🚀"]
+            end
+            Uncle["Uncle ✅"]
+        end
+        GreatUncle["GreatUncle ✅"]
+    end
+```
+
+**Decision Steps**:
+
+1. `Child` explicitly requested;
+2. No instances depend on `Child`;
+3. `Child` is child of `Parent` → `Parent` included (compositional);
+4. `Parent` is child of `GrandParent` → `GrandParent` NOT included (compositional boundary);
+5. `Uncle` and `GreatUncle` not affected.
+
+**Destroy Phase**: `Child`, `Parent`
+
+### Example 12: Diamond Dependency Pattern
+
+**Test**: `should handle diamond dependency correctly`
+
+```mermaid
+graph RL
+    A["A 🚀"]
+    B["B ✅"]
+    C["C ✅"]
+    D["D ✅"]
+
+    D --> B
+    D --> C
+    B --> A
+    C --> A
+```
+
+**Decision Steps**:
+
+1. `A` explicitly requested;
+2. `B` depends on `A`, cascade enabled → `B` included (dependent of `A`);
+3. `C` depends on `A`, cascade enabled → `C` included (dependent of `A`);
+4. `D` depends on `B`, cascade enabled → `D` included (dependent of `B`);
+5. `D` also depends on `C`, already included → no additional change.
+
+**Destroy Phase**: `D`, `B`, `C`, `A`
+
+### Example 13: Dependency Chain with Partial Destruction Disabled
+
+**Test**: `should include dependency chain and force siblings when partial destruction disabled`
+
+```mermaid
+graph RL
+    subgraph Parent
+        Child1["Child1 ✅"]
+        Child2["Child2 ✅"]
+    end
+    ExternalX["ExternalX 🚀"]
+
+    Child1 --> ExternalX
+```
+
+**Decision Steps**:
+
+1. `ExternalX` explicitly requested;
+2. `Child1` depends on `ExternalX`, cascade enabled → `Child1` included (dependent of `ExternalX`);
+3. `Parent` has child included due to external dependency, so `Parent` becomes substantive composite;
+4. `Child2` is sibling of `Child1` in substantive composite, partial destruction disabled → `Child2` included.
+
+**Destroy Phase**: `Child1`, `Child2`, `Parent`, `ExternalX`
+
+### Example 14: Dependency Chain with Partial Destruction Enabled
+
+**Test**: `should include dependency chain without forcing siblings when partial destruction enabled`
+
+```mermaid
+graph RL
+    subgraph Parent
+        Child1["Child1 ✅"]
+        Child2["Child2 ✅"]
+    end
+    ExternalX["ExternalX 🚀"]
+
+    Child1 --> ExternalX
+```
+
+**Options:**
+
+- `allowPartialCompositeInstanceDestruction`: `true`
+
+**Decision Steps**:
+
+1. `ExternalX` explicitly requested;
+2. `Child1` depends on `ExternalX`, cascade enabled → `Child1` included (dependent of `ExternalX`);
+3. `Parent` has child included due to external dependency, so `Parent` becomes substantive composite;
+4. `Child2` is sibling of `Child1` in substantive composite, partial destruction enabled → `Child2` excluded.
+
+**Destroy Phase**: `Child1`, `Parent`, `ExternalX`