@highstate/backend 0.19.1 → 0.21.1

package/src/orchestrator/operation-context.captured-outputs.test.ts

@@ -0,0 +1,118 @@
+import pino from "pino"
+import { describe, expect, test } from "vitest"
+import { OperationContext } from "./operation-context"
+
+describe("OperationContext.updateCapturedOutputValuesFromUnitOutputs", () => {
+  test("captures single + multiple outputs and clears missing ones", () => {
+    const context = new (
+      OperationContext as unknown as {
+        new (...args: unknown[]): OperationContext
+      }
+    )(
+      { id: "project" },
+      {
+        components: {
+          "component.v1": {
+            outputs: {
+              single: { type: "entity.single.v1" },
+              multi: { type: "entity.multi.v1", multiple: true },
+            },
+          },
+        },
+        entities: {},
+      },
+      pino({ level: "silent" }),
+    )
+
+    context.updateCapturedOutputValuesFromUnitOutputs({
+      instanceId: "component.v1:one",
+      instanceType: "component.v1",
+      outputs: {
+        single: { value: { a: 1 } },
+        multi: { value: [{ b: 2 }, { b: 3 }] },
+      },
+    })
+
+    expect(context.getCapturedOutputValues("component.v1:one", "single")).toEqual([
+      { ok: true, value: { a: 1 } },
+    ])
+    expect(context.getCapturedOutputValues("component.v1:one", "multi")).toEqual([
+      { ok: true, value: { b: 2 } },
+      { ok: true, value: { b: 3 } },
+    ])
+
+    context.updateCapturedOutputValuesFromUnitOutputs({
+      instanceId: "component.v1:one",
+      instanceType: "component.v1",
+      outputs: {
+        single: { value: null },
+        multi: { value: undefined },
+      },
+    })
+
+    expect(context.getCapturedOutputValues("component.v1:one", "single")).toEqual([])
+    expect(context.getCapturedOutputValues("component.v1:one", "multi")).toEqual([])
+  })
+
+  test("throws when multiple output is not an array", () => {
+    const context = new (
+      OperationContext as unknown as {
+        new (...args: unknown[]): OperationContext
+      }
+    )(
+      { id: "project" },
+      {
+        components: {
+          "component.v1": {
+            outputs: {
+              multi: { type: "entity.multi.v1", multiple: true },
+            },
+          },
+        },
+        entities: {},
+      },
+      pino({ level: "silent" }),
+    )
+
+    expect(() =>
+      context.updateCapturedOutputValuesFromUnitOutputs({
+        instanceId: "component.v1:one",
+        instanceType: "component.v1",
+        outputs: {
+          multi: { value: { b: 1 } },
+        },
+      }),
+    ).toThrow('Output "multi" for instance "component.v1:one" must be an array')
+  })
+
+  test("throws when output item is not an object", () => {
+    const context = new (
+      OperationContext as unknown as {
+        new (...args: unknown[]): OperationContext
+      }
+    )(
+      { id: "project" },
+      {
+        components: {
+          "component.v1": {
+            outputs: {
+              multi: { type: "entity.multi.v1", multiple: true },
+            },
+          },
+        },
+        entities: {},
+      },
+      pino({ level: "silent" }),
+    )
+
+    expect(() =>
+      context.updateCapturedOutputValuesFromUnitOutputs({
+        instanceId: "component.v1:one",
+        instanceType: "component.v1",
+        outputs: {
+          multi: { value: [123] },
+        },
+      }),
+    ).toThrow('Output "multi" for instance "component.v1:one" must contain objects')
+  })
+})

package/src/orchestrator/operation-context.ts

@@ -1,5 +1,10 @@
 import type { Logger } from "pino"
-import type { InstanceStateService, ProjectModelService } from "../business"
+import type {
+  CapturedEntitySnapshotValue,
+  EntitySnapshotService,
+  InstanceStateService,
+  ProjectModelService,
+} from "../business"
 import type { LibraryBackend } from "../library"
 import {
   type ComponentModel,
@@ -7,6 +12,8 @@ import {
   type InstanceInput,
   type InstanceModel,
   isUnitModel,
+  parseInstanceId,
+  type VersionedName,
 } from "@highstate/contract"
 import { BetterLock } from "better-lock"
 import { mapValues, unique } from "remeda"
@@ -24,6 +31,8 @@ import {
   type StableInstanceInput,
 } from "../shared"
 
+type RawPulumiOutputs = Record<string, { value: unknown; secret?: boolean }>
+
 export class OperationContext {
   private readonly instanceMap = new Map<InstanceId, InstanceModel>()
   private readonly instanceChildrenMap = new Map<InstanceId, InstanceModel[]>()
@@ -48,6 +57,8 @@ export class OperationContext {
     Record<InstanceId, ResolvedInstanceInput[]>
   >()
 
+  private readonly capturedOutputValueMap = new Map<string, CapturedEntitySnapshotValue[]>()
+
   private constructor(
     public readonly project: ProjectOutput,
     public readonly library: LibraryModel,
@@ -55,7 +66,7 @@ export class OperationContext {
   ) {}
 
   public getInstance(instanceId: InstanceId): InstanceModel {
-    const instance = this.instanceMap.get(instanceId)
+    const instance = this.tryGetInstance(instanceId)
     if (!instance) {
       throw new Error(`Instance with ID ${instanceId} not found in the operation context`)
     }
@@ -63,6 +74,42 @@ export class OperationContext {
     return instance
   }
 
+  public tryGetInstance(instanceId: InstanceId): InstanceModel | undefined {
+    return this.instanceMap.get(instanceId)
+  }
+
+  public tryGetInstanceForDestroy(instanceId: InstanceId): InstanceModel | undefined {
+    const modelInstance = this.tryGetInstance(instanceId)
+    if (modelInstance) {
+      return modelInstance
+    }
+
+    const state = this.stateMap.get(instanceId)
+    if (!state) {
+      return undefined
+    }
+
+    const stateModel = state.model ?? state.lastOperationState?.model
+    if (stateModel) {
+      return stateModel
+    }
+
+    const [type, name] = parseInstanceId(instanceId)
+
+    return {
+      id: instanceId,
+      name,
+      type,
+      kind: state.kind,
+      parentId: state.parentInstanceId ?? undefined,
+      inputs: {},
+      args: {},
+      outputs: {},
+      resolvedInputs: {},
+      resolvedOutputs: {},
+    }
+  }
+
   public isGhostInstance(instanceId: InstanceId): boolean {
     return this.ghostInstanceIds.has(instanceId)
   }
@@ -85,6 +132,56 @@ export class OperationContext {
     return this.resolvedInstanceInputs.get(instanceId)
   }
 
+  public getCapturedOutputValues(
+    instanceId: InstanceId,
+    output: string,
+  ): CapturedEntitySnapshotValue[] {
+    const key = `${instanceId}:${output}`
+    return this.capturedOutputValueMap.get(key) ?? []
+  }
+
+  public updateCapturedOutputValuesFromUnitOutputs(options: {
+    instanceId: InstanceId
+    instanceType: VersionedName
+    outputs: RawPulumiOutputs
+  }): void {
+    const component = this.library.components[options.instanceType]
+    if (!component) {
+      return
+    }
+
+    for (const [outputName, outputSpec] of Object.entries(component.outputs ?? {})) {
+      const raw = options.outputs[outputName]?.value
+
+      if (raw === undefined || raw === null) {
+        this.capturedOutputValueMap.set(`${options.instanceId}:${outputName}`, [])
+        continue
+      }
+
+      const items = outputSpec.multiple ? raw : [raw]
+      if (outputSpec.multiple && !Array.isArray(raw)) {
+        throw new Error(
+          `Output "${outputName}" for instance "${options.instanceId}" must be an array`,
+        )
+      }
+
+      const values = (items as unknown[]).map(item => {
+        if (typeof item !== "object" || item === null || Array.isArray(item)) {
+          throw new Error(
+            `Output "${outputName}" for instance "${options.instanceId}" must contain objects`,
+          )
+        }
+
+        return {
+          ok: true,
+          value: item as Record<string, unknown>,
+        } satisfies CapturedEntitySnapshotValue
+      })
+
+      this.capturedOutputValueMap.set(`${options.instanceId}:${outputName}`, values)
+    }
+  }
+
   public setState(state: InstanceState): void {
     this.stateMap.set(state.instanceId, state)
     this.stateIdMap.set(state.id, state.instanceId)
@@ -127,6 +224,7 @@ export class OperationContext {
         return {
           stateId: state.id,
           output: input.output,
+          path: input.path,
         }
       }),
     )
@@ -247,22 +345,49 @@ export class OperationContext {
     }
   }
 
-  /**
-   * Gets a map of instance IDs to state IDs for dependencies of the given instance.
-   *
-   * @param instanceId The instance ID to get the state ID map for.
-   * @returns A map of instance IDs to state IDs.
-   */
-  public getInstanceIdToStateIdMap(instanceId: InstanceId): Record<InstanceId, string> {
-    const map: Record<InstanceId, string> = {}
-    const dependencies = this.getDependencies(instanceId).map(i => i.id)
+  private async captureEntitySnapshotsAtOperationStart(options: {
+    projectId: string
+    entitySnapshotService: EntitySnapshotService
+  }): Promise<void> {
+    const keys: {
+      stateId: string
+      output: string
+      instanceId: InstanceId
+      operationId?: string
+    }[] = []
+
+    for (const [instanceId, inputs] of this.resolvedInstanceInputs.entries()) {
+      for (const inputGroup of Object.values(inputs ?? {})) {
+        for (const input of inputGroup) {
+          if (input.input.instanceId === instanceId) {
+            continue
+          }
 
-    for (const dep of dependencies) {
-      const state = this.getState(dep)
-      map[dep] = state.id
+          const dependencyState = this.getState(input.input.instanceId)
+          keys.push({
+            stateId: dependencyState.id,
+            output: input.input.output,
+            instanceId: input.input.instanceId,
+            operationId: dependencyState.lastOperationState?.operationId,
+          })
+        }
+      }
+    }
+
+    if (keys.length === 0) {
+      return
     }
 
-    return map
+    const captured = await options.entitySnapshotService.reconstructLatestExportedOutputValues(
+      options.projectId,
+      keys.map(k => ({ stateId: k.stateId, output: k.output, operationId: k.operationId })),
+      this.library,
+    )
+
+    for (const key of keys) {
+      const snapshotValues = captured.get(`${key.stateId}:${key.output}`) ?? []
+      this.capturedOutputValueMap.set(`${key.instanceId}:${key.output}`, snapshotValues)
+    }
   }
 
   getUnfinishedOperationStates(): InstanceState[] {
@@ -281,6 +406,7 @@ export class OperationContext {
     libraryBackend: LibraryBackend,
     instanceStateService: InstanceStateService,
     projectModelService: ProjectModelService,
+    entitySnapshotService: EntitySnapshotService | undefined,
     logger: Logger,
   ): Promise<OperationContext> {
     const [{ instances, virtualInstances, hubs, ghostInstances }, project] =
@@ -348,11 +474,16 @@ export class OperationContext {
         kind: "instance",
         instance,
         component: library.components[instance.type],
+        entities: library.entities,
       })
     }
 
     for (const hub of hubs) {
-      context.inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
+      context.inputResolverNodes.set(`hub:${hub.id}`, {
+        kind: "hub",
+        hub,
+        entities: library.entities,
+      })
     }
 
     context.inputResolver = new InputResolver(context.inputResolverNodes, logger)
@@ -388,6 +519,13 @@ export class OperationContext {
 
     await context.inputHashResolver.process()
 
+    if (entitySnapshotService) {
+      await context.captureEntitySnapshotsAtOperationStart({
+        projectId,
+        entitySnapshotService,
+      })
+    }
+
     return context
   }
 }

package/src/orchestrator/operation-plan.destroy.test.md

@@ -77,9 +77,9 @@ graph RL
 
 **Destroy Phase**: `C`, `B`
 
-### Example 4: Composite Boundary Isolation
+### Example 4: Full Ancestor Chain for Compositional Inclusion
 
-**Test**: `should not propagate beyond compositional inclusion`
+**Test**: `should include full ancestor chain for compositional inclusion`
 
 ```mermaid
 graph RL
@@ -99,10 +99,10 @@ graph RL
 1. `A` explicitly requested;
 2. `B` depends on `A`, cascade enabled → `B` included (dependent of `A`);
 3. `A` is child of `Parent` → `Parent` included (compositional);
-4. `Parent` is child of `GrandParent` → `GrandParent` NOT included (compositional boundary);
-5. `C` is sibling of `Parent` but `GrandParent` is not included → `C` NOT included.
+4. `Parent` is child of `GrandParent` → `GrandParent` included as ancestor;
+5. `C` is sibling of `Parent`, but ancestor inclusion does not auto-include siblings → `C` NOT included.
 
-**Destroy Phase**: `B`, `A`, `Parent`
+**Destroy Phase**: `B`, `A`, `Parent`, `GrandParent`
 
 ### Example 5: Substantive Composite with Mixed Child States
 
@@ -150,9 +150,9 @@ graph RL
 1. `Child1` explicitly requested;
 2. No instances depend on `Child1` (Child1 depends on Child3, not vice versa);
 3. `Child1` is child of `Parent1` → `Parent1` included (compositional);
-4. `Parent1` is child of `GrandParent` → `GrandParent` NOT included (compositional boundary).
+4. `Parent1` is child of `GrandParent` → `GrandParent` included as ancestor.
 
-**Destroy Phase**: `Child1`, `Parent1`
+**Destroy Phase**: `Child1`, `Parent1`, `GrandParent`
 
 ### Example 7: Request Child with Isolated Destroy
 
@@ -250,9 +250,9 @@ graph RL
 
 **Destroy Phase**: `C`, `B`, `A`
 
-### Example 11: Deep Nesting with Boundary Isolation
+### Example 11: Deep Nesting with Ancestor Chain Inclusion
 
-**Test**: `should isolate boundaries in deep composite hierarchies`
+**Test**: `should include deep ancestor chain without ancestor siblings`
 
 ```mermaid
 graph RL
@@ -272,10 +272,11 @@ graph RL
 1. `Child` explicitly requested;
 2. No instances depend on `Child`;
 3. `Child` is child of `Parent` → `Parent` included (compositional);
-4. `Parent` is child of `GrandParent` → `GrandParent` NOT included (compositional boundary);
-5. `Uncle` and `GreatUncle` not affected.
+4. `Parent` is child of `GrandParent` → `GrandParent` included as ancestor;
+5. `GrandParent` is child of `GreatGrandParent` → `GreatGrandParent` included as ancestor;
+6. `Uncle` and `GreatUncle` are ancestor siblings and are not auto-included.
 
-**Destroy Phase**: `Child`, `Parent`
+**Destroy Phase**: `Child`, `Parent`, `GrandParent`, `GreatGrandParent`
 
 ### Example 12: Diamond Dependency Pattern
 
@@ -355,3 +356,23 @@ graph RL
 4. `Child2` is sibling of `Child1` in substantive composite, partial destruction enabled → `Child2` excluded.
 
 **Destroy Phase**: `Child1`, `Parent`, `ExternalX`
+
+### Example 15: State-Only Dependent Missing from Model
+
+**Test**: `should include state-only dependents when model instance is missing`
+
+```mermaid
+graph RL
+    Requested["requested 🚀"]
+    Dependent["dependent ✅ (state-only)"]
+
+    Dependent --> Requested
+```
+
+**Decision Steps**:
+
+1. `requested` explicitly requested;
+2. `dependent` is not present in current project model, but exists in state with a dependency on `requested`;
+3. destroy cascade uses state relationships, so `dependent` is included as dependent of destroyed `requested`.
+
+**Destroy Phase**: `dependent`, `requested`

package/src/orchestrator/operation-plan.destroy.test.ts

@@ -1,3 +1,5 @@
+import type { InstanceState } from "../shared"
+import { getInstanceId } from "@highstate/contract"
 import { describe } from "vitest"
 import { createOperationPlan } from "./operation-plan"
 import { operationPlanTest } from "./operation-plan.fixtures"
@@ -140,7 +142,7 @@ describe("OperationPlan - Destroy Operations", () => {
   )
 
   operationPlanTest(
-    "4. should not propagate beyond compositional inclusion",
+    "4. should include full ancestor chain for compositional inclusion",
     async ({ testBuilder, expect }) => {
       // arrange
       const { context, operation } = await testBuilder()
@@ -190,6 +192,11 @@ describe("OperationPlan - Destroy Operations", () => {
                 "message": "parent of included child "component.v1:A"",
                 "parentId": "composite.v1:GrandParent",
               },
+              {
+                "id": "composite.v1:GrandParent",
+                "message": "parent of included child "composite.v1:Parent"",
+                "parentId": undefined,
+              },
             ],
             "type": "destroy",
           },
@@ -307,6 +314,11 @@ describe("OperationPlan - Destroy Operations", () => {
                 "message": "parent of included child "component.v1:Child1"",
                 "parentId": "composite.v1:GrandParent",
               },
+              {
+                "id": "composite.v1:GrandParent",
+                "message": "parent of included child "composite.v1:Parent1"",
+                "parentId": undefined,
+              },
             ],
             "type": "destroy",
           },
@@ -542,7 +554,7 @@ describe("OperationPlan - Destroy Operations", () => {
   )
 
   operationPlanTest(
-    "11. should isolate boundaries in deep composite hierarchies",
+    "11. should include deep ancestor chain without ancestor siblings",
     async ({ testBuilder, expect }) => {
       // arrange
       const { context, operation } = await testBuilder()
@@ -589,6 +601,16 @@ describe("OperationPlan - Destroy Operations", () => {
                 "message": "parent of included child "component.v1:Child"",
                 "parentId": "composite.v1:GrandParent",
               },
+              {
+                "id": "composite.v1:GrandParent",
+                "message": "parent of included child "composite.v1:Parent"",
+                "parentId": "composite.v1:GreatGrandParent",
+              },
+              {
+                "id": "composite.v1:GreatGrandParent",
+                "message": "parent of included child "composite.v1:GrandParent"",
+                "parentId": undefined,
+              },
             ],
             "type": "destroy",
           },
@@ -772,4 +794,120 @@ describe("OperationPlan - Destroy Operations", () => {
       `)
     },
   )
+
+  operationPlanTest(
+    "15. should include state-only dependents when model instance is missing",
+    async ({ createContext, createTestOperation, expect }) => {
+      // arrange
+      const requestedId = getInstanceId("component.v1", "requested")
+      const dependentId = getInstanceId("component.v1", "dependent")
+
+      const context = await createContext(
+        [
+          {
+            id: requestedId,
+            name: "requested",
+            type: "component.v1",
+            kind: "unit",
+            parentId: undefined,
+            inputs: {},
+            args: {},
+            outputs: {},
+            resolvedInputs: {},
+            resolvedOutputs: {},
+          },
+        ],
+        [
+          {
+            id: requestedId,
+            instanceId: requestedId,
+            status: "deployed",
+            source: "resident",
+            kind: "unit",
+            hasResourceHooks: false,
+            parentId: null,
+            parentInstanceId: null,
+            selfHash: null,
+            inputHash: null,
+            outputHash: null,
+            dependencyOutputHash: null,
+            statusFields: null,
+            exportedArtifactIds: null,
+            inputHashNonce: null,
+            currentResourceCount: null,
+            model: null,
+            resolvedInputs: null,
+            lastOperationState: undefined,
+            evaluationState: {} as InstanceState["evaluationState"],
+          },
+          {
+            id: dependentId,
+            instanceId: dependentId,
+            status: "deployed",
+            source: "resident",
+            kind: "unit",
+            hasResourceHooks: false,
+            parentId: null,
+            parentInstanceId: null,
+            selfHash: null,
+            inputHash: null,
+            outputHash: null,
+            dependencyOutputHash: null,
+            statusFields: null,
+            exportedArtifactIds: null,
+            inputHashNonce: null,
+            currentResourceCount: null,
+            model: {
+              id: dependentId,
+              name: "dependent",
+              type: "component.v1",
+              kind: "unit",
+              parentId: undefined,
+              inputs: {},
+              args: {},
+              outputs: {},
+              resolvedInputs: {},
+              resolvedOutputs: {},
+            },
+            resolvedInputs: {
+              dependency: [{ stateId: requestedId, output: "default" }],
+            },
+            lastOperationState: undefined,
+            evaluationState: {} as InstanceState["evaluationState"],
+          },
+        ],
+      )
+
+      const operation = createTestOperation("destroy", [requestedId])
+
+      // act
+      const plan = createOperationPlan(
+        context,
+        operation.type,
+        operation.requestedInstanceIds,
+        operation.options,
+      )
+
+      // assert
+      expect(plan).toMatchInlineSnapshot(`
+        [
+          {
+            "instances": [
+              {
+                "id": "component.v1:dependent",
+                "message": "dependent of destroyed "component.v1:requested"",
+                "parentId": undefined,
+              },
+              {
+                "id": "component.v1:requested",
+                "message": "explicitly requested",
+                "parentId": undefined,
+              },
+            ],
+            "type": "destroy",
+          },
+        ]
+      `)
+    },
+  )
 })

package/src/orchestrator/operation-plan.fixtures.ts

@@ -85,6 +85,7 @@ export const operationPlanTest = test.extend<{
       options: {
         forceUpdateDependencies: false,
         ignoreDependencies: false,
+        ignoreChangedDependencies: false,
         forceUpdateChildren: false,
         destroyDependentInstances: true,
         invokeDestroyTriggers: true,
@@ -197,6 +198,7 @@ export const operationPlanTest = test.extend<{
         libraryBackend,
         instanceStateService,
         projectModelService,
+        undefined,
         logger,
       )
     }

package/src/orchestrator/operation-plan.md

@@ -41,10 +41,12 @@ Different operation types are assembled from phases as follows:
 - Then an `update phase` containing the same instances to be recreated
 
 **Preview Operation:**
+
 - Single `update phase` calculated using update phase rules but not executed
 - **Restriction**: Only allowed for "edge" instances (instances that depend on others but no instances depend on them)
 
 **Refresh Operation:**
+
 - Single `refresh phase` calculated using update phase rules for state refresh
 - **Key difference**: No destroy phase is created, even if ghost cleanup would normally occur
 
@@ -108,7 +110,8 @@ Since composite instances cannot be directly depended upon, message conflicts ar
 ### Propagation Rules
 
 - **Substantive composite inclusions** trigger further dependency resolution and parent propagation
-- **Compositional composite inclusions** do NOT trigger further propagation (boundary isolation)
+- **Compositional composite inclusions** still propagate upward through the full parent chain
+- **Ancestor siblings are not auto-included** unless they are independently required by other rules
 
 ### Ghost Cleanup Rules