@highstate/backend 0.18.0 → 0.20.0

package/src/orchestrator/operation-workset.ts

@@ -1,16 +1,16 @@
 import type { InstanceId } from "@highstate/contract"
 import type { InstanceStateService, UpdateOperationStateOptions } from "../business"
 import type { InstanceOperationStatus } from "../database"
+import type {
+  InstanceStatus,
+  OperationPhase,
+  OperationPhaseType,
+  OperationType,
+  ProjectOutput,
+} from "../shared"
 import type { OperationContext } from "./operation-context"
 import { EventEmitter, on } from "node:events"
 import { mapValues } from "remeda"
-import {
-  type InstanceStatus,
-  isTransientInstanceOperationStatus,
-  type OperationPhase,
-  type OperationPhaseType,
-  type ProjectOutput,
-} from "../shared"
 
 type AbortControllerPair = {
   abortController: AbortController
@@ -35,6 +35,7 @@ export class OperationWorkset {
   constructor(
     readonly project: ProjectOutput,
     readonly operationId: string,
+    readonly operationType: OperationType,
     readonly phases: OperationPhase[],
     private readonly context: OperationContext,
     private readonly instanceStateService: InstanceStateService,
@@ -117,16 +118,33 @@ export class OperationWorkset {
     const patches = await this.instanceStateService.createOperationStates(
       this.project.id,
       Array.from(this.allAffectedInstanceIds).map(instanceId => {
-        const instance = this.context.getInstance(instanceId)
         const state = this.context.getState(instanceId)
-
-        const resolvedInputs = mapValues(
-          //
-          this.context.getResolvedInputs(instance.id) ?? {},
-          inputs => inputs.map(input => input.input),
+        const hasDestroyPhase = this.phases.some(
+          phase =>
+            phase.type === "destroy" &&
+            phase.instances.some(instance => instance.id === instanceId),
         )
 
-        const serializedResolvedInputs = this.context.serializeResolvedInputs(resolvedInputs)
+        // destroy plans may reference state-only instances that are no longer in the project model
+        const instance = hasDestroyPhase
+          ? (this.context.tryGetInstance(instanceId) ??
+            this.context.tryGetInstanceForDestroy(instanceId))
+          : this.context.tryGetInstance(instanceId)
+
+        if (!instance) {
+          throw new Error(`Instance with ID ${instanceId} not found in the operation context`)
+        }
+
+        const serializedResolvedInputs =
+          this.context.getResolvedInputs(instance.id) != null
+            ? this.context.serializeResolvedInputs(
+                mapValues(
+                  //
+                  this.context.getResolvedInputs(instance.id) ?? {},
+                  inputs => inputs.map(input => input.input),
+                ),
+              )
+            : (state.resolvedInputs ?? {})
 
         return [
           {
@@ -155,6 +173,10 @@ export class OperationWorkset {
   async updateState(instanceId: InstanceId, options: UpdateOperationStateOptions): Promise<void> {
     const state = this.context.getState(instanceId)
 
+    if (state.lastOperationState?.operationId !== this.operationId) {
+      return
+    }
+
     const patch = await this.instanceStateService.updateOperationState(
       this.project.id,
       state.id,
@@ -164,9 +186,11 @@ export class OperationWorkset {
 
     Object.assign(state, patch)
 
-    if (state.parentInstanceId && this.currentPhase !== "preview") {
+    const parentInstanceId = this.getPhaseParentId(instanceId)
+
+    if (parentInstanceId && this.currentPhase !== "preview") {
       // TODO: update all updates in single transaction
-      await this.recalculateCompositeInstanceState(state.parentInstanceId)
+      await this.recalculateCompositeInstanceState(parentInstanceId)
     }
   }
 
@@ -221,12 +245,6 @@ export class OperationWorkset {
 
     await this.updateState(instanceId, {
       operationState: {
-        status:
-          !state.lastOperationState?.status ||
-          // do not override final statuses
-          isTransientInstanceOperationStatus(state.lastOperationState.status)
-            ? this.getTransientStatusByOperationPhase()
-            : state.lastOperationState.status,
         currentResourceCount,
         totalResourceCount: finalTotalResourceCount,
       },
@@ -271,6 +289,7 @@ export class OperationWorkset {
 
   getNextStableInstanceStatus(instanceId: InstanceId): InstanceStatus {
     const state = this.context.getState(instanceId)
+    const modelInstance = this.context.tryGetInstance(instanceId)
 
     switch (this.currentPhase) {
       case "preview":
@@ -278,12 +297,43 @@ export class OperationWorkset {
       case "update":
         return "deployed"
       case "destroy":
+        // update operations may include a destroy phase only for ghost cleanup
+        // keep composite containers deployed only while they still have unit descendants
+        if (this.operationType === "update" && modelInstance?.kind === "composite") {
+          return this.hasActiveUnitDescendant(instanceId) ? state.status : "undeployed"
+        }
+
         return "undeployed"
       case "refresh":
         return state.status // do not change instance status when refreshing
     }
   }
 
+  private hasActiveUnitDescendant(instanceId: InstanceId): boolean {
+    const queue = [...this.context.getStateChildIds(instanceId)]
+    const visited = new Set<InstanceId>()
+
+    while (queue.length > 0) {
+      const childId = queue.shift()
+      if (!childId || visited.has(childId)) {
+        continue
+      }
+
+      visited.add(childId)
+
+      const childState = this.context.getState(childId)
+      if (childState.kind === "unit" && childState.status !== "undeployed") {
+        return true
+      }
+
+      if (childState.kind === "composite") {
+        queue.push(...this.context.getStateChildIds(childId))
+      }
+    }
+
+    return false
+  }
+
   setupAbortControllersForAllInstances(): void {
     for (const instanceId of this.allAffectedInstanceIds) {
       this.setupInstanceAbortControllers(instanceId)

package/src/orchestrator/operation.cancel.test.ts

@@ -20,6 +20,8 @@ describe("Operation - Cancel", () => {
       instanceStateService,
       projectModelService,
       unitExtraService,
+      entitySnapshotService,
+      unitOutputService,
       createUnit,
       createDeployedUnitState,
       createOperation,
@@ -86,6 +88,8 @@ describe("Operation - Cancel", () => {
         instanceStateService,
         projectModelService,
         unitExtraService,
+        entitySnapshotService,
+        unitOutputService,
         logger,
       )
 

package/src/orchestrator/operation.composite.test.ts

@@ -18,6 +18,8 @@ describe("Operation - Composite", () => {
       instanceStateService,
       projectModelService,
       unitExtraService,
+      entitySnapshotService,
+      unitOutputService,
       createComposite,
       createUnit,
       createDeployedUnitState,
@@ -79,6 +81,8 @@ describe("Operation - Composite", () => {
         instanceStateService,
         projectModelService,
         unitExtraService,
+        entitySnapshotService,
+        unitOutputService,
         logger,
       )
 
@@ -120,4 +124,341 @@ describe("Operation - Composite", () => {
       )
     },
   )
+
+  operationTest(
+    "recalculates using phase parent when state parent is stale and outside operation",
+    async ({
+      project,
+      logger,
+      runnerBackend,
+      libraryBackend,
+      artifactService,
+      instanceLockService,
+      operationService,
+      secretService,
+      instanceStateService,
+      projectModelService,
+      unitExtraService,
+      entitySnapshotService,
+      unitOutputService,
+      createComposite,
+      createUnit,
+      createDeployedUnitState,
+      createOperation,
+      createContext,
+      setupPersistenceMocks,
+      setupImmediateLocking,
+      expect,
+    }) => {
+      // arrange
+      const grandParent = createComposite("GrandParent")
+      const parent = {
+        ...createComposite("Parent"),
+        parentId: grandParent.id,
+      }
+      const oldParent = createComposite("OldParent")
+      const child = {
+        ...createUnit("Child"),
+        parentId: parent.id,
+      }
+
+      const grandParentState = createDeployedUnitState(grandParent)
+      const parentState = createDeployedUnitState(parent)
+      const oldParentState = createDeployedUnitState(oldParent)
+      const childState = createDeployedUnitState(child)
+
+      // simulate state/model drift: child state still points to old parent state
+      childState.parentInstanceId = oldParent.id
+
+      await createContext({
+        instances: [grandParent, parent, child],
+        states: [grandParentState, parentState, oldParentState, childState],
+      })
+
+      setupImmediateLocking()
+      setupPersistenceMocks({ instances: [grandParent, parent, child, oldParent] })
+
+      const inOperationStateIds = new Set([grandParent.id, parent.id, child.id])
+      const baseUpdateMock = instanceStateService.updateOperationState.getMockImplementation()
+
+      instanceStateService.updateOperationState.mockImplementation(
+        async (projectId, stateId, operationId, options) => {
+          if (!inOperationStateIds.has(stateId as typeof grandParent.id)) {
+            throw new Error(`No operation state row for stateId ${stateId}`)
+          }
+
+          if (!baseUpdateMock) {
+            throw new Error("updateOperationState base mock is not initialized")
+          }
+
+          return await baseUpdateMock(projectId, stateId, operationId, options)
+        },
+      )
+
+      const operation = createOperation({
+        type: "update",
+        requestedInstanceIds: [child.id],
+        phases: [
+          {
+            type: "update",
+            instances: [
+              { id: grandParent.id, message: "ancestor", parentId: undefined },
+              { id: parent.id, message: "parent", parentId: grandParent.id },
+              { id: child.id, message: "requested", parentId: parent.id },
+            ],
+          },
+        ],
+      })
+
+      const runtimeOperation = new RuntimeOperation(
+        project,
+        operation,
+        runnerBackend,
+        libraryBackend,
+        artifactService,
+        instanceLockService,
+        operationService,
+        secretService,
+        instanceStateService,
+        projectModelService,
+        unitExtraService,
+        entitySnapshotService,
+        unitOutputService,
+        logger,
+      )
+
+      // act
+      await runtimeOperation.operateSafe()
+
+      // assert
+      const recalculatedOldParent = instanceStateService.updateOperationState.mock.calls.some(
+        ([, stateId]) => stateId === oldParent.id,
+      )
+
+      expect(recalculatedOldParent).toBe(false)
+      expect(operationService.markOperationFinished).toHaveBeenCalledWith(
+        project.id,
+        operation.id,
+        "completed",
+      )
+    },
+  )
+
+  operationTest(
+    "keeps composite parents deployed during ghost cleanup destroy phase in update operations",
+    async ({
+      project,
+      logger,
+      runnerBackend,
+      libraryBackend,
+      artifactService,
+      instanceLockService,
+      operationService,
+      secretService,
+      instanceStateService,
+      projectModelService,
+      unitExtraService,
+      entitySnapshotService,
+      unitOutputService,
+      createComposite,
+      createUnit,
+      createDeployedUnitState,
+      createOperation,
+      createContext,
+      setupPersistenceMocks,
+      setupImmediateLocking,
+      expect,
+    }) => {
+      // arrange
+      const grandParent = createComposite("GrandParent")
+      const parent = {
+        ...createComposite("Parent"),
+        parentId: grandParent.id,
+      }
+      const child = {
+        ...createUnit("Child"),
+        parentId: parent.id,
+      }
+      const ghostChild = {
+        ...createUnit("GhostChild"),
+        parentId: parent.id,
+      }
+
+      const grandParentState = createDeployedUnitState(grandParent)
+      const parentState = createDeployedUnitState(parent)
+      const childState = createDeployedUnitState(child)
+      const ghostChildState = createDeployedUnitState(ghostChild)
+
+      await createContext({
+        instances: [grandParent, parent, child],
+        states: [grandParentState, parentState, childState, ghostChildState],
+      })
+      setupImmediateLocking()
+      setupPersistenceMocks({ instances: [grandParent, parent, child, ghostChild] })
+
+      const operation = createOperation({
+        type: "update",
+        requestedInstanceIds: [parent.id],
+        phases: [
+          {
+            type: "update",
+            instances: [
+              { id: grandParent.id, message: "ancestor", parentId: undefined },
+              { id: parent.id, message: "requested", parentId: grandParent.id },
+              { id: child.id, message: "child", parentId: parent.id },
+            ],
+          },
+          {
+            type: "destroy",
+            instances: [
+              { id: ghostChild.id, message: "ghost cleanup", parentId: parent.id },
+              { id: parent.id, message: "parent of included child", parentId: grandParent.id },
+              { id: grandParent.id, message: "parent of included child", parentId: undefined },
+            ],
+          },
+        ],
+      })
+
+      const runtimeOperation = new RuntimeOperation(
+        project,
+        operation,
+        runnerBackend,
+        libraryBackend,
+        artifactService,
+        instanceLockService,
+        operationService,
+        secretService,
+        instanceStateService,
+        projectModelService,
+        unitExtraService,
+        entitySnapshotService,
+        unitOutputService,
+        logger,
+      )
+
+      // act
+      await runtimeOperation.operateSafe()
+
+      // assert
+      const parentStateUpdates = instanceStateService.updateOperationState.mock.calls
+        .filter(([, stateId]) => stateId === parent.id)
+        .map(([, , , options]) => options.instanceState?.status)
+        .filter((status): status is NonNullable<typeof status> => status != null)
+
+      const grandParentStateUpdates = instanceStateService.updateOperationState.mock.calls
+        .filter(([, stateId]) => stateId === grandParent.id)
+        .map(([, , , options]) => options.instanceState?.status)
+        .filter((status): status is NonNullable<typeof status> => status != null)
+
+      expect(parentStateUpdates).toContain("deployed")
+      expect(grandParentStateUpdates).toContain("deployed")
+      expect(parentStateUpdates).not.toContain("undeployed")
+      expect(grandParentStateUpdates).not.toContain("undeployed")
+    },
+  )
+
+  operationTest(
+    "marks composite parents undeployed after ghost cleanup when no unit descendants remain",
+    async ({
+      project,
+      logger,
+      runnerBackend,
+      libraryBackend,
+      artifactService,
+      instanceLockService,
+      operationService,
+      secretService,
+      instanceStateService,
+      projectModelService,
+      unitExtraService,
+      entitySnapshotService,
+      unitOutputService,
+      createComposite,
+      createUnit,
+      createDeployedUnitState,
+      createOperation,
+      createContext,
+      setupPersistenceMocks,
+      setupImmediateLocking,
+      expect,
+    }) => {
+      // arrange
+      const grandParent = createComposite("GrandParent")
+      const parent = {
+        ...createComposite("Parent"),
+        parentId: grandParent.id,
+      }
+      const ghostChild = {
+        ...createUnit("GhostChild"),
+        parentId: parent.id,
+      }
+
+      const grandParentState = createDeployedUnitState(grandParent)
+      const parentState = createDeployedUnitState(parent)
+      const ghostChildState = createDeployedUnitState(ghostChild)
+
+      await createContext({
+        instances: [grandParent, parent],
+        states: [grandParentState, parentState, ghostChildState],
+      })
+      setupImmediateLocking()
+      setupPersistenceMocks({ instances: [grandParent, parent, ghostChild] })
+
+      const operation = createOperation({
+        type: "update",
+        requestedInstanceIds: [parent.id],
+        phases: [
+          {
+            type: "update",
+            instances: [
+              { id: grandParent.id, message: "ancestor", parentId: undefined },
+              { id: parent.id, message: "requested", parentId: grandParent.id },
+            ],
+          },
+          {
+            type: "destroy",
+            instances: [
+              { id: ghostChild.id, message: "ghost cleanup", parentId: parent.id },
+              { id: parent.id, message: "parent of included child", parentId: grandParent.id },
+              { id: grandParent.id, message: "parent of included child", parentId: undefined },
+            ],
+          },
+        ],
+      })
+
+      const runtimeOperation = new RuntimeOperation(
+        project,
+        operation,
+        runnerBackend,
+        libraryBackend,
+        artifactService,
+        instanceLockService,
+        operationService,
+        secretService,
+        instanceStateService,
+        projectModelService,
+        unitExtraService,
+        entitySnapshotService,
+        unitOutputService,
+        logger,
+      )
+
+      // act
+      await runtimeOperation.operateSafe()
+
+      // assert
+      const parentStateUpdates = instanceStateService.updateOperationState.mock.calls
+        .filter(([, stateId]) => stateId === parent.id)
+        .map(([, , , options]) => options.instanceState?.status)
+        .filter((status): status is NonNullable<typeof status> => status != null)
+
+      const grandParentStateUpdates = instanceStateService.updateOperationState.mock.calls
+        .filter(([, stateId]) => stateId === grandParent.id)
+        .map(([, , , options]) => options.instanceState?.status)
+        .filter((status): status is NonNullable<typeof status> => status != null)
+
+      expect(parentStateUpdates).toContain("undeployed")
+      expect(grandParentStateUpdates).toContain("undeployed")
+    },
+  )
 })

package/src/orchestrator/operation.destroy.test.ts

@@ -18,6 +18,8 @@ describe("Operation - Destroy", () => {
       instanceStateService,
       projectModelService,
       unitExtraService,
+      entitySnapshotService,
+      unitOutputService,
       createUnit,
       createDeployedUnitState,
       createOperation,
@@ -59,6 +61,8 @@ describe("Operation - Destroy", () => {
         instanceStateService,
         projectModelService,
         unitExtraService,
+        entitySnapshotService,
+        unitOutputService,
         logger,
       )
 

package/src/orchestrator/operation.output-validation.failure.test.ts

@@ -0,0 +1,124 @@
+import { describe } from "vitest"
+import { RuntimeOperation } from "./operation"
+import { operationTest } from "./operation.test-utils"
+
+describe("Operation - Output Validation Failure", () => {
+  operationTest(
+    "writes instance log and fails instance when unit outputs are invalid",
+    async ({
+      project,
+      logger,
+      runnerBackend,
+      runner,
+      libraryBackend,
+      artifactService,
+      instanceLockService,
+      operationService,
+      secretService,
+      instanceStateService,
+      projectModelService,
+      unitExtraService,
+      entitySnapshotService,
+      unitOutputService,
+      createUnit,
+      createDeployedUnitState,
+      createOperation,
+      createContext,
+      setupPersistenceMocks,
+      setupImmediateLocking,
+      expect,
+    }) => {
+      const unit = createUnit("A")
+      const state = createDeployedUnitState(unit)
+
+      await createContext({ instances: [unit], states: [state] })
+      setupImmediateLocking()
+      setupPersistenceMocks({ instances: [unit] })
+
+      unitOutputService.parseUnitOutputs.mockResolvedValue({
+        outputHash: null,
+        statusFields: null,
+        terminals: null,
+        pages: null,
+        triggers: null,
+        secrets: null,
+        workers: null,
+        exportedArtifactIds: null,
+        hasResourceHooks: false,
+        entitySnapshotError: "invalid entity schema",
+        entitySnapshotPayload: null,
+      })
+
+      runner.setAutoCompletion(false)
+      runner.emitCompletion(unit.id, {
+        operationType: "update",
+        rawOutputs: {
+          value: { value: { some: "output" } },
+        },
+      })
+
+      const operation = createOperation({
+        type: "update",
+        requestedInstanceIds: [unit.id],
+        phases: [
+          {
+            type: "update",
+            instances: [{ id: unit.id, message: "requested", parentId: undefined }],
+          },
+        ],
+      })
+
+      const runtimeOperation = new RuntimeOperation(
+        project,
+        operation,
+        runnerBackend,
+        libraryBackend,
+        artifactService,
+        instanceLockService,
+        operationService,
+        secretService,
+        instanceStateService,
+        projectModelService,
+        unitExtraService,
+        entitySnapshotService,
+        unitOutputService,
+        logger,
+      )
+
+      await runtimeOperation.operateSafe()
+
+      expect(operationService.appendLog).toHaveBeenCalledWith(
+        project.id,
+        operation.id,
+        state.id,
+        expect.stringContaining("Failed to parse unit outputs"),
+      )
+
+      expect(instanceStateService.updateOperationState).toHaveBeenCalledWith(
+        project.id,
+        state.id,
+        operation.id,
+        expect.any(Object),
+      )
+
+      const wasMarkedFailed = instanceStateService.updateOperationState.mock.calls.some(call => {
+        const patch = call[3] as unknown
+        if (!patch || typeof patch !== "object") {
+          return false
+        }
+
+        const maybePatch = patch as {
+          operationState?: { status?: unknown }
+          instanceState?: { status?: unknown }
+        }
+
+        return (
+          maybePatch.operationState?.status === "failed" &&
+          maybePatch.instanceState?.status === "failed"
+        )
+      })
+
+      expect(wasMarkedFailed).toBe(true)
+    },
+  )
+})

package/src/orchestrator/operation.preview.test.ts

@@ -18,6 +18,8 @@ describe("Operation - Preview", () => {
       instanceStateService,
       projectModelService,
       unitExtraService,
+      entitySnapshotService,
+      unitOutputService,
       createUnit,
       createDeployedUnitState,
       createOperation,
@@ -59,6 +61,8 @@ describe("Operation - Preview", () => {
         instanceStateService,
         projectModelService,
         unitExtraService,
+        entitySnapshotService,
+        unitOutputService,
         logger,
       )