@highstate/backend 0.9.4 → 0.9.6

package/src/orchestrator/operation-workset.ts

@@ -2,7 +2,12 @@ import type { LibraryBackend } from "../library"
 import type { ProjectBackend } from "../project"
 import type { StateBackend, StateManager } from "../state"
 import type { Logger } from "pino"
-import { isUnitModel, type ComponentModel, type InstanceModel } from "@highstate/contract"
+import {
+  isUnitModel,
+  parseInstanceId,
+  type ComponentModel,
+  type InstanceModel,
+} from "@highstate/contract"
 import { unique } from "remeda"
 import {
   applyPartialInstanceState,
@@ -22,8 +27,12 @@ import {
   type ResolvedInstanceInput,
 } from "../shared"
 
+export type OperationPhase = "update" | "destroy" | "refresh"
+
 export class OperationWorkset {
-  private readonly affectedInstanceIdSet = new Set<string>()
+  private readonly instanceIdsToUpdate = new Set<string>()
+  private readonly instanceIdsToDestroy = new Set<string>()
+
   private readonly instanceMap = new Map<string, InstanceModel>()
   private readonly instanceChildrenMap = new Map<string, InstanceModel[]>()
 
@@ -67,8 +76,24 @@ export class OperationWorkset {
     return instance
   }
 
-  public isAffected(instanceId: string): boolean {
-    return this.affectedInstanceIdSet.has(instanceId)
+  public getAffectedInstanceIds(phase: OperationPhase): string[] {
+    if (phase === "destroy") {
+      return Array.from(this.instanceIdsToDestroy)
+    }
+
+    return Array.from(this.instanceIdsToUpdate)
+  }
+
+  public getInstanceOrUndefined(instanceId: string): InstanceModel | undefined {
+    return this.instanceMap.get(instanceId)
+  }
+
+  public isAffected(instanceId: string, phase: OperationPhase): boolean {
+    if (phase === "destroy") {
+      return this.instanceIdsToDestroy.has(instanceId)
+    }
+
+    return this.instanceIdsToUpdate.has(instanceId)
   }
 
   public updateState(update: InstanceStateUpdate): InstanceState {
@@ -76,7 +101,7 @@ export class OperationWorkset {
     this.stateManager.emitStatePatch(this.operation.projectId, createInstanceStatePatch(update))
 
     if (finalState.parentId) {
-      this.recalculateCompositeState(finalState.parentId)
+      this.recalculateCompositeInstanceState(finalState.parentId)
     }
 
     return finalState
@@ -128,25 +153,44 @@ export class OperationWorkset {
 
   public emitAffectedInitialStates(): void {
     for (const state of this.initialStateMap.values()) {
-      if (this.affectedInstanceIdSet.has(state.id)) {
+      if (this.instanceIdsToUpdate.has(state.id)) {
         this.stateManager.emitStatePatch(this.operation.projectId, state)
       }
     }
   }
 
-  public getAffectedCompositeChildren(instanceId: string): InstanceModel[] {
+  public getAffectedCompositeChildren(instanceId: string, phase: OperationPhase): InstanceModel[] {
     const children = this.instanceChildrenMap.get(instanceId)
     if (!children) {
       return []
     }
 
-    return children.filter(child => this.affectedInstanceIdSet.has(child.id))
+    if (phase === "destroy") {
+      return children.filter(child => this.instanceIdsToDestroy.has(child.id))
+    }
+
+    return children.filter(child => this.instanceIdsToUpdate.has(child.id))
   }
 
   public getState(instanceId: string): InstanceState | undefined {
     return this.stateMap.get(instanceId)
   }
 
+  public getParentId(instanceId: string, phase: OperationPhase): string | null {
+    if (phase === "destroy") {
+      const state = this.stateMap.get(instanceId)
+      if (!state) {
+        return null
+      }
+
+      return state.parentId
+    }
+
+    const instance = this.getInstance(instanceId)
+
+    return instance.parentId ?? null
+  }
+
   public getDependentStates(instanceId: string): InstanceState[] {
     const dependentStateIds = this.dependentStateIdMap.get(instanceId)
     if (!dependentStateIds) {
@@ -158,7 +202,7 @@ export class OperationWorkset {
       .filter((state): state is InstanceState => !!state)
   }
 
-  private recalculateCompositeState(instanceId: string): void {
+  private recalculateCompositeInstanceState(instanceId: string): void {
     const state = this.stateMap.get(instanceId) ?? createInstanceState(instanceId)
     let currentResourceCount = 0
     let totalResourceCount = 0
@@ -186,7 +230,7 @@ export class OperationWorkset {
     this.stateManager.emitStatePatch(this.operation.projectId, updatedState)
 
     if (state.parentId) {
-      this.recalculateCompositeState(state.parentId)
+      this.recalculateCompositeInstanceState(state.parentId)
     }
   }
 
@@ -215,9 +259,9 @@ export class OperationWorkset {
     }
   }
 
-  private async extendForUpdate(): Promise<void> {
+  private async calculateInstanceIdsToUpdate(): Promise<void> {
     const traverse = async (instanceId: string) => {
-      if (this.affectedInstanceIdSet.has(instanceId)) {
+      if (this.instanceIdsToUpdate.has(instanceId)) {
         return
       }
 
@@ -238,51 +282,62 @@ export class OperationWorkset {
       const { inputHash: expectedInputHash } = await this.inputHashResolver(instance.id)
 
       if (this.operation.options.forceUpdateDependencies) {
-        this.affectedInstanceIdSet.add(instanceId)
+        this.instanceIdsToUpdate.add(instanceId)
         return
       }
 
       if (state?.status !== "created" || state.inputHash !== expectedInputHash) {
-        this.affectedInstanceIdSet.add(instanceId)
+        this.instanceIdsToUpdate.add(instanceId)
       }
     }
 
     // 1. extend affected instance IDs with their not-created or not-up-to-date dependencies (only for "update" operations)
-    for (const instanceId of this.operation.instanceIds) {
+    for (const instanceId of this.operation.requestedInstanceIds) {
       if (this.operation.type === "update") {
         await traverse(instanceId)
       }
 
-      this.affectedInstanceIdSet.add(instanceId)
+      this.instanceIdsToUpdate.add(instanceId)
     }
 
     // 2. extend affected instance IDs with the children of the affected composite instances
-    const compositeInstanceQueue = Array.from(this.affectedInstanceIdSet)
+    const compositeInstanceQueue = Array.from(this.instanceIdsToUpdate)
     while (compositeInstanceQueue.length > 0) {
       const childId = compositeInstanceQueue.pop()!
       const children = this.instanceChildrenMap.get(childId) ?? []
 
       for (const child of children) {
         compositeInstanceQueue.push(child.id)
-        this.affectedInstanceIdSet.add(child.id)
+
+        if (this.operation.options.forceUpdateChildren) {
+          this.instanceIdsToUpdate.add(child.id)
+          continue
+        }
+
+        const state = this.stateMap.get(child.id)
+        const { inputHash: expectedInputHash } = await this.inputHashResolver(child.id)
+
+        if (state?.status !== "created" || state.inputHash !== expectedInputHash) {
+          this.instanceIdsToUpdate.add(child.id)
+        }
       }
     }
 
     // 3. detect composite instance children and include their parents (recursively)
-    for (const instanceId of this.affectedInstanceIdSet) {
+    for (const instanceId of this.instanceIdsToUpdate) {
       let instance = this.instanceMap.get(instanceId)
       while (instance?.parentId) {
-        this.affectedInstanceIdSet.add(instance.parentId)
+        this.instanceIdsToUpdate.add(instance.parentId)
         instance = this.instanceMap.get(instance.parentId)
       }
     }
 
-    this.operation.affectedInstanceIds = Array.from(this.affectedInstanceIdSet)
+    this.operation.instanceIdsToUpdate = Array.from(this.instanceIdsToUpdate)
   }
 
-  private extendForDestroy() {
+  private calculateInstanceIdsToDestroy() {
     const traverse = (instanceKey: string) => {
-      if (this.affectedInstanceIdSet.has(instanceKey)) {
+      if (this.instanceIdsToDestroy.has(instanceKey)) {
         return
       }
 
@@ -295,46 +350,74 @@ export class OperationWorkset {
 
       for (const dependentId of dependentIds) {
         traverse(dependentId)
-        this.affectedInstanceIdSet.add(instanceKey)
+        this.instanceIdsToDestroy.add(dependentId)
       }
     }
 
-    // 1. extend affected instance IDs with their created dependents (if not forbidden by the operation options)
-    for (const instanceId of this.operation.instanceIds) {
-      const instance = this.instanceMap.get(instanceId)
-      if (!instance) {
-        throw new Error(`Instance not found: ${instanceId}`)
-      }
+    if (this.operation.type === "destroy" || this.operation.type === "recreate") {
+      // 1.a. extend affected instance IDs with their created dependents (if not forbidden by the operation options)
+      for (const instanceId of this.operation.requestedInstanceIds) {
+        const instance = this.instanceMap.get(instanceId)
+        if (!instance) {
+          throw new Error(`Instance not found: ${instanceId}`)
+        }
 
-      if (this.operation.options.destroyDependentInstances) {
-        traverse(instance.id)
+        if (this.operation.options.destroyDependentInstances) {
+          traverse(instance.id)
+        }
+
+        this.instanceIdsToDestroy.add(instanceId)
       }
+    } else if (this.operation.type === "update") {
+      // 1.b. find all children instances of the affected to-update instances which are in the state map, but not in the instance map
+      // in other words, this code cleans up child instances which are not produced by composite instances anymore
+
+      for (const instanceId of this.operation.instanceIdsToUpdate) {
+        const [type] = parseInstanceId(instanceId)
+        const component = this.library.components[type]
 
-      this.affectedInstanceIdSet.add(instanceId)
+        if (!component || isUnitModel(component)) {
+          // ignore non-composite instances
+          continue
+        }
+
+        const childrenQueue = [...(this.stateChildIdMap.get(instanceId) ?? [])]
+
+        while (childrenQueue.length > 0) {
+          const childId = childrenQueue.pop()!
+          if (!this.instanceMap.has(childId)) {
+            this.instanceIdsToDestroy.add(childId)
+          }
+
+          childrenQueue.push(...(this.stateChildIdMap.get(childId) ?? []))
+        }
+      }
+    } else {
+      return
     }
 
     // 2. extend affected instance IDs with the children of the affected composite instances
-    const compositeInstanceQueue = Array.from(this.affectedInstanceIdSet)
+    const compositeInstanceQueue = Array.from(this.instanceIdsToDestroy)
     while (compositeInstanceQueue.length > 0) {
       const childId = compositeInstanceQueue.pop()!
       const children = this.stateChildIdMap.get(childId) ?? []
 
       for (const child of children) {
         compositeInstanceQueue.push(child)
-        this.affectedInstanceIdSet.add(child)
+        this.instanceIdsToDestroy.add(child)
       }
     }
 
     // 3. detect composite instance children and include their parents (recursively)
-    for (const instanceId of this.affectedInstanceIdSet) {
+    for (const instanceId of this.instanceIdsToDestroy) {
       let state = this.stateMap.get(instanceId)
       while (state?.parentId) {
-        this.affectedInstanceIdSet.add(state.parentId)
+        this.instanceIdsToDestroy.add(state.parentId)
         state = this.stateMap.get(state.parentId)
       }
     }
 
-    this.operation.affectedInstanceIds = Array.from(this.affectedInstanceIdSet)
+    this.operation.instanceIdsToDestroy = Array.from(this.instanceIdsToDestroy)
   }
 
   private getSourceHashIfApplicable(
@@ -366,18 +449,7 @@ export class OperationWorkset {
   }
 
   public getLockInstanceIds(): string[] {
-    const instanceIds = new Set<string>(this.operation.affectedInstanceIds)
-
-    // enrich with the parent IDs of the affected instances in order to lock and update parent states
-    for (const instanceId of this.operation.affectedInstanceIds) {
-      const instance = this.getInstance(instanceId)
-
-      if (instance.parentId) {
-        instanceIds.add(instance.parentId)
-      }
-    }
-
-    return Array.from(instanceIds)
+    return Array.from(this.instanceIdsToUpdate.union(this.instanceIdsToDestroy))
   }
 
   public static async load(
@@ -422,8 +494,13 @@ export class OperationWorkset {
       if (worksetInstance) {
         worksetInstance.outputs = instance.instance.outputs
         worksetInstance.resolvedOutputs = instance.instance.resolvedOutputs
-      } else {
+      } else if (instance.instance.parentId) {
         workset.addInstance(instance.instance)
+      } else {
+        workset.logger.warn(
+          `ignoring instance "${instance.instance.id}" from composite instance state because it is not in the project or is not a part of known composite instance`,
+        )
+        continue
       }
 
       for (const child of instance.children) {
@@ -492,20 +569,12 @@ export class OperationWorkset {
       { promiseCache: workset.inputHashResolverPromiseCache },
     )
 
-    switch (operation.type) {
-      case "update":
-      case "preview":
-      case "refresh":
-        await workset.extendForUpdate()
-        break
-      case "recreate":
-        workset.extendForDestroy()
-        break
-      case "destroy":
-        workset.extendForDestroy()
-        break
+    if (operation.type !== "destroy") {
+      await workset.calculateInstanceIdsToUpdate()
     }
 
+    workset.calculateInstanceIdsToDestroy()
+
     return workset
   }
 }

package/src/orchestrator/operation.ts

@@ -24,9 +24,7 @@ import {
   tryWrapAbortErrorLike,
   valueToString,
 } from "../common"
-import { OperationWorkset } from "./operation-workset"
-
-type OperationPhase = "update" | "destroy" | "refresh"
+import { OperationWorkset, type OperationPhase } from "./operation-workset"
 
 export class RuntimeOperation {
   private readonly abortController = new AbortController()
@@ -117,10 +115,14 @@ export class RuntimeOperation {
   }
 
   private async processOperation(): Promise<void> {
-    this.operation.affectedInstanceIds = this.workset.operation.affectedInstanceIds
+    this.operation.instanceIdsToUpdate = this.workset.operation.instanceIdsToUpdate
+    this.operation.instanceIdsToDestroy = this.workset.operation.instanceIdsToDestroy
 
     this.logger.info(
-      { affectedInstanceIds: this.operation.affectedInstanceIds },
+      {
+        instanceIdsToUpdate: this.operation.instanceIdsToUpdate,
+        instanceIdsToDestroy: this.operation.instanceIdsToDestroy,
+      },
       "operation started",
     )
 
@@ -130,9 +132,9 @@ export class RuntimeOperation {
       this.currentPhase = phase
 
       const promises: Promise<void>[] = []
-      for (const instanceId of this.operation.affectedInstanceIds) {
-        const instance = this.workset.getInstance(instanceId)
-        if (instance.parentId && this.workset.isAffected(instance.parentId)) {
+      for (const instanceId of this.workset.getAffectedInstanceIds(phase)) {
+        const parentId = this.workset.getParentId(instanceId, phase)
+        if (parentId && this.workset.isAffected(parentId, phase)) {
           // do not call the operation for child instances of affected composites,
           // they will be called by their parent instance
           continue
@@ -163,19 +165,25 @@ export class RuntimeOperation {
   }
 
   private getInstancePromiseForOperation(instanceId: string): Promise<void> {
-    const instance = this.workset.getInstance(instanceId)
-    const component = this.workset.library.components[instance.type]
+    const [instanceType] = parseInstanceId(instanceId)
+    const component = this.workset.library.components[instanceType]
 
     if (isUnitModel(component)) {
-      return this.getUnitPromise(instance)
+      return this.getUnitPromise(instanceId)
     }
 
-    return this.getCompositePromise(instance)
+    return this.getCompositePromise(instanceId)
   }
 
   private getOperationPhases(): OperationPhase[] {
     switch (this.operation.type) {
-      case "update":
+      case "update": {
+        if (this.operation.instanceIdsToDestroy.length > 0) {
+          return ["destroy", "update"]
+        }
+
+        return ["update"]
+      }
       case "preview":
         return ["update"]
       case "recreate":
@@ -187,35 +195,36 @@ export class RuntimeOperation {
     }
   }
 
-  private async getUnitPromise(instance: InstanceModel): Promise<void> {
+  private async getUnitPromise(instanceId: string): Promise<void> {
     switch (this.currentPhase) {
       case "update": {
-        return this.updateUnit(instance)
+        return this.updateUnit(instanceId)
       }
       case "destroy": {
-        return this.destroyUnit(instance.id)
+        return this.destroyUnit(instanceId)
       }
       case "refresh": {
-        return this.refreshUnit(instance.id)
+        return this.refreshUnit(instanceId)
       }
     }
   }
 
-  private async getCompositePromise(instance: InstanceModel): Promise<void> {
-    const logger = this.logger.child({ instanceId: instance.id })
+  private async getCompositePromise(instanceId: string): Promise<void> {
+    const logger = this.logger.child({ instanceId })
 
-    return this.getInstancePromise(instance.id, async () => {
-      const state = this.workset.getState(instance.id) ?? createInstanceState(instance.id)
+    return this.getInstancePromise(instanceId, async () => {
+      const state = this.workset.getState(instanceId) ?? createInstanceState(instanceId)
+      const instance = this.workset.getInstanceOrUndefined(instanceId)
 
       this.updateInstanceState({
         ...state,
-        parentId: instance.parentId,
+        parentId: instance?.parentId,
         latestOperationId: this.operation.id,
         status: this.getStatusByOperationType(),
         error: null,
       })
 
-      const children = this.workset.getAffectedCompositeChildren(instance.id)
+      const children = this.workset.getAffectedCompositeChildren(instanceId, this.currentPhase)
       const childPromises: Promise<void>[] = []
 
       if (children.length) {
@@ -225,6 +234,14 @@ export class RuntimeOperation {
       }
 
       for (const child of children) {
+        if (
+          !this.operation.options.forceUpdateChildren &&
+          !this.workset.isAffected(child.id, this.currentPhase)
+        ) {
+          // skip children that are not affected by the operation
+          continue
+        }
+
         logger.debug(`waiting for child: "%s"`, child.id)
         childPromises.push(this.getInstancePromiseForOperation(child.id))
       }
@@ -237,18 +254,18 @@ export class RuntimeOperation {
         }
 
         this.updateInstanceState({
-          id: instance.id,
+          id: instanceId,
           status: this.operation.type === "destroy" ? "not_created" : "created",
-          inputHash: await this.workset.getUpToDateInputHash(instance),
+          inputHash: instance ? await this.workset.getUpToDateInputHash(instance) : undefined,
         })
       } catch (error) {
         if (isAbortErrorLike(error)) {
-          this.workset.restoreInitialStatus(instance.id)
+          this.workset.restoreInitialStatus(instanceId)
           return
         }
 
         this.updateInstanceState({
-          id: instance.id,
+          id: instanceId,
           status: "error",
           error: errorToString(error),
         })
@@ -256,8 +273,10 @@ export class RuntimeOperation {
     })
   }
 
-  private updateUnit(instance: InstanceModel): Promise<void> {
-    return this.getInstancePromise(instance.id, async logger => {
+  private updateUnit(instanceId: string): Promise<void> {
+    return this.getInstancePromise(instanceId, async logger => {
+      const instance = this.workset.getInstance(instanceId)
+
       this.updateInstanceState({
         id: instance.id,
         parentId: instance.parentId,
@@ -323,7 +342,7 @@ export class RuntimeOperation {
       const dependencyPromises: Promise<void>[] = []
 
       for (const dependency of dependencies) {
-        if (!this.operation.affectedInstanceIds.includes(dependency.id)) {
+        if (!this.operation.instanceIdsToUpdate.includes(dependency.id)) {
           // skip dependencies that are not affected by the operation
           continue
         }

package/src/project/manager.ts

@@ -19,6 +19,10 @@ import {
 } from "../shared"
 
 type CompositeInstanceEvent =
+  | {
+      type: "evaluation-started"
+      instanceId: string
+    }
   | {
       type: "updated"
       instance: CompositeInstance
@@ -157,6 +161,10 @@ export class ProjectManager {
     await this.projectLockManager.getLock(projectId).lockInstances(instanceIds, async () => {
       this.logger.debug({ instanceIds }, "evaluating composite instances")
 
+      for (const instanceId of instanceIds) {
+        this.compositeInstanceEE.emit(projectId, { type: "evaluation-started", instanceId })
+      }
+
       const [
         { instances, resolvedInputs, stateMap, resolveInputHash },
         topLevelCompositeChildrenIds,

package/src/shared/library.ts

@@ -6,6 +6,9 @@ export type LibraryModel = {
 }
 
 export type LibraryUpdate =
+  | {
+      type: "reload-started" | "reload-completed"
+    }
   | {
       type: "component-updated"
       component: ComponentModel

package/src/shared/operation.ts

@@ -19,6 +19,14 @@ export const operationOptionsSchema = z.object({
    */
   forceUpdateDependencies: z.boolean().default(false),
 
+  /**
+   * Whether to force update all children of the composite instances even if they are not changed.
+   *
+   * Only applicable for `update`, `preview`, `recreate`, and `refresh` operations.
+   * By default, `false`.
+   */
+  forceUpdateChildren: z.boolean().default(false),
+
   /**
    * Whether to destroy all dependents of the instances when destroying them.
    *
@@ -72,8 +80,11 @@ export const projectOperationSchema = z.object({
 
   projectId: z.string(),
   type: operationTypeSchema,
-  instanceIds: z.array(z.string()),
-  affectedInstanceIds: z.array(z.string()).default(() => []),
+
+  requestedInstanceIds: z.array(z.string()),
+
+  instanceIdsToUpdate: z.array(z.string()).default(() => []),
+  instanceIdsToDestroy: z.array(z.string()).default(() => []),
 
   options: operationOptionsSchema.default(() => ({})),
 

package/src/shared/resolvers/validation.ts

@@ -59,7 +59,7 @@ export const createValidationResolver = defineGraphResolver<
 
         return {
           status: "invalid-args",
-          errorText: ajv.errorsText(),
+          errorText: `invalid argument "${name}": ${ajv.errorsText()}`,
         }
       }
     }
@@ -70,14 +70,14 @@ export const createValidationResolver = defineGraphResolver<
         if (inputInstance?.status !== "ok") {
           return {
             status: "invalid-inputs",
-            errorText: `instance "${input.input.instanceId}" is invalid`,
+            errorText: `instance "${input.input.instanceId}" has errors`,
           }
         }
       }
     }
 
     for (const [name, input] of Object.entries(component.inputs)) {
-      if (!input.required || input.multiple) {
+      if (!input.required) {
         continue
       }
 

package/src/shared/state.ts

@@ -13,9 +13,16 @@ export const instanceStatusSchema = z.enum([
   "unknown",
 ])
 
+export const instanceStatusFieldValueSchema = z.union([
+  z.string(),
+  z.number(),
+  z.boolean(),
+  z.array(z.string()),
+])
+
 export const instanceStatusFieldSchema = z.object({
   name: z.string(),
-  value: z.string().optional(),
+  value: instanceStatusFieldValueSchema.optional(),
   displayName: z.string().optional(),
   sensitive: z.boolean().optional(),
   url: z.string().optional(),
@@ -138,6 +145,7 @@ export const instanceStateUpdateSchema = z
   })
   .partial()
 
+export type InstanceStatusFieldValue = z.infer<typeof instanceStatusFieldValueSchema>
 export type InstanceStatusField = z.infer<typeof instanceStatusFieldSchema>
 export type InstanceTerminal = z.infer<typeof instanceTerminalSchema>