@highstate/backend 0.9.4 → 0.9.6

package/dist/index.js

@@ -17,7 +17,7 @@ import {
   isFinalOperationStatus,
   projectOperationSchema,
   terminalSessionSchema
-} from "./chunk-I7NLZ54E.js";
+} from "./chunk-NMGIUI6X.js";
 import {
   LocalPulumiHost,
   createAsyncBatcher,
@@ -229,7 +229,7 @@ var LocalLibraryBackend = class _LocalLibraryBackend {
   async getResolvedUnitSources(unitTypes) {
     return await this.lock.acquire(async () => {
       const [library] = await this.getLibrary();
-      const units = unitTypes.map((type) => library.components[type]).filter(isUnitModel);
+      const units = unitTypes.map((type) => library.components[type]).filter(Boolean).filter(isUnitModel);
       const packageNames = Object.keys(groupBy(units, (unit) => unit.source.package));
       await this.ensureLibraryPackagesLoaded(packageNames, true);
       const result = [];
@@ -302,10 +302,11 @@ var LocalLibraryBackend = class _LocalLibraryBackend {
   }
   async reloadLibrary() {
     this.logger.info("reloading library");
+    this.eventEmitter.emit("library", [{ type: "reload-started" }]);
     await this.ensureLibraryPackagesLoaded(this.libraryPackages, true);
     this.worker = this.createLibraryWorker({
       modulePaths: this.libraryPackages,
-      logLevel: "silent"
+      logLevel: "warn"
     });
     for await (const [event] of on(this.worker, "message")) {
       const eventData = event;
@@ -322,6 +323,7 @@ var LocalLibraryBackend = class _LocalLibraryBackend {
       this.eventEmitter.emit("library", updates);
       this.library = eventData.library;
       this.logger.info("library reloaded");
+      this.eventEmitter.emit("library", [{ type: "reload-completed" }]);
       return [this.library, this.worker];
     }
     throw new Error("Worker ended without sending library model");
@@ -989,6 +991,9 @@ var ProjectManager = class _ProjectManager {
   async evaluateCompositeInstances(projectId, instanceIds) {
     await this.projectLockManager.getLock(projectId).lockInstances(instanceIds, async () => {
       this.logger.debug({ instanceIds }, "evaluating composite instances");
+      for (const instanceId of instanceIds) {
+        this.compositeInstanceEE.emit(projectId, { type: "evaluation-started", instanceId });
+      }
       const [
         { instances, resolvedInputs, stateMap, resolveInputHash },
         topLevelCompositeChildrenIds
@@ -2673,11 +2678,14 @@ import { EventEmitter as EventEmitter6, on as on6 } from "node:events";
 import { uuidv7 as uuidv72 } from "uuidv7";
 
 // src/orchestrator/operation.ts
-import { isUnitModel as isUnitModel4, parseInstanceId as parseInstanceId2 } from "@highstate/contract";
+import { isUnitModel as isUnitModel4, parseInstanceId as parseInstanceId3 } from "@highstate/contract";
 import { mapValues as mapValues3 } from "remeda";
 
 // src/orchestrator/operation-workset.ts
-import { isUnitModel as isUnitModel3 } from "@highstate/contract";
+import {
+  isUnitModel as isUnitModel3,
+  parseInstanceId as parseInstanceId2
+} from "@highstate/contract";
 import { unique as unique2 } from "remeda";
 var OperationWorkset = class _OperationWorkset {
   constructor(operation, library, stateManager, logger) {
@@ -2686,7 +2694,8 @@ var OperationWorkset = class _OperationWorkset {
     this.stateManager = stateManager;
     this.logger = logger;
   }
-  affectedInstanceIdSet = /* @__PURE__ */ new Set();
+  instanceIdsToUpdate = /* @__PURE__ */ new Set();
+  instanceIdsToDestroy = /* @__PURE__ */ new Set();
   instanceMap = /* @__PURE__ */ new Map();
   instanceChildrenMap = /* @__PURE__ */ new Map();
   initialStateMap = /* @__PURE__ */ new Map();
@@ -2708,14 +2717,26 @@ var OperationWorkset = class _OperationWorkset {
     }
     return instance;
   }
-  isAffected(instanceId) {
-    return this.affectedInstanceIdSet.has(instanceId);
+  getAffectedInstanceIds(phase) {
+    if (phase === "destroy") {
+      return Array.from(this.instanceIdsToDestroy);
+    }
+    return Array.from(this.instanceIdsToUpdate);
+  }
+  getInstanceOrUndefined(instanceId) {
+    return this.instanceMap.get(instanceId);
+  }
+  isAffected(instanceId, phase) {
+    if (phase === "destroy") {
+      return this.instanceIdsToDestroy.has(instanceId);
+    }
+    return this.instanceIdsToUpdate.has(instanceId);
   }
   updateState(update) {
     const finalState = applyPartialInstanceState(this.stateMap, update);
     this.stateManager.emitStatePatch(this.operation.projectId, createInstanceStatePatch(update));
     if (finalState.parentId) {
-      this.recalculateCompositeState(finalState.parentId);
+      this.recalculateCompositeInstanceState(finalState.parentId);
     }
     return finalState;
   }
@@ -2755,21 +2776,35 @@ var OperationWorkset = class _OperationWorkset {
   }
   emitAffectedInitialStates() {
     for (const state of this.initialStateMap.values()) {
-      if (this.affectedInstanceIdSet.has(state.id)) {
+      if (this.instanceIdsToUpdate.has(state.id)) {
         this.stateManager.emitStatePatch(this.operation.projectId, state);
       }
     }
   }
-  getAffectedCompositeChildren(instanceId) {
+  getAffectedCompositeChildren(instanceId, phase) {
     const children = this.instanceChildrenMap.get(instanceId);
     if (!children) {
       return [];
     }
-    return children.filter((child) => this.affectedInstanceIdSet.has(child.id));
+    if (phase === "destroy") {
+      return children.filter((child) => this.instanceIdsToDestroy.has(child.id));
+    }
+    return children.filter((child) => this.instanceIdsToUpdate.has(child.id));
   }
   getState(instanceId) {
     return this.stateMap.get(instanceId);
   }
+  getParentId(instanceId, phase) {
+    if (phase === "destroy") {
+      const state = this.stateMap.get(instanceId);
+      if (!state) {
+        return null;
+      }
+      return state.parentId;
+    }
+    const instance = this.getInstance(instanceId);
+    return instance.parentId ?? null;
+  }
   getDependentStates(instanceId) {
     const dependentStateIds = this.dependentStateIdMap.get(instanceId);
     if (!dependentStateIds) {
@@ -2777,7 +2812,7 @@ var OperationWorkset = class _OperationWorkset {
     }
     return Array.from(dependentStateIds).map((id) => this.stateMap.get(id)).filter((state) => !!state);
   }
-  recalculateCompositeState(instanceId) {
+  recalculateCompositeInstanceState(instanceId) {
     const state = this.stateMap.get(instanceId) ?? createInstanceState(instanceId);
     let currentResourceCount = 0;
     let totalResourceCount = 0;
@@ -2799,7 +2834,7 @@ var OperationWorkset = class _OperationWorkset {
     this.stateMap.set(instanceId, updatedState);
     this.stateManager.emitStatePatch(this.operation.projectId, updatedState);
     if (state.parentId) {
-      this.recalculateCompositeState(state.parentId);
+      this.recalculateCompositeInstanceState(state.parentId);
     }
   }
   addInstance(instance) {
@@ -2822,9 +2857,9 @@ var OperationWorkset = class _OperationWorkset {
       children.push(instance);
     }
   }
-  async extendForUpdate() {
+  async calculateInstanceIdsToUpdate() {
     const traverse = async (instanceId) => {
-      if (this.affectedInstanceIdSet.has(instanceId)) {
+      if (this.instanceIdsToUpdate.has(instanceId)) {
         return;
       }
       const instance = this.instanceMap.get(instanceId);
@@ -2840,40 +2875,48 @@ var OperationWorkset = class _OperationWorkset {
       const state = this.stateMap.get(instance.id);
       const { inputHash: expectedInputHash } = await this.inputHashResolver(instance.id);
       if (this.operation.options.forceUpdateDependencies) {
-        this.affectedInstanceIdSet.add(instanceId);
+        this.instanceIdsToUpdate.add(instanceId);
         return;
       }
       if (state?.status !== "created" || state.inputHash !== expectedInputHash) {
-        this.affectedInstanceIdSet.add(instanceId);
+        this.instanceIdsToUpdate.add(instanceId);
       }
     };
-    for (const instanceId of this.operation.instanceIds) {
+    for (const instanceId of this.operation.requestedInstanceIds) {
       if (this.operation.type === "update") {
         await traverse(instanceId);
       }
-      this.affectedInstanceIdSet.add(instanceId);
+      this.instanceIdsToUpdate.add(instanceId);
     }
-    const compositeInstanceQueue = Array.from(this.affectedInstanceIdSet);
+    const compositeInstanceQueue = Array.from(this.instanceIdsToUpdate);
     while (compositeInstanceQueue.length > 0) {
       const childId = compositeInstanceQueue.pop();
       const children = this.instanceChildrenMap.get(childId) ?? [];
       for (const child of children) {
         compositeInstanceQueue.push(child.id);
-        this.affectedInstanceIdSet.add(child.id);
+        if (this.operation.options.forceUpdateChildren) {
+          this.instanceIdsToUpdate.add(child.id);
+          continue;
+        }
+        const state = this.stateMap.get(child.id);
+        const { inputHash: expectedInputHash } = await this.inputHashResolver(child.id);
+        if (state?.status !== "created" || state.inputHash !== expectedInputHash) {
+          this.instanceIdsToUpdate.add(child.id);
+        }
       }
     }
-    for (const instanceId of this.affectedInstanceIdSet) {
+    for (const instanceId of this.instanceIdsToUpdate) {
       let instance = this.instanceMap.get(instanceId);
       while (instance?.parentId) {
-        this.affectedInstanceIdSet.add(instance.parentId);
+        this.instanceIdsToUpdate.add(instance.parentId);
         instance = this.instanceMap.get(instance.parentId);
       }
     }
-    this.operation.affectedInstanceIds = Array.from(this.affectedInstanceIdSet);
+    this.operation.instanceIdsToUpdate = Array.from(this.instanceIdsToUpdate);
   }
-  extendForDestroy() {
+  calculateInstanceIdsToDestroy() {
     const traverse = (instanceKey) => {
-      if (this.affectedInstanceIdSet.has(instanceKey)) {
+      if (this.instanceIdsToDestroy.has(instanceKey)) {
         return;
       }
       const state = this.stateMap.get(instanceKey);
@@ -2883,36 +2926,56 @@ var OperationWorkset = class _OperationWorkset {
       const dependentIds = this.dependentStateIdMap.get(instanceKey) ?? [];
       for (const dependentId of dependentIds) {
         traverse(dependentId);
-        this.affectedInstanceIdSet.add(instanceKey);
+        this.instanceIdsToDestroy.add(dependentId);
       }
     };
-    for (const instanceId of this.operation.instanceIds) {
-      const instance = this.instanceMap.get(instanceId);
-      if (!instance) {
-        throw new Error(`Instance not found: ${instanceId}`);
+    if (this.operation.type === "destroy" || this.operation.type === "recreate") {
+      for (const instanceId of this.operation.requestedInstanceIds) {
+        const instance = this.instanceMap.get(instanceId);
+        if (!instance) {
+          throw new Error(`Instance not found: ${instanceId}`);
+        }
+        if (this.operation.options.destroyDependentInstances) {
+          traverse(instance.id);
+        }
+        this.instanceIdsToDestroy.add(instanceId);
       }
-      if (this.operation.options.destroyDependentInstances) {
-        traverse(instance.id);
+    } else if (this.operation.type === "update") {
+      for (const instanceId of this.operation.instanceIdsToUpdate) {
+        const [type] = parseInstanceId2(instanceId);
+        const component = this.library.components[type];
+        if (!component || isUnitModel3(component)) {
+          continue;
+        }
+        const childrenQueue = [...this.stateChildIdMap.get(instanceId) ?? []];
+        while (childrenQueue.length > 0) {
+          const childId = childrenQueue.pop();
+          if (!this.instanceMap.has(childId)) {
+            this.instanceIdsToDestroy.add(childId);
+          }
+          childrenQueue.push(...this.stateChildIdMap.get(childId) ?? []);
+        }
       }
-      this.affectedInstanceIdSet.add(instanceId);
+    } else {
+      return;
     }
-    const compositeInstanceQueue = Array.from(this.affectedInstanceIdSet);
+    const compositeInstanceQueue = Array.from(this.instanceIdsToDestroy);
     while (compositeInstanceQueue.length > 0) {
       const childId = compositeInstanceQueue.pop();
       const children = this.stateChildIdMap.get(childId) ?? [];
       for (const child of children) {
         compositeInstanceQueue.push(child);
-        this.affectedInstanceIdSet.add(child);
+        this.instanceIdsToDestroy.add(child);
       }
     }
-    for (const instanceId of this.affectedInstanceIdSet) {
+    for (const instanceId of this.instanceIdsToDestroy) {
       let state = this.stateMap.get(instanceId);
       while (state?.parentId) {
-        this.affectedInstanceIdSet.add(state.parentId);
+        this.instanceIdsToDestroy.add(state.parentId);
         state = this.stateMap.get(state.parentId);
       }
     }
-    this.operation.affectedInstanceIds = Array.from(this.affectedInstanceIdSet);
+    this.operation.instanceIdsToDestroy = Array.from(this.instanceIdsToDestroy);
   }
   getSourceHashIfApplicable(instance, component) {
     if (isUnitModel3(component)) {
@@ -2934,14 +2997,7 @@ var OperationWorkset = class _OperationWorkset {
     return inputHash;
   }
   getLockInstanceIds() {
-    const instanceIds = new Set(this.operation.affectedInstanceIds);
-    for (const instanceId of this.operation.affectedInstanceIds) {
-      const instance = this.getInstance(instanceId);
-      if (instance.parentId) {
-        instanceIds.add(instance.parentId);
-      }
-    }
-    return Array.from(instanceIds);
+    return Array.from(this.instanceIdsToUpdate.union(this.instanceIdsToDestroy));
   }
   static async load(operation, projectBackend, libraryBackend, stateBackend, stateManager, logger, signal) {
     const [library, project, compositeInstances, states] = await Promise.all([
@@ -2970,8 +3026,13 @@ var OperationWorkset = class _OperationWorkset {
       if (worksetInstance) {
         worksetInstance.outputs = instance.instance.outputs;
         worksetInstance.resolvedOutputs = instance.instance.resolvedOutputs;
-      } else {
+      } else if (instance.instance.parentId) {
         workset.addInstance(instance.instance);
+      } else {
+        workset.logger.warn(
+          `ignoring instance "${instance.instance.id}" from composite instance state because it is not in the project or is not a part of known composite instance`
+        );
+        continue;
       }
       for (const child of instance.children) {
         if (!workset.instanceMap.has(child.id)) {
@@ -3025,19 +3086,10 @@ var OperationWorkset = class _OperationWorkset {
       logger,
       { promiseCache: workset.inputHashResolverPromiseCache }
     );
-    switch (operation.type) {
-      case "update":
-      case "preview":
-      case "refresh":
-        await workset.extendForUpdate();
-        break;
-      case "recreate":
-        workset.extendForDestroy();
-        break;
-      case "destroy":
-        workset.extendForDestroy();
-        break;
+    if (operation.type !== "destroy") {
+      await workset.calculateInstanceIdsToUpdate();
     }
+    workset.calculateInstanceIdsToDestroy();
     return workset;
   }
 };
@@ -3113,18 +3165,22 @@ var RuntimeOperation = class {
     }
   }
   async processOperation() {
-    this.operation.affectedInstanceIds = this.workset.operation.affectedInstanceIds;
+    this.operation.instanceIdsToUpdate = this.workset.operation.instanceIdsToUpdate;
+    this.operation.instanceIdsToDestroy = this.workset.operation.instanceIdsToDestroy;
     this.logger.info(
-      { affectedInstanceIds: this.operation.affectedInstanceIds },
+      {
+        instanceIdsToUpdate: this.operation.instanceIdsToUpdate,
+        instanceIdsToDestroy: this.operation.instanceIdsToDestroy
+      },
       "operation started"
     );
     const phases = this.getOperationPhases();
     for (const phase of phases) {
       this.currentPhase = phase;
       const promises = [];
-      for (const instanceId of this.operation.affectedInstanceIds) {
-        const instance = this.workset.getInstance(instanceId);
-        if (instance.parentId && this.workset.isAffected(instance.parentId)) {
+      for (const instanceId of this.workset.getAffectedInstanceIds(phase)) {
+        const parentId = this.workset.getParentId(instanceId, phase);
+        if (parentId && this.workset.isAffected(parentId, phase)) {
           continue;
         }
         promises.push(this.getInstancePromiseForOperation(instanceId));
@@ -3145,16 +3201,21 @@ var RuntimeOperation = class {
     this.abortController.abort();
   }
   getInstancePromiseForOperation(instanceId) {
-    const instance = this.workset.getInstance(instanceId);
-    const component = this.workset.library.components[instance.type];
+    const [instanceType] = parseInstanceId3(instanceId);
+    const component = this.workset.library.components[instanceType];
     if (isUnitModel4(component)) {
-      return this.getUnitPromise(instance);
+      return this.getUnitPromise(instanceId);
     }
-    return this.getCompositePromise(instance);
+    return this.getCompositePromise(instanceId);
   }
   getOperationPhases() {
     switch (this.operation.type) {
-      case "update":
+      case "update": {
+        if (this.operation.instanceIdsToDestroy.length > 0) {
+          return ["destroy", "update"];
+        }
+        return ["update"];
+      }
       case "preview":
         return ["update"];
       case "recreate":
@@ -3165,31 +3226,32 @@ var RuntimeOperation = class {
         return ["refresh"];
     }
   }
-  async getUnitPromise(instance) {
+  async getUnitPromise(instanceId) {
     switch (this.currentPhase) {
       case "update": {
-        return this.updateUnit(instance);
+        return this.updateUnit(instanceId);
       }
       case "destroy": {
-        return this.destroyUnit(instance.id);
+        return this.destroyUnit(instanceId);
       }
       case "refresh": {
-        return this.refreshUnit(instance.id);
+        return this.refreshUnit(instanceId);
       }
     }
   }
-  async getCompositePromise(instance) {
-    const logger = this.logger.child({ instanceId: instance.id });
-    return this.getInstancePromise(instance.id, async () => {
-      const state = this.workset.getState(instance.id) ?? createInstanceState(instance.id);
+  async getCompositePromise(instanceId) {
+    const logger = this.logger.child({ instanceId });
+    return this.getInstancePromise(instanceId, async () => {
+      const state = this.workset.getState(instanceId) ?? createInstanceState(instanceId);
+      const instance = this.workset.getInstanceOrUndefined(instanceId);
       this.updateInstanceState({
         ...state,
-        parentId: instance.parentId,
+        parentId: instance?.parentId,
         latestOperationId: this.operation.id,
         status: this.getStatusByOperationType(),
         error: null
       });
-      const children = this.workset.getAffectedCompositeChildren(instance.id);
+      const children = this.workset.getAffectedCompositeChildren(instanceId, this.currentPhase);
       const childPromises = [];
       if (children.length) {
         logger.info("running %s children", children.length);
@@ -3197,6 +3259,9 @@ var RuntimeOperation = class {
         logger.warn("no affected children found for composite component");
       }
       for (const child of children) {
+        if (!this.operation.options.forceUpdateChildren && !this.workset.isAffected(child.id, this.currentPhase)) {
+          continue;
+        }
         logger.debug(`waiting for child: "%s"`, child.id);
         childPromises.push(this.getInstancePromiseForOperation(child.id));
       }
@@ -3206,25 +3271,26 @@ var RuntimeOperation = class {
           logger.info("all children completed");
         }
         this.updateInstanceState({
-          id: instance.id,
+          id: instanceId,
           status: this.operation.type === "destroy" ? "not_created" : "created",
-          inputHash: await this.workset.getUpToDateInputHash(instance)
+          inputHash: instance ? await this.workset.getUpToDateInputHash(instance) : void 0
         });
       } catch (error) {
         if (isAbortErrorLike(error)) {
-          this.workset.restoreInitialStatus(instance.id);
+          this.workset.restoreInitialStatus(instanceId);
           return;
         }
         this.updateInstanceState({
-          id: instance.id,
+          id: instanceId,
           status: "error",
           error: errorToString(error)
         });
       }
     });
   }
-  updateUnit(instance) {
-    return this.getInstancePromise(instance.id, async (logger) => {
+  updateUnit(instanceId) {
+    return this.getInstancePromise(instanceId, async (logger) => {
+      const instance = this.workset.getInstance(instanceId);
       this.updateInstanceState({
         id: instance.id,
         parentId: instance.parentId,
@@ -3276,7 +3342,7 @@ var RuntimeOperation = class {
       const dependencies = this.getInstanceDependencies(instance);
       const dependencyPromises = [];
       for (const dependency of dependencies) {
-        if (!this.operation.affectedInstanceIds.includes(dependency.id)) {
+        if (!this.operation.instanceIdsToUpdate.includes(dependency.id)) {
           continue;
         }
         logger.debug(`waiting for dependency: ${dependency.id}`);
@@ -3350,7 +3416,7 @@ var RuntimeOperation = class {
       this.abortController.signal.throwIfAborted();
       await this.processBeforeDestroyTriggers(state, logger);
       logger.info("destroying unit...");
-      const [type, name] = parseInstanceId2(instanceId);
+      const [type, name] = parseInstanceId3(instanceId);
       await this.runnerBackend.destroy({
         projectId: this.operation.projectId,
         instanceType: type,
@@ -3382,7 +3448,7 @@ var RuntimeOperation = class {
         totalResourceCount: 0
       });
       logger.info("refreshing unit...");
-      const [type, name] = parseInstanceId2(instanceId);
+      const [type, name] = parseInstanceId3(instanceId);
       await this.runnerBackend.refresh({
         projectId: this.operation.projectId,
         instanceType: type,
@@ -3565,11 +3631,13 @@ var OperationManager = class _OperationManager {
       id: uuidv72(),
       projectId: request.projectId,
       type: request.type,
-      instanceIds: request.instanceIds,
-      affectedInstanceIds: request.instanceIds,
+      requestedInstanceIds: request.instanceIds,
+      instanceIdsToUpdate: [],
+      instanceIdsToDestroy: [],
       status: "pending",
       options: {
         forceUpdateDependencies: request.options?.forceUpdateDependencies ?? false,
+        forceUpdateChildren: request.options?.forceUpdateChildren ?? false,
         destroyDependentInstances: request.options?.destroyDependentInstances ?? true,
         invokeDestroyTriggers: request.options?.invokeDestroyTriggers ?? false,
         deleteUnreachableResources: request.options?.deleteUnreachableResources ?? false,