@highstate/backend 0.9.8 → 0.9.10

package/dist/library/worker/main.js

@@ -1,6 +1,6 @@
 import {
   errorToString
-} from "../../chunk-EQ4LMS7B.js";
+} from "../../chunk-WXDYCRTT.js";
 
 // src/library/worker/main.ts
 import { parentPort, workerData } from "node:worker_threads";

package/dist/shared/index.js

@@ -1,16 +1,15 @@
 import {
-  CircularDependencyError,
+  GraphResolver,
+  InputHashResolver,
+  InputResolver,
+  ValidationResolver,
   applyLibraryUpdate,
   applyPartialInstanceState,
   buildDependentInstanceStateMap,
   compositeInstanceSchema,
-  createDefaultGraphResolverBackend,
-  createInputHashResolver,
-  createInputResolver,
+  createAsyncBatcher,
   createInstanceState,
   createInstanceStatePatch,
-  createValidationResolver,
-  defineGraphResolver,
   diffLibraries,
   getAllDependentInstanceIds,
   getMatchedInjectionInstanceInputs,
@@ -48,20 +47,19 @@ import {
   projectOperationSchema,
   resolverFactories,
   terminalSessionSchema
-} from "../chunk-NMGIUI6X.js";
+} from "../chunk-DQDXRDUA.js";
 export {
-  CircularDependencyError,
+  GraphResolver,
+  InputHashResolver,
+  InputResolver,
+  ValidationResolver,
   applyLibraryUpdate,
   applyPartialInstanceState,
   buildDependentInstanceStateMap,
   compositeInstanceSchema,
-  createDefaultGraphResolverBackend,
-  createInputHashResolver,
-  createInputResolver,
+  createAsyncBatcher,
   createInstanceState,
   createInstanceStatePatch,
-  createValidationResolver,
-  defineGraphResolver,
   diffLibraries,
   getAllDependentInstanceIds,
   getMatchedInjectionInstanceInputs,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@highstate/backend",
-  "version": "0.9.8",
+  "version": "0.9.10",
   "type": "module",
   "files": [
     "dist",
@@ -26,7 +26,7 @@
     "build": "highstate build"
   },
   "dependencies": {
-    "@highstate/contract": "^0.9.8",
+    "@highstate/contract": "^0.9.10",
     "@types/node": "^22.10.1",
     "ajv": "^8.17.1",
     "better-lock": "^3.2.0",
@@ -65,5 +65,5 @@
     "rollup": "^4.28.1",
     "typescript": "^5.7.2"
   },
-  "gitHead": "036db4d9937ff30edf15f143482c5702e5b7a7fb"
+  "gitHead": "aacf8837fdf40f2bb2f83d4c11b35a358e26ec33"
 }

package/src/common/utils.ts

@@ -61,77 +61,3 @@ export function errorToString(error: unknown): string {
 
   return JSON.stringify(error)
 }
-
-export type AsyncBatcherOptions = {
-  waitMs?: number
-  maxWaitTimeMs?: number
-}
-
-export function createAsyncBatcher<T>(
-  fn: (items: T[]) => Promise<void>,
-  { waitMs = 100, maxWaitTimeMs = 1000 }: AsyncBatcherOptions = {},
-) {
-  let batch: T[] = []
-  let activeTimeout: NodeJS.Timeout | null = null
-  let maxWaitTimeout: NodeJS.Timeout | null = null
-  let firstCallTimestamp: number | null = null
-
-  async function processBatch() {
-    if (batch.length === 0) return
-
-    const currentBatch = batch
-    batch = [] // Reset batch before async call
-
-    await fn(currentBatch)
-
-    // Clear max wait timer since batch has been processed
-    if (maxWaitTimeout) {
-      clearTimeout(maxWaitTimeout)
-      maxWaitTimeout = null
-    }
-    firstCallTimestamp = null
-  }
-
-  function schedule() {
-    if (activeTimeout) clearTimeout(activeTimeout)
-    activeTimeout = setTimeout(() => {
-      activeTimeout = null
-      void processBatch()
-    }, waitMs)
-
-    // Ensure batch is executed within maxWaitTimeMs
-    if (!firstCallTimestamp) {
-      firstCallTimestamp = Date.now()
-      maxWaitTimeout = setTimeout(() => {
-        if (activeTimeout) clearTimeout(activeTimeout)
-        activeTimeout = null
-        void processBatch()
-      }, maxWaitTimeMs)
-    }
-  }
-
-  return {
-    /**
-     * Add an item to the batch.
-     */
-    call(item: T): void {
-      batch.push(item)
-      schedule()
-    },
-
-    /**
-     * Immediately flush the pending batch (if any).
-     */
-    async flush(): Promise<void> {
-      if (activeTimeout) {
-        clearTimeout(activeTimeout)
-        activeTimeout = null
-      }
-      if (maxWaitTimeout) {
-        clearTimeout(maxWaitTimeout)
-        maxWaitTimeout = null
-      }
-      await processBatch()
-    },
-  }
-}

package/src/library/abstractions.ts

@@ -42,11 +42,6 @@ export interface LibraryBackend {
    */
   watchLibrary(signal?: AbortSignal): AsyncIterable<LibraryUpdate[]>
 
-  /**
-   * Gets the currently loaded resolved unit sources.
-   */
-  getLoadedResolvedUnitSources(): Promise<ResolvedUnitSource[]>
-
   /**
    * Gets the resolved unit sources for the given unit types.
    *

package/src/orchestrator/operation-workset.ts

@@ -9,19 +9,18 @@ import {
   type InstanceModel,
 } from "@highstate/contract"
 import { unique } from "remeda"
+import { BetterLock } from "better-lock"
 import {
   applyPartialInstanceState,
-  createInputHashResolver,
-  createInputResolver,
   createInstanceState,
   createInstanceStatePatch,
-  type GraphResolver,
-  type InputHashResolverInput,
-  type InputHashResolverOutput,
-  type InputResolverInput,
-  type InputResolverOutput,
+  InputHashResolver,
+  InputResolver,
+  type InputHashNode,
+  type InputResolverNode,
   type InstanceState,
   type InstanceStateUpdate,
+  type InstanceStatus,
   type LibraryModel,
   type ProjectOperation,
   type ResolvedInstanceInput,
@@ -43,17 +42,12 @@ export class OperationWorkset {
 
   private readonly unitSourceHashMap = new Map<string, string>()
 
-  private inputResolver!: GraphResolver<InputResolverOutput>
-  private readonly inputResolverInputs = new Map<string, InputResolverInput>()
-  private readonly inputResolverPromiseCache = new Map<string, Promise<InputResolverOutput>>()
+  private inputResolver!: InputResolver
+  private readonly inputResolverNodes = new Map<string, InputResolverNode>()
 
-  private inputHashResolver!: GraphResolver<InputHashResolverOutput>
-  private readonly inputHashResolverInputs = new Map<string, InputHashResolverInput>()
-
-  private readonly inputHashResolverPromiseCache = new Map<
-    string,
-    Promise<InputHashResolverOutput>
-  >()
+  private inputHashResolver!: InputHashResolver
+  private readonly inputHashNodes = new Map<string, InputHashNode>()
+  private readonly inputHashResolverLock = new BetterLock()
 
   public readonly resolvedInstanceInputs = new Map<
     string,
@@ -96,12 +90,12 @@ export class OperationWorkset {
     return this.instanceIdsToUpdate.has(instanceId)
   }
 
-  public updateState(update: InstanceStateUpdate): InstanceState {
+  public updateState(update: InstanceStateUpdate, phase: OperationPhase): InstanceState {
     const finalState = applyPartialInstanceState(this.stateMap, update)
     this.stateManager.emitStatePatch(this.operation.projectId, createInstanceStatePatch(update))
 
     if (finalState.parentId) {
-      this.recalculateCompositeInstanceState(finalState.parentId)
+      this.recalculateCompositeInstanceState(finalState.parentId, phase)
     }
 
     return finalState
@@ -202,7 +196,7 @@ export class OperationWorkset {
       .filter((state): state is InstanceState => !!state)
   }
 
-  private recalculateCompositeInstanceState(instanceId: string): void {
+  private recalculateCompositeInstanceState(instanceId: string, phase: OperationPhase): void {
     const state = this.stateMap.get(instanceId) ?? createInstanceState(instanceId)
     let currentResourceCount = 0
     let totalResourceCount = 0
@@ -222,6 +216,7 @@ export class OperationWorkset {
 
     const updatedState = {
       ...state,
+      status: OperationWorkset.getStatusByOperationPhase(phase),
       currentResourceCount,
       totalResourceCount,
     }
@@ -230,7 +225,7 @@ export class OperationWorkset {
     this.stateManager.emitStatePatch(this.operation.projectId, updatedState)
 
     if (state.parentId) {
-      this.recalculateCompositeInstanceState(state.parentId)
+      this.recalculateCompositeInstanceState(state.parentId, phase)
     }
   }
 
@@ -279,7 +274,7 @@ export class OperationWorkset {
       }
 
       const state = this.stateMap.get(instance.id)
-      const { inputHash: expectedInputHash } = await this.inputHashResolver(instance.id)
+      const { inputHash: expectedInputHash } = this.inputHashResolver.requireOutput(instance.id)
 
       if (this.operation.options.forceUpdateDependencies) {
         this.instanceIdsToUpdate.add(instanceId)
@@ -315,7 +310,7 @@ export class OperationWorkset {
         }
 
         const state = this.stateMap.get(child.id)
-        const { inputHash: expectedInputHash } = await this.inputHashResolver(child.id)
+        const { inputHash: expectedInputHash } = this.inputHashResolver.requireOutput(child.id)
 
         if (state?.status !== "created" || state.inputHash !== expectedInputHash) {
           this.instanceIdsToUpdate.add(child.id)
@@ -431,21 +426,35 @@ export class OperationWorkset {
     return undefined
   }
 
+  private static getStatusByOperationPhase(phase: OperationPhase): InstanceStatus {
+    switch (phase) {
+      case "update":
+        return "updating"
+      case "destroy":
+        return "destroying"
+      case "refresh":
+        return "refreshing"
+    }
+  }
+
   public async getUpToDateInputHash(instance: InstanceModel): Promise<string> {
-    const component = this.library.components[instance.type]
-
-    this.inputHashResolverInputs.set(instance.id, {
-      instance,
-      component,
-      resolvedInputs: this.resolvedInstanceInputs.get(instance.id)!,
-      state: this.stateMap.get(instance.id),
-      sourceHash: this.getSourceHashIfApplicable(instance, component),
-    })
+    return await this.inputHashResolverLock.acquire(async () => {
+      const component = this.library.components[instance.type]
 
-    this.inputHashResolverPromiseCache.delete(instance.id)
+      this.inputHashNodes.set(instance.id, {
+        instance,
+        component,
+        resolvedInputs: this.resolvedInstanceInputs.get(instance.id)!,
+        state: this.stateMap.get(instance.id),
+        sourceHash: this.getSourceHashIfApplicable(instance, component),
+      })
 
-    const { inputHash } = await this.inputHashResolver(instance.id)
-    return inputHash
+      this.inputHashResolver.invalidate(instance.id)
+      await this.inputHashResolver.process()
+
+      const { inputHash } = this.inputHashResolver.requireOutput(instance.id)
+      return inputHash
+    })
   }
 
   public getLockInstanceIds(): string[] {
@@ -468,10 +477,6 @@ export class OperationWorkset {
       stateBackend.getAllInstanceStates(operation.projectId, signal),
     ])
 
-    const unitSources = await libraryBackend.getResolvedUnitSources(
-      unique(project.instances.map(i => i.type)),
-    )
-
     const workset = new OperationWorkset(
       operation,
       library,
@@ -479,10 +484,6 @@ export class OperationWorkset {
       logger.child({ operationId: operation.id, service: "OperationWorkset" }),
     )
 
-    for (const unitSource of unitSources) {
-      workset.unitSourceHashMap.set(unitSource.unitType, unitSource.sourceHash)
-    }
-
     // prepare instances
     for (const instance of project.instances) {
       workset.addInstance(instance)
@@ -510,6 +511,14 @@ export class OperationWorkset {
       }
     }
 
+    const unitSources = await libraryBackend.getResolvedUnitSources(
+      unique(Array.from(workset.instanceMap.values().map(i => i.type))),
+    )
+
+    for (const unitSource of unitSources) {
+      workset.unitSourceHashMap.set(unitSource.unitType, unitSource.sourceHash)
+    }
+
     for (const state of states) {
       if (!workset.instanceMap.has(state.id)) {
         workset.logger.warn(
@@ -523,7 +532,7 @@ export class OperationWorkset {
 
     // prepare input resolver
     for (const instance of workset.instanceMap.values()) {
-      workset.inputResolverInputs.set(`instance:${instance.id}`, {
+      workset.inputResolverNodes.set(`instance:${instance.id}`, {
         kind: "instance",
         instance,
         component: library.components[instance.type],
@@ -531,19 +540,17 @@ export class OperationWorkset {
     }
 
     for (const hub of project.hubs) {
-      workset.inputResolverInputs.set(`hub:${hub.id}`, { kind: "hub", hub })
+      workset.inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
     }
 
-    workset.inputResolver = createInputResolver(
-      //
-      workset.inputResolverInputs,
-      logger,
-      { promiseCache: workset.inputResolverPromiseCache },
-    )
+    workset.inputResolver = new InputResolver(workset.inputResolverNodes, logger)
+    workset.inputResolver.addAllNodesToWorkset()
+
+    await workset.inputResolver.process()
 
     // resolve inputs for all instances and pass outputs to input hash resolver
     for (const instance of workset.instanceMap.values()) {
-      const output = await workset.inputResolver(`instance:${instance.id}`)
+      const output = workset.inputResolver.requireOutput(`instance:${instance.id}`)
       if (output.kind !== "instance") {
         throw new Error("Unexpected output kind")
       }
@@ -552,7 +559,7 @@ export class OperationWorkset {
 
       const component = workset.library.components[instance.type]
 
-      workset.inputHashResolverInputs.set(instance.id, {
+      workset.inputHashNodes.set(instance.id, {
         instance,
         component,
         resolvedInputs: output.resolvedInputs,
@@ -562,12 +569,10 @@ export class OperationWorkset {
     }
 
     // prepare input hash resolver
-    workset.inputHashResolver = createInputHashResolver(
-      //
-      workset.inputHashResolverInputs,
-      logger,
-      { promiseCache: workset.inputHashResolverPromiseCache },
-    )
+    workset.inputHashResolver = new InputHashResolver(workset.inputHashNodes, logger)
+    workset.inputHashResolver.addAllNodesToWorkset()
+
+    await workset.inputHashResolver.process()
 
     if (operation.type !== "destroy") {
       await workset.calculateInstanceIdsToUpdate()

package/src/orchestrator/operation.ts

@@ -11,13 +11,12 @@ import { mapValues } from "remeda"
 import {
   type InstanceState,
   type InstanceStateUpdate,
-  type InstanceStatus,
   type ProjectOperation,
   type InstanceTriggerInvocation,
   createInstanceState,
+  createAsyncBatcher,
 } from "../shared"
 import {
-  createAsyncBatcher,
   errorToString,
   isAbortError,
   isAbortErrorLike,
@@ -220,7 +219,7 @@ export class RuntimeOperation {
         ...state,
         parentId: instance?.parentId,
         latestOperationId: this.operation.id,
-        status: this.getStatusByOperationType(),
+        status: "pending",
         error: null,
       })
 
@@ -575,7 +574,7 @@ export class RuntimeOperation {
       return
     }
 
-    const state = this.workset.updateState(patch)
+    const state = this.workset.updateState(patch, this.currentPhase)
 
     // do not persist anyting for preview operations
     if (this.operation.type !== "preview") {
@@ -604,21 +603,6 @@ export class RuntimeOperation {
     return instancePromise
   }
 
-  private getStatusByOperationType(): InstanceStatus {
-    switch (this.operation.type) {
-      case "update":
-        return "updating"
-      case "preview":
-        return "previewing"
-      case "recreate":
-        return "updating"
-      case "destroy":
-        return "destroying"
-      case "refresh":
-        return "refreshing"
-    }
-  }
-
   private getInstanceDependencies(instance: InstanceModel): InstanceModel[] {
     const dependencies: InstanceModel[] = []
     const instanceInputs = this.workset.resolvedInstanceInputs.get(instance.id) ?? {}