@highstate/backend 0.9.9 → 0.9.11

package/dist/library/worker/main.js

@@ -1,6 +1,6 @@
 import {
   errorToString
-} from "../../chunk-EQ4LMS7B.js";
+} from "../../chunk-WXDYCRTT.js";
 
 // src/library/worker/main.ts
 import { parentPort, workerData } from "node:worker_threads";

package/dist/shared/index.js

@@ -1,16 +1,15 @@
 import {
-  CircularDependencyError,
+  GraphResolver,
+  InputHashResolver,
+  InputResolver,
+  ValidationResolver,
   applyLibraryUpdate,
   applyPartialInstanceState,
   buildDependentInstanceStateMap,
   compositeInstanceSchema,
-  createDefaultGraphResolverBackend,
-  createInputHashResolver,
-  createInputResolver,
+  createAsyncBatcher,
   createInstanceState,
   createInstanceStatePatch,
-  createValidationResolver,
-  defineGraphResolver,
   diffLibraries,
   getAllDependentInstanceIds,
   getMatchedInjectionInstanceInputs,
@@ -48,20 +47,19 @@ import {
   projectOperationSchema,
   resolverFactories,
   terminalSessionSchema
-} from "../chunk-NMGIUI6X.js";
+} from "../chunk-DQDXRDUA.js";
 export {
-  CircularDependencyError,
+  GraphResolver,
+  InputHashResolver,
+  InputResolver,
+  ValidationResolver,
   applyLibraryUpdate,
   applyPartialInstanceState,
   buildDependentInstanceStateMap,
   compositeInstanceSchema,
-  createDefaultGraphResolverBackend,
-  createInputHashResolver,
-  createInputResolver,
+  createAsyncBatcher,
   createInstanceState,
   createInstanceStatePatch,
-  createValidationResolver,
-  defineGraphResolver,
   diffLibraries,
   getAllDependentInstanceIds,
   getMatchedInjectionInstanceInputs,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@highstate/backend",
-  "version": "0.9.9",
+  "version": "0.9.11",
   "type": "module",
   "files": [
     "dist",
@@ -26,7 +26,7 @@
     "build": "highstate build"
   },
   "dependencies": {
-    "@highstate/contract": "^0.9.9",
+    "@highstate/contract": "^0.9.11",
     "@types/node": "^22.10.1",
     "ajv": "^8.17.1",
     "better-lock": "^3.2.0",
@@ -65,5 +65,5 @@
     "rollup": "^4.28.1",
     "typescript": "^5.7.2"
   },
-  "gitHead": "1c04a713fa1bb7c0231e5e6d0537709097bb0e8e"
+  "gitHead": "d62cae30ea9fadaa1aa4fb3b5734a16cf53810ce"
 }

package/src/common/utils.ts

@@ -61,77 +61,3 @@ export function errorToString(error: unknown): string {
 
   return JSON.stringify(error)
 }
-
-export type AsyncBatcherOptions = {
-  waitMs?: number
-  maxWaitTimeMs?: number
-}
-
-export function createAsyncBatcher<T>(
-  fn: (items: T[]) => Promise<void>,
-  { waitMs = 100, maxWaitTimeMs = 1000 }: AsyncBatcherOptions = {},
-) {
-  let batch: T[] = []
-  let activeTimeout: NodeJS.Timeout | null = null
-  let maxWaitTimeout: NodeJS.Timeout | null = null
-  let firstCallTimestamp: number | null = null
-
-  async function processBatch() {
-    if (batch.length === 0) return
-
-    const currentBatch = batch
-    batch = [] // Reset batch before async call
-
-    await fn(currentBatch)
-
-    // Clear max wait timer since batch has been processed
-    if (maxWaitTimeout) {
-      clearTimeout(maxWaitTimeout)
-      maxWaitTimeout = null
-    }
-    firstCallTimestamp = null
-  }
-
-  function schedule() {
-    if (activeTimeout) clearTimeout(activeTimeout)
-    activeTimeout = setTimeout(() => {
-      activeTimeout = null
-      void processBatch()
-    }, waitMs)
-
-    // Ensure batch is executed within maxWaitTimeMs
-    if (!firstCallTimestamp) {
-      firstCallTimestamp = Date.now()
-      maxWaitTimeout = setTimeout(() => {
-        if (activeTimeout) clearTimeout(activeTimeout)
-        activeTimeout = null
-        void processBatch()
-      }, maxWaitTimeMs)
-    }
-  }
-
-  return {
-    /**
-     * Add an item to the batch.
-     */
-    call(item: T): void {
-      batch.push(item)
-      schedule()
-    },
-
-    /**
-     * Immediately flush the pending batch (if any).
-     */
-    async flush(): Promise<void> {
-      if (activeTimeout) {
-        clearTimeout(activeTimeout)
-        activeTimeout = null
-      }
-      if (maxWaitTimeout) {
-        clearTimeout(maxWaitTimeout)
-        maxWaitTimeout = null
-      }
-      await processBatch()
-    },
-  }
-}

package/src/library/abstractions.ts

@@ -42,11 +42,6 @@ export interface LibraryBackend {
    */
   watchLibrary(signal?: AbortSignal): AsyncIterable<LibraryUpdate[]>
 
-  /**
-   * Gets the currently loaded resolved unit sources.
-   */
-  getLoadedResolvedUnitSources(): Promise<ResolvedUnitSource[]>
-
   /**
    * Gets the resolved unit sources for the given unit types.
    *

package/src/orchestrator/operation-workset.ts

@@ -9,17 +9,15 @@ import {
   type InstanceModel,
 } from "@highstate/contract"
 import { unique } from "remeda"
+import { BetterLock } from "better-lock"
 import {
   applyPartialInstanceState,
-  createInputHashResolver,
-  createInputResolver,
   createInstanceState,
   createInstanceStatePatch,
-  type GraphResolver,
-  type InputHashResolverInput,
-  type InputHashResolverOutput,
-  type InputResolverInput,
-  type InputResolverOutput,
+  InputHashResolver,
+  InputResolver,
+  type InputHashNode,
+  type InputResolverNode,
   type InstanceState,
   type InstanceStateUpdate,
   type InstanceStatus,
@@ -44,17 +42,12 @@ export class OperationWorkset {
 
   private readonly unitSourceHashMap = new Map<string, string>()
 
-  private inputResolver!: GraphResolver<InputResolverOutput>
-  private readonly inputResolverInputs = new Map<string, InputResolverInput>()
-  private readonly inputResolverPromiseCache = new Map<string, Promise<InputResolverOutput>>()
+  private inputResolver!: InputResolver
+  private readonly inputResolverNodes = new Map<string, InputResolverNode>()
 
-  private inputHashResolver!: GraphResolver<InputHashResolverOutput>
-  private readonly inputHashResolverInputs = new Map<string, InputHashResolverInput>()
-
-  private readonly inputHashResolverPromiseCache = new Map<
-    string,
-    Promise<InputHashResolverOutput>
-  >()
+  private inputHashResolver!: InputHashResolver
+  private readonly inputHashNodes = new Map<string, InputHashNode>()
+  private readonly inputHashResolverLock = new BetterLock()
 
   public readonly resolvedInstanceInputs = new Map<
     string,
@@ -281,7 +274,7 @@ export class OperationWorkset {
       }
 
       const state = this.stateMap.get(instance.id)
-      const { inputHash: expectedInputHash } = await this.inputHashResolver(instance.id)
+      const { inputHash: expectedInputHash } = this.inputHashResolver.requireOutput(instance.id)
 
       if (this.operation.options.forceUpdateDependencies) {
         this.instanceIdsToUpdate.add(instanceId)
@@ -317,7 +310,7 @@ export class OperationWorkset {
         }
 
         const state = this.stateMap.get(child.id)
-        const { inputHash: expectedInputHash } = await this.inputHashResolver(child.id)
+        const { inputHash: expectedInputHash } = this.inputHashResolver.requireOutput(child.id)
 
         if (state?.status !== "created" || state.inputHash !== expectedInputHash) {
           this.instanceIdsToUpdate.add(child.id)
@@ -445,20 +438,23 @@ export class OperationWorkset {
   }
 
   public async getUpToDateInputHash(instance: InstanceModel): Promise<string> {
-    const component = this.library.components[instance.type]
-
-    this.inputHashResolverInputs.set(instance.id, {
-      instance,
-      component,
-      resolvedInputs: this.resolvedInstanceInputs.get(instance.id)!,
-      state: this.stateMap.get(instance.id),
-      sourceHash: this.getSourceHashIfApplicable(instance, component),
-    })
+    return await this.inputHashResolverLock.acquire(async () => {
+      const component = this.library.components[instance.type]
+
+      this.inputHashNodes.set(instance.id, {
+        instance,
+        component,
+        resolvedInputs: this.resolvedInstanceInputs.get(instance.id)!,
+        state: this.stateMap.get(instance.id),
+        sourceHash: this.getSourceHashIfApplicable(instance, component),
+      })
 
-    this.inputHashResolverPromiseCache.delete(instance.id)
+      this.inputHashResolver.invalidate(instance.id)
+      await this.inputHashResolver.process()
 
-    const { inputHash } = await this.inputHashResolver(instance.id)
-    return inputHash
+      const { inputHash } = this.inputHashResolver.requireOutput(instance.id)
+      return inputHash
+    })
   }
 
   public getLockInstanceIds(): string[] {
@@ -481,10 +477,6 @@ export class OperationWorkset {
       stateBackend.getAllInstanceStates(operation.projectId, signal),
     ])
 
-    const unitSources = await libraryBackend.getResolvedUnitSources(
-      unique(project.instances.map(i => i.type)),
-    )
-
     const workset = new OperationWorkset(
       operation,
       library,
@@ -492,10 +484,6 @@ export class OperationWorkset {
       logger.child({ operationId: operation.id, service: "OperationWorkset" }),
     )
 
-    for (const unitSource of unitSources) {
-      workset.unitSourceHashMap.set(unitSource.unitType, unitSource.sourceHash)
-    }
-
     // prepare instances
     for (const instance of project.instances) {
       workset.addInstance(instance)
@@ -523,6 +511,14 @@ export class OperationWorkset {
       }
     }
 
+    const unitSources = await libraryBackend.getResolvedUnitSources(
+      unique(Array.from(workset.instanceMap.values().map(i => i.type))),
+    )
+
+    for (const unitSource of unitSources) {
+      workset.unitSourceHashMap.set(unitSource.unitType, unitSource.sourceHash)
+    }
+
     for (const state of states) {
       if (!workset.instanceMap.has(state.id)) {
         workset.logger.warn(
@@ -536,7 +532,7 @@ export class OperationWorkset {
 
     // prepare input resolver
     for (const instance of workset.instanceMap.values()) {
-      workset.inputResolverInputs.set(`instance:${instance.id}`, {
+      workset.inputResolverNodes.set(`instance:${instance.id}`, {
         kind: "instance",
         instance,
         component: library.components[instance.type],
@@ -544,19 +540,17 @@ export class OperationWorkset {
     }
 
     for (const hub of project.hubs) {
-      workset.inputResolverInputs.set(`hub:${hub.id}`, { kind: "hub", hub })
+      workset.inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
     }
 
-    workset.inputResolver = createInputResolver(
-      //
-      workset.inputResolverInputs,
-      logger,
-      { promiseCache: workset.inputResolverPromiseCache },
-    )
+    workset.inputResolver = new InputResolver(workset.inputResolverNodes, logger)
+    workset.inputResolver.addAllNodesToWorkset()
+
+    await workset.inputResolver.process()
 
     // resolve inputs for all instances and pass outputs to input hash resolver
     for (const instance of workset.instanceMap.values()) {
-      const output = await workset.inputResolver(`instance:${instance.id}`)
+      const output = workset.inputResolver.requireOutput(`instance:${instance.id}`)
       if (output.kind !== "instance") {
         throw new Error("Unexpected output kind")
       }
@@ -565,7 +559,7 @@ export class OperationWorkset {
 
       const component = workset.library.components[instance.type]
 
-      workset.inputHashResolverInputs.set(instance.id, {
+      workset.inputHashNodes.set(instance.id, {
         instance,
         component,
         resolvedInputs: output.resolvedInputs,
@@ -575,12 +569,10 @@ export class OperationWorkset {
     }
 
     // prepare input hash resolver
-    workset.inputHashResolver = createInputHashResolver(
-      //
-      workset.inputHashResolverInputs,
-      logger,
-      { promiseCache: workset.inputHashResolverPromiseCache },
-    )
+    workset.inputHashResolver = new InputHashResolver(workset.inputHashNodes, logger)
+    workset.inputHashResolver.addAllNodesToWorkset()
+
+    await workset.inputHashResolver.process()
 
     if (operation.type !== "destroy") {
       await workset.calculateInstanceIdsToUpdate()

package/src/orchestrator/operation.ts

@@ -14,9 +14,9 @@ import {
   type ProjectOperation,
   type InstanceTriggerInvocation,
   createInstanceState,
+  createAsyncBatcher,
 } from "../shared"
 import {
-  createAsyncBatcher,
   errorToString,
   isAbortError,
   isAbortErrorLike,

package/src/project/manager.ts

@@ -6,16 +6,16 @@ import type { ProjectLockManager } from "./lock"
 import { EventEmitter, on } from "node:events"
 import { isUnitModel, type InstanceModel } from "@highstate/contract"
 import {
-  createInputResolver,
-  createInputHashResolver,
-  type InputResolverInput,
-  type InputHashResolverInput,
+  type InputResolverNode,
+  type InputHashNode,
   type InstanceModelPatch,
   type LibraryUpdate,
   createInstanceState,
   type CompositeInstance,
   type ResolvedInstanceInput,
   type HubModel,
+  InputResolver,
+  InputHashResolver,
 } from "../shared"
 
 type CompositeInstanceEvent =
@@ -31,6 +31,10 @@ type CompositeInstanceEvent =
       type: "deleted"
       instanceId: string
     }
+  | {
+      type: "failed"
+      instanceId: string
+    }
 
 type CompositeInstanceEvents = {
   [K in string]: [CompositeInstanceEvent]
@@ -98,7 +102,7 @@ export class ProjectManager {
 
   async createInstance(projectId: string, instance: InstanceModel): Promise<InstanceModel> {
     const createdInstance = await this.projectBackend.createInstance(projectId, instance)
-    await this.updateCompositeInstance(projectId, createdInstance)
+    await this.evaluateChangedCompositeInstances(projectId)
 
     return createdInstance
   }
@@ -109,7 +113,7 @@ export class ProjectManager {
     patch: InstanceModelPatch,
   ): Promise<InstanceModel> {
     const instance = await this.projectBackend.updateInstance(projectId, instanceId, patch)
-    await this.updateCompositeInstance(projectId, instance)
+    await this.evaluateChangedCompositeInstances(projectId)
 
     return instance
   }
@@ -120,7 +124,7 @@ export class ProjectManager {
     newName: string,
   ): Promise<InstanceModel> {
     const instance = await this.projectBackend.renameInstance(projectId, instanceId, newName)
-    await this.updateCompositeInstance(projectId, instance)
+    await this.evaluateChangedCompositeInstances(projectId)
 
     return instance
   }
@@ -132,32 +136,22 @@ export class ProjectManager {
     ])
   }
 
-  private async updateCompositeInstance(projectId: string, instance: InstanceModel): Promise<void> {
-    const { resolveInputHash, library } = await this.prepareInputHashResolver(projectId)
-
-    const component = library.components[instance.type]
-    if (!component) {
-      return
-    }
-
-    if (isUnitModel(component)) {
-      return
-    }
-
-    const { inputHash: expectedInputHash } = await resolveInputHash(instance.id)
-    const inputHash = await this.stateBackend.getCompositeInstanceInputHash(projectId, instance.id)
+  private async evaluateChangedCompositeInstances(projectId: string): Promise<void> {
+    const { inputHashResolver, instances, library, evaluatedInputHashes } =
+      await this.prepareResolvers(projectId)
 
-    if (inputHash !== expectedInputHash) {
-      this.logger.info("re-evaluating instance since input hash has changed", {
-        projectId,
-        instanceId: instance.id,
-      })
+    inputHashResolver.addAllNodesToWorkset()
+    await inputHashResolver.process()
 
-      await this.evaluateCompositeInstances(projectId, [instance.id])
-    }
-  }
+    const instanceIds = instances
+      .filter(instance => !isUnitModel(library.components[instance.type]))
+      .filter(
+        instance =>
+          evaluatedInputHashes[instance.id] !==
+          inputHashResolver.requireOutput(instance.id).inputHash,
+      )
+      .map(instance => instance.id)
 
-  private async evaluateCompositeInstances(projectId: string, instanceIds: string[]) {
     await this.projectLockManager.getLock(projectId).lockInstances(instanceIds, async () => {
       this.logger.debug({ instanceIds }, "evaluating composite instances")
 
@@ -166,10 +160,10 @@ export class ProjectManager {
       }
 
       const [
-        { instances, resolvedInputs, stateMap, resolveInputHash },
+        { instances, resolvedInputs, stateMap, inputHashResolver },
         topLevelCompositeChildrenIds,
       ] = await Promise.all([
-        this.prepareInputHashResolver(projectId),
+        this.prepareResolvers(projectId),
         this.stateBackend.getTopLevelCompositeChildrenIds(projectId, instanceIds),
       ])
 
@@ -188,18 +182,19 @@ export class ProjectManager {
         return newState
       })
 
-      const inputHashes = new Map<string, string>()
-      for (const instanceId of instanceIds) {
-        const { inputHash } = await resolveInputHash(instanceId)
-        inputHashes.set(instanceId, inputHash)
-      }
+      inputHashResolver.addAllNodesToWorkset()
+      await inputHashResolver.process()
 
       const compositeInstances = results
         .filter(result => result.success)
         .flatMap(result =>
           result.compositeInstances.map(instance => ({
             ...instance,
-            inputHash: inputHashes.get(instance.instance.id),
+            inputHash:
+              // only store inputHash for top-level composite instances
+              instance.instance.id === result.instanceId
+                ? inputHashResolver.requireOutput(instance.instance.id).inputHash
+                : "",
           })),
         )
 
@@ -235,6 +230,18 @@ export class ProjectManager {
 
       for (const state of newStates) {
         this.stateManager.emitStatePatch(projectId, state)
+
+        if (state.evaluationError) {
+          this.logger.error(
+            { projectId, instanceId: state.id, error: state.evaluationError },
+            "instance evaluation failed",
+          )
+
+          this.compositeInstanceEE.emit(projectId, {
+            type: "failed",
+            instanceId: state.id,
+          })
+        }
       }
 
       for (const instance of compositeInstances) {
@@ -270,16 +277,18 @@ export class ProjectManager {
     })
   }
 
-  private async prepareInputHashResolver(projectId: string) {
-    const { instances, hubs } = await this.projectBackend.getProject(projectId)
+  private async prepareResolvers(projectId: string) {
+    const [{ instances, hubs }, states, library, evaluatedInputHashes] = await Promise.all([
+      this.projectBackend.getProject(projectId),
+      this.stateBackend.getAllInstanceStates(projectId),
+      this.libraryBackend.loadLibrary(),
+      this.stateBackend.getCompositeInstanceInputHashes(projectId),
+    ])
 
-    const library = await this.libraryBackend.loadLibrary()
     const filteredInstances = instances.filter(instance => instance.type in library.components)
-
-    const states = await this.stateBackend.getAllInstanceStates(projectId)
     const stateMap = new Map(states.map(state => [state.id, state]))
 
-    const inputResolverNodes = new Map<string, InputResolverInput>()
+    const inputResolverNodes = new Map<string, InputResolverNode>()
 
     for (const instance of filteredInstances) {
       inputResolverNodes.set(`instance:${instance.id}`, {
@@ -293,13 +302,16 @@ export class ProjectManager {
       inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
     }
 
-    const resolveInputs = createInputResolver(inputResolverNodes, this.logger)
+    const inputResolver = new InputResolver(inputResolverNodes, this.logger)
+    inputResolver.addAllNodesToWorkset()
 
-    const inputHashInputs = new Map<string, InputHashResolverInput>()
+    const inputHashInputs = new Map<string, InputHashNode>()
     const resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>> = {}
 
+    await inputResolver.process()
+
     for (const instance of filteredInstances) {
-      const output = await resolveInputs(`instance:${instance.id}`)
+      const output = inputResolver.requireOutput(`instance:${instance.id}`)
       if (output.kind !== "instance") {
         throw new Error("Expected instance node")
       }
@@ -328,14 +340,16 @@ export class ProjectManager {
       resolvedInputs[instance.id] = output.resolvedInputs
     }
 
-    const resolveInputHash = createInputHashResolver(inputHashInputs, this.logger)
+    const inputHashResolver = new InputHashResolver(inputHashInputs, this.logger)
 
     return {
-      resolveInputHash,
+      inputHashInputs,
+      inputHashResolver,
       library,
-      instances,
+      instances: filteredInstances,
       stateMap,
       resolvedInputs,
+      evaluatedInputHashes,
     }
   }
 
@@ -377,7 +391,7 @@ export class ProjectManager {
 
     const projects = await this.projectBackend.getProjectIds()
     for (const projectId of projects) {
-      const { resolveInputHash, instances } = await this.prepareInputHashResolver(projectId)
+      const { instances } = await this.prepareResolvers(projectId)
 
       const filteredInstances = instances.filter(
         instance =>
@@ -391,17 +405,8 @@ export class ProjectManager {
         "updating composite instances for project",
       )
 
-      const inputHashMap = new Map<string, string>()
-      for (const instance of filteredInstances) {
-        const { inputHash } = await resolveInputHash(instance.id)
-        inputHashMap.set(instance.id, inputHash)
-      }
-
       try {
-        await this.evaluateCompositeInstances(
-          projectId,
-          filteredInstances.map(instance => instance.id),
-        )
+        await this.evaluateChangedCompositeInstances(projectId)
       } catch (error) {
         this.logger.error({ error }, "failed to evaluate composite instances")
       }