@highstate/backend 0.9.8 → 0.9.10

package/src/project/manager.ts

@@ -6,16 +6,16 @@ import type { ProjectLockManager } from "./lock"
 import { EventEmitter, on } from "node:events"
 import { isUnitModel, type InstanceModel } from "@highstate/contract"
 import {
-  createInputResolver,
-  createInputHashResolver,
-  type InputResolverInput,
-  type InputHashResolverInput,
+  type InputResolverNode,
+  type InputHashNode,
   type InstanceModelPatch,
   type LibraryUpdate,
   createInstanceState,
   type CompositeInstance,
   type ResolvedInstanceInput,
   type HubModel,
+  InputResolver,
+  InputHashResolver,
 } from "../shared"
 
 type CompositeInstanceEvent =
@@ -31,6 +31,10 @@ type CompositeInstanceEvent =
       type: "deleted"
       instanceId: string
     }
+  | {
+      type: "failed"
+      instanceId: string
+    }
 
 type CompositeInstanceEvents = {
   [K in string]: [CompositeInstanceEvent]
@@ -98,7 +102,7 @@ export class ProjectManager {
 
   async createInstance(projectId: string, instance: InstanceModel): Promise<InstanceModel> {
     const createdInstance = await this.projectBackend.createInstance(projectId, instance)
-    await this.updateCompositeInstance(projectId, createdInstance)
+    await this.evaluateChangedCompositeInstances(projectId)
 
     return createdInstance
   }
@@ -109,7 +113,7 @@ export class ProjectManager {
     patch: InstanceModelPatch,
   ): Promise<InstanceModel> {
     const instance = await this.projectBackend.updateInstance(projectId, instanceId, patch)
-    await this.updateCompositeInstance(projectId, instance)
+    await this.evaluateChangedCompositeInstances(projectId)
 
     return instance
   }
@@ -120,7 +124,7 @@ export class ProjectManager {
     newName: string,
   ): Promise<InstanceModel> {
     const instance = await this.projectBackend.renameInstance(projectId, instanceId, newName)
-    await this.updateCompositeInstance(projectId, instance)
+    await this.evaluateChangedCompositeInstances(projectId)
 
     return instance
   }
@@ -132,32 +136,22 @@ export class ProjectManager {
     ])
   }
 
-  private async updateCompositeInstance(projectId: string, instance: InstanceModel): Promise<void> {
-    const { resolveInputHash, library } = await this.prepareInputHashResolver(projectId)
-
-    const component = library.components[instance.type]
-    if (!component) {
-      return
-    }
-
-    if (isUnitModel(component)) {
-      return
-    }
-
-    const { inputHash: expectedInputHash } = await resolveInputHash(instance.id)
-    const inputHash = await this.stateBackend.getCompositeInstanceInputHash(projectId, instance.id)
+  private async evaluateChangedCompositeInstances(projectId: string): Promise<void> {
+    const { inputHashResolver, instances, library, evaluatedInputHashes } =
+      await this.prepareResolvers(projectId)
 
-    if (inputHash !== expectedInputHash) {
-      this.logger.info("re-evaluating instance since input hash has changed", {
-        projectId,
-        instanceId: instance.id,
-      })
+    inputHashResolver.addAllNodesToWorkset()
+    await inputHashResolver.process()
 
-      await this.evaluateCompositeInstances(projectId, [instance.id])
-    }
-  }
+    const instanceIds = instances
+      .filter(instance => !isUnitModel(library.components[instance.type]))
+      .filter(
+        instance =>
+          evaluatedInputHashes[instance.id] !==
+          inputHashResolver.requireOutput(instance.id).inputHash,
+      )
+      .map(instance => instance.id)
 
-  private async evaluateCompositeInstances(projectId: string, instanceIds: string[]) {
     await this.projectLockManager.getLock(projectId).lockInstances(instanceIds, async () => {
       this.logger.debug({ instanceIds }, "evaluating composite instances")
 
@@ -166,10 +160,10 @@ export class ProjectManager {
       }
 
       const [
-        { instances, resolvedInputs, stateMap, resolveInputHash },
+        { instances, resolvedInputs, stateMap, inputHashResolver },
         topLevelCompositeChildrenIds,
       ] = await Promise.all([
-        this.prepareInputHashResolver(projectId),
+        this.prepareResolvers(projectId),
         this.stateBackend.getTopLevelCompositeChildrenIds(projectId, instanceIds),
       ])
 
@@ -188,18 +182,19 @@ export class ProjectManager {
         return newState
       })
 
-      const inputHashes = new Map<string, string>()
-      for (const instanceId of instanceIds) {
-        const { inputHash } = await resolveInputHash(instanceId)
-        inputHashes.set(instanceId, inputHash)
-      }
+      inputHashResolver.addAllNodesToWorkset()
+      await inputHashResolver.process()
 
       const compositeInstances = results
         .filter(result => result.success)
         .flatMap(result =>
           result.compositeInstances.map(instance => ({
             ...instance,
-            inputHash: inputHashes.get(instance.instance.id),
+            inputHash:
+              // only store inputHash for top-level composite instances
+              instance.instance.id === result.instanceId
+                ? inputHashResolver.requireOutput(instance.instance.id).inputHash
+                : "",
           })),
         )
 
@@ -235,6 +230,18 @@ export class ProjectManager {
 
       for (const state of newStates) {
         this.stateManager.emitStatePatch(projectId, state)
+
+        if (state.evaluationError) {
+          this.logger.error(
+            { projectId, instanceId: state.id, error: state.evaluationError },
+            "instance evaluation failed",
+          )
+
+          this.compositeInstanceEE.emit(projectId, {
+            type: "failed",
+            instanceId: state.id,
+          })
+        }
       }
 
       for (const instance of compositeInstances) {
@@ -270,16 +277,18 @@ export class ProjectManager {
     })
   }
 
-  private async prepareInputHashResolver(projectId: string) {
-    const { instances, hubs } = await this.projectBackend.getProject(projectId)
+  private async prepareResolvers(projectId: string) {
+    const [{ instances, hubs }, states, library, evaluatedInputHashes] = await Promise.all([
+      this.projectBackend.getProject(projectId),
+      this.stateBackend.getAllInstanceStates(projectId),
+      this.libraryBackend.loadLibrary(),
+      this.stateBackend.getCompositeInstanceInputHashes(projectId),
+    ])
 
-    const library = await this.libraryBackend.loadLibrary()
     const filteredInstances = instances.filter(instance => instance.type in library.components)
-
-    const states = await this.stateBackend.getAllInstanceStates(projectId)
     const stateMap = new Map(states.map(state => [state.id, state]))
 
-    const inputResolverNodes = new Map<string, InputResolverInput>()
+    const inputResolverNodes = new Map<string, InputResolverNode>()
 
     for (const instance of filteredInstances) {
       inputResolverNodes.set(`instance:${instance.id}`, {
@@ -293,13 +302,16 @@ export class ProjectManager {
       inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
     }
 
-    const resolveInputs = createInputResolver(inputResolverNodes, this.logger)
+    const inputResolver = new InputResolver(inputResolverNodes, this.logger)
+    inputResolver.addAllNodesToWorkset()
 
-    const inputHashInputs = new Map<string, InputHashResolverInput>()
+    const inputHashInputs = new Map<string, InputHashNode>()
     const resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>> = {}
 
+    await inputResolver.process()
+
     for (const instance of filteredInstances) {
-      const output = await resolveInputs(`instance:${instance.id}`)
+      const output = inputResolver.requireOutput(`instance:${instance.id}`)
       if (output.kind !== "instance") {
         throw new Error("Expected instance node")
       }
@@ -328,14 +340,16 @@ export class ProjectManager {
       resolvedInputs[instance.id] = output.resolvedInputs
     }
 
-    const resolveInputHash = createInputHashResolver(inputHashInputs, this.logger)
+    const inputHashResolver = new InputHashResolver(inputHashInputs, this.logger)
 
     return {
-      resolveInputHash,
+      inputHashInputs,
+      inputHashResolver,
       library,
-      instances,
+      instances: filteredInstances,
       stateMap,
       resolvedInputs,
+      evaluatedInputHashes,
     }
   }
 
@@ -377,7 +391,7 @@ export class ProjectManager {
 
     const projects = await this.projectBackend.getProjectIds()
     for (const projectId of projects) {
-      const { resolveInputHash, instances } = await this.prepareInputHashResolver(projectId)
+      const { instances } = await this.prepareResolvers(projectId)
 
       const filteredInstances = instances.filter(
         instance =>
@@ -391,17 +405,8 @@ export class ProjectManager {
         "updating composite instances for project",
       )
 
-      const inputHashMap = new Map<string, string>()
-      for (const instance of filteredInstances) {
-        const { inputHash } = await resolveInputHash(instance.id)
-        inputHashMap.set(instance.id, inputHash)
-      }
-
       try {
-        await this.evaluateCompositeInstances(
-          projectId,
-          filteredInstances.map(instance => instance.id),
-        )
+        await this.evaluateChangedCompositeInstances(projectId)
       } catch (error) {
         this.logger.error({ error }, "failed to evaluate composite instances")
       }

package/src/shared/async-batcher.ts

@@ -0,0 +1,73 @@
+export type AsyncBatcherOptions = {
+  waitMs?: number
+  maxWaitTimeMs?: number
+}
+
+export function createAsyncBatcher<T>(
+  fn: (items: T[]) => Promise<void> | void,
+  { waitMs = 100, maxWaitTimeMs = 1000 }: AsyncBatcherOptions = {},
+) {
+  let batch: T[] = []
+  let activeTimeout: NodeJS.Timeout | null = null
+  let maxWaitTimeout: NodeJS.Timeout | null = null
+  let firstCallTimestamp: number | null = null
+
+  async function processBatch() {
+    if (batch.length === 0) return
+
+    const currentBatch = batch
+    batch = [] // Reset batch before async call
+
+    await fn(currentBatch)
+
+    // Clear max wait timer since batch has been processed
+    if (maxWaitTimeout) {
+      clearTimeout(maxWaitTimeout)
+      maxWaitTimeout = null
+    }
+    firstCallTimestamp = null
+  }
+
+  function schedule() {
+    if (activeTimeout) clearTimeout(activeTimeout)
+    activeTimeout = setTimeout(() => {
+      activeTimeout = null
+      void processBatch()
+    }, waitMs)
+
+    // Ensure batch is executed within maxWaitTimeMs
+    if (!firstCallTimestamp) {
+      firstCallTimestamp = Date.now()
+      maxWaitTimeout = setTimeout(() => {
+        if (activeTimeout) clearTimeout(activeTimeout)
+        activeTimeout = null
+        void processBatch()
+      }, maxWaitTimeMs)
+    }
+  }
+
+  return {
+    /**
+     * Add an item to the batch.
+     */
+    call(item: T): void {
+      batch.push(item)
+      schedule()
+    },
+
+    /**
+     * Immediately flush the pending batch (if any).
+     */
+    async flush(): Promise<void> {
+      if (activeTimeout) {
+        clearTimeout(activeTimeout)
+        activeTimeout = null
+      }
+      if (maxWaitTimeout) {
+        clearTimeout(maxWaitTimeout)
+        maxWaitTimeout = null
+      }
+      await processBatch()
+    },
+  }
+}

package/src/shared/index.ts

@@ -8,3 +8,4 @@ export * from "./resolvers/input-hash"
 export * from "./resolvers/validation"
 export * from "./terminal"
 export * from "./library"
+export * from "./async-batcher"

package/src/shared/resolvers/graph-resolver.ts

@@ -1,111 +1,178 @@
 import type { Logger } from "pino"
 import { unique } from "remeda"
 
-export type GraphResolverOptions<TNode, TOutput> = {
-  name: string
+export type ResolverOutputHandler<TOutput> = (id: string, value: TOutput) => void
 
-  getNodeId(input: TNode): string
+export abstract class GraphResolver<TNode, TOutput> {
+  private readonly workset: Set<string> = new Set()
+  private readonly dependentMap: Map<string, Set<string>> = new Map()
+  private readonly outputMap: Map<string, TOutput> = new Map()
 
-  getNodeDependencies(input: TNode): string[]
+  constructor(
+    private readonly nodes: ReadonlyMap<string, TNode>,
+    protected readonly logger: Logger,
+    private readonly outputHandler?: ResolverOutputHandler<TOutput>,
+  ) {}
 
-  process(
-    node: TNode,
-    dependencies: ReadonlyMap<string, TOutput>,
-    logger: Logger,
-  ): TOutput | Promise<TOutput>
-}
-
-export interface GraphResolverBackend<TOutput> {
-  promiseCache: Map<string, Promise<TOutput>>
-
-  setOutput?(id: string, value: TOutput): void
-  setDependencies?(id: string, dependencies: string[]): void
-}
-
-export type GraphResolverFactory<TNode, TOutput> = {
-  factoryName: string
-} & ((
-  nodes: ReadonlyMap<string, TNode>,
-  logger: Logger,
-  backend?: GraphResolverBackend<TOutput>,
-) => GraphResolver<TOutput>)
+  addToWorkset(nodeId: string): void {
+    this.workset.add(nodeId)
+  }
 
-export type GraphResolver<TOutput> = (id: string) => Promise<TOutput>
+  addAllNodesToWorkset(): void {
+    for (const nodeId of this.nodes.keys()) {
+      this.workset.add(nodeId)
+    }
+  }
 
-export class CircularDependencyError extends Error {
-  constructor(path: string[]) {
-    super(`Circular dependency detected: ${path.join(" -> ")}`)
-    this.name = "CircularDependencyError"
+  /**
+   * The map of calculated outputs.
+   */
+  get outputs(): ReadonlyMap<string, TOutput> {
+    return this.outputMap
   }
-}
 
-export function createDefaultGraphResolverBackend<TOutput>(): GraphResolverBackend<TOutput> {
-  const promiseCache = new Map<string, Promise<TOutput>>()
+  requireOutput(nodeId: string): TOutput {
+    const output = this.outputMap.get(nodeId)
+    if (!output) {
+      throw new Error(`Output for node ${nodeId} is not available`)
+    }
 
-  return {
-    promiseCache,
+    return output
   }
-}
-
-export function defineGraphResolver<TInput, TOutput>(
-  options: GraphResolverOptions<TInput, TOutput>,
-): GraphResolverFactory<TInput, TOutput> {
-  const factory: GraphResolverFactory<TInput, TOutput> = (nodes, logger, backend) => {
-    backend ??= createDefaultGraphResolverBackend<TOutput>()
-    logger = logger.child({ resolver: options.name })
 
-    const outputs = new Map<string, TOutput>()
+  /**
+   * Gets the list of the identifiers of the dependencies for the node.
+   *
+   * Used to produce the dependency graph.
+   */
+  protected abstract getNodeDependencies(node: TNode): string[]
+
+  /**
+   * Processes the node and returns the output.
+   */
+  protected abstract processNode(node: TNode, logger: Logger): TOutput | Promise<TOutput>
+
+  /**
+   * Invalidates the node and all nodes that depend on it.
+   *
+   * Also adds the node to the work set for processing.
+   */
+  invalidate(nodeId: string): void {
+    const stack = [nodeId]
+
+    while (stack.length > 0) {
+      const nodeId = stack.pop()!
+      if (!this.nodes.has(nodeId)) {
+        // it is ok to invalidate deleted nodes
+        continue
+      }
 
-    const resolver = (itemId: string, dependencyChain: string[]) => {
-      logger.trace({ itemId }, "resolving item")
+      // remove the node from the output map
+      this.outputMap.delete(nodeId)
+      this.workset.add(nodeId)
 
-      const existingPromise = backend.promiseCache.get(itemId)
-      if (existingPromise) {
-        return existingPromise
+      const dependents = this.dependentMap.get(nodeId)
+      if (!dependents) {
+        continue
       }
 
-      if (dependencyChain.includes(itemId)) {
-        throw new CircularDependencyError([...dependencyChain, itemId])
+      for (const dependentId of dependents) {
+        if (this.outputMap.has(dependentId)) {
+          // add the dependent to the stack for further processing
+          stack.push(dependentId)
+        }
       }
 
-      const item = nodes.get(itemId)
-      if (!item) {
-        return Promise.resolve(undefined!)
-      }
+      // clear the dependent map for the node
+      this.dependentMap.delete(nodeId)
+    }
+  }
 
-      const resolve = async () => {
-        const dependencies = unique(options.getNodeDependencies(item))
+  /**
+   * Resolves all not-resolved or invalidated nodes in the graph.
+   *
+   * The abort signal of the previous operation must be called before calling this method again.
+   */
+  async process(signal?: AbortSignal): Promise<void> {
+    while (this.workset.size > 0) {
+      const rootNodeId = this.workset.values().next().value!
+      const stack = [{ nodeId: rootNodeId, resolved: false, dependencies: [] as string[] }]
+
+      while (stack.length > 0) {
+        const stackItem = stack[stack.length - 1]
+        const { nodeId, resolved } = stackItem
+
+        const node = this.nodes.get(nodeId)
+        if (!node) {
+          this.logger.warn({ nodeId }, "node not found in the graph, skipping")
+          stack.pop()
+          continue
+        }
 
-        backend.setDependencies?.(itemId, dependencies)
-        logger.trace({ itemId, dependencies }, "resolving item dependencies")
+        if (this.outputMap.has(nodeId)) {
+          // already processed
+          stack.pop()
+          continue
+        }
 
-        const newChain = [...dependencyChain, itemId]
+        if (!resolved) {
+          stackItem.dependencies = unique(this.getNodeDependencies(node))
+
+          let hasUnresolvedDependencies = false
+
+          for (const depId of stackItem.dependencies) {
+            if (!this.nodes.has(depId)) {
+              this.logger.warn({ depId, nodeId }, "dependency not found in the graph, skipping")
+              continue
+            }
+
+            // if (stack.some(item => item.nodeId === depId)) {
+            //   this.logger.warn(
+            //     { depId, nodeId, stack },
+            //     "dependency is already in the stack, looks like a circular dependency, skipping",
+            //   )
+            //   continue
+            // }
+
+            if (!this.outputMap.has(depId)) {
+              stack.push({ nodeId: depId, resolved: false, dependencies: [] })
+              hasUnresolvedDependencies = true
+            }
+          }
+
+          if (hasUnresolvedDependencies) {
+            // wait for dependencies to be resolved
+            stackItem.resolved = true
+            continue
+          }
+        }
 
-        for (const depId of dependencies) {
-          await resolver(depId, newChain)
+        // all dependencies are resolved, process the node
+        const output = await this.processNode(node, this.logger)
+
+        if (signal?.aborted) {
+          this.logger.warn({ nodeId }, "processing aborted, skipping output")
+          return
         }
 
-        return await options.process(item, outputs, logger)
-      }
+        // update the dependent map
+        for (const depId of stackItem.dependencies) {
+          let dependantSet = this.dependentMap.get(depId)
+          if (!dependantSet) {
+            dependantSet = new Set()
+            this.dependentMap.set(depId, dependantSet)
+          }
 
-      const promise = resolve().then(result => {
-        if (backend.promiseCache.get(itemId) === promise) {
-          // persist the result to the state if provided
-          // ignore if the promise has been replaced (which means this calculation was invalidated)
-          backend.setOutput?.(itemId, result)
-          outputs.set(itemId, result)
+          dependantSet.add(nodeId)
         }
 
-        return result
-      })
+        this.outputMap.set(nodeId, output)
+        this.outputHandler?.(nodeId, output)
 
-      backend.promiseCache.set(itemId, promise)
-      return promise
-    }
+        stack.pop()
+      }
 
-    return id => resolver(id, [])
+      this.workset.delete(rootNodeId)
+    }
   }
-
-  factory.factoryName = options.name
-  return factory
 }

package/src/shared/resolvers/input-hash.ts

@@ -1,10 +1,10 @@
-import type { ComponentModel, InstanceModel } from "@highstate/contract"
 import type { InstanceState } from "../state"
 import type { ResolvedInstanceInput } from "./input"
+import { isUnitModel, type ComponentModel, type InstanceModel } from "@highstate/contract"
 import { sha256 } from "crypto-hash"
-import { defineGraphResolver } from "./graph-resolver"
+import { GraphResolver } from "./graph-resolver"
 
-export type InputHashResolverInput = {
+export type InputHashNode = {
   instance: InstanceModel
   component: ComponentModel
   resolvedInputs: Record<string, ResolvedInstanceInput[]>
@@ -12,7 +12,7 @@ export type InputHashResolverInput = {
   sourceHash: string | undefined
 }
 
-export type InputHashResolverOutput = {
+export type InputHashOutput = {
   inputHash: string
   outputHash: string
 }
@@ -20,14 +20,8 @@ export type InputHashResolverOutput = {
 /**
  * Resolves the hash of the instance based on its args, resolved input hashes, source hash, and the output hash.
  */
-export const createInputHashResolver = defineGraphResolver<
-  InputHashResolverInput,
-  InputHashResolverOutput
->({
-  name: "input-hash-resolver",
-  getNodeId: node => node.instance.id,
-
-  getNodeDependencies({ resolvedInputs }) {
+export class InputHashResolver extends GraphResolver<InputHashNode, InputHashOutput> {
+  getNodeDependencies({ resolvedInputs }: InputHashNode): string[] {
     const dependencies: string[] = []
 
     for (const inputs of Object.values(resolvedInputs ?? {})) {
@@ -37,13 +31,24 @@ export const createInputHashResolver = defineGraphResolver<
     }
 
     return dependencies
-  },
+  }
 
-  async process({ instance, component, resolvedInputs, sourceHash, state }, dependencies) {
+  async processNode({
+    instance,
+    component,
+    resolvedInputs,
+    sourceHash,
+    state,
+  }: InputHashNode): Promise<InputHashOutput> {
     let sink = component.definitionHash + JSON.stringify(instance.args ?? {})
 
     if (sourceHash) {
       sink += sourceHash
+    } else if (isUnitModel(component)) {
+      this.logger.warn(
+        { instanceId: instance.id },
+        "missing source hash for unit model, this may lead to incorrect input hash",
+      )
     }
 
     const sortedInputs = Object.entries(resolvedInputs)
@@ -61,7 +66,7 @@ export const createInputHashResolver = defineGraphResolver<
       instanceIds.sort()
 
       for (const instanceId of instanceIds) {
-        const dependency = dependencies.get(instanceId)
+        const dependency = this.outputs.get(instanceId)
         if (!dependency) continue
 
         sink += dependency.inputHash
@@ -73,5 +78,5 @@ export const createInputHashResolver = defineGraphResolver<
       inputHash: await sha256(sink),
       outputHash: state?.outputHash ?? "",
     }
-  },
-})
+  }
+}