@highstate/backend 0.9.9 → 0.9.10

package/src/shared/async-batcher.ts

@@ -0,0 +1,73 @@
+export type AsyncBatcherOptions = {
+  waitMs?: number
+  maxWaitTimeMs?: number
+}
+
+export function createAsyncBatcher<T>(
+  fn: (items: T[]) => Promise<void> | void,
+  { waitMs = 100, maxWaitTimeMs = 1000 }: AsyncBatcherOptions = {},
+) {
+  let batch: T[] = []
+  let activeTimeout: NodeJS.Timeout | null = null
+  let maxWaitTimeout: NodeJS.Timeout | null = null
+  let firstCallTimestamp: number | null = null
+
+  async function processBatch() {
+    if (batch.length === 0) return
+
+    const currentBatch = batch
+    batch = [] // Reset batch before async call
+
+    await fn(currentBatch)
+
+    // Clear max wait timer since batch has been processed
+    if (maxWaitTimeout) {
+      clearTimeout(maxWaitTimeout)
+      maxWaitTimeout = null
+    }
+    firstCallTimestamp = null
+  }
+
+  function schedule() {
+    if (activeTimeout) clearTimeout(activeTimeout)
+    activeTimeout = setTimeout(() => {
+      activeTimeout = null
+      void processBatch()
+    }, waitMs)
+
+    // Ensure batch is executed within maxWaitTimeMs
+    if (!firstCallTimestamp) {
+      firstCallTimestamp = Date.now()
+      maxWaitTimeout = setTimeout(() => {
+        if (activeTimeout) clearTimeout(activeTimeout)
+        activeTimeout = null
+        void processBatch()
+      }, maxWaitTimeMs)
+    }
+  }
+
+  return {
+    /**
+     * Add an item to the batch.
+     */
+    call(item: T): void {
+      batch.push(item)
+      schedule()
+    },
+
+    /**
+     * Immediately flush the pending batch (if any).
+     */
+    async flush(): Promise<void> {
+      if (activeTimeout) {
+        clearTimeout(activeTimeout)
+        activeTimeout = null
+      }
+      if (maxWaitTimeout) {
+        clearTimeout(maxWaitTimeout)
+        maxWaitTimeout = null
+      }
+      await processBatch()
+    },
+  }
+}

package/src/shared/index.ts

@@ -8,3 +8,4 @@ export * from "./resolvers/input-hash"
 export * from "./resolvers/validation"
 export * from "./terminal"
 export * from "./library"
+export * from "./async-batcher"

package/src/shared/resolvers/graph-resolver.ts

@@ -1,111 +1,178 @@
 import type { Logger } from "pino"
 import { unique } from "remeda"
 
-export type GraphResolverOptions<TNode, TOutput> = {
-  name: string
+export type ResolverOutputHandler<TOutput> = (id: string, value: TOutput) => void
 
-  getNodeId(input: TNode): string
+export abstract class GraphResolver<TNode, TOutput> {
+  private readonly workset: Set<string> = new Set()
+  private readonly dependentMap: Map<string, Set<string>> = new Map()
+  private readonly outputMap: Map<string, TOutput> = new Map()
 
-  getNodeDependencies(input: TNode): string[]
+  constructor(
+    private readonly nodes: ReadonlyMap<string, TNode>,
+    protected readonly logger: Logger,
+    private readonly outputHandler?: ResolverOutputHandler<TOutput>,
+  ) {}
 
-  process(
-    node: TNode,
-    dependencies: ReadonlyMap<string, TOutput>,
-    logger: Logger,
-  ): TOutput | Promise<TOutput>
-}
-
-export interface GraphResolverBackend<TOutput> {
-  promiseCache: Map<string, Promise<TOutput>>
-
-  setOutput?(id: string, value: TOutput): void
-  setDependencies?(id: string, dependencies: string[]): void
-}
-
-export type GraphResolverFactory<TNode, TOutput> = {
-  factoryName: string
-} & ((
-  nodes: ReadonlyMap<string, TNode>,
-  logger: Logger,
-  backend?: GraphResolverBackend<TOutput>,
-) => GraphResolver<TOutput>)
+  addToWorkset(nodeId: string): void {
+    this.workset.add(nodeId)
+  }
 
-export type GraphResolver<TOutput> = (id: string) => Promise<TOutput>
+  addAllNodesToWorkset(): void {
+    for (const nodeId of this.nodes.keys()) {
+      this.workset.add(nodeId)
+    }
+  }
 
-export class CircularDependencyError extends Error {
-  constructor(path: string[]) {
-    super(`Circular dependency detected: ${path.join(" -> ")}`)
-    this.name = "CircularDependencyError"
+  /**
+   * The map of calculated outputs.
+   */
+  get outputs(): ReadonlyMap<string, TOutput> {
+    return this.outputMap
   }
-}
 
-export function createDefaultGraphResolverBackend<TOutput>(): GraphResolverBackend<TOutput> {
-  const promiseCache = new Map<string, Promise<TOutput>>()
+  requireOutput(nodeId: string): TOutput {
+    const output = this.outputMap.get(nodeId)
+    if (!output) {
+      throw new Error(`Output for node ${nodeId} is not available`)
+    }
 
-  return {
-    promiseCache,
+    return output
   }
-}
-
-export function defineGraphResolver<TInput, TOutput>(
-  options: GraphResolverOptions<TInput, TOutput>,
-): GraphResolverFactory<TInput, TOutput> {
-  const factory: GraphResolverFactory<TInput, TOutput> = (nodes, logger, backend) => {
-    backend ??= createDefaultGraphResolverBackend<TOutput>()
-    logger = logger.child({ resolver: options.name })
 
-    const outputs = new Map<string, TOutput>()
+  /**
+   * Gets the list of the identifiers of the dependencies for the node.
+   *
+   * Used to produce the dependency graph.
+   */
+  protected abstract getNodeDependencies(node: TNode): string[]
+
+  /**
+   * Processes the node and returns the output.
+   */
+  protected abstract processNode(node: TNode, logger: Logger): TOutput | Promise<TOutput>
+
+  /**
+   * Invalidates the node and all nodes that depend on it.
+   *
+   * Also adds the node to the work set for processing.
+   */
+  invalidate(nodeId: string): void {
+    const stack = [nodeId]
+
+    while (stack.length > 0) {
+      const nodeId = stack.pop()!
+      if (!this.nodes.has(nodeId)) {
+        // it is ok to invalidate deleted nodes
+        continue
+      }
 
-    const resolver = (itemId: string, dependencyChain: string[]) => {
-      logger.trace({ itemId }, "resolving item")
+      // remove the node from the output map
+      this.outputMap.delete(nodeId)
+      this.workset.add(nodeId)
 
-      const existingPromise = backend.promiseCache.get(itemId)
-      if (existingPromise) {
-        return existingPromise
+      const dependents = this.dependentMap.get(nodeId)
+      if (!dependents) {
+        continue
       }
 
-      if (dependencyChain.includes(itemId)) {
-        throw new CircularDependencyError([...dependencyChain, itemId])
+      for (const dependentId of dependents) {
+        if (this.outputMap.has(dependentId)) {
+          // add the dependent to the stack for further processing
+          stack.push(dependentId)
+        }
       }
 
-      const item = nodes.get(itemId)
-      if (!item) {
-        return Promise.resolve(undefined!)
-      }
+      // clear the dependent map for the node
+      this.dependentMap.delete(nodeId)
+    }
+  }
 
-      const resolve = async () => {
-        const dependencies = unique(options.getNodeDependencies(item))
+  /**
+   * Resolves all not-resolved or invalidated nodes in the graph.
+   *
+   * The abort signal of the previous operation must be called before calling this method again.
+   */
+  async process(signal?: AbortSignal): Promise<void> {
+    while (this.workset.size > 0) {
+      const rootNodeId = this.workset.values().next().value!
+      const stack = [{ nodeId: rootNodeId, resolved: false, dependencies: [] as string[] }]
+
+      while (stack.length > 0) {
+        const stackItem = stack[stack.length - 1]
+        const { nodeId, resolved } = stackItem
+
+        const node = this.nodes.get(nodeId)
+        if (!node) {
+          this.logger.warn({ nodeId }, "node not found in the graph, skipping")
+          stack.pop()
+          continue
+        }
 
-        backend.setDependencies?.(itemId, dependencies)
-        logger.trace({ itemId, dependencies }, "resolving item dependencies")
+        if (this.outputMap.has(nodeId)) {
+          // already processed
+          stack.pop()
+          continue
+        }
 
-        const newChain = [...dependencyChain, itemId]
+        if (!resolved) {
+          stackItem.dependencies = unique(this.getNodeDependencies(node))
+
+          let hasUnresolvedDependencies = false
+
+          for (const depId of stackItem.dependencies) {
+            if (!this.nodes.has(depId)) {
+              this.logger.warn({ depId, nodeId }, "dependency not found in the graph, skipping")
+              continue
+            }
+
+            // if (stack.some(item => item.nodeId === depId)) {
+            //   this.logger.warn(
+            //     { depId, nodeId, stack },
+            //     "dependency is already in the stack, looks like a circular dependency, skipping",
+            //   )
+            //   continue
+            // }
+
+            if (!this.outputMap.has(depId)) {
+              stack.push({ nodeId: depId, resolved: false, dependencies: [] })
+              hasUnresolvedDependencies = true
+            }
+          }
+
+          if (hasUnresolvedDependencies) {
+            // wait for dependencies to be resolved
+            stackItem.resolved = true
+            continue
+          }
+        }
 
-        for (const depId of dependencies) {
-          await resolver(depId, newChain)
+        // all dependencies are resolved, process the node
+        const output = await this.processNode(node, this.logger)
+
+        if (signal?.aborted) {
+          this.logger.warn({ nodeId }, "processing aborted, skipping output")
+          return
         }
 
-        return await options.process(item, outputs, logger)
-      }
+        // update the dependent map
+        for (const depId of stackItem.dependencies) {
+          let dependantSet = this.dependentMap.get(depId)
+          if (!dependantSet) {
+            dependantSet = new Set()
+            this.dependentMap.set(depId, dependantSet)
+          }
 
-      const promise = resolve().then(result => {
-        if (backend.promiseCache.get(itemId) === promise) {
-          // persist the result to the state if provided
-          // ignore if the promise has been replaced (which means this calculation was invalidated)
-          backend.setOutput?.(itemId, result)
-          outputs.set(itemId, result)
+          dependantSet.add(nodeId)
         }
 
-        return result
-      })
+        this.outputMap.set(nodeId, output)
+        this.outputHandler?.(nodeId, output)
 
-      backend.promiseCache.set(itemId, promise)
-      return promise
-    }
+        stack.pop()
+      }
 
-    return id => resolver(id, [])
+      this.workset.delete(rootNodeId)
+    }
   }
-
-  factory.factoryName = options.name
-  return factory
 }

package/src/shared/resolvers/input-hash.ts

@@ -1,10 +1,10 @@
-import type { ComponentModel, InstanceModel } from "@highstate/contract"
 import type { InstanceState } from "../state"
 import type { ResolvedInstanceInput } from "./input"
+import { isUnitModel, type ComponentModel, type InstanceModel } from "@highstate/contract"
 import { sha256 } from "crypto-hash"
-import { defineGraphResolver } from "./graph-resolver"
+import { GraphResolver } from "./graph-resolver"
 
-export type InputHashResolverInput = {
+export type InputHashNode = {
   instance: InstanceModel
   component: ComponentModel
   resolvedInputs: Record<string, ResolvedInstanceInput[]>
@@ -12,7 +12,7 @@ export type InputHashResolverInput = {
   sourceHash: string | undefined
 }
 
-export type InputHashResolverOutput = {
+export type InputHashOutput = {
   inputHash: string
   outputHash: string
 }
@@ -20,14 +20,8 @@ export type InputHashResolverOutput = {
 /**
  * Resolves the hash of the instance based on its args, resolved input hashes, source hash, and the output hash.
  */
-export const createInputHashResolver = defineGraphResolver<
-  InputHashResolverInput,
-  InputHashResolverOutput
->({
-  name: "input-hash-resolver",
-  getNodeId: node => node.instance.id,
-
-  getNodeDependencies({ resolvedInputs }) {
+export class InputHashResolver extends GraphResolver<InputHashNode, InputHashOutput> {
+  getNodeDependencies({ resolvedInputs }: InputHashNode): string[] {
     const dependencies: string[] = []
 
     for (const inputs of Object.values(resolvedInputs ?? {})) {
@@ -37,13 +31,24 @@ export const createInputHashResolver = defineGraphResolver<
     }
 
     return dependencies
-  },
+  }
 
-  async process({ instance, component, resolvedInputs, sourceHash, state }, dependencies) {
+  async processNode({
+    instance,
+    component,
+    resolvedInputs,
+    sourceHash,
+    state,
+  }: InputHashNode): Promise<InputHashOutput> {
     let sink = component.definitionHash + JSON.stringify(instance.args ?? {})
 
     if (sourceHash) {
       sink += sourceHash
+    } else if (isUnitModel(component)) {
+      this.logger.warn(
+        { instanceId: instance.id },
+        "missing source hash for unit model, this may lead to incorrect input hash",
+      )
     }
 
     const sortedInputs = Object.entries(resolvedInputs)
@@ -61,7 +66,7 @@ export const createInputHashResolver = defineGraphResolver<
       instanceIds.sort()
 
       for (const instanceId of instanceIds) {
-        const dependency = dependencies.get(instanceId)
+        const dependency = this.outputs.get(instanceId)
         if (!dependency) continue
 
         sink += dependency.inputHash
@@ -73,5 +78,5 @@ export const createInputHashResolver = defineGraphResolver<
       inputHash: await sha256(sink),
       outputHash: state?.outputHash ?? "",
     }
-  },
-})
+  }
+}

package/src/shared/resolvers/input.ts

@@ -7,9 +7,9 @@ import {
   type InstanceModel,
 } from "@highstate/contract"
 import { fromEntries, mapValues } from "remeda"
-import { defineGraphResolver } from "./graph-resolver"
+import { GraphResolver } from "./graph-resolver"
 
-export type InputResolverInput =
+export type InputResolverNode =
   | {
       kind: "instance"
       instance: InstanceModel
@@ -28,6 +28,7 @@ export type ResolvedInstanceInput = {
 export type InputResolverOutput =
   | {
       kind: "instance"
+      instance: InstanceModel
       component: ComponentModel
       resolvedInputs: Record<string, ResolvedInstanceInput[]>
       resolvedOutputs: Record<string, InstanceInput[]> | undefined
@@ -42,17 +43,16 @@ export type InputResolverOutput =
 /**
  * Resolves the all recursive instance and hub inputs based on its direct inputs and injected inputs.
  */
-export const createInputResolver = defineGraphResolver<InputResolverInput, InputResolverOutput>({
-  name: "input-resolver",
-  getNodeId(node) {
-    if (node.kind === "hub") {
-      return `hub:${node.hub.id}`
-    }
-
-    return `instance:${node.instance.id}`
-  },
-
-  getNodeDependencies(node) {
+export class InputResolver extends GraphResolver<InputResolverNode, InputResolverOutput> {
+  // constructor(
+  //   nodes: ReadonlyMap<string, InputResolverNode>,
+  //   logger: Logger,
+  //   outputHandler?: ResolverOutputHandler<InputResolverOutput>,
+  // ) {
+  //   super(nodes, logger, outputHandler)
+  // }
+
+  getNodeDependencies(node: InputResolverNode): string[] {
     const dependencies: string[] = []
 
     if (node.kind === "hub") {
@@ -84,11 +84,11 @@ export const createInputResolver = defineGraphResolver<InputResolverInput, Input
     }
 
     return dependencies
-  },
+  }
 
-  process(node, dependencies, logger) {
+  processNode(node: InputResolverNode): InputResolverOutput {
     const getHubOutput = (input: HubInput) => {
-      const output = dependencies.get(`hub:${input.hubId}`)
+      const output = this.outputs.get(`hub:${input.hubId}`)
       if (!output) {
         return { resolvedInputs: [] }
       }
@@ -101,7 +101,7 @@ export const createInputResolver = defineGraphResolver<InputResolverInput, Input
     }
 
     const getInstanceOutput = (input: InstanceInput) => {
-      const output = dependencies.get(`instance:${input.instanceId}`)
+      const output = this.outputs.get(`instance:${input.instanceId}`)
       if (!output) {
         return {
           component: null,
@@ -122,7 +122,6 @@ export const createInputResolver = defineGraphResolver<InputResolverInput, Input
     }
 
     // resolve inputs for hub
-
     if (node.kind === "hub") {
       const hubResult: Map<string, ResolvedInstanceInput> = new Map()
 
@@ -135,7 +134,7 @@ export const createInputResolver = defineGraphResolver<InputResolverInput, Input
         const componentInput = component?.outputs[input.output]
 
         if (!componentInput) {
-          logger.warn({ msg: "output not found in the component", input, component })
+          this.logger.warn({ msg: "output not found in the component", input, component })
           continue
         }
 
@@ -160,11 +159,12 @@ export const createInputResolver = defineGraphResolver<InputResolverInput, Input
     if (node.instance.resolvedInputs) {
       return {
         kind: "instance",
+        instance: node.instance,
         component: node.component,
         resolvedInputs: mapValues(node.instance.resolvedInputs, (inputs, inputName) => {
           const componentInput = node.component.inputs[inputName]
           if (!componentInput) {
-            logger.warn({
+            this.logger.warn({
               msg: "input not found in the component",
               inputName,
               component: node.component,
@@ -181,7 +181,6 @@ export const createInputResolver = defineGraphResolver<InputResolverInput, Input
     }
 
     // resolve inputs for instance
-
     const resolvedInputsMap: Map<string, Map<string, ResolvedInstanceInput>> = new Map()
 
     const addInstanceResult = (inputName: string, input: ResolvedInstanceInput) => {
@@ -197,14 +196,18 @@ export const createInputResolver = defineGraphResolver<InputResolverInput, Input
     const addInstanceInput = (inputName: string, input: InstanceInput) => {
       const componentInput = node.component.inputs[inputName]
       if (!componentInput) {
-        logger.warn({ msg: "input not found in the component", input, component: node.component })
+        this.logger.warn({
+          msg: "input not found in the component",
+          input,
+          component: node.component,
+        })
         return
       }
 
       const { component, resolvedOutputs } = getInstanceOutput(input)
 
       if (!component) {
-        logger.warn({ instanceId: node.instance.id, input }, "no output found for the input")
+        this.logger.warn({ instanceId: node.instance.id, input }, "no output found for the input")
         return
       }
 
@@ -219,7 +222,6 @@ export const createInputResolver = defineGraphResolver<InputResolverInput, Input
         }
       } else {
         // if the instance is not evaluated, we a forced to use the input as is
-
         addInstanceResult(inputName, { input, type: componentInput.type })
       }
     }
@@ -272,14 +274,15 @@ export const createInputResolver = defineGraphResolver<InputResolverInput, Input
 
     return {
       kind: "instance",
+      instance: node.instance,
       component: node.component,
       resolvedInputs,
       resolvedOutputs: node.instance.resolvedOutputs,
       resolvedInjectionInputs: Array.from(injectionInputs.values()),
       matchedInjectionInputs: Array.from(matchedInjectionInputs.values()),
     }
-  },
-})
+  }
+}
 
 export function getResolvedHubInputs(output: InputResolverOutput): ResolvedInstanceInput[] {
   if (output.kind !== "hub") {

package/src/shared/resolvers/registry.ts

@@ -1,10 +1,20 @@
-import type { GraphResolverFactory } from "./graph-resolver"
-import { createInputResolver } from "./input"
-import { createInputHashResolver } from "./input-hash"
-import { createValidationResolver } from "./validation"
+import type { Logger } from "pino"
+import type { GraphResolver, ResolverOutputHandler } from "./graph-resolver"
+import { InputResolver } from "./input"
+import { InputHashResolver } from "./input-hash"
+import { ValidationResolver } from "./validation"
 
-export const resolverFactories: Record<string, GraphResolverFactory<any, any>> = {
-  [createInputResolver.factoryName]: createInputResolver,
-  [createInputHashResolver.factoryName]: createInputHashResolver,
-  [createValidationResolver.factoryName]: createValidationResolver,
-}
+export type GraphResolverType = "InputResolver" | "InputHashResolver" | "ValidationResolver"
+
+export const resolverFactories = {
+  InputResolver,
+  InputHashResolver,
+  ValidationResolver,
+} as Record<
+  GraphResolverType,
+  new (
+    nodes: ReadonlyMap<string, unknown>,
+    logger: Logger,
+    outputHandler?: ResolverOutputHandler<unknown>,
+  ) => GraphResolver<unknown, unknown>
+>

package/src/shared/resolvers/validation.ts

@@ -1,15 +1,15 @@
 import type { ComponentModel, InstanceModel } from "@highstate/contract"
 import type { ResolvedInstanceInput } from "./input"
 import { Ajv } from "ajv"
-import { defineGraphResolver } from "./graph-resolver"
+import { GraphResolver } from "./graph-resolver"
 
-export type ValidationResolverInput = {
+export type ValidationNode = {
   instance: InstanceModel
   component: ComponentModel
   resolvedInputs: Record<string, ResolvedInstanceInput[]>
 }
 
-export type ValidationResolverOutput =
+export type ValidationOutput =
   | {
       status: "ok"
     }
@@ -25,14 +25,8 @@ export type ValidationResolverOutput =
 /**
  * Validates the instance based on its arguments and inputs.
  */
-export const createValidationResolver = defineGraphResolver<
-  ValidationResolverInput,
-  ValidationResolverOutput
->({
-  name: "validation-resolver",
-  getNodeId: node => node.instance.id,
-
-  getNodeDependencies({ resolvedInputs }) {
+export class ValidationResolver extends GraphResolver<ValidationNode, ValidationOutput> {
+  getNodeDependencies({ resolvedInputs }: ValidationNode): string[] {
     const dependencies: string[] = []
 
     for (const inputs of Object.values(resolvedInputs)) {
@@ -42,12 +36,12 @@ export const createValidationResolver = defineGraphResolver<
     }
 
     return dependencies
-  },
+  }
 
-  process({ instance, component, resolvedInputs }, dependencies, logger) {
+  processNode({ instance, component, resolvedInputs }: ValidationNode): ValidationOutput {
     const ajv = new Ajv({ strict: false })
 
-    logger.debug({ instanceId: instance.id }, "validating instance")
+    this.logger.debug({ instanceId: instance.id }, "validating instance")
 
     for (const [name, argument] of Object.entries(component.args)) {
       if (!argument.required && !instance.args?.[name]) {
@@ -55,7 +49,7 @@ export const createValidationResolver = defineGraphResolver<
       }
 
       if (!ajv.validate(argument.schema, instance.args?.[name])) {
-        logger.debug({ instanceId: instance.id, argumentName: name }, "invalid argument")
+        this.logger.debug({ instanceId: instance.id, argumentName: name }, "invalid argument")
 
         return {
           status: "invalid-args",
@@ -66,7 +60,7 @@ export const createValidationResolver = defineGraphResolver<
 
     for (const inputs of Object.values(resolvedInputs)) {
       for (const input of inputs) {
-        const inputInstance = dependencies.get(input.input.instanceId)
+        const inputInstance = this.outputs.get(input.input.instanceId)
         if (inputInstance?.status !== "ok") {
           return {
             status: "invalid-inputs",
@@ -90,5 +84,5 @@ export const createValidationResolver = defineGraphResolver<
     }
 
     return { status: "ok" }
-  },
-})
+  }
+}