@highstate/backend 0.9.14 → 0.9.16

package/src/project/manager.ts

@@ -1,75 +1,43 @@
-import type { StateBackend, StateManager } from "../state"
+import type { StateManager } from "../state"
 import type { ProjectBackend } from "./abstractions"
 import type { Logger } from "pino"
-import type { LibraryBackend } from "../library"
-import type { ProjectLockManager } from "./lock"
-import { EventEmitter, on } from "node:events"
+import type { InstanceEvaluationResult, LibraryBackend } from "../library"
+import type { InstanceLockService } from "../business"
+import type { PubSubManager } from "../pubsub"
+import { randomUUID } from "node:crypto"
 import { isUnitModel, type InstanceModel } from "@highstate/contract"
 import {
   type InputResolverNode,
   type InputHashNode,
   type InstanceModelPatch,
-  type LibraryUpdate,
-  createInstanceState,
   type CompositeInstance,
   type ResolvedInstanceInput,
   type HubModel,
+  type LibraryUpdate,
   InputResolver,
   InputHashResolver,
 } from "../shared"
-
-type CompositeInstanceEvent =
-  | {
-      type: "evaluation-started"
-      instanceId: string
-    }
-  | {
-      type: "updated"
-      instance: CompositeInstance
-    }
-  | {
-      type: "deleted"
-      instanceId: string
-    }
-  | {
-      type: "failed"
-      instanceId: string
-    }
-
-type CompositeInstanceEvents = {
-  [K in string]: [CompositeInstanceEvent]
-}
+import { renderTree, type TreeNode } from "../common"
 
 export type FullProjectModel = {
+  libraryId: string
   instances: InstanceModel[]
   hubs: HubModel[]
   compositeInstances: CompositeInstance[]
 }
 
 export class ProjectManager {
+  private readonly watchedLibraries = new Set<string>()
+  private readonly libraryWatchers = new Map<string, AbortController>()
+
   private constructor(
     private readonly projectBackend: ProjectBackend,
-    private readonly stateBackend: StateBackend,
     private readonly libraryBackend: LibraryBackend,
-    private readonly projectLockManager: ProjectLockManager,
     private readonly stateManager: StateManager,
+    private readonly instanceLockService: InstanceLockService,
+    private readonly pubsubManager: PubSubManager,
     private readonly logger: Logger,
-  ) {
-    void this.watchLibraryChanges()
-  }
-
-  private readonly compositeInstanceEE = new EventEmitter<CompositeInstanceEvents>()
-
-  async *watchCompositeInstances(
-    projectId: string,
-    signal?: AbortSignal,
-  ): AsyncIterable<CompositeInstanceEvent> {
-    for await (const [children] of on(this.compositeInstanceEE, projectId, {
-      signal,
-    })) {
-      yield children
-    }
-  }
+  ) {}
 
   /**
    * Loads the full info of a project, including instances, hubs, and composite instances.
@@ -79,13 +47,22 @@ export class ProjectManager {
    * @param projectId The ID of the project to load.
    */
   async getProject(projectId: string): Promise<FullProjectModel> {
-    const [{ instances, hubs }, compositeInstances, library] = await Promise.all([
+    const [projectInfo, project] = await Promise.all([
+      this.projectBackend.getProjectInfo(projectId),
       this.projectBackend.getProject(projectId),
-      this.stateBackend.getCompositeInstances(projectId),
-      this.libraryBackend.loadLibrary(),
     ])
 
-    const filteredInstances = instances.filter(instance => instance.type in library.components)
+    // Ensure we're watching this library for changes
+    this.ensureLibraryWatched(projectInfo.libraryId)
+
+    const [compositeInstances, library] = await Promise.all([
+      this.stateManager.getCompositeInstanceRepository(projectId).getAllItems(),
+      this.libraryBackend.loadLibrary(projectInfo.libraryId),
+    ])
+
+    const filteredInstances = project.instances.filter(
+      instance => instance.type in library.components,
+    )
     const filteredCompositeInstances = compositeInstances
       .filter(instance => instance.instance.type in library.components)
       .map(instance => ({
@@ -94,8 +71,9 @@ export class ProjectManager {
       }))
 
     return {
+      libraryId: projectInfo.libraryId,
       instances: filteredInstances,
-      hubs,
+      hubs: project.hubs,
       compositeInstances: filteredCompositeInstances,
     }
   }
@@ -113,7 +91,7 @@ export class ProjectManager {
     patch: InstanceModelPatch,
   ): Promise<InstanceModel> {
     const instance = await this.projectBackend.updateInstance(projectId, instanceId, patch)
-    await this.evaluateChangedCompositeInstances(projectId)
+    // await this.evaluateChangedCompositeInstances(projectId)
 
     return instance
   }
@@ -132,10 +110,18 @@ export class ProjectManager {
   async deleteInstance(projectId: string, instanceId: string): Promise<void> {
     await Promise.all([
       this.projectBackend.deleteInstance(projectId, instanceId),
-      this.stateBackend.clearCompositeInstances(projectId, [instanceId]),
+      this.stateManager.getCompositeInstanceRepository(projectId).delete(instanceId),
     ])
   }
 
+  async createManyNodes(
+    projectId: string,
+    instances: InstanceModel[],
+    hubs: HubModel[],
+  ): Promise<void> {
+    await this.projectBackend.createManyNodes(projectId, instances, hubs)
+  }
+
   private async evaluateChangedCompositeInstances(projectId: string): Promise<void> {
     const { inputHashResolver, instances, library, evaluatedInputHashes } =
       await this.prepareResolvers(projectId)
@@ -152,22 +138,51 @@ export class ProjectManager {
       )
       .map(instance => instance.id)
 
-    await this.projectLockManager.getLock(projectId).lockInstances(instanceIds, async () => {
+    await this.instanceLockService.tryLockInstances(
+      projectId,
+      instanceIds,
+      {
+        title: "Composite Instance Evaluation",
+        description: "This instance is now being evaluated",
+        icon: "logos:typescript-icon-round",
+      },
+      {
+        type: "evaluation",
+        evaluationId: randomUUID(),
+      },
+    )
+
+    try {
       this.logger.debug({ instanceIds }, "evaluating composite instances")
 
       for (const instanceId of instanceIds) {
-        this.compositeInstanceEE.emit(projectId, { type: "evaluation-started", instanceId })
+        void this.pubsubManager.publish(`instance:state:patch:${projectId}`, {
+          patch: {
+            id: instanceId,
+            evaluationStatus: {
+              status: "evaluating",
+            },
+          },
+        })
       }
 
       const [
-        { instances, resolvedInputs, stateMap, inputHashResolver },
-        topLevelCompositeChildrenIds,
+        { instances, resolvedInputs, stateMap, inputHashResolver, libraryId },
+        existingCompositeInstances,
       ] = await Promise.all([
         this.prepareResolvers(projectId),
-        this.stateBackend.getTopLevelCompositeChildrenIds(projectId, instanceIds),
+        this.stateManager.getCompositeInstanceRepository(projectId).getAllRecord(),
       ])
 
+      // Get child instance IDs for the instances being re-evaluated
+      const oldChildInstanceIds = new Set(
+        existingCompositeInstances
+          .filter(ci => instanceIds.includes(ci.instance.id))
+          .flatMap(ci => ci.childCompositeInstanceIds ?? []),
+      )
+
       const results = await this.libraryBackend.evaluateCompositeInstances(
+        libraryId,
         instances,
         resolvedInputs,
         instanceIds,
@@ -175,9 +190,11 @@ export class ProjectManager {
 
       const newStates = results.map(result => {
         const existingState = stateMap.get(result.instanceId)
-        const newState = existingState ?? createInstanceState(result.instanceId)
+        const newState = existingState ?? { id: result.instanceId }
 
-        newState.evaluationError = result.success ? null : result.error
+        newState.evaluationStatus = result.success
+          ? { status: "evaluated", message: ProjectManager.createSuccessEvaluationMessage(result) }
+          : { status: "error", message: result.error }
 
         return newState
       })
@@ -190,76 +207,81 @@ export class ProjectManager {
         .flatMap(result =>
           result.compositeInstances.map(instance => ({
             ...instance,
+            id: instance.instance.id,
             inputHash:
               // only store inputHash for top-level composite instances
               instance.instance.id === result.instanceId
                 ? inputHashResolver.requireOutput(instance.instance.id).inputHash
                 : "",
+            // only store childCompositeInstanceIds for top-level composite instances
+            childCompositeInstanceIds:
+              instance.instance.id === result.instanceId
+                ? result.compositeInstances
+                    .filter(ci => ci.instance.id !== result.instanceId)
+                    .map(ci => ci.instance.id)
+                : undefined,
           })),
         )
 
-      const newTopLevelCompositeChildrenIds = Object.fromEntries(
+      const newChildInstanceIds = new Set(
         results
           .filter(result => result.success)
-          .map(result => [
-            result.instanceId,
+          .flatMap(result =>
             result.compositeInstances
               .filter(instance => instance.instance.id !== result.instanceId)
               .map(instance => instance.instance.id),
-          ]),
+          ),
       )
 
-      const deletedCompositeInstanceIds = new Set(
-        Object.values(topLevelCompositeChildrenIds).flat(),
-      )
+      const deletedCompositeInstanceIds = new Set<string>()
 
-      for (const childInstanceId of Object.values(newTopLevelCompositeChildrenIds).flat()) {
-        deletedCompositeInstanceIds.delete(childInstanceId)
+      // Find child instances that were deleted (existed before but don't exist now)
+      for (const oldChildId of oldChildInstanceIds) {
+        if (!newChildInstanceIds.has(oldChildId)) {
+          deletedCompositeInstanceIds.add(oldChildId)
+        }
       }
 
-      await Promise.all([
-        this.stateBackend.clearCompositeInstances(
-          projectId,
-          Array.from(deletedCompositeInstanceIds),
-        ),
-        this.stateBackend.putTopLevelCompositeChildrenIds(
-          projectId,
-          newTopLevelCompositeChildrenIds,
-        ),
-      ])
+      await this.stateManager
+        .getCompositeInstanceRepository(projectId)
+        .deleteMany(Array.from(deletedCompositeInstanceIds))
 
       for (const state of newStates) {
-        this.stateManager.emitStatePatch(projectId, state)
+        void this.pubsubManager.publish(`instance:state:patch:${projectId}`, { patch: state })
 
-        if (state.evaluationError) {
+        if (state.evaluationStatus?.status === "error") {
           this.logger.error(
-            { projectId, instanceId: state.id, error: state.evaluationError },
-            "instance evaluation failed",
+            { projectId, instanceId: state.id },
+            `evaluation error for instance "%s": %s`,
+            state.evaluationStatus.message,
           )
-
-          this.compositeInstanceEE.emit(projectId, {
-            type: "failed",
-            instanceId: state.id,
-          })
         }
       }
 
       for (const instance of compositeInstances) {
-        this.compositeInstanceEE.emit(projectId, { type: "updated", instance })
+        void this.pubsubManager.publish(`composite-instance:${projectId}`, {
+          type: "updated",
+          instance,
+        })
       }
 
       for (const instanceId of deletedCompositeInstanceIds) {
-        this.compositeInstanceEE.emit(projectId, { type: "deleted", instanceId })
+        void this.pubsubManager.publish(`composite-instance:${projectId}`, {
+          type: "deleted",
+          instanceId,
+        })
       }
 
       const promises: Promise<void>[] = []
 
       if (newStates.length > 0) {
-        promises.push(this.stateBackend.putInstanceStates(projectId, newStates))
+        promises.push(this.stateManager.getInstanceStateRepository(projectId).putMany(newStates))
       }
 
       if (compositeInstances.length > 0) {
-        promises.push(this.stateBackend.putCompositeInstances(projectId, compositeInstances))
+        promises.push(
+          this.stateManager.getCompositeInstanceRepository(projectId).putMany(compositeInstances),
+        )
       }
 
       this.logger.info(
@@ -274,17 +296,21 @@ export class ProjectManager {
       )
 
       await Promise.all(promises)
-    })
+    } finally {
+      // TODO: implement detecting of lost evaluation locks
+      await this.instanceLockService.unlockInstancesUnconditionally(projectId, instanceIds)
+    }
   }
 
   private async prepareResolvers(projectId: string) {
-    const [{ instances, hubs }, states, library, evaluatedInputHashes] = await Promise.all([
+    const [projectInfo, { instances, hubs }, states] = await Promise.all([
+      this.projectBackend.getProjectInfo(projectId),
       this.projectBackend.getProject(projectId),
-      this.stateBackend.getAllInstanceStates(projectId),
-      this.libraryBackend.loadLibrary(),
-      this.stateBackend.getCompositeInstanceInputHashes(projectId),
+      this.stateManager.getInstanceStateRepository(projectId).getAllItems(),
     ])
 
+    const library = await this.libraryBackend.loadLibrary(projectInfo.libraryId)
+
     const filteredInstances = instances.filter(instance => instance.type in library.components)
     const stateMap = new Map(states.map(state => [state.id, state]))
 
@@ -318,7 +344,10 @@ export class ProjectManager {
 
       let sourceHash: string | undefined
       if (isUnitModel(library.components[instance.type])) {
-        const resolvedUnits = await this.libraryBackend.getResolvedUnitSources([instance.type])
+        const resolvedUnits = await this.libraryBackend.getResolvedUnitSources(
+          projectInfo.libraryId,
+          [instance.type],
+        )
         const resolvedUnit = resolvedUnits.find(unit => unit.unitType === instance.type)
 
         if (resolvedUnit) {
@@ -346,28 +375,58 @@ export class ProjectManager {
       inputHashInputs,
       inputHashResolver,
       library,
+      libraryId: projectInfo.libraryId,
       instances: filteredInstances,
       stateMap,
       resolvedInputs,
-      evaluatedInputHashes,
     }
   }
 
-  private async watchLibraryChanges(): Promise<void> {
-    for await (const updates of this.libraryBackend.watchLibrary()) {
-      try {
-        await this.handleLibraryUpdates(updates)
-      } catch (error) {
-        this.logger.error({ error }, "failed to handle library updates")
+  /**
+   * Ensures a library is being watched for changes.
+   * Called when a project using this library is accessed.
+   */
+  private ensureLibraryWatched(libraryId: string): void {
+    if (this.watchedLibraries.has(libraryId)) {
+      return
+    }
+
+    this.watchedLibraries.add(libraryId)
+    const abortController = new AbortController()
+    this.libraryWatchers.set(libraryId, abortController)
+
+    this.logger.debug({ libraryId }, "starting library watcher")
+
+    // Start watching this library in the background
+    void this.watchSingleLibrary(libraryId, abortController.signal)
+  }
+
+  /**
+   * Watches a single library for changes and handles updates.
+   */
+  private async watchSingleLibrary(libraryId: string, signal: AbortSignal): Promise<void> {
+    try {
+      for await (const updates of this.libraryBackend.watchLibrary(libraryId, signal)) {
+        try {
+          await this.handleLibraryUpdates(libraryId, updates)
+        } catch (error) {
+          this.logger.error({ error, libraryId }, "failed to handle library updates")
+        }
+      }
+    } catch (error) {
+      if (signal.aborted) {
+        this.logger.debug({ libraryId }, "library watcher stopped")
+      } else {
+        this.logger.error({ error, libraryId }, "library watcher failed")
       }
+    } finally {
+      this.watchedLibraries.delete(libraryId)
+      this.libraryWatchers.delete(libraryId)
     }
   }
 
-  private async handleLibraryUpdates(updates: LibraryUpdate[]): Promise<void> {
+  private async handleLibraryUpdates(libraryId: string, updates: LibraryUpdate[]): Promise<void> {
     const changedComponents = new Set<string>()
-    const library = await this.libraryBackend.loadLibrary()
-
-    // TODO: handle entity updates
 
     for (const update of updates) {
       switch (update.type) {
@@ -377,6 +436,11 @@ export class ProjectManager {
         case "component-removed":
           changedComponents.add(update.componentType)
           break
+        // Handle entity updates if needed
+        case "entity-updated":
+        case "entity-removed":
+          // Entity updates don't directly affect composite instance evaluation
+          break
       }
     }
 
@@ -385,49 +449,126 @@ export class ProjectManager {
     }
 
     this.logger.info(
-      { changedComponents },
-      "library components changed, updating composite instances",
+      { libraryId, changedComponents: Array.from(changedComponents) },
+      "library components changed, updating composite instances for affected projects",
     )
 
-    const projects = await this.projectBackend.getProjectIds()
-    for (const projectId of projects) {
-      const { instances } = await this.prepareResolvers(projectId)
+    // Get all projects and find those using this libraryId
+    const allProjectIds = await this.projectBackend.getProjectIds()
 
-      const filteredInstances = instances.filter(
-        instance =>
-          changedComponents.has(instance.type) &&
-          library.components[instance.type] &&
-          !isUnitModel(library.components[instance.type]),
-      )
+    for (const projectId of allProjectIds) {
+      try {
+        // Check if this project uses the changed library
+        const projectInfo = await this.projectBackend.getProjectInfo(projectId)
+        if (projectInfo.libraryId !== libraryId) {
+          continue // Skip projects that don't use this library
+        }
 
-      this.logger.info(
-        { projectId, filteredInstanceIds: filteredInstances.map(instance => instance.id) },
-        "updating composite instances for project",
-      )
+        // Load the updated library for filtering
+        const library = await this.libraryBackend.loadLibrary(libraryId)
+        const { instances } = await this.prepareResolvers(projectId)
 
-      try {
-        await this.evaluateChangedCompositeInstances(projectId)
+        const affectedInstances = instances.filter(
+          instance =>
+            changedComponents.has(instance.type) &&
+            library.components[instance.type] &&
+            !isUnitModel(library.components[instance.type]),
+        )
+
+        if (affectedInstances.length > 0) {
+          this.logger.info(
+            {
+              projectId,
+              libraryId,
+              affectedInstanceIds: affectedInstances.map(instance => instance.id),
+            },
+            "updating composite instances for project due to library changes",
+          )
+
+          await this.evaluateChangedCompositeInstances(projectId)
+        }
       } catch (error) {
-        this.logger.error({ error }, "failed to evaluate composite instances")
+        this.logger.error(
+          { error, projectId, libraryId },
+          "failed to evaluate composite instances for project during library update",
+        )
       }
     }
   }
 
+  /**
+   * Cleanup method to stop all library watchers.
+   * Should be called when the ProjectManager is being shut down.
+   */
+  dispose(): void {
+    for (const [libraryId, abortController] of this.libraryWatchers.entries()) {
+      this.logger.debug({ libraryId }, "stopping library watcher")
+      abortController.abort()
+    }
+    this.libraryWatchers.clear()
+    this.watchedLibraries.clear()
+  }
+
   static create(
     projectBackend: ProjectBackend,
-    stateBackend: StateBackend,
     libraryBackend: LibraryBackend,
-    projectLockManager: ProjectLockManager,
     stateManager: StateManager,
+    instanceLockService: InstanceLockService,
+    pubsubManager: PubSubManager,
     logger: Logger,
   ): ProjectManager {
     return new ProjectManager(
       projectBackend,
-      stateBackend,
       libraryBackend,
-      projectLockManager,
       stateManager,
+      instanceLockService,
+      pubsubManager,
       logger.child({ service: "ProjectManager" }),
     )
   }
+
+  static createSuccessEvaluationMessage(
+    result: InstanceEvaluationResult & { success: true },
+  ): string {
+    const treeNodes = new Map<string, TreeNode>()
+
+    // the order of composite instances are guaranteed to be topologically sorted
+    for (const compositeInstance of result.compositeInstances) {
+      const node: TreeNode = {
+        text: compositeInstance.id,
+        children: [],
+      }
+
+      treeNodes.set(compositeInstance.id, node)
+
+      const parentNode = compositeInstance.instance.parentId
+        ? treeNodes.get(compositeInstance.instance.parentId)
+        : undefined
+
+      if (parentNode) {
+        parentNode.children.push(node)
+      }
+
+      for (const child of compositeInstance.children) {
+        const childNode: TreeNode = {
+          text: child.id,
+          children: [],
+        }
+
+        node.children.push(childNode)
+        treeNodes.set(child.id, childNode)
+      }
+    }
+
+    // get the root node
+    const rootNode = treeNodes.get(result.instanceId)
+    if (!rootNode) {
+      return `Composite instance evaluation completed successfully, but failed to build the tree structure.`
+    }
+
+    // render the tree structure
+    const tree = renderTree(rootNode)
+
+    return `Composite instance evaluation completed successfully.\n\nInstance Tree:\n${tree}`
+  }
 }

package/src/pubsub/abstractions.ts

@@ -0,0 +1,13 @@
+import type { z } from "zod"
+
+export interface PubSubBackend {
+  /**
+   * Subscribes to a topic and returns an async iterator for the messages.
+   */
+  subscribe(topic: string, schema: z.ZodType, signal?: AbortSignal): AsyncIterable<unknown>
+
+  /**
+   * Publishes a message to a topic.
+   */
+  publish(topic: string, schema: z.ZodType, message: unknown): Promise<void>
+}

package/src/pubsub/factory.ts

@@ -0,0 +1,19 @@
+import type { Logger } from "pino"
+import type { PubSubBackend } from "./abstractions"
+import { z } from "zod"
+import { LocalPubSubBackend } from "./local"
+
+export const pubSubBackendConfig = z.object({
+  HIGHSTATE_PUBSUB_BACKEND_TYPE: z.enum(["local"]).default("local"),
+})
+
+export function createPubSubBackend(
+  config: z.infer<typeof pubSubBackendConfig>,
+  logger: Logger,
+): PubSubBackend {
+  switch (config.HIGHSTATE_PUBSUB_BACKEND_TYPE) {
+    case "local": {
+      return LocalPubSubBackend.create(logger)
+    }
+  }
+}

package/src/pubsub/index.ts

@@ -0,0 +1,3 @@
+export * from "./abstractions"
+export * from "./factory"
+export * from "./manager"

package/src/pubsub/local.ts

@@ -0,0 +1,36 @@
+import type { Logger } from "pino"
+import type { PubSubBackend } from "./abstractions"
+import type { z } from "zod"
+import { EventEmitter, on } from "node:events"
+
+export class LocalPubSubBackend implements PubSubBackend {
+  private readonly eventEmitter = new EventEmitter()
+
+  private constructor(private readonly logger: Logger) {}
+
+  async *subscribe<TSchema extends z.ZodSchema>(
+    topic: string,
+    _schema: TSchema,
+    signal?: AbortSignal,
+  ): AsyncIterable<z.infer<TSchema>> {
+    for await (const [event] of on(this.eventEmitter, topic, { signal })) {
+      // do not validate here because all IPC happends in the same process
+      yield event as z.infer<TSchema>
+    }
+  }
+
+  publish<TSchema extends z.ZodSchema>(
+    topic: string,
+    _schema: TSchema,
+    message: z.infer<TSchema>,
+  ): Promise<void> {
+    this.eventEmitter.emit(topic, message)
+    this.logger.trace({ topic, message }, "published message to topic")
+
+    return Promise.resolve()
+  }
+
+  static create(logger: Logger): PubSubBackend {
+    return new LocalPubSubBackend(logger)
+  }
+}