@highstate/backend 0.9.16 → 0.9.18

package/src/project/manager.ts

@@ -1,574 +0,0 @@
-import type { StateManager } from "../state"
-import type { ProjectBackend } from "./abstractions"
-import type { Logger } from "pino"
-import type { InstanceEvaluationResult, LibraryBackend } from "../library"
-import type { InstanceLockService } from "../business"
-import type { PubSubManager } from "../pubsub"
-import { randomUUID } from "node:crypto"
-import { isUnitModel, type InstanceModel } from "@highstate/contract"
-import {
-  type InputResolverNode,
-  type InputHashNode,
-  type InstanceModelPatch,
-  type CompositeInstance,
-  type ResolvedInstanceInput,
-  type HubModel,
-  type LibraryUpdate,
-  InputResolver,
-  InputHashResolver,
-} from "../shared"
-import { renderTree, type TreeNode } from "../common"
-
-export type FullProjectModel = {
-  libraryId: string
-  instances: InstanceModel[]
-  hubs: HubModel[]
-  compositeInstances: CompositeInstance[]
-}
-
-export class ProjectManager {
-  private readonly watchedLibraries = new Set<string>()
-  private readonly libraryWatchers = new Map<string, AbortController>()
-
-  private constructor(
-    private readonly projectBackend: ProjectBackend,
-    private readonly libraryBackend: LibraryBackend,
-    private readonly stateManager: StateManager,
-    private readonly instanceLockService: InstanceLockService,
-    private readonly pubsubManager: PubSubManager,
-    private readonly logger: Logger,
-  ) {}
-
-  /**
-   * Loads the full info of a project, including instances, hubs, and composite instances.
-   *
-   * Also filters out instances that are not in the library.
-   *
-   * @param projectId The ID of the project to load.
-   */
-  async getProject(projectId: string): Promise<FullProjectModel> {
-    const [projectInfo, project] = await Promise.all([
-      this.projectBackend.getProjectInfo(projectId),
-      this.projectBackend.getProject(projectId),
-    ])
-
-    // Ensure we're watching this library for changes
-    this.ensureLibraryWatched(projectInfo.libraryId)
-
-    const [compositeInstances, library] = await Promise.all([
-      this.stateManager.getCompositeInstanceRepository(projectId).getAllItems(),
-      this.libraryBackend.loadLibrary(projectInfo.libraryId),
-    ])
-
-    const filteredInstances = project.instances.filter(
-      instance => instance.type in library.components,
-    )
-    const filteredCompositeInstances = compositeInstances
-      .filter(instance => instance.instance.type in library.components)
-      .map(instance => ({
-        ...instance,
-        children: instance.children.filter(child => child.type in library.components),
-      }))
-
-    return {
-      libraryId: projectInfo.libraryId,
-      instances: filteredInstances,
-      hubs: project.hubs,
-      compositeInstances: filteredCompositeInstances,
-    }
-  }
-
-  async createInstance(projectId: string, instance: InstanceModel): Promise<InstanceModel> {
-    const createdInstance = await this.projectBackend.createInstance(projectId, instance)
-    await this.evaluateChangedCompositeInstances(projectId)
-
-    return createdInstance
-  }
-
-  async updateInstance(
-    projectId: string,
-    instanceId: string,
-    patch: InstanceModelPatch,
-  ): Promise<InstanceModel> {
-    const instance = await this.projectBackend.updateInstance(projectId, instanceId, patch)
-    // await this.evaluateChangedCompositeInstances(projectId)
-
-    return instance
-  }
-
-  async renameInstance(
-    projectId: string,
-    instanceId: string,
-    newName: string,
-  ): Promise<InstanceModel> {
-    const instance = await this.projectBackend.renameInstance(projectId, instanceId, newName)
-    await this.evaluateChangedCompositeInstances(projectId)
-
-    return instance
-  }
-
-  async deleteInstance(projectId: string, instanceId: string): Promise<void> {
-    await Promise.all([
-      this.projectBackend.deleteInstance(projectId, instanceId),
-      this.stateManager.getCompositeInstanceRepository(projectId).delete(instanceId),
-    ])
-  }
-
-  async createManyNodes(
-    projectId: string,
-    instances: InstanceModel[],
-    hubs: HubModel[],
-  ): Promise<void> {
-    await this.projectBackend.createManyNodes(projectId, instances, hubs)
-  }
-
-  private async evaluateChangedCompositeInstances(projectId: string): Promise<void> {
-    const { inputHashResolver, instances, library, evaluatedInputHashes } =
-      await this.prepareResolvers(projectId)
-
-    inputHashResolver.addAllNodesToWorkset()
-    await inputHashResolver.process()
-
-    const instanceIds = instances
-      .filter(instance => !isUnitModel(library.components[instance.type]))
-      .filter(
-        instance =>
-          evaluatedInputHashes[instance.id] !==
-          inputHashResolver.requireOutput(instance.id).inputHash,
-      )
-      .map(instance => instance.id)
-
-    await this.instanceLockService.tryLockInstances(
-      projectId,
-      instanceIds,
-      {
-        title: "Composite Instance Evaluation",
-        description: "This instance is now being evaluated",
-        icon: "logos:typescript-icon-round",
-      },
-      {
-        type: "evaluation",
-        evaluationId: randomUUID(),
-      },
-    )
-
-    try {
-      this.logger.debug({ instanceIds }, "evaluating composite instances")
-
-      for (const instanceId of instanceIds) {
-        void this.pubsubManager.publish(`instance:state:patch:${projectId}`, {
-          patch: {
-            id: instanceId,
-            evaluationStatus: {
-              status: "evaluating",
-            },
-          },
-        })
-      }
-
-      const [
-        { instances, resolvedInputs, stateMap, inputHashResolver, libraryId },
-        existingCompositeInstances,
-      ] = await Promise.all([
-        this.prepareResolvers(projectId),
-        this.stateManager.getCompositeInstanceRepository(projectId).getAllRecord(),
-      ])
-
-      // Get child instance IDs for the instances being re-evaluated
-      const oldChildInstanceIds = new Set(
-        existingCompositeInstances
-          .filter(ci => instanceIds.includes(ci.instance.id))
-          .flatMap(ci => ci.childCompositeInstanceIds ?? []),
-      )
-
-      const results = await this.libraryBackend.evaluateCompositeInstances(
-        libraryId,
-        instances,
-        resolvedInputs,
-        instanceIds,
-      )
-
-      const newStates = results.map(result => {
-        const existingState = stateMap.get(result.instanceId)
-        const newState = existingState ?? { id: result.instanceId }
-
-        newState.evaluationStatus = result.success
-          ? { status: "evaluated", message: ProjectManager.createSuccessEvaluationMessage(result) }
-          : { status: "error", message: result.error }
-
-        return newState
-      })
-
-      inputHashResolver.addAllNodesToWorkset()
-      await inputHashResolver.process()
-
-      const compositeInstances = results
-        .filter(result => result.success)
-        .flatMap(result =>
-          result.compositeInstances.map(instance => ({
-            ...instance,
-            id: instance.instance.id,
-            inputHash:
-              // only store inputHash for top-level composite instances
-              instance.instance.id === result.instanceId
-                ? inputHashResolver.requireOutput(instance.instance.id).inputHash
-                : "",
-            // only store childCompositeInstanceIds for top-level composite instances
-            childCompositeInstanceIds:
-              instance.instance.id === result.instanceId
-                ? result.compositeInstances
-                    .filter(ci => ci.instance.id !== result.instanceId)
-                    .map(ci => ci.instance.id)
-                : undefined,
-          })),
-        )
-
-      const newChildInstanceIds = new Set(
-        results
-          .filter(result => result.success)
-          .flatMap(result =>
-            result.compositeInstances
-              .filter(instance => instance.instance.id !== result.instanceId)
-              .map(instance => instance.instance.id),
-          ),
-      )
-
-      const deletedCompositeInstanceIds = new Set<string>()
-
-      // Find child instances that were deleted (existed before but don't exist now)
-      for (const oldChildId of oldChildInstanceIds) {
-        if (!newChildInstanceIds.has(oldChildId)) {
-          deletedCompositeInstanceIds.add(oldChildId)
-        }
-      }
-
-      await this.stateManager
-        .getCompositeInstanceRepository(projectId)
-        .deleteMany(Array.from(deletedCompositeInstanceIds))
-
-      for (const state of newStates) {
-        void this.pubsubManager.publish(`instance:state:patch:${projectId}`, { patch: state })
-
-        if (state.evaluationStatus?.status === "error") {
-          this.logger.error(
-            { projectId, instanceId: state.id },
-            `evaluation error for instance "%s": %s`,
-            state.evaluationStatus.message,
-          )
-        }
-      }
-
-      for (const instance of compositeInstances) {
-        void this.pubsubManager.publish(`composite-instance:${projectId}`, {
-          type: "updated",
-          instance,
-        })
-      }
-
-      for (const instanceId of deletedCompositeInstanceIds) {
-        void this.pubsubManager.publish(`composite-instance:${projectId}`, {
-          type: "deleted",
-          instanceId,
-        })
-      }
-
-      const promises: Promise<void>[] = []
-
-      if (newStates.length > 0) {
-        promises.push(this.stateManager.getInstanceStateRepository(projectId).putMany(newStates))
-      }
-
-      if (compositeInstances.length > 0) {
-        promises.push(
-          this.stateManager.getCompositeInstanceRepository(projectId).putMany(compositeInstances),
-        )
-      }
-
-      this.logger.info(
-        { projectId },
-        "instance evaluation completed, %d instances persisted",
-        compositeInstances.length,
-      )
-
-      this.logger.debug(
-        { compositeInstanceIds: compositeInstances.map(instance => instance.instance.id) },
-        "persisted composite instances",
-      )
-
-      await Promise.all(promises)
-    } finally {
-      // TODO: implement detecting of lost evaluation locks
-      await this.instanceLockService.unlockInstancesUnconditionally(projectId, instanceIds)
-    }
-  }
-
-  private async prepareResolvers(projectId: string) {
-    const [projectInfo, { instances, hubs }, states] = await Promise.all([
-      this.projectBackend.getProjectInfo(projectId),
-      this.projectBackend.getProject(projectId),
-      this.stateManager.getInstanceStateRepository(projectId).getAllItems(),
-    ])
-
-    const library = await this.libraryBackend.loadLibrary(projectInfo.libraryId)
-
-    const filteredInstances = instances.filter(instance => instance.type in library.components)
-    const stateMap = new Map(states.map(state => [state.id, state]))
-
-    const inputResolverNodes = new Map<string, InputResolverNode>()
-
-    for (const instance of filteredInstances) {
-      inputResolverNodes.set(`instance:${instance.id}`, {
-        kind: "instance",
-        instance,
-        component: library.components[instance.type],
-      })
-    }
-
-    for (const hub of hubs) {
-      inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
-    }
-
-    const inputResolver = new InputResolver(inputResolverNodes, this.logger)
-    inputResolver.addAllNodesToWorkset()
-
-    const inputHashInputs = new Map<string, InputHashNode>()
-    const resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>> = {}
-
-    await inputResolver.process()
-
-    for (const instance of filteredInstances) {
-      const output = inputResolver.requireOutput(`instance:${instance.id}`)
-      if (output.kind !== "instance") {
-        throw new Error("Expected instance node")
-      }
-
-      let sourceHash: string | undefined
-      if (isUnitModel(library.components[instance.type])) {
-        const resolvedUnits = await this.libraryBackend.getResolvedUnitSources(
-          projectInfo.libraryId,
-          [instance.type],
-        )
-        const resolvedUnit = resolvedUnits.find(unit => unit.unitType === instance.type)
-
-        if (resolvedUnit) {
-          sourceHash = resolvedUnit.sourceHash
-        } else {
-          this.logger.warn(`resolved unit source not found for type "%s"`, instance.type)
-          sourceHash = undefined
-        }
-      }
-
-      inputHashInputs.set(instance.id, {
-        instance,
-        component: library.components[instance.type],
-        resolvedInputs: output.resolvedInputs,
-        state: stateMap.get(instance.id),
-        sourceHash,
-      })
-
-      resolvedInputs[instance.id] = output.resolvedInputs
-    }
-
-    const inputHashResolver = new InputHashResolver(inputHashInputs, this.logger)
-
-    return {
-      inputHashInputs,
-      inputHashResolver,
-      library,
-      libraryId: projectInfo.libraryId,
-      instances: filteredInstances,
-      stateMap,
-      resolvedInputs,
-    }
-  }
-
-  /**
-   * Ensures a library is being watched for changes.
-   * Called when a project using this library is accessed.
-   */
-  private ensureLibraryWatched(libraryId: string): void {
-    if (this.watchedLibraries.has(libraryId)) {
-      return
-    }
-
-    this.watchedLibraries.add(libraryId)
-    const abortController = new AbortController()
-    this.libraryWatchers.set(libraryId, abortController)
-
-    this.logger.debug({ libraryId }, "starting library watcher")
-
-    // Start watching this library in the background
-    void this.watchSingleLibrary(libraryId, abortController.signal)
-  }
-
-  /**
-   * Watches a single library for changes and handles updates.
-   */
-  private async watchSingleLibrary(libraryId: string, signal: AbortSignal): Promise<void> {
-    try {
-      for await (const updates of this.libraryBackend.watchLibrary(libraryId, signal)) {
-        try {
-          await this.handleLibraryUpdates(libraryId, updates)
-        } catch (error) {
-          this.logger.error({ error, libraryId }, "failed to handle library updates")
-        }
-      }
-    } catch (error) {
-      if (signal.aborted) {
-        this.logger.debug({ libraryId }, "library watcher stopped")
-      } else {
-        this.logger.error({ error, libraryId }, "library watcher failed")
-      }
-    } finally {
-      this.watchedLibraries.delete(libraryId)
-      this.libraryWatchers.delete(libraryId)
-    }
-  }
-
-  private async handleLibraryUpdates(libraryId: string, updates: LibraryUpdate[]): Promise<void> {
-    const changedComponents = new Set<string>()
-
-    for (const update of updates) {
-      switch (update.type) {
-        case "component-updated":
-          changedComponents.add(update.component.type)
-          break
-        case "component-removed":
-          changedComponents.add(update.componentType)
-          break
-        // Handle entity updates if needed
-        case "entity-updated":
-        case "entity-removed":
-          // Entity updates don't directly affect composite instance evaluation
-          break
-      }
-    }
-
-    if (changedComponents.size === 0) {
-      return
-    }
-
-    this.logger.info(
-      { libraryId, changedComponents: Array.from(changedComponents) },
-      "library components changed, updating composite instances for affected projects",
-    )
-
-    // Get all projects and find those using this libraryId
-    const allProjectIds = await this.projectBackend.getProjectIds()
-
-    for (const projectId of allProjectIds) {
-      try {
-        // Check if this project uses the changed library
-        const projectInfo = await this.projectBackend.getProjectInfo(projectId)
-        if (projectInfo.libraryId !== libraryId) {
-          continue // Skip projects that don't use this library
-        }
-
-        // Load the updated library for filtering
-        const library = await this.libraryBackend.loadLibrary(libraryId)
-        const { instances } = await this.prepareResolvers(projectId)
-
-        const affectedInstances = instances.filter(
-          instance =>
-            changedComponents.has(instance.type) &&
-            library.components[instance.type] &&
-            !isUnitModel(library.components[instance.type]),
-        )
-
-        if (affectedInstances.length > 0) {
-          this.logger.info(
-            {
-              projectId,
-              libraryId,
-              affectedInstanceIds: affectedInstances.map(instance => instance.id),
-            },
-            "updating composite instances for project due to library changes",
-          )
-
-          await this.evaluateChangedCompositeInstances(projectId)
-        }
-      } catch (error) {
-        this.logger.error(
-          { error, projectId, libraryId },
-          "failed to evaluate composite instances for project during library update",
-        )
-      }
-    }
-  }
-
-  /**
-   * Cleanup method to stop all library watchers.
-   * Should be called when the ProjectManager is being shut down.
-   */
-  dispose(): void {
-    for (const [libraryId, abortController] of this.libraryWatchers.entries()) {
-      this.logger.debug({ libraryId }, "stopping library watcher")
-      abortController.abort()
-    }
-    this.libraryWatchers.clear()
-    this.watchedLibraries.clear()
-  }
-
-  static create(
-    projectBackend: ProjectBackend,
-    libraryBackend: LibraryBackend,
-    stateManager: StateManager,
-    instanceLockService: InstanceLockService,
-    pubsubManager: PubSubManager,
-    logger: Logger,
-  ): ProjectManager {
-    return new ProjectManager(
-      projectBackend,
-      libraryBackend,
-      stateManager,
-      instanceLockService,
-      pubsubManager,
-      logger.child({ service: "ProjectManager" }),
-    )
-  }
-
-  static createSuccessEvaluationMessage(
-    result: InstanceEvaluationResult & { success: true },
-  ): string {
-    const treeNodes = new Map<string, TreeNode>()
-
-    // the order of composite instances are guaranteed to be topologically sorted
-    for (const compositeInstance of result.compositeInstances) {
-      const node: TreeNode = {
-        text: compositeInstance.id,
-        children: [],
-      }
-
-      treeNodes.set(compositeInstance.id, node)
-
-      const parentNode = compositeInstance.instance.parentId
-        ? treeNodes.get(compositeInstance.instance.parentId)
-        : undefined
-
-      if (parentNode) {
-        parentNode.children.push(node)
-      }
-
-      for (const child of compositeInstance.children) {
-        const childNode: TreeNode = {
-          text: child.id,
-          children: [],
-        }
-
-        node.children.push(childNode)
-        treeNodes.set(child.id, childNode)
-      }
-    }
-
-    // get the root node
-    const rootNode = treeNodes.get(result.instanceId)
-    if (!rootNode) {
-      return `Composite instance evaluation completed successfully, but failed to build the tree structure.`
-    }
-
-    // render the tree structure
-    const tree = renderTree(rootNode)
-
-    return `Composite instance evaluation completed successfully.\n\nInstance Tree:\n${tree}`
-  }
-}

package/src/shared/models/project/component.ts

@@ -1,45 +0,0 @@
-import { z } from "zod"
-
-export const metaSchema = z.object({
-  displayName: z.string().optional(),
-  description: z.string().optional(),
-  color: z.string().optional(),
-})
-
-export const entitySchema = z.object({
-  type: z.string(),
-  schema: z.any(), // TSchema from TypeBox
-  meta: metaSchema,
-  definitionHash: z.string(),
-})
-
-export const componentMetaSchema = metaSchema.extend({
-  primaryIcon: z.string().optional(),
-  primaryIconColor: z.string().optional(),
-  secondaryIcon: z.string().optional(),
-  secondaryIconColor: z.string().optional(),
-  category: z.string().optional(),
-  defaultNamePrefix: z.string().optional(),
-})
-
-export const componentArgumentSchema = z.object({
-  schema: z.any(), // ArgumentValueSchema from TypeBox
-  required: z.boolean(),
-  meta: componentMetaSchema,
-})
-
-export const componentInputSchema = z.object({
-  type: z.string(),
-  required: z.boolean(),
-  multiple: z.boolean(),
-  meta: componentMetaSchema,
-})
-
-export const componentModelSchema = z.object({
-  type: z.string(),
-  args: z.record(componentArgumentSchema),
-  inputs: z.record(componentInputSchema),
-  outputs: z.record(componentInputSchema),
-  meta: componentMetaSchema,
-  definitionHash: z.string(),
-})

package/src/shared/models/project/instance.ts

@@ -1,74 +0,0 @@
-import { z } from "zod"
-
-export const positionSchema = z.object({
-  x: z.number(),
-  y: z.number(),
-})
-
-export const instanceInputSchema = z.object({
-  instanceId: z.string(),
-  output: z.string(),
-})
-
-export const hubInstanceInputSchema = z.object({
-  hubId: z.string(),
-})
-
-export const instanceModelPatchSchema = z.object({
-  args: z.record(z.unknown()).optional(),
-  inputs: z.record(z.array(instanceInputSchema)).optional(),
-  hubInputs: z.record(z.array(hubInstanceInputSchema)).optional(),
-  injectionInputs: z.array(hubInstanceInputSchema).optional(),
-  position: positionSchema.optional(),
-})
-
-export const instanceModelSchema = z.object({
-  id: z.string(),
-  type: z.string(),
-  name: z.string(),
-
-  ...instanceModelPatchSchema.shape,
-  resolvedInputs: z.record(z.array(instanceInputSchema)).optional(),
-
-  parentId: z.string().optional(),
-  outputs: z.record(z.array(instanceInputSchema)).optional(),
-  resolvedOutputs: z.record(z.array(instanceInputSchema)).optional(),
-})
-
-export const compositeInstanceSchema = z.object({
-  instance: instanceModelSchema,
-  children: z.array(instanceModelSchema),
-  childCompositeInstanceIds: z.array(z.string()).optional(),
-  inputHash: z.string().optional(),
-})
-
-export type CompositeInstance = z.infer<typeof compositeInstanceSchema>
-
-export const compositeInstanceEventSchema = z.discriminatedUnion("type", [
-  z.object({
-    type: z.literal("updated"),
-    instance: compositeInstanceSchema,
-  }),
-  z.object({
-    type: z.literal("deleted"),
-    instanceId: z.string(),
-  }),
-])
-
-export const hubModelPatchSchema = z.object({
-  position: positionSchema.optional(),
-  inputs: z.array(instanceInputSchema).optional(),
-  injectionInputs: z.array(hubInstanceInputSchema).optional(),
-})
-
-export const hubModelSchema = z.object({
-  id: z.string().nanoid(),
-  position: positionSchema,
-
-  inputs: z.array(instanceInputSchema).optional(),
-  injectionInputs: z.array(hubInstanceInputSchema).optional(),
-})
-
-export type InstanceModelPatch = z.infer<typeof instanceModelPatchSchema>
-export type HubModel = z.infer<typeof hubModelSchema>
-export type HubModelPatch = z.infer<typeof hubModelPatchSchema>