@highstate/backend 0.9.18 → 0.9.20

package/src/business/project.ts

@@ -1,95 +1,114 @@
 import type { Logger } from "pino"
-import type { StateManager } from "../state"
-import type { InputUnlockMethod, Project } from "../shared"
-import type { RandomProvider } from "../common"
-import type { ProjectUnlockService } from "./project-unlock"
-import type { ProjectBackend, ProjectEvaluationSubsystem } from "../project"
-import type { LockManager } from "../lock"
-import type { PubSubManager } from "../pubsub"
+import type { DatabaseManager, Project } from "../database"
 import type { LibraryBackend } from "../library"
+import type { ProjectEvaluationSubsystem, ProjectModelBackend } from "../project-model"
+import type { PubSubManager } from "../pubsub"
+import type { ProjectModelService } from "./project-model"
+import type { ProjectUnlockService } from "./project-unlock"
 import {
-  isUnitModel,
   type HubModel,
   type HubModelPatch,
   type InstanceId,
   type InstanceModel,
   type InstanceModelPatch,
+  isUnitModel,
 } from "@highstate/contract"
-
-export type FullProjectModel = {
-  instances: InstanceModel[]
-  hubs: HubModel[]
-  virtualInstances: InstanceModel[]
-}
-
-export class ProjectNotFoundError extends Error {
-  constructor(readonly projectId: string) {
-    super(`Project with ID "${projectId}" not found`)
-    this.name = "ProjectNotFoundError"
-  }
-}
-
-export class ProjectNameConflictError extends Error {
-  constructor(readonly projectName: string) {
-    super(`Project with name "${projectName}" already exists`)
-    this.name = "ProjectNameConflictError"
-  }
-}
+import { createId } from "@paralleldrive/cuid2"
+import { createProjectLogger } from "../common"
+import { projectDatabaseVersion } from "../database/abstractions"
+import {
+  type FullProjectModel,
+  forSchema,
+  type ProjectInput,
+  type ProjectModelStorageSpec,
+  ProjectNotFoundError,
+  type ProjectOutput,
+  projectOutputSchema,
+  type UnlockMethodInput,
+} from "../shared"
+import { includeForInstanceState, mapInstanceStateResult } from "./instance-state"
 
 export class ProjectService {
   constructor(
-    private readonly stateManager: StateManager,
+    private readonly database: DatabaseManager,
     private readonly projectUnlockService: ProjectUnlockService,
     private readonly projectEvaluationSubsystem: ProjectEvaluationSubsystem,
-    private readonly projectBackend: ProjectBackend,
+    private readonly projectModelService: ProjectModelService,
+    private readonly projectModelBackends: Record<string, ProjectModelBackend>,
     private readonly libraryBackend: LibraryBackend,
-    private readonly lockManager: LockManager,
     private readonly pubsubManager: PubSubManager,
-    private readonly random: RandomProvider,
     private readonly logger: Logger,
   ) {}
 
+  /**
+   * Returns all projects in the system.
+   */
+  async getProjects(): Promise<ProjectOutput[]> {
+    return await this.database.backend.project.findMany({
+      select: forSchema(projectOutputSchema),
+    })
+  }
+
+  /**
+   * Creates a new project with the given input and unlock method.
+   *
+   * This will set up the project database and persist the unlock method.
+   *
+   * @param projectInput The input for the new project.
+   * @param unlockMethodInput The unlock method to use for the new project.
+   */
   async createProject(
-    name: string,
-    meta: Project["meta"],
-    unlockMethod: InputUnlockMethod,
+    projectInput: ProjectInput,
+    unlockMethodInput: UnlockMethodInput,
   ): Promise<Project> {
-    return await this.lockManager.acquire(["project-name", name], async () => {
-      const existingProject = await this.stateManager.getProjectNameIndexRepository().get(name)
-      if (existingProject) {
-        throw new ProjectNameConflictError(name)
-      }
-
-      const batch = this.stateManager.batch()
-
-      const project: Project = {
-        id: this.random.uuidv7(),
-        name,
-        meta,
-      }
-
-      // 1. create the project model
-      await this.projectBackend.createProjectModel(project)
-
-      // 2. create the project in the state manager
-      await this.stateManager.getProjectRepository().putItem(project, batch)
-      await this.stateManager
-        .getProjectNameIndexRepository()
-        .indexRepository.put(project.name, project.id, batch)
-
-      // 3. setup the project unlock state
-      await this.projectUnlockService.setupProjectState(project.id, unlockMethod, batch)
+    // start by generating a random ID
+    const projectId = createId()
+    const logger = createProjectLogger(this.logger, projectId)
+
+    logger.info("creating new project")
+
+    // setup project database
+    const [encryptedMasterKey, unlockSuite] = await this.projectUnlockService.setupProjectDatabase(
+      projectId,
+      unlockMethodInput,
+    )
+
+    logger.info("project database set up")
+
+    // finally, create the project in the database
+    const project = await this.database.backend.project.create({
+      data: {
+        id: projectId,
+        name: projectInput.name,
+        meta: projectInput.meta,
+        databaseVersion: projectDatabaseVersion,
+        encryptedMasterKey,
+        unlockSuite,
+        spaceId: projectInput.spaceId,
+        modelStorageId: projectInput.modelStorageId,
+        libraryId: projectInput.libraryId,
+        pulumiBackendId: projectInput.pulumiBackendId,
+      },
+    })
 
-      await batch.write()
+    logger.info("project successfully created")
 
-      this.logger.info({ project }, "created project")
+    return project
+  }
 
-      return project
+  /**
+   * Returns the project with the given ID.
+   *
+   * If the project does not exist, this will throw an error.
+   *
+   * @param projectId The ID of the project to get.
+   */
+  async getProjectOrThrow(projectId: string): Promise<ProjectOutput> {
+    const project = await this.database.backend.project.findUnique({
+      where: { id: projectId },
+      select: forSchema(projectOutputSchema),
     })
-  }
 
-  async getProjectOrThrow(projectId: string): Promise<Project> {
-    const project = await this.stateManager.getProjectRepository().get(projectId)
     if (!project) {
       throw new ProjectNotFoundError(projectId)
     }
@@ -98,25 +117,18 @@ export class ProjectService {
   }
 
   /**
-   * Get the project model containing instances, hubs + composite instances.
+   * Get the project model containing instances, hubs + virtual and ghost instances.
    *
    * @param projectId The ID of the project to get the model for.
    * @param signal Optional abort signal for cancellation.
    */
-  async getProjectModel(projectId: string, signal?: AbortSignal): Promise<FullProjectModel> {
-    const project = await this.getProjectOrThrow(projectId)
-
-    const [library, { instances, hubs }, virtualInstances] = await Promise.all([
-      this.libraryBackend.loadLibrary(project.libraryId),
-      this.projectBackend.getProjectModel(project, signal),
-      this.stateManager.getVirtualInstanceRepository(projectId).getAllItems(),
-    ])
-
-    return {
-      instances: instances.filter(instance => instance.type in library.components),
-      hubs,
-      virtualInstances: virtualInstances.filter(instance => instance.type in library.components),
-    }
+  async getProjectModel(projectId: string): Promise<FullProjectModel> {
+    const [projectModel] = await this.projectModelService.getProjectModel(projectId, {
+      includeVirtualInstances: true,
+      includeGhostInstances: true,
+    })
+
+    return projectModel
   }
 
   /**
@@ -131,24 +143,43 @@ export class ProjectService {
     instanceId: InstanceId,
     newName: string,
   ): Promise<InstanceModel> {
-    return await this.lockManager.acquire(["project-nodes", projectId], async () => {
-      try {
-        const project = await this.getProjectOrThrow(projectId)
-        const instance = await this.projectBackend.renameInstance(project, instanceId, newName)
-
-        await this.pubsubManager.publish(["project-model", projectId], {
-          updatedInstances: [instance],
-          deletedInstanceIds: [instanceId],
+    try {
+      // rename the instance in the model
+      const { project, backend, spec } = await this.getProjectWithBackend(projectId)
+      const instance = await backend.renameInstance(project, spec, instanceId, newName)
+
+      // rename in the database state
+      const database = await this.database.forProject(projectId)
+
+      await database.$transaction(async tx => {
+        const state = await tx.instanceState.findUnique({
+          where: { instanceId },
+          select: { id: true },
         })
 
-        void this.projectEvaluationSubsystem.evaluateProject(project)
+        if (!state) {
+          // ignore if state doesn't exist yet
+          return
+        }
 
-        return instance
-      } catch (error) {
-        this.logger.error({ error, projectId, instanceId, newName }, "failed to rename instance")
-        throw error
-      }
-    })
+        await tx.instanceState.update({
+          where: { id: state.id },
+          data: { instanceId: instance.id },
+        })
+      })
+
+      await this.pubsubManager.publish(["project-model", projectId], {
+        updatedInstances: [instance],
+        deletedInstanceIds: [instanceId],
+      })
+
+      void this.projectEvaluationSubsystem.evaluateProject(projectId)
+
+      return instance
+    } catch (error) {
+      this.logger.error({ error, projectId, instanceId, newName }, "failed to rename instance")
+      throw error
+    }
   }
 
   /**
@@ -163,30 +194,28 @@ export class ProjectService {
     instanceId: InstanceId,
     patch: InstanceModelPatch,
   ): Promise<InstanceModel> {
-    return await this.lockManager.acquire(["project-nodes", projectId], async () => {
-      try {
-        const project = await this.getProjectOrThrow(projectId)
-        const instance = await this.projectBackend.updateInstance(project, instanceId, patch)
-        const library = await this.libraryBackend.loadLibrary(project.libraryId)
-
-        await this.pubsubManager.publish(["project-model", projectId], {
-          updatedInstances: [instance],
-        })
-
-        if (!isUnitModel(library.components[instance.type]) && patch.args) {
-          // evaluate the project if arguments changed for composite instancer
-          void this.projectEvaluationSubsystem.evaluateProject(project)
-        } else if (patch.hubInputs || patch.injectionInputs || patch.inputs) {
-          // TODO: only evaluate if inputs changed for composite instances
-          void this.projectEvaluationSubsystem.evaluateProject(project)
-        }
-
-        return instance
-      } catch (error) {
-        this.logger.error({ error, projectId, instanceId }, "failed to update instance")
-        throw error
+    try {
+      const { project, backend, spec } = await this.getProjectWithBackend(projectId)
+      const instance = await backend.updateInstance(project, spec, instanceId, patch)
+      const library = await this.libraryBackend.loadLibrary(project.libraryId)
+
+      await this.pubsubManager.publish(["project-model", projectId], {
+        updatedInstances: [instance],
+      })
+
+      if (!isUnitModel(library.components[instance.type]) && patch.args) {
+        // evaluate the project if arguments changed for composite instancer
+        void this.projectEvaluationSubsystem.evaluateProject(projectId)
+      } else if (patch.hubInputs || patch.injectionInputs || patch.inputs) {
+        // TODO: only evaluate if inputs changed for composite instances
+        void this.projectEvaluationSubsystem.evaluateProject(projectId)
       }
-    })
+
+      return instance
+    } catch (error) {
+      this.logger.error({ error, projectId, instanceId }, "failed to update instance")
+      throw error
+    }
   }
 
   /**
@@ -196,21 +225,19 @@ export class ProjectService {
    * @param instanceId The ID of the instance to delete.
    */
   async deleteInstance(projectId: string, instanceId: InstanceId): Promise<void> {
-    return await this.lockManager.acquire(["project-nodes", projectId], async () => {
-      try {
-        const project = await this.getProjectOrThrow(projectId)
-        await this.projectBackend.deleteInstance(project, instanceId)
-
-        await this.pubsubManager.publish(["project-model", projectId], {
-          deletedInstanceIds: [instanceId],
-        })
-
-        void this.projectEvaluationSubsystem.evaluateProject(project)
-      } catch (error) {
-        this.logger.error({ error, projectId, instanceId }, "failed to delete instance")
-        throw error
-      }
-    })
+    try {
+      const { project, backend, spec } = await this.getProjectWithBackend(projectId)
+      await backend.deleteInstance(project, spec, instanceId)
+
+      await this.pubsubManager.publish(["project-model", projectId], {
+        deletedInstanceIds: [instanceId],
+      })
+
+      void this.projectEvaluationSubsystem.evaluateProject(projectId)
+    } catch (error) {
+      this.logger.error({ error, projectId, instanceId }, "failed to delete instance")
+      throw error
+    }
   }
 
   /**
@@ -221,23 +248,23 @@ export class ProjectService {
    * @param patch The patch to apply to the hub.
    */
   async updateHub(projectId: string, hubId: string, patch: HubModelPatch): Promise<HubModel> {
-    return await this.lockManager.acquire(["project-nodes", projectId], async () => {
-      try {
-        const project = await this.getProjectOrThrow(projectId)
-        const hub = await this.projectBackend.updateHub(project, hubId, patch)
+    try {
+      const { project, backend, spec } = await this.getProjectWithBackend(projectId)
+      const hub = await backend.updateHub(project, spec, hubId, patch)
 
-        await this.pubsubManager.publish(["project-model", projectId], {
-          updatedHubs: [hub],
-        })
-
-        void this.projectEvaluationSubsystem.evaluateProject(project)
+      await this.pubsubManager.publish(["project-model", projectId], {
+        updatedHubs: [hub],
+      })
 
-        return hub
-      } catch (error) {
-        this.logger.error({ error, projectId, hubId }, "failed to update hub")
-        throw error
+      if (patch.inputs || patch.injectionInputs) {
+        void this.projectEvaluationSubsystem.evaluateProject(projectId)
       }
-    })
+
+      return hub
+    } catch (error) {
+      this.logger.error({ error, projectId, hubId }, "failed to update hub")
+      throw error
+    }
   }
 
   /**
@@ -247,21 +274,19 @@ export class ProjectService {
    * @param hubId The ID of the hub to delete.
    */
   async deleteHub(projectId: string, hubId: string): Promise<void> {
-    return await this.lockManager.acquire(["project-nodes", projectId], async () => {
-      try {
-        const project = await this.getProjectOrThrow(projectId)
-        await this.projectBackend.deleteHub(project, hubId)
-
-        await this.pubsubManager.publish(["project-model", projectId], {
-          deletedHubIds: [hubId],
-        })
-
-        void this.projectEvaluationSubsystem.evaluateProject(project)
-      } catch (error) {
-        this.logger.error({ error, projectId, hubId }, "failed to delete hub")
-        throw error
-      }
-    })
+    try {
+      const { project, backend, spec } = await this.getProjectWithBackend(projectId)
+      await backend.deleteHub(project, spec, hubId)
+
+      await this.pubsubManager.publish(["project-model", projectId], {
+        deletedHubIds: [hubId],
+      })
+
+      void this.projectEvaluationSubsystem.evaluateProject(projectId)
+    } catch (error) {
+      this.logger.error({ error, projectId, hubId }, "failed to delete hub")
+      throw error
+    }
   }
 
   /**
@@ -276,24 +301,107 @@ export class ProjectService {
     instances: InstanceModel[],
     hubs: HubModel[],
   ): Promise<void> {
-    return await this.lockManager.acquire(["project-nodes", projectId], async () => {
-      try {
-        const project = await this.getProjectOrThrow(projectId)
-        await this.projectBackend.createNodes(project, instances, hubs)
-
-        await this.pubsubManager.publish(["project-model", projectId], {
-          updatedHubs: hubs,
-          updatedInstances: instances,
-        })
-
-        void this.projectEvaluationSubsystem.evaluateProject(project)
-      } catch (error) {
-        this.logger.error(
-          { error, projectId, instanceCount: instances.length, hubCount: hubs.length },
-          "failed to create many nodes",
+    try {
+      const database = await this.database.forProject(projectId)
+
+      const states = await database.$transaction(async tx => {
+        const { project, backend, spec } = await this.getProjectWithBackend(projectId)
+        await backend.createNodes(project, spec, instances, hubs)
+
+        // ensure instance states exist for created instances
+        return await Promise.all(
+          instances.map(async instance => {
+            const result = await tx.instanceState.upsert({
+              where: { instanceId: instance.id },
+              create: {
+                instanceId: instance.id,
+                kind: instance.kind,
+                source: "resident",
+                status: "undeployed",
+              },
+              update: {
+                // turn any virtual instance into resident
+                // the next evaluation will throw an error indicating the instance is now duplicate and will no longer produce this virtual instance
+                source: "resident",
+              },
+              // in case we restoring instance for existing state, to stream it correctly
+              include: includeForInstanceState({
+                includeEvaluationState: true,
+                includeExtra: true,
+                includeLastOperationState: true,
+                loadCustomStatuses: true,
+              }),
+            })
+
+            return mapInstanceStateResult(result)
+          }),
         )
-        throw error
+      })
+
+      void this.pubsubManager.publish(["project-model", projectId], {
+        updatedHubs: hubs,
+        updatedInstances: instances,
+      })
+
+      for (const state of states) {
+        void this.pubsubManager.publish(["instance-state", projectId], { type: "updated", state })
       }
+
+      void this.projectEvaluationSubsystem.evaluateProject(projectId)
+    } catch (error) {
+      this.logger.error(
+        { error, projectId, instanceCount: instances.length, hubCount: hubs.length },
+        "failed to create many nodes",
+      )
+      throw error
+    }
+  }
+
+  /**
+   * Get the project, backend, and storage spec in a single optimized SQL call.
+   *
+   * @param projectId The ID of the project.
+   * @returns Object containing project, backend, and spec.
+   */
+  private async getProjectWithBackend(projectId: string): Promise<{
+    project: ProjectOutput
+    backend: ProjectModelBackend
+    spec: ProjectModelStorageSpec
+  }> {
+    const result = await this.database.backend.project.findUnique({
+      where: { id: projectId },
+      select: {
+        ...forSchema(projectOutputSchema),
+        modelStorage: {
+          select: {
+            spec: true,
+          },
+        },
+      },
     })
+
+    if (!result) {
+      throw new ProjectNotFoundError(projectId)
+    }
+
+    const { modelStorage, ...project } = result
+    const spec = modelStorage.spec
+    const backend = this.getProjectModelBackend(spec.type)
+
+    return { project, backend, spec }
+  }
+
+  /**
+   * Get the appropriate project model backend for the given project.
+   *
+   * @param project The project to get the backend for.
+   * @returns The project model backend.
+   */
+  private getProjectModelBackend(type: string): ProjectModelBackend {
+    const backend = this.projectModelBackends[type]
+    if (!backend) {
+      throw new Error(`Project model backend not found for type: ${type}`)
+    }
+    return backend
   }
 }