@highstate/backend 0.9.18 → 0.9.19

package/src/business/operation.ts

@@ -1,251 +1,217 @@
+import type { InstanceId } from "@highstate/contract"
 import type { Logger } from "pino"
+import type { DatabaseManager, Operation, OperationStatus, OperationUpdateInput } from "../database"
 import type { PubSubManager } from "../pubsub"
-import type { HotStateManager } from "../hotstate"
-import { v7 as uuidv7 } from "uuid"
-import { SAME_KEY, type StateManager } from "../state"
-import { isFinalOperationStatus, type Operation } from "../shared"
+import { ulid } from "ulid"
+import {
+  type OperationMeta,
+  OperationNotFoundError,
+  type OperationOptions,
+  type OperationType,
+} from "../shared"
 
 export class OperationService {
   constructor(
-    private readonly stateManager: StateManager,
-    private readonly hotStateManager: HotStateManager,
+    private readonly database: DatabaseManager,
     private readonly pubsubManager: PubSubManager,
     private readonly logger: Logger,
   ) {}
 
-  async getActiveOperaiton(projectId: string, operationId: string): Promise<Operation | null> {
-    try {
-      return await this.hotStateManager.hget(["active-operations", projectId], operationId)
-    } catch (error) {
-      this.logger.error(
-        { projectId, operationId, error },
-        `failed to get active operation with ID "%s" in project "%s" from hot state`,
-        operationId,
-        projectId,
-      )
+  /**
+   * Creates a new operation in the database.
+   *
+   * @param projectId The project ID to which the operation belongs.
+   * @param meta The operation metadata.
+   * @param type The operation type.
+   * @param requestedInstanceIds The instance IDs that were explicitly requested.
+   * @param options The operation options.
+   * @returns The created operation.
+   */
+  async createOperation(
+    projectId: string,
+    meta: OperationMeta,
+    type: OperationType,
+    requestedInstanceIds: InstanceId[],
+    options: OperationOptions,
+  ): Promise<Operation> {
+    const database = await this.database.forProject(projectId)
+
+    const operation = await database.operation.create({
+      data: {
+        meta,
+        type,
+        options,
+        requestedInstanceIds,
+        startedAt: new Date(),
+      },
+    })
+
+    await this.pubsubManager.publish(["operation", projectId], {
+      type: "updated",
+      operation,
+    })
+
+    this.logger.info({ projectId, operationId: operation.id }, "created operation")
+    return operation
+  }
 
-      return null
-    }
+  /**
+   * Updates an existing operation in the database.
+   *
+   * @param projectId The project ID containing the operation.
+   * @param operationId The operation ID to update.
+   * @param updates The updates to apply.
+   * @returns The updated operation.
+   */
+  async updateOperation(
+    projectId: string,
+    operationId: string,
+    updates: OperationUpdateInput,
+  ): Promise<Operation> {
+    const database = await this.database.forProject(projectId)
+
+    const operation = await database.operation.update({
+      where: { id: operationId },
+      data: updates,
+    })
+
+    await this.pubsubManager.publish(["operation", projectId], {
+      type: "updated",
+      operation,
+    })
+
+    this.logger.info({ projectId, operationId }, "updated operation")
+    return operation
   }
 
   /**
-   * Persists an operation to the state and publishes an update event.
+   * Gets an operation by ID.
    *
-   * @param projectId The project ID to which the operation belongs.
-   * @param operation The operation to persist.
+   * @param projectId The project ID containing the operation.
+   * @param operationId The operation ID.
+   * @returns The operation or undefined if not found.
    */
-  async updateOperation(projectId: string, operation: Operation): Promise<void> {
-    //  1. update the operation in the hot state
-    try {
-      if (isFinalOperationStatus(operation.status)) {
-        await this.hotStateManager.hdel(["active-operations", projectId], operation.id)
-      } else {
-        await this.hotStateManager.hset(["active-operations", projectId], operation.id, operation)
-      }
-    } catch (error) {
-      this.logger.error(
-        { projectId, operationId: operation.id, error },
-        `failed to update operation with ID "%s" in project "%s" in hot state`,
-        operation.id,
-        projectId,
-      )
-    }
+  async getOperation(projectId: string, operationId: string): Promise<Operation | undefined> {
+    const database = await this.database.forProject(projectId)
 
-    // 2. persist the operation in the state repository
-    try {
-      await using batch = this.stateManager.batch()
-
-      // persist the operation to the operation repository
-      await this.stateManager.getOperationRepository(projectId).put(operation.id, operation, batch)
-
-      if (operation.status === "pending") {
-        // add the just created operation to the active operations index
-        await this.stateManager
-          .getActiveOperationIndexRepository(projectId)
-          .indexRepository.put(operation.id, operation.id, batch)
-      } else if (isFinalOperationStatus(operation.status)) {
-        // remove the finished operation from the active operations index
-        await this.stateManager
-          .getActiveOperationIndexRepository(projectId)
-          .indexRepository.delete(operation.id, batch)
-      }
-
-      await batch.write()
-
-      try {
-        await this.pubsubManager.publish(["operation", operation.id], {
-          type: "updated",
-          operation,
-        })
-      } catch (error) {
-        this.logger.error(
-          { projectId, operationId: operation.id, error },
-          `failed to publish operation update for operation with ID "%s" in project "%s"`,
-          operation.id,
-          projectId,
-        )
-      }
-
-      this.logger.info(
-        { projectId, operationId: operation.id },
-        `updated operation with ID "%s" in project "%s"`,
-        operation.id,
-        projectId,
-      )
-    } catch (error) {
-      this.logger.error(
-        { projectId, operationId: operation.id, error },
-        `failed to update operation with ID "%s" in project "%s"`,
-        operation.id,
-        projectId,
-      )
-
-      throw new Error(`Failed to update operation "${operation.id}" in project "${projectId}"`, {
-        cause: error,
-      })
-    }
+    const operation = await database.operation.findUnique({
+      where: { id: operationId },
+    })
+
+    return operation ?? undefined
+  }
+
+  /**
+   * Gets all operations for a project.
+   *
+   * @param projectId The project ID.
+   * @param limit Optional limit on number of operations to return.
+   * @returns Array of operations.
+   */
+  async getOperations(projectId: string, limit?: number): Promise<Operation[]> {
+    const database = await this.database.forProject(projectId)
+
+    return await database.operation.findMany({
+      orderBy: { startedAt: "desc" },
+      take: limit,
+    })
   }
 
   /**
-   * Appends logs for a specific operation.
-   * Creates all necessary indexes to retrieve the logs by instance ID.
+   * Appends log for a specific operation.
    *
-   * @param projectId The ID of the project to persist the logs for.
-   * @param operationId The ID of the operation to persist the logs for.
-   * @param instanceIds The IDs of the instances to persist the logs for. Should include the whole hierarchy of instances produced the logs.
-   * @param logs The logs entries to persist.
+   * @param projectId The ID of the project to persist the log for.
+   * @param operationId The ID of the operation to persist the log for.
+   * @param stateId The ID of the instance state that produced the log (optional).
+   * @param content The log content to append.
    */
-  async appendLogs(
+  async appendLog(
     projectId: string,
     operationId: string,
-    instanceIds: string[],
-    logs: string[],
+    stateId: string | null,
+    content: string,
   ): Promise<void> {
-    const records: [string, string][] = logs.map(log => [uuidv7(), log])
-
-    // 1. publish logs to the pubsub system, ignore errors
-    try {
-      const events = instanceIds.flatMap(instanceId =>
-        records.map(
-          record => [[`operation-instance-log`, operationId, instanceId], record] as const,
-        ),
-      )
-
-      await this.pubsubManager.publishMany(events)
-    } catch (error) {
-      this.logger.error(
-        { projectId, operationId, instanceIds, error },
-        `failed to publish logs for operation "%s" in project "%s"`,
-        operationId,
-        projectId,
-      )
-    }
+    const database = await this.database.forProject(projectId)
 
-    // 2. persist logs to the hot state and state repository, ignore errors
-    try {
-      // insert logs into the hotstate
-      await this.hotStateManager.hmset(["operation-logs", projectId], records)
-    } catch (error) {
-      this.logger.error(
-        { projectId, operationId, instanceIds, error },
-        `failed to append logs for operation "%s" in project "%s"`,
-        operationId,
-        projectId,
-      )
+    // verify operation exists
+    const operation = await database.operation.findUnique({
+      where: { id: operationId },
+      select: { id: true, finishedAt: true },
+    })
+
+    if (!operation) {
+      throw new OperationNotFoundError(projectId, operationId)
     }
 
-    // 3. persist logs to the state repository, fail on error
-    try {
-      await using batch = this.stateManager.batch()
-
-      // insert logs into the operation log repository
-      await this.stateManager
-        .getOperationLogRepository(projectId, operationId)
-        .putMany(records, batch)
-
-      for (const instanceId of instanceIds) {
-        // create an index for the instance logs
-        await this.stateManager
-          .getInstanceLogIndexRepository(projectId, operationId, instanceId)
-          .indexRepository.putMany(
-            records.map(([id]) => [id, SAME_KEY]),
-            batch,
-          )
-      }
-
-      await batch.write()
-    } catch (error) {
-      this.logger.error(
-        { projectId, operationId, instanceIds, error },
-        `failed to persist instance logs for operation "%s" in project "%s"`,
+    // store logs in database
+    const entry = await database.operationLog.create({
+      data: {
+        id: ulid(),
         operationId,
-        projectId,
-      )
-
-      throw new Error(
-        `Failed to persist logs for operation "${operationId}" in project "${projectId}"`,
-        { cause: error },
-      )
+        stateId,
+        content,
+      },
+    })
+
+    // publish logs via pubsub - only for logs with stateId (not system logs)
+    if (stateId) {
+      await this.pubsubManager.publish(["operation-instance-log", operationId, stateId], entry)
     }
   }
 
   /**
-   * Retrieves logs for a specific operation and instance.
-   * First tries to get logs from the hot state, then falls back to the state repository.
+   * Retrieves logs for a specific operation and optionally an instance.
    *
    * @param projectId The ID of the project to retrieve logs for.
    * @param operationId The ID of the operation to retrieve logs for.
-   * @param instanceId The ID of the instance to retrieve logs for.
-   * @returns An array of log entries as tuples of [logId, logContent].
+   * @param stateId Optional instance state ID to filter logs.
+   * @returns Array of log entries.
    */
-  async getInstanceLogs(
+  async getOperationLogs(
     projectId: string,
     operationId: string,
-    instanceId: string,
-  ): Promise<[string, string][]> {
-    // 1. first try to get logs from the hot state
-    try {
-      const logs = await this.hotStateManager.hgetall(["operation-logs", projectId])
-
-      if (logs.length > 0) {
-        // if there are logs, assume they are up-to-date
-        return logs
-      }
-
-      // otherwise, check if the operation is active to determine if logs are expected
-      const hasActiveOperation = await this.hotStateManager.hexists(
-        ["active-operations", projectId],
-        operationId,
-      )
-
-      if (hasActiveOperation) {
-        // if the operation is active, there is just no logs yet
-        return []
-      }
-    } catch (error) {
-      this.logger.error(
-        { projectId, operationId, instanceId, error },
-        `failed to get logs for operation "%s" in project "%s"`,
-        operationId,
-        projectId,
-      )
-    }
+    stateId?: string,
+  ): Promise<Array<{ id: string; stateId: string | null; content: string }>> {
+    const database = await this.database.forProject(projectId)
 
-    // 2. if not found or failed, try to get logs from the state repository
-    try {
-      // if no logs are found in the hot state, load them from the state repository
-      return await this.stateManager
-        .getInstanceLogIndexRepository(projectId, operationId, instanceId)
-        .getAll()
-    } catch (error) {
-      this.logger.error(
-        { projectId, operationId, instanceId, error },
-        `failed to get logs for operation "%s" in project "%s"`,
+    const logs = await database.operationLog.findMany({
+      where: {
         operationId,
-        projectId,
-      )
+        ...(stateId ? { stateId } : {}),
+      },
+      orderBy: { id: "asc" },
+      select: {
+        id: true,
+        stateId: true,
+        content: true,
+      },
+    })
+
+    return logs
+  }
 
-      return []
-    }
+  /**
+   * Marks an operation as finished with given status.
+   *
+   * @param projectId The project ID containing the operation.
+   * @param operationId The operation ID to complete.
+   * @returns The updated operation.
+   */
+  async markOperationFinished(
+    projectId: string,
+    operationId: string,
+    status: OperationStatus,
+  ): Promise<Operation> {
+    const result = await this.updateOperation(projectId, operationId, {
+      status,
+      finishedAt: new Date(),
+    })
+
+    this.logger.info(
+      { projectId, operationId, status },
+      `marked operation as finished with status "${status}"`,
+    )
+
+    return result
   }
 }

package/src/business/project-model.ts

@@ -0,0 +1,258 @@
+import type { InstanceModel } from "@highstate/contract"
+import type { Logger } from "pino"
+import type { DatabaseManager } from "../database"
+import type { LibraryBackend } from "../library"
+import type { ProjectModelBackend } from "../project-model"
+import type { InstanceStateService } from "./instance-state"
+import type { ProjectUnlockService } from "./project-unlock"
+import { isNonNullish } from "remeda"
+import {
+  type FullProjectModel,
+  forSchema,
+  InputResolver,
+  type InputResolverNode,
+  ProjectNotFoundError,
+  type ProjectOutput,
+  projectOutputSchema,
+  type ResolvedInstanceInput,
+} from "../shared"
+
+export type GetProjectModelOptions = {
+  /**
+   * Whether to include virtual instances in the model.
+   *
+   * By default, virtual instances are not included.
+   */
+  includeVirtualInstances?: boolean
+
+  /**
+   * Whether to include ghost instances in the model.
+   *
+   * By default, ghost instances are not included.
+   */
+  includeGhostInstances?: boolean
+}
+
+export class ProjectModelService {
+  constructor(
+    private readonly database: DatabaseManager,
+    private readonly libraryBackend: LibraryBackend,
+    private readonly instanceStateService: InstanceStateService,
+    private readonly projectModelBackends: Record<string, ProjectModelBackend>,
+    private readonly projectUnlockService: ProjectUnlockService,
+    private readonly logger: Logger,
+  ) {
+    this.projectUnlockService.registerUnlockTask(
+      //
+      "sync-instance-states",
+      projectId => this.syncInstanceStates(projectId),
+    )
+  }
+
+  /**
+   * Get the project model containing instances, hubs + virtual instances and ghost instances if requested.
+   *
+   * @param projectId The ID of the project to get the model for.
+   * @param options Options to control the model retrieval.
+   */
+  async getProjectModel(
+    projectId: string,
+    { includeVirtualInstances = false, includeGhostInstances = false }: GetProjectModelOptions = {},
+  ): Promise<[projectModel: FullProjectModel, project: ProjectOutput]> {
+    const { project, backend, spec } = await this.getProjectWithBackend(projectId)
+
+    // get base model from storage backend
+    const { instances, hubs } = await backend.getProjectModel(project, spec)
+
+    return [
+      {
+        instances,
+        hubs,
+        virtualInstances: includeVirtualInstances ? await this.getVirtualInstances(projectId) : [],
+        ghostInstances: includeGhostInstances
+          ? await this.getGhostInstances(
+              projectId,
+              instances.map(instance => instance.id),
+            )
+          : [],
+      },
+      project,
+    ]
+  }
+
+  /**
+   * Resolve a project by loading all dependencies and processing input resolution.
+   * Does not load virtual instances.
+   *
+   * @param projectId The ID of the project to resolve.
+   */
+  async resolveProject(projectId: string) {
+    const [[{ instances, hubs }, project], states] = await Promise.all([
+      this.getProjectModel(projectId),
+      this.instanceStateService.getInstanceStates(projectId, { includeEvaluationState: true }),
+    ])
+
+    const library = await this.libraryBackend.loadLibrary(project.libraryId)
+
+    const filteredInstances = instances.filter(instance => instance.type in library.components)
+    const stateMap = new Map(states.map(state => [state.id, state]))
+
+    const inputResolverNodes = new Map<string, InputResolverNode>()
+
+    for (const instance of filteredInstances) {
+      inputResolverNodes.set(`instance:${instance.id}`, {
+        kind: "instance",
+        instance,
+        component: library.components[instance.type],
+      })
+    }
+
+    for (const hub of hubs) {
+      inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
+    }
+
+    const inputResolver = new InputResolver(inputResolverNodes, this.logger)
+    inputResolver.addAllNodesToWorkset()
+
+    const resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>> = {}
+
+    await inputResolver.process()
+
+    for (const instance of filteredInstances) {
+      const output = inputResolver.requireOutput(`instance:${instance.id}`)
+      if (output.kind !== "instance") {
+        throw new Error("Expected instance node")
+      }
+
+      resolvedInputs[instance.id] = output.resolvedInputs
+    }
+
+    return {
+      project,
+      library,
+      instances: filteredInstances,
+      stateMap,
+      resolvedInputs,
+    }
+  }
+
+  /**
+  /**
+   * Get the appropriate project model backend for the given project.
+   *
+   * @param type The project storage type.
+   * @returns The project model backend.
+   */
+  private getProjectModelBackend(type: string): ProjectModelBackend {
+    const backend = this.projectModelBackends[type]
+    if (!backend) {
+      throw new Error(`Project model backend not found for type: ${type}`)
+    }
+
+    return backend
+  }
+
+  /**
+   * Get project with model backend and spec.
+   *
+   * @param projectId The project ID to get.
+   * @returns The project, backend, and spec.
+   */
+  private async getProjectWithBackend(projectId: string) {
+    const project = await this.database.backend.project.findUnique({
+      where: { id: projectId },
+      select: {
+        ...forSchema(projectOutputSchema),
+        modelStorage: {
+          select: {
+            spec: true,
+          },
+        },
+      },
+    })
+
+    if (!project) {
+      throw new ProjectNotFoundError(projectId)
+    }
+
+    const backend = this.getProjectModelBackend(project.modelStorage.spec.type)
+
+    return {
+      project,
+      backend,
+      spec: project.modelStorage.spec,
+    }
+  }
+
+  /**
+   * Get virtual instances from the database for the given project ID.
+   *
+   * @param projectId The project ID to get virtual instances for.
+   * @returns The list of virtual instances.
+   */
+  private async getVirtualInstances(projectId: string): Promise<InstanceModel[]> {
+    const database = await this.database.forProject(projectId)
+    const states = await database.instanceEvaluationState.findMany({ select: { model: true } })
+
+    return states.map(state => state.model).filter(isNonNullish)
+  }
+
+  /**
+   * Get ghost instances from the database for the given project ID.
+   *
+   * @param projectId The project ID to get ghost instances for.
+   * @param residentInstanceIds The IDs of the instances present in the model.
+   * @return The list of ghost instances.
+   */
+  private async getGhostInstances(
+    projectId: string,
+    residentInstanceIds: string[],
+  ): Promise<InstanceModel[]> {
+    const database = await this.database.forProject(projectId)
+
+    const states = await database.instanceState.findMany({
+      where: {
+        // undeployed instances cannot be considered ghost
+        status: { not: "undeployed" },
+
+        OR: [
+          // the resident instance is ghost if it is not in the model
+          { source: "resident", instanceId: { notIn: residentInstanceIds } },
+
+          // the virtual instance is ghost if it is has no evaluation state
+          { source: "virtual", evaluationState: null },
+        ],
+      },
+      select: { model: true },
+    })
+
+    return states.map(state => state.model).filter(isNonNullish)
+  }
+
+  private async syncInstanceStates(projectId: string): Promise<void> {
+    const database = await this.database.forProject(projectId)
+
+    await database.$transaction(async tx => {
+      const [{ instances }] = await this.getProjectModel(projectId)
+
+      const existingStates = await tx.instanceState.findMany({ select: { instanceId: true } })
+      const existingStateIds = new Set(existingStates.map(state => state.instanceId))
+      const missingInstances = instances.filter(instance => !existingStateIds.has(instance.id))
+
+      if (missingInstances.length === 0) {
+        return
+      }
+
+      await tx.instanceState.createMany({
+        data: missingInstances.map(instance => ({
+          instanceId: instance.id,
+          kind: instance.kind,
+          source: "resident",
+          status: "undeployed",
+        })),
+      })
+
+      this.logger.info({ projectId }, "created missing %s instance states", missingInstances.length)
+    })
+  }
+}