@highstate/backend 0.9.18 → 0.9.20

package/src/library/worker/protocol.ts

@@ -7,5 +7,4 @@ export type WorkerData = {
 
   allInstances: InstanceModel[]
   resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>
-  instanceIds: string[]
 }

package/src/lock/index.ts

@@ -1,4 +1,3 @@
 export * from "./abstractions"
 export * from "./factory"
 export * from "./manager"
-export * from "./test"

package/src/lock/manager.ts

@@ -2,11 +2,6 @@ import type { LockBackend } from "./abstractions"
 import { join } from "remeda"
 
 export type LockKeyMap = {
-  /**
-   * Lock for the project names to ensure uniqueness.
-   */
-  "project-name": [name: string]
-
   /**
    * Lock for instances and hubs of the project.
    */
@@ -32,11 +27,6 @@ export type LockKeyMap = {
    */
   "instance-lock": [projectId: string, instanceId: string]
 
-  /**
-   * Lock for a specific instance state within a project.
-   */
-  "instance-state": [projectId: string, instanceId: string]
-
   /**
    * Lock for a specific artifact within a project.
    */

package/src/orchestrator/manager.ts

@@ -1,87 +1,105 @@
-import type { LibraryBackend } from "../library"
-import type { ProjectBackend } from "../project"
-import type { RunnerBackend } from "../runner"
-import type { StateManager } from "../state"
-import type { ArtifactService } from "../business/artifact"
+import type { InstanceId } from "@highstate/contract"
 import type { Logger } from "pino"
 import type {
   InstanceLockService,
+  InstanceStateService,
   OperationService,
-  SecretService,
+  ProjectModelService,
   ProjectUnlockService,
-  InstanceStateService,
-  WorkerService,
+  SecretService,
+  UnitExtraService,
 } from "../business"
-import type { PubSubManager } from "../pubsub"
-import { v7 as uuidv7 } from "uuid"
+import type { ArtifactService } from "../business/artifact"
+import type { DatabaseManager, Operation, Project } from "../database"
+import type { LibraryBackend } from "../library"
+import type { RunnerBackend } from "../runner"
 import {
-  isFinalOperationStatus,
-  type Operation,
-  type OperationRequest,
-  type Project,
+  finalOperationStatuses,
+  type OperationInput,
+  type OperationPhase,
+  operationOptionsSchema,
 } from "../shared"
+import { finalInstanceOperationStatuses } from "../shared/models/project/state"
 import { RuntimeOperation } from "./operation"
+import { OperationContext } from "./operation-context"
+import { createOperationPlan } from "./operation-plan"
 
 export class OperationManager {
   constructor(
     private readonly runnerBackend: RunnerBackend,
     private readonly libraryBackend: LibraryBackend,
-    private readonly projectBackend: ProjectBackend,
-    private readonly artifactManager: ArtifactService,
-    private readonly stateManager: StateManager,
+    private readonly artifactService: ArtifactService,
     private readonly instanceLockService: InstanceLockService,
-    private readonly stateUnlockService: ProjectUnlockService,
+    private readonly projectUnlockService: ProjectUnlockService,
     private readonly operationService: OperationService,
     private readonly secretService: SecretService,
     private readonly instanceStateService: InstanceStateService,
-    private readonly pubsubManager: PubSubManager,
-    private readonly workerService: WorkerService,
+    private readonly projectModelService: ProjectModelService,
+    private readonly unitExtraService: UnitExtraService,
+    private readonly database: DatabaseManager,
     private readonly logger: Logger,
   ) {
-    this.stateUnlockService.registerUnlockTask(
+    this.projectUnlockService.registerUnlockTask(
       //
-      "process-lost-operations",
-      projectId => this.processLostOperations(projectId),
+      "recover-system-state",
+      projectId => this.recoverSystemState(projectId),
     )
   }
 
   private readonly runtimeOperations = new Map<string, RuntimeOperation>()
 
+  /**
+   * Plans the project operation without executing it.
+   *
+   * @param request The operation request to plan.
+   * @returns The planned phases for the operation.
+   */
+  async plan(request: OperationInput): Promise<OperationPhase[]> {
+    this.logger.info({ request }, "planning operation")
+
+    const project = await this.database.backend.project.findUnique({
+      where: { id: request.projectId },
+    })
+
+    if (!project) {
+      throw new Error(`Project with ID "${request.projectId}" not found`)
+    }
+
+    const context = await OperationContext.load(
+      request.projectId,
+      this.libraryBackend,
+      this.instanceStateService,
+      this.projectModelService,
+      this.logger,
+    )
+
+    const options = operationOptionsSchema.parse(request.options ?? {})
+
+    return createOperationPlan(context, request.type, request.instanceIds, options)
+  }
+
   /**
    * Launches the project operation.
    *
    * @param request The operation request to launch.
    */
-  async launch(request: OperationRequest): Promise<Operation> {
-    const operation: Operation = {
-      id: uuidv7(),
-      meta: {},
-      type: request.type,
-      requestedInstanceIds: request.instanceIds,
-      instanceIdsToUpdate: [],
-      instanceIdsToDestroy: [],
-      status: "pending",
-      options: {
-        forceUpdateDependencies: request.options?.forceUpdateDependencies ?? false,
-        forceUpdateChildren: request.options?.forceUpdateChildren ?? false,
-        destroyDependentInstances: request.options?.destroyDependentInstances ?? true,
-        invokeDestroyTriggers: request.options?.invokeDestroyTriggers ?? false,
-        deleteUnreachableResources: request.options?.deleteUnreachableResources ?? false,
-        forceDeleteState: request.options?.forceDeleteState ?? false,
-        refresh: request.options?.refresh ?? false,
-        allowPartialCompositeInstanceUpdates:
-          request.options?.allowPartialCompositeInstanceUpdates ?? false,
-      },
-      error: null,
-      startedAt: Date.now(),
-      completedAt: null,
-    }
+  async launch(request: OperationInput): Promise<Operation> {
+    const options = operationOptionsSchema.parse(request.options ?? {})
+
+    const operation = await this.operationService.createOperation(
+      request.projectId,
+      {},
+      request.type,
+      request.instanceIds,
+      options,
+    )
 
     this.logger.info({ operation }, "launching operation")
 
-    await this.operationService.updateOperation(request.projectId, operation)
+    const project = await this.database.backend.project.findUnique({
+      where: { id: request.projectId },
+    })
 
-    const project = await this.stateManager.getProjectRepository().get(request.projectId)
     if (!project) {
       throw new Error(`Project with ID "${request.projectId}" not found`)
     }
@@ -102,7 +120,7 @@ export class OperationManager {
     }
   }
 
-  cancelInstance(operationId: string, instanceId: string): void {
+  cancelInstance(operationId: string, instanceId: InstanceId): void {
     const runtimeOperation = this.runtimeOperations.get(operationId)
     if (runtimeOperation) {
       runtimeOperation.cancelInstance(instanceId)
@@ -115,15 +133,13 @@ export class OperationManager {
       operation,
       this.runnerBackend,
       this.libraryBackend,
-      this.projectBackend,
-      this.artifactManager,
-      this.stateManager,
+      this.artifactService,
       this.instanceLockService,
       this.operationService,
       this.secretService,
       this.instanceStateService,
-      this.pubsubManager,
-      this.workerService,
+      this.projectModelService,
+      this.unitExtraService,
       this.logger.child({ operationId: operation.id }),
     )
 
@@ -131,68 +147,138 @@ export class OperationManager {
     void runtimeOperation.operateSafe().finally(() => this.runtimeOperations.delete(operation.id))
   }
 
-  private async processLostOperations(projectId: string): Promise<void> {
-    const activeOperations = await this.stateManager
-      .getActiveOperationIndexRepository(projectId)
-      .getAllItems()
+  private async recoverSystemState(projectId: string): Promise<void> {
+    const database = await this.database.forProject(projectId)
+
+    await database.$transaction(async tx => {
+      // 1. process lost operations
+      const activeOperations = await tx.operation.findMany({
+        where: {
+          status: {
+            not: {
+              in: finalOperationStatuses,
+            },
+          },
+        },
+      })
+
+      if (activeOperations.length > 0) {
+        // mark all lost operations as failed
+        for (const operation of activeOperations) {
+          await tx.operation.update({
+            where: { id: operation.id },
+            data: {
+              status: "failed",
+              finishedAt: new Date(),
+            },
+          })
+
+          void this.operationService.appendLog(
+            projectId,
+            operation.id,
+            null,
+            "Operation was interrupted",
+          )
+        }
 
-    for (const operation of activeOperations) {
-      if (isFinalOperationStatus(operation.status)) {
         this.logger.warn(
-          { projectId, operationId: operation.id },
-          "operation is in final state but still marked as active, removing from index",
+          { projectId, operationCount: activeOperations.length },
+          "marked %s lost operations as failed",
+          activeOperations.length,
         )
-
-        await this.stateManager
-          .getActiveOperationIndexRepository(projectId)
-          .indexRepository.delete(operation.id)
-        continue
       }
 
-      const errorMessagePrefix = operation.error ? `${operation.error}\n\n` : ""
-
-      operation.status = "failed"
-      operation.error = `${errorMessagePrefix}Operation was unexpectedly interrupted. Please restart it.`
+      // 2. cleanup orphaned locks
+      const existingLocks = await tx.instanceLock.findMany({
+        select: { stateId: true },
+      })
 
-      await this.operationService.updateOperation(projectId, operation)
-    }
+      if (existingLocks.length > 0) {
+        const stateIds = existingLocks.map(lock => lock.stateId)
+        await tx.instanceLock.deleteMany({
+          where: {
+            stateId: {
+              in: stateIds,
+            },
+          },
+        })
 
-    // unlock instances that were locked by lost operations
-    const activeOperationIds = new Set(activeOperations.map(op => op.id))
+        for (const lock of existingLocks) {
+          this.logger.warn({ projectId, stateId: lock.stateId }, "removed orphaned lock")
+        }
 
-    const allLocks = await this.stateManager.getInstanceLockRepository(projectId).getAllItems()
-    const lockIdsToRemvoe: string[] = []
-
-    // clean up unexpected operation locks
-    for (const lock of allLocks) {
-      if (lock.spec.type !== "operation") {
-        continue
+        this.logger.info(
+          { projectId, lockCount: existingLocks.length },
+          "cleaned up %s locks",
+          existingLocks.length,
+        )
       }
 
-      if (activeOperationIds.has(lock.spec.operationId)) {
-        // remove locks for lost operation which is expected
-        lockIdsToRemvoe.push(lock.id)
-        continue
-      }
+      // 3. reset instances with transient status
+      const instancesToReset = await tx.instanceOperationState.findMany({
+        where: {
+          status: {
+            not: {
+              in: finalInstanceOperationStatuses,
+            },
+          },
+        },
+        select: {
+          stateId: true,
+          status: true,
+        },
+      })
 
-      // unexpected operation lock found, also remove it
-      this.logger.warn(
-        { projectId, lockId: lock.id },
-        `unexpected operation lock found for completed operation "${lock.spec.operationId}", removing lock`,
-      )
+      if (instancesToReset.length > 0) {
+        // log each instance being reset
+        for (const instance of instancesToReset) {
+          this.logger.warn(
+            { projectId, stateId: instance.stateId, previousStatus: instance.status },
+            "resetting instance with transient status from %s to failed",
+            instance.status,
+          )
+        }
 
-      lockIdsToRemvoe.push(lock.id)
-    }
+        // reset operation states that were being processed by lost operations
+        const result = await tx.instanceOperationState.updateMany({
+          where: {
+            status: {
+              not: {
+                in: finalInstanceOperationStatuses,
+              },
+            },
+          },
+          data: {
+            status: "failed",
+            finishedAt: new Date(),
+          },
+        })
 
-    if (lockIdsToRemvoe.length > 0) {
-      await this.instanceLockService.unlockInstancesUnconditionally(projectId, lockIdsToRemvoe)
-    }
+        this.logger.info(
+          { projectId, resetOperationCount: result.count },
+          "reset %s operation states",
+          result.count,
+        )
+      }
 
-    this.logger.debug(
-      { projectId, operationCount: activeOperations.length },
-      `processed %s lost operations for project "%s"`,
-      activeOperations.length,
-      projectId,
-    )
+      // Step 4: reset attempted instance states
+      const attemptedResult = await tx.instanceState.updateMany({
+        where: {
+          status: "attempted",
+        },
+        data: {
+          status: "failed",
+          message: "Operation was interrupted",
+        },
+      })
+
+      if (attemptedResult.count > 0) {
+        this.logger.info(
+          { projectId, resetAttemptedCount: attemptedResult.count },
+          "reset %s attempted instance states",
+          attemptedResult.count,
+        )
+      }
+    })
   }
 }