@highstate/backend 0.9.15 → 0.9.16

package/src/lock/manager.ts

@@ -0,0 +1,82 @@
+import type { LockBackend } from "./abstractions"
+import { join } from "remeda"
+
+export type LockKeyMap = {
+  /**
+   * Lock for a specific project.
+   */
+  project: [projectId: string]
+
+  /**
+   * Lock for a specific instance within a project.
+   */
+  instance: [projectId: string, instanceId: string]
+
+  /**
+   * Lock for a specific instance lock within a project.
+   */
+  "instance-lock": [projectId: string, instanceId: string]
+
+  /**
+   * Lock for a specific instance state within a project.
+   */
+  "instance-state": [projectId: string, instanceId: string]
+
+  /**
+   * Lock for a specific artifact within a project.
+   */
+  artifact: [projectId: string, artifactId: string]
+
+  /**
+   * Lock for a specific artifact hash within a project.
+   */
+  "artifact-hash": [projectId: string, hash: string]
+
+  /**
+   * Lock for a specific operation within a project.
+   */
+  operation: [projectId: string, operationId: string]
+
+  /**
+   * Lock for a specific api key within a project.
+   */
+  "api-key": [projectId: string, key: string]
+
+  /**
+   * Lock for a specific worker within a project.
+   */
+  worker: [workerId: string]
+
+  /**
+   * Lock for a specific worker image within a project.
+   */
+  "worker-image": [projectId: string, image: string]
+}
+
+export type LockKey = Readonly<
+  {
+    [K in keyof LockKeyMap]: [type: K, ...LockKeyMap[K]]
+  }[keyof LockKeyMap]
+>
+
+export class LockManager {
+  constructor(private readonly lockBackend: LockBackend) {}
+
+  /**
+   * Acquires locks for the given keys and executes the function.
+   *
+   * This method provides a clean interface for acquiring distributed locks across
+   * multiple keys and ensures they are properly released after execution.
+   *
+   * @param keys The keys to acquire locks for.
+   * @param fn The function to execute while holding the locks.
+   * @returns The result of the executed function.
+   */
+  public async acquire<T>(keys: LockKey | LockKey[], fn: () => Promise<T> | T): Promise<T> {
+    if (typeof keys[0] === "string") {
+      return this.lockBackend.acquire([keys.join(":")], fn)
+    }
+
+    return this.lockBackend.acquire(keys.map(join(":")), fn)
+  }
+}

package/src/lock/memory.ts

@@ -0,0 +1,19 @@
+import type { LockBackend } from "./abstractions"
+import type { BetterLock as BetterLockType } from "better-lock/dist/better_lock"
+import { BetterLock } from "better-lock"
+
+export class MemoryLockBackend implements LockBackend {
+  private readonly lock: BetterLockType
+
+  private constructor() {
+    this.lock = new BetterLock()
+  }
+
+  async acquire<T>(keys: string[], fn: () => Promise<T> | T): Promise<T> {
+    return this.lock.acquire(keys, fn)
+  }
+
+  static create(): LockBackend {
+    return new MemoryLockBackend()
+  }
+}

package/src/orchestrator/manager.ts

@@ -1,76 +1,54 @@
 import type { LibraryBackend } from "../library"
-import type { ProjectBackend, ProjectLockManager } from "../project"
+import type { ProjectBackend } from "../project"
 import type { RunnerBackend } from "../runner"
-import type { SecretBackend } from "../secret"
-import type { StateBackend, StateManager } from "../state"
+import type { StateManager } from "../state"
+import type { ArtifactService } from "../business/artifact"
 import type { Logger } from "pino"
-import { EventEmitter, on } from "node:events"
-import { uuidv7 } from "uuidv7"
-import { type ProjectOperation, type ProjectOperationRequest } from "../shared"
+import type {
+  InstanceLockService,
+  OperationService,
+  SecretService,
+  ProjectUnlockService,
+  InstanceStateService,
+} from "../business"
+import type { PubSubManager } from "../pubsub"
+import { v7 as uuidv7 } from "uuid"
+import { isFinalOperationStatus, type Operation, type OperationRequest } from "../shared"
 import { RuntimeOperation } from "./operation"
 
-export type OperationEvents = Record<string, [ProjectOperation]>
-export type InstanceLogsEvents = Record<string, [string]>
-
 export class OperationManager {
   constructor(
     private readonly runnerBackend: RunnerBackend,
-    private readonly stateBackend: StateBackend,
     private readonly libraryBackend: LibraryBackend,
     private readonly projectBackend: ProjectBackend,
-    private readonly secretBackend: SecretBackend,
-    private readonly projectLockManager: ProjectLockManager,
+    private readonly artifactManager: ArtifactService,
     private readonly stateManager: StateManager,
+    private readonly instanceLockService: InstanceLockService,
+    private readonly stateUnlockService: ProjectUnlockService,
+    private readonly operationService: OperationService,
+    private readonly secretService: SecretService,
+    private readonly instanceStateService: InstanceStateService,
+    private readonly pubsubManager: PubSubManager,
     private readonly logger: Logger,
-  ) {}
-
-  private readonly operationEE = new EventEmitter<OperationEvents>()
-  private readonly instanceLogsEE = new EventEmitter<InstanceLogsEvents>()
-
-  private readonly runtimeOperations = new Map<string, RuntimeOperation>()
-
-  /**
-   * Watches for all project operations in the project.
-   *
-   * @param projectId The project ID to watch.
-   * @param signal The signal to abort the operation.
-   */
-  public async *watchOperations(
-    projectId: string,
-    signal?: AbortSignal,
-  ): AsyncIterable<ProjectOperation> {
-    for await (const [operation] of on(this.operationEE, projectId, { signal })) {
-      yield operation
-    }
+  ) {
+    this.stateUnlockService.registerUnlockTask(
+      //
+      "process-lost-operations",
+      projectId => this.processLostOperations(projectId),
+    )
   }
 
-  /**
-   * Watches for logs of the instance in the operation.
-   *
-   * @param operationId The operation ID to watch.
-   * @param instanceId The instance ID to watch.
-   * @param signal The signal to abort the operation.
-   */
-  async *watchInstanceLogs(
-    operationId: string,
-    instanceId: string,
-    signal?: AbortSignal,
-  ): AsyncIterable<string> {
-    const eventKey = `${operationId}/${instanceId}`
-    for await (const [log] of on(this.instanceLogsEE, eventKey, { signal })) {
-      yield log
-    }
-  }
+  private readonly runtimeOperations = new Map<string, RuntimeOperation>()
 
   /**
    * Launches the project operation.
    *
    * @param request The operation request to launch.
    */
-  async launch(request: ProjectOperationRequest): Promise<ProjectOperation> {
-    const operation: ProjectOperation = {
+  async launch(request: OperationRequest): Promise<Operation> {
+    const operation: Operation = {
       id: uuidv7(),
-      projectId: request.projectId,
+      meta: {},
       type: request.type,
       requestedInstanceIds: request.instanceIds,
       instanceIdsToUpdate: [],
@@ -94,9 +72,9 @@ export class OperationManager {
 
     this.logger.info({ operation }, "launching operation")
 
-    await this.stateBackend.putOperation(operation)
+    await this.operationService.updateOperation(request.projectId, operation)
 
-    this.startOperation(operation)
+    this.startOperation(request.projectId, operation)
 
     return operation
   }
@@ -112,59 +90,128 @@ export class OperationManager {
     }
   }
 
-  private startOperation(operation: ProjectOperation): void {
+  cancelInstance(operationId: string, instanceId: string): void {
+    const runtimeOperation = this.runtimeOperations.get(operationId)
+    if (runtimeOperation) {
+      runtimeOperation.cancelInstance(instanceId)
+    }
+  }
+
+  private startOperation(projectId: string, operation: Operation): void {
     const runtimeOperation = new RuntimeOperation(
+      projectId,
       operation,
       this.runnerBackend,
-      this.stateBackend,
       this.libraryBackend,
       this.projectBackend,
-      this.secretBackend,
-      this.projectLockManager.getLock(operation.projectId),
+      this.artifactManager,
       this.stateManager,
-      this.operationEE,
-      this.instanceLogsEE,
-      this.logger.child({ service: "RuntimeOperation", operationId: operation.id }),
+      this.instanceLockService,
+      this.operationService,
+      this.secretService,
+      this.instanceStateService,
+      this.pubsubManager,
+      this.logger.child({ operationId: operation.id }),
     )
 
     this.runtimeOperations.set(operation.id, runtimeOperation)
     void runtimeOperation.operateSafe().finally(() => this.runtimeOperations.delete(operation.id))
   }
 
-  static async create(
+  private async processLostOperations(projectId: string): Promise<void> {
+    const activeOperations = await this.stateManager
+      .getActiveOperationIndexRepository(projectId)
+      .getAllItems()
+
+    for (const operation of activeOperations) {
+      if (isFinalOperationStatus(operation.status)) {
+        this.logger.warn(
+          { projectId, operationId: operation.id },
+          "operation is in final state but still marked as active, removing from index",
+        )
+
+        await this.stateManager
+          .getActiveOperationIndexRepository(projectId)
+          .indexRepository.delete(operation.id)
+        continue
+      }
+
+      const errorMessagePrefix = operation.error ? `${operation.error}\n\n` : ""
+
+      operation.status = "failed"
+      operation.error = `${errorMessagePrefix}Operation was unexpectedly interrupted. Please restart it.`
+
+      await this.operationService.updateOperation(projectId, operation)
+    }
+
+    // unlock instances that were locked by lost operations
+    const activeOperationIds = new Set(activeOperations.map(op => op.id))
+
+    const allLocks = await this.stateManager.getInstanceLockRepository(projectId).getAllItems()
+    const lockIdsToRemvoe: string[] = []
+
+    // clean up unexpected operation locks
+    for (const lock of allLocks) {
+      if (lock.spec.type !== "operation") {
+        continue
+      }
+
+      if (activeOperationIds.has(lock.spec.operationId)) {
+        // remove locks for lost operation which is expected
+        lockIdsToRemvoe.push(lock.id)
+        continue
+      }
+
+      // unexpected operation lock found, also remove it
+      this.logger.warn(
+        { projectId, lockId: lock.id },
+        `unexpected operation lock found for completed operation "${lock.spec.operationId}", removing lock`,
+      )
+
+      lockIdsToRemvoe.push(lock.id)
+    }
+
+    if (lockIdsToRemvoe.length > 0) {
+      await this.instanceLockService.unlockInstancesUnconditionally(projectId, lockIdsToRemvoe)
+    }
+
+    this.logger.debug(
+      { projectId, operationCount: activeOperations.length },
+      `processed %s lost operations for project "%s"`,
+      activeOperations.length,
+      projectId,
+    )
+  }
+
+  static create(
     runnerBackend: RunnerBackend,
-    stateBackend: StateBackend,
     libraryBackend: LibraryBackend,
     projectBackend: ProjectBackend,
-    secretBackend: SecretBackend,
-    projectLockManager: ProjectLockManager,
+    artifactManager: ArtifactService,
     stateManager: StateManager,
+    instanceLockService: InstanceLockService,
+    stateUnlockService: ProjectUnlockService,
+    operationService: OperationService,
+    secretService: SecretService,
+    instanceService: InstanceStateService,
+    pubsubManager: PubSubManager,
     logger: Logger,
-  ) {
+  ): OperationManager {
     const operator = new OperationManager(
       runnerBackend,
-      stateBackend,
       libraryBackend,
       projectBackend,
-      secretBackend,
-      projectLockManager,
+      artifactManager,
       stateManager,
+      instanceLockService,
+      stateUnlockService,
+      operationService,
+      secretService,
+      instanceService,
+      pubsubManager,
       logger.child({ service: "OperationManager" }),
     )
 
-    // relaunch all active operations
-    const activeOperations = await stateBackend.getActiveOperations()
-
-    for (const operation of activeOperations) {
-      logger.info({ msg: "relaunching operation", operationId: operation.id })
-
-      // operator.startOperation(operation)
-
-      // cancel the operation for now
-      operation.status = "cancelled"
-      await stateBackend.putOperation(operation)
-    }
-
     return operator
   }
 }