@highstate/backend 0.9.15 → 0.9.18

package/src/library/worker/protocol.ts

@@ -1,38 +1,11 @@
 import type { InstanceModel } from "@highstate/contract"
-import type { InstanceEvaluationResult, ModuleEvaluationResult } from "../abstractions"
-import type { LibraryModel, ResolvedInstanceInput } from "../../shared"
+import type { ResolvedInstanceInput } from "../../shared"
 
 export type WorkerData = {
-  modulePaths: string[]
+  libraryModulePaths: string[]
   logLevel?: string
-}
-
-export type WorkerRequest =
-  | {
-      type: "evaluate-composite-instances"
-      allInstances: InstanceModel[]
-      resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>
-      instanceIds: string[]
-    }
-  | {
-      type: "evaluate-modules"
-      modulePaths: string[]
-    }
 
-export type WorkerResponse =
-  | {
-      type: "library"
-      library: LibraryModel
-    }
-  | {
-      type: "instance-evaluation-results"
-      results: InstanceEvaluationResult[]
-    }
-  | {
-      type: "module-evaluation-result"
-      result: ModuleEvaluationResult
-    }
-  | {
-      type: "error"
-      error: string
-    }
+  allInstances: InstanceModel[]
+  resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>
+  instanceIds: string[]
+}

package/src/lock/abstractions.ts

@@ -0,0 +1,6 @@
+export interface LockBackend {
+  /**
+   * Acquires locks for the given keys and executes the function.
+   */
+  acquire<T>(keys: string[], fn: () => Promise<T> | T): Promise<T>
+}

package/src/lock/factory.ts

@@ -0,0 +1,15 @@
+import type { LockBackend } from "./abstractions"
+import { z } from "zod"
+import { MemoryLockBackend } from "./memory"
+
+export const lockBackendConfig = z.object({
+  HIGHSTATE_LOCK_BACKEND_TYPE: z.enum(["memory"]).default("memory"),
+})
+
+export function createLockBackend(config: z.infer<typeof lockBackendConfig>): LockBackend {
+  switch (config.HIGHSTATE_LOCK_BACKEND_TYPE) {
+    case "memory": {
+      return MemoryLockBackend.create()
+    }
+  }
+}

package/src/lock/index.ts

@@ -0,0 +1,4 @@
+export * from "./abstractions"
+export * from "./factory"
+export * from "./manager"
+export * from "./test"

package/src/lock/manager.ts

@@ -0,0 +1,97 @@
+import type { LockBackend } from "./abstractions"
+import { join } from "remeda"
+
+export type LockKeyMap = {
+  /**
+   * Lock for the project names to ensure uniqueness.
+   */
+  "project-name": [name: string]
+
+  /**
+   * Lock for instances and hubs of the project.
+   */
+  "project-nodes": [projectId: string]
+
+  /**
+   * Locks for the evaluation of the project.
+   */
+  "project-evaluation": [projectId: string]
+
+  /**
+   * Lock for all instance states of the project.
+   */
+  "project-instance-states": [projectId: string]
+
+  /**
+   * Lock for a specific instance within a project.
+   */
+  instance: [projectId: string, instanceId: string]
+
+  /**
+   * Lock for a specific instance lock within a project.
+   */
+  "instance-lock": [projectId: string, instanceId: string]
+
+  /**
+   * Lock for a specific instance state within a project.
+   */
+  "instance-state": [projectId: string, instanceId: string]
+
+  /**
+   * Lock for a specific artifact within a project.
+   */
+  artifact: [projectId: string, artifactId: string]
+
+  /**
+   * Lock for a specific artifact hash within a project.
+   */
+  "artifact-hash": [projectId: string, hash: string]
+
+  /**
+   * Lock for a specific operation within a project.
+   */
+  operation: [projectId: string, operationId: string]
+
+  /**
+   * Lock for a specific api key within a project.
+   */
+  "api-key": [projectId: string, key: string]
+
+  /**
+   * Lock for a specific worker within a project.
+   */
+  worker: [workerId: string]
+
+  /**
+   * Lock for a specific worker image within a project.
+   */
+  "worker-image": [projectId: string, image: string]
+}
+
+export type LockKey = Readonly<
+  {
+    [K in keyof LockKeyMap]: [type: K, ...LockKeyMap[K]]
+  }[keyof LockKeyMap]
+>
+
+export class LockManager {
+  constructor(private readonly lockBackend: LockBackend) {}
+
+  /**
+   * Acquires locks for the given keys and executes the function.
+   *
+   * This method provides a clean interface for acquiring distributed locks across
+   * multiple keys and ensures they are properly released after execution.
+   *
+   * @param keys The keys to acquire locks for.
+   * @param fn The function to execute while holding the locks.
+   * @returns The result of the executed function.
+   */
+  public async acquire<T>(keys: LockKey | LockKey[], fn: () => Promise<T> | T): Promise<T> {
+    if (typeof keys[0] === "string") {
+      return this.lockBackend.acquire([keys.join(":")], fn)
+    }
+
+    return this.lockBackend.acquire(keys.map(join(":")), fn)
+  }
+}

package/src/lock/memory.ts

@@ -0,0 +1,19 @@
+import type { LockBackend } from "./abstractions"
+import type { BetterLock as BetterLockType } from "better-lock/dist/better_lock"
+import { BetterLock } from "better-lock"
+
+export class MemoryLockBackend implements LockBackend {
+  private readonly lock: BetterLockType
+
+  private constructor() {
+    this.lock = new BetterLock()
+  }
+
+  async acquire<T>(keys: string[], fn: () => Promise<T> | T): Promise<T> {
+    return this.lock.acquire(keys, fn)
+  }
+
+  static create(): LockBackend {
+    return new MemoryLockBackend()
+  }
+}

package/src/lock/test.ts

@@ -0,0 +1,108 @@
+import type { Fixtures } from "@vitest/runner"
+import type { LockKey } from "./manager"
+import { linkTraceEntry, renderTraceEntry, type TestTracer, type TraceEntry } from "../common"
+import { LockManager } from "./manager"
+import { MemoryLockBackend } from "./memory"
+
+class LockAcquireEntry implements TraceEntry {
+  releaseEntry?: TraceEntry
+
+  constructor(
+    readonly id: number,
+    private readonly keys: string[],
+  ) {}
+
+  render(): string {
+    return renderTraceEntry({
+      icon: "🔒",
+      title: this.releaseEntry
+        ? `locked, released at ${linkTraceEntry(this.releaseEntry)}`
+        : "locked",
+      fields: {
+        keys: {
+          value: this.keys,
+        },
+      },
+    })
+  }
+}
+
+class LockReleaseEntry implements TraceEntry {
+  constructor(
+    readonly id: number,
+    private readonly keys: string[],
+    private readonly acquireEntry: TraceEntry,
+  ) {}
+
+  render(): string {
+    return renderTraceEntry({
+      icon: "🔓",
+      title: `released, locked at ${linkTraceEntry(this.acquireEntry)}`,
+      fields: {
+        keys: {
+          value: this.keys,
+        },
+      },
+    })
+  }
+}
+
+class TestLockManager {
+  constructor(
+    private readonly lockManager: LockManager,
+    private readonly tracer: TestTracer,
+  ) {}
+
+  async acquire<T>(keys: LockKey | LockKey[], fn: () => Promise<T> | T): Promise<T> {
+    const keyStrings = this.formatKeys(keys)
+
+    // add acquire trace entry
+    const acquireEntry = new LockAcquireEntry(this.tracer.nextEntryId(), keyStrings)
+    this.tracer.addEntry(acquireEntry)
+
+    try {
+      const result = await this.lockManager.acquire(keys, fn)
+
+      // add release trace entry
+      const releaseEntry = new LockReleaseEntry(this.tracer.nextEntryId(), keyStrings, acquireEntry)
+      this.tracer.addEntry(releaseEntry)
+
+      // link acquire entry to release entry
+      acquireEntry.releaseEntry = releaseEntry
+
+      return result
+    } catch (error) {
+      // add release trace entry even on error
+      const releaseEntry = new LockReleaseEntry(this.tracer.nextEntryId(), keyStrings, acquireEntry)
+      this.tracer.addEntry(releaseEntry)
+
+      // link acquire entry to release entry
+      acquireEntry.releaseEntry = releaseEntry
+
+      throw error
+    }
+  }
+
+  private formatKeys(keys: LockKey | LockKey[]): string[] {
+    if (typeof keys[0] === "string") {
+      return [keys.join(":")]
+    }
+
+    return keys.map(key => key.join(":"))
+  }
+}
+
+export const lockFixtures: Fixtures<
+  {
+    lockManager: LockManager
+  },
+  { tracer: TestTracer }
+> = {
+  lockManager: async ({ tracer }, use) => {
+    const backend = MemoryLockBackend.create()
+    const baseLockManager = new LockManager(backend)
+    const testLockManager = new TestLockManager(baseLockManager, tracer) as unknown as LockManager
+
+    await use(testLockManager)
+  },
+}

package/src/orchestrator/manager.ts

@@ -1,76 +1,61 @@
 import type { LibraryBackend } from "../library"
-import type { ProjectBackend, ProjectLockManager } from "../project"
+import type { ProjectBackend } from "../project"
 import type { RunnerBackend } from "../runner"
-import type { SecretBackend } from "../secret"
-import type { StateBackend, StateManager } from "../state"
+import type { StateManager } from "../state"
+import type { ArtifactService } from "../business/artifact"
 import type { Logger } from "pino"
-import { EventEmitter, on } from "node:events"
-import { uuidv7 } from "uuidv7"
-import { type ProjectOperation, type ProjectOperationRequest } from "../shared"
+import type {
+  InstanceLockService,
+  OperationService,
+  SecretService,
+  ProjectUnlockService,
+  InstanceStateService,
+  WorkerService,
+} from "../business"
+import type { PubSubManager } from "../pubsub"
+import { v7 as uuidv7 } from "uuid"
+import {
+  isFinalOperationStatus,
+  type Operation,
+  type OperationRequest,
+  type Project,
+} from "../shared"
 import { RuntimeOperation } from "./operation"
 
-export type OperationEvents = Record<string, [ProjectOperation]>
-export type InstanceLogsEvents = Record<string, [string]>
-
 export class OperationManager {
   constructor(
     private readonly runnerBackend: RunnerBackend,
-    private readonly stateBackend: StateBackend,
     private readonly libraryBackend: LibraryBackend,
     private readonly projectBackend: ProjectBackend,
-    private readonly secretBackend: SecretBackend,
-    private readonly projectLockManager: ProjectLockManager,
+    private readonly artifactManager: ArtifactService,
     private readonly stateManager: StateManager,
+    private readonly instanceLockService: InstanceLockService,
+    private readonly stateUnlockService: ProjectUnlockService,
+    private readonly operationService: OperationService,
+    private readonly secretService: SecretService,
+    private readonly instanceStateService: InstanceStateService,
+    private readonly pubsubManager: PubSubManager,
+    private readonly workerService: WorkerService,
     private readonly logger: Logger,
-  ) {}
-
-  private readonly operationEE = new EventEmitter<OperationEvents>()
-  private readonly instanceLogsEE = new EventEmitter<InstanceLogsEvents>()
-
-  private readonly runtimeOperations = new Map<string, RuntimeOperation>()
-
-  /**
-   * Watches for all project operations in the project.
-   *
-   * @param projectId The project ID to watch.
-   * @param signal The signal to abort the operation.
-   */
-  public async *watchOperations(
-    projectId: string,
-    signal?: AbortSignal,
-  ): AsyncIterable<ProjectOperation> {
-    for await (const [operation] of on(this.operationEE, projectId, { signal })) {
-      yield operation
-    }
+  ) {
+    this.stateUnlockService.registerUnlockTask(
+      //
+      "process-lost-operations",
+      projectId => this.processLostOperations(projectId),
+    )
   }
 
-  /**
-   * Watches for logs of the instance in the operation.
-   *
-   * @param operationId The operation ID to watch.
-   * @param instanceId The instance ID to watch.
-   * @param signal The signal to abort the operation.
-   */
-  async *watchInstanceLogs(
-    operationId: string,
-    instanceId: string,
-    signal?: AbortSignal,
-  ): AsyncIterable<string> {
-    const eventKey = `${operationId}/${instanceId}`
-    for await (const [log] of on(this.instanceLogsEE, eventKey, { signal })) {
-      yield log
-    }
-  }
+  private readonly runtimeOperations = new Map<string, RuntimeOperation>()
 
   /**
    * Launches the project operation.
    *
    * @param request The operation request to launch.
    */
-  async launch(request: ProjectOperationRequest): Promise<ProjectOperation> {
-    const operation: ProjectOperation = {
+  async launch(request: OperationRequest): Promise<Operation> {
+    const operation: Operation = {
       id: uuidv7(),
-      projectId: request.projectId,
+      meta: {},
       type: request.type,
       requestedInstanceIds: request.instanceIds,
       instanceIdsToUpdate: [],
@@ -94,9 +79,14 @@ export class OperationManager {
 
     this.logger.info({ operation }, "launching operation")
 
-    await this.stateBackend.putOperation(operation)
+    await this.operationService.updateOperation(request.projectId, operation)
 
-    this.startOperation(operation)
+    const project = await this.stateManager.getProjectRepository().get(request.projectId)
+    if (!project) {
+      throw new Error(`Project with ID "${request.projectId}" not found`)
+    }
+
+    this.startOperation(project, operation)
 
     return operation
   }
@@ -112,59 +102,97 @@ export class OperationManager {
     }
   }
 
-  private startOperation(operation: ProjectOperation): void {
+  cancelInstance(operationId: string, instanceId: string): void {
+    const runtimeOperation = this.runtimeOperations.get(operationId)
+    if (runtimeOperation) {
+      runtimeOperation.cancelInstance(instanceId)
+    }
+  }
+
+  private startOperation(project: Project, operation: Operation): void {
     const runtimeOperation = new RuntimeOperation(
+      project,
       operation,
       this.runnerBackend,
-      this.stateBackend,
       this.libraryBackend,
       this.projectBackend,
-      this.secretBackend,
-      this.projectLockManager.getLock(operation.projectId),
+      this.artifactManager,
       this.stateManager,
-      this.operationEE,
-      this.instanceLogsEE,
-      this.logger.child({ service: "RuntimeOperation", operationId: operation.id }),
+      this.instanceLockService,
+      this.operationService,
+      this.secretService,
+      this.instanceStateService,
+      this.pubsubManager,
+      this.workerService,
+      this.logger.child({ operationId: operation.id }),
     )
 
     this.runtimeOperations.set(operation.id, runtimeOperation)
     void runtimeOperation.operateSafe().finally(() => this.runtimeOperations.delete(operation.id))
   }
 
-  static async create(
-    runnerBackend: RunnerBackend,
-    stateBackend: StateBackend,
-    libraryBackend: LibraryBackend,
-    projectBackend: ProjectBackend,
-    secretBackend: SecretBackend,
-    projectLockManager: ProjectLockManager,
-    stateManager: StateManager,
-    logger: Logger,
-  ) {
-    const operator = new OperationManager(
-      runnerBackend,
-      stateBackend,
-      libraryBackend,
-      projectBackend,
-      secretBackend,
-      projectLockManager,
-      stateManager,
-      logger.child({ service: "OperationManager" }),
-    )
-
-    // relaunch all active operations
-    const activeOperations = await stateBackend.getActiveOperations()
+  private async processLostOperations(projectId: string): Promise<void> {
+    const activeOperations = await this.stateManager
+      .getActiveOperationIndexRepository(projectId)
+      .getAllItems()
 
     for (const operation of activeOperations) {
-      logger.info({ msg: "relaunching operation", operationId: operation.id })
+      if (isFinalOperationStatus(operation.status)) {
+        this.logger.warn(
+          { projectId, operationId: operation.id },
+          "operation is in final state but still marked as active, removing from index",
+        )
+
+        await this.stateManager
+          .getActiveOperationIndexRepository(projectId)
+          .indexRepository.delete(operation.id)
+        continue
+      }
 
-      // operator.startOperation(operation)
+      const errorMessagePrefix = operation.error ? `${operation.error}\n\n` : ""
 
-      // cancel the operation for now
-      operation.status = "cancelled"
-      await stateBackend.putOperation(operation)
+      operation.status = "failed"
+      operation.error = `${errorMessagePrefix}Operation was unexpectedly interrupted. Please restart it.`
+
+      await this.operationService.updateOperation(projectId, operation)
     }
 
-    return operator
+    // unlock instances that were locked by lost operations
+    const activeOperationIds = new Set(activeOperations.map(op => op.id))
+
+    const allLocks = await this.stateManager.getInstanceLockRepository(projectId).getAllItems()
+    const lockIdsToRemvoe: string[] = []
+
+    // clean up unexpected operation locks
+    for (const lock of allLocks) {
+      if (lock.spec.type !== "operation") {
+        continue
+      }
+
+      if (activeOperationIds.has(lock.spec.operationId)) {
+        // remove locks for lost operation which is expected
+        lockIdsToRemvoe.push(lock.id)
+        continue
+      }
+
+      // unexpected operation lock found, also remove it
+      this.logger.warn(
+        { projectId, lockId: lock.id },
+        `unexpected operation lock found for completed operation "${lock.spec.operationId}", removing lock`,
+      )
+
+      lockIdsToRemvoe.push(lock.id)
+    }
+
+    if (lockIdsToRemvoe.length > 0) {
+      await this.instanceLockService.unlockInstancesUnconditionally(projectId, lockIdsToRemvoe)
+    }
+
+    this.logger.debug(
+      { projectId, operationCount: activeOperations.length },
+      `processed %s lost operations for project "%s"`,
+      activeOperations.length,
+      projectId,
+    )
   }
 }