@highstate/backend 0.7.2 → 0.7.3

package/src/library/worker/main.ts

@@ -0,0 +1,82 @@
+import type { WorkerData, WorkerRequest, WorkerResponse } from "./protocol"
+import { parentPort, workerData } from "node:worker_threads"
+import { createJiti, type Jiti } from "jiti"
+import { pino } from "pino"
+import { mapValues } from "remeda"
+import { errorToString } from "../../common"
+import { loadLibrary, type Library } from "./loader"
+import { evaluateInstances, evaluateModules } from "./evaluator"
+
+const data = workerData as WorkerData
+
+const logger = pino({ name: "library-worker", level: data.logLevel })
+let jiti: Jiti
+let library: Library
+
+try {
+  logger.info("worker started")
+  logger.trace({ data }, "worker data")
+
+  jiti = createJiti(import.meta.filename)
+  logger.debug({ filename: import.meta.filename }, "jiti created")
+
+  library = await loadLibrary(jiti, logger, data.modulePaths)
+
+  parentPort!.postMessage({
+    type: "library",
+    library: {
+      components: mapValues(library.components, component => component.model),
+      entities: library.entities,
+    },
+  } satisfies WorkerResponse)
+
+  logger.info("library loaded and sent")
+} catch (error) {
+  logger.error({ error }, "failed to load library")
+
+  parentPort!.postMessage({
+    type: "error",
+    error: errorToString(error),
+  })
+}
+
+// eslint-disable-next-line @typescript-eslint/no-misused-promises
+parentPort!.on("message", async message => {
+  try {
+    const request = message as WorkerRequest
+
+    switch (request.type) {
+      case "evaluate-composite-instances": {
+        const results = evaluateInstances(
+          logger,
+          library,
+          request.allInstances,
+          request.resolvedInputs,
+          request.instanceIds,
+        )
+
+        parentPort!.postMessage({
+          type: "instance-evaluation-results",
+          results,
+        } satisfies WorkerResponse)
+        break
+      }
+      case "evaluate-modules": {
+        const result = await evaluateModules(jiti, logger, request.modulePaths)
+
+        parentPort!.postMessage({
+          type: "module-evaluation-result",
+          result,
+        } satisfies WorkerResponse)
+        break
+      }
+    }
+  } catch (error) {
+    logger.error({ error }, "failed to evaluate")
+
+    parentPort!.postMessage({
+      type: "error",
+      error: errorToString(error),
+    } satisfies WorkerResponse)
+  }
+})

package/src/library/worker/protocol.ts

@@ -0,0 +1,38 @@
+import type { InstanceModel } from "@highstate/contract"
+import type { InstanceEvaluationResult, ModuleEvaluationResult } from "../abstractions"
+import type { LibraryModel, ResolvedInstanceInput } from "../../shared"
+
+export type WorkerData = {
+  modulePaths: string[]
+  logLevel?: string
+}
+
+export type WorkerRequest =
+  | {
+      type: "evaluate-composite-instances"
+      allInstances: InstanceModel[]
+      resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>
+      instanceIds: string[]
+    }
+  | {
+      type: "evaluate-modules"
+      modulePaths: string[]
+    }
+
+export type WorkerResponse =
+  | {
+      type: "library"
+      library: LibraryModel
+    }
+  | {
+      type: "instance-evaluation-results"
+      results: InstanceEvaluationResult[]
+    }
+  | {
+      type: "module-evaluation-result"
+      result: ModuleEvaluationResult
+    }
+  | {
+      type: "error"
+      error: string
+    }

package/src/orchestrator/index.ts

@@ -0,0 +1 @@
+export * from "./manager"

package/src/orchestrator/manager.ts

@@ -0,0 +1,165 @@
+import type { LibraryBackend } from "../library"
+import type { ProjectBackend, ProjectLockManager } from "../project"
+import type { RunnerBackend } from "../runner"
+import type { SecretBackend } from "../secret"
+import type { StateBackend, StateManager } from "../state"
+import type { Logger } from "pino"
+import { EventEmitter, on } from "node:events"
+import { uuidv7 } from "uuidv7"
+import { type ProjectOperation, type ProjectOperationRequest } from "../shared"
+import { RuntimeOperation } from "./operation"
+
+export type OperationEvents = Record<string, [ProjectOperation]>
+export type InstanceLogsEvents = Record<string, [string]>
+
+export class OperationManager {
+  constructor(
+    private readonly runnerBackend: RunnerBackend,
+    private readonly stateBackend: StateBackend,
+    private readonly libraryBackend: LibraryBackend,
+    private readonly projectBackend: ProjectBackend,
+    private readonly secretBackend: SecretBackend,
+    private readonly projectLockManager: ProjectLockManager,
+    private readonly stateManager: StateManager,
+    private readonly logger: Logger,
+  ) {}
+
+  private readonly operationEE = new EventEmitter<OperationEvents>()
+  private readonly instanceLogsEE = new EventEmitter<InstanceLogsEvents>()
+
+  private readonly runtimeOperations = new Map<string, RuntimeOperation>()
+
+  /**
+   * Watches for all project operations in the project.
+   *
+   * @param projectId The project ID to watch.
+   * @param signal The signal to abort the operation.
+   */
+  public async *watchOperations(
+    projectId: string,
+    signal?: AbortSignal,
+  ): AsyncIterable<ProjectOperation> {
+    for await (const [operation] of on(this.operationEE, projectId, { signal })) {
+      yield operation
+    }
+  }
+
+  /**
+   * Watches for logs of the instance in the operation.
+   *
+   * @param operationId The operation ID to watch.
+   * @param instanceId The instance ID to watch.
+   * @param signal The signal to abort the operation.
+   */
+  async *watchInstanceLogs(
+    operationId: string,
+    instanceId: string,
+    signal?: AbortSignal,
+  ): AsyncIterable<string> {
+    const eventKey = `${operationId}/${instanceId}`
+    for await (const [log] of on(this.instanceLogsEE, eventKey, { signal })) {
+      yield log
+    }
+  }
+
+  /**
+   * Launches the project operation.
+   *
+   * @param request The operation request to launch.
+   */
+  async launch(request: ProjectOperationRequest): Promise<ProjectOperation> {
+    const operation: ProjectOperation = {
+      id: uuidv7(),
+      projectId: request.projectId,
+      type: request.type,
+      instanceIds: request.instanceIds,
+      affectedInstanceIds: request.instanceIds,
+      status: "pending",
+      options: {
+        forceUpdateDependencies: request.options?.forceUpdateDependencies ?? false,
+        destroyDependentInstances: request.options?.destroyDependentInstances ?? true,
+        invokeDestroyTriggers: request.options?.invokeDestroyTriggers ?? false,
+        deleteUnreachableResources: request.options?.deleteUnreachableResources ?? false,
+        refresh: request.options?.refresh ?? false,
+      },
+      error: null,
+      startedAt: Date.now(),
+      completedAt: null,
+    }
+
+    this.logger.info({ operation }, "launching operation")
+
+    await this.stateBackend.putOperation(operation)
+
+    this.startOperation(operation)
+
+    return operation
+  }
+
+  /**
+   * Cancels the current operation.
+   * Does nothing if no operation is running.
+   */
+  cancel(operationId: string): void {
+    const runtimeOperation = this.runtimeOperations.get(operationId)
+    if (runtimeOperation) {
+      runtimeOperation.cancel()
+    }
+  }
+
+  private startOperation(operation: ProjectOperation): void {
+    const runtimeOperation = new RuntimeOperation(
+      operation,
+      this.runnerBackend,
+      this.stateBackend,
+      this.libraryBackend,
+      this.projectBackend,
+      this.secretBackend,
+      this.projectLockManager.getLock(operation.projectId),
+      this.stateManager,
+      this.operationEE,
+      this.instanceLogsEE,
+      this.logger.child({ service: "RuntimeOperation", operationId: operation.id }),
+    )
+
+    this.runtimeOperations.set(operation.id, runtimeOperation)
+    void runtimeOperation.operateSafe().finally(() => this.runtimeOperations.delete(operation.id))
+  }
+
+  static async create(
+    runnerBackend: RunnerBackend,
+    stateBackend: StateBackend,
+    libraryBackend: LibraryBackend,
+    projectBackend: ProjectBackend,
+    secretBackend: SecretBackend,
+    projectLockManager: ProjectLockManager,
+    stateManager: StateManager,
+    logger: Logger,
+  ) {
+    const operator = new OperationManager(
+      runnerBackend,
+      stateBackend,
+      libraryBackend,
+      projectBackend,
+      secretBackend,
+      projectLockManager,
+      stateManager,
+      logger.child({ service: "OperationManager" }),
+    )
+
+    // relaunch all active operations
+    const activeOperations = await stateBackend.getActiveOperations()
+
+    for (const operation of activeOperations) {
+      logger.info({ msg: "relaunching operation", operationId: operation.id })
+
+      // operator.startOperation(operation)
+
+      // cancel the operation for now
+      operation.status = "cancelled"
+      await stateBackend.putOperation(operation)
+    }
+
+    return operator
+  }
+}