@highstate/backend 0.7.2 → 0.7.4

package/src/orchestrator/operation.ts

@@ -0,0 +1,647 @@
+import type { LibraryBackend } from "../library"
+import type { LogEntry, StateBackend, StateManager } from "../state"
+import type { ProjectBackend, ProjectLock } from "../project"
+import type { SecretBackend } from "../secret"
+import type { RunnerBackend } from "../runner"
+import type { InstanceLogsEvents, OperationEvents } from "./manager"
+import type { EventEmitter } from "node:events"
+import type { Logger } from "pino"
+import { isUnitModel, parseInstanceId, type InstanceModel } from "@highstate/contract"
+import { mapValues } from "remeda"
+import {
+  type InstanceState,
+  type InstanceStateUpdate,
+  type InstanceStatus,
+  type ProjectOperation,
+  type InstanceTriggerInvocation,
+  createInstanceState,
+} from "../shared"
+import {
+  createAsyncBatcher,
+  errorToString,
+  isAbortError,
+  isAbortErrorLike,
+  tryWrapAbortErrorLike,
+  valueToString,
+} from "../common"
+import { OperationWorkset } from "./operation-workset"
+
+type OperationPhase = "update" | "destroy" | "refresh"
+
+export class RuntimeOperation {
+  private readonly abortController = new AbortController()
+  private readonly instancePromiseMap = new Map<string, Promise<void>>()
+  private workset!: OperationWorkset
+
+  private currentPhase!: OperationPhase
+
+  constructor(
+    private readonly operation: ProjectOperation,
+    private readonly runnerBackend: RunnerBackend,
+    private readonly stateBackend: StateBackend,
+    private readonly libraryBackend: LibraryBackend,
+    private readonly projectBackend: ProjectBackend,
+    private readonly secretBackend: SecretBackend,
+    private readonly projectLock: ProjectLock,
+    private readonly stateManager: StateManager,
+    private readonly operationEE: EventEmitter<OperationEvents>,
+    private readonly instanceLogsEE: EventEmitter<InstanceLogsEvents>,
+    private readonly logger: Logger,
+  ) {}
+
+  async operateSafe(): Promise<void> {
+    try {
+      await this.operate()
+    } catch (error) {
+      if (isAbortError(error)) {
+        this.logger.info("the operation was cancelled")
+        this.operation.status = "cancelled"
+
+        await this.updateOperation()
+        return
+      }
+
+      this.logger.error({ msg: "an error occurred while running the operation", error })
+
+      this.operation.status = "failed"
+      this.operation.error = errorToString(error)
+
+      await this.updateOperation()
+    } finally {
+      await Promise.all([
+        this.persistStates.flush(),
+        this.persistLogs.flush(),
+        this.persistSecrets.flush(),
+      ])
+
+      this.logger.debug("operation finished, all entries persisted")
+    }
+  }
+
+  private async operate(): Promise<void> {
+    this.logger.info("starting operation")
+    let lockInstanceIds: string[]
+
+    // keep recalculating the workset until we can acquire the locks and actually start the operation
+    while (true) {
+      this.workset = await OperationWorkset.load(
+        this.operation,
+        this.projectBackend,
+        this.libraryBackend,
+        this.stateBackend,
+        this.stateManager,
+        this.logger,
+        this.abortController.signal,
+      )
+
+      lockInstanceIds = this.workset.getLockInstanceIds()
+
+      if (this.projectLock.canImmediatelyAcquireLocks(lockInstanceIds)) {
+        break
+      }
+
+      this.logger.info("waiting for locks to be available")
+
+      await this.projectLock.lockInstances(lockInstanceIds, () => Promise.resolve())
+    }
+
+    try {
+      // actually acquire the locks and start the operation
+      await this.projectLock.lockInstances(lockInstanceIds, () => this.processOperation())
+    } finally {
+      if (this.operation.type === "preview") {
+        // stream initial states for preview operations
+        this.workset.emitAffectedInitialStates()
+      }
+    }
+  }
+
+  private async processOperation(): Promise<void> {
+    this.operation.affectedInstanceIds = this.workset.operation.affectedInstanceIds
+
+    this.logger.info(
+      { affectedInstanceIds: this.operation.affectedInstanceIds },
+      "operation started",
+    )
+
+    const phases = this.getOperationPhases()
+
+    for (const phase of phases) {
+      this.currentPhase = phase
+
+      const promises: Promise<void>[] = []
+      for (const instanceId of this.operation.affectedInstanceIds) {
+        const instance = this.workset.getInstance(instanceId)
+        if (instance.parentId && this.workset.isAffected(instance.parentId)) {
+          // do not call the operation for child instances of affected composites,
+          // they will be called by their parent instance
+          continue
+        }
+
+        promises.push(this.getInstancePromiseForOperation(instanceId))
+      }
+
+      this.logger.info(`all operations for phase "%s" started`, phase)
+      this.operation.status = "running"
+      await this.updateOperation()
+
+      await Promise.all(promises)
+
+      this.logger.info(`all operations for phase "%s" completed`, phase)
+    }
+
+    this.operation.status = "completed"
+    this.operation.error = null
+    await this.updateOperation()
+
+    this.logger.info("operation completed")
+  }
+
+  cancel() {
+    this.logger.info("cancelling operation")
+    this.abortController.abort()
+  }
+
+  private getInstancePromiseForOperation(instanceId: string): Promise<void> {
+    const instance = this.workset.getInstance(instanceId)
+    const component = this.workset.library.components[instance.type]
+
+    if (isUnitModel(component)) {
+      return this.getUnitPromise(instance)
+    }
+
+    return this.getCompositePromise(instance)
+  }
+
+  private getOperationPhases(): OperationPhase[] {
+    switch (this.operation.type) {
+      case "update":
+      case "preview":
+        return ["update"]
+      case "recreate":
+        return ["destroy", "update"]
+      case "destroy":
+        return ["destroy"]
+      case "refresh":
+        return ["refresh"]
+    }
+  }
+
+  private async getUnitPromise(instance: InstanceModel): Promise<void> {
+    switch (this.currentPhase) {
+      case "update": {
+        return this.updateUnit(instance)
+      }
+      case "destroy": {
+        return this.destroyUnit(instance.id)
+      }
+      case "refresh": {
+        return this.refreshUnit(instance.id)
+      }
+    }
+  }
+
+  private async getCompositePromise(instance: InstanceModel): Promise<void> {
+    const logger = this.logger.child({ instanceId: instance.id })
+
+    return this.getInstancePromise(instance.id, async () => {
+      const state = this.workset.getState(instance.id) ?? createInstanceState(instance.id)
+
+      this.updateInstanceState({
+        ...state,
+        parentId: instance.parentId,
+        latestOperationId: this.operation.id,
+        status: this.getStatusByOperationType(),
+        error: null,
+      })
+
+      const children = this.workset.getAffectedCompositeChildren(instance.id)
+      const childPromises: Promise<void>[] = []
+
+      if (children.length) {
+        logger.info("running %s children", children.length)
+      } else {
+        logger.warn("no affected children found for composite component")
+      }
+
+      for (const child of children) {
+        logger.debug(`waiting for child: "%s"`, child.id)
+        childPromises.push(this.getInstancePromiseForOperation(child.id))
+      }
+
+      try {
+        await Promise.all(childPromises)
+
+        if (children.length > 0) {
+          logger.info("all children completed")
+        }
+
+        this.updateInstanceState({
+          id: instance.id,
+          status: this.operation.type === "destroy" ? "not_created" : "created",
+          inputHash: await this.workset.getUpToDateInputHash(instance),
+        })
+      } catch (error) {
+        if (isAbortErrorLike(error)) {
+          this.workset.restoreInitialStatus(instance.id)
+          return
+        }
+
+        this.updateInstanceState({
+          id: instance.id,
+          status: "error",
+          error: errorToString(error),
+        })
+      }
+    })
+  }
+
+  private updateUnit(instance: InstanceModel): Promise<void> {
+    return this.getInstancePromise(instance.id, async logger => {
+      this.updateInstanceState({
+        id: instance.id,
+        parentId: instance.parentId,
+        latestOperationId: this.operation.id,
+        status: "pending",
+        error: null,
+        currentResourceCount: 0,
+      })
+
+      let dependencyIds: string[] = []
+      try {
+        dependencyIds = await this.updateUnitDependencies(instance, logger)
+      } catch (error) {
+        // restore the initial status of the instance if one of the dependencies failed
+        this.workset.restoreInitialStatus(instance.id)
+        throw error
+      }
+
+      logger.info("updating unit")
+
+      const secrets = await this.secretBackend.get(this.operation.projectId, instance.id)
+      this.abortController.signal.throwIfAborted()
+
+      logger.debug("secrets loaded", { count: Object.keys(secrets).length })
+
+      await this.runnerBackend[this.operation.type === "preview" ? "preview" : "update"]({
+        projectId: this.operation.projectId,
+        instanceType: instance.type,
+        instanceName: instance.name,
+        config: this.prepareUnitConfig(instance),
+        refresh: this.operation.options.refresh,
+        secrets: mapValues(secrets, value => valueToString(value)),
+        signal: this.abortController.signal,
+      })
+
+      logger.debug("unit update requested")
+
+      const stream = this.runnerBackend.watch({
+        projectId: this.operation.projectId,
+        instanceType: instance.type,
+        instanceName: instance.name,
+        finalStatuses: ["created", "error"],
+      })
+
+      await this.watchStateStream(stream)
+
+      const inputHash = await this.workset.getUpToDateInputHash(instance)
+
+      this.updateInstanceState({
+        id: instance.id,
+        inputHash,
+        dependencyIds,
+      })
+
+      logger.debug("input hash after update", { inputHash })
+      logger.info("unit updated")
+    })
+  }
+
+  private async updateUnitDependencies(instance: InstanceModel, logger: Logger): Promise<string[]> {
+    try {
+      const dependencies = this.getInstanceDependencies(instance)
+      const dependencyPromises: Promise<void>[] = []
+
+      for (const dependency of dependencies) {
+        if (!this.operation.affectedInstanceIds.includes(dependency.id)) {
+          // skip dependencies that are not affected by the operation
+          continue
+        }
+
+        logger.debug(`waiting for dependency: ${dependency.id}`)
+        dependencyPromises.push(this.getInstancePromiseForOperation(dependency.id))
+      }
+
+      await Promise.all(dependencyPromises)
+
+      if (dependencies.length > 0) {
+        logger.info("all dependencies completed")
+      }
+
+      return dependencies.map(dependency => dependency.id)
+    } catch (error) {
+      // restore the initial status of the instance if one of the dependencies failed
+      this.workset.restoreInitialStatus(instance.id)
+      throw error
+    }
+  }
+
+  private async processBeforeDestroyTriggers(state: InstanceState, logger: Logger): Promise<void> {
+    if (!this.operation.options.invokeDestroyTriggers) {
+      logger.debug("destroy triggers are disabled for the operation")
+      return
+    }
+
+    const instance = this.workset.getInstance(state.id)
+
+    const triggers = state.triggers.filter(trigger => trigger.spec.type === "before-destroy")
+    if (triggers.length === 0) {
+      return
+    }
+
+    const invokedTriggers = triggers.map(trigger => ({ name: trigger.name }))
+
+    logger.info("updating unit to process before-destroy triggers...")
+
+    const secrets = await this.secretBackend.get(
+      this.operation.projectId,
+      instance.id,
+      this.abortController.signal,
+    )
+
+    await this.runnerBackend.update({
+      projectId: this.operation.projectId,
+      instanceType: instance.type,
+      instanceName: instance.name,
+      config: this.prepareUnitConfig(instance, invokedTriggers),
+      refresh: this.operation.options.refresh,
+      secrets: mapValues(secrets, value => valueToString(value)),
+      signal: this.abortController.signal,
+    })
+
+    logger.debug("unit update requested")
+
+    const stream = this.runnerBackend.watch({
+      projectId: this.operation.projectId,
+      instanceType: instance.type,
+      instanceName: instance.name,
+      finalStatuses: ["created", "error"],
+    })
+
+    await this.watchStateStream(stream)
+
+    logger.debug("before-destroy triggers processed")
+  }
+
+  private async destroyUnit(instanceId: string): Promise<void> {
+    return this.getInstancePromise(instanceId, async logger => {
+      this.updateInstanceState({
+        id: instanceId,
+        latestOperationId: this.operation.id,
+        status: "pending",
+        error: null,
+      })
+
+      const state = this.workset.getState(instanceId)
+      if (!state) {
+        logger.warn("state not found for unit, but destroy was requested")
+        return
+      }
+
+      const dependentPromises: Promise<void>[] = []
+      const dependents = this.workset.getDependentStates(state.id)
+
+      for (const dependent of dependents) {
+        dependentPromises.push(this.getInstancePromiseForOperation(dependent.id))
+      }
+
+      await Promise.all(dependentPromises)
+      this.abortController.signal.throwIfAborted()
+
+      await this.processBeforeDestroyTriggers(state, logger)
+
+      logger.info("destroying unit...")
+
+      const [type, name] = parseInstanceId(instanceId)
+
+      await this.runnerBackend.destroy({
+        projectId: this.operation.projectId,
+        instanceType: type,
+        instanceName: name,
+        refresh: this.operation.options.refresh,
+        signal: this.abortController.signal,
+        deleteUnreachable: this.operation.options.deleteUnreachableResources,
+      })
+
+      this.logger.debug("destroy request sent")
+
+      const stream = this.runnerBackend.watch({
+        projectId: this.operation.projectId,
+        instanceType: type,
+        instanceName: name,
+        finalStatuses: ["not_created", "error"],
+      })
+
+      await this.watchStateStream(stream)
+      this.logger.info("unit destroyed")
+    })
+  }
+
+  private async refreshUnit(instanceId: string) {
+    const logger = this.logger.child({ instanceId })
+
+    return this.getInstancePromise(instanceId, async () => {
+      this.updateInstanceState({
+        id: instanceId,
+        latestOperationId: this.operation.id,
+        status: "refreshing",
+        currentResourceCount: 0,
+        totalResourceCount: 0,
+      })
+
+      logger.info("refreshing unit...")
+
+      const [type, name] = parseInstanceId(instanceId)
+
+      await this.runnerBackend.refresh({
+        projectId: this.operation.projectId,
+        instanceType: type,
+        instanceName: name,
+        signal: this.abortController.signal,
+      })
+
+      logger.debug("unit refresh requested")
+
+      const stream = this.runnerBackend.watch({
+        projectId: this.operation.projectId,
+        instanceType: type,
+        instanceName: name,
+        finalStatuses: ["created", "error"],
+      })
+
+      await this.watchStateStream(stream)
+      logger.info("unit refreshed")
+    })
+  }
+
+  private async watchStateStream(stream: AsyncIterable<InstanceStateUpdate>) {
+    let statePatch: InstanceStateUpdate | undefined
+    for await (statePatch of stream) {
+      if (statePatch.status === "not_created" && this.operation.type === "recreate") {
+        // do not stream "not_created" status for recreate operation to improve UX
+        continue
+      }
+
+      this.updateInstanceState(statePatch)
+    }
+
+    if (!statePatch) {
+      throw new Error("The stream ended without emitting any state.")
+    }
+
+    if (statePatch.status === "error") {
+      throw tryWrapAbortErrorLike(
+        new Error(`The operation on unit "${statePatch.id}" failed: ${statePatch.error}`),
+      )
+    }
+  }
+
+  private prepareUnitConfig(
+    instance: InstanceModel,
+    invokedTriggers: InstanceTriggerInvocation[] = [],
+  ): Record<string, string> {
+    const config: Record<string, string> = {}
+
+    for (const [key, value] of Object.entries(instance.args ?? {})) {
+      config[key] = valueToString(value)
+    }
+
+    const instanceInputs = this.workset.resolvedInstanceInputs.get(instance.id) ?? {}
+
+    for (const [key, value] of Object.entries(instanceInputs)) {
+      config[`input.${key}`] = JSON.stringify(value.map(input => input.input))
+    }
+
+    config["$invokedTriggers"] = JSON.stringify(invokedTriggers)
+
+    return config
+  }
+
+  private async updateOperation(): Promise<void> {
+    this.operationEE.emit(this.operation.projectId, this.operation)
+    await this.stateBackend.putOperation(this.operation)
+  }
+
+  private updateInstanceState(patch: InstanceStateUpdate): void {
+    if (!patch.id) {
+      throw new Error("The ID of the instance state is required.")
+    }
+
+    if (patch.logLine) {
+      // recursively persist logs for instance and all parent instances
+
+      let instance: InstanceModel | null = this.workset.getInstance(patch.id)
+
+      for (;;) {
+        this.persistLogs.call([instance.id, patch.logLine])
+        this.instanceLogsEE.emit(`${this.operation.id}/${instance.id}`, patch.logLine)
+
+        if (!instance.parentId) {
+          break
+        }
+
+        instance = this.workset.getInstance(instance.parentId)
+      }
+      return
+    }
+
+    const state = this.workset.updateState(patch)
+
+    // do not persist anyting for preview operations
+    if (this.operation.type !== "preview") {
+      this.persistStates.call(state)
+
+      if (patch.secrets) {
+        this.persistSecrets.call([patch.id, patch.secrets])
+      }
+    }
+  }
+
+  private getInstancePromise(
+    instanceId: string,
+    fn: (logger: Logger) => Promise<void>,
+  ): Promise<void> {
+    let instancePromise = this.instancePromiseMap.get(instanceId)
+    if (instancePromise) {
+      return instancePromise
+    }
+
+    const logger = this.logger.child({ instanceId }, { msgPrefix: `[${instanceId}] ` })
+
+    instancePromise = fn(logger).finally(() => this.instancePromiseMap.delete(instanceId))
+    this.instancePromiseMap.set(instanceId, instancePromise)
+
+    return instancePromise
+  }
+
+  private getStatusByOperationType(): InstanceStatus {
+    switch (this.operation.type) {
+      case "update":
+        return "updating"
+      case "preview":
+        return "previewing"
+      case "recreate":
+        return "updating"
+      case "destroy":
+        return "destroying"
+      case "refresh":
+        return "refreshing"
+    }
+  }
+
+  private getInstanceDependencies(instance: InstanceModel): InstanceModel[] {
+    const dependencies: InstanceModel[] = []
+    const instanceInputs = this.workset.resolvedInstanceInputs.get(instance.id) ?? {}
+
+    for (const inputs of Object.values(instanceInputs)) {
+      for (const input of inputs) {
+        const dependency = this.workset.getInstance(input.input.instanceId)
+
+        dependencies.push(dependency)
+      }
+    }
+
+    return dependencies
+  }
+
+  private persistStates = createAsyncBatcher(async (states: InstanceState[]) => {
+    this.logger.debug({ msg: "persisting states", count: states.length })
+
+    await this.stateBackend.putAffectedInstanceStates(
+      this.operation.projectId,
+      this.operation.id,
+      states,
+    )
+  })
+
+  private persistLogs = createAsyncBatcher(async (entries: LogEntry[]) => {
+    this.logger.trace({ msg: "persisting logs", count: entries.length })
+
+    await this.stateBackend.appendInstanceLogs(this.operation.id, entries)
+  })
+
+  private persistSecrets = createAsyncBatcher(
+    async (entries: [string, Record<string, string>][]) => {
+      this.logger.debug({ msg: "persisting secrets", count: entries.length })
+
+      // TODO: may be batched (and patched without reading)
+      for (const [instanceId, secrets] of entries) {
+        const existingSecrets = await this.secretBackend.get(this.operation.projectId, instanceId)
+
+        Object.assign(existingSecrets, secrets)
+
+        await this.secretBackend.set(this.operation.projectId, instanceId, existingSecrets)
+      }
+    },
+  )
+}

package/src/preferences/shared.ts

@@ -0,0 +1 @@
+export interface PreferencesBackend {}