@highstate/backend 0.7.2 → 0.7.3

package/src/project/manager.ts

@@ -0,0 +1,374 @@
+import type { StateBackend, StateManager } from "../state"
+import type { ProjectBackend } from "./abstractions"
+import type { Logger } from "pino"
+import type { LibraryBackend } from "../library"
+import type { ProjectLockManager } from "./lock"
+import { EventEmitter, on } from "node:events"
+import { isUnitModel, type InstanceModel } from "@highstate/contract"
+import {
+  createInputResolver,
+  createInputHashResolver,
+  type InputResolverInput,
+  type InputHashResolverInput,
+  type InstanceModelPatch,
+  type LibraryUpdate,
+  createInstanceState,
+  type CompositeInstance,
+  type ResolvedInstanceInput,
+} from "../shared"
+
+type CompositeInstanceEvent =
+  | {
+      type: "updated"
+      instance: CompositeInstance
+    }
+  | {
+      type: "deleted"
+      instanceId: string
+    }
+
+type CompositeInstanceEvents = {
+  [K in string]: [CompositeInstanceEvent]
+}
+
+export class ProjectManager {
+  private constructor(
+    private readonly projectBackend: ProjectBackend,
+    private readonly stateBackend: StateBackend,
+    private readonly libraryBackend: LibraryBackend,
+    private readonly projectLockManager: ProjectLockManager,
+    private readonly stateManager: StateManager,
+    private readonly logger: Logger,
+  ) {
+    void this.watchLibraryChanges()
+  }
+
+  private readonly compositeInstanceEE = new EventEmitter<CompositeInstanceEvents>()
+
+  async *watchCompositeInstances(
+    projectId: string,
+    signal?: AbortSignal,
+  ): AsyncIterable<CompositeInstanceEvent> {
+    for await (const [children] of on(this.compositeInstanceEE, projectId, {
+      signal,
+    })) {
+      yield children
+    }
+  }
+
+  async createInstance(projectId: string, instance: InstanceModel): Promise<InstanceModel> {
+    const createdInstance = await this.projectBackend.createInstance(projectId, instance)
+    await this.updateCompositeInstance(projectId, createdInstance)
+
+    return createdInstance
+  }
+
+  async updateInstance(
+    projectId: string,
+    instanceId: string,
+    patch: InstanceModelPatch,
+  ): Promise<InstanceModel> {
+    const instance = await this.projectBackend.updateInstance(projectId, instanceId, patch)
+    await this.updateCompositeInstance(projectId, instance)
+
+    return instance
+  }
+
+  async renameInstance(
+    projectId: string,
+    instanceId: string,
+    newName: string,
+  ): Promise<InstanceModel> {
+    const instance = await this.projectBackend.renameInstance(projectId, instanceId, newName)
+    await this.updateCompositeInstance(projectId, instance)
+
+    return instance
+  }
+
+  async deleteInstance(projectId: string, instanceId: string): Promise<void> {
+    await Promise.all([
+      this.projectBackend.deleteInstance(projectId, instanceId),
+      this.stateBackend.clearCompositeInstances(projectId, [instanceId]),
+    ])
+  }
+
+  private async updateCompositeInstance(projectId: string, instance: InstanceModel): Promise<void> {
+    const { resolveInputHash, library } = await this.prepareInputHashResolver(projectId)
+
+    const component = library.components[instance.type]
+    if (!component) {
+      return
+    }
+
+    if (isUnitModel(component)) {
+      return
+    }
+
+    const { inputHash: expectedInputHash } = await resolveInputHash(instance.id)
+    const inputHash = await this.stateBackend.getCompositeInstanceInputHash(projectId, instance.id)
+
+    if (inputHash !== expectedInputHash) {
+      this.logger.info("re-evaluating instance since input hash has changed", {
+        projectId,
+        instanceId: instance.id,
+      })
+
+      await this.evaluateCompositeInstances(projectId, [instance.id])
+    }
+  }
+
+  private async evaluateCompositeInstances(projectId: string, instanceIds: string[]) {
+    await this.projectLockManager.getLock(projectId).lockInstances(instanceIds, async () => {
+      const [
+        { instances, resolvedInputs, stateMap, resolveInputHash },
+        topLevelCompositeChildrenIds,
+      ] = await Promise.all([
+        this.prepareInputHashResolver(projectId),
+        this.stateBackend.getTopLevelCompositeChildrenIds(projectId, instanceIds),
+      ])
+
+      const results = await this.libraryBackend.evaluateCompositeInstances(
+        instances,
+        resolvedInputs,
+        instanceIds,
+      )
+
+      const newStates = results.map(result => {
+        const existingState = stateMap.get(result.instanceId)
+        const newState = existingState ?? createInstanceState(result.instanceId)
+
+        newState.evaluationError = result.success ? null : result.error
+
+        return newState
+      })
+
+      const inputHashes = new Map<string, string>()
+      for (const instanceId of instanceIds) {
+        const { inputHash } = await resolveInputHash(instanceId)
+        inputHashes.set(instanceId, inputHash)
+      }
+
+      const compositeInstances = results
+        .filter(result => result.success)
+        .flatMap(result =>
+          result.compositeInstances.map(instance => ({
+            ...instance,
+            inputHash: inputHashes.get(instance.instance.id),
+          })),
+        )
+
+      const newTopLevelCompositeChildrenIds = Object.fromEntries(
+        results
+          .filter(result => result.success)
+          .map(result => [
+            result.instanceId,
+            result.compositeInstances
+              .filter(instance => instance.instance.id !== result.instanceId)
+              .map(instance => instance.instance.id),
+          ]),
+      )
+
+      const deletedCompositeInstanceIds = new Set(
+        Object.values(topLevelCompositeChildrenIds).flat(),
+      )
+
+      for (const childInstanceId of Object.values(newTopLevelCompositeChildrenIds).flat()) {
+        deletedCompositeInstanceIds.delete(childInstanceId)
+      }
+
+      await Promise.all([
+        this.stateBackend.clearCompositeInstances(
+          projectId,
+          Array.from(deletedCompositeInstanceIds),
+        ),
+        this.stateBackend.putTopLevelCompositeChildrenIds(
+          projectId,
+          newTopLevelCompositeChildrenIds,
+        ),
+      ])
+
+      for (const state of newStates) {
+        this.stateManager.emitStatePatch(projectId, state)
+      }
+
+      for (const instance of compositeInstances) {
+        this.compositeInstanceEE.emit(projectId, { type: "updated", instance })
+      }
+
+      for (const instanceId of deletedCompositeInstanceIds) {
+        this.compositeInstanceEE.emit(projectId, { type: "deleted", instanceId })
+      }
+
+      const promises: Promise<void>[] = []
+
+      if (newStates.length > 0) {
+        promises.push(this.stateBackend.putInstanceStates(projectId, newStates))
+      }
+
+      if (compositeInstances.length > 0) {
+        promises.push(this.stateBackend.putCompositeInstances(projectId, compositeInstances))
+      }
+
+      this.logger.info(
+        { projectId },
+        "instance evaluation completed, %d instances persisted",
+        compositeInstances.length,
+      )
+
+      await Promise.all(promises)
+    })
+  }
+
+  private async prepareInputHashResolver(projectId: string) {
+    const { instances, hubs } = await this.projectBackend.getProject(projectId)
+
+    const library = await this.libraryBackend.loadLibrary()
+    const filteredInstances = instances.filter(instance => instance.type in library.components)
+
+    const states = await this.stateBackend.getAllInstanceStates(projectId)
+    const stateMap = new Map(states.map(state => [state.id, state]))
+
+    const inputResolverNodes = new Map<string, InputResolverInput>()
+
+    for (const instance of filteredInstances) {
+      inputResolverNodes.set(`instance:${instance.id}`, {
+        kind: "instance",
+        instance,
+        component: library.components[instance.type],
+      })
+    }
+
+    for (const hub of hubs) {
+      inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
+    }
+
+    const resolveInputs = createInputResolver(inputResolverNodes, this.logger)
+
+    const inputHashInputs = new Map<string, InputHashResolverInput>()
+    const resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>> = {}
+
+    for (const instance of filteredInstances) {
+      const output = await resolveInputs(`instance:${instance.id}`)
+      if (output.kind !== "instance") {
+        throw new Error("Expected instance node")
+      }
+
+      let sourceHash: string | undefined
+      if (isUnitModel(library.components[instance.type])) {
+        const resolvedUnit = await this.libraryBackend.getResolvedUnitSource(instance.type)
+        if (!resolvedUnit) {
+          throw new Error(`Resolved unit not found: ${instance.type}`)
+        }
+
+        sourceHash = resolvedUnit.sourceHash
+      }
+
+      inputHashInputs.set(instance.id, {
+        instance,
+        component: library.components[instance.type],
+        resolvedInputs: output.resolvedInputs,
+        state: stateMap.get(instance.id),
+        sourceHash,
+      })
+
+      resolvedInputs[instance.id] = output.resolvedInputs
+    }
+
+    const resolveInputHash = createInputHashResolver(inputHashInputs, this.logger)
+
+    return {
+      resolveInputHash,
+      library,
+      instances,
+      stateMap,
+      resolvedInputs,
+    }
+  }
+
+  private async watchLibraryChanges(): Promise<void> {
+    for await (const updates of this.libraryBackend.watchLibrary()) {
+      try {
+        await this.handleLibraryUpdates(updates)
+      } catch (error) {
+        this.logger.error({ error }, "failed to handle library updates")
+      }
+    }
+  }
+
+  private async handleLibraryUpdates(updates: LibraryUpdate[]): Promise<void> {
+    const changedComponents = new Set<string>()
+    const library = await this.libraryBackend.loadLibrary()
+
+    // TODO: handle entity updates
+
+    for (const update of updates) {
+      switch (update.type) {
+        case "component-updated":
+          changedComponents.add(update.component.type)
+          break
+        case "component-removed":
+          changedComponents.add(update.componentType)
+          break
+      }
+    }
+
+    if (changedComponents.size === 0) {
+      return
+    }
+
+    this.logger.info(
+      { changedComponents },
+      "library components changed, updating composite instances",
+    )
+
+    const projects = await this.projectBackend.getProjectIds()
+    for (const projectId of projects) {
+      const { resolveInputHash, instances } = await this.prepareInputHashResolver(projectId)
+
+      const filteredInstances = instances.filter(
+        instance =>
+          changedComponents.has(instance.type) &&
+          library.components[instance.type] &&
+          !isUnitModel(library.components[instance.type]),
+      )
+
+      this.logger.info(
+        { projectId, filteredInstanceIds: filteredInstances.map(instance => instance.id) },
+        "updating composite instances for project",
+      )
+
+      const inputHashMap = new Map<string, string>()
+      for (const instance of filteredInstances) {
+        const { inputHash } = await resolveInputHash(instance.id)
+        inputHashMap.set(instance.id, inputHash)
+      }
+
+      try {
+        await this.evaluateCompositeInstances(
+          projectId,
+          filteredInstances.map(instance => instance.id),
+        )
+      } catch (error) {
+        this.logger.error({ error }, "failed to evaluate composite instances")
+      }
+    }
+  }
+
+  static create(
+    projectBackend: ProjectBackend,
+    stateBackend: StateBackend,
+    libraryBackend: LibraryBackend,
+    projectLockManager: ProjectLockManager,
+    stateManager: StateManager,
+    logger: Logger,
+  ): ProjectManager {
+    return new ProjectManager(
+      projectBackend,
+      stateBackend,
+      libraryBackend,
+      projectLockManager,
+      stateManager,
+      logger.child({ service: "ProjectManager" }),
+    )
+  }
+}

package/src/runner/abstractions.ts

@@ -0,0 +1,146 @@
+import type {
+  InstancePageBlock,
+  InstanceState,
+  InstanceStateUpdate,
+  InstanceStatus,
+  InstanceTerminal,
+} from "../shared"
+
+export type RunnerBaseOptions = {
+  /**
+   * The HighState project ID.
+   */
+  projectId: string
+
+  /**
+   * The type of the instance to run.
+   */
+  instanceType: string
+
+  /**
+   * The name of the instance to run.
+   */
+  instanceName: string
+
+  /**
+   * The signal to abort the operation.
+   */
+  signal?: AbortSignal
+}
+
+export type InstanceUpdateOptions = RunnerBaseOptions & {
+  /**
+   * The configuration of the stack.
+   */
+  config: Record<string, string>
+
+  /**
+   * The values of the secrets.
+   */
+  secrets: Record<string, string>
+
+  /**
+   * Whether to refresh the state before updating.
+   */
+  refresh?: boolean
+}
+
+export type InstanceDestroyOptions = RunnerBaseOptions & {
+  /**
+   * Whether to refresh the state before updating.
+   */
+  refresh?: boolean
+
+  /**
+   * Whether to delete the unreachable resources (e.g. k8s resources in unreachable clusters).
+   */
+  deleteUnreachable?: boolean
+}
+
+export type InstanceWatchOptions = RunnerBaseOptions & {
+  /**
+   * The final statuses to watch.
+   * If the instance reaches one of these statuses, it should return the current state and exit.
+   */
+  finalStatuses?: InstanceStatus[]
+}
+
+export class InvalidInstanceStatusError extends Error {
+  constructor(
+    public readonly currentStatus: InstanceStatus,
+    public readonly expectedStatuses: InstanceStatus[],
+  ) {
+    const expectedString = expectedStatuses.join(", ")
+    super(`The current state is "${currentStatus}", but it should be one of "${expectedString}".`)
+  }
+}
+
+export interface RunnerBackend {
+  /**
+   * Watches the instance state.
+   *
+   * Returns the async iterable of the instance state patches.
+   * The fields with `undefined` values should be ignored, and the fields with `null` values should be removed.
+   */
+  watch(options: InstanceWatchOptions): AsyncIterable<InstanceStateUpdate>
+
+  /**
+   * Gets the actual state of the instance.
+   */
+  getState(options: RunnerBaseOptions): Promise<InstanceState>
+
+  /**
+   * Gets the content of the page.
+   */
+  getPageContent(options: RunnerBaseOptions, pageName: string): Promise<InstancePageBlock[] | null>
+
+  /**
+   * Gets the content of the file.
+   */
+  getFileContent(options: RunnerBaseOptions, fileName: string): Promise<string | null>
+
+  /**
+   * Gets the terminal factory.
+   */
+  getTerminalFactory(
+    options: RunnerBaseOptions,
+    terminalName: string,
+  ): Promise<InstanceTerminal | null>
+
+  /**
+   * Updates the instance.
+   *
+   * The operation must only be aborted by the signal, not even when the connection is closed.
+   * If the instance is already updating, it should exit immediately.
+   * If the instance is running another operation, it should throw an error.
+   */
+  update(options: InstanceUpdateOptions): Promise<void>
+
+  /**
+   * Previews the instance update without actually applying the changes.
+   *
+   * The operation must only be aborted by the signal, not even when the connection is closed.
+   * If the instance is already updating, it should exit immediately.
+   * If the instance is running another operation, it should throw an error.
+   */
+  preview(options: InstanceUpdateOptions): Promise<void>
+
+  /**
+   * Destroys the instance.
+   *
+   * The operation must only be aborted by the signal, not even when the connection is closed.
+   * If the instance is not created, it should exit immediately.
+   * If the instance is running another operation, it should throw an error.
+   */
+  destroy(options: InstanceDestroyOptions): Promise<void>
+
+  /**
+   * Refreshes the instance.
+   *
+   * The operation must only be aborted by the signal, not even when the connection is closed.
+   * If the instance is not created, it should exit immediately.
+   * If the instance is running another operation, it should throw an error.
+   * If the instance is not running, it should exit immediately.
+   */
+  refresh(options: RunnerBaseOptions): Promise<void>
+}

package/src/runner/factory.ts

@@ -0,0 +1,22 @@
+import type { RunnerBackend } from "./abstractions"
+import type { LocalPulumiHost } from "../common"
+import type { LibraryBackend } from "../library"
+import { z } from "zod"
+import { LocalRunnerBackend, localRunnerBackendConfig } from "./local"
+
+export const runnerBackendConfig = z.object({
+  HIGHSTATE_BACKEND_RUNNER_TYPE: z.enum(["local"]).default("local"),
+  ...localRunnerBackendConfig.shape,
+})
+
+export function createRunnerBackend(
+  config: z.infer<typeof runnerBackendConfig>,
+  pulumiProjectHost: LocalPulumiHost,
+  libraryBackend: LibraryBackend,
+): RunnerBackend {
+  switch (config.HIGHSTATE_BACKEND_RUNNER_TYPE) {
+    case "local": {
+      return LocalRunnerBackend.create(config, pulumiProjectHost, libraryBackend)
+    }
+  }
+}

package/src/runner/index.ts

@@ -0,0 +1,2 @@
+export * from "./abstractions"
+export * from "./factory"