@highstate/backend 0.9.15 → 0.9.18

package/src/project/manager.ts

@@ -1,433 +0,0 @@
-import type { StateBackend, StateManager } from "../state"
-import type { ProjectBackend } from "./abstractions"
-import type { Logger } from "pino"
-import type { LibraryBackend } from "../library"
-import type { ProjectLockManager } from "./lock"
-import { EventEmitter, on } from "node:events"
-import { isUnitModel, type InstanceModel } from "@highstate/contract"
-import {
-  type InputResolverNode,
-  type InputHashNode,
-  type InstanceModelPatch,
-  type LibraryUpdate,
-  createInstanceState,
-  type CompositeInstance,
-  type ResolvedInstanceInput,
-  type HubModel,
-  InputResolver,
-  InputHashResolver,
-} from "../shared"
-
-type CompositeInstanceEvent =
-  | {
-      type: "evaluation-started"
-      instanceId: string
-    }
-  | {
-      type: "updated"
-      instance: CompositeInstance
-    }
-  | {
-      type: "deleted"
-      instanceId: string
-    }
-  | {
-      type: "failed"
-      instanceId: string
-    }
-
-type CompositeInstanceEvents = {
-  [K in string]: [CompositeInstanceEvent]
-}
-
-export type FullProjectModel = {
-  instances: InstanceModel[]
-  hubs: HubModel[]
-  compositeInstances: CompositeInstance[]
-}
-
-export class ProjectManager {
-  private constructor(
-    private readonly projectBackend: ProjectBackend,
-    private readonly stateBackend: StateBackend,
-    private readonly libraryBackend: LibraryBackend,
-    private readonly projectLockManager: ProjectLockManager,
-    private readonly stateManager: StateManager,
-    private readonly logger: Logger,
-  ) {
-    void this.watchLibraryChanges()
-  }
-
-  private readonly compositeInstanceEE = new EventEmitter<CompositeInstanceEvents>()
-
-  async *watchCompositeInstances(
-    projectId: string,
-    signal?: AbortSignal,
-  ): AsyncIterable<CompositeInstanceEvent> {
-    for await (const [children] of on(this.compositeInstanceEE, projectId, {
-      signal,
-    })) {
-      yield children
-    }
-  }
-
-  /**
-   * Loads the full info of a project, including instances, hubs, and composite instances.
-   *
-   * Also filters out instances that are not in the library.
-   *
-   * @param projectId The ID of the project to load.
-   */
-  async getProject(projectId: string): Promise<FullProjectModel> {
-    const [{ instances, hubs }, compositeInstances, library] = await Promise.all([
-      this.projectBackend.getProject(projectId),
-      this.stateBackend.getCompositeInstances(projectId),
-      this.libraryBackend.loadLibrary(),
-    ])
-
-    const filteredInstances = instances.filter(instance => instance.type in library.components)
-    const filteredCompositeInstances = compositeInstances
-      .filter(instance => instance.instance.type in library.components)
-      .map(instance => ({
-        ...instance,
-        children: instance.children.filter(child => child.type in library.components),
-      }))
-
-    return {
-      instances: filteredInstances,
-      hubs,
-      compositeInstances: filteredCompositeInstances,
-    }
-  }
-
-  async createInstance(projectId: string, instance: InstanceModel): Promise<InstanceModel> {
-    const createdInstance = await this.projectBackend.createInstance(projectId, instance)
-    await this.evaluateChangedCompositeInstances(projectId)
-
-    return createdInstance
-  }
-
-  async updateInstance(
-    projectId: string,
-    instanceId: string,
-    patch: InstanceModelPatch,
-  ): Promise<InstanceModel> {
-    const instance = await this.projectBackend.updateInstance(projectId, instanceId, patch)
-    await this.evaluateChangedCompositeInstances(projectId)
-
-    return instance
-  }
-
-  async renameInstance(
-    projectId: string,
-    instanceId: string,
-    newName: string,
-  ): Promise<InstanceModel> {
-    const instance = await this.projectBackend.renameInstance(projectId, instanceId, newName)
-    await this.evaluateChangedCompositeInstances(projectId)
-
-    return instance
-  }
-
-  async deleteInstance(projectId: string, instanceId: string): Promise<void> {
-    await Promise.all([
-      this.projectBackend.deleteInstance(projectId, instanceId),
-      this.stateBackend.clearCompositeInstances(projectId, [instanceId]),
-    ])
-  }
-
-  private async evaluateChangedCompositeInstances(projectId: string): Promise<void> {
-    const { inputHashResolver, instances, library, evaluatedInputHashes } =
-      await this.prepareResolvers(projectId)
-
-    inputHashResolver.addAllNodesToWorkset()
-    await inputHashResolver.process()
-
-    const instanceIds = instances
-      .filter(instance => !isUnitModel(library.components[instance.type]))
-      .filter(
-        instance =>
-          evaluatedInputHashes[instance.id] !==
-          inputHashResolver.requireOutput(instance.id).inputHash,
-      )
-      .map(instance => instance.id)
-
-    await this.projectLockManager.getLock(projectId).lockInstances(instanceIds, async () => {
-      this.logger.debug({ instanceIds }, "evaluating composite instances")
-
-      for (const instanceId of instanceIds) {
-        this.compositeInstanceEE.emit(projectId, { type: "evaluation-started", instanceId })
-      }
-
-      const [
-        { instances, resolvedInputs, stateMap, inputHashResolver },
-        topLevelCompositeChildrenIds,
-      ] = await Promise.all([
-        this.prepareResolvers(projectId),
-        this.stateBackend.getTopLevelCompositeChildrenIds(projectId, instanceIds),
-      ])
-
-      const results = await this.libraryBackend.evaluateCompositeInstances(
-        instances,
-        resolvedInputs,
-        instanceIds,
-      )
-
-      const newStates = results.map(result => {
-        const existingState = stateMap.get(result.instanceId)
-        const newState = existingState ?? createInstanceState(result.instanceId)
-
-        newState.evaluationError = result.success ? null : result.error
-
-        return newState
-      })
-
-      inputHashResolver.addAllNodesToWorkset()
-      await inputHashResolver.process()
-
-      const compositeInstances = results
-        .filter(result => result.success)
-        .flatMap(result =>
-          result.compositeInstances.map(instance => ({
-            ...instance,
-            inputHash:
-              // only store inputHash for top-level composite instances
-              instance.instance.id === result.instanceId
-                ? inputHashResolver.requireOutput(instance.instance.id).inputHash
-                : "",
-          })),
-        )
-
-      const newTopLevelCompositeChildrenIds = Object.fromEntries(
-        results
-          .filter(result => result.success)
-          .map(result => [
-            result.instanceId,
-            result.compositeInstances
-              .filter(instance => instance.instance.id !== result.instanceId)
-              .map(instance => instance.instance.id),
-          ]),
-      )
-
-      const deletedCompositeInstanceIds = new Set(
-        Object.values(topLevelCompositeChildrenIds).flat(),
-      )
-
-      for (const childInstanceId of Object.values(newTopLevelCompositeChildrenIds).flat()) {
-        deletedCompositeInstanceIds.delete(childInstanceId)
-      }
-
-      await Promise.all([
-        this.stateBackend.clearCompositeInstances(
-          projectId,
-          Array.from(deletedCompositeInstanceIds),
-        ),
-        this.stateBackend.putTopLevelCompositeChildrenIds(
-          projectId,
-          newTopLevelCompositeChildrenIds,
-        ),
-      ])
-
-      for (const state of newStates) {
-        this.stateManager.emitStatePatch(projectId, state)
-
-        if (state.evaluationError) {
-          this.logger.error(
-            { projectId, instanceId: state.id, error: state.evaluationError },
-            "instance evaluation failed",
-          )
-
-          this.compositeInstanceEE.emit(projectId, {
-            type: "failed",
-            instanceId: state.id,
-          })
-        }
-      }
-
-      for (const instance of compositeInstances) {
-        this.compositeInstanceEE.emit(projectId, { type: "updated", instance })
-      }
-
-      for (const instanceId of deletedCompositeInstanceIds) {
-        this.compositeInstanceEE.emit(projectId, { type: "deleted", instanceId })
-      }
-
-      const promises: Promise<void>[] = []
-
-      if (newStates.length > 0) {
-        promises.push(this.stateBackend.putInstanceStates(projectId, newStates))
-      }
-
-      if (compositeInstances.length > 0) {
-        promises.push(this.stateBackend.putCompositeInstances(projectId, compositeInstances))
-      }
-
-      this.logger.info(
-        { projectId },
-        "instance evaluation completed, %d instances persisted",
-        compositeInstances.length,
-      )
-
-      this.logger.debug(
-        { compositeInstanceIds: compositeInstances.map(instance => instance.instance.id) },
-        "persisted composite instances",
-      )
-
-      await Promise.all(promises)
-    })
-  }
-
-  private async prepareResolvers(projectId: string) {
-    const [{ instances, hubs }, states, library, evaluatedInputHashes] = await Promise.all([
-      this.projectBackend.getProject(projectId),
-      this.stateBackend.getAllInstanceStates(projectId),
-      this.libraryBackend.loadLibrary(),
-      this.stateBackend.getCompositeInstanceInputHashes(projectId),
-    ])
-
-    const filteredInstances = instances.filter(instance => instance.type in library.components)
-    const stateMap = new Map(states.map(state => [state.id, state]))
-
-    const inputResolverNodes = new Map<string, InputResolverNode>()
-
-    for (const instance of filteredInstances) {
-      inputResolverNodes.set(`instance:${instance.id}`, {
-        kind: "instance",
-        instance,
-        component: library.components[instance.type],
-      })
-    }
-
-    for (const hub of hubs) {
-      inputResolverNodes.set(`hub:${hub.id}`, { kind: "hub", hub })
-    }
-
-    const inputResolver = new InputResolver(inputResolverNodes, this.logger)
-    inputResolver.addAllNodesToWorkset()
-
-    const inputHashInputs = new Map<string, InputHashNode>()
-    const resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>> = {}
-
-    await inputResolver.process()
-
-    for (const instance of filteredInstances) {
-      const output = inputResolver.requireOutput(`instance:${instance.id}`)
-      if (output.kind !== "instance") {
-        throw new Error("Expected instance node")
-      }
-
-      let sourceHash: string | undefined
-      if (isUnitModel(library.components[instance.type])) {
-        const resolvedUnits = await this.libraryBackend.getResolvedUnitSources([instance.type])
-        const resolvedUnit = resolvedUnits.find(unit => unit.unitType === instance.type)
-
-        if (resolvedUnit) {
-          sourceHash = resolvedUnit.sourceHash
-        } else {
-          this.logger.warn(`resolved unit source not found for type "%s"`, instance.type)
-          sourceHash = undefined
-        }
-      }
-
-      inputHashInputs.set(instance.id, {
-        instance,
-        component: library.components[instance.type],
-        resolvedInputs: output.resolvedInputs,
-        state: stateMap.get(instance.id),
-        sourceHash,
-      })
-
-      resolvedInputs[instance.id] = output.resolvedInputs
-    }
-
-    const inputHashResolver = new InputHashResolver(inputHashInputs, this.logger)
-
-    return {
-      inputHashInputs,
-      inputHashResolver,
-      library,
-      instances: filteredInstances,
-      stateMap,
-      resolvedInputs,
-      evaluatedInputHashes,
-    }
-  }
-
-  private async watchLibraryChanges(): Promise<void> {
-    for await (const updates of this.libraryBackend.watchLibrary()) {
-      try {
-        await this.handleLibraryUpdates(updates)
-      } catch (error) {
-        this.logger.error({ error }, "failed to handle library updates")
-      }
-    }
-  }
-
-  private async handleLibraryUpdates(updates: LibraryUpdate[]): Promise<void> {
-    const changedComponents = new Set<string>()
-    const library = await this.libraryBackend.loadLibrary()
-
-    // TODO: handle entity updates
-
-    for (const update of updates) {
-      switch (update.type) {
-        case "component-updated":
-          changedComponents.add(update.component.type)
-          break
-        case "component-removed":
-          changedComponents.add(update.componentType)
-          break
-      }
-    }
-
-    if (changedComponents.size === 0) {
-      return
-    }
-
-    this.logger.info(
-      { changedComponents },
-      "library components changed, updating composite instances",
-    )
-
-    const projects = await this.projectBackend.getProjectIds()
-    for (const projectId of projects) {
-      const { instances } = await this.prepareResolvers(projectId)
-
-      const filteredInstances = instances.filter(
-        instance =>
-          changedComponents.has(instance.type) &&
-          library.components[instance.type] &&
-          !isUnitModel(library.components[instance.type]),
-      )
-
-      this.logger.info(
-        { projectId, filteredInstanceIds: filteredInstances.map(instance => instance.id) },
-        "updating composite instances for project",
-      )
-
-      try {
-        await this.evaluateChangedCompositeInstances(projectId)
-      } catch (error) {
-        this.logger.error({ error }, "failed to evaluate composite instances")
-      }
-    }
-  }
-
-  static create(
-    projectBackend: ProjectBackend,
-    stateBackend: StateBackend,
-    libraryBackend: LibraryBackend,
-    projectLockManager: ProjectLockManager,
-    stateManager: StateManager,
-    logger: Logger,
-  ): ProjectManager {
-    return new ProjectManager(
-      projectBackend,
-      stateBackend,
-      libraryBackend,
-      projectLockManager,
-      stateManager,
-      logger.child({ service: "ProjectManager" }),
-    )
-  }
-}

package/src/secret/abstractions.ts

@@ -1,59 +0,0 @@
-export class SecretAccessDeniedError extends Error {
-  constructor(projectId: string, key: string) {
-    super(`Access to the secrets of component "${projectId}.${key}" is denied.`)
-  }
-}
-
-export interface SecretBackend {
-  /**
-   * Check if the state and secrets of the project are locked.
-   * The backend may not implement this method.
-   *
-   * @param projectId The ID of the project.
-   * @param signal The signal to abort the operation.
-   *
-   * @returns `true` if the project is locked, `false` if the project is unlocked.
-   */
-  isLocked?(projectId: string, signal?: AbortSignal): Promise<boolean>
-
-  /**
-   * Unlock the state and secrets of the project.
-   * The backend may not implement this method.
-   *
-   * @param projectId The ID of the project.
-   * @param password The password to unlock the secrets.
-   * @param signal The signal to abort the operation.
-   *
-   * @returns `true` if the project is unlocked, `false` if the password is incorrect.
-   */
-  unlock?(projectId: string, password: string, signal?: AbortSignal): Promise<boolean>
-
-  /**
-   * Get the secrets of the component.
-   *
-   * @param projectId The ID of the project.
-   * @param instanceId The ID of the instance.
-   * @param signal The signal to abort the operation.
-   *
-   * @returns A record of secret key-value pairs or `null` if the secrets are not found.
-   * @throws {SecretAccessDeniedError} If access to the secrets is denied.
-   */
-  get(projectId: string, instanceId: string, signal?: AbortSignal): Promise<Record<string, unknown>>
-
-  /**
-   * Set the secrets of the component.
-   *
-   * @param projectId The ID of the project.
-   * @param instanceId The ID of the instance.
-   * @param values The record of secret key-value pairs.
-   * @param signal The signal to abort the operation.
-   *
-   * @throws {SecretAccessDeniedError} If access to the secrets is denied.
-   */
-  set(
-    projectId: string,
-    instanceId: string,
-    values: Record<string, unknown>,
-    signal?: AbortSignal,
-  ): Promise<void>
-}

package/src/secret/factory.ts

@@ -1,22 +0,0 @@
-import type { SecretBackend } from "./abstractions"
-import type { LocalPulumiHost } from "../common"
-import type { Logger } from "pino"
-import { z } from "zod"
-import { LocalSecretBackend, localSecretBackendConfig } from "./local"
-
-export const secretBackendConfig = z.object({
-  HIGHSTATE_BACKEND_SECRET_TYPE: z.enum(["local"]).default("local"),
-  ...localSecretBackendConfig.shape,
-})
-
-export function createSecretBackend(
-  config: z.infer<typeof secretBackendConfig>,
-  localPulumiHost: LocalPulumiHost,
-  logger: Logger,
-): Promise<SecretBackend> {
-  switch (config.HIGHSTATE_BACKEND_SECRET_TYPE) {
-    case "local": {
-      return LocalSecretBackend.create(config, localPulumiHost, logger)
-    }
-  }
-}

package/src/secret/local.ts

@@ -1,152 +0,0 @@
-import type { SecretBackend } from "./abstractions"
-import type { Logger } from "pino"
-import { mapKeys, mapValues, pickBy } from "remeda"
-import { z } from "zod"
-import { LocalPulumiHost, resolveMainLocalProject, stringToValue, valueToString } from "../common"
-
-export const localSecretBackendConfig = z.object({
-  HIGHSTATE_BACKEND_SECRET_PROJECT_PATH: z.string().optional(),
-  HIGHSTATE_BACKEND_SECRET_PROJECT_NAME: z.string().optional(),
-})
-
-/**
- * The backend for storing secrets in the local Pulumi project.
- * It is the simplest backend and should be used for development and single-user projects.
- */
-export class LocalSecretBackend implements SecretBackend {
-  private constructor(
-    private readonly projectPath: string,
-    private readonly projectName: string,
-    private readonly pulumiProjectHost: LocalPulumiHost,
-    private readonly logger: Logger,
-  ) {
-    this.logger.debug({ msg: "initialized", projectPath, projectName })
-  }
-
-  isLocked(projectId: string): Promise<boolean> {
-    return Promise.resolve(!this.pulumiProjectHost.hasPassword(projectId))
-  }
-
-  async unlock(projectId: string, password: string, signal?: AbortSignal): Promise<boolean> {
-    this.pulumiProjectHost.setPassword(projectId, password)
-
-    try {
-      // try to run a command to check if the password is correct
-      await this.pulumiProjectHost.runLocal(
-        {
-          projectId,
-          pulumiProjectName: this.projectName,
-          pulumiStackName: projectId,
-          projectPath: this.projectPath,
-        },
-        async stack => {
-          this.logger.debug({ projectId }, "checking password")
-
-          await stack.info(true)
-        },
-        signal,
-      )
-
-      return true
-    } catch (error) {
-      if (error instanceof Error) {
-        if (error.message.includes("incorrect passphrase")) {
-          this.logger.debug({ projectId, error }, "incorrect passphrase")
-
-          this.pulumiProjectHost.removePassword(projectId)
-          return false
-        }
-      }
-
-      throw error
-    }
-  }
-
-  get(
-    projectId: string,
-    instanceId: string,
-    signal?: AbortSignal,
-  ): Promise<Record<string, unknown>> {
-    return this.pulumiProjectHost.runLocal(
-      {
-        projectId,
-        pulumiProjectName: this.projectName,
-        pulumiStackName: projectId,
-        projectPath: this.projectPath,
-      },
-      async stack => {
-        this.logger.debug({ projectId, instanceId }, "getting secrets")
-
-        const config = await stack.getAllConfig()
-        signal?.throwIfAborted()
-
-        const prefix = this.getPrefix(projectId, instanceId)
-        const secrets = pickBy(config, (_, key) => key.startsWith(prefix))
-        const trimmedSecrets = mapKeys(secrets, key => key.slice(prefix.length))
-
-        return mapValues(trimmedSecrets, value => stringToValue(value.value))
-      },
-      signal,
-    )
-  }
-
-  set(
-    projectId: string,
-    instanceId: string,
-    values: Record<string, unknown>,
-    signal?: AbortSignal,
-  ): Promise<void> {
-    return this.pulumiProjectHost.runLocal(
-      {
-        projectId,
-        pulumiProjectName: this.projectName,
-        pulumiStackName: projectId,
-        projectPath: this.projectPath,
-      },
-      async stack => {
-        this.logger.debug({ projectId, instanceId }, "setting secrets")
-
-        const componentSecrets = mapValues(
-          mapKeys(values, key => `${this.getPrefix(projectId, instanceId)}${key}`),
-          value => ({
-            value: valueToString(value),
-            secret: true,
-          }),
-        )
-
-        const config = await stack.getAllConfig()
-        signal?.throwIfAborted()
-
-        Object.assign(config, componentSecrets)
-
-        await stack.setAllConfig(config)
-      },
-      signal,
-    )
-  }
-
-  private getPrefix(projectId: string, instanceId: string) {
-    // replace all semicolons with dashes since extra semicolons are not allowed in Pulumi config keys
-    instanceId = instanceId.replace(/:/g, "_")
-
-    return `${this.projectName}:${projectId}/${instanceId}/`
-  }
-
-  static async create(
-    config: z.infer<typeof localSecretBackendConfig>,
-    pulumiProjectHost: LocalPulumiHost,
-    logger: Logger,
-  ) {
-    const [projectPath, projectName] = await resolveMainLocalProject(
-      config.HIGHSTATE_BACKEND_SECRET_PROJECT_PATH,
-      config.HIGHSTATE_BACKEND_SECRET_PROJECT_NAME,
-    )
-
-    return new LocalSecretBackend(
-      projectPath,
-      projectName,
-      pulumiProjectHost,
-      logger.child({ backend: "SecretBackend", service: "LocalSecretBackend" }),
-    )
-  }
-}