@highstate/backend 0.7.2 → 0.7.4

package/src/secret/abstractions.ts

@@ -0,0 +1,59 @@
+export class SecretAccessDeniedError extends Error {
+  constructor(projectId: string, key: string) {
+    super(`Access to the secrets of component "${projectId}.${key}" is denied.`)
+  }
+}
+
+export interface SecretBackend {
+  /**
+   * Check if the state and secrets of the project are locked.
+   * The backend may not implement this method.
+   *
+   * @param projectId The ID of the project.
+   * @param signal The signal to abort the operation.
+   *
+   * @returns `true` if the project is locked, `false` if the project is unlocked.
+   */
+  isLocked?(projectId: string, signal?: AbortSignal): Promise<boolean>
+
+  /**
+   * Unlock the state and secrets of the project.
+   * The backend may not implement this method.
+   *
+   * @param projectId The ID of the project.
+   * @param password The password to unlock the secrets.
+   * @param signal The signal to abort the operation.
+   *
+   * @returns `true` if the project is unlocked, `false` if the password is incorrect.
+   */
+  unlock?(projectId: string, password: string, signal?: AbortSignal): Promise<boolean>
+
+  /**
+   * Get the secrets of the component.
+   *
+   * @param projectId The ID of the project.
+   * @param instanceId The ID of the instance.
+   * @param signal The signal to abort the operation.
+   *
+   * @returns A record of secret key-value pairs or `null` if the secrets are not found.
+   * @throws {SecretAccessDeniedError} If access to the secrets is denied.
+   */
+  get(projectId: string, instanceId: string, signal?: AbortSignal): Promise<Record<string, unknown>>
+
+  /**
+   * Set the secrets of the component.
+   *
+   * @param projectId The ID of the project.
+   * @param instanceId The ID of the instance.
+   * @param values The record of secret key-value pairs.
+   * @param signal The signal to abort the operation.
+   *
+   * @throws {SecretAccessDeniedError} If access to the secrets is denied.
+   */
+  set(
+    projectId: string,
+    instanceId: string,
+    values: Record<string, unknown>,
+    signal?: AbortSignal,
+  ): Promise<void>
+}

package/src/secret/factory.ts

@@ -0,0 +1,22 @@
+import type { SecretBackend } from "./abstractions"
+import type { LocalPulumiHost } from "../common"
+import type { Logger } from "pino"
+import { z } from "zod"
+import { LocalSecretBackend, localSecretBackendConfig } from "./local"
+
+export const secretBackendConfig = z.object({
+  HIGHSTATE_BACKEND_SECRET_TYPE: z.enum(["local"]).default("local"),
+  ...localSecretBackendConfig.shape,
+})
+
+export function createSecretBackend(
+  config: z.infer<typeof secretBackendConfig>,
+  localPulumiHost: LocalPulumiHost,
+  logger: Logger,
+): Promise<SecretBackend> {
+  switch (config.HIGHSTATE_BACKEND_SECRET_TYPE) {
+    case "local": {
+      return LocalSecretBackend.create(config, localPulumiHost, logger)
+    }
+  }
+}

package/src/secret/index.ts

@@ -0,0 +1,2 @@
+export * from "./abstractions"
+export * from "./factory"

package/src/secret/local.ts

@@ -0,0 +1,152 @@
+import type { SecretBackend } from "./abstractions"
+import type { Logger } from "pino"
+import { mapKeys, mapValues, pickBy } from "remeda"
+import { z } from "zod"
+import { LocalPulumiHost, resolveMainLocalProject, stringToValue, valueToString } from "../common"
+
+export const localSecretBackendConfig = z.object({
+  HIGHSTATE_BACKEND_SECRET_PROJECT_PATH: z.string().optional(),
+  HIGHSTATE_BACKEND_SECRET_PROJECT_NAME: z.string().optional(),
+})
+
+/**
+ * The backend for storing secrets in the local Pulumi project.
+ * It is the simplest backend and should be used for development and single-user projects.
+ */
+export class LocalSecretBackend implements SecretBackend {
+  private constructor(
+    private readonly projectPath: string,
+    private readonly projectName: string,
+    private readonly pulumiProjectHost: LocalPulumiHost,
+    private readonly logger: Logger,
+  ) {
+    this.logger.debug({ msg: "initialized", projectPath, projectName })
+  }
+
+  isLocked(projectId: string): Promise<boolean> {
+    return Promise.resolve(!this.pulumiProjectHost.hasPassword(projectId))
+  }
+
+  async unlock(projectId: string, password: string, signal?: AbortSignal): Promise<boolean> {
+    this.pulumiProjectHost.setPassword(projectId, password)
+
+    try {
+      // try to run a command to check if the password is correct
+      await this.pulumiProjectHost.runLocal(
+        {
+          projectId,
+          pulumiProjectName: this.projectName,
+          pulumiStackName: projectId,
+          projectPath: this.projectPath,
+        },
+        async stack => {
+          this.logger.debug({ projectId }, "checking password")
+
+          await stack.info(true)
+        },
+        signal,
+      )
+
+      return true
+    } catch (error) {
+      if (error instanceof Error) {
+        if (error.message.includes("incorrect passphrase")) {
+          this.logger.debug({ projectId, error }, "incorrect passphrase")
+
+          this.pulumiProjectHost.removePassword(projectId)
+          return false
+        }
+      }
+
+      throw error
+    }
+  }
+
+  get(
+    projectId: string,
+    instanceId: string,
+    signal?: AbortSignal,
+  ): Promise<Record<string, unknown>> {
+    return this.pulumiProjectHost.runLocal(
+      {
+        projectId,
+        pulumiProjectName: this.projectName,
+        pulumiStackName: projectId,
+        projectPath: this.projectPath,
+      },
+      async stack => {
+        this.logger.debug({ projectId, instanceId }, "getting secrets")
+
+        const config = await stack.getAllConfig()
+        signal?.throwIfAborted()
+
+        const prefix = this.getPrefix(projectId, instanceId)
+        const secrets = pickBy(config, (_, key) => key.startsWith(prefix))
+        const trimmedSecrets = mapKeys(secrets, key => key.slice(prefix.length))
+
+        return mapValues(trimmedSecrets, value => stringToValue(value.value))
+      },
+      signal,
+    )
+  }
+
+  set(
+    projectId: string,
+    instanceId: string,
+    values: Record<string, unknown>,
+    signal?: AbortSignal,
+  ): Promise<void> {
+    return this.pulumiProjectHost.runLocal(
+      {
+        projectId,
+        pulumiProjectName: this.projectName,
+        pulumiStackName: projectId,
+        projectPath: this.projectPath,
+      },
+      async stack => {
+        this.logger.debug({ projectId, instanceId }, "setting secrets")
+
+        const componentSecrets = mapValues(
+          mapKeys(values, key => `${this.getPrefix(projectId, instanceId)}${key}`),
+          value => ({
+            value: valueToString(value),
+            secret: true,
+          }),
+        )
+
+        const config = await stack.getAllConfig()
+        signal?.throwIfAborted()
+
+        Object.assign(config, componentSecrets)
+
+        await stack.setAllConfig(config)
+      },
+      signal,
+    )
+  }
+
+  private getPrefix(projectId: string, instanceId: string) {
+    // replace all semicolons with dashes since extra semicolons are not allowed in Pulumi config keys
+    instanceId = instanceId.replace(/:/g, "_")
+
+    return `${this.projectName}:${projectId}/${instanceId}/`
+  }
+
+  static async create(
+    config: z.infer<typeof localSecretBackendConfig>,
+    pulumiProjectHost: LocalPulumiHost,
+    logger: Logger,
+  ) {
+    const [projectPath, projectName] = await resolveMainLocalProject(
+      config.HIGHSTATE_BACKEND_SECRET_PROJECT_PATH,
+      config.HIGHSTATE_BACKEND_SECRET_PROJECT_NAME,
+    )
+
+    return new LocalSecretBackend(
+      projectPath,
+      projectName,
+      pulumiProjectHost,
+      logger.child({ backend: "SecretBackend", service: "LocalSecretBackend" }),
+    )
+  }
+}

package/src/services.ts

@@ -0,0 +1,133 @@
+import type { RunnerBackend } from "./runner"
+import { pino, type Logger } from "pino"
+import { createWorkspaceBackend, type WorkspaceBackend } from "./workspace"
+import { createStateBackend, StateManager, type StateBackend } from "./state"
+import { type Config, loadConfig } from "./config"
+import { type LibraryBackend, createLibraryBackend } from "./library"
+import { OperationManager } from "./orchestrator"
+import {
+  type ProjectBackend,
+  ProjectLockManager,
+  ProjectManager,
+  createProjectBackend,
+} from "./project"
+import { createRunnerBackend } from "./runner"
+import { type SecretBackend, createSecretBackend } from "./secret"
+import { type TerminalBackend, TerminalManager, createTerminalBackend } from "./terminal"
+import { LocalPulumiHost } from "./common"
+
+export type Services = {
+  readonly logger: Logger
+  readonly libraryBackend: LibraryBackend
+  readonly secretBackend: SecretBackend
+  readonly runnerBackend: RunnerBackend
+  readonly projectBackend: ProjectBackend
+  readonly projectManager: ProjectManager
+  readonly stateBackend: StateBackend
+  readonly stateManager: StateManager
+  readonly operationManager: OperationManager
+  readonly terminalBackend: TerminalBackend
+  readonly terminalManager: TerminalManager
+  readonly workspaceBackend: WorkspaceBackend
+}
+
+export interface CreateServicesOptions {
+  /**
+   * The config to use. If not provided, it will be loaded from the environment.
+   */
+  readonly config?: Config
+
+  /**
+   * The already created services to use. If the particular service is not provided, it will be created.
+   */
+  readonly services?: Partial<Services>
+}
+
+export async function createServices({
+  config,
+  services: {
+    logger,
+    libraryBackend,
+    secretBackend,
+    runnerBackend,
+    projectBackend,
+    projectManager,
+    stateBackend,
+    stateManager,
+    operationManager,
+    terminalBackend,
+    terminalManager,
+    workspaceBackend,
+  } = {},
+}: CreateServicesOptions = {}): Promise<Services> {
+  config ??= await loadConfig()
+
+  logger ??= pino({
+    name: config.HIGHSTATE_BACKEND_LOGGER_NAME,
+    level: config.HIGHSTATE_BACKEND_LOGGER_LEVEL,
+    errorKey: "error",
+  })
+
+  const localPulumiHost = LocalPulumiHost.create(logger)
+  const projectLockManager = new ProjectLockManager()
+
+  stateManager ??= new StateManager()
+
+  libraryBackend ??= await createLibraryBackend(config, logger)
+  secretBackend ??= await createSecretBackend(config, localPulumiHost, logger)
+  stateBackend ??= await createStateBackend(config, localPulumiHost, logger)
+  runnerBackend ??= createRunnerBackend(config, localPulumiHost, libraryBackend)
+  projectBackend ??= await createProjectBackend(config)
+
+  terminalBackend ??= createTerminalBackend(config, logger)
+  terminalManager ??= TerminalManager.create(terminalBackend, stateBackend, runnerBackend, logger)
+
+  operationManager ??= await OperationManager.create(
+    runnerBackend,
+    stateBackend,
+    libraryBackend,
+    projectBackend,
+    secretBackend,
+    projectLockManager,
+    stateManager,
+    logger,
+  )
+
+  projectManager ??= ProjectManager.create(
+    projectBackend,
+    stateBackend,
+    libraryBackend,
+    projectLockManager,
+    stateManager,
+    logger,
+  )
+
+  workspaceBackend ??= await createWorkspaceBackend(config)
+
+  logger.info("services created")
+
+  return {
+    logger,
+    libraryBackend,
+    secretBackend,
+    runnerBackend,
+    projectBackend,
+    projectManager,
+    stateBackend,
+    stateManager,
+    operationManager,
+    terminalBackend,
+    terminalManager,
+    workspaceBackend,
+  }
+}
+
+let sharedServicesPromise: Promise<Services> | undefined
+
+export function getSharedServices(options: CreateServicesOptions = {}): Promise<Services> {
+  if (!sharedServicesPromise) {
+    sharedServicesPromise = createServices(options)
+  }
+
+  return sharedServicesPromise
+}

package/src/shared/index.ts

@@ -0,0 +1,10 @@
+export * from "./project"
+export * from "./state"
+export * from "./operation"
+export * from "./resolvers/graph-resolver"
+export * from "./resolvers/registry"
+export * from "./resolvers/input"
+export * from "./resolvers/input-hash"
+export * from "./resolvers/validation"
+export * from "./terminal"
+export * from "./library"

package/src/shared/library.ts

@@ -0,0 +1,77 @@
+import type { ComponentModel, Entity } from "@highstate/contract"
+
+export type LibraryModel = {
+  components: Record<string, ComponentModel>
+  entities: Record<string, Entity>
+}
+
+export type LibraryUpdate =
+  | {
+      type: "component-updated"
+      component: ComponentModel
+    }
+  | {
+      type: "entity-updated"
+      entity: Entity
+    }
+  | {
+      type: "component-removed"
+      componentType: string
+    }
+  | {
+      type: "entity-removed"
+      entityType: string
+    }
+
+export function diffLibraries(oldLibrary: LibraryModel, newLibrary: LibraryModel): LibraryUpdate[] {
+  const updates: LibraryUpdate[] = []
+
+  for (const [componentType, newComponent] of Object.entries(newLibrary.components)) {
+    const existingComponent = oldLibrary.components[componentType]
+    if (existingComponent?.definitionHash !== newComponent.definitionHash) {
+      updates.push({ type: "component-updated", component: newComponent })
+    }
+  }
+
+  for (const componentType of Object.keys(oldLibrary.components)) {
+    if (!newLibrary.components[componentType]) {
+      updates.push({ type: "component-removed", componentType })
+    }
+  }
+
+  for (const [entityType, newEntity] of Object.entries(newLibrary.entities)) {
+    const existingEntity = oldLibrary.entities[entityType]
+    if (existingEntity?.definitionHash !== newEntity.definitionHash) {
+      updates.push({ type: "entity-updated", entity: newEntity })
+    }
+  }
+
+  for (const entityType of Object.keys(oldLibrary.entities)) {
+    if (!newLibrary.entities[entityType]) {
+      updates.push({ type: "entity-removed", entityType })
+    }
+  }
+
+  return updates
+}
+
+export function applyLibraryUpdate(
+  components: Record<string, ComponentModel>,
+  entities: Record<string, Entity>,
+  update: LibraryUpdate,
+): void {
+  switch (update.type) {
+    case "component-updated":
+      components[update.component.type] = update.component
+      break
+    case "entity-updated":
+      entities[update.entity.type] = update.entity
+      break
+    case "component-removed":
+      delete components[update.componentType]
+      break
+    case "entity-removed":
+      delete entities[update.entityType]
+      break
+  }
+}

package/src/shared/operation.ts

@@ -0,0 +1,85 @@
+import { z } from "zod"
+
+export const operationTypeSchema = z.enum(["update", "preview", "destroy", "recreate", "refresh"])
+
+export const operationStatusSchema = z.enum([
+  "pending",
+  "running",
+  "completed",
+  "failed",
+  "cancelled",
+])
+
+export const operationOptionsSchema = z.object({
+  /**
+   * Whether to force update all dependencies of the instances even if they are not changed.
+   *
+   * Only applicable for `update`, `preview`, `recreate`, and `refresh` operations.
+   * By default, `false`.
+   */
+  forceUpdateDependencies: z.boolean().default(false),
+
+  /**
+   * Whether to destroy all dependents of the instances when destroying them.
+   *
+   * Only applicable for `destroy`.
+   * By default, `true`.
+   */
+  destroyDependentInstances: z.boolean().default(true),
+
+  /**
+   * Whether to invoke destroy triggers when destroying the instances.
+   *
+   * Only applicable for `destroy`.
+   * By default, `true`.
+   */
+  invokeDestroyTriggers: z.boolean().default(true),
+
+  /**
+   * Whether to delete unreachable resources when updating or destroying the instances.
+   * This is potentially dangerous and should be used with caution.
+   *
+   * By default, `false`.
+   */
+  deleteUnreachableResources: z.boolean().default(false),
+
+  /**
+   * Whether to refresh the state before running the operation.
+   *
+   * By default, `false`.
+   */
+  refresh: z.boolean().default(false),
+})
+
+export const projectOperationRequestSchema = z.object({
+  projectId: z.string(),
+  type: operationTypeSchema,
+  instanceIds: z.array(z.string()),
+  options: operationOptionsSchema.partial().optional(),
+})
+
+export const projectOperationSchema = z.object({
+  id: z.string().uuid(),
+  status: operationStatusSchema,
+
+  projectId: z.string(),
+  type: operationTypeSchema,
+  instanceIds: z.array(z.string()),
+  affectedInstanceIds: z.array(z.string()).default(() => []),
+
+  options: operationOptionsSchema.default(() => ({})),
+
+  error: z.string().nullable(),
+  startedAt: z.number(),
+  completedAt: z.number().nullable(),
+})
+
+export type OperationType = z.infer<typeof operationTypeSchema>
+export type OperationStatus = z.infer<typeof operationStatusSchema>
+export type ProjectOperation = z.infer<typeof projectOperationSchema>
+export type ProjectOperationRequest = z.infer<typeof projectOperationRequestSchema>
+export type OperationOptions = z.infer<typeof operationOptionsSchema>
+
+export function isFinalOperationStatus(status: OperationStatus): boolean {
+  return status === "completed" || status === "failed" || status === "cancelled"
+}

package/src/shared/project.ts

@@ -0,0 +1,62 @@
+import { z } from "zod"
+
+export const positionSchema = z.object({
+  x: z.number(),
+  y: z.number(),
+})
+
+export const instanceInputSchema = z.object({
+  instanceId: z.string(),
+  output: z.string(),
+})
+
+export const hubInstanceInputSchema = z.object({
+  hubId: z.string(),
+})
+
+export const instanceModelPatchSchema = z.object({
+  args: z.record(z.unknown()).optional(),
+  inputs: z.record(z.array(instanceInputSchema)).optional(),
+  hubInputs: z.record(z.array(hubInstanceInputSchema)).optional(),
+  injectionInputs: z.array(hubInstanceInputSchema).optional(),
+  position: positionSchema.optional(),
+})
+
+export const instanceModelSchema = z.object({
+  id: z.string(),
+  type: z.string(),
+  name: z.string(),
+
+  ...instanceModelPatchSchema.shape,
+  resolvedInputs: z.record(z.array(instanceInputSchema)).optional(),
+
+  parentId: z.string().optional(),
+  outputs: z.record(z.array(instanceInputSchema)).optional(),
+  resolvedOutputs: z.record(z.array(instanceInputSchema)).optional(),
+})
+
+export const compositeInstanceSchema = z.object({
+  instance: instanceModelSchema,
+  children: z.array(instanceModelSchema),
+  inputHash: z.string().optional(),
+})
+
+export type CompositeInstance = z.infer<typeof compositeInstanceSchema>
+
+export const hubModelPatchSchema = z.object({
+  position: positionSchema.optional(),
+  inputs: z.array(instanceInputSchema).optional(),
+  injectionInputs: z.array(hubInstanceInputSchema).optional(),
+})
+
+export const hubModelSchema = z.object({
+  id: z.string().nanoid(),
+  position: positionSchema,
+
+  inputs: z.array(instanceInputSchema).optional(),
+  injectionInputs: z.array(hubInstanceInputSchema).optional(),
+})
+
+export type InstanceModelPatch = z.infer<typeof instanceModelPatchSchema>
+export type HubModel = z.infer<typeof hubModelSchema>
+export type HubModelPatch = z.infer<typeof hubModelPatchSchema>