@highstate/backend 0.9.15 → 0.9.18

package/src/{common → runner}/pulumi.ts

@@ -1,7 +1,17 @@
-import type { ConfigMap, OpMap, OpType, Stack, WhoAmIResult } from "@pulumi/pulumi/automation"
 import type { Logger } from "pino"
+import type { StateManager } from "../state"
+import {
+  type ConfigMap,
+  type OpMap,
+  type OpType,
+  type Stack,
+  type WhoAmIResult,
+} from "@pulumi/pulumi/automation/index.js"
 import { BetterLock } from "better-lock"
-import { AbortError, runWithRetryOnError } from "./utils"
+import { v4 as uuidv4, v7 as uuidv7 } from "uuid"
+import { formatSecretDescriptor, WellKnownSecretDescriptors, type Secret } from "../shared"
+import { AbortError, errorToString, runWithRetryOnError } from "../common/utils"
+import { createForceAbortableCommand } from "./force-abort"
 
 export type RunOptions = {
   projectId: string
@@ -18,7 +28,10 @@ export type RunLocalOptions = RunOptions & {
 export class LocalPulumiHost {
   private lock = new BetterLock()
 
-  private constructor(private readonly logger: Logger) {}
+  private constructor(
+    private readonly stateManager: StateManager,
+    private readonly logger: Logger,
+  ) {}
 
   async getCurrentUser(): Promise<WhoAmIResult | null> {
     const { LocalWorkspace } = await import("@pulumi/pulumi/automation/index.js")
@@ -55,10 +68,11 @@ export class LocalPulumiHost {
             runtime: "nodejs",
           },
           envVars: {
-            PULUMI_CONFIG_PASSPHRASE: this.getPassword(projectId),
+            PULUMI_CONFIG_PASSPHRASE: await this.getPassword(projectId),
             PULUMI_K8S_AWAIT_ALL: "true",
             ...envVars,
           },
+          pulumiCommand: await createForceAbortableCommand(),
         },
       )
 
@@ -90,6 +104,8 @@ export class LocalPulumiHost {
     return await this.lock.acquire(`${pulumiProjectName}.${pulumiStackName}`, async () => {
       const { LocalWorkspace } = await import("@pulumi/pulumi/automation/index.js")
 
+      console.log({ projectPath })
+
       const stack = await LocalWorkspace.createOrSelectStack(
         {
           stackName: pulumiStackName,
@@ -99,6 +115,7 @@ export class LocalPulumiHost {
           projectSettings: {
             name: pulumiProjectName,
             runtime: "nodejs",
+            main: "index.js",
           },
           stackSettings: stackConfig
             ? {
@@ -108,10 +125,11 @@ export class LocalPulumiHost {
               }
             : undefined,
           envVars: {
-            PULUMI_CONFIG_PASSPHRASE: this.getPassword(projectId),
+            PULUMI_CONFIG_PASSPHRASE: await this.getPassword(projectId),
             PULUMI_K8S_AWAIT_ALL: "true",
             ...envVars,
           },
+          pulumiCommand: await createForceAbortableCommand(),
         },
       )
 
@@ -132,30 +150,50 @@ export class LocalPulumiHost {
     })
   }
 
-  private sharedPassword: string = process.env.PULUMI_CONFIG_PASSPHRASE ?? ""
-  private passwords = new Map<string, string>()
+  private async getPassword(projectId: string): Promise<string> {
+    return await this.lock.acquire(`pulumi-password.${projectId}`, async () => {
+      const secretRepo = this.stateManager.getSecretRepository(projectId)
+      const secretContentRepo = this.stateManager.getSecretContentRepository(projectId)
+      const indexRepo = this.stateManager.getSecretIndexRepository(projectId)
 
-  hasPassword(projectId: string) {
-    return !!this.sharedPassword || this.passwords.has(projectId)
-  }
+      const contentKey = formatSecretDescriptor(WellKnownSecretDescriptors.PulumiPassword)
+      const pulumiPasswordContent = await secretContentRepo.get(contentKey)
+      if (typeof pulumiPasswordContent === "string") {
+        return pulumiPasswordContent
+      }
 
-  setPassword(projectId: string, password: string) {
-    this.passwords.set(projectId, password)
-  }
+      this.logger.info({ projectId }, "generating new Pulumi password")
 
-  removePassword(projectId: string) {
-    this.passwords.delete(projectId)
-  }
+      const pulumiPassword = uuidv4()
+      const batch = this.stateManager.batch()
+
+      const pulumiPasswordSecret: Secret = {
+        id: uuidv7(),
+        descriptor: WellKnownSecretDescriptors.PulumiPassword,
+        meta: {
+          title: "Pulumi Password",
+          description: "The password used to encrypt the Pulumi state.",
+          icon: "devicon:pulumi",
+        },
+      }
 
-  private getPassword(projectId: string) {
-    return this.sharedPassword || this.passwords.get(projectId) || ""
+      await Promise.all([
+        secretRepo.putItem(pulumiPasswordSecret, batch),
+        secretContentRepo.put(contentKey, pulumiPassword, batch),
+        indexRepo.indexRepository.put(contentKey, pulumiPasswordSecret.id, batch),
+      ])
+
+      await batch.write()
+
+      return pulumiPassword
+    })
   }
 
   async tryUnlockStack(stack: Stack, error: unknown) {
     if (error instanceof Error && error.message.includes("the stack is currently locked")) {
       // TODO: kill the process if the hostname matches the current hostname
 
-      this.logger.warn({ stackName: stack.name }, "inlocking stack")
+      this.logger.warn({ stackName: stack.name }, "unlocking stack")
       await stack.cancel()
       return true
     }
@@ -163,24 +201,8 @@ export class LocalPulumiHost {
     return false
   }
 
-  static create(logger: Logger) {
-    return new LocalPulumiHost(logger.child({ service: "LocalPulumiHost" }))
-  }
-}
-
-export function valueToString(value: unknown): string {
-  if (typeof value === "string") {
-    return value
-  }
-
-  return JSON.stringify(value)
-}
-
-export function stringToValue(value: string): unknown {
-  try {
-    return JSON.parse(value)
-  } catch {
-    return value
+  static create(stateManager: StateManager, logger: Logger) {
+    return new LocalPulumiHost(stateManager, logger.child({ service: "LocalPulumiHost" }))
   }
 }
 
@@ -228,3 +250,33 @@ export function calculateTotalResources(opMap: OpMap | undefined): number {
 
   return total
 }
+
+export async function pulumiErrorToString(error: unknown): Promise<string> {
+  const { CommandError } = await import("@pulumi/pulumi/automation/index.js")
+
+  if (error instanceof CommandError) {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    const stderr = error["commandResult"].stderr as string
+    let diagnosticsStart = stderr.indexOf("\u001b[38;5;13m\u001b[1mDiagnostics:")
+
+    if (diagnosticsStart === -1) {
+      diagnosticsStart = stderr.indexOf("Diagnostics:")
+    }
+
+    if (diagnosticsStart !== -1) {
+      return stderr.slice(diagnosticsStart)
+    }
+  }
+
+  return errorToString(error)
+}
+
+export function getOperationSummary(stdout: string): string | null {
+  const summaryStart = stdout.indexOf("\u001b[38;5;13m\u001b[1mOutputs:")
+
+  if (summaryStart !== -1) {
+    return stdout.slice(summaryStart)
+  }
+
+  return null
+}

package/src/services.ts

@@ -1,34 +1,77 @@
 import type { RunnerBackend } from "./runner"
 import { pino, type Logger } from "pino"
-import { createWorkspaceBackend, type WorkspaceBackend } from "./workspace"
+import { MemoryProjectUnlockBackend, type ProjectUnlockBackend } from "./unlock"
+import { createWorkerBackend, WorkerManager, type WorkerBackend } from "./worker"
+import { createHotStateBackend, HotStateManager, type HotStateBackend } from "./hotstate"
+import { createPubSubBackend, PubSubManager, type PubSubBackend } from "./pubsub"
+import { createLockBackend, type LockBackend, LockManager } from "./lock"
 import { createStateBackend, StateManager, type StateBackend } from "./state"
 import { type Config, loadConfig } from "./config"
 import { type LibraryBackend, createLibraryBackend } from "./library"
 import { OperationManager } from "./orchestrator"
-import {
-  type ProjectBackend,
-  ProjectLockManager,
-  ProjectManager,
-  createProjectBackend,
-} from "./project"
+import { type ProjectBackend, ProjectEvaluationSubsystem, createProjectBackend } from "./project"
 import { createRunnerBackend } from "./runner"
-import { type SecretBackend, createSecretBackend } from "./secret"
 import { type TerminalBackend, TerminalManager, createTerminalBackend } from "./terminal"
-import { LocalPulumiHost } from "./common"
+import { type ArtifactBackend, createArtifactBackend, ArtifactService } from "./artifact"
+import {
+  InstanceLockService,
+  OperationService,
+  SecretService,
+  ProjectUnlockService,
+  InstanceStateService,
+  ApiKeyService,
+  WorkerService,
+  ProjectService,
+} from "./business"
+import {
+  CryptoRandomProvider,
+  SystemClockProvider,
+  type ClockProvider,
+  type RandomProvider,
+} from "./common"
 
 export type Services = {
+  readonly clock: ClockProvider
+  readonly random: RandomProvider
   readonly logger: Logger
+
+  readonly projectUnlockBackend: ProjectUnlockBackend
+
+  readonly hotStateBackend: HotStateBackend
+  readonly hotStateManager: HotStateManager
+
+  readonly pubsubBackend: PubSubBackend
+  readonly pubsubManager: PubSubManager
+
+  readonly lockBackend: LockBackend
+  readonly lockManager: LockManager
+
   readonly libraryBackend: LibraryBackend
-  readonly secretBackend: SecretBackend
   readonly runnerBackend: RunnerBackend
   readonly projectBackend: ProjectBackend
-  readonly projectManager: ProjectManager
+  readonly projectEvaluationSubsystem: ProjectEvaluationSubsystem
   readonly stateBackend: StateBackend
   readonly stateManager: StateManager
   readonly operationManager: OperationManager
+
   readonly terminalBackend: TerminalBackend
   readonly terminalManager: TerminalManager
-  readonly workspaceBackend: WorkspaceBackend
+
+  readonly workerBackend: WorkerBackend
+  readonly workerManager: WorkerManager
+
+  readonly artifactBackend: ArtifactBackend
+  readonly artifactManager: ArtifactService
+
+  // business services
+  readonly instanceLockService: InstanceLockService
+  readonly projectUnlockService: ProjectUnlockService
+  readonly operationService: OperationService
+  readonly instanceStateService: InstanceStateService
+  readonly secretService: SecretService
+  readonly apiKeyService: ApiKeyService
+  readonly workerService: WorkerService
+  readonly projectService: ProjectService
 }
 
 export interface CreateServicesOptions {
@@ -46,79 +89,247 @@ export interface CreateServicesOptions {
 export async function createServices({
   config,
   services: {
+    clock,
+    random,
     logger,
+
+    projectUnlockBackend,
+
+    hotStateBackend,
+    hotStateManager,
+
+    pubsubBackend,
+    pubsubManager,
+
+    lockBackend,
+    lockManager,
+
     libraryBackend,
-    secretBackend,
     runnerBackend,
     projectBackend,
-    projectManager,
+    projectEvaluationSubsystem,
     stateBackend,
     stateManager,
     operationManager,
+
     terminalBackend,
     terminalManager,
-    workspaceBackend,
+
+    workerBackend,
+    workerManager,
+
+    artifactBackend,
+    artifactManager,
+
+    // business services
+    instanceLockService,
+    projectUnlockService,
+    operationService,
+    secretService,
+    instanceStateService,
+    apiKeyService,
+    workerService,
+    projectService,
   } = {},
 }: CreateServicesOptions = {}): Promise<Services> {
   config ??= await loadConfig()
 
-  logger ??= pino({
-    name: config.HIGHSTATE_BACKEND_LOGGER_NAME,
-    level: config.HIGHSTATE_BACKEND_LOGGER_LEVEL,
-    errorKey: "error",
-  })
+  clock ??= new SystemClockProvider()
+  random ??= new CryptoRandomProvider()
+  logger ??= pino({ level: config.HIGHSTATE_LOG_LEVEL, errorKey: "error" })
 
-  const localPulumiHost = LocalPulumiHost.create(logger)
-  const projectLockManager = new ProjectLockManager()
+  projectUnlockBackend ??= new MemoryProjectUnlockBackend()
 
-  stateManager ??= new StateManager()
+  pubsubBackend ??= createPubSubBackend(config, logger)
+  pubsubManager ??= new PubSubManager(pubsubBackend, logger)
+
+  lockBackend ??= createLockBackend(config)
+  lockManager ??= new LockManager(lockBackend)
+
+  hotStateBackend ??= createHotStateBackend(config)
+  hotStateManager ??= new HotStateManager(hotStateBackend)
+
+  stateBackend ??= await createStateBackend(config, logger)
+  stateManager ??= await StateManager.create(
+    config,
+    stateBackend,
+    projectUnlockBackend,
+    clock,
+    random,
+    logger,
+  )
 
   libraryBackend ??= await createLibraryBackend(config, logger)
-  secretBackend ??= await createSecretBackend(config, localPulumiHost, logger)
-  stateBackend ??= await createStateBackend(config, localPulumiHost, logger)
-  runnerBackend ??= createRunnerBackend(config, localPulumiHost, libraryBackend)
+
+  artifactBackend ??= await createArtifactBackend(config, stateManager, logger)
+  artifactManager ??= new ArtifactService(stateManager, artifactBackend, lockManager, logger)
+
+  runnerBackend ??= createRunnerBackend(
+    config,
+    libraryBackend,
+    artifactManager,
+    artifactBackend,
+    stateManager,
+    logger,
+  )
+
   projectBackend ??= await createProjectBackend(config)
 
-  terminalBackend ??= createTerminalBackend(config, logger)
-  terminalManager ??= TerminalManager.create(terminalBackend, stateBackend, runnerBackend, logger)
+  instanceLockService ??= new InstanceLockService(
+    stateManager,
+    lockManager,
+    pubsubManager,
+    logger.child({ service: "InstanceLockService" }),
+  )
+
+  projectUnlockService ??= new ProjectUnlockService(
+    stateManager,
+    hotStateManager,
+    pubsubManager,
+    projectUnlockBackend,
+    random,
+    logger.child({ service: "StateUnlockService" }),
+  )
+
+  operationService ??= new OperationService(
+    stateManager,
+    hotStateManager,
+    pubsubManager,
+    logger.child({ service: "OperationService" }),
+  )
 
-  operationManager ??= await OperationManager.create(
+  instanceStateService ??= new InstanceStateService(
+    stateManager,
+    hotStateManager,
+    pubsubManager,
+    lockManager,
     runnerBackend,
-    stateBackend,
+    artifactManager,
+    logger.child({ service: "InstanceService" }),
+  )
+
+  secretService ??= new SecretService(
+    stateManager,
     libraryBackend,
+    instanceStateService,
+    random,
+    logger.child({ service: "SecretService" }),
+  )
+
+  apiKeyService ??= new ApiKeyService(stateManager, lockManager)
+
+  projectEvaluationSubsystem ??= new ProjectEvaluationSubsystem(
     projectBackend,
-    secretBackend,
-    projectLockManager,
+    libraryBackend,
+    stateManager,
+    instanceLockService,
+    instanceStateService,
+    pubsubManager,
+    lockManager,
+    random,
+    logger,
+  )
+
+  terminalBackend ??= createTerminalBackend(config, logger)
+  terminalManager ??= TerminalManager.create(
+    terminalBackend,
+    stateManager,
+    pubsubManager,
+    projectUnlockService,
+    logger,
+  )
+
+  workerBackend ??= createWorkerBackend(config, logger)
+  workerManager ??= new WorkerManager(
+    config,
+    workerBackend,
     stateManager,
+    projectUnlockService,
+    lockManager,
+    apiKeyService,
     logger,
   )
 
-  projectManager ??= ProjectManager.create(
+  workerService ??= new WorkerService(
+    stateManager,
+    workerManager,
+    lockManager,
+    random,
+    logger.child({ service: "WorkerService" }),
+  )
+
+  projectService ??= new ProjectService(
+    stateManager,
+    projectUnlockService,
+    projectEvaluationSubsystem,
     projectBackend,
-    stateBackend,
     libraryBackend,
-    projectLockManager,
+    lockManager,
+    pubsubManager,
+    random,
+    logger.child({ service: "ProjectService" }),
+  )
+
+  operationManager ??= new OperationManager(
+    runnerBackend,
+    libraryBackend,
+    projectBackend,
+    artifactManager,
     stateManager,
+    instanceLockService,
+    projectUnlockService,
+    operationService,
+    secretService,
+    instanceStateService,
+    pubsubManager,
+    workerService,
     logger,
   )
 
-  workspaceBackend ??= await createWorkspaceBackend(config)
-
   logger.info("services created")
 
   return {
+    clock,
+    random,
     logger,
+
+    projectUnlockBackend,
+
+    hotStateBackend,
+    hotStateManager,
+
+    pubsubBackend,
+    pubsubManager,
+
+    lockBackend,
+    lockManager,
+
     libraryBackend,
-    secretBackend,
     runnerBackend,
     projectBackend,
-    projectManager,
+    projectEvaluationSubsystem,
     stateBackend,
     stateManager,
     operationManager,
+
     terminalBackend,
     terminalManager,
-    workspaceBackend,
+
+    workerBackend,
+    workerManager,
+
+    artifactBackend,
+    artifactManager,
+
+    // business services
+    instanceLockService,
+    projectUnlockService,
+    operationService,
+    instanceStateService,
+    secretService,
+    apiKeyService,
+    workerService,
+    projectService,
   }
 }
 

package/src/shared/index.ts

@@ -1,11 +1,3 @@
-export * from "./project"
-export * from "./state"
-export * from "./operation"
-export * from "./resolvers/graph-resolver"
-export * from "./resolvers/registry"
-export * from "./resolvers/input"
-export * from "./resolvers/input-hash"
-export * from "./resolvers/validation"
-export * from "./terminal"
-export * from "./library"
-export * from "./async-batcher"
+export * from "./models"
+export * from "./resolvers"
+export * from "./utils"

package/src/shared/models/backend/index.ts

@@ -0,0 +1,3 @@
+export * from "./project"
+export * from "./library"
+export * from "./unlock-method"

package/src/shared/{library.ts → models/backend/library.ts}

@@ -1,8 +1,8 @@
-import type { ComponentModel, Entity } from "@highstate/contract"
+import type { ComponentModel, EntityModel } from "@highstate/contract"
 
 export type LibraryModel = {
   components: Record<string, ComponentModel>
-  entities: Record<string, Entity>
+  entities: Record<string, EntityModel>
 }
 
 export type LibraryUpdate =
@@ -15,7 +15,7 @@ export type LibraryUpdate =
     }
   | {
       type: "entity-updated"
-      entity: Entity
+      entity: EntityModel
     }
   | {
       type: "component-removed"
@@ -60,7 +60,7 @@ export function diffLibraries(oldLibrary: LibraryModel, newLibrary: LibraryModel
 
 export function applyLibraryUpdate(
   components: Record<string, ComponentModel>,
-  entities: Record<string, Entity>,
+  entities: Record<string, EntityModel>,
   update: LibraryUpdate,
 ): void {
   switch (update.type) {

package/src/shared/models/backend/project.ts

@@ -0,0 +1,82 @@
+import { z } from "zod"
+import { hubModelSchema, instanceModelSchema, objectMetaSchema } from "@highstate/contract"
+
+export const projectSchema = z.object({
+  /**
+   * The identifier of the project unique across the world.
+   */
+  id: z.uuidv7(),
+
+  /**
+   * The metadata of the project.
+   */
+  meta: objectMetaSchema
+    .required({ title: true })
+    .pick({ title: true, description: true, icon: true, iconColor: true }),
+
+  /**
+   * The user-provided name of the project unique across its scope.
+   */
+  name: z
+    .string()
+    .min(1)
+    .max(64)
+    .regex(/^[a-z][a-z0-9-.]*$/),
+
+  /**
+   * The ID of the project scope (e.g., organization or user or something else not decided yet).
+   *
+   * For local deployments, this is `undefined`.
+   */
+  scopeId: z.uuidv7().optional(),
+
+  /**
+   * The identifier of the library used by the project.
+   *
+   * For local deployment is `undefined`, for cloud deployment will be the library ID.
+   */
+  libraryId: z.uuidv7().optional(),
+})
+
+export type Project = z.infer<typeof projectSchema>
+
+export const projectModelSchema = z.object({
+  instances: z.array(instanceModelSchema),
+  hubs: z.array(hubModelSchema),
+})
+
+export type ProjectModel = z.infer<typeof projectModelSchema>
+
+/**
+ * The project unlock suite is something that the frontend
+ * needs to unlock the project and provide the backend
+ * with the necessary information to decrypt the master key.
+ */
+export const projectUnlockSuiteSchema = z.object({
+  /**
+   * The list of encrypted AGE identities that can be used to decrypt the master key of the project.
+   *
+   * The frontend should try to decrypt at least one of these identities
+   * using the password or passkey.
+   */
+  encryptedIdentities: z.array(z.string()),
+
+  /**
+   * Whether one of the identities is a passkey and user should be asked to use it.
+   */
+  hasPasskey: z.boolean(),
+})
+
+export type ProjectUnlockSuite = z.infer<typeof projectUnlockSuiteSchema>
+
+export const projectUnlockStateSchema = z.discriminatedUnion("type", [
+  z.object({
+    type: z.literal("locked"),
+    unlockSuite: projectUnlockSuiteSchema.optional(),
+  }),
+  z.object({
+    type: z.literal("unlocked"),
+  }),
+])
+
+export type ProjectUnlockState = z.infer<typeof projectUnlockStateSchema>