@highstate/backend 0.7.2 → 0.7.3

package/src/common/pulumi.ts

@@ -0,0 +1,230 @@
+import type { ConfigMap, OpMap, OpType, Stack, WhoAmIResult } from "@pulumi/pulumi/automation"
+import type { Logger } from "pino"
+import { BetterLock } from "better-lock"
+import { AbortError, runWithRetryOnError } from "./utils"
+
+export type RunOptions = {
+  projectId: string
+  pulumiProjectName: string
+  pulumiStackName: string
+  envVars?: Record<string, string>
+}
+
+export type RunLocalOptions = RunOptions & {
+  projectPath: string
+  stackConfig?: ConfigMap
+}
+
+export class LocalPulumiHost {
+  private lock = new BetterLock()
+
+  private constructor(private readonly logger: Logger) {}
+
+  async getCurrentUser(): Promise<WhoAmIResult | null> {
+    const { LocalWorkspace } = await import("@pulumi/pulumi/automation/index.js")
+    const workspace = await LocalWorkspace.create({})
+
+    try {
+      return await workspace.whoAmI()
+    } catch (error) {
+      this.logger.error({ msg: "failed to get current user", error })
+
+      return null
+    }
+  }
+
+  async runEmpty<T>(
+    options: RunOptions,
+    fn: (stack: Stack) => Promise<T>,
+    signal?: AbortSignal,
+  ): Promise<T> {
+    const { projectId, pulumiProjectName, pulumiStackName, envVars } = options
+
+    return await this.lock.acquire(`${pulumiProjectName}.${pulumiStackName}`, async () => {
+      const { LocalWorkspace } = await import("@pulumi/pulumi/automation/index.js")
+
+      const stack = await LocalWorkspace.createOrSelectStack(
+        {
+          projectName: pulumiProjectName,
+          stackName: pulumiStackName,
+          program: () => Promise.resolve(),
+        },
+        {
+          projectSettings: {
+            name: pulumiProjectName,
+            runtime: "nodejs",
+          },
+          envVars: {
+            PULUMI_CONFIG_PASSPHRASE: this.getPassword(projectId),
+            PULUMI_K8S_AWAIT_ALL: "true",
+            ...envVars,
+          },
+        },
+      )
+
+      signal?.throwIfAborted()
+
+      try {
+        return await runWithRetryOnError(
+          () => fn(stack),
+          error => this.tryUnlockStack(stack, error),
+        )
+      } catch (e) {
+        if (e instanceof Error && e.message.includes("canceled")) {
+          throw new AbortError()
+        }
+
+        throw e
+      }
+    })
+  }
+
+  async runLocal<T>(
+    options: RunLocalOptions,
+    fn: (stack: Stack) => Promise<T>,
+    signal?: AbortSignal,
+  ): Promise<T> {
+    const { projectId, pulumiProjectName, pulumiStackName, projectPath, stackConfig, envVars } =
+      options
+
+    return await this.lock.acquire(`${pulumiProjectName}.${pulumiStackName}`, async () => {
+      const { LocalWorkspace } = await import("@pulumi/pulumi/automation/index.js")
+
+      const stack = await LocalWorkspace.createOrSelectStack(
+        {
+          stackName: pulumiStackName,
+          workDir: projectPath,
+        },
+        {
+          projectSettings: {
+            name: pulumiProjectName,
+            runtime: "nodejs",
+          },
+          stackSettings: stackConfig
+            ? {
+                [pulumiStackName]: {
+                  config: stackConfig,
+                },
+              }
+            : undefined,
+          envVars: {
+            PULUMI_CONFIG_PASSPHRASE: this.getPassword(projectId),
+            PULUMI_K8S_AWAIT_ALL: "true",
+            ...envVars,
+          },
+        },
+      )
+
+      signal?.throwIfAborted()
+
+      try {
+        return await runWithRetryOnError(
+          () => fn(stack),
+          error => this.tryUnlockStack(stack, error),
+        )
+      } catch (e) {
+        if (e instanceof Error && e.message.includes("canceled")) {
+          throw new AbortError()
+        }
+
+        throw e
+      }
+    })
+  }
+
+  private sharedPassword: string = process.env.PULUMI_CONFIG_PASSPHRASE ?? ""
+  private passwords = new Map<string, string>()
+
+  hasPassword(projectId: string) {
+    return !!this.sharedPassword || this.passwords.has(projectId)
+  }
+
+  setPassword(projectId: string, password: string) {
+    this.passwords.set(projectId, password)
+  }
+
+  removePassword(projectId: string) {
+    this.passwords.delete(projectId)
+  }
+
+  private getPassword(projectId: string) {
+    return this.sharedPassword || this.passwords.get(projectId) || ""
+  }
+
+  async tryUnlockStack(stack: Stack, error: unknown) {
+    if (error instanceof Error && error.message.includes("the stack is currently locked")) {
+      // TODO: kill the process if the hostname matches the current hostname
+
+      this.logger.warn({ stackName: stack.name }, "inlocking stack")
+      await stack.cancel()
+      return true
+    }
+
+    return false
+  }
+
+  static create(logger: Logger) {
+    return new LocalPulumiHost(logger.child({ service: "LocalPulumiHost" }))
+  }
+}
+
+export function valueToString(value: unknown): string {
+  if (typeof value === "string") {
+    return value
+  }
+
+  return JSON.stringify(value)
+}
+
+export function stringToValue(value: string): unknown {
+  try {
+    return JSON.parse(value)
+  } catch {
+    return value
+  }
+}
+
+export function updateResourceCount(opType: OpType, currentCount: number): number {
+  switch (opType) {
+    case "same":
+    case "create":
+    case "update":
+    case "replace":
+    case "create-replacement":
+    case "import":
+    case "import-replacement":
+      return currentCount + 1
+
+    case "delete":
+    case "delete-replaced":
+    case "discard":
+    case "discard-replaced":
+    case "remove-pending-replace":
+      return currentCount - 1
+
+    case "refresh":
+    case "read-replacement":
+    case "read":
+      return currentCount
+
+    default:
+      throw new Error(`Unknown operation type: ${opType as string}`)
+  }
+}
+
+export function calculateTotalResources(opMap: OpMap | undefined): number {
+  if (!opMap) {
+    return 0 // No operations imply no resources
+  }
+
+  let total = 0
+
+  for (const [op, count] of Object.entries(opMap)) {
+    const opType = op as OpType
+    const value = count ?? 0
+
+    total = updateResourceCount(opType, value)
+  }
+
+  return total
+}

package/src/common/utils.ts

@@ -0,0 +1,137 @@
+import { z } from "zod"
+
+export async function runWithRetryOnError<T>(
+  runner: () => T | Promise<T>,
+  tryHandleError: (error: unknown) => boolean | Promise<boolean>,
+  maxRetries: number = 1,
+): Promise<T> {
+  let lastError: unknown
+
+  for (let i = 0; i < maxRetries + 1; i++) {
+    try {
+      return await runner()
+    } catch (e) {
+      lastError = e
+
+      if (await tryHandleError(e)) {
+        continue
+      }
+
+      throw e
+    }
+  }
+
+  throw lastError
+}
+
+export class AbortError extends Error {
+  constructor(options?: ErrorOptions) {
+    super("Operation aborted", options)
+  }
+}
+
+export function isAbortError(error: unknown): boolean {
+  return error instanceof Error && error.name === "AbortError"
+}
+
+const abortMessagePatterns = ["Operation aborted", "Command was killed with SIGINT"]
+
+export function isAbortErrorLike(error: unknown): boolean {
+  if (error instanceof Error) {
+    return abortMessagePatterns.some(pattern => error.message.includes(pattern))
+  }
+
+  return false
+}
+
+export function tryWrapAbortErrorLike(error: unknown): unknown {
+  if (isAbortErrorLike(error)) {
+    return new AbortError({ cause: error })
+  }
+
+  return error
+}
+
+export const stringArrayType = z.string().transform(args => args.split(",").map(arg => arg.trim()))
+
+export function errorToString(error: unknown): string {
+  if (error instanceof Error) {
+    return error.stack || error.message
+  }
+
+  return JSON.stringify(error)
+}
+
+export type AsyncBatcherOptions = {
+  waitMs?: number
+  maxWaitTimeMs?: number
+}
+
+export function createAsyncBatcher<T>(
+  fn: (items: T[]) => Promise<void>,
+  { waitMs = 100, maxWaitTimeMs = 1000 }: AsyncBatcherOptions = {},
+) {
+  let batch: T[] = []
+  let activeTimeout: NodeJS.Timeout | null = null
+  let maxWaitTimeout: NodeJS.Timeout | null = null
+  let firstCallTimestamp: number | null = null
+
+  async function processBatch() {
+    if (batch.length === 0) return
+
+    const currentBatch = batch
+    batch = [] // Reset batch before async call
+
+    await fn(currentBatch)
+
+    // Clear max wait timer since batch has been processed
+    if (maxWaitTimeout) {
+      clearTimeout(maxWaitTimeout)
+      maxWaitTimeout = null
+    }
+    firstCallTimestamp = null
+  }
+
+  function schedule() {
+    if (activeTimeout) clearTimeout(activeTimeout)
+    activeTimeout = setTimeout(() => {
+      activeTimeout = null
+      void processBatch()
+    }, waitMs)
+
+    // Ensure batch is executed within maxWaitTimeMs
+    if (!firstCallTimestamp) {
+      firstCallTimestamp = Date.now()
+      maxWaitTimeout = setTimeout(() => {
+        if (activeTimeout) clearTimeout(activeTimeout)
+        activeTimeout = null
+        void processBatch()
+      }, maxWaitTimeMs)
+    }
+  }
+
+  return {
+    /**
+     * Add an item to the batch.
+     */
+    call(item: T): void {
+      batch.push(item)
+      schedule()
+    },
+
+    /**
+     * Immediately flush the pending batch (if any).
+     */
+    async flush(): Promise<void> {
+      if (activeTimeout) {
+        clearTimeout(activeTimeout)
+        activeTimeout = null
+      }
+      if (maxWaitTimeout) {
+        clearTimeout(maxWaitTimeout)
+        maxWaitTimeout = null
+      }
+      await processBatch()
+    },
+  }
+}

package/src/config.ts

@@ -0,0 +1,40 @@
+import { z } from "zod"
+import { libraryBackendConfig } from "./library"
+import { projectBackendConfig } from "./project"
+import { secretBackendConfig } from "./secret"
+import { terminalBackendConfig } from "./terminal"
+import { runnerBackendConfig } from "./runner"
+import { stateBackendConfig } from "./state"
+import { workspaceBackendConfig } from "./workspace"
+
+const loggerConfig = z.object({
+  HIGHSTATE_BACKEND_LOGGER_NAME: z.string().default("highstate-backend"),
+
+  HIGHSTATE_BACKEND_LOGGER_LEVEL: z
+    .enum(["fatal", "error", "warn", "info", "debug", "trace"])
+    .default("info"),
+})
+
+const configSchema = z.object({
+  ...libraryBackendConfig.shape,
+  ...projectBackendConfig.shape,
+  ...secretBackendConfig.shape,
+  ...stateBackendConfig.shape,
+  ...runnerBackendConfig.shape,
+  ...terminalBackendConfig.shape,
+  ...workspaceBackendConfig.shape,
+  ...loggerConfig.shape,
+})
+
+export type Config = z.infer<typeof configSchema>
+
+export async function loadConfig(
+  env: NodeJS.ProcessEnv = process.env,
+  useDotenv = true,
+): Promise<Config> {
+  if (useDotenv) {
+    await import("dotenv/config")
+  }
+
+  return configSchema.parse(env)
+}

package/src/index.ts

@@ -0,0 +1,6 @@
+export * from "./secret"
+export * from "./library"
+export * from "./config"
+export * from "./orchestrator"
+export * from "./terminal"
+export * from "./services"

package/src/library/abstractions.ts

@@ -0,0 +1,83 @@
+import type { InstanceModel, CompositeInstance } from "@highstate/contract"
+import type { LibraryModel, LibraryUpdate, ResolvedInstanceInput } from "../shared"
+
+export type ResolvedUnitSource = {
+  unitType: string
+  serializedSource: string
+  projectPath: string
+  packageJsonPath: string
+  allowedDependencies: string[]
+  sourceHash: string
+}
+
+export type ModuleEvaluationResult =
+  | {
+      success: true
+      compositeInstances: CompositeInstance[]
+    }
+  | {
+      success: false
+      modulePath: string
+      error: string
+    }
+
+export type InstanceEvaluationResult =
+  | {
+      success: true
+      instanceId: string
+      compositeInstances: CompositeInstance[]
+    }
+  | {
+      success: false
+      instanceId: string
+      error: string
+    }
+
+export interface LibraryBackend {
+  /**
+   * Loads the library.
+   */
+  loadLibrary(signal?: AbortSignal): Promise<LibraryModel>
+
+  /**
+   * Watches the library for changes.
+   */
+  watchLibrary(signal?: AbortSignal): AsyncIterable<LibraryUpdate[]>
+
+  /**
+   * Gets the resolved unit sources for all units in the library.
+   */
+  getResolvedUnitSources(): Promise<readonly ResolvedUnitSource[]>
+
+  /**
+   * Gets the resolved unit source for the specified unit type.
+   */
+  getResolvedUnitSource(unitType: string): Promise<ResolvedUnitSource | null>
+
+  /**
+   * Watches the resolved unit sources for changes.
+   * Returns an async iterable that emits each resolved unit source whenever it changes.
+   * Does not emit the resolved unit sources for units that have not changed even if the library was reloaded.
+   */
+  watchResolvedUnitSources(signal?: AbortSignal): AsyncIterable<ResolvedUnitSource>
+
+  /**
+   * Evaluates the instances and returns the evaluated composite instances.
+   *
+   * @param allInstances The all instances of the project.
+   * @param resolvedInputs The resolved inputs of the instances.
+   * @param instanceIds The instance ids to evaluate.
+   */
+  evaluateCompositeInstances(
+    allInstances: InstanceModel[],
+    resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
+    instanceIds: string[],
+  ): Promise<InstanceEvaluationResult[]>
+
+  /**
+   * Evaluates the modules and returns the evaluated instances.
+   *
+   * @param modulePaths The module paths to evaluate.
+   */
+  evaluateModules(modulePaths: string[]): Promise<ModuleEvaluationResult>
+}

package/src/library/factory.ts

@@ -0,0 +1,20 @@
+import type { LibraryBackend } from "./abstractions"
+import type { Logger } from "pino"
+import { z } from "zod"
+import { LocalLibraryBackend, localLibraryBackendConfig } from "./local"
+
+export const libraryBackendConfig = z.object({
+  HIGHSTATE_BACKEND_LIBRARY_TYPE: z.enum(["local"]).default("local"),
+  ...localLibraryBackendConfig.shape,
+})
+
+export async function createLibraryBackend(
+  config: z.infer<typeof libraryBackendConfig>,
+  logger: Logger,
+): Promise<LibraryBackend> {
+  switch (config.HIGHSTATE_BACKEND_LIBRARY_TYPE) {
+    case "local": {
+      return await LocalLibraryBackend.create(config, logger)
+    }
+  }
+}

package/src/library/index.ts

@@ -0,0 +1,2 @@
+export * from "./abstractions"
+export * from "./factory"