@highstate/backend 0.9.18 → 0.9.19

package/src/database/manager.ts

@@ -0,0 +1,181 @@
+import type { Logger } from "pino"
+import type { ProjectUnlockBackend } from "../unlock"
+import type { BackendDatabase, ProjectDatabase } from "./prisma"
+import { LRUCache } from "lru-cache"
+import z from "zod"
+import { createProjectLogger } from "../common"
+import { BackendError, ProjectLockedError, ProjectNotFoundError } from "../shared"
+import { type ProjectDatabaseBackend, projectDatabaseVersion } from "./abstractions"
+import { migrateDatabase } from "./migrate"
+
+export const databaseManagerConfig = z.object({
+  HIGHSTATE_ENCRYPTION_ENABLED: z.stringbool().default(true),
+})
+
+export interface DatabaseManager {
+  /**
+   * The backend database instance.
+   */
+  readonly backend: BackendDatabase
+
+  /**
+   * Whether the encryption is enabled for the backend and project databases.
+   */
+  readonly isEncryptionEnabled: boolean
+
+  /**
+   * Returns the master key of the project with the given ID.
+   *
+   * If the project does not exist or is not unlocked, this will throw an error.
+   *
+   * Returns `undefined` if the encryption is not enabled.
+   *
+   * @param projectId The ID of the project to get the master key for.
+   * @returns A Promise that resolves to the master key of the project as a Uint8Array.
+   */
+  getProjectMasterKey(projectId: string): Promise<Buffer | undefined>
+
+  /**
+   * Sets up the database for the project with the given ID.
+   *
+   * Assumes that no database exists yet.
+   *
+   * The project must be unlocked before calling this method.
+   *
+   * @param projectId The ID of the project to set up the database for.
+   */
+  setupDatabase(projectId: string): Promise<ProjectDatabase>
+
+  /**
+   * Returns the database client for the project with the given ID.
+   *
+   * Automatically migrates the project database if necessary.
+   *
+   * If the project does not exist or not unlocked, this will throw an error.
+   *
+   * @param projectId The ID of the project to get the database client for.
+   */
+  forProject(projectId: string): Promise<ProjectDatabase>
+}
+
+export class DatabaseManagerImpl implements DatabaseManager {
+  constructor(
+    readonly backend: BackendDatabase,
+    private readonly projectUnlockBackend: ProjectUnlockBackend,
+    private readonly projectDatabaseBackend: ProjectDatabaseBackend,
+    private readonly config: z.infer<typeof databaseManagerConfig>,
+    private readonly logger: Logger,
+  ) {}
+
+  // store the master keys in memory cache for 30 seconds
+  private readonly projectMasterKeys = new LRUCache<string, Buffer>({
+    ttl: 30_000,
+    ttlAutopurge: true,
+  })
+
+  // TODO: auto unload project databases after some time
+  private readonly projectDatabases = new Map<string, ProjectDatabase>()
+
+  get isEncryptionEnabled(): boolean {
+    return this.config.HIGHSTATE_ENCRYPTION_ENABLED
+  }
+
+  async getProjectMasterKey(projectId: string): Promise<Buffer | undefined> {
+    if (!this.isEncryptionEnabled) {
+      return undefined
+    }
+
+    const cachedInfo = this.projectMasterKeys.get(projectId)
+    if (cachedInfo) {
+      return cachedInfo
+    }
+
+    const masterKey = await this.projectUnlockBackend.getProjectMasterKey(projectId)
+    if (!masterKey) {
+      throw new ProjectLockedError(projectId)
+    }
+
+    this.projectMasterKeys.set(projectId, masterKey)
+    return masterKey
+  }
+
+  async setupDatabase(projectId: string): Promise<ProjectDatabase> {
+    const logger = createProjectLogger(this.logger, projectId)
+    const masterKey = await this.getProjectMasterKey(projectId)
+    const hexMasterKey = masterKey?.toString("hex")
+
+    const [database, databaseUrl] = await this.projectDatabaseBackend.openProjectDatabase(
+      projectId,
+      hexMasterKey,
+    )
+
+    // can safely apply migrations here, because no one knows about the database yet
+    await migrateDatabase(databaseUrl, "project", hexMasterKey, logger)
+
+    this.projectDatabases.set(projectId, database)
+
+    return database
+  }
+
+  async forProject(projectId: string): Promise<ProjectDatabase> {
+    const cachedDatabase = this.projectDatabases.get(projectId)
+    if (cachedDatabase) {
+      return cachedDatabase
+    }
+
+    const masterKey = await this.getProjectMasterKey(projectId)
+    const hexMasterKey = masterKey?.toString("hex")
+
+    // TODO: is it really necessary to migrate the database inside the transaction?
+    let database = await this.backend.$transaction(async tx => {
+      const databaseEntity = await tx.project.findUnique({
+        where: { id: projectId },
+        select: { databaseVersion: true },
+      })
+
+      if (!databaseEntity) {
+        throw new ProjectNotFoundError(projectId)
+      }
+
+      if (databaseEntity.databaseVersion > projectDatabaseVersion) {
+        throw new BackendError(
+          `Project database version (${databaseEntity.databaseVersion}) is newer than expected (${projectDatabaseVersion}).`,
+        )
+      }
+
+      if (databaseEntity.databaseVersion === projectDatabaseVersion) {
+        return
+      }
+
+      const [database, databaseUrl] = await this.projectDatabaseBackend.openProjectDatabase(
+        projectId,
+        hexMasterKey,
+      )
+
+      if (databaseEntity.databaseVersion < projectDatabaseVersion) {
+        await migrateDatabase(databaseUrl, "project", hexMasterKey, this.logger)
+      }
+
+      await tx.project.update({
+        where: { id: projectId },
+        data: { databaseVersion: projectDatabaseVersion },
+      })
+
+      return database
+    })
+
+    if (!database) {
+      // open database if was not migrated in the transaction
+      const [_database] = await this.projectDatabaseBackend.openProjectDatabase(
+        projectId,
+        hexMasterKey,
+      )
+
+      database = _database
+    }
+
+    this.projectDatabases.set(projectId, database)
+
+    return database
+  }
+}

package/src/database/migrate.ts

@@ -0,0 +1,35 @@
+import type { Logger } from "pino"
+import { join } from "node:path"
+import { fileURLToPath } from "node:url"
+import { execa } from "execa"
+import { resolve } from "import-meta-resolve"
+import { detectPackageManager } from "nypm"
+
+export async function migrateDatabase(
+  databaseUrl: string,
+  schemaPath: "backend/sqlite" | "project",
+  masterKey: string | undefined,
+  logger: Logger,
+): Promise<void> {
+  logger.info("applying database migrations")
+
+  const backendIndexPath = resolve("@highstate/backend", import.meta.url)
+  const backendRootPath = join(fileURLToPath(backendIndexPath), "..")
+
+  const packageManager = await detectPackageManager(backendRootPath)
+  if (!packageManager) {
+    throw new Error("Could not detect package manager to run migrations")
+  }
+
+  const hasCorepack = await execa`"corepack" --version`.then(() => true).catch(() => false)
+  const command = hasCorepack ? `corepack ${packageManager.command}` : packageManager.command
+
+  await execa({
+    cwd: backendRootPath,
+    env: {
+      HIGHSTATE_MIGRATION_DATABASE_SCHEMA_PATH: schemaPath,
+      HIGHSTATE_MIGRATION_DATABASE_URL: databaseUrl,
+      HIGHSTATE_MIGRATION_DATABASE_ENCRYPTION_KEY: masterKey ?? "",
+    },
+  })`${command} migrate`
+}

package/src/database/prisma.ts

@@ -0,0 +1,56 @@
+import type * as runtime from "@prisma/client/runtime/client"
+import { PrismaClient as BackendDatabase } from "./_generated/backend/postgresql/client"
+import { PrismaClient as ProjectDatabase } from "./_generated/project/client"
+
+export type BackendTransaction = Omit<BackendDatabase, runtime.ITXClientDenyList>
+export type ProjectTransaction = Omit<ProjectDatabase, runtime.ITXClientDenyList>
+
+export type {
+  Library,
+  Project,
+  ProjectSpace,
+  PulumiBackend,
+} from "./_generated/backend/postgresql/client"
+export type {
+  ApiKey,
+  Artifact,
+  InstanceCustomStatus,
+  InstanceEvaluationState,
+  InstanceEvaluationStatus,
+  InstanceLock,
+  InstanceOperationState,
+  InstanceOperationStatus,
+  InstanceSource,
+  InstanceState,
+  InstanceStatus,
+  Operation,
+  OperationLog,
+  OperationStatus,
+  OperationType,
+  Page,
+  Secret,
+  ServiceAccount,
+  Terminal,
+  TerminalSession,
+  TerminalSessionLog,
+  TerminalStatus,
+  Trigger,
+  UnlockMethod,
+  Worker,
+  WorkerUnitRegistration,
+  WorkerVersion,
+  WorkerVersionLog,
+} from "./_generated/project/client"
+export type {
+  InstanceEvaluationStateUncheckedCreateInput,
+  InstanceEvaluationStateUpdateInput,
+  InstanceOperationStateCreateInput,
+  InstanceOperationStateCreateManyInput,
+  InstanceOperationStateUpdateInput,
+  InstanceStateInclude,
+  InstanceStateUpdateInput,
+  OperationUpdateInput,
+} from "./_generated/project/models"
+export { DbNull } from "./_generated/project/internal/prismaNamespace"
+
+export { BackendDatabase, ProjectDatabase }

package/src/database/well-known.ts

@@ -0,0 +1,38 @@
+import type { BackendDatabase } from "./prisma"
+import {
+  codebaseLibrary,
+  codebaseProjectModelStorage,
+  databaseProjectModelStorage,
+  globalProjectSpace,
+  hostPulumiBackend,
+} from "../shared/models/backend/well-known"
+
+export async function ensureWellKnownEntitiesCreated(database: BackendDatabase): Promise<void> {
+  await database.$transaction([
+    database.projectSpace.upsert({
+      where: { id: globalProjectSpace.id },
+      create: globalProjectSpace,
+      update: globalProjectSpace,
+    }),
+    database.library.upsert({
+      where: { id: codebaseLibrary.id },
+      create: codebaseLibrary,
+      update: codebaseLibrary,
+    }),
+    database.pulumiBackend.upsert({
+      where: { id: hostPulumiBackend.id },
+      create: hostPulumiBackend,
+      update: hostPulumiBackend,
+    }),
+    database.projectModelStorage.upsert({
+      where: { id: codebaseProjectModelStorage.id },
+      create: codebaseProjectModelStorage,
+      update: codebaseProjectModelStorage,
+    }),
+    database.projectModelStorage.upsert({
+      where: { id: databaseProjectModelStorage.id },
+      create: databaseProjectModelStorage,
+      update: databaseProjectModelStorage,
+    }),
+  ])
+}

package/src/index.ts

@@ -1,7 +1,7 @@
-export * from "./library"
+export * from "./business"
 export * from "./config"
+export * from "./database"
+export * from "./library"
 export * from "./orchestrator"
-export * from "./terminal"
 export * from "./services"
-export * from "./state"
-export * from "./business"
+export * from "./terminal"

package/src/library/abstractions.ts

@@ -1,4 +1,4 @@
-import type { InstanceModel } from "@highstate/contract"
+import type { InstanceId, InstanceModel } from "@highstate/contract"
 import type { LibraryModel, LibraryUpdate, ResolvedInstanceInput } from "../shared"
 
 export type ResolvedUnitSource = {
@@ -16,7 +16,7 @@ export type ProjectEvaluationResult =
       /**
        * The mapping of top-level composite instance IDs to error messages if any.
        */
-      topLevelErrors: Record<string, string>
+      topLevelErrors: Record<InstanceId, string>
     }
   | {
       success: false
@@ -58,17 +58,15 @@ export interface LibraryBackend {
   ): AsyncIterable<ResolvedUnitSource>
 
   /**
-   * Evaluates the instances and returns the evaluated composite instances.
+   * Evaluates the composite instances of the project and returns evaluated virtual instances.
    *
    * @param libraryId The library ID to use for evaluation.
    * @param allInstances The all instances of the project.
    * @param resolvedInputs The resolved inputs of the instances.
-   * @param instanceIds The instance ids to evaluate.
    */
   evaluateCompositeInstances(
     libraryId: string | undefined,
     allInstances: InstanceModel[],
     resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
-    instanceIds: string[],
   ): Promise<ProjectEvaluationResult>
 }

package/src/library/factory.ts

@@ -1,5 +1,5 @@
-import type { LibraryBackend } from "./abstractions"
 import type { Logger } from "pino"
+import type { LibraryBackend } from "./abstractions"
 import { z } from "zod"
 import { LocalLibraryBackend, localLibraryBackendConfig } from "./local"
 

package/src/library/local.ts

@@ -1,24 +1,24 @@
-import type { LibraryBackend, ProjectEvaluationResult, ResolvedUnitSource } from "./abstractions"
 import type { Logger } from "pino"
+import type { LibraryBackend, ProjectEvaluationResult, ResolvedUnitSource } from "./abstractions"
 import type {
   PackageResolutionResponse,
   PackageResolutionWorkerData,
 } from "./package-resolution-worker"
 import type { WorkerData } from "./worker/protocol"
-import { fileURLToPath } from "node:url"
 import { EventEmitter, on } from "node:events"
-import { Worker } from "node:worker_threads"
-import { resolve } from "node:path"
 import { readFile } from "node:fs/promises"
+import { resolve } from "node:path"
+import { fileURLToPath } from "node:url"
+import { Worker } from "node:worker_threads"
 import { type InstanceModel, isUnitModel } from "@highstate/contract"
-import Watcher from "watcher"
+import { decode } from "@msgpack/msgpack"
 import { BetterLock } from "better-lock"
 import { resolve as importMetaResolve } from "import-meta-resolve"
-import { z } from "zod"
+import { addDependency, installDependencies, runScript } from "nypm"
 import { readPackageJSON } from "pkg-types"
-import { runScript, installDependencies, addDependency } from "nypm"
 import { flatMap, groupBy, map, pipe, unique } from "remeda"
-import { decode } from "@msgpack/msgpack"
+import Watcher from "watcher"
+import { z } from "zod"
 import { resolveMainLocalProject, stringArrayType } from "../common"
 import {
   diffLibraries,
@@ -48,6 +48,11 @@ type LibraryPackage = {
   dependents: Set<string>
 }
 
+type RebuildState = {
+  inProgress: boolean
+  pending: boolean
+}
+
 export class LocalLibraryBackend implements LibraryBackend {
   private readonly watcher: Watcher
 
@@ -58,6 +63,7 @@ export class LocalLibraryBackend implements LibraryBackend {
 
   private readonly packages = new Map<string, LibraryPackage>()
   private readonly resolvedUnitSources = new Map<string, ResolvedUnitSource>()
+  private readonly rebuildStates = new Map<string, RebuildState>()
 
   private constructor(
     private readonly libraryPackages: string[],
@@ -67,7 +73,7 @@ export class LocalLibraryBackend implements LibraryBackend {
     this.watcher = new Watcher(watchPaths, {
       recursive: true,
       ignoreInitial: true,
-      ignore: /\.git|node_modules|dist/,
+      ignore: /\.git|node_modules|dist|\.highstate/,
     })
 
     this.watcher.on("all", (event: string, path: string) => {
@@ -148,15 +154,11 @@ export class LocalLibraryBackend implements LibraryBackend {
     _libraryId: string,
     allInstances: InstanceModel[],
     resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
-    instanceIds: string[],
   ): Promise<ProjectEvaluationResult> {
-    this.logger.info("evaluating %d composite instances", instanceIds.length)
-
     const worker = this.createLibraryWorker({
       libraryModulePaths: this.libraryPackages,
       allInstances,
       resolvedInputs,
-      instanceIds,
     })
 
     for await (const [event] of on(worker, "message", { signal: AbortSignal.timeout(10_000) })) {
@@ -260,7 +262,7 @@ export class LocalLibraryBackend implements LibraryBackend {
 
       this.resolvedUnitSources.set(unit.type, newResolvedSource)
       this.eventEmitter.emit("resolvedUnitSource", newResolvedSource)
-      this.logger.debug(`updated source for unit: "%s"`, unit.type)
+      this.logger.trace(`updated source for unit: "%s"`, unit.type)
     }
   }
 
@@ -468,22 +470,75 @@ export class LocalLibraryBackend implements LibraryBackend {
     }
   }
 
-  private async handleFileEvent(path: string): Promise<void> {
-    await this.lock.acquire(async () => {
-      const libraryPackage = this.packages.values().find(pkg => path.startsWith(pkg.rootPath))
+  private handleFileEvent(path: string): void {
+    const libraryPackage = this.packages.values().find(pkg => path.startsWith(pkg.rootPath))
 
-      if (!libraryPackage) {
-        return
-      }
+    if (!libraryPackage) {
+      return
+    }
 
-      const builtPackages = new Set<string>()
-      await this.rebuildLibraryPackage(libraryPackage, false, false, builtPackages)
+    void this.schedulePackageRebuild(libraryPackage.name)
+  }
+
+  private async schedulePackageRebuild(packageName: string): Promise<void> {
+    const state = this.rebuildStates.get(packageName) ?? { inProgress: false, pending: false }
+
+    // if both in progress and pending, discard this request
+    if (state.inProgress && state.pending) {
+      this.logger.debug(`rebuild in progress and already pending for "%s", discarding`, packageName)
+      return
+    }
+
+    // if in progress but no pending, mark as pending
+    if (state.inProgress && !state.pending) {
+      state.pending = true
+      this.rebuildStates.set(packageName, state)
+      this.logger.debug(`rebuild in progress, scheduling pending rebuild for "%s"`, packageName)
+      return
+    }
+
+    // not in progress, start immediately
+    await this.executePackageRebuild(packageName)
+  }
 
-      if (this.libraryPackages.some(pkg => builtPackages.has(pkg))) {
-        this.logger.info("reloading library due to file change in package: %s", libraryPackage.name)
-        await this.reloadLibrary()
+  private async executePackageRebuild(packageName: string): Promise<void> {
+    const state = this.rebuildStates.get(packageName) ?? { inProgress: false, pending: false }
+
+    do {
+      // mark as in progress, clear pending
+      state.inProgress = true
+      state.pending = false
+      this.rebuildStates.set(packageName, state)
+
+      try {
+        await this.lock.acquire(async () => {
+          const libraryPackage = this.packages.get(packageName)
+
+          if (!libraryPackage) {
+            this.logger.warn(`package not found for rebuild: "%s"`, packageName)
+            return
+          }
+
+          const builtPackages = new Set<string>()
+          await this.rebuildLibraryPackage(libraryPackage, false, false, builtPackages)
+
+          if (this.libraryPackages.some(pkg => builtPackages.has(pkg))) {
+            this.logger.info(
+              "reloading library due to file change in package: %s",
+              libraryPackage.name,
+            )
+            await this.reloadLibrary()
+          }
+        })
+      } finally {
+        state.inProgress = false
       }
-    })
+
+      // continue if another rebuild was requested while we were working
+    } while (state.pending)
+
+    // clean up if no longer needed
+    this.rebuildStates.delete(packageName)
   }
 
   private createLibraryWorker(workerData: WorkerData): Worker {

package/src/library/package-resolution-worker.ts

@@ -64,7 +64,7 @@ for (const packageName of packageNames) {
   }
 }
 
-parentPort!.postMessage({
+parentPort?.postMessage({
   type: "result",
   results,
 })

package/src/library/worker/evaluator.ts

@@ -14,20 +14,18 @@ export function evaluateProject(
   components: Readonly<Record<string, Component>>,
   allInstances: InstanceModel[],
   resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
-  instanceIds: string[],
 ): ProjectEvaluationResult {
-  const errors: Record<string, string> = {}
   const allInstancesMap = new Map(allInstances.map(instance => [instance.id, instance]))
+
+  const instanceErrors: Record<string, unknown> = {}
+  const topLevelErrors: Record<string, string> = {}
+
   const instanceOutputs = new Map<string, Record<string, unknown>>()
 
-  for (const instanceId of instanceIds ?? []) {
+  for (const instance of allInstances) {
     try {
-      logger.debug({ instanceId }, "evaluating top-level instance")
-
-      evaluateInstance(instanceId as InstanceModel["id"])
+      evaluateInstance(instance.id)
     } catch (error) {
-      errors[instanceId] = errorToString(error)
-
       if (error instanceof InstanceNameConflictError) {
         // fail the whole evaluation if there's a name conflict
         return {
@@ -40,35 +38,54 @@ export function evaluateProject(
 
   return {
     success: true,
+
     virtualInstances: getRuntimeInstances()
       .map(instance => instance.instance)
       // only include top-level composite instances and their children
-      .filter(instance => instanceIds.includes(instance.id) || !allInstancesMap.has(instance.id)),
-    topLevelErrors: errors,
+      .filter(instance => instance.kind === "composite" || !allInstancesMap.has(instance.id)),
+
+    topLevelErrors,
   }
 
   function evaluateInstance(instanceId: InstanceModel["id"]): Record<string, unknown> {
     let outputs = instanceOutputs.get(instanceId)
-
-    if (!outputs) {
-      outputs = _evaluateInstance(instanceId)
-      instanceOutputs.set(instanceId, outputs)
+    if (outputs) {
+      return outputs
     }
 
-    return outputs
-  }
+    // do not evaluate instance if it has an error, just rethrow it
+    const error = instanceErrors[instanceId]
+    if (error) {
+      // eslint-disable-next-line @typescript-eslint/only-throw-error
+      throw error
+    }
 
-  function _evaluateInstance(instanceId: InstanceModel["id"]): Record<string, unknown> {
-    const inputs: Record<string, unknown> = {}
     const instance = allInstancesMap.get(instanceId)
-
-    logger.debug("evaluating instance", { instanceId })
-
     if (!instance) {
       throw new Error(`Instance not found: ${instanceId}`)
     }
 
-    for (const [inputName, input] of Object.entries(resolvedInputs[instanceId] ?? {})) {
+    try {
+      outputs = _evaluateInstance(instance)
+
+      instanceOutputs.set(instanceId, outputs)
+      return outputs
+    } catch (error) {
+      if (instance.kind === "composite" || !allInstancesMap.has(instance.id)) {
+        topLevelErrors[instance.id] = errorToString(error)
+      }
+
+      instanceErrors[instanceId] = error
+      throw error
+    }
+  }
+
+  function _evaluateInstance(instance: InstanceModel): Record<string, unknown> {
+    const inputs: Record<string, unknown> = {}
+
+    logger.debug("evaluating instance", { instanceId: instance.id })
+
+    for (const [inputName, input] of Object.entries(resolvedInputs[instance.id] ?? {})) {
       inputs[inputName] = input.map(input => {
         const evaluated = evaluateInstance(input.input.instanceId)
 
@@ -78,7 +95,7 @@ export function evaluateProject(
 
     const component = components[instance.type]
     if (!component) {
-      throw new Error(`Component not found: ${instance.type}, required by instance: ${instanceId}`)
+      throw new Error(`Component not found: ${instance.type}, required by instance: ${instance.id}`)
     }
 
     return component({

package/src/library/worker/loader.lite.ts

@@ -1,5 +1,5 @@
 import type { Logger } from "pino"
-import { isComponent, type Component } from "@highstate/contract"
+import { type Component, isComponent } from "@highstate/contract"
 
 export async function loadComponents(
   logger: Logger,

package/src/library/worker/main.ts

@@ -11,20 +11,13 @@ const logger = pino({ name: "library-worker" })
 
 try {
   const library = await loadComponents(logger, data.libraryModulePaths)
+  const result = evaluateProject(logger, library, data.allInstances, data.resolvedInputs)
 
-  const result = evaluateProject(
-    logger,
-    library,
-    data.allInstances,
-    data.resolvedInputs,
-    data.instanceIds,
-  )
-
-  parentPort!.postMessage(result)
+  parentPort?.postMessage(result)
 } catch (error) {
   logger.error({ error }, "failed to evaluate project")
 
-  parentPort!.postMessage({
+  parentPort?.postMessage({
     success: false,
     error: errorToString(error),
   })