@highstate/backend 0.9.16 → 0.9.19

package/src/database/manager.ts

@@ -0,0 +1,181 @@
+import type { Logger } from "pino"
+import type { ProjectUnlockBackend } from "../unlock"
+import type { BackendDatabase, ProjectDatabase } from "./prisma"
+import { LRUCache } from "lru-cache"
+import z from "zod"
+import { createProjectLogger } from "../common"
+import { BackendError, ProjectLockedError, ProjectNotFoundError } from "../shared"
+import { type ProjectDatabaseBackend, projectDatabaseVersion } from "./abstractions"
+import { migrateDatabase } from "./migrate"
+
+export const databaseManagerConfig = z.object({
+  HIGHSTATE_ENCRYPTION_ENABLED: z.stringbool().default(true),
+})
+
+export interface DatabaseManager {
+  /**
+   * The backend database instance.
+   */
+  readonly backend: BackendDatabase
+
+  /**
+   * Whether the encryption is enabled for the backend and project databases.
+   */
+  readonly isEncryptionEnabled: boolean
+
+  /**
+   * Returns the master key of the project with the given ID.
+   *
+   * If the project does not exist or is not unlocked, this will throw an error.
+   *
+   * Returns `undefined` if the encryption is not enabled.
+   *
+   * @param projectId The ID of the project to get the master key for.
+   * @returns A Promise that resolves to the master key of the project as a Uint8Array.
+   */
+  getProjectMasterKey(projectId: string): Promise<Buffer | undefined>
+
+  /**
+   * Sets up the database for the project with the given ID.
+   *
+   * Assumes that no database exists yet.
+   *
+   * The project must be unlocked before calling this method.
+   *
+   * @param projectId The ID of the project to set up the database for.
+   */
+  setupDatabase(projectId: string): Promise<ProjectDatabase>
+
+  /**
+   * Returns the database client for the project with the given ID.
+   *
+   * Automatically migrates the project database if necessary.
+   *
+   * If the project does not exist or not unlocked, this will throw an error.
+   *
+   * @param projectId The ID of the project to get the database client for.
+   */
+  forProject(projectId: string): Promise<ProjectDatabase>
+}
+
+export class DatabaseManagerImpl implements DatabaseManager {
+  constructor(
+    readonly backend: BackendDatabase,
+    private readonly projectUnlockBackend: ProjectUnlockBackend,
+    private readonly projectDatabaseBackend: ProjectDatabaseBackend,
+    private readonly config: z.infer<typeof databaseManagerConfig>,
+    private readonly logger: Logger,
+  ) {}
+
+  // store the master keys in memory cache for 30 seconds
+  private readonly projectMasterKeys = new LRUCache<string, Buffer>({
+    ttl: 30_000,
+    ttlAutopurge: true,
+  })
+
+  // TODO: auto unload project databases after some time
+  private readonly projectDatabases = new Map<string, ProjectDatabase>()
+
+  get isEncryptionEnabled(): boolean {
+    return this.config.HIGHSTATE_ENCRYPTION_ENABLED
+  }
+
+  async getProjectMasterKey(projectId: string): Promise<Buffer | undefined> {
+    if (!this.isEncryptionEnabled) {
+      return undefined
+    }
+
+    const cachedInfo = this.projectMasterKeys.get(projectId)
+    if (cachedInfo) {
+      return cachedInfo
+    }
+
+    const masterKey = await this.projectUnlockBackend.getProjectMasterKey(projectId)
+    if (!masterKey) {
+      throw new ProjectLockedError(projectId)
+    }
+
+    this.projectMasterKeys.set(projectId, masterKey)
+    return masterKey
+  }
+
+  async setupDatabase(projectId: string): Promise<ProjectDatabase> {
+    const logger = createProjectLogger(this.logger, projectId)
+    const masterKey = await this.getProjectMasterKey(projectId)
+    const hexMasterKey = masterKey?.toString("hex")
+
+    const [database, databaseUrl] = await this.projectDatabaseBackend.openProjectDatabase(
+      projectId,
+      hexMasterKey,
+    )
+
+    // can safely apply migrations here, because no one knows about the database yet
+    await migrateDatabase(databaseUrl, "project", hexMasterKey, logger)
+
+    this.projectDatabases.set(projectId, database)
+
+    return database
+  }
+
+  async forProject(projectId: string): Promise<ProjectDatabase> {
+    const cachedDatabase = this.projectDatabases.get(projectId)
+    if (cachedDatabase) {
+      return cachedDatabase
+    }
+
+    const masterKey = await this.getProjectMasterKey(projectId)
+    const hexMasterKey = masterKey?.toString("hex")
+
+    // TODO: is it really necessary to migrate the database inside the transaction?
+    let database = await this.backend.$transaction(async tx => {
+      const databaseEntity = await tx.project.findUnique({
+        where: { id: projectId },
+        select: { databaseVersion: true },
+      })
+
+      if (!databaseEntity) {
+        throw new ProjectNotFoundError(projectId)
+      }
+
+      if (databaseEntity.databaseVersion > projectDatabaseVersion) {
+        throw new BackendError(
+          `Project database version (${databaseEntity.databaseVersion}) is newer than expected (${projectDatabaseVersion}).`,
+        )
+      }
+
+      if (databaseEntity.databaseVersion === projectDatabaseVersion) {
+        return
+      }
+
+      const [database, databaseUrl] = await this.projectDatabaseBackend.openProjectDatabase(
+        projectId,
+        hexMasterKey,
+      )
+
+      if (databaseEntity.databaseVersion < projectDatabaseVersion) {
+        await migrateDatabase(databaseUrl, "project", hexMasterKey, this.logger)
+      }
+
+      await tx.project.update({
+        where: { id: projectId },
+        data: { databaseVersion: projectDatabaseVersion },
+      })
+
+      return database
+    })
+
+    if (!database) {
+      // open database if was not migrated in the transaction
+      const [_database] = await this.projectDatabaseBackend.openProjectDatabase(
+        projectId,
+        hexMasterKey,
+      )
+
+      database = _database
+    }
+
+    this.projectDatabases.set(projectId, database)
+
+    return database
+  }
+}

package/src/database/migrate.ts

@@ -0,0 +1,35 @@
+import type { Logger } from "pino"
+import { join } from "node:path"
+import { fileURLToPath } from "node:url"
+import { execa } from "execa"
+import { resolve } from "import-meta-resolve"
+import { detectPackageManager } from "nypm"
+
+export async function migrateDatabase(
+  databaseUrl: string,
+  schemaPath: "backend/sqlite" | "project",
+  masterKey: string | undefined,
+  logger: Logger,
+): Promise<void> {
+  logger.info("applying database migrations")
+
+  const backendIndexPath = resolve("@highstate/backend", import.meta.url)
+  const backendRootPath = join(fileURLToPath(backendIndexPath), "..")
+
+  const packageManager = await detectPackageManager(backendRootPath)
+  if (!packageManager) {
+    throw new Error("Could not detect package manager to run migrations")
+  }
+
+  const hasCorepack = await execa`"corepack" --version`.then(() => true).catch(() => false)
+  const command = hasCorepack ? `corepack ${packageManager.command}` : packageManager.command
+
+  await execa({
+    cwd: backendRootPath,
+    env: {
+      HIGHSTATE_MIGRATION_DATABASE_SCHEMA_PATH: schemaPath,
+      HIGHSTATE_MIGRATION_DATABASE_URL: databaseUrl,
+      HIGHSTATE_MIGRATION_DATABASE_ENCRYPTION_KEY: masterKey ?? "",
+    },
+  })`${command} migrate`
+}

package/src/database/prisma.ts

@@ -0,0 +1,56 @@
+import type * as runtime from "@prisma/client/runtime/client"
+import { PrismaClient as BackendDatabase } from "./_generated/backend/postgresql/client"
+import { PrismaClient as ProjectDatabase } from "./_generated/project/client"
+
+export type BackendTransaction = Omit<BackendDatabase, runtime.ITXClientDenyList>
+export type ProjectTransaction = Omit<ProjectDatabase, runtime.ITXClientDenyList>
+
+export type {
+  Library,
+  Project,
+  ProjectSpace,
+  PulumiBackend,
+} from "./_generated/backend/postgresql/client"
+export type {
+  ApiKey,
+  Artifact,
+  InstanceCustomStatus,
+  InstanceEvaluationState,
+  InstanceEvaluationStatus,
+  InstanceLock,
+  InstanceOperationState,
+  InstanceOperationStatus,
+  InstanceSource,
+  InstanceState,
+  InstanceStatus,
+  Operation,
+  OperationLog,
+  OperationStatus,
+  OperationType,
+  Page,
+  Secret,
+  ServiceAccount,
+  Terminal,
+  TerminalSession,
+  TerminalSessionLog,
+  TerminalStatus,
+  Trigger,
+  UnlockMethod,
+  Worker,
+  WorkerUnitRegistration,
+  WorkerVersion,
+  WorkerVersionLog,
+} from "./_generated/project/client"
+export type {
+  InstanceEvaluationStateUncheckedCreateInput,
+  InstanceEvaluationStateUpdateInput,
+  InstanceOperationStateCreateInput,
+  InstanceOperationStateCreateManyInput,
+  InstanceOperationStateUpdateInput,
+  InstanceStateInclude,
+  InstanceStateUpdateInput,
+  OperationUpdateInput,
+} from "./_generated/project/models"
+export { DbNull } from "./_generated/project/internal/prismaNamespace"
+
+export { BackendDatabase, ProjectDatabase }

package/src/database/well-known.ts

@@ -0,0 +1,38 @@
+import type { BackendDatabase } from "./prisma"
+import {
+  codebaseLibrary,
+  codebaseProjectModelStorage,
+  databaseProjectModelStorage,
+  globalProjectSpace,
+  hostPulumiBackend,
+} from "../shared/models/backend/well-known"
+
+export async function ensureWellKnownEntitiesCreated(database: BackendDatabase): Promise<void> {
+  await database.$transaction([
+    database.projectSpace.upsert({
+      where: { id: globalProjectSpace.id },
+      create: globalProjectSpace,
+      update: globalProjectSpace,
+    }),
+    database.library.upsert({
+      where: { id: codebaseLibrary.id },
+      create: codebaseLibrary,
+      update: codebaseLibrary,
+    }),
+    database.pulumiBackend.upsert({
+      where: { id: hostPulumiBackend.id },
+      create: hostPulumiBackend,
+      update: hostPulumiBackend,
+    }),
+    database.projectModelStorage.upsert({
+      where: { id: codebaseProjectModelStorage.id },
+      create: codebaseProjectModelStorage,
+      update: codebaseProjectModelStorage,
+    }),
+    database.projectModelStorage.upsert({
+      where: { id: databaseProjectModelStorage.id },
+      create: databaseProjectModelStorage,
+      update: databaseProjectModelStorage,
+    }),
+  ])
+}

package/src/index.ts

@@ -1,7 +1,7 @@
-export * from "./library"
+export * from "./business"
 export * from "./config"
+export * from "./database"
+export * from "./library"
 export * from "./orchestrator"
-export * from "./terminal"
 export * from "./services"
-export * from "./state"
-export * from "./business"
+export * from "./terminal"

package/src/library/abstractions.ts

@@ -1,4 +1,4 @@
-import type { InstanceModel, CompositeInstance } from "@highstate/contract"
+import type { InstanceId, InstanceModel } from "@highstate/contract"
 import type { LibraryModel, LibraryUpdate, ResolvedInstanceInput } from "../shared"
 
 export type ResolvedUnitSource = {
@@ -8,15 +8,18 @@ export type ResolvedUnitSource = {
   allowedDependencies: string[]
 }
 
-export type InstanceEvaluationResult =
+export type ProjectEvaluationResult =
   | {
       success: true
-      instanceId: string
-      compositeInstances: CompositeInstance[]
+      virtualInstances: InstanceModel[]
+
+      /**
+       * The mapping of top-level composite instance IDs to error messages if any.
+       */
+      topLevelErrors: Record<InstanceId, string>
     }
   | {
       success: false
-      instanceId: string
       error: string
     }
 
@@ -24,42 +27,46 @@ export interface LibraryBackend {
   /**
    * Loads the library.
    */
-  loadLibrary(libraryId: string, signal?: AbortSignal): Promise<LibraryModel>
+  loadLibrary(libraryId: string | undefined, signal?: AbortSignal): Promise<LibraryModel>
 
   /**
    * Watches the library for changes.
    */
-  watchLibrary(libraryId: string, signal?: AbortSignal): AsyncIterable<LibraryUpdate[]>
+  watchLibrary(libraryId: string | undefined, signal?: AbortSignal): AsyncIterable<LibraryUpdate[]>
 
   /**
    * Gets the resolved unit sources for the given unit types.
    *
    * If the packages for these units are not resolved, it will resolve them and include in watch list.
    */
-  getResolvedUnitSources(libraryId: string, unitTypes: string[]): Promise<ResolvedUnitSource[]>
+  getResolvedUnitSources(
+    libraryId: string | undefined,
+    unitTypes: string[],
+  ): Promise<ResolvedUnitSource[]>
 
   /**
    * Watches the resolved unit sources for changes.
    * Returns an async iterable that emits each resolved unit source whenever it changes.
    * Does not emit the resolved unit sources for units that have not changed even if the library was reloaded.
+   *
+   * @param libraryId The library ID to watch for resolved unit sources.
+   * @param signal Optional AbortSignal to cancel the watch.
    */
   watchResolvedUnitSources(
-    libraryId: string,
+    libraryId: string | undefined,
     signal?: AbortSignal,
   ): AsyncIterable<ResolvedUnitSource>
 
   /**
-   * Evaluates the instances and returns the evaluated composite instances.
+   * Evaluates the composite instances of the project and returns evaluated virtual instances.
    *
    * @param libraryId The library ID to use for evaluation.
    * @param allInstances The all instances of the project.
    * @param resolvedInputs The resolved inputs of the instances.
-   * @param instanceIds The instance ids to evaluate.
    */
   evaluateCompositeInstances(
-    libraryId: string,
+    libraryId: string | undefined,
     allInstances: InstanceModel[],
     resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
-    instanceIds: string[],
-  ): Promise<InstanceEvaluationResult[]>
+  ): Promise<ProjectEvaluationResult>
 }

package/src/library/factory.ts

@@ -1,5 +1,5 @@
-import type { LibraryBackend } from "./abstractions"
 import type { Logger } from "pino"
+import type { LibraryBackend } from "./abstractions"
 import { z } from "zod"
 import { LocalLibraryBackend, localLibraryBackendConfig } from "./local"
 

package/src/library/local.ts

@@ -1,24 +1,24 @@
-import type { InstanceEvaluationResult, LibraryBackend, ResolvedUnitSource } from "./abstractions"
 import type { Logger } from "pino"
+import type { LibraryBackend, ProjectEvaluationResult, ResolvedUnitSource } from "./abstractions"
 import type {
   PackageResolutionResponse,
   PackageResolutionWorkerData,
 } from "./package-resolution-worker"
-import type { WorkerData, WorkerResponse } from "./worker/protocol"
-import { fileURLToPath } from "node:url"
+import type { WorkerData } from "./worker/protocol"
 import { EventEmitter, on } from "node:events"
-import { Worker } from "node:worker_threads"
-import { resolve } from "node:path"
 import { readFile } from "node:fs/promises"
+import { resolve } from "node:path"
+import { fileURLToPath } from "node:url"
+import { Worker } from "node:worker_threads"
 import { type InstanceModel, isUnitModel } from "@highstate/contract"
-import Watcher from "watcher"
+import { decode } from "@msgpack/msgpack"
 import { BetterLock } from "better-lock"
 import { resolve as importMetaResolve } from "import-meta-resolve"
-import { z } from "zod"
+import { addDependency, installDependencies, runScript } from "nypm"
 import { readPackageJSON } from "pkg-types"
-import { runScript, installDependencies, addDependency } from "nypm"
 import { flatMap, groupBy, map, pipe, unique } from "remeda"
-import { decode } from "@msgpack/msgpack"
+import Watcher from "watcher"
+import { z } from "zod"
 import { resolveMainLocalProject, stringArrayType } from "../common"
 import {
   diffLibraries,
@@ -28,7 +28,7 @@ import {
 } from "../shared"
 
 export const localLibraryBackendConfig = z.object({
-  HIGHSTATE_LIBRARY_BACKEND_LOCAL_PACKAGES: stringArrayType.default("@highstate/library"),
+  HIGHSTATE_LIBRARY_BACKEND_LOCAL_PACKAGES: stringArrayType.default(() => ["@highstate/library"]),
   HIGHSTATE_LIBRARY_BACKEND_LOCAL_WATCH_PATHS: stringArrayType.optional(),
 })
 
@@ -48,6 +48,11 @@ type LibraryPackage = {
   dependents: Set<string>
 }
 
+type RebuildState = {
+  inProgress: boolean
+  pending: boolean
+}
+
 export class LocalLibraryBackend implements LibraryBackend {
   private readonly watcher: Watcher
 
@@ -58,6 +63,7 @@ export class LocalLibraryBackend implements LibraryBackend {
 
   private readonly packages = new Map<string, LibraryPackage>()
   private readonly resolvedUnitSources = new Map<string, ResolvedUnitSource>()
+  private readonly rebuildStates = new Map<string, RebuildState>()
 
   private constructor(
     private readonly libraryPackages: string[],
@@ -67,7 +73,7 @@ export class LocalLibraryBackend implements LibraryBackend {
     this.watcher = new Watcher(watchPaths, {
       recursive: true,
       ignoreInitial: true,
-      ignore: /\.git|node_modules|dist/,
+      ignore: /\.git|node_modules|dist|\.highstate/,
     })
 
     this.watcher.on("all", (event: string, path: string) => {
@@ -148,26 +154,15 @@ export class LocalLibraryBackend implements LibraryBackend {
     _libraryId: string,
     allInstances: InstanceModel[],
     resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
-    instanceIds: string[],
-  ): Promise<InstanceEvaluationResult[]> {
-    this.logger.info("evaluating %d composite instances", instanceIds.length)
-
+  ): Promise<ProjectEvaluationResult> {
     const worker = this.createLibraryWorker({
       libraryModulePaths: this.libraryPackages,
       allInstances,
       resolvedInputs,
-      instanceIds,
     })
 
-    for await (const [event] of on(worker, "message")) {
-      const eventData = event as WorkerResponse
-
-      if (eventData.type === "error") {
-        throw new Error(`Worker error: ${eventData.error}`)
-      }
-
-      this.logger.info("composite instances evaluated successfully")
-      return eventData.results
+    for await (const [event] of on(worker, "message", { signal: AbortSignal.timeout(10_000) })) {
+      return event as ProjectEvaluationResult
     }
 
     throw new Error("Worker ended without sending any response")
@@ -267,7 +262,7 @@ export class LocalLibraryBackend implements LibraryBackend {
 
       this.resolvedUnitSources.set(unit.type, newResolvedSource)
       this.eventEmitter.emit("resolvedUnitSource", newResolvedSource)
-      this.logger.debug(`updated source for unit: "%s"`, unit.type)
+      this.logger.trace(`updated source for unit: "%s"`, unit.type)
     }
   }
 
@@ -475,22 +470,75 @@ export class LocalLibraryBackend implements LibraryBackend {
     }
   }
 
-  private async handleFileEvent(path: string): Promise<void> {
-    await this.lock.acquire(async () => {
-      const libraryPackage = this.packages.values().find(pkg => path.startsWith(pkg.rootPath))
+  private handleFileEvent(path: string): void {
+    const libraryPackage = this.packages.values().find(pkg => path.startsWith(pkg.rootPath))
 
-      if (!libraryPackage) {
-        return
-      }
+    if (!libraryPackage) {
+      return
+    }
+
+    void this.schedulePackageRebuild(libraryPackage.name)
+  }
 
-      const builtPackages = new Set<string>()
-      await this.rebuildLibraryPackage(libraryPackage, false, false, builtPackages)
+  private async schedulePackageRebuild(packageName: string): Promise<void> {
+    const state = this.rebuildStates.get(packageName) ?? { inProgress: false, pending: false }
+
+    // if both in progress and pending, discard this request
+    if (state.inProgress && state.pending) {
+      this.logger.debug(`rebuild in progress and already pending for "%s", discarding`, packageName)
+      return
+    }
+
+    // if in progress but no pending, mark as pending
+    if (state.inProgress && !state.pending) {
+      state.pending = true
+      this.rebuildStates.set(packageName, state)
+      this.logger.debug(`rebuild in progress, scheduling pending rebuild for "%s"`, packageName)
+      return
+    }
+
+    // not in progress, start immediately
+    await this.executePackageRebuild(packageName)
+  }
 
-      if (this.libraryPackages.some(pkg => builtPackages.has(pkg))) {
-        this.logger.info("reloading library due to file change in package: %s", libraryPackage.name)
-        await this.reloadLibrary()
+  private async executePackageRebuild(packageName: string): Promise<void> {
+    const state = this.rebuildStates.get(packageName) ?? { inProgress: false, pending: false }
+
+    do {
+      // mark as in progress, clear pending
+      state.inProgress = true
+      state.pending = false
+      this.rebuildStates.set(packageName, state)
+
+      try {
+        await this.lock.acquire(async () => {
+          const libraryPackage = this.packages.get(packageName)
+
+          if (!libraryPackage) {
+            this.logger.warn(`package not found for rebuild: "%s"`, packageName)
+            return
+          }
+
+          const builtPackages = new Set<string>()
+          await this.rebuildLibraryPackage(libraryPackage, false, false, builtPackages)
+
+          if (this.libraryPackages.some(pkg => builtPackages.has(pkg))) {
+            this.logger.info(
+              "reloading library due to file change in package: %s",
+              libraryPackage.name,
+            )
+            await this.reloadLibrary()
+          }
+        })
+      } finally {
+        state.inProgress = false
       }
-    })
+
+      // continue if another rebuild was requested while we were working
+    } while (state.pending)
+
+    // clean up if no longer needed
+    this.rebuildStates.delete(packageName)
   }
 
   private createLibraryWorker(workerData: WorkerData): Worker {

package/src/library/package-resolution-worker.ts

@@ -64,7 +64,7 @@ for (const packageName of packageNames) {
   }
 }
 
-parentPort!.postMessage({
+parentPort?.postMessage({
   type: "result",
   results,
 })