@highstate/backend 0.9.26 → 0.9.28

package/src/database/local/backend.ts

@@ -1,12 +1,13 @@
 import type { Logger } from "pino"
 import type { BackendDatabase } from "../prisma"
 import { randomBytes } from "node:crypto"
+import { hostname } from "node:os"
 import { PrismaLibSQL } from "@prisma/adapter-libsql"
 import { armor, Decrypter, Encrypter, identityToRecipient } from "age-encryption"
 import { z } from "zod"
 import { codebaseConfig, getCodebaseHighstatePath } from "../../common"
 import { PrismaClient } from "../_generated/backend/sqlite/client"
-import { backendDatabaseVersion } from "../abstractions"
+import { type BackendDatabaseBackend, backendDatabaseVersion } from "../abstractions"
 import { migrateDatabase } from "../migrate"
 import { ensureWellKnownEntitiesCreated } from "../well-known"
 import { getOrCreateBackendIdentity } from "./keyring"
@@ -18,6 +19,59 @@ export const localBackendDatabaseConfig = z.object({
   HIGHSTATE_ENCRYPTION_ENABLED: z.stringbool().default(true),
 })
 
+/**
+ * Local implementation backed by a LibSQL database with optional encryption.
+ */
+class LocalBackendDatabaseBackend implements BackendDatabaseBackend {
+  constructor(
+    readonly database: BackendDatabase,
+    private readonly databasePath: string,
+    private readonly logger: Logger,
+    readonly isEncryptionEnabled: boolean,
+  ) {}
+
+  /**
+   * Rewrites the encrypted master key to match the provided recipients.
+   *
+   * @param recipients AGE recipients that should retain access to the backend master key.
+   */
+  async reencryptMasterKey(recipients: string[]): Promise<void> {
+    if (!this.isEncryptionEnabled) {
+      return
+    }
+
+    const meta = await readMetaFile(this.databasePath)
+    if (!meta?.masterKey) {
+      this.logger.warn(
+        { databasePath: this.databasePath },
+        "backend meta file does not contain a master key; skipping re-encryption",
+      )
+      return
+    }
+
+    const identity = await getOrCreateBackendIdentity(this.logger)
+    const decrypter = new Decrypter()
+    decrypter.addIdentity(identity)
+
+    const plaintextMasterKey = await decrypter.decrypt(armor.decode(meta.masterKey), "text")
+
+    const encrypter = new Encrypter()
+    const allowedRecipients = new Set<string>(recipients)
+    allowedRecipients.add(await identityToRecipient(identity))
+
+    for (const recipient of allowedRecipients) {
+      encrypter.addRecipient(recipient)
+    }
+
+    const encrypted = await encrypter.encrypt(plaintextMasterKey)
+
+    await writeMetaFile(this.databasePath, {
+      ...meta,
+      masterKey: armor.encode(encrypted),
+    })
+  }
+}
+
 async function createMasterKey(logger: Logger) {
   const identity = await getOrCreateBackendIdentity(logger)
 
@@ -30,14 +84,22 @@ async function createMasterKey(logger: Logger) {
   const encryptedMasterKey = await encrypter.encrypt(masterKey)
   const armoredMasterKey = armor.encode(encryptedMasterKey)
 
-  return { armoredMasterKey, masterKey }
+  return { armoredMasterKey, masterKey, recipient }
+}
+
+type DatabaseInitializationResult = {
+  shouldMigrate: boolean
+  masterKey?: string
+  metaFile: DatabaseMetaFile
+  created: boolean
+  initialRecipient?: string
 }
 
 async function ensureDatabaseInitialized(
   databasePath: string,
   encryptionEnabled: boolean,
   logger: Logger,
-) {
+): Promise<DatabaseInitializationResult> {
   const meta = await readMetaFile(databasePath)
 
   if (!meta) {
@@ -50,7 +112,13 @@ async function ensureDatabaseInitialized(
       masterKey: masterKey?.armoredMasterKey,
     }
 
-    return { shouldMigrate: true, masterKey: masterKey?.masterKey, metaFile }
+    return {
+      shouldMigrate: true,
+      masterKey: masterKey?.masterKey,
+      metaFile,
+      created: true,
+      initialRecipient: masterKey?.recipient,
+    }
   }
 
   if (meta.version > backendDatabaseVersion) {
@@ -64,12 +132,13 @@ async function ensureDatabaseInitialized(
       shouldMigrate: meta.version < backendDatabaseVersion,
       masterKey: undefined,
       metaFile: meta,
+      created: false,
     }
   }
 
   if (!meta.masterKey) {
     throw new Error(
-      `Database meta file at "${databasePath}/meta.yaml" does not contain a master key.`,
+      `Database meta file at "${databasePath}/backend.meta.yaml" does not contain a master key.`,
     )
   }
 
@@ -87,13 +156,21 @@ async function ensureDatabaseInitialized(
     shouldMigrate: meta.version < backendDatabaseVersion,
     masterKey,
     metaFile: meta,
+    created: false,
   }
 }
 
-export async function createLocalBackendDatabase(
+/**
+ * Creates the local backend database backend with migrations applied.
+ *
+ * @param config Backend database configuration resolved from environment variables.
+ * @param logger Logger scoped to backend startup.
+ * @returns The backend database backend bound to the local LibSQL store.
+ */
+export async function createLocalBackendDatabaseBackend(
   config: z.infer<typeof localBackendDatabaseConfig>,
   logger: Logger,
-): Promise<BackendDatabase> {
+): Promise<BackendDatabaseBackend> {
   if (!config.HIGHSTATE_ENCRYPTION_ENABLED) {
     logger.warn("local database encryption is disabled, this is not recommended for production use")
   }
@@ -101,18 +178,14 @@ export async function createLocalBackendDatabase(
   let databasePath = config.HIGHSTATE_BACKEND_DATABASE_LOCAL_PATH
   databasePath ??= await getCodebaseHighstatePath(config, logger)
 
-  const { shouldMigrate, masterKey, metaFile } = await ensureDatabaseInitialized(
-    databasePath,
-    config.HIGHSTATE_ENCRYPTION_ENABLED,
-    logger,
-  )
+  const { shouldMigrate, masterKey, metaFile, created, initialRecipient } =
+    await ensureDatabaseInitialized(databasePath, config.HIGHSTATE_ENCRYPTION_ENABLED, logger)
 
   const databaseUrl = `file:${databasePath}/backend.db`
 
   if (shouldMigrate) {
     await migrateDatabase(databaseUrl, "backend/sqlite", masterKey, logger)
 
-    // update version in meta file
     await writeMetaFile(databasePath, { ...metaFile, version: backendDatabaseVersion })
   }
 
@@ -125,10 +198,67 @@ export async function createLocalBackendDatabase(
     adapter,
   })
 
-  await ensureWellKnownEntitiesCreated(prismaClient as BackendDatabase)
-  logger.info("database is ready")
+  const database = prismaClient as BackendDatabase
+
+  await ensureWellKnownEntitiesCreated(database)
+
+  const backendLogger = logger.child({ service: "LocalBackendDatabaseBackend" })
+
+  await ensureInitialUnlockMethod(database, created, initialRecipient, backendLogger)
+
+  backendLogger.info("database is ready")
+
+  return new LocalBackendDatabaseBackend(
+    database,
+    databasePath,
+    backendLogger,
+    config.HIGHSTATE_ENCRYPTION_ENABLED,
+  )
+}
+
+/**
+ * Derives the meta payload for the auto-generated backend unlock method.
+ *
+ * @param host Raw host name captured during backend initialization.
+ */
+export function getInitialBackendUnlockMethodMeta(host: string | undefined): {
+  title: string
+  description: string
+} {
+  const trimmed = host?.trim() ?? ""
+  const title = trimmed.length > 0 ? trimmed : "initial"
+  const description =
+    trimmed.length > 0
+      ? `Identity automatically registered for ${trimmed} when this backend database was created.`
+      : "Identity automatically registered when this backend database was created."
+
+  return { title, description }
+}
+
+/**
+ * Registers the machine that initialized the backend as the first unlock method.
+ */
+async function ensureInitialUnlockMethod(
+  database: BackendDatabase,
+  created: boolean,
+  initialRecipient: string | undefined,
+  logger: Logger,
+): Promise<void> {
+  if (!created || !initialRecipient) {
+    return
+  }
+
+  const meta = getInitialBackendUnlockMethodMeta(hostname())
 
-  // yes, we return sqlite client as postgresql client
-  // TODO: https://github.com/prisma/prisma/issues/2443
-  return prismaClient as BackendDatabase
+  await database.backendUnlockMethod.create({
+    data: {
+      recipient: initialRecipient,
+      meta,
+    },
+  })
+
+  logger.info(
+    { title: meta.title, recipient: initialRecipient },
+    "registered initial backend unlock method",
+  )
 }

package/src/database/manager.ts

@@ -5,7 +5,11 @@ import { LRUCache } from "lru-cache"
 import z from "zod"
 import { createProjectLogger } from "../common"
 import { BackendError, ProjectLockedError, ProjectNotFoundError } from "../shared"
-import { type ProjectDatabaseBackend, projectDatabaseVersion } from "./abstractions"
+import {
+  type BackendDatabaseBackend,
+  type ProjectDatabaseBackend,
+  projectDatabaseVersion,
+} from "./abstractions"
 import { migrateDatabase } from "./migrate"
 
 export const databaseManagerConfig = z.object({
@@ -23,6 +27,13 @@ export interface DatabaseManager {
    */
   readonly isEncryptionEnabled: boolean
 
+  /**
+   * Re-encrypts the backend master key so it can be decrypted by the provided recipients.
+   *
+   * @param recipients AGE recipients that must be able to decrypt the backend master key.
+   */
+  updateBackendUnlockRecipients(recipients: string[]): Promise<void>
+
   /**
    * Returns the master key of the project with the given ID.
    *
@@ -60,13 +71,17 @@ export interface DatabaseManager {
 
 export class DatabaseManagerImpl implements DatabaseManager {
   constructor(
-    readonly backend: BackendDatabase,
+    private readonly backendBackend: BackendDatabaseBackend,
     private readonly projectUnlockBackend: ProjectUnlockBackend,
     private readonly projectDatabaseBackend: ProjectDatabaseBackend,
     private readonly config: z.infer<typeof databaseManagerConfig>,
     private readonly logger: Logger,
   ) {}
 
+  get backend(): BackendDatabase {
+    return this.backendBackend.database
+  }
+
   // store the master keys in memory cache for 30 seconds
   private readonly projectMasterKeys = new LRUCache<string, Buffer>({
     ttl: 30_000,
@@ -80,6 +95,19 @@ export class DatabaseManagerImpl implements DatabaseManager {
     return this.config.HIGHSTATE_ENCRYPTION_ENABLED
   }
 
+  /**
+   * Delegates backend master-key rotation to the active backend database backend.
+   *
+   * @param recipients AGE recipients that must retain access to the backend master key.
+   */
+  async updateBackendUnlockRecipients(recipients: string[]): Promise<void> {
+    if (!this.backendBackend.isEncryptionEnabled) {
+      return
+    }
+
+    await this.backendBackend.reencryptMasterKey(recipients)
+  }
+
   async getProjectMasterKey(projectId: string): Promise<Buffer | undefined> {
     if (!this.isEncryptionEnabled) {
       return undefined

package/src/database/prisma.ts

@@ -6,6 +6,7 @@ export type BackendTransaction = Omit<BackendDatabase, runtime.ITXClientDenyList
 export type ProjectTransaction = Omit<ProjectDatabase, runtime.ITXClientDenyList>
 
 export type {
+  BackendUnlockMethod,
   Library,
   Project,
   ProjectSpace,

package/src/orchestrator/operation-plan.ts

@@ -35,11 +35,6 @@ export function createOperationPlan(
   requestedInstanceIds: string[],
   options: OperationOptions,
 ): OperationPhase[] {
-  // handle preview restrictions
-  if (type === "preview") {
-    validatePreviewRestrictions(context, requestedInstanceIds)
-  }
-
   // initialize work state
   const workState: WorkState = {
     included: new Map(),
@@ -89,20 +84,6 @@ export function createOperationPlan(
   return createOrderedPhases(workState, context, type, options)
 }
 
-function validatePreviewRestrictions(
-  context: OperationContext,
-  requestedInstanceIds: string[],
-): void {
-  for (const instanceId of requestedInstanceIds) {
-    const dependents = context.getDependentStates(instanceId as InstanceId)
-    if (dependents.length > 0) {
-      throw new Error(
-        `Preview operation not allowed for instance ${instanceId} - has dependent instances`,
-      )
-    }
-  }
-}
-
 function processInstance(
   instanceId: InstanceId,
   workState: WorkState,

package/src/orchestrator/operation.ts

@@ -649,10 +649,14 @@ export class RuntimeOperation {
     state: InstanceState,
   ): Promise<void> {
     await this.workset.updateState(update.unitId, {
-      instanceState: {
-        // keep "deployed" status for initially deployed instances even if the operation was failed or cancelled
-        status: state.status === "deployed" ? "deployed" : "failed",
-      },
+      instanceState:
+        this.operation.type === "preview"
+          ? // do not change instance status in preview mode
+            undefined
+          : {
+              // keep "deployed" status for initially deployed instances even if the operation was failed or cancelled
+              status: state.status === "deployed" ? "deployed" : "failed",
+            },
       operationState: {
         status: isAbortErrorLike(update.message) ? "cancelled" : "failed",
         finishedAt: new Date(),
@@ -664,6 +668,17 @@ export class RuntimeOperation {
     update: TypedUnitStateUpdate<"completion">,
     state: InstanceState,
   ): Promise<void> {
+    if (this.operation.type === "preview") {
+      // do not change instance status in preview mode
+      await this.workset.updateState(update.unitId, {
+        operationState: {
+          status: this.workset.getStableStatusByOperationPhase(),
+          finishedAt: new Date(),
+        },
+      })
+      return
+    }
+
     const instance = this.context.getInstance(update.unitId)
 
     const data: InstanceStatePatch = {
@@ -696,6 +711,8 @@ export class RuntimeOperation {
       data.dependencyOutputHash = null
       data.outputHash = null
       data.parentId = null
+      data.model = null
+      data.resolvedInputs = null
     }
 
     // update the operation state

package/src/services.ts

@@ -4,6 +4,7 @@ import { type Logger, pino } from "pino"
 import { type ArtifactBackend, ArtifactService, createArtifactBackend } from "./artifact"
 import {
   ApiKeyService,
+  BackendUnlockService,
   InstanceLockService,
   InstanceStateService,
   OperationService,
@@ -19,7 +20,7 @@ import {
 import { ProjectEvaluationSubsystem } from "./business/evaluation"
 import { type Config, loadConfig } from "./config"
 import {
-  createBackendDatabase,
+  createBackendDatabaseBackend,
   createProjectDatabaseBackend,
   type DatabaseManager,
   DatabaseManagerImpl,
@@ -72,6 +73,7 @@ export type Services = {
   readonly artifactBackend: ArtifactBackend
 
   // business services
+  readonly backendUnlockService: BackendUnlockService
   readonly instanceLockService: InstanceLockService
   readonly projectUnlockService: ProjectUnlockService
   readonly operationService: OperationService
@@ -131,6 +133,7 @@ export async function createServices({
     artifactService,
 
     // business services
+    backendUnlockService,
     instanceLockService,
     projectUnlockService,
     operationService,
@@ -152,11 +155,11 @@ export async function createServices({
 
   projectUnlockBackend ??= new MemoryProjectUnlockBackend()
 
-  const backendDatabase = await createBackendDatabase(config, logger)
+  const backendDatabaseBackend = await createBackendDatabaseBackend(config, logger)
   const projectDatabaseBackend = await createProjectDatabaseBackend(config, logger)
 
   database ??= new DatabaseManagerImpl(
-    backendDatabase,
+    backendDatabaseBackend,
     projectUnlockBackend,
     projectDatabaseBackend,
     config,
@@ -174,6 +177,11 @@ export async function createServices({
   artifactBackend ??= await createArtifactBackend(config, database, logger)
   artifactService ??= new ArtifactService(database, artifactBackend, logger)
 
+  backendUnlockService ??= new BackendUnlockService(
+    database,
+    logger.child({ service: "BackendUnlockService" }),
+  )
+
   secretService ??= new SecretService(
     database,
     pubsubManager,
@@ -335,6 +343,7 @@ export async function createServices({
     artifactService,
 
     // business services
+    backendUnlockService,
     instanceLockService,
     projectUnlockService,
     operationService,

package/src/shared/models/backend/unlock-method.ts

@@ -1,20 +1,14 @@
 import { objectMetaSchema } from "@highstate/contract"
 import { z } from "zod"
 
-/**
- * Complete unlock method schema for database storage.
- * Contains all unlock method information including encryption details.
- *
- * Do not confuse with the project unlock method schema.
- */
-export const backendUnlockMethodSchema = z.object({
-  id: z.string(),
-  meta: objectMetaSchema,
+export const backendUnlockMethodMetaSchema = objectMetaSchema
+  .pick({ title: true, description: true })
+  .required({ title: true })
 
-  /**
-   * The AGE recipient of the unlock method to use to encrypt the master key for this unlock method.
-   */
+export const backendUnlockMethodInputSchema = z.object({
+  meta: backendUnlockMethodMetaSchema,
   recipient: z.string(),
 })
 
-export type BackendUnlockMethod = z.infer<typeof backendUnlockMethodSchema>
+export type BackendUnlockMethodMeta = z.infer<typeof backendUnlockMethodMetaSchema>
+export type BackendUnlockMethodInput = z.infer<typeof backendUnlockMethodInputSchema>

package/src/shared/models/errors.ts

@@ -84,3 +84,17 @@ export class WorkerVersionNotFoundError extends BackendError {
     this.name = "WorkerVersionNotFoundError"
   }
 }
+
+export class BackendUnlockMethodNotFoundError extends BackendError {
+  constructor(id: string) {
+    super(`Backend unlock method with ID "${id}" not found.`)
+    this.name = "BackendUnlockMethodNotFoundError"
+  }
+}
+
+export class CannotDeleteLastBackendUnlockMethodError extends BackendError {
+  constructor() {
+    super(`Refused to delete the last backend unlock method.`)
+    this.name = "CannotDeleteLastBackendUnlockMethodError"
+  }
+}

package/src/shared/models/prisma.ts

@@ -2,14 +2,22 @@ import type * as contract from "@highstate/contract"
 import type * as shared from "../../shared"
 
 declare global {
+  // Common
   namespace PrismaJson {
-    type LibrarySpec = shared.LibrarySpec
     type CommonObjectMeta = contract.CommonObjectMeta
+  }
 
-    type PulumiBackendSpec = shared.PulumiBackendSpec
+  // Backend
+  namespace PrismaJson {
+    type LibrarySpec = shared.LibrarySpec
+    type BackendUnlockMethodMeta = shared.BackendUnlockMethodMeta
 
+    type PulumiBackendSpec = shared.PulumiBackendSpec
     type ProjectModelStorageSpec = shared.ProjectModelStorageSpec
+  }
 
+  // Project
+  namespace PrismaJson {
     type InstanceId = contract.InstanceId
     type NullableInstanceId = contract.InstanceId | null
     type InstanceIds = contract.InstanceId[]

package/src/test-utils/database.ts

@@ -2,11 +2,13 @@ import type { Logger } from "pino"
 import type { BackendDatabase, ProjectDatabase } from "../database/prisma"
 import { constants } from "node:fs"
 import { access, mkdtemp, rm } from "node:fs/promises"
-import { tmpdir } from "node:os"
+import { hostname, tmpdir } from "node:os"
 import { join } from "node:path"
 import { PrismaLibSQL } from "@prisma/adapter-libsql"
+import { generateIdentity, identityToRecipient } from "age-encryption"
 import { type DatabaseManager, ensureWellKnownEntitiesCreated } from "../database"
 import { PrismaClient as BackendPrismaClient } from "../database/_generated/backend/sqlite/client"
+import { getInitialBackendUnlockMethodMeta } from "../database/local/backend"
 import { migrateDatabase } from "../database/migrate"
 import {
   type BackendDatabase as BackendDatabaseClient,
@@ -16,16 +18,25 @@ import {
 export class TestDatabaseManager implements DatabaseManager {
   private readonly projectDatabases = new Map<string, Promise<ProjectDatabase>>()
   private readonly tempDirs: string[] = []
+  readonly backendUnlockRecipientUpdates: string[][] = []
 
   constructor(
     readonly backend: BackendDatabase,
     private readonly logger: Logger,
+    readonly isEncryptionEnabled: boolean,
   ) {}
 
-  readonly isEncryptionEnabled = false
+  updateBackendUnlockRecipients(recipients: string[]): Promise<void> {
+    if (!this.isEncryptionEnabled) {
+      return Promise.resolve()
+    }
+
+    this.backendUnlockRecipientUpdates.push(recipients)
+    return Promise.resolve()
+  }
 
-  getProjectMasterKey(): Promise<Buffer> {
-    return Promise.resolve(Buffer.alloc(0))
+  getProjectMasterKey(): Promise<Buffer | undefined> {
+    return Promise.resolve(undefined)
   }
 
   setupDatabase(projectId: string): Promise<ProjectDatabase> {
@@ -55,7 +66,11 @@ export class TestDatabaseManager implements DatabaseManager {
     })
   }
 
-  static async create(logger: Logger): Promise<TestDatabaseManager> {
+  static async create(
+    logger: Logger,
+    options: { isEncryptionEnabled?: boolean } = {},
+  ): Promise<TestDatabaseManager> {
+    const { isEncryptionEnabled = false } = options
     const tempRoot = await findWritableTempDir()
     const tempPath = await mkdtemp(join(tempRoot, "highstate"))
     const backendUrl = `file:${join(tempPath, "backend.db")}`
@@ -68,7 +83,20 @@ export class TestDatabaseManager implements DatabaseManager {
 
     await ensureWellKnownEntitiesCreated(backend)
 
-    return new TestDatabaseManager(backend, logger)
+    if (isEncryptionEnabled) {
+      const identity = await generateIdentity()
+      const recipient = await identityToRecipient(identity)
+      const meta = getInitialBackendUnlockMethodMeta(hostname())
+
+      await backend.backendUnlockMethod.create({
+        data: {
+          recipient,
+          meta,
+        },
+      })
+    }
+
+    return new TestDatabaseManager(backend, logger, isEncryptionEnabled)
   }
 
   private async createTempPath(): Promise<string> {