@highstate/backend 0.9.18 → 0.9.20

package/src/database/_generated/project/models.ts

@@ -0,0 +1,35 @@
+
+/* !!! This is code generated by Prisma. Do not edit directly. !!! */
+/* eslint-disable */
+// @ts-nocheck 
+/**
+ * This is a barrel export file for all models and their related types.
+ *
+ * 🟢 You can import this file directly.
+ */
+export type * from './models/ApiKey.ts'
+export type * from './models/Artifact.ts'
+export type * from './models/InstanceCustomStatus.ts'
+export type * from './models/InstanceEvaluationState.ts'
+export type * from './models/InstanceState.ts'
+export type * from './models/UserProjectViewport.ts'
+export type * from './models/UserCompositeViewport.ts'
+export type * from './models/InstanceLock.ts'
+export type * from './models/InstanceModel.ts'
+export type * from './models/HubModel.ts'
+export type * from './models/Operation.ts'
+export type * from './models/InstanceOperationState.ts'
+export type * from './models/OperationLog.ts'
+export type * from './models/Page.ts'
+export type * from './models/Secret.ts'
+export type * from './models/ServiceAccount.ts'
+export type * from './models/Terminal.ts'
+export type * from './models/TerminalSession.ts'
+export type * from './models/TerminalSessionLog.ts'
+export type * from './models/Trigger.ts'
+export type * from './models/UnlockMethod.ts'
+export type * from './models/Worker.ts'
+export type * from './models/WorkerVersion.ts'
+export type * from './models/WorkerUnitRegistration.ts'
+export type * from './models/WorkerVersionLog.ts'
+export type * from './commonInputTypes.ts'

package/src/database/_generated/project/pjtg.ts

@@ -0,0 +1,182 @@
+declare global {
+  namespace PrismaJson {
+    // This namespace will always be empty. Definitions should be done by
+    // you manually, and merged automatically by typescript. Make sure that
+    // your declaration merging file is included in your tsconfig.json
+    //
+    // Learn more: https://github.com/arthurfiorette/prisma-json-types-generator/issues/143
+    // Declaration Merging: https://www.typescriptlang.org/docs/handbook/declaration-merging.html
+  }
+}
+
+/** A filter to be used against nullable List types. */
+export type NullableListFilter<T> = {
+  equals?: T | T[] | null;
+  has?: T | null;
+  hasEvery?: T[];
+  hasSome?: T[];
+  isEmpty?: boolean;
+};
+
+/** A type to determine how to update a json field */
+export type UpdateInput<T> = T extends object ? { [P in keyof T]?: UpdateInput<T[P]> } : T;
+
+/** A type to determine how to update a json[] field */
+export type UpdateManyInput<T> = T | T[] | { set?: T[]; push?: T | T[] };
+
+/** A type to determine how to create a json[] input */
+export type CreateManyInput<T> = T | T[] | { set?: T[] };
+
+/**
+ * A typed version of NestedStringFilter, allowing narrowing of string types to
+ * discriminated unions.
+ */
+export type TypedNestedStringFilter<S extends string> =
+  //@ts-ignore - When Prisma.StringFilter is not present, this type is not used
+  Prisma.StringFilter & {
+    equals?: S;
+    in?: S[];
+    notIn?: S[];
+    not?: TypedNestedStringFilter<S> | S;
+  };
+
+/**
+ * A typed version of StringFilter, allowing narrowing of string types to discriminated
+ * unions.
+ */
+export type TypedStringFilter<S extends string> =
+  //@ts-ignore - When Prisma.StringFilter is not present, this type is not used
+  Prisma.StringFilter & {
+    equals?: S;
+    in?: S[];
+    notIn?: S[];
+    not?: TypedNestedStringFilter<S> | S;
+  };
+
+/**
+ * A typed version of NestedStringNullableFilter, allowing narrowing of string types to
+ * discriminated unions.
+ */
+export type TypedNestedStringNullableFilter<S extends string> =
+  //@ts-ignore - When Prisma.StringNullableFilter is not present, this type is not used
+  Prisma.StringNullableFilter & {
+    equals?: S | null;
+    in?: S[] | null;
+    notIn?: S[] | null;
+    not?: TypedNestedStringNullableFilter<S> | S | null;
+  };
+
+/**
+ * A typed version of StringNullableFilter, allowing narrowing of string types to
+ * discriminated unions.
+ */
+export type TypedStringNullableFilter<S extends string> =
+  //@ts-ignore - When Prisma.StringNullableFilter is not present, this type is not used
+  Prisma.StringNullableFilter & {
+    equals?: S | null;
+    in?: S[] | null;
+    notIn?: S[] | null;
+    not?: TypedNestedStringNullableFilter<S> | S | null;
+  };
+
+/**
+ * A typed version of NestedStringWithAggregatesFilter, allowing narrowing of string types
+ * to discriminated unions.
+ */
+export type TypedNestedStringWithAggregatesFilter<S extends string> =
+  //@ts-ignore - When Prisma.NestedStringWithAggregatesFilter is not present, this type is not used
+  Prisma.NestedStringWithAggregatesFilter & {
+    equals?: S;
+    in?: S[];
+    notIn?: S[];
+    not?: TypedNestedStringWithAggregatesFilter<S> | S;
+  };
+
+/**
+ * A typed version of StringWithAggregatesFilter, allowing narrowing of string types to
+ * discriminated unions.
+ */
+export type TypedStringWithAggregatesFilter<S extends string> =
+  //@ts-ignore - When Prisma.StringWithAggregatesFilter is not present, this type is not used
+  Prisma.StringWithAggregatesFilter & {
+    equals?: S;
+    in?: S[];
+    notIn?: S[];
+    not?: TypedNestedStringWithAggregatesFilter<S> | S;
+  };
+
+/**
+ * A typed version of NestedStringNullableWithAggregatesFilter, allowing narrowing of
+ * string types to discriminated unions.
+ */
+export type TypedNestedStringNullableWithAggregatesFilter<S extends string> =
+  //@ts-ignore - When Prisma.NestedStringNullableWithAggregatesFilter is not present, this type is not used
+  Prisma.NestedStringNullableWithAggregatesFilter & {
+    equals?: S | null;
+    in?: S[] | null;
+    notIn?: S[] | null;
+    not?: TypedNestedStringNullableWithAggregatesFilter<S> | S | null;
+  };
+
+/**
+ * A typed version of StringNullableWithAggregatesFilter, allowing narrowing of string
+ * types to discriminated unions.
+ */
+export type TypedStringNullableWithAggregatesFilter<S extends string> =
+  //@ts-ignore - When Prisma.StringNullableWithAggregatesFilter is not present, this type is not used
+  Prisma.StringNullableWithAggregatesFilter & {
+    equals?: S | null;
+    in?: S[] | null;
+    notIn?: S[] | null;
+    not?: TypedNestedStringNullableWithAggregatesFilter<S> | S | null;
+  };
+
+/**
+ * A typed version of StringFieldUpdateOperationsInput, allowing narrowing of string types
+ * to discriminated unions.
+ */
+export type TypedStringFieldUpdateOperationsInput<S extends string> =
+  //@ts-ignore - When Prisma.StringFieldUpdateOperationsInput is not present, this type is not used
+  Prisma.StringFieldUpdateOperationsInput & {
+    set?: S;
+  };
+
+/**
+ * A typed version of NullableStringFieldUpdateOperationsInput, allowing narrowing of
+ * string types to discriminated unions.
+ */
+export type TypedNullableStringFieldUpdateOperationsInput<S extends string> =
+  //@ts-ignore - When Prisma.NullableStringFieldUpdateOperationsInput is not present, this type is not used
+  Prisma.NullableStringFieldUpdateOperationsInput & {
+    set?: S | null;
+  };
+
+/**
+ * A typed version of StringNullableListFilter, allowing narrowing of string types to
+ * discriminated unions.
+ */
+export type TypedStringNullableListFilter<S extends string> =
+  //@ts-ignore - When Prisma.StringNullableListFilter is not present, this type is not used
+  Prisma.StringNullableListFilter & {
+    equals?: S[] | null;
+    has?: S | null;
+    hasEvery?: S[];
+    hasSome?: S[];
+  };
+
+/**
+ * A typed version of the input type to update a string[] field, allowing narrowing of
+ * string types to discriminated unions.
+ */
+export type UpdateStringArrayInput<S extends string> = {
+  set?: S[];
+  push?: S | S[];
+};
+
+/**
+ * A typed version of the input type to create a string[] field, allowing narrowing of
+ * string types to discriminated unions.
+ */
+export type CreateStringArrayInput<S extends string> = {
+  set?: S[];
+};

package/src/database/abstractions.ts

@@ -0,0 +1,19 @@
+import type { ProjectDatabase } from "./prisma"
+
+export interface ProjectDatabaseBackend {
+  /**
+   * Opens the database for the project with the given ID.
+   *
+   * If the project does not exist, this will throw an error.
+   *
+   * @param projectId The ID of the project to open the database for.
+   * @param masterKey The master key to decrypt the project database. If not provided, the encryption is assumed to be disabled.
+   */
+  openProjectDatabase(
+    projectId: string,
+    masterKey?: string,
+  ): Promise<[database: ProjectDatabase, url: string]>
+}
+
+export const backendDatabaseVersion = 2
+export const projectDatabaseVersion = 10

package/src/database/factory.ts

@@ -0,0 +1,37 @@
+import type { Logger } from "pino"
+import type { ProjectDatabaseBackend } from "./abstractions"
+import type { BackendDatabase } from "./prisma"
+import { z } from "zod"
+import {
+  createLocalBackendDatabase,
+  LocalProjectDatabaseBackend,
+  localBackendDatabaseConfig,
+} from "./local"
+import { databaseManagerConfig } from "./manager"
+
+export const databaseConfig = z.object({
+  HIGHSTATE_BACKEND_DATABASE_TYPE: z.enum(["local"]).default("local"),
+  HIGHSTATE_PROJECT_DATABASE_TYPE: z.enum(["local"]).default("local"),
+  ...localBackendDatabaseConfig.shape,
+  ...databaseManagerConfig.shape,
+})
+
+export function createBackendDatabase(
+  config: z.infer<typeof databaseConfig>,
+  logger: Logger,
+): Promise<BackendDatabase> {
+  switch (config.HIGHSTATE_BACKEND_DATABASE_TYPE) {
+    case "local":
+      return createLocalBackendDatabase(config, logger)
+  }
+}
+
+export function createProjectDatabaseBackend(
+  config: z.infer<typeof databaseConfig>,
+  logger: Logger,
+): Promise<ProjectDatabaseBackend> {
+  switch (config.HIGHSTATE_PROJECT_DATABASE_TYPE) {
+    case "local":
+      return LocalProjectDatabaseBackend.create(config, logger)
+  }
+}

package/src/database/index.ts

@@ -0,0 +1,6 @@
+export * from "./abstractions"
+export * from "./factory"
+export { getOrCreateBackendIdentity } from "./local"
+export * from "./manager"
+export * from "./prisma"
+export * from "./well-known"

package/src/database/local/backend.ts

@@ -0,0 +1,134 @@
+import type { Logger } from "pino"
+import type { BackendDatabase } from "../prisma"
+import { randomBytes } from "node:crypto"
+import { PrismaLibSQL } from "@prisma/adapter-libsql"
+import { armor, Decrypter, Encrypter, identityToRecipient } from "age-encryption"
+import { z } from "zod"
+import { codebaseConfig, getCodebaseHighstatePath } from "../../common"
+import { PrismaClient } from "../_generated/backend/sqlite/client"
+import { backendDatabaseVersion } from "../abstractions"
+import { migrateDatabase } from "../migrate"
+import { ensureWellKnownEntitiesCreated } from "../well-known"
+import { getOrCreateBackendIdentity } from "./keyring"
+import { type DatabaseMetaFile, readMetaFile, writeMetaFile } from "./meta"
+
+export const localBackendDatabaseConfig = z.object({
+  ...codebaseConfig.shape,
+  HIGHSTATE_BACKEND_DATABASE_LOCAL_PATH: z.string().optional(),
+  HIGHSTATE_ENCRYPTION_ENABLED: z.stringbool().default(true),
+})
+
+async function createMasterKey(logger: Logger) {
+  const identity = await getOrCreateBackendIdentity(logger)
+
+  const masterKey = randomBytes(32).toString("hex")
+  const encrypter = new Encrypter()
+
+  const recipient = await identityToRecipient(identity)
+  encrypter.addRecipient(recipient)
+
+  const encryptedMasterKey = await encrypter.encrypt(masterKey)
+  const armoredMasterKey = armor.encode(encryptedMasterKey)
+
+  return { armoredMasterKey, masterKey }
+}
+
+async function ensureDatabaseInitialized(
+  databasePath: string,
+  encryptionEnabled: boolean,
+  logger: Logger,
+) {
+  const meta = await readMetaFile(databasePath)
+
+  if (!meta) {
+    logger.info("creating new database")
+
+    const masterKey = encryptionEnabled ? await createMasterKey(logger) : undefined
+
+    const metaFile: DatabaseMetaFile = {
+      version: backendDatabaseVersion,
+      masterKey: masterKey?.armoredMasterKey,
+    }
+
+    return { shouldMigrate: true, masterKey: masterKey?.masterKey, metaFile }
+  }
+
+  if (meta.version > backendDatabaseVersion) {
+    throw new Error(
+      `Database version (${meta.version}) is newer than expected (${backendDatabaseVersion}). You likely need to update the Highstate.`,
+    )
+  }
+
+  if (!encryptionEnabled) {
+    return {
+      shouldMigrate: meta.version < backendDatabaseVersion,
+      masterKey: undefined,
+      metaFile: meta,
+    }
+  }
+
+  if (!meta.masterKey) {
+    throw new Error(
+      `Database meta file at "${databasePath}/meta.yaml" does not contain a master key.`,
+    )
+  }
+
+  const identity = await getOrCreateBackendIdentity(logger)
+
+  const decrypter = new Decrypter()
+  decrypter.addIdentity(identity)
+
+  const encryptedMasterKey = armor.decode(meta.masterKey)
+  const masterKey = await decrypter.decrypt(encryptedMasterKey, "text")
+
+  logger.info("loaded backend master key using OS keyring")
+
+  return {
+    shouldMigrate: meta.version < backendDatabaseVersion,
+    masterKey,
+    metaFile: meta,
+  }
+}
+
+export async function createLocalBackendDatabase(
+  config: z.infer<typeof localBackendDatabaseConfig>,
+  logger: Logger,
+): Promise<BackendDatabase> {
+  if (!config.HIGHSTATE_ENCRYPTION_ENABLED) {
+    logger.warn("local database encryption is disabled, this is not recommended for production use")
+  }
+
+  let databasePath = config.HIGHSTATE_BACKEND_DATABASE_LOCAL_PATH
+  databasePath ??= await getCodebaseHighstatePath(config, logger)
+
+  const { shouldMigrate, masterKey, metaFile } = await ensureDatabaseInitialized(
+    databasePath,
+    config.HIGHSTATE_ENCRYPTION_ENABLED,
+    logger,
+  )
+
+  const databaseUrl = `file:${databasePath}/backend.db`
+
+  if (shouldMigrate) {
+    await migrateDatabase(databaseUrl, "backend/sqlite", masterKey, logger)
+
+    // update version in meta file
+    await writeMetaFile(databasePath, { ...metaFile, version: backendDatabaseVersion })
+  }
+
+  const adapter = new PrismaLibSQL({
+    url: databaseUrl,
+    encryptionKey: masterKey,
+  })
+
+  const prismaClient = new PrismaClient({
+    adapter,
+  })
+
+  await ensureWellKnownEntitiesCreated(prismaClient as BackendDatabase)
+  logger.info("database is ready")
+
+  // yes, we return sqlite client as postgresql client
+  // TODO: https://github.com/prisma/prisma/issues/2443
+  return prismaClient as BackendDatabase
+}

package/src/database/local/index.ts

@@ -0,0 +1,3 @@
+export * from "./backend"
+export * from "./keyring"
+export * from "./project"

package/src/database/local/meta.ts

@@ -0,0 +1,46 @@
+import { readFile, writeFile } from "node:fs/promises"
+import { parse, stringify } from "yaml"
+import { z } from "zod"
+
+const databaseMetaFileSchema = z.object({
+  version: z.number(),
+  masterKey: z.string().optional(),
+})
+
+export type DatabaseMetaFile = z.infer<typeof databaseMetaFileSchema>
+
+export async function readMetaFile(
+  databasePath: string,
+): Promise<z.infer<typeof databaseMetaFileSchema> | undefined> {
+  const metaFilePath = `${databasePath}/backend.meta.yaml`
+
+  let content: string
+  try {
+    content = await readFile(metaFilePath, "utf-8")
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "ENOENT") {
+      return undefined
+    }
+
+    throw new Error(`Failed to read database meta file at "${metaFilePath}"`, { cause: error })
+  }
+
+  try {
+    return databaseMetaFileSchema.parse(parse(content))
+  } catch (error) {
+    throw new Error(`Failed to parse database meta file at "${metaFilePath}"`, { cause: error })
+  }
+}
+
+export async function writeMetaFile(
+  databasePath: string,
+  meta: z.infer<typeof databaseMetaFileSchema>,
+): Promise<void> {
+  const metaFilePath = `${databasePath}/backend.meta.yaml`
+
+  try {
+    await writeFile(metaFilePath, stringify(meta), "utf-8")
+  } catch (error) {
+    throw new Error(`Failed to write database meta file at "${metaFilePath}"`, { cause: error })
+  }
+}

package/src/database/local/prisma.config.ts

@@ -0,0 +1,25 @@
+import { PrismaLibSQL } from "@prisma/adapter-libsql"
+import { defineConfig } from "prisma/config"
+
+if (!process.env.HIGHSTATE_MIGRATION_DATABASE_SCHEMA_PATH) {
+  throw new Error("HIGHSTATE_MIGRATION_DATABASE_SCHEMA_PATH is not set")
+}
+
+export default defineConfig({
+  experimental: {
+    adapter: true,
+  },
+
+  schema: `../../../prisma/${process.env.HIGHSTATE_MIGRATION_DATABASE_SCHEMA_PATH}`,
+
+  async adapter() {
+    if (!process.env.HIGHSTATE_MIGRATION_DATABASE_URL) {
+      throw new Error("HIGHSTATE_MIGRATION_DATABASE_URL is not set")
+    }
+
+    return new PrismaLibSQL({
+      url: process.env.HIGHSTATE_MIGRATION_DATABASE_URL,
+      encryptionKey: process.env.HIGHSTATE_MIGRATION_DATABASE_ENCRYPTION_KEY || undefined,
+    })
+  },
+})

package/src/database/local/project.ts

@@ -0,0 +1,39 @@
+import type { Logger } from "pino"
+import type { z } from "zod"
+import type { ProjectDatabaseBackend } from "../abstractions"
+import { mkdir } from "node:fs/promises"
+import { PrismaLibSQL } from "@prisma/adapter-libsql"
+import { type codebaseConfig, getCodebaseHighstatePath } from "../../common"
+import { ProjectDatabase } from "../prisma"
+
+export class LocalProjectDatabaseBackend implements ProjectDatabaseBackend {
+  constructor(private readonly highstatePath: string) {}
+
+  async openProjectDatabase(
+    projectId: string,
+    masterKey?: string,
+  ): Promise<[database: ProjectDatabase, url: string]> {
+    const databasePath = `${this.highstatePath}/projects/${projectId}`
+    await mkdir(databasePath, { recursive: true })
+
+    const databaseUrl = `file:${databasePath}/project.db`
+
+    const adapter = new PrismaLibSQL({
+      url: databaseUrl,
+      encryptionKey: masterKey,
+    })
+
+    const database = new ProjectDatabase({ adapter })
+
+    return [database, databaseUrl]
+  }
+
+  static async create(
+    config: z.infer<typeof codebaseConfig>,
+    logger: Logger,
+  ): Promise<LocalProjectDatabaseBackend> {
+    const highstatePath = await getCodebaseHighstatePath(config, logger)
+
+    return new LocalProjectDatabaseBackend(highstatePath)
+  }
+}