@highstate/backend 0.9.15 → 0.9.18

package/src/shared/{async-batcher.ts → utils/async-batcher.ts}

@@ -3,10 +3,22 @@ export type AsyncBatcherOptions = {
   maxWaitTimeMs?: number
 }
 
+export type AsyncBatcher<T> = {
+  /**
+   * Add an item to the batch.
+   */
+  call(item: T): void
+
+  /**
+   * Immediately flush the pending batch (if any).
+   */
+  flush(): Promise<void>
+}
+
 export function createAsyncBatcher<T>(
   fn: (items: T[]) => Promise<void> | void,
   { waitMs = 100, maxWaitTimeMs = 1000 }: AsyncBatcherOptions = {},
-) {
+): AsyncBatcher<T> {
   let batch: T[] = []
   let activeTimeout: NodeJS.Timeout | null = null
   let maxWaitTimeout: NodeJS.Timeout | null = null

package/src/shared/utils/hash.ts

@@ -0,0 +1,6 @@
+export function int32ToBytes(value: number): Uint8Array {
+  const buffer = new ArrayBuffer(4)
+  const view = new DataView(buffer)
+  view.setInt32(0, value, true) // true for little-endian
+  return new Uint8Array(buffer)
+}

package/src/shared/utils/index.ts

@@ -0,0 +1,4 @@
+export * from "./promise-tracker"
+export * from "./async-batcher"
+export * from "./hash"
+export * from "./args"

package/src/shared/utils/promise-tracker.ts

@@ -0,0 +1,23 @@
+export class PromiseTracker {
+  private readonly trackedPromises = new Set<Promise<unknown>>()
+
+  /**
+   * Tracks a promise to ensure its rejection is handled inside the tracker's scope.
+   */
+  track<T>(promise: Promise<T>): void {
+    this.trackedPromises.add(promise)
+
+    void promise.finally(() => this.trackedPromises.delete(promise))
+  }
+
+  /**
+   * Waits for all tracked promises to resolve or reject.
+   */
+  async waitForAll(): Promise<void> {
+    if (this.trackedPromises.size === 0) {
+      return
+    }
+
+    await Promise.allSettled(this.trackedPromises)
+  }
+}

package/src/state/abstractions.ts

@@ -1,221 +1,289 @@
-import {
-  type CompositeInstance,
-  type InstanceState,
-  type ProjectOperation,
-  type TerminalSession,
-} from "../shared"
+import type { KeyObfuscationBackend } from "./encryption"
+import { AccessError, type CollectionQueryResult } from "../shared"
 
-export type LogEntry = [instanceId: string, line: string]
-export type TerminalHistoryEntry = [sessionId: string, entry: string]
-
-export interface StateBackend {
+export type CollectionQuery = {
   /**
-   * Gets the active operations which are not completed or failed.
+   * The search string to filter documents by display name, descriptiion, or other text fields.
+   *
+   * Not implemented yet.
    */
-  getActiveOperations(signal?: AbortSignal): Promise<ProjectOperation[]>
+  search?: string
 
   /**
-   * Gets recent operations of the project.
-   * If `beforeOperationId` is provided, returns the operations before the specified operation ID.
-   * The page size is fixed to 10.
+   * The ID of the document to start the query from.
    *
-   * @param projectId The ID of the project.
-   * @param beforeOperationId The ID of the operation to get the operations before.
+   * If "asc" sorting is used, this will return results after this ID.
+   * Otherwise, it will return results before this ID.
    */
-  getOperations(projectId: string, beforeOperationId?: string): Promise<ProjectOperation[]>
+  cursor?: string
 
   /**
-   * Gets the current (cached) instance states of the project.
-   * Actual states should be retrieved from the runner (and they are still may not be up-to-date).
+   * The count of documents to return.
    *
-   * @param projectId The ID of the project.
+   * Defaults to 20 if not specified.
+   * Maximum value is 100.
    */
-  getAllInstanceStates(projectId: string, signal?: AbortSignal): Promise<InstanceState[]>
+  count?: number
 
   /**
-   * Gets the current (cached) state of the instance.
+   * The sorting order for the results.
    *
-   * @param projectId The ID of the project.
-   * @param instanceId The ID of the instance.
+   * By default, "asc" is used.
    */
-  getInstanceState(projectId: string, instanceId: string): Promise<InstanceState | null>
+  sort?: "asc" | "desc"
+}
+
+export const CollectionBackend = Symbol("CollectionBackend")
+
+/**
+ * The batch interface for write operations.
+ *
+ * It ensures that all changes made within a batch are written atomically.
+ */
+export type StateBatch = {
+  [CollectionBackend]: unknown
+  [Symbol.asyncDispose](): Promise<void>
 
   /**
-   * Gets the current (cached) state of the instances.
-   *
-   * @param projectId The ID of the project.
-   * @param instanceIds The IDs of the instances.
+   * Writes all changes made in this batch as a single atomic operation.
+   */
+  write(): Promise<void>
+}
+
+/**
+ * The snapshot interface for read operations.
+ *
+ * It ensure the consistency of read operations performed within a single snapshot.
+ */
+export type StateSnapshot = {
+  [CollectionBackend]: unknown
+  [Symbol.asyncDispose](): Promise<void>
+}
+
+/**
+ * Low-level storage abstraction that operates on raw binary data.
+ * This interface provides basic CRUD operations without any knowledge of data structure or validation.
+ */
+export interface StateCollection {
+  /**
+   * The name of the collection received from the state manager.
    */
-  getInstanceStates(projectId: string, instanceIds: string[]): Promise<InstanceState[]>
+  readonly name: string
 
   /**
-   * Gets the affected instance states at the moment of the operation completion.
-   * The logs are not included.
-   *
-   * @param operationId The ID of the operation.
+   * The parameters received from the state manager.
    */
-  getAffectedInstanceStates(operationId: string): Promise<InstanceState[]>
+  readonly params: Record<string, string>
 
   /**
-   * Gets the full logs of the instance at the moment of the operation completion.
-   *
-   * @param operationId The ID of the operation.
-   * @param instanceId The ID of the instance.
+   * The UUIDv5 namespace for this collection which combines the collection name and parameters + project namespace if applicable.
    */
-  getInstanceLogs(operationId: string, instanceId: string): Promise<string[]>
+  readonly namespace: Promise<string>
 
   /**
-   * Puts the operation to the state.
-   * The operation must have a unique ID sortable chronologically.
-   * Also, the operation must have the project ID.
+   * Retrieves a single item by its ID.
    *
-   * @param operation The operation to put.
+   * @param id The ID of the item to retrieve.
+   * @param snapshot Optional snapshot for consistent reads.
    */
-  putOperation(operation: ProjectOperation): Promise<void>
+  get(id: string, snapshot?: StateSnapshot): Promise<Uint8Array | undefined>
 
   /**
-   * Puts the instance states affected by the operation to the state.
+   * Retrieves multiple items by their IDs.
+   * Returns a record where keys are IDs and values are the data (or undefined if not found).
+   * The order of the results matches the order of the input IDs.
    *
-   * @param projectId The ID of the project.
-   * @param operationId The ID of the operation.
-   * @param states The instance states to put.
+   * @param ids An array of IDs to retrieve.
+   * @param snapshot Optional snapshot for consistent reads.
    */
-  putAffectedInstanceStates(
-    projectId: string,
-    operationId: string,
-    states: InstanceState[],
-  ): Promise<void>
+  getMany(ids: string[], snapshot?: StateSnapshot): Promise<(Uint8Array | undefined)[]>
 
   /**
-   * Puts the instance states of the project to the state.
+   * Retrieves all items in the collection as an array of tuples.
+   * Use carefully, as this may return a large amount of data.
    *
-   * @param projectId The ID of the project.
-   * @param states The instance states to put.
+   * @param snapshot Optional snapshot for consistent reads.
    */
-  putInstanceStates(projectId: string, states: InstanceState[]): Promise<void>
+  getAll(snapshot?: StateSnapshot): Promise<[string, Uint8Array][]>
 
   /**
-   * Appends the log lines to the instance logs.
+   * Retrieves items matching a query.
    *
-   * @param operationId The ID of the operation.
-   * @param logs The logs to append. Each log is a tuple of the instance ID and the log line.
+   * @param query The query object containing search, cursor, skip, count, and sort options.
+   * @param snapshot Optional snapshot for consistent reads.
    */
-  appendInstanceLogs(operationId: string, logs: LogEntry[]): Promise<void>
+  query(
+    query: CollectionQuery,
+    snapshot?: StateSnapshot,
+  ): Promise<CollectionQueryResult<[string, Uint8Array]>>
 
   /**
-   * Gets all the evaluated composite instances of the project.
+   * Retrieves the total count of items in the collection.
+   * This is useful for pagination and understanding the size of the collection.
    *
-   * @param projectId The ID of the project.
+   * @param snapshot Optional snapshot for consistent reads.
    */
-  getCompositeInstances(projectId: string, signal?: AbortSignal): Promise<CompositeInstance[]>
+  getTotalCount(snapshot?: StateSnapshot): Promise<number>
 
   /**
-   * Gets the IDs of the composite instances which are children (level 2+) of the top-level composite instances.
+   * Returns an async iterator over all items in the collection.
+   * This is more memory-efficient than getAll() for large collections.
    *
-   * Used to track the evaluated composite instances and clear them when the top-level composite instances are cleared/changed.
-   *
-   * @param projectId The ID of the project.
-   * @param instanceIds The IDs of the top-level composite instances.
+   * @param snapshot Optional snapshot for consistent reads.
    */
-  getTopLevelCompositeChildrenIds(
-    projectId: string,
-    instanceIds: string[],
-    signal?: AbortSignal,
-  ): Promise<Record<string, string[]>>
+  iterate(snapshot?: StateSnapshot): AsyncIterable<[string, Uint8Array]>
 
   /**
-   * Puts the IDs of the composite instances which are children (level 2+) of the top-level composite instances.
+   * Stores a single item with the given ID.
    *
-   * @param projectId The ID of the project.
-   * @param childrenIds The IDs of the composite instances (key is the top-level instance ID).
+   * @param id The ID of the item to store.
+   * @param value The binary data to store.
+   * @param batch Optional batch for atomic writes.
    */
-  putTopLevelCompositeChildrenIds(
-    projectId: string,
-    childrenIds: Record<string, string[]>,
-  ): Promise<void>
+  put(id: string, value: Uint8Array, batch?: StateBatch): Promise<void>
 
   /**
-   * Gets the evaluated composite instance of the project.
+   * Stores multiple items in a batch operation.
    *
-   * @param projectId The ID of the project.
-   * @param instanceId The ID of the instance.
+   * @param entries An array of key-value pairs where keys are IDs and values are binary data.
+   * @param batch Optional batch for atomic writes.
    */
-  getCompositeInstance(projectId: string, instanceId: string): Promise<CompositeInstance | null>
+  putMany(entries: Array<[string, Uint8Array]>, batch?: StateBatch): Promise<void>
 
   /**
-   * Gets the input hashes of the evaluated composite instances.
+   * Deletes a single item by its ID.
    *
-   * @param projectId The ID of the project.
+   * @param id The ID of the item to delete.
+   * @param batch Optional batch for atomic writes.
    */
-  getCompositeInstanceInputHashes(projectId: string): Promise<Record<string, string>>
+  delete(id: string, batch?: StateBatch): Promise<void>
 
   /**
-   * Puts the evaluated composite instances of the project to the state.
+   * Deletes multiple items by their IDs.
    *
-   * @param projectId The ID of the project.
-   * @param instances The instances to put.
+   * @param ids An array of IDs to delete.
+   * @param batch Optional batch for atomic writes.
    */
-  putCompositeInstances(projectId: string, instances: CompositeInstance[]): Promise<void>
+  deleteMany(ids: string[], batch?: StateBatch): Promise<void>
 
   /**
-   * Clears the evaluation state of the instance.
-   *
-   * @param projectId The ID of the project.
-   * @param instanceIds The IDs of the instances to clear.
+   * Deletes all items in the collection.
    */
-  clearCompositeInstances(projectId: string, instanceIds: string[]): Promise<void>
+  clear(): Promise<void>
+}
+
+export type CollectionMap = {
+  // Backend System
+  projects: Record<never, never>
+  "project-name-index": Record<never, never>
+  "project-master-keys": Record<never, never>
+  "project-namespaces": Record<never, never>
+  "project-unlock-suites": Record<never, never>
+  "user-layouts": Record<never, never>
+
+  // Instances
+  "instance-states": { projectId: string }
+  "instance-locks": { projectId: string }
+  "virtual-instances": { projectId: string }
 
+  // Operations
+  operations: { projectId: string }
+  "active-operations": { projectId: string }
+  "operation-logs": { projectId: string; operationId: string }
+  "instance-log-index": { projectId: string; operationId: string; instanceId: string }
+
+  // Secrets
+  secrets: { projectId: string }
+  "secret-content": { projectId: string }
+  "secret-index": { projectId: string }
+
+  // Artifacts
+  artifacts: { projectId: string }
+  "artifact-hash-index": { projectId: string }
+
+  // Terminals
+  terminals: { projectId: string }
+  "terminal-specs": { projectId: string }
+  "terminal-sessions": { projectId: string }
+  "active-terminal-sessions": { projectId: string }
+  "instance-terminal-sessions": { projectId: string; instanceId: string }
+  "terminal-session-lines": { projectId: string; sessionId: string }
+
+  // Pages
+  pages: { projectId: string }
+  "page-contents": { projectId: string }
+
+  // Triggers
+  triggers: { projectId: string }
+
+  // Authentication
+  "unlock-methods": { projectId: string }
+
+  // Service Accounts
+  "service-accounts": { projectId: string }
+
+  // API Keys
+  "api-keys": { projectId: string }
+  "api-key-token-index": { projectId: string }
+
+  // Workers
+  workers: { projectId: string }
+  "worker-registrations": { projectId: string }
+  "worker-registration-index": { projectId: string; workerId: string }
+  "worker-logs": { projectId: string; workerId: string }
+
+  // Viewports
+  "project-viewports": { projectId: string }
+  "instance-viewports": { projectId: string; instanceId: string }
+}
+
+export interface StateBackend {
   /**
-   * Gets all the terminal sessions of the instance.
+   * Returns the encrypted master key used for the encryption of backend-wide data.
    *
-   * @param projectId The ID of the project.
-   * @param instanceId The ID of the instance.
+   * The key is AGE-armored string.
    */
-  getTerminalSessions(projectId: string, instanceId: string): Promise<TerminalSession[]>
+  getEncryptedBackendMasterKey(): Promise<string | undefined>
 
   /**
-   * Gets the IDs of all the active terminal sessions.
+   * Sets the encrypted master key used for the encryption of backend-wide data.
+   *
+   * @param encryptedMasterKey The encrypted master key to set, as an AGE-armored string.
    */
-  getActiveTerminalSessions(): Promise<TerminalSession[]>
+  setEncryptedBackendMasterKey(encryptedMasterKey: string): Promise<void>
 
   /**
-   * Puts the IDs of the active terminal sessions.
+   * Returns the encrypted namespace used for the backend.
    */
-  putActiveTerminalSessions(sessions: TerminalSession[]): Promise<void>
+  getEncryptedBackendNamespace(): Promise<Uint8Array | undefined>
 
   /**
-   * Gets the terminal session.
-   *
-   * @param projectId The ID of the project.
-   * @param instanceId The ID of the instance.
-   * @param sessionId The ID of the session.
+   * Sets the static identifier for the backend.
    */
-  getTerminalSession(
-    projectId: string,
-    instanceId: string,
-    sessionId: string,
-  ): Promise<TerminalSession | null>
+  setEncryptedBackendNamespace(encryptedNamespace: Uint8Array): Promise<void>
 
   /**
-   * Puts the terminal sessions of the instance to the state.
-   *
-   * @param projectId The ID of the project.
-   * @param instanceId The ID of the instance.
-   * @param session The terminal session to put.
+   * Creates a batch for performing atomic write operations.
    */
-  putTerminalSession(projectId: string, instanceId: string, session: TerminalSession): Promise<void>
+  createStateBatch(): StateBatch
 
   /**
-   * Gets the history lines of the terminal session.
-   *
-   * @param sessionId The ID of the session.
+   * Creates a snapshot for consistent read operations.
    */
-  getTerminalSessionHistory(sessionId: string): Promise<string[]>
+  createStateSnapshot(): StateSnapshot
 
   /**
-   * Appends the history lines to the terminal sessions.
-   *
-   * @param entries The entries to append.
+   * Creates a collection by name with the appropriate parameters.
    */
-  appendTerminalSessionHistory(entries: TerminalHistoryEntry[]): Promise<void>
+  createCollection<TName extends keyof CollectionMap>(
+    name: TName,
+    params: CollectionMap[TName],
+    keyObfuscationBackend: KeyObfuscationBackend,
+  ): StateCollection
+}
+
+export class ProjectLockedError extends AccessError {
+  constructor(projectId: string) {
+    super(`The project with ID "${projectId}" is locked, decryption is not possible.`)
+  }
 }

package/src/state/encryption.ts

@@ -0,0 +1,98 @@
+import { xchacha20poly1305 } from "@noble/ciphers/chacha.js"
+import { managedNonce } from "@noble/ciphers/webcrypto.js"
+import { v5 as uuidv5 } from "uuid"
+
+export interface EncryptionBackend {
+  /**
+   * Encrypts the given binary data using the configured master key.
+   *
+   * @param data The binary data to encrypt, represented as a Uint8Array.
+   */
+  encrypt(data: Uint8Array): Promise<Uint8Array>
+
+  /**
+   * Decrypts the given binary data using the configured master key.
+   *
+   * @param data The binary data to decrypt, represented as a Uint8Array.
+   */
+  decrypt(data: Uint8Array): Promise<Uint8Array>
+}
+
+/**
+ * The no-op encryption backend that does not perform any encryption or decryption and simply
+ * returns the data as-is.
+ */
+export class PassThroughEncryptionBackend implements EncryptionBackend {
+  static readonly instance = new PassThroughEncryptionBackend()
+
+  encrypt(data: Uint8Array): Promise<Uint8Array> {
+    return Promise.resolve(data)
+  }
+
+  decrypt(data: Uint8Array): Promise<Uint8Array> {
+    return Promise.resolve(data)
+  }
+}
+
+/**
+ * The MasterKeyEncryptionBackend encryption backend that uses the noble-ciphers library to encrypt and decrypt data.
+ *
+ * It requires a master key to be provided, which is used for the actual encryption and decryption operations.
+ * The master key should be a 32-byte Uint8Array.
+ * This backend uses XChaCha20-Poly1305 for encryption and decryption.
+ * The nonce is 24 bytes long and is prepended to the ciphertext.
+ */
+export class MasterKeyEncryptionBackend implements EncryptionBackend {
+  constructor(private readonly masterKeyProvider: () => Promise<Uint8Array>) {}
+
+  async encrypt(data: Uint8Array): Promise<Uint8Array> {
+    const chacha = managedNonce(xchacha20poly1305)(await this.masterKeyProvider())
+    const encryptedData = chacha.encrypt(data)
+
+    return Promise.resolve(encryptedData)
+  }
+
+  async decrypt(data: Uint8Array): Promise<Uint8Array> {
+    const chacha = managedNonce(xchacha20poly1305)(await this.masterKeyProvider())
+    const decryptedData = chacha.decrypt(data)
+
+    return Promise.resolve(decryptedData)
+  }
+}
+
+export interface KeyObfuscationBackend {
+  /**
+   * Returns the obfuscated key for the given key.
+   *
+   * @param key The key to obfuscate, represented as a string.
+   */
+  obfuscateKey(key: string): Promise<string>
+}
+
+/**
+ * The PassThroughKeyObfuscationBackend does not obfuscate the key and returns it as-is.
+ *
+ * This is useful for cases where no obfuscation is needed.
+ */
+export class PassThroughKeyObfuscationBackend implements KeyObfuscationBackend {
+  static readonly instance = new PassThroughKeyObfuscationBackend()
+
+  obfuscateKey(key: string): Promise<string> {
+    return Promise.resolve(key)
+  }
+}
+
+/**
+ * The NamespaceKeyObfuscationBackend obfuscates keys using a namespace derived from a hash function.
+ *
+ * It uses UUIDv5 to generate a consistent obfuscated key based on the provided key and namespace.
+ */
+export class NamespaceKeyObfuscationBackend implements KeyObfuscationBackend {
+  constructor(private readonly hashNamespaceProvider: () => Promise<string>) {}
+
+  async obfuscateKey(key: string): Promise<string> {
+    const namespace = await this.hashNamespaceProvider()
+
+    return uuidv5(key, namespace)
+  }
+}

package/src/state/factory.ts

@@ -1,22 +1,20 @@
 import type { StateBackend } from "./abstractions"
 import type { Logger } from "pino"
-import type { LocalPulumiHost } from "../common"
 import { z } from "zod"
 import { LocalStateBackend, localStateBackendConfig } from "./local"
 
 export const stateBackendConfig = z.object({
-  HIGHSTATE_BACKEND_STATE_TYPE: z.enum(["local"]).default("local"),
+  HIGHSTATE_STATE_BACKEND_TYPE: z.enum(["local"]).default("local"),
   ...localStateBackendConfig.shape,
 })
 
 export function createStateBackend(
   config: z.infer<typeof stateBackendConfig>,
-  localPulumiHost: LocalPulumiHost,
   logger: Logger,
 ): Promise<StateBackend> {
-  switch (config.HIGHSTATE_BACKEND_STATE_TYPE) {
+  switch (config.HIGHSTATE_STATE_BACKEND_TYPE) {
     case "local": {
-      return LocalStateBackend.create(config, localPulumiHost, logger)
+      return LocalStateBackend.create(config, logger)
     }
   }
 }

package/src/state/index.ts

@@ -1,3 +1,7 @@
 export * from "./abstractions"
 export * from "./factory"
 export * from "./manager"
+export * from "./repository"
+export * from "./keyring"
+export * from "./encryption"
+export * from "./test"

package/src/state/keyring.ts

@@ -0,0 +1,22 @@
+import type { Logger } from "pino"
+import { AsyncEntry, findCredentialsAsync } from "@napi-rs/keyring"
+import { generateIdentity } from "age-encryption"
+
+const serviceName = "io.highstate.backend"
+const accountName = "identity"
+
+export async function getOrCreateBackendIdentity(logger: Logger): Promise<string> {
+  const credentials = await findCredentialsAsync(serviceName)
+  const entry = credentials.find(entry => entry.account === accountName)
+
+  if (entry) {
+    logger.debug("found existing backend identity in keyring")
+    return entry.password
+  }
+
+  const newIdentity = await generateIdentity()
+  await new AsyncEntry(serviceName, accountName).setPassword(newIdentity)
+
+  logger.info("created new backend identity and stored it in the OS keyring")
+  return newIdentity
+}