@highstate/backend 0.9.15 → 0.9.16

package/src/artifact/encryption.ts

@@ -0,0 +1,69 @@
+import type { EncryptionBackend, StateManager } from "../state"
+import type { ArtifactBackend } from "./abstractions"
+
+const nonceSize = 24
+
+export class EncryptionArtifactBackend implements ArtifactBackend {
+  constructor(
+    private readonly artifactBackend: ArtifactBackend,
+    private readonly stateManager: StateManager,
+  ) {}
+
+  async store(
+    projectId: string,
+    hash: string,
+    chunkSize: number,
+    content: AsyncIterable<Uint8Array>,
+  ): Promise<void> {
+    const encryptionBackend = this.stateManager.getEncryptionBackend(projectId)
+    const encryptedContent = this.getEncryptedContent(encryptionBackend, content)
+
+    await this.artifactBackend.store(projectId, hash, chunkSize + nonceSize, encryptedContent)
+  }
+
+  private async *getEncryptedContent(
+    encryptionBackend: EncryptionBackend,
+    content: AsyncIterable<Uint8Array>,
+  ): AsyncIterable<Uint8Array> {
+    for await (const chunk of content) {
+      yield await encryptionBackend.encrypt(chunk)
+    }
+  }
+
+  async retrieve(
+    projectId: string,
+    hash: string,
+    chunkSize: number,
+  ): Promise<AsyncIterable<Uint8Array> | null> {
+    const encryptionBackend = this.stateManager.getEncryptionBackend(projectId)
+
+    const encryptedContent = await this.artifactBackend.retrieve(
+      projectId,
+      hash,
+      chunkSize + nonceSize,
+    )
+
+    if (encryptedContent === null) {
+      return null
+    }
+
+    return this.getDecryptedContent(encryptionBackend, encryptedContent)
+  }
+
+  private async *getDecryptedContent(
+    encryptionBackend: EncryptionBackend,
+    encryptedContent: AsyncIterable<Uint8Array>,
+  ): AsyncIterable<Uint8Array> {
+    for await (const chunk of encryptedContent) {
+      yield await encryptionBackend.decrypt(chunk)
+    }
+  }
+
+  async delete(projectId: string, hash: string): Promise<void> {
+    return await this.artifactBackend.delete(projectId, hash)
+  }
+
+  async exists(projectId: string, hash: string): Promise<boolean> {
+    return await this.artifactBackend.exists(projectId, hash)
+  }
+}

package/src/artifact/factory.ts

@@ -0,0 +1,36 @@
+import type { ArtifactBackend } from "./abstractions"
+import type { Logger } from "pino"
+import type { StateManager } from "../state"
+import { z } from "zod"
+import { LocalArtifactBackend, localArtifactBackendConfig } from "./local"
+import { EncryptionArtifactBackend } from "./encryption"
+
+export const artifactBackendConfig = z.object({
+  HIGHSTATE_ARTIFACT_BACKEND_TYPE: z.enum(["local"]).default("local"),
+  HIGHSTATE_ARTIFACT_ENABLE_ENCRYPTION: z.coerce.boolean().default(true),
+  ...localArtifactBackendConfig.shape,
+})
+
+export async function createArtifactBackend(
+  config: z.infer<typeof artifactBackendConfig> & Record<string, unknown>,
+  stateManager: StateManager,
+  logger: Logger,
+): Promise<ArtifactBackend> {
+  let backend: ArtifactBackend
+
+  const fileExtension = config.HIGHSTATE_ARTIFACT_ENABLE_ENCRYPTION ? ".enc" : ".tgz"
+
+  switch (config.HIGHSTATE_ARTIFACT_BACKEND_TYPE) {
+    case "local": {
+      backend = await LocalArtifactBackend.create(config, fileExtension, stateManager, logger)
+    }
+  }
+
+  if (config.HIGHSTATE_ARTIFACT_ENABLE_ENCRYPTION) {
+    backend = new EncryptionArtifactBackend(backend, stateManager)
+  } else {
+    logger.warn("artifact encryption is disabled, this is not recommended")
+  }
+
+  return backend
+}

package/src/artifact/index.ts

@@ -0,0 +1,3 @@
+export * from "./abstractions"
+export * from "./factory"
+export * from "../business/artifact"

package/src/artifact/local.ts

@@ -0,0 +1,142 @@
+import type { Logger } from "pino"
+import type { StateManager } from "../state"
+import { createReadStream, createWriteStream } from "node:fs"
+import { access, mkdir, readdir, rmdir, unlink } from "node:fs/promises"
+import { join, resolve } from "node:path"
+import { z } from "zod"
+import { type ArtifactBackend } from "./abstractions"
+
+export const localArtifactBackendConfig = z.object({
+  HIGHSTATE_ARTIFACT_BACKEND_LOCAL_DIR: z.string().optional(),
+})
+
+/**
+ * A local artifact backend that stores artifacts in the filesystem.
+ *
+ * The default artifact location is `~/.highstate/artifacts`.
+ *
+ * File structure:
+ * - `{artifactDir}/{first2chars}/{hash}` - actual artifact content files
+ */
+export class LocalArtifactBackend implements ArtifactBackend {
+  constructor(
+    private readonly storageDir: string,
+    private readonly fileExtension: string,
+    private readonly stateManager: StateManager,
+    private readonly logger: Logger,
+  ) {
+    this.logger.debug({ msg: "initialized", baseDir: storageDir })
+  }
+
+  async store(
+    projectId: string,
+    hash: string,
+    chunkSize: number,
+    content: AsyncIterable<Uint8Array>,
+  ): Promise<void> {
+    const [baseDir, fileName] = this.getArtifactPath(projectId, hash)
+    await mkdir(baseDir, { recursive: true })
+
+    // check if the artifact already exists
+    try {
+      await access(fileName)
+      this.logger.debug({ msg: "artifact already exists", hash })
+
+      return
+    } catch {
+      // artifact does not exist, continue with storing
+    }
+
+    const file = createWriteStream(fileName, { highWaterMark: chunkSize })
+    this.logger.debug({ msg: "opened file for writing", hash, fileName })
+
+    for await (const chunk of content) {
+      file.write(chunk)
+    }
+
+    file.end()
+    this.logger.info({ msg: "artifact stored", hash, fileName })
+  }
+
+  async retrieve(
+    projectId: string,
+    hash: string,
+    chunkSize: number,
+  ): Promise<AsyncIterable<Uint8Array> | null> {
+    const [, fileName] = this.getArtifactPath(projectId, hash)
+
+    try {
+      return Promise.resolve(createReadStream(fileName, { highWaterMark: chunkSize }))
+    } catch (error) {
+      this.logger.debug({ msg: "artifact retrieval failed", hash, error })
+
+      return null
+    }
+  }
+
+  async delete(projectId: string, hash: string): Promise<void> {
+    const [baseDir, fileName] = this.getArtifactPath(projectId, hash)
+
+    try {
+      await unlink(fileName)
+      await this.deleteDirectoryIfEmpty(baseDir)
+
+      this.logger.info({ msg: "artifact deleted", hash, fileName })
+    } catch (error) {
+      this.logger.error({ msg: "artifact deletion failed", hash, fileName, error })
+    }
+  }
+
+  async deleteDirectoryIfEmpty(dirPath: string): Promise<void> {
+    try {
+      const files = await readdir(dirPath)
+      if (files.length === 0) {
+        await rmdir(dirPath)
+        this.logger.info({ msg: "deleted empty directory", dirPath })
+      } else {
+        this.logger.debug({ msg: "directory not empty, skipping deletion", dirPath, files })
+      }
+    } catch (error) {
+      this.logger.error({ msg: "failed to delete directory", dirPath, error })
+    }
+  }
+
+  async exists(projectId: string, hash: string): Promise<boolean> {
+    const [, fileName] = this.getArtifactPath(projectId, hash)
+
+    try {
+      await access(fileName)
+      return true
+    } catch {
+      return false
+    }
+  }
+
+  private getArtifactPath(projectId: string, hash: string): [baseDir: string, fileName: string] {
+    const hashedProjectId = this.stateManager.getHashedProjectId(projectId)
+    const baseDir = resolve(this.storageDir, hashedProjectId, hash.substring(0, 2))
+    const fileName = join(baseDir, `${hash}${this.fileExtension}`)
+
+    return [baseDir, fileName]
+  }
+
+  static async create(
+    config: z.infer<typeof localArtifactBackendConfig>,
+    fileExtension: string,
+    stateManager: StateManager,
+    logger: Logger,
+  ): Promise<LocalArtifactBackend> {
+    const childLogger = logger.child({
+      backend: "ArtifactBackend",
+      service: "LocalArtifactBackend",
+    })
+
+    let location = config.HIGHSTATE_ARTIFACT_BACKEND_LOCAL_DIR
+    if (!location) {
+      location = resolve(process.env.HOME!, ".highstate", "artifacts")
+    }
+
+    await mkdir(location, { recursive: true })
+    return new LocalArtifactBackend(location, fileExtension, stateManager, childLogger)
+  }
+}

package/src/business/api-key.ts

@@ -0,0 +1,65 @@
+import type { LockManager } from "../lock"
+import type { StateManager } from "../state"
+import { randomBytes } from "node:crypto"
+import { AccessError, type ProjectApiKey } from "../shared"
+
+export class ApiKeyService {
+  constructor(
+    private readonly stateManager: StateManager,
+    private readonly lockManager: LockManager,
+  ) {}
+
+  /**
+   * Regenerates the token for an API key in a project.
+   *
+   * @param projectId The ID of the project containing the API key.
+   * @param apiKeyId The ID of the API key to regenerate.
+   */
+  async regenerateToken(projectId: string, apiKeyId: string): Promise<ProjectApiKey> {
+    return await this.lockManager.acquire([["api-key", projectId, apiKeyId]], async () => {
+      const apiKey = await this.stateManager.getApiKeyRepository(projectId).get(apiKeyId)
+      if (!apiKey) {
+        throw new Error(`API key with ID "${apiKeyId}" not found in project "${projectId}"`)
+      }
+
+      const batch = this.stateManager.batch()
+
+      // delete the old token from the index
+      await this.stateManager
+        .getApiKeyTokenIndexRepository(projectId)
+        .indexRepository.delete(apiKey.token, batch)
+
+      // generate a new token
+      apiKey.token = randomBytes(32).toString("hex")
+
+      // update the API key with the new token and update the index
+      await Promise.all([
+        this.stateManager.getApiKeyRepository(projectId).putItem(apiKey, batch),
+        this.stateManager
+          .getApiKeyTokenIndexRepository(projectId)
+          .indexRepository.put(apiKey.token, apiKey.id, batch),
+      ])
+
+      await batch.write()
+
+      return apiKey
+    })
+  }
+
+  /**
+   * Retrieves an API key by its token for a specific project.
+   *
+   * @param projectId The ID of the project containing the API key.
+   * @param token The token of the API key to retrieve.
+   * @returns The ProjectApiKey object if found.
+   * @throws AccessError if the token is not valid for the project.
+   */
+  async getApiKeyByToken(projectId: string, token: string): Promise<ProjectApiKey> {
+    const apiKey = await this.stateManager.getApiKeyTokenIndexRepository(projectId).get(token)
+    if (!apiKey || apiKey.token !== token) {
+      throw new AccessError(`Token is not valid for project "${projectId}"`)
+    }
+
+    return apiKey
+  }
+}

package/src/business/artifact.ts

@@ -0,0 +1,288 @@
+import type { Logger } from "pino"
+import type { StateManager } from "../state"
+import type { ArtifactBackend } from "../artifact/abstractions"
+import type { LockManager } from "../lock"
+import { v7 as uuidv7 } from "uuid"
+import { unique } from "remeda"
+import { compareArtifactUsage, type Artifact, type ArtifactUsage, type ObjectMeta } from "../shared"
+
+const artifactChunkSize = 1024 * 1024 // 1 MB
+
+/**
+ * The service which handles the storage, retrieval, and management of artifacts in the system.
+ */
+export class ArtifactService {
+  constructor(
+    private readonly stateManager: StateManager,
+    private readonly artifactBackend: ArtifactBackend,
+    private readonly lockManager: LockManager,
+    private readonly logger: Logger,
+  ) {}
+
+  /**
+   * Stores an artifact in the backend and indexes it in the state manager.
+   * If the artifact already exists, it does nothing.
+   *
+   * Returns the artifact ID of the stored artifact.
+   *
+   * @param projectId The project ID to store the artifact under.
+   * @param hash The unique hash of the artifact content.
+   * @param meta The metadata for the artifact, including name, size, and other properties.
+   * @param size The total size of the artifact in bytes.
+   * @param chunkSize The size of each chunk in bytes.
+   * @param content An async iterable providing the artifact content in chunks.
+   * @param usages The list of usages for this artifact. Can be provided later.
+   */
+  async store(
+    projectId: string,
+    hash: string,
+    size: number,
+    meta: ObjectMeta,
+    content: AsyncIterable<Uint8Array>,
+    usages: ArtifactUsage[] = [],
+  ): Promise<string> {
+    return await this.lockManager.acquire(["artifact-hash", projectId, hash], async () => {
+      const existingArtifact = await this.stateManager
+        .getArtifactHashIndexRepository(projectId)
+        .get(hash)
+
+      if (existingArtifact) {
+        const fileExists = await this.artifactBackend.exists(projectId, hash)
+        if (fileExists) {
+          this.logger.debug(`artifact with hash "%s" already exists`, hash)
+          return existingArtifact.id
+        }
+
+        // TODO: make this check configurable
+        this.logger.warn(
+          `artifact with hash "%s" exists in index but not in backend, re-uploading`,
+          hash,
+        )
+      }
+
+      await this.artifactBackend.store(projectId, hash, artifactChunkSize, content)
+
+      const artifact: Artifact = {
+        id: uuidv7(),
+        hash,
+        size,
+        usages,
+        meta,
+        chunkSize: artifactChunkSize,
+      }
+
+      const batch = this.stateManager.batch()
+
+      await Promise.all([
+        this.stateManager.getArtifactRepository(projectId).putItem(artifact, batch),
+        this.stateManager
+          .getArtifactHashIndexRepository(projectId)
+          .indexRepository.put(hash, artifact.id, batch),
+      ])
+
+      await batch.write()
+
+      this.logger.info(
+        { projectId },
+        `stored artifact with hash "%s" and ID "%s"`,
+        hash,
+        artifact.id,
+      )
+
+      return artifact.id
+    })
+  }
+
+  /**
+   * Add usages for artifacts in the project.
+   *
+   * @param projectId The project ID to which the artifacts belong.
+   * @param artifactIds The IDs of the artifacts to track usages for.
+   * @param usages The list of usages to track for the artifacts.
+   * @returns
+   */
+  async addUsages(
+    projectId: string,
+    artifactIds: string[],
+    usages: ArtifactUsage[],
+  ): Promise<void> {
+    if (artifactIds.length === 0 || usages.length === 0) {
+      return
+    }
+
+    await this.lockManager.acquire(
+      artifactIds.map(id => ["artifact", projectId, id] as const),
+      () => this._addUsages(projectId, artifactIds, usages),
+    )
+  }
+
+  private async _addUsages(
+    projectId: string,
+    artifactIds: string[],
+    usages: ArtifactUsage[],
+  ): Promise<void> {
+    this.logger.debug({
+      msg: "tracking artifact usages",
+      projectId,
+      artifactIds,
+      usages,
+    })
+
+    const artifacts = await this.stateManager
+      .getArtifactRepository(projectId)
+      .getManyRecord(artifactIds)
+
+    const artifactsToUpdate: Artifact[] = []
+
+    for (const artifactId of artifactIds) {
+      const artifact = artifacts[artifactId]
+      if (!artifact) {
+        this.logger.warn({
+          msg: "artifact not found during usage addition",
+          projectId,
+          artifactId,
+        })
+        continue
+      }
+
+      // add new usages to the artifact
+      for (const usage of usages) {
+        if (!artifact.usages.some(u => compareArtifactUsage(u, usage))) {
+          artifact.usages.push(usage)
+        }
+      }
+
+      artifactsToUpdate.push(artifact)
+    }
+
+    await this.stateManager.getArtifactRepository(projectId).putManyItems(artifactsToUpdate)
+  }
+
+  /**
+   * Removes usages for artifacts in the project.
+   *
+   * If an artifact has no usages left, it will be deleted immediately.
+   *
+   * @param projectId The project ID to which the artifacts belong.
+   * @param artifactIds The IDs of the artifacts to remove usages for.
+   * @param usages The list of usages to remove from the artifacts.
+   */
+  async removeUsages(
+    projectId: string,
+    artifactIds: string[],
+    usages: ArtifactUsage[],
+  ): Promise<void> {
+    if (artifactIds.length === 0 || usages.length === 0) {
+      return
+    }
+
+    await this.lockManager.acquire(
+      artifactIds.map(id => ["artifact", projectId, id] as const),
+      () => this._removeUsages(projectId, artifactIds, usages),
+    )
+  }
+
+  private async _removeUsages(
+    projectId: string,
+    artifactIds: string[],
+    usages: ArtifactUsage[],
+  ): Promise<void> {
+    this.logger.debug({
+      msg: "removing artifact usages",
+      projectId,
+      artifactIds,
+      usages,
+    })
+
+    const artifacts = await this.stateManager
+      .getArtifactRepository(projectId)
+      .getManyRecord(artifactIds)
+
+    const artifactsToUpdate: Artifact[] = []
+    const artifactsToDelete: Artifact[] = []
+
+    for (const artifactId of artifactIds) {
+      const artifact = artifacts[artifactId]
+      if (!artifact) {
+        this.logger.warn({
+          msg: "artifact not found during usage removal",
+          projectId,
+          artifactId,
+        })
+        continue
+      }
+
+      // remove specified usages from the artifact
+      artifact.usages = artifact.usages.filter(
+        u => !usages.some(usage => compareArtifactUsage(u, usage)),
+      )
+
+      if (artifact.usages.length === 0) {
+        // if no usages left, mark for deletion
+        artifactsToDelete.push(artifact)
+      } else {
+        // otherwise, update the artifact metadata
+        artifactsToUpdate.push(artifact)
+      }
+    }
+
+    if (artifactsToUpdate.length > 0) {
+      await this.stateManager.getArtifactRepository(projectId).putManyItems(artifactsToUpdate)
+    }
+
+    if (artifactsToDelete.length > 0) {
+      await this.removeArtifacts(projectId, artifactsToDelete)
+    }
+  }
+
+  /**
+   * Updates the usage of artifacts which still have it and removes the usage from artifacts which no longer have it.
+   *
+   * @param projectId The project ID to which the artifacts belong.
+   * @param usage The usage to track for the artifacts.
+   * @param oldArtifactIds The IDs of the artifacts that previously had this usage.
+   * @param newArtifactIds The IDs of the artifacts that now have this usage.
+   */
+  async updateUsage(
+    projectId: string,
+    usage: ArtifactUsage,
+    oldArtifactIds: string[],
+    newArtifactIds: string[],
+  ): Promise<void> {
+    const removedArtifactIds = oldArtifactIds.filter(id => !newArtifactIds.includes(id))
+    const allArtifactIds = unique([...oldArtifactIds, ...newArtifactIds])
+
+    await this.lockManager.acquire(
+      allArtifactIds.map(id => ["artifact", projectId, id] as const),
+      async () => {
+        await this._addUsages(projectId, newArtifactIds, [usage])
+        await this._removeUsages(projectId, removedArtifactIds, [usage])
+      },
+    )
+  }
+
+  private async removeArtifacts(projectId: string, artifacts: Artifact[]): Promise<void> {
+    const batch = this.stateManager.batch()
+
+    await Promise.all([
+      this.stateManager.getArtifactRepository(projectId).deleteMany(
+        artifacts.map(a => a.id),
+        batch,
+      ),
+      this.stateManager.getArtifactHashIndexRepository(projectId).indexRepository.deleteMany(
+        artifacts.map(a => a.hash),
+        batch,
+      ),
+    ])
+
+    await Promise.allSettled(
+      artifacts.map(artifact => this.artifactBackend.delete(projectId, artifact.hash)),
+    )
+
+    this.logger.info({
+      msg: "deleted dangling artifacts",
+      projectId,
+      artifactHashes: artifacts.map(a => a.hash),
+    })
+  }
+}

package/src/business/backend-unlock.ts

@@ -0,0 +1,10 @@
+import type { BackendUnlockMethod } from "../shared"
+import type { StateBackend } from "../state/abstractions"
+
+export class BackendUnlockService {
+  constructor(private readonly stateBackend: StateBackend) {}
+
+  getUnlockMethods(): Promise<BackendUnlockMethod[]> {}
+
+  unlock(): Promise<string> {}
+}

package/src/business/index.ts

@@ -0,0 +1,9 @@
+export * from "./api-key"
+export * from "./artifact"
+export * from "./backend-unlock"
+export * from "./instance-lock"
+export * from "./instance-state"
+export * from "./operation"
+export * from "./project-unlock"
+export * from "./secret"
+export * from "./worker"