@highstate/backend 0.9.14 → 0.9.16

package/src/business/instance-lock.ts

@@ -0,0 +1,124 @@
+import type { Logger } from "pino"
+import type { LockManager } from "../lock"
+import type { PubSubManager } from "../pubsub"
+import type { StateManager } from "../state"
+import { compareInstanceLockSpecs, type InstanceLockSpec, type ObjectMeta } from "../shared"
+
+export class InstanceLockService {
+  constructor(
+    private readonly stateManager: StateManager,
+    private readonly lockManager: LockManager,
+    private readonly pubsubManager: PubSubManager,
+    private readonly logger: Logger,
+  ) {}
+
+  /**
+   * Tries to lock instances in the project.
+   * Returns true if the lock was acquired successfully, false otherwise.
+   *
+   * @param projectId The project ID to lock instances in.
+   * @param instanceIds The instance IDs to lock.
+   * @param lockMeta The metadata for the lock.
+   * @param spec The lock specification containing the lock type and other parameters.
+   * @param allowPartialLock Whether to allow partial locking of instances.
+   * If true, it will lock as many instances as possible even if some are already locked.
+   *
+   * @return A tuple containing a boolean indicating if the lock was acquired,
+   * and an array of instance IDs that were successfully locked.
+   * If `allowPartialLock` is false and some instances are already locked,
+   * it will return false and an empty array.
+   */
+  async tryLockInstances(
+    projectId: string,
+    instanceIds: string[],
+    lockMeta: ObjectMeta,
+    spec: InstanceLockSpec,
+    allowPartialLock = false,
+  ): Promise<[locked: boolean, lockedInstanceIds: string[]]> {
+    const lockKeys = instanceIds.map(id => ["instance-lock", projectId, id] as const)
+
+    // acquire the hard lock to ensure no other process can manage the lock entries
+    return await this.lockManager.acquire(lockKeys, async () => {
+      // check if the instances are already locked
+      const existingLocks = await this.stateManager
+        .getInstanceLockRepository(projectId)
+        .getManyItems(instanceIds)
+
+      // only the locks with another spec are considered as the conflicting locks
+      const conflictingLocks = existingLocks.filter(
+        lock => !compareInstanceLockSpecs(lock.spec, spec),
+      )
+
+      if (conflictingLocks.length > 0) {
+        this.logger.debug(
+          {
+            projectId,
+            requestedInstanceIds: instanceIds,
+            lockedInstanceIds: conflictingLocks.map(lock => lock.id),
+          },
+          `failed to lock %s of %s instances in project "%s"`,
+          conflictingLocks.length,
+          instanceIds.length,
+          projectId,
+        )
+
+        if (!allowPartialLock) {
+          // if partial locking is not allowed, return false
+          return [false, []]
+        }
+
+        // if partial locking is allowed, filter out the already locked instances and proceed with the rest
+        instanceIds = instanceIds.filter(id => !conflictingLocks.some(lock => lock.id === id))
+      }
+
+      const newLocks = instanceIds.map(id => ({ id, projectId, meta: lockMeta, spec }))
+
+      // persist the new locks
+      await this.stateManager.getInstanceLockRepository(projectId).putManyItems(newLocks)
+
+      this.logger.debug(
+        {
+          projectId,
+          instanceIds: newLocks.map(lock => lock.id),
+          spec,
+        },
+        `locked %s instances in project "%s"`,
+        newLocks.length,
+        projectId,
+      )
+
+      // publish the lock event
+      await this.pubsubManager.publish(["instance-lock", projectId], {
+        type: "locked",
+        locks: newLocks,
+      })
+
+      return [true, newLocks.map(lock => lock.id)]
+    })
+  }
+
+  /**
+   * Unlocks instances in the project unconditionally.
+   * This will remove the locks regardless of their current state.
+   *
+   * @param projectId The project ID to unlock instances in.
+   * @param instanceIds The instance IDs to unlock.
+   */
+  async unlockInstancesUnconditionally(projectId: string, instanceIds: string[]): Promise<void> {
+    this.logger.debug(
+      { projectId, instanceIds },
+      `unconditionally unlocking %s instances in project "%s"`,
+      instanceIds.length,
+      projectId,
+    )
+
+    // remove the locks from the state
+    await this.stateManager.getInstanceLockRepository(projectId).deleteMany(instanceIds)
+
+    // publish the unlock event
+    await this.pubsubManager.publish(["instance-lock", projectId], {
+      type: "unlocked",
+      instanceIds,
+    })
+  }
+}

package/src/business/instance-state.ts

@@ -0,0 +1,292 @@
+import type { HotStateManager } from "../hotstate"
+import type { LockManager } from "../lock"
+import type { PubSubManager } from "../pubsub"
+import type { StateManager } from "../state"
+import type { Logger } from "pino"
+import type { RunnerBackend } from "../runner"
+import type { ArtifactService } from "../artifact"
+import { randomBytes } from "node:crypto"
+import { parseInstanceId } from "@highstate/contract"
+import { unique } from "remeda"
+import {
+  applyStatePatch,
+  isStableOperationStatus,
+  isStateEmpty,
+  type InstanceCustomStatus,
+  type InstanceState,
+  type InstanceStatePatch,
+  type Operation,
+} from "../shared"
+
+export class InstanceStateService {
+  constructor(
+    private readonly stateManager: StateManager,
+    private readonly hotStateManager: HotStateManager,
+    private readonly pubsubManager: PubSubManager,
+    private readonly lockManager: LockManager,
+    private readonly runnerBackend: RunnerBackend,
+    private readonly artifactManager: ArtifactService,
+    private readonly logger: Logger,
+  ) {}
+
+  /**
+   * Gets the current states of all instances in a project.
+   *
+   * @param projectId The ID of the project for which to retrieve instance states.
+   */
+  async getInstanceStates(projectId: string): Promise<InstanceState[]> {
+    const entries = await this.hotStateManager.hgetall(["instance-states", projectId])
+
+    return entries.map(([, value]) => value)
+  }
+
+  async getInstanceState(projectId: string, instanceId: string): Promise<InstanceState | null> {
+    return await this.hotStateManager.hget(["instance-states", projectId], instanceId)
+  }
+
+  async updateInstanceStates(
+    projectId: string,
+    states: InstanceState[],
+    persistent = false,
+  ): Promise<InstanceState[]> {
+    const batch = this.stateManager.batch()
+    const promises: Promise<void>[] = []
+
+    for (const state of states) {
+      promises.push(this.hotStateManager.hset(["instance-states", projectId], state.id, state))
+
+      promises.push(
+        this.pubsubManager.publish(["instance-state", projectId], { type: "updated", state }),
+      )
+
+      if (persistent) {
+        promises.push(this.stateManager.getInstanceStateRepository(projectId).putItem(state, batch))
+      }
+    }
+
+    await Promise.allSettled(promises)
+    await batch.write()
+
+    this.logger.debug({ projectId, states }, `instance states updated for project "%s"`, projectId)
+
+    return states
+  }
+
+  async patchInstanceState(projectId: string, patch: InstanceStatePatch): Promise<InstanceState> {
+    return await this.lockManager.acquire(["instance-state", projectId, patch.id], async () => {
+      const state = await this.hotStateManager.hget(["instance-states", projectId], patch.id)
+
+      if (!state) {
+        throw new Error(`Instance state with ID "${patch.id}" not found in project "${projectId}"`)
+      }
+
+      const patchedState = await this.applyStatePatch(
+        projectId,
+        state,
+        patch,
+        patch.operationStatus !== undefined,
+      )
+
+      this.logger.debug(
+        { projectId, instanceId: patch.id, patch },
+        `instance state updated for instance "%s" in project "%s"`,
+        patch.id,
+        projectId,
+      )
+
+      return patchedState
+    })
+  }
+
+  async patchOperationInstanceState(
+    projectId: string,
+    operation: Operation,
+    patch: InstanceStatePatch,
+  ): Promise<InstanceState> {
+    return await this.lockManager.acquire(["instance-state", projectId, patch.id], async () => {
+      const state = await this.hotStateManager.hget(["instance-states", projectId], patch.id)
+
+      const patchedState = await this.applyStatePatch(
+        projectId,
+        state,
+        patch,
+        operation.type !== "preview",
+      )
+
+      this.logger.debug(
+        { projectId, instanceId: patch.id, patch },
+        `instance state updated for instance "%s" in project "%s"`,
+        patch.id,
+        projectId,
+      )
+
+      return patchedState
+    })
+  }
+
+  async updateStateSecretNames(
+    projectId: string,
+    instanceId: string,
+    secretNamesToAdd: string[],
+    secretNamesToRemove: string[],
+    invalidateState = true,
+  ): Promise<void> {
+    await this.lockManager.acquire(["instance-state", projectId, instanceId], async () => {
+      const state = await this.hotStateManager.hget(["instance-states", projectId], instanceId)
+
+      const secretNameSet = new Set(state?.secretNames ?? [])
+      for (const name of secretNamesToAdd) {
+        secretNameSet.add(name)
+      }
+
+      for (const name of secretNamesToRemove) {
+        secretNameSet.delete(name)
+      }
+
+      const patch: InstanceStatePatch = {
+        id: instanceId,
+        secretNames: Array.from(secretNameSet),
+      }
+
+      if (state?.operationStatus && invalidateState) {
+        // invalidate the input hash nonce if the operation status is present
+        // if not, the invalidation makes no sense
+        patch.operationStatus = {
+          inputHashNonce: randomBytes(4).readInt32LE(),
+        }
+      }
+
+      await this.applyStatePatch(projectId, state, patch, true)
+    })
+  }
+
+  /**
+   * Completely deletes the instance state from the system including all related resources.
+   *
+   * @param projectId The ID of the project from which to delete the instance state.
+   * @param instanceId The ID of the instance whose state is to be deleted.
+   */
+  async deleteInstanceState(projectId: string, instanceId: string): Promise<void> {
+    await this.lockManager.acquire(["instance-state", projectId, instanceId], async () => {
+      const [instanceType, instanceName] = parseInstanceId(instanceId)
+      const state = await this.getInstanceState(projectId, instanceId)
+
+      const batch = this.stateManager.batch()
+
+      await Promise.allSettled([
+        // remove from the state
+        this.stateManager.getInstanceStateRepository(projectId).delete(instanceId, batch),
+
+        // clear related resources
+        this.stateManager.getInstanceLockRepository(projectId).delete(instanceId, batch),
+
+        this.stateManager
+          .getPageRepository(projectId)
+          .deleteMany(Object.values(state?.extra?.pageIds ?? {}), batch),
+
+        this.stateManager
+          .getTriggerRepository(projectId)
+          .deleteMany(Object.values(state?.extra?.triggerIds ?? {}), batch),
+
+        this.stateManager
+          .getTerminalRepository(projectId)
+          .deleteMany(Object.values(state?.extra?.terminalIds ?? {}), batch),
+      ])
+
+      await Promise.allSettled([
+        // commit the state batch
+        batch.write(),
+
+        // update hotstate
+        this.hotStateManager.hdel(["instance-states", projectId], instanceId),
+
+        // publish the instance state deletion event
+        this.pubsubManager.publish(["instance-state", projectId], { type: "deleted", instanceId }),
+
+        // remove all artifact references
+        this.artifactManager.removeUsages(
+          projectId,
+          unique([
+            ...Object.values(state?.extra?.ownedArtifactIds ?? []).flat(),
+            ...(state?.extra?.usedArtifactIds ?? []),
+          ]),
+          [{ type: "instance", instanceId }],
+        ),
+
+        // clear pulumi state
+        this.runnerBackend.deleteState({ projectId, instanceName, instanceType }),
+      ])
+    })
+  }
+
+  async updateCustomStatus(
+    projectId: string,
+    instanceId: string,
+    status: InstanceCustomStatus,
+  ): Promise<void> {
+    await this.lockManager.acquire(["instance-state", projectId, instanceId], async () => {
+      const state = await this.hotStateManager.hget(["instance-states", projectId], instanceId)
+      if (!state) {
+        throw new Error(
+          `Instance state with ID "${instanceId}" not found in project "${projectId}"`,
+        )
+      }
+
+      const existingStatus = state.extra?.customStatuses?.find(s => s.name === status.name)
+      if (existingStatus) {
+        Object.assign(existingStatus, status)
+      } else {
+        state.extra ??= {}
+        state.extra.customStatuses ??= []
+        state.extra.customStatuses?.push(status)
+      }
+
+      await this.applyStatePatch(projectId, state, { id: instanceId, extra: state.extra })
+    })
+  }
+
+  private async applyStatePatch(
+    projectId: string,
+    state: InstanceState | null,
+    patch: InstanceStatePatch,
+    persistent = true,
+  ): Promise<InstanceState> {
+    const patchedState = applyStatePatch(state, patch)
+
+    const promises: Promise<void>[] = [
+      this.hotStateManager.hset(["instance-states", projectId], patchedState.id, patchedState),
+    ]
+
+    const shouldPersist =
+      persistent &&
+      (!patchedState.operationStatus ||
+        isStableOperationStatus(patchedState.operationStatus?.status))
+
+    const repo = this.stateManager.getInstanceStateRepository(projectId)
+
+    if (isStateEmpty(patchedState)) {
+      promises.push(
+        this.pubsubManager.publish(["instance-state", projectId], {
+          type: "deleted",
+          instanceId: patchedState.id,
+        }),
+      )
+
+      if (shouldPersist) {
+        promises.push(repo.delete(patchedState.id))
+      }
+    } else {
+      promises.push(
+        this.pubsubManager.publish(["instance-state", projectId], { type: "patched", patch }),
+      )
+
+      if (shouldPersist) {
+        promises.push(repo.putItem(patchedState))
+      }
+    }
+
+    await Promise.allSettled(promises)
+
+    return patchedState
+  }
+}

package/src/business/operation.ts

@@ -0,0 +1,251 @@
+import type { Logger } from "pino"
+import type { PubSubManager } from "../pubsub"
+import type { HotStateManager } from "../hotstate"
+import { v7 as uuidv7 } from "uuid"
+import { SAME_KEY, type StateManager } from "../state"
+import { isFinalOperationStatus, type Operation } from "../shared"
+
+export class OperationService {
+  constructor(
+    private readonly stateManager: StateManager,
+    private readonly hotStateManager: HotStateManager,
+    private readonly pubsubManager: PubSubManager,
+    private readonly logger: Logger,
+  ) {}
+
+  async getActiveOperaiton(projectId: string, operationId: string): Promise<Operation | null> {
+    try {
+      return await this.hotStateManager.hget(["active-operations", projectId], operationId)
+    } catch (error) {
+      this.logger.error(
+        { projectId, operationId, error },
+        `failed to get active operation with ID "%s" in project "%s" from hot state`,
+        operationId,
+        projectId,
+      )
+
+      return null
+    }
+  }
+
+  /**
+   * Persists an operation to the state and publishes an update event.
+   *
+   * @param projectId The project ID to which the operation belongs.
+   * @param operation The operation to persist.
+   */
+  async updateOperation(projectId: string, operation: Operation): Promise<void> {
+    //  1. update the operation in the hot state
+    try {
+      if (isFinalOperationStatus(operation.status)) {
+        await this.hotStateManager.hdel(["active-operations", projectId], operation.id)
+      } else {
+        await this.hotStateManager.hset(["active-operations", projectId], operation.id, operation)
+      }
+    } catch (error) {
+      this.logger.error(
+        { projectId, operationId: operation.id, error },
+        `failed to update operation with ID "%s" in project "%s" in hot state`,
+        operation.id,
+        projectId,
+      )
+    }
+
+    // 2. persist the operation in the state repository
+    try {
+      await using batch = this.stateManager.batch()
+
+      // persist the operation to the operation repository
+      await this.stateManager.getOperationRepository(projectId).put(operation.id, operation, batch)
+
+      if (operation.status === "pending") {
+        // add the just created operation to the active operations index
+        await this.stateManager
+          .getActiveOperationIndexRepository(projectId)
+          .indexRepository.put(operation.id, operation.id, batch)
+      } else if (isFinalOperationStatus(operation.status)) {
+        // remove the finished operation from the active operations index
+        await this.stateManager
+          .getActiveOperationIndexRepository(projectId)
+          .indexRepository.delete(operation.id, batch)
+      }
+
+      await batch.write()
+
+      try {
+        await this.pubsubManager.publish(["operation", operation.id], {
+          type: "updated",
+          operation,
+        })
+      } catch (error) {
+        this.logger.error(
+          { projectId, operationId: operation.id, error },
+          `failed to publish operation update for operation with ID "%s" in project "%s"`,
+          operation.id,
+          projectId,
+        )
+      }
+
+      this.logger.info(
+        { projectId, operationId: operation.id },
+        `updated operation with ID "%s" in project "%s"`,
+        operation.id,
+        projectId,
+      )
+    } catch (error) {
+      this.logger.error(
+        { projectId, operationId: operation.id, error },
+        `failed to update operation with ID "%s" in project "%s"`,
+        operation.id,
+        projectId,
+      )
+
+      throw new Error(`Failed to update operation "${operation.id}" in project "${projectId}"`, {
+        cause: error,
+      })
+    }
+  }
+
+  /**
+   * Appends logs for a specific operation.
+   * Creates all necessary indexes to retrieve the logs by instance ID.
+   *
+   * @param projectId The ID of the project to persist the logs for.
+   * @param operationId The ID of the operation to persist the logs for.
+   * @param instanceIds The IDs of the instances to persist the logs for. Should include the whole hierarchy of instances produced the logs.
+   * @param logs The logs entries to persist.
+   */
+  async appendLogs(
+    projectId: string,
+    operationId: string,
+    instanceIds: string[],
+    logs: string[],
+  ): Promise<void> {
+    const records: [string, string][] = logs.map(log => [uuidv7(), log])
+
+    // 1. publish logs to the pubsub system, ignore errors
+    try {
+      const events = instanceIds.flatMap(instanceId =>
+        records.map(
+          record => [[`operation-instance-log`, operationId, instanceId], record] as const,
+        ),
+      )
+
+      await this.pubsubManager.publishMany(events)
+    } catch (error) {
+      this.logger.error(
+        { projectId, operationId, instanceIds, error },
+        `failed to publish logs for operation "%s" in project "%s"`,
+        operationId,
+        projectId,
+      )
+    }
+
+    // 2. persist logs to the hot state and state repository, ignore errors
+    try {
+      // insert logs into the hotstate
+      await this.hotStateManager.hmset(["operation-logs", projectId], records)
+    } catch (error) {
+      this.logger.error(
+        { projectId, operationId, instanceIds, error },
+        `failed to append logs for operation "%s" in project "%s"`,
+        operationId,
+        projectId,
+      )
+    }
+
+    // 3. persist logs to the state repository, fail on error
+    try {
+      await using batch = this.stateManager.batch()
+
+      // insert logs into the operation log repository
+      await this.stateManager
+        .getOperationLogRepository(projectId, operationId)
+        .putMany(records, batch)
+
+      for (const instanceId of instanceIds) {
+        // create an index for the instance logs
+        await this.stateManager
+          .getInstanceLogIndexRepository(projectId, operationId, instanceId)
+          .indexRepository.putMany(
+            records.map(([id]) => [id, SAME_KEY]),
+            batch,
+          )
+      }
+
+      await batch.write()
+    } catch (error) {
+      this.logger.error(
+        { projectId, operationId, instanceIds, error },
+        `failed to persist instance logs for operation "%s" in project "%s"`,
+        operationId,
+        projectId,
+      )
+
+      throw new Error(
+        `Failed to persist logs for operation "${operationId}" in project "${projectId}"`,
+        { cause: error },
+      )
+    }
+  }
+
+  /**
+   * Retrieves logs for a specific operation and instance.
+   * First tries to get logs from the hot state, then falls back to the state repository.
+   *
+   * @param projectId The ID of the project to retrieve logs for.
+   * @param operationId The ID of the operation to retrieve logs for.
+   * @param instanceId The ID of the instance to retrieve logs for.
+   * @returns An array of log entries as tuples of [logId, logContent].
+   */
+  async getInstanceLogs(
+    projectId: string,
+    operationId: string,
+    instanceId: string,
+  ): Promise<[string, string][]> {
+    // 1. first try to get logs from the hot state
+    try {
+      const logs = await this.hotStateManager.hgetall(["operation-logs", projectId])
+
+      if (logs.length > 0) {
+        // if there are logs, assume they are up-to-date
+        return logs
+      }
+
+      // otherwise, check if the operation is active to determine if logs are expected
+      const hasActiveOperation = await this.hotStateManager.hexists(
+        ["active-operations", projectId],
+        operationId,
+      )
+
+      if (hasActiveOperation) {
+        // if the operation is active, there is just no logs yet
+        return []
+      }
+    } catch (error) {
+      this.logger.error(
+        { projectId, operationId, instanceId, error },
+        `failed to get logs for operation "%s" in project "%s"`,
+        operationId,
+        projectId,
+      )
+    }
+
+    // 2. if not found or failed, try to get logs from the state repository
+    try {
+      // if no logs are found in the hot state, load them from the state repository
+      return await this.stateManager
+        .getInstanceLogIndexRepository(projectId, operationId, instanceId)
+        .getAll()
+    } catch (error) {
+      this.logger.error(
+        { projectId, operationId, instanceId, error },
+        `failed to get logs for operation "%s" in project "%s"`,
+        operationId,
+        projectId,
+      )
+
+      return []
+    }
+  }
+}