@highstate/backend 0.9.15 → 0.9.18

package/src/business/instance-lock.ts

@@ -0,0 +1,125 @@
+import type { Logger } from "pino"
+import type { LockManager } from "../lock"
+import type { PubSubManager } from "../pubsub"
+import type { StateManager } from "../state"
+import type { InstanceId, ObjectMeta } from "@highstate/contract"
+import { compareInstanceLockSpecs, type InstanceLockSpec } from "../shared"
+
+export class InstanceLockService {
+  constructor(
+    private readonly stateManager: StateManager,
+    private readonly lockManager: LockManager,
+    private readonly pubsubManager: PubSubManager,
+    private readonly logger: Logger,
+  ) {}
+
+  /**
+   * Tries to lock instances in the project.
+   * Returns true if the lock was acquired successfully, false otherwise.
+   *
+   * @param projectId The project ID to lock instances in.
+   * @param instanceIds The instance IDs to lock.
+   * @param lockMeta The metadata for the lock.
+   * @param spec The lock specification containing the lock type and other parameters.
+   * @param allowPartialLock Whether to allow partial locking of instances.
+   * If true, it will lock as many instances as possible even if some are already locked.
+   *
+   * @return A tuple containing a boolean indicating if the lock was acquired,
+   * and an array of instance IDs that were successfully locked.
+   * If `allowPartialLock` is false and some instances are already locked,
+   * it will return false and an empty array.
+   */
+  async tryLockInstances(
+    projectId: string,
+    instanceIds: InstanceId[],
+    lockMeta: ObjectMeta,
+    spec: InstanceLockSpec,
+    allowPartialLock = false,
+  ): Promise<[locked: boolean, lockedInstanceIds: string[]]> {
+    const lockKeys = instanceIds.map(id => ["instance-lock", projectId, id] as const)
+
+    // acquire the hard lock to ensure no other process can manage the lock entries
+    return await this.lockManager.acquire(lockKeys, async () => {
+      // check if the instances are already locked
+      const existingLocks = await this.stateManager
+        .getInstanceLockRepository(projectId)
+        .getManyItems(instanceIds)
+
+      // only the locks with another spec are considered as the conflicting locks
+      const conflictingLocks = existingLocks.filter(
+        lock => !compareInstanceLockSpecs(lock.spec, spec),
+      )
+
+      if (conflictingLocks.length > 0) {
+        this.logger.debug(
+          {
+            projectId,
+            requestedInstanceIds: instanceIds,
+            lockedInstanceIds: conflictingLocks.map(lock => lock.id),
+          },
+          `failed to lock %s of %s instances in project "%s"`,
+          conflictingLocks.length,
+          instanceIds.length,
+          projectId,
+        )
+
+        if (!allowPartialLock) {
+          // if partial locking is not allowed, return false
+          return [false, []]
+        }
+
+        // if partial locking is allowed, filter out the already locked instances and proceed with the rest
+        instanceIds = instanceIds.filter(id => !conflictingLocks.some(lock => lock.id === id))
+      }
+
+      const newLocks = instanceIds.map(id => ({ id, projectId, meta: lockMeta, spec }))
+
+      // persist the new locks
+      await this.stateManager.getInstanceLockRepository(projectId).putManyItems(newLocks)
+
+      this.logger.debug(
+        {
+          projectId,
+          instanceIds: newLocks.map(lock => lock.id),
+          spec,
+        },
+        `locked %s instances in project "%s"`,
+        newLocks.length,
+        projectId,
+      )
+
+      // publish the lock event
+      await this.pubsubManager.publish(["instance-lock", projectId], {
+        type: "locked",
+        locks: newLocks,
+      })
+
+      return [true, newLocks.map(lock => lock.id)]
+    })
+  }
+
+  /**
+   * Unlocks instances in the project unconditionally.
+   * This will remove the locks regardless of their current state.
+   *
+   * @param projectId The project ID to unlock instances in.
+   * @param instanceIds The instance IDs to unlock.
+   */
+  async unlockInstancesUnconditionally(projectId: string, instanceIds: string[]): Promise<void> {
+    this.logger.debug(
+      { projectId, instanceIds },
+      `unconditionally unlocking %s instances in project "%s"`,
+      instanceIds.length,
+      projectId,
+    )
+
+    // remove the locks from the state
+    await this.stateManager.getInstanceLockRepository(projectId).deleteMany(instanceIds)
+
+    // publish the unlock event
+    await this.pubsubManager.publish(["instance-lock", projectId], {
+      type: "unlocked",
+      instanceIds,
+    })
+  }
+}

package/src/business/instance-state.ts

@@ -0,0 +1,434 @@
+import type { HotStateManager } from "../hotstate"
+import type { LockManager } from "../lock"
+import type { PubSubManager } from "../pubsub"
+import type { StateManager } from "../state"
+import type { Logger } from "pino"
+import type { RunnerBackend } from "../runner"
+import type { ArtifactService } from "../artifact"
+import { randomBytes } from "node:crypto"
+import { parseInstanceId } from "@highstate/contract"
+import {
+  applyStatePatch,
+  isStableOperationStatus,
+  isStateEmpty,
+  type InstanceCustomStatus,
+  type InstanceState,
+  type InstanceStatePatch,
+  type Operation,
+} from "../shared"
+
+export class InstanceStateService {
+  constructor(
+    private readonly stateManager: StateManager,
+    private readonly hotStateManager: HotStateManager,
+    private readonly pubsubManager: PubSubManager,
+    private readonly lockManager: LockManager,
+    private readonly runnerBackend: RunnerBackend,
+    private readonly artifactManager: ArtifactService,
+    private readonly logger: Logger,
+  ) {}
+
+  /**
+   * Gets the current states of all instances in a project.
+   *
+   * @param projectId The ID of the project for which to retrieve instance states.
+   * @param checkLoaded Whether to check if instance states are loaded in the hot state.
+   * Can be set to false if you are sure that the states are already loaded (e.g., if updating states within operation).
+   */
+  async getInstanceStates(projectId: string, checkLoaded = true): Promise<InstanceState[]> {
+    return await this.withLoadedInstanceStates(projectId, checkLoaded, async () => {
+      const entries = await this.hotStateManager.hgetall(["instance-states", projectId])
+
+      return entries.map(([, value]) => value)
+    })
+  }
+
+  /**
+   * Gets the state of a specific instance in a project.
+   *
+   * @param projectId The ID of the project to which the instance belongs.
+   * @param instanceId The ID of the instance to retrieve the state for.
+   * @param checkLoaded Whether to check if instance states are loaded in the hot state.
+   * Can be set to false if you are sure that the states are already loaded (e.g., if updating states within operation).
+   */
+  async getInstanceState(
+    projectId: string,
+    instanceId: string,
+    checkLoaded = true,
+  ): Promise<InstanceState | null> {
+    return await this.withLoadedInstanceStates(projectId, checkLoaded, async () => {
+      return await this.hotStateManager.hget(["instance-states", projectId], instanceId)
+    })
+  }
+
+  /**
+   * Updates the states of multiple instances in a project.
+   *
+   * @param projectId The ID of the project to which the instances belong.
+   * @param states The array of instance states to update.
+   * @param persistent Whether to persist the changes in the state backend.
+   * @param checkLoaded Whether to check if instance states are loaded in the hot state.
+   * Can be set to false if you are sure that the states are already loaded (e.g., if updating states within operation).
+   */
+  async updateInstanceStates(
+    projectId: string,
+    states: InstanceState[],
+    persistent = false,
+    checkLoaded = true,
+  ): Promise<InstanceState[]> {
+    return await this.withLoadedInstanceStates(projectId, checkLoaded, async () => {
+      const batch = this.stateManager.batch()
+      const promises: Promise<void>[] = []
+
+      for (const state of states) {
+        promises.push(
+          this.hotStateManager.hset(["instance-states", projectId], state.id, state),
+          this.pubsubManager.publish(["instance-state", projectId], { type: "updated", state }),
+        )
+
+        if (persistent) {
+          promises.push(
+            this.stateManager.getInstanceStateRepository(projectId).putItem(state, batch),
+          )
+        }
+      }
+
+      await Promise.all(promises)
+      await batch.write()
+
+      this.logger.debug(
+        { projectId, states },
+        `instance states updated for project "%s"`,
+        projectId,
+      )
+
+      return states
+    })
+  }
+
+  /**
+   * Patches multiple instance states in a project.
+   *
+   * @param projectId The ID of the project to which the instances belong.
+   * @param patches The array of instance state patches to apply.
+   * @param checkLoaded Whether to check if instance states are loaded in the hot state.
+   * Can be set to false if you are sure that the states are already loaded (e.g., if updating states within operation).
+   */
+  async patchInstanceStates(
+    projectId: string,
+    patches: InstanceStatePatch[],
+    checkLoaded = true,
+  ): Promise<InstanceState[]> {
+    return await this.withLoadedInstanceStates(projectId, checkLoaded, async () => {
+      const batch = this.stateManager.batch()
+      const promises: Promise<InstanceState>[] = []
+
+      for (const patch of patches) {
+        promises.push(
+          this.hotStateManager.hget(["instance-states", projectId], patch.id).then(state => {
+            return this.applyStatePatch(projectId, state, patch, true)
+          }),
+        )
+      }
+
+      const updatedStates = await Promise.all(promises)
+
+      await batch.write()
+
+      this.logger.debug(
+        { projectId, patches },
+        `instance states patched for project "%s"`,
+        projectId,
+      )
+
+      return updatedStates
+    })
+  }
+
+  /**
+   * Patches an instance state in a project.
+   * TODO: delete
+   *
+   * @param projectId The ID of the project to which the instance belongs.
+   * @param patch The patch to apply to the instance state.
+   * @param checkLoaded Whether to check if instance states are loaded in the hot state.
+   * Can be set to false if you are sure that the states are already loaded (e.g., if updating states within operation).
+   */
+  async patchInstanceState(
+    projectId: string,
+    patch: InstanceStatePatch,
+    checkLoaded = true,
+  ): Promise<InstanceState> {
+    return await this.withLoadedInstanceStates(projectId, checkLoaded, async () => {
+      return await this.lockManager.acquire(["instance-state", projectId, patch.id], async () => {
+        const state = await this.hotStateManager.hget(["instance-states", projectId], patch.id)
+
+        const patchedState = await this.applyStatePatch(
+          projectId,
+          state,
+          patch,
+          patch.operationStatus !== undefined,
+        )
+
+        this.logger.debug(
+          { projectId, instanceId: patch.id, patch },
+          `instance state updated for instance "%s" in project "%s"`,
+          patch.id,
+          projectId,
+        )
+
+        return patchedState
+      })
+    })
+  }
+
+  /**
+   * Patches an instance state within an operation.
+   *
+   * @param projectId The ID of the project to which the instance belongs.
+   * @param operation The operation context for which the instance state is being patched.
+   * @param patch The patch to apply to the instance state.
+   */
+  async patchOperationInstanceState(
+    projectId: string,
+    operation: Operation,
+    patch: InstanceStatePatch,
+  ): Promise<InstanceState> {
+    return await this.lockManager.acquire(["instance-state", projectId, patch.id], async () => {
+      const state = await this.hotStateManager.hget(["instance-states", projectId], patch.id)
+
+      const patchedState = await this.applyStatePatch(
+        projectId,
+        state,
+        patch,
+        operation.type !== "preview",
+      )
+
+      this.logger.debug(
+        { projectId, instanceId: patch.id, patch },
+        `instance state updated for instance "%s" in project "%s"`,
+        patch.id,
+        projectId,
+      )
+
+      return patchedState
+    })
+  }
+
+  /**
+   * Updates the secret names of an instance state.
+   *
+   * @param projectId The ID of the project to which the instance belongs.
+   * @param instanceId The ID of the instance whose state is to be updated.
+   * @param secretNamesToAdd The list of secret names to add to the instance state.
+   * @param secretNamesToRemove The list of secret names to remove from the instance state.
+   * @param invalidateState Whether to invalidate the state after updating secret names.
+   * @param checkLoaded Whether to check if instance states are loaded in the hot state.
+   */
+  async updateStateSecretNames(
+    projectId: string,
+    instanceId: string,
+    secretNamesToAdd: string[],
+    secretNamesToRemove: string[],
+    invalidateState = true,
+    checkLoaded = true,
+  ): Promise<void> {
+    return await this.withLoadedInstanceStates(projectId, checkLoaded, async () => {
+      await this.lockManager.acquire(["instance-state", projectId, instanceId], async () => {
+        const state = await this.hotStateManager.hget(["instance-states", projectId], instanceId)
+
+        const secretNameSet = new Set(state?.secretNames ?? [])
+        for (const name of secretNamesToAdd) {
+          secretNameSet.add(name)
+        }
+
+        for (const name of secretNamesToRemove) {
+          secretNameSet.delete(name)
+        }
+
+        const patch: InstanceStatePatch = {
+          id: instanceId,
+          secretNames: Array.from(secretNameSet),
+        }
+
+        if (state?.operationStatus && invalidateState) {
+          // invalidate the input hash nonce if the operation status is present
+          // if not, the invalidation makes no sense
+          patch.operationStatus = {
+            inputHashNonce: randomBytes(4).readInt32LE(),
+          }
+        }
+
+        await this.applyStatePatch(projectId, state, patch, true)
+      })
+    })
+  }
+
+  /**
+   * Completely deletes the instance state from the system including all related resources.
+   *
+   * @param projectId The ID of the project from which to delete the instance state.
+   * @param instanceId The ID of the instance whose state is to be deleted.
+   */
+  async deleteInstanceState(projectId: string, instanceId: string): Promise<void> {
+    await this.lockManager.acquire(["instance-state", projectId, instanceId], async () => {
+      const [instanceType, instanceName] = parseInstanceId(instanceId)
+      const state = await this.getInstanceState(projectId, instanceId)
+
+      const batch = this.stateManager.batch()
+
+      await Promise.allSettled([
+        // remove from the state
+        this.stateManager.getInstanceStateRepository(projectId).delete(instanceId, batch),
+
+        // clear related resources
+        this.stateManager.getInstanceLockRepository(projectId).delete(instanceId, batch),
+
+        this.stateManager
+          .getPageRepository(projectId)
+          .deleteMany(Object.values(state?.extra?.pageIds ?? {}), batch),
+
+        this.stateManager
+          .getTriggerRepository(projectId)
+          .deleteMany(Object.values(state?.extra?.triggerIds ?? {}), batch),
+
+        this.stateManager
+          .getTerminalRepository(projectId)
+          .deleteMany(Object.values(state?.extra?.terminalIds ?? {}), batch),
+      ])
+
+      await Promise.allSettled([
+        // commit the state batch
+        batch.write(),
+
+        // update hotstate
+        this.hotStateManager.hdel(["instance-states", projectId], instanceId),
+
+        // publish the instance state deletion event
+        this.pubsubManager.publish(["instance-state", projectId], { type: "deleted", instanceId }),
+
+        // remove all artifact references
+        this.artifactManager.removeUsages(
+          projectId,
+          Object.values(state?.extra?.exportedArtifactIds ?? {}).flat(),
+          [{ type: "instance", instanceId }],
+        ),
+
+        // clear pulumi state
+        this.runnerBackend.deleteState({ projectId, instanceName, instanceType }),
+      ])
+    })
+  }
+
+  async updateCustomStatus(
+    projectId: string,
+    instanceId: string,
+    status: InstanceCustomStatus,
+  ): Promise<void> {
+    await this.lockManager.acquire(["instance-state", projectId, instanceId], async () => {
+      const state = await this.hotStateManager.hget(["instance-states", projectId], instanceId)
+      if (!state) {
+        throw new Error(
+          `Instance state with ID "${instanceId}" not found in project "${projectId}"`,
+        )
+      }
+
+      const existingStatus = state.extra?.customStatuses?.find(s => s.name === status.name)
+      if (existingStatus) {
+        Object.assign(existingStatus, status)
+      } else {
+        state.extra ??= {}
+        state.extra.customStatuses ??= []
+        state.extra.customStatuses?.push(status)
+      }
+
+      await this.applyStatePatch(projectId, state, { id: instanceId, extra: state.extra })
+    })
+  }
+
+  private async withLoadedInstanceStates<T>(
+    projectId: string,
+    checkLoaded: boolean,
+    action: () => Promise<T>,
+  ): Promise<T> {
+    if (!checkLoaded) {
+      return await action()
+    }
+
+    return await this.lockManager.acquire(["project-instance-states", projectId], async () => {
+      const instanceStatesLoaded = await this.hotStateManager.get([
+        "instance-states-loaded",
+        projectId,
+      ])
+
+      if (instanceStatesLoaded) {
+        this.logger.debug(`instance states already loaded for project "%s"`, projectId)
+        return await action()
+      }
+
+      // load instance states from the storage
+      const states = await this.stateManager.getInstanceStateRepository(projectId).getAllItems()
+      // store them in the hot state
+      await this.hotStateManager.hmset(
+        ["instance-states", projectId],
+        states.map(state => [state.id, state]),
+      )
+
+      this.logger.info(
+        { projectId, count: states.length },
+        `loaded %d instance states`,
+        states.length,
+      )
+
+      // mark instance states as loaded
+      await this.hotStateManager.set(["instance-states-loaded", projectId], true)
+
+      return await action()
+    })
+  }
+
+  private async applyStatePatch(
+    projectId: string,
+    state: InstanceState | null,
+    patch: InstanceStatePatch,
+    persistent = true,
+  ): Promise<InstanceState> {
+    const patchedState = applyStatePatch(state, patch)
+
+    const promises: Promise<void>[] = [
+      this.hotStateManager.hset(["instance-states", projectId], patchedState.id, patchedState),
+    ]
+
+    const shouldPersist =
+      persistent &&
+      (!patchedState.operationStatus ||
+        isStableOperationStatus(patchedState.operationStatus?.status))
+
+    const repo = this.stateManager.getInstanceStateRepository(projectId)
+
+    if (isStateEmpty(patchedState)) {
+      promises.push(
+        this.pubsubManager.publish(["instance-state", projectId], {
+          type: "deleted",
+          instanceId: patchedState.id,
+        }),
+      )
+
+      if (shouldPersist) {
+        promises.push(repo.delete(patchedState.id))
+      }
+    } else {
+      promises.push(
+        this.pubsubManager.publish(["instance-state", projectId], { type: "patched", patch }),
+      )
+
+      if (shouldPersist) {
+        promises.push(repo.putItem(patchedState))
+      }
+    }
+
+    await Promise.allSettled(promises)
+
+    return patchedState
+  }
+}