@highstate/backend 0.9.15 → 0.9.16

package/src/common/utils.ts

@@ -34,13 +34,25 @@ export function isAbortError(error: unknown): boolean {
   return error instanceof Error && error.name === "AbortError"
 }
 
-const abortMessagePatterns = ["Operation aborted", "Command was killed with SIGINT"]
+const abortMessagePatterns = [
+  "Operation aborted",
+  "This operation was aborted",
+  "Command was killed with SIGINT",
+]
 
 export function isAbortErrorLike(error: unknown): boolean {
+  if (isAbortError(error)) {
+    return true
+  }
+
   if (error instanceof Error) {
     return abortMessagePatterns.some(pattern => error.message.includes(pattern))
   }
 
+  if (typeof error === "string") {
+    return abortMessagePatterns.some(pattern => error.includes(pattern))
+  }
+
   return false
 }
 
@@ -61,3 +73,30 @@ export function errorToString(error: unknown): string {
 
   return JSON.stringify(error)
 }
+
+export function valueToString(value: unknown): string {
+  if (typeof value === "string") {
+    return value
+  }
+
+  return JSON.stringify(value)
+}
+
+export function stringToValue(value: string): unknown {
+  try {
+    return JSON.parse(value)
+  } catch {
+    return value
+  }
+}
+
+export function waitForAbort(signal: AbortSignal): Promise<void> {
+  return new Promise(resolve => {
+    if (signal.aborted) {
+      resolve()
+      return
+    }
+
+    signal.addEventListener("abort", () => resolve(), { once: true })
+  })
+}

package/src/config.ts

@@ -1,28 +1,32 @@
 import { z } from "zod"
 import { libraryBackendConfig } from "./library"
 import { projectBackendConfig } from "./project"
-import { secretBackendConfig } from "./secret"
 import { terminalBackendConfig } from "./terminal"
 import { runnerBackendConfig } from "./runner"
-import { stateBackendConfig } from "./state"
-import { workspaceBackendConfig } from "./workspace"
+import { stateBackendConfig, stateManagerConfig } from "./state"
+import { artifactBackendConfig } from "./artifact"
+import { pubSubBackendConfig } from "./pubsub"
+import { lockBackendConfig } from "./lock"
+import { hotStateBackendConfig } from "./hotstate"
+import { workerBackendConfig, workerManagerConfig } from "./worker"
 
 const loggerConfig = z.object({
-  HIGHSTATE_BACKEND_LOGGER_NAME: z.string().default("highstate-backend"),
-
-  HIGHSTATE_BACKEND_LOGGER_LEVEL: z
-    .enum(["fatal", "error", "warn", "info", "debug", "trace"])
-    .default("info"),
+  HIGHSTATE_LOG_LEVEL: z.enum(["fatal", "error", "warn", "info", "debug", "trace"]).default("info"),
 })
 
 const configSchema = z.object({
+  ...hotStateBackendConfig.shape,
+  ...pubSubBackendConfig.shape,
+  ...lockBackendConfig.shape,
   ...libraryBackendConfig.shape,
   ...projectBackendConfig.shape,
-  ...secretBackendConfig.shape,
   ...stateBackendConfig.shape,
+  ...stateManagerConfig.shape,
   ...runnerBackendConfig.shape,
   ...terminalBackendConfig.shape,
-  ...workspaceBackendConfig.shape,
+  ...workerBackendConfig.shape,
+  ...workerManagerConfig.shape,
+  ...artifactBackendConfig.shape,
   ...loggerConfig.shape,
 })
 

package/src/hotstate/abstractions.ts

@@ -0,0 +1,48 @@
+import type { z } from "zod"
+
+export interface HotStateBackend {
+  /**
+   * Gets a value for a key.
+   */
+  get(key: string, schema: z.ZodType): Promise<unknown>
+
+  /**
+   * Checks if a key exists.
+   */
+  exists(key: string): Promise<boolean>
+
+  /**
+   * Sets a value for a key.
+   */
+  set(key: string, schema: z.ZodType, value: unknown): Promise<void>
+
+  /**
+   * Deletes a key.
+   */
+  delete(key: string): Promise<void>
+
+  /**
+   * Gets all key-value pairs from a hash set.
+   */
+  hgetall(key: string, schema: z.ZodType): Promise<Array<[string, unknown]>>
+
+  /**
+   * Checks if a field exists in a hash set.
+   */
+  hexists(key: string, field: string): Promise<boolean>
+
+  /**
+   * Sets a field in a hash set.
+   */
+  hset(key: string, field: string, schema: z.ZodType, value: unknown): Promise<void>
+
+  /**
+   * Gets a field from a hash set.
+   */
+  hget(key: string, field: string, schema: z.ZodType): Promise<unknown>
+
+  /**
+   * Deletes a field from a hash set.
+   */
+  hdel(key: string, field: string): Promise<void>
+}

package/src/hotstate/factory.ts

@@ -0,0 +1,17 @@
+import type { HotStateBackend } from "./abstractions"
+import { z } from "zod"
+import { MemoryHotStateBackend } from "./memory"
+
+export const hotStateBackendConfig = z.object({
+  HIGHSTATE_HOTSTATE_BACKEND_TYPE: z.enum(["memory"]).default("memory"),
+})
+
+export function createHotStateBackend(
+  config: z.infer<typeof hotStateBackendConfig>,
+): HotStateBackend {
+  switch (config.HIGHSTATE_HOTSTATE_BACKEND_TYPE) {
+    case "memory": {
+      return MemoryHotStateBackend.create()
+    }
+  }
+}

package/src/{secret → hotstate}/index.ts

@@ -1,2 +1,3 @@
 export * from "./abstractions"
 export * from "./factory"
+export * from "./manager"

package/src/hotstate/manager.ts

@@ -0,0 +1,192 @@
+import type { HotStateBackend } from "./abstractions"
+import { z } from "zod"
+import { instanceLockSchema, instanceStateSchema, operationSchema } from "../shared"
+
+export type DataMap = {
+  /**
+   * Master key for a project used to read and write encrypted data.
+   *
+   * If the key is not set, the project is considered locked.
+   */
+  "project-master-key": {
+    key: [projectId: string]
+    data: z.ZodType<Uint8Array>
+  }
+}
+
+const dataSchemas: Record<string, z.ZodType> = {
+  "project-master-key": z.instanceof(Uint8Array),
+}
+
+export type DataHashSetMap = {
+  "active-operations": {
+    key: [projectId: string]
+    data: typeof operationSchema
+  }
+  "instance-states": {
+    key: [projectId: string]
+    data: typeof instanceStateSchema
+  }
+  "instance-locks": {
+    key: [projectId: string]
+    data: typeof instanceLockSchema
+  }
+  "operation-logs": {
+    key: [operationId: string]
+    data: z.ZodString
+  }
+}
+
+const hashSetSchemas: Record<string, z.ZodType> = {
+  "active-operations": operationSchema,
+  "instance-states": instanceStateSchema,
+  "instance-locks": instanceLockSchema,
+  "operation-logs": z.string(),
+}
+
+type GenericKey =
+  | {
+      [K in keyof DataMap]: [type: K, ...DataMap[K]["key"]]
+    }[keyof DataMap]
+  | {
+      [K in keyof DataHashSetMap]: [type: K, ...DataHashSetMap[K]["key"]]
+    }[keyof DataHashSetMap]
+
+export class HotStateManager {
+  constructor(private readonly hotStateBackend: HotStateBackend) {}
+
+  /**
+   * Gets a value for a key.
+   *
+   * @param key The key to get the value for.
+   * @param schema The Zod schema to validate the value.
+   * @returns The value or null if not found.
+   */
+  public async get<K extends keyof DataMap>(
+    key: [type: K, ...DataMap[K]["key"]],
+  ): Promise<z.infer<DataMap[K]["data"]> | null> {
+    const result = await this.hotStateBackend.get(key.join(":"), dataSchemas[key[0]])
+
+    return result as z.infer<DataMap[K]["data"]> | null
+  }
+
+  /**
+   * Checks if a key exists.
+   *
+   * @param key The key to check.
+   * @returns True if the key exists, false otherwise.
+   */
+  public async exists(key: GenericKey): Promise<boolean> {
+    return await this.hotStateBackend.exists(key.join(":"))
+  }
+
+  /**
+   * Sets a value for a key.
+   *
+   * @param key The key to set the value for.
+   * @param value The value to set.
+   */
+  public async set<K extends keyof DataMap>(
+    key: [type: K, ...DataMap[K]["key"]],
+    value: z.infer<DataMap[K]["data"]>,
+  ): Promise<void> {
+    return await this.hotStateBackend.set(key.join(":"), dataSchemas[key[0]], value)
+  }
+
+  /**
+   * Deletes a key.
+   *
+   * @param key The key to delete.
+   */
+  public async delete(key: GenericKey): Promise<void> {
+    return await this.hotStateBackend.delete(key.join(":"))
+  }
+
+  /**
+   * Gets all key-value pairs from a hash set.
+   *
+   * @param key The hash set key.
+   * @returns All key-value pairs in the hash set.
+   */
+  public async hgetall<K extends keyof DataHashSetMap>(
+    key: [type: K, ...DataHashSetMap[K]["key"]],
+  ): Promise<[string, z.infer<DataHashSetMap[K]["data"]>][]> {
+    const result = await this.hotStateBackend.hgetall(key.join(":"), hashSetSchemas[key[0]])
+
+    return result as [string, z.infer<DataHashSetMap[K]["data"]>][]
+  }
+
+  /**
+   * Checks if a field exists in a hash set.
+   *
+   * @param key The hash set key.
+   * @param field The field name.
+   * @returns True if the field exists, false otherwise.
+   */
+  public async hexists<K extends keyof DataHashSetMap>(
+    key: [type: K, ...DataHashSetMap[K]["key"]],
+    field: string,
+  ): Promise<boolean> {
+    return await this.hotStateBackend.hexists(key.join(":"), field)
+  }
+
+  /**
+   * Sets a field in a hash set.
+   *
+   * @param key The hash set key.
+   * @param field The field name.
+   * @param value The value to set.
+   */
+  public async hset<K extends keyof DataHashSetMap>(
+    key: [type: K, ...DataHashSetMap[K]["key"]],
+    field: string,
+    value: z.infer<DataHashSetMap[K]["data"]>,
+  ): Promise<void> {
+    return await this.hotStateBackend.hset(key.join(":"), field, hashSetSchemas[key[0]], value)
+  }
+
+  /**
+   * Sets multiple fields in a hash set.
+   *
+   * @param key The hash set key.
+   * @param entries An array of field-value pairs to set.
+   */
+  public async hmset<K extends keyof DataHashSetMap>(
+    key: [type: K, ...DataHashSetMap[K]["key"]],
+    entries: Array<[string, z.infer<DataHashSetMap[K]["data"]>]>,
+  ): Promise<void> {
+    const promises = entries.map(async ([field, value]) => this.hset(key, field, value))
+
+    await Promise.allSettled(promises)
+  }
+
+  /**
+   * Gets a field from a hash set.
+   *
+   * @param key The hash set key.
+   * @param field The field name.
+   * @returns The value or null if not found.
+   */
+  public async hget<K extends keyof DataHashSetMap>(
+    key: [type: K, ...DataHashSetMap[K]["key"]],
+    field: string,
+  ): Promise<z.infer<DataHashSetMap[K]["data"]> | null> {
+    const result = await this.hotStateBackend.hget(key.join(":"), field, hashSetSchemas[key[0]])
+
+    return result as z.infer<DataHashSetMap[K]["data"]> | null
+  }
+
+  /**
+   * Deletes a field from a hash set.
+   *
+   * @param key The hash set key.
+   * @param field The field name.
+   * @returns A promise that resolves when the field is deleted.
+   */
+  public async hdel<K extends keyof DataHashSetMap>(
+    key: [type: K, ...DataHashSetMap[K]["key"]],
+    field: string,
+  ): Promise<void> {
+    return await this.hotStateBackend.hdel(key.join(":"), field)
+  }
+}

package/src/hotstate/memory.ts

@@ -0,0 +1,100 @@
+import type { z } from "zod"
+import type { HotStateBackend } from "./abstractions"
+
+export class MemoryHotStateBackend implements HotStateBackend {
+  private readonly storage = new Map<string, unknown>()
+
+  get<TSchema extends z.ZodSchema>(key: string): Promise<z.infer<TSchema> | null> {
+    const value = this.storage.get(key)
+    if (value === undefined || value instanceof Map) {
+      return Promise.resolve(null)
+    }
+    return Promise.resolve(value as z.infer<TSchema>)
+  }
+
+  exists(key: string): Promise<boolean> {
+    const exists = this.storage.has(key) && !(this.storage.get(key) instanceof Map)
+    return Promise.resolve(exists)
+  }
+
+  set<TSchema extends z.ZodSchema>(
+    key: string,
+    _schema: TSchema,
+    value: z.infer<TSchema>,
+  ): Promise<void> {
+    this.storage.set(key, value)
+    return Promise.resolve()
+  }
+
+  delete(key: string): Promise<void> {
+    this.storage.delete(key)
+    return Promise.resolve()
+  }
+
+  hgetall<TSchema extends z.ZodSchema>(key: string): Promise<Array<[string, z.infer<TSchema>]>> {
+    const value = this.storage.get(key)
+    if (!(value instanceof Map)) {
+      return Promise.resolve([])
+    }
+
+    const result: Array<[string, z.infer<TSchema>]> = []
+    for (const [field, fieldValue] of value.entries()) {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+      result.push([field, fieldValue as z.infer<TSchema>])
+    }
+    return Promise.resolve(result)
+  }
+
+  hexists(key: string, field: string): Promise<boolean> {
+    const hashSet = this.storage.get(key)
+    if (!(hashSet instanceof Map)) {
+      return Promise.resolve(false)
+    }
+
+    const exists = hashSet.has(field)
+    return Promise.resolve(exists)
+  }
+
+  hset<TSchema extends z.ZodSchema>(
+    key: string,
+    field: string,
+    _schema: TSchema,
+    value: z.infer<TSchema>,
+  ): Promise<void> {
+    let hashSet = this.storage.get(key)
+    if (!(hashSet instanceof Map)) {
+      hashSet = new Map<string, unknown>()
+      this.storage.set(key, hashSet)
+    }
+
+    ;(hashSet as Map<string, unknown>).set(field, value)
+    return Promise.resolve()
+  }
+
+  hget<TSchema extends z.ZodSchema>(key: string, field: string): Promise<z.infer<TSchema> | null> {
+    const hashSet = this.storage.get(key)
+    if (!(hashSet instanceof Map)) {
+      return Promise.resolve(null)
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+    const value = hashSet.get(field)
+    return Promise.resolve(value === undefined ? null : (value as z.infer<TSchema>))
+  }
+
+  hdel(key: string, field: string): Promise<void> {
+    const hashSet = this.storage.get(key)
+    if (hashSet instanceof Map) {
+      hashSet.delete(field)
+      // If the hash set becomes empty, remove it from storage
+      if (hashSet.size === 0) {
+        this.storage.delete(key)
+      }
+    }
+    return Promise.resolve()
+  }
+
+  static create(): HotStateBackend {
+    return new MemoryHotStateBackend()
+  }
+}

package/src/hotstate/validation.ts

@@ -0,0 +1,101 @@
+import type { Logger } from "pino"
+import type { HotStateBackend } from "./abstractions"
+import type { z } from "zod"
+
+export class HotStateValidationDecorator implements HotStateBackend {
+  constructor(
+    private readonly hotstateBackend: HotStateBackend,
+    private readonly logger: Logger,
+  ) {}
+
+  async get(key: string, schema: z.ZodType): Promise<unknown> {
+    const value = await this.hotstateBackend.get(key, schema)
+
+    if (value === null) {
+      return null
+    }
+
+    try {
+      return schema.parse(value)
+    } catch (error) {
+      this.logger.error({ key, error }, `invalid value for key "${key}", returning null`)
+      return null
+    }
+  }
+
+  async exists(key: string): Promise<boolean> {
+    return this.hotstateBackend.exists(key)
+  }
+
+  async set(key: string, schema: z.ZodType, value: unknown): Promise<void> {
+    try {
+      schema.parse(value)
+    } catch (error) {
+      throw new Error(`Failed to validate value for key "${key}"`, { cause: error })
+    }
+
+    await this.hotstateBackend.set(key, schema, value)
+  }
+
+  async delete(key: string): Promise<void> {
+    return this.hotstateBackend.delete(key)
+  }
+
+  async hgetall(key: string, schema: z.ZodType): Promise<Array<[string, unknown]>> {
+    const entries = await this.hotstateBackend.hgetall(key, schema)
+    const validEntries: Array<[string, unknown]> = []
+
+    for (const [field, value] of entries) {
+      try {
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+        const validValue = schema.parse(value)
+        validEntries.push([field, validValue])
+      } catch (error) {
+        this.logger.error(
+          { key, field, error },
+          `skipped invalid value for field "${field}" in hash "${key}"`,
+        )
+      }
+    }
+
+    return validEntries
+  }
+
+  async hexists(key: string, field: string): Promise<boolean> {
+    return this.hotstateBackend.hexists(key, field)
+  }
+
+  async hset(key: string, field: string, schema: z.ZodType, value: unknown): Promise<void> {
+    try {
+      schema.parse(value)
+    } catch (error) {
+      throw new Error(`Failed to validate value for field "${field}" in hash "${key}"`, {
+        cause: error,
+      })
+    }
+
+    await this.hotstateBackend.hset(key, field, schema, value)
+  }
+
+  async hget(key: string, field: string, schema: z.ZodType): Promise<unknown> {
+    const value = await this.hotstateBackend.hget(key, field, schema)
+
+    if (value === null) {
+      return null
+    }
+
+    try {
+      return schema.parse(value)
+    } catch (error) {
+      this.logger.error(
+        { key, field, error },
+        `invalid value for field "${field}" in hash "${key}", returning null`,
+      )
+      return null
+    }
+  }
+
+  async hdel(key: string, field: string): Promise<void> {
+    return this.hotstateBackend.hdel(key, field)
+  }
+}

package/src/index.ts

@@ -1,6 +1,7 @@
-export * from "./secret"
 export * from "./library"
 export * from "./config"
 export * from "./orchestrator"
 export * from "./terminal"
 export * from "./services"
+export * from "./state"
+export * from "./business"

package/src/library/abstractions.ts

@@ -3,22 +3,11 @@ import type { LibraryModel, LibraryUpdate, ResolvedInstanceInput } from "../shar
 
 export type ResolvedUnitSource = {
   unitType: string
-  sourceHash: string
+  sourceHash: number
   projectPath: string
   allowedDependencies: string[]
 }
 
-export type ModuleEvaluationResult =
-  | {
-      success: true
-      compositeInstances: CompositeInstance[]
-    }
-  | {
-      success: false
-      modulePath: string
-      error: string
-    }
-
 export type InstanceEvaluationResult =
   | {
       success: true
@@ -35,44 +24,42 @@ export interface LibraryBackend {
   /**
    * Loads the library.
    */
-  loadLibrary(signal?: AbortSignal): Promise<LibraryModel>
+  loadLibrary(libraryId: string, signal?: AbortSignal): Promise<LibraryModel>
 
   /**
    * Watches the library for changes.
    */
-  watchLibrary(signal?: AbortSignal): AsyncIterable<LibraryUpdate[]>
+  watchLibrary(libraryId: string, signal?: AbortSignal): AsyncIterable<LibraryUpdate[]>
 
   /**
    * Gets the resolved unit sources for the given unit types.
    *
    * If the packages for these units are not resolved, it will resolve them and include in watch list.
    */
-  getResolvedUnitSources(unitTypes: string[]): Promise<ResolvedUnitSource[]>
+  getResolvedUnitSources(libraryId: string, unitTypes: string[]): Promise<ResolvedUnitSource[]>
 
   /**
    * Watches the resolved unit sources for changes.
    * Returns an async iterable that emits each resolved unit source whenever it changes.
    * Does not emit the resolved unit sources for units that have not changed even if the library was reloaded.
    */
-  watchResolvedUnitSources(signal?: AbortSignal): AsyncIterable<ResolvedUnitSource>
+  watchResolvedUnitSources(
+    libraryId: string,
+    signal?: AbortSignal,
+  ): AsyncIterable<ResolvedUnitSource>
 
   /**
    * Evaluates the instances and returns the evaluated composite instances.
    *
+   * @param libraryId The library ID to use for evaluation.
    * @param allInstances The all instances of the project.
    * @param resolvedInputs The resolved inputs of the instances.
    * @param instanceIds The instance ids to evaluate.
    */
   evaluateCompositeInstances(
+    libraryId: string,
     allInstances: InstanceModel[],
     resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
     instanceIds: string[],
   ): Promise<InstanceEvaluationResult[]>
-
-  /**
-   * Evaluates the modules and returns the evaluated instances.
-   *
-   * @param modulePaths The module paths to evaluate.
-   */
-  evaluateModules(modulePaths: string[]): Promise<ModuleEvaluationResult>
 }

package/src/library/factory.ts

@@ -4,7 +4,7 @@ import { z } from "zod"
 import { LocalLibraryBackend, localLibraryBackendConfig } from "./local"
 
 export const libraryBackendConfig = z.object({
-  HIGHSTATE_BACKEND_LIBRARY_TYPE: z.enum(["local"]).default("local"),
+  HIGHSTATE_LIBRARY_BACKEND_TYPE: z.enum(["local"]).default("local"),
   ...localLibraryBackendConfig.shape,
 })
 
@@ -12,7 +12,7 @@ export async function createLibraryBackend(
   config: z.infer<typeof libraryBackendConfig>,
   logger: Logger,
 ): Promise<LibraryBackend> {
-  switch (config.HIGHSTATE_BACKEND_LIBRARY_TYPE) {
+  switch (config.HIGHSTATE_LIBRARY_BACKEND_TYPE) {
     case "local": {
       return await LocalLibraryBackend.create(config, logger)
     }