@highstate/backend 0.9.14 → 0.9.16

package/src/library/local.ts

@@ -1,15 +1,10 @@
-import type {
-  InstanceEvaluationResult,
-  LibraryBackend,
-  ModuleEvaluationResult,
-  ResolvedUnitSource,
-} from "./abstractions"
+import type { InstanceEvaluationResult, LibraryBackend, ResolvedUnitSource } from "./abstractions"
 import type { Logger } from "pino"
 import type {
   PackageResolutionResponse,
   PackageResolutionWorkerData,
 } from "./package-resolution-worker"
-import type { WorkerRequest, WorkerResponse } from "./worker/protocol"
+import type { WorkerData, WorkerResponse } from "./worker/protocol"
 import { fileURLToPath } from "node:url"
 import { EventEmitter, on } from "node:events"
 import { Worker } from "node:worker_threads"
@@ -23,6 +18,7 @@ import { z } from "zod"
 import { readPackageJSON } from "pkg-types"
 import { runScript, installDependencies, addDependency } from "nypm"
 import { flatMap, groupBy, map, pipe, unique } from "remeda"
+import { decode } from "@msgpack/msgpack"
 import { resolveMainLocalProject, stringArrayType } from "../common"
 import {
   diffLibraries,
@@ -32,8 +28,8 @@ import {
 } from "../shared"
 
 export const localLibraryBackendConfig = z.object({
-  HIGHSTATE_BACKEND_LIBRARY_LOCAL_PACKAGES: stringArrayType.default("@highstate/library"),
-  HIGHSTATE_BACKEND_LIBRARY_LOCAL_WATCH_PATHS: stringArrayType.optional(),
+  HIGHSTATE_LIBRARY_BACKEND_LOCAL_PACKAGES: stringArrayType.default("@highstate/library"),
+  HIGHSTATE_LIBRARY_BACKEND_LOCAL_WATCH_PATHS: stringArrayType.optional(),
 })
 
 interface Events {
@@ -42,7 +38,7 @@ interface Events {
 }
 
 type HighstateManifestJson = {
-  sourceHashes?: Record<string, string>
+  sourceHashes?: Record<string, number>
 }
 
 type LibraryPackage = {
@@ -59,7 +55,6 @@ export class LocalLibraryBackend implements LibraryBackend {
   private readonly eventEmitter = new EventEmitter<Events>()
 
   private library: LibraryModel | null = null
-  private worker: Worker | null = null
 
   private readonly packages = new Map<string, LibraryPackage>()
   private readonly resolvedUnitSources = new Map<string, ResolvedUnitSource>()
@@ -88,13 +83,12 @@ export class LocalLibraryBackend implements LibraryBackend {
 
   async loadLibrary(): Promise<LibraryModel> {
     return await this.lock.acquire(async () => {
-      const [library] = await this.getLibrary()
-
-      return library
+      return await this.getLibrary()
     })
   }
 
-  async *watchLibrary(signal?: AbortSignal): AsyncIterable<LibraryUpdate[]> {
+  async *watchLibrary(_libraryId: string, signal?: AbortSignal): AsyncIterable<LibraryUpdate[]> {
+    // For local backend, we ignore libraryId since there's only one library
     for await (const [library] of on(this.eventEmitter, "library", { signal })) {
       yield library
     }
@@ -106,9 +100,12 @@ export class LocalLibraryBackend implements LibraryBackend {
     })
   }
 
-  async getResolvedUnitSources(unitTypes: string[]): Promise<ResolvedUnitSource[]> {
+  async getResolvedUnitSources(
+    _libraryId: string,
+    unitTypes: string[],
+  ): Promise<ResolvedUnitSource[]> {
     return await this.lock.acquire(async () => {
-      const [library] = await this.getLibrary()
+      const library = await this.getLibrary()
 
       const units = unitTypes
         .map(type => library.components[type])
@@ -135,7 +132,11 @@ export class LocalLibraryBackend implements LibraryBackend {
     })
   }
 
-  async *watchResolvedUnitSources(signal?: AbortSignal): AsyncIterable<ResolvedUnitSource> {
+  async *watchResolvedUnitSources(
+    _libraryId: string,
+    signal?: AbortSignal,
+  ): AsyncIterable<ResolvedUnitSource> {
+    // For local backend, we ignore libraryId since there's only one library
     for await (const [resolvedUnitSource] of on(this.eventEmitter, "resolvedUnitSource", {
       signal,
     })) {
@@ -144,49 +145,20 @@ export class LocalLibraryBackend implements LibraryBackend {
   }
 
   async evaluateCompositeInstances(
+    _libraryId: string,
     allInstances: InstanceModel[],
     resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
     instanceIds: string[],
   ): Promise<InstanceEvaluationResult[]> {
-    return await this.lock.acquire(async () => {
-      this.logger.info("evaluating %d composite instances", instanceIds.length)
-
-      const [, worker] = await this.getLibrary()
-      worker.postMessage({
-        type: "evaluate-composite-instances",
-        allInstances,
-        resolvedInputs,
-        instanceIds,
-      } satisfies WorkerRequest)
-
-      this.logger.debug("evaluation request sent")
-
-      const { results } = await this.getResult(worker, "instance-evaluation-results")
-      return results
-    })
-  }
-
-  async evaluateModules(modulePaths: string[]): Promise<ModuleEvaluationResult> {
-    return await this.lock.acquire(async () => {
-      this.logger.info({ msg: "evaluating modules", modulePaths })
-
-      const [, worker] = await this.getLibrary()
-      worker.postMessage({
-        type: "evaluate-modules",
-        modulePaths,
-      } satisfies WorkerRequest)
+    this.logger.info("evaluating %d composite instances", instanceIds.length)
 
-      this.logger.debug("evaluation request sent")
-
-      const { result } = await this.getResult(worker, "module-evaluation-result")
-      return result
+    const worker = this.createLibraryWorker({
+      libraryModulePaths: this.libraryPackages,
+      allInstances,
+      resolvedInputs,
+      instanceIds,
     })
-  }
 
-  private async getResult<TType extends WorkerResponse["type"]>(
-    worker: Worker,
-    expectedType: TType,
-  ): Promise<WorkerResponse & { type: TType }> {
     for await (const [event] of on(worker, "message")) {
       const eventData = event as WorkerResponse
 
@@ -194,69 +166,56 @@ export class LocalLibraryBackend implements LibraryBackend {
         throw new Error(`Worker error: ${eventData.error}`)
       }
 
-      if (eventData.type !== expectedType) {
-        throw new Error(
-          `Unexpected response message type "${eventData.type}", expected "${expectedType}"`,
-        )
-      }
-
-      return eventData as WorkerResponse & { type: TType }
+      this.logger.info("composite instances evaluated successfully")
+      return eventData.results
     }
 
     throw new Error("Worker ended without sending any response")
   }
 
-  private async getLibrary(): Promise<[LibraryModel, Worker]> {
-    if (this.library && this.worker) {
-      return [this.library, this.worker]
+  private async getLibrary(): Promise<LibraryModel> {
+    if (this.library) {
+      return this.library
     }
 
     return await this.reloadLibrary()
   }
 
-  private async reloadLibrary(): Promise<[LibraryModel, Worker]> {
+  private async reloadLibrary(): Promise<LibraryModel> {
     this.logger.info("reloading library")
 
     this.eventEmitter.emit("library", [{ type: "reload-started" }])
 
     await this.ensureLibraryPackagesLoaded(this.libraryPackages, true)
 
-    this.worker = this.createLibraryWorker({
-      modulePaths: this.libraryPackages,
-      logLevel: "warn",
-    })
+    const loadedPackages = this.packages
+      .values()
+      .filter(pkg => this.libraryPackages.includes(pkg.name))
 
-    for await (const [event] of on(this.worker, "message")) {
-      const eventData = event as {
-        type: string
-        library: LibraryModel
-        error?: string
-      }
+    const mergedLibrary: LibraryModel = { components: {}, entities: {} }
 
-      if (eventData.type === "error") {
-        throw new Error(`Worker error: ${eventData.error}`)
-      }
+    for (const loadedPackage of loadedPackages) {
+      const libraryContent = await this.readLibraryContent(loadedPackage)
 
-      if (eventData.type !== "library") {
-        throw new Error(`Unexpected message type '${eventData.type}', expected 'library'`)
+      for (const [componentType, component] of Object.entries(libraryContent.components)) {
+        mergedLibrary.components[componentType] = component
       }
 
-      const updates = diffLibraries(
-        this.library ?? { components: {}, entities: {} },
-        eventData.library,
-      )
+      for (const [entityType, entity] of Object.entries(libraryContent.entities)) {
+        mergedLibrary.entities[entityType] = entity
+      }
+    }
 
-      this.eventEmitter.emit("library", updates)
-      this.library = eventData.library
+    const updates = diffLibraries(this.library ?? { components: {}, entities: {} }, mergedLibrary)
 
-      this.logger.info("library reloaded")
+    this.eventEmitter.emit("library", updates)
+    this.library = mergedLibrary
 
-      this.eventEmitter.emit("library", [{ type: "reload-completed" }])
+    this.logger.info("library reloaded")
 
-      return [this.library, this.worker]
-    }
+    this.eventEmitter.emit("library", [{ type: "reload-completed" }])
 
-    throw new Error("Worker ended without sending library model")
+    return this.library
   }
 
   private async reloadUnitManifest(libraryPackage: LibraryPackage): Promise<void> {
@@ -347,9 +306,7 @@ export class LocalLibraryBackend implements LibraryBackend {
     this.logger.info(`rebuilding library package "${libraryPackage.name}" via build script`)
     await runScript("build", { cwd: libraryPackage.rootPath })
 
-    if (this.libraryPackages.includes(libraryPackage.name)) {
-      await this.reloadLibrary()
-    } else {
+    if (!this.libraryPackages.includes(libraryPackage.name)) {
       await this.reloadUnitManifest(libraryPackage)
     }
 
@@ -361,7 +318,7 @@ export class LocalLibraryBackend implements LibraryBackend {
     const parsedName = LocalLibraryBackend.parseDependencyName(libraryPackage.name)
 
     const dependencyPackageNames = pipe(
-      [packageJson.dependencies, packageJson.devDependencies, packageJson.peerDependencies],
+      [packageJson.dependencies, packageJson.peerDependencies],
       flatMap(deps => Object.keys(deps ?? {})),
       unique(),
       map(LocalLibraryBackend.parseDependencyName),
@@ -440,8 +397,27 @@ export class LocalLibraryBackend implements LibraryBackend {
     }
   }
 
+  private async readLibraryContent(libraryPackage: LibraryPackage): Promise<LibraryModel> {
+    const contentPath = resolve(libraryPackage.rootPath, "dist", "highstate.library.msgpack")
+
+    try {
+      const contentBuffer = await readFile(contentPath)
+      const content = decode(contentBuffer) as LibraryModel
+
+      return content
+    } catch (error) {
+      this.logger.debug(
+        { error },
+        `failed to read highstate library content of package: "%s"`,
+        libraryPackage.name,
+      )
+
+      return { components: {}, entities: {} }
+    }
+  }
+
   private async loadLibraryPackages(names: string[], installIfNotFound = false): Promise<void> {
-    this.logger.info("loading library packages: %s", names.join(", "))
+    this.logger.debug("loading library packages: %s", names.join(", "))
 
     const missingPackages: string[] = []
     const packagesToUpdate: LibraryPackage[] = []
@@ -484,7 +460,10 @@ export class LocalLibraryBackend implements LibraryBackend {
     for (const libraryPackage of packagesToUpdate) {
       await this.updateLibraryPackageDependencies(libraryPackage)
 
-      if (!this.libraryPackages.includes(libraryPackage.name)) {
+      if (
+        !this.libraryPackages.includes(libraryPackage.name) &&
+        libraryPackage.name !== "@highstate/contract"
+      ) {
         await this.reloadUnitManifest(libraryPackage)
       }
     }
@@ -500,13 +479,21 @@ export class LocalLibraryBackend implements LibraryBackend {
     await this.lock.acquire(async () => {
       const libraryPackage = this.packages.values().find(pkg => path.startsWith(pkg.rootPath))
 
-      if (libraryPackage) {
-        await this.rebuildLibraryPackage(libraryPackage)
+      if (!libraryPackage) {
+        return
+      }
+
+      const builtPackages = new Set<string>()
+      await this.rebuildLibraryPackage(libraryPackage, false, false, builtPackages)
+
+      if (this.libraryPackages.some(pkg => builtPackages.has(pkg))) {
+        this.logger.info("reloading library due to file change in package: %s", libraryPackage.name)
+        await this.reloadLibrary()
       }
     })
   }
 
-  private createLibraryWorker(workerData: Record<string, unknown>) {
+  private createLibraryWorker(workerData: WorkerData): Worker {
     const workerPathUrl = importMetaResolve(`@highstate/backend/library-worker`, import.meta.url)
     const workerPath = fileURLToPath(workerPathUrl)
 
@@ -524,14 +511,14 @@ export class LocalLibraryBackend implements LibraryBackend {
   }
 
   static async create(config: z.infer<typeof localLibraryBackendConfig>, logger: Logger) {
-    let watchPaths = config.HIGHSTATE_BACKEND_LIBRARY_LOCAL_WATCH_PATHS
+    let watchPaths = config.HIGHSTATE_LIBRARY_BACKEND_LOCAL_WATCH_PATHS
     if (!watchPaths) {
       const [projectPath] = await resolveMainLocalProject()
       watchPaths = [resolve(projectPath, "packages")]
     }
 
     return new LocalLibraryBackend(
-      config.HIGHSTATE_BACKEND_LIBRARY_LOCAL_PACKAGES,
+      config.HIGHSTATE_LIBRARY_BACKEND_LOCAL_PACKAGES,
       watchPaths,
       logger.child({ backend: "LibraryBackend", service: "LocalLibraryBackend" }),
     )

package/src/library/worker/evaluator.ts

@@ -1,50 +1,17 @@
 import type { Logger } from "pino"
-import type { Library } from "./loader"
-import type { Jiti } from "jiti"
-import type { InstanceEvaluationResult, ModuleEvaluationResult } from "../abstractions"
+import type { InstanceEvaluationResult } from "../abstractions"
 import type { ResolvedInstanceInput } from "../../shared"
-import { getCompositeInstances, resetEvaluation, type InstanceModel } from "@highstate/contract"
-import { BetterLock } from "better-lock"
+import {
+  getCompositeInstances,
+  resetEvaluation,
+  type Component,
+  type InstanceModel,
+} from "@highstate/contract"
 import { errorToString } from "../../common"
 
-const lock = new BetterLock()
-
-export function evaluateModules(
-  jiti: Jiti,
-  logger: Logger,
-  modulePaths: string[],
-): Promise<ModuleEvaluationResult> {
-  return lock.acquire(async () => {
-    resetEvaluation()
-    let lastModulePath = ""
-
-    try {
-      for (const modulePath of modulePaths) {
-        logger.info("loading module: %s", modulePath)
-
-        lastModulePath = modulePath
-        await jiti.import(modulePath)
-
-        logger.debug("module loaded: %s", modulePath)
-      }
-
-      return {
-        success: true,
-        compositeInstances: getCompositeInstances(),
-      }
-    } catch (error) {
-      return {
-        success: false,
-        modulePath: lastModulePath,
-        error: errorToString(error),
-      }
-    }
-  })
-}
-
 export function evaluateInstances(
   logger: Logger,
-  library: Library,
+  components: Readonly<Record<string, Component>>,
   allInstances: InstanceModel[],
   resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
   instanceIds: string[],
@@ -53,19 +20,19 @@ export function evaluateInstances(
   const allInstancesMap = new Map(allInstances.map(instance => [instance.id, instance]))
 
   for (const instanceId of instanceIds ?? []) {
-    const externalInstanceIds = new Set(allInstances.map(instance => instance.id))
-    externalInstanceIds.delete(instanceId)
-
     try {
       logger.debug({ instanceId }, "evaluating top-level instance")
-      resetEvaluation(externalInstanceIds)
+      resetEvaluation()
 
       evaluateInstance(instanceId)
 
       results.push({
         success: true,
         instanceId,
-        compositeInstances: getCompositeInstances(),
+        compositeInstances: getCompositeInstances().filter(
+          instance =>
+            instanceId.includes(instance.instance.id) || !allInstancesMap.has(instance.instance.id),
+        ),
       })
     } catch (error) {
       results.push({
@@ -113,7 +80,7 @@ export function evaluateInstances(
       })
     }
 
-    const component = library.components[instance.type]
+    const component = components[instance.type]
     if (!component) {
       throw new Error(`Component not found: ${instance.type}, required by instance: ${instanceId}`)
     }

package/src/library/worker/loader.lite.ts

@@ -0,0 +1,41 @@
+import type { Logger } from "pino"
+import { isComponent, type Component } from "@highstate/contract"
+
+export async function loadComponents(
+  logger: Logger,
+  modulePaths: string[],
+): Promise<Readonly<Record<string, Component>>> {
+  const modules: Record<string, unknown> = {}
+  for (const modulePath of modulePaths) {
+    try {
+      logger.debug({ modulePath }, "loading module")
+      modules[modulePath] = await import(modulePath)
+
+      logger.debug({ modulePath }, "module loaded")
+    } catch (err) {
+      logger.error({ modulePath, err }, "module load failed")
+    }
+  }
+
+  const components: Record<string, Component> = {}
+
+  await _loadLibrary(modules, components)
+  logger.debug("library loaded with %s components", Object.keys(components).length)
+
+  return components
+}
+
+async function _loadLibrary(value: unknown, components: Record<string, Component>): Promise<void> {
+  if (isComponent(value)) {
+    components[value.model.type] = value
+    return
+  }
+
+  if (typeof value !== "object" || value === null) {
+    return
+  }
+
+  for (const key in value) {
+    await _loadLibrary((value as Record<string, unknown>)[key], components)
+  }
+}

package/src/library/worker/main.ts

@@ -1,82 +1,31 @@
-import type { WorkerData, WorkerRequest, WorkerResponse } from "./protocol"
+import type { WorkerData, WorkerResponse } from "./protocol"
 import { parentPort, workerData } from "node:worker_threads"
-import { createJiti, type Jiti } from "jiti"
 import { pino } from "pino"
-import { mapValues } from "remeda"
 import { errorToString } from "../../common"
-import { loadLibrary, type Library } from "./loader"
-import { evaluateInstances, evaluateModules } from "./evaluator"
+import { evaluateInstances } from "./evaluator"
+import { loadComponents } from "./loader.lite"
 
 const data = workerData as WorkerData
 
 const logger = pino({ name: "library-worker", level: data.logLevel })
-let jiti: Jiti
-let library: Library
 
 try {
-  logger.info("worker started")
-  logger.trace({ data }, "worker data")
+  const library = await loadComponents(logger, data.libraryModulePaths)
 
-  jiti = createJiti(import.meta.filename)
-  logger.debug({ filename: import.meta.filename }, "jiti created")
+  const results = evaluateInstances(
+    logger,
+    library,
+    data.allInstances,
+    data.resolvedInputs,
+    data.instanceIds,
+  )
 
-  library = await loadLibrary(jiti, logger, data.modulePaths)
-
-  parentPort!.postMessage({
-    type: "library",
-    library: {
-      components: mapValues(library.components, component => component.model),
-      entities: library.entities,
-    },
-  } satisfies WorkerResponse)
-
-  logger.info("library loaded and sent")
+  parentPort!.postMessage({ type: "results", results } satisfies WorkerResponse)
 } catch (error) {
-  logger.error({ error }, "failed to load library")
+  logger.error({ error }, "failed to evaluate")
 
   parentPort!.postMessage({
     type: "error",
     error: errorToString(error),
-  })
+  } satisfies WorkerResponse)
 }
-
-// eslint-disable-next-line @typescript-eslint/no-misused-promises
-parentPort!.on("message", async message => {
-  try {
-    const request = message as WorkerRequest
-
-    switch (request.type) {
-      case "evaluate-composite-instances": {
-        const results = evaluateInstances(
-          logger,
-          library,
-          request.allInstances,
-          request.resolvedInputs,
-          request.instanceIds,
-        )
-
-        parentPort!.postMessage({
-          type: "instance-evaluation-results",
-          results,
-        } satisfies WorkerResponse)
-        break
-      }
-      case "evaluate-modules": {
-        const result = await evaluateModules(jiti, logger, request.modulePaths)
-
-        parentPort!.postMessage({
-          type: "module-evaluation-result",
-          result,
-        } satisfies WorkerResponse)
-        break
-      }
-    }
-  } catch (error) {
-    logger.error({ error }, "failed to evaluate")
-
-    parentPort!.postMessage({
-      type: "error",
-      error: errorToString(error),
-    } satisfies WorkerResponse)
-  }
-})

package/src/library/worker/protocol.ts

@@ -1,37 +1,21 @@
 import type { InstanceModel } from "@highstate/contract"
-import type { InstanceEvaluationResult, ModuleEvaluationResult } from "../abstractions"
-import type { LibraryModel, ResolvedInstanceInput } from "../../shared"
+import type { InstanceEvaluationResult } from "../abstractions"
+import type { ResolvedInstanceInput } from "../../shared"
 
 export type WorkerData = {
-  modulePaths: string[]
+  libraryModulePaths: string[]
   logLevel?: string
-}
 
-export type WorkerRequest =
-  | {
-      type: "evaluate-composite-instances"
-      allInstances: InstanceModel[]
-      resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>
-      instanceIds: string[]
-    }
-  | {
-      type: "evaluate-modules"
-      modulePaths: string[]
-    }
+  allInstances: InstanceModel[]
+  resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>
+  instanceIds: string[]
+}
 
 export type WorkerResponse =
   | {
-      type: "library"
-      library: LibraryModel
-    }
-  | {
-      type: "instance-evaluation-results"
+      type: "results"
       results: InstanceEvaluationResult[]
     }
-  | {
-      type: "module-evaluation-result"
-      result: ModuleEvaluationResult
-    }
   | {
       type: "error"
       error: string

package/src/lock/abstractions.ts

@@ -0,0 +1,6 @@
+export interface LockBackend {
+  /**
+   * Acquires locks for the given keys and executes the function.
+   */
+  acquire<T>(keys: string[], fn: () => Promise<T> | T): Promise<T>
+}

package/src/lock/factory.ts

@@ -0,0 +1,15 @@
+import type { LockBackend } from "./abstractions"
+import { z } from "zod"
+import { MemoryLockBackend } from "./memory"
+
+export const lockBackendConfig = z.object({
+  HIGHSTATE_LOCK_BACKEND_TYPE: z.enum(["memory"]).default("memory"),
+})
+
+export function createLockBackend(config: z.infer<typeof lockBackendConfig>): LockBackend {
+  switch (config.HIGHSTATE_LOCK_BACKEND_TYPE) {
+    case "memory": {
+      return MemoryLockBackend.create()
+    }
+  }
+}

package/src/{workspace → lock}/index.ts

@@ -1,2 +1,3 @@
 export * from "./abstractions"
 export * from "./factory"
+export * from "./manager"