@highstate/backend 0.7.8 → 0.7.10

package/src/library/local.ts

@@ -5,19 +5,24 @@ import type {
   ResolvedUnitSource,
 } from "./abstractions"
 import type { Logger } from "pino"
-import type { HighstateManifestJson, SourceResolutionResult } from "./source-resolution-worker"
+import type {
+  PackageResolutionResponse,
+  PackageResolutionWorkerData,
+} from "./package-resolution-worker"
 import type { WorkerRequest, WorkerResponse } from "./worker/protocol"
 import { fileURLToPath } from "node:url"
 import { EventEmitter, on } from "node:events"
 import { Worker } from "node:worker_threads"
-import { basename, dirname, relative, resolve } from "node:path"
+import { resolve } from "node:path"
 import { readFile } from "node:fs/promises"
-import { type InstanceModel, isUnitModel, type UnitModel } from "@highstate/contract"
+import { type InstanceModel, isUnitModel } from "@highstate/contract"
 import Watcher from "watcher"
 import { BetterLock } from "better-lock"
 import { resolve as importMetaResolve } from "import-meta-resolve"
 import { z } from "zod"
-import { readPackageJSON, resolvePackageJSON } from "pkg-types"
+import { readPackageJSON } from "pkg-types"
+import { runScript, installDependencies, addDependency } from "nypm"
+import { flatMap, groupBy, map, pipe, unique } from "remeda"
 import { resolveMainLocalProject, stringArrayType } from "../common"
 import {
   diffLibraries,
@@ -27,9 +32,8 @@ import {
 } from "../shared"
 
 export const localLibraryBackendConfig = z.object({
-  HIGHSTATE_BACKEND_LIBRARY_LOCAL_MODULES: stringArrayType.default("@highstate/library"),
-  HIGHSTATE_BACKEND_LIBRARY_LOCAL_SOURCE_BASE_PATH: z.string().optional(),
-  HIGHSTATE_BACKEND_LIBRARY_LOCAL_EXTRA_SOURCE_WATCH_PATHS: stringArrayType.default(""),
+  HIGHSTATE_BACKEND_LIBRARY_LOCAL_PACKAGES: stringArrayType.default("@highstate/library"),
+  HIGHSTATE_BACKEND_LIBRARY_LOCAL_WATCH_PATHS: stringArrayType.optional(),
 })
 
 interface Events {
@@ -37,9 +41,19 @@ interface Events {
   resolvedUnitSource: [ResolvedUnitSource]
 }
 
+type HighstateManifestJson = {
+  sourceHashes?: Record<string, string>
+}
+
+type LibraryPackage = {
+  name: string
+  rootPath: string
+  dependencies: Set<string>
+  dependents: Set<string>
+}
+
 export class LocalLibraryBackend implements LibraryBackend {
-  private readonly libraryWatcher: Watcher
-  private readonly sourceWatcher: Watcher
+  private readonly watcher: Watcher
 
   private readonly lock = new BetterLock()
   private readonly eventEmitter = new EventEmitter<Events>()
@@ -47,37 +61,29 @@ export class LocalLibraryBackend implements LibraryBackend {
   private library: LibraryModel | null = null
   private worker: Worker | null = null
 
+  private readonly packages = new Map<string, LibraryPackage>()
   private readonly resolvedUnitSources = new Map<string, ResolvedUnitSource>()
 
   private constructor(
-    private readonly modulePaths: string[],
-    private readonly sourceBasePath: string,
-    extraSourceWatchPaths: string[],
+    private readonly libraryPackages: string[],
+    watchPaths: string[],
     private readonly logger: Logger,
   ) {
-    this.libraryWatcher = new Watcher(modulePaths, { recursive: true, ignoreInitial: true })
-    this.libraryWatcher.on("all", (event: string, path: string) => {
-      const prefixPath = modulePaths.find(modulePath => path.startsWith(modulePath))
-
-      this.logger.info({ msg: "library event", event, path: relative(prefixPath!, path) })
-      void this.lock.acquire(() => this.updateLibrary())
-    })
-
-    this.sourceWatcher = new Watcher([sourceBasePath, ...extraSourceWatchPaths], {
+    this.watcher = new Watcher(watchPaths, {
       recursive: true,
       ignoreInitial: true,
-      ignore: /\.git|node_modules/,
+      ignore: /\.git|node_modules|dist/,
     })
 
-    this.sourceWatcher.on("all", (_, path: string) => {
-      if (!path.endsWith("highstate.manifest.json")) {
+    this.watcher.on("all", (event: string, path: string) => {
+      this.logger.debug({ event, path }, "library event")
+
+      if (!path.endsWith(".json") && !path.endsWith(".ts")) {
         return
       }
 
-      void this.updateUnitSourceHashes(path)
+      void this.handleFileEvent(path)
     })
-
-    this.logger.debug({ msg: "initialized", modulePaths })
   }
 
   async loadLibrary(): Promise<LibraryModel> {
@@ -94,29 +100,34 @@ export class LocalLibraryBackend implements LibraryBackend {
     }
   }
 
-  async getResolvedUnitSources(): Promise<readonly ResolvedUnitSource[]> {
-    return await this.lock.acquire(async () => {
-      // ensure library is loaded
-      const [library] = await this.getLibrary()
-
-      if (!this.resolvedUnitSources.size) {
-        await this.syncUnitSources(library)
-      }
-
+  getLoadedResolvedUnitSources(): Promise<ResolvedUnitSource[]> {
+    return this.lock.acquire(() => {
       return Array.from(this.resolvedUnitSources.values())
     })
   }
 
-  async getResolvedUnitSource(unitType: string): Promise<ResolvedUnitSource | null> {
+  async getResolvedUnitSources(unitTypes: string[]): Promise<ResolvedUnitSource[]> {
     return await this.lock.acquire(async () => {
-      // ensure library is loaded
       const [library] = await this.getLibrary()
 
-      if (!this.resolvedUnitSources.size) {
-        await this.syncUnitSources(library)
+      const units = unitTypes.map(type => library.components[type]).filter(isUnitModel)
+      const packageNames = Object.keys(groupBy(units, unit => unit.source.package))
+
+      await this.ensureLibraryPackagesLoaded(packageNames, true)
+
+      const result: ResolvedUnitSource[] = []
+
+      for (const unitType of unitTypes) {
+        const resolvedUnitSource = this.resolvedUnitSources.get(unitType)
+
+        if (resolvedUnitSource) {
+          result.push(resolvedUnitSource)
+        } else {
+          this.logger.warn(`resolved unit source not found for unit: "%s"`, unitType)
+        }
       }
 
-      return this.resolvedUnitSources.get(unitType) ?? null
+      return result
     })
   }
 
@@ -128,80 +139,6 @@ export class LocalLibraryBackend implements LibraryBackend {
     }
   }
 
-  async syncUnitSources(library: LibraryModel): Promise<void> {
-    const unitsToResolve = new Map<string, UnitModel>()
-
-    for (const component of Object.values(library.components)) {
-      if (!isUnitModel(component)) {
-        continue
-      }
-
-      const existingResolvedSource = this.resolvedUnitSources.get(component.type)
-      const expectedSource = JSON.stringify(component.source)
-
-      if (existingResolvedSource?.serializedSource !== expectedSource) {
-        unitsToResolve.set(component.type, component)
-      }
-    }
-
-    await this.runSourceResolution(unitsToResolve)
-  }
-
-  private async runSourceResolution(units: Map<string, UnitModel>): Promise<void> {
-    const workerPathUrl = importMetaResolve(
-      `@highstate/backend/source-resolution-worker`,
-      import.meta.url,
-    )
-    const workerPath = fileURLToPath(workerPathUrl)
-
-    const worker = new Worker(workerPath, {
-      workerData: {
-        requests: Array.from(units.values()).map(unit => ({
-          unitType: unit.type,
-          source: unit.source,
-        })),
-        sourceBasePath: this.sourceBasePath,
-        logLevel: "error",
-      },
-    })
-
-    for await (const [event] of on(worker, "message")) {
-      const eventData = event as {
-        type: string
-        results: readonly SourceResolutionResult[]
-      }
-
-      if (eventData.type !== "result") {
-        throw new Error(`Unexpected message type '${eventData.type}', expected 'result'`)
-      }
-
-      for (const result of eventData.results) {
-        const unit = units.get(result.unitType)
-        if (!unit) {
-          this.logger.warn("unit not found for resolved source", { unitType: result.unitType })
-          continue
-        }
-
-        const resolvedSource: ResolvedUnitSource = {
-          unitType: result.unitType,
-          serializedSource: JSON.stringify(unit.source),
-          projectPath: result.projectPath,
-          packageJsonPath: result.packageJsonPath,
-          allowedDependencies: result.allowedDependencies,
-          sourceHash: result.sourceHash,
-        }
-
-        this.resolvedUnitSources.set(result.unitType, resolvedSource)
-        this.eventEmitter.emit("resolvedUnitSource", resolvedSource)
-      }
-
-      this.logger.info("unit sources synced")
-      return
-    }
-
-    throw new Error("Worker ended without sending the result")
-  }
-
   async evaluateCompositeInstances(
     allInstances: InstanceModel[],
     resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
@@ -270,12 +207,16 @@ export class LocalLibraryBackend implements LibraryBackend {
       return [this.library, this.worker]
     }
 
-    return await this.updateLibrary()
+    return await this.reloadLibrary()
   }
 
-  private async updateLibrary(): Promise<[LibraryModel, Worker]> {
-    this.logger.info("creating library worker")
-    this.worker = this.createWorker({ modulePaths: this.modulePaths, logLevel: "silent" })
+  private async reloadLibrary(): Promise<[LibraryModel, Worker]> {
+    this.logger.info("reloading library")
+
+    this.worker = this.createLibraryWorker({
+      modulePaths: this.libraryPackages,
+      logLevel: "silent",
+    })
 
     for await (const [event] of on(this.worker, "message")) {
       const eventData = event as {
@@ -302,54 +243,37 @@ export class LocalLibraryBackend implements LibraryBackend {
 
       this.logger.info("library reloaded")
 
-      await this.syncUnitSources(eventData.library)
-
       return [this.library, this.worker]
     }
 
     throw new Error("Worker ended without sending library model")
   }
 
-  private createWorker(workerData: Record<string, unknown>) {
-    const workerPathUrl = importMetaResolve(`@highstate/backend/library-worker`, import.meta.url)
-    const workerPath = fileURLToPath(workerPathUrl)
-
-    return new Worker(workerPath, { workerData })
-  }
-
-  private async updateUnitSourceHashes(path: string): Promise<void> {
-    const packageJsonPath = await resolvePackageJSON(path)
-    const packageJson = await readPackageJSON(path)
-    const library = await this.loadLibrary()
-
-    const manifestPath = resolve(dirname(packageJsonPath), "dist", "highstate.manifest.json")
-
-    let manifest: HighstateManifestJson | undefined
-    try {
-      manifest = JSON.parse(await readFile(manifestPath, "utf8")) as HighstateManifestJson
-    } catch (error) {
-      this.logger.debug(
-        { error },
-        `failed to read highstate manifest for package "%s"`,
-        packageJson.name,
+  private async reloadUnitManifest(libraryPackage: LibraryPackage): Promise<void> {
+    const library = this.library
+    if (!library) {
+      this.logger.warn(
+        `library not loaded, cannot reload unit manifest for package: "%s"`,
+        libraryPackage.name,
       )
+      return
     }
 
+    const manifest = await this.readLibraryPackageManifest(libraryPackage)
+    const packageJson = await readPackageJSON(libraryPackage.rootPath)
+
     for (const unit of Object.values(library.components)) {
       if (!isUnitModel(unit)) {
         continue
       }
 
-      if (unit.source.package !== packageJson.name) {
+      if (unit.source.package !== libraryPackage.name) {
         continue
       }
 
       // TODO: resolve the path
-      const relativePath = unit.source.path
-        ? `./dist/${unit.source.path}/index.js`
-        : `./dist/index.js`
-
-      const sourceHash = manifest?.sourceHashes?.[relativePath]
+      const relativePath = unit.source.path ? `./dist/${unit.source.path}` : `./dist`
+      const sourceHash = manifest?.sourceHashes?.[`${relativePath}/index.js`]
 
       if (!sourceHash) {
         this.logger.warn(`source hash not found for unit: "%s"`, unit.type)
@@ -357,47 +281,248 @@ export class LocalLibraryBackend implements LibraryBackend {
       }
 
       const resolvedSource = this.resolvedUnitSources.get(unit.type)
-      if (!resolvedSource) {
-        this.logger.warn(`resolved source not found for unit: "%s"`, unit.type)
-        continue
-      }
 
       const newResolvedSource: ResolvedUnitSource = {
-        ...resolvedSource,
+        unitType: unit.type,
         sourceHash,
+        projectPath: resolve(libraryPackage.rootPath, relativePath),
+        allowedDependencies: Object.keys(packageJson.peerDependencies ?? {}),
+      }
+
+      if (
+        resolvedSource?.sourceHash === newResolvedSource.sourceHash &&
+        resolvedSource?.projectPath === newResolvedSource.projectPath
+      ) {
+        continue
       }
 
       this.resolvedUnitSources.set(unit.type, newResolvedSource)
       this.eventEmitter.emit("resolvedUnitSource", newResolvedSource)
-      this.logger.info(`updated source hash for unit: "%s"`, unit.type)
+      this.logger.debug(`updated source for unit: "%s"`, unit.type)
     }
   }
 
-  static async create(config: z.infer<typeof localLibraryBackendConfig>, logger: Logger) {
-    const modulePaths: string[] = []
+  private async ensureLibraryPackagesLoaded(
+    names: string[],
+    installIfNotFound = false,
+  ): Promise<void> {
+    const packagesToLoad = names.filter(name => !this.packages.has(name))
 
-    for (const module of config.HIGHSTATE_BACKEND_LIBRARY_LOCAL_MODULES) {
-      const url = importMetaResolve(module, import.meta.url)
-      let path = fileURLToPath(url)
-      if (basename(path).includes(".")) {
-        path = dirname(path)
+    if (packagesToLoad.length > 0) {
+      await this.loadLibraryPackages(packagesToLoad, installIfNotFound)
+    }
+  }
+
+  private async rebuildLibraryPackage(
+    libraryPackage: LibraryPackage,
+    installDeps = false,
+    updateDeps = false,
+    rebuiltPackages = new Set<string>(),
+  ): Promise<void> {
+    if (rebuiltPackages.has(libraryPackage.name)) {
+      return
+    }
+
+    rebuiltPackages.add(libraryPackage.name)
+
+    if (installDeps) {
+      this.logger.info(`installing dependencies for package "${libraryPackage.name}"`)
+      await installDependencies({ cwd: libraryPackage.rootPath })
+    }
+
+    if (updateDeps) {
+      await this.updateLibraryPackageDependencies(libraryPackage)
+    }
+
+    this.logger.info(`rebuilding library package "${libraryPackage.name}" via build script`)
+    await runScript("build", { cwd: libraryPackage.rootPath })
+
+    if (this.libraryPackages.includes(libraryPackage.name)) {
+      await this.reloadLibrary()
+    } else {
+      await this.reloadUnitManifest(libraryPackage)
+    }
+
+    await this.rebuildLibraryPackageDependents(libraryPackage, rebuiltPackages)
+  }
+
+  private async updateLibraryPackageDependencies(libraryPackage: LibraryPackage): Promise<void> {
+    const packageJson = await readPackageJSON(libraryPackage.rootPath)
+    const parsedName = LocalLibraryBackend.parseDependencyName(libraryPackage.name)
+
+    const dependencyPackageNames = pipe(
+      [packageJson.dependencies, packageJson.devDependencies, packageJson.peerDependencies],
+      flatMap(deps => Object.keys(deps ?? {})),
+      unique(),
+      map(LocalLibraryBackend.parseDependencyName),
+    )
+
+    const sameScopeDependencies = dependencyPackageNames.filter(
+      dep => dep.scope === parsedName.scope && dep.name !== parsedName.name,
+    )
+
+    await this.ensureLibraryPackagesLoaded(sameScopeDependencies.map(dep => dep.name))
+
+    for (const dependency of sameScopeDependencies) {
+      const dependencyPackage = this.packages.get(dependency.name)
+      if (!dependencyPackage) {
+        this.logger.warn(`dependency package not found for graph update: "%s"`, dependency.name)
+        continue
+      }
+
+      libraryPackage.dependencies.add(dependency.name)
+      dependencyPackage.dependents.add(libraryPackage.name)
+    }
+  }
+
+  private async rebuildLibraryPackageDependents(
+    libraryPackage: LibraryPackage,
+    rebuiltPackages = new Set<string>(),
+  ): Promise<void> {
+    const promises: Promise<void>[] = []
+
+    for (const dependent of libraryPackage.dependents) {
+      const dependentPackage = this.packages.get(dependent)
+      if (!dependentPackage) {
+        this.logger.warn(`dependent package not found for rebuild: "%s"`, dependent)
+        continue
+      }
+
+      promises.push(this.rebuildLibraryPackage(dependentPackage, false, false, rebuiltPackages))
+    }
+
+    await Promise.all(promises)
+  }
+
+  private static parseDependencyName(this: void, dependency: string) {
+    if (dependency.startsWith("@")) {
+      const parts = dependency.split("/")
+
+      return {
+        name: dependency,
+        scope: parts[0],
+      }
+    }
+
+    return {
+      name: dependency,
+      scope: null,
+    }
+  }
+
+  private async readLibraryPackageManifest(
+    libraryPackage: LibraryPackage,
+  ): Promise<HighstateManifestJson | undefined> {
+    const manifestPath = resolve(libraryPackage.rootPath, "dist", "highstate.manifest.json")
+
+    try {
+      const manifest = JSON.parse(await readFile(manifestPath, "utf8")) as HighstateManifestJson
+
+      return manifest
+    } catch (error) {
+      this.logger.debug(
+        { error },
+        `failed to read highstate manifest of package: "%s"`,
+        libraryPackage.name,
+      )
+
+      return undefined
+    }
+  }
+
+  private async loadLibraryPackages(names: string[], installIfNotFound = false): Promise<void> {
+    this.logger.info("loading library packages: %s", names.join(", "))
+
+    const missingPackages: string[] = []
+    const packagesToUpdate: LibraryPackage[] = []
+
+    const worker = this.createPackageResolutionWorker({ packageNames: names })
+    for await (const [event] of on(worker, "message")) {
+      const eventData = event as PackageResolutionResponse
+
+      if (eventData.type !== "result") {
+        continue
+      }
+
+      for (const result of eventData.results) {
+        if (result.type === "success") {
+          const libraryPackage: LibraryPackage = {
+            name: result.packageName,
+            rootPath: result.packageRootPath,
+            dependencies: new Set(),
+            dependents: new Set(),
+          }
+
+          this.packages.set(result.packageName, libraryPackage)
+          packagesToUpdate.push(libraryPackage)
+
+          this.logger.info(`loaded library package: "%s"`, result.packageName)
+        } else if (result.type === "not-found") {
+          missingPackages.push(result.packageName)
+        } else {
+          this.logger.error(
+            `failed to load library package "%s": %s`,
+            result.packageName,
+            result.error,
+          )
+        }
       }
 
-      modulePaths.push(path)
+      break
     }
 
-    let sourceBasePath = config.HIGHSTATE_BACKEND_LIBRARY_LOCAL_SOURCE_BASE_PATH
-    const extraSourceWatchPaths = config.HIGHSTATE_BACKEND_LIBRARY_LOCAL_EXTRA_SOURCE_WATCH_PATHS
-    if (!sourceBasePath) {
+    for (const libraryPackage of packagesToUpdate) {
+      await this.updateLibraryPackageDependencies(libraryPackage)
+
+      if (!this.libraryPackages.includes(libraryPackage.name)) {
+        await this.reloadUnitManifest(libraryPackage)
+      }
+    }
+
+    if (installIfNotFound && missingPackages.length > 0) {
+      this.logger.info("installing missing library packages: %s", missingPackages.join(", "))
+      await addDependency(missingPackages)
+      await this.loadLibraryPackages(missingPackages)
+    }
+  }
+
+  private async handleFileEvent(path: string): Promise<void> {
+    await this.lock.acquire(async () => {
+      const libraryPackage = this.packages.values().find(pkg => path.startsWith(pkg.rootPath))
+
+      if (libraryPackage) {
+        await this.rebuildLibraryPackage(libraryPackage)
+      }
+    })
+  }
+
+  private createLibraryWorker(workerData: Record<string, unknown>) {
+    const workerPathUrl = importMetaResolve(`@highstate/backend/library-worker`, import.meta.url)
+    const workerPath = fileURLToPath(workerPathUrl)
+
+    return new Worker(workerPath, { workerData })
+  }
+
+  private createPackageResolutionWorker(workerData: PackageResolutionWorkerData) {
+    const workerPathUrl = importMetaResolve(
+      `@highstate/backend/package-resolution-worker`,
+      import.meta.url,
+    )
+    const workerPath = fileURLToPath(workerPathUrl)
+
+    return new Worker(workerPath, { workerData })
+  }
+
+  static async create(config: z.infer<typeof localLibraryBackendConfig>, logger: Logger) {
+    let watchPaths = config.HIGHSTATE_BACKEND_LIBRARY_LOCAL_WATCH_PATHS
+    if (!watchPaths) {
       const [projectPath] = await resolveMainLocalProject()
-      sourceBasePath = resolve(projectPath, "units")
-      extraSourceWatchPaths.push(projectPath)
+      watchPaths = [resolve(projectPath, "packages")]
     }
 
     return new LocalLibraryBackend(
-      modulePaths,
-      sourceBasePath,
-      extraSourceWatchPaths,
+      config.HIGHSTATE_BACKEND_LIBRARY_LOCAL_PACKAGES,
+      watchPaths,
       logger.child({ backend: "LibraryBackend", service: "LocalLibraryBackend" }),
     )
   }

package/src/library/package-resolution-worker.ts

@@ -0,0 +1,70 @@
+import { parentPort, workerData } from "node:worker_threads"
+import { dirname } from "node:path"
+import { findPackageJSON } from "node:module"
+import { realpath } from "node:fs/promises"
+import pino, { type Level } from "pino"
+
+export type PackageResolutionWorkerData = {
+  packageNames: string[]
+  logLevel?: Level
+}
+
+export type PackageResult = { packageName: string } & (
+  | {
+      type: "success"
+      packageRootPath: string
+    }
+  | {
+      type: "not-found"
+    }
+  | {
+      type: "error"
+      error: string
+    }
+)
+
+export type PackageResolutionResponse = {
+  type: "result"
+  results: PackageResult[]
+}
+
+const { packageNames, logLevel } = workerData as PackageResolutionWorkerData
+
+const logger = pino({ name: "source-resolution-worker", level: logLevel ?? "silent" })
+
+const results: PackageResult[] = []
+
+for (const packageName of packageNames) {
+  try {
+    const path = findPackageJSON(packageName, import.meta.url)
+    if (!path) {
+      throw new Error(`Package "${packageName}" not found`)
+    }
+
+    results.push({
+      type: "success",
+      packageName,
+      packageRootPath: await realpath(dirname(path)),
+    })
+  } catch (error) {
+    logger.error({ error }, `failed to resolve package "%s"`, packageName)
+
+    if (error instanceof Error && error.message.includes("not found")) {
+      results.push({
+        type: "not-found",
+        packageName,
+      })
+    } else {
+      results.push({
+        type: "error",
+        packageName,
+        error: error instanceof Error ? error.message : String(error),
+      })
+    }
+  }
+}
+
+parentPort!.postMessage({
+  type: "result",
+  results,
+})

package/src/library/worker/loader.ts

@@ -1,10 +1,6 @@
-/* eslint-disable @typescript-eslint/no-unsafe-call */
-/* eslint-disable @typescript-eslint/no-unsafe-return */
-/* eslint-disable @typescript-eslint/no-unsafe-member-access */
-/* eslint-disable @typescript-eslint/no-unsafe-assignment */
-
 import type { Logger } from "pino"
 import type { Jiti } from "jiti"
+import Module from "node:module"
 import {
   type Component,
   type Entity,
@@ -16,8 +12,13 @@ import {
 import { serializeFunction } from "@pulumi/pulumi/runtime/index.js"
 import { sha256 } from "crypto-hash"
 
-// eslint-disable-next-line @typescript-eslint/no-require-imports
-const Module = require("module")
+declare module "module" {
+  // eslint-disable-next-line @typescript-eslint/no-namespace
+  namespace Module {
+    function _load(request: unknown, parent: unknown, isMain: boolean): unknown
+  }
+}
+
 const originalLoad = Module._load
 
 Module._load = function (request: unknown, parent: unknown, isMain: boolean) {