@highstate/backend 0.7.2 → 0.7.4

package/src/library/local.ts

@@ -0,0 +1,404 @@
+import type {
+  InstanceEvaluationResult,
+  LibraryBackend,
+  ModuleEvaluationResult,
+  ResolvedUnitSource,
+} from "./abstractions"
+import type { Logger } from "pino"
+import type { HighstateManifestJson, SourceResolutionResult } from "./source-resolution-worker"
+import type { WorkerRequest, WorkerResponse } from "./worker/protocol"
+import { fileURLToPath } from "node:url"
+import { EventEmitter, on } from "node:events"
+import { Worker } from "node:worker_threads"
+import { basename, dirname, relative, resolve } from "node:path"
+import { readFile } from "node:fs/promises"
+import { type InstanceModel, isUnitModel, type UnitModel } from "@highstate/contract"
+import Watcher from "watcher"
+import { BetterLock } from "better-lock"
+import { resolve as importMetaResolve } from "import-meta-resolve"
+import { z } from "zod"
+import { readPackageJSON, resolvePackageJSON } from "pkg-types"
+import { resolveMainLocalProject, stringArrayType } from "../common"
+import {
+  diffLibraries,
+  type LibraryModel,
+  type LibraryUpdate,
+  type ResolvedInstanceInput,
+} from "../shared"
+
+export const localLibraryBackendConfig = z.object({
+  HIGHSTATE_BACKEND_LIBRARY_LOCAL_MODULES: stringArrayType.default("@highstate/library"),
+  HIGHSTATE_BACKEND_LIBRARY_LOCAL_SOURCE_BASE_PATH: z.string().optional(),
+  HIGHSTATE_BACKEND_LIBRARY_LOCAL_EXTRA_SOURCE_WATCH_PATHS: stringArrayType.default(""),
+})
+
+interface Events {
+  library: [LibraryUpdate[]]
+  resolvedUnitSource: [ResolvedUnitSource]
+}
+
+export class LocalLibraryBackend implements LibraryBackend {
+  private readonly libraryWatcher: Watcher
+  private readonly sourceWatcher: Watcher
+
+  private readonly lock = new BetterLock()
+  private readonly eventEmitter = new EventEmitter<Events>()
+
+  private library: LibraryModel | null = null
+  private worker: Worker | null = null
+
+  private readonly resolvedUnitSources = new Map<string, ResolvedUnitSource>()
+
+  private constructor(
+    private readonly modulePaths: string[],
+    private readonly sourceBasePath: string,
+    extraSourceWatchPaths: string[],
+    private readonly logger: Logger,
+  ) {
+    this.libraryWatcher = new Watcher(modulePaths, { recursive: true, ignoreInitial: true })
+    this.libraryWatcher.on("all", (event: string, path: string) => {
+      const prefixPath = modulePaths.find(modulePath => path.startsWith(modulePath))
+
+      this.logger.info({ msg: "library event", event, path: relative(prefixPath!, path) })
+      void this.lock.acquire(() => this.updateLibrary())
+    })
+
+    this.sourceWatcher = new Watcher([sourceBasePath, ...extraSourceWatchPaths], {
+      recursive: true,
+      ignoreInitial: true,
+      ignore: /\.git|node_modules/,
+    })
+
+    this.sourceWatcher.on("all", (_, path: string) => {
+      if (!path.endsWith("highstate.manifest.json")) {
+        return
+      }
+
+      void this.updateUnitSourceHashes(path)
+    })
+
+    this.logger.debug({ msg: "initialized", modulePaths })
+  }
+
+  async loadLibrary(): Promise<LibraryModel> {
+    return await this.lock.acquire(async () => {
+      const [library] = await this.getLibrary()
+
+      return library
+    })
+  }
+
+  async *watchLibrary(signal?: AbortSignal): AsyncIterable<LibraryUpdate[]> {
+    for await (const [library] of on(this.eventEmitter, "library", { signal })) {
+      yield library
+    }
+  }
+
+  async getResolvedUnitSources(): Promise<readonly ResolvedUnitSource[]> {
+    return await this.lock.acquire(async () => {
+      // ensure library is loaded
+      const [library] = await this.getLibrary()
+
+      if (!this.resolvedUnitSources.size) {
+        await this.syncUnitSources(library)
+      }
+
+      return Array.from(this.resolvedUnitSources.values())
+    })
+  }
+
+  async getResolvedUnitSource(unitType: string): Promise<ResolvedUnitSource | null> {
+    return await this.lock.acquire(async () => {
+      // ensure library is loaded
+      const [library] = await this.getLibrary()
+
+      if (!this.resolvedUnitSources.size) {
+        await this.syncUnitSources(library)
+      }
+
+      return this.resolvedUnitSources.get(unitType) ?? null
+    })
+  }
+
+  async *watchResolvedUnitSources(signal?: AbortSignal): AsyncIterable<ResolvedUnitSource> {
+    for await (const [resolvedUnitSource] of on(this.eventEmitter, "resolvedUnitSource", {
+      signal,
+    })) {
+      yield resolvedUnitSource
+    }
+  }
+
+  async syncUnitSources(library: LibraryModel): Promise<void> {
+    const unitsToResolve = new Map<string, UnitModel>()
+
+    for (const component of Object.values(library.components)) {
+      if (!isUnitModel(component)) {
+        continue
+      }
+
+      const existingResolvedSource = this.resolvedUnitSources.get(component.type)
+      const expectedSource = JSON.stringify(component.source)
+
+      if (existingResolvedSource?.serializedSource !== expectedSource) {
+        unitsToResolve.set(component.type, component)
+      }
+    }
+
+    await this.runSourceResolution(unitsToResolve)
+  }
+
+  private async runSourceResolution(units: Map<string, UnitModel>): Promise<void> {
+    const workerPathUrl = importMetaResolve(
+      `@highstate/backend/source-resolution-worker`,
+      import.meta.url,
+    )
+    const workerPath = fileURLToPath(workerPathUrl)
+
+    const worker = new Worker(workerPath, {
+      workerData: {
+        requests: Array.from(units.values()).map(unit => ({
+          unitType: unit.type,
+          source: unit.source,
+        })),
+        sourceBasePath: this.sourceBasePath,
+        logLevel: "error",
+      },
+    })
+
+    for await (const [event] of on(worker, "message")) {
+      const eventData = event as {
+        type: string
+        results: readonly SourceResolutionResult[]
+      }
+
+      if (eventData.type !== "result") {
+        throw new Error(`Unexpected message type '${eventData.type}', expected 'result'`)
+      }
+
+      for (const result of eventData.results) {
+        const unit = units.get(result.unitType)
+        if (!unit) {
+          this.logger.warn("unit not found for resolved source", { unitType: result.unitType })
+          continue
+        }
+
+        const resolvedSource: ResolvedUnitSource = {
+          unitType: result.unitType,
+          serializedSource: JSON.stringify(unit.source),
+          projectPath: result.projectPath,
+          packageJsonPath: result.packageJsonPath,
+          allowedDependencies: result.allowedDependencies,
+          sourceHash: result.sourceHash,
+        }
+
+        this.resolvedUnitSources.set(result.unitType, resolvedSource)
+        this.eventEmitter.emit("resolvedUnitSource", resolvedSource)
+      }
+
+      this.logger.info("unit sources synced")
+      return
+    }
+
+    throw new Error("Worker ended without sending the result")
+  }
+
+  async evaluateCompositeInstances(
+    allInstances: InstanceModel[],
+    resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
+    instanceIds: string[],
+  ): Promise<InstanceEvaluationResult[]> {
+    return await this.lock.acquire(async () => {
+      this.logger.info("evaluating %d composite instances", instanceIds.length)
+
+      const [, worker] = await this.getLibrary()
+      worker.postMessage({
+        type: "evaluate-composite-instances",
+        allInstances,
+        resolvedInputs,
+        instanceIds,
+      } satisfies WorkerRequest)
+
+      this.logger.debug("evaluation request sent")
+
+      const { results } = await this.getResult(worker, "instance-evaluation-results")
+      return results
+    })
+  }
+
+  async evaluateModules(modulePaths: string[]): Promise<ModuleEvaluationResult> {
+    return await this.lock.acquire(async () => {
+      this.logger.info({ msg: "evaluating modules", modulePaths })
+
+      const [, worker] = await this.getLibrary()
+      worker.postMessage({
+        type: "evaluate-modules",
+        modulePaths,
+      } satisfies WorkerRequest)
+
+      this.logger.debug("evaluation request sent")
+
+      const { result } = await this.getResult(worker, "module-evaluation-result")
+      return result
+    })
+  }
+
+  private async getResult<TType extends WorkerResponse["type"]>(
+    worker: Worker,
+    expectedType: TType,
+  ): Promise<WorkerResponse & { type: TType }> {
+    for await (const [event] of on(worker, "message")) {
+      const eventData = event as WorkerResponse
+
+      if (eventData.type === "error") {
+        throw new Error(`Worker error: ${eventData.error}`)
+      }
+
+      if (eventData.type !== expectedType) {
+        throw new Error(
+          `Unexpected response message type "${eventData.type}", expected "${expectedType}"`,
+        )
+      }
+
+      return eventData as WorkerResponse & { type: TType }
+    }
+
+    throw new Error("Worker ended without sending any response")
+  }
+
+  private async getLibrary(): Promise<[LibraryModel, Worker]> {
+    if (this.library && this.worker) {
+      return [this.library, this.worker]
+    }
+
+    return await this.updateLibrary()
+  }
+
+  private async updateLibrary(): Promise<[LibraryModel, Worker]> {
+    this.logger.info("creating library worker")
+    this.worker = this.createWorker({ modulePaths: this.modulePaths, logLevel: "silent" })
+
+    for await (const [event] of on(this.worker, "message")) {
+      const eventData = event as {
+        type: string
+        library: LibraryModel
+        error?: string
+      }
+
+      if (eventData.type === "error") {
+        throw new Error(`Worker error: ${eventData.error}`)
+      }
+
+      if (eventData.type !== "library") {
+        throw new Error(`Unexpected message type '${eventData.type}', expected 'library'`)
+      }
+
+      const updates = diffLibraries(
+        this.library ?? { components: {}, entities: {} },
+        eventData.library,
+      )
+
+      this.eventEmitter.emit("library", updates)
+      this.library = eventData.library
+
+      this.logger.info("library reloaded")
+
+      await this.syncUnitSources(eventData.library)
+
+      return [this.library, this.worker]
+    }
+
+    throw new Error("Worker ended without sending library model")
+  }
+
+  private createWorker(workerData: Record<string, unknown>) {
+    const workerPathUrl = importMetaResolve(`@highstate/backend/library-worker`, import.meta.url)
+    const workerPath = fileURLToPath(workerPathUrl)
+
+    return new Worker(workerPath, { workerData })
+  }
+
+  private async updateUnitSourceHashes(path: string): Promise<void> {
+    const packageJsonPath = await resolvePackageJSON(path)
+    const packageJson = await readPackageJSON(path)
+    const library = await this.loadLibrary()
+
+    const manifestPath = resolve(dirname(packageJsonPath), "dist", "highstate.manifest.json")
+
+    let manifest: HighstateManifestJson | undefined
+    try {
+      manifest = JSON.parse(await readFile(manifestPath, "utf8")) as HighstateManifestJson
+    } catch (error) {
+      this.logger.debug(
+        { error },
+        `failed to read highstate manifest for package "%s"`,
+        packageJson.name,
+      )
+    }
+
+    for (const unit of Object.values(library.components)) {
+      if (!isUnitModel(unit)) {
+        continue
+      }
+
+      if (unit.source.package !== packageJson.name) {
+        continue
+      }
+
+      // TODO: resolve the path
+      const relativePath = unit.source.path
+        ? `./dist/${unit.source.path}/index.js`
+        : `./dist/index.js`
+
+      const sourceHash = manifest?.sourceHashes?.[relativePath]
+
+      if (!sourceHash) {
+        this.logger.warn(`source hash not found for unit: "%s"`, unit.type)
+        continue
+      }
+
+      const resolvedSource = this.resolvedUnitSources.get(unit.type)
+      if (!resolvedSource) {
+        this.logger.warn(`resolved source not found for unit: "%s"`, unit.type)
+        continue
+      }
+
+      const newResolvedSource: ResolvedUnitSource = {
+        ...resolvedSource,
+        sourceHash,
+      }
+
+      this.resolvedUnitSources.set(unit.type, newResolvedSource)
+      this.eventEmitter.emit("resolvedUnitSource", newResolvedSource)
+      this.logger.info(`updated source hash for unit: "%s"`, unit.type)
+    }
+  }
+
+  static async create(config: z.infer<typeof localLibraryBackendConfig>, logger: Logger) {
+    const modulePaths: string[] = []
+
+    for (const module of config.HIGHSTATE_BACKEND_LIBRARY_LOCAL_MODULES) {
+      const url = importMetaResolve(module, import.meta.url)
+      let path = fileURLToPath(url)
+      if (basename(path).includes(".")) {
+        path = dirname(path)
+      }
+
+      modulePaths.push(path)
+    }
+
+    let sourceBasePath = config.HIGHSTATE_BACKEND_LIBRARY_LOCAL_SOURCE_BASE_PATH
+    const extraSourceWatchPaths = config.HIGHSTATE_BACKEND_LIBRARY_LOCAL_EXTRA_SOURCE_WATCH_PATHS
+    if (!sourceBasePath) {
+      const [projectPath] = await resolveMainLocalProject()
+      sourceBasePath = resolve(projectPath, "units")
+      extraSourceWatchPaths.push(projectPath)
+    }
+
+    return new LocalLibraryBackend(
+      modulePaths,
+      sourceBasePath,
+      extraSourceWatchPaths,
+      logger.child({ backend: "LibraryBackend", service: "LocalLibraryBackend" }),
+    )
+  }
+}

package/src/library/source-resolution-worker.ts

@@ -0,0 +1,96 @@
+import type { UnitSource } from "@highstate/contract"
+import { parentPort, workerData } from "node:worker_threads"
+import { fileURLToPath } from "node:url"
+import { dirname, relative, resolve } from "node:path"
+import { readFile } from "node:fs/promises"
+import pino, { type Level } from "pino"
+import { resolve as importMetaResolve } from "import-meta-resolve"
+import { readPackageJSON, resolvePackageJSON } from "pkg-types"
+
+export type SourceResolutionRequest = {
+  unitType: string
+  source: UnitSource
+}
+
+export type SourceResolutionResult = {
+  unitType: string
+  projectPath: string
+  packageJsonPath: string
+  sourceHash: string
+  allowedDependencies: string[]
+}
+
+export type HighstateManifestJson = {
+  sourceHashes?: Record<string, string>
+}
+
+const { requests, logLevel } = workerData as {
+  requests: SourceResolutionRequest[]
+  logLevel?: Level
+}
+
+const logger = pino({ name: "source-resolution-worker", level: logLevel ?? "silent" })
+
+const results = await Promise.all(
+  requests.map(request => resolveUnitSourceSafe(request.source, request.unitType)),
+)
+
+parentPort!.postMessage({
+  type: "result",
+  results: results.filter((result): result is SourceResolutionResult => result !== null),
+})
+
+async function resolveUnitSourceSafe(
+  source: UnitSource,
+  unitType: string,
+): Promise<SourceResolutionResult | null> {
+  try {
+    return await resolveUnitSource(source, unitType)
+  } catch (error) {
+    logger.error({ source, unitType, err: error }, "failed to resolve unit source")
+    return null
+  }
+}
+
+async function resolveUnitSource(
+  source: UnitSource,
+  unitType: string,
+): Promise<SourceResolutionResult> {
+  const fullPath = source.path ? `${source.package}/${source.path}` : source.package
+
+  const url = importMetaResolve(fullPath, import.meta.url)
+  const path = fileURLToPath(url)
+  const projectPath = dirname(path)
+
+  const packageJsonPath = await resolvePackageJSON(projectPath)
+  const packageJson = await readPackageJSON(projectPath)
+
+  const manifestPath = resolve(dirname(packageJsonPath), "dist", "highstate.manifest.json")
+  let manifest: HighstateManifestJson | undefined
+  try {
+    manifest = JSON.parse(await readFile(manifestPath, "utf8")) as HighstateManifestJson
+  } catch (error) {
+    logger.debug({ error }, "failed to read highstate manifest")
+  }
+
+  let relativePath = relative(dirname(packageJsonPath), path)
+  relativePath = relativePath.startsWith(".") ? relativePath : `./${relativePath}`
+
+  const sourceHash = manifest?.sourceHashes?.[relativePath]
+  if (!sourceHash) {
+    logger.warn({ unitType, relativePath, packageName: packageJson.name }, "source hash not found")
+  }
+
+  // only the peer dependencies of the package are allowed to auto-install when they are missing
+  const allowedDependencies = Object.keys(packageJson.peerDependencies ?? {})
+
+  logger.debug({ packageJson }, "package.json read")
+
+  return {
+    unitType,
+    projectPath,
+    packageJsonPath,
+    sourceHash: sourceHash ?? "",
+    allowedDependencies,
+  }
+}

package/src/library/worker/evaluator.ts

@@ -0,0 +1,119 @@
+import type { Logger } from "pino"
+import type { Library } from "./loader"
+import type { Jiti } from "jiti"
+import type { InstanceEvaluationResult, ModuleEvaluationResult } from "../abstractions"
+import type { ResolvedInstanceInput } from "../../shared"
+import { getCompositeInstances, resetEvaluation, type InstanceModel } from "@highstate/contract"
+import { BetterLock } from "better-lock"
+import { errorToString } from "../../common"
+
+const lock = new BetterLock()
+
+export function evaluateModules(
+  jiti: Jiti,
+  logger: Logger,
+  modulePaths: string[],
+): Promise<ModuleEvaluationResult> {
+  return lock.acquire(async () => {
+    resetEvaluation()
+    let lastModulePath = ""
+
+    try {
+      for (const modulePath of modulePaths) {
+        logger.info("loading module: %s", modulePath)
+
+        lastModulePath = modulePath
+        await jiti.import(modulePath)
+
+        logger.debug("module loaded: %s", modulePath)
+      }
+
+      return {
+        success: true,
+        compositeInstances: getCompositeInstances(),
+      }
+    } catch (error) {
+      return {
+        success: false,
+        modulePath: lastModulePath,
+        error: errorToString(error),
+      }
+    }
+  })
+}
+
+export function evaluateInstances(
+  logger: Logger,
+  library: Library,
+  allInstances: InstanceModel[],
+  resolvedInputs: Record<string, Record<string, ResolvedInstanceInput[]>>,
+  instanceIds: string[],
+): InstanceEvaluationResult[] {
+  const results: InstanceEvaluationResult[] = []
+  const allInstancesMap = new Map(allInstances.map(instance => [instance.id, instance]))
+  const instanceOutputs = new Map<string, Record<string, unknown>>()
+
+  for (const instanceId of instanceIds ?? []) {
+    try {
+      logger.debug({ instanceId }, "evaluating top-level instance")
+      resetEvaluation()
+
+      evaluateInstance(instanceId)
+
+      results.push({
+        success: true,
+        instanceId,
+        compositeInstances: getCompositeInstances(),
+      })
+    } catch (error) {
+      results.push({
+        success: false,
+        instanceId,
+        error: errorToString(error),
+      })
+    }
+  }
+
+  return results
+
+  function evaluateInstance(instanceId: string): Record<string, unknown> {
+    let outputs = instanceOutputs.get(instanceId)
+
+    if (!outputs) {
+      outputs = _evaluateInstance(instanceId)
+      instanceOutputs.set(instanceId, outputs)
+    }
+
+    return outputs
+  }
+
+  function _evaluateInstance(instanceId: string): Record<string, unknown> {
+    const inputs: Record<string, unknown> = {}
+    const instance = allInstancesMap.get(instanceId)
+
+    logger.info("evaluating instance", { instanceId })
+
+    if (!instance) {
+      throw new Error(`Instance not found: ${instanceId}`)
+    }
+
+    for (const [inputName, input] of Object.entries(resolvedInputs[instanceId] ?? {})) {
+      inputs[inputName] = input.map(input => {
+        const evaluated = evaluateInstance(input.input.instanceId)
+
+        return evaluated[input.input.output]
+      })
+    }
+
+    const component = library.components[instance.type]
+    if (!component) {
+      throw new Error(`Component not found: ${instance.type}, required by instance: ${instanceId}`)
+    }
+
+    return component({
+      name: instance.name,
+      args: instance.args as Record<string, never>,
+      inputs: inputs as Record<string, never>,
+    })
+  }
+}