@highstate/backend 0.7.8 → 0.7.10

package/src/orchestrator/operation-workset.ts

@@ -3,6 +3,7 @@ import type { ProjectBackend } from "../project"
 import type { StateBackend, StateManager } from "../state"
 import type { Logger } from "pino"
 import { isUnitModel, type ComponentModel, type InstanceModel } from "@highstate/contract"
+import { unique } from "remeda"
 import {
   applyPartialInstanceState,
   createInputHashResolver,
@@ -366,14 +367,17 @@ export class OperationWorkset {
     logger: Logger,
     signal: AbortSignal,
   ): Promise<OperationWorkset> {
-    const [library, unitSources, project, compositeInstances, states] = await Promise.all([
+    const [library, project, compositeInstances, states] = await Promise.all([
       libraryBackend.loadLibrary(signal),
-      libraryBackend.getResolvedUnitSources(),
       projectBackend.getProject(operation.projectId, signal),
       stateBackend.getCompositeInstances(operation.projectId, signal),
       stateBackend.getAllInstanceStates(operation.projectId, signal),
     ])
 
+    const unitSources = await libraryBackend.getResolvedUnitSources(
+      unique(project.instances.map(i => i.type)),
+    )
+
     const workset = new OperationWorkset(
       operation,
       library,

package/src/project/local.ts

@@ -8,6 +8,7 @@ import {
   type InstanceInput,
   type InstanceModel,
 } from "@highstate/contract"
+import { stringify, parse } from "yaml"
 import {
   type HubModel,
   type HubModelPatch,
@@ -38,7 +39,7 @@ export class LocalProjectBackend implements ProjectBackend {
     try {
       const files = await readdir(this.projectsDir)
 
-      return files.filter(file => file.endsWith(".json")).map(file => file.replace(/\.json$/, ""))
+      return files.filter(file => file.endsWith(".yaml")).map(file => file.replace(/\.yaml$/, ""))
     } catch (error) {
       throw new Error("Failed to get project names", { cause: error })
     }
@@ -345,7 +346,7 @@ export class LocalProjectBackend implements ProjectBackend {
   }
 
   private getProjectPath(projectId: string) {
-    return `${this.projectsDir}/${projectId}.json`
+    return `${this.projectsDir}/${projectId}.yaml`
   }
 
   private async loadProject(projectId: string) {
@@ -354,7 +355,7 @@ export class LocalProjectBackend implements ProjectBackend {
     try {
       const content = await readFile(projectPath, "utf-8")
 
-      return projectModelSchema.parse(JSON.parse(content))
+      return projectModelSchema.parse(parse(content))
     } catch (error) {
       if (error instanceof Error && "code" in error && error.code === "ENOENT") {
         return { instances: {}, hubs: {} }
@@ -366,7 +367,7 @@ export class LocalProjectBackend implements ProjectBackend {
 
   private async writeProject(projectId: string, project: z.infer<typeof projectModelSchema>) {
     const projectPath = this.getProjectPath(projectId)
-    const content = JSON.stringify(project, undefined, 2)
+    const content = stringify(project, undefined, 2)
 
     await writeFile(projectPath, content)
   }

package/src/project/manager.ts

@@ -15,6 +15,7 @@ import {
   createInstanceState,
   type CompositeInstance,
   type ResolvedInstanceInput,
+  type HubModel,
 } from "../shared"
 
 type CompositeInstanceEvent =
@@ -31,6 +32,12 @@ type CompositeInstanceEvents = {
   [K in string]: [CompositeInstanceEvent]
 }
 
+export type FullProjectModel = {
+  instances: InstanceModel[]
+  hubs: HubModel[]
+  compositeInstances: CompositeInstance[]
+}
+
 export class ProjectManager {
   private constructor(
     private readonly projectBackend: ProjectBackend,
@@ -56,6 +63,35 @@ export class ProjectManager {
     }
   }
 
+  /**
+   * Loads the full info of a project, including instances, hubs, and composite instances.
+   *
+   * Also filters out instances that are not in the library.
+   *
+   * @param projectId The ID of the project to load.
+   */
+  async getProject(projectId: string): Promise<FullProjectModel> {
+    const [{ instances, hubs }, compositeInstances, library] = await Promise.all([
+      this.projectBackend.getProject(projectId),
+      this.stateBackend.getCompositeInstances(projectId),
+      this.libraryBackend.loadLibrary(),
+    ])
+
+    const filteredInstances = instances.filter(instance => instance.type in library.components)
+    const filteredCompositeInstances = compositeInstances
+      .filter(instance => instance.instance.type in library.components)
+      .map(instance => ({
+        ...instance,
+        children: instance.children.filter(child => child.type in library.components),
+      }))
+
+    return {
+      instances: filteredInstances,
+      hubs,
+      compositeInstances: filteredCompositeInstances,
+    }
+  }
+
   async createInstance(projectId: string, instance: InstanceModel): Promise<InstanceModel> {
     const createdInstance = await this.projectBackend.createInstance(projectId, instance)
     await this.updateCompositeInstance(projectId, createdInstance)
@@ -255,9 +291,11 @@ export class ProjectManager {
 
       let sourceHash: string | undefined
       if (isUnitModel(library.components[instance.type])) {
-        const resolvedUnit = await this.libraryBackend.getResolvedUnitSource(instance.type)
+        const resolvedUnits = await this.libraryBackend.getResolvedUnitSources([instance.type])
+        const resolvedUnit = resolvedUnits.find(unit => unit.unitType === instance.type)
+
         if (!resolvedUnit) {
-          throw new Error(`Resolved unit not found: ${instance.type}`)
+          throw new Error(`Resolved unit not found for type "${instance.type}"`)
         }
 
         sourceHash = resolvedUnit.sourceHash

package/src/runner/local.ts

@@ -1,5 +1,5 @@
 import type { ConfigMap, Stack } from "@pulumi/pulumi/automation"
-import type { LibraryBackend } from "../library"
+import type { LibraryBackend, ResolvedUnitSource } from "../library"
 import { EventEmitter, on } from "node:events"
 import { resolve } from "node:path"
 import { getInstanceId } from "@highstate/contract"
@@ -238,10 +238,7 @@ export class LocalRunnerBackend implements RunnerBackend {
     const instanceId = LocalRunnerBackend.getInstanceId(options)
 
     try {
-      const resolvedSource = await this.libraryBackend.getResolvedUnitSource(options.instanceType)
-      if (!resolvedSource) {
-        throw new Error(`Resolved unit source not found for ${options.instanceType}`)
-      }
+      const resolvedSource = await this.getResolvedUnitSource(options.instanceType)
 
       await this.pulumiProjectHost.runLocal(
         {
@@ -366,7 +363,7 @@ export class LocalRunnerBackend implements RunnerBackend {
     const instanceId = LocalRunnerBackend.getInstanceId(options)
 
     try {
-      const resolvedSource = await this.libraryBackend.getResolvedUnitSource(options.instanceType)
+      const resolvedSource = await this.getResolvedUnitSource(options.instanceType)
       if (!resolvedSource) {
         throw new Error(`Resolved unit source not found for ${options.instanceType}`)
       }
@@ -666,6 +663,17 @@ export class LocalRunnerBackend implements RunnerBackend {
     return true
   }
 
+  private async getResolvedUnitSource(instanceType: string): Promise<ResolvedUnitSource> {
+    const sources = await this.libraryBackend.getResolvedUnitSources([instanceType])
+    const source = sources.find(source => source.unitType === instanceType)
+
+    if (!source) {
+      throw new Error(`Resolved unit source not found for ${instanceType}`)
+    }
+
+    return source
+  }
+
   private static getStackName(options: RunnerBaseOptions) {
     return `${options.projectId}_${options.instanceName}`
   }

package/dist/chunk-DGUM43GV.js

@@ -1,11 +0,0 @@
-var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
-  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
-}) : x)(function(x) {
-  if (typeof require !== "undefined") return require.apply(this, arguments);
-  throw Error('Dynamic require of "' + x + '" is not supported');
-});
-
-export {
-  __require
-};
-//# sourceMappingURL=chunk-DGUM43GV.js.map

package/dist/chunk-DGUM43GV.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/library/source-resolution-worker.js

@@ -1,56 +0,0 @@
-// src/library/source-resolution-worker.ts
-import { parentPort, workerData } from "node:worker_threads";
-import { fileURLToPath } from "node:url";
-import { dirname, relative, resolve } from "node:path";
-import { readFile } from "node:fs/promises";
-import pino from "pino";
-import { resolve as importMetaResolve } from "import-meta-resolve";
-import { readPackageJSON, resolvePackageJSON } from "pkg-types";
-var { requests, logLevel } = workerData;
-var logger = pino({ name: "source-resolution-worker", level: logLevel ?? "silent" });
-var results = await Promise.all(
-  requests.map((request) => resolveUnitSourceSafe(request.source, request.unitType))
-);
-parentPort.postMessage({
-  type: "result",
-  results: results.filter((result) => result !== null)
-});
-async function resolveUnitSourceSafe(source, unitType) {
-  try {
-    return await resolveUnitSource(source, unitType);
-  } catch (error) {
-    logger.error({ source, unitType, err: error }, "failed to resolve unit source");
-    return null;
-  }
-}
-async function resolveUnitSource(source, unitType) {
-  const fullPath = source.path ? `${source.package}/${source.path}` : source.package;
-  const url = importMetaResolve(fullPath, import.meta.url);
-  const path = fileURLToPath(url);
-  const projectPath = dirname(path);
-  const packageJsonPath = await resolvePackageJSON(projectPath);
-  const packageJson = await readPackageJSON(projectPath);
-  const manifestPath = resolve(dirname(packageJsonPath), "dist", "highstate.manifest.json");
-  let manifest;
-  try {
-    manifest = JSON.parse(await readFile(manifestPath, "utf8"));
-  } catch (error) {
-    logger.debug({ error }, "failed to read highstate manifest");
-  }
-  let relativePath = relative(dirname(packageJsonPath), path);
-  relativePath = relativePath.startsWith(".") ? relativePath : `./${relativePath}`;
-  const sourceHash = manifest?.sourceHashes?.[relativePath];
-  if (!sourceHash) {
-    logger.warn({ unitType, relativePath, packageName: packageJson.name }, "source hash not found");
-  }
-  const allowedDependencies = Object.keys(packageJson.peerDependencies ?? {});
-  logger.debug({ packageJson }, "package.json read");
-  return {
-    unitType,
-    projectPath,
-    packageJsonPath,
-    sourceHash: sourceHash ?? "",
-    allowedDependencies
-  };
-}
-//# sourceMappingURL=source-resolution-worker.js.map

package/dist/library/source-resolution-worker.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../src/library/source-resolution-worker.ts"],"sourcesContent":["import type { UnitSource } from \"@highstate/contract\"\nimport { parentPort, workerData } from \"node:worker_threads\"\nimport { fileURLToPath } from \"node:url\"\nimport { dirname, relative, resolve } from \"node:path\"\nimport { readFile } from \"node:fs/promises\"\nimport pino, { type Level } from \"pino\"\nimport { resolve as importMetaResolve } from \"import-meta-resolve\"\nimport { readPackageJSON, resolvePackageJSON } from \"pkg-types\"\n\nexport type SourceResolutionRequest = {\n  unitType: string\n  source: UnitSource\n}\n\nexport type SourceResolutionResult = {\n  unitType: string\n  projectPath: string\n  packageJsonPath: string\n  sourceHash: string\n  allowedDependencies: string[]\n}\n\nexport type HighstateManifestJson = {\n  sourceHashes?: Record<string, string>\n}\n\nconst { requests, logLevel } = workerData as {\n  requests: SourceResolutionRequest[]\n  logLevel?: Level\n}\n\nconst logger = pino({ name: \"source-resolution-worker\", level: logLevel ?? \"silent\" })\n\nconst results = await Promise.all(\n  requests.map(request => resolveUnitSourceSafe(request.source, request.unitType)),\n)\n\nparentPort!.postMessage({\n  type: \"result\",\n  results: results.filter((result): result is SourceResolutionResult => result !== null),\n})\n\nasync function resolveUnitSourceSafe(\n  source: UnitSource,\n  unitType: string,\n): Promise<SourceResolutionResult | null> {\n  try {\n    return await resolveUnitSource(source, unitType)\n  } catch (error) {\n    logger.error({ source, unitType, err: error }, \"failed to resolve unit source\")\n    return null\n  }\n}\n\nasync function resolveUnitSource(\n  source: UnitSource,\n  unitType: string,\n): Promise<SourceResolutionResult> {\n  const fullPath = source.path ? `${source.package}/${source.path}` : source.package\n\n  const url = importMetaResolve(fullPath, import.meta.url)\n  const path = fileURLToPath(url)\n  const projectPath = dirname(path)\n\n  const packageJsonPath = await resolvePackageJSON(projectPath)\n  const packageJson = await readPackageJSON(projectPath)\n\n  const manifestPath = resolve(dirname(packageJsonPath), \"dist\", \"highstate.manifest.json\")\n  let manifest: HighstateManifestJson | undefined\n  try {\n    manifest = JSON.parse(await readFile(manifestPath, \"utf8\")) as HighstateManifestJson\n  } catch (error) {\n    logger.debug({ error }, \"failed to read highstate manifest\")\n  }\n\n  let relativePath = relative(dirname(packageJsonPath), path)\n  relativePath = relativePath.startsWith(\".\") ? relativePath : `./${relativePath}`\n\n  const sourceHash = manifest?.sourceHashes?.[relativePath]\n  if (!sourceHash) {\n    logger.warn({ unitType, relativePath, packageName: packageJson.name }, \"source hash not found\")\n  }\n\n  // only the peer dependencies of the package are allowed to auto-install when they are missing\n  const allowedDependencies = Object.keys(packageJson.peerDependencies ?? {})\n\n  logger.debug({ packageJson }, \"package.json read\")\n\n  return {\n    unitType,\n    projectPath,\n    packageJsonPath,\n    sourceHash: sourceHash ?? \"\",\n    allowedDependencies,\n  }\n}\n"],"mappings":";AACA,SAAS,YAAY,kBAAkB;AACvC,SAAS,qBAAqB;AAC9B,SAAS,SAAS,UAAU,eAAe;AAC3C,SAAS,gBAAgB;AACzB,OAAO,UAA0B;AACjC,SAAS,WAAW,yBAAyB;AAC7C,SAAS,iBAAiB,0BAA0B;AAmBpD,IAAM,EAAE,UAAU,SAAS,IAAI;AAK/B,IAAM,SAAS,KAAK,EAAE,MAAM,4BAA4B,OAAO,YAAY,SAAS,CAAC;AAErF,IAAM,UAAU,MAAM,QAAQ;AAAA,EAC5B,SAAS,IAAI,aAAW,sBAAsB,QAAQ,QAAQ,QAAQ,QAAQ,CAAC;AACjF;AAEA,WAAY,YAAY;AAAA,EACtB,MAAM;AAAA,EACN,SAAS,QAAQ,OAAO,CAAC,WAA6C,WAAW,IAAI;AACvF,CAAC;AAED,eAAe,sBACb,QACA,UACwC;AACxC,MAAI;AACF,WAAO,MAAM,kBAAkB,QAAQ,QAAQ;AAAA,EACjD,SAAS,OAAO;AACd,WAAO,MAAM,EAAE,QAAQ,UAAU,KAAK,MAAM,GAAG,+BAA+B;AAC9E,WAAO;AAAA,EACT;AACF;AAEA,eAAe,kBACb,QACA,UACiC;AACjC,QAAM,WAAW,OAAO,OAAO,GAAG,OAAO,OAAO,IAAI,OAAO,IAAI,KAAK,OAAO;AAE3E,QAAM,MAAM,kBAAkB,UAAU,YAAY,GAAG;AACvD,QAAM,OAAO,cAAc,GAAG;AAC9B,QAAM,cAAc,QAAQ,IAAI;AAEhC,QAAM,kBAAkB,MAAM,mBAAmB,WAAW;AAC5D,QAAM,cAAc,MAAM,gBAAgB,WAAW;AAErD,QAAM,eAAe,QAAQ,QAAQ,eAAe,GAAG,QAAQ,yBAAyB;AACxF,MAAI;AACJ,MAAI;AACF,eAAW,KAAK,MAAM,MAAM,SAAS,cAAc,MAAM,CAAC;AAAA,EAC5D,SAAS,OAAO;AACd,WAAO,MAAM,EAAE,MAAM,GAAG,mCAAmC;AAAA,EAC7D;AAEA,MAAI,eAAe,SAAS,QAAQ,eAAe,GAAG,IAAI;AAC1D,iBAAe,aAAa,WAAW,GAAG,IAAI,eAAe,KAAK,YAAY;AAE9E,QAAM,aAAa,UAAU,eAAe,YAAY;AACxD,MAAI,CAAC,YAAY;AACf,WAAO,KAAK,EAAE,UAAU,cAAc,aAAa,YAAY,KAAK,GAAG,uBAAuB;AAAA,EAChG;AAGA,QAAM,sBAAsB,OAAO,KAAK,YAAY,oBAAoB,CAAC,CAAC;AAE1E,SAAO,MAAM,EAAE,YAAY,GAAG,mBAAmB;AAEjD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,YAAY,cAAc;AAAA,IAC1B;AAAA,EACF;AACF;","names":[]}

package/src/library/source-resolution-worker.ts

@@ -1,96 +0,0 @@
-import type { UnitSource } from "@highstate/contract"
-import { parentPort, workerData } from "node:worker_threads"
-import { fileURLToPath } from "node:url"
-import { dirname, relative, resolve } from "node:path"
-import { readFile } from "node:fs/promises"
-import pino, { type Level } from "pino"
-import { resolve as importMetaResolve } from "import-meta-resolve"
-import { readPackageJSON, resolvePackageJSON } from "pkg-types"
-
-export type SourceResolutionRequest = {
-  unitType: string
-  source: UnitSource
-}
-
-export type SourceResolutionResult = {
-  unitType: string
-  projectPath: string
-  packageJsonPath: string
-  sourceHash: string
-  allowedDependencies: string[]
-}
-
-export type HighstateManifestJson = {
-  sourceHashes?: Record<string, string>
-}
-
-const { requests, logLevel } = workerData as {
-  requests: SourceResolutionRequest[]
-  logLevel?: Level
-}
-
-const logger = pino({ name: "source-resolution-worker", level: logLevel ?? "silent" })
-
-const results = await Promise.all(
-  requests.map(request => resolveUnitSourceSafe(request.source, request.unitType)),
-)
-
-parentPort!.postMessage({
-  type: "result",
-  results: results.filter((result): result is SourceResolutionResult => result !== null),
-})
-
-async function resolveUnitSourceSafe(
-  source: UnitSource,
-  unitType: string,
-): Promise<SourceResolutionResult | null> {
-  try {
-    return await resolveUnitSource(source, unitType)
-  } catch (error) {
-    logger.error({ source, unitType, err: error }, "failed to resolve unit source")
-    return null
-  }
-}
-
-async function resolveUnitSource(
-  source: UnitSource,
-  unitType: string,
-): Promise<SourceResolutionResult> {
-  const fullPath = source.path ? `${source.package}/${source.path}` : source.package
-
-  const url = importMetaResolve(fullPath, import.meta.url)
-  const path = fileURLToPath(url)
-  const projectPath = dirname(path)
-
-  const packageJsonPath = await resolvePackageJSON(projectPath)
-  const packageJson = await readPackageJSON(projectPath)
-
-  const manifestPath = resolve(dirname(packageJsonPath), "dist", "highstate.manifest.json")
-  let manifest: HighstateManifestJson | undefined
-  try {
-    manifest = JSON.parse(await readFile(manifestPath, "utf8")) as HighstateManifestJson
-  } catch (error) {
-    logger.debug({ error }, "failed to read highstate manifest")
-  }
-
-  let relativePath = relative(dirname(packageJsonPath), path)
-  relativePath = relativePath.startsWith(".") ? relativePath : `./${relativePath}`
-
-  const sourceHash = manifest?.sourceHashes?.[relativePath]
-  if (!sourceHash) {
-    logger.warn({ unitType, relativePath, packageName: packageJson.name }, "source hash not found")
-  }
-
-  // only the peer dependencies of the package are allowed to auto-install when they are missing
-  const allowedDependencies = Object.keys(packageJson.peerDependencies ?? {})
-
-  logger.debug({ packageJson }, "package.json read")
-
-  return {
-    unitType,
-    projectPath,
-    packageJsonPath,
-    sourceHash: sourceHash ?? "",
-    allowedDependencies,
-  }
-}