@highstate/backend 0.7.2 → 0.7.4

package/dist/library/source-resolution-worker.js

@@ -0,0 +1,55 @@
+import { workerData, parentPort } from 'node:worker_threads';
+import { fileURLToPath } from 'node:url';
+import { dirname, resolve as resolve$1, relative } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import pino from 'pino';
+import { resolve } from 'import-meta-resolve';
+import { resolvePackageJSON, readPackageJSON } from 'pkg-types';
+
+const { requests, logLevel } = workerData;
+const logger = pino({ name: "source-resolution-worker", level: logLevel ?? "silent" });
+const results = await Promise.all(
+  requests.map((request) => resolveUnitSourceSafe(request.source, request.unitType))
+);
+parentPort.postMessage({
+  type: "result",
+  results: results.filter((result) => result !== null)
+});
+async function resolveUnitSourceSafe(source, unitType) {
+  try {
+    return await resolveUnitSource(source, unitType);
+  } catch (error) {
+    logger.error({ source, unitType, err: error }, "failed to resolve unit source");
+    return null;
+  }
+}
+async function resolveUnitSource(source, unitType) {
+  const fullPath = source.path ? `${source.package}/${source.path}` : source.package;
+  const url = resolve(fullPath, import.meta.url);
+  const path = fileURLToPath(url);
+  const projectPath = dirname(path);
+  const packageJsonPath = await resolvePackageJSON(projectPath);
+  const packageJson = await readPackageJSON(projectPath);
+  const manifestPath = resolve$1(dirname(packageJsonPath), "dist", "highstate.manifest.json");
+  let manifest;
+  try {
+    manifest = JSON.parse(await readFile(manifestPath, "utf8"));
+  } catch (error) {
+    logger.debug({ error }, "failed to read highstate manifest");
+  }
+  let relativePath = relative(dirname(packageJsonPath), path);
+  relativePath = relativePath.startsWith(".") ? relativePath : `./${relativePath}`;
+  const sourceHash = manifest?.sourceHashes?.[relativePath];
+  if (!sourceHash) {
+    logger.warn({ unitType, relativePath, packageName: packageJson.name }, "source hash not found");
+  }
+  const allowedDependencies = Object.keys(packageJson.peerDependencies ?? {});
+  logger.debug({ packageJson }, "package.json read");
+  return {
+    unitType,
+    projectPath,
+    packageJsonPath,
+    sourceHash: sourceHash ?? "",
+    allowedDependencies
+  };
+}

package/dist/library/worker/main.js

@@ -0,0 +1,216 @@
+import { workerData, parentPort } from 'node:worker_threads';
+import { createJiti } from 'jiti';
+import { pino } from 'pino';
+import { mapValues } from 'remeda';
+import { e as errorToString } from '../../utils-ByadNcv4.js';
+import { BetterLock } from 'better-lock';
+import 'node:path';
+import 'pkg-types';
+import require$$0 from 'module';
+import { isComponent, isEntity, isUnitModel, originalCreate, resetEvaluation, getCompositeInstances } from '@highstate/contract';
+import { serializeFunction } from '@pulumi/pulumi/runtime/index.js';
+import { sha256 } from 'crypto-hash';
+import 'zod';
+
+const Module = require$$0;
+const originalLoad = Module._load;
+Module._load = function(request, parent, isMain) {
+  if (request === "trace_events") {
+    return {};
+  }
+  return originalLoad(request, parent, isMain);
+};
+async function loadLibrary(jiti, logger, modulePaths) {
+  const modules = {};
+  for (const modulePath of modulePaths) {
+    try {
+      logger.debug("loading module", { modulePath });
+      modules[modulePath] = await jiti.import(modulePath);
+      logger.debug("module loaded", { modulePath });
+    } catch (error) {
+      logger.error("module load failed", { modulePath, error });
+    }
+  }
+  const components = {};
+  const entities = {};
+  await _loadLibrary(modules, components, entities);
+  logger.info("library loaded", {
+    componentCount: Object.keys(components).length,
+    entityCount: Object.keys(entities).length
+  });
+  logger.trace("library content", { components, entities });
+  return { components, entities };
+}
+async function _loadLibrary(value, components, entities) {
+  if (isComponent(value)) {
+    components[value.model.type] = value;
+    value.model.definitionHash = await calculateComponentDefinitionHash(value);
+    for (const entity of value.entities.values()) {
+      entity.definitionHash ??= await calculateEntityDefinitionHash(entity);
+    }
+    return;
+  }
+  if (isEntity(value)) {
+    entities[value.type] = value;
+    entities[value.type].definitionHash ??= await calculateEntityDefinitionHash(value);
+    return;
+  }
+  if (typeof value !== "object" || value === null) {
+    return;
+  }
+  for (const key in value) {
+    await _loadLibrary(value[key], components, entities);
+  }
+}
+async function calculateComponentDefinitionHash(component) {
+  if (isUnitModel(component.model)) {
+    return await sha256(JSON.stringify(component.model));
+  }
+  const serializedCreate = await serializeFunction(component[originalCreate]);
+  return await sha256(JSON.stringify(component.model) + serializedCreate.text);
+}
+async function calculateEntityDefinitionHash(entity) {
+  return await sha256(JSON.stringify(entity));
+}
+
+const lock = new BetterLock();
+function evaluateModules(jiti, logger, modulePaths) {
+  return lock.acquire(async () => {
+    resetEvaluation();
+    let lastModulePath = "";
+    try {
+      for (const modulePath of modulePaths) {
+        logger.info("loading module: %s", modulePath);
+        lastModulePath = modulePath;
+        await jiti.import(modulePath);
+        logger.debug("module loaded: %s", modulePath);
+      }
+      return {
+        success: true,
+        compositeInstances: getCompositeInstances()
+      };
+    } catch (error) {
+      return {
+        success: false,
+        modulePath: lastModulePath,
+        error: errorToString(error)
+      };
+    }
+  });
+}
+function evaluateInstances(logger, library, allInstances, resolvedInputs, instanceIds) {
+  const results = [];
+  const allInstancesMap = new Map(allInstances.map((instance) => [instance.id, instance]));
+  const instanceOutputs = /* @__PURE__ */ new Map();
+  for (const instanceId of instanceIds ?? []) {
+    try {
+      logger.debug({ instanceId }, "evaluating top-level instance");
+      resetEvaluation();
+      evaluateInstance(instanceId);
+      results.push({
+        success: true,
+        instanceId,
+        compositeInstances: getCompositeInstances()
+      });
+    } catch (error) {
+      results.push({
+        success: false,
+        instanceId,
+        error: errorToString(error)
+      });
+    }
+  }
+  return results;
+  function evaluateInstance(instanceId) {
+    let outputs = instanceOutputs.get(instanceId);
+    if (!outputs) {
+      outputs = _evaluateInstance(instanceId);
+      instanceOutputs.set(instanceId, outputs);
+    }
+    return outputs;
+  }
+  function _evaluateInstance(instanceId) {
+    const inputs = {};
+    const instance = allInstancesMap.get(instanceId);
+    logger.info("evaluating instance", { instanceId });
+    if (!instance) {
+      throw new Error(`Instance not found: ${instanceId}`);
+    }
+    for (const [inputName, input] of Object.entries(resolvedInputs[instanceId] ?? {})) {
+      inputs[inputName] = input.map((input2) => {
+        const evaluated = evaluateInstance(input2.input.instanceId);
+        return evaluated[input2.input.output];
+      });
+    }
+    const component = library.components[instance.type];
+    if (!component) {
+      throw new Error(`Component not found: ${instance.type}, required by instance: ${instanceId}`);
+    }
+    return component({
+      name: instance.name,
+      args: instance.args,
+      inputs
+    });
+  }
+}
+
+const data = workerData;
+const logger = pino({ name: "library-worker", level: data.logLevel });
+let jiti;
+let library;
+try {
+  logger.info("worker started");
+  logger.trace({ data }, "worker data");
+  jiti = createJiti(import.meta.filename);
+  logger.debug({ filename: import.meta.filename }, "jiti created");
+  library = await loadLibrary(jiti, logger, data.modulePaths);
+  parentPort.postMessage({
+    type: "library",
+    library: {
+      components: mapValues(library.components, (component) => component.model),
+      entities: library.entities
+    }
+  });
+  logger.info("library loaded and sent");
+} catch (error) {
+  logger.error({ error }, "failed to load library");
+  parentPort.postMessage({
+    type: "error",
+    error: errorToString(error)
+  });
+}
+parentPort.on("message", async (message) => {
+  try {
+    const request = message;
+    switch (request.type) {
+      case "evaluate-composite-instances": {
+        const results = evaluateInstances(
+          logger,
+          library,
+          request.allInstances,
+          request.resolvedInputs,
+          request.instanceIds
+        );
+        parentPort.postMessage({
+          type: "instance-evaluation-results",
+          results
+        });
+        break;
+      }
+      case "evaluate-modules": {
+        const result = await evaluateModules(jiti, logger, request.modulePaths);
+        parentPort.postMessage({
+          type: "module-evaluation-result",
+          result
+        });
+        break;
+      }
+    }
+  } catch (error) {
+    logger.error({ error }, "failed to evaluate");
+    parentPort.postMessage({
+      type: "error",
+      error: errorToString(error)
+    });
+  }
+});