@highstate/backend 0.9.33 → 0.9.35

package/dist/index.js

@@ -3,21 +3,21 @@ import { codebaseConfig, stringArrayType, createProjectLogger, isAbortErrorLike,
 import { __using, __callDispose } from './chunk-I7BWSAN6.js';
 import { randomBytes, createHash } from 'node:crypto';
 import { createId } from '@paralleldrive/cuid2';
-import { hubModelSchema, instanceModelSchema, parseInstanceId, isUnitModel, HighstateConfigKey, unitArtifactSchema, unitWorkerSchema, unitTriggerSchema, unitPageSchema, unitTerminalSchema, instanceStatusFieldSchema, getInstanceId } from '@highstate/contract';
-import { omit, isNonNullish, groupBy, pipe, flatMap, unique, map, mapValues, join as join$1, omitBy } from 'remeda';
+import { hubModelSchema, instanceModelSchema, parseInstanceId, isUnitModel, HighstateConfigKey, unitArtifactSchema, unitArtifactId, unitWorkerSchema, unitTriggerSchema, unitPageSchema, unitTerminalSchema, instanceStatusFieldSchema, getInstanceId } from '@highstate/contract';
+import { omit, isNonNullish, groupBy, mapValues, unique, join as join$1, omitBy } from 'remeda';
 import z3, { z } from 'zod';
 import * as os from 'node:os';
 import { tmpdir, hostname } from 'node:os';
 import { PrismaLibSQL } from '@prisma/adapter-libsql';
 import { generateIdentity, armor, Decrypter, Encrypter, identityToRecipient } from 'age-encryption';
 import * as path4 from 'node:path';
-import path4__default, { join, resolve as resolve$1 } from 'node:path';
+import path4__default, { join, resolve as resolve$1, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import * as runtime2 from '@prisma/client/runtime/client';
 import { PrismaClientKnownRequestError } from '@prisma/client/runtime/client';
 import { execa } from 'execa';
 import { resolve } from 'import-meta-resolve';
-import { detectPackageManager, installDependencies, runScript, addDependency, ensureDependencyInstalled } from 'nypm';
+import { detectPackageManager, runScript, addDependency, ensureDependencyInstalled } from 'nypm';
 import { findCredentialsAsync, AsyncEntry } from '@napi-rs/keyring';
 import { mkdir, readFile, writeFile, access, unlink, readdir, rm, mkdtemp } from 'node:fs/promises';
 import { stringify, parse } from 'yaml';
@@ -30,7 +30,9 @@ import { EventEmitter, on } from 'node:events';
 import { Worker } from 'node:worker_threads';
 import { decode, encode } from '@msgpack/msgpack';
 import { BetterLock } from 'better-lock';
-import { readPackageJSON } from 'pkg-types';
+import { glob } from 'glob';
+import PQueue from 'p-queue';
+import { readPackageJSON, resolvePackageJSON } from 'pkg-types';
 import Watcher from 'watcher';
 import { crc32 } from 'node:zlib';
 import { sha256 } from '@noble/hashes/sha2';
@@ -80,8 +82,6 @@ var ApiKeyService = class {
     return apiKey;
   }
 };
-
-// src/business/artifact.ts
 var artifactChunkSize = 1024 * 1024;
 var ArtifactService = class {
   constructor(database, artifactBackend, logger) {
@@ -104,6 +104,7 @@ var ArtifactService = class {
   async store(projectId, hash, size, meta, content, track) {
     const database = await this.database.forProject(projectId);
     const existingArtifact = await database.artifact.findUnique({ where: { hash } });
+    const artifactId = existingArtifact?.id ?? createId();
     if (!existingArtifact || !await this.artifactBackend.exists(projectId, hash)) {
       if (existingArtifact) {
         this.logger.warn(
@@ -111,12 +112,12 @@ var ArtifactService = class {
           hash
         );
       }
-      await this.artifactBackend.store(projectId, hash, artifactChunkSize, content);
+      await this.artifactBackend.store(projectId, artifactId, artifactChunkSize, content);
     }
     return await database.$transaction(async (tx) => {
       const artifact = await tx.artifact.upsert({
-        where: { hash },
-        create: { hash, size, meta, chunkSize: artifactChunkSize },
+        where: { id: artifactId },
+        create: { id: artifactId, hash, size, meta, chunkSize: artifactChunkSize },
         update: { meta }
       });
       await track(tx, artifact);
@@ -129,49 +130,6 @@ var ArtifactService = class {
       return artifact;
     });
   }
-  /**
-   * Clears all artifact references for a specific instance and runs garbage collection.
-   * This removes all associations between the instance and its artifacts, then
-   * cleans up any unreferenced artifacts.
-   *
-   * @param projectId The project ID.
-   * @param instanceId The instance ID to clear artifact references for.
-   */
-  async clearInstanceArtifactReferences(projectId, instanceId) {
-    const database = await this.database.forProject(projectId);
-    this.logger.info({ projectId, instanceId }, "clearing instance artifact references");
-    await database.$transaction(async (tx) => {
-      const artifactsConnectedToInstance = await tx.artifact.findMany({
-        where: {
-          instances: {
-            some: {
-              id: instanceId
-            }
-          }
-        },
-        select: {
-          id: true
-        }
-      });
-      await Promise.all(
-        artifactsConnectedToInstance.map(
-          (artifact) => tx.artifact.update({
-            where: { id: artifact.id },
-            data: {
-              instances: {
-                disconnect: { id: instanceId }
-              }
-            }
-          })
-        )
-      );
-    });
-    this.logger.info(
-      { projectId, instanceId },
-      "cleared instance artifact references, running garbage collection"
-    );
-    await this.collectGarbage(projectId);
-  }
   /**
    * Removes artifacts with no references and cleans up backend storage.
    *
@@ -694,8 +652,8 @@ config.compilerWasm = {
     return await decodeBase64AsWasm(wasm);
   }
 };
-function getPrismaClientClass(dirname4) {
-  config.dirname = dirname4;
+function getPrismaClientClass(dirname5) {
+  config.dirname = dirname5;
   return runtime2.getPrismaClient(config);
 }
 runtime2.Public.validator;
@@ -1040,8 +998,8 @@ config2.compilerWasm = {
     return await decodeBase64AsWasm2(wasm);
   }
 };
-function getPrismaClientClass2(dirname4) {
-  config2.dirname = dirname4;
+function getPrismaClientClass2(dirname5) {
+  config2.dirname = dirname5;
   return runtime2.getPrismaClient(config2);
 }
 runtime2.Public.validator;
@@ -1137,8 +1095,8 @@ config3.compilerWasm = {
     return await decodeBase64AsWasm3(wasm);
   }
 };
-function getPrismaClientClass3(dirname4) {
-  config3.dirname = dirname4;
+function getPrismaClientClass3(dirname5) {
+  config3.dirname = dirname5;
   return runtime2.getPrismaClient(config3);
 }
 runtime2.Public.validator;
@@ -1685,6 +1643,9 @@ var InstanceStateService = class {
           this.workerService.updateUnitRegistrations(tx, projectId, stateId, unitExtra.workers)
         ]);
         unitExtraData = { pageIds, terminalIds, triggerIds, secretNames };
+        if (unitExtra.artifactIds !== void 0) {
+          await this.unitExtraService.pruneInstanceArtifacts(tx, stateId, unitExtra.artifactIds);
+        }
       }
       if (instanceState) {
         await tx.instanceState.update({
@@ -1694,6 +1655,9 @@ var InstanceStateService = class {
       }
       return { updatedOperationState, unitExtraData };
     });
+    if (options.unitExtra?.artifactIds !== void 0) {
+      await this.artifactService.collectGarbage(projectId);
+    }
     const patch = {
       ...instanceState,
       ...result.unitExtraData,
@@ -3549,6 +3513,39 @@ var UnitExtraService = class {
     });
     return triggerIds;
   }
+  /**
+   * Disconnects artifacts that are no longer referenced by the instance.
+   *
+   * @param tx The database transaction to use.
+   * @param stateId The ID of the instance state.
+   * @param artifactIds The artifact IDs that should remain connected to the instance.
+   */
+  async pruneInstanceArtifacts(tx, stateId, artifactIds) {
+    const staleArtifacts = await tx.artifact.findMany({
+      where: {
+        instances: {
+          some: {
+            id: stateId
+          }
+        },
+        id: artifactIds.length > 0 ? { notIn: artifactIds } : void 0
+      },
+      select: {
+        id: true
+      }
+    });
+    if (staleArtifacts.length === 0) {
+      return;
+    }
+    await tx.instanceState.update({
+      where: { id: stateId },
+      data: {
+        artifacts: {
+          disconnect: staleArtifacts.map((artifact) => ({ id: artifact.id }))
+        }
+      }
+    });
+  }
   /**
    * Gets all triggers for a specific instance.
    *
@@ -3842,13 +3839,13 @@ var LocalArtifactBackend = class _LocalArtifactBackend {
     file.end();
     logger.info({ artifactId, fileName }, "artifact stored");
   }
-  async retrieve(projectId, hash, chunkSize) {
+  async retrieve(projectId, artifactId, chunkSize) {
     const logger = createProjectLogger(this.logger, projectId);
-    const [, fileName] = this.getArtifactPath(projectId, hash);
+    const [, fileName] = this.getArtifactPath(projectId, artifactId);
     try {
       return Promise.resolve(createReadStream(fileName, { highWaterMark: chunkSize }));
     } catch (error) {
-      logger.debug({ hash, error }, "artifact retrieval failed");
+      logger.debug({ hash: artifactId, error }, "artifact retrieval failed");
       return null;
     }
   }
@@ -3876,8 +3873,8 @@ var LocalArtifactBackend = class _LocalArtifactBackend {
       logger.error({ dirPath, error }, "failed to delete directory");
     }
   }
-  async exists(projectId, hash) {
-    const [, fileName] = this.getArtifactPath(projectId, hash);
+  async exists(projectId, artifactId) {
+    const [, fileName] = this.getArtifactPath(projectId, artifactId);
     try {
       await access(fileName);
       return true;
@@ -3885,9 +3882,9 @@ var LocalArtifactBackend = class _LocalArtifactBackend {
       return false;
     }
   }
-  getArtifactPath(projectId, hash) {
+  getArtifactPath(projectId, artifactId) {
     const baseDir = resolve$1(this.hsCodebasePath, "projects", projectId, "artifacts");
-    const fileName = join(baseDir, `${hash}${this.extension}`);
+    const fileName = join(baseDir, `${artifactId}${this.extension}`);
     return [baseDir, fileName];
   }
   static async create(config4, extension, logger) {
@@ -3916,17 +3913,27 @@ async function createArtifactBackend(config4, database, logger) {
   return backend;
 }
 var localLibraryBackendConfig = z.object({
-  HIGHSTATE_LIBRARY_BACKEND_LOCAL_PACKAGES: stringArrayType.default(() => ["@highstate/library"]),
-  HIGHSTATE_LIBRARY_BACKEND_LOCAL_WATCH_PATHS: stringArrayType.optional()
+  HIGHSTATE_LIBRARY_BACKEND_LOCAL_LIBRARY_PACKAGES: stringArrayType.default(() => [
+    "@highstate/library"
+  ]),
+  HIGHSTATE_LIBRARY_BACKEND_LOCAL_WORKSPACE_PATH: z.string().optional(),
+  HIGHSTATE_LIBRARY_BACKEND_LOCAL_BUILD_CONCURRENCY: z.coerce.number().int().positive().default(3),
+  HIGHSTATE_LIBRARY_BACKEND_LOCAL_BUILD_ON_STARTUP: z.stringbool().default(true),
+  HIGHSTATE_LIBRARY_BACKEND_LOCAL_BUILD_ON_CHANGES: z.stringbool().default(true)
 });
 var LocalLibraryBackend = class _LocalLibraryBackend {
-  constructor(libraryPackages, watchPaths, logger) {
+  constructor(libraryPackages, workspacePath, workspacePatterns, buildConcurrency, buildOnStartup, buildOnChanges, logger) {
     this.libraryPackages = libraryPackages;
     this.logger = logger;
-    this.watcher = new Watcher(watchPaths, {
+    this.workspacePath = workspacePath;
+    this.workspacePatterns = workspacePatterns;
+    this.buildQueue = new PQueue({ concurrency: buildConcurrency });
+    this.configBuildOnStartup = buildOnStartup;
+    this.configBuildOnChanges = buildOnChanges;
+    this.watcher = new Watcher([workspacePath], {
       recursive: true,
       ignoreInitial: true,
-      ignore: /\.git|node_modules|dist|\.highstate/
+      ignore: /\.git|node_modules|dist|\.highstate|\.nx/
     });
     this.watcher.on("all", (event, path5) => {
       this.logger.debug({ event, path: path5 }, "library event");
@@ -3937,12 +3944,18 @@ var LocalLibraryBackend = class _LocalLibraryBackend {
     });
   }
   watcher;
+  workspacePath;
+  workspacePatterns;
   lock = new BetterLock();
   eventEmitter = new EventEmitter();
   library = null;
   packages = /* @__PURE__ */ new Map();
   resolvedUnitSources = /* @__PURE__ */ new Map();
   rebuildStates = /* @__PURE__ */ new Map();
+  ignoredPackages = /* @__PURE__ */ new Set();
+  buildQueue;
+  configBuildOnStartup;
+  configBuildOnChanges;
   async loadLibrary() {
     return await this.lock.acquire(async () => await this.getLibrary());
   }
@@ -4065,72 +4078,89 @@ var LocalLibraryBackend = class _LocalLibraryBackend {
       await this.loadLibraryPackages(packagesToLoad, installIfNotFound);
     }
   }
-  async rebuildLibraryPackage(libraryPackage, installDeps = false, updateDeps = false, rebuiltPackages = /* @__PURE__ */ new Set()) {
-    if (rebuiltPackages.has(libraryPackage.name)) {
+  logIgnoredPackage(identifier, message) {
+    if (this.ignoredPackages.has(identifier)) {
       return;
     }
-    rebuiltPackages.add(libraryPackage.name);
-    if (installDeps) {
-      this.logger.info(`installing dependencies for package "${libraryPackage.name}"`);
-      await installDependencies({ cwd: libraryPackage.rootPath });
-    }
-    if (updateDeps) {
-      await this.updateLibraryPackageDependencies(libraryPackage);
+    this.logger.debug({ package: identifier }, message);
+    this.ignoredPackages.add(identifier);
+  }
+  isHighstateManagedPackage(packageName, packageJson) {
+    if (!packageJson.devDependencies?.["@highstate/cli"]) {
+      this.logIgnoredPackage(packageName, "skipping package without @highstate/cli dev dependency");
+      return false;
     }
-    this.logger.info(`rebuilding library package "${libraryPackage.name}" via build script`);
-    await runScript("build", { cwd: libraryPackage.rootPath });
-    if (!this.libraryPackages.includes(libraryPackage.name)) {
-      await this.reloadUnitManifest(libraryPackage);
+    const buildScript = packageJson.scripts?.build ?? "";
+    if (!buildScript.includes("highstate build")) {
+      this.logIgnoredPackage(
+        packageName,
+        'skipping package without "highstate build" in build script'
+      );
+      return false;
     }
-    await this.rebuildLibraryPackageDependents(libraryPackage, rebuiltPackages);
+    this.ignoredPackages.delete(packageName);
+    return true;
   }
-  async updateLibraryPackageDependencies(libraryPackage) {
-    const packageJson = await readPackageJSON(libraryPackage.rootPath);
-    const parsedName = _LocalLibraryBackend.parseDependencyName(libraryPackage.name);
-    const dependencyPackageNames = pipe(
-      [packageJson.dependencies, packageJson.peerDependencies],
-      flatMap((deps) => Object.keys(deps ?? {})),
-      unique(),
-      map(_LocalLibraryBackend.parseDependencyName)
-    );
-    const sameScopeDependencies = dependencyPackageNames.filter(
-      (dep) => dep.scope === parsedName.scope && dep.name !== parsedName.name
-    );
-    await this.ensureLibraryPackagesLoaded(sameScopeDependencies.map((dep) => dep.name));
-    for (const dependency of sameScopeDependencies) {
-      const dependencyPackage = this.packages.get(dependency.name);
-      if (!dependencyPackage) {
-        this.logger.warn(`dependency package not found for graph update: "%s"`, dependency.name);
-        continue;
+  async registerLibraryPackage(packageRootPath, declaredName, existingPackageJson) {
+    let packageJson = existingPackageJson;
+    if (!packageJson) {
+      try {
+        packageJson = await readPackageJSON(packageRootPath);
+      } catch (error) {
+        this.logger.debug({ error }, `failed to read package.json at path: "%s"`, packageRootPath);
+        this.logIgnoredPackage(packageRootPath, "skipping package without readable package.json");
+        return null;
       }
-      libraryPackage.dependencies.add(dependency.name);
-      dependencyPackage.dependents.add(libraryPackage.name);
     }
-  }
-  async rebuildLibraryPackageDependents(libraryPackage, rebuiltPackages = /* @__PURE__ */ new Set()) {
-    const promises = [];
-    for (const dependent of libraryPackage.dependents) {
-      const dependentPackage = this.packages.get(dependent);
-      if (!dependentPackage) {
-        this.logger.warn(`dependent package not found for rebuild: "%s"`, dependent);
-        continue;
-      }
-      promises.push(this.rebuildLibraryPackage(dependentPackage, false, false, rebuiltPackages));
+    if (!packageJson.name) {
+      this.logIgnoredPackage(packageRootPath, "skipping package without name in package.json");
+      return null;
     }
-    await Promise.all(promises);
-  }
-  static parseDependencyName(dependency) {
-    if (dependency.startsWith("@")) {
-      const parts = dependency.split("/");
-      return {
-        name: dependency,
-        scope: parts[0]
+    if (declaredName && declaredName !== packageJson.name) {
+      this.logger.warn(
+        {
+          declaredName,
+          packageJsonName: packageJson.name,
+          packageRootPath
+        },
+        "resolved package name does not match declared name"
+      );
+    }
+    const isHighstateManaged = this.isHighstateManagedPackage(packageJson.name, packageJson);
+    let libraryPackage = this.packages.get(packageJson.name);
+    if (libraryPackage) {
+      libraryPackage.rootPath = packageRootPath;
+      libraryPackage.isHighstateManaged = isHighstateManaged;
+    } else {
+      libraryPackage = {
+        name: packageJson.name,
+        rootPath: packageRootPath,
+        isHighstateManaged
       };
+      this.packages.set(packageJson.name, libraryPackage);
+    }
+    this.ignoredPackages.delete(packageRootPath);
+    if (libraryPackage.isHighstateManaged && this.library && !this.libraryPackages.includes(libraryPackage.name) && libraryPackage.name !== "@highstate/contract") {
+      await this.reloadUnitManifest(libraryPackage);
+    }
+    return libraryPackage;
+  }
+  async rebuildLibraryPackage(libraryPackage) {
+    if (!libraryPackage.isHighstateManaged) {
+      this.logIgnoredPackage(
+        libraryPackage.name,
+        "skipping rebuild for non Highstate-managed package"
+      );
+      return;
+    }
+    const now = Date.now();
+    this.logger.info(`rebuilding "%s"`, libraryPackage.name);
+    await runScript("build", { cwd: libraryPackage.rootPath });
+    const duration = Date.now() - now;
+    this.logger.info(`built "%s" in %dms`, libraryPackage.name, duration);
+    if (this.library && !this.libraryPackages.includes(libraryPackage.name)) {
+      await this.reloadUnitManifest(libraryPackage);
     }
-    return {
-      name: dependency,
-      scope: null
-    };
   }
   async readLibraryPackageManifest(libraryPackage) {
     const manifestPath = resolve$1(libraryPackage.rootPath, "dist", "highstate.manifest.json");
@@ -4164,7 +4194,6 @@ var LocalLibraryBackend = class _LocalLibraryBackend {
   async loadLibraryPackages(names, installIfNotFound = false) {
     this.logger.debug("loading library packages: %s", names.join(", "));
     const missingPackages = [];
-    const packagesToUpdate = [];
     const worker = this.createPackageResolutionWorker({ packageNames: names });
     for await (const [event] of on(worker, "message")) {
       const eventData = event;
@@ -4173,15 +4202,13 @@ var LocalLibraryBackend = class _LocalLibraryBackend {
       }
       for (const result of eventData.results) {
         if (result.type === "success") {
-          const libraryPackage = {
-            name: result.packageName,
-            rootPath: result.packageRootPath,
-            dependencies: /* @__PURE__ */ new Set(),
-            dependents: /* @__PURE__ */ new Set()
-          };
-          this.packages.set(result.packageName, libraryPackage);
-          packagesToUpdate.push(libraryPackage);
-          this.logger.info(`loaded library package: "%s"`, result.packageName);
+          const libraryPackage = await this.registerLibraryPackage(
+            result.packageRootPath,
+            result.packageName
+          );
+          if (libraryPackage) {
+            this.logger.info(`loaded library package: "%s"`, libraryPackage.name);
+          }
         } else if (result.type === "not-found") {
           missingPackages.push(result.packageName);
         } else {
@@ -4194,26 +4221,57 @@ var LocalLibraryBackend = class _LocalLibraryBackend {
       }
       break;
     }
-    for (const libraryPackage of packagesToUpdate) {
-      await this.updateLibraryPackageDependencies(libraryPackage);
-      if (!this.libraryPackages.includes(libraryPackage.name) && libraryPackage.name !== "@highstate/contract") {
-        await this.reloadUnitManifest(libraryPackage);
-      }
-    }
     if (installIfNotFound && missingPackages.length > 0) {
       this.logger.info("installing missing library packages: %s", missingPackages.join(", "));
       await addDependency(missingPackages);
       await this.loadLibraryPackages(missingPackages);
     }
   }
-  handleFileEvent(path5) {
-    const libraryPackage = this.packages.values().find((pkg) => path5.startsWith(pkg.rootPath));
-    if (!libraryPackage) {
-      return;
+  async handleFileEvent(path5) {
+    try {
+      const libraryPackage = await this.resolveLibraryPackageForPath(path5);
+      if (!libraryPackage) {
+        return;
+      }
+      if (!libraryPackage.isHighstateManaged) {
+        this.logIgnoredPackage(
+          libraryPackage.name,
+          "skipping file event for non Highstate-managed package"
+        );
+        return;
+      }
+      if (!this.configBuildOnChanges) {
+        return;
+      }
+      this.schedulePackageRebuild(libraryPackage.name);
+    } catch (error) {
+      this.logger.error({ error, path: path5 }, "failed to schedule library package rebuild");
+    }
+  }
+  async resolveLibraryPackageForPath(path5) {
+    const existingPackage = Array.from(this.packages.values()).find(
+      (pkg) => path5.startsWith(pkg.rootPath)
+    );
+    if (existingPackage) {
+      return existingPackage;
+    }
+    let packageJsonPath;
+    try {
+      packageJsonPath = await resolvePackageJSON(path5);
+    } catch (error) {
+      this.logger.debug({ error }, `failed to resolve package.json for path: "%s"`, path5);
+      return null;
+    }
+    const packageRootPath = dirname(packageJsonPath);
+    const alreadyLoaded = Array.from(this.packages.values()).find(
+      (pkg) => pkg.rootPath === packageRootPath
+    );
+    if (alreadyLoaded) {
+      return alreadyLoaded;
     }
-    void this.schedulePackageRebuild(libraryPackage.name);
+    return await this.registerLibraryPackage(packageRootPath);
   }
-  async schedulePackageRebuild(packageName) {
+  schedulePackageRebuild(packageName) {
     const state = this.rebuildStates.get(packageName) ?? { inProgress: false, pending: false };
     if (state.inProgress && state.pending) {
       this.logger.debug(`rebuild in progress and already pending for "%s", discarding`, packageName);
@@ -4225,36 +4283,178 @@ var LocalLibraryBackend = class _LocalLibraryBackend {
       this.logger.debug(`rebuild in progress, scheduling pending rebuild for "%s"`, packageName);
       return;
     }
-    await this.executePackageRebuild(packageName);
-  }
-  async executePackageRebuild(packageName) {
-    const state = this.rebuildStates.get(packageName) ?? { inProgress: false, pending: false };
-    do {
-      state.inProgress = true;
-      state.pending = false;
-      this.rebuildStates.set(packageName, state);
+    state.inProgress = true;
+    this.rebuildStates.set(packageName, state);
+    void this.buildQueue.add(async () => {
       try {
-        await this.lock.acquire(async () => {
-          const libraryPackage = this.packages.get(packageName);
-          if (!libraryPackage) {
-            this.logger.warn(`package not found for rebuild: "%s"`, packageName);
-            return;
-          }
-          const builtPackages = /* @__PURE__ */ new Set();
-          await this.rebuildLibraryPackage(libraryPackage, false, false, builtPackages);
-          if (this.libraryPackages.some((pkg) => builtPackages.has(pkg))) {
-            this.logger.info(
-              "reloading library due to file change in package: %s",
-              libraryPackage.name
-            );
-            await this.reloadLibrary();
-          }
-        });
+        await this.executePackageRebuild(packageName);
       } finally {
         state.inProgress = false;
+        if (state.pending) {
+          state.pending = false;
+          this.schedulePackageRebuild(packageName);
+        } else {
+          this.rebuildStates.delete(packageName);
+        }
+      }
+    });
+  }
+  async executePackageRebuild(packageName) {
+    const libraryPackage = this.packages.get(packageName);
+    if (!libraryPackage) {
+      this.logger.warn(`package not found for rebuild: "%s"`, packageName);
+      return;
+    }
+    await this.rebuildLibraryPackage(libraryPackage);
+    if (this.libraryPackages.includes(libraryPackage.name)) {
+      this.logger.info("reloading library due to file change in package: %s", libraryPackage.name);
+      await this.reloadLibrary();
+    }
+  }
+  async initialize() {
+    const workspacePackages = await this.collectWorkspacePackages();
+    if (workspacePackages.length === 0) {
+      return;
+    }
+    const managedPackageNames = /* @__PURE__ */ new Set();
+    await this.lock.acquire(async () => {
+      for (const workspacePackage of workspacePackages) {
+        const registered = await this.registerLibraryPackage(
+          workspacePackage.rootPath,
+          workspacePackage.packageJson.name,
+          workspacePackage.packageJson
+        );
+        if (registered?.isHighstateManaged) {
+          managedPackageNames.add(registered.name);
+        }
+      }
+    });
+    if (!this.configBuildOnStartup) {
+      return;
+    }
+    for (const packageName of managedPackageNames) {
+      this.schedulePackageRebuild(packageName);
+    }
+  }
+  async collectWorkspacePackages() {
+    if (this.workspacePatterns.length === 0) {
+      this.logger.warn(
+        { workspaceRoot: this.workspacePath },
+        "workspace root does not define workspaces; skipping startup builds"
+      );
+      return [];
+    }
+    return await this.expandWorkspacePatterns(this.workspacePath, this.workspacePatterns);
+  }
+  async expandWorkspacePatterns(rootPath, patterns) {
+    const includePatterns = patterns.filter((pattern) => !pattern.startsWith("!"));
+    const excludePatterns = patterns.filter((pattern) => pattern.startsWith("!")).map((pattern) => pattern.slice(1));
+    if (includePatterns.length === 0) {
+      return [];
+    }
+    let matches = [];
+    try {
+      matches = await glob(includePatterns, {
+        cwd: rootPath,
+        absolute: true,
+        dot: false,
+        ignore: excludePatterns
+      });
+    } catch (error) {
+      this.logger.error({ error, rootPath }, "failed to expand workspace patterns");
+      return [];
+    }
+    const results = [];
+    const seen = /* @__PURE__ */ new Set();
+    for (const match of matches) {
+      const packageDir = match.endsWith("package.json") ? dirname(match) : match;
+      if (seen.has(packageDir)) {
+        continue;
+      }
+      const packageJson = await this.tryReadPackageJson(packageDir);
+      if (!packageJson) {
+        continue;
+      }
+      seen.add(packageDir);
+      results.push({ rootPath: packageDir, packageJson });
+    }
+    return results;
+  }
+  async tryReadPackageJson(directory) {
+    try {
+      return await readPackageJSON(directory);
+    } catch (error) {
+      this.logger.trace(
+        { error, directory },
+        "failed to read package.json while resolving workspaces"
+      );
+      return null;
+    }
+  }
+  static extractWorkspacePatterns(packageJson) {
+    const workspaces = packageJson.workspaces;
+    if (!workspaces) {
+      return [];
+    }
+    if (Array.isArray(workspaces)) {
+      return workspaces;
+    }
+    if (typeof workspaces === "object" && Array.isArray(workspaces.packages)) {
+      return workspaces.packages;
+    }
+    return [];
+  }
+  static async resolveWorkspaceConfig(explicitWorkspace, projectRoot, logger) {
+    if (explicitWorkspace) {
+      const workspaceRoot = resolve$1(explicitWorkspace);
+      try {
+        const packageJson = await readPackageJSON(workspaceRoot);
+        const patterns = _LocalLibraryBackend.extractWorkspacePatterns(packageJson);
+        if (patterns.length === 0) {
+          logger.warn(
+            { workspaceRoot },
+            "configured workspace does not define workspaces; skipping startup builds"
+          );
+        }
+        return { root: workspaceRoot, patterns };
+      } catch (error) {
+        logger.warn(
+          { error, workspaceRoot },
+          "failed to read configured workspace package.json; skipping startup builds"
+        );
+        return { root: workspaceRoot, patterns: [] };
+      }
+    }
+    const detected = await _LocalLibraryBackend.findWorkspaceRootForPath(projectRoot);
+    if (detected) {
+      return detected;
+    }
+    logger.warn(
+      { projectRoot },
+      'no workspace root with "workspaces" field detected; skipping startup builds'
+    );
+    return { root: projectRoot, patterns: [] };
+  }
+  static async findWorkspaceRootForPath(startPath) {
+    let current = startPath;
+    const visited = /* @__PURE__ */ new Set();
+    while (!visited.has(current)) {
+      visited.add(current);
+      try {
+        const packageJson = await readPackageJSON(current);
+        const patterns = _LocalLibraryBackend.extractWorkspacePatterns(packageJson);
+        if (patterns.length > 0) {
+          return { root: current, patterns };
+        }
+      } catch {
       }
-    } while (state.pending);
-    this.rebuildStates.delete(packageName);
+      const parent = dirname(current);
+      if (parent === current) {
+        break;
+      }
+      current = parent;
+    }
+    return null;
   }
   createLibraryWorker(workerData) {
     const workerPathUrl = resolve(`@highstate/backend/library-worker`, import.meta.url);
@@ -4270,16 +4470,23 @@ var LocalLibraryBackend = class _LocalLibraryBackend {
     return new Worker(workerPath, { workerData });
   }
   static async create(config4, logger) {
-    let watchPaths = config4.HIGHSTATE_LIBRARY_BACKEND_LOCAL_WATCH_PATHS;
-    if (!watchPaths) {
-      const [projectPath] = await resolveMainLocalProject();
-      watchPaths = [resolve$1(projectPath, "packages")];
-    }
-    return new _LocalLibraryBackend(
-      config4.HIGHSTATE_LIBRARY_BACKEND_LOCAL_PACKAGES,
-      watchPaths,
+    const [projectPath] = await resolveMainLocalProject();
+    const workspaceConfig = await _LocalLibraryBackend.resolveWorkspaceConfig(
+      config4.HIGHSTATE_LIBRARY_BACKEND_LOCAL_WORKSPACE_PATH,
+      projectPath,
+      logger
+    );
+    const backend = new _LocalLibraryBackend(
+      config4.HIGHSTATE_LIBRARY_BACKEND_LOCAL_LIBRARY_PACKAGES,
+      workspaceConfig.root,
+      workspaceConfig.patterns,
+      config4.HIGHSTATE_LIBRARY_BACKEND_LOCAL_BUILD_CONCURRENCY,
+      config4.HIGHSTATE_LIBRARY_BACKEND_LOCAL_BUILD_ON_STARTUP,
+      config4.HIGHSTATE_LIBRARY_BACKEND_LOCAL_BUILD_ON_CHANGES,
       logger.child({ backend: "LibraryBackend", service: "LocalLibraryBackend" })
     );
+    void backend.initialize().catch((error) => logger.error({ error }, "failed to initialize LocalLibraryBackend"));
+    return backend;
   }
 };
 
@@ -4456,13 +4663,13 @@ async function setupArtifactEnvironment(projectId, artifacts, artifactBackend, t
     await Promise.all(
       artifacts.map(async (artifact) => {
         const artifactPath = join(readDir, `${artifact.hash}.tgz`);
-        const stream = await artifactBackend.retrieve(projectId, artifact.hash, artifact.chunkSize);
+        const stream = await artifactBackend.retrieve(projectId, artifact.id, artifact.chunkSize);
         if (!stream) {
           throw new Error(`artifact not found: ${artifact.hash}`);
         }
         const writeStream = createWriteStream(artifactPath);
         await pipeline(stream, writeStream);
-        logger.debug({ msg: "downloaded artifact", hash: artifact.hash, path: artifactPath });
+        logger.debug({ msg: "downloaded artifact", id: artifact.id, path: artifactPath });
       })
     );
   } else {
@@ -4511,23 +4718,24 @@ async function collectAndStoreArtifacts(writePath, projectId, stateId, artifactM
               return;
             }
             const readStream = __using(_stack, createReadStream(filePath), true);
-            await artifactManager.store(
+            const storedArtifact = await artifactManager.store(
               projectId,
               expectedHash,
               fileSize,
               artifact.meta ?? { title: `Artifact for "${stateId}"` },
               readStream,
-              async (tx, storedArtifact) => {
+              async (tx, storedArtifact2) => {
                 await tx.instanceState.update({
                   where: { id: stateId },
                   data: {
                     artifacts: {
-                      connect: { id: storedArtifact.id }
+                      connect: { id: storedArtifact2.id }
                     }
                   }
                 });
               }
             );
+            artifact[unitArtifactId] = storedArtifact.id;
             logger.debug({ msg: "stored artifact", hash: expectedHash, file: filePath });
           } catch (_) {
             var _error = _, _hasError = true;
@@ -4826,7 +5034,7 @@ var LocalRunnerBackend = class _LocalRunnerBackend {
     let unitTempPath = null;
     let artifactEnv = null;
     try {
-      unitTempPath = await mkdtemp(join(tmpdir(), `highstate-unit-${unitId}-`));
+      unitTempPath = await mkdtemp(join(tmpdir(), `highstate-unit-${options.stateId}-`));
       childLogger.debug({ msg: "created unit temp directory", unitTempPath });
       options.signal?.throwIfAborted();
       artifactEnv = await setupArtifactEnvironment(
@@ -4915,6 +5123,16 @@ var LocalRunnerBackend = class _LocalRunnerBackend {
                   Object.values(artifacts).flat(),
                   childLogger
                 );
+                completionUpdate.exportedArtifactIds = mapValues(artifacts, (artifacts2) => {
+                  return artifacts2.map((artifact) => {
+                    if (artifact[unitArtifactId]) {
+                      return artifact[unitArtifactId];
+                    }
+                    throw new Error(
+                      `Failed to determine artifact ID for artifact with hash ${artifact.hash}`
+                    );
+                  });
+                });
               } else if (preview && outputs["$artifacts"]) {
                 childLogger.debug({ msg: "skipping artifact persistence for preview" });
               }
@@ -6205,13 +6423,13 @@ var OperationContext = class _OperationContext {
    * @returns A map of instance IDs to state IDs.
    */
   getInstanceIdToStateIdMap(instanceId) {
-    const map2 = {};
+    const map = {};
     const dependencies = this.getDependencies(instanceId).map((i) => i.id);
     for (const dep of dependencies) {
       const state = this.getState(dep);
-      map2[dep] = state.id;
+      map[dep] = state.id;
     }
-    return map2;
+    return map;
   }
   getUnfinishedOperationStates() {
     const unfinishedStates = [];
@@ -7431,7 +7649,6 @@ ${errors.join("\n")}`
       });
       logger.debug("destroy request sent");
       await this.watchStateStream(state, type, name, logger);
-      await this.artifactService.clearInstanceArtifactReferences(this.project.id, instance.id);
       logger.info("unit destroyed");
     });
   }
@@ -7557,12 +7774,14 @@ ${errors.join("\n")}`
       status: this.workset.getNextStableInstanceStatus(instance.id),
       statusFields: update.statusFields ?? null
     };
+    const artifactIds = update.exportedArtifactIds ? Object.values(update.exportedArtifactIds).flat() : [];
     if (update.operationType !== "destroy") {
       state.outputHash = update.outputHash ?? null;
       const { inputHash, dependencyOutputHash } = await this.context.getUpToDateInputHashOutput(instance);
       data.inputHash = inputHash;
       data.dependencyOutputHash = dependencyOutputHash;
       data.outputHash = update.outputHash;
+      data.exportedArtifactIds = update.exportedArtifactIds;
       if (instance.parentId) {
         const parentState = this.context.getState(instance.parentId);
         data.parentId = parentState.id;
@@ -7576,6 +7795,7 @@ ${errors.join("\n")}`
       data.parentId = null;
       data.model = null;
       data.resolvedInputs = null;
+      data.exportedArtifactIds = null;
     }
     await this.workset.updateState(instance.id, {
       // TODO: honestly, it is not correct
@@ -7593,7 +7813,8 @@ ${errors.join("\n")}`
         terminals: update.terminals ?? [],
         triggers: update.triggers ?? [],
         workers: update.workers ?? [],
-        secrets: update.secrets ?? {}
+        secrets: update.secrets ?? {},
+        artifactIds
       } : void 0
     });
   }