@malloy-publisher/server 0.0.204 → 0.0.206

package/src/service/environment_store.spec.ts

@@ -11,12 +11,6 @@ import { EnvironmentStore } from "./environment_store";
 
 type MockData = Record<string, unknown>;
 
-const initializeDuckLakeCalls: Array<{
-   environmentId: string;
-   environmentName: string;
-   config: { catalogUrl: string; dataPath: string };
-}> = [];
-
 mock.module("../storage/StorageManager", () => {
    return {
       StorageManager: class MockStorageManager {
@@ -24,18 +18,6 @@ mock.module("../storage/StorageManager", () => {
             return;
          }
 
-         async initializeDuckLakeForEnvironment(
-            environmentId: string,
-            environmentName: string,
-            config: { catalogUrl: string; dataPath: string },
-         ): Promise<void> {
-            initializeDuckLakeCalls.push({
-               environmentId,
-               environmentName,
-               config,
-            });
-         }
-
          getRepository() {
             return {
                // ===== PROJECT METHODS =====
@@ -488,69 +470,6 @@ describe("EnvironmentStore Service", () => {
       expect(readmeContent).toBe("Updated README content");
    });
 
-   it("should propagate materializationStorage on addEnvironment for new environment", async () => {
-      writeFileSync(
-         path.join(serverRootPath, "publisher.config.json"),
-         JSON.stringify({ frozenConfig: false, environments: [] }),
-      );
-
-      await environmentStore.finishedInitialization;
-
-      const materializationStorage = {
-         catalogUrl:
-            "postgres:host=localhost port=5432 dbname=ducklake user=u password=p",
-         dataPath: "gs://test-bucket",
-      };
-
-      initializeDuckLakeCalls.length = 0;
-      const project = await environmentStore.addEnvironment({
-         name: projectName,
-         materializationStorage,
-      });
-
-      expect(project.metadata.materializationStorage).toEqual(
-         materializationStorage,
-      );
-      expect(initializeDuckLakeCalls).toHaveLength(1);
-      expect(initializeDuckLakeCalls[0].config).toEqual(materializationStorage);
-   });
-
-   it("should propagate materializationStorage on update", async () => {
-      const projectPath = path.join(serverRootPath, projectName);
-      mkdirSync(projectPath, { recursive: true });
-      writeFileSync(
-         path.join(projectPath, "publisher.json"),
-         JSON.stringify({ name: projectName, description: "Test package" }),
-      );
-      writeFileSync(
-         path.join(serverRootPath, "publisher.config.json"),
-         JSON.stringify({
-            frozenConfig: false,
-            environments: [
-               {
-                  name: projectName,
-                  packages: [{ name: projectName, location: projectPath }],
-               },
-            ],
-         }),
-      );
-
-      await environmentStore.finishedInitialization;
-      const project = await environmentStore.getEnvironment(projectName);
-
-      const materializationStorage = {
-         catalogUrl:
-            "postgres:host=localhost port=5432 dbname=ducklake user=u password=p",
-         dataPath: "gs://test-bucket",
-      };
-
-      await project.update({ name: projectName, materializationStorage });
-
-      expect(project.metadata.materializationStorage).toEqual(
-         materializationStorage,
-      );
-   });
-
    it(
       "should handle project reload",
       async () => {

package/src/service/environment_store.ts

@@ -95,6 +95,26 @@ function validateEnvironmentAzureUrls(environment: ApiEnvironment): void {
    }
 }
 
+// Safely clear a prior mount target before (re)mounting at `targetPath`.
+// Distinguishes symlinks from real dirs: fs.rm's recursive mode could otherwise
+// traverse into a symlinked source and damage it, so unlink symlinks and rm real
+// dirs. Both the in-place (symlink) and copy mount paths call this, so toggling
+// watch mode across runs against the same publisher_data can't leave a stale
+// entry that breaks the next mount (mkdir is a no-op on a symlink and fs.cp then
+// throws ERR_FS_CP_DIR_TO_NON_DIR).
+async function clearMountTarget(targetPath: string): Promise<void> {
+   try {
+      const stats = await fs.promises.lstat(targetPath);
+      if (stats.isDirectory() && !stats.isSymbolicLink()) {
+         await fs.promises.rm(targetPath, { recursive: true, force: true });
+      } else {
+         await fs.promises.unlink(targetPath);
+      }
+   } catch {
+      // Nothing there, fine.
+   }
+}
+
 export class EnvironmentStore {
    public serverRootPath: string;
    private environments: Map<string, Environment> = new Map();
@@ -112,10 +132,36 @@ export class EnvironmentStore {
    // new Environments pick it up at construction.
    private memoryGovernor: PackageMemoryGovernor | null = null;
 
+   /**
+    * Set of environment names that should be loaded "in place" — i.e. the
+    * package directories under `publisher_data/<envName>/` are symlinks to
+    * the original local source directories instead of recursive copies.
+    *
+    * In-place mode powers the `npm run dev`-style live-reload story: edits
+    * to your source repo are visible to Publisher and trigger watch events
+    * that fan out via SSE to embedded HTML pages. Production mode (the
+    * default) keeps the safe copy semantics so the source repo and the
+    * served package are decoupled.
+    *
+    * Populated from the `PUBLISHER_WATCH` env var (comma-separated env
+    * names) at construction; can also be augmented via `markInPlace()` if
+    * a future feature wants to flip an environment to dev mode at runtime.
+    *
+    * Only LOCAL-DIR package locations are eligible for symlinking; remote
+    * sources (GitHub, GCS, S3) always copy regardless.
+    */
+   private inPlaceEnvs = new Set<string>();
+
    constructor(serverRootPath: string) {
       this.serverRootPath = serverRootPath;
       this.gcsClient = new Storage();
 
+      const watchEnvList = (process.env.PUBLISHER_WATCH || "")
+         .split(",")
+         .map((s) => s.trim())
+         .filter((s) => s.length > 0);
+      for (const envName of watchEnvList) this.inPlaceEnvs.add(envName);
+
       const storageConfig: StorageConfig = {
          type: "duckdb",
          duckdb: {
@@ -127,6 +173,17 @@ export class EnvironmentStore {
       this.finishedInitialization = this.initialize();
    }
 
+   /** True if this env should mount package dirs in place (symlinks). */
+   public isInPlace(environmentName: string): boolean {
+      return this.inPlaceEnvs.has(environmentName);
+   }
+
+   /** Add an env to in-place mode at runtime. Caller is responsible for
+    *  triggering a re-mount if the env was already loaded with copies. */
+   public markInPlace(environmentName: string): void {
+      this.inPlaceEnvs.add(environmentName);
+   }
+
    /**
     * Attach (or detach with `null`) the shared {@link PackageMemoryGovernor}.
     * Propagated to every Environment so the back-pressure decision is
@@ -395,25 +452,6 @@ export class EnvironmentStore {
          dbEnvironment = await repository.createEnvironment(environmentData);
       }
 
-      // Initialize DuckLake manifest storage if configured on the environment.
-      const materializationStorage = environment.metadata
-         ?.materializationStorage as
-         | { catalogUrl?: string; dataPath?: string }
-         | undefined;
-      if (
-         materializationStorage?.catalogUrl &&
-         materializationStorage?.dataPath
-      ) {
-         await this.storageManager.initializeDuckLakeForEnvironment(
-            dbEnvironment.id,
-            dbEnvironment.name,
-            {
-               catalogUrl: materializationStorage.catalogUrl,
-               dataPath: materializationStorage.dataPath,
-            },
-         );
-      }
-
       return dbEnvironment;
    }
 
@@ -881,10 +919,6 @@ export class EnvironmentStore {
 
       if (!newEnvironment.metadata) newEnvironment.metadata = {};
       newEnvironment.metadata.location = absoluteEnvironmentPath;
-      if (environment.materializationStorage !== undefined) {
-         newEnvironment.metadata.materializationStorage =
-            environment.materializationStorage;
-      }
 
       this.environments.set(environmentName, newEnvironment);
 
@@ -1202,7 +1236,15 @@ export class EnvironmentStore {
                } else {
                   // For non-GitHub locations, use package name
                   if (this.isLocalPath(_package.location)) {
-                     sourcePath = _package.location;
+                     // Match the resolution rule used by
+                     // `downloadOrMountLocation` (line ~1352): relative
+                     // paths are anchored at `serverRootPath`. Without this
+                     // step the existing-source check below falls through
+                     // for any relative location, and the in-place mount
+                     // branch is unreachable.
+                     sourcePath = path.isAbsolute(_package.location)
+                        ? _package.location
+                        : path.join(this.serverRootPath, _package.location);
                   } else {
                      sourcePath = safeJoinUnderRoot(
                         tempDownloadPath,
@@ -1217,18 +1259,84 @@ export class EnvironmentStore {
                   .catch(() => false);
 
                if (sourceExists) {
-                  // Copy the specific directory
-                  await fs.promises.mkdir(absolutePackagePath, {
-                     recursive: true,
-                  });
-                  await fs.promises.cp(sourcePath, absolutePackagePath, {
-                     recursive: true,
-                  });
-                  logger.info(
-                     `Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`,
-                  );
+                  // In-place mount path (watch-mode-only, local-dir packages
+                  // only). Replace the recursive copy with a symlink so that
+                  // edits to the source repo are visible to Publisher and
+                  // can trigger live-reload events.
+                  //
+                  // Carefully: we MUST distinguish symlinks from real dirs
+                  // when removing the prior content, because `fs.rm`'s
+                  // recursive mode could in principle traverse into a
+                  // symlinked source dir and damage it. We lstat first and
+                  // call `unlink` for symlinks, `rm` for real directories.
+                  const isInPlace =
+                     this.inPlaceEnvs.has(environmentName) &&
+                     this.isLocalPath(_package.location);
+                  if (isInPlace) {
+                     await clearMountTarget(absolutePackagePath);
+                     const absoluteSourcePath = path.resolve(sourcePath);
+                     // On Windows a "dir" symlink needs elevation
+                     // (SeCreateSymbolicLinkPrivilege — admin or Developer
+                     // Mode); a junction does not and works for absolute
+                     // directory targets, so prefer it there. chokidar watches
+                     // through both.
+                     const linkType =
+                        process.platform === "win32" ? "junction" : "dir";
+                     try {
+                        await fs.promises.symlink(
+                           absoluteSourcePath,
+                           absolutePackagePath,
+                           linkType,
+                        );
+                        logger.info(
+                           `In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`,
+                        );
+                     } catch (linkError) {
+                        // Degrade gracefully instead of failing the env load:
+                        // copy the package so it still serves. Live reload
+                        // won't work for it (source edits aren't seen), but the
+                        // environment comes up. Hit e.g. on locked-down Windows
+                        // where even junction creation is denied.
+                        const code =
+                           (linkError as NodeJS.ErrnoException)?.code ??
+                           String(linkError);
+                        logger.warn(
+                           `In-place mount failed for package "${packageDir}" (${code}); falling back to a copy. Source-edit live reload is disabled for this package.`,
+                        );
+                        await clearMountTarget(absolutePackagePath);
+                        await fs.promises.mkdir(absolutePackagePath, {
+                           recursive: true,
+                        });
+                        await fs.promises.cp(sourcePath, absolutePackagePath, {
+                           recursive: true,
+                        });
+                     }
+                  } else {
+                     if (
+                        this.inPlaceEnvs.has(environmentName) &&
+                        !this.isLocalPath(_package.location)
+                     ) {
+                        logger.warn(
+                           `Watch mode: package "${packageDir}" has remote location "${_package.location}" — falling back to copy. Source-edit live reload won't work for this package; clone the source locally and use a local-dir location to enable it.`,
+                        );
+                     }
+                     // Copy the specific directory. Clear any stale mount target
+                     // first (e.g. a symlink left by a prior --watch-env run), or
+                     // fs.cp would throw ERR_FS_CP_DIR_TO_NON_DIR onto it.
+                     await clearMountTarget(absolutePackagePath);
+                     await fs.promises.mkdir(absolutePackagePath, {
+                        recursive: true,
+                     });
+                     await fs.promises.cp(sourcePath, absolutePackagePath, {
+                        recursive: true,
+                     });
+                     logger.info(
+                        `Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`,
+                     );
+                  }
                } else {
                   // If source doesn't exist, copy the entire download as the package
+                  await clearMountTarget(absolutePackagePath);
                   await fs.promises.mkdir(absolutePackagePath, {
                      recursive: true,
                   });