@malloy-publisher/server 0.0.203 → 0.0.205

package/src/service/compile_authorize.spec.ts

@@ -0,0 +1,85 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import { AccessDeniedError } from "../errors";
+import { Environment } from "./environment";
+
+// End-to-end gate on the /compile path. Exercises environment.compileSource
+// through a real installed package, not just the Model primitives — pins that
+// the early gate AND the compiled-source backstop fire, the latter REGARDLESS of
+// includeSql (a compile-time schema oracle is closed even with no SQL extraction).
+
+const PUBLISHER_JSON = JSON.stringify({
+   name: "pkg",
+   description: "compile-gate",
+});
+
+// `gated` is locked to $ROLE='analyst'; `open_src` is unrestricted.
+const MODEL = `##! experimental.givens
+
+given:
+  ROLE :: string
+
+#(authorize) "$ROLE = 'analyst'"
+source: gated is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
+
+source: open_src is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
+`;
+
+describe("compile-path authorize gate (compileSource)", () => {
+   let rootDir: string;
+   let env: Environment;
+
+   beforeEach(async () => {
+      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-compile-"));
+      const envPath = path.join(rootDir, "env");
+      await fs.mkdir(envPath, { recursive: true });
+      env = await Environment.create("testEnv", envPath, []);
+      await env.installPackage("pkg", async (stagingPath) => {
+         await fs.mkdir(stagingPath, { recursive: true });
+         await fs.writeFile(
+            path.join(stagingPath, "publisher.json"),
+            PUBLISHER_JSON,
+         );
+         await fs.writeFile(path.join(stagingPath, "model.malloy"), MODEL);
+      });
+   });
+
+   afterEach(async () => {
+      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
+   });
+
+   const compile = (source: string, givens?: Record<string, string>) =>
+      env.compileSource("pkg", "model.malloy", source, false, givens);
+
+   it("denies a direct gated source without the satisfying given (early gate)", async () => {
+      await expect(
+         compile("run: gated -> { aggregate: c }"),
+      ).rejects.toBeInstanceOf(AccessDeniedError);
+   });
+
+   it("denies a gated source reached via the LAST run: statement (backstop, includeSql=false)", async () => {
+      // Regression guard: the early gate only matches the first `run:` (ungated
+      // open_src here), so the gated source in the executed final statement is
+      // caught only by the compiled-source backstop — which must run even when
+      // no SQL is requested.
+      await expect(
+         compile(
+            "run: open_src -> { aggregate: c }\nrun: gated -> { aggregate: c }",
+         ),
+      ).rejects.toBeInstanceOf(AccessDeniedError);
+   });
+
+   it("allows the gated source when the given satisfies the gate", async () => {
+      const { problems } = await compile("run: gated -> { aggregate: c }", {
+         ROLE: "analyst",
+      });
+      expect(problems).toBeDefined();
+   });
+
+   it("leaves an ungated source compilable without any given", async () => {
+      const { problems } = await compile("run: open_src -> { aggregate: c }");
+      expect(problems).toBeDefined();
+   });
+});

package/src/service/connection.ts

@@ -1084,7 +1084,7 @@ export function buildEnvironmentMalloyConfig(
             ...azureDuckDBCache.values(),
          ];
          const closeResults = await Promise.allSettled([
-            malloyConfig.releaseConnections(),
+            malloyConfig.shutdown("close"),
             ...wrapperPromises.map(async (promise) => {
                const connection = await promise;
                await connection.close();

package/src/service/environment.ts

@@ -5,6 +5,7 @@ import { Mutex } from "async-mutex";
 import crypto from "crypto";
 import * as fs from "fs";
 import * as path from "path";
+import { pathToFileURL } from "url";
 import { components } from "../api";
 import { API_PREFIX, README_NAME } from "../constants";
 import {
@@ -137,6 +138,11 @@ export class Environment {
    // governor as the single owner of the back-pressure boolean.
    private memoryGovernor: PackageMemoryGovernor | null = null;
 
+   /** Absolute path on disk where this environment's package files live. */
+   public getEnvironmentPath(): string {
+      return this.environmentPath;
+   }
+
    constructor(
       environmentName: string,
       environmentPath: string,
@@ -307,10 +313,18 @@ export class Environment {
             packageName,
             modelName,
          );
-         // Place the virtual file in the model's directory so relative imports resolve correctly.
+         // Place the virtual file in the model's directory so relative imports
+         // resolve correctly. Use `pathToFileURL` rather than hand-prefixing
+         // `file://`: on Windows the latter produces a malformed URL
+         // (`file://D:\Temp\…`) that round-trips differently than the URL the
+         // Malloy runtime synthesizes from the same path, breaking the
+         // intercepting reader's string comparison below and falling through
+         // to disk for a virtual file that doesn't exist.
          const modelDir = path.dirname(modelPath);
-         const virtualUri = `file://${path.join(modelDir, "__compile_check.malloy")}`;
-         const virtualUrl = new URL(virtualUri);
+         const virtualUrl = pathToFileURL(
+            path.join(modelDir, "__compile_check.malloy"),
+         );
+         const virtualUri = virtualUrl.toString();
 
          // Read the full model file so the submitted source inherits the model's
          // complete namespace — imports, source definitions, queries, etc.
@@ -339,6 +353,21 @@ export class Environment {
          // Use the locked variant — we already hold the per-package mutex.
          const pkg = await this._loadOrGetPackageLocked(packageName);
 
+         // Authorize gate: /compile is compile-only, but it can still act
+         // as a schema oracle (a denied caller learns a gated source's columns
+         // from compile errors) and, with includeSql, leak its SQL. Gate the
+         // named source the submitted text targets BEFORE compiling — mirrors
+         // the query path's early surface-syntax gate. Unnamed/inline source
+         // text resolves to undefined, so only the model-wide file-level gate
+         // applies. The gate runs against the package's cached Model (its
+         // `given:` block + authorize annotations), independent of the virtual
+         // compile below. If the model isn't loaded, there's nothing to enforce
+         // and compilation surfaces its own error.
+         const gateModel = pkg.getModel(modelName);
+         if (gateModel) {
+            await gateModel.assertAuthorizedForText(source, givens ?? {});
+         }
+
          // Initialize Runtime with the package's active MalloyConfig so compile
          // checks see the same package-scoped duckdb as execution. This runtime
          // borrows the package config; the package/environment lifecycle owns release.
@@ -352,11 +381,40 @@ export class Environment {
             const modelMaterializer = runtime.loadModel(virtualUrl);
             const model = await modelMaterializer.getModel();
 
+            // Resolve the final query's materializer once (if there is one).
+            let queryMaterializer: ReturnType<
+               typeof modelMaterializer.loadFinalQuery
+            > | null = null;
+            try {
+               queryMaterializer = modelMaterializer.loadFinalQuery();
+            } catch {
+               // No runnable query (e.g. only source definitions) — nothing to
+               // gate or extract beyond the early text gate already applied.
+            }
+
+            // Compiled-source backstop — runs REGARDLESS of includeSql. It gates
+            // the source the COMPILED final query actually reads, closing
+            // named-query / multi-statement indirection the early surface-syntax
+            // gate misses (e.g. `run: ungated\nrun: gated` — the early gate only
+            // matches the FIRST `run:`, but the LAST statement is what executes).
+            // Compiling a gated source even without SQL is a schema oracle
+            // (field-not-found errors leak its columns), so this must not be
+            // conditional on SQL extraction. (A `source: x is gated` alias makes
+            // a new ungated source — that's the documented "extend doesn't
+            // inherit authorize" footgun, the same as the query path.) Only run
+            // when the model declares gates so ungated compiles don't pay for
+            // the extra final-query compile.
+            if (queryMaterializer && gateModel?.hasAuthorize()) {
+               await gateModel.assertAuthorizedForRunnable(
+                  queryMaterializer,
+                  givens ?? {},
+               );
+            }
+
             // If includeSql is requested and compilation succeeded, attempt to extract SQL
             let sql: string | undefined;
-            if (includeSql) {
+            if (includeSql && queryMaterializer) {
                try {
-                  const queryMaterializer = modelMaterializer.loadFinalQuery();
                   sql = await queryMaterializer.getSQL({ givens });
                } catch {
                   // Source may not contain a runnable query (e.g. only source definitions),
@@ -736,7 +794,7 @@ export class Environment {
          );
          if (existingPackage !== undefined && reload) {
             this.retireConnectionGeneration(`package ${packageName}`, () =>
-               existingPackage.getMalloyConfig().releaseConnections(),
+               existingPackage.getMalloyConfig().shutdown("close"),
             );
          }
          this.packages.set(packageName, _package);
@@ -955,7 +1013,7 @@ export class Environment {
 
          if (oldPackage) {
             this.retireConnectionGeneration(`package ${packageName}`, () =>
-               oldPackage.getMalloyConfig().releaseConnections(),
+               oldPackage.getMalloyConfig().shutdown("close"),
             );
          }
 
@@ -1139,7 +1197,7 @@ export class Environment {
          // any in-flight queries that already acquired a connection finish
          // before the underlying duckdb handle is released.
          this.retireConnectionGeneration(`package ${packageName}`, () =>
-            _package.getMalloyConfig().releaseConnections(),
+            _package.getMalloyConfig().shutdown("close"),
          );
 
          // Atomically rename the canonical tree out of the way so no reader
@@ -1235,7 +1293,7 @@ export class Environment {
       // they wrap. Without this, hard unload leaks per-package DuckDB handles.
       const packageReleases = await Promise.allSettled(
          Array.from(this.packages.values(), (pkg) =>
-            pkg.getMalloyConfig().releaseConnections(),
+            pkg.getMalloyConfig().shutdown("close"),
          ),
       );
       for (const result of packageReleases) {

package/src/service/environment_store.ts

@@ -95,6 +95,26 @@ function validateEnvironmentAzureUrls(environment: ApiEnvironment): void {
    }
 }
 
+// Safely clear a prior mount target before (re)mounting at `targetPath`.
+// Distinguishes symlinks from real dirs: fs.rm's recursive mode could otherwise
+// traverse into a symlinked source and damage it, so unlink symlinks and rm real
+// dirs. Both the in-place (symlink) and copy mount paths call this, so toggling
+// watch mode across runs against the same publisher_data can't leave a stale
+// entry that breaks the next mount (mkdir is a no-op on a symlink and fs.cp then
+// throws ERR_FS_CP_DIR_TO_NON_DIR).
+async function clearMountTarget(targetPath: string): Promise<void> {
+   try {
+      const stats = await fs.promises.lstat(targetPath);
+      if (stats.isDirectory() && !stats.isSymbolicLink()) {
+         await fs.promises.rm(targetPath, { recursive: true, force: true });
+      } else {
+         await fs.promises.unlink(targetPath);
+      }
+   } catch {
+      // Nothing there, fine.
+   }
+}
+
 export class EnvironmentStore {
    public serverRootPath: string;
    private environments: Map<string, Environment> = new Map();
@@ -112,10 +132,36 @@ export class EnvironmentStore {
    // new Environments pick it up at construction.
    private memoryGovernor: PackageMemoryGovernor | null = null;
 
+   /**
+    * Set of environment names that should be loaded "in place" — i.e. the
+    * package directories under `publisher_data/<envName>/` are symlinks to
+    * the original local source directories instead of recursive copies.
+    *
+    * In-place mode powers the `npm run dev`-style live-reload story: edits
+    * to your source repo are visible to Publisher and trigger watch events
+    * that fan out via SSE to embedded HTML pages. Production mode (the
+    * default) keeps the safe copy semantics so the source repo and the
+    * served package are decoupled.
+    *
+    * Populated from the `PUBLISHER_WATCH` env var (comma-separated env
+    * names) at construction; can also be augmented via `markInPlace()` if
+    * a future feature wants to flip an environment to dev mode at runtime.
+    *
+    * Only LOCAL-DIR package locations are eligible for symlinking; remote
+    * sources (GitHub, GCS, S3) always copy regardless.
+    */
+   private inPlaceEnvs = new Set<string>();
+
    constructor(serverRootPath: string) {
       this.serverRootPath = serverRootPath;
       this.gcsClient = new Storage();
 
+      const watchEnvList = (process.env.PUBLISHER_WATCH || "")
+         .split(",")
+         .map((s) => s.trim())
+         .filter((s) => s.length > 0);
+      for (const envName of watchEnvList) this.inPlaceEnvs.add(envName);
+
       const storageConfig: StorageConfig = {
          type: "duckdb",
          duckdb: {
@@ -127,6 +173,17 @@ export class EnvironmentStore {
       this.finishedInitialization = this.initialize();
    }
 
+   /** True if this env should mount package dirs in place (symlinks). */
+   public isInPlace(environmentName: string): boolean {
+      return this.inPlaceEnvs.has(environmentName);
+   }
+
+   /** Add an env to in-place mode at runtime. Caller is responsible for
+    *  triggering a re-mount if the env was already loaded with copies. */
+   public markInPlace(environmentName: string): void {
+      this.inPlaceEnvs.add(environmentName);
+   }
+
    /**
     * Attach (or detach with `null`) the shared {@link PackageMemoryGovernor}.
     * Propagated to every Environment so the back-pressure decision is
@@ -1202,7 +1259,15 @@ export class EnvironmentStore {
                } else {
                   // For non-GitHub locations, use package name
                   if (this.isLocalPath(_package.location)) {
-                     sourcePath = _package.location;
+                     // Match the resolution rule used by
+                     // `downloadOrMountLocation` (line ~1352): relative
+                     // paths are anchored at `serverRootPath`. Without this
+                     // step the existing-source check below falls through
+                     // for any relative location, and the in-place mount
+                     // branch is unreachable.
+                     sourcePath = path.isAbsolute(_package.location)
+                        ? _package.location
+                        : path.join(this.serverRootPath, _package.location);
                   } else {
                      sourcePath = safeJoinUnderRoot(
                         tempDownloadPath,
@@ -1217,18 +1282,84 @@ export class EnvironmentStore {
                   .catch(() => false);
 
                if (sourceExists) {
-                  // Copy the specific directory
-                  await fs.promises.mkdir(absolutePackagePath, {
-                     recursive: true,
-                  });
-                  await fs.promises.cp(sourcePath, absolutePackagePath, {
-                     recursive: true,
-                  });
-                  logger.info(
-                     `Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`,
-                  );
+                  // In-place mount path (watch-mode-only, local-dir packages
+                  // only). Replace the recursive copy with a symlink so that
+                  // edits to the source repo are visible to Publisher and
+                  // can trigger live-reload events.
+                  //
+                  // Carefully: we MUST distinguish symlinks from real dirs
+                  // when removing the prior content, because `fs.rm`'s
+                  // recursive mode could in principle traverse into a
+                  // symlinked source dir and damage it. We lstat first and
+                  // call `unlink` for symlinks, `rm` for real directories.
+                  const isInPlace =
+                     this.inPlaceEnvs.has(environmentName) &&
+                     this.isLocalPath(_package.location);
+                  if (isInPlace) {
+                     await clearMountTarget(absolutePackagePath);
+                     const absoluteSourcePath = path.resolve(sourcePath);
+                     // On Windows a "dir" symlink needs elevation
+                     // (SeCreateSymbolicLinkPrivilege — admin or Developer
+                     // Mode); a junction does not and works for absolute
+                     // directory targets, so prefer it there. chokidar watches
+                     // through both.
+                     const linkType =
+                        process.platform === "win32" ? "junction" : "dir";
+                     try {
+                        await fs.promises.symlink(
+                           absoluteSourcePath,
+                           absolutePackagePath,
+                           linkType,
+                        );
+                        logger.info(
+                           `In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`,
+                        );
+                     } catch (linkError) {
+                        // Degrade gracefully instead of failing the env load:
+                        // copy the package so it still serves. Live reload
+                        // won't work for it (source edits aren't seen), but the
+                        // environment comes up. Hit e.g. on locked-down Windows
+                        // where even junction creation is denied.
+                        const code =
+                           (linkError as NodeJS.ErrnoException)?.code ??
+                           String(linkError);
+                        logger.warn(
+                           `In-place mount failed for package "${packageDir}" (${code}); falling back to a copy. Source-edit live reload is disabled for this package.`,
+                        );
+                        await clearMountTarget(absolutePackagePath);
+                        await fs.promises.mkdir(absolutePackagePath, {
+                           recursive: true,
+                        });
+                        await fs.promises.cp(sourcePath, absolutePackagePath, {
+                           recursive: true,
+                        });
+                     }
+                  } else {
+                     if (
+                        this.inPlaceEnvs.has(environmentName) &&
+                        !this.isLocalPath(_package.location)
+                     ) {
+                        logger.warn(
+                           `Watch mode: package "${packageDir}" has remote location "${_package.location}" — falling back to copy. Source-edit live reload won't work for this package; clone the source locally and use a local-dir location to enable it.`,
+                        );
+                     }
+                     // Copy the specific directory. Clear any stale mount target
+                     // first (e.g. a symlink left by a prior --watch-env run), or
+                     // fs.cp would throw ERR_FS_CP_DIR_TO_NON_DIR onto it.
+                     await clearMountTarget(absolutePackagePath);
+                     await fs.promises.mkdir(absolutePackagePath, {
+                        recursive: true,
+                     });
+                     await fs.promises.cp(sourcePath, absolutePackagePath, {
+                        recursive: true,
+                     });
+                     logger.info(
+                        `Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`,
+                     );
+                  }
                } else {
                   // If source doesn't exist, copy the entire download as the package
+                  await clearMountTarget(absolutePackagePath);
                   await fs.promises.mkdir(absolutePackagePath, {
                      recursive: true,
                   });

package/src/service/filter.spec.ts

@@ -423,7 +423,7 @@ describe("service/filter", () => {
          const query = "run: orders -> summary";
          const clause = "`status` = 'active'";
          expect(injectFilterRefinement(query, clause)).toBe(
-            "run: orders -> summary + {where: `status` = 'active'}",
+            "run: orders -> summary\n+ {where: `status` = 'active'}",
          );
       });
 
@@ -432,7 +432,7 @@ describe("service/filter", () => {
             "run: orders -> { group_by: status; aggregate: order_count }";
          const clause = "`region` = 'US'";
          expect(injectFilterRefinement(query, clause)).toBe(
-            "run: orders -> { group_by: status; aggregate: order_count } + {where: `region` = 'US'}",
+            "run: orders -> { group_by: status; aggregate: order_count }\n+ {where: `region` = 'US'}",
          );
       });
 
@@ -440,8 +440,19 @@ describe("service/filter", () => {
          const query = "run: orders -> summary   \n  ";
          const clause = "`status` = 'active'";
          expect(injectFilterRefinement(query, clause)).toBe(
-            "run: orders -> summary + {where: `status` = 'active'}",
+            "run: orders -> summary\n+ {where: `status` = 'active'}",
          );
       });
+
+      it("places refinement on its own line so a trailing comment cannot swallow it", () => {
+         const clause = "`org_id` = 'acme'";
+         // A trailing line comment must not extend over the injected filter.
+         for (const comment of ["//", "-- sneaky"]) {
+            const query = `run: orders -> { group_by: status } ${comment}`;
+            const refined = injectFilterRefinement(query, clause);
+            const lastLine = refined.split("\n").pop() ?? "";
+            expect(lastLine).toBe(`+ {where: ${clause}}`);
+         }
+      });
    });
 });

package/src/service/filter.ts

@@ -272,6 +272,10 @@ export function buildFilterClause(
  * Append a filter refinement to a Malloy query string.
  * Uses Malloy's `+ {where: ...}` refinement syntax.
  *
+ * The refinement is placed on its own line so that a trailing line comment
+ * (`//` or `--`) on the caller's query cannot extend over it and neutralize
+ * the filter.
+ *
  * If `filterClause` is empty, returns the original query unchanged.
  */
 export function injectFilterRefinement(
@@ -281,7 +285,7 @@ export function injectFilterRefinement(
    if (!filterClause) {
       return query;
    }
-   return `${query.trimEnd()} + {where: ${filterClause}}`;
+   return `${query.trimEnd()}\n+ {where: ${filterClause}}`;
 }
 
 // ---------------------------------------------------------------------------