@malloy-publisher/server 0.0.204 → 0.0.205

package/src/service/compile_authorize.spec.ts

@@ -0,0 +1,85 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import os from "os";
+import path from "path";
+import { AccessDeniedError } from "../errors";
+import { Environment } from "./environment";
+
+// End-to-end gate on the /compile path. Exercises environment.compileSource
+// through a real installed package, not just the Model primitives — pins that
+// the early gate AND the compiled-source backstop fire, the latter REGARDLESS of
+// includeSql (a compile-time schema oracle is closed even with no SQL extraction).
+
+const PUBLISHER_JSON = JSON.stringify({
+   name: "pkg",
+   description: "compile-gate",
+});
+
+// `gated` is locked to $ROLE='analyst'; `open_src` is unrestricted.
+const MODEL = `##! experimental.givens
+
+given:
+  ROLE :: string
+
+#(authorize) "$ROLE = 'analyst'"
+source: gated is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
+
+source: open_src is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
+`;
+
+describe("compile-path authorize gate (compileSource)", () => {
+   let rootDir: string;
+   let env: Environment;
+
+   beforeEach(async () => {
+      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-compile-"));
+      const envPath = path.join(rootDir, "env");
+      await fs.mkdir(envPath, { recursive: true });
+      env = await Environment.create("testEnv", envPath, []);
+      await env.installPackage("pkg", async (stagingPath) => {
+         await fs.mkdir(stagingPath, { recursive: true });
+         await fs.writeFile(
+            path.join(stagingPath, "publisher.json"),
+            PUBLISHER_JSON,
+         );
+         await fs.writeFile(path.join(stagingPath, "model.malloy"), MODEL);
+      });
+   });
+
+   afterEach(async () => {
+      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
+   });
+
+   const compile = (source: string, givens?: Record<string, string>) =>
+      env.compileSource("pkg", "model.malloy", source, false, givens);
+
+   it("denies a direct gated source without the satisfying given (early gate)", async () => {
+      await expect(
+         compile("run: gated -> { aggregate: c }"),
+      ).rejects.toBeInstanceOf(AccessDeniedError);
+   });
+
+   it("denies a gated source reached via the LAST run: statement (backstop, includeSql=false)", async () => {
+      // Regression guard: the early gate only matches the first `run:` (ungated
+      // open_src here), so the gated source in the executed final statement is
+      // caught only by the compiled-source backstop — which must run even when
+      // no SQL is requested.
+      await expect(
+         compile(
+            "run: open_src -> { aggregate: c }\nrun: gated -> { aggregate: c }",
+         ),
+      ).rejects.toBeInstanceOf(AccessDeniedError);
+   });
+
+   it("allows the gated source when the given satisfies the gate", async () => {
+      const { problems } = await compile("run: gated -> { aggregate: c }", {
+         ROLE: "analyst",
+      });
+      expect(problems).toBeDefined();
+   });
+
+   it("leaves an ungated source compilable without any given", async () => {
+      const { problems } = await compile("run: open_src -> { aggregate: c }");
+      expect(problems).toBeDefined();
+   });
+});

package/src/service/environment.ts

@@ -5,6 +5,7 @@ import { Mutex } from "async-mutex";
 import crypto from "crypto";
 import * as fs from "fs";
 import * as path from "path";
+import { pathToFileURL } from "url";
 import { components } from "../api";
 import { API_PREFIX, README_NAME } from "../constants";
 import {
@@ -137,6 +138,11 @@ export class Environment {
    // governor as the single owner of the back-pressure boolean.
    private memoryGovernor: PackageMemoryGovernor | null = null;
 
+   /** Absolute path on disk where this environment's package files live. */
+   public getEnvironmentPath(): string {
+      return this.environmentPath;
+   }
+
    constructor(
       environmentName: string,
       environmentPath: string,
@@ -307,10 +313,18 @@ export class Environment {
             packageName,
             modelName,
          );
-         // Place the virtual file in the model's directory so relative imports resolve correctly.
+         // Place the virtual file in the model's directory so relative imports
+         // resolve correctly. Use `pathToFileURL` rather than hand-prefixing
+         // `file://`: on Windows the latter produces a malformed URL
+         // (`file://D:\Temp\…`) that round-trips differently than the URL the
+         // Malloy runtime synthesizes from the same path, breaking the
+         // intercepting reader's string comparison below and falling through
+         // to disk for a virtual file that doesn't exist.
          const modelDir = path.dirname(modelPath);
-         const virtualUri = `file://${path.join(modelDir, "__compile_check.malloy")}`;
-         const virtualUrl = new URL(virtualUri);
+         const virtualUrl = pathToFileURL(
+            path.join(modelDir, "__compile_check.malloy"),
+         );
+         const virtualUri = virtualUrl.toString();
 
          // Read the full model file so the submitted source inherits the model's
          // complete namespace — imports, source definitions, queries, etc.
@@ -339,6 +353,21 @@ export class Environment {
          // Use the locked variant — we already hold the per-package mutex.
          const pkg = await this._loadOrGetPackageLocked(packageName);
 
+         // Authorize gate: /compile is compile-only, but it can still act
+         // as a schema oracle (a denied caller learns a gated source's columns
+         // from compile errors) and, with includeSql, leak its SQL. Gate the
+         // named source the submitted text targets BEFORE compiling — mirrors
+         // the query path's early surface-syntax gate. Unnamed/inline source
+         // text resolves to undefined, so only the model-wide file-level gate
+         // applies. The gate runs against the package's cached Model (its
+         // `given:` block + authorize annotations), independent of the virtual
+         // compile below. If the model isn't loaded, there's nothing to enforce
+         // and compilation surfaces its own error.
+         const gateModel = pkg.getModel(modelName);
+         if (gateModel) {
+            await gateModel.assertAuthorizedForText(source, givens ?? {});
+         }
+
          // Initialize Runtime with the package's active MalloyConfig so compile
          // checks see the same package-scoped duckdb as execution. This runtime
          // borrows the package config; the package/environment lifecycle owns release.
@@ -352,11 +381,40 @@ export class Environment {
             const modelMaterializer = runtime.loadModel(virtualUrl);
             const model = await modelMaterializer.getModel();
 
+            // Resolve the final query's materializer once (if there is one).
+            let queryMaterializer: ReturnType<
+               typeof modelMaterializer.loadFinalQuery
+            > | null = null;
+            try {
+               queryMaterializer = modelMaterializer.loadFinalQuery();
+            } catch {
+               // No runnable query (e.g. only source definitions) — nothing to
+               // gate or extract beyond the early text gate already applied.
+            }
+
+            // Compiled-source backstop — runs REGARDLESS of includeSql. It gates
+            // the source the COMPILED final query actually reads, closing
+            // named-query / multi-statement indirection the early surface-syntax
+            // gate misses (e.g. `run: ungated\nrun: gated` — the early gate only
+            // matches the FIRST `run:`, but the LAST statement is what executes).
+            // Compiling a gated source even without SQL is a schema oracle
+            // (field-not-found errors leak its columns), so this must not be
+            // conditional on SQL extraction. (A `source: x is gated` alias makes
+            // a new ungated source — that's the documented "extend doesn't
+            // inherit authorize" footgun, the same as the query path.) Only run
+            // when the model declares gates so ungated compiles don't pay for
+            // the extra final-query compile.
+            if (queryMaterializer && gateModel?.hasAuthorize()) {
+               await gateModel.assertAuthorizedForRunnable(
+                  queryMaterializer,
+                  givens ?? {},
+               );
+            }
+
             // If includeSql is requested and compilation succeeded, attempt to extract SQL
             let sql: string | undefined;
-            if (includeSql) {
+            if (includeSql && queryMaterializer) {
                try {
-                  const queryMaterializer = modelMaterializer.loadFinalQuery();
                   sql = await queryMaterializer.getSQL({ givens });
                } catch {
                   // Source may not contain a runnable query (e.g. only source definitions),

package/src/service/environment_store.ts

@@ -95,6 +95,26 @@ function validateEnvironmentAzureUrls(environment: ApiEnvironment): void {
    }
 }
 
+// Safely clear a prior mount target before (re)mounting at `targetPath`.
+// Distinguishes symlinks from real dirs: fs.rm's recursive mode could otherwise
+// traverse into a symlinked source and damage it, so unlink symlinks and rm real
+// dirs. Both the in-place (symlink) and copy mount paths call this, so toggling
+// watch mode across runs against the same publisher_data can't leave a stale
+// entry that breaks the next mount (mkdir is a no-op on a symlink and fs.cp then
+// throws ERR_FS_CP_DIR_TO_NON_DIR).
+async function clearMountTarget(targetPath: string): Promise<void> {
+   try {
+      const stats = await fs.promises.lstat(targetPath);
+      if (stats.isDirectory() && !stats.isSymbolicLink()) {
+         await fs.promises.rm(targetPath, { recursive: true, force: true });
+      } else {
+         await fs.promises.unlink(targetPath);
+      }
+   } catch {
+      // Nothing there, fine.
+   }
+}
+
 export class EnvironmentStore {
    public serverRootPath: string;
    private environments: Map<string, Environment> = new Map();
@@ -112,10 +132,36 @@ export class EnvironmentStore {
    // new Environments pick it up at construction.
    private memoryGovernor: PackageMemoryGovernor | null = null;
 
+   /**
+    * Set of environment names that should be loaded "in place" — i.e. the
+    * package directories under `publisher_data/<envName>/` are symlinks to
+    * the original local source directories instead of recursive copies.
+    *
+    * In-place mode powers the `npm run dev`-style live-reload story: edits
+    * to your source repo are visible to Publisher and trigger watch events
+    * that fan out via SSE to embedded HTML pages. Production mode (the
+    * default) keeps the safe copy semantics so the source repo and the
+    * served package are decoupled.
+    *
+    * Populated from the `PUBLISHER_WATCH` env var (comma-separated env
+    * names) at construction; can also be augmented via `markInPlace()` if
+    * a future feature wants to flip an environment to dev mode at runtime.
+    *
+    * Only LOCAL-DIR package locations are eligible for symlinking; remote
+    * sources (GitHub, GCS, S3) always copy regardless.
+    */
+   private inPlaceEnvs = new Set<string>();
+
    constructor(serverRootPath: string) {
       this.serverRootPath = serverRootPath;
       this.gcsClient = new Storage();
 
+      const watchEnvList = (process.env.PUBLISHER_WATCH || "")
+         .split(",")
+         .map((s) => s.trim())
+         .filter((s) => s.length > 0);
+      for (const envName of watchEnvList) this.inPlaceEnvs.add(envName);
+
       const storageConfig: StorageConfig = {
          type: "duckdb",
          duckdb: {
@@ -127,6 +173,17 @@ export class EnvironmentStore {
       this.finishedInitialization = this.initialize();
    }
 
+   /** True if this env should mount package dirs in place (symlinks). */
+   public isInPlace(environmentName: string): boolean {
+      return this.inPlaceEnvs.has(environmentName);
+   }
+
+   /** Add an env to in-place mode at runtime. Caller is responsible for
+    *  triggering a re-mount if the env was already loaded with copies. */
+   public markInPlace(environmentName: string): void {
+      this.inPlaceEnvs.add(environmentName);
+   }
+
    /**
     * Attach (or detach with `null`) the shared {@link PackageMemoryGovernor}.
     * Propagated to every Environment so the back-pressure decision is
@@ -1202,7 +1259,15 @@ export class EnvironmentStore {
                } else {
                   // For non-GitHub locations, use package name
                   if (this.isLocalPath(_package.location)) {
-                     sourcePath = _package.location;
+                     // Match the resolution rule used by
+                     // `downloadOrMountLocation` (line ~1352): relative
+                     // paths are anchored at `serverRootPath`. Without this
+                     // step the existing-source check below falls through
+                     // for any relative location, and the in-place mount
+                     // branch is unreachable.
+                     sourcePath = path.isAbsolute(_package.location)
+                        ? _package.location
+                        : path.join(this.serverRootPath, _package.location);
                   } else {
                      sourcePath = safeJoinUnderRoot(
                         tempDownloadPath,
@@ -1217,18 +1282,84 @@ export class EnvironmentStore {
                   .catch(() => false);
 
                if (sourceExists) {
-                  // Copy the specific directory
-                  await fs.promises.mkdir(absolutePackagePath, {
-                     recursive: true,
-                  });
-                  await fs.promises.cp(sourcePath, absolutePackagePath, {
-                     recursive: true,
-                  });
-                  logger.info(
-                     `Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`,
-                  );
+                  // In-place mount path (watch-mode-only, local-dir packages
+                  // only). Replace the recursive copy with a symlink so that
+                  // edits to the source repo are visible to Publisher and
+                  // can trigger live-reload events.
+                  //
+                  // Carefully: we MUST distinguish symlinks from real dirs
+                  // when removing the prior content, because `fs.rm`'s
+                  // recursive mode could in principle traverse into a
+                  // symlinked source dir and damage it. We lstat first and
+                  // call `unlink` for symlinks, `rm` for real directories.
+                  const isInPlace =
+                     this.inPlaceEnvs.has(environmentName) &&
+                     this.isLocalPath(_package.location);
+                  if (isInPlace) {
+                     await clearMountTarget(absolutePackagePath);
+                     const absoluteSourcePath = path.resolve(sourcePath);
+                     // On Windows a "dir" symlink needs elevation
+                     // (SeCreateSymbolicLinkPrivilege — admin or Developer
+                     // Mode); a junction does not and works for absolute
+                     // directory targets, so prefer it there. chokidar watches
+                     // through both.
+                     const linkType =
+                        process.platform === "win32" ? "junction" : "dir";
+                     try {
+                        await fs.promises.symlink(
+                           absoluteSourcePath,
+                           absolutePackagePath,
+                           linkType,
+                        );
+                        logger.info(
+                           `In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`,
+                        );
+                     } catch (linkError) {
+                        // Degrade gracefully instead of failing the env load:
+                        // copy the package so it still serves. Live reload
+                        // won't work for it (source edits aren't seen), but the
+                        // environment comes up. Hit e.g. on locked-down Windows
+                        // where even junction creation is denied.
+                        const code =
+                           (linkError as NodeJS.ErrnoException)?.code ??
+                           String(linkError);
+                        logger.warn(
+                           `In-place mount failed for package "${packageDir}" (${code}); falling back to a copy. Source-edit live reload is disabled for this package.`,
+                        );
+                        await clearMountTarget(absolutePackagePath);
+                        await fs.promises.mkdir(absolutePackagePath, {
+                           recursive: true,
+                        });
+                        await fs.promises.cp(sourcePath, absolutePackagePath, {
+                           recursive: true,
+                        });
+                     }
+                  } else {
+                     if (
+                        this.inPlaceEnvs.has(environmentName) &&
+                        !this.isLocalPath(_package.location)
+                     ) {
+                        logger.warn(
+                           `Watch mode: package "${packageDir}" has remote location "${_package.location}" — falling back to copy. Source-edit live reload won't work for this package; clone the source locally and use a local-dir location to enable it.`,
+                        );
+                     }
+                     // Copy the specific directory. Clear any stale mount target
+                     // first (e.g. a symlink left by a prior --watch-env run), or
+                     // fs.cp would throw ERR_FS_CP_DIR_TO_NON_DIR onto it.
+                     await clearMountTarget(absolutePackagePath);
+                     await fs.promises.mkdir(absolutePackagePath, {
+                        recursive: true,
+                     });
+                     await fs.promises.cp(sourcePath, absolutePackagePath, {
+                        recursive: true,
+                     });
+                     logger.info(
+                        `Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`,
+                     );
+                  }
                } else {
                   // If source doesn't exist, copy the entire download as the package
+                  await clearMountTarget(absolutePackagePath);
                   await fs.promises.mkdir(absolutePackagePath, {
                      recursive: true,
                   });

package/src/service/model.ts

@@ -232,6 +232,18 @@ export class Model {
       );
    }
 
+   /**
+    * Whether the model declares any `#(authorize)` / `##(authorize)` gate at all
+    * (file-level or on any source). Lets callers cheaply skip authorize work for
+    * ungated models without compiling a probe.
+    */
+   public hasAuthorize(): boolean {
+      return (
+         this.fileLevelAuthorize.length > 0 ||
+         (this.sources?.some((s) => (s.authorize?.length ?? 0) > 0) ?? false)
+      );
+   }
+
    /**
     * Effective authorize expressions for whatever a query runs against:
     *  - a declared model source → its own list (file-level ++ source-level);
@@ -288,6 +300,38 @@ export class Model {
       if (!passed) deny();
    }
 
+   /**
+    * Gate ad-hoc compile/query text by the named source it targets. Resolves the
+    * source from surface syntax (`extractSourceName`) and applies the gate. An
+    * unnamed/inline source resolves to `undefined`, so only the model-wide
+    * file-level gate applies — the same top-level-only boundary as the query
+    * path's early gate. Used by the `/compile` path, which has no runnable to
+    * resolve before it decides whether to compile at all.
+    */
+   public async assertAuthorizedForText(
+      text: string,
+      givens: Record<string, GivenValue>,
+   ): Promise<void> {
+      await this.assertAuthorized(this.extractSourceName(text), givens);
+   }
+
+   /**
+    * Gate a compiled query by the source it actually reads, resolved from the
+    * prepared query's `structRef` (authoritative — survives named-query and
+    * multi-statement indirection that surface syntax misses, e.g. the executed
+    * `run:` statement isn't the first one). Used as the `/compile` backstop once
+    * a runnable exists.
+    */
+   public async assertAuthorizedForRunnable(
+      runnable: { getPreparedQuery(): Promise<unknown> },
+      givens: Record<string, GivenValue>,
+   ): Promise<void> {
+      await this.assertAuthorized(
+         await this.resolveAuthorizeSourceFromRunnable(runnable),
+         givens,
+      );
+   }
+
    /**
     * Resolve the source a compiled query reads, from its prepared query's
     * `structRef`. This is authoritative — it survives named-query indirection

package/src/service/package.ts

@@ -138,13 +138,24 @@ export class Package {
             malloy_package_name: packageName,
             status,
          });
-         // Clean up package directory on failure
+         // Clean up the package directory on failure, but NOT when packagePath
+         // is an in-place mount symlink (watch mode). Removing it would unmount
+         // the package, so a transient compile error from a half-typed model
+         // saved mid-edit would brick the package until a restart. The symlink
+         // points at the user's live source, which is left untouched; the next
+         // save recompiles against it.
          try {
-            await fs.rm(packagePath, {
-               recursive: true,
-               force: true,
-            });
-            logger.info(`Cleaned up failed package directory: ${packagePath}`);
+            const stat = await fs.lstat(packagePath).catch(() => null);
+            if (stat?.isSymbolicLink()) {
+               logger.info(
+                  `Skipping cleanup of symlinked package path on failure: ${packagePath}`,
+               );
+            } else {
+               await fs.rm(packagePath, { recursive: true, force: true });
+               logger.info(
+                  `Cleaned up failed package directory: ${packagePath}`,
+               );
+            }
          } catch (cleanupError) {
             logger.warn(`Failed to clean up package directory ${packagePath}`, {
                error: cleanupError,