@malloy-publisher/server 0.0.227 → 0.0.229

package/src/service/package_quote_bound_tables.spec.ts

@@ -0,0 +1,171 @@
+import { afterEach, describe, expect, it } from "bun:test";
+import sinon from "sinon";
+import type { Connection } from "@malloydata/malloy";
+import { logger } from "../logger";
+import type { FreshnessManifest } from "../storage/DatabaseInterface";
+import { Model } from "./model";
+import { Package } from "./package";
+
+// Unit coverage for Package.quoteBoundTableNames — the bind-time step that
+// mirrors the CREATE side's quoting onto manifest table paths (PR #904). The
+// integration test proves quote chars appear in real DuckDB SQL, but DuckDB is
+// not case-folding and never exhibited the bug; these tests pin the actual
+// quoted contract per dialect and exercise every branch (dialect selection,
+// no-connection passthrough, already-quoted passthrough, and the
+// unresolvable-connection fallback) without needing a live warehouse. The
+// quoting itself runs through the real `quoteTablePath`, so this is not a
+// re-implementation of the oracle — it drives the shipping code.
+describe("service/package quoteBoundTableNames", () => {
+   afterEach(() => sinon.restore());
+
+   function makePackage(): Package {
+      return new Package(
+         "testEnv",
+         "testPackage",
+         "/tmp/does-not-matter",
+         { name: "testPackage", description: "Test package" },
+         [],
+         new Map<string, Model>(),
+      );
+   }
+
+   // Reach the private method the same way package.spec.ts reaches private
+   // statics — this is the smallest seam that drives the real branch logic.
+   function quote(
+      pkg: Package,
+      entries: FreshnessManifest,
+   ): Promise<FreshnessManifest> {
+      // @ts-expect-error Accessing private method for testing
+      return pkg.quoteBoundTableNames(entries);
+   }
+
+   function stubDialect(pkg: Package, dialectName: string): void {
+      sinon
+         .stub(pkg, "getMalloyConnection")
+         .resolves({ dialectName } as Connection);
+   }
+
+   it("double-quotes each path segment for a case-folding dialect (Snowflake)", async () => {
+      const pkg = makePackage();
+      stubDialect(pkg, "snowflake");
+
+      const out = await quote(pkg, {
+         s1: { tableName: "schema.order_summary__g0", connectionName: "sf" },
+      });
+
+      expect(out.s1.tableName).toBe('"schema"."order_summary__g0"');
+      // connectionName and any other fields survive the rewrite.
+      expect(out.s1.connectionName).toBe("sf");
+   });
+
+   it("double-quotes for other double-quote dialects (Postgres)", async () => {
+      const pkg = makePackage();
+      stubDialect(pkg, "postgres");
+
+      const out = await quote(pkg, {
+         s1: { tableName: "public.daily", connectionName: "pg" },
+      });
+
+      expect(out.s1.tableName).toBe('"public"."daily"');
+   });
+
+   it("backticks each segment for a backtick dialect (BigQuery)", async () => {
+      const pkg = makePackage();
+      stubDialect(pkg, "standardsql");
+
+      const out = await quote(pkg, {
+         s1: { tableName: "my-proj.ds.events", connectionName: "bq" },
+      });
+
+      expect(out.s1.tableName).toBe("`my-proj`.`ds`.`events`");
+   });
+
+   it("binds verbatim when the producer recorded no connectionName", async () => {
+      const pkg = makePackage();
+      const spy = sinon.stub(pkg, "getMalloyConnection");
+
+      const out = await quote(pkg, {
+         s1: { tableName: "schema.bare_builder_table" },
+      });
+
+      expect(out.s1.tableName).toBe("schema.bare_builder_table");
+      // No connection lookup attempted — this is the benign default path.
+      expect(spy.called).toBe(false);
+   });
+
+   it("passes an already-quoted name through without double-quoting", async () => {
+      const pkg = makePackage();
+      const spy = sinon.stub(pkg, "getMalloyConnection");
+
+      const out = await quote(pkg, {
+         s1: {
+            tableName: '"schema"."already_canonical"',
+            connectionName: "sf",
+         },
+      });
+
+      expect(out.s1.tableName).toBe('"schema"."already_canonical"');
+      expect(spy.called).toBe(false);
+   });
+
+   it("passes an already-backticked name through without re-quoting", async () => {
+      const pkg = makePackage();
+      const spy = sinon.stub(pkg, "getMalloyConnection");
+
+      const out = await quote(pkg, {
+         s1: { tableName: "`ds`.`events`", connectionName: "bq" },
+      });
+
+      expect(out.s1.tableName).toBe("`ds`.`events`");
+      expect(spy.called).toBe(false);
+   });
+
+   it("degrades one entry (unquoted) and logs an actionable error when its connection is unresolvable", async () => {
+      const pkg = makePackage();
+      sinon
+         .stub(pkg, "getMalloyConnection")
+         .rejects(new Error("no connection named 'gone'"));
+      const errSpy = sinon.stub(logger, "error");
+
+      const out = await quote(pkg, {
+         s1: { tableName: "schema.orphaned", connectionName: "gone" },
+      });
+
+      // The one entry degrades to the unquoted name rather than throwing out
+      // of the loop and failing the whole package bind.
+      expect(out.s1.tableName).toBe("schema.orphaned");
+      // ...but the misconfiguration is a distinct, loud, actionable signal —
+      // not folded silently into the benign no-connection default.
+      expect(errSpy.calledOnce).toBe(true);
+      // winston's error() is overloaded, so widen the recorded args to assert
+      // the (message, context) contract this branch emits.
+      const [message, ctx] = errSpy.firstCall.args as unknown as [
+         string,
+         Record<string, unknown>,
+      ];
+      expect(message).toContain("gone");
+      expect(message).toContain("Fix:");
+      expect(ctx).toMatchObject({
+         sourceEntityId: "s1",
+         connectionName: "gone",
+      });
+   });
+
+   it("quotes resolvable entries and degrades an unresolvable one in the same bind", async () => {
+      const pkg = makePackage();
+      const lookup = sinon.stub(pkg, "getMalloyConnection");
+      lookup
+         .withArgs("sf")
+         .resolves({ dialectName: "snowflake" } as Connection);
+      lookup.withArgs("gone").rejects(new Error("missing"));
+      sinon.stub(logger, "error");
+
+      const out = await quote(pkg, {
+         good: { tableName: "schema.kept", connectionName: "sf" },
+         bad: { tableName: "schema.orphaned", connectionName: "gone" },
+      });
+
+      expect(out.good.tableName).toBe('"schema"."kept"');
+      expect(out.bad.tableName).toBe("schema.orphaned");
+   });
+});

package/src/service/package_reload_safety.spec.ts

@@ -0,0 +1,193 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import * as fs from "fs/promises";
+import * as os from "os";
+import * as path from "path";
+
+import { Environment, PackageStatus } from "./environment";
+
+/**
+ * Regression tests for RELOAD data loss.
+ *
+ * `Package.create`'s failure cleanup was written for a fresh install, where the
+ * half-built tree is Publisher's to remove. A reload runs the same code against
+ * a directory that already exists and is already serving, so a model that
+ * failed to compile deleted the user's package directory and evicted the
+ * last-good compiled model, taking the package offline until it was
+ * re-provisioned. That is reachable from the reload endpoint and the
+ * malloy_reloadPackage MCP tool by any caller that saves a broken model and
+ * reloads, which is the ordinary authoring mistake.
+ *
+ * Directory cleanup is now opt-in and only `installPackage` asks for it, so
+ * these two guard the reload path. Run against a real `Environment` and a real
+ * `Package.create` over temp dirs, the same way package_rollback.spec.ts does.
+ */
+describe("failed reload does not destroy a serving package", () => {
+   let rootDir: string;
+   let envPath: string;
+
+   const GOOD_MODEL = `source: ones is duckdb.sql("SELECT 1 as x")\n`;
+   const BROKEN_MODEL = `source: broken is @@@ not valid malloy !!!\n`;
+
+   async function writePackageDir(dir: string, model: string): Promise<void> {
+      await fs.mkdir(dir, { recursive: true });
+      await fs.writeFile(
+         path.join(dir, "publisher.json"),
+         JSON.stringify({ name: "pkg", description: "reload fixture" }),
+      );
+      await fs.writeFile(path.join(dir, "model.malloy"), model);
+   }
+
+   async function copyDir(src: string, dst: string): Promise<void> {
+      await fs.mkdir(dst, { recursive: true });
+      await fs.cp(src, dst, { recursive: true });
+   }
+
+   beforeEach(async () => {
+      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-reload-"));
+      envPath = path.join(rootDir, "env");
+      await fs.mkdir(envPath, { recursive: true });
+   });
+
+   afterEach(async () => {
+      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
+   });
+
+   it("keeps the package directory on disk when the reloaded model does not compile", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+      const pkgDir = path.join(envPath, "pkg");
+      await writePackageDir(pkgDir, GOOD_MODEL);
+      await env.addPackage("pkg");
+
+      // The ordinary authoring mistake: save something that does not compile,
+      // then reload.
+      await fs.writeFile(path.join(pkgDir, "model.malloy"), BROKEN_MODEL);
+      await expect(env.getPackage("pkg", true)).rejects.toThrow();
+
+      // The directory and the file the user was editing both survive.
+      const modelText = await fs.readFile(
+         path.join(pkgDir, "model.malloy"),
+         "utf-8",
+      );
+      expect(modelText).toBe(BROKEN_MODEL);
+   });
+
+   it("keeps serving the last good model after a failed reload", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+      const pkgDir = path.join(envPath, "pkg");
+      await writePackageDir(pkgDir, GOOD_MODEL);
+      await env.addPackage("pkg");
+
+      const servingBefore = await env.getPackage("pkg", false);
+
+      await fs.writeFile(path.join(pkgDir, "model.malloy"), BROKEN_MODEL);
+      await expect(env.getPackage("pkg", true)).rejects.toThrow();
+
+      // The package is still loaded and answerable: a failed reload reports the
+      // compile error, it does not take the package down. Assert the exact
+      // state, not just that something is there: a status stranded at LOADING
+      // would satisfy toBeDefined() while listPackages skips it, so the package
+      // would answer getPackage and be invisible to listings and discovery.
+      const stillServing = await env.getPackage("pkg", false);
+      expect(stillServing).toBe(servingBefore);
+      expect(env.getPackageStatus("pkg")?.status).toBe(PackageStatus.SERVING);
+      expect((await env.listPackages()).map((p) => p.name)).toContain("pkg");
+   });
+
+   it("keeps a rolled-back reinstall listed and serving", async () => {
+      // The other reload path. A package with an install location reloads via
+      // installPackage, whose rollback restores the previous tree but used to
+      // drop the status while `packages` kept the old package: it answered
+      // getPackage but vanished from listPackages and discovery until a
+      // restart. Both maps must agree that it is still serving.
+      const env = await Environment.create("testEnv", envPath, []);
+      const goodFixture = path.join(rootDir, "good");
+      const brokenFixture = path.join(rootDir, "broken");
+      await writePackageDir(goodFixture, GOOD_MODEL);
+      await writePackageDir(brokenFixture, BROKEN_MODEL);
+
+      await env.installPackage("pkg", (stagingPath) =>
+         copyDir(goodFixture, stagingPath),
+      );
+      const servingBefore = await env.getPackage("pkg", false);
+
+      await expect(
+         env.installPackage("pkg", (stagingPath) =>
+            copyDir(brokenFixture, stagingPath),
+         ),
+      ).rejects.toThrow();
+
+      expect(env.getPackageStatus("pkg")?.status).toBe(PackageStatus.SERVING);
+      expect((await env.listPackages()).map((p) => p.name)).toContain("pkg");
+      expect(await env.getPackage("pkg", false)).toBe(servingBefore);
+      // The on-disk half of the same claim: the rejected tree is gone and the
+      // user's is back. Without this, nothing pins "your files are left alone".
+      expect(
+         await fs.readFile(path.join(envPath, "pkg", "model.malloy"), "utf-8"),
+      ).toBe(GOOD_MODEL);
+   });
+
+   it("does not advertise a package as serving when the rollback could not restore it", async () => {
+      // Keeping the last-good package served is only honest when the rollback
+      // actually put its tree back. If nothing was restored, the cached package
+      // no longer matches disk, and claiming SERVING would advertise a package
+      // over a canonical path that is missing or still holds rejected content.
+      const env = await Environment.create("testEnv", envPath, []);
+      const pkgDir = path.join(envPath, "pkg");
+      await writePackageDir(pkgDir, GOOD_MODEL);
+      await env.addPackage("pkg");
+      expect(env.getPackageStatus("pkg")?.status).toBe(PackageStatus.SERVING);
+
+      // Record retirements rather than scheduling the real drain, which is a
+      // 30s timer this test has no reason to wait on.
+      const retiredLabels: string[] = [];
+      (
+         env as unknown as {
+            retireConnectionGeneration: (label: string) => void;
+         }
+      ).retireConnectionGeneration = (label: string) => {
+         retiredLabels.push(label);
+      };
+
+      // The tree disappears from under Publisher, so the reinstall has nothing
+      // to retire and the rollback has nothing to restore.
+      await fs.rm(pkgDir, { recursive: true, force: true });
+
+      const brokenFixture = path.join(rootDir, "broken-nofallback");
+      await writePackageDir(brokenFixture, BROKEN_MODEL);
+      await expect(
+         env.installPackage("pkg", (stagingPath) =>
+            copyDir(brokenFixture, stagingPath),
+         ),
+      ).rejects.toThrow();
+
+      // Eviction is the honest outcome here, and both maps must agree on it.
+      expect(env.getPackageStatus("pkg")).toBeUndefined();
+      expect((await env.listPackages()).map((p) => p.name)).not.toContain(
+         "pkg",
+      );
+      // Assert the packages map too, not just the status: asserting only the
+      // status leaves the half that actually strands a package untested.
+      await expect(env.getPackage("pkg", false)).rejects.toThrow();
+
+      // Evicting must retire the old package's connections. Once it leaves
+      // this.packages, closeAllConnections cannot reach its MalloyConfig, so
+      // skipping the retire orphans its native handles for the process's life.
+      // The drain is timer-based, so assert it was scheduled, not that it ran.
+      expect(retiredLabels).toContain("package pkg");
+   });
+
+   it("recovers on the next reload once the model compiles again", async () => {
+      const env = await Environment.create("testEnv", envPath, []);
+      const pkgDir = path.join(envPath, "pkg");
+      await writePackageDir(pkgDir, GOOD_MODEL);
+      await env.addPackage("pkg");
+
+      await fs.writeFile(path.join(pkgDir, "model.malloy"), BROKEN_MODEL);
+      await expect(env.getPackage("pkg", true)).rejects.toThrow();
+
+      // Because nothing was deleted, fixing the model and reloading is enough.
+      await fs.writeFile(path.join(pkgDir, "model.malloy"), GOOD_MODEL);
+      const reloaded = await env.getPackage("pkg", true);
+      expect(reloaded).toBeDefined();
+   });
+});

package/src/service/persistence_policy.spec.ts

@@ -209,6 +209,37 @@ source: f is duckdb.sql("SELECT 1 as x")
       { timeout: 30000 },
    );
 
+   // ── Rule 4: the cron must be a valid 5-field UNIX expression ──────────
+
+   it(
+      "rejects an unparseable schedule cron",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL, {
+            scope: "version",
+            materialization: { schedule: "not a cron" },
+         });
+         const joined = pkg.formatInvalidPersistencePolicy();
+         expect(joined).toContain("valid");
+         expect(joined).toContain("UNIX cron");
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "rejects a cron-parser extension the UNIX grammar lacks (L)",
+      async () => {
+         // Guards config-parity: cron-parser accepts `0 6 L * *` but the control
+         // plane's UNIX parser does not, so publish must reject it too rather
+         // than let it silently never arm in production.
+         const pkg = await loadPackage(PLAIN_MODEL, {
+            scope: "version",
+            materialization: { schedule: "0 6 L * *" },
+         });
+         expect(pkg.formatInvalidPersistencePolicy()).toContain("UNIX cron");
+      },
+      { timeout: 30000 },
+   );
+
    // ── inert when nothing is declared ────────────────────────────────────
 
    it(

package/src/service/quoting.spec.ts

@@ -1,5 +1,11 @@
 import { describe, expect, it } from "bun:test";
-import { bareTableName, quoteIdentifier, quoteTablePath } from "./quoting";
+import {
+   bareTableName,
+   isQuotedIdentifierPath,
+   quoteIdentifier,
+   quoteManifestTablePath,
+   quoteTablePath,
+} from "./quoting";
 
 describe("bareTableName", () => {
    it("returns the segment after the last dot for a qualified name", () => {
@@ -77,3 +83,53 @@ describe("quoteTablePath", () => {
       expect(quoteTablePath("t_ab12_v0", "postgres")).toBe('"t_ab12_v0"');
    });
 });
+
+describe("isQuotedIdentifierPath", () => {
+   it("is false for a logical, unquoted path (the control-plane form)", () => {
+      expect(isQuotedIdentifierPath("schema.order_summary__g0")).toBe(false);
+      expect(isQuotedIdentifierPath("order_summary")).toBe(false);
+   });
+
+   it("is true once any segment carries a dialect quote char", () => {
+      expect(isQuotedIdentifierPath('"schema"."order_summary"')).toBe(true);
+      expect(isQuotedIdentifierPath("`ds`.`events`")).toBe(true);
+   });
+});
+
+// quoteManifestTablePath is the single quoting authority both the serve-side
+// bind and the build-side manifest route through. It runs the real
+// quoteTablePath (not a re-implementation), so this pins the exact per-dialect
+// contract that keeps read (a Malloy FROM) byte-identical to write (the CREATE
+// DDL, which also quotes via quoteTablePath) on every dialect — case-folding or
+// not.
+describe("quoteManifestTablePath", () => {
+   it("double-quotes each segment on a case-folding double-quote dialect (Snowflake)", () => {
+      expect(
+         quoteManifestTablePath("schema.order_summary__g0", "snowflake"),
+      ).toBe('"schema"."order_summary__g0"');
+   });
+
+   it("double-quotes on Postgres and DuckDB (unchanged for lowercase names, correct for mixed case)", () => {
+      expect(quoteManifestTablePath("public.daily", "postgres")).toBe(
+         '"public"."daily"',
+      );
+      expect(quoteManifestTablePath("main.orders_mz", "duckdb")).toBe(
+         '"main"."orders_mz"',
+      );
+   });
+
+   it("backticks each segment on BigQuery (standardsql) — required for hyphenated ids", () => {
+      expect(quoteManifestTablePath("my-proj.ds.events", "standardsql")).toBe(
+         "`my-proj`.`ds`.`events`",
+      );
+   });
+
+   it("passes an already-quoted name through unchanged (any dialect)", () => {
+      expect(quoteManifestTablePath('"schema"."already"', "snowflake")).toBe(
+         '"schema"."already"',
+      );
+      expect(quoteManifestTablePath("`ds`.`events`", "standardsql")).toBe(
+         "`ds`.`events`",
+      );
+   });
+});

package/src/service/quoting.ts

@@ -43,3 +43,36 @@ export function quoteTablePath(tableName: string, dialect: string): string {
       .map((segment) => quoteIdentifier(segment, dialect))
       .join(".");
 }
+
+/**
+ * Whether a table path already carries a dialect quote character, in which case
+ * it is treated as canonical SQL that must not be re-quoted. The single
+ * definition of "already quoted" shared by every bind site.
+ */
+export function isQuotedIdentifierPath(tableName: string): boolean {
+   return tableName.includes('"') || tableName.includes("`");
+}
+
+/**
+ * Quote a physical table path for a manifest entry so a Malloy `FROM` resolves
+ * it correctly, unless it is already canonical SQL (passed through unchanged).
+ * This is the single quoting authority for the two places a physical name is
+ * bound into a `FROM`: the serve-side bind ({@link Package.quoteBoundTableNames})
+ * and the build-side manifest that stitches chained persist sources together
+ * (materialization_service). Both must mirror the CREATE side's
+ * {@link quoteTablePath} so a case-folding engine (Snowflake uppercases
+ * unquoted identifiers) can resolve the case-preserved table the builder wrote.
+ *
+ * Passing an already-quoted name through is safe for the names this handles:
+ * control-plane physical names are logical/unquoted (never contain a quote),
+ * and a self-assigned `#@ persist name=` value is the author's own canonical
+ * SQL (they own quoting it for the dialect).
+ */
+export function quoteManifestTablePath(
+   tableName: string,
+   dialect: string,
+): string {
+   return isQuotedIdentifierPath(tableName)
+      ? tableName
+      : quoteTablePath(tableName, dialect);
+}

package/src/storage/DatabaseInterface.ts

@@ -59,6 +59,14 @@ export interface ResourceRepository {
       packageName: string,
       options?: { limit?: number; offset?: number },
    ): Promise<Materialization[]>;
+   listMaterializationsByEnvironment(
+      environmentId: string,
+      options?: { limit?: number; offset?: number },
+   ): Promise<Materialization[]>;
+   getLatestScheduledFireAt(
+      environmentId: string,
+      packageName: string,
+   ): Promise<Date | null>;
    getMaterializationById(id: string): Promise<Materialization | null>;
    getActiveMaterialization(
       environmentId: string,
@@ -174,6 +182,13 @@ export interface BuildManifest {
  */
 export interface FreshnessAwareManifestEntry {
    tableName: string;
+   /**
+    * The connection the physical table lives on, carried from the wire
+    * manifest so the bind step can quote `tableName` for that connection's
+    * dialect (see Package.quoteBoundTableNames). Absent on entries whose
+    * producer didn't record it; those bind verbatim.
+    */
+   connectionName?: string;
    dataAsOf?: string;
    freshnessWindowSeconds?: number;
    freshnessFallback?: "live" | "stale_ok" | "fail";

package/src/storage/duckdb/DuckDBRepository.ts

@@ -153,6 +153,23 @@ export class DuckDBRepository implements ResourceRepository {
       return this.materializationRepo.list(environmentId, packageName, options);
    }
 
+   async listMaterializationsByEnvironment(
+      environmentId: string,
+      options?: { limit?: number; offset?: number },
+   ): Promise<Materialization[]> {
+      return this.materializationRepo.listByEnvironment(environmentId, options);
+   }
+
+   async getLatestScheduledFireAt(
+      environmentId: string,
+      packageName: string,
+   ): Promise<Date | null> {
+      return this.materializationRepo.getLatestScheduledFireAt(
+         environmentId,
+         packageName,
+      );
+   }
+
    async getMaterializationById(id: string): Promise<Materialization | null> {
       return this.materializationRepo.getById(id);
    }