@malloy-publisher/server 0.0.228 → 0.0.230

package/src/service/extension_fetch_policy.spec.ts

@@ -0,0 +1,256 @@
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import path from "path";
+import sinon from "sinon";
+import { getExtensionFetchPolicy } from "../config";
+import {
+   applyExtensionSessionSettings,
+   buildEnvironmentMalloyConfig,
+} from "./connection";
+
+describe("getExtensionFetchPolicy", () => {
+   const original = process.env.EXTENSION_FETCH_POLICY;
+   afterEach(() => {
+      if (original === undefined) delete process.env.EXTENSION_FETCH_POLICY;
+      else process.env.EXTENSION_FETCH_POLICY = original;
+   });
+
+   it("defaults to on-demand when unset or empty", () => {
+      delete process.env.EXTENSION_FETCH_POLICY;
+      expect(getExtensionFetchPolicy()).toBe("on-demand");
+      process.env.EXTENSION_FETCH_POLICY = "   ";
+      expect(getExtensionFetchPolicy()).toBe("on-demand");
+   });
+
+   it("accepts the two policies case-insensitively", () => {
+      process.env.EXTENSION_FETCH_POLICY = "local-only";
+      expect(getExtensionFetchPolicy()).toBe("local-only");
+      process.env.EXTENSION_FETCH_POLICY = "On-Demand";
+      expect(getExtensionFetchPolicy()).toBe("on-demand");
+   });
+
+   it("throws loudly on an unrecognised value", () => {
+      process.env.EXTENSION_FETCH_POLICY = "offline";
+      expect(() => getExtensionFetchPolicy()).toThrow(
+         /Invalid value for EXTENSION_FETCH_POLICY/,
+      );
+   });
+});
+
+describe("EXTENSION_FETCH_POLICY=local-only extension loading", () => {
+   const envPath = path.join(process.cwd(), "test-extension-policy");
+   const original = process.env.EXTENSION_FETCH_POLICY;
+
+   beforeEach(async () => {
+      await fs.mkdir(envPath, { recursive: true });
+   });
+   afterEach(async () => {
+      sinon.restore();
+      if (original === undefined) delete process.env.EXTENSION_FETCH_POLICY;
+      else process.env.EXTENSION_FETCH_POLICY = original;
+      await fs.rm(envPath, { recursive: true, force: true }).catch(() => {});
+   });
+
+   it("never runs INSTALL and raises a loud, actionable error when a required extension is missing", async () => {
+      process.env.EXTENSION_FETCH_POLICY = "local-only";
+
+      // Session PRAGMAs and metadata probes succeed; the first extension LOAD
+      // fails, simulating an extension that was never baked into the image.
+      const runSQL = sinon
+         .stub(DuckDBConnection.prototype, "runSQL")
+         .resolves({ rows: [] } as never);
+      runSQL
+         .withArgs(sinon.match(/^LOAD ducklake/))
+         .rejects(new Error('Extension "ducklake" not found'));
+
+      const config = buildEnvironmentMalloyConfig(
+         [
+            {
+               name: "ducklake_localonly",
+               type: "ducklake",
+               ducklakeConnection: {
+                  catalog: {
+                     postgresConnection: {
+                        host: "192.0.2.1",
+                        port: 5432,
+                        userName: "nobody",
+                        password: "nobody",
+                        databaseName: "catalog",
+                     },
+                  },
+                  storage: {
+                     bucketUrl: "s3://test-bucket",
+                     s3Connection: {
+                        accessKeyId: "test",
+                        secretAccessKey: "test",
+                     },
+                  },
+               },
+            },
+         ] as never,
+         envPath,
+      );
+
+      await expect(
+         config.malloyConfig.connections.lookupConnection("ducklake_localonly"),
+      ).rejects.toThrow(/local-only.*ducklake|ducklake.*local-only/s);
+
+      // local-only must never issue an INSTALL — not even for a missing extension.
+      const ranInstall = runSQL
+         .getCalls()
+         .some((c) => /^\s*INSTALL\s/i.test(String(c.args[0])));
+      expect(ranInstall).toBe(false);
+
+      // And it did pin implicit auto-install off on the session (item 3).
+      const setAutoinstallOff = runSQL
+         .getCalls()
+         .some((c) =>
+            /autoinstall_known_extensions\s*=\s*false/i.test(String(c.args[0])),
+         );
+      expect(setAutoinstallOff).toBe(true);
+
+      await config.releaseConnections().catch(() => {});
+   });
+});
+
+describe("applyExtensionSessionSettings", () => {
+   const original = process.env.EXTENSION_FETCH_POLICY;
+   afterEach(() => {
+      sinon.restore();
+      if (original === undefined) delete process.env.EXTENSION_FETCH_POLICY;
+      else process.env.EXTENSION_FETCH_POLICY = original;
+   });
+
+   // Minimal stand-in — the helper only ever calls runSQL.
+   const mockConn = (runSQL: sinon.SinonStub) =>
+      ({ runSQL }) as unknown as DuckDBConnection;
+
+   it("pins autoinstall off (autoload on) under local-only", async () => {
+      process.env.EXTENSION_FETCH_POLICY = "local-only";
+      const runSQL = sinon.stub().resolves({ rows: [] });
+      await applyExtensionSessionSettings(mockConn(runSQL));
+      const sql = runSQL.getCalls().map((c) => String(c.args[0]));
+      expect(
+         sql.some((s) => /autoinstall_known_extensions\s*=\s*false/i.test(s)),
+      ).toBe(true);
+      expect(
+         sql.some((s) => /autoload_known_extensions\s*=\s*true/i.test(s)),
+      ).toBe(true);
+   });
+
+   it("is a no-op for a generic session under on-demand", async () => {
+      delete process.env.EXTENSION_FETCH_POLICY;
+      const runSQL = sinon.stub().resolves({ rows: [] });
+      await applyExtensionSessionSettings(mockConn(runSQL));
+      expect(runSQL.callCount).toBe(0);
+   });
+
+   it("pins a tier session even under on-demand (alwaysDisableAutoinstall)", async () => {
+      delete process.env.EXTENSION_FETCH_POLICY;
+      const runSQL = sinon.stub().resolves({ rows: [] });
+      await applyExtensionSessionSettings(mockConn(runSQL), {
+         alwaysDisableAutoinstall: true,
+      });
+      expect(
+         runSQL
+            .getCalls()
+            .some((c) =>
+               /autoinstall_known_extensions\s*=\s*false/i.test(
+                  String(c.args[0]),
+               ),
+            ),
+      ).toBe(true);
+   });
+
+   it("pins each connection only once (dedup)", async () => {
+      process.env.EXTENSION_FETCH_POLICY = "local-only";
+      const runSQL = sinon.stub().resolves({ rows: [] });
+      const conn = mockConn(runSQL);
+      await applyExtensionSessionSettings(conn);
+      await applyExtensionSessionSettings(conn);
+      expect(runSQL.callCount).toBe(2); // two SETs from the first call only
+   });
+
+   it("fails closed under local-only when the pin cannot be applied", async () => {
+      process.env.EXTENSION_FETCH_POLICY = "local-only";
+      const runSQL = sinon.stub().rejects(new Error("pragma unsupported"));
+      await expect(
+         applyExtensionSessionSettings(mockConn(runSQL)),
+      ).rejects.toThrow(/EXTENSION_FETCH_POLICY=local-only/);
+   });
+
+   it("does NOT fail a tier attach under on-demand when the pin fails", async () => {
+      delete process.env.EXTENSION_FETCH_POLICY;
+      const runSQL = sinon.stub().rejects(new Error("pragma unsupported"));
+      // Resolves (warn-and-continue), does not throw.
+      await applyExtensionSessionSettings(mockConn(runSQL), {
+         alwaysDisableAutoinstall: true,
+      });
+      expect(runSQL.callCount).toBeGreaterThan(0);
+   });
+});
+
+describe("EXTENSION_FETCH_POLICY=on-demand (default) — no behaviour change", () => {
+   const envPath = path.join(process.cwd(), "test-extension-policy-ondemand");
+   const original = process.env.EXTENSION_FETCH_POLICY;
+
+   beforeEach(async () => {
+      await fs.mkdir(envPath, { recursive: true });
+   });
+   afterEach(async () => {
+      sinon.restore();
+      if (original === undefined) delete process.env.EXTENSION_FETCH_POLICY;
+      else process.env.EXTENSION_FETCH_POLICY = original;
+      await fs.rm(envPath, { recursive: true, force: true }).catch(() => {});
+   });
+
+   it("leaves autoinstall at DuckDB's default on a generic attach session", async () => {
+      // Default policy: an ordinary DuckDB connection with an attachment must
+      // behave exactly as before — Publisher installs the attachment's extension
+      // explicitly, but it must NOT disable DuckDB's implicit auto-install (that
+      // would regress a user whose query references an un-provisioned extension).
+      delete process.env.EXTENSION_FETCH_POLICY;
+
+      const runSQL = sinon
+         .stub(DuckDBConnection.prototype, "runSQL")
+         .resolves({ rows: [] } as never);
+
+      const config = buildEnvironmentMalloyConfig(
+         [
+            {
+               name: "gen_duck",
+               type: "duckdb",
+               duckdbConnection: {
+                  attachedDatabases: [
+                     {
+                        name: "pg_attached",
+                        type: "postgres",
+                        postgresConnection: {
+                           connectionString: "postgresql://localhost/test",
+                        },
+                     },
+                  ],
+               },
+            },
+         ] as never,
+         envPath,
+      );
+
+      await config.malloyConfig.connections.lookupConnection("gen_duck");
+
+      const issued = runSQL.getCalls().map((c) => String(c.args[0]));
+      // The attachment path ran (postgres explicitly installed)...
+      expect(issued.some((sql) => /^\s*INSTALL\s+postgres/i.test(sql))).toBe(
+         true,
+      );
+      // ...but implicit auto-install was left untouched — no behaviour change.
+      expect(
+         issued.some((sql) =>
+            /autoinstall_known_extensions\s*=\s*false/i.test(sql),
+         ),
+      ).toBe(false);
+
+      await config.releaseConnections().catch(() => {});
+   });
+});

package/src/service/manifest_loader.spec.ts

@@ -42,7 +42,9 @@ describe("fetchManifestEntries", () => {
 
       expect(entries).toEqual({
          b1: { tableName: "schema.orders_mz" },
-         b2: { tableName: "schema.daily_mz" },
+         // connectionName is carried through so the bind step can quote the
+         // table path for that connection's dialect (Package.quoteBoundTableNames).
+         b2: { tableName: "schema.daily_mz", connectionName: "bq" },
       });
    });
 

package/src/service/manifest_loader.ts

@@ -58,11 +58,14 @@ async function readManifestBytes(uri: string): Promise<string> {
 
 /**
  * Fetch and parse the control-plane-computed build manifest at `uri`, returning
- * the full wire binding map (`sourceEntityId -> { tableName, dataAsOf,
- * freshnessWindowSeconds, freshnessFallback }`). The wire manifest keys physical
- * tables under `physicalTableName`; the Malloy runtime consumes `tableName`, so
- * we translate that field here but otherwise carry the control-plane freshness
- * fields verbatim so the serve path can gate `age vs window` per query. Entries
+ * the full wire binding map (`sourceEntityId -> { tableName, connectionName,
+ * dataAsOf, freshnessWindowSeconds, freshnessFallback }`). The wire manifest
+ * keys physical tables under `physicalTableName`; the Malloy runtime consumes
+ * `tableName`, so we translate that field here but otherwise carry the
+ * control-plane fields verbatim — the freshness fields so the serve path can
+ * gate `age vs window` per query, and `connectionName` so the bind step can
+ * quote the path for that connection's dialect (see
+ * Package.quoteBoundTableNames). Entries
  * missing a physical table are skipped. This is a pure fetch + parse: it does
  * **not** filter on freshness (that happens per query on the serve path). Throws
  * if the URI can't be read or parsed.
@@ -95,6 +98,7 @@ export async function fetchManifestEntries(
       }
       entries[sourceEntityId] = {
          tableName: physicalTableName,
+         connectionName: entry.connectionName,
          dataAsOf: entry.dataAsOf,
          freshnessWindowSeconds: entry.freshnessWindowSeconds,
          freshnessFallback: entry.freshnessFallback,

package/src/service/materialization_scheduler.spec.ts

@@ -0,0 +1,289 @@
+import { describe, expect, it } from "bun:test";
+import { MaterializationConflictError } from "../errors";
+import type { CronEvaluator } from "./cron_evaluator";
+import type { EnvironmentStore } from "./environment_store";
+import type { MaterializationService } from "./materialization_service";
+import { MaterializationScheduler } from "./materialization_scheduler";
+
+// ── Fakes ────────────────────────────────────────────────────────────────
+
+interface FakePkgOpts {
+   schedule?: string | null;
+   manifestLocation?: string | null;
+   policyWarnings?: string[];
+}
+
+function fakePackage(name: string, opts: FakePkgOpts = {}) {
+   const {
+      schedule = null,
+      manifestLocation = null,
+      policyWarnings = [],
+   } = opts;
+   return {
+      getPackageName: () => name,
+      getPackageMetadata: () => ({
+         materialization: schedule ? { schedule } : null,
+         manifestLocation,
+      }),
+      persistencePolicyWarnings: () => policyWarnings,
+   };
+}
+
+function fakeEnv(name: string, pkgs: ReturnType<typeof fakePackage>[]) {
+   return {
+      getEnvironmentName: () => name,
+      getLoadedPackages: () => pkgs,
+   };
+}
+
+function fakeStore(envs: ReturnType<typeof fakeEnv>[]): EnvironmentStore {
+   return {
+      getLoadedEnvironments: () => envs,
+   } as unknown as EnvironmentStore;
+}
+
+interface FireCall {
+   env: string;
+   pkg: string;
+   opts: { forceRefresh?: boolean; trigger?: string };
+}
+
+function fakeService(
+   opts: {
+      throwConflict?: boolean;
+      throwError?: boolean;
+      // Newest recorded SCHEDULER fire, used by the restart-recovery anchor.
+      lastFireAt?: Date | null;
+   } = {},
+) {
+   const calls: FireCall[] = [];
+   const service = {
+      calls,
+      createMaterialization: (
+         env: string,
+         pkg: string,
+         o: { forceRefresh?: boolean; trigger?: string },
+      ) => {
+         calls.push({ env, pkg, opts: o });
+         if (opts.throwConflict) {
+            return Promise.reject(new MaterializationConflictError("busy"));
+         }
+         if (opts.throwError) {
+            return Promise.reject(new Error("boom"));
+         }
+         return Promise.resolve({});
+      },
+      getLatestScheduledFireAt: () => Promise.resolve(opts.lastFireAt ?? null),
+   };
+   return service as unknown as MaterializationService & { calls: FireCall[] };
+}
+
+// A deterministic cron: nextAfter is always `from + 60s`; isValid configurable.
+function fakeCron(
+   isValid: (expr: string) => boolean = () => true,
+): CronEvaluator {
+   return {
+      isValid,
+      nextAfter: (_expr: string, from: Date) =>
+         new Date(from.getTime() + 60_000),
+   } as unknown as CronEvaluator;
+}
+
+const CONFIG = { tickIntervalMs: 60_000, maxFiresPerTick: 10 };
+
+function makeScheduler(
+   pkgs: ReturnType<typeof fakePackage>[],
+   service: ReturnType<typeof fakeService>,
+   cron: CronEvaluator = fakeCron(),
+   config = CONFIG,
+) {
+   const store = fakeStore([fakeEnv("env1", pkgs)]);
+   return new MaterializationScheduler(store, service, config, cron);
+}
+
+// ── Tests ────────────────────────────────────────────────────────────────
+
+describe("MaterializationScheduler", () => {
+   const t0 = 1_000_000_000_000; // fixed epoch ms
+   const dueLater = t0 + 60_001; // just past the armed nextFire (t0 + 60s)
+
+   it("does not fire on the arming tick (nextFire is strictly future)", async () => {
+      const service = fakeService();
+      const sched = makeScheduler(
+         [fakePackage("p", { schedule: "* * * * *" })],
+         service,
+      );
+      await sched.tick(t0);
+      expect(service.calls.length).toBe(0);
+   });
+
+   it("recovers a missed occurrence on first arm, firing once then jumping forward", async () => {
+      // A prior SCHEDULER fire well in the past: anchoring nextFire to
+      // nextAfter(lastFire) lands before t0, so the occurrence missed during
+      // "downtime" is due on the very first tick (a restart recovering).
+      const service = fakeService({ lastFireAt: new Date(t0 - 10 * 60_000) });
+      const sched = makeScheduler(
+         [fakePackage("p", { schedule: "* * * * *" })],
+         service,
+      );
+      await sched.tick(t0);
+      expect(service.calls.length).toBe(1); // one catch-up fire
+      // Jumped forward to nextAfter(now); not due again on the next tick.
+      await sched.tick(t0 + 1);
+      expect(service.calls.length).toBe(1);
+   });
+
+   it("does not catch up a never-fired schedule on first arm", async () => {
+      // No prior SCHEDULER fire on record -> anchor is now -> strictly future,
+      // so a freshly-scheduled (never-fired) package does not fire on arm.
+      const service = fakeService({ lastFireAt: null });
+      const sched = makeScheduler(
+         [fakePackage("p", { schedule: "* * * * *" })],
+         service,
+      );
+      await sched.tick(t0);
+      expect(service.calls.length).toBe(0);
+   });
+
+   it("fires a due, valid, standalone package as SCHEDULER + forceRefresh", async () => {
+      const service = fakeService();
+      const sched = makeScheduler(
+         [fakePackage("p", { schedule: "* * * * *" })],
+         service,
+      );
+      await sched.tick(t0); // arm
+      await sched.tick(dueLater); // due -> fire
+      expect(service.calls).toEqual([
+         {
+            env: "env1",
+            pkg: "p",
+            opts: { forceRefresh: true, trigger: "SCHEDULER" },
+         },
+      ]);
+   });
+
+   it("skips a package with no schedule", async () => {
+      const service = fakeService();
+      const sched = makeScheduler([fakePackage("p", {})], service);
+      await sched.tick(t0);
+      await sched.tick(dueLater);
+      expect(service.calls.length).toBe(0);
+   });
+
+   it("skips an orchestrated package (manifestLocation set) — Guard 2", async () => {
+      const service = fakeService();
+      const sched = makeScheduler(
+         [
+            fakePackage("p", {
+               schedule: "* * * * *",
+               manifestLocation: "gs://cp/manifest.json",
+            }),
+         ],
+         service,
+      );
+      await sched.tick(t0);
+      await sched.tick(dueLater);
+      expect(service.calls.length).toBe(0);
+   });
+
+   it("skips a package whose persistence policy is invalid", async () => {
+      const service = fakeService();
+      const sched = makeScheduler(
+         [
+            fakePackage("p", {
+               schedule: "* * * * *",
+               policyWarnings: ["schedule requires scope: version"],
+            }),
+         ],
+         service,
+      );
+      await sched.tick(t0);
+      await sched.tick(dueLater);
+      expect(service.calls.length).toBe(0);
+   });
+
+   it("skips an invalid cron", async () => {
+      const service = fakeService();
+      const sched = makeScheduler(
+         [fakePackage("p", { schedule: "not a cron" })],
+         service,
+         fakeCron(() => false),
+      );
+      await sched.tick(t0);
+      await sched.tick(dueLater);
+      expect(service.calls.length).toBe(0);
+   });
+
+   it("coalesces when a materialization is already active (no throw)", async () => {
+      const service = fakeService({ throwConflict: true });
+      const sched = makeScheduler(
+         [fakePackage("p", { schedule: "* * * * *" })],
+         service,
+      );
+      await sched.tick(t0);
+      await sched.tick(dueLater); // due -> fire attempt -> conflict, swallowed
+      expect(service.calls.length).toBe(1); // attempted once
+      // Advanced past this occurrence: not due again until the next window.
+      await sched.tick(dueLater + 1);
+      expect(service.calls.length).toBe(1);
+   });
+
+   it("isolates an unexpected fire error (sweep keeps going)", async () => {
+      const service = fakeService({ throwError: true });
+      const sched = makeScheduler(
+         [fakePackage("p", { schedule: "* * * * *" })],
+         service,
+      );
+      await sched.tick(t0);
+      await expect(sched.tick(dueLater)).resolves.toBeUndefined();
+      expect(service.calls.length).toBe(1);
+   });
+
+   it("caps fires per tick (stampede guard)", async () => {
+      const service = fakeService();
+      const pkgs = [
+         fakePackage("a", { schedule: "* * * * *" }),
+         fakePackage("b", { schedule: "* * * * *" }),
+         fakePackage("c", { schedule: "* * * * *" }),
+      ];
+      const sched = makeScheduler(pkgs, service, fakeCron(), {
+         tickIntervalMs: 60_000,
+         maxFiresPerTick: 2,
+      });
+      await sched.tick(t0); // arm all
+      await sched.tick(dueLater); // all due, but cap = 2
+      expect(service.calls.length).toBe(2);
+      // The capped one is still due and fires on the next tick.
+      await sched.tick(dueLater + 1);
+      expect(service.calls.length).toBe(3);
+   });
+
+   it("prunes arming state when a package unloads, then re-anchors on reload without a stale fire", async () => {
+      // A never-fired schedule (fakeService default lastFireAt = null), so a
+      // fresh arm anchors from `now` and does not catch up.
+      const service = fakeService();
+      const p = fakePackage("p", { schedule: "* * * * *" });
+      // makeScheduler closes over this array via getLoadedPackages, so mutating
+      // it simulates the package leaving and re-entering the loaded set.
+      const loaded = [p];
+      const sched = makeScheduler(loaded, service);
+
+      await sched.tick(t0); // arm: nextFire = t0 + 60s
+
+      // Unload: the package leaves the loaded set, so the sweep prunes its state
+      // (tick's not-seen cleanup) instead of leaving a stale nextFire behind.
+      loaded.length = 0;
+      await sched.tick(dueLater); // past the old nextFire, but the package is gone
+      expect(service.calls.length).toBe(0);
+
+      // Reload: with the state pruned, arm is fresh — anchored from `now`, so the
+      // pre-unload nextFire can't resurface as a spurious catch-up.
+      loaded.push(p);
+      await sched.tick(dueLater + 1); // now < fresh nextFire -> no fire
+      expect(service.calls.length).toBe(0);
+
+      // It is genuinely re-armed (not inert): the next occurrence still fires.
+      await sched.tick(dueLater + 1 + 60_001);
+      expect(service.calls.length).toBe(1);
+   });
+});