@malloy-publisher/server 0.0.229 → 0.0.230

package/src/service/extension_fetch_policy.spec.ts

@@ -0,0 +1,256 @@
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import path from "path";
+import sinon from "sinon";
+import { getExtensionFetchPolicy } from "../config";
+import {
+   applyExtensionSessionSettings,
+   buildEnvironmentMalloyConfig,
+} from "./connection";
+
+describe("getExtensionFetchPolicy", () => {
+   const original = process.env.EXTENSION_FETCH_POLICY;
+   afterEach(() => {
+      if (original === undefined) delete process.env.EXTENSION_FETCH_POLICY;
+      else process.env.EXTENSION_FETCH_POLICY = original;
+   });
+
+   it("defaults to on-demand when unset or empty", () => {
+      delete process.env.EXTENSION_FETCH_POLICY;
+      expect(getExtensionFetchPolicy()).toBe("on-demand");
+      process.env.EXTENSION_FETCH_POLICY = "   ";
+      expect(getExtensionFetchPolicy()).toBe("on-demand");
+   });
+
+   it("accepts the two policies case-insensitively", () => {
+      process.env.EXTENSION_FETCH_POLICY = "local-only";
+      expect(getExtensionFetchPolicy()).toBe("local-only");
+      process.env.EXTENSION_FETCH_POLICY = "On-Demand";
+      expect(getExtensionFetchPolicy()).toBe("on-demand");
+   });
+
+   it("throws loudly on an unrecognised value", () => {
+      process.env.EXTENSION_FETCH_POLICY = "offline";
+      expect(() => getExtensionFetchPolicy()).toThrow(
+         /Invalid value for EXTENSION_FETCH_POLICY/,
+      );
+   });
+});
+
+describe("EXTENSION_FETCH_POLICY=local-only extension loading", () => {
+   const envPath = path.join(process.cwd(), "test-extension-policy");
+   const original = process.env.EXTENSION_FETCH_POLICY;
+
+   beforeEach(async () => {
+      await fs.mkdir(envPath, { recursive: true });
+   });
+   afterEach(async () => {
+      sinon.restore();
+      if (original === undefined) delete process.env.EXTENSION_FETCH_POLICY;
+      else process.env.EXTENSION_FETCH_POLICY = original;
+      await fs.rm(envPath, { recursive: true, force: true }).catch(() => {});
+   });
+
+   it("never runs INSTALL and raises a loud, actionable error when a required extension is missing", async () => {
+      process.env.EXTENSION_FETCH_POLICY = "local-only";
+
+      // Session PRAGMAs and metadata probes succeed; the first extension LOAD
+      // fails, simulating an extension that was never baked into the image.
+      const runSQL = sinon
+         .stub(DuckDBConnection.prototype, "runSQL")
+         .resolves({ rows: [] } as never);
+      runSQL
+         .withArgs(sinon.match(/^LOAD ducklake/))
+         .rejects(new Error('Extension "ducklake" not found'));
+
+      const config = buildEnvironmentMalloyConfig(
+         [
+            {
+               name: "ducklake_localonly",
+               type: "ducklake",
+               ducklakeConnection: {
+                  catalog: {
+                     postgresConnection: {
+                        host: "192.0.2.1",
+                        port: 5432,
+                        userName: "nobody",
+                        password: "nobody",
+                        databaseName: "catalog",
+                     },
+                  },
+                  storage: {
+                     bucketUrl: "s3://test-bucket",
+                     s3Connection: {
+                        accessKeyId: "test",
+                        secretAccessKey: "test",
+                     },
+                  },
+               },
+            },
+         ] as never,
+         envPath,
+      );
+
+      await expect(
+         config.malloyConfig.connections.lookupConnection("ducklake_localonly"),
+      ).rejects.toThrow(/local-only.*ducklake|ducklake.*local-only/s);
+
+      // local-only must never issue an INSTALL — not even for a missing extension.
+      const ranInstall = runSQL
+         .getCalls()
+         .some((c) => /^\s*INSTALL\s/i.test(String(c.args[0])));
+      expect(ranInstall).toBe(false);
+
+      // And it did pin implicit auto-install off on the session (item 3).
+      const setAutoinstallOff = runSQL
+         .getCalls()
+         .some((c) =>
+            /autoinstall_known_extensions\s*=\s*false/i.test(String(c.args[0])),
+         );
+      expect(setAutoinstallOff).toBe(true);
+
+      await config.releaseConnections().catch(() => {});
+   });
+});
+
+describe("applyExtensionSessionSettings", () => {
+   const original = process.env.EXTENSION_FETCH_POLICY;
+   afterEach(() => {
+      sinon.restore();
+      if (original === undefined) delete process.env.EXTENSION_FETCH_POLICY;
+      else process.env.EXTENSION_FETCH_POLICY = original;
+   });
+
+   // Minimal stand-in — the helper only ever calls runSQL.
+   const mockConn = (runSQL: sinon.SinonStub) =>
+      ({ runSQL }) as unknown as DuckDBConnection;
+
+   it("pins autoinstall off (autoload on) under local-only", async () => {
+      process.env.EXTENSION_FETCH_POLICY = "local-only";
+      const runSQL = sinon.stub().resolves({ rows: [] });
+      await applyExtensionSessionSettings(mockConn(runSQL));
+      const sql = runSQL.getCalls().map((c) => String(c.args[0]));
+      expect(
+         sql.some((s) => /autoinstall_known_extensions\s*=\s*false/i.test(s)),
+      ).toBe(true);
+      expect(
+         sql.some((s) => /autoload_known_extensions\s*=\s*true/i.test(s)),
+      ).toBe(true);
+   });
+
+   it("is a no-op for a generic session under on-demand", async () => {
+      delete process.env.EXTENSION_FETCH_POLICY;
+      const runSQL = sinon.stub().resolves({ rows: [] });
+      await applyExtensionSessionSettings(mockConn(runSQL));
+      expect(runSQL.callCount).toBe(0);
+   });
+
+   it("pins a tier session even under on-demand (alwaysDisableAutoinstall)", async () => {
+      delete process.env.EXTENSION_FETCH_POLICY;
+      const runSQL = sinon.stub().resolves({ rows: [] });
+      await applyExtensionSessionSettings(mockConn(runSQL), {
+         alwaysDisableAutoinstall: true,
+      });
+      expect(
+         runSQL
+            .getCalls()
+            .some((c) =>
+               /autoinstall_known_extensions\s*=\s*false/i.test(
+                  String(c.args[0]),
+               ),
+            ),
+      ).toBe(true);
+   });
+
+   it("pins each connection only once (dedup)", async () => {
+      process.env.EXTENSION_FETCH_POLICY = "local-only";
+      const runSQL = sinon.stub().resolves({ rows: [] });
+      const conn = mockConn(runSQL);
+      await applyExtensionSessionSettings(conn);
+      await applyExtensionSessionSettings(conn);
+      expect(runSQL.callCount).toBe(2); // two SETs from the first call only
+   });
+
+   it("fails closed under local-only when the pin cannot be applied", async () => {
+      process.env.EXTENSION_FETCH_POLICY = "local-only";
+      const runSQL = sinon.stub().rejects(new Error("pragma unsupported"));
+      await expect(
+         applyExtensionSessionSettings(mockConn(runSQL)),
+      ).rejects.toThrow(/EXTENSION_FETCH_POLICY=local-only/);
+   });
+
+   it("does NOT fail a tier attach under on-demand when the pin fails", async () => {
+      delete process.env.EXTENSION_FETCH_POLICY;
+      const runSQL = sinon.stub().rejects(new Error("pragma unsupported"));
+      // Resolves (warn-and-continue), does not throw.
+      await applyExtensionSessionSettings(mockConn(runSQL), {
+         alwaysDisableAutoinstall: true,
+      });
+      expect(runSQL.callCount).toBeGreaterThan(0);
+   });
+});
+
+describe("EXTENSION_FETCH_POLICY=on-demand (default) — no behaviour change", () => {
+   const envPath = path.join(process.cwd(), "test-extension-policy-ondemand");
+   const original = process.env.EXTENSION_FETCH_POLICY;
+
+   beforeEach(async () => {
+      await fs.mkdir(envPath, { recursive: true });
+   });
+   afterEach(async () => {
+      sinon.restore();
+      if (original === undefined) delete process.env.EXTENSION_FETCH_POLICY;
+      else process.env.EXTENSION_FETCH_POLICY = original;
+      await fs.rm(envPath, { recursive: true, force: true }).catch(() => {});
+   });
+
+   it("leaves autoinstall at DuckDB's default on a generic attach session", async () => {
+      // Default policy: an ordinary DuckDB connection with an attachment must
+      // behave exactly as before — Publisher installs the attachment's extension
+      // explicitly, but it must NOT disable DuckDB's implicit auto-install (that
+      // would regress a user whose query references an un-provisioned extension).
+      delete process.env.EXTENSION_FETCH_POLICY;
+
+      const runSQL = sinon
+         .stub(DuckDBConnection.prototype, "runSQL")
+         .resolves({ rows: [] } as never);
+
+      const config = buildEnvironmentMalloyConfig(
+         [
+            {
+               name: "gen_duck",
+               type: "duckdb",
+               duckdbConnection: {
+                  attachedDatabases: [
+                     {
+                        name: "pg_attached",
+                        type: "postgres",
+                        postgresConnection: {
+                           connectionString: "postgresql://localhost/test",
+                        },
+                     },
+                  ],
+               },
+            },
+         ] as never,
+         envPath,
+      );
+
+      await config.malloyConfig.connections.lookupConnection("gen_duck");
+
+      const issued = runSQL.getCalls().map((c) => String(c.args[0]));
+      // The attachment path ran (postgres explicitly installed)...
+      expect(issued.some((sql) => /^\s*INSTALL\s+postgres/i.test(sql))).toBe(
+         true,
+      );
+      // ...but implicit auto-install was left untouched — no behaviour change.
+      expect(
+         issued.some((sql) =>
+            /autoinstall_known_extensions\s*=\s*false/i.test(sql),
+         ),
+      ).toBe(false);
+
+      await config.releaseConnections().catch(() => {});
+   });
+});

package/src/service/materialization_scheduler.spec.ts

@@ -257,4 +257,33 @@ describe("MaterializationScheduler", () => {
       await sched.tick(dueLater + 1);
       expect(service.calls.length).toBe(3);
    });
+
+   it("prunes arming state when a package unloads, then re-anchors on reload without a stale fire", async () => {
+      // A never-fired schedule (fakeService default lastFireAt = null), so a
+      // fresh arm anchors from `now` and does not catch up.
+      const service = fakeService();
+      const p = fakePackage("p", { schedule: "* * * * *" });
+      // makeScheduler closes over this array via getLoadedPackages, so mutating
+      // it simulates the package leaving and re-entering the loaded set.
+      const loaded = [p];
+      const sched = makeScheduler(loaded, service);
+
+      await sched.tick(t0); // arm: nextFire = t0 + 60s
+
+      // Unload: the package leaves the loaded set, so the sweep prunes its state
+      // (tick's not-seen cleanup) instead of leaving a stale nextFire behind.
+      loaded.length = 0;
+      await sched.tick(dueLater); // past the old nextFire, but the package is gone
+      expect(service.calls.length).toBe(0);
+
+      // Reload: with the state pruned, arm is fresh — anchored from `now`, so the
+      // pre-unload nextFire can't resurface as a spurious catch-up.
+      loaded.push(p);
+      await sched.tick(dueLater + 1); // now < fresh nextFire -> no fire
+      expect(service.calls.length).toBe(0);
+
+      // It is genuinely re-armed (not inert): the next occurrence still fires.
+      await sched.tick(dueLater + 1 + 60_001);
+      expect(service.calls.length).toBe(1);
+   });
 });

package/src/service/materialization_service.spec.ts

@@ -1,6 +1,6 @@
 import type { Connection as MalloyConnection } from "@malloydata/malloy";
 import { Manifest } from "@malloydata/malloy";
-import { beforeEach, describe, expect, it } from "bun:test";
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
 import * as sinon from "sinon";
 import {
    BadRequestError,
@@ -27,6 +27,11 @@ import {
    MaterializationService,
    stagingSuffix,
 } from "./materialization_service";
+import { resetMaterializationTelemetryForTesting } from "../materialization_metrics";
+import {
+   startMetricsHarness,
+   type MetricsHarness,
+} from "../test_helpers/metrics_harness";
 
 type MockRepo = sinon.SinonStubbedInstance<ResourceRepository>;
 
@@ -500,6 +505,86 @@ describe("MaterializationService", () => {
    });
 });
 
+describe("deleteMaterialization telemetry", () => {
+   let ctx: ReturnType<typeof createMocks>;
+   let harness: MetricsHarness;
+
+   beforeEach(async () => {
+      ctx = createMocks();
+      harness = await startMetricsHarness();
+      // Drop cached instruments so recordDropTables re-binds to this provider.
+      resetMaterializationTelemetryForTesting();
+   });
+
+   afterEach(async () => {
+      resetMaterializationTelemetryForTesting();
+      await harness.shutdown();
+   });
+
+   const DROP_COUNTER = "publisher_materialization_drop_tables_total";
+
+   // A terminal materialization whose manifest names one physical table, so
+   // dropMaterializedTables issues exactly one DROP — the emission under test.
+   function dropTablesFixture(runSQL: sinon.SinonStub) {
+      const getMalloyConnection = sinon
+         .stub()
+         .resolves({ runSQL, dialectName: "duckdb" });
+      (ctx.environmentStore.getEnvironment as sinon.SinonStub).resolves({
+         getPackage: sinon.stub().resolves({ getMalloyConnection }),
+         withPackageLock: async (_n: string, fn: () => Promise<unknown>) =>
+            fn(),
+      });
+      ctx.repository.getMaterializationById.resolves(
+         makeMaterialization({
+            status: "MANIFEST_FILE_READY",
+            manifest: {
+               builtAt: new Date().toISOString(),
+               strict: false,
+               entries: {
+                  b1: {
+                     sourceEntityId: "b1",
+                     physicalTableName: "orders_mz",
+                     connectionName: "duckdb",
+                  },
+               },
+            },
+         }),
+      );
+   }
+
+   it("meters a successful physical-table drop as outcome=success", async () => {
+      dropTablesFixture(sinon.stub().resolves());
+
+      await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
+         dropTables: true,
+      });
+
+      expect(
+         await harness.collectCounter(DROP_COUNTER, { outcome: "success" }),
+      ).toBe(1);
+      expect(
+         await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
+      ).toBe(0);
+   });
+
+   it("meters a failed drop as outcome=failure and still deletes the record", async () => {
+      dropTablesFixture(sinon.stub().rejects(new Error("drop boom")));
+
+      // The drop is best-effort: a failure is metered but never surfaces to the
+      // caller, and the record is still removed.
+      await ctx.service.deleteMaterialization("my-env", "pkg", "mat-1", {
+         dropTables: true,
+      });
+
+      expect(
+         await harness.collectCounter(DROP_COUNTER, { outcome: "failure" }),
+      ).toBe(1);
+      expect(ctx.repository.deleteMaterialization.calledOnceWith("mat-1")).toBe(
+         true,
+      );
+   });
+});
+
 describe("stagingSuffix", () => {
    it("derives a short, stable suffix from the sourceEntityId", () => {
       expect(stagingSuffix("abcdef1234567890")).toBe("_abcdef123456");
@@ -1002,6 +1087,39 @@ describe("buildOneSource", () => {
       );
    });
 
+   it("rethrows the original build error when staging cleanup also fails", async () => {
+      const runSQL = sinon.stub();
+      runSQL.onCall(0).resolves(); // initial staging drop
+      runSQL.onCall(1).rejects(new Error("create boom")); // CREATE TABLE AS
+      runSQL.onCall(2).rejects(new Error("cleanup boom")); // cleanup drop fails
+      // The cleanup failure is swallowed (logged as a physical leak); the
+      // original build error is what propagates, so the run is classified by its
+      // real cause rather than the cleanup noise.
+      await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
+         "create boom",
+      );
+      expect(runSQL.callCount).toBe(3);
+      expect(runSQL.lastCall.args[0]).toBe(
+         'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
+      );
+   });
+
+   it("drops staging when the build is cancelled mid-run (no _staging orphan)", async () => {
+      // buildOneSource is cancellation-agnostic: a cancel surfaces as a thrown
+      // build SQL and takes the same staging-cleanup path as a failure, so a
+      // cancelled build leaves no orphaned _staging table behind.
+      const runSQL = sinon.stub();
+      runSQL.onCall(0).resolves(); // initial staging drop
+      runSQL.onCall(1).rejects(new Error("Build cancelled")); // aborted mid-CREATE
+      runSQL.onCall(2).resolves(); // cleanup staging drop
+      await expect(callBuildOneSource({ runSQL }, "orders_v1")).rejects.toThrow(
+         "Build cancelled",
+      );
+      expect(runSQL.lastCall.args[0]).toBe(
+         'DROP TABLE IF EXISTS "orders_v1_abcdef123456"',
+      );
+   });
+
    it("quotes each segment of a container path for a backtick dialect", async () => {
       const runSQL = sinon.stub().resolves();
       // Container path (dataset.table) on a backtick dialect (BigQuery): each

package/src/service/model.spec.ts

@@ -4,6 +4,7 @@ import fs from "fs/promises";
 import sinon from "sinon";
 
 import {
+   AccessDeniedError,
    BadRequestError,
    ModelNotFoundError,
    PayloadTooLargeError,
@@ -420,6 +421,11 @@ describe("service/model", () => {
                undefined,
                undefined,
                undefined,
+               undefined,
+               // Model surfaces `region` so filterGivensToModelSurface (see
+               // model.ts) forwards it rather than dropping it as unknown.
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               [{ name: "region", type: "string" }] as any,
             );
 
             await expect(
@@ -716,6 +722,11 @@ describe("service/model", () => {
                // eslint-disable-next-line @typescript-eslint/no-explicit-any
                runnableCells as any,
                undefined,
+               undefined,
+               // Model surfaces `target_code` so filterGivensToModelSurface
+               // (see model.ts) forwards it rather than dropping it as unknown.
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               [{ name: "target_code", type: "string" }] as any,
             );
 
             await expect(
@@ -819,6 +830,62 @@ describe("service/model", () => {
       });
    });
 
+   describe("authorize struct-resolution drift invariant", () => {
+      // collectAllReachableGates walks struct.fields using Malloy's own
+      // isJoined ('join' in sd) / isSourceDef (struct.type is a known source
+      // kind) guards. If a future Malloy version adds a new source `type`
+      // that isSourceDef doesn't yet recognize, a genuinely-joined field
+      // would fail isSourceDef and used to be silently `continue`d past —
+      // failing OPEN (the joined source's gate is never found, so it's never
+      // evaluated). This confirms the walk instead denies loudly when it
+      // finds a field that IS a join but can't be resolved to a walkable
+      // SourceDef, rather than silently treating it as "nothing to gate".
+      it("denies rather than silently skipping when a joined field doesn't resolve to a walkable SourceDef", async () => {
+         const driftedJoinField = {
+            name: "mystery_join",
+            join: "one", // isJoined: true ('join' in sd)
+            // A source `type` isSourceDef doesn't (yet) recognize -- stands
+            // in for a future Malloy source kind the duck-typed guard hasn't
+            // been updated to cover.
+            type: "future_source_kind",
+            fields: [],
+         };
+         const runStruct = {
+            type: "table",
+            name: "root",
+            fields: [driftedJoinField],
+         };
+         const modelDef = { contents: {}, exports: [], queryList: [] };
+         const runnable = {
+            getPreparedQuery: sinon.stub().resolves({
+               _query: { structRef: runStruct },
+               _modelDef: modelDef,
+            }),
+         };
+
+         const model = new Model(
+            packageName,
+            mockModelPath,
+            {},
+            "model",
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            { loadQuery: sinon.stub() } as any, // modelMaterializer
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            modelDef as any,
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+         );
+
+         await expect(
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            model.assertAuthorizedForAllSources(runnable as any, {}),
+         ).rejects.toThrow(AccessDeniedError);
+      });
+   });
+
    describe("static methods", () => {
       describe("getModelRuntime", () => {
          it("should throw ModelNotFoundError for invalid modelPath", async () => {