@malloy-publisher/server 0.0.229 → 0.0.230

package/src/service/package.spec.ts

@@ -95,6 +95,44 @@ describe("service/package", () => {
       expect(pkg).toBeInstanceOf(Package);
    });
 
+   describe("per-package sandbox extension pinning", () => {
+      const originalPolicy = process.env.EXTENSION_FETCH_POLICY;
+      afterEach(() => {
+         if (originalPolicy === undefined) {
+            delete process.env.EXTENSION_FETCH_POLICY;
+         } else {
+            process.env.EXTENSION_FETCH_POLICY = originalPolicy;
+         }
+      });
+
+      it(
+         "pins autoinstall off on the :memory: sandbox under local-only",
+         async () => {
+            // Regression guard for the sandbox bypass: the per-package sandbox
+            // has no attached databases, so the environment attach paths never
+            // touch it. Under local-only its session must still be pinned
+            // against DuckDB's implicit auto-install, or a sandbox query
+            // referencing an unbaked extension could silently fetch from the
+            // CDN. Uses a spy (not a stub) so real DuckDB runs the pragma.
+            process.env.EXTENSION_FETCH_POLICY = "local-only";
+            const spy = sinon.spy(DuckDBConnection.prototype, "runSQL");
+            await Package.create(
+               "testProject",
+               "testPackage",
+               testPackageDirectory,
+               new Map(),
+            );
+            const issued = spy.getCalls().map((c) => String(c.args[0]));
+            expect(
+               issued.some((s) =>
+                  /autoinstall_known_extensions\s*=\s*false/i.test(s),
+               ),
+            ).toBe(true);
+         },
+         { timeout: 30000 },
+      );
+   });
+
    describe("instance methods", () => {
       describe("create", () => {
          it("should throw PackageNotFoundError if the package manifest does not exist", async () => {

package/src/service/package.ts

@@ -2,6 +2,7 @@ import * as fs from "fs/promises";
 import * as path from "path";
 
 import "@malloydata/db-duckdb/native";
+import { DuckDBConnection } from "@malloydata/db-duckdb";
 import {
    Connection,
    ConnectionRuntime,
@@ -27,6 +28,7 @@ import {
    PackageNotFoundError,
    ServiceUnavailableError,
 } from "../errors";
+import { applyExtensionSessionSettings } from "./connection";
 import { formatDuration, logger } from "../logger";
 import {
    recordBuildPlanComputeDuration,
@@ -1256,7 +1258,16 @@ export class Package {
       malloyConfig.wrapConnections((base) => ({
          lookupConnection: async (name?: string) => {
             if (!name || name === "duckdb") {
-               return base.lookupConnection(name);
+               const connection = await base.lookupConnection(name);
+               // The per-package :memory: sandbox is a Publisher-owned DuckDB
+               // session too. Pin it against implicit auto-install so
+               // EXTENSION_FETCH_POLICY covers it — otherwise local-only's
+               // no-network guarantee had a hole here (the sandbox has no
+               // attached databases, so the env attach paths never touch it).
+               if (connection instanceof DuckDBConnection) {
+                  await applyExtensionSessionSettings(connection);
+               }
+               return connection;
             }
             // Resolve against the *current* environment MalloyConfig so a
             // connection-generation swap on Environment propagates without a

package/src/storage/duckdb/MaterializationRepository.spec.ts

@@ -3,16 +3,21 @@ import { DuckDBConnection } from "./DuckDBConnection";
 import { MaterializationRepository } from "./MaterializationRepository";
 import { initializeSchema } from "./schema";
 
-// Capture the SQL + params of the last query without touching a real DuckDB.
+// Capture the SQL + params of reads (all) and writes (run) without touching a
+// real DuckDB.
 function fakeRepo() {
    const calls: Array<{ sql: string; params: unknown[] }> = [];
+   const runCalls: Array<{ sql: string; params: unknown[] }> = [];
    const db = {
       all: async (sql: string, params: unknown[]) => {
          calls.push({ sql, params });
          return [];
       },
+      run: async (sql: string, params: unknown[]) => {
+         runCalls.push({ sql, params });
+      },
    } as unknown as DuckDBConnection;
-   return { repo: new MaterializationRepository(db), calls };
+   return { repo: new MaterializationRepository(db), calls, runCalls };
 }
 
 describe("MaterializationRepository list bounds", () => {
@@ -152,3 +157,35 @@ describe("MaterializationRepository.getLatestScheduledFireAt (real DuckDB)", ()
       expect(await repo.getLatestScheduledFireAt(ENV_ID, PKG)).toBeNull();
    });
 });
+
+// The publisher tracks materialization *records*; dropping the physical tables
+// is opt-in (deleteMaterialization({ dropTables })) and otherwise the caller's
+// responsibility. These cascade deletes therefore remove records only — they
+// must never issue DDL — so a materialized table intentionally outlives the
+// record when an environment or package is deleted. Locking that in guards the
+// contract against a future refactor that "helpfully" adds a DROP here (which
+// would fire against a possibly-already-torn-down connection on env delete).
+//
+// This pins DDL out of the low-level record deletes; it does not close the
+// standalone GC gap (#901: nothing drops those tables today, so they accumulate
+// in the warehouse). #901's best-effort drop-before-teardown lives ABOVE these
+// methods, so adding it completes the design and leaves this test green.
+describe("MaterializationRepository cascade deletes are records-only", () => {
+   it("deleteByEnvironmentId issues a single DELETE and no DDL", async () => {
+      const { repo, runCalls } = fakeRepo();
+      await repo.deleteByEnvironmentId("env-1");
+      expect(runCalls.length).toBe(1);
+      expect(runCalls[0].sql).toContain("DELETE FROM materializations");
+      expect(runCalls[0].sql).not.toContain("DROP");
+      expect(runCalls[0].params).toEqual(["env-1"]);
+   });
+
+   it("deleteByPackage issues a single DELETE and no DDL", async () => {
+      const { repo, runCalls } = fakeRepo();
+      await repo.deleteByPackage("env-1", "pkg-a");
+      expect(runCalls.length).toBe(1);
+      expect(runCalls[0].sql).toContain("DELETE FROM materializations");
+      expect(runCalls[0].sql).not.toContain("DROP");
+      expect(runCalls[0].params).toEqual(["env-1", "pkg-a"]);
+   });
+});

package/src/storage/duckdb/MaterializationRepository.ts

@@ -263,6 +263,18 @@ export class MaterializationRepository {
       await this.db.run("DELETE FROM materializations WHERE id = ?", [id]);
    }
 
+   // Cascade deletes remove the publisher's records only; they intentionally
+   // issue no DDL. Dropping the physical tables is opt-in
+   // (deleteMaterialization({ dropTables })) and otherwise the caller's
+   // responsibility, so a materialized table outlives its record when an
+   // environment or package is deleted. Adding a DROP here would also risk
+   // firing against a connection already torn down by the delete path.
+   //
+   // Standalone caveat (#901): with no control plane owning physical GC,
+   // nothing drops those tables today, so they accumulate in the warehouse.
+   // That is an open follow-up, not a settled contract — its best-effort
+   // drop-before-teardown belongs ABOVE these methods, so it completes this
+   // design rather than moving DDL into the low-level record deletes.
    async deleteByEnvironmentId(environmentId: string): Promise<void> {
       await this.db.run(
          "DELETE FROM materializations WHERE environment_id = ?",

package/tests/integration/materialization/scheduler_transitions.integration.spec.ts

@@ -0,0 +1,256 @@
+/// <reference types="bun-types" />
+
+import { afterAll, afterEach, beforeAll, describe, expect, it } from "bun:test";
+import path from "path";
+import { fileURLToPath } from "url";
+import { MaterializationScheduler } from "../../../src/service/materialization_scheduler";
+import { MaterializationService } from "../../../src/service/materialization_service";
+import { environmentStore } from "../../../src/server";
+import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const FIXTURES = path.resolve(__dirname, "../../fixtures");
+const SCHEDULED_FIXTURE = path.join(FIXTURES, "persist-schedule-test"); // cron "0 6 * * *", scope: version
+const PERSIST_FIXTURE = path.join(FIXTURES, "persist-test"); // same persist source, no schedule
+
+// Deterministic clock: tick(nowMs) is called explicitly so the daily
+// "0 6 * * *" cron fires without waiting on a wall-clock minute.
+const T0 = Date.parse("2026-01-01T00:00:00.000Z");
+const MIN = 60_000;
+const H = 60 * MIN;
+const D = 24 * H;
+
+const TERMINAL = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
+
+// Integration coverage for the scheduler's state-machine transitions against the
+// real store (#895): the unit spec covers the per-tick guards; these drive the
+// transitions end to end through the REST-created packages the scheduler sweeps.
+// Each test uses its own environment and tears it down in afterEach — the
+// scheduler sweeps every loaded environment, so a leftover env would leak fires
+// into another test (and steal the per-tick cap in the carryover case).
+describe("MaterializationScheduler transitions (integration, real store)", () => {
+   let e2e: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
+   let baseUrl: string;
+   let currentEnv: string | null = null;
+
+   beforeAll(async () => {
+      e2e = await startRestE2E();
+      baseUrl = e2e.baseUrl;
+   });
+
+   afterEach(async () => {
+      if (currentEnv) {
+         try {
+            await fetch(`${baseUrl}/api/v0/environments/${currentEnv}`, {
+               method: "DELETE",
+            });
+         } catch {
+            // best-effort teardown
+         }
+         currentEnv = null;
+      }
+   });
+
+   afterAll(async () => {
+      await e2e?.stop();
+      e2e = null;
+   });
+
+   // ── helpers ──────────────────────────────────────────────────────────
+
+   function newScheduler(maxFiresPerTick = 10): MaterializationScheduler {
+      return new MaterializationScheduler(
+         environmentStore,
+         new MaterializationService(environmentStore),
+         { tickIntervalMs: 60_000, maxFiresPerTick },
+      );
+   }
+
+   async function createEnv(
+      name: string,
+      packages: Array<{ name: string; location: string }>,
+   ): Promise<void> {
+      const res = await fetch(`${baseUrl}/api/v0/environments`, {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({ name, packages, connections: [] }),
+      });
+      if (!res.ok) {
+         throw new Error(
+            `create env ${name} failed (${res.status}): ${await res.text()}`,
+         );
+      }
+      currentEnv = name;
+   }
+
+   async function waitForPackage(env: string, pkg: string): Promise<void> {
+      const deadline = Date.now() + 30_000;
+      while (Date.now() < deadline) {
+         const res = await fetch(
+            `${baseUrl}/api/v0/environments/${env}/packages/${pkg}`,
+         );
+         if (res.ok) return;
+         await new Promise((r) => setTimeout(r, 500));
+      }
+      throw new Error(`package ${env}/${pkg} did not load in time`);
+   }
+
+   // Set scope: version + a cron schedule on a package (makes persist-test
+   // standalone-schedulable). editingPolicy is true, so the publish-gate rules
+   // run and must pass (version scope + valid cron, no freshness).
+   async function setSchedule(
+      env: string,
+      pkg: string,
+      schedule: string,
+   ): Promise<void> {
+      const res = await fetch(
+         `${baseUrl}/api/v0/environments/${env}/packages/${pkg}`,
+         {
+            method: "PATCH",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+               name: pkg,
+               scope: "version",
+               materialization: { schedule },
+            }),
+         },
+      );
+      if (!res.ok) {
+         throw new Error(
+            `set schedule on ${pkg} failed (${res.status}): ${await res.text()}`,
+         );
+      }
+   }
+
+   type Row = Record<string, unknown>;
+
+   async function envMaterializations(env: string): Promise<Row[]> {
+      const res = await fetch(
+         `${baseUrl}/api/v0/environments/${env}/packages/materializations`,
+      );
+      expect(res.status).toBe(200);
+      return (await res.json()) as Row[];
+   }
+
+   function triggerOf(m: Row): string | undefined {
+      return (m.metadata as { trigger?: string } | null)?.trigger;
+   }
+
+   async function scheduledFireCount(env: string): Promise<number> {
+      return (await envMaterializations(env)).filter(
+         (m) => triggerOf(m) === "SCHEDULER",
+      ).length;
+   }
+
+   // Let every fired build settle so env teardown doesn't race an in-flight one.
+   async function waitAllTerminal(env: string): Promise<void> {
+      const deadline = Date.now() + 90_000;
+      while (Date.now() < deadline) {
+         const rows = await envMaterializations(env);
+         if (rows.every((m) => TERMINAL.includes(m.status as string))) return;
+         await new Promise((r) => setTimeout(r, 250));
+      }
+      throw new Error(`materializations in ${env} did not all settle`);
+   }
+
+   // ── transitions ─────────────────────────────────────────────────────
+
+   it(
+      "a cron edit between ticks re-arms from now (no stale fire from the old cadence)",
+      async () => {
+         const ENV = "sched-tx-cron-edit";
+         await createEnv(ENV, [
+            { name: "persist-schedule-test", location: SCHEDULED_FIXTURE },
+         ]);
+         await waitForPackage(ENV, "persist-schedule-test");
+         const sched = newScheduler();
+
+         // Arm on the original "0 6 * * *" cadence: nextFire = Jan-01 06:00.
+         await sched.tick(T0);
+
+         // Edit the cron to midnight before the old 06:00 occurrence fires.
+         await setSchedule(ENV, "persist-schedule-test", "0 0 * * *");
+
+         // Tick just past the OLD 06:00 occurrence. If the edit didn't re-arm,
+         // the stale nextFire would fire here; instead arm() re-anchors from now
+         // on the new cron (next = Jan-02 00:00), so nothing fires.
+         await sched.tick(T0 + 6 * H + MIN);
+         expect(await scheduledFireCount(ENV)).toBe(0);
+
+         // Past the NEW cron's next occurrence (Jan-02 00:00): it fires, proving
+         // the edit re-armed rather than silently dropping the schedule.
+         await sched.tick(T0 + D + MIN);
+         expect(await scheduledFireCount(ENV)).toBe(1);
+
+         await waitAllTerminal(ENV);
+      },
+      { timeout: 120_000 },
+   );
+
+   // The unload/reload prune + re-anchor transition is covered as a deterministic
+   // unit test (materialization_scheduler.spec.ts): it is pure in-memory arming
+   // state, and driving it here via a package delete + same-name re-add races the
+   // package-install staging rename on Windows (POSIX is fine), which is unrelated
+   // to the scheduler behavior under test.
+
+   it(
+      "a fired occurrence advances to the next occurrence (fires once per occurrence, not every tick)",
+      async () => {
+         const ENV = "sched-tx-cadence";
+         await createEnv(ENV, [
+            { name: "persist-schedule-test", location: SCHEDULED_FIXTURE },
+         ]);
+         await waitForPackage(ENV, "persist-schedule-test");
+         const sched = newScheduler();
+
+         await sched.tick(T0); // arm
+         await sched.tick(T0 + 6 * H + MIN); // due -> fire once
+         expect(await scheduledFireCount(ENV)).toBe(1);
+         await waitAllTerminal(ENV);
+
+         // Immediately after, still inside the same daily window: the scheduler
+         // advanced nextFire to the next occurrence, so it does NOT re-fire every
+         // tick. (The advance is outcome-independent — a FAILED fire advances the
+         // same way; that isolation is covered in the scheduler unit spec.)
+         await sched.tick(T0 + 6 * H + 2 * MIN);
+         expect(await scheduledFireCount(ENV)).toBe(1);
+
+         // The next day's 06:00 occurrence fires again.
+         await sched.tick(T0 + D + 6 * H + MIN);
+         expect(await scheduledFireCount(ENV)).toBe(2);
+         await waitAllTerminal(ENV);
+      },
+      { timeout: 180_000 },
+   );
+
+   it(
+      "maxFiresPerTick caps a tick and the capped package fires on the next tick",
+      async () => {
+         const ENV = "sched-tx-cap";
+         await createEnv(ENV, [
+            { name: "persist-schedule-test", location: SCHEDULED_FIXTURE },
+            { name: "persist-test", location: PERSIST_FIXTURE },
+         ]);
+         await waitForPackage(ENV, "persist-schedule-test");
+         await waitForPackage(ENV, "persist-test");
+         // Make the second package standalone-schedulable on the same cadence.
+         await setSchedule(ENV, "persist-test", "0 6 * * *");
+
+         const sched = newScheduler(1); // cap = 1 fire per tick
+
+         await sched.tick(T0); // arm both
+         // Both due at 06:00, but the cap lets only one fire this tick.
+         await sched.tick(T0 + 6 * H + MIN);
+         expect(await scheduledFireCount(ENV)).toBe(1);
+
+         // The capped package is still due and fires on the following tick.
+         await sched.tick(T0 + 6 * H + 2 * MIN);
+         expect(await scheduledFireCount(ENV)).toBe(2);
+
+         await waitAllTerminal(ENV);
+      },
+      { timeout: 180_000 },
+   );
+});