@malloy-publisher/server 0.0.214 → 0.0.216

package/dist/app/api-doc.yaml

@@ -2443,7 +2443,12 @@ components:
           description: |
             Package-level Malloy Persistence policy declared in
             malloy-publisher.json. The control plane reads it to drive scheduled
-            re-materialization. Null/absent = no package-level policy.
+            re-materialization. The object is present whenever the package has
+            loaded — with `schedule: null` when the manifest declares no
+            schedule — so the caller can treat object-present as the
+            authoritative manifest policy. Null/absent only when package
+            metadata is unavailable for this request (not loaded), which the
+            caller must treat as "unknown", not as a removal.
         manifestBindingStatus:
           type: string
           readOnly: true
@@ -2499,14 +2504,17 @@ components:
         Package-level Malloy Persistence policy from malloy-publisher.json's
         `materialization` block. Surfaced verbatim so the control plane can drive
         scheduled version-level re-materialization without re-reading the package
-        files.
+        files. The object is emitted whenever the package is loaded, even when no
+        policy is declared (all fields null), so its presence is an authoritative
+        "the manifest says this" signal rather than "policy missing".
       properties:
         schedule:
           type: ["string", "null"]
           description: >-
             5-field UNIX cron controlling how often the control plane
-            re-materializes this package's published versions. Null/absent = no
-            scheduled re-materialization (publish / on-demand only).
+            re-materializes this package's published versions. Null = no
+            scheduled re-materialization declared in the manifest (publish /
+            on-demand only).
 
     Model:
       type: object

package/dist/server.mjs

@@ -239566,7 +239566,9 @@ class Package {
       explores: outcome.packageMetadata.explores,
       queryableSources: outcome.packageMetadata.queryableSources,
       manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
-      materialization: outcome.packageMetadata.materialization ?? null
+      materialization: outcome.packageMetadata.materialization ?? {
+        schedule: null
+      }
     };
     const models = new Map;
     for (const sm of outcome.models) {
@@ -240514,7 +240516,8 @@ ${source}` : source;
         location: body.location,
         explores,
         queryableSources,
-        manifestLocation
+        manifestLocation,
+        materialization: existing.materialization
       });
       const invalidMsg = _package.formatInvalidExplores();
       if (invalidMsg) {
@@ -248857,8 +248860,12 @@ class MaterializationService {
   }
   async executeInstructedBuild(compiled, instructions, seedEntries, signal) {
     const { graphs, sources, connectionDigests, connections } = compiled;
+    const bySourceID = new Map;
     const byBuildId = new Map;
     for (const instruction of instructions) {
+      if (instruction.sourceID) {
+        bySourceID.set(instruction.sourceID, instruction);
+      }
       byBuildId.set(instruction.buildId, instruction);
     }
     const manifest = new Manifest;
@@ -248878,7 +248885,7 @@ class MaterializationService {
         if (signal.aborted)
           throw new Error("Build cancelled");
         const buildId = computeBuildId(persistSource, connectionDigests);
-        const instruction = byBuildId.get(buildId);
+        const instruction = bySourceID.get(persistSource.sourceID) ?? byBuildId.get(buildId);
         if (!instruction)
           continue;
         const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest);

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.214",
+   "version": "0.0.216",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"

package/src/service/environment.ts

@@ -1340,6 +1340,13 @@ export class Environment {
             explores,
             queryableSources,
             manifestLocation,
+            // Carry the manifest-derived materialization policy through the
+            // PATCH. `setPackageMetadata` replaces the whole object, so omitting
+            // this wipes the in-memory `materialization` until the next reload —
+            // which makes a later getPackage report no schedule and lets the
+            // control plane misread the gap as a schedule removal. It is not a
+            // PATCH-editable field, so always preserve the existing value.
+            materialization: existing.materialization,
          });
 
          // Strict-reject, symmetric with the publish path

package/src/service/materialization_schedule_surface.spec.ts

@@ -0,0 +1,103 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import * as fs from "fs/promises";
+import * as os from "os";
+import * as path from "path";
+
+import { Environment } from "./environment";
+
+/**
+ * The publisher must surface a non-null `materialization` object on a loaded
+ * package's metadata so the control plane can treat object-present as the
+ * authoritative manifest policy ("this is what the manifest says") and
+ * object-absent as "metadata not loaded this request" — never as a schedule
+ * removal. A metadata PATCH must not drop that policy from the in-memory
+ * metadata. Regression coverage for the re-materialization schedule self-wipe
+ * (docs/bugs/materialization-schedule-self-wipe.md).
+ *
+ * Runs against a real `Environment` + real `Package.create` over temp dirs.
+ */
+describe("materialization schedule surfacing", () => {
+   const MODEL = `source: ones is duckdb.sql("SELECT 1 as x")\n`;
+   let rootDir: string;
+   let envPath: string;
+
+   async function writePackageDir(
+      manifest: Record<string, unknown>,
+   ): Promise<void> {
+      const dir = path.join(envPath, "pkg");
+      await fs.mkdir(dir, { recursive: true });
+      await fs.writeFile(
+         path.join(dir, "publisher.json"),
+         JSON.stringify({ name: "pkg", description: "fixture", ...manifest }),
+      );
+      await fs.writeFile(path.join(dir, "model.malloy"), MODEL);
+   }
+
+   beforeEach(async () => {
+      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-mat-"));
+      envPath = path.join(rootDir, "env");
+      await fs.mkdir(envPath, { recursive: true });
+   });
+
+   afterEach(async () => {
+      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
+   });
+
+   it(
+      "surfaces the manifest's materialization.schedule on load",
+      async () => {
+         const env = await Environment.create("testEnv", envPath, []);
+         await writePackageDir({ materialization: { schedule: "0 6 * * *" } });
+         await env.addPackage("pkg");
+
+         const pkg = await env.getPackage("pkg", false);
+         expect(pkg.getPackageMetadata().materialization).toEqual({
+            schedule: "0 6 * * *",
+         });
+      },
+      { timeout: 20000 },
+   );
+
+   it(
+      "surfaces a non-null materialization object even when the manifest declares none",
+      async () => {
+         const env = await Environment.create("testEnv", envPath, []);
+         await writePackageDir({});
+         await env.addPackage("pkg");
+
+         // Object present with a null schedule — NOT a null object — so the
+         // control plane reads it as an authoritative "no schedule declared",
+         // not as "metadata unavailable".
+         const pkg = await env.getPackage("pkg", false);
+         expect(pkg.getPackageMetadata().materialization).toEqual({
+            schedule: null,
+         });
+      },
+      { timeout: 20000 },
+   );
+
+   it(
+      "preserves the materialization policy across a metadata PATCH",
+      async () => {
+         const env = await Environment.create("testEnv", envPath, []);
+         await writePackageDir({ materialization: { schedule: "0 6 * * *" } });
+         await env.addPackage("pkg");
+
+         // A description-only PATCH must not wipe the schedule from the
+         // in-memory metadata. The bug: setPackageMetadata replaces the whole
+         // object, so a later getPackage reported no schedule and the control
+         // plane misread the gap as a removal and cleared the cadence.
+         const updated = await env.updatePackage("pkg", {
+            name: "pkg",
+            description: "updated",
+         });
+         expect(updated.materialization).toEqual({ schedule: "0 6 * * *" });
+
+         const pkg = await env.getPackage("pkg", false);
+         expect(pkg.getPackageMetadata().materialization).toEqual({
+            schedule: "0 6 * * *",
+         });
+      },
+      { timeout: 20000 },
+   );
+});

package/src/service/materialization_service.ts

@@ -560,8 +560,20 @@ export class MaterializationService {
    ): Promise<Record<string, ManifestEntry>> {
       const { graphs, sources, connectionDigests, connections } = compiled;
 
+      // Index instructions by sourceID (the stable per-source handle) so the
+      // build no longer recomputes the buildId to find an instruction.
+      // Recomputing it here forced a caller's buildId to equal the publisher's
+      // content hash, so a caller that derives buildIds by any other scheme
+      // would have its sources silently skipped (the recomputed buildId would
+      // not match the instruction). buildId is treated as opaque, caller-assigned
+      // identity. A buildId index is kept as a fallback for instructions without
+      // a sourceID (e.g. standalone auto-run).
+      const bySourceID = new Map<string, BuildInstruction>();
       const byBuildId = new Map<string, BuildInstruction>();
       for (const instruction of instructions) {
+         if (instruction.sourceID) {
+            bySourceID.set(instruction.sourceID, instruction);
+         }
          byBuildId.set(instruction.buildId, instruction);
       }
 
@@ -587,8 +599,15 @@ export class MaterializationService {
          for (const persistSource of iterGraphSources(graph, sources)) {
             if (signal.aborted) throw new Error("Build cancelled");
 
+            // The manifest is keyed by the content buildId — what Malloy
+            // recomputes to resolve upstream persist references during SQL
+            // generation — independent of the instruction's identity buildId.
             const buildId = computeBuildId(persistSource, connectionDigests);
-            const instruction = byBuildId.get(buildId);
+            // Prefer sourceID matching (so the caller's buildId scheme stays
+            // opaque to the build); fall back to buildId for instructions
+            // without a sourceID (auto-run).
+            const instruction =
+               bySourceID.get(persistSource.sourceID) ?? byBuildId.get(buildId);
             if (!instruction) continue;
 
             const entry = await this.buildOneSource(

package/src/service/package.ts

@@ -331,7 +331,16 @@ export class Package {
          explores: outcome.packageMetadata.explores,
          queryableSources: outcome.packageMetadata.queryableSources,
          manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
-         materialization: outcome.packageMetadata.materialization ?? null,
+         // Always surface a non-null `materialization` object once the package
+         // has loaded (schedule null when the manifest declares no policy). The
+         // control plane treats object-present as the authoritative "this is
+         // what the manifest says" signal and object-absent as "metadata not
+         // available this request" — so it must never be dropped to null on a
+         // successfully loaded package, or the CP can misread a transient
+         // absence as a schedule removal. See `parsePackageMaterialization`.
+         materialization: outcome.packageMetadata.materialization ?? {
+            schedule: null,
+         },
       };
 
       // Build live `Model`s from worker output. Any per-model compile