@malloy-publisher/server 0.0.224 → 0.0.225

package/dist/app/api-doc.yaml

@@ -4124,6 +4124,25 @@ components:
             The source's declared `#@ persist ... refresh=...` value
             ("full" | "incremental"), reported verbatim; null = unset.
             Metadata pass-through — inert to the publisher today.
+        freshness:
+          oneOf:
+            - $ref: "#/components/schemas/Freshness"
+            - type: "null"
+          description: >-
+            The source's EFFECTIVE freshness objective after most-specific-wins
+            resolution (source > model-file > package). Null = unset at every
+            level; the control plane applies the platform default. Reported
+            verbatim (invalid fields dropped, never defaulted).
+        schedule:
+          type: ["string", "null"]
+          description: >-
+            The source's EFFECTIVE power-tier cron after most-specific-wins
+            resolution of the source's OWN declaration (source > model-file).
+            The package-level cron (PackageMaterializationConfig.schedule) is a
+            distinct package-grain concept and is NOT folded in here. Null =
+            unset. Valid only when the source resolves to sharing="private" (a
+            cron on a shared or unset source is rejected at publish; declare
+            freshness.window instead).
         columns:
           type: array
           description: Output schema of the source.
@@ -4162,6 +4181,45 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/BuildInstruction"
+        referenceManifest:
+          type: array
+          description: >-
+            Already-materialized persist upstreams the built sources may
+            reference but which are NOT rebuilt in this run. The publisher seeds
+            the build Manifest with these so a downstream source's upstream
+            persist reference resolves to the existing physical table instead of
+            recomputing it live. Only consumed on the orchestrated
+            (buildInstructions) path; auto-run seeds its own reference set from
+            the most-recent manifest.
+          items:
+            $ref: "#/components/schemas/ManifestReference"
+        strictUpstreams:
+          type: boolean
+          default: false
+          description: >-
+            When true, a persist upstream that is neither built here nor present
+            in referenceManifest fails the build (compiler strict mode) instead
+            of silently recomputing it live. Per-unit dispatch should set this
+            true.
+
+    ManifestReference:
+      type: object
+      description: >-
+        A reference to an already-materialized persist upstream that the built
+        sources may read but which is not rebuilt in this run.
+      required: [sourceEntityId, physicalTableName]
+      properties:
+        sourceEntityId:
+          type: string
+          description: >-
+            The upstream's content id AS REPORTED BY THE PUBLISHER in
+            PersistSourcePlan.sourceEntityId. It MUST equal what the compiler
+            recomputes for the manifest lookup (mkBuildID over the upstream's
+            manifest-ignorant SQL); do not substitute any other identity here.
+        physicalTableName:
+          type: string
+          description: >-
+            Fully-qualified physical table the upstream currently serves.
 
     BuildInstruction:
       type: object

package/dist/server.mjs

@@ -158407,6 +158407,11 @@ var require_utils74 = __commonJS((exports, module) => {
   };
 });
 
+// ../../node_modules/ssh2/lib/protocol/crypto/build/Release/sshcrypto.node
+var require_sshcrypto = __commonJS((exports, module) => {
+  module.exports = __require("./sshcrypto-8m50vnmb.node");
+});
+
 // ../../node_modules/ssh2/lib/protocol/crypto/poly1305.js
 var require_poly1305 = __commonJS((exports, module) => {
   var __dirname = "/home/runner/work/publisher/publisher/node_modules/ssh2/lib/protocol/crypto", __filename = "/home/runner/work/publisher/publisher/node_modules/ssh2/lib/protocol/crypto/poly1305.js";
@@ -158893,7 +158898,7 @@ var require_crypto = __commonJS((exports, module) => {
   var ChaChaPolyDecipher;
   var GenericDecipher;
   try {
-    binding = (()=>{throw new Error("Cannot require module "+"./crypto/build/Release/sshcrypto.node");})();
+    binding = require_sshcrypto();
     ({
       AESGCMCipher,
       ChaChaPolyCipher,
@@ -258421,6 +258426,7 @@ function hydrateMarkdownOnlyCells(notebookCells) {
 }
 
 // src/service/build_plan.ts
+var FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"];
 function deriveColumns(persistSource) {
   try {
     return persistSource._explore.intrinsicFields.filter((f) => f.isAtomicField()).map((f) => ({
@@ -258447,6 +258453,65 @@ function deriveAnnotationFields(persistSource) {
   } catch {}
   return out;
 }
+function tagFreshnessScheduleLayer(tag) {
+  if (!tag || typeof tag.text !== "function")
+    return {};
+  const layer = {};
+  const window2 = tag.text("freshness", "window");
+  if (typeof window2 === "string")
+    layer.window = window2;
+  const fallback = tag.text("freshness", "fallback");
+  if (typeof fallback === "string" && FRESHNESS_FALLBACKS.includes(fallback)) {
+    layer.fallback = fallback;
+  }
+  const schedule = tag.text("schedule");
+  if (typeof schedule === "string")
+    layer.schedule = schedule;
+  return layer;
+}
+function packageFreshnessLayer(cfg) {
+  if (!cfg)
+    return {};
+  const layer = {};
+  if (cfg.freshness?.window)
+    layer.window = cfg.freshness.window;
+  const fallback = cfg.freshness?.fallback;
+  if (typeof fallback === "string" && FRESHNESS_FALLBACKS.includes(fallback)) {
+    layer.fallback = fallback;
+  }
+  return layer;
+}
+function safeSourceTag(source) {
+  try {
+    return source.annotations.parseAsTag("@").tag;
+  } catch {
+    return;
+  }
+}
+function safeModelTag(source) {
+  try {
+    return source.modelAnnotations.parseAsTag().tag;
+  } catch {
+    return;
+  }
+}
+function resolveFreshnessSchedule(source, packageMaterialization) {
+  const sourceLayer = tagFreshnessScheduleLayer(safeSourceTag(source));
+  const modelLayer = tagFreshnessScheduleLayer(safeModelTag(source));
+  const pkgLayer = packageFreshnessLayer(packageMaterialization);
+  const window2 = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
+  const fallback = sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
+  const schedule = sourceLayer.schedule ?? modelLayer.schedule ?? null;
+  let freshness = null;
+  if (window2 !== undefined || fallback !== undefined) {
+    freshness = {};
+    if (window2 !== undefined)
+      freshness.window = window2;
+    if (fallback !== undefined)
+      freshness.fallback = fallback;
+  }
+  return { freshness, schedule };
+}
 function flattenDependsOn(node) {
   return node.dependsOn.map((d) => d.sourceID);
 }
@@ -258539,7 +258604,7 @@ async function compilePackageBuildPlan(pkg, signal) {
     sourceModelPaths
   };
 }
-function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, sourceModelPaths) {
+function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, sourceModelPaths, packageMaterialization) {
   const include = sourceNames ? new Set(sourceNames) : null;
   const wireGraphs = graphs.map((graph) => ({
     connectionName: graph.connectionName,
@@ -258553,6 +258618,7 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
     if (include && !include.has(source.name))
       continue;
     const annotationFields = deriveAnnotationFields(source);
+    const { freshness, schedule } = resolveFreshnessSchedule(source, packageMaterialization);
     wireSources[sourceID] = {
       name: source.name,
       sourceID: source.sourceID,
@@ -258562,6 +258628,8 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
       sql: source.getSQL(),
       sharing: annotationFields.sharing ?? null,
       refresh: annotationFields.refresh ?? null,
+      freshness,
+      schedule,
       columns: deriveColumns(source),
       annotationFields,
       modelPath: sourceModelPaths?.[sourceID]
@@ -258574,7 +258642,7 @@ async function computePackageBuildPlan(pkg, signal) {
   if (compiled.graphs.length === 0) {
     return null;
   }
-  return deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, undefined, compiled.sourceModelPaths);
+  return deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, undefined, compiled.sourceModelPaths, pkg.getMaterializationConfig?.() ?? null);
 }
 
 // src/service/freshness.ts
@@ -258851,6 +258919,9 @@ class Package {
   getBuildPlan() {
     return this.buildPlan;
   }
+  getMaterializationConfig() {
+    return this.packageMetadata.materialization ?? null;
+  }
   getPackageMetadata() {
     const metadata = {
       ...this.packageMetadata,
@@ -258936,12 +259007,22 @@ class Package {
 `);
   }
   scheduleWarnings() {
-    const schedule = this.packageMetadata.materialization?.schedule;
-    if (!schedule)
-      return [];
-    return [
-      `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is not accepted: ` + `package-level crons are rejected in Phase B. Declare ` + `'materialization.freshness.window' instead (the control plane schedules ` + `refreshes from it).`
-    ];
+    const warnings = [];
+    const sources = this.buildPlan?.sources ? Object.values(this.buildPlan.sources) : [];
+    for (const source of sources) {
+      if (source.schedule && source.sharing !== "private") {
+        warnings.push(`#@ persist source "${source.name}" declares a schedule (cron) but ` + `resolves to sharing="${source.sharing ?? "shared (unset default)"}": a per-source cron is valid only for sharing="private". ` + `Declare 'freshness.window' instead (the control plane schedules ` + `refreshes from it).`);
+      }
+    }
+    const packageSchedule = this.packageMetadata.materialization?.schedule;
+    if (packageSchedule) {
+      const nonPrivate = sources.filter((s) => s.sharing !== "private");
+      if (sources.length === 0 || nonPrivate.length > 0) {
+        const detail = sources.length === 0 ? "the package declares no persist source for it to govern" : `sources [${nonPrivate.map((s) => s.name).join(", ")}] resolve to shared/unset`;
+        warnings.push(`materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is valid ` + `only when every persist source resolves to sharing="private": ` + `${detail}. Declare 'materialization.freshness.window' instead ` + `(the control plane schedules refreshes from it).`);
+      }
+    }
+    return warnings;
   }
   formatInvalidSchedule() {
     return this.scheduleWarnings().join(`
@@ -261261,7 +261342,14 @@ class MaterializationController {
   validateCreateBody(body) {
     const result = {};
     if (body.buildInstructions !== undefined && body.buildInstructions !== null) {
-      result.buildInstructions = this.validateBuildInstructions(body.buildInstructions);
+      const parsed = this.validateBuildInstructions(body.buildInstructions);
+      result.buildInstructions = parsed.sources;
+      if (parsed.referenceManifest !== undefined) {
+        result.referenceManifest = parsed.referenceManifest;
+      }
+      if (parsed.strictUpstreams !== undefined) {
+        result.strictUpstreams = parsed.strictUpstreams;
+      }
     }
     if (body.forceRefresh !== undefined) {
       if (typeof body.forceRefresh !== "boolean") {
@@ -261281,11 +261369,42 @@ class MaterializationController {
     if (typeof raw !== "object" || raw === null) {
       throw new BadRequestError("buildInstructions must be an object");
     }
-    const sources = raw.sources;
+    const obj = raw;
+    const sources = obj.sources;
     if (!Array.isArray(sources) || sources.length === 0) {
       throw new BadRequestError("buildInstructions requires a non-empty 'sources' array of BuildInstruction");
     }
-    return sources.map((instruction) => this.validateInstruction(instruction));
+    const result = {
+      sources: sources.map((instruction) => this.validateInstruction(instruction))
+    };
+    if (obj.referenceManifest !== undefined && obj.referenceManifest !== null) {
+      if (!Array.isArray(obj.referenceManifest)) {
+        throw new BadRequestError("buildInstructions.referenceManifest must be an array of ManifestReference");
+      }
+      result.referenceManifest = obj.referenceManifest.map((ref) => this.validateManifestReference(ref));
+    }
+    if (obj.strictUpstreams !== undefined) {
+      if (typeof obj.strictUpstreams !== "boolean") {
+        throw new BadRequestError("buildInstructions.strictUpstreams must be a boolean");
+      }
+      result.strictUpstreams = obj.strictUpstreams;
+    }
+    return result;
+  }
+  validateManifestReference(raw) {
+    if (typeof raw !== "object" || raw === null) {
+      throw new BadRequestError("Each manifest reference must be an object");
+    }
+    const ref = raw;
+    for (const field of ["sourceEntityId", "physicalTableName"]) {
+      if (typeof ref[field] !== "string") {
+        throw new BadRequestError(`Manifest reference is missing required string field '${field}'`);
+      }
+    }
+    return {
+      sourceEntityId: ref.sourceEntityId,
+      physicalTableName: ref.physicalTableName
+    };
   }
   validateInstruction(raw) {
     if (typeof raw !== "object" || raw === null) {
@@ -268174,7 +268293,9 @@ class MaterializationService {
     this.runInBackground(created.id, (signal) => this.runBuild(created.id, environmentName, packageName, {
       sourceNames: options.sourceNames,
       forceRefresh,
-      buildInstructions
+      buildInstructions,
+      referenceManifest: options.referenceManifest,
+      strictUpstreams: options.strictUpstreams
     }, signal));
     return created;
   }
@@ -268199,12 +268320,12 @@ class MaterializationService {
       let carried;
       if (orchestrated) {
         instructions = opts.buildInstructions;
-        carried = {};
+        carried = this.referenceManifestToEntries(opts.referenceManifest);
       } else {
         const priorEntries = opts.forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
         ({ instructions, carried } = this.deriveSelfInstructions(compiled, opts.sourceNames, priorEntries));
       }
-      const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal);
+      const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal, opts.strictUpstreams ?? false);
       const sourcesBuilt = instructions.length;
       const sourcesReused = Object.keys(carried).length;
       const durationMs = Date.now() - startedAt;
@@ -268263,6 +268384,16 @@ class MaterializationService {
     }
     return { instructions, carried };
   }
+  referenceManifestToEntries(referenceManifest) {
+    const entries = {};
+    for (const ref of referenceManifest ?? []) {
+      entries[ref.sourceEntityId] = {
+        sourceEntityId: ref.sourceEntityId,
+        physicalTableName: ref.physicalTableName
+      };
+    }
+    return entries;
+  }
   async getMostRecentManifestEntries(environmentId, packageName, excludeId) {
     const list = await this.repository.listMaterializations(environmentId, packageName);
     for (const m of list) {
@@ -268315,7 +268446,7 @@ class MaterializationService {
       }
     }
   }
-  async executeInstructedBuild(compiled, instructions, seedEntries, signal) {
+  async executeInstructedBuild(compiled, instructions, seedEntries, signal, strict = false) {
     const { graphs, sources, connectionDigests, connections } = compiled;
     const bySourceID = new Map;
     const bySourceEntityId = new Map;
@@ -268326,6 +268457,7 @@ class MaterializationService {
       bySourceEntityId.set(instruction.sourceEntityId, instruction);
     }
     const manifest = new Manifest;
+    manifest.strict = strict;
     const entries = {};
     for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
       if (entry.physicalTableName) {

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.224",
+   "version": "0.0.225",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"

package/src/controller/materialization.controller.spec.ts

@@ -85,6 +85,78 @@ describe("MaterializationController.createMaterialization validation", () => {
       expect(await parse({ buildInstructions: null })).toEqual({});
    });
 
+   it("parses referenceManifest and strictUpstreams alongside sources", async () => {
+      const parsed = await parse({
+         buildInstructions: {
+            sources: [
+               {
+                  sourceEntityId: "b2",
+                  materializedTableId: "mt-2",
+                  physicalTableName: "downstream_v1",
+                  realization: "COPY",
+               },
+            ],
+            referenceManifest: [
+               { sourceEntityId: "b1", physicalTableName: "upstream_table" },
+            ],
+            strictUpstreams: true,
+         },
+      });
+      expect(parsed).toEqual({
+         buildInstructions: [
+            {
+               sourceEntityId: "b2",
+               sourceID: undefined,
+               materializedTableId: "mt-2",
+               physicalTableName: "downstream_v1",
+               realization: "COPY",
+            },
+         ],
+         referenceManifest: [
+            { sourceEntityId: "b1", physicalTableName: "upstream_table" },
+         ],
+         strictUpstreams: true,
+      });
+   });
+
+   it("rejects a referenceManifest entry missing a required field", async () => {
+      const { controller } = build();
+      await expect(
+         controller.createMaterialization("env", "pkg", {
+            buildInstructions: {
+               sources: [
+                  {
+                     sourceEntityId: "b2",
+                     materializedTableId: "mt-2",
+                     physicalTableName: "downstream_v1",
+                     realization: "COPY",
+                  },
+               ],
+               referenceManifest: [{ sourceEntityId: "b1" }],
+            },
+         }),
+      ).rejects.toThrow(BadRequestError);
+   });
+
+   it("rejects a non-boolean strictUpstreams", async () => {
+      const { controller } = build();
+      await expect(
+         controller.createMaterialization("env", "pkg", {
+            buildInstructions: {
+               sources: [
+                  {
+                     sourceEntityId: "b2",
+                     materializedTableId: "mt-2",
+                     physicalTableName: "downstream_v1",
+                     realization: "COPY",
+                  },
+               ],
+               strictUpstreams: "yes",
+            },
+         }),
+      ).rejects.toThrow(BadRequestError);
+   });
+
    it("rejects buildInstructions without a non-empty sources array", async () => {
       const { controller } = build();
       await expect(

package/src/controller/materialization.controller.ts

@@ -1,5 +1,8 @@
 import { BadRequestError } from "../errors";
-import { BuildInstruction } from "../storage/DatabaseInterface";
+import {
+   BuildInstruction,
+   ManifestReference,
+} from "../storage/DatabaseInterface";
 import { MaterializationService } from "../service/materialization_service";
 
 export class MaterializationController {
@@ -21,19 +24,28 @@ export class MaterializationController {
       forceRefresh?: boolean;
       sourceNames?: string[];
       buildInstructions?: BuildInstruction[];
+      referenceManifest?: ManifestReference[];
+      strictUpstreams?: boolean;
    } {
       const result: {
          forceRefresh?: boolean;
          sourceNames?: string[];
          buildInstructions?: BuildInstruction[];
+         referenceManifest?: ManifestReference[];
+         strictUpstreams?: boolean;
       } = {};
       if (
          body.buildInstructions !== undefined &&
          body.buildInstructions !== null
       ) {
-         result.buildInstructions = this.validateBuildInstructions(
-            body.buildInstructions,
-         );
+         const parsed = this.validateBuildInstructions(body.buildInstructions);
+         result.buildInstructions = parsed.sources;
+         if (parsed.referenceManifest !== undefined) {
+            result.referenceManifest = parsed.referenceManifest;
+         }
+         if (parsed.strictUpstreams !== undefined) {
+            result.strictUpstreams = parsed.strictUpstreams;
+         }
       }
       if (body.forceRefresh !== undefined) {
          if (typeof body.forceRefresh !== "boolean") {
@@ -57,22 +69,74 @@ export class MaterializationController {
 
    /**
     * Validate the orchestrated `buildInstructions` payload (BuildInstructions:
-    * `{ sources: BuildInstruction[] }`) and flatten it to the instruction list
-    * the service consumes.
+    * `{ sources: BuildInstruction[], referenceManifest?, strictUpstreams? }`)
+    * into the parts the service consumes: the flattened instruction list, the
+    * optional upstream reference manifest, and the strict flag.
     */
-   private validateBuildInstructions(raw: unknown): BuildInstruction[] {
+   private validateBuildInstructions(raw: unknown): {
+      sources: BuildInstruction[];
+      referenceManifest?: ManifestReference[];
+      strictUpstreams?: boolean;
+   } {
       if (typeof raw !== "object" || raw === null) {
          throw new BadRequestError("buildInstructions must be an object");
       }
-      const sources = (raw as Record<string, unknown>).sources;
+      const obj = raw as Record<string, unknown>;
+      const sources = obj.sources;
       if (!Array.isArray(sources) || sources.length === 0) {
          throw new BadRequestError(
             "buildInstructions requires a non-empty 'sources' array of BuildInstruction",
          );
       }
-      return sources.map((instruction) =>
-         this.validateInstruction(instruction),
-      );
+      const result: {
+         sources: BuildInstruction[];
+         referenceManifest?: ManifestReference[];
+         strictUpstreams?: boolean;
+      } = {
+         sources: sources.map((instruction) =>
+            this.validateInstruction(instruction),
+         ),
+      };
+      if (
+         obj.referenceManifest !== undefined &&
+         obj.referenceManifest !== null
+      ) {
+         if (!Array.isArray(obj.referenceManifest)) {
+            throw new BadRequestError(
+               "buildInstructions.referenceManifest must be an array of ManifestReference",
+            );
+         }
+         result.referenceManifest = obj.referenceManifest.map((ref) =>
+            this.validateManifestReference(ref),
+         );
+      }
+      if (obj.strictUpstreams !== undefined) {
+         if (typeof obj.strictUpstreams !== "boolean") {
+            throw new BadRequestError(
+               "buildInstructions.strictUpstreams must be a boolean",
+            );
+         }
+         result.strictUpstreams = obj.strictUpstreams;
+      }
+      return result;
+   }
+
+   private validateManifestReference(raw: unknown): ManifestReference {
+      if (typeof raw !== "object" || raw === null) {
+         throw new BadRequestError("Each manifest reference must be an object");
+      }
+      const ref = raw as Record<string, unknown>;
+      for (const field of ["sourceEntityId", "physicalTableName"] as const) {
+         if (typeof ref[field] !== "string") {
+            throw new BadRequestError(
+               `Manifest reference is missing required string field '${field}'`,
+            );
+         }
+      }
+      return {
+         sourceEntityId: ref.sourceEntityId as string,
+         physicalTableName: ref.physicalTableName as string,
+      };
    }
 
    private validateInstruction(raw: unknown): BuildInstruction {