@malloy-publisher/server 0.0.219 → 0.0.221

package/src/service/materialization_service.ts

@@ -34,7 +34,7 @@ import { errMessage } from "../utils";
 import {
    CompiledBuildPlan,
    compilePackageBuildPlan,
-   computeBuildId,
+   computeSourceEntityId,
    deriveAnnotationFields,
    iterGraphSources,
 } from "./build_plan";
@@ -43,16 +43,17 @@ import { bareTableName, quoteIdentifier, quoteTablePath } from "./quoting";
 import { resolveEnvironmentId } from "./resolve_environment";
 
 /**
- * Length of the buildId prefix used when synthesizing staging table names.
- * buildId is a 64-char SHA-256 hex string; 12 hex chars is 48 bits of
- * entropy, well inside every dialect's identifier limit (Postgres is the
- * tightest at 63).
+ * Length of the sourceEntityId prefix used when synthesizing staging table
+ * names. 12 hex chars is 48 bits of entropy, well inside every dialect's
+ * identifier limit (Postgres is the tightest at 63).
  */
-const STAGING_BUILD_ID_LEN = 12;
+const STAGING_ID_LEN = 12;
 
 /** Staging suffix appended to a table name while it is being built. */
-export function stagingSuffix(buildId: string): string {
-   return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
+export function stagingSuffix(sourceEntityId: string): string {
+   // Drop hyphens so the suffix stays a bare identifier fragment when the id
+   // becomes a UUID5 (a no-op for the current hex ids).
+   return `_${sourceEntityId.replace(/-/g, "").substring(0, STAGING_ID_LEN)}`;
 }
 
 /**
@@ -334,7 +335,7 @@ export class MaterializationService {
             carried = {};
          } else {
             // Skip-if-unchanged: reuse tables from the most recent successful
-            // manifest for sources whose buildId is unchanged, unless
+            // manifest for sources whose sourceEntityId is unchanged, unless
             // forceRefresh.
             const priorEntries = opts.forceRefresh
                ? {}
@@ -396,7 +397,7 @@ export class MaterializationService {
    /**
     * Derive the publisher's own build instructions for auto-run. Each persist
     * source (respecting the optional sourceNames filter) gets a self-assigned
-    * physical table name and COPY realization, unless its buildId is unchanged
+    * physical table name and COPY realization, unless its sourceEntityId is unchanged
     * since `priorEntries` — those are carried forward (reused) instead of
     * rebuilt.
     */
@@ -420,24 +421,24 @@ export class MaterializationService {
          )) {
             if (include && !include.has(persistSource.name)) continue;
 
-            const buildId = computeBuildId(
+            const sourceEntityId = computeSourceEntityId(
                persistSource,
                compiled.connectionDigests,
             );
-            if (seen.has(buildId)) continue;
-            seen.add(buildId);
+            if (seen.has(sourceEntityId)) continue;
+            seen.add(sourceEntityId);
 
-            const prior = priorEntries[buildId];
+            const prior = priorEntries[sourceEntityId];
             if (prior && prior.physicalTableName) {
-               carried[buildId] = prior;
+               carried[sourceEntityId] = prior;
                continue;
             }
 
             instructions.push({
-               buildId,
-               materializedTableId: `local-${buildId.substring(
+               sourceEntityId,
+               materializedTableId: `local-${sourceEntityId.substring(
                   0,
-                  STAGING_BUILD_ID_LEN,
+                  STAGING_ID_LEN,
                )}`,
                physicalTableName: selfAssignTableName(persistSource),
                realization: "COPY",
@@ -486,9 +487,11 @@ export class MaterializationService {
       entries: Record<string, ManifestEntry>,
    ): Promise<void> {
       const manifestEntries: BuildManifest["entries"] = {};
-      for (const [buildId, entry] of Object.entries(entries)) {
+      for (const [sourceEntityId, entry] of Object.entries(entries)) {
          if (entry.physicalTableName) {
-            manifestEntries[buildId] = { tableName: entry.physicalTableName };
+            manifestEntries[sourceEntityId] = {
+               tableName: entry.physicalTableName,
+            };
          }
       }
       try {
@@ -512,7 +515,7 @@ export class MaterializationService {
 
    /**
     * Validate caller-supplied build instructions against the package's compiled
-    * build plan: every instructed buildId must be a planned source, and only
+    * build plan: every instructed sourceEntityId must be a planned source, and only
     * COPY realization is supported. Throws when the package declares no persist
     * source (no plan to build against).
     */
@@ -525,15 +528,15 @@ export class MaterializationService {
             "Package has no persist sources; buildInstructions cannot be applied",
          );
       }
-      const plannedBuildIds = new Set<string>();
+      const plannedSourceEntityIds = new Set<string>();
       for (const source of Object.values(plan.sources)) {
-         plannedBuildIds.add(source.buildId);
+         plannedSourceEntityIds.add(source.sourceEntityId);
       }
 
       for (const instruction of instructions) {
-         if (!plannedBuildIds.has(instruction.buildId)) {
+         if (!plannedSourceEntityIds.has(instruction.sourceEntityId)) {
             throw new BadRequestError(
-               `Instruction references unknown buildId '${instruction.buildId}'`,
+               `Instruction references unknown sourceEntityId '${instruction.sourceEntityId}'`,
             );
          }
          // COPY-only for now; SNAPSHOT lands once clone semantics are defined.
@@ -561,20 +564,20 @@ export class MaterializationService {
       const { graphs, sources, connectionDigests, connections } = compiled;
 
       // Index instructions by sourceID (the stable per-source handle) so the
-      // build no longer recomputes the buildId to find an instruction.
-      // Recomputing it here forced a caller's buildId to equal the publisher's
-      // content hash, so a caller that derives buildIds by any other scheme
-      // would have its sources silently skipped (the recomputed buildId would
-      // not match the instruction). buildId is treated as opaque, caller-assigned
-      // identity. A buildId index is kept as a fallback for instructions without
+      // build no longer recomputes the sourceEntityId to find an instruction.
+      // Recomputing it here forced a caller's sourceEntityId to equal the publisher's
+      // content hash, so a caller that derives sourceEntityIds by any other scheme
+      // would have its sources silently skipped (the recomputed sourceEntityId would
+      // not match the instruction). sourceEntityId is treated as opaque, caller-assigned
+      // identity. A sourceEntityId index is kept as a fallback for instructions without
       // a sourceID (e.g. standalone auto-run).
       const bySourceID = new Map<string, BuildInstruction>();
-      const byBuildId = new Map<string, BuildInstruction>();
+      const bySourceEntityId = new Map<string, BuildInstruction>();
       for (const instruction of instructions) {
          if (instruction.sourceID) {
             bySourceID.set(instruction.sourceID, instruction);
          }
-         byBuildId.set(instruction.buildId, instruction);
+         bySourceEntityId.set(instruction.sourceEntityId, instruction);
       }
 
       // Accumulates physical names as sources are built so downstream sources
@@ -582,11 +585,13 @@ export class MaterializationService {
       // it with carried-forward entries so reused upstreams resolve too.
       const manifest = new Manifest();
       const entries: Record<string, ManifestEntry> = {};
-      for (const [buildId, entry] of Object.entries(seedEntries)) {
+      for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
          if (entry.physicalTableName) {
-            manifest.update(buildId, { tableName: entry.physicalTableName });
+            manifest.update(sourceEntityId, {
+               tableName: entry.physicalTableName,
+            });
          }
-         entries[buildId] = entry;
+         entries[sourceEntityId] = entry;
       }
 
       for (const graph of graphs) {
@@ -599,15 +604,19 @@ export class MaterializationService {
          for (const persistSource of iterGraphSources(graph, sources)) {
             if (signal.aborted) throw new Error("Build cancelled");
 
-            // The manifest is keyed by the content buildId — what Malloy
+            // The manifest is keyed by the content sourceEntityId — what Malloy
             // recomputes to resolve upstream persist references during SQL
-            // generation — independent of the instruction's identity buildId.
-            const buildId = computeBuildId(persistSource, connectionDigests);
-            // Prefer sourceID matching (so the caller's buildId scheme stays
-            // opaque to the build); fall back to buildId for instructions
+            // generation — independent of the instruction's identity sourceEntityId.
+            const sourceEntityId = computeSourceEntityId(
+               persistSource,
+               connectionDigests,
+            );
+            // Prefer sourceID matching (so the caller's sourceEntityId scheme stays
+            // opaque to the build); fall back to sourceEntityId for instructions
             // without a sourceID (auto-run).
             const instruction =
-               bySourceID.get(persistSource.sourceID) ?? byBuildId.get(buildId);
+               bySourceID.get(persistSource.sourceID) ??
+               bySourceEntityId.get(sourceEntityId);
             if (!instruction) continue;
 
             const entry = await this.buildOneSource(
@@ -617,7 +626,7 @@ export class MaterializationService {
                connectionDigests,
                manifest,
             );
-            entries[buildId] = entry;
+            entries[sourceEntityId] = entry;
          }
       }
 
@@ -627,7 +636,7 @@ export class MaterializationService {
    /**
     * Build a single instructed source into its assigned physical table.
     * COPY uses a staging table + atomic rename for crash-safety; the staging
-    * name derives from the buildId. Records and returns the manifest entry.
+    * name derives from the sourceEntityId. Records and returns the manifest entry.
     */
    private async buildOneSource(
       persistSource: PersistSource,
@@ -636,7 +645,7 @@ export class MaterializationService {
       connectionDigests: Record<string, string>,
       manifest: Manifest,
    ): Promise<ManifestEntry> {
-      const buildId = instruction.buildId;
+      const sourceEntityId = instruction.sourceEntityId;
       const physicalTableName = instruction.physicalTableName;
       const buildSQL = persistSource.getSQL({
          buildManifest: manifest.buildManifest,
@@ -644,7 +653,7 @@ export class MaterializationService {
       });
 
       const bareName = bareTableName(physicalTableName);
-      const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
+      const stagingTableName = `${physicalTableName}${stagingSuffix(sourceEntityId)}`;
       // The control plane sends the logical (unquoted) physical name; dialect-
       // quote each identifier here so a container path or quote-requiring name
       // (e.g. a hyphenated BigQuery project id) produces valid DDL. The manifest
@@ -681,7 +690,7 @@ export class MaterializationService {
       }
 
       // Make this table visible to downstream sources built later in this run.
-      manifest.update(buildId, { tableName: physicalTableName });
+      manifest.update(sourceEntityId, { tableName: physicalTableName });
 
       const durationMs = Math.round(performance.now() - startTime);
       recordSourceBuildDuration(durationMs);
@@ -691,7 +700,7 @@ export class MaterializationService {
       });
 
       return {
-         buildId,
+         sourceEntityId,
          sourceName: persistSource.name,
          materializedTableId: instruction.materializedTableId,
          physicalTableName,
@@ -799,7 +808,7 @@ export class MaterializationService {
          if (!connectionName || !physicalTableName) {
             logger.warn("Skipping manifest entry with no connection/table", {
                materializationId: m.id,
-               buildId: entry.buildId,
+               sourceEntityId: entry.sourceEntityId,
             });
             continue;
          }
@@ -825,7 +834,7 @@ export class MaterializationService {
             // table behind; clean it up too while we hold the connection.
             await connection.runSQL(
                `DROP TABLE IF EXISTS ${quoteTablePath(
-                  `${physicalTableName}${stagingSuffix(entry.buildId)}`,
+                  `${physicalTableName}${stagingSuffix(entry.sourceEntityId)}`,
                   dialect,
                )}`,
             );

package/src/service/materialization_test_fixtures.ts

@@ -48,7 +48,7 @@ export function makeBuildPlan(overrides: Partial<BuildPlan> = {}): BuildPlan {
             sourceID: "orders@m.malloy",
             connectionName: "duckdb",
             dialect: "duckdb",
-            buildId: "build-orders",
+            sourceEntityId: "build-orders",
             sql: "SELECT 1",
             columns: [],
          },
@@ -62,7 +62,7 @@ export function makeInstruction(
    overrides: Partial<BuildInstruction> = {},
 ): BuildInstruction {
    return {
-      buildId: "build-orders",
+      sourceEntityId: "build-orders",
       materializedTableId: "mt-1",
       physicalTableName: '"orders_v1"',
       realization: "COPY",
@@ -72,25 +72,41 @@ export function makeInstruction(
 
 /**
  * A minimal stand-in for a Malloy {@link PersistSource} exposing only what the
- * build internals touch (name/id, deterministic buildId, SQL, and the
+ * build internals touch (name/id, deterministic sourceEntityId, SQL, and the
  * `#@ persist name=` annotation reader, defaulted to "unset").
  */
 export function fakeSource(opts: {
    name: string;
-   buildId: string;
+   sourceEntityId: string;
    sql?: string;
    connectionName?: string;
    dialectName?: string;
+   /** key=value fields of the `#@ persist` annotation (e.g. sharing). */
+   annotationFields?: Record<string, string>;
 }): PersistSource {
+   const fields = opts.annotationFields;
    return {
       name: opts.name,
       sourceID: opts.name,
       connectionName: opts.connectionName ?? "duckdb",
       dialectName: opts.dialectName ?? "duckdb",
-      makeBuildId: () => opts.buildId,
+      makeBuildId: () => opts.sourceEntityId,
       getSQL: () => opts.sql ?? "SELECT 1",
       annotations: {
-         parseAsTag: () => ({ tag: { text: () => undefined } }),
+         parseAsTag: () =>
+            fields
+               ? {
+                    tag: {
+                       *entries() {
+                          for (const [key, value] of Object.entries(fields)) {
+                             yield [key, { text: () => value }];
+                          }
+                       },
+                    },
+                 }
+               : // No annotation fields: entries() is absent, and
+                 // deriveAnnotationFields degrades to {}.
+                 { tag: { text: () => undefined } },
       },
    } as unknown as PersistSource;
 }

package/src/service/model.ts

@@ -1931,7 +1931,7 @@ function makeHydrationRuntime(
    // (getSQL) time, gated on `prepareResultOptions.buildManifest`; without this
    // the hydrated model always recomputes from the base tables even though the
    // manifest was bound at load. `strict: false` keeps serving live for any
-   // source whose buildId is absent from the manifest.
+   // source whose sourceEntityId is absent from the manifest.
    return new Runtime({
       urlReader,
       config,

package/src/service/package.ts

@@ -61,7 +61,7 @@ export class Package {
    private packagePath: string;
    private malloyConfig: MalloyConfig;
    // Build-manifest binding state (Malloy Persistence v0). When bound, these
-   // entries (buildId -> { tableName }) are what served queries use to route
+   // entries (sourceEntityId -> { tableName }) are what served queries use to route
    // persist sources to their materialized physical tables; they are also reused
    // by the /compile preview so previewed SQL matches executed SQL. Surfaced on
    // /status (via getPackageMetadata) so the control plane can confirm a worker
@@ -72,7 +72,7 @@ export class Package {
    private manifestEntryCount = 0;
    private boundManifestUri: string | null = null;
    // The package's persist build plan: a deterministic property of the compiled
-   // package (per-source buildId, columns, build SQL, dependency graphs),
+   // package (per-source sourceEntityId, columns, build SQL, dependency graphs),
    // computed once at load from the live (unbound) models so it is stable for a
    // given (package version, connection config). Null when the package declares
    // no persist source. Surfaced read-only on getPackageMetadata() so a caller
@@ -347,6 +347,7 @@ export class Package {
          // absence as a schedule removal. See `parsePackageMaterialization`.
          materialization: outcome.packageMetadata.materialization ?? {
             schedule: null,
+            freshness: null,
          },
       };
 
@@ -450,6 +451,16 @@ export class Package {
             detail: invalidMsg,
          });
       }
+      // Same fail-safe split for the materialization cron gate: an existing
+      // package whose manifest violates the private-only cron rule still loads
+      // (warn), but a publish of it is rejected (see package.controller).
+      const invalidSchedule = pkg.formatInvalidSchedule();
+      if (invalidSchedule) {
+         logger.warn(`Package ${packageName} has an invalid cron schedule`, {
+            packageName,
+            detail: invalidSchedule,
+         });
+      }
       pkg.logEmptyDiscoveryWarnings();
 
       return pkg;
@@ -460,7 +471,7 @@ export class Package {
    }
 
    /**
-    * The package's persist build plan (per-source buildId, columns, build SQL,
+    * The package's persist build plan (per-source sourceEntityId, columns, build SQL,
     * dependency graphs), or null when the package declares no persist source.
     * A deterministic property of the compiled package; callers derive build
     * instructions from it for an orchestrated materialization.
@@ -503,7 +514,7 @@ export class Package {
    }
 
    /**
-    * The currently-bound build-manifest entries (buildId -> { tableName }), or
+    * The currently-bound build-manifest entries (sourceEntityId -> { tableName }), or
     * undefined when the package is serving live. Reused by the /compile preview
     * so previewed SQL gets the same persist-source -> physical-table routing as
     * execution.
@@ -603,6 +614,41 @@ export class Package {
       return this.exploreWarnings(exploresOverride).join("\n");
    }
 
+   /**
+    * Publish-gate for the package-level materialization cron (the power tier
+    * of artifact-anchored scheduling): a declared `materialization.schedule`
+    * governs every persist source in the package, so it is valid only when
+    * each source in the compiled build plan resolves to an explicit
+    * `sharing=private`. A cron over any shared/unset source would let one
+    * package dictate the refresh cadence of an artifact other packages share —
+    * those packages declare `materialization.freshness.window` instead and the
+    * control plane schedules the shared artifact from the tightest window.
+    * One actionable message per offending source (empty when the cron is
+    * valid or no cron is declared). Strict at publish (package.controller),
+    * warn-only at load/reload (loadViaWorker) — same split as explores.
+    */
+   public scheduleWarnings(): string[] {
+      const schedule = this.packageMetadata.materialization?.schedule;
+      if (!schedule) return [];
+      const sources = Object.values(this.buildPlan?.sources ?? {});
+      return sources
+         .filter((source) => source.sharing !== "private")
+         .map(
+            (source) =>
+               `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires every ` +
+               `persist source to declare '#@ persist ... sharing=private'; source ` +
+               `'${source.name}' resolves to ${
+                  source.sharing ? `'${source.sharing}'` : "unset"
+               }. Declare 'materialization.freshness.window' instead (the control plane ` +
+               `schedules refreshes from it), or mark every persist source sharing=private.`,
+         );
+   }
+
+   /** The {@link scheduleWarnings} joined into one string, or "" if none. */
+   public formatInvalidSchedule(): string {
+      return this.scheduleWarnings().join("\n");
+   }
+
    /**
     * One message per LISTED model whose discovery surface is empty because it
     * is import-only (imports other files, declares/re-exports nothing). Such a

package/src/service/package_manifest.spec.ts

@@ -5,7 +5,7 @@ describe("service/package_manifest", () => {
    describe("parsePackageMaterialization", () => {
       it("extracts a string schedule", () => {
          expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
-            { schedule: "0 6 * * *" },
+            { schedule: "0 6 * * *", freshness: null },
          );
       });
 
@@ -18,21 +18,59 @@ describe("service/package_manifest", () => {
          expect(
             parsePackageMaterialization({
                schedule: "*/15 * * * *",
-               freshness: { window: "24h" },
+               retries: 3,
             }),
-         ).toEqual({ schedule: "*/15 * * * *" });
+         ).toEqual({ schedule: "*/15 * * * *", freshness: null });
       });
 
       it("degrades a non-string schedule to null", () => {
          expect(parsePackageMaterialization({ schedule: 42 })).toEqual({
             schedule: null,
+            freshness: null,
+         });
+         expect(parsePackageMaterialization({})).toEqual({
+            schedule: null,
+            freshness: null,
          });
-         expect(parsePackageMaterialization({})).toEqual({ schedule: null });
       });
 
       it("returns null for non-object input", () => {
          expect(parsePackageMaterialization("0 6 * * *")).toBeNull();
          expect(parsePackageMaterialization(7)).toBeNull();
       });
+
+      it("extracts a full freshness block verbatim", () => {
+         expect(
+            parsePackageMaterialization({
+               freshness: { window: "24h", fallback: "stale_ok" },
+            }),
+         ).toEqual({
+            schedule: null,
+            freshness: { window: "24h", fallback: "stale_ok" },
+         });
+      });
+
+      it("keeps a partial freshness block (window only)", () => {
+         expect(
+            parsePackageMaterialization({ freshness: { window: "1h" } }),
+         ).toEqual({ schedule: null, freshness: { window: "1h" } });
+      });
+
+      it("drops invalid freshness fields rather than defaulting them", () => {
+         // A bad value is reported as absent — never substituted — so the
+         // control plane sees exactly what was (validly) declared.
+         expect(
+            parsePackageMaterialization({
+               freshness: { window: 24, fallback: "retry" },
+            }),
+         ).toEqual({ schedule: null, freshness: {} });
+      });
+
+      it("degrades a non-object freshness to null", () => {
+         expect(parsePackageMaterialization({ freshness: "24h" })).toEqual({
+            schedule: null,
+            freshness: null,
+         });
+      });
    });
 });

package/src/service/package_manifest.ts

@@ -4,18 +4,61 @@
  * testable in isolation from the package-load worker that consumes it.
  */
 
+const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
+export type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
+
+/**
+ * The manifest's `materialization.freshness` block, surfaced verbatim for the
+ * control plane (which owns the scheduling and query-time gating logic).
+ * Fields are kept only when valid — an invalid value is dropped, never
+ * defaulted, so absence on the wire always means "not declared".
+ */
+export interface PackageFreshnessConfig {
+   /** Maximum acceptable staleness, as a duration string (e.g. "24h"). */
+   window?: string;
+   /** Declared query-time behavior when the window is missed. */
+   fallback?: FreshnessFallback;
+}
+
 export interface PackageMaterializationConfig {
    /**
     * 5-field UNIX cron the control plane uses to schedule version-level
-    * re-materialization. Null when absent or not a string.
+    * re-materialization. Null when absent or not a string. Publish-gated:
+    * only valid when every persist source resolves to explicit
+    * `sharing=private` (see Package.scheduleWarnings).
     */
    schedule: string | null;
+   /**
+    * Freshness policy ({ window, fallback }). Null when the manifest declares
+    * none — distinct from an empty object, so the control plane can tell
+    * "no policy" from "policy with no valid fields".
+    */
+   freshness: PackageFreshnessConfig | null;
+}
+
+function parseFreshness(raw: unknown): PackageFreshnessConfig | null {
+   if (!raw || typeof raw !== "object") {
+      return null;
+   }
+   const { window, fallback } = raw as { window?: unknown; fallback?: unknown };
+   const freshness: PackageFreshnessConfig = {};
+   if (typeof window === "string") {
+      freshness.window = window;
+   }
+   if (
+      typeof fallback === "string" &&
+      (FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
+   ) {
+      freshness.fallback = fallback as FreshnessFallback;
+   }
+   return freshness;
 }
 
 /**
  * Read the manifest's `materialization` object, keeping only recognized fields.
  * Returns null when the block is absent so the API field is null rather than an
- * empty object; a non-string schedule degrades to null.
+ * empty object; a non-string schedule degrades to null, and an absent or
+ * non-object freshness degrades to null.
  */
 export function parsePackageMaterialization(
    raw: unknown,
@@ -23,6 +66,12 @@ export function parsePackageMaterialization(
    if (!raw || typeof raw !== "object") {
       return null;
    }
-   const schedule = (raw as { schedule?: unknown }).schedule;
-   return { schedule: typeof schedule === "string" ? schedule : null };
+   const { schedule, freshness } = raw as {
+      schedule?: unknown;
+      freshness?: unknown;
+   };
+   return {
+      schedule: typeof schedule === "string" ? schedule : null,
+      freshness: parseFreshness(freshness),
+   };
 }