@malloy-publisher/server 0.0.230 → 0.0.232

package/src/service/build_plan.spec.ts

@@ -10,9 +10,11 @@ import {
    deriveBuildPlan,
    flattenDependsOn,
    iterGraphSources,
+   projectToPublicColumns,
    resolveFreshness,
    resolvePackageConnections,
 } from "./build_plan";
+import { MaterializationEligibilityError } from "../errors";
 import { fakeSource } from "./materialization_test_fixtures";
 import { Model } from "./model";
 
@@ -150,6 +152,69 @@ describe("deriveAnnotationFields", () => {
    });
 });
 
+describe("projectToPublicColumns", () => {
+   // A source whose PUBLIC surface (intrinsic atomic fields) is `cols` — i.e. any
+   // `except:`-ed / access-restricted column is already absent here, as Malloy
+   // reflects it. deriveColumns reads exactly this.
+   const sourceWithPublicCols = (cols: string[]): PersistSource =>
+      ({
+         dialectName: "postgres",
+         _explore: {
+            intrinsicFields: cols.map((name) => ({
+               name,
+               isAtomicField: () => true,
+               type: "string",
+            })),
+         },
+      }) as unknown as PersistSource;
+
+   it("wraps the build SQL to project only the source's public columns", () => {
+      const src = sourceWithPublicCols(["order_date", "amount"]); // `region` hidden → absent
+      const out = projectToPublicColumns(
+         src,
+         "SELECT order_date, region, amount FROM t",
+      );
+      // Outer projection lists ONLY the public columns; the hidden one is dropped.
+      expect(out).toMatch(/^SELECT\b/);
+      expect(out).toContain("order_date");
+      expect(out).toContain("amount");
+      expect(out).toContain(
+         "FROM (SELECT order_date, region, amount FROM t) AS __public",
+      );
+      // `region` must not appear in the OUTER projection (before the subquery).
+      const outerProjection = out.slice(0, out.indexOf("FROM ("));
+      expect(outerProjection).not.toContain("region");
+   });
+
+   // Fails closed, like the rest of the eligibility surface: a public surface we
+   // can't determine refuses the build rather than widening to everything getSQL
+   // projects (which is where the hidden columns are).
+   it("refuses the build when the field list can't be read", () => {
+      const noExplore = {} as unknown as PersistSource;
+      expect(() => projectToPublicColumns(noExplore, "SELECT 1")).toThrow(
+         /public column surface could not be determined/,
+      );
+   });
+
+   it("refuses the build when the source exposes no public atomic columns", () => {
+      // Everything getSQL projects is hidden — the worst case to materialize.
+      expect(() =>
+         projectToPublicColumns(sourceWithPublicCols([]), "SELECT 1"),
+      ).toThrow(/no public atomic columns/);
+   });
+
+   it("refuses with an eligibility error (422), naming the source", () => {
+      const src = { ...sourceWithPublicCols([]), name: "daily" };
+      try {
+         projectToPublicColumns(src as unknown as PersistSource, "SELECT 1");
+         throw new Error("expected a refusal");
+      } catch (err) {
+         expect(err).toBeInstanceOf(MaterializationEligibilityError);
+         expect((err as Error).message).toContain("'daily'");
+      }
+   });
+});
+
 describe("computeSourceEntityId", () => {
    it("delegates to PersistSource.makeBuildId with the connection digest and SQL", () => {
       const makeBuildId = sinon.stub().returns("computed-id");
@@ -418,10 +483,68 @@ describe("compilePackageBuildPlan", () => {
          getModelRuntime.restore();
       }
    });
+
+   it("skips a model lacking ##! experimental.persistence instead of aborting the package", async () => {
+      // getBuildPlan() THROWS on a model without the flag (it does not return
+      // empty), so a header-less non-persist model — e.g. an imported base that
+      // only defines raw sources — must be skipped, or it would abort the whole
+      // package plan and drop the persist source in the sibling model.
+      const fakeModel = (hasFlag: boolean, graphs: MalloyBuildGraph[]) => ({
+         modelAnnotations: {
+            parseAsTag: () => ({ tag: { has: () => hasFlag } }),
+         },
+         getBuildPlan: () => {
+            if (!hasFlag) {
+               throw new Error(
+                  "Model must have ##! experimental.persistence to use getBuildPlan()",
+               );
+            }
+            return { graphs, sources: {}, tagParseLog: [] };
+         },
+      });
+      const models: Record<string, ReturnType<typeof fakeModel>> = {
+         "base.malloy": fakeModel(false, []), // header-less: must be skipped
+         "agg.malloy": fakeModel(true, [
+            {
+               connectionName: "duckdb",
+               nodes: [[{ sourceID: "daily@agg", dependsOn: [] }]],
+            },
+         ] as unknown as MalloyBuildGraph[]),
+      };
+      const getModelRuntime = sinon
+         .stub(Model, "getModelRuntime")
+         .callsFake((async (_path: unknown, modelPath: unknown) => ({
+            runtime: {
+               loadModel: () => ({
+                  getModel: async () => models[modelPath as string],
+               }),
+            },
+            modelURL: new URL(`file:///${modelPath}`),
+            importBaseURL: new URL("file:///"),
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+         })) as any);
+      try {
+         const pkg = {
+            getModelPaths: () => ["base.malloy", "agg.malloy"],
+            getPackagePath: () => "/test",
+            getMalloyConfig: () => ({}),
+            getMalloyConnection: async () => ({ getDigest: async () => "dig" }),
+         } as unknown as Parameters<typeof compilePackageBuildPlan>[0];
+
+         // Would throw here before the guard (base.malloy's getBuildPlan).
+         const compiled = await compilePackageBuildPlan(pkg);
+
+         // The header-less model is skipped; the persist source in agg survives.
+         expect(compiled.graphs).toHaveLength(1);
+         expect(compiled.graphs[0].nodes[0][0].sourceID).toBe("daily@agg");
+      } finally {
+         getModelRuntime.restore();
+      }
+   });
 });
 
 describe("computePackageBuildPlan", () => {
-   it("returns null when the package declares no persist sources", async () => {
+   it("returns a null plan and no dropped sources when the package declares no persist sources", async () => {
       const pkg = {
          getModelPaths: () => [],
          getPackagePath: () => "/test",
@@ -429,6 +552,9 @@ describe("computePackageBuildPlan", () => {
          getMalloyConnection: async () => ({}),
       } as unknown as Parameters<typeof computePackageBuildPlan>[0];
 
-      expect(await computePackageBuildPlan(pkg)).toBeNull();
+      const { plan, droppedPersistSources } =
+         await computePackageBuildPlan(pkg);
+      expect(plan).toBeNull();
+      expect(droppedPersistSources).toEqual([]);
    });
 });

package/src/service/build_plan.ts

@@ -6,12 +6,19 @@ import type {
    Connection as MalloyConnection,
    PersistSource,
 } from "@malloydata/malloy";
+import { Annotations } from "@malloydata/malloy";
 import { components } from "../api";
+import { MaterializationEligibilityError } from "../errors";
 import { MODEL_FILE_SUFFIX } from "../constants";
 import { logger } from "../logger";
-import { recordConnectionDigestSkipped } from "../materialization_metrics";
+import {
+   recordConnectionDigestSkipped,
+   recordEligibilityRefused,
+} from "../materialization_metrics";
 import { errMessage } from "../utils";
+import { assertMaterializationEligible } from "./materialization_eligibility";
 import { Model } from "./model";
+import { quoteIdentifier } from "./quoting";
 
 type WireBuildGraph = components["schemas"]["BuildGraph"];
 type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
@@ -74,6 +81,15 @@ export interface CompiledBuildPlan {
     * Optional so existing fixtures/callers that don't track it still typecheck.
     */
    sourceModelPaths?: Record<string, string>;
+   /**
+    * Sources carrying a `#@ persist` annotation that Malloy's getBuildPlan() did
+    * NOT recognize as a materializable build root, so they produced no plan entry
+    * and would otherwise be a silent no-op (served live, never materialized). See
+    * {@link detectDroppedPersistSources}. Callers surface these: a load-time
+    * warning and a hard build failure, so a persist annotation is never silently
+    * dropped.
+    */
+   droppedPersistSources?: { name: string; modelPath: string }[];
 }
 
 /** Output columns of a persist source, degrading to [] if unavailable. */
@@ -94,6 +110,85 @@ export function deriveColumns(persistSource: PersistSource): WireColumn[] {
    }
 }
 
+/**
+ * A source's PUBLIC column names, or a throw if they can't be determined. The
+ * strict counterpart to {@link deriveColumns}, which degrades to `[]` because its
+ * caller (the wire build plan) only reports columns. Here the list decides what
+ * gets WRITTEN, so "unknown" must not read as "nothing to narrow".
+ *
+ * A name is returned only for a field that is demonstrably a public atomic field;
+ * an unreadable field list, an unreadable individual field, or an empty result all
+ * throw. Empty is a throw rather than a pass-through because a source with no
+ * public atomic columns means everything `getSQL` projects is hidden — the worst
+ * case to materialize, not a benign one.
+ */
+function publicColumnNames(persistSource: PersistSource): string[] {
+   let names: string[];
+   try {
+      names = persistSource._explore.intrinsicFields
+         .filter((f) => f.isAtomicField())
+         .map((f) => f.name)
+         .filter((n): n is string => typeof n === "string" && n.length > 0);
+   } catch (err) {
+      throw new Error(
+         `the compiled source's field list could not be read (${errMessage(err)})`,
+      );
+   }
+   if (names.length === 0) {
+      throw new Error("the compiled source exposes no public atomic columns");
+   }
+   return names;
+}
+
+/**
+ * Wrap a source's build SQL to project only its PUBLIC columns. `getSQL` emits
+ * every underlying column, including ones the source hides (`except:`, non-public
+ * access modifiers); this narrows the physical table to the public surface.
+ *
+ * A `storage=` build must not materialize a hidden column. Reachability THROUGH
+ * the source is separately bounded by the declared serve shape (see
+ * `narrowSchemaToPublic`, and the `shape-bounds-physical-columns` scenario that
+ * proves a physical column absent from the shape does not resolve). What this
+ * projection prevents is the hidden column's values sitting AT REST in the
+ * destination store, where direct catalog access reaches them and where the data
+ * may have crossed a trust boundary the source's visibility rules were meant to
+ * hold. Applies to BOTH storage build paths (the warehouse-passthrough
+ * single-source build and the chained "stack on the parent" downstream build).
+ *
+ * Fails CLOSED: if the public surface can't be determined the build is refused,
+ * rather than widening to everything `getSQL` projects. This matches the rest of
+ * the eligibility surface (see `assertMaterializationEligible`), and it fires
+ * before any warehouse or destination SQL runs, so a refusal writes nothing and
+ * leaves the previous generation serving.
+ *
+ * @throws {MaterializationEligibilityError} (HTTP 422) naming the source.
+ */
+export function projectToPublicColumns(
+   persistSource: PersistSource,
+   buildSQL: string,
+): string {
+   let cols: string[];
+   try {
+      cols = publicColumnNames(persistSource);
+   } catch (err) {
+      recordEligibilityRefused("public_surface_unknown");
+      throw new MaterializationEligibilityError({
+         message:
+            `Source '${persistSource.name}' cannot be materialized into a ` +
+            `storage destination: its public column surface could not be ` +
+            `determined — ${errMessage(err)}. The publisher narrows a stored ` +
+            `table to the source's public columns, so it refuses the build ` +
+            `rather than materialize columns the source hides. This usually ` +
+            `means the compiled-source shape changed; drop 'storage=' to serve ` +
+            `this source live in the meantime.`,
+      });
+   }
+   const projection = cols
+      .map((n) => quoteIdentifier(n, persistSource.dialectName))
+      .join(", ");
+   return `SELECT ${projection} FROM (${buildSQL}) AS __public`;
+}
+
 /**
  * All key=value fields of a source's `#@ persist` annotation (e.g.
  * `{ name: "engaged_events", realization: "COPY" }`), degrading to {} if the
@@ -320,6 +415,56 @@ export async function resolvePackageConnections(
  * connections. The build plan is a pure function of the compiled model plus
  * connection config (no warehouse access).
  */
+/**
+ * Names of sources in a compiled model that carry a `#@ persist` annotation but
+ * are absent from the model's build plan — i.e. Malloy's getBuildPlan() did not
+ * recognize them as a materializable build root and silently returned no graph
+ * for them.
+ *
+ * BACKSTOP for a Malloy getBuildPlan() gap: it silently returns no graph for a
+ * `#@ persist` source whose shape it doesn't treat as a build root (observed for
+ * a filtered pass-through `X is <table> extend { where … }`, which stays type
+ * `table`; only query-shaped sources are treated as roots). Without this check
+ * the annotation is a silent no-op (served live, no build, no error). The primary
+ * fix is in Malloy (recognize the shape or emit a diagnostic); until then, detect
+ * the annotated-but-absent source here so callers can refuse loudly.
+ *
+ * Reads the annotation exactly as Malloy's own checkPersistAnnotation does
+ * (`Annotations(def.annotations).parseAsTag('@').tag.has('persist')`). Best-effort
+ * and fail-open: any introspection failure yields no dropped names rather than
+ * risking a false positive that would wrongly fail a healthy build.
+ */
+function detectDroppedPersistSources(
+   // eslint-disable-next-line @typescript-eslint/no-explicit-any
+   malloyModel: any,
+   recognizedNames: Set<string>,
+): string[] {
+   const dropped: string[] = [];
+   try {
+      const contents = malloyModel?._modelDef?.contents as
+         | Record<string, { type?: string; annotations?: unknown }>
+         | undefined;
+      if (!contents) return dropped;
+      for (const [name, def] of Object.entries(contents)) {
+         if (recognizedNames.has(name)) continue;
+         if (!def || !def.annotations) continue;
+         let isPersist = false;
+         try {
+            isPersist = new Annotations(def.annotations)
+               .parseAsTag("@")
+               .tag.has("persist");
+         } catch {
+            continue;
+         }
+         if (isPersist) dropped.push(name);
+      }
+   } catch {
+      // Fail open: never let introspection break the build plan.
+      return [];
+   }
+   return dropped;
+}
+
 export async function compilePackageBuildPlan(
    pkg: BuildPlanPackage,
    signal?: AbortSignal,
@@ -327,6 +472,7 @@ export async function compilePackageBuildPlan(
    const allGraphs: MalloyBuildGraph[] = [];
    const allSources: Record<string, PersistSource> = {};
    const sourceModelPaths: Record<string, string> = {};
+   const droppedPersistSources: { name: string; modelPath: string }[] = [];
 
    for (const modelPath of pkg.getModelPaths()) {
       // Only `.malloy` models declare persist sources. Skip `.malloynb`
@@ -345,8 +491,18 @@ export async function compilePackageBuildPlan(
          .loadModel(modelURL, { importBaseURL })
          .getModel();
 
-      // getBuildPlan() returns empty graphs for models with no #@ persist
-      // sources, so non-persist models are simply skipped below.
+      // getBuildPlan() THROWS "Model must have ##! experimental.persistence"
+      // on any model that lacks the flag — it does NOT return empty. So a
+      // header-less non-persist model in the package (e.g. an imported base
+      // model that only defines raw sources) would abort the entire package
+      // build plan, silently dropping every persist source in every other
+      // model. Mirror Malloy's own guard and skip such models: without the flag
+      // a model cannot carry a functioning persist source anyway. Models that
+      // DO have the flag but declare no persist source return empty graphs and
+      // are skipped by the `graphs.length === 0` check below.
+      const modelTag = malloyModel.modelAnnotations.parseAsTag("!").tag;
+      if (!modelTag.has("experimental", "persistence")) continue;
+
       const buildPlan = malloyModel.getBuildPlan();
       for (const msg of buildPlan.tagParseLog) {
          logger.warn("Persist annotation issue", {
@@ -355,6 +511,20 @@ export async function compilePackageBuildPlan(
             severity: msg.severity,
          });
       }
+
+      // Detect `#@ persist` sources the plan didn't recognize (see
+      // detectDroppedPersistSources). Runs BEFORE the empty-graphs `continue`, so
+      // a model whose ONLY persist source is a dropped shape is still caught.
+      const recognizedNames = new Set(
+         Object.values(buildPlan.sources).map((s) => s.name),
+      );
+      for (const name of detectDroppedPersistSources(
+         malloyModel,
+         recognizedNames,
+      )) {
+         droppedPersistSources.push({ name, modelPath });
+      }
+
       if (buildPlan.graphs.length === 0) continue;
 
       allGraphs.push(...buildPlan.graphs);
@@ -396,6 +566,7 @@ export async function compilePackageBuildPlan(
       connectionDigests,
       connections,
       sourceModelPaths,
+      droppedPersistSources,
    };
 }
 
@@ -451,24 +622,75 @@ export function deriveBuildPlan(
 }
 
 /**
- * Compile and project a package's build plan, or null when the package
- * declares no persist source. Convenience for the read-only `Package.buildPlan`
- * field, which is a deterministic property of the compiled package.
+ * Compile and project a package's build plan (null when the package declares no
+ * materializable persist source), plus any `#@ persist` sources that were
+ * silently dropped from the plan (see {@link detectDroppedPersistSources}) so
+ * the caller can surface a load-time warning. A deterministic property of the
+ * compiled package; feeds the read-only `Package.buildPlan` field.
  */
 export async function computePackageBuildPlan(
    pkg: BuildPlanPackage,
    signal?: AbortSignal,
-): Promise<BuildPlan | null> {
+): Promise<{
+   plan: BuildPlan | null;
+   droppedPersistSources: { name: string; modelPath: string }[];
+   sourceEligibility: SourceEligibility;
+}> {
    const compiled = await compilePackageBuildPlan(pkg, signal);
-   if (compiled.graphs.length === 0) {
-      return null;
+   const droppedPersistSources = compiled.droppedPersistSources ?? [];
+   const plan =
+      compiled.graphs.length === 0
+         ? null
+         : deriveBuildPlan(
+              compiled.graphs,
+              compiled.sources,
+              compiled.connectionDigests,
+              undefined,
+              compiled.sourceModelPaths,
+              pkg.getMaterializationConfig?.() ?? null,
+           );
+   return {
+      plan,
+      droppedPersistSources,
+      sourceEligibility: collectSourceEligibility(compiled.sources),
+   };
+}
+
+/**
+ * Which sources may be served from a materialized table, decided HERE because
+ * this is where the compiled sources exist. The serve side needs the answer
+ * without them: binding a host's manifest deliberately skips the recompile, so it
+ * cannot re-derive eligibility and would otherwise have to take the host's word
+ * for it.
+ *
+ * Carries the ELIGIBLE names, not just the refusals, so the serve side can
+ * require a positive result. Absence is not eligibility: Malloy admits only
+ * query-shaped sources as build roots, so a `#@ persist` on a filtered
+ * pass-through (`X is <table> extend { where … }`, which stays type `table`)
+ * never reaches this record at all — and that is the row-level-access shape. A
+ * refusal-only view would read that silence as consent.
+ *
+ * Keyed by source NAME because that is what a serve binding carries.
+ */
+export type SourceEligibility = {
+   /** Sources compiled, examined, and found eligible. */
+   eligible: string[];
+   /** Source name -> why it was refused; for the operator-facing log. */
+   refused: Record<string, string>;
+};
+
+function collectSourceEligibility(
+   sources: Record<string, PersistSource>,
+): SourceEligibility {
+   const eligible: string[] = [];
+   const refused: Record<string, string> = {};
+   for (const source of Object.values(sources)) {
+      try {
+         assertMaterializationEligible(source);
+         eligible.push(source.name);
+      } catch (err) {
+         refused[source.name] = errMessage(err);
+      }
    }
-   return deriveBuildPlan(
-      compiled.graphs,
-      compiled.sources,
-      compiled.connectionDigests,
-      undefined,
-      compiled.sourceModelPaths,
-      pkg.getMaterializationConfig?.() ?? null,
-   );
+   return { eligible, refused };
 }