@malloy-publisher/server 0.0.228 → 0.0.230

package/src/service/materialization_service.ts

@@ -14,13 +14,13 @@ import {
    MaterializationMode,
    recordAutoLoadOutcome,
    recordDropTables,
+   recordManifestBindDegraded,
    recordMaterializationRun,
    recordSourceBuildDuration,
    recordSourcesOutcome,
 } from "../materialization_metrics";
 import {
    BuildInstruction,
-   BuildManifest,
    BuildManifestResult,
    BuildPlan,
    FreshnessManifest,
@@ -41,7 +41,12 @@ import {
    iterGraphSources,
 } from "./build_plan";
 import { EnvironmentStore } from "./environment_store";
-import { bareTableName, quoteIdentifier, quoteTablePath } from "./quoting";
+import {
+   bareTableName,
+   quoteIdentifier,
+   quoteManifestTablePath,
+   quoteTablePath,
+} from "./quoting";
 import { resolveEnvironmentId } from "./resolve_environment";
 
 /**
@@ -177,6 +182,38 @@ export class MaterializationService {
       );
    }
 
+   /**
+    * Every materialization across all packages in an environment, newest first.
+    * Each record carries its `packageName`, so an env-scoped view can group or
+    * label by package without a per-package fan-out.
+    */
+   async listEnvironmentMaterializations(
+      environmentName: string,
+      options?: { limit?: number; offset?: number },
+   ): Promise<Materialization[]> {
+      const environmentId = await this.resolveEnvironmentId(environmentName);
+      return this.repository.listMaterializationsByEnvironment(
+         environmentId,
+         options,
+      );
+   }
+
+   /**
+    * `created_at` of the newest scheduler-fired materialization for a package,
+    * or null if none. The standalone scheduler uses this on its first arm to
+    * recover a fire missed during downtime (see MaterializationScheduler.arm).
+    */
+   async getLatestScheduledFireAt(
+      environmentName: string,
+      packageName: string,
+   ): Promise<Date | null> {
+      const environmentId = await this.resolveEnvironmentId(environmentName);
+      return this.repository.getLatestScheduledFireAt(
+         environmentId,
+         packageName,
+      );
+   }
+
    async getMaterialization(
       environmentName: string,
       packageName: string,
@@ -219,6 +256,13 @@ export class MaterializationService {
          buildInstructions?: BuildInstruction[];
          referenceManifest?: ManifestReference[];
          strictUpstreams?: boolean;
+         /**
+          * What initiated this run. `ON_DEMAND` (default) = a manual/API create;
+          * `SCHEDULER` = the standalone materialization scheduler firing a
+          * package's `materialization.schedule` cron. Recorded on the run
+          * metadata so a scheduled rebuild is distinguishable from a manual one.
+          */
+         trigger?: "ON_DEMAND" | "SCHEDULER";
       } = {},
    ): Promise<Materialization> {
       const environmentId = await this.resolveEnvironmentId(environmentName);
@@ -244,10 +288,12 @@ export class MaterializationService {
       }
 
       const forceRefresh = options.forceRefresh ?? false;
+      const trigger = options.trigger ?? "ON_DEMAND";
       const metadata = {
          forceRefresh,
          sourceNames: options.sourceNames ?? null,
          mode: orchestrated ? "orchestrated" : "auto",
+         trigger,
       };
 
       let created: Materialization;
@@ -280,6 +326,7 @@ export class MaterializationService {
                buildInstructions,
                referenceManifest: options.referenceManifest,
                strictUpstreams: options.strictUpstreams,
+               trigger,
             },
             signal,
          ),
@@ -306,6 +353,7 @@ export class MaterializationService {
          buildInstructions: BuildInstruction[] | undefined;
          referenceManifest: ManifestReference[] | undefined;
          strictUpstreams: boolean | undefined;
+         trigger: "ON_DEMAND" | "SCHEDULER";
       },
       signal: AbortSignal,
    ): Promise<void> {
@@ -379,6 +427,7 @@ export class MaterializationService {
             forceRefresh: opts.forceRefresh,
             sourceNames: opts.sourceNames ?? null,
             mode,
+            trigger: opts.trigger,
             sourcesBuilt,
             sourcesReused,
             durationMs,
@@ -466,9 +515,13 @@ export class MaterializationService {
    /**
     * Project the caller-supplied upstream reference manifest into the seed
     * entry map `executeInstructedBuild` consumes. Each reference is keyed by the
-    * compiler's manifest-lookup sourceEntityId and carries only the physical
-    * table name — enough for the build Manifest to resolve a downstream persist
-    * reference to the existing table without rebuilding the upstream.
+    * compiler's manifest-lookup sourceEntityId and carries the physical table
+    * name plus the connection it lives on — enough for the build Manifest to
+    * resolve a downstream persist reference to the existing table, and to quote
+    * that reference for the connection's dialect so it resolves on a
+    * case-folding engine (see {@link quoteSeedTablePath}). `connectionName` is
+    * optional on the wire: an older control plane that omits it seeds unquoted,
+    * exactly as before.
     */
    private referenceManifestToEntries(
       referenceManifest: ManifestReference[] | undefined,
@@ -478,6 +531,7 @@ export class MaterializationService {
          entries[ref.sourceEntityId] = {
             sourceEntityId: ref.sourceEntityId,
             physicalTableName: ref.physicalTableName,
+            connectionName: ref.connectionName,
          };
       }
       return entries;
@@ -524,11 +578,17 @@ export class MaterializationService {
       // stamps freshness (dataAsOf/window/fallback) on the wire manifest it
       // distributes, not on this in-memory post-build load, so these sources are
       // bound un-gated (always serve the freshly-built table).
-      const manifestEntries: BuildManifest["entries"] = {};
+      const manifestEntries: FreshnessManifest = {};
       for (const [sourceEntityId, entry] of Object.entries(entries)) {
          if (entry.physicalTableName) {
             manifestEntries[sourceEntityId] = {
                tableName: entry.physicalTableName,
+               // Carried so the bind step can quote the physical path for the
+               // connection's dialect (Package.quoteBoundTableNames) — the
+               // build CREATEd it quoted, so an unquoted read would miss on a
+               // case-folding engine in the window before the control plane's
+               // wire-manifest rebind.
+               connectionName: entry.connectionName,
             };
          }
       }
@@ -586,6 +646,43 @@ export class MaterializationService {
       }
    }
 
+   /**
+    * Quote a seeded upstream's physical path for the in-memory build Manifest,
+    * mirroring the CREATE side ({@link quoteManifestTablePath}) so a downstream
+    * persist source resolves the reference on a case-folding engine. The seed's
+    * own connection (carried on the entry) supplies the dialect. If it can't be
+    * resolved — an older control plane omitted `connectionName`, or the named
+    * connection isn't part of this build — the path binds unquoted: the build
+    * then fails loudly at the downstream CREATE on a case-folding engine (a
+    * self-signaling miss, unlike the serve path), and is unaffected elsewhere.
+    */
+   private quoteSeedTablePath(
+      sourceEntityId: string,
+      physicalTableName: string,
+      connectionName: string | undefined,
+      connections: Map<string, MalloyConnection>,
+   ): string {
+      const connection = connectionName
+         ? connections.get(connectionName)
+         : undefined;
+      if (!connection) {
+         // A named-but-absent connection is a real gap (the seed can't be
+         // quoted); a missing name is the benign older-CP default. Only the
+         // former is worth a signal.
+         if (connectionName) {
+            recordManifestBindDegraded();
+            logger.warn(
+               "Seeded upstream names a connection not present in this build; " +
+                  "leaving its manifest path unquoted (a downstream build will " +
+                  "fail on a case-folding engine)",
+               { sourceEntityId, connectionName },
+            );
+         }
+         return physicalTableName;
+      }
+      return quoteManifestTablePath(physicalTableName, connection.dialectName);
+   }
+
    /**
     * Shared build loop for both auto-run and orchestrated builds. Seeds the
     * manifest with carried-forward (reused) upstream entries so downstream
@@ -630,8 +727,19 @@ export class MaterializationService {
       const entries: Record<string, ManifestEntry> = {};
       for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
          if (entry.physicalTableName) {
+            // The build Manifest feeds a downstream persist's `FROM` verbatim,
+            // so a seeded upstream must carry the SAME quoting the builder
+            // CREATEd it with — else the downstream CREATE misses the
+            // case-preserved table on a case-folding engine. The seed keeps its
+            // logical (unquoted) name in `entries` (the committed manifest, in
+            // logical-name space); only the in-memory build Manifest is quoted.
             manifest.update(sourceEntityId, {
-               tableName: entry.physicalTableName,
+               tableName: this.quoteSeedTablePath(
+                  sourceEntityId,
+                  entry.physicalTableName,
+                  entry.connectionName,
+                  connections,
+               ),
             });
          }
          entries[sourceEntityId] = entry;
@@ -733,7 +841,12 @@ export class MaterializationService {
       }
 
       // Make this table visible to downstream sources built later in this run.
-      manifest.update(sourceEntityId, { tableName: physicalTableName });
+      // Record the SAME quoted path the CREATE used (not the logical name): the
+      // build Manifest is pasted into a downstream `FROM` verbatim, so on a
+      // case-folding engine an unquoted name would miss the case-preserved table
+      // just written. The returned entry (below) keeps the logical name for the
+      // committed manifest; only this in-memory build Manifest is quoted.
+      manifest.update(sourceEntityId, { tableName: quotedPhysical });
 
       const durationMs = Math.round(performance.now() - startTime);
       recordSourceBuildDuration(durationMs);

package/src/service/materialization_test_fixtures.ts

@@ -120,6 +120,12 @@ export function fakeSource(opts: {
    freshnessSchedule?: FakeFreshnessSchedule;
    /** Model-file-level (`##`) freshness default. */
    modelFreshnessSchedule?: FakeFreshnessSchedule;
+   /**
+    * Spy on the args Malloy's SQL generation is handed — chiefly the
+    * `buildManifest` that resolves upstream persist references — so a test can
+    * assert what physical name a downstream build sees for its upstream.
+    */
+   onGetSQL?: (sqlOpts: unknown) => void;
 }): PersistSource {
    const fields = opts.annotationFields;
    return {
@@ -128,7 +134,10 @@ export function fakeSource(opts: {
       connectionName: opts.connectionName ?? "duckdb",
       dialectName: opts.dialectName ?? "duckdb",
       makeBuildId: () => opts.sourceEntityId,
-      getSQL: () => opts.sql ?? "SELECT 1",
+      getSQL: (sqlOpts?: unknown) => {
+         opts.onGetSQL?.(sqlOpts);
+         return opts.sql ?? "SELECT 1";
+      },
       annotations: {
          parseAsTag: () => ({
             tag: fakeTag(fields, opts.freshnessSchedule),

package/src/service/model.spec.ts

@@ -4,6 +4,7 @@ import fs from "fs/promises";
 import sinon from "sinon";
 
 import {
+   AccessDeniedError,
    BadRequestError,
    ModelNotFoundError,
    PayloadTooLargeError,
@@ -420,6 +421,11 @@ describe("service/model", () => {
                undefined,
                undefined,
                undefined,
+               undefined,
+               // Model surfaces `region` so filterGivensToModelSurface (see
+               // model.ts) forwards it rather than dropping it as unknown.
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               [{ name: "region", type: "string" }] as any,
             );
 
             await expect(
@@ -716,6 +722,11 @@ describe("service/model", () => {
                // eslint-disable-next-line @typescript-eslint/no-explicit-any
                runnableCells as any,
                undefined,
+               undefined,
+               // Model surfaces `target_code` so filterGivensToModelSurface
+               // (see model.ts) forwards it rather than dropping it as unknown.
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               [{ name: "target_code", type: "string" }] as any,
             );
 
             await expect(
@@ -819,6 +830,62 @@ describe("service/model", () => {
       });
    });
 
+   describe("authorize struct-resolution drift invariant", () => {
+      // collectAllReachableGates walks struct.fields using Malloy's own
+      // isJoined ('join' in sd) / isSourceDef (struct.type is a known source
+      // kind) guards. If a future Malloy version adds a new source `type`
+      // that isSourceDef doesn't yet recognize, a genuinely-joined field
+      // would fail isSourceDef and used to be silently `continue`d past —
+      // failing OPEN (the joined source's gate is never found, so it's never
+      // evaluated). This confirms the walk instead denies loudly when it
+      // finds a field that IS a join but can't be resolved to a walkable
+      // SourceDef, rather than silently treating it as "nothing to gate".
+      it("denies rather than silently skipping when a joined field doesn't resolve to a walkable SourceDef", async () => {
+         const driftedJoinField = {
+            name: "mystery_join",
+            join: "one", // isJoined: true ('join' in sd)
+            // A source `type` isSourceDef doesn't (yet) recognize -- stands
+            // in for a future Malloy source kind the duck-typed guard hasn't
+            // been updated to cover.
+            type: "future_source_kind",
+            fields: [],
+         };
+         const runStruct = {
+            type: "table",
+            name: "root",
+            fields: [driftedJoinField],
+         };
+         const modelDef = { contents: {}, exports: [], queryList: [] };
+         const runnable = {
+            getPreparedQuery: sinon.stub().resolves({
+               _query: { structRef: runStruct },
+               _modelDef: modelDef,
+            }),
+         };
+
+         const model = new Model(
+            packageName,
+            mockModelPath,
+            {},
+            "model",
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            { loadQuery: sinon.stub() } as any, // modelMaterializer
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            modelDef as any,
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+            undefined,
+         );
+
+         await expect(
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            model.assertAuthorizedForAllSources(runnable as any, {}),
+         ).rejects.toThrow(AccessDeniedError);
+      });
+   });
+
    describe("static methods", () => {
       describe("getModelRuntime", () => {
          it("should throw ModelNotFoundError for invalid modelPath", async () => {