@malloy-publisher/server 0.0.210 → 0.0.211

package/src/service/materialization_service.ts

@@ -30,13 +30,16 @@ import {
    ResourceRepository,
 } from "../storage/DatabaseInterface";
 import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
+import { errMessage } from "../utils";
 import {
    CompiledBuildPlan,
    compilePackageBuildPlan,
    computeBuildId,
+   deriveAnnotationFields,
+   iterGraphSources,
 } from "./build_plan";
 import { EnvironmentStore } from "./environment_store";
-import { splitTablePath } from "./quoting";
+import { bareTableName, quoteIdentifier, quoteTablePath } from "./quoting";
 import { resolveEnvironmentId } from "./resolve_environment";
 
 /**
@@ -58,14 +61,7 @@ export function stagingSuffix(buildId: string): string {
  * The author owns quoting the `name=` value for the dialect.
  */
 function selfAssignTableName(persistSource: PersistSource): string {
-   try {
-      return (
-         persistSource.annotations.parseAsTag("@").tag.text("name") ||
-         persistSource.name
-      );
-   } catch {
-      return persistSource.name;
-   }
+   return deriveAnnotationFields(persistSource).name || persistSource.name;
 }
 
 /** Classify a thrown build error as cancelled (cooperative abort) or failed. */
@@ -239,9 +235,7 @@ export class MaterializationService {
          packageName,
       );
       if (active) {
-         throw new MaterializationConflictError(
-            `Package ${packageName} already has an active materialization (${active.id})`,
-         );
+         throw this.activeConflict(packageName, active.id);
       }
 
       const forceRefresh = options.forceRefresh ?? false;
@@ -265,11 +259,7 @@ export class MaterializationService {
                environmentId,
                packageName,
             );
-            throw new MaterializationConflictError(
-               winner
-                  ? `Package ${packageName} already has an active materialization (${winner.id})`
-                  : `Package ${packageName} already has an active materialization`,
-            );
+            throw this.activeConflict(packageName, winner?.id);
          }
          throw err;
       }
@@ -320,6 +310,12 @@ export class MaterializationService {
       const startedAt = Date.now();
 
       try {
+         // Persist the run's start time so the UI can compute a duration
+         // (start -> now while in-flight, start -> completed once terminal).
+         await this.repository.updateMaterialization(id, {
+            startedAt: new Date(startedAt),
+         });
+
          const environmentId = await this.resolveEnvironmentId(environmentName);
          const environment = await this.environmentStore.getEnvironment(
             environmentName,
@@ -418,35 +414,34 @@ export class MaterializationService {
       const seen = new Set<string>();
 
       for (const graph of compiled.graphs) {
-         for (const level of graph.nodes) {
-            for (const node of level) {
-               const persistSource = compiled.sources[node.sourceID];
-               if (!persistSource) continue;
-               if (include && !include.has(persistSource.name)) continue;
-
-               const buildId = computeBuildId(
-                  persistSource,
-                  compiled.connectionDigests,
-               );
-               if (seen.has(buildId)) continue;
-               seen.add(buildId);
-
-               const prior = priorEntries[buildId];
-               if (prior && prior.physicalTableName) {
-                  carried[buildId] = prior;
-                  continue;
-               }
-
-               instructions.push({
-                  buildId,
-                  materializedTableId: `local-${buildId.substring(
-                     0,
-                     STAGING_BUILD_ID_LEN,
-                  )}`,
-                  physicalTableName: selfAssignTableName(persistSource),
-                  realization: "COPY",
-               });
+         for (const persistSource of iterGraphSources(
+            graph,
+            compiled.sources,
+         )) {
+            if (include && !include.has(persistSource.name)) continue;
+
+            const buildId = computeBuildId(
+               persistSource,
+               compiled.connectionDigests,
+            );
+            if (seen.has(buildId)) continue;
+            seen.add(buildId);
+
+            const prior = priorEntries[buildId];
+            if (prior && prior.physicalTableName) {
+               carried[buildId] = prior;
+               continue;
             }
+
+            instructions.push({
+               buildId,
+               materializedTableId: `local-${buildId.substring(
+                  0,
+                  STAGING_BUILD_ID_LEN,
+               )}`,
+               physicalTableName: selfAssignTableName(persistSource),
+               realization: "COPY",
+            });
          }
       }
 
@@ -589,25 +584,21 @@ export class MaterializationService {
                `Connection '${graph.connectionName}' not found`,
             );
          }
-         for (const level of graph.nodes) {
-            for (const node of level) {
-               if (signal.aborted) throw new Error("Build cancelled");
-               const persistSource = sources[node.sourceID];
-               if (!persistSource) continue;
-
-               const buildId = computeBuildId(persistSource, connectionDigests);
-               const instruction = byBuildId.get(buildId);
-               if (!instruction) continue;
-
-               const entry = await this.buildOneSource(
-                  persistSource,
-                  instruction,
-                  connection,
-                  connectionDigests,
-                  manifest,
-               );
-               entries[buildId] = entry;
-            }
+         for (const persistSource of iterGraphSources(graph, sources)) {
+            if (signal.aborted) throw new Error("Build cancelled");
+
+            const buildId = computeBuildId(persistSource, connectionDigests);
+            const instruction = byBuildId.get(buildId);
+            if (!instruction) continue;
+
+            const entry = await this.buildOneSource(
+               persistSource,
+               instruction,
+               connection,
+               connectionDigests,
+               manifest,
+            );
+            entries[buildId] = entry;
          }
       }
 
@@ -633,32 +624,37 @@ export class MaterializationService {
          connectionDigests,
       });
 
-      const { bareName } = splitTablePath(physicalTableName);
+      const bareName = bareTableName(physicalTableName);
       const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
+      // The control plane sends the logical (unquoted) physical name; dialect-
+      // quote each identifier here so a container path or quote-requiring name
+      // (e.g. a hyphenated BigQuery project id) produces valid DDL. The manifest
+      // echoes the logical name (below) so the CP stays in logical-name space.
+      const dialect = persistSource.dialectName;
+      const quotedStaging = quoteTablePath(stagingTableName, dialect);
+      const quotedPhysical = quoteTablePath(physicalTableName, dialect);
+      const quotedBareName = quoteIdentifier(bareName, dialect);
 
       const startTime = performance.now();
-      await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
+      await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`);
       try {
          await connection.runSQL(
-            `CREATE TABLE ${stagingTableName} AS (${buildSQL})`,
+            `CREATE TABLE ${quotedStaging} AS (${buildSQL})`,
          );
-         await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
+         await connection.runSQL(`DROP TABLE IF EXISTS ${quotedPhysical}`);
          await connection.runSQL(
-            `ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`,
+            `ALTER TABLE ${quotedStaging} RENAME TO ${quotedBareName}`,
          );
       } catch (err) {
          try {
-            await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
+            await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`);
          } catch (cleanupErr) {
             logger.warn(
                "Failed to clean up staging table after a failed build; physical leak",
                {
                   stagingTableName,
                   connectionName: persistSource.connectionName,
-                  cleanupError:
-                     cleanupErr instanceof Error
-                        ? cleanupErr.message
-                        : String(cleanupErr),
+                  cleanupError: errMessage(cleanupErr),
                },
             );
          }
@@ -795,14 +791,23 @@ export class MaterializationService {
                connection = await pkg.getMalloyConnection(connectionName);
                connectionCache.set(connectionName, connection);
             }
+            // Dialect-quote from the live connection, the same way
+            // buildOneSource quoted at build time, so a name that built
+            // successfully also drops successfully (container paths, hyphenated
+            // BigQuery project ids, etc.).
+            const dialect = connection.dialectName;
             await connection.runSQL(
-               `DROP TABLE IF EXISTS ${physicalTableName}`,
+               `DROP TABLE IF EXISTS ${quoteTablePath(
+                  physicalTableName,
+                  dialect,
+               )}`,
             );
             // A crash between staging-create and rename can leave the staging
             // table behind; clean it up too while we hold the connection.
             await connection.runSQL(
-               `DROP TABLE IF EXISTS ${physicalTableName}${stagingSuffix(
-                  entry.buildId,
+               `DROP TABLE IF EXISTS ${quoteTablePath(
+                  `${physicalTableName}${stagingSuffix(entry.buildId)}`,
+                  dialect,
                )}`,
             );
             recordDropTables("success");
@@ -817,7 +822,7 @@ export class MaterializationService {
                materializationId: m.id,
                physicalTableName,
                connectionName,
-               error: err instanceof Error ? err.message : String(err),
+               error: errMessage(err),
             });
          }
       }
@@ -848,6 +853,17 @@ export class MaterializationService {
 
    // ==================== HELPERS ====================
 
+   /** The single-active conflict error, with the winning run's id when known. */
+   private activeConflict(
+      packageName: string,
+      activeId?: string,
+   ): MaterializationConflictError {
+      const suffix = activeId ? ` (${activeId})` : "";
+      return new MaterializationConflictError(
+         `Package ${packageName} already has an active materialization${suffix}`,
+      );
+   }
+
    private recordRun(
       mode: MaterializationMode,
       outcome: "success" | "failed" | "cancelled",
@@ -865,7 +881,7 @@ export class MaterializationService {
 
       run(abortController.signal)
          .catch(async (err) => {
-            const message = err instanceof Error ? err.message : String(err);
+            const message = errMessage(err);
             const next = abortController.signal.aborted
                ? "CANCELLED"
                : "FAILED";
@@ -879,10 +895,7 @@ export class MaterializationService {
                logger.error("Failed to record materialization failure", {
                   materializationId: id,
                   originalError: message,
-                  transitionError:
-                     transitionErr instanceof Error
-                        ? transitionErr.message
-                        : String(transitionErr),
+                  transitionError: errMessage(transitionErr),
                });
             }
          })

package/src/service/materialization_test_fixtures.ts

@@ -80,11 +80,13 @@ export function fakeSource(opts: {
    buildId: string;
    sql?: string;
    connectionName?: string;
+   dialectName?: string;
 }): PersistSource {
    return {
       name: opts.name,
       sourceID: opts.name,
       connectionName: opts.connectionName ?? "duckdb",
+      dialectName: opts.dialectName ?? "duckdb",
       makeBuildId: () => opts.buildId,
       getSQL: () => opts.sql ?? "SELECT 1",
       annotations: {

package/src/service/package.ts

@@ -31,9 +31,10 @@ import { formatDuration, logger } from "../logger";
 import { recordBuildPlanComputeDuration } from "../materialization_metrics";
 import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
 import { BuildManifest, BuildPlan } from "../storage/DatabaseInterface";
-import { ignoreDotfiles } from "../utils";
+import { errMessage, ignoreDotfiles } from "../utils";
 import { computePackageBuildPlan } from "./build_plan";
 import { Model } from "./model";
+import { assertPersistNamesQuoted } from "./persist_annotation_validation";
 
 type ApiDatabase = components["schemas"]["Database"];
 type ApiModel = components["schemas"]["Model"];
@@ -361,6 +362,17 @@ export class Package {
          // to load inside the pure-CPU package-load worker). A misconfigured tag
          // is logged as a warning naming the target; it does not fail the load.
          await model.validateRenderTags();
+         // Reject unquoted `#@ persist name=` annotations the same way: an
+         // unquoted name is dropped from the build plan, so the source would
+         // publish but never materialize. Scan the raw `.malloy` source (the
+         // ground truth for quoting); throws a ModelCompilationError (424).
+         if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
+            const modelSource = await fs.readFile(
+               path.join(packagePath, sm.modelPath),
+               "utf-8",
+            );
+            assertPersistNamesQuoted(modelSource, sm.modelPath);
+         }
          models.set(sm.modelPath, model);
       }
 
@@ -400,7 +412,7 @@ export class Package {
             `Failed to compute build plan for package ${packageName}`,
             {
                packageName,
-               error: err instanceof Error ? err.message : String(err),
+               error: errMessage(err),
             },
          );
       }

package/src/service/persist_annotation_validation.spec.ts

@@ -0,0 +1,77 @@
+import { describe, expect, it } from "bun:test";
+import { ModelCompilationError } from "../errors";
+import { assertPersistNamesQuoted } from "./persist_annotation_validation";
+
+describe("assertPersistNamesQuoted", () => {
+   it("throws ModelCompilationError on a bare (unquoted) persist name", () => {
+      expect(() =>
+         assertPersistNamesQuoted(
+            `#@ persist name=engaged_events\nsource: engaged_events is x -> { select: * }`,
+            "m.malloy",
+         ),
+      ).toThrow(ModelCompilationError);
+   });
+
+   it("tolerates whitespace around = but still flags an unquoted value", () => {
+      expect(() =>
+         assertPersistNamesQuoted(
+            `#@ persist name = engaged_events`,
+            "m.malloy",
+         ),
+      ).toThrow(/must be quoted/);
+   });
+
+   it("accepts a double-quoted name", () => {
+      expect(() =>
+         assertPersistNamesQuoted(
+            `#@ persist name="engaged_events"`,
+            "m.malloy",
+         ),
+      ).not.toThrow();
+   });
+
+   it("accepts a single-quoted name", () => {
+      expect(() =>
+         assertPersistNamesQuoted(
+            `#@ persist name='engaged_events'`,
+            "m.malloy",
+         ),
+      ).not.toThrow();
+   });
+
+   it("accepts a dotted, quoted dialect path", () => {
+      expect(() =>
+         assertPersistNamesQuoted(
+            `#@ persist name="my_dataset.engaged_events"`,
+            "m.malloy",
+         ),
+      ).not.toThrow();
+   });
+
+   it("ignores non-persist annotations and persist lines with no name field", () => {
+      expect(() =>
+         assertPersistNamesQuoted(
+            `# number=2\n#@ persist realization="COPY"\nsource: x is y -> { select: * }`,
+            "m.malloy",
+         ),
+      ).not.toThrow();
+   });
+
+   it("does not mistake a neighbouring key for the name field", () => {
+      expect(() =>
+         assertPersistNamesQuoted(
+            `#@ persist tablename=foo name="bar"`,
+            "m.malloy",
+         ),
+      ).not.toThrow();
+   });
+
+   it("reports every offending annotation in the message", () => {
+      expect(() =>
+         assertPersistNamesQuoted(
+            `#@ persist name=a\nsource: a is x -> { select: * }\n#@ persist name=b\nsource: b is x -> { select: * }`,
+            "m.malloy",
+         ),
+      ).toThrow(/name=a.*name=b/s);
+   });
+});

package/src/service/persist_annotation_validation.ts

@@ -0,0 +1,54 @@
+import { ModelCompilationError } from "../errors";
+
+/** A line whose first non-whitespace content is a `#@ persist` directive. */
+const PERSIST_LINE_PATTERN = /^\s*#@\s+persist\b/;
+/**
+ * A `name=` key whose value is NOT immediately opened by a single or double
+ * quote — i.e. a bare `name=engaged_events` (whitespace around `=` tolerated).
+ * `name="..."` and `name='...'` pass. The `\bname` word boundary requires a
+ * standalone key, so neighbours like `tablename=` / `realization_name=` are
+ * never mistaken for it.
+ */
+const UNQUOTED_NAME_PATTERN = /\bname\s*=\s*(?!["'])/;
+
+/**
+ * Reject `#@ persist name=<value>` annotations whose name is unquoted.
+ *
+ * The persist build plan requires a quoted name — a dialect-style table path
+ * (`name="engaged_events"`, or `name="my_dataset.engaged_events"`). A bare
+ * value is dropped from the build plan, so the source publishes and serves but
+ * is never materialized, with no error anywhere. Throwing a
+ * `ModelCompilationError` (HTTP 424) fails the publish/load with a clear,
+ * actionable message — the same hard-stop `Model.validateRenderTags` applies to
+ * a misconfigured render tag.
+ *
+ * Scans the raw model source line-by-line rather than the compiled annotation
+ * objects: the check must fire regardless of how the compiler attaches the
+ * annotation, and the raw text is the ground truth for whether the author
+ * quoted the value (the tag parser discards quote information once parsed).
+ *
+ * @throws {ModelCompilationError} listing every offending annotation.
+ */
+export function assertPersistNamesQuoted(
+   modelSource: string,
+   modelPath: string,
+): void {
+   const offenders: string[] = [];
+   for (const rawLine of modelSource.split("\n")) {
+      if (!PERSIST_LINE_PATTERN.test(rawLine)) continue;
+      if (UNQUOTED_NAME_PATTERN.test(rawLine)) {
+         offenders.push(rawLine.trim());
+      }
+   }
+   if (offenders.length > 0) {
+      throw new ModelCompilationError({
+         message:
+            `${modelPath}: persist annotation name must be quoted. Write a quoted ` +
+            `value like name="engaged_events" (or a dialect table path such as ` +
+            `name="my_dataset.engaged_events"), not a bare value — an unquoted ` +
+            `persist name is dropped from the build plan, so the source would ` +
+            `publish but never materialize. Offending annotation(s): ` +
+            `${offenders.join("; ")}.`,
+      });
+   }
+}

package/src/service/quoting.spec.ts

@@ -0,0 +1,50 @@
+import { describe, expect, it } from "bun:test";
+import { bareTableName, quoteIdentifier, quoteTablePath } from "./quoting";
+
+describe("bareTableName", () => {
+   it("returns the segment after the last dot for a qualified name", () => {
+      expect(bareTableName("my_schema.my_table")).toBe("my_table");
+   });
+
+   it("returns the name unchanged when it is unqualified", () => {
+      expect(bareTableName("my_table")).toBe("my_table");
+   });
+});
+
+describe("quoteIdentifier", () => {
+   it("backticks for backtick dialects (BigQuery / MySQL / Databricks)", () => {
+      expect(quoteIdentifier("my-table", "standardsql")).toBe("`my-table`");
+      expect(quoteIdentifier("t", "mysql")).toBe("`t`");
+      expect(quoteIdentifier("t", "databricks")).toBe("`t`");
+   });
+
+   it("double-quotes for everything else (Postgres / DuckDB / Snowflake)", () => {
+      expect(quoteIdentifier("my-table", "postgres")).toBe('"my-table"');
+      expect(quoteIdentifier("t", "duckdb")).toBe('"t"');
+      expect(quoteIdentifier("t", "snowflake")).toBe('"t"');
+   });
+
+   it("escapes an embedded quote char by doubling it", () => {
+      expect(quoteIdentifier("a`b", "standardsql")).toBe("`a``b`");
+      expect(quoteIdentifier('a"b', "postgres")).toBe('"a""b"');
+   });
+});
+
+describe("quoteTablePath", () => {
+   it("quotes each segment of a container path independently", () => {
+      expect(
+         quoteTablePath(
+            "my-proj.mydataset.engaged_events_ab12_v0",
+            "standardsql",
+         ),
+      ).toBe("`my-proj`.`mydataset`.`engaged_events_ab12_v0`");
+      expect(quoteTablePath("mydataset.t_ab12_v0", "postgres")).toBe(
+         '"mydataset"."t_ab12_v0"',
+      );
+   });
+
+   it("quotes a bare (unqualified) name", () => {
+      expect(quoteTablePath("t_ab12_v0", "standardsql")).toBe("`t_ab12_v0`");
+      expect(quoteTablePath("t_ab12_v0", "postgres")).toBe('"t_ab12_v0"');
+   });
+});

package/src/service/quoting.ts

@@ -1,21 +1,42 @@
 /**
- * Split a possibly schema-qualified table name into its schema prefix
- * (including the trailing dot) and the bare table name.
- *
- * Examples:
- *   "my_schema.my_table" -> { schemaPrefix: "my_schema.", bareName: "my_table" }
- *   "my_table"           -> { schemaPrefix: "", bareName: "my_table" }
+ * The bare (unqualified) name of a possibly container-qualified table path:
+ * the segment after the last dot, e.g. `my_schema.my_table` -> `my_table` and
+ * `my_table` -> `my_table`. Used as the RENAME target, which names a table
+ * within its existing schema rather than re-stating the full path.
  */
-export function splitTablePath(tableName: string): {
-   schemaPrefix: string;
-   bareName: string;
-} {
+export function bareTableName(tableName: string): string {
    const lastDot = tableName.lastIndexOf(".");
-   if (lastDot >= 0) {
-      return {
-         schemaPrefix: tableName.substring(0, lastDot + 1),
-         bareName: tableName.substring(lastDot + 1),
-      };
+   return lastDot >= 0 ? tableName.substring(lastDot + 1) : tableName;
+}
+
+// Dialects whose identifier quote character is a backtick; everything else uses
+// the SQL-standard double quote. Keyed by Malloy `dialectName`.
+const BACKTICK_DIALECTS = new Set(["standardsql", "mysql", "databricks"]);
+
+/**
+ * Quote a single SQL identifier for {@code dialect}, escaping any embedded quote
+ * character by doubling it.
+ */
+export function quoteIdentifier(identifier: string, dialect: string): string {
+   if (BACKTICK_DIALECTS.has(dialect)) {
+      return "`" + identifier.replace(/`/g, "``") + "`";
    }
-   return { schemaPrefix: "", bareName: tableName };
+   return '"' + identifier.replace(/"/g, '""') + '"';
+}
+
+/**
+ * Dialect-quote a (possibly container-qualified) table path so it can be inlined
+ * into DDL. Each dot segment is quoted independently and rejoined with dots, so
+ * a path like {@code my-proj.mydataset.engaged_events_v0} becomes
+ * `` `my-proj`.`mydataset`.`engaged_events_v0` `` on BigQuery or
+ * {@code "my-proj"."mydataset"."engaged_events_v0"} on Postgres — handling
+ * container hierarchies and quote-requiring names (e.g. hyphenated BigQuery
+ * project ids) uniformly. The control plane provides the logical (unquoted) path
+ * in `physicalTableName`; quoting for the warehouse is the publisher's job.
+ */
+export function quoteTablePath(tableName: string, dialect: string): string {
+   return tableName
+      .split(".")
+      .map((segment) => quoteIdentifier(segment, dialect))
+      .join(".");
 }

package/src/utils.ts

@@ -22,3 +22,8 @@ export const URL_READER: URLReader = {
 export function ignoreDotfiles(file: string): boolean {
    return path.basename(file).startsWith(".");
 }
+
+/** The message of a thrown value, stringifying non-Error throws. */
+export function errMessage(err: unknown): string {
+   return err instanceof Error ? err.message : String(err);
+}

package/dist/app/assets/HomePage-DQzgkiMI.js

@@ -1 +0,0 @@
-import{$ as s,j as t,s as o}from"./index-DuqTjxM_.js";function e(){const n=s();return t.jsx(o,{onClickEnvironment:n})}export{e as default};