@malloy-publisher/server 0.0.210 → 0.0.211

package/dist/server.mjs

@@ -237699,97 +237699,52 @@ import { pathToFileURL as pathToFileURL2 } from "url";
 
 // src/materialization_metrics.ts
 var import_api4 = __toESM(require_src(), 1);
-var runCounter = null;
-var runDuration = null;
-var sourcesCounter = null;
-var buildPlanComputeDuration = null;
-var autoLoadCounter = null;
-var connectionDigestSkipCounter = null;
-var manifestBindCounter = null;
-var sourceBuildDuration = null;
-var dropTablesCounter = null;
-function ensureTelemetry() {
-  if (runCounter && runDuration) {
-    return { counter: runCounter, duration: runDuration };
-  }
-  const meter2 = import_api4.metrics.getMeter("publisher");
-  if (!runCounter) {
-    runCounter = meter2.createCounter("publisher_materialization_runs_total", {
-      description: "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled')."
-    });
-  }
-  if (!runDuration) {
-    runDuration = meter2.createHistogram("publisher_materialization_run_duration_ms", {
-      description: "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').",
-      unit: "ms"
-    });
-  }
-  return { counter: runCounter, duration: runDuration };
-}
+var resetHooks = [];
+function lazyCounter(name, description) {
+  let instrument2 = null;
+  resetHooks.push(() => instrument2 = null);
+  return () => instrument2 ??= import_api4.metrics.getMeter("publisher").createCounter(name, { description });
+}
+function lazyHistogram(name, description, unit) {
+  let instrument2 = null;
+  resetHooks.push(() => instrument2 = null);
+  return () => instrument2 ??= import_api4.metrics.getMeter("publisher").createHistogram(name, { description, unit });
+}
+var runCounter = lazyCounter("publisher_materialization_runs_total", "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').");
+var runDuration = lazyHistogram("publisher_materialization_run_duration_ms", "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').", "ms");
+var sourcesCounter = lazyCounter("publisher_materialization_sources_total", "Persist sources processed by a materialization run. Label: outcome ('built'|'reused').");
+var buildPlanComputeDuration = lazyHistogram("publisher_materialization_build_plan_compute_duration_ms", "Wall-clock duration of compiling a package's build plan (Package.buildPlan).", "ms");
+var autoLoadCounter = lazyCounter("publisher_materialization_auto_load_total", "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').");
+var connectionDigestSkipCounter = lazyCounter("publisher_materialization_connection_digest_skipped_total", "Connection digests skipped during build-plan compile because the connection did not resolve.");
+var manifestBindCounter = lazyCounter("publisher_materialization_manifest_bind_total", "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').");
+var sourceBuildDuration = lazyHistogram("publisher_materialization_source_build_duration_ms", "Wall-clock duration of building a single persist source.", "ms");
+var dropTablesCounter = lazyCounter("publisher_materialization_drop_tables_total", "Physical tables dropped on delete. Label: outcome ('success'|'failure').");
 function recordMaterializationRun(mode, outcome, durationMs) {
-  const { counter, duration } = ensureTelemetry();
-  counter.add(1, { mode, outcome });
-  duration.record(durationMs, { mode });
+  runCounter().add(1, { mode, outcome });
+  runDuration().record(durationMs, { mode });
 }
 function recordSourcesOutcome(outcome, count) {
   if (count <= 0)
     return;
-  if (!sourcesCounter) {
-    sourcesCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_sources_total", {
-      description: "Persist sources processed by a materialization run. Label: outcome ('built'|'reused')."
-    });
-  }
-  sourcesCounter.add(count, { outcome });
+  sourcesCounter().add(count, { outcome });
 }
 function recordBuildPlanComputeDuration(durationMs) {
-  if (!buildPlanComputeDuration) {
-    buildPlanComputeDuration = import_api4.metrics.getMeter("publisher").createHistogram("publisher_materialization_build_plan_compute_duration_ms", {
-      description: "Wall-clock duration of compiling a package's build plan (Package.buildPlan).",
-      unit: "ms"
-    });
-  }
-  buildPlanComputeDuration.record(durationMs);
+  buildPlanComputeDuration().record(durationMs);
 }
 function recordAutoLoadOutcome(outcome) {
-  if (!autoLoadCounter) {
-    autoLoadCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_auto_load_total", {
-      description: "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure')."
-    });
-  }
-  autoLoadCounter.add(1, { outcome });
+  autoLoadCounter().add(1, { outcome });
 }
 function recordConnectionDigestSkipped() {
-  if (!connectionDigestSkipCounter) {
-    connectionDigestSkipCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_connection_digest_skipped_total", {
-      description: "Connection digests skipped during build-plan compile because the connection did not resolve."
-    });
-  }
-  connectionDigestSkipCounter.add(1);
+  connectionDigestSkipCounter().add(1);
 }
 function recordManifestBind(outcome) {
-  if (!manifestBindCounter) {
-    manifestBindCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_manifest_bind_total", {
-      description: "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout')."
-    });
-  }
-  manifestBindCounter.add(1, { outcome });
+  manifestBindCounter().add(1, { outcome });
 }
 function recordSourceBuildDuration(durationMs) {
-  if (!sourceBuildDuration) {
-    sourceBuildDuration = import_api4.metrics.getMeter("publisher").createHistogram("publisher_materialization_source_build_duration_ms", {
-      description: "Wall-clock duration of building a single persist source.",
-      unit: "ms"
-    });
-  }
-  sourceBuildDuration.record(durationMs);
+  sourceBuildDuration().record(durationMs);
 }
 function recordDropTables(outcome) {
-  if (!dropTablesCounter) {
-    dropTablesCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_drop_tables_total", {
-      description: "Physical tables dropped on delete. Label: outcome ('success'|'failure')."
-    });
-  }
-  dropTablesCounter.add(1, { outcome });
+  dropTablesCounter().add(1, { outcome });
 }
 
 // src/utils.ts
@@ -237808,6 +237763,9 @@ var URL_READER = {
 function ignoreDotfiles(file) {
   return path5.basename(file).startsWith(".");
 }
+function errMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
 
 // src/service/manifest_loader.ts
 init_logger();
@@ -237889,6 +237847,7 @@ import {
 } from "@malloydata/malloy";
 
 // src/service/build_plan.ts
+init_constants();
 init_logger();
 
 // src/service/model.ts
@@ -239307,14 +239266,35 @@ function deriveColumns(persistSource) {
   } catch (err) {
     logger.warn("Failed to derive columns for persist source", {
       sourceID: persistSource.sourceID,
-      error: err instanceof Error ? err.message : String(err)
+      error: errMessage(err)
     });
     return [];
   }
 }
+function deriveAnnotationFields(persistSource) {
+  const out = {};
+  try {
+    const tag = persistSource.annotations.parseAsTag("@").tag;
+    for (const [key, value] of tag.entries()) {
+      const text = value.text();
+      if (text !== undefined)
+        out[key] = text;
+    }
+  } catch {}
+  return out;
+}
 function flattenDependsOn(node) {
   return node.dependsOn.map((d) => d.sourceID);
 }
+function* iterGraphSources(graph, sources) {
+  for (const level of graph.nodes) {
+    for (const node of level) {
+      const source = sources[node.sourceID];
+      if (source)
+        yield source;
+    }
+  }
+}
 function computeBuildId(source, connectionDigests) {
   return source.makeBuildId(connectionDigests[source.connectionName], source.getSQL());
 }
@@ -239329,7 +239309,7 @@ async function resolvePackageConnections(pkg, names) {
       map.set(name, await pkg.getMalloyConnection(name));
     } catch (err) {
       logger.warn(`Failed to resolve connection ${name}`, {
-        error: err instanceof Error ? err.message : String(err)
+        error: errMessage(err)
       });
     }
   }
@@ -239339,6 +239319,8 @@ async function compilePackageBuildPlan(pkg, signal) {
   const allGraphs = [];
   const allSources = {};
   for (const modelPath of pkg.getModelPaths()) {
+    if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
+      continue;
     if (signal?.aborted)
       throw new Error("Build cancelled");
     const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(pkg.getPackagePath(), modelPath, pkg.getMalloyConfig());
@@ -239400,7 +239382,8 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames) {
       dialect: source.dialectName,
       buildId: computeBuildId(source, connectionDigests),
       sql: source.getSQL(),
-      columns: deriveColumns(source)
+      columns: deriveColumns(source),
+      annotationFields: deriveAnnotationFields(source)
     };
   }
   return { graphs: wireGraphs, sources: wireSources };
@@ -239413,6 +239396,27 @@ async function computePackageBuildPlan(pkg, signal) {
   return deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests);
 }
 
+// src/service/persist_annotation_validation.ts
+init_errors();
+var PERSIST_LINE_PATTERN = /^\s*#@\s+persist\b/;
+var UNQUOTED_NAME_PATTERN = /\bname\s*=\s*(?!["'])/;
+function assertPersistNamesQuoted(modelSource, modelPath) {
+  const offenders = [];
+  for (const rawLine of modelSource.split(`
+`)) {
+    if (!PERSIST_LINE_PATTERN.test(rawLine))
+      continue;
+    if (UNQUOTED_NAME_PATTERN.test(rawLine)) {
+      offenders.push(rawLine.trim());
+    }
+  }
+  if (offenders.length > 0) {
+    throw new ModelCompilationError({
+      message: `${modelPath}: persist annotation name must be quoted. Write a quoted ` + `value like name="engaged_events" (or a dialect table path such as ` + `name="my_dataset.engaged_events"), not a bare value — an unquoted ` + `persist name is dropped from the build plan, so the source would ` + `publish but never materialize. Offending annotation(s): ` + `${offenders.join("; ")}.`
+    });
+  }
+}
+
 // src/service/package.ts
 class Package {
   environmentName;
@@ -239556,6 +239560,10 @@ class Package {
       }
       const model = Model.fromSerialized(packageName, packagePath, malloyConfig, sm);
       await model.validateRenderTags();
+      if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
+        const modelSource = await fs6.readFile(path7.join(packagePath, sm.modelPath), "utf-8");
+        assertPersistNamesQuoted(modelSource, sm.modelPath);
+      }
       models.set(sm.modelPath, model);
     }
     const endTime = performance.now();
@@ -239576,7 +239584,7 @@ class Package {
     } catch (err) {
       logger.warn(`Failed to compute build plan for package ${packageName}`, {
         packageName,
-        error: err instanceof Error ? err.message : String(err)
+        error: errMessage(err)
       });
     }
     const invalidMsg = pkg.formatInvalidExplores();
@@ -248569,15 +248577,19 @@ init_logger();
 import { Manifest } from "@malloydata/malloy";
 
 // src/service/quoting.ts
-function splitTablePath(tableName) {
+function bareTableName(tableName) {
   const lastDot = tableName.lastIndexOf(".");
-  if (lastDot >= 0) {
-    return {
-      schemaPrefix: tableName.substring(0, lastDot + 1),
-      bareName: tableName.substring(lastDot + 1)
-    };
+  return lastDot >= 0 ? tableName.substring(lastDot + 1) : tableName;
+}
+var BACKTICK_DIALECTS = new Set(["standardsql", "mysql", "databricks"]);
+function quoteIdentifier(identifier, dialect) {
+  if (BACKTICK_DIALECTS.has(dialect)) {
+    return "`" + identifier.replace(/`/g, "``") + "`";
   }
-  return { schemaPrefix: "", bareName: tableName };
+  return '"' + identifier.replace(/"/g, '""') + '"';
+}
+function quoteTablePath(tableName, dialect) {
+  return tableName.split(".").map((segment) => quoteIdentifier(segment, dialect)).join(".");
 }
 
 // src/service/resolve_environment.ts
@@ -248596,11 +248608,7 @@ function stagingSuffix(buildId) {
   return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
 }
 function selfAssignTableName(persistSource) {
-  try {
-    return persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
-  } catch {
-    return persistSource.name;
-  }
+  return deriveAnnotationFields(persistSource).name || persistSource.name;
 }
 function outcomeFor(_err, signal) {
   return signal?.aborted ? "cancelled" : "failed";
@@ -248668,7 +248676,7 @@ class MaterializationService {
     }
     const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
     if (active2) {
-      throw new MaterializationConflictError(`Package ${packageName} already has an active materialization (${active2.id})`);
+      throw this.activeConflict(packageName, active2.id);
     }
     const forceRefresh = options.forceRefresh ?? false;
     const metadata = {
@@ -248682,7 +248690,7 @@ class MaterializationService {
     } catch (err) {
       if (err instanceof DuplicateActiveMaterializationError) {
         const winner = await this.repository.getActiveMaterialization(environmentId, packageName);
-        throw new MaterializationConflictError(winner ? `Package ${packageName} already has an active materialization (${winner.id})` : `Package ${packageName} already has an active materialization`);
+        throw this.activeConflict(packageName, winner?.id);
       }
       throw err;
     }
@@ -248703,6 +248711,9 @@ class MaterializationService {
     });
     const startedAt = Date.now();
     try {
+      await this.repository.updateMaterialization(id, {
+        startedAt: new Date(startedAt)
+      });
       const environmentId = await this.resolveEnvironmentId(environmentName);
       const environment = await this.environmentStore.getEnvironment(environmentName, false);
       const pkg = await environment.getPackage(packageName, false);
@@ -248753,29 +248764,24 @@ class MaterializationService {
     const carried = {};
     const seen = new Set;
     for (const graph of compiled.graphs) {
-      for (const level of graph.nodes) {
-        for (const node of level) {
-          const persistSource = compiled.sources[node.sourceID];
-          if (!persistSource)
-            continue;
-          if (include && !include.has(persistSource.name))
-            continue;
-          const buildId = computeBuildId(persistSource, compiled.connectionDigests);
-          if (seen.has(buildId))
-            continue;
-          seen.add(buildId);
-          const prior = priorEntries[buildId];
-          if (prior && prior.physicalTableName) {
-            carried[buildId] = prior;
-            continue;
-          }
-          instructions.push({
-            buildId,
-            materializedTableId: `local-${buildId.substring(0, STAGING_BUILD_ID_LEN)}`,
-            physicalTableName: selfAssignTableName(persistSource),
-            realization: "COPY"
-          });
+      for (const persistSource of iterGraphSources(graph, compiled.sources)) {
+        if (include && !include.has(persistSource.name))
+          continue;
+        const buildId = computeBuildId(persistSource, compiled.connectionDigests);
+        if (seen.has(buildId))
+          continue;
+        seen.add(buildId);
+        const prior = priorEntries[buildId];
+        if (prior && prior.physicalTableName) {
+          carried[buildId] = prior;
+          continue;
         }
+        instructions.push({
+          buildId,
+          materializedTableId: `local-${buildId.substring(0, STAGING_BUILD_ID_LEN)}`,
+          physicalTableName: selfAssignTableName(persistSource),
+          realization: "COPY"
+        });
       }
     }
     return { instructions, carried };
@@ -248849,20 +248855,15 @@ class MaterializationService {
       if (!connection) {
         throw new BadRequestError(`Connection '${graph.connectionName}' not found`);
       }
-      for (const level of graph.nodes) {
-        for (const node of level) {
-          if (signal.aborted)
-            throw new Error("Build cancelled");
-          const persistSource = sources[node.sourceID];
-          if (!persistSource)
-            continue;
-          const buildId = computeBuildId(persistSource, connectionDigests);
-          const instruction = byBuildId.get(buildId);
-          if (!instruction)
-            continue;
-          const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest);
-          entries[buildId] = entry;
-        }
+      for (const persistSource of iterGraphSources(graph, sources)) {
+        if (signal.aborted)
+          throw new Error("Build cancelled");
+        const buildId = computeBuildId(persistSource, connectionDigests);
+        const instruction = byBuildId.get(buildId);
+        if (!instruction)
+          continue;
+        const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest);
+        entries[buildId] = entry;
       }
     }
     return entries;
@@ -248874,22 +248875,26 @@ class MaterializationService {
       buildManifest: manifest.buildManifest,
       connectionDigests
     });
-    const { bareName } = splitTablePath(physicalTableName);
+    const bareName = bareTableName(physicalTableName);
     const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
+    const dialect = persistSource.dialectName;
+    const quotedStaging = quoteTablePath(stagingTableName, dialect);
+    const quotedPhysical = quoteTablePath(physicalTableName, dialect);
+    const quotedBareName = quoteIdentifier(bareName, dialect);
     const startTime = performance.now();
-    await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
+    await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`);
     try {
-      await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
-      await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
-      await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
+      await connection.runSQL(`CREATE TABLE ${quotedStaging} AS (${buildSQL})`);
+      await connection.runSQL(`DROP TABLE IF EXISTS ${quotedPhysical}`);
+      await connection.runSQL(`ALTER TABLE ${quotedStaging} RENAME TO ${quotedBareName}`);
     } catch (err) {
       try {
-        await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
+        await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`);
       } catch (cleanupErr) {
         logger.warn("Failed to clean up staging table after a failed build; physical leak", {
           stagingTableName,
           connectionName: persistSource.connectionName,
-          cleanupError: cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)
+          cleanupError: errMessage(cleanupErr)
         });
       }
       throw err;
@@ -248969,8 +248974,9 @@ class MaterializationService {
           connection = await pkg.getMalloyConnection(connectionName);
           connectionCache.set(connectionName, connection);
         }
-        await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}`);
-        await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName}${stagingSuffix(entry.buildId)}`);
+        const dialect = connection.dialectName;
+        await connection.runSQL(`DROP TABLE IF EXISTS ${quoteTablePath(physicalTableName, dialect)}`);
+        await connection.runSQL(`DROP TABLE IF EXISTS ${quoteTablePath(`${physicalTableName}${stagingSuffix(entry.buildId)}`, dialect)}`);
         recordDropTables("success");
         logger.info("Dropped materialized table on delete", {
           materializationId: m.id,
@@ -248983,7 +248989,7 @@ class MaterializationService {
           materializationId: m.id,
           physicalTableName,
           connectionName,
-          error: err instanceof Error ? err.message : String(err)
+          error: errMessage(err)
         });
       }
     }
@@ -249001,6 +249007,10 @@ class MaterializationService {
       metadata
     });
   }
+  activeConflict(packageName, activeId) {
+    const suffix = activeId ? ` (${activeId})` : "";
+    return new MaterializationConflictError(`Package ${packageName} already has an active materialization${suffix}`);
+  }
   recordRun(mode, outcome, startedAtMs) {
     recordMaterializationRun(mode, outcome, Date.now() - startedAtMs);
   }
@@ -249008,7 +249018,7 @@ class MaterializationService {
     const abortController = new AbortController;
     this.runningAbortControllers.set(id, abortController);
     run(abortController.signal).catch(async (err) => {
-      const message = err instanceof Error ? err.message : String(err);
+      const message = errMessage(err);
       const next = abortController.signal.aborted ? "CANCELLED" : "FAILED";
       try {
         await this.repository.updateMaterialization(id, {
@@ -249020,7 +249030,7 @@ class MaterializationService {
         logger.error("Failed to record materialization failure", {
           materializationId: id,
           originalError: message,
-          transitionError: transitionErr instanceof Error ? transitionErr.message : String(transitionErr)
+          transitionError: errMessage(transitionErr)
         });
       }
     }).finally(() => {

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.210",
+   "version": "0.0.211",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"

package/src/materialization_metrics.spec.ts

@@ -0,0 +1,84 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+
+import {
+   recordConnectionDigestSkipped,
+   recordDropTables,
+   recordMaterializationRun,
+   recordSourcesOutcome,
+   resetMaterializationTelemetryForTesting,
+} from "./materialization_metrics";
+import {
+   startMetricsHarness,
+   type MetricsHarness,
+} from "./test_helpers/metrics_harness";
+
+describe("materialization_metrics", () => {
+   let harness: MetricsHarness;
+
+   beforeEach(async () => {
+      harness = await startMetricsHarness();
+      // Drop cached instruments so they re-bind to this test's provider.
+      resetMaterializationTelemetryForTesting();
+   });
+
+   afterEach(async () => {
+      resetMaterializationTelemetryForTesting();
+      await harness.shutdown();
+   });
+
+   it("counts runs labeled by mode and outcome", async () => {
+      recordMaterializationRun("auto", "success", 10);
+      recordMaterializationRun("auto", "success", 20);
+      recordMaterializationRun("orchestrated", "failed", 5);
+
+      expect(
+         await harness.collectCounter("publisher_materialization_runs_total", {
+            mode: "auto",
+            outcome: "success",
+         }),
+      ).toBe(2);
+      expect(
+         await harness.collectCounter("publisher_materialization_runs_total", {
+            mode: "orchestrated",
+            outcome: "failed",
+         }),
+      ).toBe(1);
+   });
+
+   it("adds the source count labeled by outcome, ignoring non-positive counts", async () => {
+      recordSourcesOutcome("built", 3);
+      recordSourcesOutcome("reused", 2);
+      recordSourcesOutcome("built", 0); // guarded: no emission
+
+      expect(
+         await harness.collectCounter(
+            "publisher_materialization_sources_total",
+            { outcome: "built" },
+         ),
+      ).toBe(3);
+      expect(
+         await harness.collectCounter(
+            "publisher_materialization_sources_total",
+            { outcome: "reused" },
+         ),
+      ).toBe(2);
+   });
+
+   it("counts drop-table outcomes and connection-digest skips", async () => {
+      recordDropTables("success");
+      recordDropTables("failure");
+      recordConnectionDigestSkipped();
+
+      expect(
+         await harness.collectCounter(
+            "publisher_materialization_drop_tables_total",
+            { outcome: "failure" },
+         ),
+      ).toBe(1);
+      expect(
+         await harness.collectCounter(
+            "publisher_materialization_connection_digest_skipped_total",
+         ),
+      ).toBe(1);
+   });
+});