@malloy-publisher/server 0.0.243 → 0.0.244

package/dist/server.mjs

@@ -269386,7 +269386,10 @@ class QueryController {
         serializedResult,
         rowLimit,
         rowLimitSource,
-        queryCorrelationId
+        queryCorrelationId,
+        servedFrom,
+        executionTimeMs,
+        queryCostBytes
       } = await runWithQueryTimeout((abortSignal) => model.getQueryResults(sourceName, queryName, query, filterParams, bypassFilters, givens, abortSignal, {
         request: requestMetadata,
         queryClass,
@@ -269413,7 +269416,10 @@ class QueryController {
         renderLogs: renderLogs.length > 0 ? renderLogs : undefined,
         queryRowLimit: rowLimit,
         queryRowLimitSource: rowLimitSource,
-        queryCorrelationId
+        queryCorrelationId,
+        servedFrom,
+        executionTimeMs,
+        queryCostBytes
       };
     }
   }
@@ -277262,7 +277268,8 @@ var sourceBuildDuration = lazyHistogram("publisher_materialization_source_build_
 var dropTablesCounter = lazyCounter2("publisher_materialization_drop_tables_total", "Physical tables dropped on delete. Label: outcome ('success'|'failure').");
 var scheduledFireCounter = lazyCounter2("publisher_materialization_scheduled_fires_total", "Standalone-scheduler attempts to fire a package's materialization.schedule. " + "Label: outcome ('fired'|'conflict'|'error').");
 var storageServeRoutingCounter = lazyCounter2("publisher_storage_serve_routing_total", "storage= serve routing decisions. Label: outcome ('storage'|'live_fallback'|'runtime_live_fallback').");
-var storageBuildFailureCounter = lazyCounter2("publisher_storage_build_failures_total", "storage= build failures (federation/passthrough/attach/CTAS), distinct from " + "in-warehouse build failures. Label: destination (connection name).");
+var storageBuildFailureCounter = lazyCounter2("publisher_storage_build_failures_total", "storage= build failures (federation/passthrough/attach/CTAS), distinct from " + "in-warehouse build failures. Labels: destination (connection name), " + "reason ('build_failed'|'billed_read_not_captured'). The second is the " + "expensive one: the warehouse read ran and was charged, and the rows could " + "not be captured — so a re-drive pays for it again. Worth alerting on " + "separately from a failure that costs only a retry.");
+var attributionSkippedCounter = lazyCounter2("publisher_storage_build_attribution_skipped_total", "storage= builds whose warehouse read went out UNATTRIBUTED while tagging was " + "on. Label: reason ('job_listing_unavailable'|'tag_failed'|" + "'read_row_not_found'|'read_row_ambiguous'|'cost_query_failed'). " + "The read still ran and the " + "build still succeeded — what was lost is the label in the customer's own " + "query history, and the cost on this side. Without this an operator who " + "turns tagging on and sees nothing has a single log line to go on.");
 var eligibilityRefusedCounter = lazyCounter2("publisher_materialization_eligibility_refused_total", "storage= materialization-eligibility refusals. Label: reason " + "('free_parameter'|'given'|'authorize'|'not_duckdb_portable'|" + "'public_surface_unknown').");
 var serveShapeTierDropCounter = lazyCounter2("publisher_storage_serve_shape_tier_drop_total", "storage serve-shape compile escalations: a refinement tier failed to " + "compile and the riskiest category was dropped. Label: tier (the failed " + "tier index, 0=full).");
 var serveShapeTypeFallbackCounter = lazyCounter2("publisher_storage_serve_shape_type_fallback_total", "Captured DuckDB column types mapped to json in the serve shape (type " + "fidelity loss). Label: kind ('array'|'unrecognized').");
@@ -277303,8 +277310,11 @@ function recordSourceBuildDuration(durationMs, engine) {
 function recordDropTables(outcome, engine) {
   dropTablesCounter().add(1, { outcome, engine });
 }
-function recordStorageBuildFailure(destination) {
-  storageBuildFailureCounter().add(1, { destination });
+function recordStorageBuildFailure(destination, reason = "build_failed") {
+  storageBuildFailureCounter().add(1, { destination, reason });
+}
+function recordAttributionSkipped(reason) {
+  attributionSkippedCounter().add(1, { reason });
 }
 function recordEligibilityRefused(reason) {
   eligibilityRefusedCounter().add(1, { reason });
@@ -278685,6 +278695,10 @@ class Model {
     description: "How long it takes to execute a Malloy model query",
     unit: "ms"
   });
+  queryScannedBytesCounter = this.meter.createCounter("malloy_model_query_scanned_bytes", {
+    description: "Warehouse bytes scanned by Malloy model queries, where the backend reports them. NOT bytes billed: BigQuery bills a 10MB minimum per query, so spend exceeds this on small reads.",
+    unit: "By"
+  });
   constructor(packageName, modelPath, dataStyles, modelType, modelMaterializer, modelDef, sources, queries, sourceInfos, runnableNotebookCells, compilationError, filterMap, givens, modelInfo) {
     this.packageName = packageName;
     this.modelPath = modelPath;
@@ -279460,13 +279474,12 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
       } else {
         const endTime = performance.now();
         const executionTime2 = endTime - startTime;
-        this.queryExecutionHistogram.record(executionTime2, {
-          "malloy.model.path": this.modelPath,
-          "malloy.model.query.name": queryName,
-          "malloy.model.query.source": sourceName,
-          "malloy.model.query.query": query,
-          "malloy.model.query.status": "error"
-        });
+        this.queryExecutionHistogram.record(executionTime2, this.queryMetricAttributes({
+          environment: queryMetadataInput?.environment,
+          queryName,
+          sourceName,
+          status: "error"
+        }));
         throw new BadRequestError("Invalid query request. (Query AND !sourceName) OR (queryName AND sourceName) must be defined.");
       }
       const isAdHocQuery = !sourceName && !queryName && !!query;
@@ -279553,7 +279566,6 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
         maxRows
       });
       appliedQueryMetadata = this.resolveQueryMetadata(queryMetadataInput, preparedResult.connectionName);
-      executionTime = performance.now() - startTime;
       queryResults = await runnable.run({
         rowLimit,
         givens: effectiveGivens,
@@ -279562,19 +279574,19 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
         virtualMap: serveVirtualMap,
         queryMetadata: appliedQueryMetadata
       });
+      executionTime = performance.now() - startTime;
     } catch (error) {
       const canDegradeToLive = !!serveVirtualMap && !!liveRunnable && !abortSignal?.aborted && !String(error?.code ?? "").startsWith("runtime-given-") && serveShapeBindings.length > 0 && serveShapeBindings.every((b) => b.freshnessFallback === "live");
       const failQuery = (err) => {
         const errorEndTime = performance.now();
         const errorExecutionTime = errorEndTime - startTime;
-        this.queryExecutionHistogram.record(errorExecutionTime, {
-          "malloy.model.path": this.modelPath,
-          "malloy.model.query.name": queryName,
-          "malloy.model.query.source": sourceName,
-          "malloy.model.query.query": query,
-          "malloy.model.query.status": "error",
-          ...servedFrom ? { "malloy.model.query.served_from": servedFrom } : {}
-        });
+        this.queryExecutionHistogram.record(errorExecutionTime, this.queryMetricAttributes({
+          environment: queryMetadataInput?.environment,
+          queryName,
+          sourceName,
+          status: "error",
+          servedFrom
+        }));
         const givenCode = err?.code;
         if (typeof givenCode === "string" && givenCode.startsWith("runtime-given-")) {
           logger.debug("Rejected client-supplied given", {
@@ -279635,17 +279647,20 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
     const wrappedResult = API.util.wrapResult(queryResults);
     const serializedResult = stringifyQueryResponse(responseShape === "compact" ? queryResults.data.value : wrappedResult, queryResults.totalRows, maxBytes, "model_query", responseShape === "compact" ? bigIntReplacer : undefined);
     assertWithinModelByteLimit(serializedResult, maxBytes, "model_query");
-    this.queryExecutionHistogram.record(executionTime, {
-      "malloy.model.path": this.modelPath,
-      "malloy.model.query.name": queryName,
-      "malloy.model.query.source": sourceName,
-      "malloy.model.query.query": query,
-      "malloy.model.query.rows_limit": rowLimit,
-      "malloy.model.query.rows_total": queryResults.totalRows,
-      "malloy.model.query.connection": queryResults.connectionName,
-      "malloy.model.query.status": "success",
-      ...servedFrom ? { "malloy.model.query.served_from": servedFrom } : {}
+    const metricAttributes = this.queryMetricAttributes({
+      environment: queryMetadataInput?.environment,
+      queryName,
+      sourceName,
+      status: "success",
+      connection: queryResults.connectionName,
+      servedFrom,
+      rowsLimit: rowLimit
     });
+    this.queryExecutionHistogram.record(executionTime, metricAttributes);
+    const queryCostBytes = queryResults.runStats?.queryCostBytes;
+    if (queryCostBytes !== undefined) {
+      this.queryScannedBytesCounter.add(queryCostBytes, metricAttributes);
+    }
     return {
       result: wrappedResult,
       serializedResult,
@@ -279654,7 +279669,23 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
       dataStyles: this.dataStyles,
       rowLimit,
       rowLimitSource,
-      queryCorrelationId: appliedQueryMetadata?.query_id ?? null
+      queryCorrelationId: appliedQueryMetadata?.query_id ?? null,
+      servedFrom: servedFrom ?? null,
+      executionTimeMs: Math.round(executionTime),
+      queryCostBytes: queryCostBytes ?? null
+    };
+  }
+  queryMetricAttributes(args) {
+    return {
+      "malloy.model.path": this.modelPath,
+      "malloy.package": this.packageName,
+      "malloy.model.query.name": args.queryName,
+      "malloy.model.query.source": args.sourceName,
+      "malloy.model.query.status": args.status,
+      ...args.environment ? { "malloy.environment": args.environment } : {},
+      ...args.connection ? { "malloy.model.query.connection": args.connection } : {},
+      ...args.rowsLimit === undefined ? {} : { "malloy.model.query.rows_limit": args.rowsLimit },
+      ...args.servedFrom ? { "malloy.model.query.served_from": args.servedFrom } : {}
     };
   }
   resolveQueryMetadata(input, connectionName) {
@@ -293040,6 +293071,130 @@ import {
 import { mkdirSync as mkdirSync2, mkdtempSync, rmSync } from "node:fs";
 import os4 from "node:os";
 import path13 from "node:path";
+
+// src/service/build_query_tag.ts
+var MAX_QUERY_TAG_LENGTH = 2000;
+var BQ_MAX_LEN = 63;
+function snowflakeQueryTagValue(metadata) {
+  if (metadata === undefined || Object.keys(metadata).length === 0) {
+    return;
+  }
+  const tag = JSON.stringify(metadata);
+  return tag.length > MAX_QUERY_TAG_LENGTH ? undefined : tag;
+}
+function snowflakeSetQueryTagSQL(metadata, secretName) {
+  const tag = snowflakeQueryTagValue(metadata);
+  if (tag === undefined)
+    return;
+  const inner = `ALTER SESSION SET QUERY_TAG = '${sqlLiteral(tag, "snowflake")}'`;
+  return `SELECT * FROM snowflake_query('${inner.replace(/'/g, "''")}', '${secretName.replace(/'/g, "''")}')`;
+}
+function sanitizeBigQueryValue(value) {
+  return value.toLowerCase().replace(/[^a-z0-9_-]/g, "_").slice(0, BQ_MAX_LEN);
+}
+function sanitizeBigQueryKey(key) {
+  const sanitized = sanitizeBigQueryValue(key);
+  return /^[a-z]/.test(sanitized) ? sanitized : undefined;
+}
+function bigQueryQueryLabelValue(metadata) {
+  if (metadata === undefined)
+    return;
+  const rendered = new Map;
+  for (const [key, value] of Object.entries(metadata)) {
+    const sanitizedKey = sanitizeBigQueryKey(key);
+    if (sanitizedKey === undefined)
+      continue;
+    rendered.set(sanitizedKey, sanitizeBigQueryValue(value));
+  }
+  if (rendered.size === 0)
+    return;
+  return [...rendered].map(([key, value]) => `${key}:${value}`).join(",");
+}
+
+// src/service/build_read_cost.ts
+var SNOWFLAKE_HISTORY_LIMIT = 1e4;
+var BIGQUERY_COST_COLUMNS = `
+   total_slot_time_ms,
+   json_extract_string(statistics, '$.query.totalBytesProcessed') AS bytes_processed,
+   json_extract_string(statistics, '$.query.totalBytesBilled')    AS bytes_billed,
+   json_extract_string(statistics, '$.query.cacheHit')            AS cache_hit`;
+function bigQueryReadCost(row, jobId, parentJobId) {
+  return {
+    engine: "bigquery",
+    jobId,
+    parentJobId,
+    bytesScanned: num(row.bytes_processed),
+    bytesBilled: num(row.bytes_billed),
+    slotTimeMs: num(row.total_slot_time_ms),
+    executionTimeMs: null,
+    cacheHit: bool(row.cache_hit)
+  };
+}
+function snowflakeDatabaseQualifier(database) {
+  return typeof database === "string" && /^[A-Za-z_][A-Za-z0-9_$]*$/.test(database) ? database : "SNOWFLAKE";
+}
+function snowflakeCostSQL(queryTag, database) {
+  const qualifier = `${snowflakeDatabaseQualifier(database)}.INFORMATION_SCHEMA`;
+  return `
+      SELECT QUERY_ID       AS "job_id",
+             QUERY_TEXT     AS "query_text",
+             BYTES_SCANNED  AS "bytes_scanned",
+             EXECUTION_TIME AS "execution_time_ms",
+             ROWS_PRODUCED  AS "rows_produced"
+      FROM TABLE(${qualifier}.QUERY_HISTORY_BY_SESSION(
+                 RESULT_LIMIT => ${SNOWFLAKE_HISTORY_LIMIT}))
+      WHERE QUERY_TAG = '${sqlLiteral(queryTag, "snowflake")}'
+        AND EXECUTION_STATUS = 'SUCCESS'`;
+}
+function pickSnowflakeReadRow(rows, buildSQL) {
+  const matches = rows.filter((row) => str(col(row, "query_text"))?.trim() === buildSQL.trim());
+  return matches.length === 1 ? matches[0] : null;
+}
+function snowflakeReadCost(row) {
+  const scanned = num(col(row, "bytes_scanned"));
+  const produced = num(col(row, "rows_produced"));
+  return {
+    engine: "snowflake",
+    jobId: str(col(row, "job_id")),
+    parentJobId: null,
+    bytesScanned: scanned,
+    bytesBilled: null,
+    slotTimeMs: null,
+    executionTimeMs: num(col(row, "execution_time_ms")),
+    cacheHit: scanned === null || produced === null ? null : scanned === 0 && produced > 0
+  };
+}
+function col(row, key) {
+  if (key in row)
+    return row[key];
+  const upper = key.toUpperCase();
+  if (upper in row)
+    return row[upper];
+  const hit = Object.keys(row).find((k) => k.toLowerCase() === key);
+  return hit === undefined ? undefined : row[hit];
+}
+function num(value) {
+  if (value === null || value === undefined)
+    return null;
+  const n = Number(value);
+  return Number.isFinite(n) ? n : null;
+}
+function str(value) {
+  return typeof value === "string" && value.length > 0 ? value : null;
+}
+function bool(value) {
+  if (typeof value === "boolean")
+    return value;
+  if (typeof value === "string") {
+    if (value === "true")
+      return true;
+    if (value === "false")
+      return false;
+  }
+  return null;
+}
+
+// src/service/materialization_build_session.ts
 var sharedGateSession;
 var PASSTHROUGH_SOURCE_TYPES = [
   "bigquery",
@@ -293063,6 +293218,157 @@ function wrapPassthrough(sourceType, handle, innerSQL) {
     }
   }
 }
+async function tagSnowflakeSession(session, sourceType, handle, queryMetadata) {
+  if (sourceType !== "snowflake")
+    return;
+  const sql = snowflakeSetQueryTagSQL(queryMetadata, handle);
+  if (sql === undefined)
+    return;
+  try {
+    await session.runSQL(sql);
+  } catch (error) {
+    logger.warn("Could not tag the Snowflake build session", {
+      error: error instanceof Error ? error.message : String(error)
+    });
+    recordAttributionSkipped("tag_failed");
+  }
+}
+async function canSplitBigQueryRead(session, handle) {
+  try {
+    await session.runSQL(`SELECT job_id FROM bigquery_jobs('${escapeSQL(handle)}', maxResults := 1)`);
+    return true;
+  } catch (error) {
+    logger.warn("Cannot list BigQuery jobs for this connection, so this build's " + "warehouse read will not be labelled or costed", {
+      error: error instanceof Error ? error.message : String(error)
+    });
+    recordAttributionSkipped("job_listing_unavailable");
+    return false;
+  }
+}
+
+class BilledReadNotCapturedError extends Error {
+  constructor(message, options) {
+    super(message, options);
+    this.name = "BilledReadNotCapturedError";
+  }
+}
+var BIGQUERY_LOOKUP_ATTEMPTS = 3;
+var BIGQUERY_LOOKUP_BACKOFF_MS = 250;
+async function withBigQueryLookupRetry(work, parentJobId) {
+  let lastError;
+  for (let attempt = 1;attempt <= BIGQUERY_LOOKUP_ATTEMPTS; attempt++) {
+    try {
+      return await work();
+    } catch (error) {
+      lastError = error;
+      if (attempt < BIGQUERY_LOOKUP_ATTEMPTS) {
+        logger.warn("Could not read the BigQuery job record for a build's read; retrying", {
+          attempt,
+          of: BIGQUERY_LOOKUP_ATTEMPTS,
+          parentJobId,
+          error: error instanceof Error ? error.message : String(error)
+        });
+        await new Promise((resolve6) => setTimeout(resolve6, BIGQUERY_LOOKUP_BACKOFF_MS * attempt));
+      }
+    }
+  }
+  throw new BilledReadNotCapturedError(`Could not read the BigQuery job record for a build's read (parent job ` + `${parentJobId}) after ${BIGQUERY_LOOKUP_ATTEMPTS} attempts, so its ` + `rows cannot be captured: ` + `${lastError instanceof Error ? lastError.message : String(lastError)}`, { cause: lastError });
+}
+async function issuePassthroughRead(session, sourceType, handle, buildSQL, queryMetadata) {
+  const label = sourceType === "bigquery" ? bigQueryQueryLabelValue(queryMetadata) : undefined;
+  if (label === undefined || !await canSplitBigQueryRead(session, handle)) {
+    return {
+      selectSQL: wrapPassthrough(sourceType, handle, buildSQL),
+      jobId: null,
+      cost: null
+    };
+  }
+  const script = `SET @@query_label = "${label}";
+${buildSQL};`;
+  const executed = await session.runSQL(`SELECT * FROM bigquery_execute('${escapeSQL(handle)}', '${escapeSQL(script)}')`);
+  const parentJobId = firstColumn(executed, "job_id");
+  if (parentJobId === null) {
+    throw new BilledReadNotCapturedError("bigquery_execute returned no job_id for the build's read, so its " + "result table cannot be located");
+  }
+  const located = await withBigQueryLookupRetry(async () => {
+    const child = await session.runSQL(`
+      SELECT job_id,
+             json_extract_string(configuration, '$.query.destinationTable.projectId') AS project,
+             json_extract_string(configuration, '$.query.destinationTable.datasetId') AS dataset,
+             json_extract_string(configuration, '$.query.destinationTable.tableId')   AS table,
+             ${BIGQUERY_COST_COLUMNS}
+      FROM (SELECT * FROM bigquery_jobs('${escapeSQL(handle)}',
+                                        parentJobId := '${escapeSQL(parentJobId)}')
+            WHERE job_type = 'QUERY')
+      ORDER BY creation_time DESC`);
+    const row = resultRows(child).find((r) => str2(r.dataset) && str2(r.table));
+    if (row === undefined) {
+      throw new Error(`no child job with a result table under parent ${parentJobId}`);
+    }
+    return row;
+  }, parentJobId);
+  const path14 = [
+    str2(located.project) ?? handle,
+    str2(located.dataset),
+    str2(located.table)
+  ].join(".");
+  const jobId = str2(located.job_id);
+  return {
+    selectSQL: `SELECT * FROM bigquery_scan('${escapeSQL(path14)}')`,
+    jobId,
+    cost: bigQueryReadCost(located, jobId, parentJobId)
+  };
+}
+async function snowflakeReadCostAfterBuild(session, sourceType, handle, buildSQL, queryMetadata, database) {
+  if (sourceType !== "snowflake")
+    return null;
+  const tag = snowflakeQueryTagValue(queryMetadata);
+  if (tag === undefined)
+    return null;
+  try {
+    const costResult = await session.runSQL(passthroughSnowflake(snowflakeCostSQL(tag, database), handle));
+    const candidates = resultRows(costResult);
+    const row = pickSnowflakeReadRow(candidates, buildSQL);
+    if (row === null) {
+      recordAttributionSkipped(candidates.length === 0 ? "read_row_not_found" : "read_row_ambiguous");
+      return null;
+    }
+    return snowflakeReadCost(row);
+  } catch (error) {
+    logger.warn("Could not read back what the Snowflake build read cost", {
+      error: error instanceof Error ? error.message : String(error)
+    });
+    recordAttributionSkipped("cost_query_failed");
+    return null;
+  }
+}
+function passthroughSnowflake(innerSQL, handle) {
+  return `SELECT * FROM snowflake_query('${escapeSQL(innerSQL)}', '${escapeSQL(handle)}')`;
+}
+function resultRows(result) {
+  if (Array.isArray(result))
+    return result;
+  const rows = result?.rows;
+  return Array.isArray(rows) ? rows : [];
+}
+function firstColumn(result, column) {
+  const rows = resultRows(result);
+  return rows.length > 0 ? str2(rows[0][column]) : null;
+}
+function str2(value) {
+  return typeof value === "string" && value.length > 0 ? value : null;
+}
+async function clearSnowflakeSessionTag(session, sourceType, handle) {
+  if (sourceType !== "snowflake" || handle === undefined)
+    return;
+  try {
+    await session.runSQL(passthroughSnowflake("ALTER SESSION UNSET QUERY_TAG", handle));
+  } catch (error) {
+    logger.warn("Could not clear the Snowflake build session's query tag", {
+      error: error instanceof Error ? error.message : String(error)
+    });
+  }
+}
 function passthroughSourceType(sourceConnection) {
   const type = sourceConnection.type;
   if (PASSTHROUGH_SOURCE_TYPES.includes(type ?? "")) {
@@ -293116,22 +293422,30 @@ async function buildSourceIntoStorage(params) {
     sourceConnection,
     buildSQL,
     physicalTableName,
-    environmentPath
+    environmentPath,
+    queryMetadata
   } = params;
   assertSupportedDestination(destinationName, destinationConnection);
   const sourceType = passthroughSourceType(sourceConnection);
   const { session, dispose } = createIsolatedBuildSession(`build_${destinationName}`);
+  let federatedHandle;
   try {
     await attachDestinationReadWrite(session, destinationName, destinationConnection, environmentPath);
     if (destinationConnection.type === "ducklake") {
       await session.runSQL("SET ducklake_default_data_inlining_row_limit=0");
     }
     const federated = await federateSourceForPassthrough(session, sourceType, sourceFederationConfig(sourceConnection));
+    await tagSnowflakeSession(session, sourceType, federated.handle, queryMetadata);
     const target = quoteManifestTablePath(`${destinationName}.${physicalTableName}`, "duckdb");
-    const passthrough = wrapPassthrough(sourceType, federated.handle, buildSQL);
-    const schema = await createTableAndDescribe(session, target, passthrough);
-    return { storageDestinationName: destinationName, schema };
+    const read = await issuePassthroughRead(session, sourceType, federated.handle, buildSQL, queryMetadata);
+    const schema = await createTableAndDescribe(session, target, read.selectSQL);
+    return {
+      storageDestinationName: destinationName,
+      schema,
+      readCost: read.cost ?? await snowflakeReadCostAfterBuild(session, sourceType, federated.handle, buildSQL, queryMetadata, sourceConnection.snowflakeConnection?.database)
+    };
   } finally {
+    await clearSnowflakeSessionTag(session, sourceType, federatedHandle);
     await dispose();
   }
 }
@@ -293182,7 +293496,11 @@ async function buildDownstreamIntoStorage(params) {
     const sql = projectToPublicColumns(downstream, downstream.getSQL({ virtualMap }));
     const target = quoteManifestTablePath(`${destinationName}.${physicalTableName}`, "duckdb");
     const schema = await createTableAndDescribe(session, target, sql);
-    return { storageDestinationName: destinationName, schema };
+    return {
+      storageDestinationName: destinationName,
+      schema,
+      readCost: null
+    };
   } finally {
     await dispose();
   }
@@ -293981,15 +294299,15 @@ class MaterializationService {
       buildManifest,
       connectionDigests
     });
+    const runOptions = this.buildRunSQLOptions(persistSource, environment, buildMetadata);
     if (isStorageBuild) {
       const dependsOnStorageUpstream = persistSource.getSQL({
         buildManifest: manifest.buildManifest,
         connectionDigests
       }) !== buildSQL;
       const publicBuildSQL = projectToPublicColumns(persistSource, buildSQL);
-      return this.buildOneSourceIntoStorage(persistSource, instruction, manifest, environment, publicBuildSQL, builtEntries, dependsOnStorageUpstream);
+      return this.buildOneSourceIntoStorage(persistSource, instruction, manifest, environment, publicBuildSQL, builtEntries, dependsOnStorageUpstream, runOptions.queryMetadata);
     }
-    const runOptions = this.buildRunSQLOptions(persistSource, environment, buildMetadata);
     const dialect = persistSource.dialectName;
     const quotedPhysicalPath = quoteTablePath(physicalTableName, dialect);
     const lineage = incremental && contentSourceEntityId ? incrementalLineage({
@@ -294028,8 +294346,10 @@ class MaterializationService {
     }
     const startTime = performance.now();
     await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`, runOptions);
+    let buildCostBytes;
     try {
-      await connection.runSQL(`CREATE TABLE ${quotedStaging} AS (${buildSQL})`, runOptions);
+      const ctas = await connection.runSQL(`CREATE TABLE ${quotedStaging} AS (${buildSQL})`, runOptions);
+      buildCostBytes = ctas?.runStats?.queryCostBytes;
       await connection.runSQL(`DROP TABLE IF EXISTS ${quotedPhysical}`, runOptions);
       await connection.runSQL(`ALTER TABLE ${quotedStaging} RENAME TO ${quotedBareName}`, runOptions);
     } catch (err) {
@@ -294067,7 +294387,9 @@ class MaterializationService {
       realization: instruction.realization,
       ...ledgerFields(incrementalRefresh?.lineage, seededThrough),
       ...refreshFields(incrementalRefresh ? "full" : undefined),
-      rowCount: null
+      rowCount: null,
+      buildDurationMs: durationMs,
+      queryCostBytes: buildCostBytes ?? null
     };
   }
   async refreshOneSourceIncrementally(params) {
@@ -294096,6 +294418,7 @@ class MaterializationService {
     if (step.mode === "seed")
       return;
     const startTime = performance.now();
+    let appliedDurationMs;
     if (step.mode === "delta") {
       await applyDeltaScript(runner, dialect, step.statements);
       await advanceLedger({
@@ -294104,6 +294427,7 @@ class MaterializationService {
         coveredThrough: step.coveredThrough
       });
       const durationMs = Math.round(performance.now() - startTime);
+      appliedDurationMs = durationMs;
       recordSourceBuildDuration(durationMs, "delta");
       reportDeltaApplied({
         packageName: context.packageName,
@@ -294125,11 +294449,13 @@ class MaterializationService {
       connectionName: persistSource.connectionName,
       realization: instruction.realization,
       rowCount: null,
+      buildDurationMs: appliedDurationMs ?? null,
+      queryCostBytes: null,
       ...ledgerFields(lineage, step.coveredThrough),
       ...refreshFields(step.mode === "delta" ? "delta" : "none")
     };
   }
-  async buildOneSourceIntoStorage(persistSource, instruction, manifest, environment, buildSQL, builtEntries, dependsOnStorageUpstream) {
+  async buildOneSourceIntoStorage(persistSource, instruction, manifest, environment, buildSQL, builtEntries, dependsOnStorageUpstream, queryMetadata) {
     const sourceEntityId = instruction.sourceEntityId;
     const physicalTableName = instruction.physicalTableName;
     const destinationName = instruction.destination;
@@ -294169,17 +294495,21 @@ class MaterializationService {
           sourceConnection,
           buildSQL,
           physicalTableName,
-          environmentPath: environment.getEnvironmentPath()
+          environmentPath: environment.getEnvironmentPath(),
+          queryMetadata
         });
       } catch (err) {
         const safeDetail = redactConnectionSecrets(errMessage(err), sourceConnection, destinationConnection);
-        recordStorageBuildFailure(destinationName);
+        const alreadyBilled = err instanceof BilledReadNotCapturedError;
+        recordStorageBuildFailure(destinationName, alreadyBilled ? "billed_read_not_captured" : "build_failed");
         logger.warn("Storage materialization build failed", {
           sourceName: persistSource.name,
           destinationName,
+          alreadyBilled,
           error: safeDetail
         });
-        throw new Error(`Failed to materialize source '${persistSource.name}' into ` + `storage destination '${destinationName}': ${safeDetail}`);
+        const failure = `Failed to materialize source '${persistSource.name}' into ` + `storage destination '${destinationName}': ${safeDetail}`;
+        throw alreadyBilled ? new BilledReadNotCapturedError(failure, { cause: err }) : new Error(failure, { cause: err });
       }
     }
     try {
@@ -294215,7 +294545,8 @@ class MaterializationService {
       physicalTableName,
       storageDestinationName: result.storageDestinationName,
       columns: result.schema.length,
-      durationMs
+      durationMs,
+      readCost: result.readCost
     });
     return {
       sourceEntityId,
@@ -294226,7 +294557,9 @@ class MaterializationService {
       storageDestinationName: result.storageDestinationName,
       schema: result.schema,
       realization: instruction.realization,
-      rowCount: null
+      rowCount: null,
+      buildDurationMs: durationMs,
+      queryCostBytes: result.readCost?.bytesScanned ?? null
     };
   }
   async buildDownstreamViaParents(persistSource, destinationName, destinationConnection, builtEntries, environment, physicalTableName) {

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.243",
+   "version": "0.0.244",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"