@malloy-publisher/server 0.0.225 → 0.0.227

package/src/package_load/package_load_worker.ts

@@ -85,7 +85,9 @@ import { validateAuthorizeProbes } from "../service/authorize";
 import { type FilterDefinition } from "../service/filter";
 import {
    PackageMaterializationConfig,
+   PackageScope,
    parsePackageMaterialization,
+   parsePackageScope,
 } from "../service/package_manifest";
 import {
    extractQueriesFromModelDef,
@@ -97,6 +99,7 @@ import {
    type MalloyGivenApi,
 } from "../service/given";
 import { ignoreDotfiles } from "../utils";
+import { RpcWaitAccountant } from "./rpc_wait_accountant";
 import type {
    ConnectionMetadata,
    ConnectionMetadataRequest,
@@ -150,6 +153,13 @@ function callMain<T>(send: (requestId: string) => void): Promise<T> {
    });
 }
 
+// Per-load phase timing: brackets the wall-clock spent awaiting proxied schema
+// fetches so `loadPackage` can separate connection I/O from compile CPU. The
+// pool runs one load per worker at a time, so a single module-level instance
+// is safe; it's still keyed by jobId to shrug off a straggler fetch from a
+// prior, reused-worker load. See {@link RpcWaitAccountant}.
+const schemaWait = new RpcWaitAccountant();
+
 function dispatchMainResponse(message: MainToWorkerMessage): void {
    if (
       message.type === "schema-for-tables-response" ||
@@ -200,18 +210,25 @@ class ProxyConnection {
       schemas: Record<string, TableSourceDef>;
       errors: Record<string, string>;
    }> {
-      const response = await callMain<SchemaForTablesResponse>((requestId) => {
-         const req: SchemaForTablesRequest = {
-            type: "schema-for-tables",
-            requestId,
-            jobId: this.jobId,
-            connectionName: this.name,
-            tables,
-            options: serializeFetchOptions(options),
-         };
-         port.postMessage(req);
-      });
-      return { schemas: response.schemas, errors: response.errors };
+      schemaWait.noteStart(this.jobId);
+      try {
+         const response = await callMain<SchemaForTablesResponse>(
+            (requestId) => {
+               const req: SchemaForTablesRequest = {
+                  type: "schema-for-tables",
+                  requestId,
+                  jobId: this.jobId,
+                  connectionName: this.name,
+                  tables,
+                  options: serializeFetchOptions(options),
+               };
+               port.postMessage(req);
+            },
+         );
+         return { schemas: response.schemas, errors: response.errors };
+      } finally {
+         schemaWait.noteSettle(this.jobId);
+      }
    }
 
    async fetchSchemaForSQLStruct(
@@ -221,17 +238,23 @@ class ProxyConnection {
       | { structDef: SQLSourceDef; error?: undefined }
       | { error: string; structDef?: undefined }
    > {
-      const response = await callMain<SchemaForSqlResponse>((requestId) => {
-         const req: SchemaForSqlRequest = {
-            type: "schema-for-sql",
-            requestId,
-            jobId: this.jobId,
-            connectionName: this.name,
-            sentence: sentence as unknown,
-            options: serializeFetchOptions(options),
-         };
-         port.postMessage(req);
-      });
+      schemaWait.noteStart(this.jobId);
+      let response: SchemaForSqlResponse;
+      try {
+         response = await callMain<SchemaForSqlResponse>((requestId) => {
+            const req: SchemaForSqlRequest = {
+               type: "schema-for-sql",
+               requestId,
+               jobId: this.jobId,
+               connectionName: this.name,
+               sentence: sentence as unknown,
+               options: serializeFetchOptions(options),
+            };
+            port.postMessage(req);
+         });
+      } finally {
+         schemaWait.noteSettle(this.jobId);
+      }
       if (response.error !== undefined) return { error: response.error };
       if (response.structDef === undefined) {
          return { error: "Empty SQL schema response from main thread" };
@@ -387,6 +410,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
    queryableSources?: "declared" | "all";
    manifestLocation?: string | null;
    materialization?: PackageMaterializationConfig | null;
+   scope?: PackageScope;
 }> {
    const manifestPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
    const contents = await fs.promises.readFile(manifestPath, "utf8");
@@ -397,6 +421,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
       queryableSources?: unknown;
       manifestLocation?: unknown;
       materialization?: unknown;
+      scope?: unknown;
    };
    return {
       name: parsed.name,
@@ -416,6 +441,9 @@ async function readPackageMetadata(packagePath: string): Promise<{
       // Package-level Malloy Persistence policy; surfaced to the control plane,
       // which owns scheduling. Only `schedule` is read today.
       materialization: parsePackageMaterialization(parsed.materialization),
+      // Package-level persist scope mode; defaults to "package". An invalid
+      // value throws here and fails the load (scope is load-bearing).
+      scope: parsePackageScope(parsed.scope),
    };
 }
 
@@ -849,18 +877,41 @@ async function loadPackage(
    const allFiles = await listPackageFiles(job.packagePath);
    const modelPaths = filterModelPaths(allFiles);
 
+   // Bracket the compile region: only work from here on is compilation +
+   // proxied schema fetches. The setup above (manifest read + file listing)
+   // is excluded so it can't inflate the compile figure — it stays in the
+   // derivable remainder (loadDuration - compile - schemaFetch). Begin the
+   // schema-fetch accounting HERE, at the region boundary, not at load start:
+   // that keeps `compileDurationMs = compileRegion - schemaFetchWait` exact
+   // regardless of whether any fetch ever happens during setup (none do
+   // today, but this stops that assumption from silently mattering).
+   const compileRegionStart = performance.now();
+   schemaWait.begin(job.requestId);
    const models = await Promise.all(
       modelPaths.map((modelPath) =>
          compileOneModel(job, malloyConfig, modelPath),
       ),
    );
 
+   const loadEnd = performance.now();
+   const schemaFetchDurationMs = schemaWait.waitMs;
    return {
       type: "load-package-result",
       requestId: job.requestId,
       packageMetadata,
       models,
-      loadDurationMs: performance.now() - loadStart,
+      loadDurationMs: loadEnd - loadStart,
+      timings: {
+         // Compile-region wall minus the schema-fetch wait it contains — a
+         // conservative ceiling on compile CPU (clamped: rounding can nudge
+         // it slightly negative).
+         compileDurationMs: Math.max(
+            0,
+            loadEnd - compileRegionStart - schemaFetchDurationMs,
+         ),
+         schemaFetchDurationMs,
+         schemaFetchCount: schemaWait.fetches,
+      },
    };
 }
 

package/src/package_load/protocol.ts

@@ -65,7 +65,10 @@
  */
 
 import type { SQLSourceDef, TableSourceDef } from "@malloydata/malloy";
-import type { PackageMaterializationConfig } from "../service/package_manifest";
+import type {
+   PackageMaterializationConfig,
+   PackageScope,
+} from "../service/package_manifest";
 
 // ──────────────────────────────────────────────────────────────────────
 // Direction: main ──▶ worker (load-package job)
@@ -181,10 +184,27 @@ export interface LoadPackageResult {
       queryableSources?: "declared" | "all";
       manifestLocation?: string | null;
       materialization?: PackageMaterializationConfig | null;
+      scope?: PackageScope;
    };
    models: SerializedModel[];
    /** Wall-clock ms inside the worker for the full package load. */
    loadDurationMs: number;
+   /**
+    * Per-load timing breakdown (subset of {@link loadDurationMs}; the
+    * remainder is worker setup + post-compile extraction). See
+    * `rpc_wait_accountant.ts` for how compile time is separated from I/O.
+    */
+   timings: {
+      /**
+       * Compile-region wall-clock minus time awaiting proxied schema fetches —
+       * a conservative ceiling on Malloy compile CPU.
+       */
+      compileDurationMs: number;
+      /** Wall-clock awaiting proxied connection schema fetches (union). */
+      schemaFetchDurationMs: number;
+      /** Number of proxied connection schema fetches the load drove. */
+      schemaFetchCount: number;
+   };
 }
 
 export interface LoadPackageError {

package/src/package_load/rpc_wait_accountant.spec.ts

@@ -0,0 +1,109 @@
+import { describe, expect, it } from "bun:test";
+
+import { RpcWaitAccountant } from "./rpc_wait_accountant";
+
+/** Accountant driven by a settable fake clock. */
+function withClock(): { acc: RpcWaitAccountant; at: (ms: number) => void } {
+   let t = 0;
+   const acc = new RpcWaitAccountant(() => t);
+   return { acc, at: (ms) => (t = ms) };
+}
+
+describe("RpcWaitAccountant", () => {
+   it("brackets a single fetch's wait", () => {
+      const { acc, at } = withClock();
+      at(0);
+      acc.begin("A");
+      at(10);
+      acc.noteStart("A");
+      at(60);
+      acc.noteSettle("A");
+      expect(acc.waitMs).toBe(50);
+   });
+
+   it("counts overlapping fetches as their union, not the sum", () => {
+      const { acc, at } = withClock();
+      at(0);
+      acc.begin("A");
+      at(0);
+      acc.noteStart("A"); // window opens
+      at(10);
+      acc.noteStart("A"); // second fetch, window stays open
+      at(40);
+      acc.noteSettle("A"); // one still outstanding
+      at(60);
+      acc.noteSettle("A"); // window closes
+      expect(acc.waitMs).toBe(60); // union [0,60], not 30+50
+   });
+
+   it("accumulates disjoint fetch windows", () => {
+      const { acc, at } = withClock();
+      at(0);
+      acc.begin("A");
+      at(0);
+      acc.noteStart("A");
+      at(20);
+      acc.noteSettle("A");
+      at(50);
+      acc.noteStart("A");
+      at(70);
+      acc.noteSettle("A");
+      expect(acc.waitMs).toBe(40);
+   });
+
+   it("ignores a straggler fetch from a prior reused-worker load", () => {
+      const { acc, at } = withClock();
+      at(0);
+      acc.begin("A");
+      at(0);
+      acc.noteStart("A"); // load A opens a fetch and never settles it (failed)
+
+      at(100);
+      acc.begin("B"); // worker reused for load B
+      at(110);
+      acc.noteStart("B");
+      at(130);
+      acc.noteSettle("B");
+
+      at(140);
+      acc.noteSettle("A"); // A's late response — must be a no-op
+
+      expect(acc.waitMs).toBe(20); // only B's [110,130]
+   });
+
+   it("ignores a stray start from a non-active load", () => {
+      const { acc, at } = withClock();
+      at(0);
+      acc.begin("B");
+      at(10);
+      acc.noteStart("A"); // wrong load
+      at(50);
+      acc.noteSettle("A");
+      expect(acc.waitMs).toBe(0);
+   });
+
+   it("counts active-load fetches only", () => {
+      const { acc, at } = withClock();
+      at(0);
+      acc.begin("B");
+      acc.noteStart("B");
+      acc.noteSettle("B");
+      acc.noteStart("B");
+      acc.noteStart("A"); // straggler — not counted
+      expect(acc.fetches).toBe(2);
+   });
+
+   it("begin() discards prior-load state", () => {
+      const { acc, at } = withClock();
+      at(0);
+      acc.begin("A");
+      at(0);
+      acc.noteStart("A");
+      at(50);
+      acc.noteSettle("A");
+      expect(acc.waitMs).toBe(50);
+      at(60);
+      acc.begin("B");
+      expect(acc.waitMs).toBe(0);
+   });
+});

package/src/package_load/rpc_wait_accountant.ts

@@ -0,0 +1,76 @@
+/**
+ * Per-load "wait" accounting for the package-load worker.
+ *
+ * A package load spends its time in two very different ways: CPU-bound Malloy
+ * compilation, and I/O waiting on connection schema fetches that are proxied
+ * back to the main thread. Today only the *total* load time is observable, so
+ * an operator seeing slow loads can't tell which half dominates. This tracks
+ * the wall-clock during which at least one proxied schema fetch is outstanding,
+ * so the load handler can report `compile = compileRegion - schemaFetchWait`
+ * alongside the fetch time itself.
+ *
+ * Why wall-minus-wait and not a CPU counter: worker threads share the host
+ * process, so `process.cpuUsage()` is process-wide and would fold in every
+ * other worker's CPU — useless for a per-load number. Wall-minus-wait is the
+ * honest per-load proxy. Treat the compute figure as a conservative *ceiling*
+ * on compile CPU: only proxied waits are subtracted, so any non-fetch stall
+ * (GC, scheduler preemption) also lands in "compute".
+ *
+ * Overlapping fetches are counted as their union (the wall-time with >=1
+ * outstanding), never the sum, so concurrent fetches during one compile don't
+ * over-subtract.
+ *
+ * Scoped by `jobId`: the pool runs one load per worker at a time, but reuses a
+ * worker across loads, and a load that fails can leave a proxied fetch in
+ * flight. Gating every start/settle on the active job makes such a straggler's
+ * late response a no-op instead of corrupting the next load's numbers. (If the
+ * one-load-per-worker rule ever changes, this stays correct per job but the
+ * compute ceiling would then also fold in a sibling load's CPU.)
+ */
+export class RpcWaitAccountant {
+   private inFlight = 0;
+   private waitStartMs = 0;
+   private accumMs = 0;
+   private startedCount = 0;
+   private activeJobId: string | undefined;
+
+   constructor(private readonly now: () => number = () => performance.now()) {}
+
+   /** Begin accounting for `jobId`, discarding any prior-load state. */
+   begin(jobId: string): void {
+      this.activeJobId = jobId;
+      this.inFlight = 0;
+      this.waitStartMs = 0;
+      this.accumMs = 0;
+      this.startedCount = 0;
+   }
+
+   /** A proxied fetch for `jobId` was issued. Ignored if `jobId` isn't active. */
+   noteStart(jobId: string): void {
+      if (jobId !== this.activeJobId) return;
+      if (this.inFlight === 0) this.waitStartMs = this.now();
+      this.inFlight += 1;
+      this.startedCount += 1;
+   }
+
+   /**
+    * A proxied fetch for `jobId` settled (resolved or rejected). Ignored if
+    * `jobId` isn't active, so a straggler from a prior reused-worker load
+    * can't drive the counter negative or close the active load's bracket.
+    */
+   noteSettle(jobId: string): void {
+      if (jobId !== this.activeJobId) return;
+      this.inFlight -= 1;
+      if (this.inFlight === 0) this.accumMs += this.now() - this.waitStartMs;
+   }
+
+   /** Accumulated wall-time with >=1 active-load fetch outstanding (union). */
+   get waitMs(): number {
+      return this.accumMs;
+   }
+
+   /** Number of proxied fetches started for the active load. */
+   get fetches(): number {
+      return this.startedCount;
+   }
+}

package/src/package_load_metrics.spec.ts

@@ -0,0 +1,114 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+
+import {
+   recordPackageLoadPhases,
+   resetPackageLoadMetricsForTesting,
+} from "./package_load_metrics";
+import {
+   startMetricsHarness,
+   type MetricsHarness,
+} from "./test_helpers/metrics_harness";
+
+describe("package_load_metrics", () => {
+   let harness: MetricsHarness;
+
+   beforeEach(async () => {
+      harness = await startMetricsHarness();
+      resetPackageLoadMetricsForTesting();
+   });
+
+   afterEach(async () => {
+      resetPackageLoadMetricsForTesting();
+      await harness.shutdown();
+   });
+
+   it("records the three phase histograms, labeled by status", async () => {
+      recordPackageLoadPhases(
+         {
+            compileDurationMs: 120,
+            schemaFetchDurationMs: 80,
+            schemaFetchCount: 4,
+         },
+         "success",
+      );
+      recordPackageLoadPhases(
+         {
+            compileDurationMs: 40,
+            schemaFetchDurationMs: 10,
+            schemaFetchCount: 1,
+         },
+         "success",
+      );
+
+      const compile = await harness.collectHistogram(
+         "malloy_package_load_compile_duration",
+         { status: "success" },
+      );
+      expect(compile.count).toBe(2);
+      expect(compile.sum).toBe(160);
+
+      const fetchDuration = await harness.collectHistogram(
+         "malloy_package_load_schema_fetch_duration",
+         { status: "success" },
+      );
+      expect(fetchDuration.count).toBe(2);
+      expect(fetchDuration.sum).toBe(90);
+
+      const fetches = await harness.collectHistogram(
+         "malloy_package_load_schema_fetches",
+         { status: "success" },
+      );
+      expect(fetches.count).toBe(2);
+      expect(fetches.sum).toBe(5);
+   });
+
+   it("labels phases by terminal status, so failures are separable from successes", async () => {
+      recordPackageLoadPhases(
+         {
+            compileDurationMs: 30,
+            schemaFetchDurationMs: 5,
+            schemaFetchCount: 1,
+         },
+         "success",
+      );
+      recordPackageLoadPhases(
+         {
+            compileDurationMs: 12,
+            schemaFetchDurationMs: 3,
+            schemaFetchCount: 1,
+         },
+         "compilation_error",
+      );
+
+      const ok = await harness.collectHistogram(
+         "malloy_package_load_compile_duration",
+         { status: "success" },
+      );
+      const failed = await harness.collectHistogram(
+         "malloy_package_load_compile_duration",
+         { status: "compilation_error" },
+      );
+      expect(ok.count).toBe(1);
+      expect(ok.sum).toBe(30);
+      expect(failed.count).toBe(1);
+      expect(failed.sum).toBe(12);
+   });
+
+   it("resolves the slow-load tail past OTel's default 10s cap", async () => {
+      // A 90s load would fall in the +Inf bucket under OTel's default
+      // boundaries (top = 10000ms); the explicit buckets must extend further.
+      recordPackageLoadPhases(
+         {
+            compileDurationMs: 90_000,
+            schemaFetchDurationMs: 0,
+            schemaFetchCount: 0,
+         },
+         "success",
+      );
+      const compile = await harness.collectHistogram(
+         "malloy_package_load_compile_duration",
+      );
+      expect(compile.boundaries.some((b) => b > 10_000)).toBe(true);
+      expect(compile.boundaries[compile.boundaries.length - 1]).toBe(300_000);
+   });
+});

package/src/package_load_metrics.ts

@@ -0,0 +1,127 @@
+/**
+ * Per-load phase timing for package loads.
+ *
+ * `malloy_package_load_duration` already reports the *total* time to load a
+ * package, but an operator watching a slow load can't tell whether it's the
+ * Malloy compile or the connection schema fetches that dominate — the two are
+ * tuned very differently (CPU/worker sizing vs. warehouse round-trips). These
+ * histograms split the load into those phases so a single dashboard answers
+ * "where did the time go?". The worker measures the phases and reports them on
+ * its result; see `package_load/package_load_worker.ts` and
+ * `package_load/rpc_wait_accountant.ts`.
+ *
+ * Labels: `status` only. Deliberately NOT labelled by package name — unlike
+ * `malloy_package_load_duration`, which carries `malloy_package_name` — to keep
+ * cardinality bounded on deployments with many packages. Per-package latency,
+ * when needed, is better answered by exemplars/traces than a label explosion.
+ *
+ * Lazy init for the same reason as `query_cap_metrics.ts` / `query_timeout.ts`:
+ * instruments created before `setGlobalMeterProvider` bind to a NoOp meter
+ * (https://github.com/open-telemetry/opentelemetry-js/issues/3505).
+ */
+
+import { type Histogram } from "@opentelemetry/api";
+import { publisherMeter } from "./telemetry";
+
+/**
+ * Explicit bucket boundaries (ms) for the load-duration histograms. OTel's
+ * default boundaries top out at 10 000 ms, so any load slower than 10 s falls
+ * in the `+Inf` bucket and `histogram_quantile` can't resolve the tail — which
+ * is exactly the tail (large/slow packages) worth seeing. These extend to
+ * 5 minutes while keeping sub-10 ms resolution for the fast common case.
+ * Shared with the total `malloy_package_load_duration` histogram so all
+ * load-timing histograms bucket consistently.
+ */
+export const LOAD_DURATION_BUCKETS_MS = [
+   1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,
+   120_000, 300_000,
+];
+
+/** Bucket boundaries for the per-load schema-fetch count. */
+const SCHEMA_FETCH_COUNT_BUCKETS = [0, 1, 2, 5, 10, 25, 50, 100, 250, 500];
+
+/**
+ * Terminal status of a package load, mirroring the `status` label on the
+ * existing `malloy_package_load_duration` histogram so the phase metrics slice
+ * the same way. In practice the phase histograms only ever carry `success`,
+ * `compilation_error`, or `error` — a `pool_unavailable` failure happens before
+ * the worker returns any timings, so there is nothing to record for it.
+ */
+export type PackageLoadStatus =
+   | "success"
+   | "compilation_error"
+   | "pool_unavailable"
+   | "error";
+
+/** Phase timings a single package load reports (subset of the total). */
+export interface PackageLoadPhaseTimings {
+   /** Ceiling on Malloy compile CPU (compile region minus schema-fetch wait). */
+   compileDurationMs: number;
+   /** Wall-clock awaiting proxied connection schema fetches. */
+   schemaFetchDurationMs: number;
+   /** Number of proxied connection schema fetches the load drove. */
+   schemaFetchCount: number;
+}
+
+let compileDuration: Histogram | null = null;
+let schemaFetchDuration: Histogram | null = null;
+let schemaFetches: Histogram | null = null;
+
+function ensureInstruments(): void {
+   if (compileDuration && schemaFetchDuration && schemaFetches) return;
+   const meter = publisherMeter();
+   compileDuration ??= meter.createHistogram(
+      "malloy_package_load_compile_duration",
+      {
+         description:
+            "Per-load Malloy compile time (compile region minus proxied schema-fetch wait; a ceiling on compile CPU). Label: status.",
+         unit: "ms",
+         advice: { explicitBucketBoundaries: LOAD_DURATION_BUCKETS_MS },
+      },
+   );
+   schemaFetchDuration ??= meter.createHistogram(
+      "malloy_package_load_schema_fetch_duration",
+      {
+         description:
+            "Per-load wall-clock awaiting proxied connection schema fetches. Label: status.",
+         unit: "ms",
+         advice: { explicitBucketBoundaries: LOAD_DURATION_BUCKETS_MS },
+      },
+   );
+   schemaFetches ??= meter.createHistogram(
+      "malloy_package_load_schema_fetches",
+      {
+         description:
+            "Per-load count of proxied connection schema fetches. Label: status.",
+         advice: { explicitBucketBoundaries: SCHEMA_FETCH_COUNT_BUCKETS },
+      },
+   );
+}
+
+/**
+ * Record one load's phase breakdown. Call once the worker has returned timings,
+ * with the load's terminal `status` (so the phases can be sliced by outcome the
+ * same way the total duration is). The terms are meaningless for a load that
+ * never produced a worker result (e.g. a pool failure), so don't call it there.
+ */
+export function recordPackageLoadPhases(
+   timings: PackageLoadPhaseTimings,
+   status: PackageLoadStatus,
+): void {
+   ensureInstruments();
+   const attrs = { status };
+   compileDuration?.record(timings.compileDurationMs, attrs);
+   schemaFetchDuration?.record(timings.schemaFetchDurationMs, attrs);
+   schemaFetches?.record(timings.schemaFetchCount, attrs);
+}
+
+/**
+ * Visible for tests. Drops cached instruments so a fresh `MeterProvider`
+ * (installed via `startMetricsHarness`) captures future emissions. Do NOT call
+ * from production code.
+ */
+export function resetPackageLoadMetricsForTesting(): void {
+   compileDuration = null;
+   schemaFetchDuration = null;
+   schemaFetches = null;
+}