@malloy-publisher/server 0.0.198-dev1 → 0.0.198-dev3

package/dist/service/schema_worker.mjs

@@ -0,0 +1,61 @@
+// src/service/schema_worker.ts
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import"@malloydata/db-duckdb/native";
+import {
+  ConnectionRuntime,
+  EmptyURLReader
+} from "@malloydata/malloy";
+import * as path from "path";
+import { parentPort } from "worker_threads";
+if (!parentPort) {
+  throw new Error("schema_worker.ts loaded outside a worker thread");
+}
+var connection = new DuckDBConnection({
+  name: "duckdb",
+  databasePath: ":memory:",
+  threads: 1,
+  memoryLimit: "256MB"
+});
+async function handleRequest(req) {
+  try {
+    const fullPath = path.join(req.packagePath, req.databasePath);
+    const normalizedPath = fullPath.replace(/\\/g, "/");
+    const runtime = new ConnectionRuntime({
+      urlReader: new EmptyURLReader,
+      connections: [connection]
+    });
+    const model = runtime.loadModel(`source: temp is duckdb.table('${normalizedPath}')`);
+    const modelDef = await model.getModel();
+    const fields = modelDef._modelDef.contents["temp"].fields;
+    const columns = fields.map((field) => ({
+      type: String(field.type),
+      name: field.name
+    }));
+    const runner = model.loadQuery("run: temp->{aggregate: row_count is count()}");
+    const result = await runner.run();
+    const rowCount = result.data.value[0].row_count?.valueOf();
+    return {
+      id: req.id,
+      ok: true,
+      result: { name: req.databasePath, rowCount, columns }
+    };
+  } catch (err) {
+    const error = err instanceof Error ? err : new Error(String(err));
+    return {
+      id: req.id,
+      ok: false,
+      error: { message: error.message, stack: error.stack }
+    };
+  }
+}
+parentPort.on("message", async (msg) => {
+  const response = await handleRequest(msg);
+  parentPort.postMessage(response);
+});
+var shutdown = async () => {
+  try {
+    await connection.close();
+  } catch {}
+  process.exit(0);
+};
+parentPort.on("close", () => void shutdown());

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.198-dev1",
+   "version": "0.0.198-dev3",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -51,7 +51,6 @@
       "@opentelemetry/sdk-metrics": "^2.0.0",
       "@opentelemetry/sdk-node": "^0.200.0",
       "@opentelemetry/sdk-trace-node": "^2.0.0",
-      "adm-zip": "^0.5.16",
       "async-mutex": "^0.5.0",
       "aws-sdk": "^2.1692.0",
       "body-parser": "^1.20.2",
@@ -61,6 +60,7 @@
       "cors": "^2.8.5",
       "duckdb": "1.4.4",
       "express": "^4.21.0",
+      "extract-zip": "^2.0.1",
       "globals": "^15.9.0",
       "handlebars": "^4.7.8",
       "http-proxy-middleware": "^3.0.5",
@@ -76,7 +76,6 @@
       "@eslint/eslintrc": "^3.3.1",
       "@eslint/js": "^9.23.0",
       "@faker-js/faker": "^9.4.0",
-      "@types/adm-zip": "^0.5.7",
       "@types/bun": "^1.2.20",
       "@types/cors": "^2.8.12",
       "@types/express": "^4.17.14",

package/src/health.ts

@@ -143,19 +143,6 @@ export async function performGracefulShutdownAfterDrain(
       /* do nothing */
    }
 
-   try {
-      // Drain in-flight compiles and terminate worker_threads before
-      // we exit so a slow compile doesn't leave orphan worker
-      // processes. Lazy-imported to avoid pulling the pool module
-      // into the health.ts dep graph for tests that don't exercise
-      // the compile path.
-      const { getCompilePool } = await import("./compile/compile_pool");
-      await getCompilePool().shutdown();
-      logger.info("Malloy compile worker pool shut down");
-   } catch (_error) {
-      /* do nothing */
-   }
-
    if (shutdownGracefulCloseTimeoutSeconds > 0) {
       logger.info(
          `Waiting ${shutdownGracefulCloseTimeoutSeconds} seconds after server close before exit...`,

package/src/instrumentation.ts

@@ -1,3 +1,4 @@
+import { monitorEventLoopDelay } from "node:perf_hooks";
 import { metrics } from "@opentelemetry/api";
 import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
 import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
@@ -116,6 +117,55 @@ const httpRequestCount = meter.createCounter("http_server_requests_total", {
    description: "Total number of HTTP requests",
 });
 
+// Event-loop-delay metrics. A blocked event loop is the only way the
+// /health/liveness probe (a pure synchronous 200 handler) can fail under K8s,
+// so we surface p50/p99/max so an operator can correlate liveness restarts
+// with sustained event-loop pressure (large Malloy compiles, GC, etc.).
+const eventLoopHistogram = monitorEventLoopDelay({ resolution: 20 });
+eventLoopHistogram.enable();
+
+const eventLoopLagP50 = meter.createObservableGauge(
+   "publisher_event_loop_lag_p50_ms",
+   {
+      description:
+         "Event loop delay p50 since the last scrape, in milliseconds",
+      unit: "ms",
+   },
+);
+const eventLoopLagP99 = meter.createObservableGauge(
+   "publisher_event_loop_lag_p99_ms",
+   {
+      description:
+         "Event loop delay p99 since the last scrape, in milliseconds",
+      unit: "ms",
+   },
+);
+const eventLoopLagMax = meter.createObservableGauge(
+   "publisher_event_loop_lag_max_ms",
+   {
+      description:
+         "Event loop delay max since the last scrape, in milliseconds",
+      unit: "ms",
+   },
+);
+
+// Sample all three in one batch so the histogram reset can't race the reads.
+meter.addBatchObservableCallback(
+   (observableResult) => {
+      observableResult.observe(
+         eventLoopLagP50,
+         eventLoopHistogram.percentile(50) / 1e6,
+      );
+      observableResult.observe(
+         eventLoopLagP99,
+         eventLoopHistogram.percentile(99) / 1e6,
+      );
+      observableResult.observe(eventLoopLagMax, eventLoopHistogram.max / 1e6);
+      eventLoopHistogram.reset();
+   },
+   [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax],
+);
+
 const IGNORED_PATHS = new Set([
    "/health",
    "/health/liveness",

package/src/server.ts

@@ -43,6 +43,7 @@ import { EnvironmentStore } from "./service/environment_store";
 import { ManifestService } from "./service/manifest_service";
 import { MaterializationService } from "./service/materialization_service";
 import { PackageMemoryGovernor } from "./service/package_memory_governor";
+import { ProcessStatsReporter } from "./service/process_stats_reporter";
 
 /** Normalize an Express query param into a string[] or undefined. */
 export function normalizeQueryArray(value: unknown): string[] | undefined {
@@ -172,6 +173,10 @@ const memoryGovernor = memoryGovernorConfig
    : null;
 memoryGovernor?.start();
 environmentStore.setMemoryGovernor(memoryGovernor);
+// Always-on process-stats heartbeat so we can correlate RSS / thread
+// counts / heap usage with traffic in prod. Logs every 30s at info.
+const processStatsReporter = new ProcessStatsReporter(memoryGovernor);
+processStatsReporter.start();
 const packageController = new PackageController(
    environmentStore,
    manifestService,

package/src/service/environment_store.ts

@@ -1,8 +1,8 @@
 import { GetObjectCommand, S3 } from "@aws-sdk/client-s3";
 import { Storage } from "@google-cloud/storage";
-import AdmZip from "adm-zip";
 import { Mutex } from "async-mutex";
 import crypto from "crypto";
+import extract from "extract-zip";
 import * as fs from "fs";
 import * as path from "path";
 import simpleGit from "simple-git";
@@ -703,6 +703,15 @@ export class EnvironmentStore {
    }
 
    public async getStatus() {
+      const memoryGovernorStatus = this.memoryGovernor?.getStatus() ?? null;
+      // Log every /status hit so we have a trace of RSS / back-pressure
+      // state to correlate against pod OOMs and request-driven leaks.
+      // Logged at info so it shows up in prod without LOG_LEVEL changes;
+      // the endpoint is low-frequency (monitoring/UI), so volume is fine.
+      logger.info("Memory governor status", {
+         memoryGovernor: memoryGovernorStatus,
+      });
+
       const status = {
          timestamp: Date.now(),
          environments: [] as Array<components["schemas"]["Environment"]>,
@@ -884,6 +893,7 @@ export class EnvironmentStore {
    }
 
    public async unzipEnvironment(absoluteEnvironmentPath: string) {
+      const startedAt = Date.now();
       logger.info(
          `Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`,
       );
@@ -897,8 +907,28 @@ export class EnvironmentStore {
       });
       await fs.promises.mkdir(unzippedEnvironmentPath, { recursive: true });
 
-      const zip = new AdmZip(absoluteEnvironmentPath);
-      zip.extractAllTo(unzippedEnvironmentPath, true);
+      // Stream-extract via yauzl (wrapped by extract-zip). Each entry's
+      // inflate and write are dispatched to the libuv thread pool, so the
+      // main event loop stays responsive even for very large archives.
+      // The previous adm-zip path used fs.readFileSync + zlib.inflateRawSync
+      // on the main thread, which parked the loop long enough on multi-
+      // hundred-MB packages to fail Kubernetes liveness probes mid-extract.
+      let entryCount = 0;
+      let totalUncompressedBytes = 0;
+      await extract(absoluteEnvironmentPath, {
+         dir: path.resolve(unzippedEnvironmentPath),
+         onEntry: (entry) => {
+            entryCount += 1;
+            totalUncompressedBytes += entry.uncompressedSize ?? 0;
+         },
+      });
+
+      const mib = (totalUncompressedBytes / (1024 * 1024)).toFixed(1);
+      logger.info(
+         `Unzipped "${absoluteEnvironmentPath}" -> "${unzippedEnvironmentPath}" ` +
+            `(${entryCount} entries, ${mib} MiB uncompressed) in ` +
+            `${formatDuration(Date.now() - startedAt)}`,
+      );
 
       return unzippedEnvironmentPath;
    }

package/src/service/model.ts

@@ -29,7 +29,6 @@ import * as fs from "fs/promises";
 import { createRequire } from "module";
 import * as path from "path";
 import { components } from "../api";
-import { getCompilePool } from "../compile/compile_pool";
 import {
    MODEL_FILE_SUFFIX,
    NOTEBOOK_FILE_SUFFIX,
@@ -141,31 +140,12 @@ interface RunnableNotebookCell {
    queryInfo?: Malloy.QueryInfo;
 }
 
-/**
- * Lazily produces a `ModelMaterializer` on demand. Used by the worker-
- * compile path: the worker returns a fully-built `modelDef` but cannot
- * ship the materializer (it binds to a Runtime that holds live native
- * connection handles and would not survive a structured-clone). The
- * first query that actually needs to execute calls this builder,
- * which constructs the materializer in-process. After construction
- * Malloy caches the compiled model internally on the materializer,
- * so subsequent queries pay no recompile cost.
- */
-type MaterializerBuilder = () => Promise<ModelMaterializer>;
-
 export class Model {
    private packageName: string;
    private modelPath: string;
    private dataStyles: DataStyles;
    private modelType: ModelType;
    private modelMaterializer: ModelMaterializer | undefined;
-   /**
-    * Lazy builder used when the model was compiled in a worker_threads
-    * worker. The first `getQueryResults`/`executeNotebookCell` call
-    * invokes this and caches the result in `modelMaterializer`.
-    */
-   private materializerBuilder: MaterializerBuilder | undefined;
-   private materializerBuildPromise: Promise<ModelMaterializer> | undefined;
    private modelDef: ModelDef | undefined;
    private modelInfo: Malloy.ModelInfo | undefined;
    private sources: ApiSource[] | undefined;
@@ -179,8 +159,6 @@ export class Model {
     *  `Model.givens` already collapses inheritance; we just stash the list
     *  for surfacing on the compiled-model response. */
    private givens: ApiGiven[] | undefined;
-   /** Cached responses from `getStandardModel()` so we don't re-stringify a multi-MB modelDef on every GET. */
-   private cachedStandardModel: ApiCompiledModel | undefined;
    private meter = metrics.getMeter("publisher");
    private queryExecutionHistogram = this.meter.createHistogram(
       "malloy_model_query_duration",
@@ -205,7 +183,6 @@ export class Model {
       compilationError: MalloyError | Error | undefined,
       filterMap?: Map<string, FilterDefinition[]>,
       givens?: ApiGiven[],
-      materializerBuilder?: MaterializerBuilder,
    ) {
       this.packageName = packageName;
       this.modelPath = modelPath;
@@ -213,7 +190,6 @@ export class Model {
       this.modelType = modelType;
       this.modelDef = modelDef;
       this.modelMaterializer = modelMaterializer;
-      this.materializerBuilder = materializerBuilder;
       this.sources = sources;
       this.queries = queries;
       this.sourceInfos = sourceInfos;
@@ -226,28 +202,6 @@ export class Model {
          : undefined;
    }
 
-   /**
-    * Resolve the in-process `ModelMaterializer`, building it lazily if
-    * the model was compiled in a worker_threads worker. Memoizes both
-    * the materializer and the in-flight build promise so concurrent
-    * queries on the same model share a single construction.
-    */
-   private async ensureMaterializer(): Promise<ModelMaterializer> {
-      if (this.modelMaterializer) return this.modelMaterializer;
-      if (!this.materializerBuilder) {
-         throw new BadRequestError("Model has no queryable entities.");
-      }
-      if (!this.materializerBuildPromise) {
-         this.materializerBuildPromise = this.materializerBuilder().then(
-            (mm) => {
-               this.modelMaterializer = mm;
-               return mm;
-            },
-         );
-      }
-      return this.materializerBuildPromise;
-   }
-
    /**
     * Get the parsed filter definitions for a given source name.
     * Returns an empty array if no filters are declared.
@@ -273,158 +227,6 @@ export class Model {
       modelPath: string,
       malloyConfig: ModelConnectionInput,
       options?: { buildManifest?: BuildManifest["entries"] },
-   ): Promise<Model> {
-      // Worker-pool fast path for plain `.malloy` files. Notebooks
-      // stay in-process for v1 — their per-cell ModelMaterializer
-      // chain is too entangled to ship across a worker boundary.
-      // The MALLOY_COMPILE_WORKERS=0 kill switch / pool.enabled check
-      // funnels everything through the legacy in-process path when
-      // the pool is disabled, so this is safe to land dark.
-      const pool = getCompilePool();
-      if (pool.enabled && modelPath.endsWith(MODEL_FILE_SUFFIX)) {
-         try {
-            return await Model.createViaWorker(
-               packageName,
-               packagePath,
-               modelPath,
-               malloyConfig,
-               pool,
-               options,
-            );
-         } catch (poolError) {
-            // Real compile errors propagate to the caller as a Model
-            // with `compilationError` populated, matching the
-            // in-process path's contract.
-            if (
-               poolError instanceof ModelCompilationError ||
-               poolError instanceof MalloyError
-            ) {
-               return Model.makeErrorModel(
-                  packageName,
-                  modelPath,
-                  poolError instanceof MalloyError
-                     ? new ModelCompilationError(poolError)
-                     : poolError,
-               );
-            }
-            // Anything else (worker exited, RPC timeout) — fall back
-            // to in-process compile so a transient pool failure
-            // doesn't take a package down.
-            logger.warn(
-               "Compile worker failed; falling back to in-process compile",
-               { packageName, modelPath, error: poolError },
-            );
-         }
-      }
-      return Model.createInProcess(
-         packageName,
-         packagePath,
-         modelPath,
-         malloyConfig,
-         options,
-      );
-   }
-
-   /**
-    * Compile via the {@link CompileWorkerPool}. Builds a `Model` whose
-    * `modelDef` / `sources` / `queries` / `sourceInfos` / `givens` are
-    * populated up-front, but whose `ModelMaterializer` is constructed
-    * lazily on the first query through {@link ensureMaterializer}.
-    * This keeps the heavy CPU work (parse, type-check, IR build) off
-    * the main event loop so the K8s liveness probe stays responsive.
-    */
-   private static async createViaWorker(
-      packageName: string,
-      packagePath: string,
-      modelPath: string,
-      malloyConfig: ModelConnectionInput,
-      pool: ReturnType<typeof getCompilePool>,
-      options?: { buildManifest?: BuildManifest["entries"] },
-   ): Promise<Model> {
-      const resolvedConfig = Model.toMalloyConfig(malloyConfig);
-      const outcome = await pool.compile({
-         packagePath,
-         modelPath,
-         malloyConfig: resolvedConfig,
-         // Package-level configs wrap a "duckdb" default; matches
-         // Package.buildPackageMalloyConfig.
-         defaultConnectionName: "duckdb",
-         urlReader: URL_READER,
-         buildManifest: options?.buildManifest,
-      });
-
-      // Materializer construction is deferred until a query actually
-      // runs. Build it the same way the in-process path does so
-      // execution semantics stay identical.
-      const materializerBuilder: MaterializerBuilder = async () => {
-         const { runtime, modelURL, importBaseURL } =
-            await Model.getModelRuntime(
-               packagePath,
-               modelPath,
-               malloyConfig,
-               options,
-            );
-         return Model.getStandardModelMaterializer(
-            runtime,
-            importBaseURL,
-            modelURL,
-            modelPath,
-         );
-      };
-
-      return new Model(
-         packageName,
-         modelPath,
-         {} as DataStyles,
-         "model",
-         undefined, // modelMaterializer — built lazily
-         outcome.modelDef,
-         outcome.sources as ApiSource[],
-         outcome.queries as ApiQuery[],
-         outcome.sourceInfos.length > 0 ? outcome.sourceInfos : undefined,
-         undefined, // runnableNotebookCells — .malloy is not a notebook
-         undefined, // compilationError
-         outcome.filterMap,
-         outcome.givens as ApiGiven[] | undefined,
-         materializerBuilder,
-      );
-   }
-
-   private static makeErrorModel(
-      packageName: string,
-      modelPath: string,
-      error: Error,
-   ): Model {
-      const isNotebook = modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
-      return new Model(
-         packageName,
-         modelPath,
-         {} as DataStyles,
-         isNotebook ? "notebook" : "model",
-         undefined,
-         undefined,
-         undefined,
-         undefined,
-         undefined,
-         undefined,
-         error,
-      );
-   }
-
-   /**
-    * Legacy in-process compile path. Retained for:
-    *   - notebooks (`.malloynb`), whose per-cell materializer chain
-    *     is too coupled to the Runtime to ship to a worker for v1.
-    *   - environments where MALLOY_COMPILE_WORKERS=0.
-    *   - fallback when the worker pool encounters a non-compile
-    *     failure (worker exit, RPC timeout).
-    */
-   private static async createInProcess(
-      packageName: string,
-      packagePath: string,
-      modelPath: string,
-      malloyConfig: ModelConnectionInput,
-      options?: { buildManifest?: BuildManifest["entries"] },
    ): Promise<Model> {
       // getModelRuntime might throw a ModelNotFoundError. It's the callers responsibility
       // to pass a valid model path or handle the error.
@@ -635,25 +437,9 @@ export class Model {
          );
       }
       let runnable: QueryMaterializer;
-      if (!this.modelDef || !this.modelInfo)
+      if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
          throw new BadRequestError("Model has no queryable entities.");
 
-      // Resolve the materializer — either already-built (in-process
-      // create path, or a previous query on this Model) or lazily
-      // constructed now (worker-compile path on first query).
-      let materializer: ModelMaterializer;
-      try {
-         materializer = await this.ensureMaterializer();
-      } catch (error) {
-         if (error instanceof BadRequestError) throw error;
-         if (error instanceof MalloyError) throw error;
-         throw new BadRequestError(
-            error instanceof Error
-               ? `Failed to prepare model: ${error.message}`
-               : "Failed to prepare model.",
-         );
-      }
-
       // Wrap loadQuery calls in try-catch to handle query parsing errors
       try {
          let queryString: string;
@@ -694,7 +480,7 @@ export class Model {
             }
          }
 
-         runnable = materializer.loadQuery(queryString);
+         runnable = this.modelMaterializer.loadQuery(queryString);
       } catch (error) {
          // Re-throw BadRequestError as-is
          if (error instanceof BadRequestError) {
@@ -783,14 +569,7 @@ export class Model {
    }
 
    private getStandardModel(): ApiCompiledModel {
-      // modelDef is immutable for the lifetime of this Model, so the
-      // (potentially multi-MB) JSON.stringify result can be memoised.
-      // Without this cache every `GET /environments/:e/packages/:p/
-      // models/:m` re-stringifies the whole tree on the main thread —
-      // a known source of multi-hundred-ms event-loop pauses that
-      // chips away at the K8s liveness budget.
-      if (this.cachedStandardModel) return this.cachedStandardModel;
-      const compiled: ApiCompiledModel = {
+      return {
          type: "source",
          packageName: this.packageName,
          modelPath: this.modelPath,
@@ -807,8 +586,6 @@ export class Model {
          queries: this.queries,
          givens: this.givens,
       } as ApiCompiledModel;
-      this.cachedStandardModel = compiled;
-      return compiled;
    }
 
    private async getNotebookModel(): Promise<ApiRawNotebook> {

package/src/service/package.spec.ts

@@ -1,5 +1,4 @@
 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-import { Stats } from "fs";
 import fs from "fs/promises";
 import { join, resolve } from "path";
 import sinon from "sinon";
@@ -336,12 +335,17 @@ describe("service/package", () => {
          });
       });
 
-      describe("getDatabaseInfo", () => {
-         it("should return the size of the database file", async () => {
-            sinon.stub(fs, "stat").resolves({ size: 13 } as Stats);
-
-            // @ts-expect-error Accessing private static method for testing
-            const info = await Package.getDatabaseInfo(
+      describe("schema introspection (via worker pool)", () => {
+         it("returns columns and rowCount for a csv database", async () => {
+            // Schema introspection moved off the main thread into a
+            // worker pool to isolate DuckDB's native thread pool (see
+            // schema_worker_pool.ts). Hit the pool directly here so
+            // the test exercises the same code path prod uses.
+            const { getSchemaWorkerPool } = await import(
+               "./schema_worker_pool"
+            );
+            const pool = getSchemaWorkerPool();
+            const info = await pool.submit(
                testPackageDirectory,
                "database.csv",
             );