@malloy-publisher/server 0.0.198-dev → 0.0.198-dev2

package/src/service/model.spec.ts

@@ -234,6 +234,111 @@ describe("service/model", () => {
 
             sinon.restore();
          });
+
+         it("forwards givens to runnable.getPreparedResult and .run", async () => {
+            const givensArg = { region: "EU" };
+            const preparedResultStub = sinon
+               .stub()
+               .resolves({ resultExplore: { limit: 10 } });
+            const runStub = sinon
+               .stub()
+               .rejects(new MalloyError("stub-stop", []));
+            const modelMaterializer = {
+               loadQuery: sinon.stub().returns({
+                  getPreparedResult: preparedResultStub,
+                  run: runStub,
+               }),
+            };
+
+            const model = new Model(
+               packageName,
+               mockModelPath,
+               {},
+               "model",
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               modelMaterializer as any,
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               { contents: {}, exports: [], queryList: [] } as any,
+               undefined,
+               undefined,
+               undefined,
+               undefined,
+               undefined,
+            );
+
+            await expect(
+               model.getQueryResults(
+                  undefined,
+                  undefined,
+                  "run: orders -> summary",
+                  undefined,
+                  undefined,
+                  givensArg,
+               ),
+            ).rejects.toThrow(MalloyError);
+
+            expect(preparedResultStub.calledOnce).toBe(true);
+            expect(preparedResultStub.firstCall.args[0]).toEqual({
+               givens: givensArg,
+            });
+            expect(runStub.firstCall.args[0]).toMatchObject({
+               givens: givensArg,
+            });
+
+            sinon.restore();
+         });
+      });
+
+      describe("executeNotebookCell", () => {
+         it("forwards givens to runnable.getPreparedResult and .run", async () => {
+            const givensArg = { target_code: "AA" };
+            const preparedResultStub = sinon
+               .stub()
+               .resolves({ resultExplore: { limit: 10 } });
+            const runStub = sinon
+               .stub()
+               .rejects(new MalloyError("stub-stop", []));
+            const cellRunnable = {
+               getPreparedResult: preparedResultStub,
+               run: runStub,
+            };
+            const runnableCells = [
+               {
+                  type: "code" as const,
+                  text: "run: orders -> by_code",
+                  runnable: cellRunnable,
+               },
+            ];
+
+            const model = new Model(
+               packageName,
+               "test.malloynb",
+               {},
+               "notebook",
+               undefined,
+               undefined,
+               undefined,
+               undefined,
+               undefined,
+               // eslint-disable-next-line @typescript-eslint/no-explicit-any
+               runnableCells as any,
+               undefined,
+            );
+
+            await expect(
+               model.executeNotebookCell(0, undefined, undefined, givensArg),
+            ).rejects.toThrow(MalloyError);
+
+            expect(preparedResultStub.calledOnce).toBe(true);
+            expect(preparedResultStub.firstCall.args[0]).toEqual({
+               givens: givensArg,
+            });
+            expect(runStub.firstCall.args[0]).toMatchObject({
+               givens: givensArg,
+            });
+
+            sinon.restore();
+         });
       });
    });
 

package/src/service/model.ts

@@ -3,6 +3,7 @@ import {
    API,
    Connection,
    FixedConnectionMap,
+   GivenValue,
    isSourceDef,
    MalloyConfig,
    MalloyError,
@@ -56,6 +57,7 @@ type ApiNotebookCell = components["schemas"]["NotebookCell"];
 type ApiRawNotebook = components["schemas"]["RawNotebook"];
 type ApiSource = components["schemas"]["Source"];
 type ApiFilter = components["schemas"]["Filter"];
+type ApiGiven = components["schemas"]["Given"];
 type ApiView = components["schemas"]["View"];
 type ApiQuery = components["schemas"]["Query"];
 export type ApiConnection = components["schemas"]["Connection"];
@@ -73,6 +75,61 @@ const MALLOY_VERSION = (
 export type ModelType = "model" | "notebook";
 type ModelConnectionInput = MalloyConfig | Map<string, Connection>;
 
+/**
+ * Structural type for a Malloy SDK `Given` instance (the value type of
+ * `Model.givens`). The `Given` class is declared in
+ * `@malloydata/malloy/dist/api/foundation/core.d.ts` but is not re-exported
+ * from the package root, so we duck-type against the surface we use rather
+ * than importing it.
+ */
+interface MalloyGiven {
+   readonly name: string;
+   readonly type: { type: string; filterType?: string };
+   getTaglines(prefix?: RegExp): string[];
+}
+
+/**
+ * Convert a Malloy SDK `Given` (returned from `Model.givens`) to the wire
+ * shape declared in `api-doc.yaml`. Two fields are deliberately not surfaced:
+ *
+ * - `location` — Malloy's `DocumentLocation.url` is an absolute `file://`
+ *   path on the publisher's filesystem. Surfacing it would leak the OS user,
+ *   install directory, and internal layout. Existing `Filter` introspection
+ *   does not expose location either; matching that floor. A future PR can
+ *   add a sanitized package-relative path if a client needs it.
+ *
+ * - `default` / `defaultText` — Malloy's API only exposes the parsed
+ *   `ConstantExpr` AST, not a rendered source string. Rendering it here
+ *   would duplicate the Malloy printer. Add when Malloy surfaces a
+ *   stringified accessor.
+ *
+ * `annotations` is restricted to `#(...)` declaration annotations (the
+ * caller-facing kind, e.g. `#(doc)`). `getTaglines()` with no prefix would
+ * also return `##` doc-comment lines and the model-level `##!` pragma,
+ * which aren't part of the given's surface contract.
+ *
+ * Type rendering: `GivenTypeDef` is typed as `AtomicTypeDef |
+ * FilterExpressionParamTypeDef`, but Malloy's grammar only emits the
+ * scalar parameter types (`string` | `number` | `boolean` | `date` |
+ * `timestamp` | `timestamptz` | `filter expression` | `error`) for
+ * given declarations today. If the grammar expands to allow array or
+ * record givens, the bare `type.type` discriminator (`'array'`,
+ * `'record'`) will land in the wire response with no element info —
+ * revisit when that happens.
+ */
+function malloyGivenToApi(given: MalloyGiven): ApiGiven {
+   const type = given.type;
+   const renderedType =
+      type.type === "filter expression"
+         ? `filter<${type.filterType}>`
+         : type.type;
+   return {
+      name: given.name,
+      type: renderedType,
+      annotations: given.getTaglines(/^#\(/),
+   };
+}
+
 interface RunnableNotebookCell {
    type: "code" | "markdown";
    text: string;
@@ -98,6 +155,10 @@ export class Model {
    private compilationError: MalloyError | Error | undefined;
    /** Parsed #(filter) definitions keyed by source name. */
    private filterMap: Map<string, FilterDefinition[]>;
+   /** Givens declared on the model, in declaration order. Malloy's
+    *  `Model.givens` already collapses inheritance; we just stash the list
+    *  for surfacing on the compiled-model response. */
+   private givens: ApiGiven[] | undefined;
    private meter = metrics.getMeter("publisher");
    private queryExecutionHistogram = this.meter.createHistogram(
       "malloy_model_query_duration",
@@ -121,6 +182,7 @@ export class Model {
       runnableNotebookCells: RunnableNotebookCell[] | undefined,
       compilationError: MalloyError | Error | undefined,
       filterMap?: Map<string, FilterDefinition[]>,
+      givens?: ApiGiven[],
    ) {
       this.packageName = packageName;
       this.modelPath = modelPath;
@@ -134,6 +196,7 @@ export class Model {
       this.runnableNotebookCells = runnableNotebookCells;
       this.compilationError = compilationError;
       this.filterMap = filterMap ?? new Map();
+      this.givens = givens;
       this.modelInfo = this.modelDef
          ? modelDefToModelInfo(this.modelDef)
          : undefined;
@@ -188,10 +251,19 @@ export class Model {
          let sources = undefined;
          let queries = undefined;
          let filterMap: Map<string, FilterDefinition[]> | undefined;
+         let givens: ApiGiven[] | undefined;
          const sourceInfos: Malloy.SourceInfo[] = [];
          if (modelMaterializer) {
-            modelDef = (await modelMaterializer.getModel())._modelDef;
-            const sourceResult = Model.getSources(modelPath, modelDef);
+            const compiledModel = await modelMaterializer.getModel();
+            modelDef = compiledModel._modelDef;
+            // Malloy's `Model.givens` already collapses inheritance from imports
+            // and applies any `finalizeGivens` runtime config. Just read it.
+            const malloyGivens = Array.from(compiledModel.givens.values());
+            givens =
+               malloyGivens.length > 0
+                  ? malloyGivens.map(malloyGivenToApi)
+                  : undefined;
+            const sourceResult = Model.getSources(modelPath, modelDef, givens);
             sources = sourceResult.sources;
             filterMap = sourceResult.filterMap;
             queries = Model.getQueries(modelPath, modelDef);
@@ -255,6 +327,7 @@ export class Model {
             runnableNotebookCells,
             undefined,
             filterMap,
+            givens,
          );
       } catch (error) {
          let computedError = error;
@@ -342,6 +415,7 @@ export class Model {
       query?: string,
       filterParams?: FilterParams,
       bypassFilters?: boolean,
+      givens?: Record<string, GivenValue>,
    ): Promise<{
       result: Malloy.Result;
       compactResult: QueryData;
@@ -436,13 +510,14 @@ export class Model {
       }
 
       const rowLimit =
-         (await runnable.getPreparedResult()).resultExplore.limit || ROW_LIMIT;
+         (await runnable.getPreparedResult({ givens })).resultExplore.limit ||
+         ROW_LIMIT;
       const endTime = performance.now();
       const executionTime = endTime - startTime;
 
       let queryResults;
       try {
-         queryResults = await runnable.run({ rowLimit });
+         queryResults = await runnable.run({ rowLimit, givens });
       } catch (error) {
          // Record error metrics
          const errorEndTime = performance.now();
@@ -509,6 +584,7 @@ export class Model {
          ),
          sources: this.sources,
          queries: this.queries,
+         givens: this.givens,
       } as ApiCompiledModel;
    }
 
@@ -568,6 +644,7 @@ export class Model {
       cellIndex: number,
       filterParams?: FilterParams,
       bypassFilters?: boolean,
+      givens?: Record<string, GivenValue>,
    ): Promise<{
       type: "code" | "markdown";
       text: string;
@@ -629,9 +706,9 @@ export class Model {
             }
 
             const rowLimit =
-               (await runnableToExecute.getPreparedResult()).resultExplore
-                  .limit || ROW_LIMIT;
-            const result = await runnableToExecute.run({ rowLimit });
+               (await runnableToExecute.getPreparedResult({ givens }))
+                  .resultExplore.limit || ROW_LIMIT;
+            const result = await runnableToExecute.run({ rowLimit, givens });
             const query = (await runnableToExecute.getPreparedQuery())._query;
             queryName = (query as NamedQueryDef).as || query.name;
             queryResult =
@@ -758,6 +835,7 @@ export class Model {
    private static getSources(
       modelPath: string,
       modelDef: ModelDef,
+      givens?: ApiGiven[],
    ): {
       sources: ApiSource[];
       filterMap: Map<string, FilterDefinition[]>;
@@ -846,6 +924,12 @@ export class Model {
                annotations,
                views,
                filters,
+               // Malloy exposes givens at the model level, not per-source.
+               // First pass: surface the full model-level list on every source
+               // — matches how filter introspection already collapses
+               // inheritance into the per-source list. Refine to view-scoped
+               // filtering if a customer asks.
+               givens,
             } as ApiSource;
          });
 

package/src/service/package.spec.ts

@@ -1,5 +1,4 @@
 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-import { Stats } from "fs";
 import fs from "fs/promises";
 import { join, resolve } from "path";
 import sinon from "sinon";
@@ -336,12 +335,17 @@ describe("service/package", () => {
          });
       });
 
-      describe("getDatabaseInfo", () => {
-         it("should return the size of the database file", async () => {
-            sinon.stub(fs, "stat").resolves({ size: 13 } as Stats);
-
-            // @ts-expect-error Accessing private static method for testing
-            const info = await Package.getDatabaseInfo(
+      describe("schema introspection (via worker pool)", () => {
+         it("returns columns and rowCount for a csv database", async () => {
+            // Schema introspection moved off the main thread into a
+            // worker pool to isolate DuckDB's native thread pool (see
+            // schema_worker_pool.ts). Hit the pool directly here so
+            // the test exercises the same code path prod uses.
+            const { getSchemaWorkerPool } = await import(
+               "./schema_worker_pool"
+            );
+            const pool = getSchemaWorkerPool();
+            const info = await pool.submit(
                testPackageDirectory,
                "database.csv",
             );

package/src/service/package.ts

@@ -1,16 +1,11 @@
 import * as fs from "fs/promises";
 import * as path from "path";
 
-import { DuckDBConnection } from "@malloydata/db-duckdb";
-import "@malloydata/db-duckdb/native";
 import {
    Connection,
-   ConnectionRuntime,
    contextOverlay,
-   EmptyURLReader,
    FixedConnectionMap,
    MalloyConfig,
-   SourceDef,
 } from "@malloydata/malloy";
 import { metrics } from "@opentelemetry/api";
 import recursive from "recursive-readdir";
@@ -24,14 +19,14 @@ import {
 import { PackageNotFoundError } from "../errors";
 import { formatDuration, logger } from "../logger";
 import { BuildManifest } from "../storage/DatabaseInterface";
+import { ignoreDotfiles } from "../utils";
 import { Model } from "./model";
+import { getSchemaWorkerPool } from "./schema_worker_pool";
 
 type ApiDatabase = components["schemas"]["Database"];
 type ApiModel = components["schemas"]["Model"];
 type ApiNotebook = components["schemas"]["Notebook"];
 export type ApiPackage = components["schemas"]["Package"];
-type ApiColumn = components["schemas"]["Column"];
-type ApiTableDescription = components["schemas"]["TableDescription"];
 // A thunk lets callers pass a live reference to the *current* environment
 // MalloyConfig so the package wrapper resolves environment connections against the
 // generation that's active at lookup time, not the one that was current when
@@ -42,6 +37,7 @@ type PackageConnectionInput =
    | (() => MalloyConfig);
 
 const ENABLE_LIST_MODEL_COMPILATION = true;
+
 export class Package {
    private environmentName: string;
    private packageName: string;
@@ -91,6 +87,8 @@ export class Package {
          duration: formatDuration(manifestValidationTime - startTime),
       });
 
+      let packageMalloyConfig: MalloyConfig | undefined;
+
       try {
          const packageConfig = await Package.readPackageConfig(packagePath);
          const packageConfigTime = performance.now();
@@ -179,6 +177,17 @@ export class Package {
             malloy_package_name: packageName,
             status: "error",
          });
+
+         if (packageMalloyConfig) {
+            try {
+               await packageMalloyConfig.shutdown("close");
+            } catch (releaseError) {
+               logger.warn(
+                  `Failed to release package-local DuckDB for ${packageName}`,
+                  { error: releaseError },
+               );
+            }
+         }
          // Clean up package directory on failure
          try {
             await fs.rm(packagePath, {
@@ -380,7 +389,7 @@ export class Package {
    private static async getModelPaths(packagePath: string): Promise<string[]> {
       let files = undefined;
       try {
-         files = await recursive(packagePath);
+         files = await recursive(packagePath, [ignoreDotfiles]);
       } catch (error) {
          logger.error(error);
          throw new PackageNotFoundError(
@@ -428,29 +437,49 @@ export class Package {
    private static async readDatabases(
       packagePath: string,
    ): Promise<ApiDatabase[]> {
-      return await Promise.all(
-         (await Package.getDatabasePaths(packagePath)).map(
-            async (databasePath) => {
-               const databaseInfo = await Package.getDatabaseInfo(
-                  packagePath,
-                  databasePath,
-               );
-
-               return {
-                  path: databasePath,
-                  info: databaseInfo,
-                  type: "embedded",
-               };
-            },
+      const databasePaths = await Package.getDatabasePaths(packagePath);
+      if (databasePaths.length === 0) return [];
+
+      // Off-main-thread: schema introspection runs in the
+      // SchemaWorkerPool so DuckDB's native thread pool lives inside
+      // a worker we control. This is the leak class that OOM-killed
+      // prod (466 leaked Bun Pool threads on worker-76b49bdb89-8bsv4)
+      // — worker isolation puts a hard ceiling on per-package native
+      // thread usage and the worker's connection is reused across all
+      // schema queries for the life of the process.
+      const pool = getSchemaWorkerPool();
+      const settled = await Promise.allSettled(
+         databasePaths.map((databasePath) =>
+            pool.submit(packagePath, databasePath),
          ),
       );
+
+      const results: ApiDatabase[] = [];
+      for (let i = 0; i < settled.length; i++) {
+         const outcome = settled[i];
+         if (outcome.status === "fulfilled") {
+            results.push({
+               path: databasePaths[i],
+               info: outcome.value,
+               type: "embedded",
+            });
+         } else {
+            // A single bad parquet (corrupt footer, unsupported type)
+            // must not fail the whole package load. Log and skip.
+            logger.warn("Schema introspection failed for database", {
+               packagePath,
+               databasePath: databasePaths[i],
+               error: outcome.reason,
+            });
+         }
+      }
+      return results;
    }
 
    private static async getDatabasePaths(
       packagePath: string,
    ): Promise<string[]> {
-      let files = undefined;
-      files = await recursive(packagePath);
+      const files = await recursive(packagePath, [ignoreDotfiles]);
       return files
          .map((fullPath: string) => {
             return path.relative(packagePath, fullPath).replace(/\\/g, "/");
@@ -461,38 +490,6 @@ export class Package {
          );
    }
 
-   private static async getDatabaseInfo(
-      packagePath: string,
-      databasePath: string,
-   ): Promise<ApiTableDescription> {
-      const fullPath = path.join(packagePath, databasePath);
-
-      // Create a DuckDB source then:
-      // 1. Load the model and get the table schema from model
-      // 2. Run a query to get the row count from the table
-      const runtime = new ConnectionRuntime({
-         urlReader: new EmptyURLReader(),
-         connections: [new DuckDBConnection("duckdb")],
-      });
-      // Normalize path to use forward slashes for cross-platform compatibility
-      // DuckDB on Windows supports forward slashes, and this avoids escaping issues
-      const normalizedPath = fullPath.replace(/\\/g, "/");
-      const model = runtime.loadModel(
-         `source: temp is duckdb.table('${normalizedPath}')`,
-      );
-      const modelDef = await model.getModel();
-      const fields = (modelDef._modelDef.contents["temp"] as SourceDef).fields;
-      const schema = fields.map((field): ApiColumn => {
-         return { type: field.type, name: field.name };
-      });
-      const runner = model.loadQuery(
-         "run: temp->{aggregate: row_count is count()}",
-      );
-      const result = await runner.run();
-      const rowCount = result.data.value[0].row_count?.valueOf() as number;
-      return { name: databasePath, rowCount, columns: schema };
-   }
-
    public setName(name: string) {
       this.packageName = name;
    }