@malloy-publisher/server 0.0.198-dev3 → 0.0.198-dev6

package/src/service/model.ts

@@ -29,6 +29,11 @@ import * as fs from "fs/promises";
 import { createRequire } from "module";
 import * as path from "path";
 import { components } from "../api";
+import { deserializeError } from "../package_load/package_load_pool";
+import type {
+   SerializedModel,
+   SerializedNotebookCell,
+} from "../package_load/protocol";
 import {
    MODEL_FILE_SUFFIX,
    NOTEBOOK_FILE_SUFFIX,
@@ -51,6 +56,7 @@ import {
    type FilterDefinition,
    type FilterParams,
 } from "./filter";
+import { malloyGivenToApi, type MalloyGiven } from "./given";
 
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiNotebookCell = components["schemas"]["NotebookCell"];
@@ -75,61 +81,6 @@ const MALLOY_VERSION = (
 export type ModelType = "model" | "notebook";
 type ModelConnectionInput = MalloyConfig | Map<string, Connection>;
 
-/**
- * Structural type for a Malloy SDK `Given` instance (the value type of
- * `Model.givens`). The `Given` class is declared in
- * `@malloydata/malloy/dist/api/foundation/core.d.ts` but is not re-exported
- * from the package root, so we duck-type against the surface we use rather
- * than importing it.
- */
-interface MalloyGiven {
-   readonly name: string;
-   readonly type: { type: string; filterType?: string };
-   getTaglines(prefix?: RegExp): string[];
-}
-
-/**
- * Convert a Malloy SDK `Given` (returned from `Model.givens`) to the wire
- * shape declared in `api-doc.yaml`. Two fields are deliberately not surfaced:
- *
- * - `location` — Malloy's `DocumentLocation.url` is an absolute `file://`
- *   path on the publisher's filesystem. Surfacing it would leak the OS user,
- *   install directory, and internal layout. Existing `Filter` introspection
- *   does not expose location either; matching that floor. A future PR can
- *   add a sanitized package-relative path if a client needs it.
- *
- * - `default` / `defaultText` — Malloy's API only exposes the parsed
- *   `ConstantExpr` AST, not a rendered source string. Rendering it here
- *   would duplicate the Malloy printer. Add when Malloy surfaces a
- *   stringified accessor.
- *
- * `annotations` is restricted to `#(...)` declaration annotations (the
- * caller-facing kind, e.g. `#(doc)`). `getTaglines()` with no prefix would
- * also return `##` doc-comment lines and the model-level `##!` pragma,
- * which aren't part of the given's surface contract.
- *
- * Type rendering: `GivenTypeDef` is typed as `AtomicTypeDef |
- * FilterExpressionParamTypeDef`, but Malloy's grammar only emits the
- * scalar parameter types (`string` | `number` | `boolean` | `date` |
- * `timestamp` | `timestamptz` | `filter expression` | `error`) for
- * given declarations today. If the grammar expands to allow array or
- * record givens, the bare `type.type` discriminator (`'array'`,
- * `'record'`) will land in the wire response with no element info —
- * revisit when that happens.
- */
-function malloyGivenToApi(given: MalloyGiven): ApiGiven {
-   const type = given.type;
-   const renderedType =
-      type.type === "filter expression"
-         ? `filter<${type.filterType}>`
-         : type.type;
-   return {
-      name: given.name,
-      type: renderedType,
-      annotations: given.getTaglines(/^#\(/),
-   };
-}
-
 interface RunnableNotebookCell {
    type: "code" | "markdown";
    text: string;
@@ -183,6 +134,15 @@ export class Model {
       compilationError: MalloyError | Error | undefined,
       filterMap?: Map<string, FilterDefinition[]>,
       givens?: ApiGiven[],
+      /**
+       * Precomputed `modelDefToModelInfo(modelDef)`. The package-load
+       * worker emits it as part of `SerializedModel` so we don't
+       * re-derive it on every package load. Callers that build a
+       * `Model` from a raw `modelDef` (e.g. test fixtures via
+       * `Model.create`) can omit this and let the constructor
+       * derive it lazily.
+       */
+      modelInfo?: Malloy.ModelInfo,
    ) {
       this.packageName = packageName;
       this.modelPath = modelPath;
@@ -197,9 +157,9 @@ export class Model {
       this.compilationError = compilationError;
       this.filterMap = filterMap ?? new Map();
       this.givens = givens;
-      this.modelInfo = this.modelDef
-         ? modelDefToModelInfo(this.modelDef)
-         : undefined;
+      this.modelInfo =
+         modelInfo ??
+         (this.modelDef ? modelDefToModelInfo(this.modelDef) : undefined);
    }
 
    /**
@@ -221,6 +181,15 @@ export class Model {
       return runMatch?.[1] ?? arrowMatch?.[1];
    }
 
+   /**
+    * Compile a single model in-process. Kept as a library entry point
+    * for test fixtures and any future caller that needs an ad-hoc
+    * `Model` from a `.malloy` / `.malloynb` file. Production package
+    * loads (`Package.create`) and reloads (`Package.reloadAllModels`)
+    * route through the package-load worker pool and dispatch through
+    * {@link Model.fromSerialized} instead — neither calls this on the
+    * main thread.
+    */
    public static async create(
       packageName: string,
       packagePath: string,
@@ -258,10 +227,12 @@ export class Model {
             modelDef = compiledModel._modelDef;
             // Malloy's `Model.givens` already collapses inheritance from imports
             // and applies any `finalizeGivens` runtime config. Just read it.
-            const malloyGivens = Array.from(compiledModel.givens.values());
+            const malloyGivens = Array.from(
+               compiledModel.givens.values(),
+            ) as MalloyGiven[];
             givens =
                malloyGivens.length > 0
-                  ? malloyGivens.map(malloyGivenToApi)
+                  ? (malloyGivens.map(malloyGivenToApi) as ApiGiven[])
                   : undefined;
             const sourceResult = Model.getSources(modelPath, modelDef, givens);
             sources = sourceResult.sources;
@@ -358,6 +329,123 @@ export class Model {
       }
    }
 
+   /**
+    * Construct a `Model` from a worker-compiled `SerializedModel`. All
+    * the heavy compile work (parse, type check, IR build, sourceInfo
+    * extraction, per-cell notebook compile) already ran inside a
+    * `worker_threads` worker; this factory just rewraps the wire data
+    * into a live `Model`.
+    *
+    * Hydrates the `ModelMaterializer` (and, for notebooks, the
+    * per-cell materializers + runnables) **eagerly** via
+    * `Runtime._loadModelFromModelDef` /
+    * `ModelMaterializer._loadQueryFromQueryDef`. These are constant-time
+    * wraps around the worker's pre-compiled `modelDef` / `queryDef` —
+    * no parse, no type-check, no schema fetch — so doing it here at
+    * package-load time costs microseconds per model and keeps the
+    * resulting `Model` interchangeable with one produced by
+    * `Model.create` (no lazy-init branches in the hot path).
+    */
+   public static fromSerialized(
+      packageName: string,
+      _packagePath: string,
+      malloyConfig: ModelConnectionInput,
+      data: SerializedModel,
+   ): Model {
+      const modelDef = data.modelDef as ModelDef | undefined;
+      const modelInfo = data.modelInfo as Malloy.ModelInfo | undefined;
+      const dataStyles = (data.dataStyles ?? {}) as DataStyles;
+      const sources = data.sources as ApiSource[] | undefined;
+      const queries = data.queries as ApiQuery[] | undefined;
+      const sourceInfos = data.sourceInfos as Malloy.SourceInfo[] | undefined;
+      const givens = data.givens as ApiGiven[] | undefined;
+      const filterMap = data.filterMap
+         ? new Map(data.filterMap as Array<[string, FilterDefinition[]]>)
+         : undefined;
+
+      // No modelDef → either an empty notebook (no MALLOY statements)
+      // or a corrupt worker payload. Build a Model with no materializer;
+      // downstream getQueryResults / executeNotebookCell will throw a
+      // clean BadRequestError if a caller tries to run a query. We
+      // still preserve markdown cells for an all-markdown notebook so
+      // `getNotebook()` can serve raw text.
+      if (!modelDef) {
+         return new Model(
+            packageName,
+            data.modelPath,
+            dataStyles,
+            data.modelType,
+            undefined,
+            undefined,
+            sources,
+            queries,
+            sourceInfos,
+            data.modelType === "notebook"
+               ? hydrateMarkdownOnlyCells(data.notebookCells)
+               : undefined,
+            undefined,
+            filterMap,
+            givens,
+            modelInfo,
+         );
+      }
+
+      const runtime = makeHydrationRuntime(malloyConfig);
+      const modelMaterializer = runtime._loadModelFromModelDef(modelDef);
+      const runnableNotebookCells =
+         data.modelType === "notebook"
+            ? hydrateNotebookCells(runtime, data.notebookCells)
+            : undefined;
+
+      return new Model(
+         packageName,
+         data.modelPath,
+         dataStyles,
+         data.modelType,
+         modelMaterializer,
+         modelDef,
+         sources,
+         queries,
+         sourceInfos,
+         runnableNotebookCells,
+         undefined, // compilationError
+         filterMap,
+         givens,
+         modelInfo,
+      );
+   }
+
+   /**
+    * Build a Model representing a compilation failure (no modelDef,
+    * no materializer). Matches the shape `Model.create` returns when
+    * it catches a `MalloyError`, so the rest of the system handles
+    * both paths uniformly (the iteration loop in `Package.create`
+    * reads `compilationError` via a structural cast).
+    */
+   public static fromCompilationError(
+      packageName: string,
+      modelPath: string,
+      modelType: ModelType,
+      error: Error,
+   ): Model {
+      return new Model(
+         packageName,
+         modelPath,
+         {} as DataStyles,
+         modelType,
+         undefined,
+         undefined,
+         undefined,
+         undefined,
+         undefined,
+         undefined,
+         error,
+      );
+   }
+
+   /** Look up the deserialized error helper for callers (e.g. Package.create). */
+   public static deserializeCompilationError = deserializeError;
+
    public getPath(): string {
       return this.modelPath;
    }
@@ -576,9 +664,10 @@ export class Model {
          malloyVersion: MALLOY_VERSION,
          dataStyles: JSON.stringify(this.dataStyles),
          modelDef: JSON.stringify(this.modelDef),
-         modelInfo: JSON.stringify(
-            this.modelDef ? modelDefToModelInfo(this.modelDef) : {},
-         ),
+         // `this.modelInfo` is precomputed once at construction (either
+         // by the worker or in the Model.create constructor); don't
+         // re-run `modelDefToModelInfo` on every API hit.
+         modelInfo: JSON.stringify(this.modelInfo ?? {}),
          sourceInfos: this.getSourceInfos()?.map((sourceInfo) =>
             JSON.stringify(sourceInfo),
          ),
@@ -630,9 +719,7 @@ export class Model {
          packageName: this.packageName,
          modelPath: this.modelPath,
          malloyVersion: MALLOY_VERSION,
-         modelInfo: JSON.stringify(
-            this.modelDef ? modelDefToModelInfo(this.modelDef) : {},
-         ),
+         modelInfo: JSON.stringify(this.modelInfo ?? {}),
          sources: this.modelDef && this.sources,
          queries: this.modelDef && this.queries,
          annotations: allAnnotations,
@@ -1152,3 +1239,103 @@ export class Model {
       }
    }
 }
+
+// ──────────────────────────────────────────────────────────────────────
+// Helpers for hydrating worker-compiled models on the main thread
+// ──────────────────────────────────────────────────────────────────────
+
+/**
+ * Minimal subset of `Runtime` we use here. The `_` methods are
+ * marked `@internal` in Malloy but are the only API for constructing
+ * a materializer / query materializer from an existing `modelDef` /
+ * queryDef — the public `loadModel(url)` path always recompiles.
+ */
+type HydrationRuntime = Runtime & {
+   _loadModelFromModelDef(modelDef: ModelDef): ModelMaterializer;
+};
+type HydrationMaterializer = ModelMaterializer & {
+   _loadQueryFromQueryDef(query: unknown): QueryMaterializer;
+};
+
+function makeHydrationRuntime(
+   malloyConfig: ModelConnectionInput,
+): HydrationRuntime {
+   const urlReader = new HackyDataStylesAccumulator(URL_READER);
+   const config =
+      malloyConfig instanceof MalloyConfig
+         ? malloyConfig
+         : (() => {
+              const c = new MalloyConfig({ connections: {} });
+              c.wrapConnections(
+                 () => new FixedConnectionMap(malloyConfig, "duckdb"),
+              );
+              return c;
+           })();
+   return new Runtime({ urlReader, config }) as HydrationRuntime;
+}
+
+/**
+ * Build the live `RunnableNotebookCell[]` from worker-emitted
+ * per-cell data. Each MALLOY cell is hydrated via
+ * `Runtime._loadModelFromModelDef` (for the cell's scope) and
+ * `ModelMaterializer._loadQueryFromQueryDef` (for the cell's
+ * runnable) — no recompile.
+ */
+function hydrateNotebookCells(
+   runtime: HydrationRuntime,
+   notebookCells: SerializedNotebookCell[] | undefined,
+): RunnableNotebookCell[] {
+   if (!notebookCells) return [];
+   return notebookCells.map((sc): RunnableNotebookCell => {
+      if (sc.type === "markdown") {
+         return { type: "markdown", text: sc.text };
+      }
+      const cellModelDef = sc.cellModelDef as ModelDef | undefined;
+      let modelMaterializer: ModelMaterializer | undefined;
+      let runnable: QueryMaterializer | undefined;
+      if (cellModelDef) {
+         modelMaterializer = runtime._loadModelFromModelDef(cellModelDef);
+         if (sc.cellQueryDef !== undefined) {
+            try {
+               runnable = (
+                  modelMaterializer as HydrationMaterializer
+               )._loadQueryFromQueryDef(sc.cellQueryDef);
+            } catch (error) {
+               // Hydration shouldn't fail for a queryDef the worker
+               // already prepared, but if Malloy's internal shape
+               // drifts we'd rather drop the runnable than crash the
+               // whole notebook. The cell remains markdown-runnable.
+               logger.warn("Failed to hydrate notebook cell queryDef", {
+                  error,
+               });
+            }
+         }
+      }
+      return {
+         type: "code",
+         text: sc.text,
+         runnable,
+         modelMaterializer,
+         newSources: sc.newSources as Malloy.SourceInfo[] | undefined,
+         queryInfo: sc.queryInfo as Malloy.QueryInfo | undefined,
+      };
+   });
+}
+
+/**
+ * For an all-markdown notebook (no MALLOY statements → no
+ * `modelDef`), we still want to preserve the cell list so
+ * `getNotebook()` can serve raw text. This skips materializer
+ * hydration (there's nothing to hydrate) and returns markdown-only
+ * cells.
+ */
+function hydrateMarkdownOnlyCells(
+   notebookCells: SerializedNotebookCell[] | undefined,
+): RunnableNotebookCell[] | undefined {
+   if (!notebookCells) return undefined;
+   return notebookCells.map((sc): RunnableNotebookCell => {
+      if (sc.type === "markdown") return { type: "markdown", text: sc.text };
+      // A code cell without a hydratable scope — surface text only.
+      return { type: "code", text: sc.text };
+   });
+}

package/src/service/package.spec.ts

@@ -1,4 +1,7 @@
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import "@malloydata/db-duckdb/native";
 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import { Stats } from "fs";
 import fs from "fs/promises";
 import { join, resolve } from "path";
 import sinon from "sinon";
@@ -335,19 +338,22 @@ describe("service/package", () => {
          });
       });
 
-      describe("schema introspection (via worker pool)", () => {
-         it("returns columns and rowCount for a csv database", async () => {
-            // Schema introspection moved off the main thread into a
-            // worker pool to isolate DuckDB's native thread pool (see
-            // schema_worker_pool.ts). Hit the pool directly here so
-            // the test exercises the same code path prod uses.
-            const { getSchemaWorkerPool } = await import(
-               "./schema_worker_pool"
-            );
-            const pool = getSchemaWorkerPool();
-            const info = await pool.submit(
+      describe("getDatabaseInfo", () => {
+         it("should return the size of the database file", async () => {
+            sinon.stub(fs, "stat").resolves({ size: 13 } as Stats);
+
+            // `getDatabaseInfo` now requires the caller to pass in the
+            // shared DuckDB connection (resolved once by `readDatabases`
+            // off the package's MalloyConfig). For this isolated unit
+            // test we mint a fresh ephemeral one — production paths
+            // reuse a single connection per package via `Package.create`.
+            const conn = new DuckDBConnection("duckdb");
+
+            // @ts-expect-error Accessing private static method for testing
+            const info = await Package.getDatabaseInfo(
                testPackageDirectory,
                "database.csv",
+               conn,
             );
 
             expect(info).toEqual({