@malloy-publisher/server 0.0.198-dev4 → 0.0.198-dev6

package/src/{compile/compile_worker.ts → package_load/package_load_worker.ts}

@@ -1,61 +1,103 @@
 /**
- * Compile worker entry point.
+ * Package-load worker entry point.
  *
- * Runs inside a worker_threads `Worker`. Owns no DuckDB / native
- * connection state — every schema lookup is proxied back to the
- * main thread via {@link SchemaForTablesRequest} / {@link
- * SchemaForSqlRequest}. The point of this file is to take the
- * dominant CPU cost of a Malloy compile (parser, type checker, IR
- * builder, sourceInfo extraction) off the main event loop so the
- * Kubernetes liveness probe on `/health/liveness` never gets parked
- * behind a multi-second compile.
+ * Runs inside a worker_threads `Worker`. Owns **no native connection
+ * state of any kind** — every connection lookup, schema fetch, SQL
+ * block schema, and non-file URL read is proxied back to the main
+ * thread via correlated RPC messages. The worker is intentionally
+ * pure-CPU: only Malloy parser / type-checker / IR-builder runs here.
  *
- * Contract:
- *   - Receives {@link CompileJobRequest} messages from the parent
- *     port. Dispatches one compile per message.
- *   - Proxies schema and URL-reader operations back to the parent
- *     via correlated RPC requests; awaits matching responses.
- *   - Sends back exactly one {@link CompileJobResult} or {@link
- *     CompileJobError} per job.
- *   - Honours a graceful {@link ShutdownRequest} so the pool can
- *     drain on SIGTERM.
+ * Why no DuckDB in the worker?
+ * ----------------------------
+ * DuckDB's native bindings cannot be safely loaded into more than one
+ * isolate of the same Node/Bun process (we hit Bun crash 0x20131 when
+ * the worker isolate and the main isolate both `dlopen` duckdb-native).
+ * Even if Bun fixes that, holding a duckdb handle in the worker
+ * duplicates the in-memory DB state and adds native-module load
+ * latency to every worker spawn. Database probing (`readDatabases`)
+ * stays on the main thread where it can reuse the package's existing
+ * DuckDB connection (see PR #772).
  *
- * This file is bundled separately by build.ts and shipped as
- * `dist/compile_worker.mjs`.
+ * Boundary
+ * --------
+ *   1. Worker: read `publisher.json` (package manifest).
+ *   2. Worker: compile every `.malloy` and `.malloynb` via Malloy.
+ *      All `lookupConnection(name)` / schema-fetch calls Malloy makes
+ *      during compile are proxied to the main thread's live
+ *      connection pool over RPC.
+ *   3. Worker: for each model, capture the structured-clonable fields
+ *      the main-thread `Model` constructor needs: `modelDef`,
+ *      `sourceInfos`, `sources`, `queries`, `filterMap`, `givens`,
+ *      dataStyles, plus per-cell `modelDef` + `queryDef` for
+ *      notebooks (so per-cell materializers / runnables can be
+ *      hydrated on the main thread via `Runtime._loadModelFromModelDef`
+ *      / `ModelMaterializer._loadQueryFromQueryDef` — no recompile).
+ *   4. Main thread: probe embedded `.parquet` / `.csv` databases
+ *      against the package's existing DuckDB connection.
+ *
+ * Per-model compile failures are returned in-band on
+ * `SerializedModel.compilationError`; the rest of the package keeps
+ * loading. Whole-package failures (e.g. manifest missing) come back
+ * as `LoadPackageError`.
+ *
+ * Bundled separately by `build.ts` as `dist/package_load_worker.mjs`
+ * so `new Worker(...)` can load it without dragging in the entire
+ * server module graph.
  */
 import {
    contextOverlay,
-   MalloyConfig,
-   MalloyError,
-   Runtime,
-   isSourceDef,
-   modelDefToModelInfo,
    type Annotation,
+   type BuildManifestEntry,
    type Connection,
    type FetchSchemaOptions,
    type LookupConnection,
+   MalloyConfig,
+   MalloyError,
    type ModelDef,
    type ModelMaterializer,
+   modelDefToModelInfo,
    type NamedModelObject,
    type NamedQueryDef,
+   type Query,
+   Runtime,
    type SQLSourceDef,
    type SQLSourceRequest,
    type StructDef,
    type TableSourceDef,
    type TurtleDef,
+   isSourceDef,
 } from "@malloydata/malloy";
 import * as Malloy from "@malloydata/malloy-interfaces";
+import {
+   MalloySQLParser,
+   MalloySQLStatementType,
+} from "@malloydata/malloy-sql";
 import * as fs from "fs";
+import * as path from "path";
 import { parentPort, threadId } from "node:worker_threads";
-import { fileURLToPath } from "url";
+import recursive from "recursive-readdir";
+import { fileURLToPath, pathToFileURL } from "url";
+
+import {
+   MODEL_FILE_SUFFIX,
+   NOTEBOOK_FILE_SUFFIX,
+   PACKAGE_MANIFEST_NAME,
+} from "../constants";
+import { HackyDataStylesAccumulator } from "../data_styles";
 import { parseFilters, type FilterDefinition } from "../service/filter";
+import {
+   malloyGivenToApi,
+   type MalloyGiven,
+   type MalloyGivenApi,
+} from "../service/given";
+import { ignoreDotfiles } from "../utils";
 import type {
-   CompileJobError,
-   CompileJobRequest,
-   CompileJobResult,
    ConnectionMetadata,
    ConnectionMetadataRequest,
    ConnectionMetadataResponse,
+   LoadPackageError,
+   LoadPackageRequest,
+   LoadPackageResult,
    MainToWorkerMessage,
    ReadUrlRequest,
    ReadUrlResponse,
@@ -64,11 +106,13 @@ import type {
    SchemaForTablesRequest,
    SchemaForTablesResponse,
    SerializedError,
+   SerializedModel,
+   SerializedNotebookCell,
 } from "./protocol";
 
 if (!parentPort) {
    throw new Error(
-      "compile_worker.ts must be loaded inside a worker_threads Worker",
+      "package_load_worker.ts must be loaded inside a worker_threads Worker",
    );
 }
 
@@ -123,17 +167,9 @@ function dispatchMainResponse(message: MainToWorkerMessage): void {
 }
 
 // ──────────────────────────────────────────────────────────────────────
-// Stub InfoConnection that proxies schema fetches to the main thread
+// Proxy connection: stand-in for non-duckdb connections at compile time
 // ──────────────────────────────────────────────────────────────────────
 
-/**
- * Minimal `Connection` implementation that satisfies Malloy's compile
- * pipeline. Only the methods called during compile are implemented
- * meaningfully; the rest throw, since the worker never executes SQL.
- *
- * Holds the `jobId` so the main thread can route the schema RPC to
- * the right environment-side `MalloyConfig`.
- */
 class ProxyConnection {
    public readonly name: string;
    public readonly dialectName: string;
@@ -190,21 +226,19 @@ class ProxyConnection {
          };
          port.postMessage(req);
       });
-      if (response.error !== undefined) {
-         return { error: response.error };
-      }
+      if (response.error !== undefined) return { error: response.error };
       if (response.structDef === undefined) {
          return { error: "Empty SQL schema response from main thread" };
       }
       return { structDef: response.structDef };
    }
 
-   // Compile path never calls these. We intentionally throw rather
-   // than silently no-op so a misrouted query in a worker surfaces
-   // as a loud error rather than a wrong-answer bug.
+   // Compile path never calls these on a non-duckdb connection (the
+   // worker doesn't execute non-duckdb SQL). We throw rather than no-op
+   // so a misrouted call surfaces loudly.
    async runSQL(): Promise<never> {
       throw new Error(
-         `ProxyConnection(${this.name}): runSQL is not available in compile workers`,
+         `ProxyConnection(${this.name}): runSQL is not available in package-load workers`,
       );
    }
    isPool(): false {
@@ -224,7 +258,7 @@ class ProxyConnection {
    }
    async estimateQueryCost(): Promise<never> {
       throw new Error(
-         `ProxyConnection(${this.name}): estimateQueryCost not available in compile workers`,
+         `ProxyConnection(${this.name}): estimateQueryCost not available in package-load workers`,
       );
    }
    async fetchMetadata(): Promise<Record<string, unknown>> {
@@ -250,19 +284,18 @@ function serializeFetchOptions(options: FetchSchemaOptions): {
 }
 
 // ──────────────────────────────────────────────────────────────────────
-// URLReader: try fs first, fall back to main-thread RPC for non-file URLs
+// URLReader: file:// → fs; everything else proxies to main thread
 // ──────────────────────────────────────────────────────────────────────
 
-function makeWorkerUrlReader(jobId: string) {
+function makeWorkerUrlReader(jobId: string): {
+   readURL: (url: URL) => Promise<string>;
+} {
    return {
       readURL: async (url: URL): Promise<string> => {
          if (url.protocol === "file:") {
             const filePath = fileURLToPath(url);
             return fs.promises.readFile(filePath, "utf8");
          }
-         // Non-file URL — delegate to main so semantics stay
-         // identical to the in-process URL_READER (e.g. allow
-         // future https:// imports).
          const response = await callMain<ReadUrlResponse>((requestId) => {
             const req: ReadUrlRequest = {
                type: "read-url",
@@ -281,25 +314,25 @@ function makeWorkerUrlReader(jobId: string) {
 // MalloyConfig assembly inside the worker
 // ──────────────────────────────────────────────────────────────────────
 
-function buildWorkerMalloyConfig(job: CompileJobRequest): MalloyConfig {
-   // Connections are resolved lazily on first lookup via a metadata
-   // RPC back to the main thread — see ConnectionMetadataRequest.
-   // We never enumerate the connection list upfront; the package
-   // layer doesn't always have one (e.g. environment-wrapped
-   // connections appear only when Malloy compiles a `table('...')`
-   // reference that names them).
-   //
-   // Concurrent lookups for the same name are deduped via
-   // `inflight` — Malloy's compile pipeline can fan-out multiple
-   // schema fetches that all hit `lookupConnection(name)` before
-   // any of them resolve, and we don't want to N-multiply the RPC.
+/**
+ * Build the package's MalloyConfig inside the worker. Every
+ * connection name — including `"duckdb"` — resolves to a
+ * {@link ProxyConnection} that RPCs schema fetches back to the main
+ * thread. The worker never holds a native connection handle, which
+ * keeps it pure-CPU and avoids dlopen'ing duckdb in a second isolate.
+ *
+ * Concurrent `lookupConnection(name)` calls for the same name are
+ * deduped via `inflight` — Malloy's compile pipeline can fan out
+ * many schema fetches that all hit `lookupConnection` before any
+ * resolve, and we don't want to N-multiply the metadata RPC.
+ */
+function buildWorkerMalloyConfig(job: LoadPackageRequest): MalloyConfig {
    const proxies = new Map<string, ProxyConnection>();
    const inflight = new Map<string, Promise<ProxyConnection>>();
+
    const config = new MalloyConfig(
       { connections: {} },
-      {
-         config: contextOverlay({ rootDirectory: job.packagePath }),
-      },
+      { config: contextOverlay({ rootDirectory: job.packagePath }) },
    );
    config.wrapConnections(
       (_base: LookupConnection<Connection>): LookupConnection<Connection> => ({
@@ -331,8 +364,7 @@ function buildWorkerMalloyConfig(job: CompileJobRequest): MalloyConfig {
                })();
                inflight.set(effectiveName, pending);
             }
-            const proxy = await pending;
-            return proxy as unknown as Connection;
+            return (await pending) as unknown as Connection;
          },
       }),
    );
@@ -340,96 +372,82 @@ function buildWorkerMalloyConfig(job: CompileJobRequest): MalloyConfig {
 }
 
 // ──────────────────────────────────────────────────────────────────────
-// The actual compile — mirrors Model.create's in-process flow but
-// only the parts that produce data shippable across postMessage.
+// Filesystem helpers (replicated from service/package.ts so the worker
+// stays decoupled from the main-thread service module graph)
 // ──────────────────────────────────────────────────────────────────────
 
-async function compile(job: CompileJobRequest): Promise<CompileJobResult> {
-   const compileStart = performance.now();
+async function readPackageMetadata(
+   packagePath: string,
+): Promise<{ name?: string; description?: string }> {
+   const manifestPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
+   const contents = await fs.promises.readFile(manifestPath, "utf8");
+   const parsed = JSON.parse(contents) as {
+      name?: string;
+      description?: string;
+   };
+   return { name: parsed.name, description: parsed.description };
+}
 
-   const malloyConfig = buildWorkerMalloyConfig(job);
-   const urlReader = makeWorkerUrlReader(job.requestId);
+async function listPackageFiles(packagePath: string): Promise<string[]> {
+   const files = await recursive(packagePath, [ignoreDotfiles]);
+   return files.map((full: string) =>
+      path.relative(packagePath, full).replace(/\\/g, "/"),
+   );
+}
 
-   const runtime = new Runtime({
-      urlReader,
-      config: malloyConfig,
-      // job.buildManifest is wire-typed as `unknown` because the
-      // worker protocol doesn't depend on Malloy types — assert
-      // the shape Malloy's Runtime expects here. We only pass the
-      // wrapper when the caller actually supplied entries.
-      buildManifest:
-         job.buildManifest !== undefined && job.buildManifest !== null
-            ? {
-                 entries: job.buildManifest as Record<
-                    string,
-                    import("@malloydata/malloy").BuildManifestEntry
-                 >,
-                 strict: false,
-              }
-            : undefined,
-   });
+function filterModelPaths(allRelative: string[]): string[] {
+   return allRelative.filter(
+      (p) => p.endsWith(MODEL_FILE_SUFFIX) || p.endsWith(NOTEBOOK_FILE_SUFFIX),
+   );
+}
 
-   // Two compile shapes:
-   //   1. File-backed: `modelPath` resolves to a file:// URL the runtime
-   //      reads via urlReader. The importBaseURL is the model's
-   //      directory.
-   //   2. Inline-source: `inlineSource` is a Malloy string the runtime
-   //      compiles directly. Mostly used by synthesized snippets like
-   //      `source: temp is duckdb.table('<path>')` from the package
-   //      database-info probe. We use the caller-provided importBaseURL
-   //      (or fall back to the package root) so any `import "…"`
-   //      statements in the snippet resolve correctly.
-   const isInline = typeof job.inlineSource === "string";
-   if (!isInline && typeof job.modelPath !== "string") {
-      throw new Error(
-         "CompileJobRequest must supply either inlineSource or modelPath",
-      );
-   }
-   const importBaseURL = isInline
-      ? new URL(job.importBaseURL ?? `file://${job.packagePath}/`)
-      : new URL(".", new URL(`file://${job.packagePath}/${job.modelPath}`));
-
-   const mm: ModelMaterializer = isInline
-      ? runtime.loadModel(job.inlineSource as string, { importBaseURL })
-      : runtime.loadModel(
-           new URL(`file://${job.packagePath}/${job.modelPath}`),
-           { importBaseURL },
-        );
-
-   const compiledModel = await mm.getModel();
-   const modelDef = compiledModel._modelDef as ModelDef;
-
-   // Givens — converted to API shape here so the main thread can
-   // stash them on the Model without needing Malloy's MalloyGiven
-   // type (which has non-serializable methods).
-   const malloyGivens = Array.from(compiledModel.givens.values());
-   const givens: ApiGivenWire[] | undefined =
-      malloyGivens.length > 0
-         ? malloyGivens.map((g) => malloyGivenToWire(g))
-         : undefined;
+// ──────────────────────────────────────────────────────────────────────
+// Model compile (mirrors service/model.ts but produces SerializedModel)
+// ──────────────────────────────────────────────────────────────────────
 
-   // Imported sourceInfos — mirrors Model.create line 199–242. We
-   // collect them here so the main thread doesn't have to recompile
-   // imports just to fill in the response.
+interface ApiSourceWire {
+   name: string;
+   annotations?: string[];
+   views?: { name: string; annotations?: string[] }[];
+   filters?: unknown[];
+   givens?: unknown[];
+}
+interface ApiQueryWire {
+   name: string;
+   sourceName?: string;
+   annotations?: string[];
+}
+type ApiGivenWire = MalloyGivenApi;
+
+async function collectImportedSourceInfos(
+   modelDef: ModelDef,
+   runtime: Runtime,
+   importBaseURL: URL,
+): Promise<{
+   sourceInfos: Malloy.SourceInfo[];
+   importedNames: Set<string>;
+}> {
    const sourceInfos: Malloy.SourceInfo[] = [];
-   const importedSourceNames = new Set<string>();
+   const importedNames = new Set<string>();
    const imports = modelDef.imports ?? [];
    for (const importLocation of imports) {
       try {
-         const modelString = await urlReader.readURL(
+         const modelString = await runtime.urlReader.readURL(
             new URL(importLocation.importURL),
          );
          const importedModelDef = (
-            await runtime.loadModel(modelString, { importBaseURL }).getModel()
+            await runtime
+               .loadModel(modelString as string, { importBaseURL })
+               .getModel()
          )._modelDef;
          const importedInfo = modelDefToModelInfo(importedModelDef);
          const importedSources = importedInfo.entries.filter(
             (entry) => entry.kind === "source",
          ) as Malloy.SourceInfo[];
          for (const source of importedSources) {
-            if (!importedSourceNames.has(source.name)) {
+            if (!importedNames.has(source.name)) {
                sourceInfos.push(source);
-               importedSourceNames.add(source.name);
+               importedNames.add(source.name);
             }
          }
       } catch {
@@ -437,102 +455,30 @@ async function compile(job: CompileJobRequest): Promise<CompileJobResult> {
          // of warning-and-skipping when an import can't be loaded.
       }
    }
+   return { sourceInfos, importedNames };
+}
+
+function appendLocalSourceInfos(
+   modelDef: ModelDef,
+   target: Malloy.SourceInfo[],
+   importedNames: Set<string>,
+): void {
    const localInfo = modelDefToModelInfo(modelDef);
    const localSources = localInfo.entries.filter(
       (entry) => entry.kind === "source",
    ) as Malloy.SourceInfo[];
    for (const source of localSources) {
-      if (!importedSourceNames.has(source.name)) {
-         sourceInfos.push(source);
-      }
+      if (!importedNames.has(source.name)) target.push(source);
    }
-
-   // Inline-source compiles have no on-disk modelPath. extractSources /
-   // extractQueries use modelPath only to filter annotations by the URL
-   // they came from; the inline path has no such annotations to filter,
-   // so `""` (matches everything via `includes`) is the correct neutral.
-   const modelPathForAnnotations = job.modelPath ?? "";
-   const { sources, filterMap } = extractSources(
-      modelPathForAnnotations,
-      modelDef,
-   );
-   const queries = extractQueries(modelPathForAnnotations, modelDef);
-   const filterMapEntries: Array<[string, FilterDefinition[]]> = Array.from(
-      filterMap.entries(),
-   );
-
-   return {
-      type: "compile-result",
-      requestId: job.requestId,
-      modelDef,
-      sourceInfos,
-      sources,
-      queries,
-      filterMap: filterMapEntries,
-      givens,
-      // dataStyles: the in-process HackyDataStylesAccumulator is fed
-      // by the URLReader. We don't reuse it here — main thread will
-      // accumulate its own when it builds the lazy materializer.
-      dataStyles: {},
-      compileDurationMs: performance.now() - compileStart,
-   };
-}
-
-/**
- * Wire-friendly mirror of the publisher's `ApiGiven`. Inlined here so
- * the worker doesn't import the OpenAPI-generated `components` map
- * (which would drag the whole api.ts surface into the worker bundle).
- * Kept structurally identical to `ApiGiven` so the main thread can
- * type-assert it without conversion.
- */
-interface ApiGivenWire {
-   name: string;
-   type: string;
-   annotations?: string[];
-}
-
-interface MalloyGivenLike {
-   name: string;
-   type: { type: string; filterType?: string };
-   getTaglines(regex: RegExp): string[];
-}
-
-function malloyGivenToWire(given: MalloyGivenLike): ApiGivenWire {
-   const t = given.type;
-   const renderedType =
-      t.type === "filter expression" ? `filter<${t.filterType}>` : t.type;
-   return {
-      name: given.name,
-      type: renderedType,
-      annotations: given.getTaglines(/^#\(/),
-   };
-}
-
-// ──────────────────────────────────────────────────────────────────────
-// extractSources / extractQueries — direct copies of the static
-// helpers in Model.ts. Inlined here to keep the worker independent
-// of the main-thread module graph (smaller bundle, fewer imports of
-// things that pull in DuckDB or AWS SDK by transitive include).
-// ──────────────────────────────────────────────────────────────────────
-
-interface ApiSource {
-   name: string;
-   annotations?: string[];
-   views?: { name: string; annotations?: string[] }[];
-   filters?: unknown[];
-}
-interface ApiQuery {
-   name: string;
-   sourceName?: string;
-   annotations?: string[];
 }
 
 function extractSources(
    modelPath: string,
    modelDef: ModelDef,
-): { sources: ApiSource[]; filterMap: Map<string, FilterDefinition[]> } {
+   givens: ApiGivenWire[] | undefined,
+): { sources: ApiSourceWire[]; filterMap: Map<string, FilterDefinition[]> } {
    const filterMap = new Map<string, FilterDefinition[]>();
-   const sources: ApiSource[] = Object.values(modelDef.contents)
+   const sources: ApiSourceWire[] = Object.values(modelDef.contents)
       .filter((obj) => isSourceDef(obj))
       .map((sourceObj) => {
          const sourceName =
@@ -572,8 +518,7 @@ function extractSources(
                   });
                }
             } catch {
-               // Mirrors the in-process behaviour: filter parse
-               // errors are warnings, not fatal compile failures.
+               /* parse errors are warnings; matches in-process */
             }
          }
 
@@ -596,13 +541,14 @@ function extractSources(
             annotations,
             views,
             filters,
-         } as ApiSource;
+            givens,
+         } as ApiSourceWire;
       });
 
    return { sources, filterMap };
 }
 
-function extractQueries(modelPath: string, modelDef: ModelDef): ApiQuery[] {
+function extractQueries(modelPath: string, modelDef: ModelDef): ApiQueryWire[] {
    const isNamedQuery = (obj: NamedModelObject): obj is NamedQueryDef =>
       obj.type === "query";
    return Object.values(modelDef.contents)
@@ -618,17 +564,333 @@ function extractQueries(modelPath: string, modelDef: ModelDef): ApiQuery[] {
       }));
 }
 
+function buildRuntimeForModel(
+   job: LoadPackageRequest,
+   malloyConfig: MalloyConfig,
+   jobId: string,
+): { runtime: Runtime; urlReader: HackyDataStylesAccumulator } {
+   const urlReader = new HackyDataStylesAccumulator(makeWorkerUrlReader(jobId));
+   const runtime = new Runtime({
+      urlReader,
+      config: malloyConfig,
+      buildManifest:
+         job.buildManifest !== undefined && job.buildManifest !== null
+            ? {
+                 entries: job.buildManifest as Record<
+                    string,
+                    BuildManifestEntry
+                 >,
+                 strict: false,
+              }
+            : undefined,
+   });
+   return { runtime, urlReader };
+}
+
+async function compileMalloyModel(
+   job: LoadPackageRequest,
+   malloyConfig: MalloyConfig,
+   modelPath: string,
+): Promise<SerializedModel> {
+   const compileStart = performance.now();
+   const fullPath = path.join(job.packagePath, modelPath);
+   // `pathToFileURL` produces a valid URL on every platform; the
+   // naïve `file://${fullPath}` template parses host=`D:` on Windows.
+   const modelURL = pathToFileURL(fullPath);
+   const importBaseURL = new URL(".", modelURL);
+
+   const { runtime, urlReader } = buildRuntimeForModel(
+      job,
+      malloyConfig,
+      job.requestId,
+   );
+   const mm = runtime.loadModel(modelURL, { importBaseURL });
+   const compiled = await mm.getModel();
+   const modelDef = compiled._modelDef;
+
+   const malloyGivens = Array.from(compiled.givens.values());
+   const givens =
+      malloyGivens.length > 0
+         ? malloyGivens.map((g) => malloyGivenToApi(g as MalloyGiven))
+         : undefined;
+
+   const { sourceInfos, importedNames } = await collectImportedSourceInfos(
+      modelDef,
+      runtime,
+      importBaseURL,
+   );
+   appendLocalSourceInfos(modelDef, sourceInfos, importedNames);
+
+   const { sources, filterMap } = extractSources(modelPath, modelDef, givens);
+   const queries = extractQueries(modelPath, modelDef);
+
+   return {
+      modelPath,
+      modelType: "model",
+      modelDef,
+      modelInfo: modelDefToModelInfo(modelDef),
+      sourceInfos,
+      sources,
+      queries,
+      filterMap: Array.from(filterMap.entries()),
+      givens,
+      dataStyles: urlReader.getHackyAccumulatedDataStyles(),
+      compileDurationMs: performance.now() - compileStart,
+   };
+}
+
+async function compileNotebookModel(
+   job: LoadPackageRequest,
+   malloyConfig: MalloyConfig,
+   modelPath: string,
+): Promise<SerializedModel> {
+   const compileStart = performance.now();
+   const fullPath = path.join(job.packagePath, modelPath);
+   // See compileMalloyModel above: `pathToFileURL` is the only
+   // cross-platform way to build a file URL from an OS path.
+   const modelURL = pathToFileURL(fullPath);
+   const importBaseURL = new URL(".", modelURL);
+
+   const { runtime, urlReader } = buildRuntimeForModel(
+      job,
+      malloyConfig,
+      job.requestId,
+   );
+
+   const fileContents = await fs.promises.readFile(modelURL, "utf8");
+   const parse = MalloySQLParser.parse(fileContents, modelPath);
+
+   // Build the extendModel chain synchronously so per-cell materializers
+   // line up with statement order. Matches the in-process flow in
+   // Model.getNotebookModelMaterializer.
+   let mm: ModelMaterializer | undefined;
+   const perCellMM: (ModelMaterializer | undefined)[] = parse.statements.map(
+      (stmt) => {
+         if (stmt.type === MalloySQLStatementType.MALLOY) {
+            mm =
+               mm === undefined
+                  ? runtime.loadModel(stmt.text, { importBaseURL })
+                  : mm.extendModel(stmt.text, { importBaseURL });
+         }
+         return mm;
+      },
+   );
+
+   const oldImports: string[] = [];
+   const oldSources: Record<string, Malloy.SourceInfo> = {};
+   const notebookCells: SerializedNotebookCell[] = [];
+   for (let i = 0; i < parse.statements.length; i++) {
+      const stmt = parse.statements[i];
+      if (stmt.type === MalloySQLStatementType.MARKDOWN) {
+         notebookCells.push({ type: "markdown", text: stmt.text });
+         continue;
+      }
+      if (stmt.type !== MalloySQLStatementType.MALLOY) continue;
+
+      const localMM = perCellMM[i];
+      if (!localMM) {
+         // Shouldn't happen for a MALLOY statement, but guard rather
+         // than crash a whole notebook compile on one corrupt cell.
+         continue;
+      }
+      const currentModelDef = (await localMM.getModel())._modelDef;
+
+      // newSources via the import chain — mirrors in-process logic.
+      let newSources: Malloy.SourceInfo[] = [];
+      const newImports = currentModelDef.imports?.slice(oldImports.length);
+      if (newImports) {
+         for (const importLocation of newImports) {
+            try {
+               const modelString = await runtime.urlReader.readURL(
+                  new URL(importLocation.importURL),
+               );
+               const importModel = (
+                  await runtime
+                     .loadModel(modelString as string, { importBaseURL })
+                     .getModel()
+               )._modelDef;
+               const importInfo = modelDefToModelInfo(importModel);
+               newSources = importInfo.entries
+                  .filter((e) => e.kind === "source")
+                  .filter(
+                     (s) => !(s.name in oldSources),
+                  ) as Malloy.SourceInfo[];
+               oldImports.push(importLocation.importURL.toString());
+            } catch {
+               // Same best-effort policy as the in-process path.
+            }
+         }
+      }
+      const currentInfo = modelDefToModelInfo(currentModelDef);
+      newSources = newSources.concat(
+         currentInfo.entries
+            .filter((e) => e.kind === "source")
+            .filter((s) => !(s.name in oldSources)) as Malloy.SourceInfo[],
+      );
+      for (const s of newSources) oldSources[s.name] = s;
+
+      // Capture the per-cell final-query queryDef so the main thread can
+      // hydrate a QueryMaterializer via
+      // `ModelMaterializer._loadQueryFromQueryDef` without a recompile.
+      const runnable = localMM.loadFinalQuery();
+      let cellQueryDef: Query | undefined;
+      let queryInfo: Malloy.QueryInfo | undefined;
+      try {
+         const prepared = await runnable.getPreparedQuery();
+         cellQueryDef = prepared._query;
+         const queryName =
+            (prepared._query as NamedQueryDef).as ||
+            (prepared._query as NamedQueryDef).name;
+         const anonymous =
+            currentInfo.anonymous_queries[
+               currentInfo.anonymous_queries.length - 1
+            ];
+         if (anonymous) {
+            queryInfo = {
+               name: queryName,
+               schema: anonymous.schema,
+               annotations: anonymous.annotations,
+               definition: anonymous.definition,
+               code: anonymous.code,
+               location: anonymous.location,
+            } as Malloy.QueryInfo;
+         }
+      } catch {
+         // Some cells (source-only) have no final query; that's fine.
+      }
+
+      notebookCells.push({
+         type: "code",
+         text: stmt.text,
+         cellModelDef: currentModelDef,
+         cellQueryDef,
+         newSources,
+         queryInfo,
+      });
+   }
+
+   // Aggregate (notebook-level) artifacts — derived from the final mm
+   // if any MALLOY statements were present. If the notebook is all
+   // markdown, modelDef stays undefined and the main thread treats
+   // this as a notebook with no compiled content.
+   let finalModelDef: ModelDef | undefined;
+   let finalSources: ApiSourceWire[] | undefined;
+   let finalQueries: ApiQueryWire[] | undefined;
+   let finalSourceInfos: Malloy.SourceInfo[] | undefined;
+   let finalFilterMap: Map<string, FilterDefinition[]> | undefined;
+   let finalGivens: ApiGivenWire[] | undefined;
+   if (mm) {
+      const compiled = await mm.getModel();
+      finalModelDef = compiled._modelDef;
+      const malloyGivens = Array.from(compiled.givens.values());
+      finalGivens =
+         malloyGivens.length > 0
+            ? malloyGivens.map((g) => malloyGivenToApi(g as MalloyGiven))
+            : undefined;
+      const collected = await collectImportedSourceInfos(
+         finalModelDef,
+         runtime,
+         importBaseURL,
+      );
+      appendLocalSourceInfos(
+         finalModelDef,
+         collected.sourceInfos,
+         collected.importedNames,
+      );
+      finalSourceInfos = collected.sourceInfos;
+      const extracted = extractSources(modelPath, finalModelDef, finalGivens);
+      finalSources = extracted.sources;
+      finalFilterMap = extracted.filterMap;
+      finalQueries = extractQueries(modelPath, finalModelDef);
+   }
+
+   return {
+      modelPath,
+      modelType: "notebook",
+      modelDef: finalModelDef,
+      modelInfo: finalModelDef ? modelDefToModelInfo(finalModelDef) : undefined,
+      sourceInfos: finalSourceInfos,
+      sources: finalSources,
+      queries: finalQueries,
+      filterMap: finalFilterMap
+         ? Array.from(finalFilterMap.entries())
+         : undefined,
+      givens: finalGivens,
+      notebookCells,
+      dataStyles: urlReader.getHackyAccumulatedDataStyles(),
+      compileDurationMs: performance.now() - compileStart,
+   };
+}
+
+async function compileOneModel(
+   job: LoadPackageRequest,
+   malloyConfig: MalloyConfig,
+   modelPath: string,
+): Promise<SerializedModel> {
+   try {
+      if (modelPath.endsWith(MODEL_FILE_SUFFIX)) {
+         return await compileMalloyModel(job, malloyConfig, modelPath);
+      }
+      if (modelPath.endsWith(NOTEBOOK_FILE_SUFFIX)) {
+         return await compileNotebookModel(job, malloyConfig, modelPath);
+      }
+      return {
+         modelPath,
+         modelType: "model",
+         compilationError: {
+            name: "Error",
+            message: `Unknown model file suffix: ${modelPath}`,
+         },
+      };
+   } catch (error) {
+      const modelType: SerializedModel["modelType"] = modelPath.endsWith(
+         NOTEBOOK_FILE_SUFFIX,
+      )
+         ? "notebook"
+         : "model";
+      return {
+         modelPath,
+         modelType,
+         compilationError: serializeError(error),
+      };
+   }
+}
+
+// ──────────────────────────────────────────────────────────────────────
+// The actual load-package job
+// ──────────────────────────────────────────────────────────────────────
+
+async function loadPackage(
+   job: LoadPackageRequest,
+): Promise<LoadPackageResult> {
+   const loadStart = performance.now();
+
+   const packageMetadata = await readPackageMetadata(job.packagePath);
+   const malloyConfig = buildWorkerMalloyConfig(job);
+
+   const allFiles = await listPackageFiles(job.packagePath);
+   const modelPaths = filterModelPaths(allFiles);
+   const models = await Promise.all(
+      modelPaths.map((modelPath) =>
+         compileOneModel(job, malloyConfig, modelPath),
+      ),
+   );
+
+   return {
+      type: "load-package-result",
+      requestId: job.requestId,
+      packageMetadata,
+      models,
+      loadDurationMs: performance.now() - loadStart,
+   };
+}
+
 // ──────────────────────────────────────────────────────────────────────
 // Error serialization
 // ──────────────────────────────────────────────────────────────────────
 
 function serializeError(error: unknown): SerializedError {
    if (error instanceof MalloyError) {
-      // MalloyError is what Malloy throws for compile failures
-      // (parse / type / unresolved-reference errors). Flagging
-      // `isCompilationError` lets the main thread re-wrap it as a
-      // `ModelCompilationError` so callers' instanceof checks for
-      // that type still fire after a worker-side compile.
       return {
          name: error.name,
          message: error.message,
@@ -664,20 +926,17 @@ const inFlightJobs = new Set<string>();
 port.on("message", (message: MainToWorkerMessage) => {
    if (message.type === "shutdown") {
       shuttingDown = true;
-      // Don't exit until in-flight jobs finish. Once empty we exit
-      // via the explicit process.exit() below; until then we just
-      // keep servicing message responses.
       maybeExit();
       return;
    }
-   if (message.type === "compile") {
+   if (message.type === "load-package") {
       if (shuttingDown) {
-         const errMsg: CompileJobError = {
-            type: "compile-error",
+         const errMsg: LoadPackageError = {
+            type: "load-package-error",
             requestId: message.requestId,
             error: {
                name: "ShuttingDown",
-               message: "Compile worker is shutting down",
+               message: "Package-load worker is shutting down",
             },
          };
          port.postMessage(errMsg);
@@ -690,13 +949,13 @@ port.on("message", (message: MainToWorkerMessage) => {
    dispatchMainResponse(message);
 });
 
-async function runJob(job: CompileJobRequest): Promise<void> {
+async function runJob(job: LoadPackageRequest): Promise<void> {
    try {
-      const result = await compile(job);
+      const result = await loadPackage(job);
       port.postMessage(result);
    } catch (error) {
-      const errMsg: CompileJobError = {
-         type: "compile-error",
+      const errMsg: LoadPackageError = {
+         type: "load-package-error",
          requestId: job.requestId,
          error: serializeError(error),
       };
@@ -715,7 +974,7 @@ function maybeExit(): void {
    }
 }
 
-// Announce readiness — the pool waits for this before dispatching
-// jobs to a newly-spawned worker so we don't race the worker's
-// module-init time.
+// Announce readiness — the pool waits for this before dispatching jobs
+// to a newly-spawned worker so we don't race the worker's module-init
+// time.
 port.postMessage({ type: "ready" });