@malloy-publisher/server 0.0.203 → 0.0.205

package/src/service/source_extraction.ts

@@ -0,0 +1,226 @@
+/**
+ * Shared source / query introspection extracted from a compiled `ModelDef`.
+ *
+ * Both the in-process `Model.create` path (`service/model.ts`) and the
+ * package-load worker (`package_load/package_load_worker.ts`, which runs in a
+ * separate bundle and serializes the result over the worker protocol) need to
+ * walk a `ModelDef` and produce the same `sources` / `queries` shapes plus the
+ * `#(filter)` `filterMap`. These two call sites used to carry byte-for-byte
+ * copies of this logic; keeping them in lockstep by hand was a standing hazard
+ * (a change to one silently diverged from the other). This module is the single
+ * source of truth — the two callers differ only in how they type the result
+ * (generated API types vs. worker wire types — structurally identical, so each
+ * casts at its boundary) and in how they report a filter parse failure (the
+ * service logs a warning; the worker has no logger and stays silent), which is
+ * threaded through the optional `onParseError` callback.
+ */
+
+import {
+   isSourceDef,
+   ModelDef,
+   NamedModelObject,
+   NamedQueryDef,
+   StructDef,
+   TurtleDef,
+} from "@malloydata/malloy";
+import { annotationTexts, modelAnnotations } from "./annotations";
+import { collectAuthorizeExprs, type AuthorizeMap } from "./authorize";
+import { parseFilters, type FilterDefinition } from "./filter";
+
+/** A `#(filter)` definition enriched with the dimension's Malloy type. */
+export interface ExtractedFilter {
+   name: string;
+   dimension: string;
+   type: string;
+   implicit: boolean;
+   required: boolean;
+   dimensionType: string | undefined;
+}
+
+export interface ExtractedView {
+   name: string;
+   annotations: string[] | undefined;
+}
+
+/**
+ * Structural source shape both callers cast to their own typed view
+ * (`ApiSource` in the service, `ApiSourceWire` in the worker). `givens` is
+ * attached verbatim from the caller-supplied list, so it stays `unknown` here.
+ */
+export interface ExtractedSource {
+   name: string;
+   annotations: string[] | undefined;
+   views: ExtractedView[];
+   filters: ExtractedFilter[] | undefined;
+   givens: unknown;
+   /**
+    * Effective `#(authorize)` / `##(authorize)` expressions gating this source:
+    * file-level expressions first, then the source's own. Undefined when the
+    * source carries no authorize annotations. Surfaced for introspection;
+    * enforcement happens server-side.
+    */
+   authorize: string[] | undefined;
+}
+
+export interface ExtractedQuery {
+   name: string;
+   sourceName: string | undefined;
+   annotations: string[] | undefined;
+}
+
+/**
+ * Extract every source from a compiled model, parsing `#(filter)` annotations
+ * along the way.
+ *
+ * Filters are collected by walking the `annotations.inherits` chain so that
+ * filters declared on a base source flow to an extending source. The chain runs
+ * child → parent, so we collect child-first then reverse — `parseFilters` uses
+ * "last wins" dedup, which lets a child's `#(filter)` override the base's.
+ *
+ * `givens` is attached unchanged to every source (Malloy exposes givens at the
+ * model level, not per-source). `onParseError`, when supplied, is invoked with
+ * the source name and error if a source's `#(filter)` annotations fail to parse;
+ * filter extraction then continues. Authorize parse errors are NOT routed here —
+ * they propagate (a malformed gate fails model load) so a security gate is never
+ * silently dropped.
+ *
+ * Authorize (`#(authorize)` / `##(authorize)`) is collected from the source's
+ * own `blockNotes` only — we do NOT walk the `inherits` chain. Note Malloy's
+ * behavior for `X is Y extend {...}`: if X declares its own `#(authorize)`,
+ * X.blockNotes holds only X's gates (Y's are dropped — the intended "curated
+ * re-exposure"); if X declares none, Malloy surfaces Y's blockNotes on X, so
+ * the base gate carries to the un-annotated extension (a safe default — a
+ * locked base stays locked unless an extension explicitly re-exposes itself).
+ * This carry happens through `blockNotes`, not the `inherits` chain, so reading
+ * own-blockNotes is sufficient. Joins are a separate concern and are not gated.
+ * The effective list per source is the file-level `##(authorize)` expressions
+ * (from `modelDef.annotations.notes`) followed by the source's own
+ * `#(authorize)` expressions, evaluated as one OR disjunction at request time.
+ */
+export function extractSourcesFromModelDef(
+   modelDef: ModelDef,
+   givens: unknown,
+   onParseError?: (sourceName: string, err: unknown) => void,
+): {
+   sources: ExtractedSource[];
+   filterMap: Map<string, FilterDefinition[]>;
+   authorizeMap: AuthorizeMap;
+} {
+   const filterMap = new Map<string, FilterDefinition[]>();
+   const authorizeMap: AuthorizeMap = new Map();
+
+   // File-level ##(authorize) is collected once and prepended to every source.
+   // Unlike filters, a malformed authorize annotation is NOT swallowed: the
+   // parse error propagates so the model fails to load loudly (caught per-model
+   // upstream and turned into a compilationError). Silently dropping a gate —
+   // and in the worker path there is no onParseError callback, so it would be
+   // truly silent — could leave a source that the author meant to lock looking
+   // unrestricted.
+   const fileLevelAuthorize = collectAuthorizeExprs(
+      (modelAnnotations(modelDef).notes ?? []).map((note) => note.text),
+   );
+
+   const sources: ExtractedSource[] = Object.values(modelDef.contents)
+      .filter((obj) => isSourceDef(obj))
+      .map((sourceObj) => {
+         const struct = sourceObj as StructDef;
+         const sourceName = struct.as || struct.name;
+         const annotations = annotationTexts(struct.annotations);
+
+         const collected: string[][] = [];
+         let cur = struct.annotations;
+         while (cur) {
+            if (cur.blockNotes) {
+               collected.push(cur.blockNotes.map((note) => note.text));
+            }
+            cur = cur.inherits;
+         }
+         const allAnnotations = collected.reverse().flat();
+
+         let filters: ExtractedFilter[] | undefined;
+         if (allAnnotations.length > 0) {
+            try {
+               const parsed = parseFilters(allAnnotations);
+               if (parsed.length > 0) {
+                  filterMap.set(sourceName, parsed);
+                  const fields = struct.fields;
+                  filters = parsed.map((f) => {
+                     const field = fields.find(
+                        (fd) => (fd.as || fd.name) === f.dimension,
+                     );
+                     return {
+                        name: f.name,
+                        dimension: f.dimension,
+                        type: f.type,
+                        implicit: f.implicit,
+                        required: f.required,
+                        dimensionType: field?.type as string | undefined,
+                     };
+                  });
+               }
+            } catch (err) {
+               onParseError?.(sourceName, err);
+            }
+         }
+
+         // Authorize: the source's OWN #(authorize) annotations only — no
+         // inherits walk. File-level ##(authorize) is prepended so file gates
+         // and source gates form one OR disjunction. A malformed annotation
+         // propagates (model fails to load) rather than silently dropping the
+         // gate — see the file-level note above.
+         const ownNotes = (struct.annotations?.blockNotes ?? []).map(
+            (note) => note.text,
+         );
+         const effective = [
+            ...fileLevelAuthorize,
+            ...collectAuthorizeExprs(ownNotes),
+         ];
+         let authorize: string[] | undefined;
+         if (effective.length > 0) {
+            authorizeMap.set(sourceName, effective);
+            authorize = effective;
+         }
+
+         const views: ExtractedView[] = struct.fields
+            .filter((field) => field.type === "turtle")
+            .filter((turtle) =>
+               // Filter out non-reduce views (e.g. indexes).
+               (turtle as TurtleDef).pipeline
+                  .map((stage) => stage.type)
+                  .every((type) => type === "reduce"),
+            )
+            .map((turtle) => ({
+               name: turtle.as || turtle.name,
+               annotations: annotationTexts(turtle.annotations),
+            }));
+
+         return {
+            name: sourceName,
+            annotations,
+            views,
+            filters,
+            givens,
+            authorize,
+         };
+      });
+
+   return { sources, filterMap, authorizeMap };
+}
+
+/** Extract every named query from a compiled model. */
+export function extractQueriesFromModelDef(
+   modelDef: ModelDef,
+): ExtractedQuery[] {
+   const isNamedQuery = (obj: NamedModelObject): obj is NamedQueryDef =>
+      obj.type === "query";
+   return Object.values(modelDef.contents)
+      .filter(isNamedQuery)
+      .map((queryObj) => ({
+         name: queryObj.as || queryObj.name,
+         sourceName:
+            typeof queryObj.structRef === "string"
+               ? queryObj.structRef
+               : undefined,
+         annotations: annotationTexts(queryObj.annotations),
+      }));
+}

package/src/storage/StorageManager.ts

@@ -1,5 +1,9 @@
 import { Mutex } from "async-mutex";
 import * as crypto from "crypto";
+import {
+   isCatalogVersionSupported,
+   SUPPORTED_CATALOG_VERSIONS,
+} from "../ducklake_version";
 import { ConnectionAuthError } from "../errors";
 import { logger } from "../logger";
 import {
@@ -64,6 +68,53 @@ function catalogNameForConfig(c: DuckLakeManifestConfig): string {
    return `manifest_lake_${hash}`;
 }
 
+// Read the catalog's recorded DuckLake format version from its
+// `ducklake_metadata` table via a plain postgres ATTACH (does NOT invoke
+// the DuckLake extension on the catalog). Returns the version string on
+// success, or `undefined` on any failure (missing table, query timeout,
+// connect failure) so the main ATTACH path stays the source of truth for
+// unrelated errors. Only meaningful for postgres-backed catalogs; the
+// caller must guard with `isPostgres`.
+async function readDuckLakeCatalogVersion(
+   connection: DuckDBConnection,
+   catalogUrl: string,
+   catalogName: string,
+): Promise<string | undefined> {
+   if (!catalogUrl.startsWith("postgres:")) {
+      return undefined;
+   }
+   const pgConnString = catalogUrl.slice("postgres:".length);
+   const tempDb = `${catalogName}_preflight`;
+   const escaped = escapeSQL(pgConnString);
+   try {
+      await connection.run(
+         `ATTACH '${escaped}' AS ${tempDb} (TYPE postgres, READ_ONLY);`,
+      );
+      const rows = await connection.all<{ value: string }>(
+         `SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`,
+      );
+      const value = rows[0]?.value;
+      return typeof value === "string" ? value : undefined;
+   } catch (error) {
+      logger.warn(
+         "DuckLake catalog version preflight failed; falling back to ATTACH",
+         {
+            catalogName,
+            error: redactPgSecrets(
+               error instanceof Error ? error.message : String(error),
+            ),
+         },
+      );
+      return undefined;
+   } finally {
+      try {
+         await connection.run(`DETACH ${tempDb};`);
+      } catch {
+         // ATTACH may have failed, so DETACH may have nothing to do.
+      }
+   }
+}
+
 /**
  * Manages the storage backend (DuckDB, Postgres, etc.) and per-environment
  * manifest stores. Environments without `materializationStorage` config use
@@ -222,6 +273,28 @@ export class StorageManager {
          await connection.run("INSTALL httpfs; LOAD httpfs;");
       }
 
+      // Preflight: read the catalog's recorded format version via the
+      // postgres extension (not DuckLake) and fail fast with a non-retryable
+      // 422 if the baked DuckLake extension can't read it. Without this,
+      // an unsupported catalog would surface as a generic DuckDB error
+      // from the ATTACH below, which retry loops misclassify as transient.
+      if (isPostgres) {
+         const catalogVersion = await readDuckLakeCatalogVersion(
+            connection,
+            catalogUrl,
+            catalogName,
+         );
+         if (catalogVersion && !isCatalogVersionSupported(catalogVersion)) {
+            const supportedMax =
+               SUPPORTED_CATALOG_VERSIONS[
+                  SUPPORTED_CATALOG_VERSIONS.length - 1
+               ];
+            throw new ConnectionAuthError(
+               `DuckLake catalog version ${catalogVersion} is newer than this Publisher's extension supports (max ${supportedMax}). Upgrade the Publisher image or downgrade the catalog.`,
+            );
+         }
+      }
+
       let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
       const attachOpts: string[] = [
          `DATA_PATH '${escapedDataPath}'`,

package/src/storage/duckdb/DuckDBConnection.ts

@@ -1,11 +1,30 @@
 import { Mutex } from "async-mutex";
-import duckdb from "duckdb";
+import {
+   DuckDBConnection as NeoConnection,
+   DuckDBInstance,
+   type DuckDBValue,
+} from "@duckdb/node-api";
 import * as path from "path";
 import { DatabaseConnection } from "../DatabaseInterface";
 
+/**
+ * Embedded persistence layer for the publisher's own metadata (environments,
+ * packages, connections, materializations, build manifests) in `publisher.db`.
+ *
+ * This is a plain DAO over a durable, exclusively-owned DuckDB handle -- it is
+ * deliberately NOT Malloy's `@malloydata/db-duckdb` connection, which is an
+ * analytical query connection (no prepared-statement parameter binding, a
+ * `:memory:` primary with ATTACH/DETACH/idle lifecycle, pooled/shared
+ * instances, and a poison-pill close). Those semantics are wrong for a
+ * source-of-truth store that must hold one handle open for the server's
+ * lifetime and run parameterized CRUD.
+ *
+ * It wraps `@duckdb/node-api` (the same DuckDB engine Malloy pulls in), so the
+ * repo carries a single DuckDB engine rather than a second, redundant driver.
+ */
 export class DuckDBConnection implements DatabaseConnection {
-   private db: duckdb.Database | null = null;
-   private connection: duckdb.Connection | null = null;
+   private instance: DuckDBInstance | null = null;
+   private connection: NeoConnection | null = null;
    private dbPath: string;
    private mutex: Mutex = new Mutex();
 
@@ -15,94 +34,47 @@ export class DuckDBConnection implements DatabaseConnection {
    }
 
    async initialize(): Promise<void> {
-      return new Promise((resolve, reject) => {
-         this.db = new duckdb.Database(this.dbPath, {}, (err) => {
-            if (err) {
-               console.error("Failed to create DuckDB database:", err);
-               reject(new Error(`Failed to initialize DuckDB: ${err.message}`));
-               return;
-            }
-
-            // Connect synchronously
-            this.connection = (
-               this.db as duckdb.Database & { connect(): duckdb.Connection }
-            ).connect();
-
-            if (!this.connection) {
-               reject(new Error("Failed to create connection object"));
-               return;
-            }
-
-            // Verify connection works
-            this.connection.all("SELECT 42 as answer", (testErr, _rows) => {
-               if (testErr) {
-                  console.error("Connection test failed:", testErr);
-                  reject(
-                     new Error(
-                        `Failed to verify DuckDB connection: ${testErr.message}`,
-                     ),
-                  );
-                  return;
-               }
-
-               resolve();
-            });
-         });
-      });
+      try {
+         this.instance = await DuckDBInstance.create(this.dbPath);
+         this.connection = await this.instance.connect();
+         // Verify the connection works
+         await this.connection.run("SELECT 42 as answer");
+      } catch (err) {
+         const message = err instanceof Error ? err.message : String(err);
+         console.error("Failed to create DuckDB database:", err);
+         throw new Error(`Failed to initialize DuckDB: ${message}`);
+      }
    }
 
    async close(): Promise<void> {
-      return new Promise((resolve, reject) => {
+      try {
          if (this.connection) {
-            this.connection.close((err) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Failed to close DuckDB connection: ${err.message}`,
-                     ),
-                  );
-                  return;
-               }
-
-               if (this.db) {
-                  this.db.close((dbErr) => {
-                     if (dbErr) {
-                        reject(
-                           new Error(
-                              `Failed to close DuckDB: ${dbErr.message}`,
-                           ),
-                        );
-                        return;
-                     }
-                     console.log("DuckDB connection closed");
-                     resolve();
-                  });
-               } else {
-                  resolve();
-               }
-            });
-         } else {
-            resolve();
+            this.connection.closeSync();
+            this.connection = null;
          }
-      });
+         if (this.instance) {
+            this.instance.closeSync();
+            this.instance = null;
+         }
+         console.log("DuckDB connection closed");
+      } catch (err) {
+         const message = err instanceof Error ? err.message : String(err);
+         throw new Error(`Failed to close DuckDB connection: ${message}`);
+      }
    }
 
    async isInitialized(): Promise<boolean> {
       if (!this.connection) return false;
 
       return this.mutex.runExclusive(async () => {
-         return new Promise<boolean>((resolve) => {
-            this.connection!.all(
+         try {
+            const reader = await this.connection!.runAndReadAll(
                "SELECT name FROM sqlite_master WHERE type='table' AND name='environments'",
-               (err, rows) => {
-                  if (err) {
-                     resolve(false);
-                     return;
-                  }
-                  resolve(rows && rows.length > 0);
-               },
             );
-         });
+            return reader.getRowObjectsJS().length > 0;
+         } catch {
+            return false;
+         }
       });
    }
 
@@ -112,26 +84,14 @@ export class DuckDBConnection implements DatabaseConnection {
       }
 
       return this.mutex.runExclusive(async () => {
-         return new Promise<void>((resolve, reject) => {
-            const callback = (err: Error | null) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Query execution failed: ${err.message}\nQuery: ${query}`,
-                     ),
-                  );
-                  return;
-               }
-               resolve();
-            };
-
-            // Pass params directly without the params argument if empty
-            if (params && params.length > 0) {
-               this.connection!.run(query, ...params, callback);
-            } else {
-               this.connection!.run(query, callback);
-            }
-         });
+         try {
+            await this.connection!.run(query, params as DuckDBValue[]);
+         } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            throw new Error(
+               `Query execution failed: ${message}\nQuery: ${query}`,
+            );
+         }
       });
    }
 
@@ -141,25 +101,18 @@ export class DuckDBConnection implements DatabaseConnection {
       }
 
       return this.mutex.runExclusive(async () => {
-         return new Promise<T[]>((resolve, reject) => {
-            const callback = (err: Error | null, rows: unknown[]) => {
-               if (err) {
-                  reject(
-                     new Error(
-                        `Query execution failed: ${err.message}\nQuery: ${query}`,
-                     ),
-                  );
-                  return;
-               }
-               resolve((rows || []) as T[]);
-            };
-
-            if (params && params.length > 0) {
-               this.connection!.all(query, ...params, callback);
-            } else {
-               this.connection!.all(query, callback);
-            }
-         });
+         try {
+            const reader = await this.connection!.runAndReadAll(
+               query,
+               params as DuckDBValue[],
+            );
+            return reader.getRowObjectsJS() as T[];
+         } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            throw new Error(
+               `Query execution failed: ${message}\nQuery: ${query}`,
+            );
+         }
       });
    }
 
@@ -167,11 +120,4 @@ export class DuckDBConnection implements DatabaseConnection {
       const rows = await this.all<T>(query, params);
       return rows.length > 0 ? rows[0] : null;
    }
-
-   getConnection(): duckdb.Connection {
-      if (!this.connection) {
-         throw new Error("Database not initialized");
-      }
-      return this.connection;
-   }
 }

package/tests/fixtures/authorize-compile/model.malloy

@@ -0,0 +1,9 @@
+##! experimental.givens
+
+given:
+  ROLE :: string
+
+#(authorize) "$ROLE = 'analyst'"
+source: gated is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }
+
+source: open_src is duckdb.sql("SELECT 1 as x") extend { measure: c is count() }

package/tests/fixtures/authorize-compile/publisher.json

@@ -0,0 +1,4 @@
+{
+   "name": "authorize-compile",
+   "description": "Fixture: a gated source for the /compile authorize HTTP integration test."
+}

package/tests/fixtures/html-pages-nopublic/model.malloy

@@ -0,0 +1 @@
+source: nums is duckdb.sql("SELECT 1 as n")

package/tests/fixtures/html-pages-nopublic/publisher.json

@@ -0,0 +1,5 @@
+{
+  "name": "html-pages-nopublic",
+  "version": "0.0.1",
+  "description": "Package with no public/ directory; static file requests return 404."
+}

package/tests/fixtures/html-pages-test/data.csv

@@ -0,0 +1,3 @@
+id,name
+1,alpha
+2,beta

package/tests/fixtures/html-pages-test/public/assets/app.css

@@ -0,0 +1,3 @@
+body {
+   font-family: system-ui, sans-serif;
+}

package/tests/fixtures/html-pages-test/public/data.json

@@ -0,0 +1 @@
+{ "ok": true }

package/tests/fixtures/html-pages-test/public/index.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+   <head>
+      <title>Carrier Dashboard</title>
+   </head>
+   <body>
+      <h1>Hello from the in-package data app</h1>
+   </body>
+</html>

package/tests/fixtures/html-pages-test/public/sub/page2.html

@@ -0,0 +1,9 @@
+<!doctype html>
+<html>
+   <head>
+      <title>Second Page</title>
+   </head>
+   <body>
+      <p>A page in a subdirectory.</p>
+   </body>
+</html>

package/tests/fixtures/html-pages-test/publisher.json

@@ -0,0 +1,5 @@
+{
+  "name": "html-pages-test",
+  "version": "1.0.0",
+  "description": "Fixture for in-package HTML static-file serving + /pages + /events"
+}

package/tests/fixtures/html-pages-test/report.malloy

@@ -0,0 +1 @@
+source: report is duckdb.sql("SELECT 1 as n")