@malloy-publisher/server 0.0.182 → 0.0.183-dev

package/src/service/model.ts

@@ -22,7 +22,7 @@ import {
    MalloySQLParser,
    MalloySQLStatementType,
 } from "@malloydata/malloy-sql";
-import malloyPackage from "@malloydata/malloy/package.json";
+import { createRequire } from "module";
 import { DataStyles } from "@malloydata/render";
 import { metrics } from "@opentelemetry/api";
 import * as fs from "fs/promises";
@@ -41,12 +41,13 @@ import {
    ModelNotFoundError,
 } from "../errors";
 import { logger } from "../logger";
+import { BuildManifest } from "../storage/DatabaseInterface";
 import { URL_READER } from "../utils";
 import {
    buildFilterClause,
+   FilterValidationError,
    injectFilterRefinement,
    parseFilters,
-   FilterValidationError,
    type FilterDefinition,
    type FilterParams,
 } from "./filter";
@@ -64,7 +65,11 @@ export type PostgresConnection = components["schemas"]["PostgresConnection"];
 export type BigqueryConnection = components["schemas"]["BigqueryConnection"];
 export type TrinoConnection = components["schemas"]["TrinoConnection"];
 
-const MALLOY_VERSION = malloyPackage.version;
+const MALLOY_VERSION = (
+   createRequire(import.meta.url)("@malloydata/malloy/package.json") as {
+      version: string;
+   }
+).version;
 
 export type ModelType = "model" | "notebook";
 
@@ -158,11 +163,17 @@ export class Model {
       packagePath: string,
       modelPath: string,
       connections: Map<string, Connection>,
+      options?: { buildManifest?: BuildManifest["entries"] },
    ): Promise<Model> {
       // getModelRuntime might throw a ModelNotFoundError. It's the callers responsibility
       // to pass a valid model path or handle the error.
       const { runtime, modelURL, importBaseURL, dataStyles, modelType } =
-         await Model.getModelRuntime(packagePath, modelPath, connections);
+         await Model.getModelRuntime(
+            packagePath,
+            modelPath,
+            connections,
+            options,
+         );
 
       try {
          const { modelMaterializer, runnableNotebookCells } =
@@ -291,7 +302,7 @@ export class Model {
    }
 
    public getQueries(): ApiQuery[] | undefined {
-      return this.sources;
+      return this.queries;
    }
 
    public async getModel(): Promise<ApiCompiledModel> {
@@ -662,6 +673,7 @@ export class Model {
       packagePath: string,
       modelPath: string,
       connections: Map<string, Connection>,
+      options?: { buildManifest?: BuildManifest["entries"] },
    ): Promise<{
       runtime: Runtime;
       modelURL: URL;
@@ -701,10 +713,23 @@ export class Model {
          `SET FILE_SEARCH_PATH='${workingDirectory}';`,
       );
 
-      const runtime = new Runtime({
+      const runtimeOptions: {
+         urlReader: typeof urlReader;
+         connections: FixedConnectionMap;
+         buildManifest?: BuildManifest;
+      } = {
          urlReader,
          connections: new FixedConnectionMap(connections, "duckdb"),
-      });
+      };
+
+      if (options?.buildManifest) {
+         runtimeOptions.buildManifest = {
+            entries: options.buildManifest,
+            strict: false,
+         };
+      }
+
+      const runtime = new Runtime(runtimeOptions);
       const dataStyles = urlReader.getHackyAccumulatedDataStyles();
       return { runtime, modelURL, importBaseURL, dataStyles, modelType };
    }
@@ -750,13 +775,29 @@ export class Model {
                ?.filter((note) => note.at.url.includes(modelPath))
                .map((note) => note.text);
 
-            // Parse #(filter) from ALL annotations (including imports)
-            // so filters defined on an imported source are honored by notebooks
-            const allAnnotations = (
-               sourceObj as StructDef
-            ).annotation?.blockNotes?.map((note) => note.text);
+            // Parse #(filter) from ALL annotations, traversing the inherits
+            // chain so that filters on a base source (e.g. `recalls`) are
+            // picked up by an extending source (`manufacturer_recalls is
+            // recalls extend {}`).  The Malloy compiler stores the base
+            // source's annotations in `annotation.inherits`.
+            //
+            // The chain goes child → parent, so we collect child-first.
+            // parseFilters uses "last wins" dedup, so we reverse to put
+            // parent annotations first and child annotations last (winning).
+            const collectedAnnotations: string[][] = [];
+            let curAnnotation: Annotation | undefined = (sourceObj as StructDef)
+               .annotation;
+            while (curAnnotation) {
+               if (curAnnotation.blockNotes) {
+                  collectedAnnotations.push(
+                     curAnnotation.blockNotes.map((note) => note.text),
+                  );
+               }
+               curAnnotation = curAnnotation.inherits;
+            }
+            const allAnnotations = collectedAnnotations.reverse().flat();
             let filters: ApiFilter[] | undefined;
-            if (allAnnotations && allAnnotations.length > 0) {
+            if (allAnnotations.length > 0) {
                try {
                   const parsed = parseFilters(allAnnotations);
                   if (parsed.length > 0) {

package/src/service/package.ts

@@ -19,6 +19,7 @@ import {
 } from "../constants";
 import { PackageNotFoundError } from "../errors";
 import { formatDuration, logger } from "../logger";
+import { BuildManifest } from "../storage/DatabaseInterface";
 import { Model } from "./model";
 
 type ApiDatabase = components["schemas"]["Database"];
@@ -189,6 +190,10 @@ export class Package {
       return this.packageName;
    }
 
+   public getPackagePath(): string {
+      return this.packagePath;
+   }
+
    public getPackageMetadata(): ApiPackage {
       return this.packageMetadata;
    }
@@ -201,6 +206,51 @@ export class Package {
       return this.models.get(modelPath);
    }
 
+   public getModelPaths(): string[] {
+      return Array.from(this.models.keys());
+   }
+
+   /**
+    * Recompile every model in the package with the given build manifest
+    * so queries resolve persist references to materialized tables.
+    *
+    * Builds a fresh map off to the side and swaps it in at the end. If any
+    * recompile fails the whole call rejects before the swap and the live
+    * `this.models` reference remains untouched — no half-loaded state is
+    * ever observable to concurrent readers.
+    */
+   public async reloadAllModels(
+      buildManifest: BuildManifest["entries"],
+   ): Promise<void> {
+      const modelPaths = Array.from(this.models.keys());
+      logger.info("Reloading all models with build manifest", {
+         packageName: this.packageName,
+         modelCount: modelPaths.length,
+         manifestEntryCount: Object.keys(buildManifest).length,
+      });
+
+      const reloaded = await Promise.all(
+         modelPaths.map((modelPath) =>
+            Model.create(
+               this.packageName,
+               this.packagePath,
+               modelPath,
+               this.connections,
+               { buildManifest },
+            ),
+         ),
+      );
+      const nextModels = new Map<string, Model>();
+      for (const model of reloaded) {
+         nextModels.set(model.getPath(), model);
+      }
+      this.models = nextModels;
+   }
+
+   public getConnections(): Map<string, Connection> {
+      return this.connections;
+   }
+
    public getMalloyConnection(connectionName: string): Connection {
       const connection = this.connections.get(connectionName);
       if (!connection) {

package/src/service/project_store.ts

@@ -320,6 +320,7 @@ export class ProjectStore {
       };
       const existingProject = await repository.getProjectByName(projectName);
 
+      let dbProject: { id: string; name: string };
       if (existingProject) {
          const updateData = {
             description: projectDescription,
@@ -327,10 +328,27 @@ export class ProjectStore {
          };
 
          await repository.updateProject(existingProject.id, updateData);
-         return { id: existingProject.id, name: projectName };
+         dbProject = { id: existingProject.id, name: projectName };
       } else {
-         return await repository.createProject(projectData);
+         dbProject = await repository.createProject(projectData);
       }
+
+      // Initialize DuckLake manifest storage if configured on the project.
+      const materializationStorage = project.metadata
+         ?.materializationStorage as
+         | { catalogUrl?: string; dataPath?: string }
+         | undefined;
+      if (
+         materializationStorage?.catalogUrl &&
+         materializationStorage?.dataPath
+      ) {
+         await this.storageManager.initializeDuckLakeForProject(dbProject.id, {
+            catalogUrl: materializationStorage.catalogUrl,
+            dataPath: materializationStorage.dataPath,
+         });
+      }
+
+      return dbProject;
    }
 
    private async addPackages(
@@ -917,6 +935,7 @@ export class ProjectStore {
    private isLocalPath(location: string) {
       return (
          location.startsWith("./") ||
+         location.startsWith("../") ||
          location.startsWith("~/") ||
          location.startsWith("/") ||
          path.isAbsolute(location)

package/src/service/quoting.ts

@@ -0,0 +1,41 @@
+/**
+ * Minimal identifier-quoting surface. Every `Dialect` in `@malloydata/malloy`
+ * implements this; we accept the duck type so tests can inject a fake without
+ * instantiating a full dialect.
+ */
+export interface Quoter {
+   quoteTablePath(seg: string): string;
+}
+
+/**
+ * Quote a potentially schema-qualified table path (e.g. "schema.table")
+ * by quoting each segment individually with the dialect's quoteTablePath.
+ */
+export function quoteTablePath(path: string, dialect: Quoter): string {
+   return path
+      .split(".")
+      .map((seg) => dialect.quoteTablePath(seg))
+      .join(".");
+}
+
+/**
+ * Split a possibly schema-qualified table name into its schema prefix
+ * (including the trailing dot) and the bare table name.
+ *
+ * Examples:
+ *   "my_schema.my_table" -> { schemaPrefix: "my_schema.", bareName: "my_table" }
+ *   "my_table"           -> { schemaPrefix: "", bareName: "my_table" }
+ */
+export function splitTablePath(tableName: string): {
+   schemaPrefix: string;
+   bareName: string;
+} {
+   const lastDot = tableName.lastIndexOf(".");
+   if (lastDot >= 0) {
+      return {
+         schemaPrefix: tableName.substring(0, lastDot + 1),
+         bareName: tableName.substring(lastDot + 1),
+      };
+   }
+   return { schemaPrefix: "", bareName: tableName };
+}

package/src/service/resolve_project.ts

@@ -0,0 +1,13 @@
+import { ProjectNotFoundError } from "../errors";
+import { ResourceRepository } from "../storage/DatabaseInterface";
+
+export async function resolveProjectId(
+   repository: ResourceRepository,
+   projectName: string,
+): Promise<string> {
+   const dbProject = await repository.getProjectByName(projectName);
+   if (!dbProject) {
+      throw new ProjectNotFoundError(`Project '${projectName}' not found`);
+   }
+   return dbProject.id;
+}

package/src/storage/DatabaseInterface.ts

@@ -34,7 +34,7 @@ export interface ResourceRepository {
    getConnectionById(id: string): Promise<Connection | null>;
    getConnectionByName(
       projectId: string,
-      id: string,
+      name: string,
    ): Promise<Connection | null>;
    createConnection(
       connection: Omit<Connection, "id" | "createdAt" | "updatedAt">,
@@ -44,6 +44,44 @@ export interface ResourceRepository {
       updates: Partial<Connection>,
    ): Promise<Connection>;
    deleteConnection(id: string): Promise<void>;
+
+   // Materializations
+   listMaterializations(
+      projectId: string,
+      packageName: string,
+      options?: { limit?: number; offset?: number },
+   ): Promise<Materialization[]>;
+   getMaterializationById(id: string): Promise<Materialization | null>;
+   getActiveMaterialization(
+      projectId: string,
+      packageName: string,
+   ): Promise<Materialization | null>;
+   createMaterialization(
+      projectId: string,
+      packageName: string,
+      status?: MaterializationStatus,
+      metadata?: Record<string, unknown> | null,
+   ): Promise<Materialization>;
+   updateMaterialization(
+      id: string,
+      updates: {
+         status?: MaterializationStatus;
+         startedAt?: Date;
+         completedAt?: Date;
+         error?: string | null;
+         metadata?: Record<string, unknown> | null;
+      },
+   ): Promise<Materialization>;
+   deleteMaterialization(id: string): Promise<void>;
+   // Build Manifests
+   listManifestEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]>;
+   upsertManifestEntry(
+      entry: Omit<ManifestEntry, "id" | "createdAt" | "updatedAt">,
+   ): Promise<ManifestEntry>;
+   deleteManifestEntry(id: string): Promise<void>;
 }
 
 export interface Project {
@@ -76,3 +114,67 @@ export interface Connection {
    createdAt: Date;
    updatedAt: Date;
 }
+
+export type MaterializationStatus =
+   | "PENDING"
+   | "RUNNING"
+   | "SUCCESS"
+   | "FAILED"
+   | "CANCELLED";
+
+export interface Materialization {
+   id: string;
+   projectId: string;
+   packageName: string;
+   status: MaterializationStatus;
+   startedAt: Date | null;
+   completedAt: Date | null;
+   error: string | null;
+   metadata: Record<string, unknown> | null;
+   createdAt: Date;
+   updatedAt: Date;
+}
+
+export interface ManifestEntry {
+   id: string;
+   projectId: string;
+   packageName: string;
+   buildId: string;
+   tableName: string;
+   sourceName: string;
+   connectionName: string;
+   createdAt: Date;
+   updatedAt: Date;
+}
+
+// ==================== MANIFEST STORE ====================
+
+export interface BuildManifestEntry {
+   tableName: string;
+}
+
+export interface BuildManifest {
+   entries: Record<string, BuildManifestEntry>;
+   strict?: boolean;
+}
+
+/**
+ * Abstraction for manifest storage. Standalone mode uses DuckDB;
+ * orchestrated mode swaps in a DuckLakeManifestStore.
+ */
+export interface ManifestStore {
+   getManifest(projectId: string, packageName: string): Promise<BuildManifest>;
+   writeEntry(
+      projectId: string,
+      packageName: string,
+      buildId: string,
+      tableName: string,
+      sourceName: string,
+      connectionName: string,
+   ): Promise<void>;
+   deleteEntry(id: string): Promise<void>;
+   listEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]>;
+}

package/src/storage/{StorageManager.spec.ts → StorageManager.mock.ts}

@@ -47,4 +47,13 @@ export class StorageManager {
          deleteConnection: async (): Promise<void> => {},
       };
    }
+
+   getManifestStore() {
+      return {
+         getManifest: async () => ({ entries: {}, strict: false }),
+         writeEntry: async () => {},
+         deleteEntry: async () => {},
+         listEntries: async () => [],
+      };
+   }
 }

package/src/storage/StorageManager.ts

@@ -1,11 +1,22 @@
 import { logger } from "../logger";
-import { DatabaseConnection, ResourceRepository } from "./DatabaseInterface";
+import {
+   DatabaseConnection,
+   ManifestStore,
+   ResourceRepository,
+} from "./DatabaseInterface";
 import { DuckDBConnection } from "./duckdb/DuckDBConnection";
+import { DuckDBManifestStore } from "./duckdb/DuckDBManifestStore";
 import { DuckDBRepository } from "./duckdb/DuckDBRepository";
 import { initializeSchema } from "./duckdb/schema";
+import { DuckLakeManifestStore } from "./ducklake/DuckLakeManifestStore";
 
 export type StorageType = "duckdb" | "postgres" | "mysql";
 
+export interface DuckLakeManifestConfig {
+   catalogUrl: string;
+   dataPath: string;
+}
+
 export interface StorageConfig {
    type: StorageType;
    duckdb?: {
@@ -27,9 +38,28 @@ export interface StorageConfig {
    };
 }
 
+function escapeSQL(value: string): string {
+   return value.replace(/'/g, "''");
+}
+
+/**
+ * Manages the storage backend (DuckDB, Postgres, etc.) and per-project
+ * manifest stores. Projects without `materializationStorage` config use
+ * the default DuckDB manifest store. Projects with the config get a
+ * DuckLake-backed store attached lazily on first access.
+ */
 export class StorageManager {
    private connection: DatabaseConnection | null = null;
+   private duckDbConnection: DuckDBConnection | null = null;
    private repository: ResourceRepository | null = null;
+   private defaultManifestStore: ManifestStore | null = null;
+
+   /** Per-project DuckLake manifest stores, keyed by projectId. */
+   private projectManifestStores = new Map<string, ManifestStore>();
+
+   /** Tracks which catalogs have been attached to avoid duplicate ATTACHes. */
+   private attachedCatalogs = new Set<string>();
+
    private config: StorageConfig;
 
    constructor(config: StorageConfig) {
@@ -68,7 +98,73 @@ export class StorageManager {
       await initializeSchema(connection, reinit);
 
       this.connection = connection;
+      this.duckDbConnection = connection;
       this.repository = new DuckDBRepository(connection);
+      this.defaultManifestStore = new DuckDBManifestStore(this.repository);
+   }
+
+   /**
+    * Lazily initializes a DuckLake manifest store for a project.
+    * The DuckLake catalog is attached to the shared DuckDB connection
+    * and persists for the lifetime of the process.
+    */
+   async initializeDuckLakeForProject(
+      projectId: string,
+      config: DuckLakeManifestConfig,
+   ): Promise<void> {
+      if (!this.duckDbConnection) {
+         throw new Error("Storage not initialized. Call initialize() first.");
+      }
+
+      const catalogName = `manifest_lake_${projectId.replace(/[^a-zA-Z0-9_]/g, "_")}`;
+
+      if (!this.attachedCatalogs.has(catalogName)) {
+         await this.attachDuckLakeCatalog(config, catalogName);
+      }
+
+      const store = new DuckLakeManifestStore(
+         this.duckDbConnection,
+         catalogName,
+      );
+      await store.bootstrapSchema();
+
+      this.projectManifestStores.set(projectId, store);
+      logger.info("DuckLake manifest store initialized for project", {
+         projectId,
+         catalogName,
+      });
+   }
+
+   private async attachDuckLakeCatalog(
+      config: DuckLakeManifestConfig,
+      catalogName: string,
+   ): Promise<void> {
+      const connection = this.duckDbConnection!;
+
+      await connection.run("INSTALL ducklake; LOAD ducklake;");
+
+      const isPostgres = config.catalogUrl.startsWith("postgres:");
+      if (isPostgres) {
+         await connection.run("INSTALL postgres; LOAD postgres;");
+      }
+
+      const escapedCatalogUrl = escapeSQL(config.catalogUrl);
+      const escapedDataPath = escapeSQL(config.dataPath);
+      const isCloudStorage =
+         config.dataPath.startsWith("gs://") ||
+         config.dataPath.startsWith("s3://");
+
+      let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
+      const attachOpts: string[] = [`DATA_PATH '${escapedDataPath}'`];
+      if (isCloudStorage) {
+         attachOpts.push("OVERRIDE_DATA_PATH true");
+      }
+      attachCmd += ` (${attachOpts.join(", ")});`;
+
+      logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
+      await connection.run(attachCmd);
+
+      this.attachedCatalogs.add(catalogName);
    }
 
    getRepository(): ResourceRepository {
@@ -78,11 +174,33 @@ export class StorageManager {
       return this.repository;
    }
 
+   /**
+    * Returns the manifest store for a project. If the project has a
+    * DuckLake store configured, returns that; otherwise returns the
+    * default DuckDB-backed store.
+    */
+   getManifestStore(projectId?: string): ManifestStore {
+      if (projectId) {
+         const projectStore = this.projectManifestStores.get(projectId);
+         if (projectStore) {
+            return projectStore;
+         }
+      }
+      if (!this.defaultManifestStore) {
+         throw new Error("Storage not initialized. Call initialize() first.");
+      }
+      return this.defaultManifestStore;
+   }
+
    async close(): Promise<void> {
       if (this.connection) {
          await this.connection.close();
          this.connection = null;
+         this.duckDbConnection = null;
          this.repository = null;
+         this.defaultManifestStore = null;
+         this.projectManifestStores.clear();
+         this.attachedCatalogs.clear();
       }
    }
 

package/src/storage/duckdb/DuckDBConnection.ts

@@ -1,5 +1,5 @@
 import { Mutex } from "async-mutex";
-import * as duckdb from "duckdb";
+import duckdb from "duckdb";
 import * as path from "path";
 import { DatabaseConnection } from "../DatabaseInterface";
 

package/src/storage/duckdb/DuckDBManifestStore.ts

@@ -0,0 +1,70 @@
+import {
+   BuildManifest,
+   ManifestEntry,
+   ManifestStore,
+   ResourceRepository,
+} from "../DatabaseInterface";
+
+/**
+ * DuckDB-backed ManifestStore that delegates to the local ResourceRepository.
+ *
+ * In standalone mode this is the active store. Orchestrated deployments swap
+ * in a DuckLake-backed implementation that reads/writes manifests through the
+ * shared catalog instead of the local DuckDB build_manifests table.
+ */
+export class DuckDBManifestStore implements ManifestStore {
+   constructor(private repository: ResourceRepository) {}
+
+   /**
+    * Assembles a {@link BuildManifest} by folding all manifest rows for the
+    * given package into a build-ID-keyed map. `strict: false` lets the Malloy
+    * runtime fall back to executing the underlying query when a persist
+    * reference has no manifest entry (e.g. before the first materialization).
+    */
+   async getManifest(
+      projectId: string,
+      packageName: string,
+   ): Promise<BuildManifest> {
+      const entries = await this.repository.listManifestEntries(
+         projectId,
+         packageName,
+      );
+      const manifest: BuildManifest = {
+         entries: {},
+         strict: false,
+      };
+      for (const entry of entries) {
+         manifest.entries[entry.buildId] = { tableName: entry.tableName };
+      }
+      return manifest;
+   }
+
+   async writeEntry(
+      projectId: string,
+      packageName: string,
+      buildId: string,
+      tableName: string,
+      sourceName: string,
+      connectionName: string,
+   ): Promise<void> {
+      await this.repository.upsertManifestEntry({
+         projectId,
+         packageName,
+         buildId,
+         tableName,
+         sourceName,
+         connectionName,
+      });
+   }
+
+   async deleteEntry(id: string): Promise<void> {
+      await this.repository.deleteManifestEntry(id);
+   }
+
+   async listEntries(
+      projectId: string,
+      packageName: string,
+   ): Promise<ManifestEntry[]> {
+      return this.repository.listManifestEntries(projectId, packageName);
+   }
+}