@malloy-publisher/server 0.0.382-dev → 0.2.0

package/src/service/package.spec.ts

@@ -1,236 +0,0 @@
-import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-import { Stats } from "fs";
-import fs from "fs/promises";
-import { join } from "path";
-import sinon from "sinon";
-import { PackageNotFoundError } from "../errors";
-import { Model } from "./model";
-import { Package } from "./package";
-
-// Minimal partial types for mocking
-type PartialModel = Pick<Model, "getPath">;
-
-describe("service/package", () => {
-   const testPackageDirectory = "testPackage";
-
-   beforeEach(async () => {
-      await fs.mkdir(testPackageDirectory, { recursive: true });
-      await fs.writeFile(join(testPackageDirectory, "model1.model"), "");
-      // Create a simple parquet file with schema "name: string, value: int"
-      const parquetBuffer = Buffer.from("Name,Value\nJohn,10\nJane,20\nJim,30");
-
-      await fs.writeFile(
-         join(testPackageDirectory, "database.csv"),
-         parquetBuffer,
-      );
-      const publisherContent = JSON.stringify({ description: "Test package" });
-      await fs.writeFile(
-         join(testPackageDirectory, "publisher.json"),
-         publisherContent,
-      );
-   });
-
-   afterEach(async () => {
-      sinon.restore();
-      // On Windows, DuckDB connections may still have file handles open,
-      // causing EBUSY errors. Retry deletion with exponential backoff.
-      const maxRetries = 3;
-      const delay = 50; // Start with 50ms
-      let lastError: unknown;
-
-      for (let attempt = 0; attempt < maxRetries; attempt++) {
-         try {
-            await fs.rm(testPackageDirectory, { recursive: true, force: true });
-            return; // Success, exit retry loop
-         } catch (error) {
-            lastError = error;
-            const errnoError = error as NodeJS.ErrnoException;
-            // Only retry on EBUSY errors (Windows file handle still open)
-            if (errnoError.code !== "EBUSY") {
-               throw error; // Non-EBUSY errors should be thrown immediately
-            }
-            // Exponential backoff: 50ms, 100ms, 200ms, 400ms, 800ms
-            if (attempt < maxRetries - 1) {
-               await new Promise((resolve) => setTimeout(resolve, delay));
-            }
-         }
-      }
-      // If we've exhausted retries, ignore EBUSY errors as they don't affect test results
-      // and the files will be cleaned up eventually
-      if ((lastError as NodeJS.ErrnoException).code !== "EBUSY") {
-         throw lastError;
-      }
-   });
-
-   it("should create a package instance", async () => {
-      // Using 'as any' for simplified mock Map value in test
-      const pkg = new Package(
-         "testProject",
-         "testPackage",
-         testPackageDirectory,
-         { name: "testPackage", description: "Test package" },
-         [],
-         new Map([
-            [
-               "model1.malloy",
-               { getPath: () => "model1.malloy" } as unknown as Model,
-            ],
-            [
-               "model2.malloynb",
-               { getPath: () => "model2.malloynb" } as unknown as Model,
-            ],
-         ]),
-      );
-
-      expect(pkg).toBeInstanceOf(Package);
-   });
-
-   describe("instance methods", () => {
-      describe("create", () => {
-         it("should throw PackageNotFoundError if the package manifest does not exist", async () => {
-            await fs.rm(join(testPackageDirectory, "publisher.json"));
-            sinon.stub(fs, "stat").rejects(new Error("File not found"));
-
-            await expect(
-               Package.create(
-                  "testProject",
-                  "testPackage",
-                  testPackageDirectory,
-                  new Map(),
-               ),
-            ).rejects.toThrowError(
-               new PackageNotFoundError(
-                  "Package manifest for testPackage does not exist.",
-               ),
-            );
-         });
-         it(
-            "should return a Package object if the package exists",
-            async () => {
-               sinon.stub(fs, "stat").resolves();
-               const readFileStub = sinon
-                  .stub(fs, "readFile")
-                  .resolves(
-                     Buffer.from(
-                        JSON.stringify({ description: "Test package" }),
-                     ),
-                  );
-
-               // Still use Partial<Model> for the stub resolution type
-               sinon
-                  .stub(Model, "create")
-                  // @ts-expect-error PartialModel is a partial type
-                  .resolves({ getPath: () => "model1.model" } as PartialModel);
-
-               readFileStub.restore();
-               readFileStub.resolves(Buffer.from(JSON.stringify([])));
-
-               const packageInstance = await Package.create(
-                  "testProject",
-                  "testPackage",
-                  testPackageDirectory,
-                  new Map(),
-               );
-
-               expect(packageInstance).toBeInstanceOf(Package);
-               expect(packageInstance.getPackageName()).toBe("testPackage");
-               expect(packageInstance.getPackageMetadata().description).toBe(
-                  "Test package",
-               );
-               expect(packageInstance.listDatabases()).toEqual([
-                  {
-                     path: "database.csv",
-                     type: "embedded",
-                     info: {
-                        name: "database.csv",
-                        columns: [
-                           { name: "Name", type: "string" },
-                           { name: "Value", type: "number" },
-                        ],
-                        rowCount: 3,
-                     },
-                  },
-               ]);
-               expect(packageInstance.listModels()).toBeEmpty();
-            },
-            { timeout: 20000 },
-         );
-      });
-
-      describe("listModels", () => {
-         it("should return a list of models with their paths and types", async () => {
-            // Using 'as any' for simplified mock Map value in test
-            const packageInstance = new Package(
-               "testProject",
-               "testPackage",
-               testPackageDirectory,
-               { name: "testPackage", description: "Test package" },
-               [],
-               new Map([
-                  [
-                     "model1.malloy",
-                     {
-                        getPath: () => "model1.malloy",
-                        getModel: () => "foo",
-                     } as unknown as Model,
-                  ],
-                  [
-                     "model2.malloynb",
-                     {
-                        getPath: () => "model2.malloynb",
-                        getNotebookError: () => {
-                           return {
-                              message: "This is the error",
-                           };
-                        },
-                     } as unknown as Model,
-                  ],
-               ]),
-            );
-
-            const models = await packageInstance.listModels();
-            expect(models).toEqual([
-               {
-                  // @ts-expect-error TODO: Fix missing projectName type in API
-                  projectName: "testProject",
-                  packageName: "testPackage",
-                  path: "model1.malloy",
-                  error: undefined,
-               },
-            ]);
-
-            const notebooks = await packageInstance.listNotebooks();
-            expect(notebooks).toEqual([
-               {
-                  // @ts-expect-error TODO: Fix missing projectName type in API
-                  projectName: "testProject",
-                  packageName: "testPackage",
-                  path: "model2.malloynb",
-                  error: "This is the error",
-               },
-            ]);
-         });
-      });
-
-      describe("getDatabaseInfo", () => {
-         it("should return the size of the database file", async () => {
-            sinon.stub(fs, "stat").resolves({ size: 13 } as Stats);
-
-            // @ts-expect-error Accessing private static method for testing
-            const info = await Package.getDatabaseInfo(
-               testPackageDirectory,
-               "database.csv",
-            );
-
-            expect(info).toEqual({
-               name: "database.csv",
-               columns: [
-                  { name: "Name", type: "string" },
-                  { name: "Value", type: "number" },
-               ],
-               rowCount: 3,
-            });
-         });
-      });
-   });
-});

package/src/service/package.ts

@@ -1,463 +0,0 @@
-import * as fs from "fs/promises";
-import * as path from "path";
-
-import { DuckDBConnection } from "@malloydata/db-duckdb";
-import {
-   Connection,
-   ConnectionRuntime,
-   EmptyURLReader,
-   SourceDef,
-} from "@malloydata/malloy";
-import { metrics } from "@opentelemetry/api";
-import recursive from "recursive-readdir";
-import { components } from "../api";
-import {
-   API_PREFIX,
-   MODEL_FILE_SUFFIX,
-   NOTEBOOK_FILE_SUFFIX,
-   PACKAGE_MANIFEST_NAME,
-} from "../constants";
-import { PackageNotFoundError } from "../errors";
-import { formatDuration, logger } from "../logger";
-import { BuildManifest } from "../storage/DatabaseInterface";
-import { Model } from "./model";
-
-type ApiDatabase = components["schemas"]["Database"];
-type ApiModel = components["schemas"]["Model"];
-type ApiNotebook = components["schemas"]["Notebook"];
-export type ApiPackage = components["schemas"]["Package"];
-type ApiColumn = components["schemas"]["Column"];
-type ApiTableDescription = components["schemas"]["TableDescription"];
-
-const ENABLE_LIST_MODEL_COMPILATION = true;
-export class Package {
-   private projectName: string;
-   private packageName: string;
-   private packageMetadata: ApiPackage;
-   private databases: ApiDatabase[];
-   private models: Map<string, Model> = new Map();
-   private packagePath: string;
-   private connections: Map<string, Connection> = new Map();
-   private static meter = metrics.getMeter("publisher");
-   private static packageLoadHistogram = this.meter.createHistogram(
-      "malloy_package_load_duration",
-      {
-         description: "Time taken to load a Malloy package",
-         unit: "ms",
-      },
-   );
-
-   constructor(
-      projectName: string,
-      packageName: string,
-      packagePath: string,
-      packageMetadata: ApiPackage,
-      databases: ApiDatabase[],
-      models: Map<string, Model>,
-      connections: Map<string, Connection> = new Map(),
-   ) {
-      this.projectName = projectName;
-      this.packageName = packageName;
-      this.packagePath = packagePath;
-      this.packageMetadata = packageMetadata;
-      this.databases = databases;
-      this.models = models;
-      this.connections = connections;
-   }
-
-   static async create(
-      projectName: string,
-      packageName: string,
-      packagePath: string,
-      projectConnections: Map<string, Connection>,
-   ): Promise<Package> {
-      const startTime = performance.now();
-      await Package.validatePackageManifestExistsOrThrowError(packagePath);
-      const manifestValidationTime = performance.now();
-      logger.info("Package manifest validation completed", {
-         packageName,
-         duration: formatDuration(manifestValidationTime - startTime),
-      });
-
-      try {
-         const packageConfig = await Package.readPackageConfig(packagePath);
-         const packageConfigTime = performance.now();
-         logger.info("Package config read completed", {
-            packageName,
-            duration: formatDuration(
-               packageConfigTime - manifestValidationTime,
-            ),
-         });
-         packageConfig.resource = `${API_PREFIX}/projects/${projectName}/packages/${packageName}`;
-
-         const databases = await Package.readDatabases(packagePath);
-         const databasesTime = performance.now();
-         logger.info("Databases read completed", {
-            packageName,
-            databaseCount: databases.length,
-            duration: formatDuration(databasesTime - packageConfigTime),
-         });
-         const connections = new Map<string, Connection>(projectConnections);
-
-         // Add a duckdb connection for the package.
-         const duckdbConnection = new DuckDBConnection(
-            "duckdb",
-            ":memory:",
-            packagePath,
-         );
-         connections.set("duckdb", duckdbConnection);
-
-         const models = await Package.loadModels(
-            packageName,
-            packagePath,
-            connections,
-         );
-         const modelsTime = performance.now();
-         logger.info("Models loaded", {
-            packageName,
-            modelCount: models.size,
-            duration: formatDuration(modelsTime - databasesTime),
-         });
-         for (const [modelPath, model] of models.entries()) {
-            const maybeModel = model as unknown as {
-               compilationError?: unknown;
-            };
-            if (maybeModel.compilationError) {
-               const err = maybeModel.compilationError;
-               const message =
-                  err instanceof Error
-                     ? err.message
-                     : `Unknown compilation error in ${modelPath}`;
-
-               logger.error("Model compilation failed", {
-                  packageName,
-                  modelPath,
-                  error: message,
-               });
-
-               this.packageLoadHistogram.record(performance.now() - startTime, {
-                  malloy_package_name: packageName,
-                  status: "compilation_error",
-               });
-               throw err;
-            }
-         }
-         const endTime = performance.now();
-         const executionTime = endTime - startTime;
-         this.packageLoadHistogram.record(executionTime, {
-            malloy_package_name: packageName,
-            status: "success",
-         });
-         logger.info(`Successfully loaded package ${packageName}`, {
-            packageName,
-            duration: formatDuration(executionTime),
-         });
-         return new Package(
-            projectName,
-            packageName,
-            packagePath,
-            packageConfig,
-            databases,
-            models,
-            connections,
-         );
-      } catch (error) {
-         logger.error(`Error loading package ${packageName}`, { error });
-         console.error(error);
-         const endTime = performance.now();
-         const executionTime = endTime - startTime;
-         this.packageLoadHistogram.record(executionTime, {
-            malloy_package_name: packageName,
-            status: "error",
-         });
-         // Clean up package directory on failure
-         try {
-            await fs.rm(packagePath, {
-               recursive: true,
-               force: true,
-            });
-            logger.info(`Cleaned up failed package directory: ${packagePath}`);
-         } catch (cleanupError) {
-            logger.warn(`Failed to clean up package directory ${packagePath}`, {
-               error: cleanupError,
-            });
-         }
-         throw error;
-      }
-   }
-
-   public getPackageName(): string {
-      return this.packageName;
-   }
-
-   public getPackagePath(): string {
-      return this.packagePath;
-   }
-
-   public getPackageMetadata(): ApiPackage {
-      return this.packageMetadata;
-   }
-
-   public listDatabases(): ApiDatabase[] {
-      return this.databases;
-   }
-
-   public getModel(modelPath: string): Model | undefined {
-      return this.models.get(modelPath);
-   }
-
-   public getModelPaths(): string[] {
-      return Array.from(this.models.keys());
-   }
-
-   /**
-    * Recompile every model in the package with the given build manifest
-    * so queries resolve persist references to materialized tables.
-    *
-    * Builds a fresh map off to the side and swaps it in at the end. If any
-    * recompile fails the whole call rejects before the swap and the live
-    * `this.models` reference remains untouched — no half-loaded state is
-    * ever observable to concurrent readers.
-    */
-   public async reloadAllModels(
-      buildManifest: BuildManifest["entries"],
-   ): Promise<void> {
-      const modelPaths = Array.from(this.models.keys());
-      logger.info("Reloading all models with build manifest", {
-         packageName: this.packageName,
-         modelCount: modelPaths.length,
-         manifestEntryCount: Object.keys(buildManifest).length,
-      });
-
-      const reloaded = await Promise.all(
-         modelPaths.map((modelPath) =>
-            Model.create(
-               this.packageName,
-               this.packagePath,
-               modelPath,
-               this.connections,
-               { buildManifest },
-            ),
-         ),
-      );
-      const nextModels = new Map<string, Model>();
-      for (const model of reloaded) {
-         nextModels.set(model.getPath(), model);
-      }
-      this.models = nextModels;
-   }
-
-   public getConnections(): Map<string, Connection> {
-      return this.connections;
-   }
-
-   public getMalloyConnection(connectionName: string): Connection {
-      const connection = this.connections.get(connectionName);
-      if (!connection) {
-         throw new Error(
-            `Connection ${connectionName} not found in package ${this.packageName}`,
-         );
-      }
-      return connection;
-   }
-
-   public async getModelFileText(modelPath: string): Promise<string> {
-      const model = this.getModel(modelPath);
-      if (!model) {
-         throw new Error(`Model not found: ${modelPath}`);
-      }
-      return await model.getFileText(this.packagePath);
-   }
-
-   public async listModels(): Promise<ApiModel[]> {
-      const values = await Promise.all(
-         Array.from(this.models.keys())
-            .filter((modelPath) => {
-               return modelPath.endsWith(MODEL_FILE_SUFFIX);
-            })
-            .map(async (modelPath) => {
-               let error: string | undefined;
-               if (ENABLE_LIST_MODEL_COMPILATION) {
-                  try {
-                     await this.models.get(modelPath)?.getModel();
-                  } catch (modelError) {
-                     error =
-                        modelError instanceof Error
-                           ? modelError.message
-                           : undefined;
-                  }
-               }
-               return {
-                  projectName: this.projectName,
-                  path: modelPath,
-                  packageName: this.packageName,
-                  error,
-               };
-            }),
-      );
-      return values;
-   }
-
-   public async listNotebooks(): Promise<ApiNotebook[]> {
-      return await Promise.all(
-         Array.from(this.models.keys())
-            .filter((modelPath) => {
-               return modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
-            })
-            .map(async (modelPath) => {
-               let error: Error | undefined;
-               if (ENABLE_LIST_MODEL_COMPILATION) {
-                  error = this.models.get(modelPath)?.getNotebookError();
-               }
-               return {
-                  projectName: this.projectName,
-                  packageName: this.packageName,
-                  path: modelPath,
-                  error: error?.message,
-               };
-            }),
-      );
-   }
-
-   private static async loadModels(
-      packageName: string,
-      packagePath: string,
-      connections: Map<string, Connection>,
-   ): Promise<Map<string, Model>> {
-      const modelPaths = await Package.getModelPaths(packagePath);
-      const models = await Promise.all(
-         modelPaths.map((modelPath) =>
-            Model.create(packageName, packagePath, modelPath, connections),
-         ),
-      );
-      return new Map(models.map((model) => [model.getPath(), model]));
-   }
-
-   private static async getModelPaths(packagePath: string): Promise<string[]> {
-      let files = undefined;
-      try {
-         files = await recursive(packagePath);
-      } catch (error) {
-         logger.error(error);
-         throw new PackageNotFoundError(
-            `Package config for ${packagePath} does not exist.`,
-         );
-      }
-      return files
-         .map((fullPath: string) => {
-            return path.relative(packagePath, fullPath).replace(/\\/g, "/");
-         })
-         .filter(
-            (modelPath: string) =>
-               modelPath.endsWith(MODEL_FILE_SUFFIX) ||
-               modelPath.endsWith(NOTEBOOK_FILE_SUFFIX),
-         );
-   }
-
-   private static async validatePackageManifestExistsOrThrowError(
-      packagePath: string,
-   ) {
-      const packageConfigPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
-      try {
-         await fs.stat(packageConfigPath);
-      } catch {
-         logger.error(`Can't find ${packageConfigPath}`);
-         throw new PackageNotFoundError(
-            `Package manifest for ${packagePath} does not exist.`,
-         );
-      }
-   }
-
-   private static async readPackageConfig(
-      packagePath: string,
-   ): Promise<ApiPackage> {
-      const packageConfigPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
-      const packageConfigContents = await fs.readFile(packageConfigPath);
-      // TODO: Validate package manifest.  Define manifest type in public API.
-      const packageManifest = JSON.parse(packageConfigContents.toString());
-      return {
-         name: packageManifest.name,
-         description: packageManifest.description,
-      };
-   }
-
-   private static async readDatabases(
-      packagePath: string,
-   ): Promise<ApiDatabase[]> {
-      return await Promise.all(
-         (await Package.getDatabasePaths(packagePath)).map(
-            async (databasePath) => {
-               const databaseInfo = await Package.getDatabaseInfo(
-                  packagePath,
-                  databasePath,
-               );
-
-               return {
-                  path: databasePath,
-                  info: databaseInfo,
-                  type: "embedded",
-               };
-            },
-         ),
-      );
-   }
-
-   private static async getDatabasePaths(
-      packagePath: string,
-   ): Promise<string[]> {
-      let files = undefined;
-      files = await recursive(packagePath);
-      return files
-         .map((fullPath: string) => {
-            return path.relative(packagePath, fullPath).replace(/\\/g, "/");
-         })
-         .filter(
-            (modelPath: string) =>
-               modelPath.endsWith(".parquet") || modelPath.endsWith(".csv"),
-         );
-   }
-
-   private static async getDatabaseInfo(
-      packagePath: string,
-      databasePath: string,
-   ): Promise<ApiTableDescription> {
-      const fullPath = path.join(packagePath, databasePath);
-
-      // Create a DuckDB source then:
-      // 1. Load the model and get the table schema from model
-      // 2. Run a query to get the row count from the table
-      const runtime = new ConnectionRuntime({
-         urlReader: new EmptyURLReader(),
-         connections: [new DuckDBConnection("duckdb")],
-      });
-      // Normalize path to use forward slashes for cross-platform compatibility
-      // DuckDB on Windows supports forward slashes, and this avoids escaping issues
-      const normalizedPath = fullPath.replace(/\\/g, "/");
-      const model = runtime.loadModel(
-         `source: temp is duckdb.table('${normalizedPath}')`,
-      );
-      const modelDef = await model.getModel();
-      const fields = (modelDef._modelDef.contents["temp"] as SourceDef).fields;
-      const schema = fields.map((field): ApiColumn => {
-         return { type: field.type, name: field.name };
-      });
-      const runner = model.loadQuery(
-         "run: temp->{aggregate: row_count is count()}",
-      );
-      const result = await runner.run();
-      const rowCount = result.data.value[0].row_count?.valueOf() as number;
-      return { name: databasePath, rowCount, columns: schema };
-   }
-
-   public setName(name: string) {
-      this.packageName = name;
-   }
-
-   public setProjectName(projectName: string) {
-      this.projectName = projectName;
-   }
-
-   public setPackageMetadata(packageMetadata: ApiPackage) {
-      this.packageMetadata = packageMetadata;
-   }
-}