@malloy-publisher/server 0.0.382-dev → 0.2.0

package/src/controller/model.controller.ts

@@ -1,107 +0,0 @@
-import { components } from "../api";
-import { ModelNotFoundError } from "../errors";
-import { ProjectStore } from "../service/project_store";
-import type { FilterParams } from "../service/filter";
-
-type ApiNotebook = components["schemas"]["Notebook"];
-type ApiModel = components["schemas"]["Model"];
-type ApiCompiledModel = components["schemas"]["CompiledModel"];
-type ApiRawNotebook = components["schemas"]["RawNotebook"];
-export class ModelController {
-   private projectStore: ProjectStore;
-
-   constructor(projectStore: ProjectStore) {
-      this.projectStore = projectStore;
-   }
-
-   public async listModels(
-      projectName: string,
-      packageName: string,
-   ): Promise<ApiModel[]> {
-      const project = await this.projectStore.getProject(projectName, false);
-      const p = await project.getPackage(packageName, false);
-      return p.listModels();
-   }
-
-   public async listNotebooks(
-      projectName: string,
-      packageName: string,
-   ): Promise<ApiNotebook[]> {
-      const project = await this.projectStore.getProject(projectName, false);
-      const p = await project.getPackage(packageName, false);
-      return p.listNotebooks();
-   }
-
-   public async getModel(
-      projectName: string,
-      packageName: string,
-      modelPath: string,
-   ): Promise<ApiCompiledModel> {
-      try {
-         const project = await this.projectStore.getProject(projectName, false);
-         const p = await project.getPackage(packageName, false);
-         const model = p.getModel(modelPath);
-         if (!model) {
-            throw new ModelNotFoundError(`${modelPath} does not exist`);
-         }
-         if (model.getType() === "notebook") {
-            throw new ModelNotFoundError(`${modelPath} is a notebook`);
-         }
-         return await model.getModel();
-      } catch (error) {
-         // Re-throw ModelNotFoundError as-is
-         if (error instanceof ModelNotFoundError) {
-            throw error;
-         }
-         // Wrap other errors with more context
-         throw new Error(
-            `Failed to get model ${modelPath} from package ${packageName} in project ${projectName}: ${error}`,
-         );
-      }
-   }
-
-   public async getNotebook(
-      projectName: string,
-      packageName: string,
-      notebookPath: string,
-   ): Promise<ApiRawNotebook> {
-      const project = await this.projectStore.getProject(projectName, false);
-      const p = await project.getPackage(packageName, false);
-      const model = p.getModel(notebookPath);
-      if (!model) {
-         throw new ModelNotFoundError(`${notebookPath} does not exist`);
-      }
-      if (model.getType() === "model") {
-         throw new ModelNotFoundError(`${notebookPath} is a model`);
-      }
-
-      return model.getNotebook();
-   }
-
-   public async executeNotebookCell(
-      projectName: string,
-      packageName: string,
-      notebookPath: string,
-      cellIndex: number,
-      filterParams?: FilterParams,
-      bypassFilters?: boolean,
-   ): Promise<{
-      type: "code" | "markdown";
-      text: string;
-      queryName?: string;
-      result?: string;
-      newSources?: string[];
-   }> {
-      const project = await this.projectStore.getProject(projectName, false);
-      const p = await project.getPackage(packageName, false);
-      const model = p.getModel(notebookPath);
-      if (!model) {
-         throw new ModelNotFoundError(`${notebookPath} does not exist`);
-      }
-      if (model.getType() === "model") {
-         throw new ModelNotFoundError(`${notebookPath} is a model`);
-      }
-
-      return model.executeNotebookCell(cellIndex, filterParams, bypassFilters);
-   }
-}

package/src/controller/package.controller.ts

@@ -1,182 +0,0 @@
-import * as path from "path";
-import { components } from "../api";
-import { PUBLISHER_DATA_DIR } from "../constants";
-import { BadRequestError, FrozenConfigError } from "../errors";
-import { logger } from "../logger";
-import { ManifestService } from "../service/manifest_service";
-import { ProjectStore } from "../service/project_store";
-
-type ApiPackage = components["schemas"]["Package"];
-
-export class PackageController {
-   private projectStore: ProjectStore;
-   private manifestService: ManifestService;
-
-   constructor(projectStore: ProjectStore, manifestService: ManifestService) {
-      this.projectStore = projectStore;
-      this.manifestService = manifestService;
-   }
-
-   public async listPackages(projectName: string): Promise<ApiPackage[]> {
-      const project = await this.projectStore.getProject(projectName, false);
-      return project.listPackages();
-   }
-
-   public async getPackage(
-      projectName: string,
-      packageName: string,
-      reload: boolean,
-   ): Promise<ApiPackage> {
-      const project = await this.projectStore.getProject(projectName, false);
-      const _package = await project.getPackage(packageName, reload);
-      const packageLocation = _package.getPackageMetadata().location;
-      if (reload && packageLocation) {
-         await this.downloadPackage(projectName, packageName, packageLocation);
-      }
-      return _package.getPackageMetadata();
-   }
-
-   async addPackage(
-      projectName: string,
-      body: ApiPackage,
-      options?: { autoLoadManifest?: boolean },
-   ) {
-      if (this.projectStore.publisherConfigIsFrozen) {
-         throw new FrozenConfigError();
-      }
-      if (!body.name) {
-         throw new BadRequestError("Package name is required");
-      }
-      const project = await this.projectStore.getProject(projectName, false);
-      if (body.location) {
-         await this.downloadPackage(projectName, body.name, body.location);
-      }
-      const result = await project.addPackage(body.name);
-      await this.projectStore.addPackageToDatabase(projectName, body.name);
-
-      if (options?.autoLoadManifest === true) {
-         await this.tryLoadExistingManifest(projectName, body.name);
-      }
-
-      return result;
-   }
-
-   /**
-    * If there are already manifest entries for this package (e.g. from a
-    * previous materialization run), reload all models with the manifest so
-    * persist references resolve to the materialized tables immediately.
-    */
-   private async tryLoadExistingManifest(
-      projectName: string,
-      packageName: string,
-   ): Promise<void> {
-      try {
-         const repository = this.projectStore.storageManager.getRepository();
-         const dbProject = await repository.getProjectByName(projectName);
-         if (!dbProject) return;
-
-         const manifest = await this.manifestService.getManifest(
-            dbProject.id,
-            packageName,
-         );
-         if (Object.keys(manifest.entries).length === 0) return;
-
-         await this.manifestService.reloadManifest(
-            dbProject.id,
-            packageName,
-            projectName,
-         );
-         logger.info("Auto-loaded existing manifest for added package", {
-            projectName,
-            packageName,
-            entryCount: Object.keys(manifest.entries).length,
-         });
-      } catch (error) {
-         logger.warn("Failed to auto-load manifest for package", {
-            projectName,
-            packageName,
-            error,
-         });
-      }
-   }
-
-   public async deletePackage(projectName: string, packageName: string) {
-      if (this.projectStore.publisherConfigIsFrozen) {
-         throw new FrozenConfigError();
-      }
-      const project = await this.projectStore.getProject(projectName, false);
-      const result = await project.deletePackage(packageName);
-      await this.projectStore.deletePackageFromDatabase(
-         projectName,
-         packageName,
-      );
-
-      return result;
-   }
-
-   public async updatePackage(
-      projectName: string,
-      packageName: string,
-      body: ApiPackage,
-   ) {
-      if (this.projectStore.publisherConfigIsFrozen) {
-         throw new FrozenConfigError();
-      }
-      const project = await this.projectStore.getProject(projectName, false);
-      if (body.location) {
-         await this.downloadPackage(projectName, packageName, body.location);
-      }
-      const result = await project.updatePackage(packageName, body);
-      await this.projectStore.addPackageToDatabase(projectName, packageName);
-
-      return result;
-   }
-
-   private async downloadPackage(
-      projectName: string,
-      packageName: string,
-      packageLocation: string,
-   ) {
-      const absoluteTargetPath = path.join(
-         this.projectStore.serverRootPath,
-         PUBLISHER_DATA_DIR,
-         projectName,
-         packageName,
-      );
-      const isCompressedFile = packageLocation.endsWith(".zip");
-      if (
-         packageLocation.startsWith("https://") ||
-         packageLocation.startsWith("git@")
-      ) {
-         await this.projectStore.downloadGitHubDirectory(
-            packageLocation,
-            absoluteTargetPath,
-         );
-      } else if (packageLocation.startsWith("gs://")) {
-         await this.projectStore.downloadGcsDirectory(
-            packageLocation,
-            projectName,
-            absoluteTargetPath,
-            isCompressedFile,
-         );
-      } else if (packageLocation.startsWith("s3://")) {
-         await this.projectStore.downloadS3Directory(
-            packageLocation,
-            projectName,
-            absoluteTargetPath,
-            isCompressedFile,
-         );
-      }
-
-      if (packageLocation.startsWith("/")) {
-         // Absolute paths from the publisher.config could be placed outside of /etc/publisher,
-         // so we need to mount them on the right place.
-         await this.projectStore.mountLocalDirectory(
-            packageLocation,
-            absoluteTargetPath,
-            projectName,
-            packageName,
-         );
-      }
-   }
-}

package/src/controller/query.controller.ts

@@ -1,60 +0,0 @@
-import { validateRenderTags } from "@malloydata/render-validator";
-import { components } from "../api";
-import { API_PREFIX } from "../constants";
-import { ModelNotFoundError } from "../errors";
-import { ProjectStore } from "../service/project_store";
-import type { FilterParams } from "../service/filter";
-
-type ApiQuery = components["schemas"]["QueryResult"];
-
-// Replacer function to handle BigInt serialization
-function bigIntReplacer(_key: string, value: unknown): unknown {
-   if (typeof value === "bigint") {
-      return Number(value);
-   }
-   return value;
-}
-
-export class QueryController {
-   private projectStore: ProjectStore;
-
-   constructor(projectStore: ProjectStore) {
-      this.projectStore = projectStore;
-   }
-
-   public async getQuery(
-      projectName: string,
-      packageName: string,
-      modelPath: string,
-      sourceName: string,
-      queryName: string,
-      query: string,
-      compactJson: boolean = false,
-      filterParams?: FilterParams,
-      bypassFilters?: boolean,
-   ): Promise<ApiQuery> {
-      const project = await this.projectStore.getProject(projectName, false);
-      const p = await project.getPackage(packageName, false);
-      const model = p.getModel(modelPath);
-
-      if (!model) {
-         throw new ModelNotFoundError(`${modelPath} does not exist`);
-      } else {
-         const { result, compactResult } = await model.getQueryResults(
-            sourceName,
-            queryName,
-            query,
-            filterParams,
-            bypassFilters,
-         );
-         const renderLogs = validateRenderTags(result);
-         return {
-            result: compactJson
-               ? JSON.stringify(compactResult, bigIntReplacer)
-               : JSON.stringify(result),
-            resource: `${API_PREFIX}/projects/${projectName}/packages/${packageName}/models/${modelPath}/query`,
-            renderLogs: renderLogs.length > 0 ? renderLogs : undefined,
-         } as ApiQuery;
-      }
-   }
-}

package/src/controller/watch-mode.controller.ts

@@ -1,98 +0,0 @@
-import chokidar, { FSWatcher } from "chokidar";
-import { RequestHandler } from "express";
-import path from "path";
-import { components } from "../api";
-import { logger } from "../logger";
-import { ProjectStore } from "../service/project_store";
-
-type StartWatchReq = components["schemas"]["StartWatchRequest"];
-type WatchStatusRes = components["schemas"]["WatchStatus"];
-type Handler<Req = object, Res = void> = RequestHandler<object, Res, Req>;
-
-export class WatchModeController {
-   watchingPath: string | null;
-   watchingProjectName: string | null;
-   watcher: FSWatcher;
-
-   constructor(private projectStore: ProjectStore) {
-      this.watchingPath = null;
-      this.watchingProjectName = null;
-   }
-
-   public getWatchStatus: Handler<void, WatchStatusRes> = async (_req, res) => {
-      return res.json({
-         enabled: !!this.watchingPath,
-         watchingPath: this.watchingPath ?? "",
-         projectName: this.watchingProjectName ?? "",
-      });
-   };
-
-   public startWatching: Handler<StartWatchReq, { error: string }> = async (
-      req,
-      res,
-   ) => {
-      const projectManifest = await ProjectStore.reloadProjectManifest(
-         this.projectStore.serverRootPath,
-      );
-      this.watchingProjectName = req.body.projectName;
-
-      // Find the project in the new array structure
-      const project = projectManifest.projects.find(
-         (p) => p.name === req.body.projectName,
-      );
-      if (!project || !project.packages || project.packages.length === 0) {
-         res.status(404).json({
-            error: `Project ${req.body.projectName} not found or has no packages`,
-         });
-         return;
-      }
-
-      this.watchingPath = path.join(
-         this.projectStore.serverRootPath,
-         req.body.projectName,
-      );
-      this.watcher = chokidar.watch(this.watchingPath, {
-         ignored: (path, stats) =>
-            !!stats?.isFile() &&
-            !path.endsWith(".malloy") &&
-            !path.endsWith(".md"),
-         ignoreInitial: true,
-      });
-      const reloadProject = async () => {
-         // Overwrite the project with it's existing metadata to trigger a re-read
-         const project = await this.projectStore.getProject(
-            req.body.projectName,
-            true,
-         );
-         await this.projectStore.addProject(project.metadata);
-         logger.info(`Reloaded ${req.body.projectName}`);
-      };
-
-      this.watcher.on("add", async (path) => {
-         logger.info(
-            `Detected new file ${path}, reloading ${req.body.projectName}`,
-         );
-         await reloadProject();
-      });
-      this.watcher.on("unlink", async (path) => {
-         logger.info(
-            `Detected deletion of ${path}, reloading ${req.body.projectName}`,
-         );
-         await reloadProject();
-      });
-      this.watcher.on("change", async (path) => {
-         logger.info(
-            `Detected change on ${path}, reloading ${req.body.projectName}`,
-         );
-         await reloadProject();
-      });
-      res.json();
-   };
-
-   public stopWatchMode: Handler = async (_req, res) => {
-      this.watcher.close();
-      this.watchingPath = null;
-      this.watchingProjectName = null;
-      res.json();
-   };
-}

package/src/data_styles.ts

@@ -1,71 +0,0 @@
-import { URLReader } from "@malloydata/malloy";
-import { DataStyles } from "@malloydata/render";
-import { logger } from "./logger";
-
-export function compileDataStyles(styles: string): DataStyles {
-   try {
-      return JSON.parse(styles) as DataStyles;
-   } catch (error) {
-      throw new Error(`Error compiling data styles: ${error}`);
-   }
-}
-
-// TODO replace this with actual JSON metadata import functionality, when it exists
-export async function dataStylesForFile(
-   url: string,
-   text: string,
-   urlReader: URLReader,
-): Promise<DataStyles> {
-   const PREFIX = "--! styles ";
-   let styles: DataStyles = {};
-   for (const line of text.split("\n")) {
-      if (line.startsWith(PREFIX)) {
-         const fileName = line.trimEnd().substring(PREFIX.length);
-         // TODO instead of failing silently when the file does not exist, perform this after the WebView has been
-         //      created, so that the error can be shown there.
-         let stylesText;
-         try {
-            stylesText = (await urlReader.readURL(
-               new URL(fileName, url),
-            )) as string;
-         } catch (error) {
-            logger.error(`Error loading data style '${fileName}': ${error}`);
-            stylesText = "{}";
-         }
-         styles = { ...styles, ...compileDataStyles(stylesText) };
-      }
-   }
-
-   return styles;
-}
-
-// TODO Come up with a better way to handle data styles. Perhaps this is
-//      an in-language understanding of model "metadata". For now,
-//      we abuse the `URLReader` API to keep track of requested URLs
-//      and accumulate data styles for those files.
-export class HackyDataStylesAccumulator implements URLReader {
-   private urlReader: URLReader;
-   private dataStyles: DataStyles = {};
-
-   constructor(urlReader: URLReader) {
-      this.urlReader = urlReader;
-   }
-
-   async readURL(url: URL): Promise<string> {
-      const contents = await this.urlReader.readURL(url);
-      this.dataStyles = {
-         ...this.dataStyles,
-         ...(await dataStylesForFile(
-            url.toString(),
-            contents as string,
-            this.urlReader,
-         )),
-      };
-
-      return contents as string;
-   }
-
-   getHackyAccumulatedDataStyles(): DataStyles {
-      return this.dataStyles;
-   }
-}

package/src/dto/connection.dto.spec.ts

@@ -1,131 +0,0 @@
-import { expect, it, describe } from "bun:test";
-
-import { plainToInstance } from "class-transformer";
-import { validate } from "class-validator";
-import {
-   BigqueryConnectionDto,
-   ConnectionDto,
-   PostgresConnectionDto,
-   SnowflakeConnectionDto,
-   MysqlConnectionDto,
-} from "./connection.dto";
-
-describe("dto/connection", () => {
-   describe("Connection Validation", () => {
-      it("should validate a valid MysqlConnection object", async () => {
-         const validData = {
-            host: "localhost",
-            port: 3306,
-            database: "testdb",
-            user: "user",
-            password: "pass",
-         };
-         const mysqlConnection = plainToInstance(MysqlConnectionDto, validData);
-
-         const errors = await validate(mysqlConnection);
-         expect(errors).toHaveLength(0);
-      });
-
-      it("should validate a valid PostgresConnection object", async () => {
-         const validData = {
-            host: "localhost",
-            port: 5432,
-            databaseName: "testdb",
-            userName: "user",
-            password: "pass",
-            connectionString: "postgres://user:pass@localhost:5432/testdb",
-         };
-         const postgresConnection = plainToInstance(
-            PostgresConnectionDto,
-            validData,
-         );
-
-         const errors = await validate(postgresConnection);
-         expect(errors).toHaveLength(0);
-      });
-
-      it("should return errors for invalid PostgresConnection object", async () => {
-         const invalidData = {
-            host: 123, // Invalid type
-            port: "not-a-number", // Invalid type
-         };
-         const postgresConnection = plainToInstance(
-            PostgresConnectionDto,
-            invalidData,
-         );
-
-         const errors = await validate(postgresConnection);
-         expect(errors).not.toHaveLength(0);
-         expect(errors).toHaveLength(2);
-      });
-
-      it("should validate a valid BigqueryConnection object", async () => {
-         const validData = {
-            defaultProjectId: "default-project",
-            billingProjectId: "billing-project",
-            location: "US",
-            serviceAccountKeyJson: "{}",
-            maximumBytesBilled: "1000000",
-            queryTimeoutMilliseconds: "1000",
-         };
-         const bigqueryConnection = plainToInstance(
-            BigqueryConnectionDto,
-            validData,
-         );
-
-         const errors = await validate(bigqueryConnection);
-         expect(errors).toHaveLength(0);
-      });
-
-      it("should validate a valid SnowflakeConnection object", async () => {
-         const validData = {
-            account: "my-account",
-            username: "user",
-            password: "pass",
-            warehouse: "my-warehouse",
-            database: "my-database",
-            schema: "my-schema",
-            responseTimeoutMilliseconds: 5000,
-         };
-         const snowflakeConnection = plainToInstance(
-            SnowflakeConnectionDto,
-            validData,
-         );
-
-         const errors = await validate(snowflakeConnection);
-         expect(errors).toHaveLength(0);
-      });
-
-      it("should validate a valid Connection object with postgres type", async () => {
-         const validData = {
-            name: "My Postgres Connection",
-            type: "postgres",
-            postgresConnection: {
-               host: "localhost",
-               port: 5432,
-               databaseName: "testdb",
-               userName: "user",
-               password: "pass",
-            },
-         };
-         const connection = plainToInstance(ConnectionDto, validData);
-
-         const errors = await validate(connection);
-         expect(errors).toHaveLength(0);
-      });
-
-      it("should return errors for invalid Connection object", async () => {
-         const invalidData = {
-            type: "invalid-type", // Invalid enum value
-            postgresConnection: {
-               port: "invalid-port", // Invalid type
-            },
-         };
-         const connection = plainToInstance(ConnectionDto, invalidData);
-
-         const errors = await validate(connection);
-         expect(errors).not.toHaveLength(0);
-         expect(errors.length).toBeGreaterThan(0);
-      });
-   });
-});