@malloy-publisher/server 0.0.94 → 0.0.97

package/dist/app/index.html

@@ -12,7 +12,7 @@
          href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap"
       />
       <title>Malloy Publisher</title>
-     <script type="module" crossorigin src="/assets/index-TVOAGCOZ.js"></script>
+     <script type="module" crossorigin src="/assets/index-BIibKHZO.js"></script>
      <link rel="modulepreload" crossorigin href="/assets/vendor-c5ypKtDW.js">
      <link rel="modulepreload" crossorigin href="/assets/mui-UpaxdnvH.js">
      <link rel="stylesheet" crossorigin href="/assets/index-CcIq0aEZ.css">

package/dist/server.js

@@ -231306,18 +231306,13 @@ var TEMP_DIR_PATH = import_os.default.tmpdir();
 var publisherPath;
 if (process.platform === "win32") {
   publisherPath = import_path.default.join(process.env.PROGRAMDATA || "C:\\ProgramData", "publisher");
-  try {
-    import_fs.default.accessSync(publisherPath, import_fs.default.constants.W_OK);
-  } catch {
-    publisherPath = import_path.default.join(import_os.default.tmpdir(), "publisher");
-  }
 } else {
   publisherPath = "/etc/publisher";
-  try {
-    import_fs.default.accessSync(publisherPath, import_fs.default.constants.W_OK);
-  } catch {
-    publisherPath = import_path.default.join(import_os.default.tmpdir(), "publisher");
-  }
+}
+try {
+  import_fs.default.accessSync(publisherPath, import_fs.default.constants.W_OK);
+} catch {
+  publisherPath = import_path.default.join(import_os.default.tmpdir(), "publisher");
 }
 
 // src/errors.ts
@@ -235087,8 +235082,8 @@ async function testConnectionConfig(connectionConfig) {
         throw new Error("Invalid connection configuration. No postgres connection.");
       }
       const postgresConfig = connectionConfig.postgresConnection;
-      if (!postgresConfig.host || !postgresConfig.port || !postgresConfig.userName || !postgresConfig.password || !postgresConfig.databaseName) {
-        throw new Error("PostgreSQL connection requires: host, port, userName, password, and databaseName");
+      if (!postgresConfig.host || !postgresConfig.port || !postgresConfig.userName || !postgresConfig.databaseName) {
+        throw new Error("PostgreSQL connection requires: host, port, userName, and databaseName");
       }
       const configReader = async () => {
         return {
@@ -235122,7 +235117,7 @@ async function testConnectionConfig(connectionConfig) {
         throw new Error("Invalid connection configuration. No snowflake connection.");
       }
       const snowflakeConfig = connectionConfig.snowflakeConnection;
-      if (!snowflakeConfig.account || !snowflakeConfig.username || !snowflakeConfig.password || !snowflakeConfig.warehouse || !snowflakeConfig.database || !snowflakeConfig.schema) {
+      if (!snowflakeConfig.account || !snowflakeConfig.username || !snowflakeConfig.password || !snowflakeConfig.warehouse) {
         throw new Error("Snowflake connection requires: account, username, password, warehouse, database, and schema");
       }
       const snowflakeConnectionOptions = {
@@ -235158,12 +235153,12 @@ async function testConnectionConfig(connectionConfig) {
         throw new Error("Invalid connection configuration. No bigquery connection.");
       }
       const bigqueryConfig = connectionConfig.bigqueryConnection;
-      if (!bigqueryConfig.serviceAccountKeyJson || !bigqueryConfig.defaultProjectId) {
-        throw new Error("BigQuery connection requires: serviceAccountKeyJson and defaultProjectId");
-      }
-      const serviceAccountKeyPath = import_path3.default.join(TEMP_DIR_PATH, `test-${v4_default()}-service-account-key.json`);
+      let serviceAccountKeyPath = undefined;
       try {
-        await import_promises.default.writeFile(serviceAccountKeyPath, connectionConfig.bigqueryConnection.serviceAccountKeyJson);
+        if (bigqueryConfig.serviceAccountKeyJson) {
+          serviceAccountKeyPath = import_path3.default.join(TEMP_DIR_PATH, `test-${v4_default()}-service-account-key.json`);
+          await import_promises.default.writeFile(serviceAccountKeyPath, bigqueryConfig.serviceAccountKeyJson);
+        }
         const bigqueryConnectionOptions = {
           projectId: connectionConfig.bigqueryConnection.defaultProjectId,
           serviceAccountKeyPath,
@@ -235187,7 +235182,9 @@ async function testConnectionConfig(connectionConfig) {
         };
       } finally {
         try {
-          await import_promises.default.unlink(serviceAccountKeyPath);
+          if (serviceAccountKeyPath) {
+            await import_promises.default.unlink(serviceAccountKeyPath);
+          }
         } catch (cleanupError) {
           logger2.warn(`Failed to cleanup temporary file ${serviceAccountKeyPath}:`, cleanupError);
         }
@@ -235199,8 +235196,8 @@ async function testConnectionConfig(connectionConfig) {
         throw new Error("Trino connection configuration is missing.");
       }
       const trinoConfig = connectionConfig.trinoConnection;
-      if (!trinoConfig.server || !trinoConfig.port || !trinoConfig.catalog || !trinoConfig.schema || !trinoConfig.user || !trinoConfig.password) {
-        throw new Error("Trino connection requires: server, port, catalog, schema, user, and password");
+      if (!trinoConfig.server || !trinoConfig.port || !trinoConfig.catalog || !trinoConfig.schema || !trinoConfig.user) {
+        throw new Error("Trino connection requires: server, port, catalog, schema, and user");
       }
       const trinoConnectionOptions = {
         server: trinoConfig.server,
@@ -235478,16 +235475,15 @@ class PackageController {
   }
   async downloadPackage(projectName, packageName, packageLocation) {
     const absoluteTargetPath = `${publisherPath}/${projectName}/${packageName}`;
+    const isCompressedFile = packageLocation.endsWith(".zip");
     if (packageLocation.startsWith("https://") || packageLocation.startsWith("git@")) {
       await this.projectStore.downloadGitHubDirectory(packageLocation, absoluteTargetPath);
     } else if (packageLocation.startsWith("gs://")) {
-      await this.projectStore.downloadGcsDirectory(packageLocation, projectName, absoluteTargetPath);
+      await this.projectStore.downloadGcsDirectory(packageLocation, projectName, absoluteTargetPath, isCompressedFile);
     } else if (packageLocation.startsWith("s3://")) {
       await this.projectStore.downloadS3Directory(packageLocation, projectName, absoluteTargetPath);
-    } else if (packageLocation.startsWith("/")) {
-      if (packageLocation.endsWith(".zip")) {
-        packageLocation = await this.projectStore.unzipProject(packageLocation);
-      }
+    }
+    if (packageLocation.startsWith("/")) {
       await this.projectStore.mountLocalDirectory(packageLocation, absoluteTargetPath, projectName, packageName);
     }
   }
@@ -241097,6 +241093,12 @@ var import_fs5 = __toESM(require("fs"));
 var import_path4 = __toESM(require("path"));
 var getPublisherConfig = (serverRoot) => {
   const publisherConfigPath = import_path4.default.join(serverRoot, PUBLISHER_CONFIG_NAME);
+  if (!import_fs5.default.existsSync(publisherConfigPath)) {
+    return {
+      frozenConfig: false,
+      projects: []
+    };
+  }
   const rawConfig = JSON.parse(import_fs5.default.readFileSync(publisherConfigPath, "utf8"));
   if (rawConfig.projects && typeof rawConfig.projects === "object" && !Array.isArray(rawConfig.projects)) {
     throw new Error("Config has changed. Please update your config to the new format.");
@@ -242645,10 +242647,11 @@ class ProjectStore {
     return absoluteTargetPath;
   }
   async downloadOrMountLocation(location, targetPath, projectName, packageName) {
+    const isCompressedFile = location.endsWith(".zip");
     if (location.startsWith("gs://")) {
       try {
         logger2.info(`Downloading GCS directory from "${location}" to "${targetPath}"`);
-        await this.downloadGcsDirectory(location, projectName, targetPath);
+        await this.downloadGcsDirectory(location, projectName, targetPath, isCompressedFile);
         return;
       } catch (error) {
         logger2.error(`Failed to download GCS directory "${location}"`, {
@@ -242715,7 +242718,7 @@ class ProjectStore {
       throw new PackageNotFoundError(`Package ${packageName} for project ${projectName} not found in "${projectPath}"`);
     }
   }
-  async downloadGcsDirectory(gcsPath, projectName, absoluteDirPath) {
+  async downloadGcsDirectory(gcsPath, projectName, absoluteDirPath, isCompressedFile) {
     const trimmedPath = gcsPath.slice(5);
     const [bucketName, ...prefixParts] = trimmedPath.split("/");
     const prefix = prefixParts.join("/");
@@ -242725,11 +242728,18 @@ class ProjectStore {
     if (files.length === 0) {
       throw new ProjectNotFoundError(`Project ${projectName} not found in ${gcsPath}`);
     }
-    await fs9.promises.rm(absoluteDirPath, { recursive: true, force: true });
-    await fs9.promises.mkdir(absoluteDirPath, { recursive: true });
+    if (!isCompressedFile) {
+      await fs9.promises.rm(absoluteDirPath, {
+        recursive: true,
+        force: true
+      });
+      await fs9.promises.mkdir(absoluteDirPath, { recursive: true });
+    } else {
+      absoluteDirPath = `${absoluteDirPath}.zip`;
+    }
     await Promise.all(files.map(async (file) => {
       const relativeFilePath = file.name.replace(prefix, "");
-      const absoluteFilePath = path8.join(absoluteDirPath, relativeFilePath);
+      const absoluteFilePath = isCompressedFile ? absoluteDirPath : path8.join(absoluteDirPath, relativeFilePath);
       if (file.name.endsWith("/")) {
         return;
       }
@@ -242738,6 +242748,10 @@ class ProjectStore {
       });
       return fs9.promises.writeFile(absoluteFilePath, await file.download());
     }));
+    if (isCompressedFile) {
+      await this.unzipProject(absoluteDirPath);
+    }
+    logger2.info(`Downloaded GCS directory ${gcsPath} to ${absoluteDirPath}`);
   }
   async downloadS3Directory(s3Path, projectName, absoluteDirPath) {
     const trimmedPath = s3Path.slice(5);

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.94",
+   "version": "0.0.97",
    "main": "dist/server.js",
    "bin": {
       "malloy-publisher": "dist/server.js"
@@ -23,15 +23,15 @@
    },
    "dependencies": {
       "@google-cloud/storage": "^7.16.0",
-      "@malloydata/db-bigquery": "^0.0.295",
-      "@malloydata/db-duckdb": "^0.0.295",
-      "@malloydata/db-mysql": "^0.0.295",
-      "@malloydata/db-postgres": "^0.0.295",
-      "@malloydata/db-snowflake": "^0.0.295",
-      "@malloydata/db-trino": "^0.0.295",
-      "@malloydata/malloy": "^0.0.295",
-      "@malloydata/malloy-sql": "^0.0.295",
-      "@malloydata/render": "^0.0.295",
+      "@malloydata/db-bigquery": "^0.0.307",
+      "@malloydata/db-duckdb": "^0.0.307",
+      "@malloydata/db-mysql": "^0.0.307",
+      "@malloydata/db-postgres": "^0.0.307",
+      "@malloydata/db-snowflake": "^0.0.307",
+      "@malloydata/db-trino": "^0.0.307",
+      "@malloydata/malloy": "^0.0.307",
+      "@malloydata/malloy-sql": "^0.0.307",
+      "@malloydata/render": "^0.0.307",
       "@modelcontextprotocol/sdk": "^1.13.2",
       "@opentelemetry/api": "^1.9.0",
       "@opentelemetry/auto-instrumentations-node": "^0.57.0",

package/src/config.ts

@@ -22,6 +22,12 @@ export type PublisherConfig = {
 
 export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
    const publisherConfigPath = path.join(serverRoot, PUBLISHER_CONFIG_NAME);
+   if (!fs.existsSync(publisherConfigPath)) {
+      return {
+         frozenConfig: false,
+         projects: [],
+      };
+   }
    const rawConfig = JSON.parse(fs.readFileSync(publisherConfigPath, "utf8"));
 
    if (

package/src/constants.ts

@@ -10,6 +10,7 @@ export const MODEL_FILE_SUFFIX = ".malloy";
 export const NOTEBOOK_FILE_SUFFIX = ".malloynb";
 export const ROW_LIMIT = 1000;
 export const TEMP_DIR_PATH = os.tmpdir();
+
 export let publisherPath: string;
 
 if (process.platform === "win32") {
@@ -17,16 +18,12 @@ if (process.platform === "win32") {
       process.env.PROGRAMDATA || "C:\\ProgramData",
       "publisher",
    );
-   try {
-      fs.accessSync(publisherPath, fs.constants.W_OK);
-   } catch {
-      publisherPath = path.join(os.tmpdir(), "publisher");
-   }
 } else {
    publisherPath = "/etc/publisher";
-   try {
-      fs.accessSync(publisherPath, fs.constants.W_OK);
-   } catch {
-      publisherPath = path.join(os.tmpdir(), "publisher");
-   }
+}
+
+try {
+   fs.accessSync(publisherPath, fs.constants.W_OK);
+} catch {
+   publisherPath = path.join(os.tmpdir(), "publisher");
 }

package/src/controller/package.controller.ts

@@ -74,6 +74,7 @@ export class PackageController {
       packageLocation: string,
    ) {
       const absoluteTargetPath = `${publisherPath}/${projectName}/${packageName}`;
+      const isCompressedFile = packageLocation.endsWith(".zip");
       if (
          packageLocation.startsWith("https://") ||
          packageLocation.startsWith("git@")
@@ -87,6 +88,7 @@ export class PackageController {
             packageLocation,
             projectName,
             absoluteTargetPath,
+            isCompressedFile,
          );
       } else if (packageLocation.startsWith("s3://")) {
          await this.projectStore.downloadS3Directory(
@@ -94,11 +96,11 @@ export class PackageController {
             projectName,
             absoluteTargetPath,
          );
-      } else if (packageLocation.startsWith("/")) {
-         if (packageLocation.endsWith(".zip")) {
-            packageLocation =
-               await this.projectStore.unzipProject(packageLocation);
-         }
+      }
+
+      if (packageLocation.startsWith("/")) {
+         // Absolute paths from the publisher.config could be placed outside of /etc/publisher,
+         // so we need to mount them on the right place.
          await this.projectStore.mountLocalDirectory(
             packageLocation,
             absoluteTargetPath,

package/src/data_styles.ts

@@ -1,5 +1,4 @@
 import { URLReader } from "@malloydata/malloy";
-// @ts-expect-error until this gets merged https://github.com/malloydata/malloy/pull/2484
 import { DataStyles } from "@malloydata/render";
 import { logger } from "./logger";
 

package/src/service/connection.ts

@@ -277,11 +277,10 @@ export async function testConnectionConfig(
             !postgresConfig.host ||
             !postgresConfig.port ||
             !postgresConfig.userName ||
-            !postgresConfig.password ||
             !postgresConfig.databaseName
          ) {
             throw new Error(
-               "PostgreSQL connection requires: host, port, userName, password, and databaseName",
+               "PostgreSQL connection requires: host, port, userName, and databaseName",
             );
          }
 
@@ -331,9 +330,7 @@ export async function testConnectionConfig(
             !snowflakeConfig.account ||
             !snowflakeConfig.username ||
             !snowflakeConfig.password ||
-            !snowflakeConfig.warehouse ||
-            !snowflakeConfig.database ||
-            !snowflakeConfig.schema
+            !snowflakeConfig.warehouse
          ) {
             throw new Error(
                "Snowflake connection requires: account, username, password, warehouse, database, and schema",
@@ -381,26 +378,18 @@ export async function testConnectionConfig(
          }
 
          const bigqueryConfig = connectionConfig.bigqueryConnection;
-         if (
-            !bigqueryConfig.serviceAccountKeyJson ||
-            !bigqueryConfig.defaultProjectId
-         ) {
-            throw new Error(
-               "BigQuery connection requires: serviceAccountKeyJson and defaultProjectId",
-            );
-         }
-
-         const serviceAccountKeyPath = path.join(
-            TEMP_DIR_PATH,
-            `test-${uuidv4()}-service-account-key.json`,
-         );
-
+         let serviceAccountKeyPath = undefined;
          try {
-            await fs.writeFile(
-               serviceAccountKeyPath,
-               connectionConfig.bigqueryConnection
-                  .serviceAccountKeyJson as string,
-            );
+            if (bigqueryConfig.serviceAccountKeyJson) {
+               serviceAccountKeyPath = path.join(
+                  TEMP_DIR_PATH,
+                  `test-${uuidv4()}-service-account-key.json`,
+               );
+               await fs.writeFile(
+                  serviceAccountKeyPath,
+                  bigqueryConfig.serviceAccountKeyJson as string,
+               );
+            }
 
             const bigqueryConnectionOptions = {
                projectId: connectionConfig.bigqueryConnection.defaultProjectId,
@@ -433,7 +422,9 @@ export async function testConnectionConfig(
             };
          } finally {
             try {
-               await fs.unlink(serviceAccountKeyPath);
+               if (serviceAccountKeyPath) {
+                  await fs.unlink(serviceAccountKeyPath);
+               }
             } catch (cleanupError) {
                logger.warn(
                   `Failed to cleanup temporary file ${serviceAccountKeyPath}:`,
@@ -455,11 +446,10 @@ export async function testConnectionConfig(
             !trinoConfig.port ||
             !trinoConfig.catalog ||
             !trinoConfig.schema ||
-            !trinoConfig.user ||
-            !trinoConfig.password
+            !trinoConfig.user
          ) {
             throw new Error(
-               "Trino connection requires: server, port, catalog, schema, user, and password",
+               "Trino connection requires: server, port, catalog, schema, and user",
             );
          }
 

package/src/service/model.ts

@@ -20,7 +20,6 @@ import {
    MalloySQLStatementType,
 } from "@malloydata/malloy-sql";
 import malloyPackage from "@malloydata/malloy/package.json";
-// @ts-expect-error until this gets merged https://github.com/malloydata/malloy/pull/2484
 import { DataStyles } from "@malloydata/render";
 import { metrics } from "@opentelemetry/api";
 import * as fs from "fs/promises";

package/src/service/project_store.ts

@@ -473,13 +473,19 @@ export class ProjectStore {
       projectName: string,
       packageName: string,
    ) {
+      const isCompressedFile = location.endsWith(".zip");
       // Handle GCS paths
       if (location.startsWith("gs://")) {
          try {
             logger.info(
                `Downloading GCS directory from "${location}" to "${targetPath}"`,
             );
-            await this.downloadGcsDirectory(location, projectName, targetPath);
+            await this.downloadGcsDirectory(
+               location,
+               projectName,
+               targetPath,
+               isCompressedFile,
+            );
             return;
          } catch (error) {
             logger.error(`Failed to download GCS directory "${location}"`, {
@@ -591,6 +597,7 @@ export class ProjectStore {
       gcsPath: string,
       projectName: string,
       absoluteDirPath: string,
+      isCompressedFile: boolean,
    ) {
       const trimmedPath = gcsPath.slice(5);
       const [bucketName, ...prefixParts] = trimmedPath.split("/");
@@ -603,15 +610,21 @@ export class ProjectStore {
             `Project ${projectName} not found in ${gcsPath}`,
          );
       }
-      await fs.promises.rm(absoluteDirPath, { recursive: true, force: true });
-      await fs.promises.mkdir(absoluteDirPath, { recursive: true });
+      if (!isCompressedFile) {
+         await fs.promises.rm(absoluteDirPath, {
+            recursive: true,
+            force: true,
+         });
+         await fs.promises.mkdir(absoluteDirPath, { recursive: true });
+      } else {
+         absoluteDirPath = `${absoluteDirPath}.zip`;
+      }
       await Promise.all(
          files.map(async (file) => {
             const relativeFilePath = file.name.replace(prefix, "");
-            const absoluteFilePath = path.join(
-               absoluteDirPath,
-               relativeFilePath,
-            );
+            const absoluteFilePath = isCompressedFile
+               ? absoluteDirPath
+               : path.join(absoluteDirPath, relativeFilePath);
             if (file.name.endsWith("/")) {
                return;
             }
@@ -624,6 +637,10 @@ export class ProjectStore {
             );
          }),
       );
+      if (isCompressedFile) {
+         await this.unzipProject(absoluteDirPath);
+      }
+      logger.info(`Downloaded GCS directory ${gcsPath} to ${absoluteDirPath}`);
    }
 
    async downloadS3Directory(