@malloy-publisher/server 0.0.91 → 0.0.92

package/eslint.config.mjs

@@ -50,6 +50,14 @@ export default [
 
       rules: {
          "react/react-in-jsx-scope": "off",
+         "@typescript-eslint/no-unused-vars": [
+            "error",
+            {
+               argsIgnorePattern: "^_",
+               varsIgnorePattern: "^_",
+               caughtErrorsIgnorePattern: "^_",
+            },
+         ],
       },
    },
 ];

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.91",
+   "version": "0.0.92",
    "main": "dist/server.js",
    "bin": {
       "malloy-publisher": "dist/server.js"
@@ -63,6 +63,7 @@
       "@eslint/js": "^9.23.0",
       "@faker-js/faker": "^9.4.0",
       "@types/adm-zip": "^0.5.7",
+      "@types/bun": "^1.2.20",
       "@types/cors": "^2.8.12",
       "@types/express": "^4.17.14",
       "@types/morgan": "^1.9.9",

package/publisher.config.json

@@ -1,6 +1,26 @@
 {
-   "frozenConfig": false,
-   "projects": {
-      "malloy-samples": "https://github.com/ms2data/malloy-samples"
-   }
-}
+    "frozenConfig": false,
+    "projects": [
+      {
+        "name": "malloy-samples",
+        "packages": [
+            {
+            "name": "ecommerce",
+            "location": "https://github.com/ms2data/malloy-samples"
+          },
+          {
+            "name": "imdb",
+            "location": "https://github.com/ms2data/malloy-samples"
+          },
+          {
+            "name": "faa",
+            "location": "https://github.com/ms2data/malloy-samples"
+          },
+          {
+            "name": "bigquery-hackernews",
+            "location": "https://github.com/ms2data/malloy-samples"
+          }
+        ]
+      }
+    ]
+  }

package/src/config.ts

@@ -4,14 +4,37 @@ import { PUBLISHER_CONFIG_NAME } from "./constants";
 
 type FilesystemPath = `./${string}` | `../${string}` | `/${string}`;
 type GcsPath = `gs://${string}`;
+
+export type Package = {
+   name: string;
+   location: FilesystemPath | GcsPath;
+};
+
+export type Project = {
+   name: string;
+   packages: Package[];
+};
+
 export type PublisherConfig = {
    frozenConfig: boolean;
-   projects: Record<string, FilesystemPath | GcsPath>;
+   projects: Project[];
 };
 
 export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
    const publisherConfigPath = path.join(serverRoot, PUBLISHER_CONFIG_NAME);
-   return JSON.parse(fs.readFileSync(publisherConfigPath, "utf8"));
+   const rawConfig = JSON.parse(fs.readFileSync(publisherConfigPath, "utf8"));
+
+   if (
+      rawConfig.projects &&
+      typeof rawConfig.projects === "object" &&
+      !Array.isArray(rawConfig.projects)
+   ) {
+      throw new Error(
+         "Config has changed. Please update your config to the new format.",
+      );
+   }
+
+   return rawConfig as PublisherConfig;
 };
 
 export const isPublisherConfigFrozen = (serverRoot: string) => {

package/src/constants.ts

@@ -11,6 +11,6 @@ export const ROW_LIMIT = 1000;
 export let publisherPath = "/etc/publisher";
 try {
    fs.accessSync(publisherPath, fs.constants.W_OK);
-} catch (e) {
+} catch {
    publisherPath = "/tmp/publisher";
 }

package/src/controller/package.controller.ts

@@ -103,6 +103,7 @@ export class PackageController {
             packageLocation,
             absoluteTargetPath,
             projectName,
+            packageName,
          );
       }
    }

package/src/controller/watch-mode.controller.ts

@@ -7,7 +7,7 @@ import { ProjectStore } from "../service/project_store";
 
 type StartWatchReq = components["schemas"]["StartWatchRequest"];
 type WatchStatusRes = components["schemas"]["WatchStatus"];
-type Handler<Req = {}, Res = void> = RequestHandler<{}, Res, Req>;
+type Handler<Req = object, Res = void> = RequestHandler<object, Res, Req>;
 
 export class WatchModeController {
    watchingPath: string | null;
@@ -19,7 +19,7 @@ export class WatchModeController {
       this.watchingProjectName = null;
    }
 
-   public getWatchStatus: Handler<{}, WatchStatusRes> = async (_req, res) => {
+   public getWatchStatus: Handler<void, WatchStatusRes> = async (_req, res) => {
       return res.json({
          enabled: !!this.watchingPath,
          watchingPath: this.watchingPath,
@@ -27,14 +27,29 @@ export class WatchModeController {
       });
    };
 
-   public startWatching: Handler<StartWatchReq> = async (req, res) => {
+   public startWatching: Handler<StartWatchReq, { error: string }> = async (
+      req,
+      res,
+   ) => {
       const projectManifest = await ProjectStore.reloadProjectManifest(
          this.projectStore.serverRootPath,
       );
       this.watchingProjectName = req.body.projectName;
+
+      // Find the project in the new array structure
+      const project = projectManifest.projects.find(
+         (p) => p.name === req.body.projectName,
+      );
+      if (!project || !project.packages || project.packages.length === 0) {
+         res.status(404).json({
+            error: `Project ${req.body.projectName} not found or has no packages`,
+         });
+         return;
+      }
+
       this.watchingPath = path.join(
          this.projectStore.serverRootPath,
-         projectManifest.projects[req.body.projectName],
+         req.body.projectName,
       );
       this.watcher = chokidar.watch(this.watchingPath, {
          ignored: (path, stats) =>

package/src/data_styles.ts

@@ -1,4 +1,5 @@
 import { URLReader } from "@malloydata/malloy";
+// @ts-expect-error until this gets merged https://github.com/malloydata/malloy/pull/2484
 import { DataStyles } from "@malloydata/render";
 import { logger } from "./logger";
 
@@ -25,7 +26,9 @@ export async function dataStylesForFile(
          //      created, so that the error can be shown there.
          let stylesText;
          try {
-            stylesText = await urlReader.readURL(new URL(fileName, url));
+            stylesText = (await urlReader.readURL(
+               new URL(fileName, url),
+            )) as string;
          } catch (error) {
             logger.error(`Error loading data style '${fileName}': ${error}`);
             stylesText = "{}";
@@ -53,10 +56,14 @@ export class HackyDataStylesAccumulator implements URLReader {
       const contents = await this.urlReader.readURL(url);
       this.dataStyles = {
          ...this.dataStyles,
-         ...(await dataStylesForFile(url.toString(), contents, this.urlReader)),
+         ...(await dataStylesForFile(
+            url.toString(),
+            contents as string,
+            this.urlReader,
+         )),
       };
 
-      return contents;
+      return contents as string;
    }
 
    getHackyAccumulatedDataStyles(): DataStyles {

package/src/mcp/prompts/index.ts

@@ -1,3 +1,11 @@
-export { withVersion, PROMPTS, MALLOY_PROMPTS, ExplainArgsSchema, GenerateQueryArgsSchema, TranslateSqlArgsSchema, SummarizeModelArgsSchema } from "./prompt_definitions";
+export {
+   withVersion,
+   PROMPTS,
+   MALLOY_PROMPTS,
+   ExplainArgsSchema,
+   GenerateQueryArgsSchema,
+   TranslateSqlArgsSchema,
+   SummarizeModelArgsSchema,
+} from "./prompt_definitions";
 export { promptHandlerMap, makePromptHandler } from "./handlers";
 export { registerPromptCapability } from "./prompt_service";

package/src/mcp/resources/package_resource.ts

@@ -38,13 +38,14 @@ async function handleGetPackageContents(
       const packageInstance = await project.getPackage(packageName, false);
 
       // Use listModels() which returns { path: string, type: 'source' | 'notebook' }[]
-      const entries = packageInstance.listModels();
+      const entries = await packageInstance.listModels();
 
       // Use a type that includes metadata explicitly
       const resourceDefinitions: { uri: string; metadata: unknown }[] = [];
 
       for (const entry of entries) {
          const entryPath = entry.path; // e.g., "flights.malloy", "overview.malloynb"
+         // @ts-expect-error TODO: Fix missing type in API
          const entryType = entry.type; // 'source' or 'notebook'
 
          if (typeof entryPath !== "string" || entryPath === "") {

package/src/mcp/resources/source_resource.ts

@@ -91,6 +91,7 @@ export function registerSourceResource(
                   }
                   // Add type annotation for 's'
                   const source = sources.find(
+                     // @ts-expect-error TODO: Fix missing Source type in API
                      (s: components["schemas"]["Source"]) =>
                         s.name === sourceName,
                   );

package/src/mcp/resources/view_resource.ts

@@ -83,6 +83,7 @@ export function registerViewResource(
                      throw new Error("Could not retrieve sources from model.");
                   }
                   const source = sources.find(
+                     // @ts-expect-error TODO: Fix missing Source type in API
                      (s: components["schemas"]["Source"]) =>
                         s.name === sourceName,
                   );

package/src/server.ts

@@ -13,11 +13,7 @@ import { PackageController } from "./controller/package.controller";
 import { QueryController } from "./controller/query.controller";
 import { ScheduleController } from "./controller/schedule.controller";
 import { WatchModeController } from "./controller/watch-mode.controller";
-import {
-   internalErrorToHttpError,
-   NotImplementedError,
-   PackageNotFoundError,
-} from "./errors";
+import { internalErrorToHttpError, NotImplementedError } from "./errors";
 import { logger, loggerMiddleware } from "./logger";
 import { initializeMcpServer } from "./mcp/server";
 import { ProjectStore } from "./service/project_store";
@@ -213,10 +209,14 @@ app.use(cors());
 app.use(bodyParser.json());
 
 app.get(`${API_PREFIX}/status`, async (_req, res) => {
-   res.status(200).json({
-      timestamp: Date.now(),
-      projects: await projectStore.listProjects(),
-   });
+   try {
+      const status = await projectStore.getStatus();
+      res.status(200).json(status);
+   } catch (error) {
+      logger.error("Error getting status", { error });
+      const { json, status } = internalErrorToHttpError(error as Error);
+      res.status(status).json(json);
+   }
 });
 
 app.get(`${API_PREFIX}/watch-mode/status`, watchModeController.getWatchStatus);

package/src/service/connection.ts

@@ -53,16 +53,26 @@ export async function createConnections(
    const connectionMap = new Map<string, BaseConnection>();
    const connectionConfig = await readConnectionConfig(basePath);
 
-   for (const connection of [...defaultConnections, ...connectionConfig]) {
+   const allConnections = [...defaultConnections, ...connectionConfig];
+
+   const processedConnections = new Set<string>();
+   const apiConnections: InternalConnection[] = [];
+
+   for (const connection of allConnections) {
+      if (connection.name && processedConnections.has(connection.name)) {
+         continue;
+      }
+
       logger.info(`Adding connection ${connection.name}`, {
          connection,
       });
-      // This case shouldn't happen.  The package validation logic should
-      // catch it.
+
       if (!connection.name) {
          throw "Invalid connection configuration.  No name.";
       }
 
+      processedConnections.add(connection.name);
+
       switch (connection.type) {
          case "postgres": {
             const configReader = async () => {
@@ -218,11 +228,14 @@ export async function createConnections(
             throw new Error(`Unsupported connection type: ${connection.type}`);
          }
       }
+
+      // Add the connection to apiConnections (this will be sanitized when returned)
+      apiConnections.push(connection);
    }
 
    return {
       malloyConnections: connectionMap,
-      apiConnections: connectionConfig,
+      apiConnections: apiConnections,
    };
 }
 

package/src/service/db_utils.ts

@@ -9,6 +9,7 @@ import { components } from "../api";
 import { logger } from "../logger";
 import {
    ApiConnection,
+   // @ts-expect-error TODO: Fix missing MysqlConnection type in API
    MysqlConnection,
    PostgresConnection,
    SnowflakeConnection,
@@ -68,7 +69,7 @@ async function getSnowflakeConnection(
    }
    return new Promise((resolve, reject) => {
       const connection = snowflake.createConnection({
-         account: apiSnowflakeConnection.account,
+         account: apiSnowflakeConnection.account as string,
          username: apiSnowflakeConnection.username,
          password: apiSnowflakeConnection.password,
          database: apiSnowflakeConnection.database,
@@ -318,7 +319,7 @@ async function getSnowflakeSchemas(
    return new Promise((resolve, reject) => {
       connection.execute({
          sqlText: "SHOW SCHEMAS",
-         complete: (err, stmt, rows) => {
+         complete: (err, _stmt, rows) => {
             if (err) {
                reject(err);
             } else {

package/src/service/model.ts

@@ -20,6 +20,7 @@ import {
    MalloySQLStatementType,
 } from "@malloydata/malloy-sql";
 import malloyPackage from "@malloydata/malloy/package.json";
+// @ts-expect-error until this gets merged https://github.com/malloydata/malloy/pull/2484
 import { DataStyles } from "@malloydata/render";
 import { metrics } from "@opentelemetry/api";
 import * as fs from "fs/promises";
@@ -42,6 +43,7 @@ import { URL_READER } from "../utils";
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiNotebookCell = components["schemas"]["NotebookCell"];
 type ApiCompiledNotebook = components["schemas"]["CompiledNotebook"];
+// @ts-expect-error TODO: Fix missing Source type in API
 type ApiSource = components["schemas"]["Source"];
 type ApiView = components["schemas"]["View"];
 type ApiQuery = components["schemas"]["Query"];

package/src/service/package.spec.ts

@@ -7,6 +7,7 @@ import { readConnectionConfig } from "./connection";
 import { ApiConnection, Model } from "./model";
 import { Package } from "./package";
 import { Scheduler } from "./scheduler";
+import { Stats } from "fs";
 
 // Minimal partial types for mocking
 type PartialScheduler = Pick<Scheduler, "list">;
@@ -55,8 +56,16 @@ describe("service/package", () => {
          { name: "testPackage", description: "Test package" },
          [],
          new Map([
-            ["model1.malloy", { getPath: () => "model1.malloy" } as any],
-            ["model2.malloynb", { getPath: () => "model2.malloynb" } as any],
+            [
+               "model1.malloy",
+               // @ts-expect-error PartialModel is a partial type
+               { getPath: () => "model1.malloy" } as PartialModel,
+            ],
+            [
+               "model2.malloynb",
+               // @ts-expect-error PartialModel is a partial type
+               { getPath: () => "model2.malloynb" } as PartialModel,
+            ],
          ]),
          undefined,
       );
@@ -78,60 +87,69 @@ describe("service/package", () => {
                   new Map(),
                ),
             ).rejects.toThrowError(
-               PackageNotFoundError,
-               "Package manifest for testPackage does not exist.",
+               new PackageNotFoundError(
+                  "Package manifest for testPackage does not exist.",
+               ),
             );
          });
-         it("should return a Package object if the package exists", async () => {
-            sinon.stub(fs, "stat").resolves();
-            const readFileStub = sinon
-               .stub(fs, "readFile")
-               .resolves(
-                  Buffer.from(JSON.stringify({ description: "Test package" })),
+         it(
+            "should return a Package object if the package exists",
+            async () => {
+               sinon.stub(fs, "stat").resolves();
+               const readFileStub = sinon
+                  .stub(fs, "readFile")
+                  .resolves(
+                     Buffer.from(
+                        JSON.stringify({ description: "Test package" }),
+                     ),
+                  );
+
+               // Still use Partial<Model> for the stub resolution type
+               type PartialModel = Pick<Model, "getPath">;
+               sinon
+                  .stub(Model, "create")
+                  // @ts-expect-error PartialModel is a partial type
+                  .resolves({ getPath: () => "model1.model" } as PartialModel);
+
+               // @ts-expect-error PartialScheduler is a partial type
+               sinon.stub(Scheduler, "create").returns({
+                  list: () => [],
+               } as PartialScheduler);
+
+               readFileStub.restore();
+               readFileStub.resolves(Buffer.from(JSON.stringify([])));
+
+               const packageInstance = await Package.create(
+                  "testProject",
+                  "testPackage",
+                  testPackageDirectory,
+                  new Map(),
                );
 
-            // Still use Partial<Model> for the stub resolution type
-            type PartialModel = Pick<Model, "getPath">;
-            sinon
-               .stub(Model, "create")
-               .resolves({ getPath: () => "model1.model" } as PartialModel);
-
-            sinon.stub(Scheduler, "create").returns({
-               list: () => [],
-            } as PartialScheduler);
-
-            readFileStub.restore();
-            readFileStub.resolves(Buffer.from(JSON.stringify([])));
-
-            const packageInstance = await Package.create(
-               "testProject",
-               "testPackage",
-               testPackageDirectory,
-               new Map(),
-            );
-
-            expect(packageInstance).toBeInstanceOf(Package);
-            expect(packageInstance.getPackageName()).toBe("testPackage");
-            expect(packageInstance.getPackageMetadata().description).toBe(
-               "Test package",
-            );
-            expect(packageInstance.listDatabases()).toEqual([
-               {
-                  path: "database.csv",
-                  type: "embedded",
-                  info: {
-                     name: "database.csv",
-                     columns: [
-                        { name: "Name", type: "string" },
-                        { name: "Value", type: "number" },
-                     ],
-                     rowCount: 3,
+               expect(packageInstance).toBeInstanceOf(Package);
+               expect(packageInstance.getPackageName()).toBe("testPackage");
+               expect(packageInstance.getPackageMetadata().description).toBe(
+                  "Test package",
+               );
+               expect(packageInstance.listDatabases()).toEqual([
+                  {
+                     path: "database.csv",
+                     type: "embedded",
+                     info: {
+                        name: "database.csv",
+                        columns: [
+                           { name: "Name", type: "string" },
+                           { name: "Value", type: "number" },
+                        ],
+                        rowCount: 3,
+                     },
                   },
-               },
-            ]);
-            expect(packageInstance.listModels()).toBeEmpty();
-            expect(packageInstance.listSchedules()).toBeEmpty();
-         });
+               ]);
+               expect(packageInstance.listModels()).toBeEmpty();
+               expect(packageInstance.listSchedules()).toBeEmpty();
+            },
+            { timeout: 15000 },
+         );
       });
 
       describe("listModels", () => {
@@ -149,7 +167,8 @@ describe("service/package", () => {
                      {
                         getPath: () => "model1.malloy",
                         getModel: () => "foo",
-                     } as any,
+                        // @ts-expect-error PartialModel is a partial type
+                     } as PartialModel,
                   ],
                   [
                      "model2.malloynb",
@@ -160,7 +179,8 @@ describe("service/package", () => {
                               message: "This is the error",
                            };
                         },
-                     } as any,
+                        // @ts-expect-error PartialModel is a partial type
+                     } as PartialModel,
                   ],
                ]),
                undefined,
@@ -169,6 +189,7 @@ describe("service/package", () => {
             const models = await packageInstance.listModels();
             expect(models).toEqual([
                {
+                  // @ts-expect-error TODO: Fix missing projectName type in API
                   projectName: "testProject",
                   packageName: "testPackage",
                   path: "model1.malloy",
@@ -179,6 +200,7 @@ describe("service/package", () => {
             const notebooks = await packageInstance.listNotebooks();
             expect(notebooks).toEqual([
                {
+                  // @ts-expect-error TODO: Fix missing projectName type in API
                   projectName: "testProject",
                   packageName: "testPackage",
                   path: "model2.malloynb",
@@ -190,7 +212,7 @@ describe("service/package", () => {
 
       describe("getDatabaseInfo", () => {
          it("should return the size of the database file", async () => {
-            sinon.stub(fs, "stat").resolves({ size: 13 } as { size: number });
+            sinon.stub(fs, "stat").resolves({ size: 13 } as Stats);
 
             // @ts-expect-error Accessing private static method for testing
             const info = await Package.getDatabaseInfo(