@malloy-publisher/server 0.0.119 → 0.0.120

package/src/service/package.spec.ts

@@ -1,16 +1,14 @@
 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import { Stats } from "fs";
 import fs from "fs/promises";
 import { join } from "path";
 import sinon from "sinon";
 import { PackageNotFoundError } from "../errors";
-import { readConnectionConfig } from "./connection";
 import { Model } from "./model";
 import { Package } from "./package";
-import { Scheduler } from "./scheduler";
-import { Stats } from "fs";
 
 // Minimal partial types for mocking
-type PartialScheduler = Pick<Scheduler, "list">;
+type PartialModel = Pick<Model, "getPath">;
 
 describe("service/package", () => {
    const testPackageDirectory = "testPackage";
@@ -48,16 +46,13 @@ describe("service/package", () => {
          new Map([
             [
                "model1.malloy",
-               // @ts-expect-error PartialModel is a partial type
-               { getPath: () => "model1.malloy" } as PartialModel,
+               { getPath: () => "model1.malloy" } as unknown as Model,
             ],
             [
                "model2.malloynb",
-               // @ts-expect-error PartialModel is a partial type
-               { getPath: () => "model2.malloynb" } as PartialModel,
+               { getPath: () => "model2.malloynb" } as unknown as Model,
             ],
          ]),
-         undefined,
       );
 
       expect(pkg).toBeInstanceOf(Package);
@@ -75,7 +70,7 @@ describe("service/package", () => {
                   "testPackage",
                   testPackageDirectory,
                   new Map(),
-                  "/server/root",
+                  [],
                ),
             ).rejects.toThrowError(
                new PackageNotFoundError(
@@ -96,17 +91,11 @@ describe("service/package", () => {
                   );
 
                // Still use Partial<Model> for the stub resolution type
-               type PartialModel = Pick<Model, "getPath">;
                sinon
                   .stub(Model, "create")
                   // @ts-expect-error PartialModel is a partial type
                   .resolves({ getPath: () => "model1.model" } as PartialModel);
 
-               // @ts-expect-error PartialScheduler is a partial type
-               sinon.stub(Scheduler, "create").returns({
-                  list: () => [],
-               } as PartialScheduler);
-
                readFileStub.restore();
                readFileStub.resolves(Buffer.from(JSON.stringify([])));
 
@@ -115,7 +104,7 @@ describe("service/package", () => {
                   "testPackage",
                   testPackageDirectory,
                   new Map(),
-                  "/server/root",
+                  [],
                );
 
                expect(packageInstance).toBeInstanceOf(Package);
@@ -138,7 +127,6 @@ describe("service/package", () => {
                   },
                ]);
                expect(packageInstance.listModels()).toBeEmpty();
-               expect(packageInstance.listSchedules()).toBeEmpty();
             },
             { timeout: 15000 },
          );
@@ -159,8 +147,7 @@ describe("service/package", () => {
                      {
                         getPath: () => "model1.malloy",
                         getModel: () => "foo",
-                        // @ts-expect-error PartialModel is a partial type
-                     } as PartialModel,
+                     } as unknown as Model,
                   ],
                   [
                      "model2.malloynb",
@@ -171,11 +158,9 @@ describe("service/package", () => {
                               message: "This is the error",
                            };
                         },
-                        // @ts-expect-error PartialModel is a partial type
-                     } as PartialModel,
+                     } as unknown as Model,
                   ],
                ]),
-               undefined,
             );
 
             const models = await packageInstance.listModels();
@@ -222,79 +207,5 @@ describe("service/package", () => {
             });
          });
       });
-
-      describe("readConnectionConfig", () => {
-         it("should return an empty array if no project name or server root path is provided", async () => {
-            const config = await readConnectionConfig(testPackageDirectory);
-            expect(Array.isArray(config)).toBe(true);
-            expect(config).toHaveLength(0);
-         });
-
-         it("should return an empty array if project name or server root path is missing", async () => {
-            const config1 = await readConnectionConfig(
-               testPackageDirectory,
-               "testProject",
-            );
-            expect(Array.isArray(config1)).toBe(true);
-            expect(config1).toHaveLength(0);
-
-            const config2 = await readConnectionConfig(
-               testPackageDirectory,
-               undefined,
-               "/server/root",
-            );
-            expect(Array.isArray(config2)).toBe(true);
-            expect(config2).toHaveLength(0);
-         });
-
-         it("should return connections from publisher config when project name and server root are provided", async () => {
-            const tempServerRoot = "/tmp/test-server-root";
-            const tempConfigPath = join(
-               tempServerRoot,
-               "publisher.config.json",
-            );
-
-            // Create temp directory and config file
-            await fs.mkdir(tempServerRoot, { recursive: true });
-            await fs.writeFile(
-               tempConfigPath,
-               JSON.stringify({
-                  frozenConfig: false,
-                  projects: [
-                     {
-                        name: "testProject",
-                        packages: [],
-                        connections: [
-                           { name: "test-conn", type: "postgres" },
-                           { name: "test-conn2", type: "bigquery" },
-                        ],
-                     },
-                  ],
-               }),
-            );
-
-            const config = await readConnectionConfig(
-               testPackageDirectory,
-               "testProject",
-               tempServerRoot,
-            );
-
-            expect(Array.isArray(config)).toBe(true);
-            expect(config).toHaveLength(2);
-            expect(config[0]).toMatchObject({
-               name: "test-conn",
-               type: "postgres",
-               resource: "/api/v0/connections/test-conn",
-            });
-            expect(config[1]).toMatchObject({
-               name: "test-conn2",
-               type: "bigquery",
-               resource: "/api/v0/connections/test-conn2",
-            });
-
-            // Clean up
-            await fs.rm(tempServerRoot, { recursive: true, force: true });
-         });
-      });
    });
 });

package/src/service/package.ts

@@ -19,15 +19,13 @@ import {
 } from "../constants";
 import { PackageNotFoundError } from "../errors";
 import { logger } from "../logger";
-import { createConnections } from "./connection";
-import { Model } from "./model";
-import { Scheduler } from "./scheduler";
+import { createPackageDuckDBConnections } from "./connection";
+import { ApiConnection, Model } from "./model";
 
 type ApiDatabase = components["schemas"]["Database"];
 type ApiModel = components["schemas"]["Model"];
 type ApiNotebook = components["schemas"]["Notebook"];
 export type ApiPackage = components["schemas"]["Package"];
-type ApiSchedule = components["schemas"]["Schedule"];
 type ApiColumn = components["schemas"]["Column"];
 type ApiTableDescription = components["schemas"]["TableDescription"];
 
@@ -38,8 +36,8 @@ export class Package {
    private packageMetadata: ApiPackage;
    private databases: ApiDatabase[];
    private models: Map<string, Model> = new Map();
-   private scheduler: Scheduler | undefined;
    private packagePath: string;
+   private connections: Map<string, Connection> = new Map();
    private static meter = metrics.getMeter("publisher");
    private static packageLoadHistogram = this.meter.createHistogram(
       "malloy_package_load_duration",
@@ -56,7 +54,7 @@ export class Package {
       packageMetadata: ApiPackage,
       databases: ApiDatabase[],
       models: Map<string, Model>,
-      scheduler: Scheduler | undefined,
+      connections: Map<string, Connection> = new Map(),
    ) {
       this.projectName = projectName;
       this.packageName = packageName;
@@ -64,7 +62,7 @@ export class Package {
       this.packageMetadata = packageMetadata;
       this.databases = databases;
       this.models = models;
-      this.scheduler = scheduler;
+      this.connections = connections;
    }
 
    static async create(
@@ -72,7 +70,7 @@ export class Package {
       packageName: string,
       packagePath: string,
       projectConnections: Map<string, Connection>,
-      serverRootPath: string,
+      packageConnections: ApiConnection[],
    ): Promise<Package> {
       const startTime = performance.now();
       await Package.validatePackageManifestExistsOrThrowError(packagePath);
@@ -102,34 +100,15 @@ export class Package {
             unit: "ms",
          });
          const connections = new Map<string, Connection>(projectConnections);
-         logger.info(`Project connections: ${connections.size}`, {
-            connections,
-            projectConnections,
-         });
-         // Package connections override project connections.
-         const { malloyConnections: packageConnections } =
-            await createConnections(
-               packagePath,
-               [],
-               projectName,
-               serverRootPath,
-            );
-         const connectionsTime = performance.now();
-         logger.info("Package connections created", {
-            packageName,
-            connectionCount: packageConnections.size,
-            duration: connectionsTime - databasesTime,
-            unit: "ms",
-         });
-         packageConnections.forEach((connection) => {
-            connections.set(connection.name, connection);
-         });
 
          // Add a duckdb connection for the package.
-         connections.set(
-            "duckdb",
-            new DuckDBConnection("duckdb", ":memory:", packagePath),
+         const duckdbConnections = await createPackageDuckDBConnections(
+            packageConnections,
+            packagePath,
          );
+         duckdbConnections.malloyConnections.forEach((connection, name) => {
+            connections.set(name, connection);
+         });
 
          const models = await Package.loadModels(
             packageName,
@@ -140,14 +119,7 @@ export class Package {
          logger.info("Models loaded", {
             packageName,
             modelCount: models.size,
-            duration: modelsTime - connectionsTime,
-            unit: "ms",
-         });
-         const scheduler = Scheduler.create(models);
-         const schedulerTime = performance.now();
-         logger.info("Scheduler created", {
-            packageName,
-            duration: schedulerTime - modelsTime,
+            duration: modelsTime - databasesTime,
             unit: "ms",
          });
          const endTime = performance.now();
@@ -168,7 +140,7 @@ export class Package {
             packageConfig,
             databases,
             models,
-            scheduler,
+            connections,
          );
       } catch (error) {
          logger.error(`Error loading package ${packageName}`, { error });
@@ -195,14 +167,20 @@ export class Package {
       return this.databases;
    }
 
-   public listSchedules(): ApiSchedule[] {
-      return this.scheduler ? this.scheduler.list() : [];
-   }
-
    public getModel(modelPath: string): Model | undefined {
       return this.models.get(modelPath);
    }
 
+   public getMalloyConnection(connectionName: string): Connection {
+      const connection = this.connections.get(connectionName);
+      if (!connection) {
+         throw new Error(
+            `Connection ${connectionName} not found in package ${this.packageName}`,
+         );
+      }
+      return connection;
+   }
+
    public async getModelFileText(modelPath: string): Promise<string> {
       const model = this.getModel(modelPath);
       if (!model) {

package/src/service/project.ts

@@ -10,7 +10,7 @@ import {
    ProjectNotFoundError,
 } from "../errors";
 import { logger } from "../logger";
-import { createConnections, InternalConnection } from "./connection";
+import { createProjectConnections, InternalConnection } from "./connection";
 import { ApiConnection } from "./model";
 import { Package } from "./package";
 
@@ -37,7 +37,6 @@ export class Project {
    private apiConnections: ApiConnection[];
    private projectPath: string;
    private projectName: string;
-   private serverRootPath: string;
    public metadata: ApiProject;
 
    constructor(
@@ -45,11 +44,9 @@ export class Project {
       projectPath: string,
       malloyConnections: Map<string, BaseConnection>,
       apiConnections: InternalConnection[],
-      serverRootPath: string,
    ) {
       this.projectName = projectName;
       this.projectPath = projectPath;
-      this.serverRootPath = serverRootPath;
       this.malloyConnections = malloyConnections;
       this.apiConnections = apiConnections;
       this.metadata = {
@@ -73,10 +70,8 @@ export class Project {
          );
 
          // Reload connections with full config
-         const { malloyConnections, apiConnections } = await createConnections(
-            this.projectPath,
-            payload.connections,
-         );
+         const { malloyConnections, apiConnections } =
+            await createProjectConnections(payload.connections);
 
          // Update the project's connection maps
          this.malloyConnections = malloyConnections;
@@ -98,8 +93,7 @@ export class Project {
    static async create(
       projectName: string,
       projectPath: string,
-      defaultConnections: ApiConnection[],
-      serverRootPath?: string,
+      connections: ApiConnection[],
    ): Promise<Project> {
       if (!(await fs.promises.stat(projectPath)).isDirectory()) {
          throw new ProjectNotFoundError(
@@ -107,18 +101,9 @@ export class Project {
          );
       }
 
-      let malloyConnections: Map<string, BaseConnection> = new Map();
-      let apiConnections: InternalConnection[] = [];
-
       logger.info(`Creating project with connection configuration`);
-      const result = await createConnections(
-         projectPath,
-         defaultConnections,
-         projectName,
-         serverRootPath,
-      );
-      malloyConnections = result.malloyConnections;
-      apiConnections = result.apiConnections;
+      const { malloyConnections, apiConnections } =
+         await createProjectConnections(connections);
 
       logger.info(
          `Loaded ${malloyConnections.size + apiConnections.length} connections for project ${projectName}`,
@@ -133,7 +118,6 @@ export class Project {
          projectPath,
          malloyConnections,
          apiConnections,
-         serverRootPath || "",
       );
    }
 
@@ -264,7 +248,7 @@ export class Project {
                packageName,
                path.join(this.projectPath, packageName),
                this.malloyConnections,
-               this.serverRootPath,
+               this.apiConnections,
             );
             this.packages.set(packageName, _package);
 
@@ -309,7 +293,7 @@ export class Project {
                packageName,
                packagePath,
                this.malloyConnections,
-               this.serverRootPath,
+               this.apiConnections,
             ),
          );
       } catch (error) {

package/src/service/project_store.ts

@@ -232,7 +232,6 @@ export class ProjectStore {
          projectName,
          absoluteProjectPath,
          project.connections || [],
-         this.serverRootPath,
       );
       this.projects.set(projectName, newProject);
       projectConfig?.packages.forEach((_package) => {

package/dist/app/assets/ProjectPage-DA66xbmQ.js

@@ -1 +0,0 @@
-import{G as t,p as n,j as e,F as o,N as c}from"./index-DjbXd602.js";function j(){const r=t(),{projectName:s}=n();if(s){const a=o({projectName:s});return e.jsx(c,{onSelectPackage:r,resourceUri:a})}else return e.jsx("div",{children:e.jsx("h2",{children:"Missing project name"})})}export{j as default};

package/src/controller/schedule.controller.ts

@@ -1,21 +0,0 @@
-import { components } from "../api";
-import { ProjectStore } from "../service/project_store";
-
-type ApiSchedule = components["schemas"]["Schedule"];
-
-export class ScheduleController {
-   private projectStore: ProjectStore;
-
-   constructor(projectStore: ProjectStore) {
-      this.projectStore = projectStore;
-   }
-
-   public async listSchedules(
-      projectName: string,
-      packageName: string,
-   ): Promise<ApiSchedule[]> {
-      const project = await this.projectStore?.getProject?.(projectName);
-      const p = await project?.getPackage?.(packageName);
-      return p?.listSchedules?.();
-   }
-}

package/src/service/scheduler.ts

@@ -1,190 +0,0 @@
-import cron from "node-cron";
-import { components } from "../api";
-import { logger } from "../logger";
-import { Model } from "./model";
-
-// @ts-expect-error TODO: Fix missing Source type in API
-type ApiSource = components["schemas"]["Source"];
-type ApiView = components["schemas"]["View"];
-type ApiQuery = components["schemas"]["Query"];
-type ApiSchedule = components["schemas"]["Schedule"];
-
-const SCHEDULE_ANNOTATION = "# schedule";
-
-class Schedule {
-   private model: Model;
-   private source: ApiSource | undefined;
-   private view: ApiView | undefined;
-   private query: ApiQuery | undefined;
-
-   private schedule: string;
-   private action: string;
-   private connection: string;
-   private task: cron.ScheduledTask;
-   private lastRunTime: number | undefined;
-   private lastRunStatus: string;
-
-   public constructor(
-      model: Model,
-      source: ApiSource | undefined,
-      view: ApiView | undefined,
-      query: ApiQuery | undefined,
-      annotation: string,
-   ) {
-      this.model = model;
-      this.source = source;
-      this.view = view;
-      this.query = query;
-      const { origSchedule, cronSchedule, action, connection } =
-         Schedule.parseAnnotation(annotation);
-      this.schedule = origSchedule;
-      this.action = action;
-      this.connection = connection;
-      this.lastRunTime = undefined;
-      this.lastRunStatus = "unknown";
-
-      this.task = cron.schedule(cronSchedule, () => {
-         this.lastRunTime = Date.now();
-         this.lastRunStatus = "ok";
-      });
-   }
-
-   public stop() {
-      this.task.stop();
-   }
-
-   public get(): ApiSchedule {
-      const query = this.source
-         ? `${this.source.name} > ${this.view?.name}`
-         : this.query?.name;
-      return {
-         resource: `${this.model.getPath()}  > ${query}`,
-         schedule: this.schedule,
-         action: this.action,
-         connection: this.connection,
-         lastRunTime: this.lastRunTime,
-         lastRunStatus: this.lastRunStatus,
-      };
-   }
-
-   private static parseAnnotation(annotation: string) {
-      // Example schedule annotation
-      // # schedule @hourly materialize duckdb
-      // # schedule "0 * * * *" materialize duckdb
-      // TODO: Don't split quoted strings.  Use regex instead of split.
-      const annotationSplit = annotation.split(/\s+/);
-      if (annotationSplit.length != 6) {
-         logger.info("Length: " + annotationSplit.length);
-         throw new Error(
-            "Invalid annotation string does not have enough parts: " +
-               annotation,
-         );
-      }
-
-      if (annotationSplit[0] != "#") {
-         throw new Error(
-            "Invalid annotation string does not have start with #: " +
-               annotationSplit[0],
-         );
-      }
-
-      if (annotationSplit[1] != "schedule") {
-         throw new Error(
-            "Invalid annotation string does not start with schedule command: " +
-               annotationSplit[1],
-         );
-      }
-
-      const standardCron = Schedule.translateNonStandardCron(
-         annotationSplit[2],
-      );
-      if (!cron.validate(standardCron)) {
-         throw new Error(
-            "Invalid annotation string does not have valid cron schedule: " +
-               standardCron,
-         );
-      }
-
-      if (
-         annotationSplit[3] != "materialize" &&
-         annotationSplit[3] != "report"
-      ) {
-         throw new Error(
-            "Invalid annotation string unrecognized command: " + annotation,
-         );
-      }
-
-      // TODO: Validate connection exists.
-
-      return {
-         origSchedule: annotationSplit[2],
-         cronSchedule: standardCron,
-         action: annotationSplit[3],
-         connection: annotationSplit[4],
-      };
-   }
-
-   public static translateNonStandardCron(schedule: string): string {
-      let standardCron = schedule;
-      switch (schedule) {
-         case "@yearly":
-         case "@anually":
-            standardCron = "0 0 1 1 *";
-            break;
-         case "@monthly":
-            standardCron = "0 0 1 * *";
-            break;
-         case "@weekly":
-            standardCron = "0 0 * * 0";
-            break;
-         case "@daily":
-         case "@midnight":
-            standardCron = "0 0 * * *";
-            break;
-         case "@hourly":
-            standardCron = "0 * * * *";
-            break;
-         case "@minutely":
-            standardCron = "* * * * *";
-      }
-      return standardCron;
-   }
-}
-
-export class Scheduler {
-   private schedules: Schedule[];
-
-   private constructor(schedules: Schedule[]) {
-      this.schedules = schedules;
-   }
-
-   public static create(models: Map<string, Model>): Scheduler {
-      const schedules: Schedule[] = [];
-
-      models.forEach((m) => {
-         m.getSources()?.forEach((s) => {
-            s.views?.forEach((v: ApiView) => {
-               v.annotations?.forEach((a: string) => {
-                  if (a.startsWith(SCHEDULE_ANNOTATION)) {
-                     schedules.push(new Schedule(m, s, v, undefined, a));
-                  }
-               });
-            });
-         });
-
-         m.getQueries()?.forEach((q) => {
-            q.annotations?.forEach((a) => {
-               if (a.startsWith(SCHEDULE_ANNOTATION)) {
-                  schedules.push(new Schedule(m, undefined, undefined, q, a));
-               }
-            });
-         });
-      });
-
-      return new Scheduler(schedules);
-   }
-
-   public list(): ApiSchedule[] {
-      return this.schedules.map((s) => s.get());
-   }
-}