@malloy-publisher/server 0.0.219 → 0.0.220

package/src/service/theme_store.ts

@@ -0,0 +1,199 @@
+import { Mutex } from "async-mutex";
+import { getInstanceTheme, sanitizeTheme, Theme } from "../config";
+import { logger } from "../logger";
+import { DuckDBConnection } from "../storage/duckdb/DuckDBConnection";
+import { StorageManager } from "../storage/StorageManager";
+
+const THEME_ROW_ID = "default";
+
+/**
+ * Persistence for the instance-wide theme edited via the in-app Theme
+ * Editor. Backed by a single row in the `themes` table (DuckDB).
+ *
+ * Boot semantics: on first read after process start, if the table is
+ * empty, the row is seeded from `publisher.config.json`'s top-level
+ * `theme` block (if any). Subsequent writes go to DuckDB; the JSON file
+ * remains the boot seed only. This mirrors how environments + packages
+ * are handled. The JSON config is the seed; runtime mutations live in
+ * the DB and survive across restarts.
+ *
+ * Every payload read out of the DB is run through `sanitizeTheme` so a
+ * schema change can't put a stale shape in front of the editor (e.g.
+ * old per-mode `series` arrays after the simplification revert). The
+ * editor receives only fields that match the current type.
+ *
+ * Single in-process consumer; a Mutex serializes set / reset against
+ * concurrent reads to avoid the cache + DB diverging.
+ */
+export class ThemeStore {
+   private mutex = new Mutex();
+   private cached: Theme | undefined;
+   private cacheLoaded = false;
+
+   constructor(
+      private storageManager: StorageManager,
+      private serverRoot: string,
+   ) {}
+
+   private get db(): DuckDBConnection {
+      return this.storageManager.getDuckDbConnection();
+   }
+
+   /**
+    * Idempotent. Loads the persisted theme into memory. If the table is
+    * empty, seeds from `publisher.config.json`'s instance theme (if
+    * present) and persists the seed so subsequent reads + the editor see
+    * the same baseline.
+    */
+   async initialize(): Promise<void> {
+      await this.mutex.runExclusive(async () => {
+         await this.loadLocked();
+      });
+   }
+
+   async get(): Promise<Theme | undefined> {
+      if (!this.cacheLoaded) {
+         await this.initialize();
+      }
+      return this.cached;
+   }
+
+   async set(theme: Theme): Promise<Theme> {
+      return this.mutex.runExclusive(async () => {
+         await this.persistLocked(theme);
+         this.cached = theme;
+         this.cacheLoaded = true;
+         return theme;
+      });
+   }
+
+   /**
+    * Drops the persisted row so the next read falls back to the boot
+    * seed (or to no theme at all if the config doesn't define one).
+    */
+   async reset(): Promise<Theme | undefined> {
+      return this.mutex.runExclusive(async () => {
+         await this.db.run("DELETE FROM themes WHERE id = ?", [THEME_ROW_ID]);
+         this.cached = undefined;
+         this.cacheLoaded = false;
+         await this.loadLocked();
+         return this.cached;
+      });
+   }
+
+   /**
+    * Single source of truth for loading the cache. Must be called with
+    * the mutex held. Idempotent: returns immediately if already loaded.
+    */
+   private async loadLocked(): Promise<void> {
+      if (this.cacheLoaded) return;
+      const row = await this.db.get<{ payload: string }>(
+         "SELECT payload FROM themes WHERE id = ?",
+         [THEME_ROW_ID],
+      );
+      if (row?.payload) {
+         let parsed: unknown;
+         try {
+            parsed = JSON.parse(row.payload);
+         } catch (error) {
+            logger.warn(
+               "ThemeStore: stored theme JSON is unparseable; treating as empty",
+               { error },
+            );
+            this.cacheLoaded = true;
+            return;
+         }
+         // Run through sanitizeTheme so stale schema shapes (e.g. an
+         // old per-mode `series` object) get dropped before the editor
+         // ever sees them. Operators can keep editing without hitting
+         // Reset; fields that match the current shape survive.
+         const sanitized = sanitizeTheme(parsed, "ThemeStore.load");
+         const dropped = listSanitizerDrops(parsed, sanitized);
+         if (dropped.length > 0) {
+            // Surface what disappeared so operators reading the server
+            // log can see why the editor doesn't show their old custom
+            // values after an upgrade. Re-saving from the editor will
+            // overwrite the on-disk payload with the current shape.
+            logger.info(
+               "ThemeStore: dropped stale fields from stored theme during load",
+               { dropped },
+            );
+         }
+         this.cached = sanitized;
+      } else {
+         const seed = getInstanceTheme(this.serverRoot);
+         if (seed) {
+            this.cached = seed;
+            await this.persistLocked(seed);
+         }
+      }
+      this.cacheLoaded = true;
+   }
+
+   private async persistLocked(theme: Theme): Promise<void> {
+      const now = new Date().toISOString();
+      const payload = JSON.stringify(theme);
+      await this.db.run(
+         `INSERT INTO themes (id, payload, created_at, updated_at)
+          VALUES (?, ?, ?, ?)
+          ON CONFLICT (id) DO UPDATE SET
+            payload = EXCLUDED.payload,
+            updated_at = EXCLUDED.updated_at`,
+         [THEME_ROW_ID, payload, now, now],
+      );
+   }
+}
+
+/**
+ * Return the list of top-level Theme keys present on `raw` but absent
+ * from `sanitized` (i.e. dropped by `sanitizeTheme`). Reports dotted
+ * paths down one level so the log says e.g. `palette.series` rather
+ * than just `palette`. Returns an empty array when nothing was lost.
+ */
+function listSanitizerDrops(
+   raw: unknown,
+   sanitized: Theme | undefined,
+): string[] {
+   if (!raw || typeof raw !== "object" || Array.isArray(raw)) return [];
+   const r = raw as Record<string, unknown>;
+   const drops: string[] = [];
+
+   const topKeys = ["defaultMode", "allowUserToggle"] as const;
+   for (const k of topKeys) {
+      if (r[k] !== undefined && sanitized?.[k] === undefined) drops.push(k);
+   }
+
+   if (
+      r.palette &&
+      typeof r.palette === "object" &&
+      !Array.isArray(r.palette)
+   ) {
+      const rp = r.palette as Record<string, unknown>;
+      const sp = (sanitized?.palette ?? {}) as Record<string, unknown>;
+      for (const k of Object.keys(rp)) {
+         if (rp[k] !== undefined && sp[k] === undefined)
+            drops.push(`palette.${k}`);
+      }
+   } else if (
+      r.palette !== undefined &&
+      (!sanitized || sanitized.palette === undefined)
+   ) {
+      drops.push("palette");
+   }
+
+   if (r.font && typeof r.font === "object" && !Array.isArray(r.font)) {
+      const rf = r.font as Record<string, unknown>;
+      const sf = (sanitized?.font ?? {}) as Record<string, unknown>;
+      for (const k of Object.keys(rf)) {
+         if (rf[k] !== undefined && sf[k] === undefined)
+            drops.push(`font.${k}`);
+      }
+   } else if (
+      r.font !== undefined &&
+      (!sanitized || sanitized.font === undefined)
+   ) {
+      drops.push("font");
+   }
+
+   return drops;
+}

package/src/storage/DatabaseInterface.ts

@@ -143,7 +143,7 @@ export interface MaterializationUpdate {
 }
 
 /**
- * Malloy-facing build manifest: maps a buildId to the physical table backing
+ * Malloy-facing build manifest: maps a sourceEntityId to the physical table backing
  * that persist source. This is the shape the Malloy runtime consumes when
  * (re)loading models so persist references resolve to materialized tables.
  * Distinct from {@link BuildManifestResult} (the wire build output, which also

package/src/storage/StorageManager.ts

@@ -75,6 +75,23 @@ export class StorageManager {
       this.repository = new DuckDBRepository(connection);
    }
 
+   /**
+    * Access the underlying DuckDB connection for callers that need raw
+    * SQL access (e.g. the ThemeStore singleton) and don't fit the
+    * structured `ResourceRepository` interface. Use sparingly; prefer
+    * `getRepository()` for anything that maps cleanly onto the existing
+    * repository methods.
+    */
+   getDuckDbConnection(): DuckDBConnection {
+      if (!this.connection) {
+         throw new Error("Storage not initialized");
+      }
+      if (!(this.connection instanceof DuckDBConnection)) {
+         throw new Error("Underlying storage connection is not DuckDB");
+      }
+      return this.connection;
+   }
+
    getRepository(): ResourceRepository {
       if (!this.repository) {
          throw new Error("Storage not initialized. Call initialize() first.");

package/src/storage/duckdb/schema.ts

@@ -7,27 +7,28 @@ export async function initializeSchema(
 ): Promise<void> {
    const initialized = await db.isInitialized();
 
-   if (initialized && !force) {
-      return;
-   }
-
    if (force) {
       logger.info(
          "Reinitializing database schema dropping and recreating all tables",
       );
       await dropAllTables(db);
-   } else {
+   } else if (!initialized) {
       // TODO: Remove this during projects cleanup
       // If a pre-rename `projects` schema is on disk, the new
       // CREATE TABLE IF NOT EXISTS pass below would silently leave child
       // tables on the old `project_id` column and the first query against
       // `environment_id` would crash. Drop the legacy tables (with a loud
       // warning) so the fresh schema can be created cleanly. This is
-      // destructive — operators upgrading should re-create their environments
+      // destructive: operators upgrading should re-create their environments
       // and packages via the API after the upgrade.
       await dropLegacyProjectSchema(db);
       logger.info("Creating database schema for the first time...");
    }
+   // Always fall through to the CREATE TABLE IF NOT EXISTS pass below.
+   // The statements are idempotent and let an already-initialized DB pick
+   // up new tables added in later builds (e.g., the `themes` table for
+   // the in-app Theme Editor) without forcing operators to run --init
+   // and lose their existing environments / packages / materializations.
 
    // Environments table
    await db.run(`
@@ -102,6 +103,25 @@ export async function initializeSchema(
     )
   `);
 
+   // Themes table.
+   //
+   // Singleton storage for the instance-wide theme edited by the in-app
+   // Theme Editor. At most one row (id = "default"); we use INSERT ON
+   // CONFLICT to keep the table effectively a key/value cell.
+   //
+   // publisher.config.json's optional `theme` block is a BOOT SEED only —
+   // on first run, if this table is empty, ThemeStore writes the seed
+   // here. Subsequent reads come from this table so the editor's saves
+   // win over the file.
+   await db.run(`
+    CREATE TABLE IF NOT EXISTS themes (
+      id VARCHAR PRIMARY KEY,
+      payload JSON NOT NULL,
+      created_at TIMESTAMP NOT NULL,
+      updated_at TIMESTAMP NOT NULL
+    )
+  `);
+
    // Create indexes for better query performance
    await db.run(
       "CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)",
@@ -156,6 +176,7 @@ async function dropAllTables(db: DuckDBConnection): Promise<void> {
       "packages",
       "connections",
       "environments",
+      "themes",
    ];
 
    logger.info("Dropping tables:", tables.join(", "));

package/src/theme_key_parity.spec.ts

@@ -0,0 +1,57 @@
+import { describe, expect, it } from "bun:test";
+import { readFileSync } from "fs";
+import { resolve } from "path";
+import { PER_MODE_COLOR_KEYS } from "./config";
+
+/**
+ * The per-mode palette key list is hand-copied in three places: this server
+ * copy (config.ts), the SDK copy (packages/sdk/src/theme/keys.ts), and the
+ * api-doc.yaml Theme.palette schema. The server and the SDK are intentionally
+ * decoupled (neither imports the other), and api-doc.yaml is their only shared
+ * contract, so nothing but this test keeps the three in step. A silent drift
+ * drops a colour: the server sanitizer or the SDK resolver ignores a key that
+ * is not in its own copy.
+ */
+
+function extractTsArray(source: string, constName: string): string[] {
+   const decl = source.indexOf(`${constName} = [`);
+   if (decl === -1) throw new Error(`${constName} array not found`);
+   const open = source.indexOf("[", decl);
+   const close = source.indexOf("]", open);
+   return [...source.slice(open + 1, close).matchAll(/"([^"]+)"/g)].map(
+      (m) => m[1],
+   );
+}
+
+function extractApiDocPaletteKeys(apiDoc: string): string[] {
+   const start = apiDoc.indexOf("\n        palette:");
+   if (start === -1) throw new Error("palette block not found in api-doc.yaml");
+   const end = apiDoc.indexOf("\n        font:", start);
+   const block = apiDoc.slice(start, end === -1 ? undefined : end);
+   // Direct children of `palette.properties` sit at 12-space indentation;
+   // deeper keys (type, properties, light, dark) are more indented and skipped.
+   return [...block.matchAll(/^ {12}(\w+):$/gm)].map((m) => m[1]);
+}
+
+describe("per-mode palette key parity", () => {
+   const serverKeys = [...PER_MODE_COLOR_KEYS];
+
+   it("server config.ts matches the SDK keys.ts copy", () => {
+      const sdk = readFileSync(
+         resolve(import.meta.dir, "../../sdk/src/theme/keys.ts"),
+         "utf8",
+      );
+      expect(extractTsArray(sdk, "PER_MODE_COLOR_KEYS")).toEqual(serverKeys);
+   });
+
+   it("server config.ts matches the api-doc.yaml Theme.palette schema", () => {
+      const apiDoc = readFileSync(
+         resolve(import.meta.dir, "../../../api-doc.yaml"),
+         "utf8",
+      );
+      // palette also declares `series` (a shared array, not a per-mode object).
+      expect(new Set(extractApiDocPaletteKeys(apiDoc))).toEqual(
+         new Set([...serverKeys, "series"]),
+      );
+   });
+});

package/tests/integration/materialization/manifest_binding.integration.spec.ts

@@ -111,7 +111,7 @@ describe("Manifest binding via Package.manifestLocation (E2E)", () => {
             strict: false,
             entries: {
                build123: {
-                  buildId: "build123",
+                  sourceEntityId: "build123",
                   sourceName: "order_summary",
                   physicalTableName: "main.order_summary_mz",
                   connectionName: "duckdb",

package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts

@@ -206,14 +206,14 @@ describe("Materialization REST API: single-call (E2E)", () => {
             // instruction, and create the materialization already building it.
             const source = await firstPlanSource();
             expect(source.name).toBe("order_summary");
-            expect(typeof source.buildId).toBe("string");
+            expect(typeof source.sourceEntityId).toBe("string");
 
             const physicalTableName = "order_summary_built";
             const createRes = await createMaterialization({
                buildInstructions: {
                   sources: [
                      {
-                        buildId: source.buildId,
+                        sourceEntityId: source.sourceEntityId,
                         sourceID: source.sourceID,
                         materializedTableId: "mt-order-summary",
                         physicalTableName,
@@ -241,7 +241,7 @@ describe("Materialization REST API: single-call (E2E)", () => {
                string,
                Record<string, unknown>
             >;
-            const entry = entries[source.buildId as string];
+            const entry = entries[source.sourceEntityId as string];
             expect(entry).toBeDefined();
             expect(entry.physicalTableName).toBe(physicalTableName);
             expect(entry.sourceName).toBe("order_summary");
@@ -261,12 +261,12 @@ describe("Materialization REST API: single-call (E2E)", () => {
          { timeout: 90_000 },
       );
 
-      it("rejects buildInstructions with an unknown buildId at create (400)", async () => {
+      it("rejects buildInstructions with an unknown sourceEntityId at create (400)", async () => {
          const createRes = await createMaterialization({
             buildInstructions: {
                sources: [
                   {
-                     buildId: "not-a-real-build-id",
+                     sourceEntityId: "not-a-real-build-id",
                      materializedTableId: "mt",
                      physicalTableName: "t",
                      realization: "COPY",

package/tests/unit/controller/theme.controller.spec.ts

@@ -0,0 +1,141 @@
+// Lives under `tests/unit/` (not `src/`) on purpose: a sibling spec
+// (environment_store.spec.ts) calls
+// `mock.module("../storage/StorageManager", ...)`. Bun's module mocks
+// persist process-wide across spec files, so a spec under `src/` that
+// imports the real StorageManager would silently get the mock (which
+// doesn't expose `getDuckDbConnection`). The integration runner gives
+// us a clean module cache.
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs";
+import path from "path";
+import { PUBLISHER_CONFIG_NAME } from "../../../src/constants";
+import { ThemeController } from "../../../src/controller/theme.controller";
+import { BadRequestError, FrozenConfigError } from "../../../src/errors";
+import { ThemeStore } from "../../../src/service/theme_store";
+import { StorageManager } from "../../../src/storage/StorageManager";
+
+const TEST_ROOT = path.join(process.cwd(), "test-temp-theme-controller");
+
+function writeConfig(json: object) {
+   if (!fs.existsSync(TEST_ROOT)) fs.mkdirSync(TEST_ROOT, { recursive: true });
+   fs.writeFileSync(
+      path.join(TEST_ROOT, PUBLISHER_CONFIG_NAME),
+      JSON.stringify(json),
+   );
+}
+
+async function build(
+   configJson: object,
+): Promise<{ controller: ThemeController; sm: StorageManager }> {
+   writeConfig(configJson);
+   const sm = new StorageManager({
+      type: "duckdb",
+      duckdb: { path: ":memory:" },
+   });
+   await sm.initialize(true);
+   const store = new ThemeStore(sm, TEST_ROOT);
+   return { controller: new ThemeController(store, TEST_ROOT), sm };
+}
+
+describe("ThemeController", () => {
+   beforeEach(() => {
+      if (fs.existsSync(TEST_ROOT)) {
+         fs.rmSync(TEST_ROOT, { recursive: true, force: true });
+      }
+   });
+   afterEach(() => {
+      if (fs.existsSync(TEST_ROOT)) {
+         fs.rmSync(TEST_ROOT, { recursive: true, force: true });
+      }
+   });
+
+   it("getTheme returns the boot-seeded theme", async () => {
+      const { controller } = await build({
+         frozenConfig: false,
+         theme: { palette: { series: ["#abc"] } },
+         environments: [],
+      });
+      const theme = await controller.getTheme();
+      expect(theme.palette?.series).toEqual(["#abc"]);
+   });
+
+   it("getTheme returns an empty object when no theme configured", async () => {
+      const { controller } = await build({
+         frozenConfig: false,
+         environments: [],
+      });
+      expect(await controller.getTheme()).toEqual({});
+   });
+
+   it("putTheme writes and returns the theme", async () => {
+      const { controller } = await build({
+         frozenConfig: false,
+         environments: [],
+      });
+      const saved = await controller.putTheme({
+         palette: { series: ["#ff0080"] },
+      });
+      expect(saved.palette?.series).toEqual(["#ff0080"]);
+      // The next GET returns the same value.
+      expect((await controller.getTheme()).palette?.series).toEqual([
+         "#ff0080",
+      ]);
+   });
+
+   it("putTheme accepts an empty object as a clear-all-overrides signal", async () => {
+      const { controller } = await build({
+         frozenConfig: false,
+         theme: { palette: { series: ["#aaa"] } },
+         environments: [],
+      });
+      // Seed the editor's draft, then PUT {} to clear it.
+      await controller.putTheme({ palette: { series: ["#edit"] } });
+      const cleared = await controller.putTheme({});
+      expect(cleared).toEqual({});
+      expect(await controller.getTheme()).toEqual({});
+   });
+
+   it("putTheme rejects non-object payloads", async () => {
+      const { controller } = await build({
+         frozenConfig: false,
+         environments: [],
+      });
+      await expect(controller.putTheme("nope")).rejects.toBeInstanceOf(
+         BadRequestError,
+      );
+      await expect(controller.putTheme([1, 2, 3])).rejects.toBeInstanceOf(
+         BadRequestError,
+      );
+   });
+
+   it("putTheme throws FrozenConfigError when frozenConfig is true", async () => {
+      const { controller } = await build({
+         frozenConfig: true,
+         environments: [],
+      });
+      await expect(
+         controller.putTheme({ palette: { series: ["#abc"] } }),
+      ).rejects.toBeInstanceOf(FrozenConfigError);
+   });
+
+   it("resetTheme falls back to the boot seed", async () => {
+      const { controller } = await build({
+         frozenConfig: false,
+         theme: { palette: { series: ["#seed"] } },
+         environments: [],
+      });
+      await controller.putTheme({ palette: { series: ["#edit"] } });
+      const reseeded = await controller.resetTheme();
+      expect(reseeded.palette?.series).toEqual(["#seed"]);
+   });
+
+   it("resetTheme throws FrozenConfigError when frozenConfig is true", async () => {
+      const { controller } = await build({
+         frozenConfig: true,
+         environments: [],
+      });
+      await expect(controller.resetTheme()).rejects.toBeInstanceOf(
+         FrozenConfigError,
+      );
+   });
+});