@malloy-publisher/server 0.0.235 → 0.0.236

package/src/service/source_extraction.ts

@@ -1,225 +0,0 @@
-/**
- * Shared source / query introspection extracted from a compiled `ModelDef`.
- *
- * Both the in-process `Model.create` path (`service/model.ts`) and the
- * package-load worker (`package_load/package_load_worker.ts`, which runs in a
- * separate bundle and serializes the result over the worker protocol) need to
- * walk a `ModelDef` and produce the same `sources` / `queries` shapes plus the
- * `#(filter)` `filterMap`. This module is the single source of truth for that
- * walk so the two call sites can't drift out of lockstep — the two callers
- * differ only in how they type the result
- * (generated API types vs. worker wire types — structurally identical, so each
- * casts at its boundary) and in how they report a filter parse failure (the
- * service logs a warning; the worker has no logger and stays silent), which is
- * threaded through the optional `onParseError` callback.
- */
-
-import {
-   isSourceDef,
-   ModelDef,
-   NamedModelObject,
-   NamedQueryDef,
-   StructDef,
-   TurtleDef,
-} from "@malloydata/malloy";
-import { annotationTexts, modelAnnotations } from "./annotations";
-import { collectAuthorizeExprs, type AuthorizeMap } from "./authorize";
-import { parseFilters, type FilterDefinition } from "./filter";
-
-/** A `#(filter)` definition enriched with the dimension's Malloy type. */
-export interface ExtractedFilter {
-   name: string;
-   dimension: string;
-   type: string;
-   implicit: boolean;
-   required: boolean;
-   dimensionType: string | undefined;
-}
-
-export interface ExtractedView {
-   name: string;
-   annotations: string[] | undefined;
-}
-
-/**
- * Structural source shape both callers cast to their own typed view
- * (`ApiSource` in the service, `ApiSourceWire` in the worker). `givens` is
- * attached verbatim from the caller-supplied list, so it stays `unknown` here.
- */
-export interface ExtractedSource {
-   name: string;
-   annotations: string[] | undefined;
-   views: ExtractedView[];
-   filters: ExtractedFilter[] | undefined;
-   givens: unknown;
-   /**
-    * Effective `#(authorize)` / `##(authorize)` expressions gating this source:
-    * file-level expressions first, then the source's own. Undefined when the
-    * source carries no authorize annotations. Surfaced for introspection;
-    * enforcement happens server-side.
-    */
-   authorize: string[] | undefined;
-}
-
-export interface ExtractedQuery {
-   name: string;
-   sourceName: string | undefined;
-   annotations: string[] | undefined;
-}
-
-/**
- * Extract every source from a compiled model, parsing `#(filter)` annotations
- * along the way.
- *
- * Filters are collected by walking the `annotations.inherits` chain so that
- * filters declared on a base source flow to an extending source. The chain runs
- * child → parent, so we collect child-first then reverse — `parseFilters` uses
- * "last wins" dedup, which lets a child's `#(filter)` override the base's.
- *
- * `givens` is attached unchanged to every source (Malloy exposes givens at the
- * model level, not per-source). `onParseError`, when supplied, is invoked with
- * the source name and error if a source's `#(filter)` annotations fail to parse;
- * filter extraction then continues. Authorize parse errors are NOT routed here —
- * they propagate (a malformed gate fails model load) so a security gate is never
- * silently dropped.
- *
- * Authorize (`#(authorize)` / `##(authorize)`) is collected from the source's
- * own `blockNotes` only — we do NOT walk the `inherits` chain. Note Malloy's
- * behavior for `X is Y extend {...}`: if X declares its own `#(authorize)`,
- * X.blockNotes holds only X's gates (Y's are dropped — the intended "curated
- * re-exposure"); if X declares none, Malloy surfaces Y's blockNotes on X, so
- * the base gate carries to the un-annotated extension (a safe default — a
- * locked base stays locked unless an extension explicitly re-exposes itself).
- * This carry happens through `blockNotes`, not the `inherits` chain, so reading
- * own-blockNotes is sufficient. Joins are a separate concern and are not gated.
- * The effective list per source is the file-level `##(authorize)` expressions
- * (from `modelDef.annotations.notes`) followed by the source's own
- * `#(authorize)` expressions, evaluated as one OR disjunction at request time.
- */
-export function extractSourcesFromModelDef(
-   modelDef: ModelDef,
-   givens: unknown,
-   onParseError?: (sourceName: string, err: unknown) => void,
-): {
-   sources: ExtractedSource[];
-   filterMap: Map<string, FilterDefinition[]>;
-   authorizeMap: AuthorizeMap;
-} {
-   const filterMap = new Map<string, FilterDefinition[]>();
-   const authorizeMap: AuthorizeMap = new Map();
-
-   // File-level ##(authorize) is collected once and prepended to every source.
-   // Unlike filters, a malformed authorize annotation is NOT swallowed: the
-   // parse error propagates so the model fails to load loudly (caught per-model
-   // upstream and turned into a compilationError). Silently dropping a gate —
-   // and in the worker path there is no onParseError callback, so it would be
-   // truly silent — could leave a source that the author meant to lock looking
-   // unrestricted.
-   const fileLevelAuthorize = collectAuthorizeExprs(
-      (modelAnnotations(modelDef).notes ?? []).map((note) => note.text),
-   );
-
-   const sources: ExtractedSource[] = Object.values(modelDef.contents)
-      .filter((obj) => isSourceDef(obj))
-      .map((sourceObj) => {
-         const struct = sourceObj as StructDef;
-         const sourceName = struct.as || struct.name;
-         const annotations = annotationTexts(struct.annotations);
-
-         const collected: string[][] = [];
-         let cur = struct.annotations;
-         while (cur) {
-            if (cur.blockNotes) {
-               collected.push(cur.blockNotes.map((note) => note.text));
-            }
-            cur = cur.inherits;
-         }
-         const allAnnotations = collected.reverse().flat();
-
-         let filters: ExtractedFilter[] | undefined;
-         if (allAnnotations.length > 0) {
-            try {
-               const parsed = parseFilters(allAnnotations);
-               if (parsed.length > 0) {
-                  filterMap.set(sourceName, parsed);
-                  const fields = struct.fields;
-                  filters = parsed.map((f) => {
-                     const field = fields.find(
-                        (fd) => (fd.as || fd.name) === f.dimension,
-                     );
-                     return {
-                        name: f.name,
-                        dimension: f.dimension,
-                        type: f.type,
-                        implicit: f.implicit,
-                        required: f.required,
-                        dimensionType: field?.type as string | undefined,
-                     };
-                  });
-               }
-            } catch (err) {
-               onParseError?.(sourceName, err);
-            }
-         }
-
-         // Authorize: the source's OWN #(authorize) annotations only — no
-         // inherits walk. File-level ##(authorize) is prepended so file gates
-         // and source gates form one OR disjunction. A malformed annotation
-         // propagates (model fails to load) rather than silently dropping the
-         // gate — see the file-level note above.
-         const ownNotes = (struct.annotations?.blockNotes ?? []).map(
-            (note) => note.text,
-         );
-         const effective = [
-            ...fileLevelAuthorize,
-            ...collectAuthorizeExprs(ownNotes),
-         ];
-         let authorize: string[] | undefined;
-         if (effective.length > 0) {
-            authorizeMap.set(sourceName, effective);
-            authorize = effective;
-         }
-
-         const views: ExtractedView[] = struct.fields
-            .filter((field) => field.type === "turtle")
-            .filter((turtle) =>
-               // Filter out non-reduce views (e.g. indexes).
-               (turtle as TurtleDef).pipeline
-                  .map((stage) => stage.type)
-                  .every((type) => type === "reduce"),
-            )
-            .map((turtle) => ({
-               name: turtle.as || turtle.name,
-               annotations: annotationTexts(turtle.annotations),
-            }));
-
-         return {
-            name: sourceName,
-            annotations,
-            views,
-            filters,
-            givens,
-            authorize,
-         };
-      });
-
-   return { sources, filterMap, authorizeMap };
-}
-
-/** Extract every named query from a compiled model. */
-export function extractQueriesFromModelDef(
-   modelDef: ModelDef,
-): ExtractedQuery[] {
-   const isNamedQuery = (obj: NamedModelObject): obj is NamedQueryDef =>
-      obj.type === "query";
-   return Object.values(modelDef.contents)
-      .filter(isNamedQuery)
-      .map((queryObj) => ({
-         name: queryObj.as || queryObj.name,
-         sourceName:
-            typeof queryObj.structRef === "string"
-               ? queryObj.structRef
-               : undefined,
-         annotations: annotationTexts(queryObj.annotations),
-      }));
-}

package/src/service/theme_store.ts

@@ -1,199 +0,0 @@
-import { Mutex } from "async-mutex";
-import { getInstanceTheme, sanitizeTheme, Theme } from "../config";
-import { logger } from "../logger";
-import { DuckDBConnection } from "../storage/duckdb/DuckDBConnection";
-import { StorageManager } from "../storage/StorageManager";
-
-const THEME_ROW_ID = "default";
-
-/**
- * Persistence for the instance-wide theme edited via the in-app Theme
- * Editor. Backed by a single row in the `themes` table (DuckDB).
- *
- * Boot semantics: on first read after process start, if the table is
- * empty, the row is seeded from `publisher.config.json`'s top-level
- * `theme` block (if any). Subsequent writes go to DuckDB; the JSON file
- * remains the boot seed only. This mirrors how environments + packages
- * are handled. The JSON config is the seed; runtime mutations live in
- * the DB and survive across restarts.
- *
- * Every payload read out of the DB is run through `sanitizeTheme` so a
- * schema change can't put a stale shape in front of the editor (e.g.
- * old per-mode `series` arrays after the simplification revert). The
- * editor receives only fields that match the current type.
- *
- * Single in-process consumer; a Mutex serializes set / reset against
- * concurrent reads to avoid the cache + DB diverging.
- */
-export class ThemeStore {
-   private mutex = new Mutex();
-   private cached: Theme | undefined;
-   private cacheLoaded = false;
-
-   constructor(
-      private storageManager: StorageManager,
-      private serverRoot: string,
-   ) {}
-
-   private get db(): DuckDBConnection {
-      return this.storageManager.getDuckDbConnection();
-   }
-
-   /**
-    * Idempotent. Loads the persisted theme into memory. If the table is
-    * empty, seeds from `publisher.config.json`'s instance theme (if
-    * present) and persists the seed so subsequent reads + the editor see
-    * the same baseline.
-    */
-   async initialize(): Promise<void> {
-      await this.mutex.runExclusive(async () => {
-         await this.loadLocked();
-      });
-   }
-
-   async get(): Promise<Theme | undefined> {
-      if (!this.cacheLoaded) {
-         await this.initialize();
-      }
-      return this.cached;
-   }
-
-   async set(theme: Theme): Promise<Theme> {
-      return this.mutex.runExclusive(async () => {
-         await this.persistLocked(theme);
-         this.cached = theme;
-         this.cacheLoaded = true;
-         return theme;
-      });
-   }
-
-   /**
-    * Drops the persisted row so the next read falls back to the boot
-    * seed (or to no theme at all if the config doesn't define one).
-    */
-   async reset(): Promise<Theme | undefined> {
-      return this.mutex.runExclusive(async () => {
-         await this.db.run("DELETE FROM themes WHERE id = ?", [THEME_ROW_ID]);
-         this.cached = undefined;
-         this.cacheLoaded = false;
-         await this.loadLocked();
-         return this.cached;
-      });
-   }
-
-   /**
-    * Single source of truth for loading the cache. Must be called with
-    * the mutex held. Idempotent: returns immediately if already loaded.
-    */
-   private async loadLocked(): Promise<void> {
-      if (this.cacheLoaded) return;
-      const row = await this.db.get<{ payload: string }>(
-         "SELECT payload FROM themes WHERE id = ?",
-         [THEME_ROW_ID],
-      );
-      if (row?.payload) {
-         let parsed: unknown;
-         try {
-            parsed = JSON.parse(row.payload);
-         } catch (error) {
-            logger.warn(
-               "ThemeStore: stored theme JSON is unparseable; treating as empty",
-               { error },
-            );
-            this.cacheLoaded = true;
-            return;
-         }
-         // Run through sanitizeTheme so stale schema shapes (e.g. an
-         // old per-mode `series` object) get dropped before the editor
-         // ever sees them. Operators can keep editing without hitting
-         // Reset; fields that match the current shape survive.
-         const sanitized = sanitizeTheme(parsed, "ThemeStore.load");
-         const dropped = listSanitizerDrops(parsed, sanitized);
-         if (dropped.length > 0) {
-            // Surface what disappeared so operators reading the server
-            // log can see why the editor doesn't show their old custom
-            // values after an upgrade. Re-saving from the editor will
-            // overwrite the on-disk payload with the current shape.
-            logger.info(
-               "ThemeStore: dropped stale fields from stored theme during load",
-               { dropped },
-            );
-         }
-         this.cached = sanitized;
-      } else {
-         const seed = getInstanceTheme(this.serverRoot);
-         if (seed) {
-            this.cached = seed;
-            await this.persistLocked(seed);
-         }
-      }
-      this.cacheLoaded = true;
-   }
-
-   private async persistLocked(theme: Theme): Promise<void> {
-      const now = new Date().toISOString();
-      const payload = JSON.stringify(theme);
-      await this.db.run(
-         `INSERT INTO themes (id, payload, created_at, updated_at)
-          VALUES (?, ?, ?, ?)
-          ON CONFLICT (id) DO UPDATE SET
-            payload = EXCLUDED.payload,
-            updated_at = EXCLUDED.updated_at`,
-         [THEME_ROW_ID, payload, now, now],
-      );
-   }
-}
-
-/**
- * Return the list of top-level Theme keys present on `raw` but absent
- * from `sanitized` (i.e. dropped by `sanitizeTheme`). Reports dotted
- * paths down one level so the log says e.g. `palette.series` rather
- * than just `palette`. Returns an empty array when nothing was lost.
- */
-function listSanitizerDrops(
-   raw: unknown,
-   sanitized: Theme | undefined,
-): string[] {
-   if (!raw || typeof raw !== "object" || Array.isArray(raw)) return [];
-   const r = raw as Record<string, unknown>;
-   const drops: string[] = [];
-
-   const topKeys = ["defaultMode", "allowUserToggle"] as const;
-   for (const k of topKeys) {
-      if (r[k] !== undefined && sanitized?.[k] === undefined) drops.push(k);
-   }
-
-   if (
-      r.palette &&
-      typeof r.palette === "object" &&
-      !Array.isArray(r.palette)
-   ) {
-      const rp = r.palette as Record<string, unknown>;
-      const sp = (sanitized?.palette ?? {}) as Record<string, unknown>;
-      for (const k of Object.keys(rp)) {
-         if (rp[k] !== undefined && sp[k] === undefined)
-            drops.push(`palette.${k}`);
-      }
-   } else if (
-      r.palette !== undefined &&
-      (!sanitized || sanitized.palette === undefined)
-   ) {
-      drops.push("palette");
-   }
-
-   if (r.font && typeof r.font === "object" && !Array.isArray(r.font)) {
-      const rf = r.font as Record<string, unknown>;
-      const sf = (sanitized?.font ?? {}) as Record<string, unknown>;
-      for (const k of Object.keys(rf)) {
-         if (rf[k] !== undefined && sf[k] === undefined)
-            drops.push(`font.${k}`);
-      }
-   } else if (
-      r.font !== undefined &&
-      (!sanitized || sanitized.font === undefined)
-   ) {
-      drops.push("font");
-   }
-
-   return drops;
-}

package/src/storage/DatabaseInterface.ts

@@ -1,198 +0,0 @@
-import { components } from "../api";
-
-/**
- * Generic database interface for storage abstraction
- * This allows switching between DuckDB, PostgreSQL, MySQL, etc.
- */
-export interface DatabaseConnection {
-   initialize(): Promise<void>;
-   close(): Promise<void>;
-   isInitialized(): Promise<boolean>;
-}
-
-export interface ResourceRepository {
-   // Environments
-   listEnvironments(): Promise<Environment[]>;
-   getEnvironmentById(id: string): Promise<Environment | null>;
-   getEnvironmentByName(name: string): Promise<Environment | null>;
-   createEnvironment(
-      environment: Omit<Environment, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Environment>;
-   updateEnvironment(
-      id: string,
-      updates: Partial<Environment>,
-   ): Promise<Environment>;
-   deleteEnvironment(id: string): Promise<void>;
-
-   // Packages
-   listPackages(environmentId: string): Promise<Package[]>;
-   getPackageById(id: string): Promise<Package | null>;
-   getPackageByName(
-      environmentId: string,
-      name: string,
-   ): Promise<Package | null>;
-   createPackage(
-      pkg: Omit<Package, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Package>;
-   updatePackage(id: string, updates: Partial<Package>): Promise<Package>;
-   deletePackage(id: string): Promise<void>;
-
-   // Connections
-   listConnections(environmentId: string): Promise<Connection[]>;
-   getConnectionById(id: string): Promise<Connection | null>;
-   getConnectionByName(
-      environmentId: string,
-      name: string,
-   ): Promise<Connection | null>;
-   createConnection(
-      connection: Omit<Connection, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Connection>;
-   updateConnection(
-      id: string,
-      updates: Partial<Connection>,
-   ): Promise<Connection>;
-   deleteConnection(id: string): Promise<void>;
-
-   // Materializations
-   listMaterializations(
-      environmentId: string,
-      packageName: string,
-      options?: { limit?: number; offset?: number },
-   ): Promise<Materialization[]>;
-   listMaterializationsByEnvironment(
-      environmentId: string,
-      options?: { limit?: number; offset?: number },
-   ): Promise<Materialization[]>;
-   getLatestScheduledFireAt(
-      environmentId: string,
-      packageName: string,
-   ): Promise<Date | null>;
-   getMaterializationById(id: string): Promise<Materialization | null>;
-   getActiveMaterialization(
-      environmentId: string,
-      packageName: string,
-   ): Promise<Materialization | null>;
-   createMaterialization(
-      environmentId: string,
-      packageName: string,
-      status?: MaterializationStatus,
-      metadata?: Record<string, unknown> | null,
-   ): Promise<Materialization>;
-   updateMaterialization(
-      id: string,
-      updates: MaterializationUpdate,
-   ): Promise<Materialization>;
-   deleteMaterialization(id: string): Promise<void>;
-}
-
-export interface Environment {
-   id: string;
-   name: string;
-   path: string;
-   description?: string;
-   createdAt: Date;
-   updatedAt: Date;
-   metadata?: Record<string, unknown>;
-}
-
-export interface Package {
-   id: string;
-   environmentId: string;
-   name: string;
-   description?: string;
-   manifestPath: string;
-   createdAt: Date;
-   updatedAt: Date;
-   metadata?: Record<string, unknown>;
-}
-
-export interface Connection {
-   id: string;
-   environmentId: string;
-   name: string;
-   type: "bigquery" | "postgres" | "duckdb" | "mysql" | "snowflake" | "trino";
-   config: Record<string, unknown>;
-   createdAt: Date;
-   updatedAt: Date;
-}
-
-// Wire types for the build protocol, kept in sync with the OpenAPI spec via
-// the generated `api.ts`.
-export type MaterializationStatus =
-   components["schemas"]["MaterializationStatus"];
-export type BuildPlan = components["schemas"]["BuildPlan"];
-export type BuildManifestResult = components["schemas"]["BuildManifest"];
-export type ManifestEntry = components["schemas"]["ManifestEntry"];
-export type BuildInstruction = components["schemas"]["BuildInstruction"];
-export type ManifestReference = components["schemas"]["ManifestReference"];
-export type Realization = components["schemas"]["Realization"];
-
-export interface Materialization {
-   id: string;
-   environmentId: string;
-   packageName: string;
-   status: MaterializationStatus;
-   /** Build output. Null until status = MANIFEST_FILE_READY. */
-   manifest: BuildManifestResult | null;
-   startedAt: Date | null;
-   completedAt: Date | null;
-   error: string | null;
-   metadata: Record<string, unknown> | null;
-   createdAt: Date;
-   updatedAt: Date;
-}
-
-export interface MaterializationUpdate {
-   status?: MaterializationStatus;
-   manifest?: BuildManifestResult | null;
-   startedAt?: Date;
-   completedAt?: Date;
-   error?: string | null;
-   metadata?: Record<string, unknown> | null;
-}
-
-/**
- * Malloy-facing build manifest: maps a sourceEntityId to the physical table backing
- * that persist source. This is the shape the Malloy runtime consumes when
- * (re)loading models so persist references resolve to materialized tables.
- * Distinct from {@link BuildManifestResult} (the wire build output, which also
- * carries caller bookkeeping like materializedTableId and rowCount).
- */
-export interface BuildManifestEntry {
-   tableName: string;
-}
-
-export interface BuildManifest {
-   entries: Record<string, BuildManifestEntry>;
-   strict?: boolean;
-}
-
-/**
- * A wire manifest entry carrying the control-plane freshness fields alongside
- * the physical `tableName`. This is what the publisher retains at bind so the
- * serve path can re-evaluate `age vs window` per query (see
- * {@link ../service/freshness}). A `{ tableName }`-only value is structurally
- * assignable here (the freshness fields are optional), so callers that only
- * know a table name — e.g. the post-build auto-load — bind un-gated entries.
- *
- *  - `dataAsOf`   the artifact's data-as-of instant (snapshot start); the age
- *                 anchor. `age = now - dataAsOf`. Absent ⇒ never stale.
- *  - `freshnessWindowSeconds`  the version's effective window. Absent ⇒ un-gated.
- *  - `freshnessFallback`       per-version read-time policy when stale.
- */
-export interface FreshnessAwareManifestEntry {
-   tableName: string;
-   /**
-    * The connection the physical table lives on, carried from the wire
-    * manifest so the bind step can quote `tableName` for that connection's
-    * dialect (see Package.quoteBoundTableNames). Absent on entries whose
-    * producer didn't record it; those bind verbatim.
-    */
-   connectionName?: string;
-   dataAsOf?: string;
-   freshnessWindowSeconds?: number;
-   freshnessFallback?: "live" | "stale_ok" | "fail";
-}
-
-/** Full (unfiltered) manifest map keyed by sourceEntityId. */
-export type FreshnessManifest = Record<string, FreshnessAwareManifestEntry>;