@malloy-publisher/server 0.0.235 → 0.0.236

package/src/storage/duckdb/PackageRepository.ts

@@ -1,135 +0,0 @@
-import { Package } from "../DatabaseInterface";
-import { DuckDBConnection } from "./DuckDBConnection";
-
-export class PackageRepository {
-   constructor(private db: DuckDBConnection) {}
-
-   private generateId(): string {
-      return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
-   }
-
-   private now(): Date {
-      return new Date();
-   }
-
-   async listPackages(environmentId: string): Promise<Package[]> {
-      const rows = await this.db.all<Record<string, unknown>>(
-         "SELECT * FROM packages WHERE environment_id = ? ORDER BY name",
-         [environmentId],
-      );
-      return rows.map(this.mapToPackage);
-   }
-
-   async getPackageById(id: string): Promise<Package | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM packages WHERE id = ?",
-         [id],
-      );
-      return row ? this.mapToPackage(row) : null;
-   }
-
-   async getPackageByName(
-      environmentId: string,
-      name: string,
-   ): Promise<Package | null> {
-      const row = await this.db.get<Record<string, unknown>>(
-         "SELECT * FROM packages WHERE environment_id = ? AND name = ?",
-         [environmentId, name],
-      );
-      return row ? this.mapToPackage(row) : null;
-   }
-
-   async createPackage(
-      pkg: Omit<Package, "id" | "createdAt" | "updatedAt">,
-   ): Promise<Package> {
-      const id = this.generateId();
-      const now = this.now();
-
-      await this.db.run(
-         `INSERT INTO packages (id, environment_id, name, description, manifest_path, metadata, created_at, updated_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
-         [
-            id,
-            pkg.environmentId,
-            pkg.name,
-            pkg.description || null,
-            pkg.manifestPath,
-            pkg.metadata ? JSON.stringify(pkg.metadata) : null,
-            now.toISOString(),
-            now.toISOString(),
-         ],
-      );
-
-      return {
-         id,
-         ...pkg,
-         createdAt: now,
-         updatedAt: now,
-      };
-   }
-
-   async updatePackage(
-      id: string,
-      updates: Partial<Package>,
-   ): Promise<Package> {
-      const existing = await this.getPackageById(id);
-      if (!existing) {
-         throw new Error(`Package with id ${id} not found`);
-      }
-
-      const now = this.now();
-      const setClauses: string[] = [];
-      const params: unknown[] = [];
-
-      if (updates.name !== undefined) {
-         setClauses.push(`name = ?`);
-         params.push(updates.name);
-      }
-      if (updates.description !== undefined) {
-         setClauses.push(`description = ?`);
-         params.push(updates.description);
-      }
-      if (updates.manifestPath !== undefined) {
-         setClauses.push(`manifest_path = ?`);
-         params.push(updates.manifestPath);
-      }
-      if (updates.metadata !== undefined) {
-         setClauses.push(`metadata = ?`);
-         params.push(JSON.stringify(updates.metadata));
-      }
-
-      setClauses.push(`updated_at = ?`);
-      params.push(now.toISOString());
-      params.push(id);
-
-      await this.db.run(
-         `UPDATE packages SET ${setClauses.join(", ")} WHERE id = ?`,
-         params,
-      );
-
-      return this.getPackageById(id) as Promise<Package>;
-   }
-
-   async deletePackage(id: string): Promise<void> {
-      await this.db.run("DELETE FROM packages WHERE id = ?", [id]);
-   }
-
-   async deletePackagesByEnvironmentId(id: string): Promise<void> {
-      await this.db.run("DELETE FROM packages WHERE environment_id = ?", [id]);
-   }
-
-   private mapToPackage(row: Record<string, unknown>): Package {
-      return {
-         id: row.id as string,
-         environmentId: row.environment_id as string,
-         name: row.name as string,
-         description: row.description as string | undefined,
-         manifestPath: row.manifest_path as string,
-         metadata: row.metadata
-            ? JSON.parse(row.metadata as string)
-            : undefined,
-         createdAt: new Date(row.created_at as string),
-         updatedAt: new Date(row.updated_at as string),
-      };
-   }
-}

package/src/storage/duckdb/schema.ts

@@ -1,229 +0,0 @@
-import { logger } from "../../logger";
-import { DuckDBConnection } from "./DuckDBConnection";
-
-export async function initializeSchema(
-   db: DuckDBConnection,
-   force: boolean = false,
-): Promise<void> {
-   const initialized = await db.isInitialized();
-
-   if (force) {
-      logger.info(
-         "Reinitializing database schema dropping and recreating all tables",
-      );
-      await dropAllTables(db);
-   } else if (!initialized) {
-      // TODO: Remove this during projects cleanup
-      // If a pre-rename `projects` schema is on disk, the new
-      // CREATE TABLE IF NOT EXISTS pass below would silently leave child
-      // tables on the old `project_id` column and the first query against
-      // `environment_id` would crash. Drop the legacy tables (with a loud
-      // warning) so the fresh schema can be created cleanly. This is
-      // destructive: operators upgrading should re-create their environments
-      // and packages via the API after the upgrade.
-      await dropLegacyProjectSchema(db);
-      logger.info("Creating database schema for the first time...");
-   }
-   // Always fall through to the CREATE TABLE IF NOT EXISTS pass below.
-   // The statements are idempotent and let an already-initialized DB pick
-   // up new tables added in later builds (e.g., the `themes` table for
-   // the in-app Theme Editor) without forcing operators to run --init
-   // and lose their existing environments / packages / materializations.
-
-   // Environments table
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS environments (
-      id VARCHAR PRIMARY KEY,
-      name VARCHAR NOT NULL UNIQUE,
-      path VARCHAR NOT NULL,
-      description VARCHAR,
-      metadata JSON,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL
-    )
-  `);
-
-   // Packages table
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS packages (
-      id VARCHAR PRIMARY KEY,
-      environment_id VARCHAR NOT NULL,
-      name VARCHAR NOT NULL,
-      description VARCHAR,
-      manifest_path VARCHAR NOT NULL,
-      metadata JSON,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (environment_id) REFERENCES environments(id),
-      UNIQUE (environment_id, name)
-    )
-  `);
-
-   // Connections table
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS connections (
-      id VARCHAR PRIMARY KEY,
-      environment_id VARCHAR NOT NULL,
-      name VARCHAR NOT NULL,
-      type VARCHAR NOT NULL,
-      config JSON NOT NULL,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (environment_id) REFERENCES environments(id),
-      UNIQUE (environment_id, name)
-    )
-  `);
-
-   // Materializations table.
-   //
-   // `active_key` enforces at-most-one active (non-terminal) materialization
-   // per (environment, package) at the DB layer. It is set to
-   // `{environment_id}|{package_name}` while the row is active and cleared
-   // to NULL on transition to any terminal state. A unique index on
-   // `active_key` (see below) makes the insert-then-check race impossible —
-   // a second concurrent create fails with a constraint violation, which the
-   // service layer translates to `MaterializationConflictError`.
-   // `manifest` is a JSON blob holding the build output returned inline on the
-   // resource.
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS materializations (
-      id VARCHAR PRIMARY KEY,
-      environment_id VARCHAR NOT NULL,
-      package_name VARCHAR NOT NULL,
-      status VARCHAR NOT NULL,
-      active_key VARCHAR,
-      started_at TIMESTAMP,
-      completed_at TIMESTAMP,
-      error TEXT,
-      metadata JSON,
-      manifest JSON,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL,
-      FOREIGN KEY (environment_id) REFERENCES environments(id)
-    )
-  `);
-
-   // Themes table.
-   //
-   // Singleton storage for the instance-wide theme edited by the in-app
-   // Theme Editor. At most one row (id = "default"); we use INSERT ON
-   // CONFLICT to keep the table effectively a key/value cell.
-   //
-   // publisher.config.json's optional `theme` block is a BOOT SEED only —
-   // on first run, if this table is empty, ThemeStore writes the seed
-   // here. Subsequent reads come from this table so the editor's saves
-   // win over the file.
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS themes (
-      id VARCHAR PRIMARY KEY,
-      payload JSON NOT NULL,
-      created_at TIMESTAMP NOT NULL,
-      updated_at TIMESTAMP NOT NULL
-    )
-  `);
-
-   await createEntityEmbeddingsTable(db);
-
-   // Create indexes for better query performance
-   await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)",
-   );
-   await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_connections_environment_id ON connections(environment_id)",
-   );
-   await db.run(
-      "CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)",
-   );
-   await db.run(
-      "CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)",
-   );
-}
-
-/**
- * Vector cache for semantic `malloy_getContext` retrieval (see
- * mcp/tools/embedding_index.ts). One row per discoverable model entity;
- * the primary key mirrors the tool's in-memory dedup key. Rows are
- * content-addressed: the index sync compares `content_hash` (over the
- * embedded text) plus `embedding_model` and re-embeds on mismatch, while
- * a dimensionality change is detected at query time by the stale-row
- * heal, so staleness always self-corrects and wiping the table only
- * ever costs re-embedding. `embedding` is a LIST column searched with
- * list_cosine_similarity; at the entity counts a single Publisher
- * serves, a brute-force scan is faster and simpler than a vector-index
- * extension.
- */
-export async function createEntityEmbeddingsTable(
-   db: DuckDBConnection,
-): Promise<void> {
-   await db.run(`
-    CREATE TABLE IF NOT EXISTS entity_embeddings (
-      environment_name VARCHAR NOT NULL,
-      package_name VARCHAR NOT NULL,
-      entity_kind VARCHAR NOT NULL,
-      entity_source VARCHAR NOT NULL,
-      entity_name VARCHAR NOT NULL,
-      model_path VARCHAR NOT NULL,
-      content_hash VARCHAR NOT NULL,
-      embedding_model VARCHAR NOT NULL,
-      dims INTEGER NOT NULL,
-      embedding FLOAT[] NOT NULL,
-      updated_at TIMESTAMP NOT NULL,
-      PRIMARY KEY (environment_name, package_name, entity_kind, entity_source, entity_name)
-    )
-  `);
-}
-
-// TODO: Remove this during projects cleanup
-// Tables in the pre-rename schema, listed children-first so DROP order
-// satisfies foreign-key dependencies on the legacy `projects` table.
-const LEGACY_TABLES_DROP_ORDER = [
-   "build_manifests",
-   "materializations",
-   "packages",
-   "connections",
-   "projects",
-] as const;
-
-async function dropLegacyProjectSchema(db: DuckDBConnection): Promise<void> {
-   const legacy = await db.all<{ name: string }>(
-      "SELECT name FROM sqlite_master WHERE type='table' AND name='projects'",
-   );
-   if (!legacy || legacy.length === 0) {
-      return;
-   }
-
-   logger.warn(
-      "Detected legacy 'projects' schema. Dropping legacy tables; existing environments/packages/connections/materializations data will be lost. Re-create them via the API after upgrade.",
-   );
-
-   for (const table of LEGACY_TABLES_DROP_ORDER) {
-      try {
-         await db.run(`DROP TABLE IF EXISTS ${table}`);
-      } catch (err) {
-         logger.warn(`Failed to drop legacy table ${table}:`, err);
-      }
-   }
-}
-
-async function dropAllTables(db: DuckDBConnection): Promise<void> {
-   const tables = [
-      "build_manifests",
-      "materializations",
-      "packages",
-      "connections",
-      "environments",
-      "themes",
-      "entity_embeddings",
-   ];
-
-   logger.info("Dropping tables:", tables.join(", "));
-
-   for (const table of tables) {
-      try {
-         await db.run(`DROP TABLE IF EXISTS ${table} `);
-         logger.info(`Dropped table: ${table}`);
-      } catch (err) {
-         logger.warn(` Warning: Could not drop table ${table}:`, err);
-      }
-   }
-}

package/src/stream_helpers.spec.ts

@@ -1,280 +0,0 @@
-import type {
-   QueryRecord,
-   RunSQLOptions,
-   StreamingConnection,
-} from "@malloydata/malloy";
-import { describe, expect, it } from "bun:test";
-
-import { PayloadTooLargeError } from "./errors";
-import { isStreamingConnection, streamSqlWithBudget } from "./stream_helpers";
-
-/**
- * Build a fake StreamingConnection backed by a fixed row array. The
- * fake records the abort signal so tests can confirm
- * `streamSqlWithBudget` actually signaled the driver to stop —
- * client-side counting alone is not enough; the whole point of the
- * helper is to terminate fetching early.
- */
-function fakeStreamingConnection(opts: {
-   rows: QueryRecord[];
-   /**
-    * When true (the default), the fake honors `RunSQLOptions.rowLimit`
-    * by slicing — same as real Postgres/DuckDB. Set false to drive
-    * the helper's own overflow detection (the (cap+1)-th row check).
-    */
-   honorRowLimit?: boolean;
-   /**
-    * Test hook: set to `true` when the fake observes `signal.aborted`
-    * flip via the listener it installed at the start of streaming.
-    */
-   abortObserved?: { value: boolean };
-   /**
-    * Test hook: captures the options the helper passed to
-    * `runSQLStream`, so tests can assert it preserved caller-supplied
-    * `rowLimit` (and the abortSignal it wired in).
-    */
-   capturedOptions?: { value: RunSQLOptions | undefined };
-}): StreamingConnection {
-   const { rows, honorRowLimit = true, abortObserved, capturedOptions } = opts;
-   return {
-      canStream(): true {
-         return true;
-      },
-      async *runSQLStream(
-         _sql: string,
-         options?: RunSQLOptions,
-      ): AsyncIterableIterator<QueryRecord> {
-         if (capturedOptions) capturedOptions.value = options;
-         if (abortObserved) {
-            options?.abortSignal?.addEventListener("abort", () => {
-               abortObserved.value = true;
-            });
-         }
-         const limit =
-            honorRowLimit && typeof options?.rowLimit === "number"
-               ? options.rowLimit
-               : rows.length;
-         for (let i = 0; i < Math.min(rows.length, limit); i += 1) {
-            yield rows[i];
-         }
-      },
-   } as unknown as StreamingConnection;
-}
-
-describe("isStreamingConnection", () => {
-   it("returns true for a connection whose canStream() returns true", () => {
-      const conn = fakeStreamingConnection({ rows: [] });
-      expect(isStreamingConnection(conn)).toBe(true);
-   });
-
-   it("returns false for a connection without canStream", () => {
-      expect(isStreamingConnection({} as never)).toBe(false);
-   });
-
-   it("returns false for a connection whose canStream() returns false", () => {
-      const conn = {
-         canStream() {
-            return false;
-         },
-      } as never;
-      expect(isStreamingConnection(conn)).toBe(false);
-   });
-});
-
-describe("streamSqlWithBudget", () => {
-   it("returns all rows when both budgets are comfortably above the stream", async () => {
-      const rows: QueryRecord[] = [{ a: 1 }, { a: 2 }, { a: 3 }];
-      const conn = fakeStreamingConnection({ rows });
-      const result = await streamSqlWithBudget(
-         conn,
-         "SELECT a FROM t",
-         { rowLimit: 10 },
-         { maxRows: 10, maxBytes: 1_000_000 },
-      );
-      expect(result.rows).toEqual(rows);
-      expect(result.totalRows).toBe(3);
-   });
-
-   it("forwards caller-supplied runSQLOptions (rowLimit) to the driver", async () => {
-      const captured = { value: undefined as RunSQLOptions | undefined };
-      const conn = fakeStreamingConnection({
-         rows: [{ a: 1 }, { a: 2 }],
-         capturedOptions: captured,
-      });
-      await streamSqlWithBudget(
-         conn,
-         "SELECT 1",
-         { rowLimit: 6 },
-         { maxRows: 5, maxBytes: 0 },
-      );
-      expect(captured.value?.rowLimit).toBe(6);
-      // abortSignal must be wired in so the helper can abort the
-      // iterator on overflow.
-      expect(captured.value?.abortSignal).toBeDefined();
-   });
-
-   it("composes the caller-supplied abortSignal with its internal cap-abort signal", async () => {
-      // Step 5: the caller's signal is the query timeout. Composing
-      // both sources means EITHER an external timeout OR an internal
-      // cap overflow terminates the iterator. The combined signal
-      // must be a fresh AbortSignal (not either input by reference);
-      // aborting either input must mark the composed signal aborted.
-      const captured = { value: undefined as RunSQLOptions | undefined };
-      const callerAc = new AbortController();
-      const conn = fakeStreamingConnection({
-         rows: [{ a: 1 }],
-         capturedOptions: captured,
-      });
-      await streamSqlWithBudget(
-         conn,
-         "SELECT 1",
-         { rowLimit: 10, abortSignal: callerAc.signal },
-         { maxRows: 5, maxBytes: 0 },
-      );
-      const observed = captured.value?.abortSignal;
-      expect(observed).toBeInstanceOf(AbortSignal);
-      // Composed signal is a new object (`AbortSignal.any` returns a
-      // fresh signal), not the caller's signal by reference.
-      expect(observed).not.toBe(callerAc.signal);
-      expect(observed?.aborted).toBe(false);
-   });
-
-   it("composed signal aborts when the caller's signal aborts", async () => {
-      // Drive the external (caller) signal manually and confirm the
-      // composed signal that reached the driver tracks it. This is the
-      // half of composition that runWithQueryTimeout depends on for
-      // 504 to actually cancel an in-flight query.
-      if (typeof AbortSignal.any !== "function") return;
-      const captured = { value: undefined as RunSQLOptions | undefined };
-      const callerAc = new AbortController();
-      const conn = fakeStreamingConnection({
-         rows: [{ a: 1 }],
-         capturedOptions: captured,
-      });
-      await streamSqlWithBudget(
-         conn,
-         "SELECT 1",
-         { rowLimit: 10, abortSignal: callerAc.signal },
-         { maxRows: 5, maxBytes: 0 },
-      );
-      const observed = captured.value?.abortSignal;
-      expect(observed?.aborted).toBe(false);
-      callerAc.abort();
-      expect(observed?.aborted).toBe(true);
-   });
-
-   it("throws PayloadTooLargeError and aborts the iterator on the (cap+1)-th row", async () => {
-      const rows: QueryRecord[] = [
-         { a: 1 },
-         { a: 2 },
-         { a: 3 },
-         { a: 4 },
-         { a: 5 },
-      ];
-      const abortObserved = { value: false };
-      const conn = fakeStreamingConnection({
-         rows,
-         abortObserved,
-         honorRowLimit: false,
-      });
-      await expect(
-         streamSqlWithBudget(
-            conn,
-            "SELECT a FROM t",
-            {},
-            { maxRows: 2, maxBytes: 1_000_000 },
-         ),
-      ).rejects.toBeInstanceOf(PayloadTooLargeError);
-      await expect(
-         streamSqlWithBudget(
-            conn,
-            "SELECT a FROM t",
-            {},
-            { maxRows: 2, maxBytes: 1_000_000 },
-         ),
-      ).rejects.toThrow("more than 2 rows");
-      // The helper must have fired the abort signal so a real driver
-      // (pg-query-stream / duckdb) would stop producing rows server-
-      // side, not just be discarded client-side.
-      expect(abortObserved.value).toBe(true);
-   });
-
-   it("throws PayloadTooLargeError when summed JSON byte size exceeds the cap", async () => {
-      const big = "x".repeat(40);
-      const rows: QueryRecord[] = [{ s: big }, { s: big }, { s: big }];
-      const abortObserved = { value: false };
-      const conn = fakeStreamingConnection({
-         rows,
-         abortObserved,
-         honorRowLimit: false,
-      });
-      await expect(
-         streamSqlWithBudget(
-            conn,
-            "SELECT s FROM t",
-            {},
-            { maxRows: 100, maxBytes: 60 },
-         ),
-      ).rejects.toThrow("exceeded 60 bytes");
-      expect(abortObserved.value).toBe(true);
-   });
-
-   it("returns all rows when the byte cap is disabled (maxBytes = 0)", async () => {
-      const big = "x".repeat(10_000);
-      const rows: QueryRecord[] = Array.from({ length: 3 }, () => ({ s: big }));
-      const conn = fakeStreamingConnection({ rows, honorRowLimit: false });
-      const result = await streamSqlWithBudget(
-         conn,
-         "SELECT s FROM t",
-         {},
-         { maxRows: 100, maxBytes: 0 },
-      );
-      expect(result.rows.length).toBe(3);
-   });
-
-   it("returns all rows when the row cap is disabled (maxRows = 0)", async () => {
-      const rows: QueryRecord[] = Array.from({ length: 50 }, (_, i) => ({
-         a: i,
-      }));
-      const conn = fakeStreamingConnection({ rows, honorRowLimit: false });
-      const result = await streamSqlWithBudget(
-         conn,
-         "SELECT a FROM t",
-         {},
-         { maxRows: 0, maxBytes: 1_000_000 },
-      );
-      expect(result.rows.length).toBe(50);
-   });
-
-   it("returns rows when count equals the cap exactly (not an overflow)", async () => {
-      const rows: QueryRecord[] = [{ a: 1 }, { a: 2 }, { a: 3 }];
-      const conn = fakeStreamingConnection({ rows, honorRowLimit: false });
-      const result = await streamSqlWithBudget(
-         conn,
-         "SELECT a FROM t",
-         {},
-         { maxRows: 3, maxBytes: 1_000_000 },
-      );
-      expect(result.rows.length).toBe(3);
-   });
-
-   it("re-throws non-overflow errors from the driver", async () => {
-      const conn = {
-         canStream(): true {
-            return true;
-         },
-         // eslint-disable-next-line require-yield
-         async *runSQLStream(): AsyncIterableIterator<QueryRecord> {
-            throw new Error("connection reset");
-         },
-      } as unknown as StreamingConnection;
-      await expect(
-         streamSqlWithBudget(
-            conn,
-            "SELECT 1",
-            {},
-            { maxRows: 10, maxBytes: 1_000 },
-         ),
-      ).rejects.toThrow("connection reset");
-   });
-});