@malloy-publisher/server 0.0.235 → 0.0.236

package/src/service/givens_integration.spec.ts

@@ -1,219 +0,0 @@
-import { DuckDBConnection } from "@malloydata/db-duckdb";
-import { Connection } from "@malloydata/malloy";
-import { afterAll, beforeAll, describe, expect, it } from "bun:test";
-import fs from "fs/promises";
-import os from "os";
-import path from "path";
-import { Model } from "./model";
-
-const TEST_DIR = path.join(os.tmpdir(), "givens-integration-tests");
-const TEST_DB_DIR = path.join(TEST_DIR, "db");
-const TEST_DB_PATH = path.join(TEST_DB_DIR, "test.duckdb");
-const TEST_PKG_DIR = path.join(TEST_DIR, "pkg");
-
-let duckdbConnection: DuckDBConnection;
-
-const SEED_SQL = `
-CREATE TABLE IF NOT EXISTS orders (
-   order_id INTEGER,
-   region VARCHAR,
-   order_date DATE
-);
-INSERT INTO orders VALUES
-   (1, 'US', '2024-01-15'),
-   (2, 'EU', '2024-02-10'),
-   (3, 'APAC', '2024-03-05');
-`;
-
-const MODEL_WITH_GIVENS = `
-##! experimental.givens
-
-given: region_filter :: string is 'US'
-given: cutoff_date :: date is @2024-02-01
-
-source: orders is duckdb.table('orders') extend {
-   primary_key: order_id
-
-   measure: order_count is count()
-}
-`;
-
-const MODEL_WITHOUT_GIVENS = `
-source: orders is duckdb.table('orders') extend {
-   primary_key: order_id
-
-   measure: order_count is count()
-}
-`;
-
-const MODEL_WITH_ANNOTATED_GIVEN = `
-##! experimental.givens
-
-#(doc) Region code, e.g. US, EU
-#(label) Region
-given: region_filter :: string is 'US'
-
-source: orders is duckdb.table('orders') extend {
-   primary_key: order_id
-}
-`;
-
-beforeAll(async () => {
-   await fs.mkdir(TEST_DB_DIR, { recursive: true });
-   await fs.mkdir(TEST_PKG_DIR, { recursive: true });
-   duckdbConnection = new DuckDBConnection("duckdb", TEST_DB_PATH, TEST_DB_DIR);
-   for (const stmt of SEED_SQL.trim().split(";").filter(Boolean)) {
-      await duckdbConnection.runSQL(stmt.trim() + ";");
-   }
-   // Each fixture lives in its own file. Tests share `beforeAll` for harness
-   // setup but never edit these files at runtime, so no `beforeEach` /
-   // `afterEach` cleanup is needed.
-   await fs.writeFile(
-      path.join(TEST_PKG_DIR, "orders.malloy"),
-      MODEL_WITH_GIVENS,
-      "utf-8",
-   );
-   await fs.writeFile(
-      path.join(TEST_PKG_DIR, "orders_no_givens.malloy"),
-      MODEL_WITHOUT_GIVENS,
-      "utf-8",
-   );
-   await fs.writeFile(
-      path.join(TEST_PKG_DIR, "orders_annotated.malloy"),
-      MODEL_WITH_ANNOTATED_GIVEN,
-      "utf-8",
-   );
-});
-
-afterAll(async () => {
-   try {
-      await duckdbConnection.close();
-      await new Promise((resolve) => setTimeout(resolve, 100));
-      await fs.rm(TEST_DIR, { recursive: true, force: true });
-   } catch {
-      // Ignore cleanup errors
-   }
-});
-
-function getConnections(): Map<string, Connection> {
-   const map = new Map<string, Connection>();
-   map.set("duckdb", duckdbConnection);
-   return map;
-}
-
-describe("givens introspection", () => {
-   it("surfaces declared givens on the compiled-model response", async () => {
-      const model = await Model.create(
-         "test-pkg",
-         TEST_PKG_DIR,
-         "orders.malloy",
-         getConnections(),
-      );
-
-      const compiledModel = await model.getModel();
-
-      expect(compiledModel.givens).toBeDefined();
-      expect(compiledModel.givens).toHaveLength(2);
-
-      const byName = new Map(
-         (compiledModel.givens ?? []).map((g) => [g.name, g]),
-      );
-      const region = byName.get("region_filter");
-      const cutoff = byName.get("cutoff_date");
-
-      expect(region).toBeDefined();
-      expect(region?.type).toBe("string");
-      expect(region?.default).toBe("'US'");
-      expect(cutoff).toBeDefined();
-      expect(cutoff?.type).toBe("date");
-      expect(cutoff?.default).toBe("@2024-02-01");
-   });
-
-   it("omits default for a given declared without one", async () => {
-      await fs.writeFile(
-         path.join(TEST_PKG_DIR, "mixed_defaults.malloy"),
-         `##! experimental.givens
-
-given: with_default :: string is 'WN'
-given: no_default :: string
-
-source: orders is duckdb.table('orders') extend {
-   primary_key: order_id
-}
-`,
-      );
-      const model = await Model.create(
-         "test-pkg",
-         TEST_PKG_DIR,
-         "mixed_defaults.malloy",
-         getConnections(),
-      );
-      const byName = new Map(
-         ((await model.getModel()).givens ?? []).map((g) => [g.name, g]),
-      );
-      expect(byName.get("with_default")?.default).toBe("'WN'");
-      expect(byName.get("no_default")?.default).toBeUndefined();
-   });
-
-   it("attaches the model-level givens list to every source", async () => {
-      const model = await Model.create(
-         "test-pkg",
-         TEST_PKG_DIR,
-         "orders.malloy",
-         getConnections(),
-      );
-
-      const sources = model.getSources();
-      expect(sources).toBeDefined();
-      expect(sources).toHaveLength(1);
-
-      const ordersSource = sources?.[0];
-      expect(ordersSource?.name).toBe("orders");
-      expect(ordersSource?.givens).toBeDefined();
-      expect(ordersSource?.givens).toHaveLength(2);
-
-      const names = (ordersSource?.givens ?? []).map((g) => g.name).sort();
-      expect(names).toEqual(["cutoff_date", "region_filter"]);
-   });
-
-   it("returns undefined for givens when the model declares none", async () => {
-      const model = await Model.create(
-         "test-pkg",
-         TEST_PKG_DIR,
-         "orders_no_givens.malloy",
-         getConnections(),
-      );
-
-      const compiledModel = await model.getModel();
-
-      // Absent rather than empty: matches how `sources`/`queries` behave when
-      // there are none, and lets OpenAPI clients distinguish "feature
-      // unsupported" from "supported but no declarations."
-      expect(compiledModel.givens).toBeUndefined();
-      expect(model.getSources()?.[0]?.givens).toBeUndefined();
-   });
-
-   it("surfaces only `#(...)` annotations, not pragmas or doc comments", async () => {
-      const model = await Model.create(
-         "test-pkg",
-         TEST_PKG_DIR,
-         "orders_annotated.malloy",
-         getConnections(),
-      );
-
-      const compiledModel = await model.getModel();
-
-      expect(compiledModel.givens).toHaveLength(1);
-      const region = compiledModel.givens?.[0];
-      expect(region?.name).toBe("region_filter");
-
-      // The given declares two app-route annotations (`#(doc)`, `#(label)`).
-      // Only app routes land on the wire; Malloy-reserved routes — the
-      // model-level `##!` pragma, plain `#` tags, `#"` doc strings — must
-      // not leak onto the given's surface.
-      const annotations = region?.annotations ?? [];
-      expect(annotations.length).toBeGreaterThanOrEqual(2);
-      expect(annotations.some((a) => a.startsWith("##"))).toBe(false);
-      expect(annotations.some((a) => a.startsWith('#"'))).toBe(false);
-   });
-});

package/src/service/manifest_loader.spec.ts

@@ -1,189 +0,0 @@
-/// <reference types="bun-types" />
-
-import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-import * as fs from "fs/promises";
-import os from "os";
-import path from "path";
-import { pathToFileURL } from "url";
-import { fetchManifestEntries } from "./manifest_loader";
-
-describe("fetchManifestEntries", () => {
-   let dir: string;
-
-   beforeEach(async () => {
-      dir = await fs.mkdtemp(path.join(os.tmpdir(), "manifest-loader-"));
-   });
-
-   afterEach(async () => {
-      await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
-   });
-
-   async function writeManifest(contents: unknown): Promise<string> {
-      const file = path.join(dir, "manifest.json");
-      await fs.writeFile(file, JSON.stringify(contents), "utf8");
-      return file;
-   }
-
-   it("maps physicalTableName to the Malloy runtime tableName (colocated entries)", async () => {
-      const file = await writeManifest({
-         builtAt: new Date().toISOString(),
-         strict: false,
-         entries: {
-            b1: { sourceEntityId: "b1", physicalTableName: "schema.orders_mz" },
-            b2: {
-               sourceEntityId: "b2",
-               physicalTableName: "schema.daily_mz",
-               connectionName: "bq",
-            },
-         },
-      });
-
-      const { tableNameManifest, storageEntries } =
-         await fetchManifestEntries(file);
-
-      expect(tableNameManifest).toEqual({
-         b1: { tableName: "schema.orders_mz" },
-         // connectionName is carried through so the bind step can quote the
-         // table path for that connection's dialect (Package.quoteBoundTableNames).
-         b2: { tableName: "schema.daily_mz", connectionName: "bq" },
-      });
-      expect(storageEntries).toEqual({});
-   });
-
-   it("routes storage= entries (with storageConnectionName) to storageEntries, not the tableName manifest", async () => {
-      const schema = [
-         { name: "d", type: "DATE" },
-         { name: "n", type: "BIGINT" },
-      ];
-      const file = await writeManifest({
-         entries: {
-            // colocated: in-warehouse substitution
-            wh: {
-               sourceEntityId: "wh",
-               physicalTableName: "orders_v1",
-               connectionName: "bq",
-            },
-            // storage tier: cross-connection serve binding (full entry retained)
-            lake: {
-               sourceEntityId: "lake",
-               sourceName: "daily_orders",
-               physicalTableName: "daily_orders",
-               connectionName: "bq",
-               storageConnectionName: "lake",
-               schema,
-            },
-         },
-      });
-
-      const { tableNameManifest, storageEntries } =
-         await fetchManifestEntries(file);
-
-      // The storage entry must NOT enter the same-connection tableName manifest.
-      expect(tableNameManifest).toEqual({
-         wh: { tableName: "orders_v1", connectionName: "bq" },
-      });
-      // …and is preserved as a full entry (schema + sourceName + destination).
-      expect(storageEntries).toEqual({
-         lake: {
-            sourceEntityId: "lake",
-            sourceName: "daily_orders",
-            physicalTableName: "daily_orders",
-            connectionName: "bq",
-            storageConnectionName: "lake",
-            schema,
-         },
-      });
-   });
-
-   it("carries the host-supplied freshness fields verbatim (colocated)", async () => {
-      const dataAsOf = "2026-07-07T00:00:00.000Z";
-      const file = await writeManifest({
-         entries: {
-            gated: {
-               sourceEntityId: "gated",
-               physicalTableName: "schema.gated_mz",
-               dataAsOf,
-               freshnessWindowSeconds: 3600,
-               freshnessFallback: "stale_ok",
-            },
-            ungated: {
-               sourceEntityId: "ungated",
-               physicalTableName: "schema.ungated_mz",
-            },
-         },
-      });
-
-      const { tableNameManifest } = await fetchManifestEntries(file);
-
-      // Filter-free: freshness fields are retained for the serve-path gate; a
-      // window-less entry is left un-gated.
-      expect(tableNameManifest).toEqual({
-         gated: {
-            tableName: "schema.gated_mz",
-            dataAsOf,
-            freshnessWindowSeconds: 3600,
-            freshnessFallback: "stale_ok",
-         },
-         ungated: { tableName: "schema.ungated_mz" },
-      });
-   });
-
-   it("reads via a file:// URI", async () => {
-      const file = await writeManifest({
-         entries: { b1: { sourceEntityId: "b1", physicalTableName: "t1" } },
-      });
-
-      const { tableNameManifest } = await fetchManifestEntries(
-         pathToFileURL(file).href,
-      );
-
-      expect(tableNameManifest).toEqual({ b1: { tableName: "t1" } });
-   });
-
-   it("skips entries without a physicalTableName", async () => {
-      const file = await writeManifest({
-         entries: {
-            good: { sourceEntityId: "good", physicalTableName: "t1" },
-            bad: { sourceEntityId: "bad" },
-         },
-      });
-
-      const { tableNameManifest } = await fetchManifestEntries(file);
-
-      expect(tableNameManifest).toEqual({ good: { tableName: "t1" } });
-   });
-
-   it("returns empty maps when there are no entries", async () => {
-      const file = await writeManifest({ entries: {} });
-      expect(await fetchManifestEntries(file)).toEqual({
-         tableNameManifest: {},
-         storageEntries: {},
-      });
-
-      const file2 = await writeManifest({});
-      expect(await fetchManifestEntries(file2)).toEqual({
-         tableNameManifest: {},
-         storageEntries: {},
-      });
-   });
-
-   it("throws on malformed JSON", async () => {
-      const file = path.join(dir, "bad.json");
-      await fs.writeFile(file, "{ not json", "utf8");
-      await expect(fetchManifestEntries(file)).rejects.toThrow(
-         /Failed to parse build manifest/,
-      );
-   });
-
-   it("throws when the file does not exist", async () => {
-      await expect(
-         fetchManifestEntries(path.join(dir, "missing.json")),
-      ).rejects.toThrow();
-   });
-
-   it("rejects a malformed gs:// URI without a key", async () => {
-      await expect(fetchManifestEntries("gs://bucket-only")).rejects.toThrow(
-         /Malformed manifest URI/,
-      );
-   });
-});

package/src/service/manifest_loader.ts

@@ -1,156 +0,0 @@
-import { GetObjectCommand, S3 } from "@aws-sdk/client-s3";
-import { Storage } from "@google-cloud/storage";
-import * as fs from "fs/promises";
-import { fileURLToPath } from "url";
-import { components } from "../api";
-import { logger } from "../logger";
-import { FreshnessManifest, ManifestEntry } from "../storage/DatabaseInterface";
-
-type WireBuildManifest = components["schemas"]["BuildManifest"];
-
-/**
- * A fetched build manifest, split by tier. A storage-materialized entry (one
- * that carries `storageConnectionName`) is served cross-connection via the
- * virtual-source transform, so it becomes a serve BINDING — it must never enter
- * the same-connection `tableName` substitution. Everything else is an
- * colocated entry the Malloy runtime resolves by substituting its
- * `tableName` at compile time.
- */
-export interface FetchedManifest {
-   /** colocated entries: same-connection `tableName` substitution. */
-   tableNameManifest: FreshnessManifest;
-   /**
-    * `storage=` entries keyed by sourceEntityId, carried as full
-    * {@link ManifestEntry}s (with `storageConnectionName` + captured `schema` +
-    * `sourceName`) so the bind step can derive cross-connection serve bindings.
-    */
-   storageEntries: Record<string, ManifestEntry>;
-}
-
-// Lazily-created clients reused across fetches. Both use the ambient credential
-// chain (ADC for GCS; the default provider chain for S3) — the same auth the
-// package downloader already relies on.
-let gcsClient: Storage | undefined;
-let s3Client: S3 | undefined;
-
-function parseBucketUri(
-   uri: string,
-   scheme: "gs://" | "s3://",
-): { bucket: string; key: string } {
-   const rest = uri.slice(scheme.length);
-   const slash = rest.indexOf("/");
-   if (slash <= 0 || slash === rest.length - 1) {
-      throw new Error(`Malformed manifest URI: ${uri}`);
-   }
-   return { bucket: rest.slice(0, slash), key: rest.slice(slash + 1) };
-}
-
-/** Read the raw bytes of a manifest URI (gs://, s3://, file://, or local path). */
-async function readManifestBytes(uri: string): Promise<string> {
-   if (uri.startsWith("gs://")) {
-      const { bucket, key } = parseBucketUri(uri, "gs://");
-      gcsClient ??= new Storage();
-      const [contents] = await gcsClient.bucket(bucket).file(key).download();
-      return contents.toString("utf8");
-   }
-   if (uri.startsWith("s3://")) {
-      const { bucket, key } = parseBucketUri(uri, "s3://");
-      s3Client ??= new S3({ followRegionRedirects: true });
-      const res = await s3Client.send(
-         new GetObjectCommand({ Bucket: bucket, Key: key }),
-      );
-      if (!res.Body) {
-         throw new Error(`Empty S3 response for manifest URI: ${uri}`);
-      }
-      return res.Body.transformToString();
-   }
-   if (uri.startsWith("file://")) {
-      return fs.readFile(fileURLToPath(uri), "utf8");
-   }
-   // Bare local path (used by tests and local/mounted deployments). The URI is
-   // operator-supplied config (publisher.json or an authenticated PATCH), not
-   // end-user input, and the file's contents are never returned to the caller —
-   // only physicalTableName strings are extracted — so the CodeQL
-   // js/path-injection alert here is a dismissed false positive.
-   return fs.readFile(uri, "utf8");
-}
-
-/**
- * Fetch and parse the host-computed build manifest at `uri`, split by
- * tier into {@link FetchedManifest}. The wire manifest keys physical tables
- * under `physicalTableName`; for an colocated entry the Malloy
- * runtime consumes `tableName`, so that field is translated here and the entry
- * lands in `tableNameManifest`, carrying the freshness fields verbatim (so the
- * serve path can gate `age vs window` per query) and `connectionName` (so the
- * bind step can quote the path for that connection's dialect — see
- * Package.quoteBoundTableNames).
- *
- * A `storage=`-materialized entry (one carrying `storageConnectionName`) is
- * routed instead to `storageEntries` as its full {@link ManifestEntry}: it lives
- * on a DIFFERENT connection and is served through the virtual-source transform
- * from its captured `schema`, so it must never become a same-connection
- * `tableName` substitution. Entries missing a physical table are skipped. This
- * is a pure fetch + parse: it does **not** filter on freshness (that happens per
- * query on the serve path). Throws if the URI can't be read or parsed.
- */
-export async function fetchManifestEntries(
-   uri: string,
-): Promise<FetchedManifest> {
-   const raw = await readManifestBytes(uri);
-
-   let parsed: WireBuildManifest;
-   try {
-      parsed = JSON.parse(raw) as WireBuildManifest;
-   } catch (err) {
-      throw new Error(
-         `Failed to parse build manifest at ${uri}: ${
-            err instanceof Error ? err.message : String(err)
-         }`,
-      );
-   }
-
-   return splitManifestEntries(parsed.entries ?? {}, uri);
-}
-
-/**
- * Split an already-in-hand manifest entry map by tier into
- * {@link FetchedManifest}, applying the same wire→runtime translation as
- * {@link fetchManifestEntries} (physicalTableName → colocated `tableName`
- * substitution carrying freshness + connectionName; a `storageConnectionName`
- * entry stays a full {@link ManifestEntry} for the virtual-source transform).
- * Pure: no I/O, no freshness filtering. Shared by the URI-fetch path (host
- * manifest) and the local-store rebind (a package's own latest persisted
- * materialization, re-applied on load). `source` is a label for the skip log.
- */
-export function splitManifestEntries(
-   entries: Record<string, ManifestEntry>,
-   source: string,
-): FetchedManifest {
-   const tableNameManifest: FreshnessManifest = {};
-   const storageEntries: Record<string, ManifestEntry> = {};
-   for (const [sourceEntityId, entry] of Object.entries(entries)) {
-      const physicalTableName = entry?.physicalTableName;
-      if (!physicalTableName) {
-         logger.warn("Manifest entry has no physicalTableName; skipping", {
-            source,
-            sourceEntityId,
-         });
-         continue;
-      }
-      if (entry.storageConnectionName) {
-         // Cross-connection storage tier: keep the full entry (schema +
-         // sourceName + storageConnectionName) for the serve-binding derivation;
-         // never enter the same-connection tableName manifest.
-         storageEntries[sourceEntityId] = entry;
-         continue;
-      }
-      tableNameManifest[sourceEntityId] = {
-         tableName: physicalTableName,
-         connectionName: entry.connectionName,
-         dataAsOf: entry.dataAsOf,
-         freshnessWindowSeconds: entry.freshnessWindowSeconds,
-         freshnessFallback: entry.freshnessFallback,
-      };
-   }
-   return { tableNameManifest, storageEntries };
-}