@malloy-publisher/server 0.0.199 → 0.0.201

package/src/server-old.ts

@@ -41,6 +41,7 @@ import {
    NotImplementedError,
 } from "./errors";
 import { logger } from "./logger";
+import { queryConcurrency } from "./query_concurrency";
 import { normalizeQueryArray } from "./query_param_utils";
 import { EnvironmentStore } from "./service/environment_store";
 
@@ -459,8 +460,18 @@ export function registerLegacyRoutes(
 
    // queryData (deprecated GET) + sqlQuery (supported POST), per-project +
    // per-package
+   // Legacy `/projects/...` query routes keep the GET `queryData`
+   // endpoints (unlike the modern `/environments/...` surface, which
+   // removed them) so existing SDK clients are not broken. The
+   // missing protection is concurrency: without `queryConcurrency()`
+   // a flood of legacy clients can saturate the pod even while the
+   // modern routes are properly gated. Apply the same per-pod cap
+   // here so the legacy surface respects PUBLISHER_MAX_CONCURRENT_QUERIES.
+   // Admission, timeout, and row/byte caps are already enforced by
+   // the shared controllers downstream.
    app.get(
       `${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/queryData`,
+      queryConcurrency(),
       async (req, res) => {
          try {
             res.status(200).json(
@@ -481,6 +492,7 @@ export function registerLegacyRoutes(
 
    app.get(
       `${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/queryData`,
+      queryConcurrency(),
       async (req, res) => {
          try {
             res.status(200).json(
@@ -502,6 +514,7 @@ export function registerLegacyRoutes(
 
    app.post(
       `${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlQuery`,
+      queryConcurrency(),
       async (req, res) => {
          try {
             let options: string | ParsedQs | (string | ParsedQs)[] | undefined;
@@ -528,6 +541,7 @@ export function registerLegacyRoutes(
 
    app.post(
       `${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlQuery`,
+      queryConcurrency(),
       async (req, res) => {
          try {
             let options: string | ParsedQs | (string | ParsedQs)[] | undefined;
@@ -556,6 +570,7 @@ export function registerLegacyRoutes(
    // temporaryTable (deprecated GET) + sqlTemporaryTable (supported POST)
    app.get(
       `${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/temporaryTable`,
+      queryConcurrency(),
       async (req, res) => {
          try {
             res.status(200).json(
@@ -575,6 +590,7 @@ export function registerLegacyRoutes(
 
    app.get(
       `${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/temporaryTable`,
+      queryConcurrency(),
       async (req, res) => {
          try {
             res.status(200).json(
@@ -595,6 +611,7 @@ export function registerLegacyRoutes(
 
    app.post(
       `${LEGACY_API_PREFIX}/projects/:projectName/connections/:connectionName/sqlTemporaryTable`,
+      queryConcurrency(),
       async (req, res) => {
          try {
             res.status(200).json(
@@ -614,6 +631,7 @@ export function registerLegacyRoutes(
 
    app.post(
       `${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/connections/:connectionName/sqlTemporaryTable`,
+      queryConcurrency(),
       async (req, res) => {
          try {
             res.status(200).json(
@@ -780,6 +798,7 @@ export function registerLegacyRoutes(
 
    app.post(
       `${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/models/*?/query`,
+      queryConcurrency(),
       async (req, res) => {
          if (req.body.versionId) {
             setVersionIdError(res);
@@ -856,6 +875,7 @@ export function registerLegacyRoutes(
    // Cell execution route comes BEFORE the general getNotebook wildcard
    app.get(
       `${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/notebooks/*/cells/:cellIndex`,
+      queryConcurrency(),
       async (req, res) => {
          if (req.query.versionId) {
             setVersionIdError(res);

package/src/server.ts

@@ -35,6 +35,9 @@ import {
 import { logger, loggerMiddleware } from "./logger";
 
 import { getMemoryGovernorConfig } from "./config";
+import { setFilterDeprecationHeaders } from "./filter_deprecation";
+import { checkHeapConfiguration } from "./heap_check";
+import { queryConcurrency } from "./query_concurrency";
 import { ManifestController } from "./controller/manifest.controller";
 import { MaterializationController } from "./controller/materialization.controller";
 import { initializeMcpServer } from "./mcp/server";
@@ -154,6 +157,12 @@ const isDevelopment = process.env["NODE_ENV"] === "development";
 export const app = express();
 app.use(loggerMiddleware);
 app.use(httpMetricsMiddleware);
+// Probe the V8 heap ceiling once at startup and warn if it's below
+// the recommended floor. The row/byte caps from Steps 1–3 still
+// bound per-request memory; this is a "your --max-old-space-size
+// looks low for the default caps" advisory so operators don't
+// chase OOMKills before checking the obvious config.
+checkHeapConfiguration();
 const environmentStore = new EnvironmentStore(SERVER_ROOT);
 const manifestService = new ManifestService(environmentStore);
 const watchModeController = new WatchModeController(environmentStore);
@@ -713,55 +722,18 @@ app.post(
    },
 );
 
-/**
- * @deprecated Use /environments/:environmentName/connections/:connectionName/sqlQuery POST method instead
- */
-app.get(
-   `${API_PREFIX}/environments/:environmentName/connections/:connectionName/queryData`,
-   async (req, res) => {
-      try {
-         res.status(200).json(
-            await connectionController.getConnectionQueryData(
-               req.params.environmentName,
-               req.params.connectionName,
-               req.query.sqlStatement as string,
-               req.query.options as string,
-            ),
-         );
-      } catch (error) {
-         logger.error(error);
-         const { json, status } = internalErrorToHttpError(error as Error);
-         res.status(status).json(json);
-      }
-   },
-);
-
-/**
- * @deprecated Use /environments/:environmentName/packages/:packageName/connections/:connectionName/sqlQuery
- */
-app.get(
-   `${API_PREFIX}/environments/:environmentName/packages/:packageName/connections/:connectionName/queryData`,
-   async (req, res) => {
-      try {
-         res.status(200).json(
-            await connectionController.getConnectionQueryData(
-               req.params.environmentName,
-               req.params.connectionName,
-               req.query.sqlStatement as string,
-               req.query.options as string,
-               req.params.packageName,
-            ),
-         );
-      } catch (error) {
-         logger.error(error);
-         const { json, status } = internalErrorToHttpError(error as Error);
-         res.status(status).json(json);
-      }
-   },
-);
-
+// NOTE: The deprecated `GET …/connections/:connectionName/queryData`
+// and `GET …/packages/:packageName/connections/:connectionName/queryData`
+// routes were removed in the operational-guards changeset.
+// They had been marked `@deprecated` for several releases; clients
+// must now use the POST `…/sqlQuery` endpoints below, which take the
+// SQL in the request body so the row/byte caps and query-timeout
+// signals introduced in the OOM-mitigation work apply uniformly.
+// The legacy `GET /projects/…/queryData` twins under `server-old.ts`
+// remain in place for now.
 app.post(
    `${API_PREFIX}/environments/:environmentName/connections/:connectionName/sqlQuery`,
+   queryConcurrency(),
    async (req, res) => {
       try {
          let options: string | ParsedQs | (string | ParsedQs)[] | undefined;
@@ -791,6 +763,7 @@ app.post(
 
 app.post(
    `${API_PREFIX}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlQuery`,
+   queryConcurrency(),
    async (req, res) => {
       try {
          let options: string | ParsedQs | (string | ParsedQs)[] | undefined;
@@ -821,6 +794,7 @@ app.post(
  */
 app.get(
    `${API_PREFIX}/environments/:environmentName/connections/:connectionName/temporaryTable`,
+   queryConcurrency(),
    async (req, res) => {
       try {
          res.status(200).json(
@@ -843,6 +817,7 @@ app.get(
  */
 app.get(
    `${API_PREFIX}/environments/:environmentName/packages/:packageName/connections/:connectionName/temporaryTable`,
+   queryConcurrency(),
    async (req, res) => {
       try {
          res.status(200).json(
@@ -863,6 +838,7 @@ app.get(
 
 app.post(
    `${API_PREFIX}/environments/:environmentName/connections/:connectionName/sqlTemporaryTable`,
+   queryConcurrency(),
    async (req, res) => {
       try {
          res.status(200).json(
@@ -882,6 +858,7 @@ app.post(
 
 app.post(
    `${API_PREFIX}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlTemporaryTable`,
+   queryConcurrency(),
    async (req, res) => {
       try {
          res.status(200).json(
@@ -1076,6 +1053,7 @@ app.get(
 // to avoid the wildcard matching incorrectly
 app.get(
    `${API_PREFIX}/environments/:environmentName/packages/:packageName/notebooks/*/cells/:cellIndex`,
+   queryConcurrency(),
    async (req, res) => {
       if (req.query.versionId) {
          setVersionIdError(res);
@@ -1121,17 +1099,20 @@ app.get(
             }
          }
 
-         res.status(200).json(
-            await modelController.executeNotebookCell(
-               req.params.environmentName,
-               req.params.packageName,
-               notebookPath,
-               cellIndex,
-               filterParams,
-               bypassFilters,
-               givens,
-            ),
+         const result = await modelController.executeNotebookCell(
+            req.params.environmentName,
+            req.params.packageName,
+            notebookPath,
+            cellIndex,
+            filterParams,
+            bypassFilters,
+            givens,
          );
+         setFilterDeprecationHeaders(res, {
+            filterParams,
+            bypassFilters,
+         });
+         res.status(200).json(result);
       } catch (error) {
          logger.error(error);
          const { json, status } = internalErrorToHttpError(error as Error);
@@ -1168,6 +1149,7 @@ app.get(
 
 app.post(
    `${API_PREFIX}/environments/:environmentName/packages/:packageName/models/*?/query`,
+   queryConcurrency(),
    async (req, res) => {
       if (req.body.versionId) {
          setVersionIdError(res);
@@ -1177,22 +1159,25 @@ app.post(
       try {
          // Express stores wildcard matches in params['0']
          const modelPath = (req.params as Record<string, string>)["0"];
-         res.status(200).json(
-            await queryController.getQuery(
-               req.params.environmentName,
-               req.params.packageName,
-               modelPath,
-               req.body.sourceName as string,
-               req.body.queryName as string,
-               req.body.query as string,
-               req.body.compactJson === true,
-               (req.body.filterParams ?? req.body.sourceFilters) as
-                  | Record<string, string | string[]>
-                  | undefined,
-               req.body.bypassFilters === true ? true : undefined,
-               req.body.givens as Record<string, GivenValue> | undefined,
-            ),
+         const result = await queryController.getQuery(
+            req.params.environmentName,
+            req.params.packageName,
+            modelPath,
+            req.body.sourceName as string,
+            req.body.queryName as string,
+            req.body.query as string,
+            req.body.compactJson === true,
+            (req.body.filterParams ?? req.body.sourceFilters) as
+               | Record<string, string | string[]>
+               | undefined,
+            req.body.bypassFilters === true ? true : undefined,
+            req.body.givens as Record<string, GivenValue> | undefined,
          );
+         setFilterDeprecationHeaders(res, {
+            filterParams: req.body.filterParams ?? req.body.sourceFilters,
+            bypassFilters: req.body.bypassFilters === true ? true : undefined,
+         });
+         res.status(200).json(result);
       } catch (error) {
          logger.error(error);
          const { json, status } = internalErrorToHttpError(error as Error);

package/src/service/connection.spec.ts

@@ -9,6 +9,7 @@ import {
    testConnectionConfig,
 } from "./connection";
 import { assembleEnvironmentConnections } from "./connection_config";
+import { EnvironmentStore } from "./environment_store";
 
 type ApiConnection = components["schemas"]["Connection"];
 type AttachedDatabase = components["schemas"]["AttachedDatabase"];
@@ -45,6 +46,21 @@ const hasGCSCredentials = () =>
 const readBigQueryServiceAccountJson = async (): Promise<string> =>
    fs.readFile(process.env.GOOGLE_APPLICATION_CREDENTIALS!, "utf-8");
 
+// `BIGQUERY_PUBLIC_DATA_*` env vars are populated by the
+// `Setup BigQuery public-data credentials` step in
+// `.github/workflows/connection-integration-tests.yml`. Kept separate
+// from the existing `BIGQUERY_TEST_*` vars (which point at the
+// org-scoped BQ_PRESTO_TRINO_KEY service account) so the two SAs can
+// coexist without one overwriting the other.
+const hasPublicDataBigQueryCredentials = () =>
+   !!(
+      process.env.BIGQUERY_PUBLIC_DATA_CREDENTIALS &&
+      process.env.BIGQUERY_PUBLIC_DATA_PROJECT_ID
+   );
+
+const readPublicDataBigQueryServiceAccountJson = async (): Promise<string> =>
+   fs.readFile(process.env.BIGQUERY_PUBLIC_DATA_CREDENTIALS!, "utf-8");
+
 describe("connection integration tests", () => {
    const testEnvironmentPath = path.join(
       process.cwd(),
@@ -672,6 +688,234 @@ describe("connection integration tests", () => {
          );
       });
 
+      describe("BigQuery direct connection (public-data)", () => {
+         // Single end-to-end check that a real `bigquery`-typed
+         // connection authenticated by the BIGQUERY_PUBLIC_DATA_SA
+         // service account can actually round-trip a query against
+         // bigquery-public-data. Skips locally if the creds aren't
+         // set; runs in CI under .github/workflows/connection-integration-tests.yml.
+         //
+         // Picked `samples.shakespeare` because (a) it's tiny (~6 MB
+         // / ~165k rows) so the BigQuery byte-scanned cost is well
+         // under the 1 TB/month free tier even across many PR runs,
+         // (b) it's a long-stable public table so the assertion stays
+         // valid across years of reruns. If this assertion ever fails
+         // the cause is almost certainly auth, not data drift.
+         it(
+            "should query bigquery-public-data via real BigQuery connection",
+            async () => {
+               if (!hasPublicDataBigQueryCredentials()) {
+                  console.log(
+                     "Skipping: BIGQUERY_PUBLIC_DATA_CREDENTIALS or BIGQUERY_PUBLIC_DATA_PROJECT_ID not configured",
+                  );
+                  return;
+               }
+
+               const serviceAccountJson =
+                  await readPublicDataBigQueryServiceAccountJson();
+
+               const bqConnection: ApiConnection = {
+                  name: "bq_public_data",
+                  type: "bigquery",
+                  bigqueryConnection: {
+                     // Billing/auth project (the SA's own project_id);
+                     // the queried tables live in bigquery-public-data
+                     // and are referenced explicitly in the SQL below.
+                     defaultProjectId:
+                        process.env.BIGQUERY_PUBLIC_DATA_PROJECT_ID!,
+                     serviceAccountKeyJson: serviceAccountJson,
+                  },
+               };
+
+               const { malloyConnections } = await createEnvironmentConnections(
+                  [bqConnection],
+                  testEnvironmentPath,
+               );
+
+               const connection = malloyConnections.get("bq_public_data");
+               expect(connection).toBeDefined();
+
+               try {
+                  const result = await connection!.runSQL(
+                     "SELECT COUNT(*) AS row_count FROM `bigquery-public-data.samples.shakespeare`",
+                  );
+                  expect(result.rows.length).toBe(1);
+                  // Shakespeare has ~165k rows; bound on both sides so
+                  // we catch both "auth succeeded but query returned
+                  // nothing" and "got a confusingly large value" failure
+                  // modes, while staying tolerant of any minor row-count
+                  // jitter Google might introduce.
+                  const row = result.rows[0] as Record<string, unknown>;
+                  const rowCount = Number(row.row_count);
+                  expect(rowCount).toBeGreaterThan(100_000);
+                  expect(rowCount).toBeLessThan(200_000);
+               } finally {
+                  // BigQuery driver holds an HTTP/2 client + auth refresh
+                  // state; close it explicitly so we don't leak across
+                  // the rest of the test run. (createdConnections is
+                  // typed for DuckDBConnection, so we can't use the
+                  // shared cleanup array here.)
+                  await connection?.close();
+               }
+            },
+            { timeout: 60000 },
+         );
+      });
+
+      describe("BigQuery package end-to-end (bq-hackernews)", () => {
+         // Step 2 of Sagar's bun-setup-fixes ask: not just "does the
+         // BigQuery driver round-trip a SQL query" (the previous
+         // describe block covers that), but "does the publisher's
+         // package-loading path successfully use the BigQuery
+         // connection on behalf of a Malloy package."
+         //
+         // Mechanism: stand up a real EnvironmentStore against a
+         // temp publisher.config.json that declares both the BQ
+         // connection (with the BIGQUERY_PUBLIC_DATA_SA injected via
+         // serviceAccountKeyJson) AND the bq-hackernews package
+         // (loaded from credibledata/malloy-samples on GitHub).
+         // EnvironmentStore.initialize then: clones the malloy-samples
+         // repo, extracts the bigquery-hackernews subdirectory, parses
+         // the package's Malloy models, and — crucially — introspects
+         // their BigQuery table schemas during model compilation. A
+         // successful Package.listModels() call therefore proves the
+         // entire package-uses-connection path, not just the bare
+         // driver auth.
+         //
+         // Cost: ~50 MB git clone (one time per test run since
+         // publisher_data lives under testEnvironmentPath which the
+         // afterEach cleans up) + ~6 MB BQ schema scan. Both well
+         // under any meaningful free-tier limit. Test budget: 3 min
+         // (clone is ~30-60s on GH runners; compile is ~10s).
+         it(
+            "should load bq-hackernews package and compile its models via real BQ",
+            async () => {
+               if (!hasPublicDataBigQueryCredentials()) {
+                  console.log(
+                     "Skipping: BIGQUERY_PUBLIC_DATA_CREDENTIALS or BIGQUERY_PUBLIC_DATA_PROJECT_ID not configured",
+                  );
+                  return;
+               }
+
+               const serviceAccountJson =
+                  await readPublicDataBigQueryServiceAccountJson();
+
+               // Each test gets its own serverRoot so publisher_data
+               // doesn't leak across tests; afterEach removes
+               // testEnvironmentPath which carries the whole tree
+               // away.
+               const tempServerRoot = path.join(
+                  testEnvironmentPath,
+                  "bq-hackernews-pkg-test",
+               );
+               await fs.mkdir(tempServerRoot, { recursive: true });
+
+               const config = {
+                  frozenConfig: false,
+                  environments: [
+                     {
+                        name: "malloy-samples",
+                        packages: [
+                           {
+                              name: "bigquery-hackernews",
+                              location:
+                                 "https://github.com/credibledata/malloy-samples/tree/main/bigquery-hackernews",
+                           },
+                        ],
+                        connections: [
+                           {
+                              name: "bigquery",
+                              type: "bigquery",
+                              bigqueryConnection: {
+                                 defaultProjectId:
+                                    process.env
+                                       .BIGQUERY_PUBLIC_DATA_PROJECT_ID!,
+                                 serviceAccountKeyJson: serviceAccountJson,
+                              },
+                           },
+                        ],
+                     },
+                  ],
+               };
+               await fs.writeFile(
+                  path.join(tempServerRoot, "publisher.config.json"),
+                  JSON.stringify(config),
+               );
+
+               // Force the load-from-config init path. Without this,
+               // EnvironmentStore.initialize() takes the load-from-DB
+               // branch and only falls back to config when the DB is
+               // empty (which it is here, by construction, but relying
+               // on that is brittle if the test pattern is reused).
+               //
+               // SAFETY PRECONDITION: this flag is the SAME one users
+               // opt into via `--init` / `start:init` to wipe persisted
+               // storage and re-initialize from config (see CLAUDE.md
+               // and `environment_store.ts:158`). It is destructive on
+               // a real serverRoot. We only set it here because
+               // `tempServerRoot` is a freshly-created empty directory
+               // (path.join(testEnvironmentPath, "bq-hackernews-pkg-test")
+               // mkdir'd ~20 lines above), so there is nothing to wipe.
+               // DO NOT copy this pattern into a test that points
+               // EnvironmentStore at a non-empty or shared serverRoot
+               // — it will delete state.
+               const previousInitializeStorage = process.env.INITIALIZE_STORAGE;
+               process.env.INITIALIZE_STORAGE = "true";
+
+               try {
+                  const envStore = new EnvironmentStore(tempServerRoot);
+                  await envStore.finishedInitialization;
+
+                  // operationalState=serving is the only signal that
+                  // initialize() actually succeeded. initialize swallows
+                  // top-level errors and just calls markNotReady() (see
+                  // environment_store.ts:297-301), so
+                  // finishedInitialization always resolves regardless of
+                  // success. By construction (env_store:288-292), serving
+                  // also implies all configured environments loaded.
+                  const status = await envStore.getStatus();
+                  expect(status.operationalState).toBe("serving");
+
+                  const env = await envStore.getEnvironment("malloy-samples");
+                  const apiPackages = await env.listPackages();
+                  expect(apiPackages.map((p) => p.name)).toContain(
+                     "bigquery-hackernews",
+                  );
+
+                  const apiConnections = env.listApiConnections();
+                  expect(apiConnections.map((c) => c.name)).toContain(
+                     "bigquery",
+                  );
+
+                  // The actual integration assertion. Package.listModels()
+                  // collects compile errors per-model and returns ALL
+                  // models in the result (with `error` populated on
+                  // failures) — so models.length>0 alone would pass even
+                  // if every BQ schema introspection failed. Filter for
+                  // models that compiled cleanly: at least one is the
+                  // real proof that BQ-on-behalf-of-a-package works.
+                  const pkg = await env.getPackage("bigquery-hackernews");
+                  const models = await pkg.listModels();
+                  const okModels = models.filter((m) => !m.error);
+                  if (okModels.length === 0) {
+                     console.error(
+                        "All bq-hackernews model compilations failed:",
+                        models.map((m) => `${m.path}: ${m.error}`),
+                     );
+                  }
+                  expect(okModels.length).toBeGreaterThan(0);
+               } finally {
+                  if (previousInitializeStorage === undefined) {
+                     delete process.env.INITIALIZE_STORAGE;
+                  } else {
+                     process.env.INITIALIZE_STORAGE = previousInitializeStorage;
+                  }
+               }
+            },
+            { timeout: 180000 },
+         );
+      });
+
       describe("DuckDB with Snowflake attachment", () => {
          it(
             "should create DuckDB connection with attached Snowflake database",

package/src/service/connection.ts

@@ -22,10 +22,14 @@ import {
 import type { LookupConnection } from "@malloydata/malloy/connection";
 import { AxiosError } from "axios";
 import fs from "fs/promises";
-import path from "path";
 import { components } from "../api";
 import { logAxiosError, logger } from "../logger";
 import { redactPgSecrets } from "../pg_helpers";
+import {
+   assertSafeEnvironmentPath,
+   assertSafePackageName,
+   safeJoinUnderRoot,
+} from "../path_safety";
 import {
    assembleEnvironmentConnections,
    CoreConnectionEntry,
@@ -91,7 +95,9 @@ async function isDatabaseAttached(
          ? existingDatabases
          : existingDatabases.rows || [];
 
-      logger.debug(`Existing databases:`, rows);
+      logger.debug("connection.duckdb.databases.queried", {
+         count: rows.length,
+      });
 
       return rows.some((row: Record<string, unknown>) =>
          Object.values(row).some(
@@ -814,7 +820,9 @@ export async function deleteDuckLakeConnectionFile(
    connectionName: string,
    environmentPath: string,
 ): Promise<void> {
-   const ducklakePath = path.join(
+   assertSafePackageName(connectionName);
+   assertSafeEnvironmentPath(environmentPath);
+   const ducklakePath = safeJoinUnderRoot(
       environmentPath,
       `${connectionName}_ducklake.duckdb`,
    );
@@ -1126,7 +1134,9 @@ export async function createEnvironmentConnections(
 
    for (const connection of environmentConfig.apiConnections) {
       if (!connection.name) continue;
-      logger.info(`Adding connection ${connection.name}`, { connection });
+      logger.info(`Adding connection ${connection.name}`, {
+         type: connection.type,
+      });
       const malloyConnection =
          await environmentConfig.malloyConfig.connections.lookupConnection(
             connection.name,