@malloy-publisher/server 0.0.235 → 0.0.236

package/src/instrumentation.ts

@@ -1,214 +0,0 @@
-import { monitorEventLoopDelay } from "node:perf_hooks";
-import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
-import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
-import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
-import {
-   ExpressInstrumentation,
-   ExpressLayerType,
-} from "@opentelemetry/instrumentation-express";
-import { resourceFromAttributes } from "@opentelemetry/resources";
-import { BatchLogRecordProcessor } from "@opentelemetry/sdk-logs";
-import { NodeSDK } from "@opentelemetry/sdk-node";
-import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
-import {
-   ATTR_SERVICE_NAME,
-   ATTR_SERVICE_VERSION,
-} from "@opentelemetry/semantic-conventions";
-import type { NextFunction, Request, Response } from "express";
-import { logger } from "./logger";
-import { publisherMeter } from "./telemetry";
-
-let prometheusExporter: PrometheusExporter | null = null;
-let sdk: NodeSDK | null = null;
-
-export function getPrometheusMetricsHandler() {
-   if (!prometheusExporter) {
-      throw new Error("Prometheus exporter not initialized");
-   }
-   return (req: Request, res: Response) => {
-      prometheusExporter!.getMetricsRequestHandler(req, res);
-   };
-}
-
-/**
- * Shuts down the OpenTelemetry SDK gracefully.
- * Should be called during application shutdown.
- */
-export async function shutdownSDK(): Promise<void> {
-   if (sdk) {
-      await sdk.shutdown();
-      sdk = null;
-   }
-}
-
-function instrument() {
-   const otelCollectorUrl = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
-
-   prometheusExporter = new PrometheusExporter({
-      preventServerStart: true,
-   });
-
-   const instrumentations = [
-      getNodeAutoInstrumentations(),
-      new ExpressInstrumentation({
-         ignoreLayersType: [ExpressLayerType.MIDDLEWARE],
-         ignoreLayers: [/\/health/, /\/metrics/],
-      }),
-   ];
-
-   sdk = new NodeSDK({
-      resource: resourceFromAttributes({
-         [ATTR_SERVICE_NAME]: "publisher",
-         [ATTR_SERVICE_VERSION]: "1.0.0",
-      }),
-      autoDetectResources: true,
-      metricReader: prometheusExporter,
-      instrumentations,
-      ...(otelCollectorUrl && {
-         spanProcessors: [
-            new BatchSpanProcessor(
-               new OTLPTraceExporter({
-                  url: `${otelCollectorUrl}/v1/traces`,
-               }),
-            ),
-         ],
-         logRecordProcessors: [
-            new BatchLogRecordProcessor(
-               new OTLPLogExporter({
-                  url: `${otelCollectorUrl}/v1/logs`,
-               }),
-            ),
-         ],
-      }),
-   });
-
-   sdk.start();
-
-   if (otelCollectorUrl) {
-      logger.info(
-         `OpenTelemetry SDK initialized with OTLP collector at ${otelCollectorUrl}`,
-      );
-   } else {
-      logger.info("OpenTelemetry SDK initialized with Prometheus metrics only");
-   }
-}
-
-instrument();
-
-// --- HTTP metrics middleware ---
-
-const meter = publisherMeter();
-
-const httpRequestDuration = meter.createHistogram(
-   "http_server_request_duration_ms",
-   {
-      description: "Duration of HTTP requests in milliseconds",
-      unit: "ms",
-      advice: {
-         explicitBucketBoundaries: [
-            5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 60000,
-         ],
-      },
-   },
-);
-
-const httpRequestCount = meter.createCounter("http_server_requests_total", {
-   description: "Total number of HTTP requests",
-});
-
-// Event-loop-delay metrics. A blocked event loop is the only way the
-// /health/liveness probe (a pure synchronous 200 handler) can fail under K8s,
-// so we surface p50/p99/max so an operator can correlate liveness restarts
-// with sustained event-loop pressure (large Malloy compiles, GC, etc.).
-//
-// `monitorEventLoopDelay` is a Node-only API; some runtimes (e.g. Bun on
-// certain platforms, used by the integration test suite) don't implement it.
-// Degrade gracefully there rather than crashing module load: the metrics are
-// simply not registered.
-const eventLoopHistogram = (() => {
-   try {
-      const histogram = monitorEventLoopDelay({ resolution: 20 });
-      histogram.enable();
-      return histogram;
-   } catch {
-      return null;
-   }
-})();
-
-const eventLoopLagP50 = meter.createObservableGauge(
-   "publisher_event_loop_lag_p50_ms",
-   {
-      description:
-         "Event loop delay p50 since the last scrape, in milliseconds",
-      unit: "ms",
-   },
-);
-const eventLoopLagP99 = meter.createObservableGauge(
-   "publisher_event_loop_lag_p99_ms",
-   {
-      description:
-         "Event loop delay p99 since the last scrape, in milliseconds",
-      unit: "ms",
-   },
-);
-const eventLoopLagMax = meter.createObservableGauge(
-   "publisher_event_loop_lag_max_ms",
-   {
-      description:
-         "Event loop delay max since the last scrape, in milliseconds",
-      unit: "ms",
-   },
-);
-
-// Sample all three in one batch so the histogram reset can't race the reads.
-// Skipped entirely when the runtime lacks event-loop-delay monitoring.
-if (eventLoopHistogram) {
-   const histogram = eventLoopHistogram;
-   meter.addBatchObservableCallback(
-      (observableResult) => {
-         observableResult.observe(
-            eventLoopLagP50,
-            histogram.percentile(50) / 1e6,
-         );
-         observableResult.observe(
-            eventLoopLagP99,
-            histogram.percentile(99) / 1e6,
-         );
-         observableResult.observe(eventLoopLagMax, histogram.max / 1e6);
-         histogram.reset();
-      },
-      [eventLoopLagP50, eventLoopLagP99, eventLoopLagMax],
-   );
-}
-
-const IGNORED_PATHS = new Set([
-   "/health",
-   "/health/liveness",
-   "/health/readiness",
-   "/metrics",
-]);
-
-export function httpMetricsMiddleware(
-   req: Request,
-   res: Response,
-   next: NextFunction,
-) {
-   const start = performance.now();
-
-   res.on("finish", () => {
-      if (IGNORED_PATHS.has(req.path)) return;
-
-      const duration = performance.now() - start;
-      const attrs = {
-         "http.method": req.method,
-         "http.route": req.route?.path ?? req.path,
-         "http.status_code": res.statusCode,
-      };
-
-      httpRequestDuration.record(duration, attrs);
-      httpRequestCount.add(1, attrs);
-   });
-
-   next();
-}

package/src/json_utils.spec.ts

@@ -1,51 +0,0 @@
-import { describe, expect, it } from "bun:test";
-import { bigIntReplacer } from "./json_utils";
-
-const ser = (v: unknown) => JSON.stringify(v, bigIntReplacer);
-
-describe("bigIntReplacer", () => {
-   it("still renders ordinary BigInt ids as numbers", () => {
-      // The common case, and the one that must not change: a DuckDB BIGINT
-      // column of ordinary ids has always reached callers as JSON numbers, and
-      // the MCP rows are byte-compared against the REST payload.
-      expect(ser({ id: 100001n })).toBe('{"id":100001}');
-      expect(ser({ c: 0n })).toBe('{"c":0}');
-      expect(ser({ c: -42n })).toBe('{"c":-42}');
-   });
-
-   it("keeps the largest exactly-representable values as numbers", () => {
-      expect(ser({ v: 9007199254740991n })).toBe('{"v":9007199254740991}');
-      expect(ser({ v: -9007199254740991n })).toBe('{"v":-9007199254740991}');
-   });
-
-   /**
-    * The regression this guards. Number() rounds past 2^53, so an id came back
-    * altered with no error: an agent filtering or joining on it gets a wrong or
-    * empty answer. Malloy's own wrapResult keeps an exact string for the same
-    * case, so a string here matches what the typed-cell path already did.
-    */
-   it("preserves values past the safe range exactly, as strings", () => {
-      expect(ser({ id: 9007199254740993n })).toBe('{"id":"9007199254740993"}');
-      expect(ser({ id: 1234567890123456789n })).toBe(
-         '{"id":"1234567890123456789"}',
-      );
-      expect(ser({ id: -9007199254740993n })).toBe(
-         '{"id":"-9007199254740993"}',
-      );
-   });
-
-   it("round-trips a large id without losing a digit", () => {
-      const exact = "9223372036854775807"; // int64 max
-      expect(JSON.parse(ser({ id: BigInt(exact) })).id).toBe(exact);
-   });
-
-   it("leaves non-BigInt values alone", () => {
-      expect(ser({ a: 1, b: "x", c: null, d: 1.5 })).toBe(
-         '{"a":1,"b":"x","c":null,"d":1.5}',
-      );
-   });
-
-   it("does not throw on the BigInt a plain stringify would reject", () => {
-      expect(() => ser({ c: 150930n })).not.toThrow();
-   });
-});

package/src/json_utils.ts

@@ -1,33 +0,0 @@
-/**
- * JSON.stringify replacer for raw driver output, which can carry BigInt.
- *
- * Load-bearing wherever `compactResult` is serialized: it is
- * `queryResults.data.value` straight from the connector, and a DuckDB BIGINT
- * column arrives as a JS BigInt, so a plain JSON.stringify throws
- * `TypeError: Do not know how to serialize a BigInt`. The wrapped
- * `Malloy.Result` path does not need this, because API.util.wrapResult has
- * already normalized values into typed cells.
- *
- * Values inside the safe integer range become numbers, which is what callers
- * have always received and what keeps the MCP and REST payloads byte-identical.
- * Outside it `Number()` rounds silently: 9007199254740993 becomes ...992, and an
- * agent that filters or joins on that id gets a wrong or empty answer with no
- * error. Those are emitted as strings instead, which is what Malloy itself does
- * for the same case (API.util.wrapResult attaches an exact `string_value`
- * beside the lossy `number_value` whenever the subtype is bigint).
- *
- * So the JSON type varies with magnitude. That is deliberate: a column only
- * yields a string for a value a JSON number cannot represent, and every such
- * value was previously being corrupted, so nothing a caller correctly receives
- * today changes.
- */
-const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
-
-export function bigIntReplacer(_key: string, value: unknown): unknown {
-   if (typeof value === "bigint") {
-      return value > MAX_SAFE_BIGINT || value < -MAX_SAFE_BIGINT
-         ? value.toString()
-         : Number(value);
-   }
-   return value;
-}

package/src/logger.spec.ts

@@ -1,210 +0,0 @@
-import { describe, expect, it } from "bun:test";
-import type { AxiosError } from "axios";
-import {
-   SENSITIVE_KEY_NAMES,
-   buildAxiosErrorLog,
-   redactSensitive,
-} from "./logger";
-
-describe("redactSensitive", () => {
-   it("redacts every name in SENSITIVE_KEY_NAMES (reflective lock)", () => {
-      // Iterates the list itself so a newly-added credential field cannot
-      // be introduced without redaction, and pins apiKey/api_key.
-      const input: Record<string, string> = {};
-      for (const name of SENSITIVE_KEY_NAMES) input[name] = "leak-me";
-      const out = redactSensitive(input) as Record<string, unknown>;
-      for (const name of SENSITIVE_KEY_NAMES) {
-         expect(out[name]).toBe("[REDACTED]");
-      }
-      expect(SENSITIVE_KEY_NAMES).toContain("apiKey");
-      expect(SENSITIVE_KEY_NAMES).toContain("api_key");
-   });
-
-   it("masks every known credential field at the top level", () => {
-      const input = {
-         password: "hunter2",
-         connectionString: "postgresql://u:p@h/db",
-         serviceAccountKeyJson: '{"private_key":"-----BEGIN..."}',
-         privateKey: "-----BEGIN PRIVATE KEY-----",
-         privateKeyPass: "passphrase",
-         secret: "gcs-hmac-secret",
-         secretAccessKey: "aws-secret",
-         sessionToken: "aws-session",
-         sasUrl: "https://a.blob.core.windows.net/c?sig=abc",
-         clientSecret: "azure-client-secret",
-         oauthClientSecret: "databricks-oauth-secret",
-         peakaKey: "peaka-api-key",
-         token: "databricks-pat",
-         accessToken: "motherduck-token",
-      };
-      const out = redactSensitive(input) as Record<string, unknown>;
-      for (const key of Object.keys(input)) {
-         expect(out[key]).toBe("[REDACTED]");
-      }
-   });
-
-   it("keeps non-secret fields intact", () => {
-      const input = {
-         name: "bigquery",
-         type: "bigquery",
-         defaultProjectId: "my-project",
-         host: "db.example.com",
-         port: 5432,
-      };
-      expect(redactSensitive(input)).toEqual(input);
-   });
-
-   it("redacts secrets nested inside the environment PATCH payload shape", () => {
-      const input = {
-         name: "demo___malloy-samples",
-         connections: [
-            {
-               name: "bigquery",
-               type: "bigquery",
-               bigqueryConnection: {
-                  defaultProjectId: "vscode-demo",
-                  serviceAccountKeyJson: '{"private_key":"-----BEGIN..."}',
-               },
-            },
-            {
-               name: "warehouse",
-               type: "snowflake",
-               snowflakeConnection: {
-                  account: "acct",
-                  username: "svc",
-                  password: "s3cret",
-                  privateKey: "-----BEGIN PRIVATE KEY-----",
-               },
-            },
-         ],
-      };
-      expect(redactSensitive(input)).toEqual({
-         name: "demo___malloy-samples",
-         connections: [
-            {
-               name: "bigquery",
-               type: "bigquery",
-               bigqueryConnection: {
-                  defaultProjectId: "vscode-demo",
-                  serviceAccountKeyJson: "[REDACTED]",
-               },
-            },
-            {
-               name: "warehouse",
-               type: "snowflake",
-               snowflakeConnection: {
-                  account: "acct",
-                  username: "svc",
-                  password: "[REDACTED]",
-                  privateKey: "[REDACTED]",
-               },
-            },
-         ],
-      });
-   });
-
-   it("recurses through arrays of secrets", () => {
-      const out = redactSensitive([{ password: "a" }, { password: "b" }]);
-      expect(out).toEqual([
-         { password: "[REDACTED]" },
-         { password: "[REDACTED]" },
-      ]);
-   });
-
-   it("passes primitives through unchanged", () => {
-      expect(redactSensitive("plain")).toBe("plain");
-      expect(redactSensitive(42)).toBe(42);
-      expect(redactSensitive(null)).toBeNull();
-      expect(redactSensitive(undefined)).toBeUndefined();
-   });
-
-   it("preserves Date values instead of destructuring them", () => {
-      const d = new Date("2026-07-08T00:00:00.000Z");
-      const out = redactSensitive({ createdAt: d }) as { createdAt: Date };
-      expect(out.createdAt).toBeInstanceOf(Date);
-      expect(out.createdAt.getTime()).toBe(d.getTime());
-   });
-
-   it("does not mutate the input object", () => {
-      const input = { password: "hunter2", nested: { token: "abc" } };
-      redactSensitive(input);
-      expect(input.password).toBe("hunter2");
-      expect(input.nested.token).toBe("abc");
-   });
-
-   it("tolerates circular references", () => {
-      const input: Record<string, unknown> = { password: "hunter2" };
-      input.self = input;
-      const out = redactSensitive(input) as Record<string, unknown>;
-      expect(out.password).toBe("[REDACTED]");
-      expect(out.self).toBe("[Circular]");
-   });
-
-   it("matches credential keys case-insensitively", () => {
-      const out = redactSensitive({
-         Password: "hunter2",
-         Authorization: "Bearer abc",
-      }) as Record<string, unknown>;
-      expect(out.Password).toBe("[REDACTED]");
-      expect(out.Authorization).toBe("[REDACTED]");
-   });
-
-   it("masks credential HTTP headers", () => {
-      const out = redactSensitive({
-         "content-type": "application/json",
-         authorization: "Bearer secret-token",
-         cookie: "session=abc",
-      }) as Record<string, unknown>;
-      expect(out["content-type"]).toBe("application/json");
-      expect(out.authorization).toBe("[REDACTED]");
-      expect(out.cookie).toBe("[REDACTED]");
-   });
-});
-
-describe("buildAxiosErrorLog", () => {
-   it("redacts response data and headers on a server-side error", () => {
-      const error = {
-         response: {
-            config: { url: "http://worker/api" },
-            status: 500,
-            headers: { authorization: "Bearer abc", "content-type": "json" },
-            data: { connection: { serviceAccountKeyJson: "PRIVATE_KEY" } },
-         },
-      } as unknown as AxiosError;
-      const { message, meta } = buildAxiosErrorLog(error);
-      expect(message).toBe("Axios server-side error");
-      expect(meta).toEqual({
-         url: "http://worker/api",
-         status: 500,
-         headers: { authorization: "[REDACTED]", "content-type": "json" },
-         data: { connection: { serviceAccountKeyJson: "[REDACTED]" } },
-      });
-   });
-
-   it("never logs the raw request object on a client-side error", () => {
-      const error = {
-         request: { _header: "GET / HTTP/1.1\r\nAuthorization: Bearer abc" },
-         config: { method: "get", url: "http://worker/api" },
-         code: "ECONNABORTED",
-      } as unknown as AxiosError;
-      const { message, meta } = buildAxiosErrorLog(error);
-      expect(message).toBe("Axios client-side error");
-      expect(meta).toEqual({
-         method: "get",
-         url: "http://worker/api",
-         code: "ECONNABORTED",
-      });
-      expect(JSON.stringify(meta)).not.toContain("Bearer abc");
-   });
-
-   it("logs only the message on a setup error", () => {
-      const error = {
-         message: "getaddrinfo ENOTFOUND",
-         config: { headers: { Authorization: "Bearer abc" } },
-      } as unknown as AxiosError;
-      const { message, meta } = buildAxiosErrorLog(error);
-      expect(message).toBe("Axios unknown error");
-      expect(meta).toEqual({ message: "getaddrinfo ENOTFOUND" });
-      expect(JSON.stringify(meta)).not.toContain("Bearer abc");
-   });
-});