@malloy-publisher/server 0.0.234 → 0.0.236

package/k6-tests/utils/common.ts

@@ -1,489 +0,0 @@
-import { check, sleep } from "k6";
-import type {
-   CompiledModel,
-   Model,
-   Package,
-} from "../clients/malloyPublisherSemanticModelServingAPI.schemas.ts";
-import {
-   getModelsClient,
-   getPackagesClient,
-   getPublisherClient,
-} from "./client_factory.ts";
-import { logger } from "./logger.ts";
-export const PUBLISHER_URL = __ENV.K6_PUBLISHER_URL || "http://localhost:4000";
-export const PROJECT_NAME = __ENV.K6_PROJECT_NAME || "malloy-samples";
-export const AUTH_TOKEN = __ENV.K6_AUTH_TOKEN || "";
-
-export const WHITELISTED_PACKAGES = __ENV.K6_WHITELISTED_PACKAGES
-   ? (JSON.parse(__ENV.K6_WHITELISTED_PACKAGES) as Array<string>)
-   : ["ecommerce", "faa", "imdb", "bigquery-hackernews"];
-
-export const USE_VERSION_ID = __ENV.K6_USE_VERSION_ID === "true" ? true : false;
-/**
- * Available packages in the k6-tests/packages directory
- * These are the actual package directories that exist
- * Can be set via AVAILABLE_PACKAGES environment variable (JSON array)
- * If not set, defaults to common packages
- *
- * Note: Since k6 can't read directories at runtime, this should be set via
- * a pre-build script that scans the packages directory, or manually via env var
- */
-const AVAILABLE_PACKAGES = __ENV.K6_AVAILABLE_PACKAGES
-   ? (JSON.parse(__ENV.K6_AVAILABLE_PACKAGES) as Array<string>)
-   : ["ecommerce", "faa", "imdb", "bigquery-hackernews"];
-
-/**
- * Single source of truth for getting available packages for testing
- *
- * This function:
- * 1. Reads from AVAILABLE_PACKAGES (env var or default list)
- * 2. Filters by WHITELISTED_PACKAGES
- * 3. Filters BigQuery packages based on credential availability:
- *    - If HAS_BIGQUERY_CREDENTIALS is true: includes all BigQuery packages
- *    - If HAS_BIGQUERY_CREDENTIALS is false: excludes BigQuery packages
- *
- * @returns Array of package names that are available, whitelisted, and compatible with current credentials
- */
-export function getAvailablePackages(): Array<string> {
-   let packages = AVAILABLE_PACKAGES;
-
-   // Step 1: Filter by whitelist if provided
-   if (WHITELISTED_PACKAGES.length > 0) {
-      packages = packages.filter((pkg) => WHITELISTED_PACKAGES.includes(pkg));
-   }
-
-   // Step 2: Filter BigQuery packages based on credential availability
-   if (!HAS_BIGQUERY_CREDENTIALS) {
-      // Remove all BigQuery packages if credentials are not available
-      packages = packages.filter((pkg) => !pkg.startsWith("bigquery-"));
-   }
-   // If credentials are available, keep all packages (including BigQuery ones)
-
-   return packages;
-}
-
-// Initialize clients with base URL including /api/v0 prefix
-export const BASE_URL = `${PUBLISHER_URL}/api/v0`;
-const packagesClient = getPackagesClient(BASE_URL, AUTH_TOKEN);
-const modelsClient = getModelsClient(BASE_URL, AUTH_TOKEN);
-const publisherClient = getPublisherClient(BASE_URL, AUTH_TOKEN);
-
-// Check if BigQuery credentials are available
-export const HAS_BIGQUERY_CREDENTIALS =
-   __ENV.GOOGLE_APPLICATION_CREDENTIALS !== undefined &&
-   __ENV.GOOGLE_APPLICATION_CREDENTIALS !== "";
-
-export const getPackages = (): Array<{
-   name: string;
-   description: string;
-   versionId?: string;
-}> => {
-   const { response, data } = packagesClient.listPackages(PROJECT_NAME, {
-      tags: { name: "list_packages" },
-   });
-   check(response, {
-      "packages list request successful": (r) => r.status === 200,
-   });
-
-   if (response.status !== 200) {
-      console.error(
-         `Failed to get packages: ${response.status} - ${response.body}`,
-      );
-      return [];
-   }
-
-   if (!Array.isArray(data)) {
-      console.error(`Expected array but got: ${typeof data}`, data);
-      return [];
-   }
-
-   return data.map((pkg: Package) => ({
-      name: pkg.name || "",
-      description: pkg.description || "",
-      versionId: USE_VERSION_ID ? "latest" : undefined,
-   }));
-};
-
-export const loadPackage = (packageName: string, location: string): Package => {
-   const apiPackage: Package = {
-      name: packageName,
-      description: "",
-      location: location,
-   };
-
-   const { response } = packagesClient.createPackage(PROJECT_NAME, apiPackage);
-
-   check(response, {
-      [`package ${packageName} can upload`]: (r) => r.status === 200,
-      [`package ${packageName} upload time < 500ms`]: (r) =>
-         r.timings.duration < 500,
-   });
-
-   return apiPackage;
-};
-
-export const queryPackage = (packageName: string): Package => {
-   const { response, data } = packagesClient.getPackage(
-      PROJECT_NAME,
-      packageName,
-   );
-
-   check(response, {
-      "package query successful": (r) => r.status === 200,
-      "package query response time < 500ms": (r) => r.timings.duration < 500,
-   });
-
-   return data;
-};
-
-export const unloadPackage = (packageName: string, packageId: string) => {
-   // Note: The generated client uses projectName/packageName pattern
-   // If packageId is different from packageName, this may need adjustment
-   const { response } = packagesClient.deletePackage(PROJECT_NAME, packageId);
-
-   check(response, {
-      [`package ${packageName} can unload`]: (r) =>
-         r.status === 200 || r.status === 204,
-      [`package ${packageName} unload time < 500ms`]: (r) =>
-         r.timings.duration < 500,
-   });
-};
-
-export const getModels = (
-   packageName: string,
-   versionId?: string,
-): Array<{
-   path: string;
-   type: string;
-}> => {
-   const params = USE_VERSION_ID && versionId ? { versionId } : {};
-   const { response, data } = modelsClient.listModels(
-      PROJECT_NAME,
-      packageName,
-      params,
-      { tags: { name: "list_models" } },
-   );
-
-   check(response, {
-      "list models request successful": (r) => r.status === 200,
-      "list models response time < 2s": (r) => r.timings.duration < 2000,
-   });
-
-   // Handle case where API call failed or returned no data
-   if (!data || response.status !== 200) {
-      return [];
-   }
-
-   const code = (data as { code?: number })["code"];
-   if (code !== undefined && code !== 200) {
-      console.error(`Failed to list models: ${code} - ${JSON.stringify(data)}`);
-      return [];
-   }
-   if (!Array.isArray(data)) {
-      console.log(`Data: ${JSON.stringify(data)}`);
-      console.error(`Expected array but got: ${typeof data}`, data);
-      return [];
-   }
-
-   return data.map((model: Model) => ({
-      path: model.path || "",
-      type: model.error ? "error" : "source", // Infer type from presence of error
-   }));
-};
-
-type ModelData = {
-   type: "notebook" | "source";
-   sources?: Array<{
-      name: string;
-      views: Array<{
-         name: string;
-      }>;
-   }>;
-};
-
-export const getModelData = (
-   packageName: string,
-   modelPath: string,
-   versionId?: string,
-): ModelData => {
-   const params = USE_VERSION_ID && versionId ? { versionId } : {};
-   const { response, data } = modelsClient.getModel(
-      PROJECT_NAME,
-      packageName,
-      modelPath,
-      params,
-      { tags: { name: "get_model" } },
-   );
-
-   check(response, {
-      "get model data request successful": (r) => r.status === 200,
-   });
-
-   // Handle case where API call failed or returned no data
-   if (!data || response.status !== 200) {
-      return {
-         type: "source",
-         sources: undefined,
-      };
-   }
-
-   // Transform CompiledModel to ModelData format expected by getViews
-
-   // The API returns CompiledModel with sourceInfos array containing JSON strings
-   // Each string contains source information with views in schema.fields
-   const compiledModel = data as CompiledModel & {
-      sourceInfos?: string[];
-   };
-
-   const sources: Array<{
-      name: string;
-      views: Array<{ name: string }>;
-   }> = [];
-
-   // Parse sourceInfos array (primary method - more reliable than sources)
-   if (
-      compiledModel &&
-      compiledModel.sourceInfos &&
-      Array.isArray(compiledModel.sourceInfos)
-   ) {
-      for (const sourceInfoJson of compiledModel.sourceInfos) {
-         try {
-            const sourceInfo = JSON.parse(sourceInfoJson) as {
-               kind?: string;
-               name?: string;
-               schema?: {
-                  fields?: Array<{
-                     kind?: string;
-                     name?: string;
-                  }>;
-               };
-            };
-
-            // Only process source entries
-            if (sourceInfo.kind === "source" && sourceInfo.name) {
-               const views: Array<{ name: string }> = [];
-
-               // Extract views from schema.fields where kind === "view"
-               if (
-                  sourceInfo.schema?.fields &&
-                  Array.isArray(sourceInfo.schema.fields)
-               ) {
-                  for (const field of sourceInfo.schema.fields) {
-                     if (field.kind === "view" && field.name) {
-                        views.push({ name: field.name });
-                     }
-                  }
-               }
-
-               // Only include sources that have views (matching original behavior)
-               if (views.length > 0) {
-                  sources.push({
-                     name: sourceInfo.name,
-                     views: views,
-                  });
-               }
-            }
-         } catch (error) {
-            console.warn(
-               `Failed to parse sourceInfo for ${packageName}/${modelPath}:`,
-               error,
-            );
-         }
-      }
-   }
-
-   // Fallback to sources array only if sourceInfos is empty or failed to parse anything
-   const compiledModelWithSources = data as CompiledModel & {
-      sources?: Array<{
-         name?: string;
-         views?: Array<{ name?: string }>;
-      }>;
-   };
-
-   if (
-      sources.length === 0 &&
-      compiledModelWithSources &&
-      compiledModelWithSources.sources
-   ) {
-      for (const source of compiledModelWithSources.sources) {
-         if (source && source.name) {
-            const views: Array<{ name: string }> = [];
-
-            // Extract views from source
-            if (source.views && Array.isArray(source.views)) {
-               for (const view of source.views) {
-                  if (view && view.name) {
-                     views.push({ name: view.name });
-                  }
-               }
-            }
-
-            if (views.length > 0) {
-               sources.push({
-                  name: source.name,
-                  views: views,
-               });
-            }
-         }
-      }
-   }
-
-   // Note: We exclude queries from views - queries are named queries that can be executed,
-   // but they are not views. Views are defined in the sourceInfos array.
-
-   return {
-      type: "source",
-      sources: sources.length > 0 ? sources : undefined,
-   };
-};
-
-export function* getViews(modelData: ModelData) {
-   if (
-      modelData.type === "source" &&
-      modelData.sources &&
-      modelData.sources.length > 0
-   ) {
-      for (const source of modelData.sources) {
-         if (source && source.name && source.views && source.views.length > 0) {
-            for (const view of source.views) {
-               if (view && view.name) {
-                  yield {
-                     sourceName: source.name,
-                     viewName: view.name,
-                  };
-               }
-            }
-         }
-      }
-   }
-}
-
-export const isServerAvailableAndInitialized = (): boolean => {
-   const { response, data } = publisherClient.getStatus({
-      tags: { name: "get_server_status" },
-   });
-   check(response, {
-      "server status request successful": (r) => r.status === 200,
-      "server is initialized": (_) => data?.initialized === true,
-   });
-   console.log(`Server status: ${response.status}`);
-   if (response.status === 200) {
-      console.log(`Server is initialized: ${data?.initialized}`);
-   }
-
-   return response.status === 200 && data?.initialized === true;
-};
-
-/**
- * Validates that the server is up and initialized before running tests.
- * Retries every 5 seconds for up to 60 seconds.
- * Throws an error if the server is not initialized after max wait time.
- */
-export function validateServerIsUpAndInitialized(): void {
-   const MAX_WAIT_TIME_SEC = 60; // 60 seconds
-   const RETRY_INTERVAL_SEC = 5; // 5 seconds
-   const startTime = Date.now();
-   let attempt = 0;
-   let serverReady = false;
-
-   while (!serverReady) {
-      attempt++;
-      const { response, data } = publisherClient.getStatus({
-         tags: { name: "get_server_status" },
-      });
-
-      if (response.status === 200 && data?.initialized === true) {
-         console.log(
-            `Server is up and initialized (attempt ${attempt}, elapsed: ${Math.round((Date.now() - startTime) / 1000)}s)`,
-         );
-         serverReady = true;
-         return;
-      }
-
-      if (response.status == 401 || response.status == 403) {
-         throw new Error(
-            `Seems like Auth token is invalid or expired, status: ${response.status}). Please update the auth token and try again.`,
-         );
-      }
-
-      const elapsedTimeSec = (Date.now() - startTime) / 1000;
-      if (elapsedTimeSec >= MAX_WAIT_TIME_SEC) {
-         throw new Error(
-            `Server not initialized after ${MAX_WAIT_TIME_SEC}s (${attempt} attempts). Status: ${response.status}, Initialized: ${data?.initialized}`,
-         );
-      }
-
-      const remainingTimeSec = MAX_WAIT_TIME_SEC - elapsedTimeSec;
-      const waitTimeSec = Math.min(RETRY_INTERVAL_SEC, remainingTimeSec);
-      logger.info(
-         `Server not ready (attempt ${attempt}, status: ${response.status}, initialized: ${data?.initialized}). Waiting ${waitTimeSec}s before retry... (${Math.round(remainingTimeSec)}s remaining)`,
-      );
-
-      // Sleep for the wait time (k6 sleep takes seconds)
-      sleep(waitTimeSec);
-   }
-}
-
-export const queryModelView = (
-   packageName: string,
-   modelPath: string,
-   sourceName: string,
-   queryName: string,
-   versionId?: string,
-) => {
-   // For top-level queries (empty sourceName), only pass queryName
-   // For queries on sources, pass both sourceName and queryName
-   const queryRequest: {
-      sourceName?: string;
-      queryName: string;
-      versionId?: string;
-   } = {
-      queryName: queryName,
-   };
-
-   // Only include sourceName if it's not empty (top-level queries have empty sourceName)
-   if (sourceName && sourceName.trim() !== "") {
-      queryRequest.sourceName = sourceName;
-   }
-
-   // Only include versionId if USE_VERSION_ID flag is enabled
-   if (USE_VERSION_ID && versionId) {
-      queryRequest.versionId = versionId;
-   }
-
-   const { response } = modelsClient.executeQueryModel(
-      PROJECT_NAME,
-      packageName,
-      modelPath,
-      queryRequest,
-      { tags: { name: "execute_query" } },
-   );
-
-   // Log error details if request failed
-   if (response.status >= 400) {
-      console.error(
-         `Query failed for ${packageName}/${modelPath}:`,
-         JSON.stringify({
-            status: response.status,
-            statusText: response.status_text,
-            error: response.error,
-            errorCode: response.error_code,
-            body: response.body,
-            request: queryRequest,
-         }),
-      );
-   }
-
-   return response;
-};
-
-// ============================================================================
-// Helper Functions
-// ============================================================================
-
-/**
- * Generate unique test resource name
- */
-export function generateTestName(prefix: string): string {
-   return `${prefix}-${Date.now()}-${Math.random().toString(36).substring(7)}`;
-}

package/k6-tests/utils/logger.ts

@@ -1,62 +0,0 @@
-function shouldLog(level: LogLevel): boolean {
-   return (
-      LEVEL_PRIORITY[level] !== undefined &&
-      LEVEL_PRIORITY[CURRENT_LEVEL] !== undefined &&
-      LEVEL_PRIORITY[level] <= LEVEL_PRIORITY[CURRENT_LEVEL]
-   );
-}
-
-const LOG_LEVELS = ["error", "warn", "info", "debug"] as const;
-
-type LogLevel = (typeof LOG_LEVELS)[number];
-
-const LEVEL_PRIORITY: Record<LogLevel, number> = {
-   error: 0,
-   warn: 1,
-   info: 2,
-   debug: 3,
-};
-
-let CURRENT_LEVEL: LogLevel = "debug";
-
-function isLogLevel(value: string): value is LogLevel {
-   return LOG_LEVELS.includes(value as LogLevel);
-}
-
-export const setLogLevel = (level: string) => {
-   if (isLogLevel(level)) {
-      CURRENT_LEVEL = level;
-   }
-};
-export function error(msg: string) {
-   if (shouldLog("error")) {
-      console.error(`[k6][ERROR] ${msg}`);
-   }
-}
-
-export function warn(msg: string) {
-   if (shouldLog("warn")) {
-      console.warn(`[k6][WARN] ${msg}`);
-   }
-}
-
-export function info(msg: string) {
-   if (shouldLog("info")) {
-      console.info(`[k6][INFO] ${msg}`);
-   }
-}
-
-export function debug(msg: string) {
-   if (shouldLog("debug")) {
-      console.debug(`[k6][DEBUG] ${msg}`);
-   }
-}
-
-// Export logger object with methods
-export const logger = {
-   error,
-   warn,
-   info,
-   debug,
-   setLogLevel,
-};

package/k6-tests/utils/types.d.ts

@@ -1,35 +0,0 @@
-type KnownThresholds = {
-   http_req_duration?: string[];
-   http_req_failed?: string[];
-   http_req_waiting?: string[];
-   dropped_iterations?: string[];
-   checks?: string[];
-};
-
-type Thresholds = KnownThresholds & {
-   [metricName: string]: string[];
-};
-
-/**
- * Configuration options for K6 tests
- * Can be either a simple test with fixed VUs and duration,
- * or a staged test with varying number of virtual users over time
- */
-type TestOptions = {
-   thresholds?: Thresholds;
-} & (
-   | { vus: number; duration: string }
-   | { stages: Array<{ duration: string; target: number }> }
-);
-
-/**
- * Interface defining a K6 test preset
- */
-interface TestPreset {
-   defaultOptions: TestOptions;
-   run: (data?: unknown | undefined) => void;
-}
-
-declare module "https://jslib.k6.io/k6-utils/1.4.0/index.js" {
-   export const uuidv4: () => string;
-}

package/publisher.config.example.bigquery.json

@@ -1,33 +0,0 @@
-{
-  "frozenConfig": false,
-  "environments": [
-    {
-      "name": "examples",
-      "packages": [
-        {
-          "name": "storefront",
-          "location": "https://github.com/malloydata/publisher/tree/main/examples/storefront"
-        },
-        {
-          "name": "governed-analytics",
-          "location": "https://github.com/malloydata/publisher/tree/main/examples/governed-analytics"
-        },
-        {
-          "name": "html-data-app",
-          "location": "https://github.com/malloydata/publisher/tree/main/examples/html-data-app"
-        },
-        {
-          "name": "bigquery-hackernews",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/bigquery-hackernews"
-        }
-      ],
-      "connections": [
-        {
-          "name": "bigquery",
-          "type": "bigquery",
-          "bigqueryConnection": {}
-        }
-      ]
-    }
-  ]
-}

package/publisher.config.example.duckdb.json

@@ -1,23 +0,0 @@
-{
-  "frozenConfig": false,
-  "environments": [
-    {
-      "name": "examples",
-      "packages": [
-        {
-          "name": "storefront",
-          "location": "https://github.com/malloydata/publisher/tree/main/examples/storefront"
-        },
-        {
-          "name": "governed-analytics",
-          "location": "https://github.com/malloydata/publisher/tree/main/examples/governed-analytics"
-        },
-        {
-          "name": "html-data-app",
-          "location": "https://github.com/malloydata/publisher/tree/main/examples/html-data-app"
-        }
-      ],
-      "connections": []
-    }
-  ]
-}

package/publisher.config.json

@@ -1,23 +0,0 @@
-{
-  "frozenConfig": false,
-  "environments": [
-    {
-      "name": "examples",
-      "packages": [
-        {
-          "name": "storefront",
-          "location": "../../examples/storefront"
-        },
-        {
-          "name": "governed-analytics",
-          "location": "../../examples/governed-analytics"
-        },
-        {
-          "name": "html-data-app",
-          "location": "../../examples/html-data-app"
-        }
-      ],
-      "connections": []
-    }
-  ]
-}