@malloy-publisher/server 0.0.235 → 0.0.236

package/package.json

@@ -1,11 +1,14 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.235",
+   "version": "0.0.236",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
    },
+   "files": [
+      "dist"
+   ],
    "repository": {
       "type": "git",
       "url": "https://github.com/malloydata/publisher.git"
@@ -35,6 +38,7 @@
       "@azure/identity": "^4.13.0",
       "@azure/storage-blob": "^12.26.0",
       "@duckdb/node-api": "1.5.3-r.2",
+      "@google-cloud/bigquery": "^7.9.4",
       "@google-cloud/storage": "^7.16.0",
       "@malloydata/db-bigquery": "^0.0.427",
       "@malloydata/db-databricks": "^0.0.427",
@@ -55,7 +59,6 @@
       "@opentelemetry/sdk-node": "^0.200.0",
       "@opentelemetry/sdk-trace-node": "^2.0.0",
       "async-mutex": "^0.5.0",
-      "aws-sdk": "^2.1692.0",
       "body-parser": "^1.20.2",
       "chokidar": "^4.0.3",
       "cors": "^2.8.5",

package/.eslintrc.json

@@ -1,44 +0,0 @@
-{
-   // Specify the environments where the code will run
-   "env": {
-      "jest": true, // Enable Jest for testing
-      "browser": true // Enable browser environment
-   },
-
-   // Stop ESLint from searching for configuration in parent folders
-   "root": true,
-
-   // Specify the parser for TypeScript (using @typescript-eslint/parser)
-   "parser": "@typescript-eslint/parser", // Leverages TS ESTree to lint TypeScript
-
-   // Add additional rules and configuration options
-   "plugins": ["@typescript-eslint"],
-
-   // Extend various ESLint configurations and plugins
-   "extends": [
-      "eslint:recommended", // ESLint recommended rules
-      "plugin:react/recommended", // React recommended rules
-      "plugin:@typescript-eslint/recommended", // TypeScript recommended rules
-      "plugin:@typescript-eslint/eslint-recommended", // ESLint overrides for TypeScript
-      "prettier", // Prettier rules
-      "plugin:prettier/recommended", // Prettier plugin integration
-      "plugin:react-hooks/recommended", // Recommended rules for React hooks
-      "plugin:storybook/recommended" // Recommended rules for Storybook
-   ],
-   "settings": {
-      "react": {
-         "version": "detect"
-      }
-   },
-   "rules": {
-      "react/react-in-jsx-scope": "off",
-      "@typescript-eslint/no-unused-vars": [
-         "error",
-         {
-            "argsIgnorePattern": "^_",
-            "varsIgnorePattern": "^_",
-            "caughtErrorsIgnorePattern": "^_"
-         }
-      ]
-   }
-}

package/.prettierignore

@@ -1 +0,0 @@
-src/api.ts

package/.prettierrc

@@ -1,4 +0,0 @@
-{
-   "printWidth": 80,
-   "tabWidth": 3
-}

package/build.ts

@@ -1,97 +0,0 @@
-import { build } from "bun";
-import fs from "fs";
-
-fs.rmSync("./dist", { recursive: true, force: true });
-fs.mkdirSync("./dist");
-
-await build({
-   // package_load_worker.ts is bundled as a SEPARATE entrypoint so it
-   // can be loaded by `new Worker(...)` at runtime. It must NOT be
-   // inlined into server.mjs (workers can't share module state with
-   // the parent process — they get their own JS realm).
-   entrypoints: [
-      "./src/server.ts",
-      "./src/instrumentation.ts",
-      "./src/package_load/package_load_worker.ts",
-   ],
-   outdir: "./dist",
-   target: "node",
-   format: "esm",
-   external: [
-      "@malloydata/db-duckdb",
-      // The DuckDB engine binding (used by the storage layer). Externalized,
-      // not bundled, because its loader require()s platform-specific native
-      // .node files for every OS via a runtime switch; the bundler can't
-      // statically resolve the ones absent on the build host. Left as a
-      // runtime require resolved from node_modules, like the Malloy drivers.
-      "@duckdb/node-api",
-      "@duckdb/node-bindings",
-      "@malloydata/malloy",
-      "@malloydata/malloy-sql",
-      "@malloydata/render",
-      "@malloydata/db-bigquery",
-      "@malloydata/db-mysql",
-      "@malloydata/db-postgres",
-      "@malloydata/db-snowflake",
-      "@malloydata/db-trino",
-      "@malloydata/db-databricks",
-      "@google-cloud/storage",
-      "@azure/identity",
-      "@azure/storage-blob",
-   ],
-});
-
-fs.cpSync("../app/dist", "./dist/app", { recursive: true });
-
-// Copy hand-authored vanilla-JS runtime served at /sdk/publisher.js.
-fs.cpSync("./src/runtime", "./dist/runtime", { recursive: true });
-
-// Ship a default publisher.config.json inside the bundle so that
-// `npx @malloy-publisher/server` works with zero args (uses
-// DuckDB-only samples). config.ts looks for this file next to
-// server.mjs.
-//
-// IMPORTANT: keep `packages/server/src/default-publisher.config.json`
-// in sync with `packages/server/publisher.config.json`. The two files
-// have intentionally identical content — the committed one drives the
-// dev workflow (`bun run start`, integration tests), the bundled one
-// is the fallback for npx users with no config of their own. A drift
-// here surfaces as "npx users see different samples than dev users."
-fs.copyFileSync(
-   "./src/default-publisher.config.json",
-   "./dist/default-publisher.config.json",
-);
-
-// Rename ESM outputs to .mjs so both Node and Bun can execute them
-fs.renameSync("./dist/server.js", "./dist/server.mjs");
-fs.renameSync("./dist/instrumentation.js", "./dist/instrumentation.mjs");
-// Bun emits package_load_worker into its source-relative subdir;
-// flatten so package_load_pool.ts's `resolveWorkerScript()` finds it
-// as a sibling of server.mjs. The path layout match is intentional —
-// keep these two in sync or the worker pool throws at boot (the
-// in-process fallback was removed; missing worker = no service).
-if (fs.existsSync("./dist/package_load/package_load_worker.js")) {
-   fs.renameSync(
-      "./dist/package_load/package_load_worker.js",
-      "./dist/package_load_worker.mjs",
-   );
-   try {
-      fs.rmdirSync("./dist/package_load");
-   } catch {
-      /* directory may be non-empty if Bun produced sourcemaps; leave it */
-   }
-} else if (fs.existsSync("./dist/package_load_worker.js")) {
-   fs.renameSync(
-      "./dist/package_load_worker.js",
-      "./dist/package_load_worker.mjs",
-   );
-}
-
-// Add shebang to server.mjs for npx/bunx compatibility
-const serverJsPath = "./dist/server.mjs";
-const serverJsContent = fs.readFileSync(serverJsPath, "utf8");
-const shebangContent = "#!/usr/bin/env node\n" + serverJsContent;
-fs.writeFileSync(serverJsPath, shebangContent);
-
-// Make the file executable
-fs.chmodSync(serverJsPath, "755");

package/eslint.config.mjs

@@ -1,76 +0,0 @@
-import { fixupConfigRules, fixupPluginRules } from "@eslint/compat";
-import { FlatCompat } from "@eslint/eslintrc";
-import js from "@eslint/js";
-import typescriptEslint from "@typescript-eslint/eslint-plugin";
-import tsParser from "@typescript-eslint/parser";
-import globals from "globals";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-const compat = new FlatCompat({
-   baseDirectory: __dirname,
-   recommendedConfig: js.configs.recommended,
-   allConfig: js.configs.all,
-});
-
-export default [
-   {
-      ignores: [
-         "**/node_modules/**",
-         "**/dist/**",
-         "**/build/**",
-         "**/.DS_Store",
-         "**/__pycache__/**",
-         "**/*.py[cod]",
-         "**/*.egg-info/**",
-         "**/.env",
-         "**/*.log",
-      ],
-   },
-   ...fixupConfigRules(
-      compat.extends(
-         "eslint:recommended",
-         "plugin:react/recommended",
-         "plugin:@typescript-eslint/recommended",
-         "plugin:@typescript-eslint/eslint-recommended",
-         "prettier",
-         "plugin:prettier/recommended",
-         "plugin:react-hooks/recommended",
-         "plugin:storybook/recommended",
-      ),
-   ),
-   {
-      plugins: {
-         "@typescript-eslint": fixupPluginRules(typescriptEslint),
-      },
-
-      languageOptions: {
-         globals: {
-            ...globals.jest,
-            ...globals.browser,
-         },
-
-         parser: tsParser,
-      },
-
-      settings: {
-         react: {
-            version: "detect",
-         },
-      },
-
-      rules: {
-         "react/react-in-jsx-scope": "off",
-         "@typescript-eslint/no-unused-vars": [
-            "error",
-            {
-               argsIgnorePattern: "^_",
-               varsIgnorePattern: "^_",
-               caughtErrorsIgnorePattern: "^_",
-            },
-         ],
-      },
-   },
-];

package/k6-tests/README.md

@@ -1,144 +0,0 @@
-# k6-tests
-
-Performance and load testing suite for the Malloy Publisher API using k6.
-
-## Installation
-
-To install dependencies:
-
-```bash
-bun install
-```
-
-## Prerequisites
-
-**Important:** Before running tests, ensure the publisher server is running with the following environment variables to limit log output:
-
-```bash
-DISABLE_RESPONSE_LOGGING=true
-LOG_LEVEL=info  # or LOG_LEVEL=warn
-```
-
-Start the server before running any k6 tests.
-
-## Available Tests
-
-### Smoke Test
-
-Basic functionality verification under minimal load:
-
-```bash
-bun run smoke-test              # Standard test
-./scripts/run-with-otel.sh smoke-test/smoke-test.ts  # With OpenTelemetry export
-```
-
-### Load Tests
-
-#### CRUD Load Tests
-
-Specific tests for CRUD (Create, Read, Update, Delete) operations on individual resource types:
-
-```bash
-bun run load-test-crud-projects      # Projects CRUD load test
-bun run load-test-crud-connections   # Connections CRUD load test
-bun run load-test-crud-packages      # Packages CRUD load test
-bun run load-test-crud               # Run all three CRUD tests sequentially
-```
-
-Run CRUD tests with OpenTelemetry export:
-
-```bash
-./scripts/run-with-otel.sh load-test/load-test-crud-projects.ts
-./scripts/run-with-otel.sh load-test/load-test-crud-connections.ts
-./scripts/run-with-otel.sh load-test/load-test-crud-packages.ts
-```
-
-#### Comprehensive Load Test
-
-Focused on comprehensive API read operations and query execution. Tests listing, getting, and querying various resources (projects, packages, models, notebooks, connections, databases, queries, SQL sources) under normal load:
-
-```bash
-bun run load-test               # Standard test
-./scripts/run-with-otel.sh load-test/load-test.ts  # With OTEL export
-```
-
-## OpenTelemetry & Prometheus Integration
-
-Export k6 metrics to Prometheus via OpenTelemetry for visualization in Grafana.
-
-
-**Usage:**
-Use the `run-with-otel.sh` script to run any test with OpenTelemetry export:
-
-```bash
-./scripts/run-with-otel.sh <test-file> [additional-k6-args...]
-```
-
-Examples:
-```bash
-./scripts/run-with-otel.sh smoke-test/smoke-test.ts
-./scripts/run-with-otel.sh load-test/load-test.ts --duration 5m
-./scripts/run-with-otel.sh load-test/load-test-crud-projects.ts
-```
-
-## Environment Variables
-
-### Test Configuration
-
-- `K6_PUBLISHER_URL` - Base URL of the publisher service (default: `http://localhost:4000`)
-- `K6_PROJECT_NAME` - Project name for tests (default: `malloy-samples`)
-- `K6_AUTH_TOKEN` - Authorization token for API requests
-- `GOOGLE_APPLICATION_CREDENTIALS` - BigQuery service account JSON (for BigQuery tests)
-- `K6_USE_VERSION_ID` - Set to `"true"` to use version IDs in API calls - uses "latest" as versionId when set to true
-- `K6_MAX_VIEWS_PER_MODEL` - Maximum views to test per model (default: `5`)
-- `K6_DEBUG` - Set to `"true"` for verbose logging
-- `K6_WHITELISTED_PACKAGES` - JSON array of package names to test
-- `K6_AVAILABLE_PACKAGES` - JSON array of available package names
-
-### OpenTelemetry Export (when using `run-with-otel.sh`)
-
-These environment variables are automatically set by `run-with-otel.sh` but can be overridden:
-
-- `K6_OTEL_GRPC_EXPORTER_ENDPOINT` - OTEL collector gRPC endpoint (default: `localhost:4317`)
-- `K6_OTEL_GRPC_EXPORTER_INSECURE` - Disable TLS for plain gRPC (default: `true` - set automatically by script)
-- `K6_OTEL_SERVICE_NAME` - Service name for metrics (default: `k6-load-test`)
-- `K6_OTEL_EXPORTER_PROTOCOL` - Exporter protocol: `grpc` or `http/protobuf` (default: `grpc`)
-- `K6_OTEL_HTTP_EXPORTER_ENDPOINT` - OTEL collector HTTP endpoint (default: `localhost:4318`)
-- `K6_OTEL_FLUSH_INTERVAL` - How often k6 flushes internal metrics (default: `1s`)
-- `K6_OTEL_EXPORT_INTERVAL` - How often k6 exports to OTEL collector (default: `2s` in script, `10s` otherwise)
-
-## Setup & Build
-
-### Clone Malloy Samples
-
-Clone the malloy-samples repository (required for tests):
-
-```bash
-bun run clone-malloy-samples
-```
-
-This clones the [malloy-samples](https://github.com/credibledata/malloy-samples) repository into `./packages` directory.
-
-### Client Generation
-
-To regenerate API clients from OpenAPI spec:
-
-```bash
-bun run generate-clients
-```
-
-### Build
-
-Run the full build process (cleanup, clone samples, and generate clients):
-
-```bash
-bun run build
-```
-
-## Cleanup
-
-Remove generated files and dependencies:
-
-```bash
-bun run clean
-```