@rulebricks/cli 2.0.1 → 2.0.3

package/benchmarks/qps-test.js

@@ -0,0 +1,136 @@
+/**
+ * QPS (Queries Per Second) Benchmark Test
+ *
+ * Measures API responsiveness by sending individual payload requests at a constant rate.
+ * This test helps you understand:
+ * - How many individual requests your deployment can handle per second
+ * - API gateway and connection handling performance
+ * - Response latency under load
+ *
+ * Test Structure:
+ * - 1 minute warm-up phase (allows cluster to scale, excluded from results)
+ * - 4 minutes measurement phase (steady-state performance)
+ *
+ * Usage:
+ *   k6 run -e API_URL=https://your-instance.com/api/v1/flows/flow_id \
+ *          -e API_KEY=your-api-key \
+ *          qps-test.js
+ *
+ * Optional environment variables:
+ *   TEST_DURATION - Measurement duration after warm-up (default: 4m)
+ *   TARGET_RPS    - Target requests per second (default: 500)
+ */
+
+import { check } from "k6";
+import http from "k6/http";
+import { Counter, Rate, Trend } from "k6/metrics";
+import {
+  generatePayload,
+  getConfig,
+  createRequestParams,
+} from "./lib/payload.js";
+import { generateQpsReport, generateQpsConsoleSummary } from "./lib/report.js";
+
+// Load configuration
+const config = getConfig({
+  testDuration: "4m",
+  targetRps: 500,
+});
+
+// Custom metrics (only for measurement phase)
+const errorRate = new Rate("errors");
+const successRate = new Rate("successes");
+const requestDuration = new Trend("request_duration");
+const droppedRequests = new Counter("dropped_requests");
+
+// k6 options with warm-up and measurement phases
+export const options = {
+  scenarios: {
+    // Warm-up phase: 1 minute to let cluster scale
+    warm_up: {
+      executor: "constant-arrival-rate",
+      rate: config.targetRps,
+      timeUnit: "1s",
+      duration: "1m",
+      preAllocatedVUs: Math.min(config.targetRps, 200),
+      maxVUs: Math.min(config.targetRps * 2, 1000),
+      exec: "warmUp",
+      tags: { phase: "warmup" },
+    },
+    // Measurement phase: actual test after warm-up
+    qps_test: {
+      executor: "constant-arrival-rate",
+      rate: config.targetRps,
+      timeUnit: "1s",
+      duration: config.testDuration,
+      preAllocatedVUs: Math.min(config.targetRps, 200),
+      maxVUs: Math.min(config.targetRps * 2, 1000),
+      startTime: "1m", // Start after warm-up
+      exec: "measureTest",
+      tags: { phase: "measurement" },
+    },
+  },
+  thresholds: {
+    // Only apply thresholds to measurement phase
+    "http_req_duration{phase:measurement}": ["p(95)<500", "p(99)<1000"],
+    "errors{phase:measurement}": ["rate<0.05"],
+  },
+};
+
+// Request parameters
+const params = createRequestParams(config.apiKey);
+
+/**
+ * Warm-up function - same as test but metrics tagged differently
+ */
+export function warmUp() {
+  const payload = generatePayload();
+
+  try {
+    http.post(config.apiUrl, JSON.stringify(payload), params);
+  } catch (error) {
+    // Ignore errors during warm-up
+  }
+}
+
+/**
+ * Measurement test function - sends individual requests
+ */
+export function measureTest() {
+  const payload = generatePayload();
+  const start = Date.now();
+
+  try {
+    const response = http.post(config.apiUrl, JSON.stringify(payload), params);
+
+    const duration = Date.now() - start;
+    requestDuration.add(duration);
+
+    const success = check(response, {
+      "status is 200": (r) => r.status === 200,
+      "valid response": (r) => r.body && r.body.length > 0,
+    });
+
+    errorRate.add(!success);
+    successRate.add(success);
+
+    if (!success) {
+      droppedRequests.add(1);
+    }
+  } catch (error) {
+    errorRate.add(1);
+    successRate.add(0);
+    droppedRequests.add(1);
+  }
+}
+
+/**
+ * Generate summary report
+ */
+export function handleSummary(data) {
+  return {
+    stdout: generateQpsConsoleSummary(data, config),
+    "qps-report.html": generateQpsReport(data, config),
+    "qps-results.json": JSON.stringify(data, null, 2),
+  };
+}

package/benchmarks/run-qps-test.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+#
+# QPS Benchmark Test Runner
+# Runs the QPS test and opens the HTML report in your browser
+#
+# Usage: ./run-qps-test.sh -u <API_URL> -k <API_KEY> [options]
+#
+# Options:
+#   -u, --url        API URL (required)
+#   -k, --key        API key (required)
+#   -d, --duration   Measurement duration after 1m warm-up (default: 4m)
+#   -r, --rps        Target RPS (default: 500)
+#   -n, --no-open    Don't open report in browser
+#
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+# Defaults
+DURATION="4m"
+RPS="500"
+OPEN_REPORT=true
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -u|--url)
+      API_URL="$2"
+      shift 2
+      ;;
+    -k|--key)
+      API_KEY="$2"
+      shift 2
+      ;;
+    -d|--duration)
+      DURATION="$2"
+      shift 2
+      ;;
+    -r|--rps)
+      RPS="$2"
+      shift 2
+      ;;
+    -n|--no-open)
+      OPEN_REPORT=false
+      shift
+      ;;
+    -h|--help)
+      echo "Usage: ./run-qps-test.sh -u <API_URL> -k <API_KEY> [options]"
+      echo ""
+      echo "Options:"
+      echo "  -u, --url        API URL (required)"
+      echo "  -k, --key        API key (required)"
+      echo "  -d, --duration   Test duration (default: 2m)"
+      echo "  -r, --rps        Target RPS (default: 500)"
+      echo "  -n, --no-open    Don't open report in browser"
+      exit 0
+      ;;
+    *)
+      echo -e "${RED}Unknown option: $1${NC}"
+      exit 1
+      ;;
+  esac
+done
+
+# Validate required args
+if [ -z "$API_URL" ]; then
+  echo -e "${RED}Error: API_URL is required (-u)${NC}"
+  exit 1
+fi
+
+if [ -z "$API_KEY" ]; then
+  echo -e "${RED}Error: API_KEY is required (-k)${NC}"
+  exit 1
+fi
+
+# Get script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo -e "${CYAN}Running QPS Benchmark...${NC}"
+echo "  URL:      $API_URL"
+echo "  Warm-up:  1m"
+echo "  Duration: $DURATION (after warm-up)"
+echo "  Target:   $RPS RPS"
+echo ""
+
+# Run k6 test
+k6 run \
+  -e API_URL="$API_URL" \
+  -e API_KEY="$API_KEY" \
+  -e TEST_DURATION="$DURATION" \
+  -e TARGET_RPS="$RPS" \
+  "$SCRIPT_DIR/qps-test.js" || true
+
+# Open report in browser
+if [ "$OPEN_REPORT" = true ] && [ -f "$SCRIPT_DIR/qps-report.html" ]; then
+  echo ""
+  echo -e "${GREEN}Opening report in browser...${NC}"
+  
+  # Cross-platform open command
+  if command -v open &> /dev/null; then
+    open "$SCRIPT_DIR/qps-report.html"
+  elif command -v xdg-open &> /dev/null; then
+    xdg-open "$SCRIPT_DIR/qps-report.html"
+  elif command -v start &> /dev/null; then
+    start "$SCRIPT_DIR/qps-report.html"
+  else
+    echo "Report saved to: $SCRIPT_DIR/qps-report.html"
+  fi
+fi
+

package/benchmarks/run-throughput-test.sh

@@ -0,0 +1,123 @@
+#!/bin/bash
+#
+# Throughput Benchmark Test Runner
+# Runs the throughput test and opens the HTML report in your browser
+#
+# Usage: ./run-throughput-test.sh -u <API_URL> -k <API_KEY> [options]
+#
+# Options:
+#   -u, --url        API URL (required)
+#   -k, --key        API key (required)
+#   -d, --duration   Measurement duration after 1m warm-up (default: 4m)
+#   -r, --rps        Target RPS (default: 100)
+#   -b, --bulk       Bulk size (default: 50)
+#   -n, --no-open    Don't open report in browser
+#
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+# Defaults
+DURATION="4m"
+RPS="100"
+BULK_SIZE="50"
+OPEN_REPORT=true
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -u|--url)
+      API_URL="$2"
+      shift 2
+      ;;
+    -k|--key)
+      API_KEY="$2"
+      shift 2
+      ;;
+    -d|--duration)
+      DURATION="$2"
+      shift 2
+      ;;
+    -r|--rps)
+      RPS="$2"
+      shift 2
+      ;;
+    -b|--bulk)
+      BULK_SIZE="$2"
+      shift 2
+      ;;
+    -n|--no-open)
+      OPEN_REPORT=false
+      shift
+      ;;
+    -h|--help)
+      echo "Usage: ./run-throughput-test.sh -u <API_URL> -k <API_KEY> [options]"
+      echo ""
+      echo "Options:"
+      echo "  -u, --url        API URL (required)"
+      echo "  -k, --key        API key (required)"
+      echo "  -d, --duration   Test duration (default: 2m)"
+      echo "  -r, --rps        Target RPS (default: 100)"
+      echo "  -b, --bulk       Bulk size (default: 50)"
+      echo "  -n, --no-open    Don't open report in browser"
+      exit 0
+      ;;
+    *)
+      echo -e "${RED}Unknown option: $1${NC}"
+      exit 1
+      ;;
+  esac
+done
+
+# Validate required args
+if [ -z "$API_URL" ]; then
+  echo -e "${RED}Error: API_URL is required (-u)${NC}"
+  exit 1
+fi
+
+if [ -z "$API_KEY" ]; then
+  echo -e "${RED}Error: API_KEY is required (-k)${NC}"
+  exit 1
+fi
+
+# Get script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo -e "${CYAN}Running Throughput Benchmark...${NC}"
+echo "  URL:       $API_URL"
+echo "  Warm-up:   1m"
+echo "  Duration:  $DURATION (after warm-up)"
+echo "  Target:    $RPS RPS × $BULK_SIZE = $((RPS * BULK_SIZE)) solutions/sec"
+echo ""
+
+# Run k6 test (don't exit on threshold failures - we still want to open the report)
+k6 run \
+  -e API_URL="$API_URL" \
+  -e API_KEY="$API_KEY" \
+  -e TEST_DURATION="$DURATION" \
+  -e TARGET_RPS="$RPS" \
+  -e BULK_SIZE="$BULK_SIZE" \
+  "$SCRIPT_DIR/throughput-test.js" || true
+
+# Open report in browser
+if [ "$OPEN_REPORT" = true ] && [ -f "$SCRIPT_DIR/throughput-report.html" ]; then
+  echo ""
+  echo -e "${GREEN}Opening report in browser...${NC}"
+  
+  # Cross-platform open command
+  if command -v open &> /dev/null; then
+    open "$SCRIPT_DIR/throughput-report.html"
+  elif command -v xdg-open &> /dev/null; then
+    xdg-open "$SCRIPT_DIR/throughput-report.html"
+  elif command -v start &> /dev/null; then
+    start "$SCRIPT_DIR/throughput-report.html"
+  else
+    echo "Report saved to: $SCRIPT_DIR/throughput-report.html"
+  fi
+fi
+