@rulebricks/cli 2.0.0 → 2.0.2

package/benchmarks/throughput-results.json

@@ -0,0 +1,298 @@
+{
+  "root_group": {
+    "name": "",
+    "path": "",
+    "id": "d41d8cd98f00b204e9800998ecf8427e",
+    "groups": [],
+    "checks": [
+        {
+          "name": "status is 200",
+          "path": "::status is 200",
+          "id": "6210a8cd14cd70477eba5c5e4cb3fb5f",
+          "passes": 24000,
+          "fails": 0
+        },
+        {
+          "name": "valid response",
+          "path": "::valid response",
+          "id": "74e6940490a7b557672e61f02e36f4e8",
+          "passes": 24000,
+          "fails": 0
+        },
+        {
+          "name": "no error in response",
+          "path": "::no error in response",
+          "id": "04d657588d6c66446d4937ecba2292a2",
+          "passes": 24000,
+          "fails": 0
+        }
+      ]
+  },
+  "options": {
+    "summaryTrendStats": [
+      "avg",
+      "min",
+      "med",
+      "max",
+      "p(90)",
+      "p(95)"
+    ],
+    "summaryTimeUnit": "",
+    "noColor": false
+  },
+  "state": {
+    "isStdOutTTY": true,
+    "isStdErrTTY": true,
+    "testRunDurationMs": 300227.96
+  },
+  "metrics": {
+    "request_duration": {
+      "type": "trend",
+      "contains": "default",
+      "values": {
+        "p(95)": 318,
+        "avg": 207.51141666666666,
+        "min": 98,
+        "med": 195,
+        "max": 1305,
+        "p(90)": 285
+      }
+    },
+    "http_req_sending": {
+      "values": {
+        "p(90)": 0.443,
+        "p(95)": 0.575,
+        "avg": 0.3505282157261422,
+        "min": 0.029,
+        "med": 0.265,
+        "max": 197.681
+      },
+      "type": "trend",
+      "contains": "time"
+    },
+    "http_req_failed": {
+      "type": "rate",
+      "contains": "default",
+      "values": {
+        "rate": 0,
+        "passes": 0,
+        "fails": 30001
+      }
+    },
+    "checks": {
+      "type": "rate",
+      "contains": "default",
+      "values": {
+        "rate": 1,
+        "passes": 72000,
+        "fails": 0
+      }
+    },
+    "successes": {
+      "type": "rate",
+      "contains": "default",
+      "values": {
+        "rate": 1,
+        "passes": 24000,
+        "fails": 0
+      }
+    },
+    "http_req_blocked": {
+      "contains": "time",
+      "values": {
+        "med": 0.001,
+        "max": 446.733,
+        "p(90)": 0.001,
+        "p(95)": 0.002,
+        "avg": 1.6850438318703274,
+        "min": 0
+      },
+      "type": "trend"
+    },
+    "http_req_receiving": {
+      "type": "trend",
+      "contains": "time",
+      "values": {
+        "avg": 2.110890670310989,
+        "min": 0.019,
+        "med": 0.203,
+        "max": 206.634,
+        "p(90)": 3.99,
+        "p(95)": 5.81
+      }
+    },
+    "http_req_duration": {
+      "values": {
+        "avg": 204.29118289390323,
+        "min": 97.82,
+        "med": 193.159,
+        "max": 990.433,
+        "p(90)": 283.411,
+        "p(95)": 316.351
+      },
+      "type": "trend",
+      "contains": "time"
+    },
+    "vus_max": {
+      "type": "gauge",
+      "contains": "default",
+      "values": {
+        "value": 200,
+        "min": 200,
+        "max": 200
+      }
+    },
+    "errors{phase:measurement}": {
+      "values": {
+        "rate": 0,
+        "passes": 0,
+        "fails": 24000
+      },
+      "thresholds": {
+        "rate<0.05": {
+          "ok": true
+        }
+      },
+      "type": "rate",
+      "contains": "default"
+    },
+    "http_reqs": {
+      "contains": "default",
+      "values": {
+        "count": 30001,
+        "rate": 99.9274018315949
+      },
+      "type": "counter"
+    },
+    "data_received": {
+      "type": "counter",
+      "contains": "data",
+      "values": {
+        "count": 409778192,
+        "rate": 1364890.172121211
+      }
+    },
+    "iterations": {
+      "type": "counter",
+      "contains": "default",
+      "values": {
+        "count": 30001,
+        "rate": 99.9274018315949
+      }
+    },
+    "data_sent": {
+      "type": "counter",
+      "contains": "data",
+      "values": {
+        "count": 104185619,
+        "rate": 347021.7064393336
+      }
+    },
+    "http_req_tls_handshaking": {
+      "type": "trend",
+      "contains": "time",
+      "values": {
+        "avg": 0.9106618112729572,
+        "min": 0,
+        "med": 0,
+        "max": 306.889,
+        "p(90)": 0,
+        "p(95)": 0
+      }
+    },
+    "http_req_waiting": {
+      "contains": "time",
+      "values": {
+        "p(90)": 280.835,
+        "p(95)": 313.283,
+        "avg": 201.82976400786623,
+        "min": 94.379,
+        "med": 190.745,
+        "max": 933.167
+      },
+      "type": "trend"
+    },
+    "iteration_duration": {
+      "type": "trend",
+      "contains": "time",
+      "values": {
+        "p(95)": 324.982625,
+        "avg": 209.38244802586436,
+        "min": 100.226958,
+        "med": 196.689375,
+        "max": 1306.521125,
+        "p(90)": 288.073125
+      }
+    },
+    "errors": {
+      "type": "rate",
+      "contains": "default",
+      "values": {
+        "rate": 0,
+        "passes": 0,
+        "fails": 24000
+      }
+    },
+    "http_req_duration{phase:measurement}": {
+      "type": "trend",
+      "contains": "time",
+      "values": {
+        "p(95)": 316.0782,
+        "avg": 205.77155337500048,
+        "min": 97.82,
+        "med": 194.721,
+        "max": 990.433,
+        "p(90)": 283.5602
+      },
+      "thresholds": {
+        "p(95)<2000": {
+          "ok": true
+        },
+        "p(99)<5000": {
+          "ok": true
+        }
+      }
+    },
+    "http_req_connecting": {
+      "type": "trend",
+      "contains": "time",
+      "values": {
+        "avg": 0.7645731142295258,
+        "min": 0,
+        "med": 0,
+        "max": 257.371,
+        "p(90)": 0,
+        "p(95)": 0
+      }
+    },
+    "http_req_duration{expected_response:true}": {
+      "contains": "time",
+      "values": {
+        "avg": 204.29118289390323,
+        "min": 97.82,
+        "med": 193.159,
+        "max": 990.433,
+        "p(90)": 283.411,
+        "p(95)": 316.351
+      },
+      "type": "trend"
+    },
+    "vus": {
+      "type": "gauge",
+      "contains": "default",
+      "values": {
+        "max": 71,
+        "value": 21,
+        "min": 13
+      }
+    },
+    "total_payloads": {
+      "contains": "default",
+      "values": {
+        "count": 1200000,
+        "rate": 3996.9628411690906
+      },
+      "type": "counter"
+    }
+  }
+}

package/benchmarks/throughput-test.js

@@ -0,0 +1,159 @@
+/**
+ * Throughput (Solutions Per Second) Benchmark Test
+ *
+ * Measures rule engine capacity by sending bulk payload requests at a constant rate.
+ * This test helps you understand:
+ * - How many rule evaluations your deployment can process per second
+ * - Maximum throughput for batch workloads
+ * - Engine performance under sustained load
+ *
+ * Test Structure:
+ * - 1 minute warm-up phase (allows cluster to scale, excluded from results)
+ * - 4 minutes measurement phase (steady-state performance)
+ *
+ * Usage:
+ *   k6 run -e API_URL=https://your-instance.com/api/v1/flows/flow_id \
+ *          -e API_KEY=your-api-key \
+ *          throughput-test.js
+ *
+ * Optional environment variables:
+ *   TEST_DURATION - Measurement duration after warm-up (default: 4m)
+ *   TARGET_RPS    - Target bulk requests per second (default: 100)
+ *   BULK_SIZE     - Number of payloads per request (default: 50)
+ */
+
+import { check } from "k6";
+import http from "k6/http";
+import { Counter, Rate, Trend } from "k6/metrics";
+import {
+  generateBulkPayload,
+  getConfig,
+  createRequestParams,
+} from "./lib/payload.js";
+import {
+  generateThroughputReport,
+  generateThroughputConsoleSummary,
+} from "./lib/report.js";
+
+// Load configuration
+const config = getConfig({
+  testDuration: "4m",
+  targetRps: 100,
+  bulkSize: 50,
+});
+
+// Custom metrics (only for measurement phase)
+const errorRate = new Rate("errors");
+const successRate = new Rate("successes");
+const requestDuration = new Trend("request_duration");
+const droppedRequests = new Counter("dropped_requests");
+const totalPayloads = new Counter("total_payloads");
+const failedPayloads = new Counter("failed_payloads");
+
+// k6 options with warm-up and measurement phases
+export const options = {
+  scenarios: {
+    // Warm-up phase: 1 minute to let cluster scale
+    warm_up: {
+      executor: "constant-arrival-rate",
+      rate: config.targetRps,
+      timeUnit: "1s",
+      duration: "1m",
+      preAllocatedVUs: Math.min(config.targetRps, 100),
+      maxVUs: Math.min(config.targetRps * 3, 500),
+      exec: "warmUp",
+      tags: { phase: "warmup" },
+    },
+    // Measurement phase: actual test after warm-up
+    throughput_test: {
+      executor: "constant-arrival-rate",
+      rate: config.targetRps,
+      timeUnit: "1s",
+      duration: config.testDuration,
+      preAllocatedVUs: Math.min(config.targetRps, 100),
+      maxVUs: Math.min(config.targetRps * 3, 500),
+      startTime: "1m", // Start after warm-up
+      exec: "measureTest",
+      tags: { phase: "measurement" },
+    },
+  },
+  thresholds: {
+    // Only apply thresholds to measurement phase
+    "http_req_duration{phase:measurement}": ["p(95)<2000", "p(99)<5000"],
+    "errors{phase:measurement}": ["rate<0.05"],
+  },
+};
+
+// Request parameters (longer timeout for bulk requests)
+const params = createRequestParams(config.apiKey, "30s");
+
+/**
+ * Warm-up function - same as test but metrics tagged differently
+ */
+export function warmUp() {
+  const bulkPayload = generateBulkPayload(config.bulkSize);
+
+  try {
+    http.post(config.apiUrl, JSON.stringify(bulkPayload), params);
+  } catch (error) {
+    // Ignore errors during warm-up
+  }
+}
+
+/**
+ * Measurement test function - sends bulk requests
+ */
+export function measureTest() {
+  const bulkPayload = generateBulkPayload(config.bulkSize);
+  const start = Date.now();
+
+  try {
+    const response = http.post(
+      config.apiUrl,
+      JSON.stringify(bulkPayload),
+      params
+    );
+
+    const duration = Date.now() - start;
+    requestDuration.add(duration);
+
+    const success = check(response, {
+      "status is 200": (r) => r.status === 200,
+      "valid response": (r) => r.body && r.body.length > 0,
+      "no error in response": (r) => {
+        try {
+          const body = JSON.parse(r.body);
+          return !body.error;
+        } catch (e) {
+          return false;
+        }
+      },
+    });
+
+    errorRate.add(!success);
+    successRate.add(success);
+    totalPayloads.add(config.bulkSize);
+
+    if (!success) {
+      droppedRequests.add(1);
+      failedPayloads.add(config.bulkSize);
+    }
+  } catch (error) {
+    errorRate.add(1);
+    successRate.add(0);
+    droppedRequests.add(1);
+    totalPayloads.add(config.bulkSize);
+    failedPayloads.add(config.bulkSize);
+  }
+}
+
+/**
+ * Generate summary report
+ */
+export function handleSummary(data) {
+  return {
+    stdout: generateThroughputConsoleSummary(data, config),
+    "throughput-report.html": generateThroughputReport(data, config),
+    "throughput-results.json": JSON.stringify(data, null, 2),
+  };
+}

package/dist/commands/benchmark.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Benchmark Command
+ *
+ * Interactive wizard for configuring and running k6 load tests
+ * against Rulebricks deployments.
+ */
+interface BenchmarkCommandProps {
+    name?: string;
+}
+export declare function BenchmarkCommand(props: BenchmarkCommandProps): import("react/jsx-runtime").JSX.Element;
+export {};

package/dist/commands/benchmark.js

@@ -0,0 +1,173 @@
+import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
+/**
+ * Benchmark Command
+ *
+ * Interactive wizard for configuring and running k6 load tests
+ * against Rulebricks deployments.
+ */
+import { useState, useEffect, useCallback } from "react";
+import { Box, Text, useApp, useInput } from "ink";
+import { BorderBox, Spinner, ThemeProvider, useTheme, Logo, } from "../components/common/index.js";
+import { DeploymentSelectStep, ApiKeyStep, FlowSlugStep, TestModeStep, PresetsStep, ReviewStep, createInitialBenchmarkState, } from "../components/Wizard/steps/BenchmarkSteps.js";
+import { isK6Installed, getK6InstallInstructions, runBenchmark, buildApiUrl, openInBrowser, formatDuration, } from "../lib/benchmark.js";
+function BenchmarkCommandInner({ name }) {
+    const { exit } = useApp();
+    const { colors } = useTheme();
+    const [step, setStep] = useState("preflight");
+    const [wizardState, setWizardState] = useState(createInitialBenchmarkState());
+    const [error, setError] = useState(null);
+    const [k6Output, setK6Output] = useState([]);
+    const [result, setResult] = useState(null);
+    // Preflight check for k6
+    useEffect(() => {
+        if (step !== "preflight")
+            return;
+        (async () => {
+            const installed = await isK6Installed();
+            if (!installed) {
+                setError(`k6 is not installed.\n\n${getK6InstallInstructions()}\n\nVisit https://k6.io/docs/get-started/installation/ for more options.`);
+                setStep("error");
+                return;
+            }
+            // If a deployment name was provided via CLI, skip the selection step
+            if (name) {
+                setWizardState((s) => ({ ...s, deploymentName: name }));
+                // We still need to load the deployment URL, which the DeploymentSelectStep does
+                // So we'll go to that step anyway to validate and load the URL
+            }
+            setStep("select-deployment");
+        })();
+    }, [step, name]);
+    // Handle wizard step completion
+    const handleStepComplete = useCallback((data) => {
+        setWizardState((s) => ({ ...s, ...data }));
+        // Progress to next step
+        switch (step) {
+            case "select-deployment":
+                setStep("api-key");
+                break;
+            case "api-key":
+                setStep("flow-slug");
+                break;
+            case "flow-slug":
+                setStep("test-mode");
+                break;
+            case "test-mode":
+                setStep("presets");
+                break;
+            case "presets":
+                setStep("review");
+                break;
+            case "review":
+                setStep("running");
+                break;
+        }
+    }, [step]);
+    // Handle going back
+    const handleBack = useCallback(() => {
+        switch (step) {
+            case "select-deployment":
+                exit();
+                break;
+            case "api-key":
+                setStep("select-deployment");
+                break;
+            case "flow-slug":
+                setStep("api-key");
+                break;
+            case "test-mode":
+                setStep("flow-slug");
+                break;
+            case "presets":
+                setStep("test-mode");
+                break;
+            case "review":
+                setStep("presets");
+                break;
+        }
+    }, [step, exit]);
+    // Run the benchmark
+    useEffect(() => {
+        if (step !== "running")
+            return;
+        (async () => {
+            const config = {
+                deploymentName: wizardState.deploymentName,
+                apiUrl: buildApiUrl(wizardState.deploymentUrl, wizardState.flowSlug),
+                apiKey: wizardState.apiKey,
+                testMode: wizardState.testMode,
+                testDuration: wizardState.testDuration,
+                targetRps: wizardState.targetRps,
+                bulkSize: wizardState.testMode === "throughput"
+                    ? wizardState.bulkSize
+                    : undefined,
+            };
+            const benchmarkResult = await runBenchmark(config, {
+                onOutput: (line) => {
+                    setK6Output((prev) => {
+                        // Keep only last 15 lines to avoid memory issues
+                        const newOutput = [...prev, line];
+                        if (newOutput.length > 15) {
+                            return newOutput.slice(-15);
+                        }
+                        return newOutput;
+                    });
+                },
+            });
+            setResult(benchmarkResult);
+            if (benchmarkResult.success) {
+                setStep("complete");
+                // Try to open report in browser
+                try {
+                    await openInBrowser(benchmarkResult.reportPath);
+                }
+                catch {
+                    // Ignore browser open errors
+                }
+            }
+            else {
+                setError(benchmarkResult.error || "Benchmark failed");
+                setStep("error");
+            }
+        })();
+    }, [step, wizardState]);
+    // Handle key input for error/complete screens
+    useInput((input, key) => {
+        if (key.escape && (step === "error" || step === "complete")) {
+            exit();
+        }
+    });
+    // Render preflight check
+    if (step === "preflight") {
+        return (_jsx(BorderBox, { title: "Benchmark", children: _jsx(Box, { marginY: 1, children: _jsx(Spinner, { label: "Checking prerequisites..." }) }) }));
+    }
+    // Render error screen
+    if (step === "error") {
+        return (_jsx(BorderBox, { title: "Benchmark Failed", children: _jsxs(Box, { flexDirection: "column", marginY: 1, children: [_jsx(Text, { color: colors.error, bold: true, children: "Error" }), _jsx(Text, { color: colors.error, children: error }), result?.outputDir && (_jsx(Box, { marginTop: 1, children: _jsxs(Text, { color: colors.muted, dimColor: true, children: ["Output directory: ", result.outputDir] }) })), _jsx(Box, { marginTop: 1, children: _jsx(Text, { color: colors.muted, dimColor: true, children: "Press Esc to exit" }) })] }) }));
+    }
+    // Render running screen
+    if (step === "running") {
+        const expectedThroughput = wizardState.testMode === "throughput"
+            ? wizardState.targetRps * wizardState.bulkSize
+            : wizardState.targetRps;
+        return (_jsx(BorderBox, { title: "Running Benchmark", children: _jsxs(Box, { flexDirection: "column", marginY: 1, children: [_jsx(Box, { marginBottom: 1, children: _jsx(Spinner, { label: `Running ${wizardState.testMode.toUpperCase()} test against ${wizardState.deploymentName}...` }) }), _jsxs(Box, { flexDirection: "column", marginBottom: 1, children: [_jsxs(Text, { color: colors.muted, children: ["Target: ", expectedThroughput.toLocaleString(), " ", wizardState.testMode === "throughput" ? "solutions" : "requests", "/sec"] }), _jsxs(Text, { color: colors.muted, children: ["Duration: 1m warm-up + ", formatDuration(wizardState.testDuration)] })] }), k6Output.length > 0 && (_jsx(Box, { flexDirection: "column", borderStyle: "single", paddingX: 1, children: k6Output.map((line, i) => (_jsx(Text, { color: colors.muted, dimColor: true, children: line.length > 80 ? line.slice(0, 77) + "..." : line }, i))) })), _jsx(Box, { marginTop: 1, children: _jsx(Text, { color: colors.warning, dimColor: true, children: "This may take several minutes. Please wait..." }) })] }) }));
+    }
+    // Render complete screen
+    if (step === "complete" && result) {
+        const metrics = result.metrics;
+        return (_jsx(BorderBox, { title: "Benchmark Complete", children: _jsxs(Box, { flexDirection: "column", marginY: 1, children: [_jsx(Text, { color: colors.success, bold: true, children: "Benchmark completed successfully!" }), metrics && (_jsxs(Box, { flexDirection: "column", marginTop: 1, children: [_jsx(Text, { bold: true, children: "Results Summary:" }), _jsxs(Box, { marginTop: 1, flexDirection: "column", children: [_jsxs(Box, { children: [_jsx(Text, { color: colors.muted, children: "Success Rate: " }), _jsxs(Text, { color: metrics.successRate >= 99
+                                                    ? colors.success
+                                                    : metrics.successRate >= 95
+                                                        ? colors.warning
+                                                        : colors.error, bold: true, children: [metrics.successRate.toFixed(1), "%"] })] }), _jsxs(Box, { children: [_jsx(Text, { color: colors.muted, children: "Actual RPS: " }), _jsx(Text, { color: colors.accent, bold: true, children: metrics.actualRps.toFixed(1) })] }), metrics.actualThroughput && (_jsxs(Box, { children: [_jsx(Text, { color: colors.muted, children: "Throughput: " }), _jsxs(Text, { color: colors.accent, bold: true, children: [metrics.actualThroughput.toFixed(0), " solutions/sec"] })] })), _jsxs(Box, { children: [_jsx(Text, { color: colors.muted, children: "P95 Latency: " }), _jsxs(Text, { color: metrics.p95Latency < 200
+                                                    ? colors.success
+                                                    : metrics.p95Latency < 500
+                                                        ? colors.warning
+                                                        : colors.error, children: [metrics.p95Latency.toFixed(0), "ms"] })] }), _jsxs(Box, { children: [_jsx(Text, { color: colors.muted, children: "P99 Latency: " }), _jsxs(Text, { children: [metrics.p99Latency.toFixed(0), "ms"] })] }), _jsxs(Box, { children: [_jsx(Text, { color: colors.muted, children: "Total Requests: " }), _jsx(Text, { children: metrics.totalRequests.toLocaleString() })] })] })] })), _jsxs(Box, { flexDirection: "column", marginTop: 1, children: [_jsx(Text, { color: colors.muted, children: "Results saved to:" }), _jsx(Text, { color: colors.accent, children: result.outputDir })] }), _jsx(Box, { marginTop: 1, children: _jsxs(Text, { color: colors.muted, children: ["Report: ", result.reportPath.split("/").pop()] }) }), _jsx(Box, { marginTop: 1, children: _jsx(Text, { color: colors.success, children: "The HTML report should open in your browser automatically." }) }), _jsx(Box, { marginTop: 1, children: _jsx(Text, { color: colors.muted, dimColor: true, children: "Press Esc to exit" }) })] }) }));
+    }
+    // Render wizard steps
+    return (_jsxs(_Fragment, { children: [step === "select-deployment" && (_jsx(DeploymentSelectStep, { onComplete: handleStepComplete, onBack: handleBack, state: wizardState })), step === "api-key" && (_jsx(ApiKeyStep, { onComplete: handleStepComplete, onBack: handleBack, state: wizardState })), step === "flow-slug" && (_jsx(FlowSlugStep, { onComplete: handleStepComplete, onBack: handleBack, state: wizardState })), step === "test-mode" && (_jsx(TestModeStep, { onComplete: handleStepComplete, onBack: handleBack, state: wizardState })), step === "presets" && (_jsx(PresetsStep, { onComplete: handleStepComplete, onBack: handleBack, state: wizardState })), step === "review" && (_jsx(ReviewStep, { onComplete: handleStepComplete, onBack: handleBack, state: wizardState }))] }));
+}
+export function BenchmarkCommand(props) {
+    return (_jsxs(ThemeProvider, { theme: "status", children: [_jsx(Logo, {}), _jsx(BenchmarkCommandInner, { ...props })] }));
+}