@flotorch/loadtest 0.2.2 → 0.2.4

package/dist/index.js

@@ -222,6 +222,7 @@ function StatsPanel({
   recentTtft,
   recentE2eLatency,
   errors,
+  emptyResponses,
   completed
 }) {
   const sortedTtft = [...recentTtft].sort((a, b) => a - b);
@@ -268,6 +269,14 @@ function StatsPanel({
         " (",
         errRate,
         "%)"
+      ] }),
+      /* @__PURE__ */ jsx4(Text4, { children: "  " }),
+      /* @__PURE__ */ jsx4(Text4, { dimColor: true, children: "Empty " }),
+      /* @__PURE__ */ jsxs4(Text4, { color: emptyResponses > 0 ? "yellow" : void 0, bold: emptyResponses > 0, children: [
+        emptyResponses,
+        " (",
+        completed > 0 ? (emptyResponses / completed * 100).toFixed(1) : "0.0",
+        "%)"
       ] })
     ] })
   ] });
@@ -357,6 +366,7 @@ function App({ store }) {
         recentTtft: snap.recentTtft,
         recentE2eLatency: snap.recentE2eLatency,
         errors: snap.errors,
+        emptyResponses: snap.emptyResponses,
         completed: snap.completed
       }
     ) }),
@@ -455,7 +465,7 @@ var cyan = wrap("36", "39");
 var magenta = wrap("35", "39");
 
 // src/cli/args.ts
-var VERSION = true ? "0.2.2" : "dev";
+var VERSION = true ? "0.2.4" : "dev";
 var VALID_COMMANDS = /* @__PURE__ */ new Set(["run", "generate", "bench", "report", "init"]);
 var HELP_TEXT = `
 ${bold("FLOTorch Load Tester")} ${dim(`v${VERSION}`)}
@@ -882,12 +892,13 @@ var OpenAIBackend = class _OpenAIBackend {
     }
     messages.push({ role: "user", content: prompt2 });
     const body = {
-      model,
       messages,
       stream: streaming,
-      ...params
+      ...params,
+      model,
+      max_tokens: maxTokens
     };
-    if (body.max_tokens && this.isOpenAIHost()) {
+    if (this.isOpenAIHost()) {
       body.max_completion_tokens = body.max_tokens;
       delete body.max_tokens;
     }
@@ -1017,7 +1028,7 @@ var SageMakerBackend = class _SageMakerBackend {
   }
   constructor(config) {
     this.baseURL = config.baseURL ?? `https://runtime.sagemaker.${config.region}.amazonaws.com`;
-    this.requestFormat = config.requestFormat ?? "sagemaker" /* Sagemaker */;
+    this.requestFormat = config.requestFormat ?? "openai" /* OpenAI */;
     this.signer = new SignatureV4({
       service: "sagemaker",
       region: config.region,
@@ -1068,22 +1079,25 @@ var SageMakerBackend = class _SageMakerBackend {
     return this.parseResponse(response);
   }
   buildRequestBody(prompt2, maxTokens, systemPrompt, params, streaming) {
-    const messages = [];
-    if (systemPrompt) messages.push({ role: "system", content: systemPrompt });
-    messages.push({ role: "user", content: prompt2 });
     if (this.requestFormat === "openai" /* OpenAI */) {
+      const messages = [];
+      if (systemPrompt) messages.push({ role: "system", content: systemPrompt });
+      messages.push({ role: "user", content: prompt2 });
       return {
         messages,
-        max_tokens: maxTokens,
         stream: streaming,
-        ...params
+        ...params,
+        max_tokens: maxTokens
       };
     }
+    const rawPrompt = systemPrompt ? `${systemPrompt}
+
+${prompt2}` : prompt2;
     return {
-      inputs: [messages],
+      inputs: rawPrompt,
       parameters: {
-        max_new_tokens: maxTokens,
-        ...params
+        ...params,
+        max_new_tokens: maxTokens
       }
     };
   }
@@ -1262,7 +1276,7 @@ function createBackend(config) {
     case "openai":
       return OpenAIBackend.create(baseURL);
     case "sagemaker": {
-      const requestFormat = config.provider.config?.["requestFormat"] ?? "sagemaker" /* Sagemaker */;
+      const requestFormat = config.provider.config?.["requestFormat"] ?? "openai" /* OpenAI */;
       return SageMakerBackend.create(baseURL, requestFormat);
     }
     default:
@@ -1397,17 +1411,18 @@ async function executeRequest(backend, prompt2, config, requestId, phase, cacheH
     );
     const endTime = performance.now();
     const e2eLatencyMs = endTime - startTime;
+    const isEmpty = response.outputTokens === 0 && response.generatedText === "";
     return {
       requestId,
       startTime,
       endTime,
-      ttftMs: response.ttftMs,
+      ttftMs: isEmpty ? -1 : response.ttftMs,
       e2eLatencyMs,
       interTokenLatencies: response.interTokenLatencies,
       inputText: prompt2.text,
       inputTokens: prompt2.tokenCount,
       outputTokens: response.outputTokens,
-      outputThroughputTps: e2eLatencyMs > 0 ? response.outputTokens / (e2eLatencyMs / 1e3) : 0,
+      outputThroughputTps: isEmpty ? 0 : e2eLatencyMs > 0 ? response.outputTokens / (e2eLatencyMs / 1e3) : 0,
       generatedText: response.generatedText,
       phase,
       cacheHit
@@ -1559,13 +1574,15 @@ function aggregate(values) {
 
 // src/reporter/aggregator.ts
 function computeSummary(requests) {
-  const successful = requests.filter((r) => !r.error);
   const failed = requests.filter((r) => !!r.error);
+  const successful = requests.filter((r) => !r.error);
+  const empty = successful.filter((r) => r.ttftMs === -1);
+  const completed = successful.filter((r) => r.ttftMs !== -1);
   const startTime = Math.min(...requests.map((r) => r.startTime));
   const endTime = Math.max(...requests.map((r) => r.endTime));
   const durationMs = endTime - startTime;
   const durationMin = durationMs / 6e4;
-  const totalOutputTokens = successful.reduce((sum, r) => sum + r.outputTokens, 0);
+  const totalOutputTokens = completed.reduce((sum, r) => sum + r.outputTokens, 0);
   const errorCodeFrequency = {};
   for (const r of failed) {
     const code = r.errorCode ?? "unknown";
@@ -1584,24 +1601,25 @@ function computeSummary(requests) {
       };
     }
   }
-  const ttfntValues = successful.map((r) => r.ttfntMs).filter((v) => v !== void 0 && v > 0);
-  const itlValues = successful.flatMap((r) => r.interTokenLatencies);
+  const ttfntValues = completed.map((r) => r.ttfntMs).filter((v) => v !== void 0 && v > 0);
+  const itlValues = completed.flatMap((r) => r.interTokenLatencies);
   return {
     startTime,
     endTime,
     totalRequests: requests.length,
-    successfulRequests: successful.length,
+    successfulRequests: completed.length,
     failedRequests: failed.length,
+    emptyResponses: empty.length,
     errorRate: requests.length > 0 ? failed.length / requests.length : 0,
     rpm: durationMin > 0 ? requests.length / durationMin : 0,
     overallTps: durationMs > 0 ? totalOutputTokens / (durationMs / 1e3) : 0,
-    ttft: aggregate(successful.map((r) => r.ttftMs)),
+    ttft: aggregate(completed.map((r) => r.ttftMs)),
     ttfnt: ttfntValues.length > 0 ? aggregate(ttfntValues) : void 0,
-    e2eLatency: aggregate(successful.map((r) => r.e2eLatencyMs)),
-    outputThroughput: aggregate(successful.map((r) => r.outputThroughputTps)),
+    e2eLatency: aggregate(completed.map((r) => r.e2eLatencyMs)),
+    outputThroughput: aggregate(completed.map((r) => r.outputThroughputTps)),
     interTokenLatency: aggregate(itlValues),
-    inputTokens: aggregate(successful.map((r) => r.inputTokens)),
-    outputTokens: aggregate(successful.map((r) => r.outputTokens)),
+    inputTokens: aggregate(completed.map((r) => r.inputTokens)),
+    outputTokens: aggregate(completed.map((r) => r.outputTokens)),
     errorCodeFrequency,
     cacheHitRate: requests.length > 0 ? cacheHits / requests.length : 0,
     phaseBreakdown
@@ -1659,6 +1677,7 @@ var CsvExporter = class {
       "totalRequests",
       "successfulRequests",
       "failedRequests",
+      "emptyResponses",
       "errorRate",
       "rpm",
       "overallTps",
@@ -1668,6 +1687,7 @@ var CsvExporter = class {
       summary.totalRequests,
       summary.successfulRequests,
       summary.failedRequests,
+      summary.emptyResponses,
       summary.errorRate,
       summary.rpm,
       summary.overallTps,
@@ -1776,6 +1796,7 @@ var BenchmarkStore = class {
   completed = 0;
   totalTarget;
   errors = 0;
+  emptyResponses = 0;
   totalOutputTokens = 0;
   totalInputTokens = 0;
   startTime = 0;
@@ -1801,8 +1822,9 @@ var BenchmarkStore = class {
       this.errors++;
       if (this.recentErrors.length >= 5) this.recentErrors.shift();
       this.recentErrors.push(metrics.error);
-    }
-    if (!metrics.error) {
+    } else if (metrics.ttftMs === -1) {
+      this.emptyResponses++;
+    } else {
       if (this.recentTtft.length >= ROLLING_CAP) this.recentTtft.shift();
       this.recentTtft.push(metrics.ttftMs);
       if (this.recentE2eLatency.length >= ROLLING_CAP) this.recentE2eLatency.shift();
@@ -1825,6 +1847,7 @@ var BenchmarkStore = class {
       completed: this.completed,
       totalTarget: this.totalTarget,
       errors: this.errors,
+      emptyResponses: this.emptyResponses,
       totalOutputTokens: this.totalOutputTokens,
       totalInputTokens: this.totalInputTokens,
       startTime: this.startTime,
@@ -1874,7 +1897,7 @@ var FallbackDisplay = class {
     const bar = "=".repeat(filled) + ".".repeat(barWidth - filled);
     process.stderr.write(
       `  [${bar}] ${s.completed}/${totalStr}${pctStr}
-  ${formatDuration(elapsed)} | ${rps.toFixed(1)} req/s | ${tps.toFixed(0)} tok/s | ${s.errors} err
+  ${formatDuration(elapsed)} | ${rps.toFixed(1)} req/s | ${tps.toFixed(0)} tok/s | ${s.errors} err | ${s.emptyResponses} empty
 `
     );
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flotorch/loadtest",
-  "version": "0.2.2",
+  "version": "0.2.4",
   "description": "LLM inference load testing and benchmarking tool",
   "license": "MIT",
   "repository": {