@camstack/addon-benchmark 1.1.30 → 1.1.31

package/dist/index.js

@@ -551,7 +551,7 @@ __export(core_exports2, {
   parse: () => parse,
   parseAsync: () => parseAsync,
   prettifyError: () => prettifyError,
-  process: () => process,
+  process: () => process2,
   regexes: () => regexes_exports,
   registry: () => registry,
   safeDecode: () => safeDecode,
@@ -11476,7 +11476,7 @@ function initializeContext(params) {
     external: params?.external ?? void 0
   };
 }
-function process(schema, ctx, _params = { path: [], schemaPath: [] }) {
+function process2(schema, ctx, _params = { path: [], schemaPath: [] }) {
   var _a3;
   const def = schema._zod.def;
   const seen = ctx.seen.get(schema);
@@ -11513,7 +11513,7 @@ function process(schema, ctx, _params = { path: [], schemaPath: [] }) {
     if (parent) {
       if (!result.ref)
         result.ref = parent;
-      process(parent, ctx, params);
+      process2(parent, ctx, params);
       ctx.seen.get(parent).isParent = true;
     }
   }
@@ -11801,14 +11801,14 @@ function isTransforming(_schema, _ctx) {
 }
 var createToJSONSchemaMethod = (schema, processors = {}) => (params) => {
   const ctx = initializeContext({ ...params, processors });
-  process(schema, ctx);
+  process2(schema, ctx);
   extractDefs(ctx, schema);
   return finalize(ctx, schema);
 };
 var createStandardJSONSchemaMethod = (schema, io, processors = {}) => (params) => {
   const { libraryOptions, target } = params ?? {};
   const ctx = initializeContext({ ...libraryOptions ?? {}, target, io, processors });
-  process(schema, ctx);
+  process2(schema, ctx);
   extractDefs(ctx, schema);
   return finalize(ctx, schema);
 };
@@ -12054,7 +12054,7 @@ var arrayProcessor = (schema, ctx, _json, params) => {
   if (typeof maximum === "number")
     json2.maxItems = maximum;
   json2.type = "array";
-  json2.items = process(def.element, ctx, {
+  json2.items = process2(def.element, ctx, {
     ...params,
     path: [...params.path, "items"]
   });
@@ -12066,7 +12066,7 @@ var objectProcessor = (schema, ctx, _json, params) => {
   json2.properties = {};
   const shape = def.shape;
   for (const key in shape) {
-    json2.properties[key] = process(shape[key], ctx, {
+    json2.properties[key] = process2(shape[key], ctx, {
       ...params,
       path: [...params.path, "properties", key]
     });
@@ -12089,7 +12089,7 @@ var objectProcessor = (schema, ctx, _json, params) => {
     if (ctx.io === "output")
       json2.additionalProperties = false;
   } else if (def.catchall) {
-    json2.additionalProperties = process(def.catchall, ctx, {
+    json2.additionalProperties = process2(def.catchall, ctx, {
       ...params,
       path: [...params.path, "additionalProperties"]
     });
@@ -12098,7 +12098,7 @@ var objectProcessor = (schema, ctx, _json, params) => {
 var unionProcessor = (schema, ctx, json2, params) => {
   const def = schema._zod.def;
   const isExclusive = def.inclusive === false;
-  const options = def.options.map((x, i) => process(x, ctx, {
+  const options = def.options.map((x, i) => process2(x, ctx, {
     ...params,
     path: [...params.path, isExclusive ? "oneOf" : "anyOf", i]
   }));
@@ -12110,11 +12110,11 @@ var unionProcessor = (schema, ctx, json2, params) => {
 };
 var intersectionProcessor = (schema, ctx, json2, params) => {
   const def = schema._zod.def;
-  const a = process(def.left, ctx, {
+  const a = process2(def.left, ctx, {
     ...params,
     path: [...params.path, "allOf", 0]
   });
-  const b = process(def.right, ctx, {
+  const b = process2(def.right, ctx, {
     ...params,
     path: [...params.path, "allOf", 1]
   });
@@ -12131,11 +12131,11 @@ var tupleProcessor = (schema, ctx, _json, params) => {
   json2.type = "array";
   const prefixPath = ctx.target === "draft-2020-12" ? "prefixItems" : "items";
   const restPath = ctx.target === "draft-2020-12" ? "items" : ctx.target === "openapi-3.0" ? "items" : "additionalItems";
-  const prefixItems = def.items.map((x, i) => process(x, ctx, {
+  const prefixItems = def.items.map((x, i) => process2(x, ctx, {
     ...params,
     path: [...params.path, prefixPath, i]
   }));
-  const rest = def.rest ? process(def.rest, ctx, {
+  const rest = def.rest ? process2(def.rest, ctx, {
     ...params,
     path: [...params.path, restPath, ...ctx.target === "openapi-3.0" ? [def.items.length] : []]
   }) : null;
@@ -12175,7 +12175,7 @@ var recordProcessor = (schema, ctx, _json, params) => {
   const keyBag = keyType._zod.bag;
   const patterns = keyBag?.patterns;
   if (def.mode === "loose" && patterns && patterns.size > 0) {
-    const valueSchema = process(def.valueType, ctx, {
+    const valueSchema = process2(def.valueType, ctx, {
       ...params,
       path: [...params.path, "patternProperties", "*"]
     });
@@ -12185,12 +12185,12 @@ var recordProcessor = (schema, ctx, _json, params) => {
     }
   } else {
     if (ctx.target === "draft-07" || ctx.target === "draft-2020-12") {
-      json2.propertyNames = process(def.keyType, ctx, {
+      json2.propertyNames = process2(def.keyType, ctx, {
         ...params,
         path: [...params.path, "propertyNames"]
       });
     }
-    json2.additionalProperties = process(def.valueType, ctx, {
+    json2.additionalProperties = process2(def.valueType, ctx, {
       ...params,
       path: [...params.path, "additionalProperties"]
     });
@@ -12205,7 +12205,7 @@ var recordProcessor = (schema, ctx, _json, params) => {
 };
 var nullableProcessor = (schema, ctx, json2, params) => {
   const def = schema._zod.def;
-  const inner = process(def.innerType, ctx, params);
+  const inner = process2(def.innerType, ctx, params);
   const seen = ctx.seen.get(schema);
   if (ctx.target === "openapi-3.0") {
     seen.ref = def.innerType;
@@ -12216,20 +12216,20 @@ var nullableProcessor = (schema, ctx, json2, params) => {
 };
 var nonoptionalProcessor = (schema, ctx, _json, params) => {
   const def = schema._zod.def;
-  process(def.innerType, ctx, params);
+  process2(def.innerType, ctx, params);
   const seen = ctx.seen.get(schema);
   seen.ref = def.innerType;
 };
 var defaultProcessor = (schema, ctx, json2, params) => {
   const def = schema._zod.def;
-  process(def.innerType, ctx, params);
+  process2(def.innerType, ctx, params);
   const seen = ctx.seen.get(schema);
   seen.ref = def.innerType;
   json2.default = JSON.parse(JSON.stringify(def.defaultValue));
 };
 var prefaultProcessor = (schema, ctx, json2, params) => {
   const def = schema._zod.def;
-  process(def.innerType, ctx, params);
+  process2(def.innerType, ctx, params);
   const seen = ctx.seen.get(schema);
   seen.ref = def.innerType;
   if (ctx.io === "input")
@@ -12237,7 +12237,7 @@ var prefaultProcessor = (schema, ctx, json2, params) => {
 };
 var catchProcessor = (schema, ctx, json2, params) => {
   const def = schema._zod.def;
-  process(def.innerType, ctx, params);
+  process2(def.innerType, ctx, params);
   const seen = ctx.seen.get(schema);
   seen.ref = def.innerType;
   let catchValue;
@@ -12252,32 +12252,32 @@ var pipeProcessor = (schema, ctx, _json, params) => {
   const def = schema._zod.def;
   const inIsTransform = def.in._zod.traits.has("$ZodTransform");
   const innerType = ctx.io === "input" ? inIsTransform ? def.out : def.in : def.out;
-  process(innerType, ctx, params);
+  process2(innerType, ctx, params);
   const seen = ctx.seen.get(schema);
   seen.ref = innerType;
 };
 var readonlyProcessor = (schema, ctx, json2, params) => {
   const def = schema._zod.def;
-  process(def.innerType, ctx, params);
+  process2(def.innerType, ctx, params);
   const seen = ctx.seen.get(schema);
   seen.ref = def.innerType;
   json2.readOnly = true;
 };
 var promiseProcessor = (schema, ctx, _json, params) => {
   const def = schema._zod.def;
-  process(def.innerType, ctx, params);
+  process2(def.innerType, ctx, params);
   const seen = ctx.seen.get(schema);
   seen.ref = def.innerType;
 };
 var optionalProcessor = (schema, ctx, _json, params) => {
   const def = schema._zod.def;
-  process(def.innerType, ctx, params);
+  process2(def.innerType, ctx, params);
   const seen = ctx.seen.get(schema);
   seen.ref = def.innerType;
 };
 var lazyProcessor = (schema, ctx, _json, params) => {
   const innerType = schema._zod.innerType;
-  process(innerType, ctx, params);
+  process2(innerType, ctx, params);
   const seen = ctx.seen.get(schema);
   seen.ref = innerType;
 };
@@ -12329,7 +12329,7 @@ function toJSONSchema(input, params) {
     const defs = {};
     for (const entry of registry2._idmap.entries()) {
       const [_, schema] = entry;
-      process(schema, ctx2);
+      process2(schema, ctx2);
     }
     const schemas = {};
     const external = {
@@ -12352,7 +12352,7 @@ function toJSONSchema(input, params) {
     return { schemas };
   }
   const ctx = initializeContext({ ...params, processors: allProcessors });
-  process(input, ctx);
+  process2(input, ctx);
   extractDefs(ctx, input);
   return finalize(ctx, input);
 }
@@ -12410,7 +12410,7 @@ var JSONSchemaGenerator = class {
    * This must be called before emit().
    */
   process(schema, _params = { path: [], schemaPath: [] }) {
-    return process(schema, this.ctx, _params);
+    return process2(schema, this.ctx, _params);
   }
   /**
    * Emit the final JSON Schema after processing.
@@ -14775,6 +14775,62 @@ var MultiCameraStressResultSchema = external_exports.object({
   inferMsMean: external_exports.number(),
   errors: external_exports.number()
 });
+var SustainedEngineSchema = external_exports.object({
+  runtime: external_exports.enum(["python", "node"]),
+  backend: external_exports.string(),
+  format: external_exports.enum(["onnx", "coreml", "openvino"]),
+  device: external_exports.string().optional()
+}).optional();
+var SustainedNodeTargetSchema = external_exports.object({
+  nodeId: external_exports.string().optional(),
+  engine: SustainedEngineSchema,
+  steps: external_exports.array(import_types.PipelineStepInputSchema).readonly(),
+  label: external_exports.string().optional()
+});
+var SustainedThroughputInputSchema = external_exports.object({
+  frameId: external_exports.string().optional(),
+  imageBase64: external_exports.string().optional(),
+  durationSec: external_exports.number().min(1).max(3600),
+  batchSize: external_exports.number().int().min(1).max(255),
+  concurrency: external_exports.number().int().min(1).max(64),
+  targetSize: external_exports.number().int().min(64).max(2048).optional(),
+  sessionId: external_exports.string(),
+  nodes: external_exports.array(SustainedNodeTargetSchema).min(1).readonly()
+});
+var SustainedLatencyStatsSchema = external_exports.object({
+  p50: external_exports.number(),
+  p95: external_exports.number(),
+  p99: external_exports.number(),
+  mean: external_exports.number(),
+  min: external_exports.number(),
+  max: external_exports.number()
+});
+var SustainedNodeResultSchema = external_exports.object({
+  nodeId: external_exports.string().nullable(),
+  label: external_exports.string().optional(),
+  /** Total frames processed by this node's loop. */
+  totalFrames: external_exports.number(),
+  /** Total completed batch calls. */
+  totalBatches: external_exports.number(),
+  /** Measured wall-clock window in ms. */
+  wallMs: external_exports.number(),
+  /** Sustained throughput. */
+  fps: external_exports.number(),
+  detsPerSec: external_exports.number(),
+  /** Mean detections per frame — stable since the frame is fixed. */
+  avgDetections: external_exports.number(),
+  latency: SustainedLatencyStatsSchema,
+  /** Batch calls that errored (transport / inference). */
+  errors: external_exports.number(),
+  /** True when this node ended via the stop button rather than the deadline. */
+  aborted: external_exports.boolean()
+});
+var SustainedThroughputResultSchema = external_exports.object({
+  perNode: external_exports.array(SustainedNodeResultSchema).readonly(),
+  durationSec: external_exports.number(),
+  batchSize: external_exports.number(),
+  concurrency: external_exports.number()
+});
 
 // src/benchmark-actions.ts
 var import_types2 = require("@camstack/types");
@@ -14823,6 +14879,25 @@ var benchmarkActions = (0, import_types2.defineCustomActions)({
     MultiCameraStressResultSchema,
     { kind: "mutation" }
   ),
+  /**
+   * Sustained-throughput ROOT-detection e2e run. Pins ONE frame (decoded
+   * once), then feeds the identical frame to root object-detection for
+   * `durationSec` seconds — driven by BATCHES (`runPipelineBatch`,
+   * `batchSize` frames/call) with `concurrency` calls in flight, fanned out
+   * across `nodes`, each on its resolved runtime/model + root-only steps.
+   * Returns per-node sustained fps / det/s + latency percentiles on the
+   * identical pinned frame. Bounded by N + `abortSustainedThroughput`.
+   */
+  runSustainedThroughput: (0, import_types2.customAction)(
+    SustainedThroughputInputSchema,
+    SustainedThroughputResultSchema,
+    { kind: "mutation" }
+  ),
+  abortSustainedThroughput: (0, import_types2.customAction)(
+    external_exports.object({ sessionId: external_exports.string() }),
+    external_exports.object({ aborted: external_exports.boolean() }),
+    { kind: "mutation" }
+  ),
   // History (shared across pipeline + decoder-perf entries)
   listBenchmarkHistory: (0, import_types2.customAction)(external_exports.void(), external_exports.array(HistoryEntrySchema).readonly()),
   saveBenchmarkResult: (0, import_types2.customAction)(
@@ -15153,10 +15228,110 @@ async function runDecoderPerfTest(input, deps) {
   };
 }
 
+// src/throughput/sustained-runner.ts
+async function runSustainedLoop(opts) {
+  const concurrency = Math.max(1, Math.floor(opts.concurrency));
+  const start = opts.now();
+  const deadline = start + Math.max(0, opts.durationMs);
+  const callMsList = [];
+  let totalFrames = 0;
+  let totalBatches = 0;
+  let totalDetections = 0;
+  let detectionSamples = 0;
+  let errors = 0;
+  const worker = async () => {
+    while (!opts.shouldStop() && opts.now() < deadline) {
+      const t0 = opts.now();
+      try {
+        const out = await opts.dispatchBatch();
+        const t1 = opts.now();
+        callMsList.push(t1 - t0);
+        totalBatches += 1;
+        totalFrames += out.frameCount;
+        for (const c of out.detectionCounts) {
+          totalDetections += c;
+          detectionSamples += 1;
+        }
+      } catch {
+        errors += 1;
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: concurrency }, () => worker()));
+  const wallMs = opts.now() - start;
+  return {
+    callMsList,
+    totalFrames,
+    totalBatches,
+    totalDetections,
+    detectionSamples,
+    errors,
+    wallMs,
+    aborted: opts.shouldStop()
+  };
+}
+
+// src/throughput/throughput-stats.ts
+var EMPTY_LATENCY = { p50: 0, p95: 0, p99: 0, mean: 0, min: 0, max: 0 };
+function percentile(sortedAsc, p) {
+  const n = sortedAsc.length;
+  if (n === 0) return 0;
+  const clamped = p < 0 ? 0 : p > 1 ? 1 : p;
+  if (clamped <= 0) return sortedAsc[0] ?? 0;
+  if (clamped >= 1) return sortedAsc[n - 1] ?? 0;
+  const rank = Math.ceil(clamped * n);
+  const idx = Math.min(n - 1, Math.max(0, rank - 1));
+  return sortedAsc[idx] ?? 0;
+}
+function computeLatencyStats(samplesMs) {
+  if (samplesMs.length === 0) return EMPTY_LATENCY;
+  const sorted = [...samplesMs].sort((a, b) => a - b);
+  const sum = sorted.reduce((acc, x) => acc + x, 0);
+  return {
+    p50: percentile(sorted, 0.5),
+    p95: percentile(sorted, 0.95),
+    p99: percentile(sorted, 0.99),
+    mean: sum / sorted.length,
+    min: sorted[0] ?? 0,
+    max: sorted[sorted.length - 1] ?? 0
+  };
+}
+function computeFps(totalFrames, wallMs) {
+  if (wallMs <= 0) return 0;
+  return totalFrames * 1e3 / wallMs;
+}
+function computeDetsPerSec(totalDetections, wallMs) {
+  if (wallMs <= 0) return 0;
+  return totalDetections * 1e3 / wallMs;
+}
+function computeAvgDetections(totalDetections, sampledFrames) {
+  if (sampledFrames <= 0) return 0;
+  return totalDetections / sampledFrames;
+}
+
 // src/benchmark-addon.ts
 var KEY_BENCHMARK_HISTORY = "benchmarkHistory";
 var MAX_HISTORY_ENTRIES = 100;
 var EMPTY_HISTORY_JSON = "[]";
+var BENCH_RUN_TIMEOUT_MS = Math.max(
+  1e4,
+  Number(process.env["CAMSTACK_BENCH_RUN_TIMEOUT_MS"]) || 12e4
+);
+async function withHardTimeout(promise2, ms, label) {
+  let timer;
+  const timeout = new Promise((_resolve, reject) => {
+    timer = setTimeout(
+      () => reject(new Error(`${label} exceeded ${ms}ms \u2014 aborting (likely a wedged step)`)),
+      ms
+    );
+    timer.unref?.();
+  });
+  try {
+    return await Promise.race([promise2, timeout]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
 var HistoryArraySchema = external_exports.array(HistoryEntrySchema);
 var FIXTURE_META = {
   "camera-sim-360p.h264": {
@@ -15184,6 +15359,8 @@ var BenchmarkAddon = class extends import_types4.BaseAddon {
   id = "benchmark";
   /** Per-sessionId abort flags for in-flight decoder perf runs. */
   decoderPerfAbortFlags = /* @__PURE__ */ new Map();
+  /** Per-sessionId abort flags for in-flight sustained-throughput runs. */
+  sustainedAbortFlags = /* @__PURE__ */ new Map();
   /**
    * In-memory cache of original JPEG bytes keyed by `frameId`. Populated by
    * `cacheBenchFrame` (UI uploads JPEG once at bench start). Subsequent
@@ -15226,6 +15403,8 @@ var BenchmarkAddon = class extends import_types4.BaseAddon {
         runPipelineTest: async (input) => this.runPipelineTest(input),
         runPipelineBatchTest: async (input) => this.runPipelineBatchTest(input),
         runMultiCameraStressTest: async (input) => this.runMultiCameraStressTest(input),
+        runSustainedThroughput: async (input) => this.runSustainedThroughput(input),
+        abortSustainedThroughput: async (input) => this.abortSustainedThroughput(input),
         cacheBenchFrame: async (input) => this.cacheBenchFrame(input),
         releaseBenchFrame: async (input) => this.releaseBenchFrame(input),
         listBenchmarkHistory: async () => this.listBenchmarkHistory(),
@@ -15245,6 +15424,8 @@ var BenchmarkAddon = class extends import_types4.BaseAddon {
       this.frameCacheSweeper = null;
     }
     this.frameCache.clear();
+    for (const flag of this.sustainedAbortFlags.values()) flag.aborted = true;
+    this.sustainedAbortFlags.clear();
   }
   // ── Frame cache (raw RGB) ────────────────────────────────────────────────
   async cacheBenchFrame(input) {
@@ -15314,15 +15495,19 @@ var BenchmarkAddon = class extends import_types4.BaseAddon {
       });
       return globalSteps.map(toInput);
     })();
-    return this.api.pipelineExecutor.runPipeline.mutate({
-      nodeId,
-      steps: stepsToRun,
-      ...image ? { image } : {},
-      ...!image && imageBase64 ? { imageBase64 } : {},
-      referenceImage,
-      sessionId,
-      ...engine ? { engine } : {}
-    });
+    return withHardTimeout(
+      this.api.pipelineExecutor.runPipeline.mutate({
+        nodeId,
+        steps: stepsToRun,
+        ...image ? { image } : {},
+        ...!image && imageBase64 ? { imageBase64 } : {},
+        referenceImage,
+        sessionId,
+        ...engine ? { engine } : {}
+      }),
+      BENCH_RUN_TIMEOUT_MS,
+      "runPipelineTest"
+    );
   }
   /**
    * Batched pipeline tester.
@@ -15375,13 +15560,17 @@ var BenchmarkAddon = class extends import_types4.BaseAddon {
       timestamp: Date.now()
     }));
     const callStart = Date.now();
-    const out = await this.api.pipelineExecutor.runPipelineBatch.mutate({
-      nodeId,
-      steps: [...steps],
-      frames,
-      sessionId,
-      ...engine ? { engine } : {}
-    });
+    const out = await withHardTimeout(
+      this.api.pipelineExecutor.runPipelineBatch.mutate({
+        nodeId,
+        steps: [...steps],
+        frames,
+        sessionId,
+        ...engine ? { engine } : {}
+      }),
+      BENCH_RUN_TIMEOUT_MS,
+      "runPipelineBatchTest"
+    );
     const callMs = Date.now() - callStart;
     const inferMs = out.results.reduce((acc, r) => Math.max(acc, r.debug?.totalInferenceMs ?? 0), 0);
     return {
@@ -15526,6 +15715,133 @@ var BenchmarkAddon = class extends import_types4.BaseAddon {
       errors
     };
   }
+  // ── Sustained throughput (root-detection e2e) ────────────────────────────
+  /**
+   * SUSTAINED THROUGHPUT test of ROOT detection.
+   *
+   * 1. Pin ONE frame in memory: resolve `frameId` (from `cacheBenchFrame`)
+   *    or the one-shot `imageBase64`, decode + square-resize ONCE to raw
+   *    RGB. Every node + every iteration consumes the IDENTICAL buffer — no
+   *    per-iteration decode or upload.
+   * 2. For each target node, run a `concurrency`-bounded loop of
+   *    `runPipelineBatch` (batchSize frames/call) on the node's ROOT-ONLY
+   *    steps until `durationSec` elapses (or the stop flag fires). Root-only
+   *    steps stay on the batch single-root fast path == live `plane:'frame'`
+   *    root detection.
+   * 3. Aggregate sustained fps / det/s + latency percentiles per node on the
+   *    identical pinned frame for apples-to-apples comparison.
+   *
+   * Non-destructive: targets the benchmark's own cached frame (never a live
+   * camera session), is explicitly started, and is bounded by N + the stop
+   * button (`abortSustainedThroughput`).
+   */
+  async runSustainedThroughput(input) {
+    const { frameId, imageBase64, durationSec, batchSize, concurrency, sessionId, nodes } = input;
+    const targetSize = input.targetSize ?? 640;
+    const cached2 = frameId ? this.frameCache.get(frameId) : void 0;
+    if (frameId && !cached2)
+      throw new Error(
+        `runSustainedThroughput: frameId ${frameId} not in cache (expired or never cached)`
+      );
+    const jpegSource = cached2 ? cached2.jpeg : imageBase64 ? Buffer.from(imageBase64, "base64") : null;
+    if (!jpegSource)
+      throw new Error("runSustainedThroughput: no frame source (need frameId or imageBase64)");
+    const sharp = (await import("sharp")).default;
+    const rgb = await sharp(jpegSource).resize({ width: targetSize, height: targetSize, fit: "fill" }).raw().toColorspace("srgb").toBuffer();
+    const rgbBytes = new Uint8Array(rgb.buffer, rgb.byteOffset, rgb.byteLength);
+    const batchFrames = Array.from({ length: batchSize }, () => ({
+      data: rgbBytes,
+      format: "rgb",
+      width: targetSize,
+      height: targetSize,
+      timestamp: Date.now()
+    }));
+    const benchFrameId = Math.floor(Math.random() * 4294967294) + 1 >>> 0;
+    const abortFlag = { aborted: false };
+    this.sustainedAbortFlags.set(sessionId, abortFlag);
+    this.ctx.logger.info("runSustainedThroughput start", {
+      meta: {
+        sessionId,
+        durationSec,
+        batchSize,
+        concurrency,
+        targetSize,
+        nodeCount: nodes.length,
+        nodes: nodes.map((n) => n.nodeId ?? "hub")
+      }
+    });
+    try {
+      const perNode = await Promise.all(
+        nodes.map(async (node) => {
+          if (node.steps.length === 0)
+            throw new Error(
+              `runSustainedThroughput: node ${node.nodeId ?? "hub"} has no root steps`
+            );
+          const dispatchBatch = async () => {
+            const out = await this.api.pipelineExecutor.runPipelineBatch.mutate({
+              nodeId: node.nodeId,
+              steps: [...node.steps],
+              frames: [...batchFrames],
+              sessionId,
+              frameId: benchFrameId,
+              ...node.engine ? { engine: node.engine } : {}
+            });
+            return {
+              frameCount: out.results.length,
+              detectionCounts: out.results.map((r) => r.detections.length)
+            };
+          };
+          let raw;
+          try {
+            raw = await runSustainedLoop({
+              dispatchBatch,
+              concurrency,
+              durationMs: durationSec * 1e3,
+              now: () => Date.now(),
+              shouldStop: () => abortFlag.aborted
+            });
+          } finally {
+            await this.api.pipelineExecutor.uncacheFrame.mutate({ nodeId: node.nodeId, frameId: benchFrameId }).catch(() => {
+            });
+          }
+          const summary = {
+            nodeId: node.nodeId ?? null,
+            ...node.label ? { label: node.label } : {},
+            totalFrames: raw.totalFrames,
+            totalBatches: raw.totalBatches,
+            wallMs: raw.wallMs,
+            fps: computeFps(raw.totalFrames, raw.wallMs),
+            detsPerSec: computeDetsPerSec(raw.totalDetections, raw.wallMs),
+            avgDetections: computeAvgDetections(raw.totalDetections, raw.detectionSamples),
+            latency: computeLatencyStats(raw.callMsList),
+            errors: raw.errors,
+            aborted: raw.aborted
+          };
+          this.ctx.logger.info("runSustainedThroughput node done", {
+            meta: {
+              nodeId: summary.nodeId,
+              fps: Number(summary.fps.toFixed(2)),
+              detsPerSec: Number(summary.detsPerSec.toFixed(2)),
+              p95Ms: Number(summary.latency.p95.toFixed(2)),
+              totalFrames: summary.totalFrames,
+              errors: summary.errors,
+              aborted: summary.aborted
+            }
+          });
+          return summary;
+        })
+      );
+      return { perNode, durationSec, batchSize, concurrency };
+    } finally {
+      this.sustainedAbortFlags.delete(sessionId);
+    }
+  }
+  async abortSustainedThroughput(input) {
+    const flag = this.sustainedAbortFlags.get(input.sessionId);
+    if (!flag) return { aborted: false };
+    flag.aborted = true;
+    return { aborted: true };
+  }
   // ── Decoder perf — fixture discovery + run orchestration ────────────────
   async listDecoderFixtures() {
     const dir = resolveFixturesDir();