@camstack/addon-pipeline 0.1.18 → 0.1.20

package/dist/pipeline-runner/index.mjs

@@ -1,4 +1,4 @@
-import { e as errMsg, o as object, s as string, _ as _enum, l as lazy, a as array, b as boolean, n as number, d as defineCustomActions, c as customAction, B as BaseAddon, E as EventCategory, p as pipelineRunnerCapability, f as createEvent } from "../index-asZs8U_s.mjs";
+import { e as errMsg, o as object, s as string, _ as _enum, l as lazy, a as array, b as boolean, n as number, d as defineCustomActions, c as customAction, B as BaseAddon, E as EventCategory, p as pipelineRunnerCapability, m as makeSourceBrokerId, f as createEvent } from "../index-5aYef068.mjs";
 import { FrameRingReaderCache } from "@camstack/shm-ring";
 class FrameQueue {
   /**
@@ -113,9 +113,9 @@ class PipelineTimingSampler {
     if (!this.detSamples.has(deviceId)) this.detSamples.set(deviceId, []);
     this.detSamples.get(deviceId).push(s);
   }
-  addMotionSample(deviceId, ms) {
+  addMotionSample(deviceId, ms, frameAge = -1) {
     if (!this.motSamples.has(deviceId)) this.motSamples.set(deviceId, []);
-    this.motSamples.get(deviceId).push(ms);
+    this.motSamples.get(deviceId).push({ ms, frameAge });
   }
   addAudioSample(deviceId, s) {
     if (!this.audioSamples.has(deviceId)) this.audioSamples.set(deviceId, []);
@@ -140,6 +140,7 @@ class PipelineTimingSampler {
       if (det.length === 0) continue;
       const e2e = det.map((s) => s.endToEnd);
       const inf = det.map((s) => s.inference);
+      const frameAge = det.map((s) => s.frameAge).filter((v) => v >= 0);
       const totalDet = det.reduce((s, d) => s + d.detections, 0);
       this.log.info(
         "pipeline stats",
@@ -148,7 +149,20 @@ class PipelineTimingSampler {
           meta: {
             frames: det.length,
             intervalSec: REPORT_INTERVAL_MS / 1e3,
+            // enqueue → emit, in ms. Stage breakdown (avg) exposes WHERE the
+            // latency sits: queue backlog vs semaphore contention vs inference
+            // vs result-emit. inference is usually the floor (model chain).
             e2e: { avg: avg(e2e), p95: p95(e2e), max: max(e2e) },
+            // Frame age (capture→inference-pick): if large, the analyzed frame is
+            // already stale (decoder/ring behind), which delays the overlay
+            // regardless of how fast the result is delivered.
+            frameAge: { avg: avg(frameAge), p95: p95(frameAge), max: max(frameAge) },
+            stagesMs: {
+              queueWait: avg(det.map((s) => s.queueWait)),
+              semaphoreWait: avg(det.map((s) => s.semaphoreWait)),
+              inference: avg(inf),
+              resultToEmit: avg(det.map((s) => s.resultToEmit))
+            },
             inference: { avg: avg(inf), p95: p95(inf) },
             detections: totalDet,
             dropped,
@@ -161,6 +175,8 @@ class PipelineTimingSampler {
     this.detSamples.clear();
     for (const [deviceId, mot] of this.motSamples) {
       if (mot.length === 0) continue;
+      const ms = mot.map((s) => s.ms);
+      const frameAge = mot.map((s) => s.frameAge).filter((v) => v >= 0);
       this.log.info(
         "motion stats",
         {
@@ -168,10 +184,12 @@ class PipelineTimingSampler {
           meta: {
             frames: mot.length,
             intervalSec: REPORT_INTERVAL_MS / 1e3,
-            avg: avg(mot),
-            p95: p95(mot),
-            max: max(mot)
-            // motionAddon: rt.motionAddon ?? null,
+            avg: avg(ms),
+            p95: p95(ms),
+            max: max(ms),
+            // Frame age at motion analysis (capture→analysis). Large = stale
+            // input frame (decoder/ring behind) → motion box lags real movement.
+            frameAge: { avg: avg(frameAge), p95: p95(frameAge), max: max(frameAge) }
           }
         }
       );
@@ -517,8 +535,7 @@ class PipelineRunner {
   async processWithSemaphore(deviceId, entry, frameInput, state, streamType) {
     const pickedAt = Date.now();
     const { frame, handle } = entry;
-    const captureTs = frame.timestamp;
-    const enqueuedAt = frame._enqueuedAt ?? captureTs;
+    const enqueuedAt = frame._enqueuedAt ?? pickedAt;
     const release = await this.semaphore.acquire();
     const semAcquiredAt = Date.now();
     try {
@@ -533,13 +550,19 @@ class PipelineRunner {
       if (result) {
         await this.notifyCallbacks(deviceId, frame, result, streamType, handle);
         const emittedAt = Date.now();
+        const capturedAt = frame.capturedAt;
         this.timingSampler.addSample(deviceId, {
-          captureToEnqueue: enqueuedAt - captureTs,
           queueWait: pickedAt - enqueuedAt,
           semaphoreWait: semAcquiredAt - pickedAt,
           inference: inferenceMs,
           resultToEmit: emittedAt - inferDoneAt,
-          endToEnd: emittedAt - captureTs,
+          // capturedAt is the shm-ring commit wall-clock; pickedAt − capturedAt
+          // is how stale the frame was when inference started. <0/absent → unknown.
+          frameAge: typeof capturedAt === "number" && capturedAt > 0 ? pickedAt - capturedAt : -1,
+          // Wall-clock pipeline latency: enqueue → result emitted. (Capture →
+          // enqueue, i.e. RTSP decode + shm-ring write, is not measured here —
+          // the shm frame carries only a PTS, no wall-clock capture stamp.)
+          endToEnd: emittedAt - enqueuedAt,
           detections: result.detections?.length ?? 0
         });
       }
@@ -583,36 +606,87 @@ class PipelineRunner {
 const PULL_MAX_COUNT = 4;
 const MIN_POLL_INTERVAL_MS = 20;
 const FALLBACK_POLL_INTERVAL_MS = 200;
+const INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS = 250;
+const MAX_SUBSCRIBE_RETRY_BACKOFF_MS = 5e3;
 async function startFrameHandlePoller(options) {
-  const { api, brokerId, format, maxFps, tag, onFrame, logger } = options;
-  let result;
-  try {
-    result = await api.streamBroker.subscribeFrames.mutate({
-      brokerId,
-      format,
-      maxFps,
-      tag
-    });
-  } catch (err) {
-    logger.warn("frame-handle poller: subscribeFrames failed", {
-      meta: { brokerId, format, tag, error: errMsg(err) }
-    });
-    return null;
+  const lifecycle = {
+    stopped: false,
+    retryTimer: void 0,
+    activeTeardown: null
+  };
+  const teardown = () => {
+    if (lifecycle.stopped) return;
+    lifecycle.stopped = true;
+    if (lifecycle.retryTimer) {
+      clearTimeout(lifecycle.retryTimer);
+      lifecycle.retryTimer = void 0;
+    }
+    lifecycle.activeTeardown?.();
+  };
+  void subscribeWithRetry(options, lifecycle);
+  return teardown;
+}
+async function subscribeWithRetry(options, lifecycle) {
+  const { api, brokerId, format, maxFps, tag, logger } = options;
+  let backoffMs = INITIAL_SUBSCRIBE_RETRY_BACKOFF_MS;
+  let attempt = 0;
+  while (!lifecycle.stopped) {
+    attempt += 1;
+    try {
+      const result = await api.streamBroker.subscribeFrames.mutate({
+        brokerId,
+        format,
+        maxFps,
+        tag
+      });
+      if (lifecycle.stopped) {
+        await api.streamBroker.unsubscribeFrames.mutate({ subscriptionId: result.subscriptionId }).catch((err) => {
+          logger.warn("frame-handle poller: late unsubscribe failed", {
+            meta: { brokerId, subscriptionId: result.subscriptionId, error: errMsg(err) }
+          });
+        });
+        return;
+      }
+      lifecycle.activeTeardown = startPolling(options, result.subscriptionId, result.maxFps, lifecycle);
+      return;
+    } catch (err) {
+      if (lifecycle.stopped) return;
+      if (attempt === 1) {
+        logger.warn("frame-handle poller: subscribeFrames failed, retrying", {
+          meta: { brokerId, format, tag, error: errMsg(err), nextRetryInMs: backoffMs }
+        });
+      } else {
+        logger.debug("frame-handle poller: subscribeFrames still failing", {
+          meta: { brokerId, format, tag, attempt, error: errMsg(err), nextRetryInMs: backoffMs }
+        });
+      }
+      await sleep(backoffMs, lifecycle);
+      backoffMs = Math.min(MAX_SUBSCRIBE_RETRY_BACKOFF_MS, backoffMs * 2);
+    }
   }
-  const { subscriptionId } = result;
+}
+function sleep(ms, lifecycle) {
+  return new Promise((resolve) => {
+    lifecycle.retryTimer = setTimeout(() => {
+      lifecycle.retryTimer = void 0;
+      resolve();
+    }, ms);
+  });
+}
+function startPolling(options, subscriptionId, resolvedMaxFps, lifecycle) {
+  const { api, brokerId, onFrame, logger } = options;
   const readers = new FrameRingReaderCache(logger);
-  const pollIntervalMs = result.maxFps > 0 ? Math.max(MIN_POLL_INTERVAL_MS, Math.round(1e3 / result.maxFps)) : FALLBACK_POLL_INTERVAL_MS;
-  let stopped = false;
+  const pollIntervalMs = resolvedMaxFps > 0 ? Math.max(MIN_POLL_INTERVAL_MS, Math.round(1e3 / resolvedMaxFps)) : FALLBACK_POLL_INTERVAL_MS;
   let timer;
   const tick = async () => {
-    if (stopped) return;
+    if (lifecycle.stopped) return;
     try {
       const handles = await api.streamBroker.pullFrameHandles.query({
         subscriptionId,
         maxCount: PULL_MAX_COUNT
       });
       for (const handle of handles) {
-        if (stopped) break;
+        if (lifecycle.stopped) break;
         const frame = readers.read(handle);
         if (frame) onFrame(frame, handle);
       }
@@ -621,14 +695,12 @@ async function startFrameHandlePoller(options) {
         meta: { brokerId, subscriptionId, error: errMsg(err) }
       });
     }
-    if (!stopped) {
+    if (!lifecycle.stopped) {
       timer = setTimeout(() => void tick(), pollIntervalMs);
     }
   };
   void tick();
   return () => {
-    if (stopped) return;
-    stopped = true;
     if (timer) {
       clearTimeout(timer);
       timer = void 0;
@@ -730,6 +802,11 @@ const DEFAULT_CONFIG = {
   targetLoadPercent: 80,
   minThrottledFps: 1
 };
+function shouldStartOnboardAnalyzer(config) {
+  if (!config.onboardMotionDrivesAnalyzer) return false;
+  if (config.motionSources.includes("analyzer")) return false;
+  return true;
+}
 function toFrameInput(frame) {
   return {
     data: frame.data,
@@ -739,14 +816,12 @@ function toFrameInput(frame) {
     timestamp: frame.timestamp
   };
 }
-const STEP_LOG_INTERVAL_MS = 3e4;
 const METRICS_SNAPSHOT_INTERVAL_MS = 1e3;
 const METRICS_HEARTBEAT_MS = 3e4;
 class PipelineRunnerAddon extends BaseAddon {
   runner = null;
   attached = /* @__PURE__ */ new Map();
   nodeId = "unknown";
-  stepLogTimer = null;
   metricsSnapshotTimer = null;
   unsubMotionEvents = null;
   /** Last analyzer-detected state per device — gates the
@@ -760,6 +835,20 @@ class PipelineRunnerAddon extends BaseAddon {
    * detach.
    */
   lastMotionAt = /* @__PURE__ */ new Map();
+  /**
+   * Dynamic analyzer subscriptions opened on `MotionOnMotionChanged
+   * source:'onboard'` when `onboardMotionDrivesAnalyzer === true`. Each
+   * entry is the unsubscribe handle returned by `subscribeMotionFrames`.
+   * Cleared on teardown timer fire, detach, and shutdown.
+   */
+  onboardAnalyzerSubs = /* @__PURE__ */ new Map();
+  /**
+   * Teardown timers that close the dynamic analyzer subscription after
+   * `motionCooldownMs` without a new motion event. Re-armed on every
+   * `MotionOnMotionChanged source:'onboard'` call so the sub stays open
+   * while motion persists.
+   */
+  onboardAnalyzerTeardownTimers = /* @__PURE__ */ new Map();
   /**
    * Snapshot-equality cache for metrics-snapshot defer. The runner
    * fires per-camera metrics every `METRICS_SNAPSHOT_INTERVAL_MS`;
@@ -824,6 +913,9 @@ class PipelineRunnerAddon extends BaseAddon {
           const attachment = this.attached.get(deviceId);
           if (!attachment) return;
           const source = data.source;
+          if (source === "onboard") {
+            void this.handleOnboardMotionAnalyzer(deviceId, data.detected);
+          }
           if (!attachment.config.motionSources.includes(source)) return;
           this.runner?.reportMotion(
             deviceId,
@@ -834,7 +926,6 @@ class PipelineRunnerAddon extends BaseAddon {
         }
       );
     }
-    this.stepLogTimer = setInterval(() => this.logAttachedSteps(), STEP_LOG_INTERVAL_MS);
     this.metricsSnapshotTimer = setInterval(
       () => this.emitMetricsSnapshot(),
       METRICS_SNAPSHOT_INTERVAL_MS
@@ -854,10 +945,6 @@ class PipelineRunnerAddon extends BaseAddon {
       clearInterval(this.metricsSnapshotTimer);
       this.metricsSnapshotTimer = null;
     }
-    if (this.stepLogTimer) {
-      clearInterval(this.stepLogTimer);
-      this.stepLogTimer = null;
-    }
     if (this.benchFrameSweeper) {
       clearInterval(this.benchFrameSweeper);
       this.benchFrameSweeper = null;
@@ -868,6 +955,9 @@ class PipelineRunnerAddon extends BaseAddon {
       this.unsubMotionEvents = null;
     }
     this.lastAnalyzerDetected.clear();
+    for (const deviceId of [...this.onboardAnalyzerTeardownTimers.keys(), ...this.onboardAnalyzerSubs.keys()]) {
+      this.clearOnboardAnalyzer(deviceId);
+    }
     if (this.runner) {
       this.runner.stop();
       this.runner = null;
@@ -1254,54 +1344,10 @@ class PipelineRunnerAddon extends BaseAddon {
     this.runner?.reportMotion(input.deviceId, input.detected, input.source, input.regions);
     return { success: true };
   }
-  /**
-   * Periodic per-camera step roster dump. Once every
-   * STEP_LOG_INTERVAL_MS (30s) emits one log line per attached camera
-   * with the configured detection step tree + audio classifier branch
-   * so an operator looking at the agent log can quickly see what each
-   * camera is currently running without crossing tRPC. Skips when no
-   * cameras are attached so quiet dev runs stay silent.
-   */
-  logAttachedSteps() {
-    if (this.attached.size === 0) return;
-    for (const [deviceId, attachment] of this.attached) {
-      const cfg = attachment.config;
-      const detectionSteps = cfg.steps && cfg.steps.length > 0 ? this.flattenSteps(cfg.steps).filter((s) => s.enabled) : [];
-      const detectionLabel = detectionSteps.length > 0 ? detectionSteps.map((s) => `${s.addonId}/${s.modelId}`).join(" → ") : "<none>";
-      const audioLabel = cfg.audio && cfg.audio.enabled ? `${cfg.audio.engine.runtime}/${cfg.audio.engine.backend}/${cfg.audio.modelId}` : "<off>";
-      const engineLabel = cfg.engine ? `${cfg.engine.runtime}/${cfg.engine.backend}${cfg.engine.device ? `/${cfg.engine.device}` : ""}` : "<unset>";
-      this.ctx.logger.info("Camera pipeline roster", {
-        tags: { deviceId },
-        meta: {
-          phase: "roster",
-          intervalSec: STEP_LOG_INTERVAL_MS / 1e3,
-          pipelineEnabled: cfg.pipelineEnabled,
-          motionSources: cfg.motionSources,
-          motionFps: cfg.motionFps,
-          detectionFps: cfg.detectionFps,
-          engine: engineLabel,
-          videoSteps: detectionLabel,
-          videoStepCount: detectionSteps.length,
-          audio: audioLabel
-        }
-      });
-    }
-  }
-  /** Recursively flatten the step tree → ordered list of every node. */
-  flattenSteps(steps) {
-    const out = [];
-    const walk = (s) => {
-      out.push(s);
-      if (s.children) {
-        for (const c of s.children) walk(c);
-      }
-    };
-    for (const s of steps) walk(s);
-    return out;
-  }
   detachInternal(deviceId) {
     const attachment = this.attached.get(deviceId);
     if (!attachment) return;
+    this.clearOnboardAnalyzer(deviceId);
     attachment.motionUnsubscribe?.();
     attachment.detectionUnsubscribe?.();
     this.attached.delete(deviceId);
@@ -1310,6 +1356,70 @@ class PipelineRunnerAddon extends BaseAddon {
     this.runner?.unregisterCamera(deviceId);
     this.ctx?.logger.info("detachCamera", { tags: { deviceId } });
   }
+  /**
+   * Synchronously cancel the teardown timer and call the unsubscribe
+   * handle for the dynamic onboard analyzer, if one is open. Safe to
+   * call when no subscription exists.
+   */
+  clearOnboardAnalyzer(deviceId) {
+    const timer = this.onboardAnalyzerTeardownTimers.get(deviceId);
+    if (timer !== void 0) {
+      clearTimeout(timer);
+      this.onboardAnalyzerTeardownTimers.delete(deviceId);
+    }
+    const unsub = this.onboardAnalyzerSubs.get(deviceId);
+    if (unsub !== void 0) {
+      try {
+        unsub();
+      } catch {
+      }
+      this.onboardAnalyzerSubs.delete(deviceId);
+    }
+  }
+  /**
+   * Dynamic analyzer gate for onboard-motion cameras.
+   *
+   * Called from the `MotionOnMotionChanged` subscriber whenever
+   * `source === 'onboard'`. Opens a `subscribeMotionFrames` subscription
+   * the first time motion is detected (idempotent — a second `detected:true`
+   * while the sub is already open is a no-op). Always re-arms the teardown
+   * timer so the subscription stays open as long as motion events keep
+   * arriving and tears down `motionCooldownMs` after the last event.
+   *
+   * No-op when:
+   * - The camera is not currently attached.
+   * - `shouldStartOnboardAnalyzer(config)` returns false (flag off or
+   *   `motionSources` already includes `'analyzer'`).
+   */
+  async handleOnboardMotionAnalyzer(deviceId, detected) {
+    const attachment = this.attached.get(deviceId);
+    if (!attachment) return;
+    const config = attachment.config;
+    if (!shouldStartOnboardAnalyzer(config)) return;
+    const log = this.ctx.logger.withTags({ deviceId });
+    if (detected && !this.onboardAnalyzerSubs.has(deviceId)) {
+      const unsub = await this.subscribeMotionFrames(config);
+      if (unsub) {
+        this.onboardAnalyzerSubs.set(deviceId, unsub);
+        log.debug("onboard-analyzer: opened motion-frame subscription");
+      }
+    }
+    const existing = this.onboardAnalyzerTeardownTimers.get(deviceId);
+    if (existing !== void 0) clearTimeout(existing);
+    const cooldownMs = config.motionCooldownMs;
+    const timer = setTimeout(() => {
+      this.onboardAnalyzerTeardownTimers.delete(deviceId);
+      const unsub = this.onboardAnalyzerSubs.get(deviceId);
+      if (!unsub) return;
+      try {
+        unsub();
+      } catch {
+      }
+      this.onboardAnalyzerSubs.delete(deviceId);
+      log.debug("onboard-analyzer: closed after motion cooldown", { meta: { cooldownMs } });
+    }, cooldownMs);
+    this.onboardAnalyzerTeardownTimers.set(deviceId, timer);
+  }
   async getLocalLoad() {
     const metrics = this.runner?.getMetrics() ?? { avgInferenceTimeMs: 0, queueDepth: 0 };
     const allCameraMetrics = this.runner?.getAllCameraMetrics() ?? [];
@@ -1358,7 +1468,7 @@ class PipelineRunnerAddon extends BaseAddon {
     }
     return startFrameHandlePoller({
       api,
-      brokerId: `${config.deviceId}/${config.motionStreamId}`,
+      brokerId: makeSourceBrokerId(config.deviceId, config.motionStreamId),
       format: "gray",
       maxFps: config.motionFps,
       tag: "motion",
@@ -1444,7 +1554,7 @@ class PipelineRunnerAddon extends BaseAddon {
     }
     return startFrameHandlePoller({
       api,
-      brokerId: `${config.deviceId}/${config.detectionStreamId}`,
+      brokerId: makeSourceBrokerId(config.deviceId, config.detectionStreamId),
       format: "rgb",
       maxFps: config.detectionFps,
       tag: "detection",
@@ -1566,20 +1676,24 @@ class PipelineRunnerAddon extends BaseAddon {
           zonesPayload
         ));
       }
-      runner.timingSampler.addMotionSample(deviceId, Date.now() - motionStart);
+      const capturedAt = frame.capturedAt;
+      const motionFrameAge = typeof capturedAt === "number" && capturedAt > 0 ? motionStart - capturedAt : -1;
+      runner.timingSampler.addMotionSample(deviceId, Date.now() - motionStart, motionFrameAge);
     } catch (error) {
       const msg = errMsg(error);
       log.error("runMotionAnalysis failed", { meta: { error: msg } });
     }
   }
-  emitInferenceResult(deviceId, _frame, result, handle) {
+  emitInferenceResult(deviceId, frame, result, handle) {
     const ctx = this.ctx;
     if (!ctx?.eventBus) return;
+    const capturedAt = frame.capturedAt;
     const payload = {
       deviceId,
       frame: result,
       nodeId: this.nodeId,
-      frameHandle: handle
+      frameHandle: handle,
+      ...typeof capturedAt === "number" && capturedAt > 0 ? { capturedAt } : {}
     };
     this.ctx.eventBus.emit(createEvent(
       EventCategory.PipelineInferenceResult,
@@ -1720,6 +1834,7 @@ export {
   PipelineTimingSampler,
   Semaphore,
   pipelineRunnerBenchActions as customActions,
-  PipelineRunnerAddon as default
+  PipelineRunnerAddon as default,
+  shouldStartOnboardAnalyzer
 };
 //# sourceMappingURL=index.mjs.map