@camstack/server 0.1.3

package/src/core/moleculer/moleculer.service.ts

@@ -0,0 +1,612 @@
+import { ServiceBroker, type Service, type ServiceSchema } from 'moleculer'
+
+/**
+ * Narrow-typed view of the Moleculer surface this file actually uses.
+ * Moleculer's published `.d.ts` chains through `eventemitter2` whose
+ * package.json has no `types` field; typescript-eslint's `no-unsafe-*`
+ * rules flag every `broker.X` access. Casting once at the boundary
+ * collapses the rule violations into one documented narrowing.
+ */
+interface BrokerLike {
+  readonly nodeID: string
+  readonly logger: Record<string, unknown>
+  start(): Promise<void>
+  stop(): Promise<void>
+  createService(svc: Service | unknown): unknown
+  call(action: string, params?: unknown, opts?: unknown): Promise<unknown>
+  waitForServices(services: string[], timeout?: number): Promise<unknown>
+}
+import { createBroker, createHubService, createProcessService, isInfraCapability, registerEventBusService, createReadinessServiceForRegistry, createStreamProbeBrokerService, createHwAccelService, createKernelHwAccel, HubNodeRegistry, serializeTypedArrays, callWithServiceDiscovery, hashClusterSecret } from '@camstack/kernel'
+import type { HubServiceDeps, CallFn, RegisterNodeParams, RegisteredAddonManifest } from '@camstack/kernel'
+import { EventCategory, expandCapMethods, ReadinessRegistry, emitReadiness } from '@camstack/types'
+import type { CapabilityDefinition } from '@camstack/types'
+import { randomUUID } from 'node:crypto'
+import { EventBusService } from '../events/event-bus.service'
+import { ConfigService } from '../config/config.service'
+import { LoggingService } from '../logging/logging.service'
+import { CapabilityService } from '../capability/capability.service'
+import { StreamProbeService } from '../streaming/stream-probe.service'
+
+/**
+ * Narrow-typed view of the Moleculer broker surface used by the
+ * `$node.disconnected` listener — extends `BrokerLike` with `localBus`
+ * so the handler can be fully typed (same pattern as agent-registry.service.ts).
+ */
+interface BrokerWithLocalBus {
+  localBus: {
+    on(event: string, handler: (payload: { node: { id: string } }) => void): void
+  }
+}
+
+/**
+ * One `(addon, capability)` pair that a node manifest applies onto the
+ * CapabilityRegistry, with its resolved `CapabilityDefinition`. Built by
+ * `applyNodeManifest`'s diff so the register loop never re-resolves the
+ * cap def. Keyed in the diff map by `` `${registryKey}::${capName}` ``.
+ */
+interface AppliedCapEntry {
+  readonly addonId: string
+  readonly capName: string
+  readonly capDef: CapabilityDefinition
+}
+
+export class MoleculerService {
+  readonly broker: ServiceBroker
+  /** Narrow-typed view of `this.broker` — see `BrokerLike` doc above. */
+  private get brokerSafe(): BrokerLike { return this.broker as unknown as BrokerLike }
+  private readonly logger: ReturnType<LoggingService['createLogger']>
+  /**
+   * D3 authority: union of every node's manifest delivered via
+   * `$hub.registerNode`. Populated by `onRegisterNode` in `hubDeps`.
+   */
+  private readonly nodeRegistry = new HubNodeRegistry()
+  private readonly nodeCallFns = new Map<string, CallFn>()
+  /**
+   * D3: callback invoked whenever a bare-ID agent node completes the
+   * `$hub.registerNode` handshake. Registered by `AgentRegistryService`
+   * via `setOnAgentRegistered()`. The handshake is the authoritative
+   * completeness signal — the hub has the full manifest at this point
+   * and reconciliation can run immediately (no grace delay needed).
+   */
+  private onAgentRegisteredCb: ((nodeId: string) => void) | null = null
+  /**
+   * Hub-side authoritative readiness registry. Subscribed to the
+   * shared `EventBusService` so it ingests both hub-local emits and
+   * remote emits forwarded via `$hub.event`. Exposed to:
+   *  - the `$readiness.getSnapshot` Moleculer action (consumed by
+   *    workers / agents on boot)
+   *  - `ctx.kernel.readinessRegistry` on every hub addon context so
+   *    hub consumers share the same snapshot.
+   */
+  readonly readinessRegistry: ReadinessRegistry
+  /**
+   * Resolved cluster secret (`CAMSTACK_CLUSTER_SECRET` env, else
+   * `cluster.secret` config), or `undefined` when none is configured.
+   * Threaded both into the broker factory and the hub's
+   * `expectedClusterSecretHash` so `$hub.registerNode` can gate on it.
+   */
+  private readonly clusterSecret: string | undefined
+
+  constructor(
+    private readonly eventBus: EventBusService,
+    private readonly config: ConfigService,
+    private readonly logging: LoggingService,
+    private readonly capabilityService: CapabilityService,
+    private readonly streamProbe: StreamProbeService,
+  ) {
+    this.logger = this.logging.createLogger('moleculer')
+    // Optional port overrides. Live primarily for the e2e harness: when
+    // a developer's dev:full is up on the default 6000/4445 ports, an
+    // isolated test hub can't reuse them. Production keeps the defaults
+    // so cluster discovery + agent-to-hub connections keep working
+    // without per-deploy config.
+    const tcpPortEnv = process.env['CAMSTACK_HUB_TCP_PORT']
+    const udpPortEnv = process.env['CAMSTACK_HUB_UDP_PORT']
+    const tcpPort = tcpPortEnv ? Number(tcpPortEnv) : undefined
+    const udpPort = udpPortEnv ? Number(udpPortEnv) : undefined
+    // Two-step cast: createBroker's dist `.d.ts` chains through
+    // moleculer→eventemitter2 whose types are unresolvable at this
+    // boundary, so the inference falls to `error` and trips
+    // `no-unsafe-assignment`. Going via `unknown` documents the boundary.
+    this.clusterSecret = process.env['CAMSTACK_CLUSTER_SECRET'] ?? this.config.get<string>('cluster.secret')
+    const broker = createBroker({
+      nodeID: 'hub',
+      mode: 'hub',
+      logLevel: this.config.get<string>('moleculer.logLevel') ?? 'warn',
+      secret: this.clusterSecret,
+      ...(tcpPort && !Number.isNaN(tcpPort) ? { tcpPort } : {}),
+      ...(udpPort && !Number.isNaN(udpPort) ? { udpPort } : {}),
+    }) as unknown
+    // `ServiceBroker` itself surfaces as `error`-typed at this boundary
+    // (eventemitter2 chain unresolvable). Documented + single-site cast.
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+    this.broker = broker as ServiceBroker
+    this.readinessRegistry = new ReadinessRegistry({
+      eventBus: this.eventBus,
+      sourceNodeId: this.brokerSafe.nodeID,
+      logger: this.logging.createLogger('readiness'),
+    })
+  }
+
+  /**
+   * D3: register the callback that fires when an agent node completes the
+   * `$hub.registerNode` handshake. Called by `AgentRegistryService` during
+   * its own `onModuleInit` — after `MoleculerService.onModuleInit` has
+   * returned, so the broker is already live. Option (b) direct-callback
+   * wiring: no event-bus round-trip needed for internal core wiring.
+   */
+  setOnAgentRegistered(cb: (nodeId: string) => void): void {
+    this.onAgentRegisteredCb = cb
+  }
+
+  async onModuleInit(): Promise<void> {
+    const logger = this.logging.createLogger('moleculer')
+
+    const hubDeps: HubServiceDeps = {
+      getAddonConfig: (addonId) => {
+        return this.config.getAddonConfig(addonId)
+      },
+      getSettings: (scope, key) => {
+        return this.config.get(key ? `${scope}.${key}` : scope)
+      },
+      getRecentEvents: (category, limit) => {
+        return this.eventBus.getRecent(category ? { category } : undefined, limit)
+      },
+      onLog: (entry) => {
+        this.logging.writeFromWorker({
+          addonId: entry.addonId,
+          nodeId: entry.nodeId,
+          level: entry.level,
+          message: entry.message,
+          ...(entry.scope !== undefined ? { scope: entry.scope } : {}),
+          ...(entry.tags ? { tags: entry.tags } : {}),
+          ...(entry.meta ? { meta: entry.meta } : {}),
+        })
+      },
+      onSetLogLevel: (level) => {
+        const factory = this.brokerSafe.logger
+        const appenders = factory['appenders'] as Array<{ opts: { level: string } }> | undefined
+        if (appenders) {
+          for (const appender of appenders) {
+            appender.opts.level = level
+          }
+        }
+        const cache = factory['cache'] as Map<string, unknown> | undefined
+        if (cache) {
+          cache.clear()
+        }
+        logger.info('Moleculer log level changed', { meta: { level } })
+        this.brokerSafe.call('$process.setLogLevel', { level }).catch(() => {})
+      },
+      // D3: registration-handshake path. Nodes send $hub.registerNode with
+      // their complete capability manifest; the hub applies it immediately.
+      onRegisterNode: (params) => {
+        this.onRegisterNode(params)
+      },
+      onUnregisterNode: (nodeId) => {
+        this.removeNodeFromRegistry(nodeId)
+      },
+      expectedClusterSecretHash: this.clusterSecret ? hashClusterSecret(this.clusterSecret) : undefined,
+    }
+
+    const hubService: unknown = createHubService(hubDeps)
+    this.brokerSafe.createService(hubService)
+
+    const dataDir = this.config.get<string>('dataDir') ?? 'camstack-data'
+    const processService: unknown = createProcessService(this.brokerSafe.nodeID, dataDir)
+    this.brokerSafe.createService(processService)
+
+    // $addonHost — REMOVED (Sprint 6). Three-level settings are now
+    // served by the `addon-settings` singleton capability. Per-addon
+    // Moleculer services expose `settings.*` actions for remote agents.
+
+    // D3: mirror $node.disconnected onto the registry path so nodes that
+    // sent a $hub.registerNode manifest get cleaned up on disconnect.
+    const bridgeBus = this.broker as unknown as BrokerWithLocalBus
+    bridgeBus.localBus.on('$node.disconnected', ({ node }: { node: { id: string } }) => {
+      this.removeNodeFromRegistry(node.id)
+    })
+
+    // Register the $event-bus service BEFORE broker.start(). Moleculer
+    // announces service subscriptions to remote nodes only during discovery
+    // handshake, not dynamically after post-start `createService()`.
+    // Without this, cross-node broadcasts (camstack.evt.<category>) would
+    // arrive unreliably for the first ~10s after each node joins.
+    registerEventBusService(this.broker)
+
+    // Register the hub-authoritative `$readiness.getSnapshot` service
+    // BEFORE broker.start() so workers / agents see it in their initial
+    // INFO packet — post-start `createService` calls propagate via
+    // heartbeat (several seconds) and would force workers to poll.
+    this.brokerSafe.createService(createReadinessServiceForRegistry(this.readinessRegistry))
+
+    // Register the hub-authoritative `$stream-probe` service —
+    // workers route RTSP probe + field-probe through this action when
+    // the tRPC WSS link isn't available (default path for forked
+    // workers). Keeps ffprobe + HTTP-reachability as a single
+    // hub-side implementation; see `createStreamProbeBrokerService`
+    // for the shape.
+    this.brokerSafe.createService(createStreamProbeBrokerService({
+      probe: (url, options) => this.streamProbe.probe(url, options),
+      probeField: (key, value) => this.streamProbe.probeField(key, value),
+    }))
+
+    // Register `$hwaccel` on hub — every node in the cluster does the
+    // same so `broker.call('$hwaccel.resolve', params, { nodeID })`
+    // returns the backend list for whichever host the caller targets.
+    // Admin UI uses this to show per-agent hwaccel info on the
+    // pipeline / NodeDetail pages.
+    this.brokerSafe.createService(createHwAccelService(createKernelHwAccel()))
+
+    await this.brokerSafe.start()
+    logger.info('Moleculer broker started (TCP transport)')
+
+    // Wire the hub's EventBusService into the broker so hub-addon
+    // emissions fan out to every remote process via
+    // `camstack.evt.<category>`, and incoming $event-bus events land on
+    // the same local bus that subscribers already use. The
+    // EventBusService's `emit` override handles the "only broadcast
+    // locally-originated events" guard, and id-based dedup absorbs the
+    // duplicate delivery when `createBrokerEventBus` on a remote uses
+    // both broadcast + `$hub.event` for back-compat.
+    this.eventBus.attachBroker(this.broker)
+  }
+
+  /**
+   * Register the log-receiver service for agent log forwarding.
+   * Must be called AFTER app.init() so Moleculer re-advertises
+   * the updated service list to the network.
+   */
+  registerLogReceiver(): void {
+    this.brokerSafe.createService({
+      name: 'log-receiver',
+      actions: {
+        ingest: {
+          handler: (ctx: {
+            params: {
+              level: string
+              message: string
+              addonId: string
+              nodeId: string
+              scope?: string
+              tags?: import('@camstack/types').LogTags
+              meta?: Record<string, unknown>
+            }
+          }) => {
+            this.logging.writeFromWorker({
+              addonId: ctx.params.addonId,
+              nodeId: ctx.params.nodeId,
+              level: ctx.params.level,
+              message: ctx.params.message,
+              ...(ctx.params.scope !== undefined ? { scope: ctx.params.scope } : {}),
+              ...(ctx.params.tags ? { tags: ctx.params.tags } : {}),
+              ...(ctx.params.meta ? { meta: ctx.params.meta } : {}),
+            })
+            return true
+          },
+        },
+      },
+    })
+  }
+
+  /**
+   * Register the `$core-caps` Moleculer service that bridges the hub's
+   * core (non-addon) tRPC routers onto the cluster mesh.
+   *
+   * Called from `main.ts` after the appRouter is built — that happens
+   * after `app.init()`, so the broker is already started. Post-start
+   * `createService` is fine: the service propagates to remote nodes via
+   * heartbeat and `brokerTransportLink` polls discovery, so forked
+   * addons and late-joining agents still resolve `ctx.api.<coreCap>`.
+   * `registerLogReceiver` relies on the same post-init registration.
+   */
+  registerCoreCapService(service: ServiceSchema): void {
+    this.brokerSafe.createService(service)
+  }
+
+  /**
+   * Call a capability method on a specific node.
+   * Hub: calls the local provider directly (same process, no Moleculer overhead).
+   * Remote: routes through the stored Moleculer CallFn for that node.
+   */
+  async callCapabilityOnNode(
+    nodeId: string,
+    capabilityName: string,
+    methodName: string,
+    params: unknown,
+  ): Promise<unknown> {
+    // Hub — call local provider directly when registered on hub root.
+    // For caps hosted in a per-addon runner (e.g. 'hub/detection-pipeline'),
+    // the hub root registry has nothing; we then fall through to the
+    // remote findCallFn prefix lookup which routes to the subnode.
+    if (nodeId === 'hub' || nodeId === this.brokerSafe.nodeID) {
+      const registry = this.capabilityService.getRegistry()
+      const provider = registry?.getSingleton(capabilityName) as Record<string, unknown> | null
+      if (provider) {
+        const fn = provider[methodName]
+        if (typeof fn !== 'function') throw new Error(`Method "${methodName}" not found on "${capabilityName}"`)
+        return (fn as (p: unknown) => unknown).call(provider, params)
+      }
+      // Fall through to the prefix-fallback findCallFn below.
+    }
+
+    // Remote — find stored callFn. The closure already captures the
+    // FULL nodeId at registration time (e.g. `dev-agent-1/detection-pipeline`
+    // for forkable addons). Passing the caller's `nodeId` here as the
+    // 3rd arg would override the closure's capture with the bare parent
+    // prefix (`dev-agent-1`), which Moleculer doesn't know about → the
+    // call fails with "Service not found on '<prefix>' node". Omit it
+    // so the closure routes to the correct worker node.
+    const callFn = this.findCallFn(nodeId, capabilityName)
+    if (callFn) {
+      return callFn(methodName, params)
+    }
+
+    throw new Error(`Capability "${capabilityName}" not available on node "${nodeId}"`)
+  }
+
+  /**
+   * D3 registration-handshake entrypoint — invoked by the `$hub.registerNode`
+   * Moleculer action (wired through `hubDeps.onRegisterNode`).
+   *
+   * Captures the node's PREVIOUS manifest before `nodeRegistry.registerNode`
+   * overwrites it, then hands both manifests to `applyNodeManifest` so the
+   * CapabilityRegistry update is a diff (atomic replace) rather than an
+   * unconditional re-register — see `applyNodeManifest` for the rationale.
+   */
+  private onRegisterNode(params: RegisterNodeParams): void {
+    const previousManifest = this.nodeRegistry.getNodeManifest(params.nodeId)
+    this.nodeRegistry.registerNode(params)
+    this.applyNodeManifest(params, previousManifest)
+    // Notify AgentRegistryService to reconcile placement for bare-ID
+    // agent nodes (no '/' = not a hub child worker, not the hub itself).
+    // The handshake is the authoritative completeness signal — the full
+    // manifest is available here so reconciliation runs without delay.
+    const { nodeId } = params
+    if (nodeId !== 'hub' && !nodeId.includes('/') && this.onAgentRegisteredCb) {
+      this.onAgentRegisteredCb(nodeId)
+    }
+  }
+
+  /**
+   * D3: apply a node's registered manifest onto the CapabilityRegistry.
+   * Builds a method proxy for each capability — same registryKey rule
+   * (local child → bare addonId, remote agent → addonId@nodeId), same
+   * `broker.call` routing shape, same `expandCapMethods` method surface.
+   *
+   * Called from `onRegisterNode` whenever a node handshakes.
+   *
+   * Diff-based / idempotent: the D3 protocol legitimately re-handshakes (a
+   * node re-sends its COMPLETE manifest — e.g. the post-device-restore
+   * `nativeCaps` re-handshake). `registerProvider` throws on a duplicate
+   * `(cap, addonId)` pair, so a blind re-register would throw on every
+   * re-handshake and trip the registering node's retry loop into a storm.
+   * Instead this diffs the NEW manifest against `previousManifest`:
+   * unchanged caps are left untouched, dropped caps are unregistered, new
+   * caps are registered. This honours the invariant "`registerNode`
+   * replaces the node's entire cap set atomically".
+   *
+   * NOTE: `params.nativeCaps` is stored by `nodeRegistry.registerNode()`
+   * already; this method handles only `params.addons` (system caps).
+   * Native-cap wiring into device-manager is done in a later task.
+   */
+  private applyNodeManifest(params: RegisterNodeParams, previousManifest?: readonly RegisteredAddonManifest[]): void {
+    const { nodeId, addons } = params
+    const hubNodeId = this.brokerSafe.nodeID
+    const isLocalChild = nodeId.startsWith(hubNodeId + '/')
+    const isHubInProcess = nodeId === hubNodeId
+
+    // Hub in-process addons register themselves during initialize() — skip.
+    if (isHubInProcess) return
+
+    const registry = this.capabilityService.getRegistry()
+    if (!registry) return
+
+    // Same registryKey rule as CapabilityBridge / onProviderConnected:
+    //   local child → bare addonId (one instance per forked child)
+    //   remote agent → addonId@nodeId (unique per agent node)
+    // `nodeId` is identical for the previous and new manifest (same node
+    // re-handshaking), so the rule resolves the same key on both sides.
+    const registryKeyFor = (addonId: string): string =>
+      isLocalChild ? addonId : `${addonId}@${nodeId}`
+
+    // Collect the `(registryKey, capName)` pairs a manifest would APPLY —
+    // applying the SAME `isInfraCapability` skip and `capDef` existence
+    // check the register block below uses, so the set reflects exactly
+    // what is (or would have been) registered. Keyed `${registryKey}::${capName}`.
+    // The resolved `capDef` is carried through so the register loop never
+    // re-looks it up (and never needs a non-null assertion).
+    const appliedKeys = (manifest: readonly RegisteredAddonManifest[]): Map<string, AppliedCapEntry> => {
+      const keys = new Map<string, AppliedCapEntry>()
+      for (const addon of manifest) {
+        const registryKey = registryKeyFor(addon.addonId)
+        for (const capName of addon.capabilities) {
+          if (isInfraCapability(capName)) continue
+          const capDef = registry.getDefinition(capName)
+          if (!capDef) continue
+          keys.set(`${registryKey}::${capName}`, { addonId: addon.addonId, capName, capDef })
+        }
+      }
+      return keys
+    }
+
+    const desired = appliedKeys(addons)
+    const previous = appliedKeys(previousManifest ?? [])
+
+    // ── UNREGISTER ── caps the previous manifest applied but the new one
+    // does not. Quiet — readiness is driven by `$node.connected/disconnected`,
+    // not by a re-handshake, so no readiness events here.
+    for (const [key, { addonId, capName }] of previous) {
+      if (desired.has(key)) continue
+      registry.unregisterProvider(capName, registryKeyFor(addonId))
+      this.nodeCallFns.delete(`${nodeId}::${capName}`)
+    }
+
+    // ── REGISTER ── caps the new manifest applies that the previous one
+    // did not. Caps present in BOTH sets are left untouched — zero churn,
+    // no duplicate `registerProvider`, no spurious page/widget re-emit.
+    for (const [key, { addonId, capName, capDef }] of desired) {
+      if (previous.has(key)) continue
+
+      const registryKey = registryKeyFor(addonId)
+
+      // Build a callFn that routes through Moleculer, matching the shape
+      // produced by CapabilityBridge: service name = addonId, action path
+      // = `<addonId>.<capName>.<method>`, nodeID = the registering node.
+      const callFn: CallFn =
+        (method: string, methodParams: unknown, targetNodeId?: string) =>
+          callWithServiceDiscovery(
+            this.brokerSafe,
+            addonId,
+            `${addonId}.${capName}.${method}`,
+            serializeTypedArrays(methodParams),
+            { nodeID: targetNodeId ?? nodeId, timeout: 60_000 },
+          )
+
+      const proxy: Record<string, unknown> = { id: addonId, nodeId }
+      for (const methodName of Object.keys(expandCapMethods(capDef))) {
+        proxy[methodName] = (methodParams: unknown) => callFn(methodName, methodParams)
+      }
+      registry.registerProvider(capName, registryKey, proxy)
+
+      // Emit AddonPageReady / AddonWidgetReady so the admin-UI sidebar
+      // refreshes its page/widget registry for cross-process addons.
+      if (capName === 'addon-pages-source') {
+        this.eventBus.emit({
+          id: randomUUID(),
+          timestamp: new Date(),
+          source: { type: 'addon', id: addonId },
+          category: EventCategory.AddonPageReady,
+          data: { addonId, nodeId },
+        })
+      }
+      if (capName === 'addon-widgets-source') {
+        this.eventBus.emit({
+          id: randomUUID(),
+          timestamp: new Date(),
+          source: { type: 'addon', id: addonId },
+          category: EventCategory.AddonWidgetReady,
+          data: { addonId, nodeId },
+        })
+      }
+
+      // Store callFn so `callCapabilityOnNode` and `createCapabilityProxy`
+      // can reach manifest-registered nodes.
+      this.nodeCallFns.set(`${nodeId}::${capName}`, callFn)
+    }
+  }
+
+  /**
+   * D3: remove a node's manifest from the CapabilityRegistry on disconnect.
+   * Unregisters every cap the node's last manifest declared and emits
+   * synthetic readiness-down events for each.
+   */
+  private removeNodeFromRegistry(nodeId: string): void {
+    const manifest = this.nodeRegistry.getNodeManifest(nodeId)
+    if (!manifest) return  // node never sent a handshake — nothing to do
+
+    const hubNodeId = this.brokerSafe.nodeID
+    const isLocalChild = nodeId.startsWith(hubNodeId + '/')
+    const disconnectGen = `disconnect-${nodeId}-${randomUUID()}`
+    const agentNodeId = nodeId.includes('/') ? nodeId.split('/')[0]! : nodeId
+
+    const registry = this.capabilityService.getRegistry()
+
+    for (const addon of manifest) {
+      const { addonId, capabilities } = addon
+      const registryKey = isLocalChild ? addonId : `${addonId}@${nodeId}`
+
+      if (registry) {
+        for (const capName of capabilities) {
+          registry.unregisterProvider(capName, registryKey)
+        }
+      }
+
+      for (const capName of capabilities) {
+        this.nodeCallFns.delete(`${nodeId}::${capName}`)
+
+        try {
+          emitReadiness(this.eventBus, {
+            capName,
+            scope: { type: 'node', nodeId: agentNodeId },
+            state: 'down',
+            generation: disconnectGen,
+            sourceNodeId: hubNodeId,
+          })
+        } catch (err) {
+          this.logger.warn('Failed to emit synthetic readiness down', {
+            tags: { addonId, nodeId },
+            meta: { capName, err: err instanceof Error ? err.message : String(err) },
+          })
+        }
+      }
+    }
+
+    this.nodeRegistry.removeNode(nodeId)
+  }
+
+  private findCallFn(nodeId: string, capabilityName: string): CallFn | undefined {
+    // Exact match first — direct hit when the caller knows the full
+    // nodeId (e.g. `hub/detection-pipeline`) or when the cap is hosted
+    // on a bare top-level node (remote agent with in-process addons).
+    const direct = this.nodeCallFns.get(`${nodeId}::${capabilityName}`)
+    if (direct) return direct
+    // Prefix fallback: forkable addons register under
+    // `<parent>/<processName>` (e.g. `dev-agent-0/detection-pipeline`).
+    // UI callers typically pass the bare parent nodeId (`dev-agent-0`)
+    // because that's what they get from AgentOnline events and the
+    // orchestrator assignments. Resolve by finding any registered node
+    // whose id starts with `<nodeId>/` and hosts the cap.
+    const prefix = `${nodeId}/`
+    for (const key of this.nodeCallFns.keys()) {
+      const sep = key.lastIndexOf('::')
+      if (sep < 0) continue
+      const keyNode = key.slice(0, sep)
+      const keyCap = key.slice(sep + 2)
+      if (keyCap !== capabilityName) continue
+      if (keyNode.startsWith(prefix)) return this.nodeCallFns.get(key)
+    }
+    return undefined
+  }
+
+
+  /**
+   * Build a proxy object that forwards every method call on a capability
+   * to a remote node via Moleculer. Returns null if the capability is not
+   * reachable on that node. Used by the generated cap routers when a
+   * request includes a `nodeId` field for transparent node routing.
+   */
+  createCapabilityProxy(capabilityName: string, nodeId: string): Record<string, (params: unknown) => Promise<unknown>> | null {
+    // Check if we can reach this capability on the target node
+    const callFn = this.findCallFn(nodeId, capabilityName)
+    if (!callFn) return null
+
+    // Build a dynamic proxy: every property access returns a function that
+    // calls the remote capability method via the stored callFn.
+    return new Proxy<Record<string, (params: unknown) => Promise<unknown>>>({}, {
+      get: (_target, methodName: string) => {
+        return (params: unknown): Promise<unknown> =>
+          this.callCapabilityOnNode(nodeId, capabilityName, methodName, params)
+      },
+    })
+  }
+
+  /**
+   * D3 handshake-fed native-cap view of the whole cluster.
+   * Returns every `(nodeId, addonId, capName, deviceId)` tuple stored by
+   * `onRegisterNode` — updated atomically each time a node re-handshakes
+   * (e.g. after device restore completes). Used by `device-manager.addon.ts`
+   * as a reliable fallback when push-based `DeviceBindingsChanged` events
+   * were lost in the Moleculer transport handshake window.
+   *
+   * NOT a Moleculer action — only the hub process calls this directly
+   * through the `ctx.kernel.listClusterNativeCaps` injection.
+   */
+  listClusterNativeCaps(): readonly import('@camstack/kernel').NodeNativeCapEntry[] {
+    return this.nodeRegistry.listNativeCapEntries()
+  }
+
+  async onModuleDestroy(): Promise<void> {
+    await this.brokerSafe.stop()
+  }
+}

package/src/core/network/network-quality.service.spec.ts

@@ -0,0 +1,47 @@
+import { describe, it, expect, beforeEach } from 'vitest'
+import { NetworkQualityService } from './network-quality.service'
+
+describe('NetworkQualityService', () => {
+  let service: NetworkQualityService
+
+  beforeEach(() => {
+    service = new NetworkQualityService()
+  })
+
+  it('should return null for unknown device', () => {
+    expect(service.getDeviceStats(999)).toBeNull()
+  })
+
+  it('should track stream bitrate with rolling average', () => {
+    service.reportStreamStats(1, 'main', 8000)
+    service.reportStreamStats(1, 'main', 10000)
+    service.reportStreamStats(1, 'main', 9000)
+
+    const stats = service.getDeviceStats(1)
+    expect(stats).not.toBeNull()
+    expect(stats!.streams['main']!.observedBitrateKbps).toBe(9000)
+    expect(stats!.streams['main']!.peakBitrateKbps).toBe(10000)
+  })
+
+  it('should track packet loss', () => {
+    service.reportStreamStats(1, 'main', 8000, 0.5)
+    service.reportStreamStats(1, 'main', 8000, 1.5)
+
+    const stats = service.getDeviceStats(1)
+    expect(stats!.streams['main']!.packetLossPercent).toBe(1.0)
+  })
+
+  it('should track client stats', () => {
+    service.reportClientStats(1, { rttMs: 50, jitterMs: 5, estimatedBandwidthKbps: 20000 })
+    const stats = service.getDeviceStats(1)
+    expect(stats!.client?.rttMs).toBe(50)
+    expect(stats!.client?.estimatedBandwidthKbps).toBe(20000)
+  })
+
+  it('should list all device stats', () => {
+    service.reportStreamStats(1, 'main', 8000)
+    service.reportStreamStats(2, 'sub', 2000)
+    const all = service.getAllStats()
+    expect(all).toHaveLength(2)
+  })
+})

package/src/core/network/network-quality.service.ts

@@ -0,0 +1,5 @@
+import { NetworkQualityTracker } from '@camstack/core'
+
+export class NetworkQualityService extends NetworkQualityTracker {
+  // NestJS DI wrapper — delegates entirely to core NetworkQualityTracker
+}