@camstack/server 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/.env.example +17 -0
  2. package/package.json +55 -0
  3. package/src/__tests__/addon-install-e2e.test.ts +75 -0
  4. package/src/__tests__/addon-pages-e2e.test.ts +178 -0
  5. package/src/__tests__/addon-route-session.test.ts +17 -0
  6. package/src/__tests__/addon-settings-router.spec.ts +62 -0
  7. package/src/__tests__/addon-upload.spec.ts +355 -0
  8. package/src/__tests__/agent-registry.spec.ts +162 -0
  9. package/src/__tests__/agent-status-page.spec.ts +84 -0
  10. package/src/__tests__/auth-session-cookie.test.ts +21 -0
  11. package/src/__tests__/cap-providers/cap-usage-graph.spec.ts +23 -0
  12. package/src/__tests__/cap-providers/compute-topology-categories.spec.ts +64 -0
  13. package/src/__tests__/cap-routers/_meta.spec.ts +200 -0
  14. package/src/__tests__/cap-routers/addon-settings.router.spec.ts +106 -0
  15. package/src/__tests__/cap-routers/device-manager-aggregate.router.spec.ts +142 -0
  16. package/src/__tests__/cap-routers/harness.ts +159 -0
  17. package/src/__tests__/cap-routers/metrics-provider.router.spec.ts +119 -0
  18. package/src/__tests__/cap-routers/null-provider-guard.spec.ts +66 -0
  19. package/src/__tests__/cap-routers/pipeline-executor.router.spec.ts +135 -0
  20. package/src/__tests__/cap-routers/settings-store.router.spec.ts +247 -0
  21. package/src/__tests__/capability-e2e.test.ts +386 -0
  22. package/src/__tests__/cli-e2e.test.ts +129 -0
  23. package/src/__tests__/core-cap-bridge.spec.ts +89 -0
  24. package/src/__tests__/embedded-deps-e2e.test.ts +109 -0
  25. package/src/__tests__/event-bus-proxy-router.spec.ts +72 -0
  26. package/src/__tests__/fixtures/mock-analysis-addon-a.ts +37 -0
  27. package/src/__tests__/fixtures/mock-analysis-addon-b.ts +37 -0
  28. package/src/__tests__/fixtures/mock-log-addon.ts +37 -0
  29. package/src/__tests__/fixtures/mock-storage-addon.ts +40 -0
  30. package/src/__tests__/framework-allowlist.spec.ts +95 -0
  31. package/src/__tests__/https-e2e.test.ts +118 -0
  32. package/src/__tests__/lifecycle-e2e.test.ts +140 -0
  33. package/src/__tests__/live-events-subscription.spec.ts +150 -0
  34. package/src/__tests__/moleculer-register-node-idempotency.spec.ts +229 -0
  35. package/src/__tests__/oauth2-account-linking.spec.ts +736 -0
  36. package/src/__tests__/post-boot-restart.spec.ts +161 -0
  37. package/src/__tests__/singleton-contention.test.ts +487 -0
  38. package/src/__tests__/streaming-diagnostic.test.ts +512 -0
  39. package/src/__tests__/streaming-scale.test.ts +280 -0
  40. package/src/agent-status-page.ts +121 -0
  41. package/src/api/__tests__/addons-custom.spec.ts +134 -0
  42. package/src/api/__tests__/capabilities.router.test.ts +47 -0
  43. package/src/api/addon-upload.ts +472 -0
  44. package/src/api/addons-custom.router.ts +100 -0
  45. package/src/api/auth-whoami.ts +99 -0
  46. package/src/api/bridge-addons.router.ts +120 -0
  47. package/src/api/capabilities.router.ts +226 -0
  48. package/src/api/core/__tests__/auth-router-totp.spec.ts +256 -0
  49. package/src/api/core/addon-settings.router.ts +124 -0
  50. package/src/api/core/agents.router.ts +87 -0
  51. package/src/api/core/auth.router.ts +303 -0
  52. package/src/api/core/cap-providers.ts +993 -0
  53. package/src/api/core/capabilities.router.ts +119 -0
  54. package/src/api/core/collection-preference.ts +40 -0
  55. package/src/api/core/event-bus-proxy.router.ts +45 -0
  56. package/src/api/core/hwaccel.router.ts +81 -0
  57. package/src/api/core/live-events.router.ts +60 -0
  58. package/src/api/core/logs.router.ts +162 -0
  59. package/src/api/core/notifications.router.ts +65 -0
  60. package/src/api/core/repl.router.ts +41 -0
  61. package/src/api/core/settings-backend.router.ts +142 -0
  62. package/src/api/core/stream-probe.router.ts +57 -0
  63. package/src/api/core/system-events.router.ts +116 -0
  64. package/src/api/health/health.routes.ts +123 -0
  65. package/src/api/oauth2/__tests__/oauth2-routes.spec.ts +52 -0
  66. package/src/api/oauth2/consent-page.ts +42 -0
  67. package/src/api/oauth2/oauth2-routes.ts +248 -0
  68. package/src/api/trpc/__tests__/scope-access-device.spec.ts +223 -0
  69. package/src/api/trpc/__tests__/scope-access.spec.ts +107 -0
  70. package/src/api/trpc/cap-mount-helpers.ts +225 -0
  71. package/src/api/trpc/core-cap-bridge.ts +152 -0
  72. package/src/api/trpc/generated-cap-mounts.ts +707 -0
  73. package/src/api/trpc/generated-cap-routers.ts +6340 -0
  74. package/src/api/trpc/scope-access.ts +110 -0
  75. package/src/api/trpc/trpc.context.ts +255 -0
  76. package/src/api/trpc/trpc.middleware.ts +140 -0
  77. package/src/api/trpc/trpc.router.ts +275 -0
  78. package/src/auth/session-cookie.ts +44 -0
  79. package/src/boot/boot-config.ts +278 -0
  80. package/src/boot/post-boot.service.ts +103 -0
  81. package/src/core/addon/__tests__/addon-registry-capability.test.ts +53 -0
  82. package/src/core/addon/addon-package.service.ts +1684 -0
  83. package/src/core/addon/addon-registry.service.ts +2926 -0
  84. package/src/core/addon/addon-search.service.ts +90 -0
  85. package/src/core/addon/addon-settings-provider.ts +276 -0
  86. package/src/core/addon/addon.tokens.ts +2 -0
  87. package/src/core/addon-bridge/addon-bridge.service.ts +125 -0
  88. package/src/core/addon-pages/addon-pages.service.spec.ts +117 -0
  89. package/src/core/addon-pages/addon-pages.service.ts +80 -0
  90. package/src/core/addon-widgets/addon-widgets.service.ts +92 -0
  91. package/src/core/agent/agent-registry.service.ts +507 -0
  92. package/src/core/auth/auth.service.spec.ts +88 -0
  93. package/src/core/auth/auth.service.ts +8 -0
  94. package/src/core/capability/capability.service.ts +57 -0
  95. package/src/core/config/config.schema.ts +3 -0
  96. package/src/core/config/config.service.spec.ts +175 -0
  97. package/src/core/config/config.service.ts +7 -0
  98. package/src/core/events/event-bus.service.spec.ts +212 -0
  99. package/src/core/events/event-bus.service.ts +85 -0
  100. package/src/core/feature/feature.service.spec.ts +96 -0
  101. package/src/core/feature/feature.service.ts +8 -0
  102. package/src/core/lifecycle/lifecycle-state-machine.spec.ts +168 -0
  103. package/src/core/lifecycle/lifecycle-state-machine.ts +3 -0
  104. package/src/core/logging/log-ring-buffer.ts +3 -0
  105. package/src/core/logging/logging.service.spec.ts +247 -0
  106. package/src/core/logging/logging.service.ts +129 -0
  107. package/src/core/logging/scoped-logger.ts +3 -0
  108. package/src/core/moleculer/moleculer.service.ts +612 -0
  109. package/src/core/network/network-quality.service.spec.ts +47 -0
  110. package/src/core/network/network-quality.service.ts +5 -0
  111. package/src/core/notification/notification-wrapper.service.ts +36 -0
  112. package/src/core/notification/toast-wrapper.service.ts +31 -0
  113. package/src/core/provider/provider.tokens.ts +1 -0
  114. package/src/core/repl/repl-engine.service.spec.ts +417 -0
  115. package/src/core/repl/repl-engine.service.ts +156 -0
  116. package/src/core/storage/fs-storage-backend.spec.ts +70 -0
  117. package/src/core/storage/fs-storage-backend.ts +3 -0
  118. package/src/core/storage/settings-store.spec.ts +213 -0
  119. package/src/core/storage/settings-store.ts +2 -0
  120. package/src/core/storage/sql-schema.spec.ts +140 -0
  121. package/src/core/storage/sql-schema.ts +3 -0
  122. package/src/core/storage/storage-location-manager.spec.ts +121 -0
  123. package/src/core/storage/storage-location-manager.ts +3 -0
  124. package/src/core/storage/storage.service.spec.ts +73 -0
  125. package/src/core/storage/storage.service.ts +3 -0
  126. package/src/core/streaming/stream-probe.service.ts +212 -0
  127. package/src/core/topology/topology-emitter.service.ts +101 -0
  128. package/src/launcher.ts +309 -0
  129. package/src/main.ts +1049 -0
  130. package/src/manual-boot.ts +322 -0
  131. package/tsconfig.build.json +8 -0
  132. package/tsconfig.json +21 -0
  133. package/vitest.config.ts +26 -0
@@ -0,0 +1,2926 @@
1
+ /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access -- pre-existing lint debt across this 2200-line orchestration class. The flagged sites (StorageService.setLocationManager / setSettingsBackend, LoggingService.addDestination, RouteRegistry, etc.) are typed as `unknown` by their owning services to break circular construction-order dependencies; runtime contracts are validated structurally. Tracked separately; do not amend in unrelated edits. */
2
+ import * as os from "node:os";
3
+ import { ConfigService } from "../config/config.service";
4
+ import { LoggingService } from "../logging/logging.service";
5
+ import { EventBusService } from "../events/event-bus.service";
6
+ import { StorageService } from "../storage/storage.service";
7
+ import { StreamProbeService } from "../streaming/stream-probe.service";
8
+
9
+ import { CapabilityService } from "../capability/capability.service";
10
+ import {
11
+ EventCategory,
12
+ createDeviceProxy,
13
+ normalizeAddonInitResult,
14
+ errMsg,
15
+ isAgentOnlyPlacement,
16
+ resolveRunnerId,
17
+ resolveAddonPlacement,
18
+ } from "@camstack/types";
19
+ import type {
20
+ AddonDeclaration,
21
+ CapabilityDeclaration,
22
+ CapabilityDefinition,
23
+ InferProvider,
24
+ RunnerPlan,
25
+ RunnerAddonPlacement,
26
+ } from "@camstack/types";
27
+ import type {
28
+ ICamstackAddon,
29
+ AddonContext,
30
+ InternalAddonContext,
31
+ } from "@camstack/types";
32
+ import type { IScopedLogger } from "@camstack/types";
33
+ import {
34
+ CapabilityRegistry,
35
+ CustomActionRegistry,
36
+ INFRA_CAPABILITIES,
37
+ AddonLoader,
38
+ AddonHealthMonitor,
39
+
40
+ DeviceRegistry,
41
+ adaptBrokerToCluster,
42
+ createAddonService,
43
+ createBrokerDeviceManagerApi,
44
+ createKernelHwAccel,
45
+ validateProviderRegistrations,
46
+ CapabilityHandle,
47
+ scopeKey,
48
+ describeProviderKindDrift,
49
+ type ISettingsStore,
50
+ type AddonSettingsView,
51
+ type AddonHealthSnapshot,
52
+ } from "@camstack/kernel";
53
+ import { localProviderLink, brokerTransportLink } from "@camstack/kernel";
54
+ import { createTRPCClient } from "@trpc/client";
55
+ import { MoleculerService } from "../moleculer/moleculer.service";
56
+ import { AddonDepsManager } from "@camstack/kernel";
57
+ import type { SavedDevice, ReadinessScope } from "@camstack/types";
58
+ import { IntegrationRegistry } from "@camstack/core";
59
+ import type {
60
+ IStorageProvider as INewStorageProvider,
61
+ ISettingsBackend,
62
+ } from "@camstack/types";
63
+ import { AddonRouteRegistry } from "@camstack/core";
64
+ import { randomUUID } from "node:crypto";
65
+ import * as path from "node:path";
66
+ import * as fs from "node:fs";
67
+ import { pathToFileURL } from "node:url";
68
+ import { createAddonSettingsProvider } from "./addon-settings-provider.js";
69
+ import { addonSettingsCapability } from "@camstack/types";
70
+ import { DisposerChain } from "@camstack/types";
71
+
72
+ type AddonSource = "core" | "installed";
73
+
74
+ /**
75
+ * Local narrowing of the Moleculer ServiceBroker surface we actually use.
76
+ * See the `broker` getter docstring below for why this is necessary.
77
+ */
78
+ interface BrokerLike {
79
+ call<T = unknown>(action: string, params?: unknown, opts?: { nodeID?: string; timeout?: number }): Promise<T>
80
+ nodeID: string
81
+ createService(schema: unknown): unknown
82
+ }
83
+
84
+ /**
85
+ * Type predicate: true when an `ISettingsBackend` also satisfies the
86
+ * sync `ISettingsStore` contract (system/addon/device key-value ops)
87
+ * that ConfigManager expects. The default `SqliteSettingsBackend` in
88
+ * `@camstack/core` implements both surfaces; forked addon backends may
89
+ * not, in which case we skip the ConfigService wiring.
90
+ */
91
+ function isSettingsStore(
92
+ backend: ISettingsBackend,
93
+ ): backend is ISettingsBackend & ISettingsStore {
94
+ const required: readonly (keyof ISettingsStore)[] = [
95
+ "getSystem",
96
+ "setSystem",
97
+ "getAllSystem",
98
+ "getAllAddon",
99
+ "setAllAddon",
100
+ "getAllProvider",
101
+ "setProvider",
102
+ "getAllDevice",
103
+ "setDevice",
104
+ "getAddonDevice",
105
+ "setAddonDevice",
106
+ "clearAddonDevice",
107
+ ];
108
+ for (const key of required) {
109
+ if (typeof Reflect.get(backend, key) !== "function") return false;
110
+ }
111
+ return true;
112
+ }
113
+
114
+ interface AddonEntry {
115
+ readonly addon: ICamstackAddon;
116
+ initialized: boolean;
117
+ source: AddonSource;
118
+ /** npm package name from package.json (e.g. '@camstack/addon-detection-pipeline') */
119
+ packageName: string;
120
+ /** npm package version from package.json */
121
+ packageVersion: string;
122
+ /** Human-readable package name from camstack.displayName */
123
+ packageDisplayName?: string;
124
+ /** Optional bundle metadata when the package ships multiple addon entries. */
125
+ bundle?: { displayName: string; description?: string; icon?: string };
126
+ /** Capabilities declared in package.json camstack.addons (source of truth) */
127
+ declaredCapabilities: readonly CapabilityDeclaration[];
128
+ /** Addon directory on disk */
129
+ addonDir?: string;
130
+ /** Full package.json declaration (AddonDeclaration from @camstack/types) */
131
+ declaration?: AddonDeclaration;
132
+ }
133
+
134
+ // Phase 11 (settings redesign): `stripValue` helper and the legacy
135
+ // `getAddonConfigSchema / getAddonConfig / updateAddonConfig` adapter
136
+ // shim that consumed it were deleted. All settings flow through the
137
+ // new `getAddonSettings / getGlobalSettings / getDeviceSettings`
138
+ // endpoints on the `addon-settings` singleton capability.
139
+
140
+ export class AddonRegistryService {
141
+ private readonly addonEntries = new Map<string, AddonEntry>();
142
+ private readonly capabilityRegistry: CapabilityRegistry;
143
+ // Task 7.1: hub-wide registry of addon custom actions. Populated on
144
+ // each addon's initialize() (in-process path); Task 7.2 will dispatch
145
+ // through this from the `api.addons.custom` tRPC procedure.
146
+ private readonly customActionRegistry = new CustomActionRegistry();
147
+ /**
148
+ * AddonIds whose group-runner disconnect is operator-initiated (update /
149
+ * restart / uninstall). The Moleculer `$node.disconnected` handler skips
150
+ * these so the AddonCard doesn't flash a spurious "Failed to load" health
151
+ * banner during the kill→respawn window. The set is cleared as soon as
152
+ * `restartAddon` completes (success or failure) or by a 90s safety timer.
153
+ */
154
+ private readonly restartingAddons = new Map<string, NodeJS.Timeout>();
155
+ private readonly logger: IScopedLogger;
156
+ private addonLoader!: AddonLoader;
157
+ private healthMonitor!: AddonHealthMonitor;
158
+ private addonRouteRegistry: AddonRouteRegistry | null = null;
159
+
160
+ // Broker-routed AddonApi proxy — every addon's `ctx.api` resolves
161
+ // to this. Calls go through `broker.call('${addonId}.${capName}.${method}')`
162
+ // which Moleculer routes to whichever process actually hosts the
163
+ // capability (in-process services on the hub, forked workers, or
164
+ // remote agents). The hub uses the exact same transport as agents
165
+ // and forked workers — `ctx.api` is uniform across all deployment
166
+ // shapes. Lazily constructed so tests that don't wire the broker
167
+ // still instantiate the service.
168
+ private brokerApi: import("@camstack/types").AddonApi | null = null;
169
+ /**
170
+ * In-process tRPC client over the broker — same `AddonApi` shape every
171
+ * addon sees via `ctx.api`. Exposed as public so the REPL service can
172
+ * build a `SystemManager` from it without crossing the network boundary.
173
+ */
174
+ public getBrokerApi(): import("@camstack/types").AddonApi {
175
+ if (!this.brokerApi) {
176
+ const reg = this.capabilityRegistry;
177
+ const resolver = {
178
+ getByName: (capName: string): unknown | null =>
179
+ reg.getSingleton(capName) ??
180
+ (reg.getAllProviders(capName)[0] as unknown) ??
181
+ null,
182
+ };
183
+ const client: unknown = createTRPCClient({
184
+ links: [
185
+ localProviderLink(resolver),
186
+ brokerTransportLink(this.moleculer.broker),
187
+ ],
188
+ });
189
+ this.brokerApi = client as import("@camstack/types").AddonApi;
190
+ }
191
+ return this.brokerApi;
192
+ }
193
+
194
+ // Common settings resolver (3-level: defaults → global → per-device).
195
+ // Lazily constructed on first call to `getSettingsResolver()` so tests
196
+ // that don't exercise the settings API can instantiate the service
197
+ // without wiring ConfigService into the resolver.
198
+
199
+ // Active capability providers (set by consumers when capabilities are wired)
200
+ private activeStorageProvider: INewStorageProvider | null = null;
201
+ private activeSettingsBackend: ISettingsBackend | null = null;
202
+ private integrationRegistry:
203
+ | import("@camstack/core").IntegrationRegistry
204
+ | null = null;
205
+
206
+ // Device architecture — in-memory registry of live IDevice instances.
207
+ // Persistence is owned by the `device-manager` capability addon.
208
+ private readonly deviceRegistry = new DeviceRegistry();
209
+
210
+ /**
211
+ * Typed accessor for the Moleculer broker. The Moleculer `index.d.ts`
212
+ * chains through `eventemitter2`, whose package.json has no `types`
213
+ * field; under `moduleResolution: node` typescript-eslint's parser
214
+ * loses the type chain even though `tsc` accepts it via `skipLibCheck`.
215
+ * Going through `this.moleculer.broker` directly produces "type cannot
216
+ * be resolved" errors at every call site. The local `BrokerLike`
217
+ * interface narrows down to the surface we actually use, and the
218
+ * double-cast forces the parser to materialize types locally so
219
+ * `this.broker.call(...)` / `this.broker.nodeID` lint clean.
220
+ */
221
+ private get broker(): BrokerLike {
222
+ return this.moleculer.broker as unknown as BrokerLike;
223
+ }
224
+
225
+ constructor(
226
+ private readonly loggingService: LoggingService,
227
+ private readonly eventBusService: EventBusService,
228
+ private readonly configService: ConfigService,
229
+ private readonly storageService: StorageService,
230
+ private readonly capabilityService: CapabilityService,
231
+ private readonly moleculer: MoleculerService,
232
+ private readonly streamProbe: StreamProbeService,
233
+ ) {
234
+ this.logger = this.loggingService.createLogger("AddonRegistry");
235
+ this.addonLoader = new AddonLoader(
236
+ this.loggingService.createLogger("AddonLoader"),
237
+ );
238
+
239
+ // Kernel-level addon health monitor — drives the 5-min boot grace
240
+ // window + aggressive auto-retry loop (60s/120s/300s, never gives
241
+ // up). Failures and recoveries are recorded by the load + init
242
+ // paths; the monitor's tick loop calls `tryReloadPackage` to
243
+ // reattempt failed addons. Operator visibility comes through
244
+ // AddonLoadFailed / AddonLoadRecovered events consumed by
245
+ // AlertCenter (see addon-health-monitor.ts spec).
246
+ this.healthMonitor = new AddonHealthMonitor({
247
+ eventBus: this.eventBusService as unknown as import("@camstack/types").IEventBus,
248
+ logger: this.loggingService.createLogger("AddonHealthMonitor"),
249
+ retryFn: (packageName) => this.tryReloadPackage(packageName),
250
+ });
251
+
252
+ // Create capability registry with config reader for singleton preferences
253
+ this.capabilityRegistry = new CapabilityRegistry(
254
+ this.loggingService.createLogger("CapabilityRegistry"),
255
+ this.eventBusService,
256
+ );
257
+ this.capabilityRegistry.setConfigReader(
258
+ (capability: string): string | undefined => {
259
+ try {
260
+ return (
261
+ this.configService.get<string>(
262
+ `capabilities.singleton.${capability}`,
263
+ ) ?? undefined
264
+ );
265
+ } catch (err) {
266
+ this.logger.debug(
267
+ 'settings-store not wired yet during early boot',
268
+ { meta: { capability, error: errMsg(err) } },
269
+ );
270
+ return undefined;
271
+ }
272
+ },
273
+ );
274
+
275
+ // Collection-provider enable/disable persistence. Reads the same
276
+ // `capabilities.collection.<cap>` ConfigService key the `capabilities`
277
+ // router writes (`JSON.stringify({ disabled: string[] })`), so a
278
+ // collection provider an operator disabled survives a hub reboot.
279
+ this.capabilityRegistry.setCollectionConfigReader(
280
+ (capability: string): readonly string[] | undefined => {
281
+ try {
282
+ const raw = this.configService.get<string>(
283
+ `capabilities.collection.${capability}`,
284
+ );
285
+ if (!raw) return undefined;
286
+ const parsed: unknown = JSON.parse(raw);
287
+ if (
288
+ typeof parsed === "object" &&
289
+ parsed !== null &&
290
+ "disabled" in parsed &&
291
+ Array.isArray((parsed as { disabled: unknown }).disabled)
292
+ ) {
293
+ const disabled = (parsed as { disabled: unknown[] }).disabled;
294
+ return disabled.filter(
295
+ (id): id is string => typeof id === "string",
296
+ );
297
+ }
298
+ return undefined;
299
+ } catch (err) {
300
+ this.logger.debug(
301
+ "settings-store not wired yet or malformed collection preference",
302
+ { meta: { capability, error: errMsg(err) } },
303
+ );
304
+ return undefined;
305
+ }
306
+ },
307
+ );
308
+
309
+ // Wire the registry into the CapabilityService so all server services can use it
310
+ this.capabilityService.setRegistry(this.capabilityRegistry);
311
+
312
+ // Register kernel-provided infrastructure capabilities so addons
313
+ // can resolve them via the capability registry like any other cap.
314
+ // These are kernel pseudo-caps (no tRPC methods, object-reference only),
315
+ // so we declare them with empty method maps before registering providers.
316
+ this.capabilityRegistry.declareCapability({
317
+ name: "device-registry",
318
+ scope: "system",
319
+ mode: "singleton",
320
+ methods: {},
321
+ });
322
+ this.capabilityRegistry.declareCapability({
323
+ name: "cluster-broker",
324
+ scope: "system",
325
+ mode: "singleton",
326
+ methods: {},
327
+ });
328
+ this.capabilityRegistry.registerProvider(
329
+ "device-registry",
330
+ "$kernel",
331
+ this.deviceRegistry,
332
+ );
333
+ this.capabilityRegistry.registerProvider("cluster-broker", "$kernel", {
334
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- moleculer types unresolvable; see `broker` getter docstring
335
+ broker: this.moleculer.broker,
336
+ });
337
+ }
338
+
339
+ async onModuleInit(): Promise<void> {
340
+ // Register the addon-settings singleton provider — replaces the
341
+ // former `$addonHost` Moleculer service. The provider resolves
342
+ // addonId → local addon instance (hub) or remote Moleculer call.
343
+ this.capabilityRegistry.declareCapability(addonSettingsCapability);
344
+ const settingsProvider = createAddonSettingsProvider({
345
+ getAddon: (addonId) => {
346
+ const entry = this.addonEntries.get(addonId);
347
+ return entry?.addon ?? null;
348
+ },
349
+ resolveNode: (addonId) => {
350
+ // Group-runner addons live in a child Moleculer node (e.g.
351
+ // `hub/pipeline`). The hub-side `entry.addon` instance never
352
+ // runs `initialize()` and has no `_ctx.settings`, so calling
353
+ // `addon.getGlobalSettings()` on it returns defaults only —
354
+ // the real store lives in the worker. Route those via the
355
+ // remote path so the worker's `<addonId>.settings.*` handler
356
+ // serves the canonical response.
357
+ const entry = this.addonEntries.get(addonId);
358
+ if (entry?.declaration && entry.addonDir) {
359
+ const runnerId = resolveRunnerId(entry.declaration, addonId);
360
+ return `${this.broker.nodeID}/${runnerId}`;
361
+ }
362
+ return "hub";
363
+ },
364
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- moleculer types unresolvable; see `broker` getter docstring
365
+ broker: this.moleculer.broker,
366
+ hubNodeId: this.broker.nodeID,
367
+ });
368
+ this.capabilityRegistry.registerProvider(
369
+ "addon-settings",
370
+ "$hub",
371
+ settingsProvider,
372
+ );
373
+
374
+ // Wire capability consumer actions via EventBus
375
+ this.wireCapabilityConsumers();
376
+
377
+ // Subscribe to capability.binding-changed so hub-side capability
378
+ // overrides take effect on the fly. The orchestrator addon emits
379
+ // these when the operator swaps the addon implementing a cap on a
380
+ // node; we only react when the target node is the hub's own nodeID.
381
+ this.eventBusService.subscribe(
382
+ { category: "capability.binding-changed" },
383
+ (event) => {
384
+ const data = (event.data ?? {}) as {
385
+ nodeId?: string;
386
+ capName?: string;
387
+ addonId?: string | null;
388
+ };
389
+ if (!data.capName) return;
390
+ const localNodeId = this.broker.nodeID;
391
+ if (data.nodeId && data.nodeId !== localNodeId) return;
392
+ this.capabilityRegistry.setSingletonActiveAddon(
393
+ data.capName,
394
+ data.addonId ?? null,
395
+ );
396
+ },
397
+ );
398
+
399
+ // 2. Load all addons from data/addons/ directory (including core builtins)
400
+ const dataDir = path.resolve(process.env.CAMSTACK_DATA ?? "camstack-data");
401
+ const addonsDir = path.resolve(dataDir, "addons");
402
+
403
+ await this.addonLoader.loadFromDirectory(addonsDir);
404
+
405
+ // Hand pre-init load failures (broken package.json, missing
406
+ // dist, import-time errors) to the health monitor so its retry
407
+ // loop and post-grace alerting cover them. Failures during the
408
+ // init phase below are recorded separately.
409
+ for (const fail of this.addonLoader.listLoadFailures()) {
410
+ this.healthMonitor.recordFailure(fail.packageName, fail.error, fail.addonId);
411
+ }
412
+
413
+ const loadedAddons = this.addonLoader.listAddons();
414
+
415
+ for (const registered of loadedAddons) {
416
+ // Skip agent-only addons — they never run on the hub, only on remote
417
+ // agents that opt in via `execution.placement: 'agent-only'`.
418
+ if (isAgentOnlyPlacement(registered.declaration)) {
419
+ this.logger.info(
420
+ 'Skipping agent-only addon on hub',
421
+ { tags: { addonId: registered.declaration.id }, meta: { packageName: registered.packageName } },
422
+ );
423
+ continue;
424
+ }
425
+ if (registered.declaration.capabilities) {
426
+ for (const cap of registered.declaration.capabilities) {
427
+ this.capabilityRegistry.declareFromManifest(
428
+ cap,
429
+ registered.declaration.id,
430
+ );
431
+ }
432
+ }
433
+ // Create and store addon instance — cast from @camstack/types to server's
434
+ // local ICamstackAddon (structurally compatible at runtime).
435
+ // Wrap in try/catch so a single broken addon doesn't prevent the rest from loading.
436
+ try {
437
+ const addon = this.addonLoader.createInstance(
438
+ registered.declaration.id,
439
+ );
440
+ // Capabilities come from package.json declaration (source of truth),
441
+ // merged with any declared in the addon class manifest.
442
+ const declCaps = (registered.declaration.capabilities ?? []).map(
443
+ (c: string | CapabilityDeclaration) =>
444
+ typeof c === "string" ? { name: c, mode: "singleton" as const } : c,
445
+ );
446
+ this.addonEntries.set(registered.declaration.id, {
447
+ addon,
448
+ initialized: false,
449
+ source: "installed",
450
+ packageName: registered.packageName,
451
+ packageVersion: registered.packageVersion,
452
+ packageDisplayName: registered.packageDisplayName,
453
+ ...(registered.bundle !== undefined ? { bundle: registered.bundle } : {}),
454
+ declaredCapabilities: declCaps,
455
+ addonDir: path.join(addonsDir, registered.packageName),
456
+ declaration: registered.declaration,
457
+ });
458
+ } catch (err) {
459
+ const msg = errMsg(err);
460
+ this.logger.error(
461
+ 'Failed to create instance of addon',
462
+ { tags: { addonId: registered.declaration.id }, meta: { error: msg } },
463
+ );
464
+ }
465
+ }
466
+
467
+ this.logger.info(
468
+ 'Loaded addons from directory',
469
+ { meta: { count: loadedAddons.length, addonsDir } },
470
+ );
471
+ // Platform probing is no longer orchestrated by the backend. The
472
+ // `platform-probe` capability (shipped as a core builtin under
473
+ // `@camstack/core/dist/builtins/platform-probe/` since Phase B of
474
+ // the bundles refactor) exposes hardware + scored backends via
475
+ // `ctx.api.platformProbe.*`; inference addons auto-configure
476
+ // inside their own `onInitialize`.
477
+
478
+ // 4. Initialize all installed addons in correct order (installed = active)
479
+ const allIds = [...this.addonEntries.keys()];
480
+
481
+ // Mark registry as ready BEFORE any addon initializes. Addons' onInitialize
482
+ // may call ctx.api.<cap>.* which routes through localProviderLink; that
483
+ // resolver depends on getAllProviders/getSingleton which gate on `_ready`.
484
+ // Without this, Phase 1 and Phase 2 addons would all fall through to the
485
+ // Moleculer broker transport and fail with "service not registered".
486
+ this.capabilityRegistry.ready();
487
+
488
+ // Native-cap cross-process bridge: when a hub consumer resolves a native
489
+ // provider for a device whose IDevice lives in a forked worker, we
490
+ // return a broker proxy that routes calls to
491
+ // `<addonId>.native-provider.<capName>.<method>` via standard Moleculer RPC.
492
+ //
493
+ // The resolver keys off `(capName, deviceId)`, NOT device ownership.
494
+ // An addon can own a device without registering every conceivable cap
495
+ // natively (e.g. RtspCamera without `snapshotUrl` does not publish a
496
+ // snapshot provider). Synthesizing a proxy against the device owner
497
+ // would point at a Moleculer service that was never mounted and make
498
+ // every call throw "service not found" at runtime.
499
+ //
500
+ // `resolveNativeCapOwnerSync` consults both hub-local native
501
+ // registrations and `remoteNativeCaps` (populated via
502
+ // DeviceBindingsChanged events emitted by every worker). It returns
503
+ // null when no provider published the cap for that device — the
504
+ // fallback surfaces that as null so wrappers cleanly fall through to
505
+ // whatever secondary strategy they own (e.g. snapshot → ffmpeg frame
506
+ // grab), instead of throwing against a phantom service.
507
+ {
508
+ const { buildNativeCapProxy } = await import("@camstack/kernel");
509
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- moleculer types unresolvable; see `broker` getter docstring
510
+ const broker = this.moleculer.broker;
511
+ this.capabilityRegistry.setNativeFallback(
512
+ (capName: string, deviceId: number): unknown | null => {
513
+ const dm = this.capabilityRegistry.getSingleton<{
514
+ resolveNativeCapOwnerSync?: (
515
+ capName: string,
516
+ deviceId: number,
517
+ ) => { addonId: string; nodeId: string } | null;
518
+ }>("device-manager");
519
+ const owner =
520
+ dm?.resolveNativeCapOwnerSync?.(capName, deviceId) ?? null;
521
+ if (!owner) return null;
522
+ return buildNativeCapProxy(broker, owner.addonId, capName, deviceId);
523
+ },
524
+ );
525
+ }
526
+
527
+ // Runner spawn: every non-core, placement-eligible addon runs in its
528
+ // OWN forked runner subprocess by default (base-layer D2/D9 — one
529
+ // addon, one process). No shipped addon declares a shared
530
+ // `execution.group`: Phase 5 dissolved the `pipeline` media group
531
+ // once frames travel as shm `FrameHandle`s and audio over tRPC.
532
+ // `buildAddonGroupPlan` keys the plan by runner id (the addon id);
533
+ // cross-runner / cross-node cap calls travel over Moleculer TCP.
534
+ // `resolveRunnerId` still collapses a shared `execution.group` if
535
+ // one were declared — the mechanism stays available as an explicit
536
+ // opt-in for future co-location needs.
537
+ //
538
+ // `@camstack/core` builtins and `placement: 'agent-only'` addons are
539
+ // excluded from the plan (filtered inside `buildAddonGroupPlan`).
540
+ // A failed runner spawn is logged and the addon surfaces as
541
+ // `addon.error` — there is NO in-process-on-the-hub fallback (it
542
+ // would violate one-addon-one-process). Retry policy for a failed
543
+ // spawn is governed by the kernel circuit-breaker.
544
+ {
545
+ const plan = this.buildAddonGroupPlan(allIds);
546
+ for (const [runnerId, runnerAddons] of plan) {
547
+ try {
548
+ await this.initializeAddonGroup(runnerId, runnerAddons);
549
+ for (const { addonId } of runnerAddons) {
550
+ this.wireCapabilities(addonId);
551
+ }
552
+ } catch (error: unknown) {
553
+ const msg = errMsg(error);
554
+ for (const { addonId } of runnerAddons) {
555
+ this.emitAddonLifecycleEvent("addon.error", addonId, {
556
+ error: msg,
557
+ phase: "init",
558
+ });
559
+ }
560
+ this.logger.error(
561
+ "Runner spawn failed — addons on this runner will be skipped",
562
+ { meta: { runnerId, addonIds: runnerAddons.map((a) => a.addonId), error: msg } },
563
+ );
564
+ }
565
+ }
566
+ }
567
+
568
+ // In-process boot for `@camstack/core` builtins. Core builtins stay
569
+ // resident on the hub process — they provide the storage / settings
570
+ // / logging infrastructure every forked runner depends on, reachable
571
+ // via the hub broker, so `buildAddonGroupPlan` excludes them from
572
+ // the runner plan above. They boot in two ordered passes:
573
+ // infrastructure caps first (`INFRA_CAPABILITIES`,
574
+ // storage→settings→logging), then the remaining builtins in
575
+ // capability-dependency order.
576
+ //
577
+ // A core builtin is identified by `packageName === "@camstack/core"`,
578
+ // NOT by the absence of an `execution` declaration — a builtin may
579
+ // still declare `execution` (e.g. `platform-probe`); the package
580
+ // boundary is the selection criterion.
581
+ const isCoreBuiltin = (id: string): boolean =>
582
+ this.addonEntries.get(id)?.packageName === "@camstack/core";
583
+
584
+ // Pass 1 — infrastructure builtins. A failed REQUIRED infra cap
585
+ // aborts boot: nothing downstream can run without storage/settings.
586
+ for (const infra of INFRA_CAPABILITIES) {
587
+ const addonId = this.findAddonForCapability(infra.name, allIds);
588
+ if (addonId) {
589
+ const entry = this.addonEntries.get(addonId);
590
+ // Only core builtins boot in-process here. A non-core addon that
591
+ // happens to provide an infra cap (e.g. an alternate
592
+ // storage-provider) lives in its own runner — never in-process.
593
+ if (!entry || entry.initialized || !isCoreBuiltin(addonId)) continue;
594
+ try {
595
+ await this.initializeAddon(addonId);
596
+ this.wireCapabilities(addonId);
597
+ } catch (error: unknown) {
598
+ const msg = errMsg(error);
599
+ this.emitAddonLifecycleEvent("addon.error", addonId, {
600
+ error: msg,
601
+ phase: "init",
602
+ });
603
+ if (infra.required) {
604
+ throw new Error(
605
+ `Required infrastructure addon "${addonId}" failed: ${msg}`,
606
+ { cause: error },
607
+ );
608
+ }
609
+ this.logger.warn(
610
+ 'Optional infra addon failed -- continuing',
611
+ { tags: { addonId }, meta: { error: msg } },
612
+ );
613
+ }
614
+ } else if (infra.required) {
615
+ throw new Error(
616
+ `No addon provides required infrastructure capability "${infra.name}"`,
617
+ );
618
+ }
619
+ }
620
+
621
+ // Pass 2 — remaining core builtins, in capability-dependency order.
622
+ const bootOrder = this.capabilityRegistry.getBootOrder();
623
+ const infraNames = new Set(INFRA_CAPABILITIES.map((c) => c.name));
624
+
625
+ for (const capName of bootOrder) {
626
+ if (infraNames.has(capName)) continue; // Handled in pass 1
627
+
628
+ for (const id of allIds) {
629
+ const entry = this.addonEntries.get(id);
630
+ if (!entry || entry.initialized || !isCoreBuiltin(id)) continue;
631
+
632
+ // Check if this addon provides this capability
633
+ const provides = this.getAddonCapabilities(entry.addon);
634
+ if (!provides.some((c) => c.name === capName)) continue;
635
+
636
+ try {
637
+ await this.initializeAddon(id);
638
+ this.wireCapabilities(id);
639
+ } catch (error: unknown) {
640
+ const msg = errMsg(error);
641
+ this.emitAddonLifecycleEvent("addon.error", id, {
642
+ error: msg,
643
+ phase: "init",
644
+ });
645
+ this.logger.error(
646
+ 'Core builtin failed to initialize -- skipping',
647
+ { tags: { addonId: id }, meta: { error: msg } },
648
+ );
649
+ }
650
+ }
651
+ }
652
+
653
+ // Pass 3 — core builtins that declare no capabilities at all.
654
+ for (const id of allIds) {
655
+ const entry = this.addonEntries.get(id);
656
+ if (entry && !entry.initialized && isCoreBuiltin(id)) {
657
+ try {
658
+ await this.initializeAddon(id);
659
+ this.wireCapabilities(id);
660
+ } catch (error: unknown) {
661
+ const msg = errMsg(error);
662
+ this.emitAddonLifecycleEvent("addon.error", id, {
663
+ error: msg,
664
+ phase: "init",
665
+ });
666
+ this.logger.error(
667
+ 'Core builtin failed to initialize -- skipping',
668
+ { tags: { addonId: id }, meta: { error: msg } },
669
+ );
670
+ }
671
+ }
672
+ }
673
+
674
+ const initializedIds = [...this.addonEntries.entries()]
675
+ .filter(([, e]) => e.initialized)
676
+ .map(([id]) => id);
677
+
678
+ this.logger.info(
679
+ 'Addons initialized',
680
+ { meta: { initializedCount: initializedIds.length, totalCount: this.addonEntries.size } },
681
+ );
682
+
683
+ // Health snapshot: every addon entry that survived the loader is
684
+ // either initialized (record success) or failed init (record
685
+ // failure with the captured error). Pre-load failures were already
686
+ // recorded above from `addonLoader.listLoadFailures()`.
687
+ for (const [id, entry] of this.addonEntries.entries()) {
688
+ if (entry.initialized) {
689
+ this.healthMonitor.recordSuccess(entry.packageName, id);
690
+ } else {
691
+ // Init failed (caught by either Phase 1 or Phase 2 catch
692
+ // blocks above). The catch handlers don't carry the error
693
+ // forward to here, so we synthesize a generic one — the real
694
+ // error is in the addon log. Operators see the alert + can
695
+ // open the per-addon logs to investigate.
696
+ this.healthMonitor.recordFailure(
697
+ entry.packageName,
698
+ new Error(`Addon "${id}" failed to initialize — see addon logs for details`),
699
+ id,
700
+ );
701
+ }
702
+ }
703
+
704
+ // Start the kernel-level retry tick (30s). Fires immediately
705
+ // on the first interval boundary; the 5-min boot grace window
706
+ // suppresses alerts until the system has had time to stabilize.
707
+ this.healthMonitor.start();
708
+
709
+ // Group-runner / forked-addon crash detection (point 5 of the
710
+ // operator's spec — "ogni fork/moleculer deve essere considerato").
711
+ // Moleculer's process-service.ts already handles respawn with
712
+ // exponential backoff; we hook the broker's localBus events to
713
+ // GIVE VISIBILITY of those crashes through the same monitor —
714
+ // every crash records a failure, every reconnect records a
715
+ // recovery. The respawn itself stays in process-service.ts.
716
+ //
717
+ // Runner node ids look like `hub/<runnerId>` where `runnerId` is the
718
+ // addon id for a solo runner (the common case — one addon, one
719
+ // process) or the co-location group name for a shared runner (e.g.
720
+ // `hub/pipeline`). We map a disconnected/connected node back to the
721
+ // underlying addons by walking `addonEntries` for any addon whose
722
+ // resolved runner id matches the event's nodeId — every addon on
723
+ // that runner gets a recordFailure / recordSuccess.
724
+ type LocalBusEvent = { node: { id: string } }
725
+ type LocalBus = { on: (event: string, handler: (payload: LocalBusEvent) => void) => void }
726
+ const localBus = (this.moleculer.broker as unknown as { localBus?: LocalBus }).localBus
727
+ if (localBus) {
728
+ localBus.on('$node.disconnected', (payload) => {
729
+ const nodeId = payload.node?.id
730
+ if (!nodeId || nodeId === this.broker.nodeID) return
731
+ for (const [id, entry] of this.addonEntries.entries()) {
732
+ if (!entry.declaration) continue
733
+ const runnerId = resolveRunnerId(entry.declaration, id)
734
+ if (`${this.broker.nodeID}/${runnerId}` !== nodeId) continue
735
+ // Skip operator-initiated restarts — `restartAddon` already
736
+ // waits for caps to re-register and surfaces its own failure
737
+ // path. Recording a transient failure here would flash the
738
+ // "Failed to load" banner on AddonCard during a routine update.
739
+ if (this.restartingAddons.has(id)) continue
740
+ // Record one failure per addon on the disconnected runner.
741
+ // The error message references the nodeId — the operator
742
+ // can correlate with process-service.ts respawn logs.
743
+ this.healthMonitor.recordFailure(
744
+ entry.packageName,
745
+ new Error(`Addon runner ${nodeId} disconnected`),
746
+ id,
747
+ );
748
+ }
749
+ })
750
+ localBus.on('$node.connected', (payload) => {
751
+ const nodeId = payload.node?.id
752
+ if (!nodeId || nodeId === this.broker.nodeID) return
753
+ for (const [id, entry] of this.addonEntries.entries()) {
754
+ if (!entry.declaration) continue
755
+ const runnerId = resolveRunnerId(entry.declaration, id)
756
+ if (`${this.broker.nodeID}/${runnerId}` !== nodeId) continue
757
+ this.healthMonitor.recordSuccess(entry.packageName, id);
758
+ }
759
+ })
760
+ }
761
+
762
+ this.eventBusService.emit({
763
+ id: randomUUID(),
764
+ timestamp: new Date(),
765
+ source: { type: "core", id: "addon-registry" },
766
+ category: EventCategory.SystemAddonsReady,
767
+ data: { activeAddons: initializedIds },
768
+ });
769
+ }
770
+
771
+ /**
772
+ * Reload a single package — invoked by AddonHealthMonitor's retry
773
+ * loop and by the operator-facing `addons.retryLoad` tRPC procedure.
774
+ *
775
+ * Two paths:
776
+ * 1. Package not yet in the registry (e.g. import failed at boot
777
+ * time): re-walk the addons dir, pick up any new package, run
778
+ * `initializeAddon` for each contained addon.
779
+ * 2. Package already in the registry but its addon entries failed
780
+ * init: call `restartAddon` for each addon in that package.
781
+ *
782
+ * On success the monitor automatically records the addon healthy
783
+ * (no explicit recordSuccess needed — the `attemptRetry` no-throw
784
+ * fast path handles it). On failure this method throws and the
785
+ * monitor's catch path schedules the next retry.
786
+ */
787
+ async tryReloadPackage(packageName: string): Promise<void> {
788
+ const dataDir = path.resolve(process.env.CAMSTACK_DATA ?? "camstack-data");
789
+ const addonsDir = path.resolve(dataDir, "addons");
790
+ const addonDir = path.join(addonsDir, packageName);
791
+
792
+ if (!fs.existsSync(addonDir)) {
793
+ throw new Error(`Package directory missing: ${addonDir}`);
794
+ }
795
+
796
+ // Branch 1: package known to the registry already → restart each
797
+ // addon entry. `restartAddon` rebuilds AddonContext + runs init
798
+ // again; on success the entry's `initialized` flag flips back to
799
+ // true and the monitor's auto-resolve path records success.
800
+ const knownAddonIds: string[] = [];
801
+ for (const [id, entry] of this.addonEntries.entries()) {
802
+ if (entry.packageName === packageName) {
803
+ knownAddonIds.push(id);
804
+ }
805
+ }
806
+
807
+ if (knownAddonIds.length > 0) {
808
+ const errors: string[] = [];
809
+ for (const id of knownAddonIds) {
810
+ try {
811
+ await this.restartAddon(id);
812
+ this.healthMonitor.recordSuccess(packageName, id);
813
+ } catch (err) {
814
+ errors.push(`${id}: ${errMsg(err)}`);
815
+ this.healthMonitor.recordFailure(packageName, err, id);
816
+ }
817
+ }
818
+ if (errors.length > 0) {
819
+ throw new Error(`tryReloadPackage failed for ${packageName}: ${errors.join('; ')}`);
820
+ }
821
+ return;
822
+ }
823
+
824
+ // Branch 2: package not yet known → run loadNewAddons which
825
+ // walks disk, instantiates new entries, and initializes them.
826
+ // It records its own loaded/failed in the return; we re-throw
827
+ // when our package is in the failed list so the monitor knows.
828
+ const result = await this.loadNewAddons();
829
+ if (result.failed.length > 0) {
830
+ const failed = result.failed.join(', ');
831
+ throw new Error(`loadNewAddons reported failures: ${failed}`);
832
+ }
833
+ if (result.loaded.length === 0) {
834
+ throw new Error(`No new addons loaded for ${packageName} — package may still have a broken manifest`);
835
+ }
836
+ // Match loaded addons to this package + record success.
837
+ for (const id of result.loaded) {
838
+ const entry = this.addonEntries.get(id);
839
+ if (entry?.packageName === packageName) {
840
+ this.healthMonitor.recordSuccess(packageName, id);
841
+ }
842
+ }
843
+ }
844
+
845
+ /** Health snapshot — exposed to the addons.list tRPC procedure. */
846
+ getAddonHealthSnapshot(): readonly AddonHealthSnapshot[] {
847
+ return this.healthMonitor.getHealthSnapshot();
848
+ }
849
+
850
+ /** Manual user-triggered retry (resets retry counter). */
851
+ async retryAddonLoad(packageName: string): Promise<void> {
852
+ return this.healthMonitor.retryNow(packageName);
853
+ }
854
+
855
+ async onModuleDestroy(): Promise<void> {
856
+ this.logger.info("Shutting down all addons…");
857
+ this.healthMonitor.stop();
858
+ await this.shutdownAll();
859
+ this.logger.info("All addons shut down");
860
+ }
861
+
862
+ /** Set the AddonRouteRegistry for wiring addon-routes capabilities */
863
+ setAddonRouteRegistry(registry: AddonRouteRegistry): void {
864
+ this.addonRouteRegistry = registry;
865
+ }
866
+
867
+ /**
868
+ * Called after app.init() when the tRPC router is available.
869
+ * No-op now: addon `ctx.api` resolves to a broker-routed proxy, so
870
+ * the direct tRPC caller is no longer constructed. Kept for backward
871
+ * compat with `main.ts`'s bootstrap order.
872
+ */
873
+ async setAppRouter(_router: unknown): Promise<void> {
874
+ this.logger.debug(
875
+ "setAppRouter called — broker-routed addon API in use, no direct caller needed",
876
+ );
877
+ }
878
+
879
+ /** Get the CapabilityRegistry for external consumers to register */
880
+ getCapabilityRegistry(): CapabilityRegistry {
881
+ return this.capabilityRegistry;
882
+ }
883
+
884
+ /** Get the CustomActionRegistry — Task 7.2 will use this from the `api.addons.custom` tRPC procedure. */
885
+ getCustomActionRegistry(): CustomActionRegistry {
886
+ return this.customActionRegistry;
887
+ }
888
+
889
+ getDeviceRegistry(): DeviceRegistry {
890
+ return this.deviceRegistry;
891
+ }
892
+
893
+ /** Load persisted collection disabled-lists from settings-store into the registry */
894
+ private loadCollectionPreferences(): void {
895
+ // TODO: implement CapabilityRegistry.loadDisabledProviders() to restore persisted preferences
896
+ }
897
+
898
+ /**
899
+ * Returns the IntegrationRegistry filtered to exclude orphaned integrations
900
+ * (where the addon is not currently installed). Orphaned data stays in DB
901
+ * and reconnects automatically when the addon is reinstalled.
902
+ */
903
+ getIntegrationRegistry():
904
+ | import("@camstack/types").IIntegrationRegistry
905
+ | null {
906
+ if (!this.integrationRegistry) return null;
907
+ return this.createFilteredRegistry(this.integrationRegistry);
908
+ }
909
+
910
+ /**
911
+ * Return the currently-active settings backend (the `settings-store`
912
+ * capability provider). Null if no provider has been registered yet
913
+ * (e.g. during early boot before builtins are loaded). Used by the
914
+ * multi-level addon settings router to read/write addon-device
915
+ * overrides directly.
916
+ */
917
+ getSettingsBackend(): ISettingsBackend | null {
918
+ return this.activeSettingsBackend;
919
+ }
920
+
921
+ /** Get the raw (unfiltered) registry — only for internal addon wiring */
922
+ getRawIntegrationRegistry():
923
+ | import("@camstack/types").IIntegrationRegistry
924
+ | null {
925
+ return this.integrationRegistry;
926
+ }
927
+
928
+ private createFilteredRegistry(
929
+ raw: import("@camstack/types").IIntegrationRegistry,
930
+ ): import("@camstack/types").IIntegrationRegistry {
931
+ const installedAddonIds = new Set([...this.addonEntries.keys()]);
932
+
933
+ // Build set of integration IDs whose addon IS installed
934
+ // Cache per call — lightweight, called infrequently
935
+ let activeIntegrationIds: Set<string> | null = null;
936
+ const ensureActiveIds = async () => {
937
+ if (activeIntegrationIds) return activeIntegrationIds;
938
+ const all = await raw.listIntegrations();
939
+ activeIntegrationIds = new Set(
940
+ all.filter((i) => installedAddonIds.has(i.addonId)).map((i) => i.id),
941
+ );
942
+ return activeIntegrationIds;
943
+ };
944
+
945
+ return {
946
+ // Integrations: filter out orphaned
947
+ createIntegration: (input) => raw.createIntegration(input),
948
+ getIntegration: async (id) => {
949
+ const i = await raw.getIntegration(id);
950
+ return i && installedAddonIds.has(i.addonId) ? i : null;
951
+ },
952
+ getIntegrationByAddonId: async (addonId) => {
953
+ if (!installedAddonIds.has(addonId)) return null;
954
+ return raw.getIntegrationByAddonId(addonId);
955
+ },
956
+ listIntegrations: async () => {
957
+ const all = await raw.listIntegrations();
958
+ return all.filter((i) => installedAddonIds.has(i.addonId));
959
+ },
960
+ updateIntegration: (id, updates) => raw.updateIntegration(id, updates),
961
+ deleteIntegration: (id) => raw.deleteIntegration(id),
962
+
963
+ // Integration settings: passthrough (already gated by getIntegration)
964
+ getIntegrationSettings: (id) => raw.getIntegrationSettings(id),
965
+ setIntegrationSetting: (id, key, value) =>
966
+ raw.setIntegrationSetting(id, key, value),
967
+ setIntegrationSettings: (id, settings) =>
968
+ raw.setIntegrationSettings(id, settings),
969
+
970
+ // Devices: filter out devices belonging to orphaned integrations
971
+ createDevice: (input) => raw.createDevice(input),
972
+ getDevice: async (id) => {
973
+ const d = await raw.getDevice(id);
974
+ if (!d) return null;
975
+ const ids = await ensureActiveIds();
976
+ return ids.has(d.integrationId) ? d : null;
977
+ },
978
+ getDeviceByStableId: async (stableId) => {
979
+ const d = await raw.getDeviceByStableId(stableId);
980
+ if (!d) return null;
981
+ const ids = await ensureActiveIds();
982
+ return ids.has(d.integrationId) ? d : null;
983
+ },
984
+ listDevices: async (integrationId) => {
985
+ const devices = await raw.listDevices(integrationId);
986
+ const ids = await ensureActiveIds();
987
+ return devices.filter((d) => ids.has(d.integrationId));
988
+ },
989
+ listCameras: async () => {
990
+ const cameras = await raw.listCameras();
991
+ const ids = await ensureActiveIds();
992
+ return cameras.filter((d) => ids.has(d.integrationId));
993
+ },
994
+ updateDevice: (id, updates) => raw.updateDevice(id, updates),
995
+ deleteDevice: (id) => raw.deleteDevice(id),
996
+
997
+ // Device settings: passthrough
998
+ getDeviceSettings: (id) => raw.getDeviceSettings(id),
999
+ setDeviceSetting: (id, key, value) =>
1000
+ raw.setDeviceSetting(id, key, value),
1001
+ setDeviceSettings: (id, settings) => raw.setDeviceSettings(id, settings),
1002
+ };
1003
+ }
1004
+
1005
+ // InferenceCapabilitiesService removed — now lives in pipeline-executor addon.
1006
+ // Use capabilityRegistry.getSingleton('pipeline-executor') instead.
1007
+
1008
+ /** Log a standardized lifecycle line for addon start/restart */
1009
+ private logAddonLifecycle(
1010
+ event: "started" | "restarted",
1011
+ id: string,
1012
+ mode: "in-process" | "isolated",
1013
+ ): void {
1014
+ const entry = this.addonEntries.get(id);
1015
+ if (!entry) return;
1016
+ const agentName = process.env.CAMSTACK_AGENT_NAME ?? "hub";
1017
+ const platform = `${os.platform()}/${os.arch()}`;
1018
+ const message = `Addon ${event} — v${entry.packageVersion} (${entry.packageName}), ${mode}, ${platform}, agent: ${agentName}`;
1019
+ // Log under AddonRegistry scope
1020
+ this.logger.info(message, { tags: { addonId: id, agentId: agentName }, meta: { phase: 'lifecycle' } });
1021
+ // Also log under the addon's own tagged logger so it appears in the per-addon
1022
+ // log viewer. No scope — brand bracket shows the addon id.
1023
+ const addonLogger = this.loggingService.createLogger();
1024
+ addonLogger.info(message, { tags: { addonId: id, agentId: agentName } });
1025
+ }
1026
+
1027
+ // Platform probing and inference-config resolution have moved into the
1028
+ // `platform-probe` capability. Addons call `ctx.api.platformProbe.*`
1029
+ // during their own initialize(). The backend no longer loops addons
1030
+ // looking for `getModelRequirements` / `configure` — those hooks were
1031
+ // removed from the ICamstackAddon contract.
1032
+
1033
+ registerAddon(
1034
+ addon: ICamstackAddon,
1035
+ source: AddonSource = "installed",
1036
+ packageName?: string,
1037
+ packageVersion?: string,
1038
+ ): void {
1039
+ const manifest = addon.manifest;
1040
+ if (!manifest) {
1041
+ throw new Error(
1042
+ "Cannot register addon without manifest — was it created via AddonLoader.createInstance()?",
1043
+ );
1044
+ }
1045
+ this.addonEntries.set(manifest.id, {
1046
+ addon,
1047
+ initialized: false,
1048
+ source,
1049
+ packageName: packageName ?? manifest.name ?? manifest.id,
1050
+ packageVersion: packageVersion ?? manifest.version ?? "0.0.0",
1051
+ declaredCapabilities: this.getAddonCapabilities(addon),
1052
+ });
1053
+ }
1054
+
1055
+ async initializeAddon(id: string): Promise<void> {
1056
+ const entry = this.addonEntries.get(id);
1057
+ if (!entry) {
1058
+ throw new Error(`Addon "${id}" is not registered`);
1059
+ }
1060
+
1061
+ if (!entry.addon?.manifest?.id) {
1062
+ throw new Error(
1063
+ `Addon "${id}" has no manifest.id — check the addon class default export`,
1064
+ );
1065
+ }
1066
+
1067
+ // Decide whether this addon should boot in its own forked runner
1068
+ // subprocess. The runner-spawn pass during initial boot already
1069
+ // spawned every eligible addon; `initializeAddon` is the per-addon
1070
+ // entry point reached by the hot-load / restart flows (and by the
1071
+ // Pass 1/2/3 in-process boot for `@camstack/core` builtins). Fork
1072
+ // only when the addon declares an `execution` block, ships an
1073
+ // on-disk `addonDir`, and is not `agent-only`; otherwise (only
1074
+ // `@camstack/core` builtins reach the in-process path below) it
1075
+ // boots in-process on the hub.
1076
+ if (this.isForkedAddonEntry(entry)) {
1077
+ // D5: one-addon-one-process. A forked addon ALWAYS boots in its
1078
+ // own runner — there is NO in-process-on-the-hub fallback. If the
1079
+ // runner spawn fails, the addon is `failed`, surfaced as
1080
+ // `addon.error` + recorded on the health monitor; it does not
1081
+ // silently run on the hub. (Task 7's circuit breaker governs the
1082
+ // retry policy on top of this.)
1083
+ try {
1084
+ await this.broker.call("$process.spawnRunner", {
1085
+ runnerId: id,
1086
+ addons: [{ addonId: id, addonDir: entry.addonDir }],
1087
+ });
1088
+ } catch (err) {
1089
+ const msg = errMsg(err);
1090
+ this.logger.error(
1091
+ 'Failed to spawn isolated runner for addon',
1092
+ { tags: { addonId: id }, meta: { error: msg } },
1093
+ );
1094
+ this.emitAddonLifecycleEvent("addon.error", id, {
1095
+ error: msg,
1096
+ action: "initialize",
1097
+ });
1098
+ this.healthMonitor.recordFailure(entry.packageName, err, id);
1099
+ throw new Error(
1100
+ `Failed to spawn runner for addon "${id}": ${msg}`,
1101
+ { cause: err },
1102
+ );
1103
+ }
1104
+
1105
+ // Provider registration for forkable addons is delegated to the
1106
+ // `CapabilityBridge` (see `MoleculerService.onProviderConnected`).
1107
+ // When the spawned child announces its Moleculer service via
1108
+ // NODE_INFO, the bridge builds a proxy from the capability
1109
+ // definition (`{ id, nodeId, ...methods }`) and registers it.
1110
+ //
1111
+ // Custom actions: read the catalog fresh from the on-disk bundle
1112
+ // and register it against the hub-side `CustomActionRegistry`.
1113
+ // `registerForkedAddonCustomActions` re-`import()`s the entry
1114
+ // (cache-busted), so it covers the hot-load path — where the addon
1115
+ // was registered via `freshLoader` and `this.addonLoader`'s cached
1116
+ // `module` namespace is stale or absent. Dispatch routes through
1117
+ // `broker.call('<addonId>.custom.<action>')`, the only divergence
1118
+ // vs in-process being the transport, exactly like cap methods.
1119
+ await this.registerForkedAddonCustomActions(
1120
+ id,
1121
+ // `shouldFork` already asserted `entry.declaration?.execution`.
1122
+ resolveRunnerId(entry.declaration!, id),
1123
+ );
1124
+
1125
+ entry.initialized = true;
1126
+ this.logger.info('Addon spawned as isolated process', { tags: { addonId: id } });
1127
+ this.emitAddonLifecycleEvent("addon.started", id);
1128
+ return;
1129
+ }
1130
+
1131
+ // In-process initialization — capture providers from initialize() return value
1132
+ const context = await this.createAddonContext(entry.addon);
1133
+ const capturedProviders = new Map<string, unknown>();
1134
+ // Device-manager needs privileged access to CapabilityRegistry — it
1135
+ // resolves native addon ids for getBindings, lists registered wrappers,
1136
+ // etc. Inject before initialize() runs so the addon's onInitialize can
1137
+ // rely on it.
1138
+ const initResult = normalizeAddonInitResult(
1139
+ await entry.addon.initialize(context),
1140
+ );
1141
+ // NOTE: `postBrokerStart()` is deliberately NOT called here. It used to
1142
+ // fire right after `initialize()` and would emit readiness events whose
1143
+ // subscribers (remote workers) immediately called `ctx.api.<addon>.*` —
1144
+ // hitting "Service not found" because `broker.createService()` had not
1145
+ // been called yet. See below: postBrokerStart fires after the Moleculer
1146
+ // service mount so the advertised service is live before readiness.
1147
+
1148
+ for (const reg of initResult?.providers ?? []) {
1149
+ const capName = reg.capability.name;
1150
+ capturedProviders.set(capName, reg.provider);
1151
+ context.registerProvider(capName, reg.provider);
1152
+ // D4: wrapper behaviour is read from the cap DEFINITION. The runtime
1153
+ // ProviderRegistration.kind/defaultActive are deprecated hints — if an
1154
+ // addon still sets them and they disagree, warn (the cap def wins).
1155
+ const kindDrift = describeProviderKindDrift(capName, reg.capability, {
1156
+ kind: reg.kind,
1157
+ defaultActive: reg.defaultActive,
1158
+ });
1159
+ if (kindDrift) {
1160
+ this.logger.warn(kindDrift, {
1161
+ tags: { addonId: id },
1162
+ meta: { capability: capName },
1163
+ });
1164
+ }
1165
+ if (reg.capability.kind === "wrapper") {
1166
+ this.capabilityRegistry.registerWrapper(capName, id, {
1167
+ defaultActive: reg.capability.defaultActive === true,
1168
+ });
1169
+ }
1170
+ }
1171
+
1172
+ // Task 7.1: register system-level custom actions if the addon declares any.
1173
+ if (initResult?.customActions && initResult.actionHandlers) {
1174
+ const handlers = initResult.actionHandlers;
1175
+ try {
1176
+ this.customActionRegistry.registerAddon(
1177
+ id,
1178
+ initResult.customActions,
1179
+ (action, input) => {
1180
+ const fn = handlers[action];
1181
+ if (!fn)
1182
+ throw new Error(
1183
+ `addon '${id}' has no handler for custom action '${action}'`,
1184
+ );
1185
+ return fn(input);
1186
+ },
1187
+ );
1188
+ } catch (err) {
1189
+ this.logger.error(
1190
+ 'Failed to register custom actions for addon',
1191
+ { tags: { addonId: id }, meta: { error: errMsg(err) } },
1192
+ );
1193
+ throw err;
1194
+ }
1195
+ }
1196
+
1197
+ // Validate all declared capabilities have providers registered.
1198
+ // Include caps registered via `context.registerProvider()` side-effects
1199
+ // (e.g. sub-components of an addon calling ctx.registerProvider directly
1200
+ // without returning via ProviderRegistration[]), by reading back from the
1201
+ // CapabilityRegistry for this addonId. This keeps the validation accurate
1202
+ // without forcing addons into a single return-path pattern.
1203
+ if (entry.declaration) {
1204
+ for (const cap of entry.declaration.capabilities ?? []) {
1205
+ const capName = typeof cap === "string" ? cap : cap.name;
1206
+ if (capturedProviders.has(capName)) continue;
1207
+ const provider = this.capabilityRegistry.getProviderByAddon(
1208
+ capName,
1209
+ id,
1210
+ );
1211
+ if (provider) capturedProviders.set(capName, provider);
1212
+ }
1213
+ validateProviderRegistrations(
1214
+ id,
1215
+ entry.declaration,
1216
+ capturedProviders,
1217
+ this.logger,
1218
+ );
1219
+ }
1220
+
1221
+ await this.restoreAddonDevices(id, entry.addon);
1222
+ entry.initialized = true;
1223
+
1224
+ // Register as Moleculer service for remote discoverability
1225
+ if (entry.declaration) {
1226
+ try {
1227
+ // Resolve method names from CapabilityDefinitions registered in the registry
1228
+ const methodResolver = (capName: string): readonly string[] => {
1229
+ const def = this.capabilityRegistry.getDefinition(capName);
1230
+ if (!def?.methods) return [];
1231
+ return Object.keys(def.methods);
1232
+ };
1233
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- createAddonService return type unresolvable due to upstream eventemitter2/moleculer type chain
1234
+ const schema = createAddonService(entry.addon, entry.declaration!, {
1235
+ methodResolver,
1236
+ providers: capturedProviders,
1237
+ });
1238
+ this.broker.createService(schema);
1239
+ } catch (err) {
1240
+ this.logger.warn(
1241
+ 'Failed to register Moleculer service',
1242
+ { tags: { addonId: id }, meta: { error: errMsg(err) } },
1243
+ );
1244
+ }
1245
+ }
1246
+
1247
+ // Post-broker-start readiness — emitted AFTER the Moleculer service is
1248
+ // mounted so that any remote subscriber reacting to a `ready` event can
1249
+ // immediately call `ctx.api.<addon>.*` without hitting a phantom
1250
+ // service window. Hub addons with `autoEmitReadiness=true` emit here;
1251
+ // addons that manage their own readiness override `autoEmitReadiness`.
1252
+ entry.addon.postBrokerStart();
1253
+
1254
+ this.logger.info(
1255
+ 'Addon initialized in-process',
1256
+ { tags: { addonId: id }, meta: { packageName: entry.packageName } },
1257
+ );
1258
+ this.logAddonLifecycle("started", id, "in-process");
1259
+ this.emitAddonLifecycleEvent("addon.started", id);
1260
+ }
1261
+
1262
+ /**
1263
+ * Restore persisted devices for an addon after it has been initialized.
1264
+ * Calls addon.restoreDevices() with the list of devices saved to the DB.
1265
+ * No-op if the addon does not implement restoreDevices() or stores are not available.
1266
+ */
1267
+ private async restoreAddonDevices(
1268
+ addonId: string,
1269
+ addon: ICamstackAddon,
1270
+ ): Promise<void> {
1271
+ if (typeof addon.restoreDevices !== "function") return;
1272
+
1273
+ // Route through the device-persistence capability instead of
1274
+ // accessing DeviceStore/ConfigStore directly. This keeps
1275
+ // AddonRegistryService decoupled from the persistence layer —
1276
+ // the capability addon owns the stores.
1277
+ const api = this.getBrokerApi();
1278
+ try {
1279
+ const listResult: unknown = await (
1280
+ Reflect.get(
1281
+ Reflect.get(api, "deviceManager") as object,
1282
+ "listPersistedByAddon",
1283
+ ) as { query: (input: unknown) => Promise<unknown> }
1284
+ ).query({ addonId });
1285
+
1286
+ const rows = Array.isArray(listResult)
1287
+ ? (listResult as Array<{
1288
+ id: number;
1289
+ stableId: string;
1290
+ type: string;
1291
+ name: string;
1292
+ location?: string | null;
1293
+ disabled?: boolean;
1294
+ parentDeviceId: number | null;
1295
+ }>)
1296
+ : [];
1297
+ if (rows.length === 0) return;
1298
+
1299
+ const savedDevices: SavedDevice[] = await Promise.all(
1300
+ rows.map(async (row) => {
1301
+ const configResult: unknown = await (
1302
+ Reflect.get(
1303
+ Reflect.get(api, "deviceManager") as object,
1304
+ "loadConfig",
1305
+ ) as { query: (input: unknown) => Promise<unknown> }
1306
+ ).query({ deviceId: row.id });
1307
+
1308
+ return {
1309
+ id: row.id,
1310
+ stableId: row.stableId,
1311
+ type: row.type as import("@camstack/types").DeviceType,
1312
+ name: row.name,
1313
+ location: row.location ?? null,
1314
+ disabled: row.disabled ?? false,
1315
+ parentDeviceId: row.parentDeviceId,
1316
+ config: (configResult ?? {}) as Record<string, unknown>,
1317
+ };
1318
+ }),
1319
+ );
1320
+
1321
+ await addon.restoreDevices!(savedDevices);
1322
+ this.logger.info(
1323
+ 'Restored devices for addon',
1324
+ { tags: { addonId }, meta: { count: savedDevices.length } },
1325
+ );
1326
+ } catch (err) {
1327
+ this.logger.error(
1328
+ 'restoreDevices failed for addon',
1329
+ { tags: { addonId }, meta: { error: errMsg(err) } },
1330
+ );
1331
+ }
1332
+ }
1333
+
1334
+ /**
1335
+ * Refresh the package version for all addons from a given package.
1336
+ * Call this after an update to ensure the UI shows the new version.
1337
+ */
1338
+ refreshPackageVersion(packageName: string, newVersion: string): void {
1339
+ for (const [, entry] of this.addonEntries) {
1340
+ if (entry.packageName === packageName) {
1341
+ entry.packageVersion = newVersion;
1342
+ this.logger.info(
1343
+ 'Updated addon packageVersion',
1344
+ { tags: { addonId: entry.addon.manifest!.id }, meta: { newVersion } },
1345
+ );
1346
+ }
1347
+ }
1348
+ }
1349
+
1350
+ /** Emit addon.uninstalled lifecycle event for all addons belonging to a package */
1351
+ emitUninstallEvent(packageName: string): void {
1352
+ for (const [id, entry] of this.addonEntries) {
1353
+ if (entry.packageName === packageName) {
1354
+ this.emitAddonLifecycleEvent("addon.uninstalled", id);
1355
+ }
1356
+ }
1357
+ // Drop the package from the health monitor so its retry loop
1358
+ // doesn't keep trying to reload an addon the operator explicitly
1359
+ // removed. Matching `clearLoadFailures` purges any pre-init
1360
+ // failures still tracked by the addon-loader.
1361
+ this.healthMonitor.forget(packageName);
1362
+ this.addonLoader.clearLoadFailures(packageName);
1363
+ }
1364
+
1365
+ /** Emit addon.updated lifecycle event for all addons belonging to a package */
1366
+ emitUpdateEvent(
1367
+ packageName: string,
1368
+ fromVersion: string,
1369
+ toVersion: string,
1370
+ ): void {
1371
+ for (const [id, entry] of this.addonEntries) {
1372
+ if (entry.packageName === packageName) {
1373
+ this.emitAddonLifecycleEvent("addon.updated", id, {
1374
+ fromVersion,
1375
+ toVersion,
1376
+ });
1377
+ }
1378
+ }
1379
+ }
1380
+
1381
+ /**
1382
+ * Restart an addon by ID.
1383
+ * Shuts down, unregisters capabilities, re-initializes, and re-wires.
1384
+ */
1385
+ async restartAddon(
1386
+ addonId: string,
1387
+ ): Promise<{ success: boolean; error?: string }> {
1388
+ const entry = this.addonEntries.get(addonId);
1389
+ if (!entry) {
1390
+ return { success: false, error: `Addon "${addonId}" not found` };
1391
+ }
1392
+
1393
+ this.logger.info('Addon restarting...', { tags: { addonId }, meta: { phase: 'lifecycle' } });
1394
+
1395
+ // Suppress the "Failed to load" health banner during operator-initiated
1396
+ // restarts. The Moleculer `$node.disconnected` handler skips entries
1397
+ // present in `restartingAddons`; a 90s safety timer clears the flag
1398
+ // even if the restart path throws before the finally block runs.
1399
+ const existingTimer = this.restartingAddons.get(addonId);
1400
+ if (existingTimer) clearTimeout(existingTimer);
1401
+ const safetyTimer = setTimeout(() => {
1402
+ this.restartingAddons.delete(addonId);
1403
+ }, 90_000);
1404
+ this.restartingAddons.set(addonId, safetyTimer);
1405
+
1406
+ try {
1407
+ // Group-runner-hosted addon — delegate to $process.restart for the group
1408
+ if (this.isForkedAddonEntry(entry)) {
1409
+ const result = (await this.broker.call("$process.restart", {
1410
+ name: addonId,
1411
+ })) as { success: boolean; reason?: string };
1412
+ if (!result.success) {
1413
+ throw new Error(
1414
+ `Process restart failed: ${result.reason ?? "unknown"}`,
1415
+ );
1416
+ }
1417
+
1418
+ // $process.restart resolves as soon as the child is respawned, not when its
1419
+ // capabilities are re-registered. Callers (integrations.create, UI forms) may
1420
+ // immediately try to route to the provider and hit a transient null. Block here
1421
+ // until every declared capability is back on the registry so the restart is
1422
+ // observable-consistent from the caller's perspective.
1423
+ const declared = entry.declaredCapabilities;
1424
+ if (declared.length > 0) {
1425
+ // Group-runner restarts pay a multi-step cost: child fork
1426
+ // (~1s) → addon init (Python pool warmup ~5-10s, or a CLI
1427
+ // subprocess probe for tailscale-ingress etc.) → capability
1428
+ // advertisement to hub via Moleculer service discovery
1429
+ // (~5s on cold start). Any fixed ceiling is wrong — a slow
1430
+ // restart that DID eventually succeed produces a misleading
1431
+ // "did not re-register in time" error. Wait indefinitely: if
1432
+ // the group-runner crashed for real, `$node.disconnected`
1433
+ // surfaces it through the health monitor; the operator can
1434
+ // always click Cancel on the UI mutation.
1435
+ const waits = declared.map((cap) =>
1436
+ this.capabilityRegistry.waitForProvider(cap.name, addonId, Number.POSITIVE_INFINITY),
1437
+ );
1438
+ const settled = await Promise.all(waits);
1439
+ const missing = declared
1440
+ .map((cap, i) => (settled[i] == null ? cap.name : null))
1441
+ .filter((name): name is string => name !== null);
1442
+ if (missing.length > 0) {
1443
+ throw new Error(
1444
+ `Addon "${addonId}" restarted but did not re-register capabilities in time: ${missing.join(", ")}`,
1445
+ );
1446
+ }
1447
+ }
1448
+
1449
+ // Re-register the addon's custom-action catalog. `$process.restart`
1450
+ // respawns the group child and `CapabilityBridge` re-registers cap
1451
+ // providers — but custom actions live in the hub-side
1452
+ // `CustomActionRegistry`, which the restart path never touched.
1453
+ // Without this, every hot-update of a group-hosted addon silently
1454
+ // drops its custom actions (the catalog is only registered once,
1455
+ // at boot, in `initializeAddonGroup`). Reads a fresh catalog from
1456
+ // the just-updated on-disk bundle.
1457
+ const runnerId = resolveRunnerId(entry.declaration!, addonId);
1458
+ await this.registerForkedAddonCustomActions(addonId, runnerId);
1459
+
1460
+ this.logAddonLifecycle("restarted", addonId, "isolated");
1461
+ this.emitAddonLifecycleEvent("addon.restarted", addonId);
1462
+ return { success: true };
1463
+ }
1464
+
1465
+ // D5: a forked addon ALWAYS restarts via `$process.restart` above
1466
+ // (process restart — no in-process hot-reload of a live forked
1467
+ // addon). This branch is reached only by `@camstack/core` builtins
1468
+ // — they have no runner and legitimately reload in-process:
1469
+ // shutdown, unregister, re-initialize, re-wire.
1470
+ if (entry.initialized && entry.addon.shutdown) {
1471
+ await entry.addon.shutdown();
1472
+ }
1473
+ // Drain disposers registered via ctx.addDisposer(...)
1474
+ await this.drainDisposerChain(addonId);
1475
+
1476
+ // Unregister all capabilities provided by this addon
1477
+ for (const cap of entry.declaredCapabilities) {
1478
+ this.capabilityRegistry.unregisterProvider(cap.name, addonId);
1479
+ }
1480
+ const manifestCaps = this.getAddonCapabilities(entry.addon);
1481
+ for (const cap of manifestCaps) {
1482
+ this.capabilityRegistry.unregisterProvider(cap.name, addonId);
1483
+ }
1484
+ this.capabilityRegistry.unregisterAllWrappersForAddon(addonId);
1485
+
1486
+ // Task 7.1: drop any registered custom actions so re-initialization re-registers cleanly.
1487
+ this.customActionRegistry.unregisterAddon(addonId);
1488
+
1489
+ // Destroy existing Moleculer service for this addon
1490
+ try {
1491
+ await (
1492
+ this.moleculer.broker as unknown as {
1493
+ destroyService(name: string): Promise<void>;
1494
+ }
1495
+ ).destroyService(addonId);
1496
+ } catch {
1497
+ // Service may not exist if it was never registered
1498
+ }
1499
+
1500
+ entry.initialized = false;
1501
+
1502
+ // Re-initialize and re-wire capabilities
1503
+ await this.initializeAddon(addonId);
1504
+ this.wireCapabilities(addonId);
1505
+
1506
+ this.logAddonLifecycle("restarted", addonId, "in-process");
1507
+ this.emitAddonLifecycleEvent("addon.restarted", addonId);
1508
+ return { success: true };
1509
+ } catch (err) {
1510
+ const msg = errMsg(err);
1511
+ this.logger.error('Failed to restart addon', { tags: { addonId }, meta: { error: msg } });
1512
+ this.emitAddonLifecycleEvent("addon.error", addonId, {
1513
+ error: msg,
1514
+ action: "restart",
1515
+ });
1516
+ // Record the failure on the health monitor so the addon shows up
1517
+ // `failed` with its `lastError` instead of a stale `healthy`. The
1518
+ // in-process branch already unregistered every capability before
1519
+ // `initializeAddon` threw — without this the addon is invisible-
1520
+ // broken: caps gone, health green, nothing on the Addons page.
1521
+ // The "operator sees the error via the mutation result" assumption
1522
+ // only holds for a UI-clicked restart; a `camstack deploy` or a
1523
+ // `requiresRestart` auto-restart is fire-and-forget — the failure
1524
+ // would otherwise vanish. recordFailure also arms the retry loop.
1525
+ this.healthMonitor.recordFailure(entry.packageName, err, addonId);
1526
+ return { success: false, error: msg };
1527
+ } finally {
1528
+ // Clear the suppression flag regardless of success/failure — if the
1529
+ // restart failed, the operator will see the real error via the
1530
+ // mutation result rather than a misleading transient health blip.
1531
+ const timer = this.restartingAddons.get(addonId);
1532
+ if (timer) clearTimeout(timer);
1533
+ this.restartingAddons.delete(addonId);
1534
+ }
1535
+ }
1536
+
1537
+ getAddon(id: string): ICamstackAddon | null {
1538
+ const entry = this.addonEntries.get(id);
1539
+ return entry?.addon ?? null;
1540
+ }
1541
+
1542
+ getAddonEntry(id: string): { addon: ICamstackAddon } | null {
1543
+ const entry = this.addonEntries.get(id);
1544
+ return entry ? { addon: entry.addon } : null;
1545
+ }
1546
+
1547
+ /**
1548
+ * Returns the on-disk package directory for the given addon ID.
1549
+ * Used by the /api/addon-assets route to serve static files (e.g. SVG icons).
1550
+ *
1551
+ * Falls back to a disk scan when the addon isn't in `addonEntries`
1552
+ * (typical for failed-to-load packages: dist/manifest still on disk
1553
+ * but the import threw, so the registry never recorded an entry).
1554
+ * Without this fallback the icon endpoint 404s and the operator
1555
+ * sees a broken-image placeholder for any failed addon.
1556
+ */
1557
+ getAddonPackageDir(id: string): string | null {
1558
+ const entry = this.addonEntries.get(id);
1559
+ if (entry?.addonDir) return entry.addonDir;
1560
+ return this.findAddonDirOnDisk(id);
1561
+ }
1562
+
1563
+ private findAddonDirOnDisk(addonId: string): string | null {
1564
+ const dataDir = path.resolve(process.env.CAMSTACK_DATA ?? "camstack-data");
1565
+ const addonsDir = path.resolve(dataDir, "addons");
1566
+ if (!fs.existsSync(addonsDir)) return null;
1567
+
1568
+ const visit = (pkgDir: string): string | null => {
1569
+ const pkgJsonPath = path.join(pkgDir, "package.json");
1570
+ if (!fs.existsSync(pkgJsonPath)) return null;
1571
+ try {
1572
+ const pkg = JSON.parse(fs.readFileSync(pkgJsonPath, "utf-8")) as Record<string, unknown>;
1573
+ const camstack = (pkg["camstack"] as { addons?: unknown[] } | undefined) ?? {};
1574
+ const addons = Array.isArray(camstack.addons) ? camstack.addons as Record<string, unknown>[] : [];
1575
+ if (addons.some((a) => a["id"] === addonId)) return pkgDir;
1576
+ } catch {
1577
+ return null;
1578
+ }
1579
+ return null;
1580
+ };
1581
+
1582
+ const entries = fs.readdirSync(addonsDir, { withFileTypes: true });
1583
+ for (const entry of entries) {
1584
+ if (!entry.isDirectory()) continue;
1585
+ const dirName = entry.name;
1586
+ if (dirName === "node_modules" || dirName.startsWith(".")) continue;
1587
+ const fullPath = path.join(addonsDir, dirName);
1588
+ if (dirName.startsWith("@")) {
1589
+ for (const sub of fs.readdirSync(fullPath, { withFileTypes: true })) {
1590
+ if (!sub.isDirectory()) continue;
1591
+ const found = visit(path.join(fullPath, sub.name));
1592
+ if (found) return found;
1593
+ }
1594
+ } else {
1595
+ const found = visit(fullPath);
1596
+ if (found) return found;
1597
+ }
1598
+ }
1599
+ return null;
1600
+ }
1601
+
1602
+ /**
1603
+ * Resolve an addon's on-disk installation directory and the dist
1604
+ * sub-directory that contains its built entry. For singleton-package
1605
+ * addons the dist sub-directory is `dist/`; for bundled addons (where
1606
+ * one npm package ships multiple addon entries) it's `dist/<entryId>/`.
1607
+ *
1608
+ * Used by the static-file route handler that serves widget bundles
1609
+ * (`/api/addon-widgets/:addonId/*`) — post bundle merge, the addonId →
1610
+ * package-path mapping is no longer 1:1, so callers MUST go through
1611
+ * the registry instead of guessing `addons/@camstack/addon-<id>/dist/`.
1612
+ *
1613
+ * Returns null if the addonId is unknown or the entry's package.json
1614
+ * has no `entry` field. The returned `distDir` is an absolute path.
1615
+ */
1616
+ getAddonInstallPath(addonId: string): { addonDir: string; distDir: string } | null {
1617
+ const entry = this.addonEntries.get(addonId);
1618
+ if (!entry?.addonDir || !entry.declaration?.entry) return null;
1619
+ const distSubdir = path.dirname(entry.declaration.entry);
1620
+ return {
1621
+ addonDir: entry.addonDir,
1622
+ distDir: path.resolve(entry.addonDir, distSubdir),
1623
+ };
1624
+ }
1625
+
1626
+ /**
1627
+ * Build an AddonContext for a given addon — same as what initialize() receives.
1628
+ * Used by the benchmark system to create fresh addon instances with custom config.
1629
+ */
1630
+ async buildAddonContext(addonId: string): Promise<AddonContext | null> {
1631
+ const entry = this.addonEntries.get(addonId);
1632
+ if (!entry) return null;
1633
+ return this.createAddonContext(entry.addon);
1634
+ }
1635
+
1636
+ /**
1637
+ * True when the entry's manifest opted into protection (uninstall
1638
+ * disabled). The flag is per-addon (`camstack.addons[N].protected`)
1639
+ * but the kernel uninstall API operates on packages — so a package
1640
+ * is treated as protected if ANY of its addons declares the flag.
1641
+ * Computed at call time from the running registry; no hardcoded
1642
+ * package list to keep in sync.
1643
+ */
1644
+ isPackageProtected(packageName: string): boolean {
1645
+ for (const entry of this.addonEntries.values()) {
1646
+ if (entry.packageName !== packageName) continue;
1647
+ if (entry.declaration?.protected === true) return true;
1648
+ }
1649
+ return false;
1650
+ }
1651
+
1652
+ /**
1653
+ * True when the entry boots in its own forked runner: it declares an
1654
+ * `execution` block, ships an on-disk `addonDir`, and is not
1655
+ * `agent-only`. Everything else (only `@camstack/core` builtins)
1656
+ * boots in-process on the hub. The type predicate narrows `addonDir`
1657
+ * to `string` for callers.
1658
+ */
1659
+ private isForkedAddonEntry(
1660
+ entry: AddonEntry,
1661
+ ): entry is AddonEntry & { addonDir: string } {
1662
+ return !!(
1663
+ entry.declaration?.execution &&
1664
+ entry.addonDir &&
1665
+ resolveAddonPlacement(entry.declaration) !== 'agent-only'
1666
+ );
1667
+ }
1668
+
1669
+ /** Per-entry helper used by `listAddons()` to mark a row removable / not. */
1670
+ private isRequiredEntry(entry: AddonEntry): boolean {
1671
+ if (entry.declaration?.protected === true) return true;
1672
+ // Fall back to a sibling addon in the same package — the
1673
+ // protected-by-association rule above. Catches the case where the
1674
+ // SAME package ships an aggregator addon (protected: true) plus
1675
+ // a source addon (protected: false): both rows should render
1676
+ // as non-removable because uninstalling the package would tear
1677
+ // out both.
1678
+ return this.isPackageProtected(entry.packageName);
1679
+ }
1680
+
1681
+ listAddons(): Array<{
1682
+ manifest: AddonDeclaration & {
1683
+ packageName: string;
1684
+ packageVersion: string;
1685
+ packageDisplayName?: string;
1686
+ bundle?: { displayName: string; description?: string; icon?: string };
1687
+ protected?: boolean;
1688
+ removable?: boolean;
1689
+ };
1690
+ declaration?: AddonDeclaration;
1691
+ source: AddonSource;
1692
+ installSource?: "npm" | "local" | "upload";
1693
+ process?: { pid?: number; mode: "in-process"; state: string };
1694
+ }> {
1695
+ // Build process info map from in-memory data (sync, no stats)
1696
+ const processMap = new Map<
1697
+ string,
1698
+ { pid?: number; mode: "in-process"; state: string }
1699
+ >();
1700
+ for (const [id, entry] of this.addonEntries) {
1701
+ if (entry.initialized) {
1702
+ processMap.set(`addon:${id}`, {
1703
+ pid: process.pid,
1704
+ mode: "in-process",
1705
+ state: "running",
1706
+ });
1707
+ }
1708
+ }
1709
+
1710
+ const live = Array.from(this.addonEntries.values())
1711
+ .filter((entry) => entry.addon?.manifest?.id)
1712
+ .map((entry) => {
1713
+ let installSource: "npm" | "local" | "upload" | undefined;
1714
+ if (entry.addonDir) {
1715
+ try {
1716
+ const markerPath = path.join(entry.addonDir, ".install-source");
1717
+ if (fs.existsSync(markerPath)) {
1718
+ const raw = fs.readFileSync(markerPath, "utf-8").trim();
1719
+ // Normalize legacy 'workspace' → 'local'
1720
+ const normalized = raw === "workspace" ? "local" : raw;
1721
+ if (
1722
+ normalized === "npm" ||
1723
+ normalized === "local" ||
1724
+ normalized === "upload"
1725
+ ) {
1726
+ installSource = normalized;
1727
+ }
1728
+ }
1729
+ } catch (err) {
1730
+ this.logger.debug(
1731
+ 'Failed to read install-source marker for addon',
1732
+ { meta: { error: errMsg(err) } },
1733
+ );
1734
+ }
1735
+ }
1736
+ return {
1737
+ manifest: {
1738
+ ...entry.addon.manifest!,
1739
+ packageName: entry.packageName,
1740
+ packageVersion: entry.packageVersion,
1741
+ packageDisplayName: entry.packageDisplayName,
1742
+ ...(entry.bundle !== undefined ? { bundle: entry.bundle } : {}),
1743
+ protected: this.isRequiredEntry(entry) || undefined,
1744
+ removable: this.isRequiredEntry(entry) ? false : undefined,
1745
+ },
1746
+ declaration: entry.declaration,
1747
+ source: entry.source,
1748
+ installSource,
1749
+ process: processMap.get(`addon:${entry.addon.manifest!.id}`),
1750
+ };
1751
+ });
1752
+
1753
+ // Surface packages that exist on disk but failed to load — typical
1754
+ // causes are dep-version mismatch, missing native binary, or a
1755
+ // corrupted dist. Without this, an operator sees "installed" via
1756
+ // the package manifest but can't find the row in the addons UI to
1757
+ // diagnose or uninstall, leaving the package stranded.
1758
+ const seenPackages = new Set(live.map((row) => row.manifest.packageName));
1759
+ const failed = this.scanFailedToLoadPackages(seenPackages);
1760
+ return [...live, ...failed];
1761
+ }
1762
+
1763
+ /**
1764
+ * Walk `<dataDir>/addons/` for `package.json` files whose declared
1765
+ * `camstack.addons[]` entries don't appear in `addonEntries`. Returns
1766
+ * synthetic listing rows so the UI can show + delete them. The rows
1767
+ * carry a `process.state: 'failed'` flag so the AddonCard can render
1768
+ * the diagnostic state distinctly from a normal addon.
1769
+ */
1770
+ private scanFailedToLoadPackages(
1771
+ seenPackages: ReadonlySet<string>,
1772
+ ): Array<{
1773
+ manifest: AddonDeclaration & {
1774
+ packageName: string;
1775
+ packageVersion: string;
1776
+ packageDisplayName?: string;
1777
+ protected?: boolean;
1778
+ removable?: boolean;
1779
+ };
1780
+ declaration?: AddonDeclaration;
1781
+ source: AddonSource;
1782
+ installSource?: "npm" | "local" | "upload";
1783
+ process?: { pid?: number; mode: "in-process"; state: string };
1784
+ }> {
1785
+ const dataDir = path.resolve(process.env.CAMSTACK_DATA ?? "camstack-data");
1786
+ const addonsDir = path.resolve(dataDir, "addons");
1787
+ if (!fs.existsSync(addonsDir)) return [];
1788
+
1789
+ type Row = ReturnType<AddonRegistryService["scanFailedToLoadPackages"]> extends Array<infer R> ? R : never;
1790
+ const out: Row[] = [];
1791
+ const visit = (pkgDir: string): void => {
1792
+ const pkgJsonPath = path.join(pkgDir, "package.json");
1793
+ if (!fs.existsSync(pkgJsonPath)) return;
1794
+ let pkg: Record<string, unknown>;
1795
+ try {
1796
+ pkg = JSON.parse(fs.readFileSync(pkgJsonPath, "utf-8")) as Record<string, unknown>;
1797
+ } catch {
1798
+ return;
1799
+ }
1800
+ const packageName = typeof pkg["name"] === "string" ? pkg["name"] : "";
1801
+ const packageVersion = typeof pkg["version"] === "string" ? pkg["version"] : "0.0.0";
1802
+ if (!packageName || seenPackages.has(packageName)) return;
1803
+ const camstack = (pkg["camstack"] as { addons?: unknown[] } | undefined) ?? {};
1804
+ const addons = Array.isArray(camstack.addons) ? camstack.addons as Record<string, unknown>[] : [];
1805
+ if (addons.length === 0) return;
1806
+
1807
+ let installSource: "npm" | "local" | "upload" | undefined;
1808
+ try {
1809
+ const markerPath = path.join(pkgDir, ".install-source");
1810
+ if (fs.existsSync(markerPath)) {
1811
+ const raw = fs.readFileSync(markerPath, "utf-8").trim();
1812
+ const normalized = raw === "workspace" ? "local" : raw;
1813
+ if (normalized === "npm" || normalized === "local" || normalized === "upload") {
1814
+ installSource = normalized;
1815
+ }
1816
+ }
1817
+ } catch { /* non-critical */ }
1818
+
1819
+ // Manifest-driven protection — agree with the live-rows path.
1820
+ const isProtected = addons.some((a) => a["protected"] === true);
1821
+ const packageDisplayName = (camstack as { displayName?: string }).displayName;
1822
+ // Bundle metadata — present when the package ships multiple addon
1823
+ // entries that should render as collapsible children in the UI.
1824
+ const bundleRaw = (camstack as { bundle?: { displayName?: string; description?: string; icon?: string } }).bundle;
1825
+ const bundle = bundleRaw && typeof bundleRaw.displayName === 'string'
1826
+ ? {
1827
+ displayName: bundleRaw.displayName,
1828
+ ...(bundleRaw.description !== undefined ? { description: bundleRaw.description } : {}),
1829
+ ...(bundleRaw.icon !== undefined ? { icon: bundleRaw.icon } : {}),
1830
+ }
1831
+ : undefined;
1832
+
1833
+ for (const decl of addons) {
1834
+ const id = typeof decl["id"] === "string" ? decl["id"] : "";
1835
+ const name = typeof decl["name"] === "string" ? decl["name"] : id;
1836
+ if (!id) continue;
1837
+ const manifest: Row["manifest"] = {
1838
+ ...(decl as unknown as AddonDeclaration),
1839
+ id,
1840
+ name,
1841
+ packageName,
1842
+ packageVersion,
1843
+ ...(packageDisplayName ? { packageDisplayName } : {}),
1844
+ ...(bundle !== undefined ? { bundle } : {}),
1845
+ protected: isProtected || undefined,
1846
+ removable: !isProtected,
1847
+ };
1848
+ out.push({
1849
+ manifest,
1850
+ declaration: decl as unknown as AddonDeclaration,
1851
+ source: "installed" as AddonSource,
1852
+ ...(installSource ? { installSource } : {}),
1853
+ process: { mode: "in-process" as const, state: "failed" },
1854
+ });
1855
+ }
1856
+ };
1857
+
1858
+ const entries = fs.readdirSync(addonsDir, { withFileTypes: true });
1859
+ for (const entry of entries) {
1860
+ if (!entry.isDirectory()) continue;
1861
+ const dirName = entry.name;
1862
+ if (dirName === "node_modules" || dirName.startsWith(".")) continue;
1863
+ const fullPath = path.join(addonsDir, dirName);
1864
+ // Scoped: walk one level deeper.
1865
+ if (dirName.startsWith("@")) {
1866
+ const scopedEntries = fs.readdirSync(fullPath, { withFileTypes: true });
1867
+ for (const sub of scopedEntries) {
1868
+ if (sub.isDirectory()) visit(path.join(fullPath, sub.name));
1869
+ }
1870
+ } else {
1871
+ visit(fullPath);
1872
+ }
1873
+ }
1874
+ return out;
1875
+ }
1876
+
1877
+ /**
1878
+ * List all addons including core builtins.
1879
+ * Each addon carries packageName/packageVersion from its package.json,
1880
+ * so the frontend can group by package automatically.
1881
+ */
1882
+ listAllAddons() {
1883
+ return this.listAddons();
1884
+ }
1885
+
1886
+ /**
1887
+ * List all addon processes (in-process addons)
1888
+ * as ManagedProcessStatus-compatible entries for the process tree.
1889
+ */
1890
+ async listAddonProcesses(): Promise<
1891
+ ReadonlyArray<{
1892
+ id: string;
1893
+ label: string;
1894
+ state: "running" | "stopped" | "crashed" | "starting";
1895
+ pid?: number;
1896
+ stats?: {
1897
+ pid: number;
1898
+ cpu: number;
1899
+ memory: number;
1900
+ uptime: number;
1901
+ restartCount: number;
1902
+ };
1903
+ restartCount: number;
1904
+ mode: "in-process";
1905
+ }>
1906
+ > {
1907
+ const result: Array<{
1908
+ id: string;
1909
+ label: string;
1910
+ state: "running" | "stopped" | "crashed" | "starting";
1911
+ pid?: number;
1912
+ stats?: {
1913
+ pid: number;
1914
+ cpu: number;
1915
+ memory: number;
1916
+ uptime: number;
1917
+ restartCount: number;
1918
+ };
1919
+ restartCount: number;
1920
+ mode: "in-process";
1921
+ }> = [];
1922
+
1923
+ // Collect hub PID and fetch stats
1924
+ const { getPidStats } = await import("@camstack/core");
1925
+ const pidStats = await getPidStats([process.pid]);
1926
+
1927
+ // In-process addons
1928
+ const hubStats = pidStats.get(process.pid);
1929
+ for (const [id, entry] of this.addonEntries) {
1930
+ if (entry.initialized) {
1931
+ result.push({
1932
+ id: `addon:${id}`,
1933
+ label: `Addon: ${id} (in-process)`,
1934
+ state: "running",
1935
+ pid: process.pid,
1936
+ stats: {
1937
+ pid: process.pid,
1938
+ cpu: hubStats?.cpu ?? 0,
1939
+ memory: hubStats?.memory ?? 0,
1940
+ uptime: Math.round(process.uptime()),
1941
+ restartCount: 0,
1942
+ },
1943
+ restartCount: 0,
1944
+ mode: "in-process",
1945
+ });
1946
+ }
1947
+ }
1948
+
1949
+ return result;
1950
+ }
1951
+
1952
+ /**
1953
+ * Phase 11 (settings redesign): `getAddonConfigSchema`,
1954
+ * `getAddonConfig`, and `updateAddonConfig` — the legacy adapter
1955
+ * shim that synthesised a plain `ConfigUISchema` view from the new
1956
+ * three-level methods — are gone. All settings traffic goes through
1957
+ * the six Phase 5 tRPC endpoints (`addons.getAddonSettings`,
1958
+ * `updateAddonSettings`, `getGlobalSettings`, `updateGlobalSettings`,
1959
+ * `getDeviceSettings`, `updateDeviceSettings`) which call the
1960
+ * addon's methods directly via the `addon-settings` singleton capability.
1961
+ */
1962
+
1963
+ /**
1964
+ * Scan data/addons/ for new packages and initialize any addon not already registered.
1965
+ * Called after install/uninstall to hot-load new addons without server restart.
1966
+ */
1967
+ async loadNewAddons(): Promise<{ loaded: string[]; failed: string[] }> {
1968
+ const dataDir = path.resolve(process.env.CAMSTACK_DATA ?? "camstack-data");
1969
+ const addonsDir = path.resolve(dataDir, "addons");
1970
+
1971
+ // Create a fresh loader to discover what's on disk (for diffing against existing)
1972
+ const freshLoader = new AddonLoader(
1973
+ this.loggingService.createLogger("AddonLoader"),
1974
+ );
1975
+ await freshLoader.loadFromDirectory(addonsDir);
1976
+
1977
+ const loaded: string[] = [];
1978
+ const failed: string[] = [];
1979
+
1980
+ for (const registered of freshLoader.listAddons()) {
1981
+ // Already-registered addon: refresh the in-memory metadata (the
1982
+ // `packageVersion`/`packageDisplayName`/`declaration`/`bundle`
1983
+ // fields stored on the entry) from the fresh disk scan. Without
1984
+ // this step a hot-update via `installFromTgz` would leave
1985
+ // `entry.packageVersion` stale — the forked group-runner picks up
1986
+ // the new code on `restartAddon` but the UI + listAddons() keep
1987
+ // showing the OLD version. We deliberately keep `entry.addon`
1988
+ // (the running instance) and `entry.initialized` untouched so
1989
+ // `restartAddon` still shuts the live instance down cleanly.
1990
+ const existing = this.addonEntries.get(registered.declaration.id);
1991
+ if (existing) {
1992
+ existing.packageVersion = registered.packageVersion;
1993
+ existing.packageName = registered.packageName;
1994
+ existing.packageDisplayName = registered.packageDisplayName;
1995
+ if (registered.bundle !== undefined) {
1996
+ existing.bundle = registered.bundle;
1997
+ }
1998
+ existing.declaration = registered.declaration;
1999
+ existing.declaredCapabilities = (registered.declaration.capabilities ?? []).map(
2000
+ (c: string | CapabilityDeclaration) =>
2001
+ typeof c === "string" ? { name: c, mode: "singleton" as const } : c,
2002
+ );
2003
+ continue;
2004
+ }
2005
+
2006
+ // Skip agent-only addons on the hub. Same rule as the boot loader
2007
+ // path above; without it, post-install reload would re-init
2008
+ // hub-side every agent-only provider (e.g. `hub-forwarder`),
2009
+ // and a hub-resident `hub-forwarder` creates a write→ingest→
2010
+ // write feedback loop because the hub IS the log-receiver.
2011
+ if (isAgentOnlyPlacement(registered.declaration)) {
2012
+ this.logger.debug(
2013
+ 'loadNewAddons: skipping agent-only addon on hub',
2014
+ { tags: { addonId: registered.declaration.id } },
2015
+ );
2016
+ continue;
2017
+ }
2018
+
2019
+ if (registered.declaration.capabilities) {
2020
+ for (const cap of registered.declaration.capabilities) {
2021
+ this.capabilityRegistry.declareFromManifest(
2022
+ cap,
2023
+ registered.declaration.id,
2024
+ );
2025
+ }
2026
+ }
2027
+
2028
+ try {
2029
+ const addon = freshLoader.createInstance(registered.declaration.id);
2030
+ const declCaps = (registered.declaration.capabilities ?? []).map(
2031
+ (c: string | CapabilityDeclaration) =>
2032
+ typeof c === "string" ? { name: c, mode: "singleton" as const } : c,
2033
+ );
2034
+
2035
+ this.addonEntries.set(registered.declaration.id, {
2036
+ addon,
2037
+ initialized: false,
2038
+ source: "installed",
2039
+ packageName: registered.packageName,
2040
+ packageVersion: registered.packageVersion,
2041
+ packageDisplayName: registered.packageDisplayName,
2042
+ ...(registered.bundle !== undefined ? { bundle: registered.bundle } : {}),
2043
+ declaredCapabilities: declCaps,
2044
+ addonDir: path.join(addonsDir, registered.packageName),
2045
+ declaration: registered.declaration,
2046
+ });
2047
+
2048
+ // Initialize the new addon
2049
+ await this.initializeAddon(registered.declaration.id);
2050
+ this.wireCapabilities(registered.declaration.id);
2051
+ loaded.push(registered.declaration.id);
2052
+ this.logger.info(
2053
+ 'Hot-loaded addon',
2054
+ { tags: { addonId: registered.declaration.id }, meta: { packageName: registered.packageName } },
2055
+ );
2056
+ this.emitAddonLifecycleEvent(
2057
+ "addon.installed",
2058
+ registered.declaration.id,
2059
+ );
2060
+ } catch (err) {
2061
+ const msg = errMsg(err);
2062
+ const stack = err instanceof Error ? err.stack : undefined;
2063
+ this.logger.error(
2064
+ 'Failed to hot-load addon',
2065
+ {
2066
+ tags: { addonId: registered.declaration.id },
2067
+ meta: stack ? { error: msg, stack } : { error: msg },
2068
+ },
2069
+ );
2070
+ this.emitAddonLifecycleEvent(
2071
+ "addon.crashed",
2072
+ registered.declaration.id,
2073
+ { error: msg },
2074
+ );
2075
+ failed.push(registered.declaration.id);
2076
+ }
2077
+ }
2078
+
2079
+ // Remove addons that are no longer on disk — the uninstall path.
2080
+ // D5: every non-core addon lives in its own runner subprocess. The
2081
+ // uninstall consolidation is "stop the runner"; the runner exits and
2082
+ // re-handshakes the hub WITHOUT the removed addon (D3 machinery).
2083
+ // Only `@camstack/core` builtins — which have no runner — take the
2084
+ // in-process `shutdown()` + disposer-drain path.
2085
+ const onDiskIds = new Set(
2086
+ freshLoader
2087
+ .listAddons()
2088
+ .map((a: { declaration: { id: string } }) => a.declaration.id),
2089
+ );
2090
+ for (const [id, entry] of this.addonEntries) {
2091
+ if (entry.source === "installed" && !onDiskIds.has(id)) {
2092
+ // Unregister capabilities before stopping the addon.
2093
+ for (const cap of entry.declaredCapabilities) {
2094
+ this.capabilityRegistry.unregisterProvider(cap.name, id);
2095
+ }
2096
+ const manifestCaps = this.getAddonCapabilities(entry.addon);
2097
+ for (const cap of manifestCaps) {
2098
+ this.capabilityRegistry.unregisterProvider(cap.name, id);
2099
+ }
2100
+ this.capabilityRegistry.unregisterAllWrappersForAddon(id);
2101
+ // Task 7.1: drop custom actions for removed addons.
2102
+ this.customActionRegistry.unregisterAddon(id);
2103
+
2104
+ if (this.isForkedAddonEntry(entry) && entry.initialized) {
2105
+ // Forked addon — stop the runner subprocess. `$process.stop`
2106
+ // kills the child and force-evicts its Moleculer node, so the
2107
+ // hub's cap registry drops the gone provider with no per-
2108
+ // operation hub bookkeeping.
2109
+ try {
2110
+ await this.broker.call("$process.stop", { name: id });
2111
+ } catch (err) {
2112
+ this.logger.warn(
2113
+ 'Non-fatal: failed to stop runner for removed addon',
2114
+ { tags: { addonId: id }, meta: { error: errMsg(err) } },
2115
+ );
2116
+ }
2117
+ } else if (entry.initialized) {
2118
+ // `@camstack/core` builtin — reload model is in-process.
2119
+ try {
2120
+ await entry.addon.shutdown();
2121
+ } catch (err) {
2122
+ this.logger.debug(
2123
+ 'Non-fatal: shutdown error for removed addon',
2124
+ { tags: { addonId: id }, meta: { error: errMsg(err) } },
2125
+ );
2126
+ }
2127
+ await this.drainDisposerChain(id);
2128
+ }
2129
+ this.addonEntries.delete(id);
2130
+ this.logger.info('Removed addon (no longer on disk)', { tags: { addonId: id } });
2131
+ }
2132
+ }
2133
+
2134
+ return { loaded, failed };
2135
+ }
2136
+
2137
+ async shutdownAll(): Promise<void> {
2138
+ for (const [id, entry] of this.addonEntries) {
2139
+ if (entry.initialized) {
2140
+ const capabilities = this.getAddonCapabilities(entry.addon);
2141
+ for (const cap of capabilities) {
2142
+ this.capabilityRegistry.unregisterProvider(cap.name, id);
2143
+ }
2144
+ this.capabilityRegistry.unregisterAllWrappersForAddon(id);
2145
+ // Task 7.1: drop custom actions on full shutdown.
2146
+ this.customActionRegistry.unregisterAddon(id);
2147
+ await entry.addon.shutdown();
2148
+ await this.drainDisposerChain(id);
2149
+ entry.initialized = false;
2150
+ }
2151
+ }
2152
+ }
2153
+
2154
+ // --- Private helpers ---
2155
+
2156
+ // registerConsumers() — DELETED. Consumer wiring now in wireCapabilityConsumers().
2157
+
2158
+ /**
2159
+ * v2: wire capability consumer actions via EventBus.
2160
+ * Listens to 'capability:provider-registered' events from CapabilityRegistry
2161
+ * and dispatches setup actions (storage wiring, decoder↔broker, etc).
2162
+ *
2163
+ * Replaces the 11 registerConsumer callbacks from v1.
2164
+ */
2165
+ private wireCapabilityConsumers(): void {
2166
+ if (!this.capabilityRegistry) return;
2167
+
2168
+ this.eventBusService.subscribe(
2169
+ { category: "capability:provider-registered" },
2170
+ (event) => {
2171
+ const rawCapability = event.data["capability"];
2172
+ const rawAddonId = event.data["addonId"];
2173
+ if (typeof rawCapability !== "string" || typeof rawAddonId !== "string")
2174
+ return;
2175
+ const capability = rawCapability;
2176
+ const addonId = rawAddonId;
2177
+
2178
+ // Broadcast readiness on the hub-node scope so subprocess
2179
+ // brokers waiting on this cap (e.g. provider-rtsp's
2180
+ // `system.ready-state` listener for stream-broker, or
2181
+ // `runWorkerDeviceRestoreWithRetry` waiting on
2182
+ // device-manager) wake up. We emit ONLY `{ type: 'node',
2183
+ // nodeId: 'hub' }` — most consumer filters are scope-agnostic
2184
+ // (they match on capName + state), so emitting both `node` and
2185
+ // `global` causes duplicate fan-out (e.g. provider-rtsp's
2186
+ // `republishAll` running twice → repeated `dispatchCamera`
2187
+ // loops). The node-scoped emit is sufficient: the hydrate
2188
+ // path on subprocess brokers replays whichever records the
2189
+ // hub's `$readiness.getSnapshot` returns, scope and all.
2190
+ try {
2191
+ this.moleculer.readinessRegistry.emitReady(capability, { type: 'node', nodeId: 'hub' });
2192
+ } catch (err) {
2193
+ this.logger.warn('emitReady failed', {
2194
+ meta: { capability, addonId, error: errMsg(err) },
2195
+ });
2196
+ }
2197
+
2198
+ switch (capability) {
2199
+ case "storage": {
2200
+ // Storage-unification refactor (Task 8) — the consumer-
2201
+ // facing `storage` cap is now a singleton owned by the
2202
+ // `storage-orchestrator` builtin, exposing the codegen'd
2203
+ // async `IStorageCapProvider` surface. The legacy
2204
+ // synchronous `INewStorageProvider` (filesystem-only) used
2205
+ // by `StorageService.setNewStorageProvider` and the
2206
+ // `addons-data` dataDir resolution at boot is no longer
2207
+ // wired here — filesystem-storage now registers under
2208
+ // `storage-provider` (the upstream collection cap), and
2209
+ // legacy callers fall back to the deterministic
2210
+ // `camstack-data/addons-data/<addonId>` path. Task 17 will
2211
+ // migrate those callers off `INewStorageProvider`
2212
+ // entirely.
2213
+ break;
2214
+ }
2215
+
2216
+ case "settings-store": {
2217
+ const provider = this.capabilityRegistry.getProviderByAddon(
2218
+ "settings-store",
2219
+ addonId,
2220
+ );
2221
+ if (!provider) return;
2222
+ this.activeSettingsBackend = provider;
2223
+ if (isSettingsStore(provider)) {
2224
+ this.configService.setSettingsStore(provider);
2225
+ }
2226
+ this.storageService.setSettingsBackend(provider);
2227
+ this.integrationRegistry = new IntegrationRegistry(provider);
2228
+ void this.integrationRegistry.initialize().then(() => {
2229
+ this.logger.info("IntegrationRegistry initialized", { meta: { phase: 'v2' } });
2230
+ });
2231
+ this.loadCollectionPreferences();
2232
+ // DeviceStore/ConfigStore are owned by the `device-persistence`
2233
+ // capability addon — no longer created here. The addon boots
2234
+ // after `sqlite-settings` and extracts the DB handle via the
2235
+ // capability registry.
2236
+ this.logger.info("Settings backend wired", { meta: { phase: 'v2' } });
2237
+ break;
2238
+ }
2239
+
2240
+ case "log-destination": {
2241
+ const provider = this.capabilityRegistry.getProviderByAddon(
2242
+ "log-destination",
2243
+ addonId,
2244
+ );
2245
+ if (!provider) return;
2246
+ this.loggingService.addDestination(provider);
2247
+ this.logger.info("Log destination added", { meta: { phase: 'v2' } });
2248
+ break;
2249
+ }
2250
+
2251
+ case "restreamer":
2252
+ case "webrtc":
2253
+ case "decoder":
2254
+ case "stream-broker":
2255
+ // No wiring needed — consumers read from capabilityRegistry on demand.
2256
+ break;
2257
+
2258
+ case "addon-routes": {
2259
+ // Route mounting is async for forked/group addons — the
2260
+ // provider is a Moleculer proxy whose `getRoutes()` returns
2261
+ // a Promise (the wire round-trips through the worker). The
2262
+ // EventBus subscriber callback is synchronous, so delegate
2263
+ // to an async helper and surface any failure via the
2264
+ // logger instead of letting a rejected promise (or a
2265
+ // `liveRoutes.some is not a function` TypeError) escape the
2266
+ // subscriber unobserved.
2267
+ void this.mountAddonRoutes(addonId).catch((err: unknown) => {
2268
+ this.logger.error('Failed to mount addon routes', {
2269
+ meta: { phase: 'v2', addonId, error: errMsg(err) },
2270
+ })
2271
+ })
2272
+ break;
2273
+ }
2274
+
2275
+ default:
2276
+ break;
2277
+ }
2278
+ },
2279
+ );
2280
+ }
2281
+
2282
+ /**
2283
+ * Mount the `addon-routes` provider for `addonId` into the
2284
+ * `AddonRouteRegistry`. Handles both co-located (in-process) and
2285
+ * forked/group addons:
2286
+ *
2287
+ * - For a forked addon the provider is a Moleculer proxy whose
2288
+ * `getRoutes()` / `invoke()` return Promises and whose route
2289
+ * descriptors have their `handler` functions stripped by JSON
2290
+ * serialization. We `await getRoutes()`, synthesize bridge
2291
+ * handlers that dispatch through `provider.invoke(...)`, and
2292
+ * translate the captured envelope back onto the Fastify reply.
2293
+ * - For a co-located addon `getRoutes()` resolves to the live route
2294
+ * list with real handlers, so we register the provider directly.
2295
+ */
2296
+ private async mountAddonRoutes(addonId: string): Promise<void> {
2297
+ if (!this.capabilityRegistry) return;
2298
+ const routeProvider = this.capabilityRegistry.getProviderByAddon(
2299
+ "addon-routes",
2300
+ addonId,
2301
+ );
2302
+ if (!routeProvider || !this.addonRouteRegistry) return;
2303
+
2304
+ // `getRoutes()` is synchronous on a co-located provider but
2305
+ // returns a Promise on the Moleculer proxy of a forked addon —
2306
+ // `await` normalizes both. If any route is missing its `handler`
2307
+ // we synthesize one that dispatches via `provider.invoke(...)`
2308
+ // and translates the captured envelope back to the Fastify reply.
2309
+ const liveRoutes = await routeProvider.getRoutes()
2310
+ const handlersMissing = liveRoutes.some((r: { handler?: unknown }) => typeof r.handler !== 'function')
2311
+ if (handlersMissing) {
2312
+ const invokeFn = (routeProvider as { invoke?: (input: unknown) => Promise<unknown> }).invoke
2313
+ if (typeof invokeFn !== 'function') {
2314
+ this.logger.warn('Forked addon-routes provider missing `invoke` method — routes will not dispatch. Use `buildAddonRouteProvider()` from @camstack/types.', {
2315
+ meta: { phase: 'v2', routeProviderId: routeProvider.id },
2316
+ })
2317
+ }
2318
+ // Wrap each route with a bridge handler so the
2319
+ // AddonRouteRegistry can dispatch through the worker.
2320
+ const bridgeRoutes: import('@camstack/types').IAddonHttpRoute[] = (liveRoutes as ReadonlyArray<{ method: string; path: string; access?: string; description?: string }>).map((route) => ({
2321
+ method: route.method as 'GET' | 'POST' | 'PUT' | 'DELETE',
2322
+ path: route.path,
2323
+ access: (route.access ?? 'public') as 'public' | 'authenticated' | 'admin',
2324
+ ...(route.description !== undefined ? { description: route.description } : {}),
2325
+ handler: async (req: { params: Record<string, string>; query: Record<string, string>; body: unknown; headers: Record<string, string>; user?: { id: string; username: string; isAdmin: boolean }; scopedToken?: unknown }, reply: { code: (n: number) => unknown; header: (k: string, v: string) => unknown; type: (m: string) => unknown; send: (data: unknown) => void; redirect: (url: string) => void }) => {
2326
+ if (typeof invokeFn !== 'function') {
2327
+ reply.code(500)
2328
+ reply.send({ error: 'Forked addon-routes provider missing invoke' })
2329
+ return
2330
+ }
2331
+ const envelope = await invokeFn({
2332
+ method: route.method,
2333
+ path: route.path,
2334
+ params: req.params,
2335
+ query: req.query,
2336
+ body: req.body,
2337
+ headers: req.headers,
2338
+ ...(req.user ? { user: req.user } : {}),
2339
+ ...(req.scopedToken ? { scopedToken: req.scopedToken } : {}),
2340
+ }) as { status: number; headers: Record<string, string>; redirectUrl: string | null; body?: unknown; contentType?: string }
2341
+ reply.code(envelope.status)
2342
+ if (envelope.contentType) reply.type(envelope.contentType)
2343
+ for (const [k, v] of Object.entries(envelope.headers)) reply.header(k, v)
2344
+ if (envelope.redirectUrl !== null) {
2345
+ reply.header('Location', envelope.redirectUrl)
2346
+ reply.send('')
2347
+ } else {
2348
+ reply.send(envelope.body)
2349
+ }
2350
+ },
2351
+ }))
2352
+ this.addonRouteRegistry.registerRoutes(
2353
+ routeProvider.id,
2354
+ { id: routeProvider.id, getRoutes: () => bridgeRoutes },
2355
+ )
2356
+ this.logger.info('Addon routes mounted (forked-bridge)', { meta: { phase: 'v2', routeProviderId: routeProvider.id, routes: bridgeRoutes.length } })
2357
+ } else {
2358
+ this.addonRouteRegistry.registerRoutes(
2359
+ routeProvider.id,
2360
+ routeProvider,
2361
+ );
2362
+ this.logger.info('Addon routes mounted', { meta: { phase: 'v2', routeProviderId: routeProvider.id } });
2363
+ }
2364
+ }
2365
+
2366
+ // Cleanup: `addonHasConfigFields` deleted. It was the last reader of
2367
+ // the legacy `ICamstackAddon.getConfigSchema()` method, exposed via
2368
+ // `listAddons().hasConfigSchema` for the old AddonCard settings gate.
2369
+ // `NodeAddonsSettingsPanel` now gracefully shows an empty state when
2370
+ // an addon doesn't implement `getGlobalSettings`, so the backend no
2371
+ // longer needs to pre-compute this flag.
2372
+
2373
+ private emitAddonLifecycleEvent(
2374
+ eventType:
2375
+ | "addon.started"
2376
+ | "addon.stopped"
2377
+ | "addon.restarted"
2378
+ | "addon.updated"
2379
+ | "addon.installed"
2380
+ | "addon.uninstalled"
2381
+ | "addon.crashed"
2382
+ | "addon.error",
2383
+ addonId: string,
2384
+ data?: Record<string, unknown>,
2385
+ ): void {
2386
+ const entry = this.addonEntries.get(addonId);
2387
+ this.eventBusService.emit({
2388
+ id: randomUUID(),
2389
+ timestamp: new Date(),
2390
+ source: {
2391
+ type: "addon",
2392
+ id: addonId,
2393
+ nodeId: this.broker.nodeID,
2394
+ },
2395
+ category: eventType,
2396
+ data: {
2397
+ addonId,
2398
+ packageName: entry?.packageName,
2399
+ packageVersion: entry?.packageVersion,
2400
+ agent: process.env.CAMSTACK_AGENT_NAME ?? "hub",
2401
+ ...data,
2402
+ },
2403
+ });
2404
+ }
2405
+
2406
+ /**
2407
+ * Post-init hook: emit events for newly available capabilities.
2408
+ * Provider registration now happens in initialize() via context.registerProvider().
2409
+ */
2410
+ private wireCapabilities(addonId: string): void {
2411
+ const entry = this.addonEntries.get(addonId);
2412
+ if (!entry?.initialized) return;
2413
+
2414
+ // Emit addon-pages event for UI notification. Cap name is
2415
+ // `addon-pages-source` after the consolidation split — collection
2416
+ // providers register on the source cap; the singleton aggregator
2417
+ // owns `addon-pages` cluster-wide.
2418
+ const declaredCaps = entry.declaredCapabilities.map((c) => c.name);
2419
+ if (declaredCaps.includes("addon-pages-source")) {
2420
+ this.eventBusService.emit({
2421
+ id: randomUUID(),
2422
+ timestamp: new Date(),
2423
+ source: { type: "addon", id: addonId },
2424
+ category: EventCategory.AddonPageReady,
2425
+ data: { addonId, packageName: entry.packageName },
2426
+ });
2427
+ }
2428
+ // Symmetric for `addon-widgets-source` — addons that contribute
2429
+ // widget bundles emit `AddonWidgetReady` so the
2430
+ // <WidgetRegistryProvider> in admin-ui invalidates its aggregator
2431
+ // query and the new bundle becomes available without a page
2432
+ // reload.
2433
+ if (declaredCaps.includes("addon-widgets-source")) {
2434
+ this.eventBusService.emit({
2435
+ id: randomUUID(),
2436
+ timestamp: new Date(),
2437
+ source: { type: "addon", id: addonId },
2438
+ category: EventCategory.AddonWidgetReady,
2439
+ data: { addonId, packageName: entry.packageName },
2440
+ });
2441
+ }
2442
+ }
2443
+
2444
+ private getAddonCapabilities(addon: ICamstackAddon): CapabilityDeclaration[] {
2445
+ const caps = addon?.manifest?.capabilities;
2446
+ if (!caps) return [];
2447
+
2448
+ return caps.map(
2449
+ (cap): CapabilityDeclaration =>
2450
+ typeof cap === "string" ? { name: cap } : cap,
2451
+ );
2452
+ }
2453
+
2454
+ private findAddonForCapability(
2455
+ capName: string,
2456
+ addonIds: string[],
2457
+ ): string | null {
2458
+ for (const id of addonIds) {
2459
+ const entry = this.addonEntries.get(id);
2460
+ if (!entry) continue;
2461
+ // Use declaredCapabilities (from package.json) as source of truth
2462
+ if (entry.declaredCapabilities.some((c) => c.name === capName)) return id;
2463
+ }
2464
+ return null;
2465
+ }
2466
+
2467
+ /**
2468
+ * Build the addonConfig for a given addon — strictly per-addon now.
2469
+ *
2470
+ * The legacy cross-addon merge (hardcoded `ADDON_SYSTEM_SETTINGS` map
2471
+ * copying `system_settings.<section>.*` into the bootstrap config) has
2472
+ * been removed. Addons that need to read shared system settings
2473
+ * sections (`ffmpeg`, `logging`, `recording`, …) do so at runtime via
2474
+ * `ctx.settings.getGlobal({ section: '<name>' })`, which delegates to
2475
+ * `ConfigManager.getSection()` and covers the full SQL → YAML →
2476
+ * RUNTIME_DEFAULTS fallback chain.
2477
+ *
2478
+ * This keeps the kernel agnostic: every addon declares its own needs
2479
+ * in its own `initialize(ctx)` code, with zero kernel-side addon id
2480
+ * special-casing. The only remaining exception is `sqlite-settings`,
2481
+ * which receives `_runtimeDefaults` for first-boot seeding (tracked
2482
+ * separately under the kernel-cleanup roadmap — Point 3d).
2483
+ */
2484
+ private buildAddonConfig(addonId: string): Record<string, unknown> {
2485
+ // Start with per-addon SQL settings (may be empty for most addons)
2486
+ let addonSpecific: Record<string, unknown> = {};
2487
+ try {
2488
+ addonSpecific = this.configService.getAddonConfig(addonId);
2489
+ } catch (err) {
2490
+ this.logger.debug(
2491
+ 'ConfigManager not ready for addon',
2492
+ { tags: { addonId }, meta: { error: errMsg(err) } },
2493
+ );
2494
+ }
2495
+
2496
+ // No addon id special-casing here anymore. Bootstrap-level
2497
+ // concerns (e.g. first-boot seeding from RUNTIME_DEFAULTS) are
2498
+ // owned by the addon itself — sqlite-settings imports the
2499
+ // constant directly from `@camstack/types` instead of relying on
2500
+ // a kernel-injected bootstrap field.
2501
+ return addonSpecific;
2502
+ }
2503
+
2504
+ private async createAddonContext(
2505
+ addon: ICamstackAddon,
2506
+ ): Promise<InternalAddonContext> {
2507
+ const addonId = addon.manifest!.id;
2508
+ const brokerNodeId = this.broker.nodeID;
2509
+ const agentId = brokerNodeId.includes("/")
2510
+ ? brokerNodeId.split("/")[0]!
2511
+ : brokerNodeId;
2512
+ // No scope on the addon root logger — the brand bracket already shows
2513
+ // `[agent/addonId]`, so `(addon:<addonId>)` was pure duplication.
2514
+ // Sub-components add their own scope via `.child('<name>')`.
2515
+ const logger = this.loggingService
2516
+ .createLogger()
2517
+ .withTags({ addonId, nodeId: brokerNodeId, agentId });
2518
+ const bootstrapConfig = this.buildAddonConfig(addonId);
2519
+
2520
+ // Per-addon private data directory — resolved from active storage
2521
+ // provider if available, otherwise falls back to a deterministic
2522
+ // hardcoded path. Addons that need the full storage provider now
2523
+ // resolve it via the capability registry.
2524
+ const storageProvider = this.activeStorageProvider;
2525
+ const dataDir = storageProvider
2526
+ ? await storageProvider.resolve({
2527
+ location: "addons-data",
2528
+ relativePath: addonId,
2529
+ })
2530
+ : `camstack-data/addons-data/${addonId}`;
2531
+
2532
+ const registerProvider = (
2533
+ capabilityName: string,
2534
+ provider: unknown,
2535
+ ): void => {
2536
+ this.capabilityRegistry.registerProvider(
2537
+ capabilityName,
2538
+ addonId,
2539
+ provider,
2540
+ );
2541
+ logger.info(
2542
+ 'Registered provider via context.registerProvider()',
2543
+ { meta: { capabilityName } },
2544
+ );
2545
+ };
2546
+
2547
+ // Raw three-level settings store API. The resolver service is a
2548
+ // thin wrapper over `ConfigManager` that exposes raw reads/writes
2549
+ // for the addon store, the per-device store, and the cluster-wide
2550
+ // yml-backed sections. No schema merging happens here — the addon
2551
+ // combines these raw reads with its own schema via
2552
+ // `hydrateSchema()` inside `getAddonSettings / getGlobalSettings
2553
+ // / getDeviceSettings`.
2554
+ const settingsView: AddonSettingsView =
2555
+ this.configService.createSettingsView(addonId);
2556
+
2557
+ // Device management — unified path for hub and worker addons.
2558
+ // Persistence routes through the `device-manager` capability
2559
+ // addon via `ctx.api.deviceManager.*`. On the hub, `ctx.api` is
2560
+ // a broker-routed proxy that resolves the local
2561
+ // `device-manager` Moleculer service in-process — no network
2562
+ // hop. On a forked worker, the same proxy routes through the TCP
2563
+ // transport to the hub's service. Zero custom $hub.* actions needed.
2564
+ const kernelStreamProbe: import("@camstack/types").IKernelStreamProbe = {
2565
+ probe: (url, options) => this.streamProbe.probe(url, options),
2566
+ probeField: (key, value) => this.streamProbe.probeField(key, value),
2567
+ };
2568
+ const deviceManagerApi: import("@camstack/types").DeviceManagerApi =
2569
+ createBrokerDeviceManagerApi({
2570
+ api: this.getBrokerApi(),
2571
+ addonId,
2572
+ nodeId: this.broker.nodeID,
2573
+ logger,
2574
+ eventBus: this.eventBusService,
2575
+ registry: this.deviceRegistry,
2576
+ capabilityRegistry: this.capabilityRegistry,
2577
+ streamProbe: kernelStreamProbe,
2578
+ });
2579
+
2580
+ const registry = this;
2581
+ const rr = this.moleculer.readinessRegistry;
2582
+ const capHandleCache = new Map<string, CapabilityHandle<unknown>>();
2583
+ function getOrCreateHandle<T>(capName: string, scope: ReadinessScope, timeoutMs: number): CapabilityHandle<T> {
2584
+ const key = `${capName}::${scopeKey(scope)}`;
2585
+ const existing = capHandleCache.get(key);
2586
+ if (existing) return existing as CapabilityHandle<T>;
2587
+ const handle = new CapabilityHandle<T>(capName, scope, rr, timeoutMs);
2588
+ capHandleCache.set(key, handle as CapabilityHandle<unknown>);
2589
+ return handle;
2590
+ }
2591
+ const ctx: InternalAddonContext & {
2592
+ integrationRegistry?: unknown;
2593
+ capabilities: import("@camstack/types").CapabilitiesAccess;
2594
+ } = {
2595
+ id: `addon:${addonId}`,
2596
+ logger,
2597
+ eventBus: this.eventBusService,
2598
+ addonConfig: bootstrapConfig,
2599
+ dataDir,
2600
+ get api() {
2601
+ return registry.getBrokerApi();
2602
+ },
2603
+ integrationRegistry: this.integrationRegistry ?? undefined,
2604
+ // Live capability-collection accessor used by addons like stream-broker
2605
+ // (webrtc fan-out), enrichment-engine, and snapshot orchestrator. Reads
2606
+ // through the hub's CapabilityRegistry so the collection reflects
2607
+ // every addon that has declared itself since the consumer initialized.
2608
+ // Without this, those addons silently saw 0 providers and quietly
2609
+ // skipped the fan-out — no errors, just missing streams/wiring.
2610
+ capabilities: {
2611
+ getCollection: <T = unknown>(
2612
+ capName: string,
2613
+ ): readonly T[] | undefined => {
2614
+ const items = registry.capabilityRegistry.getCollection<T>(capName);
2615
+ return items ?? undefined;
2616
+ },
2617
+ getCollectionEntries: <T = unknown>(
2618
+ capName: string,
2619
+ ): readonly (readonly [string, T])[] | undefined => {
2620
+ const items = registry.capabilityRegistry.getCollectionEntries<T>(capName);
2621
+ return items ?? undefined;
2622
+ },
2623
+ get: <T = unknown>(capName: string): T | undefined => {
2624
+ return registry.capabilityRegistry.getSingleton<T>(capName) ?? undefined;
2625
+ },
2626
+ },
2627
+ deps: new AddonDepsManager(dataDir, logger),
2628
+ kernel: {
2629
+ localNodeId: this.broker.nodeID,
2630
+ storage: storageProvider ?? undefined,
2631
+ deviceRegistry: this.deviceRegistry ?? undefined,
2632
+ devices: deviceManagerApi,
2633
+ cluster: { broker: adaptBrokerToCluster(this.moleculer.broker) },
2634
+ streamProbe: kernelStreamProbe,
2635
+ hwaccel: createKernelHwAccel(),
2636
+ capabilityRegistry: this.capabilityRegistry,
2637
+ readinessRegistry: this.moleculer.readinessRegistry,
2638
+ // D3: handshake-fed native-cap view of the whole cluster. Backed by
2639
+ // `HubNodeRegistry.listNativeCapEntries()` populated by every
2640
+ // `$hub.registerNode` re-handshake. Used by device-manager as the
2641
+ // reliable fallback when push events were lost mid-transport.
2642
+ listClusterNativeCaps: () => this.moleculer.listClusterNativeCaps(),
2643
+ },
2644
+ registerProvider,
2645
+ resolveProvider: <T = unknown>(capName: string): T | null => {
2646
+ return (
2647
+ (registry.capabilityRegistry.getSingleton<T>(capName) as T | null) ??
2648
+ null
2649
+ );
2650
+ },
2651
+ getNativeProvider: <TCap extends CapabilityDefinition>(
2652
+ cap: TCap,
2653
+ deviceId: number,
2654
+ ): InferProvider<TCap> => {
2655
+ // Hub-side addons live on the same process as the CapabilityRegistry;
2656
+ // native providers are always resolvable locally.
2657
+ const local = registry.capabilityRegistry.getNativeProvider<
2658
+ InferProvider<TCap>
2659
+ >(cap.name, deviceId);
2660
+ if (!local) {
2661
+ throw new Error(
2662
+ `no native provider for capability '${cap.name}' on device '${deviceId}'`,
2663
+ );
2664
+ }
2665
+ return local;
2666
+ },
2667
+ fetchDevice: async (deviceId: number) => {
2668
+ const api = registry.getBrokerApi();
2669
+ const binding = await api.deviceManager.getBindings.query({ deviceId });
2670
+ return createDeviceProxy(api, binding);
2671
+ },
2672
+ settings: settingsView,
2673
+ useCapability<T = unknown>(capName: string, scope: ReadinessScope = { type: 'global' }) {
2674
+ return getOrCreateHandle<T>(capName, scope, 15_000);
2675
+ },
2676
+ async acquireCapability<T = unknown>(
2677
+ capName: string,
2678
+ scope: ReadinessScope = { type: 'global' },
2679
+ opts: { timeoutMs?: number } = {},
2680
+ ) {
2681
+ const timeoutMs = opts.timeoutMs ?? 15_000;
2682
+ const handle = getOrCreateHandle<T>(capName, scope, timeoutMs);
2683
+ return handle;
2684
+ },
2685
+ onCapabilityStateChange(
2686
+ capName: string,
2687
+ scope: ReadinessScope,
2688
+ handler: (state: 'ready' | 'down') => void,
2689
+ ) {
2690
+ return rr.onReadyState(capName, scope, (t) => {
2691
+ handler(t.state === 'ready' ? 'ready' : 'down');
2692
+ });
2693
+ },
2694
+ addDisposer(fn: () => void | Promise<void>) {
2695
+ return registry.getOrCreateDisposerChain(addonId).add(fn);
2696
+ },
2697
+ };
2698
+
2699
+ return ctx;
2700
+ }
2701
+
2702
+ /**
2703
+ * Per-addon disposer chain. Created lazily the first time an addon
2704
+ * calls `ctx.addDisposer(...)`. Drained by `restartAddon()` /
2705
+ * `unregisterAddon()` so cleanup callbacks run before the new addon
2706
+ * instance comes up.
2707
+ */
2708
+ private readonly disposerChains = new Map<string, DisposerChain>();
2709
+
2710
+ private getOrCreateDisposerChain(addonId: string): DisposerChain {
2711
+ let chain = this.disposerChains.get(addonId);
2712
+ if (chain == null) {
2713
+ const log = this.loggingService.createLogger().withTags({ addonId });
2714
+ chain = new DisposerChain({
2715
+ onError: (err, index) => {
2716
+ log.error(`Disposer #${index} threw during teardown`, {
2717
+ meta: { error: errMsg(err) },
2718
+ });
2719
+ },
2720
+ });
2721
+ this.disposerChains.set(addonId, chain);
2722
+ }
2723
+ return chain;
2724
+ }
2725
+
2726
+ /**
2727
+ * Drain (and remove) the disposer chain for an addon. Called by the
2728
+ * addon shutdown / restart flow so resources registered via
2729
+ * `ctx.addDisposer(...)` clean up before the next instance boots.
2730
+ */
2731
+ private async drainDisposerChain(addonId: string): Promise<void> {
2732
+ const chain = this.disposerChains.get(addonId);
2733
+ if (chain == null) return;
2734
+ this.disposerChains.delete(addonId);
2735
+ await chain.dispose();
2736
+ }
2737
+
2738
+ // ── Group orchestration (Phase G3 — opt-in, not yet wired into boot) ──
2739
+
2740
+ /**
2741
+ * Compute the runner plan for a set of addon ids (base-layer D2).
2742
+ *
2743
+ * The runner contract is one-addon-one-process by default (D2/D9):
2744
+ * - every addon gets its OWN runner, keyed by the addon id via
2745
+ * `resolveRunnerId` — no shipped addon declares `execution.group`
2746
+ * today;
2747
+ * - the group-collapse path (same `group` → shared runner keyed by
2748
+ * group name) is an explicit opt-in mechanism; Phase 5 dissolved
2749
+ * the last real co-location group (`pipeline`) once frames travel
2750
+ * as shared-memory `FrameHandle`s and audio over tRPC — nothing
2751
+ * passes a live JS reference across a process boundary any more.
2752
+ *
2753
+ * Addons with `placement: 'agent-only'` are dropped (they don't run on
2754
+ * the hub). `@camstack/core` builtins are dropped too — they stay
2755
+ * in-process on the hub (every forked runner connects to the hub
2756
+ * broker via static-URL TCP transit, so cap providers hosted on the
2757
+ * hub broker are reachable from any runner with one hop; moving them
2758
+ * into a sibling subprocess would isolate them on a separate Moleculer
2759
+ * node that child brokers cannot reach).
2760
+ *
2761
+ * Read-only. Used by the bootstrap to plan spawns up-front and by
2762
+ * diagnostics that want to render the current topology.
2763
+ */
2764
+ buildAddonGroupPlan(addonIds: readonly string[]): RunnerPlan {
2765
+ const plan = new Map<string, RunnerAddonPlacement[]>();
2766
+ for (const id of addonIds) {
2767
+ const entry = this.addonEntries.get(id);
2768
+ if (!entry?.declaration || !entry.addonDir) continue;
2769
+ if (entry.packageName === "@camstack/core") continue;
2770
+ const placement = resolveAddonPlacement(entry.declaration);
2771
+ if (placement === "agent-only") continue;
2772
+ const runnerId = resolveRunnerId(entry.declaration, id);
2773
+ const bucket = plan.get(runnerId) ?? [];
2774
+ bucket.push({ addonId: id, addonDir: entry.addonDir });
2775
+ plan.set(runnerId, bucket);
2776
+ }
2777
+ return plan;
2778
+ }
2779
+
2780
+ /**
2781
+ * Spawn ONE runner subprocess that hosts every addon in `addons` and
2782
+ * register their custom-actions catalogs against the shared
2783
+ * CustomActionRegistry. Cap providers register themselves via
2784
+ * `CapabilityBridge.onProviderConnected` — the bridge listens for any
2785
+ * new Moleculer node (the runner's nodeID is `${parent}/${runnerId}`)
2786
+ * and proxies its services into the local `CapabilityRegistry`.
2787
+ *
2788
+ * `runnerId` is the addon id (a solo runner — one addon per process,
2789
+ * D2/D9). It can structurally also be a co-location group name if an
2790
+ * addon ever declared `execution.group`, but no shipped manifest does.
2791
+ *
2792
+ * No-op when the runner is already running (idempotent for tests +
2793
+ * crash respawn paths). Errors propagate so the bootstrap can log the
2794
+ * failed runner and surface every hosted addon as `addon.error`.
2795
+ */
2796
+ async initializeAddonGroup(
2797
+ runnerId: string,
2798
+ addons: readonly RunnerAddonPlacement[],
2799
+ ): Promise<void> {
2800
+ if (addons.length === 0) {
2801
+ throw new Error(`initializeAddonGroup("${runnerId}") requires at least one addon`);
2802
+ }
2803
+ try {
2804
+ await this.broker.call("$process.spawnRunner", {
2805
+ runnerId,
2806
+ addons,
2807
+ });
2808
+ } catch (err: unknown) {
2809
+ throw new Error(
2810
+ `Failed to spawn runner "${runnerId}" (${addons.length} addons): ${errMsg(err)}`,
2811
+ { cause: err },
2812
+ );
2813
+ }
2814
+
2815
+ // Register custom actions for each addon on the runner. Provider
2816
+ // registration for cap methods is handled by
2817
+ // `CapabilityBridge.onProviderConnected` once the runner's
2818
+ // INFO heartbeat lands.
2819
+ for (const { addonId } of addons) {
2820
+ await this.registerForkedAddonCustomActions(addonId, runnerId);
2821
+ // Mark the entry initialized so the in-process core-builtin boot
2822
+ // passes skip it (those passes only touch `@camstack/core`).
2823
+ const entry = this.addonEntries.get(addonId);
2824
+ if (entry) {
2825
+ entry.initialized = true;
2826
+ this.emitAddonLifecycleEvent("addon.started", addonId);
2827
+ }
2828
+ }
2829
+
2830
+ this.logger.info("Addon runner spawned", {
2831
+ meta: { runnerId, addonCount: addons.length, addonIds: addons.map((a) => a.addonId) },
2832
+ });
2833
+ }
2834
+
2835
+ /**
2836
+ * (Re-)register the custom-action catalog for a forked / group-hosted
2837
+ * addon against the shared `CustomActionRegistry`.
2838
+ *
2839
+ * The catalog (zod `input`/`output` specs) is read STATICALLY from the
2840
+ * addon module's `customActions` named export — the handler dispatches
2841
+ * via `broker.call('<addonId>.custom.<action>')`, so the only divergence
2842
+ * vs an in-process addon is the transport, exactly like cap methods.
2843
+ *
2844
+ * Why a fresh import: `this.addonLoader`'s `module` namespace is
2845
+ * captured once at boot. After a hot-update (`installFromTgz` →
2846
+ * `restartAddon`) the on-disk bundle is newer than that cached module,
2847
+ * so re-reading the boot-time `module` would register a STALE catalog
2848
+ * (or none at all, if the addon was first installed after boot). We
2849
+ * re-`import()` the entry with a cache-busting query so Node's ESM
2850
+ * loader hands back the current bundle.
2851
+ *
2852
+ * Idempotent: drops any prior registration first. No-op (with a debug
2853
+ * log) when the addon exports no `customActions` — most addons don't.
2854
+ */
2855
+ private async registerForkedAddonCustomActions(
2856
+ addonId: string,
2857
+ runnerId: string,
2858
+ ): Promise<void> {
2859
+ // Always clear first so a restart that REMOVES custom actions (or an
2860
+ // addon whose entry no longer exports them) doesn't leave stale
2861
+ // entries resolvable.
2862
+ this.customActionRegistry.unregisterAddon(addonId);
2863
+
2864
+ const entry = this.addonEntries.get(addonId);
2865
+ const addonDir = entry?.addonDir;
2866
+ const declarationEntry = entry?.declaration?.entry;
2867
+ if (!addonDir || !declarationEntry) return;
2868
+
2869
+ // Resolve the built entry the same way `AddonLoader.loadDeclaration`
2870
+ // does: `./src/x.ts` → `dist/x.js`, with index.js fallbacks.
2871
+ const entryFile = declarationEntry
2872
+ .replace(/^\.\//, "")
2873
+ .replace(/^src\//, "dist/")
2874
+ .replace(/\.ts$/, ".js");
2875
+ let entryPath = path.resolve(addonDir, entryFile);
2876
+ if (!fs.existsSync(entryPath)) {
2877
+ const base = entryPath.replace(/\.(js|cjs|mjs)$/, "");
2878
+ const alternatives = [
2879
+ `${base}.cjs`,
2880
+ `${base}.mjs`,
2881
+ path.resolve(addonDir, "dist", "index.js"),
2882
+ path.resolve(addonDir, "dist", "index.cjs"),
2883
+ path.resolve(addonDir, "dist", "index.mjs"),
2884
+ path.resolve(addonDir, declarationEntry),
2885
+ ];
2886
+ entryPath = alternatives.find((p) => fs.existsSync(p)) ?? entryPath;
2887
+ }
2888
+ if (!fs.existsSync(entryPath)) return;
2889
+
2890
+ let catalog: unknown;
2891
+ try {
2892
+ // Cache-bust so a hot-updated bundle is re-read instead of served
2893
+ // from Node's ESM module cache.
2894
+ const cacheBustedUrl = `${pathToFileURL(entryPath).href}?t=${Date.now()}`;
2895
+ const modUnknown: unknown = await import(cacheBustedUrl);
2896
+ catalog =
2897
+ modUnknown && typeof modUnknown === "object"
2898
+ ? (modUnknown as Record<string, unknown>)["customActions"]
2899
+ : undefined;
2900
+ } catch (err) {
2901
+ this.logger.warn(
2902
+ "Failed to load custom-action catalog for forked addon",
2903
+ { tags: { addonId }, meta: { error: errMsg(err) } },
2904
+ );
2905
+ return;
2906
+ }
2907
+
2908
+ if (!catalog || typeof catalog !== "object") {
2909
+ this.logger.debug("Forked addon exports no custom actions", {
2910
+ tags: { addonId },
2911
+ meta: { runnerId },
2912
+ });
2913
+ return;
2914
+ }
2915
+
2916
+ this.customActionRegistry.registerAddon(
2917
+ addonId,
2918
+ catalog as import("@camstack/types").CustomActionsSpec,
2919
+ (action, input) => this.broker.call(`${addonId}.custom.${action}`, input),
2920
+ );
2921
+ this.logger.info("Runner addon custom actions registered", {
2922
+ tags: { addonId },
2923
+ meta: { runnerId },
2924
+ });
2925
+ }
2926
+ }