@camstack/server 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +17 -0
- package/package.json +55 -0
- package/src/__tests__/addon-install-e2e.test.ts +75 -0
- package/src/__tests__/addon-pages-e2e.test.ts +178 -0
- package/src/__tests__/addon-route-session.test.ts +17 -0
- package/src/__tests__/addon-settings-router.spec.ts +62 -0
- package/src/__tests__/addon-upload.spec.ts +355 -0
- package/src/__tests__/agent-registry.spec.ts +162 -0
- package/src/__tests__/agent-status-page.spec.ts +84 -0
- package/src/__tests__/auth-session-cookie.test.ts +21 -0
- package/src/__tests__/cap-providers/cap-usage-graph.spec.ts +23 -0
- package/src/__tests__/cap-providers/compute-topology-categories.spec.ts +64 -0
- package/src/__tests__/cap-routers/_meta.spec.ts +200 -0
- package/src/__tests__/cap-routers/addon-settings.router.spec.ts +106 -0
- package/src/__tests__/cap-routers/device-manager-aggregate.router.spec.ts +142 -0
- package/src/__tests__/cap-routers/harness.ts +159 -0
- package/src/__tests__/cap-routers/metrics-provider.router.spec.ts +119 -0
- package/src/__tests__/cap-routers/null-provider-guard.spec.ts +66 -0
- package/src/__tests__/cap-routers/pipeline-executor.router.spec.ts +135 -0
- package/src/__tests__/cap-routers/settings-store.router.spec.ts +247 -0
- package/src/__tests__/capability-e2e.test.ts +386 -0
- package/src/__tests__/cli-e2e.test.ts +129 -0
- package/src/__tests__/core-cap-bridge.spec.ts +89 -0
- package/src/__tests__/embedded-deps-e2e.test.ts +109 -0
- package/src/__tests__/event-bus-proxy-router.spec.ts +72 -0
- package/src/__tests__/fixtures/mock-analysis-addon-a.ts +37 -0
- package/src/__tests__/fixtures/mock-analysis-addon-b.ts +37 -0
- package/src/__tests__/fixtures/mock-log-addon.ts +37 -0
- package/src/__tests__/fixtures/mock-storage-addon.ts +40 -0
- package/src/__tests__/framework-allowlist.spec.ts +95 -0
- package/src/__tests__/https-e2e.test.ts +118 -0
- package/src/__tests__/lifecycle-e2e.test.ts +140 -0
- package/src/__tests__/live-events-subscription.spec.ts +150 -0
- package/src/__tests__/moleculer-register-node-idempotency.spec.ts +229 -0
- package/src/__tests__/oauth2-account-linking.spec.ts +736 -0
- package/src/__tests__/post-boot-restart.spec.ts +161 -0
- package/src/__tests__/singleton-contention.test.ts +487 -0
- package/src/__tests__/streaming-diagnostic.test.ts +512 -0
- package/src/__tests__/streaming-scale.test.ts +280 -0
- package/src/agent-status-page.ts +121 -0
- package/src/api/__tests__/addons-custom.spec.ts +134 -0
- package/src/api/__tests__/capabilities.router.test.ts +47 -0
- package/src/api/addon-upload.ts +472 -0
- package/src/api/addons-custom.router.ts +100 -0
- package/src/api/auth-whoami.ts +99 -0
- package/src/api/bridge-addons.router.ts +120 -0
- package/src/api/capabilities.router.ts +226 -0
- package/src/api/core/__tests__/auth-router-totp.spec.ts +256 -0
- package/src/api/core/addon-settings.router.ts +124 -0
- package/src/api/core/agents.router.ts +87 -0
- package/src/api/core/auth.router.ts +303 -0
- package/src/api/core/cap-providers.ts +993 -0
- package/src/api/core/capabilities.router.ts +119 -0
- package/src/api/core/collection-preference.ts +40 -0
- package/src/api/core/event-bus-proxy.router.ts +45 -0
- package/src/api/core/hwaccel.router.ts +81 -0
- package/src/api/core/live-events.router.ts +60 -0
- package/src/api/core/logs.router.ts +162 -0
- package/src/api/core/notifications.router.ts +65 -0
- package/src/api/core/repl.router.ts +41 -0
- package/src/api/core/settings-backend.router.ts +142 -0
- package/src/api/core/stream-probe.router.ts +57 -0
- package/src/api/core/system-events.router.ts +116 -0
- package/src/api/health/health.routes.ts +123 -0
- package/src/api/oauth2/__tests__/oauth2-routes.spec.ts +52 -0
- package/src/api/oauth2/consent-page.ts +42 -0
- package/src/api/oauth2/oauth2-routes.ts +248 -0
- package/src/api/trpc/__tests__/scope-access-device.spec.ts +223 -0
- package/src/api/trpc/__tests__/scope-access.spec.ts +107 -0
- package/src/api/trpc/cap-mount-helpers.ts +225 -0
- package/src/api/trpc/core-cap-bridge.ts +152 -0
- package/src/api/trpc/generated-cap-mounts.ts +707 -0
- package/src/api/trpc/generated-cap-routers.ts +6340 -0
- package/src/api/trpc/scope-access.ts +110 -0
- package/src/api/trpc/trpc.context.ts +255 -0
- package/src/api/trpc/trpc.middleware.ts +140 -0
- package/src/api/trpc/trpc.router.ts +275 -0
- package/src/auth/session-cookie.ts +44 -0
- package/src/boot/boot-config.ts +278 -0
- package/src/boot/post-boot.service.ts +103 -0
- package/src/core/addon/__tests__/addon-registry-capability.test.ts +53 -0
- package/src/core/addon/addon-package.service.ts +1684 -0
- package/src/core/addon/addon-registry.service.ts +2926 -0
- package/src/core/addon/addon-search.service.ts +90 -0
- package/src/core/addon/addon-settings-provider.ts +276 -0
- package/src/core/addon/addon.tokens.ts +2 -0
- package/src/core/addon-bridge/addon-bridge.service.ts +125 -0
- package/src/core/addon-pages/addon-pages.service.spec.ts +117 -0
- package/src/core/addon-pages/addon-pages.service.ts +80 -0
- package/src/core/addon-widgets/addon-widgets.service.ts +92 -0
- package/src/core/agent/agent-registry.service.ts +507 -0
- package/src/core/auth/auth.service.spec.ts +88 -0
- package/src/core/auth/auth.service.ts +8 -0
- package/src/core/capability/capability.service.ts +57 -0
- package/src/core/config/config.schema.ts +3 -0
- package/src/core/config/config.service.spec.ts +175 -0
- package/src/core/config/config.service.ts +7 -0
- package/src/core/events/event-bus.service.spec.ts +212 -0
- package/src/core/events/event-bus.service.ts +85 -0
- package/src/core/feature/feature.service.spec.ts +96 -0
- package/src/core/feature/feature.service.ts +8 -0
- package/src/core/lifecycle/lifecycle-state-machine.spec.ts +168 -0
- package/src/core/lifecycle/lifecycle-state-machine.ts +3 -0
- package/src/core/logging/log-ring-buffer.ts +3 -0
- package/src/core/logging/logging.service.spec.ts +247 -0
- package/src/core/logging/logging.service.ts +129 -0
- package/src/core/logging/scoped-logger.ts +3 -0
- package/src/core/moleculer/moleculer.service.ts +612 -0
- package/src/core/network/network-quality.service.spec.ts +47 -0
- package/src/core/network/network-quality.service.ts +5 -0
- package/src/core/notification/notification-wrapper.service.ts +36 -0
- package/src/core/notification/toast-wrapper.service.ts +31 -0
- package/src/core/provider/provider.tokens.ts +1 -0
- package/src/core/repl/repl-engine.service.spec.ts +417 -0
- package/src/core/repl/repl-engine.service.ts +156 -0
- package/src/core/storage/fs-storage-backend.spec.ts +70 -0
- package/src/core/storage/fs-storage-backend.ts +3 -0
- package/src/core/storage/settings-store.spec.ts +213 -0
- package/src/core/storage/settings-store.ts +2 -0
- package/src/core/storage/sql-schema.spec.ts +140 -0
- package/src/core/storage/sql-schema.ts +3 -0
- package/src/core/storage/storage-location-manager.spec.ts +121 -0
- package/src/core/storage/storage-location-manager.ts +3 -0
- package/src/core/storage/storage.service.spec.ts +73 -0
- package/src/core/storage/storage.service.ts +3 -0
- package/src/core/streaming/stream-probe.service.ts +212 -0
- package/src/core/topology/topology-emitter.service.ts +101 -0
- package/src/launcher.ts +309 -0
- package/src/main.ts +1049 -0
- package/src/manual-boot.ts +322 -0
- package/tsconfig.build.json +8 -0
- package/tsconfig.json +21 -0
- package/vitest.config.ts +26 -0
|
@@ -0,0 +1,2926 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-member-access -- pre-existing lint debt across this 2200-line orchestration class. The flagged sites (StorageService.setLocationManager / setSettingsBackend, LoggingService.addDestination, RouteRegistry, etc.) are typed as `unknown` by their owning services to break circular construction-order dependencies; runtime contracts are validated structurally. Tracked separately; do not amend in unrelated edits. */
|
|
2
|
+
import * as os from "node:os";
|
|
3
|
+
import { ConfigService } from "../config/config.service";
|
|
4
|
+
import { LoggingService } from "../logging/logging.service";
|
|
5
|
+
import { EventBusService } from "../events/event-bus.service";
|
|
6
|
+
import { StorageService } from "../storage/storage.service";
|
|
7
|
+
import { StreamProbeService } from "../streaming/stream-probe.service";
|
|
8
|
+
|
|
9
|
+
import { CapabilityService } from "../capability/capability.service";
|
|
10
|
+
import {
|
|
11
|
+
EventCategory,
|
|
12
|
+
createDeviceProxy,
|
|
13
|
+
normalizeAddonInitResult,
|
|
14
|
+
errMsg,
|
|
15
|
+
isAgentOnlyPlacement,
|
|
16
|
+
resolveRunnerId,
|
|
17
|
+
resolveAddonPlacement,
|
|
18
|
+
} from "@camstack/types";
|
|
19
|
+
import type {
|
|
20
|
+
AddonDeclaration,
|
|
21
|
+
CapabilityDeclaration,
|
|
22
|
+
CapabilityDefinition,
|
|
23
|
+
InferProvider,
|
|
24
|
+
RunnerPlan,
|
|
25
|
+
RunnerAddonPlacement,
|
|
26
|
+
} from "@camstack/types";
|
|
27
|
+
import type {
|
|
28
|
+
ICamstackAddon,
|
|
29
|
+
AddonContext,
|
|
30
|
+
InternalAddonContext,
|
|
31
|
+
} from "@camstack/types";
|
|
32
|
+
import type { IScopedLogger } from "@camstack/types";
|
|
33
|
+
import {
|
|
34
|
+
CapabilityRegistry,
|
|
35
|
+
CustomActionRegistry,
|
|
36
|
+
INFRA_CAPABILITIES,
|
|
37
|
+
AddonLoader,
|
|
38
|
+
AddonHealthMonitor,
|
|
39
|
+
|
|
40
|
+
DeviceRegistry,
|
|
41
|
+
adaptBrokerToCluster,
|
|
42
|
+
createAddonService,
|
|
43
|
+
createBrokerDeviceManagerApi,
|
|
44
|
+
createKernelHwAccel,
|
|
45
|
+
validateProviderRegistrations,
|
|
46
|
+
CapabilityHandle,
|
|
47
|
+
scopeKey,
|
|
48
|
+
describeProviderKindDrift,
|
|
49
|
+
type ISettingsStore,
|
|
50
|
+
type AddonSettingsView,
|
|
51
|
+
type AddonHealthSnapshot,
|
|
52
|
+
} from "@camstack/kernel";
|
|
53
|
+
import { localProviderLink, brokerTransportLink } from "@camstack/kernel";
|
|
54
|
+
import { createTRPCClient } from "@trpc/client";
|
|
55
|
+
import { MoleculerService } from "../moleculer/moleculer.service";
|
|
56
|
+
import { AddonDepsManager } from "@camstack/kernel";
|
|
57
|
+
import type { SavedDevice, ReadinessScope } from "@camstack/types";
|
|
58
|
+
import { IntegrationRegistry } from "@camstack/core";
|
|
59
|
+
import type {
|
|
60
|
+
IStorageProvider as INewStorageProvider,
|
|
61
|
+
ISettingsBackend,
|
|
62
|
+
} from "@camstack/types";
|
|
63
|
+
import { AddonRouteRegistry } from "@camstack/core";
|
|
64
|
+
import { randomUUID } from "node:crypto";
|
|
65
|
+
import * as path from "node:path";
|
|
66
|
+
import * as fs from "node:fs";
|
|
67
|
+
import { pathToFileURL } from "node:url";
|
|
68
|
+
import { createAddonSettingsProvider } from "./addon-settings-provider.js";
|
|
69
|
+
import { addonSettingsCapability } from "@camstack/types";
|
|
70
|
+
import { DisposerChain } from "@camstack/types";
|
|
71
|
+
|
|
72
|
+
type AddonSource = "core" | "installed";
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Local narrowing of the Moleculer ServiceBroker surface we actually use.
|
|
76
|
+
* See the `broker` getter docstring below for why this is necessary.
|
|
77
|
+
*/
|
|
78
|
+
interface BrokerLike {
|
|
79
|
+
call<T = unknown>(action: string, params?: unknown, opts?: { nodeID?: string; timeout?: number }): Promise<T>
|
|
80
|
+
nodeID: string
|
|
81
|
+
createService(schema: unknown): unknown
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Type predicate: true when an `ISettingsBackend` also satisfies the
|
|
86
|
+
* sync `ISettingsStore` contract (system/addon/device key-value ops)
|
|
87
|
+
* that ConfigManager expects. The default `SqliteSettingsBackend` in
|
|
88
|
+
* `@camstack/core` implements both surfaces; forked addon backends may
|
|
89
|
+
* not, in which case we skip the ConfigService wiring.
|
|
90
|
+
*/
|
|
91
|
+
function isSettingsStore(
|
|
92
|
+
backend: ISettingsBackend,
|
|
93
|
+
): backend is ISettingsBackend & ISettingsStore {
|
|
94
|
+
const required: readonly (keyof ISettingsStore)[] = [
|
|
95
|
+
"getSystem",
|
|
96
|
+
"setSystem",
|
|
97
|
+
"getAllSystem",
|
|
98
|
+
"getAllAddon",
|
|
99
|
+
"setAllAddon",
|
|
100
|
+
"getAllProvider",
|
|
101
|
+
"setProvider",
|
|
102
|
+
"getAllDevice",
|
|
103
|
+
"setDevice",
|
|
104
|
+
"getAddonDevice",
|
|
105
|
+
"setAddonDevice",
|
|
106
|
+
"clearAddonDevice",
|
|
107
|
+
];
|
|
108
|
+
for (const key of required) {
|
|
109
|
+
if (typeof Reflect.get(backend, key) !== "function") return false;
|
|
110
|
+
}
|
|
111
|
+
return true;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
interface AddonEntry {
|
|
115
|
+
readonly addon: ICamstackAddon;
|
|
116
|
+
initialized: boolean;
|
|
117
|
+
source: AddonSource;
|
|
118
|
+
/** npm package name from package.json (e.g. '@camstack/addon-detection-pipeline') */
|
|
119
|
+
packageName: string;
|
|
120
|
+
/** npm package version from package.json */
|
|
121
|
+
packageVersion: string;
|
|
122
|
+
/** Human-readable package name from camstack.displayName */
|
|
123
|
+
packageDisplayName?: string;
|
|
124
|
+
/** Optional bundle metadata when the package ships multiple addon entries. */
|
|
125
|
+
bundle?: { displayName: string; description?: string; icon?: string };
|
|
126
|
+
/** Capabilities declared in package.json camstack.addons (source of truth) */
|
|
127
|
+
declaredCapabilities: readonly CapabilityDeclaration[];
|
|
128
|
+
/** Addon directory on disk */
|
|
129
|
+
addonDir?: string;
|
|
130
|
+
/** Full package.json declaration (AddonDeclaration from @camstack/types) */
|
|
131
|
+
declaration?: AddonDeclaration;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// Phase 11 (settings redesign): `stripValue` helper and the legacy
|
|
135
|
+
// `getAddonConfigSchema / getAddonConfig / updateAddonConfig` adapter
|
|
136
|
+
// shim that consumed it were deleted. All settings flow through the
|
|
137
|
+
// new `getAddonSettings / getGlobalSettings / getDeviceSettings`
|
|
138
|
+
// endpoints on the `addon-settings` singleton capability.
|
|
139
|
+
|
|
140
|
+
export class AddonRegistryService {
|
|
141
|
+
private readonly addonEntries = new Map<string, AddonEntry>();
|
|
142
|
+
private readonly capabilityRegistry: CapabilityRegistry;
|
|
143
|
+
// Task 7.1: hub-wide registry of addon custom actions. Populated on
|
|
144
|
+
// each addon's initialize() (in-process path); Task 7.2 will dispatch
|
|
145
|
+
// through this from the `api.addons.custom` tRPC procedure.
|
|
146
|
+
private readonly customActionRegistry = new CustomActionRegistry();
|
|
147
|
+
/**
|
|
148
|
+
* AddonIds whose group-runner disconnect is operator-initiated (update /
|
|
149
|
+
* restart / uninstall). The Moleculer `$node.disconnected` handler skips
|
|
150
|
+
* these so the AddonCard doesn't flash a spurious "Failed to load" health
|
|
151
|
+
* banner during the kill→respawn window. The set is cleared as soon as
|
|
152
|
+
* `restartAddon` completes (success or failure) or by a 90s safety timer.
|
|
153
|
+
*/
|
|
154
|
+
private readonly restartingAddons = new Map<string, NodeJS.Timeout>();
|
|
155
|
+
private readonly logger: IScopedLogger;
|
|
156
|
+
private addonLoader!: AddonLoader;
|
|
157
|
+
private healthMonitor!: AddonHealthMonitor;
|
|
158
|
+
private addonRouteRegistry: AddonRouteRegistry | null = null;
|
|
159
|
+
|
|
160
|
+
// Broker-routed AddonApi proxy — every addon's `ctx.api` resolves
|
|
161
|
+
// to this. Calls go through `broker.call('${addonId}.${capName}.${method}')`
|
|
162
|
+
// which Moleculer routes to whichever process actually hosts the
|
|
163
|
+
// capability (in-process services on the hub, forked workers, or
|
|
164
|
+
// remote agents). The hub uses the exact same transport as agents
|
|
165
|
+
// and forked workers — `ctx.api` is uniform across all deployment
|
|
166
|
+
// shapes. Lazily constructed so tests that don't wire the broker
|
|
167
|
+
// still instantiate the service.
|
|
168
|
+
private brokerApi: import("@camstack/types").AddonApi | null = null;
|
|
169
|
+
/**
|
|
170
|
+
* In-process tRPC client over the broker — same `AddonApi` shape every
|
|
171
|
+
* addon sees via `ctx.api`. Exposed as public so the REPL service can
|
|
172
|
+
* build a `SystemManager` from it without crossing the network boundary.
|
|
173
|
+
*/
|
|
174
|
+
public getBrokerApi(): import("@camstack/types").AddonApi {
|
|
175
|
+
if (!this.brokerApi) {
|
|
176
|
+
const reg = this.capabilityRegistry;
|
|
177
|
+
const resolver = {
|
|
178
|
+
getByName: (capName: string): unknown | null =>
|
|
179
|
+
reg.getSingleton(capName) ??
|
|
180
|
+
(reg.getAllProviders(capName)[0] as unknown) ??
|
|
181
|
+
null,
|
|
182
|
+
};
|
|
183
|
+
const client: unknown = createTRPCClient({
|
|
184
|
+
links: [
|
|
185
|
+
localProviderLink(resolver),
|
|
186
|
+
brokerTransportLink(this.moleculer.broker),
|
|
187
|
+
],
|
|
188
|
+
});
|
|
189
|
+
this.brokerApi = client as import("@camstack/types").AddonApi;
|
|
190
|
+
}
|
|
191
|
+
return this.brokerApi;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// Common settings resolver (3-level: defaults → global → per-device).
|
|
195
|
+
// Lazily constructed on first call to `getSettingsResolver()` so tests
|
|
196
|
+
// that don't exercise the settings API can instantiate the service
|
|
197
|
+
// without wiring ConfigService into the resolver.
|
|
198
|
+
|
|
199
|
+
// Active capability providers (set by consumers when capabilities are wired)
|
|
200
|
+
private activeStorageProvider: INewStorageProvider | null = null;
|
|
201
|
+
private activeSettingsBackend: ISettingsBackend | null = null;
|
|
202
|
+
private integrationRegistry:
|
|
203
|
+
| import("@camstack/core").IntegrationRegistry
|
|
204
|
+
| null = null;
|
|
205
|
+
|
|
206
|
+
// Device architecture — in-memory registry of live IDevice instances.
|
|
207
|
+
// Persistence is owned by the `device-manager` capability addon.
|
|
208
|
+
private readonly deviceRegistry = new DeviceRegistry();
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Typed accessor for the Moleculer broker. The Moleculer `index.d.ts`
|
|
212
|
+
* chains through `eventemitter2`, whose package.json has no `types`
|
|
213
|
+
* field; under `moduleResolution: node` typescript-eslint's parser
|
|
214
|
+
* loses the type chain even though `tsc` accepts it via `skipLibCheck`.
|
|
215
|
+
* Going through `this.moleculer.broker` directly produces "type cannot
|
|
216
|
+
* be resolved" errors at every call site. The local `BrokerLike`
|
|
217
|
+
* interface narrows down to the surface we actually use, and the
|
|
218
|
+
* double-cast forces the parser to materialize types locally so
|
|
219
|
+
* `this.broker.call(...)` / `this.broker.nodeID` lint clean.
|
|
220
|
+
*/
|
|
221
|
+
private get broker(): BrokerLike {
|
|
222
|
+
return this.moleculer.broker as unknown as BrokerLike;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
constructor(
|
|
226
|
+
private readonly loggingService: LoggingService,
|
|
227
|
+
private readonly eventBusService: EventBusService,
|
|
228
|
+
private readonly configService: ConfigService,
|
|
229
|
+
private readonly storageService: StorageService,
|
|
230
|
+
private readonly capabilityService: CapabilityService,
|
|
231
|
+
private readonly moleculer: MoleculerService,
|
|
232
|
+
private readonly streamProbe: StreamProbeService,
|
|
233
|
+
) {
|
|
234
|
+
this.logger = this.loggingService.createLogger("AddonRegistry");
|
|
235
|
+
this.addonLoader = new AddonLoader(
|
|
236
|
+
this.loggingService.createLogger("AddonLoader"),
|
|
237
|
+
);
|
|
238
|
+
|
|
239
|
+
// Kernel-level addon health monitor — drives the 5-min boot grace
|
|
240
|
+
// window + aggressive auto-retry loop (60s/120s/300s, never gives
|
|
241
|
+
// up). Failures and recoveries are recorded by the load + init
|
|
242
|
+
// paths; the monitor's tick loop calls `tryReloadPackage` to
|
|
243
|
+
// reattempt failed addons. Operator visibility comes through
|
|
244
|
+
// AddonLoadFailed / AddonLoadRecovered events consumed by
|
|
245
|
+
// AlertCenter (see addon-health-monitor.ts spec).
|
|
246
|
+
this.healthMonitor = new AddonHealthMonitor({
|
|
247
|
+
eventBus: this.eventBusService as unknown as import("@camstack/types").IEventBus,
|
|
248
|
+
logger: this.loggingService.createLogger("AddonHealthMonitor"),
|
|
249
|
+
retryFn: (packageName) => this.tryReloadPackage(packageName),
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
// Create capability registry with config reader for singleton preferences
|
|
253
|
+
this.capabilityRegistry = new CapabilityRegistry(
|
|
254
|
+
this.loggingService.createLogger("CapabilityRegistry"),
|
|
255
|
+
this.eventBusService,
|
|
256
|
+
);
|
|
257
|
+
this.capabilityRegistry.setConfigReader(
|
|
258
|
+
(capability: string): string | undefined => {
|
|
259
|
+
try {
|
|
260
|
+
return (
|
|
261
|
+
this.configService.get<string>(
|
|
262
|
+
`capabilities.singleton.${capability}`,
|
|
263
|
+
) ?? undefined
|
|
264
|
+
);
|
|
265
|
+
} catch (err) {
|
|
266
|
+
this.logger.debug(
|
|
267
|
+
'settings-store not wired yet during early boot',
|
|
268
|
+
{ meta: { capability, error: errMsg(err) } },
|
|
269
|
+
);
|
|
270
|
+
return undefined;
|
|
271
|
+
}
|
|
272
|
+
},
|
|
273
|
+
);
|
|
274
|
+
|
|
275
|
+
// Collection-provider enable/disable persistence. Reads the same
|
|
276
|
+
// `capabilities.collection.<cap>` ConfigService key the `capabilities`
|
|
277
|
+
// router writes (`JSON.stringify({ disabled: string[] })`), so a
|
|
278
|
+
// collection provider an operator disabled survives a hub reboot.
|
|
279
|
+
this.capabilityRegistry.setCollectionConfigReader(
|
|
280
|
+
(capability: string): readonly string[] | undefined => {
|
|
281
|
+
try {
|
|
282
|
+
const raw = this.configService.get<string>(
|
|
283
|
+
`capabilities.collection.${capability}`,
|
|
284
|
+
);
|
|
285
|
+
if (!raw) return undefined;
|
|
286
|
+
const parsed: unknown = JSON.parse(raw);
|
|
287
|
+
if (
|
|
288
|
+
typeof parsed === "object" &&
|
|
289
|
+
parsed !== null &&
|
|
290
|
+
"disabled" in parsed &&
|
|
291
|
+
Array.isArray((parsed as { disabled: unknown }).disabled)
|
|
292
|
+
) {
|
|
293
|
+
const disabled = (parsed as { disabled: unknown[] }).disabled;
|
|
294
|
+
return disabled.filter(
|
|
295
|
+
(id): id is string => typeof id === "string",
|
|
296
|
+
);
|
|
297
|
+
}
|
|
298
|
+
return undefined;
|
|
299
|
+
} catch (err) {
|
|
300
|
+
this.logger.debug(
|
|
301
|
+
"settings-store not wired yet or malformed collection preference",
|
|
302
|
+
{ meta: { capability, error: errMsg(err) } },
|
|
303
|
+
);
|
|
304
|
+
return undefined;
|
|
305
|
+
}
|
|
306
|
+
},
|
|
307
|
+
);
|
|
308
|
+
|
|
309
|
+
// Wire the registry into the CapabilityService so all server services can use it
|
|
310
|
+
this.capabilityService.setRegistry(this.capabilityRegistry);
|
|
311
|
+
|
|
312
|
+
// Register kernel-provided infrastructure capabilities so addons
|
|
313
|
+
// can resolve them via the capability registry like any other cap.
|
|
314
|
+
// These are kernel pseudo-caps (no tRPC methods, object-reference only),
|
|
315
|
+
// so we declare them with empty method maps before registering providers.
|
|
316
|
+
this.capabilityRegistry.declareCapability({
|
|
317
|
+
name: "device-registry",
|
|
318
|
+
scope: "system",
|
|
319
|
+
mode: "singleton",
|
|
320
|
+
methods: {},
|
|
321
|
+
});
|
|
322
|
+
this.capabilityRegistry.declareCapability({
|
|
323
|
+
name: "cluster-broker",
|
|
324
|
+
scope: "system",
|
|
325
|
+
mode: "singleton",
|
|
326
|
+
methods: {},
|
|
327
|
+
});
|
|
328
|
+
this.capabilityRegistry.registerProvider(
|
|
329
|
+
"device-registry",
|
|
330
|
+
"$kernel",
|
|
331
|
+
this.deviceRegistry,
|
|
332
|
+
);
|
|
333
|
+
this.capabilityRegistry.registerProvider("cluster-broker", "$kernel", {
|
|
334
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- moleculer types unresolvable; see `broker` getter docstring
|
|
335
|
+
broker: this.moleculer.broker,
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
async onModuleInit(): Promise<void> {
|
|
340
|
+
// Register the addon-settings singleton provider — replaces the
|
|
341
|
+
// former `$addonHost` Moleculer service. The provider resolves
|
|
342
|
+
// addonId → local addon instance (hub) or remote Moleculer call.
|
|
343
|
+
this.capabilityRegistry.declareCapability(addonSettingsCapability);
|
|
344
|
+
const settingsProvider = createAddonSettingsProvider({
|
|
345
|
+
getAddon: (addonId) => {
|
|
346
|
+
const entry = this.addonEntries.get(addonId);
|
|
347
|
+
return entry?.addon ?? null;
|
|
348
|
+
},
|
|
349
|
+
resolveNode: (addonId) => {
|
|
350
|
+
// Group-runner addons live in a child Moleculer node (e.g.
|
|
351
|
+
// `hub/pipeline`). The hub-side `entry.addon` instance never
|
|
352
|
+
// runs `initialize()` and has no `_ctx.settings`, so calling
|
|
353
|
+
// `addon.getGlobalSettings()` on it returns defaults only —
|
|
354
|
+
// the real store lives in the worker. Route those via the
|
|
355
|
+
// remote path so the worker's `<addonId>.settings.*` handler
|
|
356
|
+
// serves the canonical response.
|
|
357
|
+
const entry = this.addonEntries.get(addonId);
|
|
358
|
+
if (entry?.declaration && entry.addonDir) {
|
|
359
|
+
const runnerId = resolveRunnerId(entry.declaration, addonId);
|
|
360
|
+
return `${this.broker.nodeID}/${runnerId}`;
|
|
361
|
+
}
|
|
362
|
+
return "hub";
|
|
363
|
+
},
|
|
364
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- moleculer types unresolvable; see `broker` getter docstring
|
|
365
|
+
broker: this.moleculer.broker,
|
|
366
|
+
hubNodeId: this.broker.nodeID,
|
|
367
|
+
});
|
|
368
|
+
this.capabilityRegistry.registerProvider(
|
|
369
|
+
"addon-settings",
|
|
370
|
+
"$hub",
|
|
371
|
+
settingsProvider,
|
|
372
|
+
);
|
|
373
|
+
|
|
374
|
+
// Wire capability consumer actions via EventBus
|
|
375
|
+
this.wireCapabilityConsumers();
|
|
376
|
+
|
|
377
|
+
// Subscribe to capability.binding-changed so hub-side capability
|
|
378
|
+
// overrides take effect on the fly. The orchestrator addon emits
|
|
379
|
+
// these when the operator swaps the addon implementing a cap on a
|
|
380
|
+
// node; we only react when the target node is the hub's own nodeID.
|
|
381
|
+
this.eventBusService.subscribe(
|
|
382
|
+
{ category: "capability.binding-changed" },
|
|
383
|
+
(event) => {
|
|
384
|
+
const data = (event.data ?? {}) as {
|
|
385
|
+
nodeId?: string;
|
|
386
|
+
capName?: string;
|
|
387
|
+
addonId?: string | null;
|
|
388
|
+
};
|
|
389
|
+
if (!data.capName) return;
|
|
390
|
+
const localNodeId = this.broker.nodeID;
|
|
391
|
+
if (data.nodeId && data.nodeId !== localNodeId) return;
|
|
392
|
+
this.capabilityRegistry.setSingletonActiveAddon(
|
|
393
|
+
data.capName,
|
|
394
|
+
data.addonId ?? null,
|
|
395
|
+
);
|
|
396
|
+
},
|
|
397
|
+
);
|
|
398
|
+
|
|
399
|
+
// 2. Load all addons from data/addons/ directory (including core builtins)
|
|
400
|
+
const dataDir = path.resolve(process.env.CAMSTACK_DATA ?? "camstack-data");
|
|
401
|
+
const addonsDir = path.resolve(dataDir, "addons");
|
|
402
|
+
|
|
403
|
+
await this.addonLoader.loadFromDirectory(addonsDir);
|
|
404
|
+
|
|
405
|
+
// Hand pre-init load failures (broken package.json, missing
|
|
406
|
+
// dist, import-time errors) to the health monitor so its retry
|
|
407
|
+
// loop and post-grace alerting cover them. Failures during the
|
|
408
|
+
// init phase below are recorded separately.
|
|
409
|
+
for (const fail of this.addonLoader.listLoadFailures()) {
|
|
410
|
+
this.healthMonitor.recordFailure(fail.packageName, fail.error, fail.addonId);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
const loadedAddons = this.addonLoader.listAddons();
|
|
414
|
+
|
|
415
|
+
for (const registered of loadedAddons) {
|
|
416
|
+
// Skip agent-only addons — they never run on the hub, only on remote
|
|
417
|
+
// agents that opt in via `execution.placement: 'agent-only'`.
|
|
418
|
+
if (isAgentOnlyPlacement(registered.declaration)) {
|
|
419
|
+
this.logger.info(
|
|
420
|
+
'Skipping agent-only addon on hub',
|
|
421
|
+
{ tags: { addonId: registered.declaration.id }, meta: { packageName: registered.packageName } },
|
|
422
|
+
);
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
if (registered.declaration.capabilities) {
|
|
426
|
+
for (const cap of registered.declaration.capabilities) {
|
|
427
|
+
this.capabilityRegistry.declareFromManifest(
|
|
428
|
+
cap,
|
|
429
|
+
registered.declaration.id,
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
// Create and store addon instance — cast from @camstack/types to server's
|
|
434
|
+
// local ICamstackAddon (structurally compatible at runtime).
|
|
435
|
+
// Wrap in try/catch so a single broken addon doesn't prevent the rest from loading.
|
|
436
|
+
try {
|
|
437
|
+
const addon = this.addonLoader.createInstance(
|
|
438
|
+
registered.declaration.id,
|
|
439
|
+
);
|
|
440
|
+
// Capabilities come from package.json declaration (source of truth),
|
|
441
|
+
// merged with any declared in the addon class manifest.
|
|
442
|
+
const declCaps = (registered.declaration.capabilities ?? []).map(
|
|
443
|
+
(c: string | CapabilityDeclaration) =>
|
|
444
|
+
typeof c === "string" ? { name: c, mode: "singleton" as const } : c,
|
|
445
|
+
);
|
|
446
|
+
this.addonEntries.set(registered.declaration.id, {
|
|
447
|
+
addon,
|
|
448
|
+
initialized: false,
|
|
449
|
+
source: "installed",
|
|
450
|
+
packageName: registered.packageName,
|
|
451
|
+
packageVersion: registered.packageVersion,
|
|
452
|
+
packageDisplayName: registered.packageDisplayName,
|
|
453
|
+
...(registered.bundle !== undefined ? { bundle: registered.bundle } : {}),
|
|
454
|
+
declaredCapabilities: declCaps,
|
|
455
|
+
addonDir: path.join(addonsDir, registered.packageName),
|
|
456
|
+
declaration: registered.declaration,
|
|
457
|
+
});
|
|
458
|
+
} catch (err) {
|
|
459
|
+
const msg = errMsg(err);
|
|
460
|
+
this.logger.error(
|
|
461
|
+
'Failed to create instance of addon',
|
|
462
|
+
{ tags: { addonId: registered.declaration.id }, meta: { error: msg } },
|
|
463
|
+
);
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
this.logger.info(
|
|
468
|
+
'Loaded addons from directory',
|
|
469
|
+
{ meta: { count: loadedAddons.length, addonsDir } },
|
|
470
|
+
);
|
|
471
|
+
// Platform probing is no longer orchestrated by the backend. The
|
|
472
|
+
// `platform-probe` capability (shipped as a core builtin under
|
|
473
|
+
// `@camstack/core/dist/builtins/platform-probe/` since Phase B of
|
|
474
|
+
// the bundles refactor) exposes hardware + scored backends via
|
|
475
|
+
// `ctx.api.platformProbe.*`; inference addons auto-configure
|
|
476
|
+
// inside their own `onInitialize`.
|
|
477
|
+
|
|
478
|
+
// 4. Initialize all installed addons in correct order (installed = active)
|
|
479
|
+
const allIds = [...this.addonEntries.keys()];
|
|
480
|
+
|
|
481
|
+
// Mark registry as ready BEFORE any addon initializes. Addons' onInitialize
|
|
482
|
+
// may call ctx.api.<cap>.* which routes through localProviderLink; that
|
|
483
|
+
// resolver depends on getAllProviders/getSingleton which gate on `_ready`.
|
|
484
|
+
// Without this, Phase 1 and Phase 2 addons would all fall through to the
|
|
485
|
+
// Moleculer broker transport and fail with "service not registered".
|
|
486
|
+
this.capabilityRegistry.ready();
|
|
487
|
+
|
|
488
|
+
// Native-cap cross-process bridge: when a hub consumer resolves a native
|
|
489
|
+
// provider for a device whose IDevice lives in a forked worker, we
|
|
490
|
+
// return a broker proxy that routes calls to
|
|
491
|
+
// `<addonId>.native-provider.<capName>.<method>` via standard Moleculer RPC.
|
|
492
|
+
//
|
|
493
|
+
// The resolver keys off `(capName, deviceId)`, NOT device ownership.
|
|
494
|
+
// An addon can own a device without registering every conceivable cap
|
|
495
|
+
// natively (e.g. RtspCamera without `snapshotUrl` does not publish a
|
|
496
|
+
// snapshot provider). Synthesizing a proxy against the device owner
|
|
497
|
+
// would point at a Moleculer service that was never mounted and make
|
|
498
|
+
// every call throw "service not found" at runtime.
|
|
499
|
+
//
|
|
500
|
+
// `resolveNativeCapOwnerSync` consults both hub-local native
|
|
501
|
+
// registrations and `remoteNativeCaps` (populated via
|
|
502
|
+
// DeviceBindingsChanged events emitted by every worker). It returns
|
|
503
|
+
// null when no provider published the cap for that device — the
|
|
504
|
+
// fallback surfaces that as null so wrappers cleanly fall through to
|
|
505
|
+
// whatever secondary strategy they own (e.g. snapshot → ffmpeg frame
|
|
506
|
+
// grab), instead of throwing against a phantom service.
|
|
507
|
+
{
|
|
508
|
+
const { buildNativeCapProxy } = await import("@camstack/kernel");
|
|
509
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- moleculer types unresolvable; see `broker` getter docstring
|
|
510
|
+
const broker = this.moleculer.broker;
|
|
511
|
+
this.capabilityRegistry.setNativeFallback(
|
|
512
|
+
(capName: string, deviceId: number): unknown | null => {
|
|
513
|
+
const dm = this.capabilityRegistry.getSingleton<{
|
|
514
|
+
resolveNativeCapOwnerSync?: (
|
|
515
|
+
capName: string,
|
|
516
|
+
deviceId: number,
|
|
517
|
+
) => { addonId: string; nodeId: string } | null;
|
|
518
|
+
}>("device-manager");
|
|
519
|
+
const owner =
|
|
520
|
+
dm?.resolveNativeCapOwnerSync?.(capName, deviceId) ?? null;
|
|
521
|
+
if (!owner) return null;
|
|
522
|
+
return buildNativeCapProxy(broker, owner.addonId, capName, deviceId);
|
|
523
|
+
},
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
// Runner spawn: every non-core, placement-eligible addon runs in its
|
|
528
|
+
// OWN forked runner subprocess by default (base-layer D2/D9 — one
|
|
529
|
+
// addon, one process). No shipped addon declares a shared
|
|
530
|
+
// `execution.group`: Phase 5 dissolved the `pipeline` media group
|
|
531
|
+
// once frames travel as shm `FrameHandle`s and audio over tRPC.
|
|
532
|
+
// `buildAddonGroupPlan` keys the plan by runner id (the addon id);
|
|
533
|
+
// cross-runner / cross-node cap calls travel over Moleculer TCP.
|
|
534
|
+
// `resolveRunnerId` still collapses a shared `execution.group` if
|
|
535
|
+
// one were declared — the mechanism stays available as an explicit
|
|
536
|
+
// opt-in for future co-location needs.
|
|
537
|
+
//
|
|
538
|
+
// `@camstack/core` builtins and `placement: 'agent-only'` addons are
|
|
539
|
+
// excluded from the plan (filtered inside `buildAddonGroupPlan`).
|
|
540
|
+
// A failed runner spawn is logged and the addon surfaces as
|
|
541
|
+
// `addon.error` — there is NO in-process-on-the-hub fallback (it
|
|
542
|
+
// would violate one-addon-one-process). Retry policy for a failed
|
|
543
|
+
// spawn is governed by the kernel circuit-breaker.
|
|
544
|
+
{
|
|
545
|
+
const plan = this.buildAddonGroupPlan(allIds);
|
|
546
|
+
for (const [runnerId, runnerAddons] of plan) {
|
|
547
|
+
try {
|
|
548
|
+
await this.initializeAddonGroup(runnerId, runnerAddons);
|
|
549
|
+
for (const { addonId } of runnerAddons) {
|
|
550
|
+
this.wireCapabilities(addonId);
|
|
551
|
+
}
|
|
552
|
+
} catch (error: unknown) {
|
|
553
|
+
const msg = errMsg(error);
|
|
554
|
+
for (const { addonId } of runnerAddons) {
|
|
555
|
+
this.emitAddonLifecycleEvent("addon.error", addonId, {
|
|
556
|
+
error: msg,
|
|
557
|
+
phase: "init",
|
|
558
|
+
});
|
|
559
|
+
}
|
|
560
|
+
this.logger.error(
|
|
561
|
+
"Runner spawn failed — addons on this runner will be skipped",
|
|
562
|
+
{ meta: { runnerId, addonIds: runnerAddons.map((a) => a.addonId), error: msg } },
|
|
563
|
+
);
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// In-process boot for `@camstack/core` builtins. Core builtins stay
|
|
569
|
+
// resident on the hub process — they provide the storage / settings
|
|
570
|
+
// / logging infrastructure every forked runner depends on, reachable
|
|
571
|
+
// via the hub broker, so `buildAddonGroupPlan` excludes them from
|
|
572
|
+
// the runner plan above. They boot in two ordered passes:
|
|
573
|
+
// infrastructure caps first (`INFRA_CAPABILITIES`,
|
|
574
|
+
// storage→settings→logging), then the remaining builtins in
|
|
575
|
+
// capability-dependency order.
|
|
576
|
+
//
|
|
577
|
+
// A core builtin is identified by `packageName === "@camstack/core"`,
|
|
578
|
+
// NOT by the absence of an `execution` declaration — a builtin may
|
|
579
|
+
// still declare `execution` (e.g. `platform-probe`); the package
|
|
580
|
+
// boundary is the selection criterion.
|
|
581
|
+
const isCoreBuiltin = (id: string): boolean =>
|
|
582
|
+
this.addonEntries.get(id)?.packageName === "@camstack/core";
|
|
583
|
+
|
|
584
|
+
// Pass 1 — infrastructure builtins. A failed REQUIRED infra cap
|
|
585
|
+
// aborts boot: nothing downstream can run without storage/settings.
|
|
586
|
+
for (const infra of INFRA_CAPABILITIES) {
|
|
587
|
+
const addonId = this.findAddonForCapability(infra.name, allIds);
|
|
588
|
+
if (addonId) {
|
|
589
|
+
const entry = this.addonEntries.get(addonId);
|
|
590
|
+
// Only core builtins boot in-process here. A non-core addon that
|
|
591
|
+
// happens to provide an infra cap (e.g. an alternate
|
|
592
|
+
// storage-provider) lives in its own runner — never in-process.
|
|
593
|
+
if (!entry || entry.initialized || !isCoreBuiltin(addonId)) continue;
|
|
594
|
+
try {
|
|
595
|
+
await this.initializeAddon(addonId);
|
|
596
|
+
this.wireCapabilities(addonId);
|
|
597
|
+
} catch (error: unknown) {
|
|
598
|
+
const msg = errMsg(error);
|
|
599
|
+
this.emitAddonLifecycleEvent("addon.error", addonId, {
|
|
600
|
+
error: msg,
|
|
601
|
+
phase: "init",
|
|
602
|
+
});
|
|
603
|
+
if (infra.required) {
|
|
604
|
+
throw new Error(
|
|
605
|
+
`Required infrastructure addon "${addonId}" failed: ${msg}`,
|
|
606
|
+
{ cause: error },
|
|
607
|
+
);
|
|
608
|
+
}
|
|
609
|
+
this.logger.warn(
|
|
610
|
+
'Optional infra addon failed -- continuing',
|
|
611
|
+
{ tags: { addonId }, meta: { error: msg } },
|
|
612
|
+
);
|
|
613
|
+
}
|
|
614
|
+
} else if (infra.required) {
|
|
615
|
+
throw new Error(
|
|
616
|
+
`No addon provides required infrastructure capability "${infra.name}"`,
|
|
617
|
+
);
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
// Pass 2 — remaining core builtins, in capability-dependency order.
|
|
622
|
+
const bootOrder = this.capabilityRegistry.getBootOrder();
|
|
623
|
+
const infraNames = new Set(INFRA_CAPABILITIES.map((c) => c.name));
|
|
624
|
+
|
|
625
|
+
for (const capName of bootOrder) {
|
|
626
|
+
if (infraNames.has(capName)) continue; // Handled in pass 1
|
|
627
|
+
|
|
628
|
+
for (const id of allIds) {
|
|
629
|
+
const entry = this.addonEntries.get(id);
|
|
630
|
+
if (!entry || entry.initialized || !isCoreBuiltin(id)) continue;
|
|
631
|
+
|
|
632
|
+
// Check if this addon provides this capability
|
|
633
|
+
const provides = this.getAddonCapabilities(entry.addon);
|
|
634
|
+
if (!provides.some((c) => c.name === capName)) continue;
|
|
635
|
+
|
|
636
|
+
try {
|
|
637
|
+
await this.initializeAddon(id);
|
|
638
|
+
this.wireCapabilities(id);
|
|
639
|
+
} catch (error: unknown) {
|
|
640
|
+
const msg = errMsg(error);
|
|
641
|
+
this.emitAddonLifecycleEvent("addon.error", id, {
|
|
642
|
+
error: msg,
|
|
643
|
+
phase: "init",
|
|
644
|
+
});
|
|
645
|
+
this.logger.error(
|
|
646
|
+
'Core builtin failed to initialize -- skipping',
|
|
647
|
+
{ tags: { addonId: id }, meta: { error: msg } },
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
// Pass 3 — core builtins that declare no capabilities at all.
|
|
654
|
+
for (const id of allIds) {
|
|
655
|
+
const entry = this.addonEntries.get(id);
|
|
656
|
+
if (entry && !entry.initialized && isCoreBuiltin(id)) {
|
|
657
|
+
try {
|
|
658
|
+
await this.initializeAddon(id);
|
|
659
|
+
this.wireCapabilities(id);
|
|
660
|
+
} catch (error: unknown) {
|
|
661
|
+
const msg = errMsg(error);
|
|
662
|
+
this.emitAddonLifecycleEvent("addon.error", id, {
|
|
663
|
+
error: msg,
|
|
664
|
+
phase: "init",
|
|
665
|
+
});
|
|
666
|
+
this.logger.error(
|
|
667
|
+
'Core builtin failed to initialize -- skipping',
|
|
668
|
+
{ tags: { addonId: id }, meta: { error: msg } },
|
|
669
|
+
);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
const initializedIds = [...this.addonEntries.entries()]
|
|
675
|
+
.filter(([, e]) => e.initialized)
|
|
676
|
+
.map(([id]) => id);
|
|
677
|
+
|
|
678
|
+
this.logger.info(
|
|
679
|
+
'Addons initialized',
|
|
680
|
+
{ meta: { initializedCount: initializedIds.length, totalCount: this.addonEntries.size } },
|
|
681
|
+
);
|
|
682
|
+
|
|
683
|
+
// Health snapshot: every addon entry that survived the loader is
|
|
684
|
+
// either initialized (record success) or failed init (record
|
|
685
|
+
// failure with the captured error). Pre-load failures were already
|
|
686
|
+
// recorded above from `addonLoader.listLoadFailures()`.
|
|
687
|
+
for (const [id, entry] of this.addonEntries.entries()) {
|
|
688
|
+
if (entry.initialized) {
|
|
689
|
+
this.healthMonitor.recordSuccess(entry.packageName, id);
|
|
690
|
+
} else {
|
|
691
|
+
// Init failed (caught by either Phase 1 or Phase 2 catch
|
|
692
|
+
// blocks above). The catch handlers don't carry the error
|
|
693
|
+
// forward to here, so we synthesize a generic one — the real
|
|
694
|
+
// error is in the addon log. Operators see the alert + can
|
|
695
|
+
// open the per-addon logs to investigate.
|
|
696
|
+
this.healthMonitor.recordFailure(
|
|
697
|
+
entry.packageName,
|
|
698
|
+
new Error(`Addon "${id}" failed to initialize — see addon logs for details`),
|
|
699
|
+
id,
|
|
700
|
+
);
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
// Start the kernel-level retry tick (30s). Fires immediately
|
|
705
|
+
// on the first interval boundary; the 5-min boot grace window
|
|
706
|
+
// suppresses alerts until the system has had time to stabilize.
|
|
707
|
+
this.healthMonitor.start();
|
|
708
|
+
|
|
709
|
+
// Group-runner / forked-addon crash detection (point 5 of the
|
|
710
|
+
// operator's spec — "ogni fork/moleculer deve essere considerato").
|
|
711
|
+
// Moleculer's process-service.ts already handles respawn with
|
|
712
|
+
// exponential backoff; we hook the broker's localBus events to
|
|
713
|
+
// GIVE VISIBILITY of those crashes through the same monitor —
|
|
714
|
+
// every crash records a failure, every reconnect records a
|
|
715
|
+
// recovery. The respawn itself stays in process-service.ts.
|
|
716
|
+
//
|
|
717
|
+
// Runner node ids look like `hub/<runnerId>` where `runnerId` is the
|
|
718
|
+
// addon id for a solo runner (the common case — one addon, one
|
|
719
|
+
// process) or the co-location group name for a shared runner (e.g.
|
|
720
|
+
// `hub/pipeline`). We map a disconnected/connected node back to the
|
|
721
|
+
// underlying addons by walking `addonEntries` for any addon whose
|
|
722
|
+
// resolved runner id matches the event's nodeId — every addon on
|
|
723
|
+
// that runner gets a recordFailure / recordSuccess.
|
|
724
|
+
type LocalBusEvent = { node: { id: string } }
|
|
725
|
+
type LocalBus = { on: (event: string, handler: (payload: LocalBusEvent) => void) => void }
|
|
726
|
+
const localBus = (this.moleculer.broker as unknown as { localBus?: LocalBus }).localBus
|
|
727
|
+
if (localBus) {
|
|
728
|
+
localBus.on('$node.disconnected', (payload) => {
|
|
729
|
+
const nodeId = payload.node?.id
|
|
730
|
+
if (!nodeId || nodeId === this.broker.nodeID) return
|
|
731
|
+
for (const [id, entry] of this.addonEntries.entries()) {
|
|
732
|
+
if (!entry.declaration) continue
|
|
733
|
+
const runnerId = resolveRunnerId(entry.declaration, id)
|
|
734
|
+
if (`${this.broker.nodeID}/${runnerId}` !== nodeId) continue
|
|
735
|
+
// Skip operator-initiated restarts — `restartAddon` already
|
|
736
|
+
// waits for caps to re-register and surfaces its own failure
|
|
737
|
+
// path. Recording a transient failure here would flash the
|
|
738
|
+
// "Failed to load" banner on AddonCard during a routine update.
|
|
739
|
+
if (this.restartingAddons.has(id)) continue
|
|
740
|
+
// Record one failure per addon on the disconnected runner.
|
|
741
|
+
// The error message references the nodeId — the operator
|
|
742
|
+
// can correlate with process-service.ts respawn logs.
|
|
743
|
+
this.healthMonitor.recordFailure(
|
|
744
|
+
entry.packageName,
|
|
745
|
+
new Error(`Addon runner ${nodeId} disconnected`),
|
|
746
|
+
id,
|
|
747
|
+
);
|
|
748
|
+
}
|
|
749
|
+
})
|
|
750
|
+
localBus.on('$node.connected', (payload) => {
|
|
751
|
+
const nodeId = payload.node?.id
|
|
752
|
+
if (!nodeId || nodeId === this.broker.nodeID) return
|
|
753
|
+
for (const [id, entry] of this.addonEntries.entries()) {
|
|
754
|
+
if (!entry.declaration) continue
|
|
755
|
+
const runnerId = resolveRunnerId(entry.declaration, id)
|
|
756
|
+
if (`${this.broker.nodeID}/${runnerId}` !== nodeId) continue
|
|
757
|
+
this.healthMonitor.recordSuccess(entry.packageName, id);
|
|
758
|
+
}
|
|
759
|
+
})
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
this.eventBusService.emit({
|
|
763
|
+
id: randomUUID(),
|
|
764
|
+
timestamp: new Date(),
|
|
765
|
+
source: { type: "core", id: "addon-registry" },
|
|
766
|
+
category: EventCategory.SystemAddonsReady,
|
|
767
|
+
data: { activeAddons: initializedIds },
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
/**
|
|
772
|
+
* Reload a single package — invoked by AddonHealthMonitor's retry
|
|
773
|
+
* loop and by the operator-facing `addons.retryLoad` tRPC procedure.
|
|
774
|
+
*
|
|
775
|
+
* Two paths:
|
|
776
|
+
* 1. Package not yet in the registry (e.g. import failed at boot
|
|
777
|
+
* time): re-walk the addons dir, pick up any new package, run
|
|
778
|
+
* `initializeAddon` for each contained addon.
|
|
779
|
+
* 2. Package already in the registry but its addon entries failed
|
|
780
|
+
* init: call `restartAddon` for each addon in that package.
|
|
781
|
+
*
|
|
782
|
+
* On success the monitor automatically records the addon healthy
|
|
783
|
+
* (no explicit recordSuccess needed — the `attemptRetry` no-throw
|
|
784
|
+
* fast path handles it). On failure this method throws and the
|
|
785
|
+
* monitor's catch path schedules the next retry.
|
|
786
|
+
*/
|
|
787
|
+
async tryReloadPackage(packageName: string): Promise<void> {
|
|
788
|
+
const dataDir = path.resolve(process.env.CAMSTACK_DATA ?? "camstack-data");
|
|
789
|
+
const addonsDir = path.resolve(dataDir, "addons");
|
|
790
|
+
const addonDir = path.join(addonsDir, packageName);
|
|
791
|
+
|
|
792
|
+
if (!fs.existsSync(addonDir)) {
|
|
793
|
+
throw new Error(`Package directory missing: ${addonDir}`);
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
// Branch 1: package known to the registry already → restart each
|
|
797
|
+
// addon entry. `restartAddon` rebuilds AddonContext + runs init
|
|
798
|
+
// again; on success the entry's `initialized` flag flips back to
|
|
799
|
+
// true and the monitor's auto-resolve path records success.
|
|
800
|
+
const knownAddonIds: string[] = [];
|
|
801
|
+
for (const [id, entry] of this.addonEntries.entries()) {
|
|
802
|
+
if (entry.packageName === packageName) {
|
|
803
|
+
knownAddonIds.push(id);
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
if (knownAddonIds.length > 0) {
|
|
808
|
+
const errors: string[] = [];
|
|
809
|
+
for (const id of knownAddonIds) {
|
|
810
|
+
try {
|
|
811
|
+
await this.restartAddon(id);
|
|
812
|
+
this.healthMonitor.recordSuccess(packageName, id);
|
|
813
|
+
} catch (err) {
|
|
814
|
+
errors.push(`${id}: ${errMsg(err)}`);
|
|
815
|
+
this.healthMonitor.recordFailure(packageName, err, id);
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
if (errors.length > 0) {
|
|
819
|
+
throw new Error(`tryReloadPackage failed for ${packageName}: ${errors.join('; ')}`);
|
|
820
|
+
}
|
|
821
|
+
return;
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
// Branch 2: package not yet known → run loadNewAddons which
|
|
825
|
+
// walks disk, instantiates new entries, and initializes them.
|
|
826
|
+
// It records its own loaded/failed in the return; we re-throw
|
|
827
|
+
// when our package is in the failed list so the monitor knows.
|
|
828
|
+
const result = await this.loadNewAddons();
|
|
829
|
+
if (result.failed.length > 0) {
|
|
830
|
+
const failed = result.failed.join(', ');
|
|
831
|
+
throw new Error(`loadNewAddons reported failures: ${failed}`);
|
|
832
|
+
}
|
|
833
|
+
if (result.loaded.length === 0) {
|
|
834
|
+
throw new Error(`No new addons loaded for ${packageName} — package may still have a broken manifest`);
|
|
835
|
+
}
|
|
836
|
+
// Match loaded addons to this package + record success.
|
|
837
|
+
for (const id of result.loaded) {
|
|
838
|
+
const entry = this.addonEntries.get(id);
|
|
839
|
+
if (entry?.packageName === packageName) {
|
|
840
|
+
this.healthMonitor.recordSuccess(packageName, id);
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
/** Health snapshot — exposed to the addons.list tRPC procedure. */
|
|
846
|
+
getAddonHealthSnapshot(): readonly AddonHealthSnapshot[] {
|
|
847
|
+
return this.healthMonitor.getHealthSnapshot();
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/** Manual user-triggered retry (resets retry counter). */
|
|
851
|
+
async retryAddonLoad(packageName: string): Promise<void> {
|
|
852
|
+
return this.healthMonitor.retryNow(packageName);
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
async onModuleDestroy(): Promise<void> {
|
|
856
|
+
this.logger.info("Shutting down all addons…");
|
|
857
|
+
this.healthMonitor.stop();
|
|
858
|
+
await this.shutdownAll();
|
|
859
|
+
this.logger.info("All addons shut down");
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
/** Set the AddonRouteRegistry for wiring addon-routes capabilities */
|
|
863
|
+
setAddonRouteRegistry(registry: AddonRouteRegistry): void {
|
|
864
|
+
this.addonRouteRegistry = registry;
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
/**
|
|
868
|
+
* Called after app.init() when the tRPC router is available.
|
|
869
|
+
* No-op now: addon `ctx.api` resolves to a broker-routed proxy, so
|
|
870
|
+
* the direct tRPC caller is no longer constructed. Kept for backward
|
|
871
|
+
* compat with `main.ts`'s bootstrap order.
|
|
872
|
+
*/
|
|
873
|
+
async setAppRouter(_router: unknown): Promise<void> {
|
|
874
|
+
this.logger.debug(
|
|
875
|
+
"setAppRouter called — broker-routed addon API in use, no direct caller needed",
|
|
876
|
+
);
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
/** Get the CapabilityRegistry for external consumers to register */
|
|
880
|
+
getCapabilityRegistry(): CapabilityRegistry {
|
|
881
|
+
return this.capabilityRegistry;
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
/** Get the CustomActionRegistry — Task 7.2 will use this from the `api.addons.custom` tRPC procedure. */
|
|
885
|
+
getCustomActionRegistry(): CustomActionRegistry {
|
|
886
|
+
return this.customActionRegistry;
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
getDeviceRegistry(): DeviceRegistry {
|
|
890
|
+
return this.deviceRegistry;
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
/** Load persisted collection disabled-lists from settings-store into the registry */
|
|
894
|
+
private loadCollectionPreferences(): void {
|
|
895
|
+
// TODO: implement CapabilityRegistry.loadDisabledProviders() to restore persisted preferences
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
/**
|
|
899
|
+
* Returns the IntegrationRegistry filtered to exclude orphaned integrations
|
|
900
|
+
* (where the addon is not currently installed). Orphaned data stays in DB
|
|
901
|
+
* and reconnects automatically when the addon is reinstalled.
|
|
902
|
+
*/
|
|
903
|
+
getIntegrationRegistry():
|
|
904
|
+
| import("@camstack/types").IIntegrationRegistry
|
|
905
|
+
| null {
|
|
906
|
+
if (!this.integrationRegistry) return null;
|
|
907
|
+
return this.createFilteredRegistry(this.integrationRegistry);
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
/**
|
|
911
|
+
* Return the currently-active settings backend (the `settings-store`
|
|
912
|
+
* capability provider). Null if no provider has been registered yet
|
|
913
|
+
* (e.g. during early boot before builtins are loaded). Used by the
|
|
914
|
+
* multi-level addon settings router to read/write addon-device
|
|
915
|
+
* overrides directly.
|
|
916
|
+
*/
|
|
917
|
+
getSettingsBackend(): ISettingsBackend | null {
|
|
918
|
+
return this.activeSettingsBackend;
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
/** Get the raw (unfiltered) registry — only for internal addon wiring */
|
|
922
|
+
getRawIntegrationRegistry():
|
|
923
|
+
| import("@camstack/types").IIntegrationRegistry
|
|
924
|
+
| null {
|
|
925
|
+
return this.integrationRegistry;
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
private createFilteredRegistry(
|
|
929
|
+
raw: import("@camstack/types").IIntegrationRegistry,
|
|
930
|
+
): import("@camstack/types").IIntegrationRegistry {
|
|
931
|
+
const installedAddonIds = new Set([...this.addonEntries.keys()]);
|
|
932
|
+
|
|
933
|
+
// Build set of integration IDs whose addon IS installed
|
|
934
|
+
// Cache per call — lightweight, called infrequently
|
|
935
|
+
let activeIntegrationIds: Set<string> | null = null;
|
|
936
|
+
const ensureActiveIds = async () => {
|
|
937
|
+
if (activeIntegrationIds) return activeIntegrationIds;
|
|
938
|
+
const all = await raw.listIntegrations();
|
|
939
|
+
activeIntegrationIds = new Set(
|
|
940
|
+
all.filter((i) => installedAddonIds.has(i.addonId)).map((i) => i.id),
|
|
941
|
+
);
|
|
942
|
+
return activeIntegrationIds;
|
|
943
|
+
};
|
|
944
|
+
|
|
945
|
+
return {
|
|
946
|
+
// Integrations: filter out orphaned
|
|
947
|
+
createIntegration: (input) => raw.createIntegration(input),
|
|
948
|
+
getIntegration: async (id) => {
|
|
949
|
+
const i = await raw.getIntegration(id);
|
|
950
|
+
return i && installedAddonIds.has(i.addonId) ? i : null;
|
|
951
|
+
},
|
|
952
|
+
getIntegrationByAddonId: async (addonId) => {
|
|
953
|
+
if (!installedAddonIds.has(addonId)) return null;
|
|
954
|
+
return raw.getIntegrationByAddonId(addonId);
|
|
955
|
+
},
|
|
956
|
+
listIntegrations: async () => {
|
|
957
|
+
const all = await raw.listIntegrations();
|
|
958
|
+
return all.filter((i) => installedAddonIds.has(i.addonId));
|
|
959
|
+
},
|
|
960
|
+
updateIntegration: (id, updates) => raw.updateIntegration(id, updates),
|
|
961
|
+
deleteIntegration: (id) => raw.deleteIntegration(id),
|
|
962
|
+
|
|
963
|
+
// Integration settings: passthrough (already gated by getIntegration)
|
|
964
|
+
getIntegrationSettings: (id) => raw.getIntegrationSettings(id),
|
|
965
|
+
setIntegrationSetting: (id, key, value) =>
|
|
966
|
+
raw.setIntegrationSetting(id, key, value),
|
|
967
|
+
setIntegrationSettings: (id, settings) =>
|
|
968
|
+
raw.setIntegrationSettings(id, settings),
|
|
969
|
+
|
|
970
|
+
// Devices: filter out devices belonging to orphaned integrations
|
|
971
|
+
createDevice: (input) => raw.createDevice(input),
|
|
972
|
+
getDevice: async (id) => {
|
|
973
|
+
const d = await raw.getDevice(id);
|
|
974
|
+
if (!d) return null;
|
|
975
|
+
const ids = await ensureActiveIds();
|
|
976
|
+
return ids.has(d.integrationId) ? d : null;
|
|
977
|
+
},
|
|
978
|
+
getDeviceByStableId: async (stableId) => {
|
|
979
|
+
const d = await raw.getDeviceByStableId(stableId);
|
|
980
|
+
if (!d) return null;
|
|
981
|
+
const ids = await ensureActiveIds();
|
|
982
|
+
return ids.has(d.integrationId) ? d : null;
|
|
983
|
+
},
|
|
984
|
+
listDevices: async (integrationId) => {
|
|
985
|
+
const devices = await raw.listDevices(integrationId);
|
|
986
|
+
const ids = await ensureActiveIds();
|
|
987
|
+
return devices.filter((d) => ids.has(d.integrationId));
|
|
988
|
+
},
|
|
989
|
+
listCameras: async () => {
|
|
990
|
+
const cameras = await raw.listCameras();
|
|
991
|
+
const ids = await ensureActiveIds();
|
|
992
|
+
return cameras.filter((d) => ids.has(d.integrationId));
|
|
993
|
+
},
|
|
994
|
+
updateDevice: (id, updates) => raw.updateDevice(id, updates),
|
|
995
|
+
deleteDevice: (id) => raw.deleteDevice(id),
|
|
996
|
+
|
|
997
|
+
// Device settings: passthrough
|
|
998
|
+
getDeviceSettings: (id) => raw.getDeviceSettings(id),
|
|
999
|
+
setDeviceSetting: (id, key, value) =>
|
|
1000
|
+
raw.setDeviceSetting(id, key, value),
|
|
1001
|
+
setDeviceSettings: (id, settings) => raw.setDeviceSettings(id, settings),
|
|
1002
|
+
};
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
// InferenceCapabilitiesService removed — now lives in pipeline-executor addon.
|
|
1006
|
+
// Use capabilityRegistry.getSingleton('pipeline-executor') instead.
|
|
1007
|
+
|
|
1008
|
+
/** Log a standardized lifecycle line for addon start/restart */
|
|
1009
|
+
private logAddonLifecycle(
|
|
1010
|
+
event: "started" | "restarted",
|
|
1011
|
+
id: string,
|
|
1012
|
+
mode: "in-process" | "isolated",
|
|
1013
|
+
): void {
|
|
1014
|
+
const entry = this.addonEntries.get(id);
|
|
1015
|
+
if (!entry) return;
|
|
1016
|
+
const agentName = process.env.CAMSTACK_AGENT_NAME ?? "hub";
|
|
1017
|
+
const platform = `${os.platform()}/${os.arch()}`;
|
|
1018
|
+
const message = `Addon ${event} — v${entry.packageVersion} (${entry.packageName}), ${mode}, ${platform}, agent: ${agentName}`;
|
|
1019
|
+
// Log under AddonRegistry scope
|
|
1020
|
+
this.logger.info(message, { tags: { addonId: id, agentId: agentName }, meta: { phase: 'lifecycle' } });
|
|
1021
|
+
// Also log under the addon's own tagged logger so it appears in the per-addon
|
|
1022
|
+
// log viewer. No scope — brand bracket shows the addon id.
|
|
1023
|
+
const addonLogger = this.loggingService.createLogger();
|
|
1024
|
+
addonLogger.info(message, { tags: { addonId: id, agentId: agentName } });
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
// Platform probing and inference-config resolution have moved into the
|
|
1028
|
+
// `platform-probe` capability. Addons call `ctx.api.platformProbe.*`
|
|
1029
|
+
// during their own initialize(). The backend no longer loops addons
|
|
1030
|
+
// looking for `getModelRequirements` / `configure` — those hooks were
|
|
1031
|
+
// removed from the ICamstackAddon contract.
|
|
1032
|
+
|
|
1033
|
+
registerAddon(
|
|
1034
|
+
addon: ICamstackAddon,
|
|
1035
|
+
source: AddonSource = "installed",
|
|
1036
|
+
packageName?: string,
|
|
1037
|
+
packageVersion?: string,
|
|
1038
|
+
): void {
|
|
1039
|
+
const manifest = addon.manifest;
|
|
1040
|
+
if (!manifest) {
|
|
1041
|
+
throw new Error(
|
|
1042
|
+
"Cannot register addon without manifest — was it created via AddonLoader.createInstance()?",
|
|
1043
|
+
);
|
|
1044
|
+
}
|
|
1045
|
+
this.addonEntries.set(manifest.id, {
|
|
1046
|
+
addon,
|
|
1047
|
+
initialized: false,
|
|
1048
|
+
source,
|
|
1049
|
+
packageName: packageName ?? manifest.name ?? manifest.id,
|
|
1050
|
+
packageVersion: packageVersion ?? manifest.version ?? "0.0.0",
|
|
1051
|
+
declaredCapabilities: this.getAddonCapabilities(addon),
|
|
1052
|
+
});
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
async initializeAddon(id: string): Promise<void> {
|
|
1056
|
+
const entry = this.addonEntries.get(id);
|
|
1057
|
+
if (!entry) {
|
|
1058
|
+
throw new Error(`Addon "${id}" is not registered`);
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
if (!entry.addon?.manifest?.id) {
|
|
1062
|
+
throw new Error(
|
|
1063
|
+
`Addon "${id}" has no manifest.id — check the addon class default export`,
|
|
1064
|
+
);
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
// Decide whether this addon should boot in its own forked runner
|
|
1068
|
+
// subprocess. The runner-spawn pass during initial boot already
|
|
1069
|
+
// spawned every eligible addon; `initializeAddon` is the per-addon
|
|
1070
|
+
// entry point reached by the hot-load / restart flows (and by the
|
|
1071
|
+
// Pass 1/2/3 in-process boot for `@camstack/core` builtins). Fork
|
|
1072
|
+
// only when the addon declares an `execution` block, ships an
|
|
1073
|
+
// on-disk `addonDir`, and is not `agent-only`; otherwise (only
|
|
1074
|
+
// `@camstack/core` builtins reach the in-process path below) it
|
|
1075
|
+
// boots in-process on the hub.
|
|
1076
|
+
if (this.isForkedAddonEntry(entry)) {
|
|
1077
|
+
// D5: one-addon-one-process. A forked addon ALWAYS boots in its
|
|
1078
|
+
// own runner — there is NO in-process-on-the-hub fallback. If the
|
|
1079
|
+
// runner spawn fails, the addon is `failed`, surfaced as
|
|
1080
|
+
// `addon.error` + recorded on the health monitor; it does not
|
|
1081
|
+
// silently run on the hub. (Task 7's circuit breaker governs the
|
|
1082
|
+
// retry policy on top of this.)
|
|
1083
|
+
try {
|
|
1084
|
+
await this.broker.call("$process.spawnRunner", {
|
|
1085
|
+
runnerId: id,
|
|
1086
|
+
addons: [{ addonId: id, addonDir: entry.addonDir }],
|
|
1087
|
+
});
|
|
1088
|
+
} catch (err) {
|
|
1089
|
+
const msg = errMsg(err);
|
|
1090
|
+
this.logger.error(
|
|
1091
|
+
'Failed to spawn isolated runner for addon',
|
|
1092
|
+
{ tags: { addonId: id }, meta: { error: msg } },
|
|
1093
|
+
);
|
|
1094
|
+
this.emitAddonLifecycleEvent("addon.error", id, {
|
|
1095
|
+
error: msg,
|
|
1096
|
+
action: "initialize",
|
|
1097
|
+
});
|
|
1098
|
+
this.healthMonitor.recordFailure(entry.packageName, err, id);
|
|
1099
|
+
throw new Error(
|
|
1100
|
+
`Failed to spawn runner for addon "${id}": ${msg}`,
|
|
1101
|
+
{ cause: err },
|
|
1102
|
+
);
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
// Provider registration for forkable addons is delegated to the
|
|
1106
|
+
// `CapabilityBridge` (see `MoleculerService.onProviderConnected`).
|
|
1107
|
+
// When the spawned child announces its Moleculer service via
|
|
1108
|
+
// NODE_INFO, the bridge builds a proxy from the capability
|
|
1109
|
+
// definition (`{ id, nodeId, ...methods }`) and registers it.
|
|
1110
|
+
//
|
|
1111
|
+
// Custom actions: read the catalog fresh from the on-disk bundle
|
|
1112
|
+
// and register it against the hub-side `CustomActionRegistry`.
|
|
1113
|
+
// `registerForkedAddonCustomActions` re-`import()`s the entry
|
|
1114
|
+
// (cache-busted), so it covers the hot-load path — where the addon
|
|
1115
|
+
// was registered via `freshLoader` and `this.addonLoader`'s cached
|
|
1116
|
+
// `module` namespace is stale or absent. Dispatch routes through
|
|
1117
|
+
// `broker.call('<addonId>.custom.<action>')`, the only divergence
|
|
1118
|
+
// vs in-process being the transport, exactly like cap methods.
|
|
1119
|
+
await this.registerForkedAddonCustomActions(
|
|
1120
|
+
id,
|
|
1121
|
+
// `shouldFork` already asserted `entry.declaration?.execution`.
|
|
1122
|
+
resolveRunnerId(entry.declaration!, id),
|
|
1123
|
+
);
|
|
1124
|
+
|
|
1125
|
+
entry.initialized = true;
|
|
1126
|
+
this.logger.info('Addon spawned as isolated process', { tags: { addonId: id } });
|
|
1127
|
+
this.emitAddonLifecycleEvent("addon.started", id);
|
|
1128
|
+
return;
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
// In-process initialization — capture providers from initialize() return value
|
|
1132
|
+
const context = await this.createAddonContext(entry.addon);
|
|
1133
|
+
const capturedProviders = new Map<string, unknown>();
|
|
1134
|
+
// Device-manager needs privileged access to CapabilityRegistry — it
|
|
1135
|
+
// resolves native addon ids for getBindings, lists registered wrappers,
|
|
1136
|
+
// etc. Inject before initialize() runs so the addon's onInitialize can
|
|
1137
|
+
// rely on it.
|
|
1138
|
+
const initResult = normalizeAddonInitResult(
|
|
1139
|
+
await entry.addon.initialize(context),
|
|
1140
|
+
);
|
|
1141
|
+
// NOTE: `postBrokerStart()` is deliberately NOT called here. It used to
|
|
1142
|
+
// fire right after `initialize()` and would emit readiness events whose
|
|
1143
|
+
// subscribers (remote workers) immediately called `ctx.api.<addon>.*` —
|
|
1144
|
+
// hitting "Service not found" because `broker.createService()` had not
|
|
1145
|
+
// been called yet. See below: postBrokerStart fires after the Moleculer
|
|
1146
|
+
// service mount so the advertised service is live before readiness.
|
|
1147
|
+
|
|
1148
|
+
for (const reg of initResult?.providers ?? []) {
|
|
1149
|
+
const capName = reg.capability.name;
|
|
1150
|
+
capturedProviders.set(capName, reg.provider);
|
|
1151
|
+
context.registerProvider(capName, reg.provider);
|
|
1152
|
+
// D4: wrapper behaviour is read from the cap DEFINITION. The runtime
|
|
1153
|
+
// ProviderRegistration.kind/defaultActive are deprecated hints — if an
|
|
1154
|
+
// addon still sets them and they disagree, warn (the cap def wins).
|
|
1155
|
+
const kindDrift = describeProviderKindDrift(capName, reg.capability, {
|
|
1156
|
+
kind: reg.kind,
|
|
1157
|
+
defaultActive: reg.defaultActive,
|
|
1158
|
+
});
|
|
1159
|
+
if (kindDrift) {
|
|
1160
|
+
this.logger.warn(kindDrift, {
|
|
1161
|
+
tags: { addonId: id },
|
|
1162
|
+
meta: { capability: capName },
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
1165
|
+
if (reg.capability.kind === "wrapper") {
|
|
1166
|
+
this.capabilityRegistry.registerWrapper(capName, id, {
|
|
1167
|
+
defaultActive: reg.capability.defaultActive === true,
|
|
1168
|
+
});
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
// Task 7.1: register system-level custom actions if the addon declares any.
|
|
1173
|
+
if (initResult?.customActions && initResult.actionHandlers) {
|
|
1174
|
+
const handlers = initResult.actionHandlers;
|
|
1175
|
+
try {
|
|
1176
|
+
this.customActionRegistry.registerAddon(
|
|
1177
|
+
id,
|
|
1178
|
+
initResult.customActions,
|
|
1179
|
+
(action, input) => {
|
|
1180
|
+
const fn = handlers[action];
|
|
1181
|
+
if (!fn)
|
|
1182
|
+
throw new Error(
|
|
1183
|
+
`addon '${id}' has no handler for custom action '${action}'`,
|
|
1184
|
+
);
|
|
1185
|
+
return fn(input);
|
|
1186
|
+
},
|
|
1187
|
+
);
|
|
1188
|
+
} catch (err) {
|
|
1189
|
+
this.logger.error(
|
|
1190
|
+
'Failed to register custom actions for addon',
|
|
1191
|
+
{ tags: { addonId: id }, meta: { error: errMsg(err) } },
|
|
1192
|
+
);
|
|
1193
|
+
throw err;
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
// Validate all declared capabilities have providers registered.
|
|
1198
|
+
// Include caps registered via `context.registerProvider()` side-effects
|
|
1199
|
+
// (e.g. sub-components of an addon calling ctx.registerProvider directly
|
|
1200
|
+
// without returning via ProviderRegistration[]), by reading back from the
|
|
1201
|
+
// CapabilityRegistry for this addonId. This keeps the validation accurate
|
|
1202
|
+
// without forcing addons into a single return-path pattern.
|
|
1203
|
+
if (entry.declaration) {
|
|
1204
|
+
for (const cap of entry.declaration.capabilities ?? []) {
|
|
1205
|
+
const capName = typeof cap === "string" ? cap : cap.name;
|
|
1206
|
+
if (capturedProviders.has(capName)) continue;
|
|
1207
|
+
const provider = this.capabilityRegistry.getProviderByAddon(
|
|
1208
|
+
capName,
|
|
1209
|
+
id,
|
|
1210
|
+
);
|
|
1211
|
+
if (provider) capturedProviders.set(capName, provider);
|
|
1212
|
+
}
|
|
1213
|
+
validateProviderRegistrations(
|
|
1214
|
+
id,
|
|
1215
|
+
entry.declaration,
|
|
1216
|
+
capturedProviders,
|
|
1217
|
+
this.logger,
|
|
1218
|
+
);
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
await this.restoreAddonDevices(id, entry.addon);
|
|
1222
|
+
entry.initialized = true;
|
|
1223
|
+
|
|
1224
|
+
// Register as Moleculer service for remote discoverability
|
|
1225
|
+
if (entry.declaration) {
|
|
1226
|
+
try {
|
|
1227
|
+
// Resolve method names from CapabilityDefinitions registered in the registry
|
|
1228
|
+
const methodResolver = (capName: string): readonly string[] => {
|
|
1229
|
+
const def = this.capabilityRegistry.getDefinition(capName);
|
|
1230
|
+
if (!def?.methods) return [];
|
|
1231
|
+
return Object.keys(def.methods);
|
|
1232
|
+
};
|
|
1233
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment -- createAddonService return type unresolvable due to upstream eventemitter2/moleculer type chain
|
|
1234
|
+
const schema = createAddonService(entry.addon, entry.declaration!, {
|
|
1235
|
+
methodResolver,
|
|
1236
|
+
providers: capturedProviders,
|
|
1237
|
+
});
|
|
1238
|
+
this.broker.createService(schema);
|
|
1239
|
+
} catch (err) {
|
|
1240
|
+
this.logger.warn(
|
|
1241
|
+
'Failed to register Moleculer service',
|
|
1242
|
+
{ tags: { addonId: id }, meta: { error: errMsg(err) } },
|
|
1243
|
+
);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
// Post-broker-start readiness — emitted AFTER the Moleculer service is
|
|
1248
|
+
// mounted so that any remote subscriber reacting to a `ready` event can
|
|
1249
|
+
// immediately call `ctx.api.<addon>.*` without hitting a phantom
|
|
1250
|
+
// service window. Hub addons with `autoEmitReadiness=true` emit here;
|
|
1251
|
+
// addons that manage their own readiness override `autoEmitReadiness`.
|
|
1252
|
+
entry.addon.postBrokerStart();
|
|
1253
|
+
|
|
1254
|
+
this.logger.info(
|
|
1255
|
+
'Addon initialized in-process',
|
|
1256
|
+
{ tags: { addonId: id }, meta: { packageName: entry.packageName } },
|
|
1257
|
+
);
|
|
1258
|
+
this.logAddonLifecycle("started", id, "in-process");
|
|
1259
|
+
this.emitAddonLifecycleEvent("addon.started", id);
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
/**
|
|
1263
|
+
* Restore persisted devices for an addon after it has been initialized.
|
|
1264
|
+
* Calls addon.restoreDevices() with the list of devices saved to the DB.
|
|
1265
|
+
* No-op if the addon does not implement restoreDevices() or stores are not available.
|
|
1266
|
+
*/
|
|
1267
|
+
private async restoreAddonDevices(
|
|
1268
|
+
addonId: string,
|
|
1269
|
+
addon: ICamstackAddon,
|
|
1270
|
+
): Promise<void> {
|
|
1271
|
+
if (typeof addon.restoreDevices !== "function") return;
|
|
1272
|
+
|
|
1273
|
+
// Route through the device-persistence capability instead of
|
|
1274
|
+
// accessing DeviceStore/ConfigStore directly. This keeps
|
|
1275
|
+
// AddonRegistryService decoupled from the persistence layer —
|
|
1276
|
+
// the capability addon owns the stores.
|
|
1277
|
+
const api = this.getBrokerApi();
|
|
1278
|
+
try {
|
|
1279
|
+
const listResult: unknown = await (
|
|
1280
|
+
Reflect.get(
|
|
1281
|
+
Reflect.get(api, "deviceManager") as object,
|
|
1282
|
+
"listPersistedByAddon",
|
|
1283
|
+
) as { query: (input: unknown) => Promise<unknown> }
|
|
1284
|
+
).query({ addonId });
|
|
1285
|
+
|
|
1286
|
+
const rows = Array.isArray(listResult)
|
|
1287
|
+
? (listResult as Array<{
|
|
1288
|
+
id: number;
|
|
1289
|
+
stableId: string;
|
|
1290
|
+
type: string;
|
|
1291
|
+
name: string;
|
|
1292
|
+
location?: string | null;
|
|
1293
|
+
disabled?: boolean;
|
|
1294
|
+
parentDeviceId: number | null;
|
|
1295
|
+
}>)
|
|
1296
|
+
: [];
|
|
1297
|
+
if (rows.length === 0) return;
|
|
1298
|
+
|
|
1299
|
+
const savedDevices: SavedDevice[] = await Promise.all(
|
|
1300
|
+
rows.map(async (row) => {
|
|
1301
|
+
const configResult: unknown = await (
|
|
1302
|
+
Reflect.get(
|
|
1303
|
+
Reflect.get(api, "deviceManager") as object,
|
|
1304
|
+
"loadConfig",
|
|
1305
|
+
) as { query: (input: unknown) => Promise<unknown> }
|
|
1306
|
+
).query({ deviceId: row.id });
|
|
1307
|
+
|
|
1308
|
+
return {
|
|
1309
|
+
id: row.id,
|
|
1310
|
+
stableId: row.stableId,
|
|
1311
|
+
type: row.type as import("@camstack/types").DeviceType,
|
|
1312
|
+
name: row.name,
|
|
1313
|
+
location: row.location ?? null,
|
|
1314
|
+
disabled: row.disabled ?? false,
|
|
1315
|
+
parentDeviceId: row.parentDeviceId,
|
|
1316
|
+
config: (configResult ?? {}) as Record<string, unknown>,
|
|
1317
|
+
};
|
|
1318
|
+
}),
|
|
1319
|
+
);
|
|
1320
|
+
|
|
1321
|
+
await addon.restoreDevices!(savedDevices);
|
|
1322
|
+
this.logger.info(
|
|
1323
|
+
'Restored devices for addon',
|
|
1324
|
+
{ tags: { addonId }, meta: { count: savedDevices.length } },
|
|
1325
|
+
);
|
|
1326
|
+
} catch (err) {
|
|
1327
|
+
this.logger.error(
|
|
1328
|
+
'restoreDevices failed for addon',
|
|
1329
|
+
{ tags: { addonId }, meta: { error: errMsg(err) } },
|
|
1330
|
+
);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
/**
|
|
1335
|
+
* Refresh the package version for all addons from a given package.
|
|
1336
|
+
* Call this after an update to ensure the UI shows the new version.
|
|
1337
|
+
*/
|
|
1338
|
+
refreshPackageVersion(packageName: string, newVersion: string): void {
|
|
1339
|
+
for (const [, entry] of this.addonEntries) {
|
|
1340
|
+
if (entry.packageName === packageName) {
|
|
1341
|
+
entry.packageVersion = newVersion;
|
|
1342
|
+
this.logger.info(
|
|
1343
|
+
'Updated addon packageVersion',
|
|
1344
|
+
{ tags: { addonId: entry.addon.manifest!.id }, meta: { newVersion } },
|
|
1345
|
+
);
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
/** Emit addon.uninstalled lifecycle event for all addons belonging to a package */
|
|
1351
|
+
emitUninstallEvent(packageName: string): void {
|
|
1352
|
+
for (const [id, entry] of this.addonEntries) {
|
|
1353
|
+
if (entry.packageName === packageName) {
|
|
1354
|
+
this.emitAddonLifecycleEvent("addon.uninstalled", id);
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
// Drop the package from the health monitor so its retry loop
|
|
1358
|
+
// doesn't keep trying to reload an addon the operator explicitly
|
|
1359
|
+
// removed. Matching `clearLoadFailures` purges any pre-init
|
|
1360
|
+
// failures still tracked by the addon-loader.
|
|
1361
|
+
this.healthMonitor.forget(packageName);
|
|
1362
|
+
this.addonLoader.clearLoadFailures(packageName);
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
/** Emit addon.updated lifecycle event for all addons belonging to a package */
|
|
1366
|
+
emitUpdateEvent(
|
|
1367
|
+
packageName: string,
|
|
1368
|
+
fromVersion: string,
|
|
1369
|
+
toVersion: string,
|
|
1370
|
+
): void {
|
|
1371
|
+
for (const [id, entry] of this.addonEntries) {
|
|
1372
|
+
if (entry.packageName === packageName) {
|
|
1373
|
+
this.emitAddonLifecycleEvent("addon.updated", id, {
|
|
1374
|
+
fromVersion,
|
|
1375
|
+
toVersion,
|
|
1376
|
+
});
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
/**
|
|
1382
|
+
* Restart an addon by ID.
|
|
1383
|
+
* Shuts down, unregisters capabilities, re-initializes, and re-wires.
|
|
1384
|
+
*/
|
|
1385
|
+
async restartAddon(
|
|
1386
|
+
addonId: string,
|
|
1387
|
+
): Promise<{ success: boolean; error?: string }> {
|
|
1388
|
+
const entry = this.addonEntries.get(addonId);
|
|
1389
|
+
if (!entry) {
|
|
1390
|
+
return { success: false, error: `Addon "${addonId}" not found` };
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
this.logger.info('Addon restarting...', { tags: { addonId }, meta: { phase: 'lifecycle' } });
|
|
1394
|
+
|
|
1395
|
+
// Suppress the "Failed to load" health banner during operator-initiated
|
|
1396
|
+
// restarts. The Moleculer `$node.disconnected` handler skips entries
|
|
1397
|
+
// present in `restartingAddons`; a 90s safety timer clears the flag
|
|
1398
|
+
// even if the restart path throws before the finally block runs.
|
|
1399
|
+
const existingTimer = this.restartingAddons.get(addonId);
|
|
1400
|
+
if (existingTimer) clearTimeout(existingTimer);
|
|
1401
|
+
const safetyTimer = setTimeout(() => {
|
|
1402
|
+
this.restartingAddons.delete(addonId);
|
|
1403
|
+
}, 90_000);
|
|
1404
|
+
this.restartingAddons.set(addonId, safetyTimer);
|
|
1405
|
+
|
|
1406
|
+
try {
|
|
1407
|
+
// Group-runner-hosted addon — delegate to $process.restart for the group
|
|
1408
|
+
if (this.isForkedAddonEntry(entry)) {
|
|
1409
|
+
const result = (await this.broker.call("$process.restart", {
|
|
1410
|
+
name: addonId,
|
|
1411
|
+
})) as { success: boolean; reason?: string };
|
|
1412
|
+
if (!result.success) {
|
|
1413
|
+
throw new Error(
|
|
1414
|
+
`Process restart failed: ${result.reason ?? "unknown"}`,
|
|
1415
|
+
);
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
// $process.restart resolves as soon as the child is respawned, not when its
|
|
1419
|
+
// capabilities are re-registered. Callers (integrations.create, UI forms) may
|
|
1420
|
+
// immediately try to route to the provider and hit a transient null. Block here
|
|
1421
|
+
// until every declared capability is back on the registry so the restart is
|
|
1422
|
+
// observable-consistent from the caller's perspective.
|
|
1423
|
+
const declared = entry.declaredCapabilities;
|
|
1424
|
+
if (declared.length > 0) {
|
|
1425
|
+
// Group-runner restarts pay a multi-step cost: child fork
|
|
1426
|
+
// (~1s) → addon init (Python pool warmup ~5-10s, or a CLI
|
|
1427
|
+
// subprocess probe for tailscale-ingress etc.) → capability
|
|
1428
|
+
// advertisement to hub via Moleculer service discovery
|
|
1429
|
+
// (~5s on cold start). Any fixed ceiling is wrong — a slow
|
|
1430
|
+
// restart that DID eventually succeed produces a misleading
|
|
1431
|
+
// "did not re-register in time" error. Wait indefinitely: if
|
|
1432
|
+
// the group-runner crashed for real, `$node.disconnected`
|
|
1433
|
+
// surfaces it through the health monitor; the operator can
|
|
1434
|
+
// always click Cancel on the UI mutation.
|
|
1435
|
+
const waits = declared.map((cap) =>
|
|
1436
|
+
this.capabilityRegistry.waitForProvider(cap.name, addonId, Number.POSITIVE_INFINITY),
|
|
1437
|
+
);
|
|
1438
|
+
const settled = await Promise.all(waits);
|
|
1439
|
+
const missing = declared
|
|
1440
|
+
.map((cap, i) => (settled[i] == null ? cap.name : null))
|
|
1441
|
+
.filter((name): name is string => name !== null);
|
|
1442
|
+
if (missing.length > 0) {
|
|
1443
|
+
throw new Error(
|
|
1444
|
+
`Addon "${addonId}" restarted but did not re-register capabilities in time: ${missing.join(", ")}`,
|
|
1445
|
+
);
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
// Re-register the addon's custom-action catalog. `$process.restart`
|
|
1450
|
+
// respawns the group child and `CapabilityBridge` re-registers cap
|
|
1451
|
+
// providers — but custom actions live in the hub-side
|
|
1452
|
+
// `CustomActionRegistry`, which the restart path never touched.
|
|
1453
|
+
// Without this, every hot-update of a group-hosted addon silently
|
|
1454
|
+
// drops its custom actions (the catalog is only registered once,
|
|
1455
|
+
// at boot, in `initializeAddonGroup`). Reads a fresh catalog from
|
|
1456
|
+
// the just-updated on-disk bundle.
|
|
1457
|
+
const runnerId = resolveRunnerId(entry.declaration!, addonId);
|
|
1458
|
+
await this.registerForkedAddonCustomActions(addonId, runnerId);
|
|
1459
|
+
|
|
1460
|
+
this.logAddonLifecycle("restarted", addonId, "isolated");
|
|
1461
|
+
this.emitAddonLifecycleEvent("addon.restarted", addonId);
|
|
1462
|
+
return { success: true };
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
// D5: a forked addon ALWAYS restarts via `$process.restart` above
|
|
1466
|
+
// (process restart — no in-process hot-reload of a live forked
|
|
1467
|
+
// addon). This branch is reached only by `@camstack/core` builtins
|
|
1468
|
+
// — they have no runner and legitimately reload in-process:
|
|
1469
|
+
// shutdown, unregister, re-initialize, re-wire.
|
|
1470
|
+
if (entry.initialized && entry.addon.shutdown) {
|
|
1471
|
+
await entry.addon.shutdown();
|
|
1472
|
+
}
|
|
1473
|
+
// Drain disposers registered via ctx.addDisposer(...)
|
|
1474
|
+
await this.drainDisposerChain(addonId);
|
|
1475
|
+
|
|
1476
|
+
// Unregister all capabilities provided by this addon
|
|
1477
|
+
for (const cap of entry.declaredCapabilities) {
|
|
1478
|
+
this.capabilityRegistry.unregisterProvider(cap.name, addonId);
|
|
1479
|
+
}
|
|
1480
|
+
const manifestCaps = this.getAddonCapabilities(entry.addon);
|
|
1481
|
+
for (const cap of manifestCaps) {
|
|
1482
|
+
this.capabilityRegistry.unregisterProvider(cap.name, addonId);
|
|
1483
|
+
}
|
|
1484
|
+
this.capabilityRegistry.unregisterAllWrappersForAddon(addonId);
|
|
1485
|
+
|
|
1486
|
+
// Task 7.1: drop any registered custom actions so re-initialization re-registers cleanly.
|
|
1487
|
+
this.customActionRegistry.unregisterAddon(addonId);
|
|
1488
|
+
|
|
1489
|
+
// Destroy existing Moleculer service for this addon
|
|
1490
|
+
try {
|
|
1491
|
+
await (
|
|
1492
|
+
this.moleculer.broker as unknown as {
|
|
1493
|
+
destroyService(name: string): Promise<void>;
|
|
1494
|
+
}
|
|
1495
|
+
).destroyService(addonId);
|
|
1496
|
+
} catch {
|
|
1497
|
+
// Service may not exist if it was never registered
|
|
1498
|
+
}
|
|
1499
|
+
|
|
1500
|
+
entry.initialized = false;
|
|
1501
|
+
|
|
1502
|
+
// Re-initialize and re-wire capabilities
|
|
1503
|
+
await this.initializeAddon(addonId);
|
|
1504
|
+
this.wireCapabilities(addonId);
|
|
1505
|
+
|
|
1506
|
+
this.logAddonLifecycle("restarted", addonId, "in-process");
|
|
1507
|
+
this.emitAddonLifecycleEvent("addon.restarted", addonId);
|
|
1508
|
+
return { success: true };
|
|
1509
|
+
} catch (err) {
|
|
1510
|
+
const msg = errMsg(err);
|
|
1511
|
+
this.logger.error('Failed to restart addon', { tags: { addonId }, meta: { error: msg } });
|
|
1512
|
+
this.emitAddonLifecycleEvent("addon.error", addonId, {
|
|
1513
|
+
error: msg,
|
|
1514
|
+
action: "restart",
|
|
1515
|
+
});
|
|
1516
|
+
// Record the failure on the health monitor so the addon shows up
|
|
1517
|
+
// `failed` with its `lastError` instead of a stale `healthy`. The
|
|
1518
|
+
// in-process branch already unregistered every capability before
|
|
1519
|
+
// `initializeAddon` threw — without this the addon is invisible-
|
|
1520
|
+
// broken: caps gone, health green, nothing on the Addons page.
|
|
1521
|
+
// The "operator sees the error via the mutation result" assumption
|
|
1522
|
+
// only holds for a UI-clicked restart; a `camstack deploy` or a
|
|
1523
|
+
// `requiresRestart` auto-restart is fire-and-forget — the failure
|
|
1524
|
+
// would otherwise vanish. recordFailure also arms the retry loop.
|
|
1525
|
+
this.healthMonitor.recordFailure(entry.packageName, err, addonId);
|
|
1526
|
+
return { success: false, error: msg };
|
|
1527
|
+
} finally {
|
|
1528
|
+
// Clear the suppression flag regardless of success/failure — if the
|
|
1529
|
+
// restart failed, the operator will see the real error via the
|
|
1530
|
+
// mutation result rather than a misleading transient health blip.
|
|
1531
|
+
const timer = this.restartingAddons.get(addonId);
|
|
1532
|
+
if (timer) clearTimeout(timer);
|
|
1533
|
+
this.restartingAddons.delete(addonId);
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
getAddon(id: string): ICamstackAddon | null {
|
|
1538
|
+
const entry = this.addonEntries.get(id);
|
|
1539
|
+
return entry?.addon ?? null;
|
|
1540
|
+
}
|
|
1541
|
+
|
|
1542
|
+
getAddonEntry(id: string): { addon: ICamstackAddon } | null {
|
|
1543
|
+
const entry = this.addonEntries.get(id);
|
|
1544
|
+
return entry ? { addon: entry.addon } : null;
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1547
|
+
/**
|
|
1548
|
+
* Returns the on-disk package directory for the given addon ID.
|
|
1549
|
+
* Used by the /api/addon-assets route to serve static files (e.g. SVG icons).
|
|
1550
|
+
*
|
|
1551
|
+
* Falls back to a disk scan when the addon isn't in `addonEntries`
|
|
1552
|
+
* (typical for failed-to-load packages: dist/manifest still on disk
|
|
1553
|
+
* but the import threw, so the registry never recorded an entry).
|
|
1554
|
+
* Without this fallback the icon endpoint 404s and the operator
|
|
1555
|
+
* sees a broken-image placeholder for any failed addon.
|
|
1556
|
+
*/
|
|
1557
|
+
getAddonPackageDir(id: string): string | null {
|
|
1558
|
+
const entry = this.addonEntries.get(id);
|
|
1559
|
+
if (entry?.addonDir) return entry.addonDir;
|
|
1560
|
+
return this.findAddonDirOnDisk(id);
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
private findAddonDirOnDisk(addonId: string): string | null {
|
|
1564
|
+
const dataDir = path.resolve(process.env.CAMSTACK_DATA ?? "camstack-data");
|
|
1565
|
+
const addonsDir = path.resolve(dataDir, "addons");
|
|
1566
|
+
if (!fs.existsSync(addonsDir)) return null;
|
|
1567
|
+
|
|
1568
|
+
const visit = (pkgDir: string): string | null => {
|
|
1569
|
+
const pkgJsonPath = path.join(pkgDir, "package.json");
|
|
1570
|
+
if (!fs.existsSync(pkgJsonPath)) return null;
|
|
1571
|
+
try {
|
|
1572
|
+
const pkg = JSON.parse(fs.readFileSync(pkgJsonPath, "utf-8")) as Record<string, unknown>;
|
|
1573
|
+
const camstack = (pkg["camstack"] as { addons?: unknown[] } | undefined) ?? {};
|
|
1574
|
+
const addons = Array.isArray(camstack.addons) ? camstack.addons as Record<string, unknown>[] : [];
|
|
1575
|
+
if (addons.some((a) => a["id"] === addonId)) return pkgDir;
|
|
1576
|
+
} catch {
|
|
1577
|
+
return null;
|
|
1578
|
+
}
|
|
1579
|
+
return null;
|
|
1580
|
+
};
|
|
1581
|
+
|
|
1582
|
+
const entries = fs.readdirSync(addonsDir, { withFileTypes: true });
|
|
1583
|
+
for (const entry of entries) {
|
|
1584
|
+
if (!entry.isDirectory()) continue;
|
|
1585
|
+
const dirName = entry.name;
|
|
1586
|
+
if (dirName === "node_modules" || dirName.startsWith(".")) continue;
|
|
1587
|
+
const fullPath = path.join(addonsDir, dirName);
|
|
1588
|
+
if (dirName.startsWith("@")) {
|
|
1589
|
+
for (const sub of fs.readdirSync(fullPath, { withFileTypes: true })) {
|
|
1590
|
+
if (!sub.isDirectory()) continue;
|
|
1591
|
+
const found = visit(path.join(fullPath, sub.name));
|
|
1592
|
+
if (found) return found;
|
|
1593
|
+
}
|
|
1594
|
+
} else {
|
|
1595
|
+
const found = visit(fullPath);
|
|
1596
|
+
if (found) return found;
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
return null;
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1602
|
+
/**
|
|
1603
|
+
* Resolve an addon's on-disk installation directory and the dist
|
|
1604
|
+
* sub-directory that contains its built entry. For singleton-package
|
|
1605
|
+
* addons the dist sub-directory is `dist/`; for bundled addons (where
|
|
1606
|
+
* one npm package ships multiple addon entries) it's `dist/<entryId>/`.
|
|
1607
|
+
*
|
|
1608
|
+
* Used by the static-file route handler that serves widget bundles
|
|
1609
|
+
* (`/api/addon-widgets/:addonId/*`) — post bundle merge, the addonId →
|
|
1610
|
+
* package-path mapping is no longer 1:1, so callers MUST go through
|
|
1611
|
+
* the registry instead of guessing `addons/@camstack/addon-<id>/dist/`.
|
|
1612
|
+
*
|
|
1613
|
+
* Returns null if the addonId is unknown or the entry's package.json
|
|
1614
|
+
* has no `entry` field. The returned `distDir` is an absolute path.
|
|
1615
|
+
*/
|
|
1616
|
+
getAddonInstallPath(addonId: string): { addonDir: string; distDir: string } | null {
|
|
1617
|
+
const entry = this.addonEntries.get(addonId);
|
|
1618
|
+
if (!entry?.addonDir || !entry.declaration?.entry) return null;
|
|
1619
|
+
const distSubdir = path.dirname(entry.declaration.entry);
|
|
1620
|
+
return {
|
|
1621
|
+
addonDir: entry.addonDir,
|
|
1622
|
+
distDir: path.resolve(entry.addonDir, distSubdir),
|
|
1623
|
+
};
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
/**
|
|
1627
|
+
* Build an AddonContext for a given addon — same as what initialize() receives.
|
|
1628
|
+
* Used by the benchmark system to create fresh addon instances with custom config.
|
|
1629
|
+
*/
|
|
1630
|
+
async buildAddonContext(addonId: string): Promise<AddonContext | null> {
|
|
1631
|
+
const entry = this.addonEntries.get(addonId);
|
|
1632
|
+
if (!entry) return null;
|
|
1633
|
+
return this.createAddonContext(entry.addon);
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
/**
|
|
1637
|
+
* True when the entry's manifest opted into protection (uninstall
|
|
1638
|
+
* disabled). The flag is per-addon (`camstack.addons[N].protected`)
|
|
1639
|
+
* but the kernel uninstall API operates on packages — so a package
|
|
1640
|
+
* is treated as protected if ANY of its addons declares the flag.
|
|
1641
|
+
* Computed at call time from the running registry; no hardcoded
|
|
1642
|
+
* package list to keep in sync.
|
|
1643
|
+
*/
|
|
1644
|
+
isPackageProtected(packageName: string): boolean {
|
|
1645
|
+
for (const entry of this.addonEntries.values()) {
|
|
1646
|
+
if (entry.packageName !== packageName) continue;
|
|
1647
|
+
if (entry.declaration?.protected === true) return true;
|
|
1648
|
+
}
|
|
1649
|
+
return false;
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
/**
|
|
1653
|
+
* True when the entry boots in its own forked runner: it declares an
|
|
1654
|
+
* `execution` block, ships an on-disk `addonDir`, and is not
|
|
1655
|
+
* `agent-only`. Everything else (only `@camstack/core` builtins)
|
|
1656
|
+
* boots in-process on the hub. The type predicate narrows `addonDir`
|
|
1657
|
+
* to `string` for callers.
|
|
1658
|
+
*/
|
|
1659
|
+
private isForkedAddonEntry(
|
|
1660
|
+
entry: AddonEntry,
|
|
1661
|
+
): entry is AddonEntry & { addonDir: string } {
|
|
1662
|
+
return !!(
|
|
1663
|
+
entry.declaration?.execution &&
|
|
1664
|
+
entry.addonDir &&
|
|
1665
|
+
resolveAddonPlacement(entry.declaration) !== 'agent-only'
|
|
1666
|
+
);
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
/** Per-entry helper used by `listAddons()` to mark a row removable / not. */
|
|
1670
|
+
private isRequiredEntry(entry: AddonEntry): boolean {
|
|
1671
|
+
if (entry.declaration?.protected === true) return true;
|
|
1672
|
+
// Fall back to a sibling addon in the same package — the
|
|
1673
|
+
// protected-by-association rule above. Catches the case where the
|
|
1674
|
+
// SAME package ships an aggregator addon (protected: true) plus
|
|
1675
|
+
// a source addon (protected: false): both rows should render
|
|
1676
|
+
// as non-removable because uninstalling the package would tear
|
|
1677
|
+
// out both.
|
|
1678
|
+
return this.isPackageProtected(entry.packageName);
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
listAddons(): Array<{
|
|
1682
|
+
manifest: AddonDeclaration & {
|
|
1683
|
+
packageName: string;
|
|
1684
|
+
packageVersion: string;
|
|
1685
|
+
packageDisplayName?: string;
|
|
1686
|
+
bundle?: { displayName: string; description?: string; icon?: string };
|
|
1687
|
+
protected?: boolean;
|
|
1688
|
+
removable?: boolean;
|
|
1689
|
+
};
|
|
1690
|
+
declaration?: AddonDeclaration;
|
|
1691
|
+
source: AddonSource;
|
|
1692
|
+
installSource?: "npm" | "local" | "upload";
|
|
1693
|
+
process?: { pid?: number; mode: "in-process"; state: string };
|
|
1694
|
+
}> {
|
|
1695
|
+
// Build process info map from in-memory data (sync, no stats)
|
|
1696
|
+
const processMap = new Map<
|
|
1697
|
+
string,
|
|
1698
|
+
{ pid?: number; mode: "in-process"; state: string }
|
|
1699
|
+
>();
|
|
1700
|
+
for (const [id, entry] of this.addonEntries) {
|
|
1701
|
+
if (entry.initialized) {
|
|
1702
|
+
processMap.set(`addon:${id}`, {
|
|
1703
|
+
pid: process.pid,
|
|
1704
|
+
mode: "in-process",
|
|
1705
|
+
state: "running",
|
|
1706
|
+
});
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
|
|
1710
|
+
const live = Array.from(this.addonEntries.values())
|
|
1711
|
+
.filter((entry) => entry.addon?.manifest?.id)
|
|
1712
|
+
.map((entry) => {
|
|
1713
|
+
let installSource: "npm" | "local" | "upload" | undefined;
|
|
1714
|
+
if (entry.addonDir) {
|
|
1715
|
+
try {
|
|
1716
|
+
const markerPath = path.join(entry.addonDir, ".install-source");
|
|
1717
|
+
if (fs.existsSync(markerPath)) {
|
|
1718
|
+
const raw = fs.readFileSync(markerPath, "utf-8").trim();
|
|
1719
|
+
// Normalize legacy 'workspace' → 'local'
|
|
1720
|
+
const normalized = raw === "workspace" ? "local" : raw;
|
|
1721
|
+
if (
|
|
1722
|
+
normalized === "npm" ||
|
|
1723
|
+
normalized === "local" ||
|
|
1724
|
+
normalized === "upload"
|
|
1725
|
+
) {
|
|
1726
|
+
installSource = normalized;
|
|
1727
|
+
}
|
|
1728
|
+
}
|
|
1729
|
+
} catch (err) {
|
|
1730
|
+
this.logger.debug(
|
|
1731
|
+
'Failed to read install-source marker for addon',
|
|
1732
|
+
{ meta: { error: errMsg(err) } },
|
|
1733
|
+
);
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1736
|
+
return {
|
|
1737
|
+
manifest: {
|
|
1738
|
+
...entry.addon.manifest!,
|
|
1739
|
+
packageName: entry.packageName,
|
|
1740
|
+
packageVersion: entry.packageVersion,
|
|
1741
|
+
packageDisplayName: entry.packageDisplayName,
|
|
1742
|
+
...(entry.bundle !== undefined ? { bundle: entry.bundle } : {}),
|
|
1743
|
+
protected: this.isRequiredEntry(entry) || undefined,
|
|
1744
|
+
removable: this.isRequiredEntry(entry) ? false : undefined,
|
|
1745
|
+
},
|
|
1746
|
+
declaration: entry.declaration,
|
|
1747
|
+
source: entry.source,
|
|
1748
|
+
installSource,
|
|
1749
|
+
process: processMap.get(`addon:${entry.addon.manifest!.id}`),
|
|
1750
|
+
};
|
|
1751
|
+
});
|
|
1752
|
+
|
|
1753
|
+
// Surface packages that exist on disk but failed to load — typical
|
|
1754
|
+
// causes are dep-version mismatch, missing native binary, or a
|
|
1755
|
+
// corrupted dist. Without this, an operator sees "installed" via
|
|
1756
|
+
// the package manifest but can't find the row in the addons UI to
|
|
1757
|
+
// diagnose or uninstall, leaving the package stranded.
|
|
1758
|
+
const seenPackages = new Set(live.map((row) => row.manifest.packageName));
|
|
1759
|
+
const failed = this.scanFailedToLoadPackages(seenPackages);
|
|
1760
|
+
return [...live, ...failed];
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
/**
|
|
1764
|
+
* Walk `<dataDir>/addons/` for `package.json` files whose declared
|
|
1765
|
+
* `camstack.addons[]` entries don't appear in `addonEntries`. Returns
|
|
1766
|
+
* synthetic listing rows so the UI can show + delete them. The rows
|
|
1767
|
+
* carry a `process.state: 'failed'` flag so the AddonCard can render
|
|
1768
|
+
* the diagnostic state distinctly from a normal addon.
|
|
1769
|
+
*/
|
|
1770
|
+
private scanFailedToLoadPackages(
|
|
1771
|
+
seenPackages: ReadonlySet<string>,
|
|
1772
|
+
): Array<{
|
|
1773
|
+
manifest: AddonDeclaration & {
|
|
1774
|
+
packageName: string;
|
|
1775
|
+
packageVersion: string;
|
|
1776
|
+
packageDisplayName?: string;
|
|
1777
|
+
protected?: boolean;
|
|
1778
|
+
removable?: boolean;
|
|
1779
|
+
};
|
|
1780
|
+
declaration?: AddonDeclaration;
|
|
1781
|
+
source: AddonSource;
|
|
1782
|
+
installSource?: "npm" | "local" | "upload";
|
|
1783
|
+
process?: { pid?: number; mode: "in-process"; state: string };
|
|
1784
|
+
}> {
|
|
1785
|
+
const dataDir = path.resolve(process.env.CAMSTACK_DATA ?? "camstack-data");
|
|
1786
|
+
const addonsDir = path.resolve(dataDir, "addons");
|
|
1787
|
+
if (!fs.existsSync(addonsDir)) return [];
|
|
1788
|
+
|
|
1789
|
+
type Row = ReturnType<AddonRegistryService["scanFailedToLoadPackages"]> extends Array<infer R> ? R : never;
|
|
1790
|
+
const out: Row[] = [];
|
|
1791
|
+
const visit = (pkgDir: string): void => {
|
|
1792
|
+
const pkgJsonPath = path.join(pkgDir, "package.json");
|
|
1793
|
+
if (!fs.existsSync(pkgJsonPath)) return;
|
|
1794
|
+
let pkg: Record<string, unknown>;
|
|
1795
|
+
try {
|
|
1796
|
+
pkg = JSON.parse(fs.readFileSync(pkgJsonPath, "utf-8")) as Record<string, unknown>;
|
|
1797
|
+
} catch {
|
|
1798
|
+
return;
|
|
1799
|
+
}
|
|
1800
|
+
const packageName = typeof pkg["name"] === "string" ? pkg["name"] : "";
|
|
1801
|
+
const packageVersion = typeof pkg["version"] === "string" ? pkg["version"] : "0.0.0";
|
|
1802
|
+
if (!packageName || seenPackages.has(packageName)) return;
|
|
1803
|
+
const camstack = (pkg["camstack"] as { addons?: unknown[] } | undefined) ?? {};
|
|
1804
|
+
const addons = Array.isArray(camstack.addons) ? camstack.addons as Record<string, unknown>[] : [];
|
|
1805
|
+
if (addons.length === 0) return;
|
|
1806
|
+
|
|
1807
|
+
let installSource: "npm" | "local" | "upload" | undefined;
|
|
1808
|
+
try {
|
|
1809
|
+
const markerPath = path.join(pkgDir, ".install-source");
|
|
1810
|
+
if (fs.existsSync(markerPath)) {
|
|
1811
|
+
const raw = fs.readFileSync(markerPath, "utf-8").trim();
|
|
1812
|
+
const normalized = raw === "workspace" ? "local" : raw;
|
|
1813
|
+
if (normalized === "npm" || normalized === "local" || normalized === "upload") {
|
|
1814
|
+
installSource = normalized;
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
} catch { /* non-critical */ }
|
|
1818
|
+
|
|
1819
|
+
// Manifest-driven protection — agree with the live-rows path.
|
|
1820
|
+
const isProtected = addons.some((a) => a["protected"] === true);
|
|
1821
|
+
const packageDisplayName = (camstack as { displayName?: string }).displayName;
|
|
1822
|
+
// Bundle metadata — present when the package ships multiple addon
|
|
1823
|
+
// entries that should render as collapsible children in the UI.
|
|
1824
|
+
const bundleRaw = (camstack as { bundle?: { displayName?: string; description?: string; icon?: string } }).bundle;
|
|
1825
|
+
const bundle = bundleRaw && typeof bundleRaw.displayName === 'string'
|
|
1826
|
+
? {
|
|
1827
|
+
displayName: bundleRaw.displayName,
|
|
1828
|
+
...(bundleRaw.description !== undefined ? { description: bundleRaw.description } : {}),
|
|
1829
|
+
...(bundleRaw.icon !== undefined ? { icon: bundleRaw.icon } : {}),
|
|
1830
|
+
}
|
|
1831
|
+
: undefined;
|
|
1832
|
+
|
|
1833
|
+
for (const decl of addons) {
|
|
1834
|
+
const id = typeof decl["id"] === "string" ? decl["id"] : "";
|
|
1835
|
+
const name = typeof decl["name"] === "string" ? decl["name"] : id;
|
|
1836
|
+
if (!id) continue;
|
|
1837
|
+
const manifest: Row["manifest"] = {
|
|
1838
|
+
...(decl as unknown as AddonDeclaration),
|
|
1839
|
+
id,
|
|
1840
|
+
name,
|
|
1841
|
+
packageName,
|
|
1842
|
+
packageVersion,
|
|
1843
|
+
...(packageDisplayName ? { packageDisplayName } : {}),
|
|
1844
|
+
...(bundle !== undefined ? { bundle } : {}),
|
|
1845
|
+
protected: isProtected || undefined,
|
|
1846
|
+
removable: !isProtected,
|
|
1847
|
+
};
|
|
1848
|
+
out.push({
|
|
1849
|
+
manifest,
|
|
1850
|
+
declaration: decl as unknown as AddonDeclaration,
|
|
1851
|
+
source: "installed" as AddonSource,
|
|
1852
|
+
...(installSource ? { installSource } : {}),
|
|
1853
|
+
process: { mode: "in-process" as const, state: "failed" },
|
|
1854
|
+
});
|
|
1855
|
+
}
|
|
1856
|
+
};
|
|
1857
|
+
|
|
1858
|
+
const entries = fs.readdirSync(addonsDir, { withFileTypes: true });
|
|
1859
|
+
for (const entry of entries) {
|
|
1860
|
+
if (!entry.isDirectory()) continue;
|
|
1861
|
+
const dirName = entry.name;
|
|
1862
|
+
if (dirName === "node_modules" || dirName.startsWith(".")) continue;
|
|
1863
|
+
const fullPath = path.join(addonsDir, dirName);
|
|
1864
|
+
// Scoped: walk one level deeper.
|
|
1865
|
+
if (dirName.startsWith("@")) {
|
|
1866
|
+
const scopedEntries = fs.readdirSync(fullPath, { withFileTypes: true });
|
|
1867
|
+
for (const sub of scopedEntries) {
|
|
1868
|
+
if (sub.isDirectory()) visit(path.join(fullPath, sub.name));
|
|
1869
|
+
}
|
|
1870
|
+
} else {
|
|
1871
|
+
visit(fullPath);
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
return out;
|
|
1875
|
+
}
|
|
1876
|
+
|
|
1877
|
+
/**
|
|
1878
|
+
* List all addons including core builtins.
|
|
1879
|
+
* Each addon carries packageName/packageVersion from its package.json,
|
|
1880
|
+
* so the frontend can group by package automatically.
|
|
1881
|
+
*/
|
|
1882
|
+
listAllAddons() {
|
|
1883
|
+
return this.listAddons();
|
|
1884
|
+
}
|
|
1885
|
+
|
|
1886
|
+
/**
|
|
1887
|
+
* List all addon processes (in-process addons)
|
|
1888
|
+
* as ManagedProcessStatus-compatible entries for the process tree.
|
|
1889
|
+
*/
|
|
1890
|
+
async listAddonProcesses(): Promise<
|
|
1891
|
+
ReadonlyArray<{
|
|
1892
|
+
id: string;
|
|
1893
|
+
label: string;
|
|
1894
|
+
state: "running" | "stopped" | "crashed" | "starting";
|
|
1895
|
+
pid?: number;
|
|
1896
|
+
stats?: {
|
|
1897
|
+
pid: number;
|
|
1898
|
+
cpu: number;
|
|
1899
|
+
memory: number;
|
|
1900
|
+
uptime: number;
|
|
1901
|
+
restartCount: number;
|
|
1902
|
+
};
|
|
1903
|
+
restartCount: number;
|
|
1904
|
+
mode: "in-process";
|
|
1905
|
+
}>
|
|
1906
|
+
> {
|
|
1907
|
+
const result: Array<{
|
|
1908
|
+
id: string;
|
|
1909
|
+
label: string;
|
|
1910
|
+
state: "running" | "stopped" | "crashed" | "starting";
|
|
1911
|
+
pid?: number;
|
|
1912
|
+
stats?: {
|
|
1913
|
+
pid: number;
|
|
1914
|
+
cpu: number;
|
|
1915
|
+
memory: number;
|
|
1916
|
+
uptime: number;
|
|
1917
|
+
restartCount: number;
|
|
1918
|
+
};
|
|
1919
|
+
restartCount: number;
|
|
1920
|
+
mode: "in-process";
|
|
1921
|
+
}> = [];
|
|
1922
|
+
|
|
1923
|
+
// Collect hub PID and fetch stats
|
|
1924
|
+
const { getPidStats } = await import("@camstack/core");
|
|
1925
|
+
const pidStats = await getPidStats([process.pid]);
|
|
1926
|
+
|
|
1927
|
+
// In-process addons
|
|
1928
|
+
const hubStats = pidStats.get(process.pid);
|
|
1929
|
+
for (const [id, entry] of this.addonEntries) {
|
|
1930
|
+
if (entry.initialized) {
|
|
1931
|
+
result.push({
|
|
1932
|
+
id: `addon:${id}`,
|
|
1933
|
+
label: `Addon: ${id} (in-process)`,
|
|
1934
|
+
state: "running",
|
|
1935
|
+
pid: process.pid,
|
|
1936
|
+
stats: {
|
|
1937
|
+
pid: process.pid,
|
|
1938
|
+
cpu: hubStats?.cpu ?? 0,
|
|
1939
|
+
memory: hubStats?.memory ?? 0,
|
|
1940
|
+
uptime: Math.round(process.uptime()),
|
|
1941
|
+
restartCount: 0,
|
|
1942
|
+
},
|
|
1943
|
+
restartCount: 0,
|
|
1944
|
+
mode: "in-process",
|
|
1945
|
+
});
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
return result;
|
|
1950
|
+
}
|
|
1951
|
+
|
|
1952
|
+
/**
|
|
1953
|
+
* Phase 11 (settings redesign): `getAddonConfigSchema`,
|
|
1954
|
+
* `getAddonConfig`, and `updateAddonConfig` — the legacy adapter
|
|
1955
|
+
* shim that synthesised a plain `ConfigUISchema` view from the new
|
|
1956
|
+
* three-level methods — are gone. All settings traffic goes through
|
|
1957
|
+
* the six Phase 5 tRPC endpoints (`addons.getAddonSettings`,
|
|
1958
|
+
* `updateAddonSettings`, `getGlobalSettings`, `updateGlobalSettings`,
|
|
1959
|
+
* `getDeviceSettings`, `updateDeviceSettings`) which call the
|
|
1960
|
+
* addon's methods directly via the `addon-settings` singleton capability.
|
|
1961
|
+
*/
|
|
1962
|
+
|
|
1963
|
+
/**
|
|
1964
|
+
* Scan data/addons/ for new packages and initialize any addon not already registered.
|
|
1965
|
+
* Called after install/uninstall to hot-load new addons without server restart.
|
|
1966
|
+
*/
|
|
1967
|
+
async loadNewAddons(): Promise<{ loaded: string[]; failed: string[] }> {
|
|
1968
|
+
const dataDir = path.resolve(process.env.CAMSTACK_DATA ?? "camstack-data");
|
|
1969
|
+
const addonsDir = path.resolve(dataDir, "addons");
|
|
1970
|
+
|
|
1971
|
+
// Create a fresh loader to discover what's on disk (for diffing against existing)
|
|
1972
|
+
const freshLoader = new AddonLoader(
|
|
1973
|
+
this.loggingService.createLogger("AddonLoader"),
|
|
1974
|
+
);
|
|
1975
|
+
await freshLoader.loadFromDirectory(addonsDir);
|
|
1976
|
+
|
|
1977
|
+
const loaded: string[] = [];
|
|
1978
|
+
const failed: string[] = [];
|
|
1979
|
+
|
|
1980
|
+
for (const registered of freshLoader.listAddons()) {
|
|
1981
|
+
// Already-registered addon: refresh the in-memory metadata (the
|
|
1982
|
+
// `packageVersion`/`packageDisplayName`/`declaration`/`bundle`
|
|
1983
|
+
// fields stored on the entry) from the fresh disk scan. Without
|
|
1984
|
+
// this step a hot-update via `installFromTgz` would leave
|
|
1985
|
+
// `entry.packageVersion` stale — the forked group-runner picks up
|
|
1986
|
+
// the new code on `restartAddon` but the UI + listAddons() keep
|
|
1987
|
+
// showing the OLD version. We deliberately keep `entry.addon`
|
|
1988
|
+
// (the running instance) and `entry.initialized` untouched so
|
|
1989
|
+
// `restartAddon` still shuts the live instance down cleanly.
|
|
1990
|
+
const existing = this.addonEntries.get(registered.declaration.id);
|
|
1991
|
+
if (existing) {
|
|
1992
|
+
existing.packageVersion = registered.packageVersion;
|
|
1993
|
+
existing.packageName = registered.packageName;
|
|
1994
|
+
existing.packageDisplayName = registered.packageDisplayName;
|
|
1995
|
+
if (registered.bundle !== undefined) {
|
|
1996
|
+
existing.bundle = registered.bundle;
|
|
1997
|
+
}
|
|
1998
|
+
existing.declaration = registered.declaration;
|
|
1999
|
+
existing.declaredCapabilities = (registered.declaration.capabilities ?? []).map(
|
|
2000
|
+
(c: string | CapabilityDeclaration) =>
|
|
2001
|
+
typeof c === "string" ? { name: c, mode: "singleton" as const } : c,
|
|
2002
|
+
);
|
|
2003
|
+
continue;
|
|
2004
|
+
}
|
|
2005
|
+
|
|
2006
|
+
// Skip agent-only addons on the hub. Same rule as the boot loader
|
|
2007
|
+
// path above; without it, post-install reload would re-init
|
|
2008
|
+
// hub-side every agent-only provider (e.g. `hub-forwarder`),
|
|
2009
|
+
// and a hub-resident `hub-forwarder` creates a write→ingest→
|
|
2010
|
+
// write feedback loop because the hub IS the log-receiver.
|
|
2011
|
+
if (isAgentOnlyPlacement(registered.declaration)) {
|
|
2012
|
+
this.logger.debug(
|
|
2013
|
+
'loadNewAddons: skipping agent-only addon on hub',
|
|
2014
|
+
{ tags: { addonId: registered.declaration.id } },
|
|
2015
|
+
);
|
|
2016
|
+
continue;
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
if (registered.declaration.capabilities) {
|
|
2020
|
+
for (const cap of registered.declaration.capabilities) {
|
|
2021
|
+
this.capabilityRegistry.declareFromManifest(
|
|
2022
|
+
cap,
|
|
2023
|
+
registered.declaration.id,
|
|
2024
|
+
);
|
|
2025
|
+
}
|
|
2026
|
+
}
|
|
2027
|
+
|
|
2028
|
+
try {
|
|
2029
|
+
const addon = freshLoader.createInstance(registered.declaration.id);
|
|
2030
|
+
const declCaps = (registered.declaration.capabilities ?? []).map(
|
|
2031
|
+
(c: string | CapabilityDeclaration) =>
|
|
2032
|
+
typeof c === "string" ? { name: c, mode: "singleton" as const } : c,
|
|
2033
|
+
);
|
|
2034
|
+
|
|
2035
|
+
this.addonEntries.set(registered.declaration.id, {
|
|
2036
|
+
addon,
|
|
2037
|
+
initialized: false,
|
|
2038
|
+
source: "installed",
|
|
2039
|
+
packageName: registered.packageName,
|
|
2040
|
+
packageVersion: registered.packageVersion,
|
|
2041
|
+
packageDisplayName: registered.packageDisplayName,
|
|
2042
|
+
...(registered.bundle !== undefined ? { bundle: registered.bundle } : {}),
|
|
2043
|
+
declaredCapabilities: declCaps,
|
|
2044
|
+
addonDir: path.join(addonsDir, registered.packageName),
|
|
2045
|
+
declaration: registered.declaration,
|
|
2046
|
+
});
|
|
2047
|
+
|
|
2048
|
+
// Initialize the new addon
|
|
2049
|
+
await this.initializeAddon(registered.declaration.id);
|
|
2050
|
+
this.wireCapabilities(registered.declaration.id);
|
|
2051
|
+
loaded.push(registered.declaration.id);
|
|
2052
|
+
this.logger.info(
|
|
2053
|
+
'Hot-loaded addon',
|
|
2054
|
+
{ tags: { addonId: registered.declaration.id }, meta: { packageName: registered.packageName } },
|
|
2055
|
+
);
|
|
2056
|
+
this.emitAddonLifecycleEvent(
|
|
2057
|
+
"addon.installed",
|
|
2058
|
+
registered.declaration.id,
|
|
2059
|
+
);
|
|
2060
|
+
} catch (err) {
|
|
2061
|
+
const msg = errMsg(err);
|
|
2062
|
+
const stack = err instanceof Error ? err.stack : undefined;
|
|
2063
|
+
this.logger.error(
|
|
2064
|
+
'Failed to hot-load addon',
|
|
2065
|
+
{
|
|
2066
|
+
tags: { addonId: registered.declaration.id },
|
|
2067
|
+
meta: stack ? { error: msg, stack } : { error: msg },
|
|
2068
|
+
},
|
|
2069
|
+
);
|
|
2070
|
+
this.emitAddonLifecycleEvent(
|
|
2071
|
+
"addon.crashed",
|
|
2072
|
+
registered.declaration.id,
|
|
2073
|
+
{ error: msg },
|
|
2074
|
+
);
|
|
2075
|
+
failed.push(registered.declaration.id);
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
|
|
2079
|
+
// Remove addons that are no longer on disk — the uninstall path.
|
|
2080
|
+
// D5: every non-core addon lives in its own runner subprocess. The
|
|
2081
|
+
// uninstall consolidation is "stop the runner"; the runner exits and
|
|
2082
|
+
// re-handshakes the hub WITHOUT the removed addon (D3 machinery).
|
|
2083
|
+
// Only `@camstack/core` builtins — which have no runner — take the
|
|
2084
|
+
// in-process `shutdown()` + disposer-drain path.
|
|
2085
|
+
const onDiskIds = new Set(
|
|
2086
|
+
freshLoader
|
|
2087
|
+
.listAddons()
|
|
2088
|
+
.map((a: { declaration: { id: string } }) => a.declaration.id),
|
|
2089
|
+
);
|
|
2090
|
+
for (const [id, entry] of this.addonEntries) {
|
|
2091
|
+
if (entry.source === "installed" && !onDiskIds.has(id)) {
|
|
2092
|
+
// Unregister capabilities before stopping the addon.
|
|
2093
|
+
for (const cap of entry.declaredCapabilities) {
|
|
2094
|
+
this.capabilityRegistry.unregisterProvider(cap.name, id);
|
|
2095
|
+
}
|
|
2096
|
+
const manifestCaps = this.getAddonCapabilities(entry.addon);
|
|
2097
|
+
for (const cap of manifestCaps) {
|
|
2098
|
+
this.capabilityRegistry.unregisterProvider(cap.name, id);
|
|
2099
|
+
}
|
|
2100
|
+
this.capabilityRegistry.unregisterAllWrappersForAddon(id);
|
|
2101
|
+
// Task 7.1: drop custom actions for removed addons.
|
|
2102
|
+
this.customActionRegistry.unregisterAddon(id);
|
|
2103
|
+
|
|
2104
|
+
if (this.isForkedAddonEntry(entry) && entry.initialized) {
|
|
2105
|
+
// Forked addon — stop the runner subprocess. `$process.stop`
|
|
2106
|
+
// kills the child and force-evicts its Moleculer node, so the
|
|
2107
|
+
// hub's cap registry drops the gone provider with no per-
|
|
2108
|
+
// operation hub bookkeeping.
|
|
2109
|
+
try {
|
|
2110
|
+
await this.broker.call("$process.stop", { name: id });
|
|
2111
|
+
} catch (err) {
|
|
2112
|
+
this.logger.warn(
|
|
2113
|
+
'Non-fatal: failed to stop runner for removed addon',
|
|
2114
|
+
{ tags: { addonId: id }, meta: { error: errMsg(err) } },
|
|
2115
|
+
);
|
|
2116
|
+
}
|
|
2117
|
+
} else if (entry.initialized) {
|
|
2118
|
+
// `@camstack/core` builtin — reload model is in-process.
|
|
2119
|
+
try {
|
|
2120
|
+
await entry.addon.shutdown();
|
|
2121
|
+
} catch (err) {
|
|
2122
|
+
this.logger.debug(
|
|
2123
|
+
'Non-fatal: shutdown error for removed addon',
|
|
2124
|
+
{ tags: { addonId: id }, meta: { error: errMsg(err) } },
|
|
2125
|
+
);
|
|
2126
|
+
}
|
|
2127
|
+
await this.drainDisposerChain(id);
|
|
2128
|
+
}
|
|
2129
|
+
this.addonEntries.delete(id);
|
|
2130
|
+
this.logger.info('Removed addon (no longer on disk)', { tags: { addonId: id } });
|
|
2131
|
+
}
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
return { loaded, failed };
|
|
2135
|
+
}
|
|
2136
|
+
|
|
2137
|
+
async shutdownAll(): Promise<void> {
|
|
2138
|
+
for (const [id, entry] of this.addonEntries) {
|
|
2139
|
+
if (entry.initialized) {
|
|
2140
|
+
const capabilities = this.getAddonCapabilities(entry.addon);
|
|
2141
|
+
for (const cap of capabilities) {
|
|
2142
|
+
this.capabilityRegistry.unregisterProvider(cap.name, id);
|
|
2143
|
+
}
|
|
2144
|
+
this.capabilityRegistry.unregisterAllWrappersForAddon(id);
|
|
2145
|
+
// Task 7.1: drop custom actions on full shutdown.
|
|
2146
|
+
this.customActionRegistry.unregisterAddon(id);
|
|
2147
|
+
await entry.addon.shutdown();
|
|
2148
|
+
await this.drainDisposerChain(id);
|
|
2149
|
+
entry.initialized = false;
|
|
2150
|
+
}
|
|
2151
|
+
}
|
|
2152
|
+
}
|
|
2153
|
+
|
|
2154
|
+
// --- Private helpers ---
|
|
2155
|
+
|
|
2156
|
+
// registerConsumers() — DELETED. Consumer wiring now in wireCapabilityConsumers().
|
|
2157
|
+
|
|
2158
|
+
/**
|
|
2159
|
+
* v2: wire capability consumer actions via EventBus.
|
|
2160
|
+
* Listens to 'capability:provider-registered' events from CapabilityRegistry
|
|
2161
|
+
* and dispatches setup actions (storage wiring, decoder↔broker, etc).
|
|
2162
|
+
*
|
|
2163
|
+
* Replaces the 11 registerConsumer callbacks from v1.
|
|
2164
|
+
*/
|
|
2165
|
+
private wireCapabilityConsumers(): void {
|
|
2166
|
+
if (!this.capabilityRegistry) return;
|
|
2167
|
+
|
|
2168
|
+
this.eventBusService.subscribe(
|
|
2169
|
+
{ category: "capability:provider-registered" },
|
|
2170
|
+
(event) => {
|
|
2171
|
+
const rawCapability = event.data["capability"];
|
|
2172
|
+
const rawAddonId = event.data["addonId"];
|
|
2173
|
+
if (typeof rawCapability !== "string" || typeof rawAddonId !== "string")
|
|
2174
|
+
return;
|
|
2175
|
+
const capability = rawCapability;
|
|
2176
|
+
const addonId = rawAddonId;
|
|
2177
|
+
|
|
2178
|
+
// Broadcast readiness on the hub-node scope so subprocess
|
|
2179
|
+
// brokers waiting on this cap (e.g. provider-rtsp's
|
|
2180
|
+
// `system.ready-state` listener for stream-broker, or
|
|
2181
|
+
// `runWorkerDeviceRestoreWithRetry` waiting on
|
|
2182
|
+
// device-manager) wake up. We emit ONLY `{ type: 'node',
|
|
2183
|
+
// nodeId: 'hub' }` — most consumer filters are scope-agnostic
|
|
2184
|
+
// (they match on capName + state), so emitting both `node` and
|
|
2185
|
+
// `global` causes duplicate fan-out (e.g. provider-rtsp's
|
|
2186
|
+
// `republishAll` running twice → repeated `dispatchCamera`
|
|
2187
|
+
// loops). The node-scoped emit is sufficient: the hydrate
|
|
2188
|
+
// path on subprocess brokers replays whichever records the
|
|
2189
|
+
// hub's `$readiness.getSnapshot` returns, scope and all.
|
|
2190
|
+
try {
|
|
2191
|
+
this.moleculer.readinessRegistry.emitReady(capability, { type: 'node', nodeId: 'hub' });
|
|
2192
|
+
} catch (err) {
|
|
2193
|
+
this.logger.warn('emitReady failed', {
|
|
2194
|
+
meta: { capability, addonId, error: errMsg(err) },
|
|
2195
|
+
});
|
|
2196
|
+
}
|
|
2197
|
+
|
|
2198
|
+
switch (capability) {
|
|
2199
|
+
case "storage": {
|
|
2200
|
+
// Storage-unification refactor (Task 8) — the consumer-
|
|
2201
|
+
// facing `storage` cap is now a singleton owned by the
|
|
2202
|
+
// `storage-orchestrator` builtin, exposing the codegen'd
|
|
2203
|
+
// async `IStorageCapProvider` surface. The legacy
|
|
2204
|
+
// synchronous `INewStorageProvider` (filesystem-only) used
|
|
2205
|
+
// by `StorageService.setNewStorageProvider` and the
|
|
2206
|
+
// `addons-data` dataDir resolution at boot is no longer
|
|
2207
|
+
// wired here — filesystem-storage now registers under
|
|
2208
|
+
// `storage-provider` (the upstream collection cap), and
|
|
2209
|
+
// legacy callers fall back to the deterministic
|
|
2210
|
+
// `camstack-data/addons-data/<addonId>` path. Task 17 will
|
|
2211
|
+
// migrate those callers off `INewStorageProvider`
|
|
2212
|
+
// entirely.
|
|
2213
|
+
break;
|
|
2214
|
+
}
|
|
2215
|
+
|
|
2216
|
+
case "settings-store": {
|
|
2217
|
+
const provider = this.capabilityRegistry.getProviderByAddon(
|
|
2218
|
+
"settings-store",
|
|
2219
|
+
addonId,
|
|
2220
|
+
);
|
|
2221
|
+
if (!provider) return;
|
|
2222
|
+
this.activeSettingsBackend = provider;
|
|
2223
|
+
if (isSettingsStore(provider)) {
|
|
2224
|
+
this.configService.setSettingsStore(provider);
|
|
2225
|
+
}
|
|
2226
|
+
this.storageService.setSettingsBackend(provider);
|
|
2227
|
+
this.integrationRegistry = new IntegrationRegistry(provider);
|
|
2228
|
+
void this.integrationRegistry.initialize().then(() => {
|
|
2229
|
+
this.logger.info("IntegrationRegistry initialized", { meta: { phase: 'v2' } });
|
|
2230
|
+
});
|
|
2231
|
+
this.loadCollectionPreferences();
|
|
2232
|
+
// DeviceStore/ConfigStore are owned by the `device-persistence`
|
|
2233
|
+
// capability addon — no longer created here. The addon boots
|
|
2234
|
+
// after `sqlite-settings` and extracts the DB handle via the
|
|
2235
|
+
// capability registry.
|
|
2236
|
+
this.logger.info("Settings backend wired", { meta: { phase: 'v2' } });
|
|
2237
|
+
break;
|
|
2238
|
+
}
|
|
2239
|
+
|
|
2240
|
+
case "log-destination": {
|
|
2241
|
+
const provider = this.capabilityRegistry.getProviderByAddon(
|
|
2242
|
+
"log-destination",
|
|
2243
|
+
addonId,
|
|
2244
|
+
);
|
|
2245
|
+
if (!provider) return;
|
|
2246
|
+
this.loggingService.addDestination(provider);
|
|
2247
|
+
this.logger.info("Log destination added", { meta: { phase: 'v2' } });
|
|
2248
|
+
break;
|
|
2249
|
+
}
|
|
2250
|
+
|
|
2251
|
+
case "restreamer":
|
|
2252
|
+
case "webrtc":
|
|
2253
|
+
case "decoder":
|
|
2254
|
+
case "stream-broker":
|
|
2255
|
+
// No wiring needed — consumers read from capabilityRegistry on demand.
|
|
2256
|
+
break;
|
|
2257
|
+
|
|
2258
|
+
case "addon-routes": {
|
|
2259
|
+
// Route mounting is async for forked/group addons — the
|
|
2260
|
+
// provider is a Moleculer proxy whose `getRoutes()` returns
|
|
2261
|
+
// a Promise (the wire round-trips through the worker). The
|
|
2262
|
+
// EventBus subscriber callback is synchronous, so delegate
|
|
2263
|
+
// to an async helper and surface any failure via the
|
|
2264
|
+
// logger instead of letting a rejected promise (or a
|
|
2265
|
+
// `liveRoutes.some is not a function` TypeError) escape the
|
|
2266
|
+
// subscriber unobserved.
|
|
2267
|
+
void this.mountAddonRoutes(addonId).catch((err: unknown) => {
|
|
2268
|
+
this.logger.error('Failed to mount addon routes', {
|
|
2269
|
+
meta: { phase: 'v2', addonId, error: errMsg(err) },
|
|
2270
|
+
})
|
|
2271
|
+
})
|
|
2272
|
+
break;
|
|
2273
|
+
}
|
|
2274
|
+
|
|
2275
|
+
default:
|
|
2276
|
+
break;
|
|
2277
|
+
}
|
|
2278
|
+
},
|
|
2279
|
+
);
|
|
2280
|
+
}
|
|
2281
|
+
|
|
2282
|
+
/**
|
|
2283
|
+
* Mount the `addon-routes` provider for `addonId` into the
|
|
2284
|
+
* `AddonRouteRegistry`. Handles both co-located (in-process) and
|
|
2285
|
+
* forked/group addons:
|
|
2286
|
+
*
|
|
2287
|
+
* - For a forked addon the provider is a Moleculer proxy whose
|
|
2288
|
+
* `getRoutes()` / `invoke()` return Promises and whose route
|
|
2289
|
+
* descriptors have their `handler` functions stripped by JSON
|
|
2290
|
+
* serialization. We `await getRoutes()`, synthesize bridge
|
|
2291
|
+
* handlers that dispatch through `provider.invoke(...)`, and
|
|
2292
|
+
* translate the captured envelope back onto the Fastify reply.
|
|
2293
|
+
* - For a co-located addon `getRoutes()` resolves to the live route
|
|
2294
|
+
* list with real handlers, so we register the provider directly.
|
|
2295
|
+
*/
|
|
2296
|
+
private async mountAddonRoutes(addonId: string): Promise<void> {
|
|
2297
|
+
if (!this.capabilityRegistry) return;
|
|
2298
|
+
const routeProvider = this.capabilityRegistry.getProviderByAddon(
|
|
2299
|
+
"addon-routes",
|
|
2300
|
+
addonId,
|
|
2301
|
+
);
|
|
2302
|
+
if (!routeProvider || !this.addonRouteRegistry) return;
|
|
2303
|
+
|
|
2304
|
+
// `getRoutes()` is synchronous on a co-located provider but
|
|
2305
|
+
// returns a Promise on the Moleculer proxy of a forked addon —
|
|
2306
|
+
// `await` normalizes both. If any route is missing its `handler`
|
|
2307
|
+
// we synthesize one that dispatches via `provider.invoke(...)`
|
|
2308
|
+
// and translates the captured envelope back to the Fastify reply.
|
|
2309
|
+
const liveRoutes = await routeProvider.getRoutes()
|
|
2310
|
+
const handlersMissing = liveRoutes.some((r: { handler?: unknown }) => typeof r.handler !== 'function')
|
|
2311
|
+
if (handlersMissing) {
|
|
2312
|
+
const invokeFn = (routeProvider as { invoke?: (input: unknown) => Promise<unknown> }).invoke
|
|
2313
|
+
if (typeof invokeFn !== 'function') {
|
|
2314
|
+
this.logger.warn('Forked addon-routes provider missing `invoke` method — routes will not dispatch. Use `buildAddonRouteProvider()` from @camstack/types.', {
|
|
2315
|
+
meta: { phase: 'v2', routeProviderId: routeProvider.id },
|
|
2316
|
+
})
|
|
2317
|
+
}
|
|
2318
|
+
// Wrap each route with a bridge handler so the
|
|
2319
|
+
// AddonRouteRegistry can dispatch through the worker.
|
|
2320
|
+
const bridgeRoutes: import('@camstack/types').IAddonHttpRoute[] = (liveRoutes as ReadonlyArray<{ method: string; path: string; access?: string; description?: string }>).map((route) => ({
|
|
2321
|
+
method: route.method as 'GET' | 'POST' | 'PUT' | 'DELETE',
|
|
2322
|
+
path: route.path,
|
|
2323
|
+
access: (route.access ?? 'public') as 'public' | 'authenticated' | 'admin',
|
|
2324
|
+
...(route.description !== undefined ? { description: route.description } : {}),
|
|
2325
|
+
handler: async (req: { params: Record<string, string>; query: Record<string, string>; body: unknown; headers: Record<string, string>; user?: { id: string; username: string; isAdmin: boolean }; scopedToken?: unknown }, reply: { code: (n: number) => unknown; header: (k: string, v: string) => unknown; type: (m: string) => unknown; send: (data: unknown) => void; redirect: (url: string) => void }) => {
|
|
2326
|
+
if (typeof invokeFn !== 'function') {
|
|
2327
|
+
reply.code(500)
|
|
2328
|
+
reply.send({ error: 'Forked addon-routes provider missing invoke' })
|
|
2329
|
+
return
|
|
2330
|
+
}
|
|
2331
|
+
const envelope = await invokeFn({
|
|
2332
|
+
method: route.method,
|
|
2333
|
+
path: route.path,
|
|
2334
|
+
params: req.params,
|
|
2335
|
+
query: req.query,
|
|
2336
|
+
body: req.body,
|
|
2337
|
+
headers: req.headers,
|
|
2338
|
+
...(req.user ? { user: req.user } : {}),
|
|
2339
|
+
...(req.scopedToken ? { scopedToken: req.scopedToken } : {}),
|
|
2340
|
+
}) as { status: number; headers: Record<string, string>; redirectUrl: string | null; body?: unknown; contentType?: string }
|
|
2341
|
+
reply.code(envelope.status)
|
|
2342
|
+
if (envelope.contentType) reply.type(envelope.contentType)
|
|
2343
|
+
for (const [k, v] of Object.entries(envelope.headers)) reply.header(k, v)
|
|
2344
|
+
if (envelope.redirectUrl !== null) {
|
|
2345
|
+
reply.header('Location', envelope.redirectUrl)
|
|
2346
|
+
reply.send('')
|
|
2347
|
+
} else {
|
|
2348
|
+
reply.send(envelope.body)
|
|
2349
|
+
}
|
|
2350
|
+
},
|
|
2351
|
+
}))
|
|
2352
|
+
this.addonRouteRegistry.registerRoutes(
|
|
2353
|
+
routeProvider.id,
|
|
2354
|
+
{ id: routeProvider.id, getRoutes: () => bridgeRoutes },
|
|
2355
|
+
)
|
|
2356
|
+
this.logger.info('Addon routes mounted (forked-bridge)', { meta: { phase: 'v2', routeProviderId: routeProvider.id, routes: bridgeRoutes.length } })
|
|
2357
|
+
} else {
|
|
2358
|
+
this.addonRouteRegistry.registerRoutes(
|
|
2359
|
+
routeProvider.id,
|
|
2360
|
+
routeProvider,
|
|
2361
|
+
);
|
|
2362
|
+
this.logger.info('Addon routes mounted', { meta: { phase: 'v2', routeProviderId: routeProvider.id } });
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2365
|
+
|
|
2366
|
+
// Cleanup: `addonHasConfigFields` deleted. It was the last reader of
|
|
2367
|
+
// the legacy `ICamstackAddon.getConfigSchema()` method, exposed via
|
|
2368
|
+
// `listAddons().hasConfigSchema` for the old AddonCard settings gate.
|
|
2369
|
+
// `NodeAddonsSettingsPanel` now gracefully shows an empty state when
|
|
2370
|
+
// an addon doesn't implement `getGlobalSettings`, so the backend no
|
|
2371
|
+
// longer needs to pre-compute this flag.
|
|
2372
|
+
|
|
2373
|
+
private emitAddonLifecycleEvent(
|
|
2374
|
+
eventType:
|
|
2375
|
+
| "addon.started"
|
|
2376
|
+
| "addon.stopped"
|
|
2377
|
+
| "addon.restarted"
|
|
2378
|
+
| "addon.updated"
|
|
2379
|
+
| "addon.installed"
|
|
2380
|
+
| "addon.uninstalled"
|
|
2381
|
+
| "addon.crashed"
|
|
2382
|
+
| "addon.error",
|
|
2383
|
+
addonId: string,
|
|
2384
|
+
data?: Record<string, unknown>,
|
|
2385
|
+
): void {
|
|
2386
|
+
const entry = this.addonEntries.get(addonId);
|
|
2387
|
+
this.eventBusService.emit({
|
|
2388
|
+
id: randomUUID(),
|
|
2389
|
+
timestamp: new Date(),
|
|
2390
|
+
source: {
|
|
2391
|
+
type: "addon",
|
|
2392
|
+
id: addonId,
|
|
2393
|
+
nodeId: this.broker.nodeID,
|
|
2394
|
+
},
|
|
2395
|
+
category: eventType,
|
|
2396
|
+
data: {
|
|
2397
|
+
addonId,
|
|
2398
|
+
packageName: entry?.packageName,
|
|
2399
|
+
packageVersion: entry?.packageVersion,
|
|
2400
|
+
agent: process.env.CAMSTACK_AGENT_NAME ?? "hub",
|
|
2401
|
+
...data,
|
|
2402
|
+
},
|
|
2403
|
+
});
|
|
2404
|
+
}
|
|
2405
|
+
|
|
2406
|
+
/**
|
|
2407
|
+
* Post-init hook: emit events for newly available capabilities.
|
|
2408
|
+
* Provider registration now happens in initialize() via context.registerProvider().
|
|
2409
|
+
*/
|
|
2410
|
+
private wireCapabilities(addonId: string): void {
|
|
2411
|
+
const entry = this.addonEntries.get(addonId);
|
|
2412
|
+
if (!entry?.initialized) return;
|
|
2413
|
+
|
|
2414
|
+
// Emit addon-pages event for UI notification. Cap name is
|
|
2415
|
+
// `addon-pages-source` after the consolidation split — collection
|
|
2416
|
+
// providers register on the source cap; the singleton aggregator
|
|
2417
|
+
// owns `addon-pages` cluster-wide.
|
|
2418
|
+
const declaredCaps = entry.declaredCapabilities.map((c) => c.name);
|
|
2419
|
+
if (declaredCaps.includes("addon-pages-source")) {
|
|
2420
|
+
this.eventBusService.emit({
|
|
2421
|
+
id: randomUUID(),
|
|
2422
|
+
timestamp: new Date(),
|
|
2423
|
+
source: { type: "addon", id: addonId },
|
|
2424
|
+
category: EventCategory.AddonPageReady,
|
|
2425
|
+
data: { addonId, packageName: entry.packageName },
|
|
2426
|
+
});
|
|
2427
|
+
}
|
|
2428
|
+
// Symmetric for `addon-widgets-source` — addons that contribute
|
|
2429
|
+
// widget bundles emit `AddonWidgetReady` so the
|
|
2430
|
+
// <WidgetRegistryProvider> in admin-ui invalidates its aggregator
|
|
2431
|
+
// query and the new bundle becomes available without a page
|
|
2432
|
+
// reload.
|
|
2433
|
+
if (declaredCaps.includes("addon-widgets-source")) {
|
|
2434
|
+
this.eventBusService.emit({
|
|
2435
|
+
id: randomUUID(),
|
|
2436
|
+
timestamp: new Date(),
|
|
2437
|
+
source: { type: "addon", id: addonId },
|
|
2438
|
+
category: EventCategory.AddonWidgetReady,
|
|
2439
|
+
data: { addonId, packageName: entry.packageName },
|
|
2440
|
+
});
|
|
2441
|
+
}
|
|
2442
|
+
}
|
|
2443
|
+
|
|
2444
|
+
private getAddonCapabilities(addon: ICamstackAddon): CapabilityDeclaration[] {
|
|
2445
|
+
const caps = addon?.manifest?.capabilities;
|
|
2446
|
+
if (!caps) return [];
|
|
2447
|
+
|
|
2448
|
+
return caps.map(
|
|
2449
|
+
(cap): CapabilityDeclaration =>
|
|
2450
|
+
typeof cap === "string" ? { name: cap } : cap,
|
|
2451
|
+
);
|
|
2452
|
+
}
|
|
2453
|
+
|
|
2454
|
+
private findAddonForCapability(
|
|
2455
|
+
capName: string,
|
|
2456
|
+
addonIds: string[],
|
|
2457
|
+
): string | null {
|
|
2458
|
+
for (const id of addonIds) {
|
|
2459
|
+
const entry = this.addonEntries.get(id);
|
|
2460
|
+
if (!entry) continue;
|
|
2461
|
+
// Use declaredCapabilities (from package.json) as source of truth
|
|
2462
|
+
if (entry.declaredCapabilities.some((c) => c.name === capName)) return id;
|
|
2463
|
+
}
|
|
2464
|
+
return null;
|
|
2465
|
+
}
|
|
2466
|
+
|
|
2467
|
+
/**
|
|
2468
|
+
* Build the addonConfig for a given addon — strictly per-addon now.
|
|
2469
|
+
*
|
|
2470
|
+
* The legacy cross-addon merge (hardcoded `ADDON_SYSTEM_SETTINGS` map
|
|
2471
|
+
* copying `system_settings.<section>.*` into the bootstrap config) has
|
|
2472
|
+
* been removed. Addons that need to read shared system settings
|
|
2473
|
+
* sections (`ffmpeg`, `logging`, `recording`, …) do so at runtime via
|
|
2474
|
+
* `ctx.settings.getGlobal({ section: '<name>' })`, which delegates to
|
|
2475
|
+
* `ConfigManager.getSection()` and covers the full SQL → YAML →
|
|
2476
|
+
* RUNTIME_DEFAULTS fallback chain.
|
|
2477
|
+
*
|
|
2478
|
+
* This keeps the kernel agnostic: every addon declares its own needs
|
|
2479
|
+
* in its own `initialize(ctx)` code, with zero kernel-side addon id
|
|
2480
|
+
* special-casing. The only remaining exception is `sqlite-settings`,
|
|
2481
|
+
* which receives `_runtimeDefaults` for first-boot seeding (tracked
|
|
2482
|
+
* separately under the kernel-cleanup roadmap — Point 3d).
|
|
2483
|
+
*/
|
|
2484
|
+
private buildAddonConfig(addonId: string): Record<string, unknown> {
|
|
2485
|
+
// Start with per-addon SQL settings (may be empty for most addons)
|
|
2486
|
+
let addonSpecific: Record<string, unknown> = {};
|
|
2487
|
+
try {
|
|
2488
|
+
addonSpecific = this.configService.getAddonConfig(addonId);
|
|
2489
|
+
} catch (err) {
|
|
2490
|
+
this.logger.debug(
|
|
2491
|
+
'ConfigManager not ready for addon',
|
|
2492
|
+
{ tags: { addonId }, meta: { error: errMsg(err) } },
|
|
2493
|
+
);
|
|
2494
|
+
}
|
|
2495
|
+
|
|
2496
|
+
// No addon id special-casing here anymore. Bootstrap-level
|
|
2497
|
+
// concerns (e.g. first-boot seeding from RUNTIME_DEFAULTS) are
|
|
2498
|
+
// owned by the addon itself — sqlite-settings imports the
|
|
2499
|
+
// constant directly from `@camstack/types` instead of relying on
|
|
2500
|
+
// a kernel-injected bootstrap field.
|
|
2501
|
+
return addonSpecific;
|
|
2502
|
+
}
|
|
2503
|
+
|
|
2504
|
+
private async createAddonContext(
|
|
2505
|
+
addon: ICamstackAddon,
|
|
2506
|
+
): Promise<InternalAddonContext> {
|
|
2507
|
+
const addonId = addon.manifest!.id;
|
|
2508
|
+
const brokerNodeId = this.broker.nodeID;
|
|
2509
|
+
const agentId = brokerNodeId.includes("/")
|
|
2510
|
+
? brokerNodeId.split("/")[0]!
|
|
2511
|
+
: brokerNodeId;
|
|
2512
|
+
// No scope on the addon root logger — the brand bracket already shows
|
|
2513
|
+
// `[agent/addonId]`, so `(addon:<addonId>)` was pure duplication.
|
|
2514
|
+
// Sub-components add their own scope via `.child('<name>')`.
|
|
2515
|
+
const logger = this.loggingService
|
|
2516
|
+
.createLogger()
|
|
2517
|
+
.withTags({ addonId, nodeId: brokerNodeId, agentId });
|
|
2518
|
+
const bootstrapConfig = this.buildAddonConfig(addonId);
|
|
2519
|
+
|
|
2520
|
+
// Per-addon private data directory — resolved from active storage
|
|
2521
|
+
// provider if available, otherwise falls back to a deterministic
|
|
2522
|
+
// hardcoded path. Addons that need the full storage provider now
|
|
2523
|
+
// resolve it via the capability registry.
|
|
2524
|
+
const storageProvider = this.activeStorageProvider;
|
|
2525
|
+
const dataDir = storageProvider
|
|
2526
|
+
? await storageProvider.resolve({
|
|
2527
|
+
location: "addons-data",
|
|
2528
|
+
relativePath: addonId,
|
|
2529
|
+
})
|
|
2530
|
+
: `camstack-data/addons-data/${addonId}`;
|
|
2531
|
+
|
|
2532
|
+
const registerProvider = (
|
|
2533
|
+
capabilityName: string,
|
|
2534
|
+
provider: unknown,
|
|
2535
|
+
): void => {
|
|
2536
|
+
this.capabilityRegistry.registerProvider(
|
|
2537
|
+
capabilityName,
|
|
2538
|
+
addonId,
|
|
2539
|
+
provider,
|
|
2540
|
+
);
|
|
2541
|
+
logger.info(
|
|
2542
|
+
'Registered provider via context.registerProvider()',
|
|
2543
|
+
{ meta: { capabilityName } },
|
|
2544
|
+
);
|
|
2545
|
+
};
|
|
2546
|
+
|
|
2547
|
+
// Raw three-level settings store API. The resolver service is a
|
|
2548
|
+
// thin wrapper over `ConfigManager` that exposes raw reads/writes
|
|
2549
|
+
// for the addon store, the per-device store, and the cluster-wide
|
|
2550
|
+
// yml-backed sections. No schema merging happens here — the addon
|
|
2551
|
+
// combines these raw reads with its own schema via
|
|
2552
|
+
// `hydrateSchema()` inside `getAddonSettings / getGlobalSettings
|
|
2553
|
+
// / getDeviceSettings`.
|
|
2554
|
+
const settingsView: AddonSettingsView =
|
|
2555
|
+
this.configService.createSettingsView(addonId);
|
|
2556
|
+
|
|
2557
|
+
// Device management — unified path for hub and worker addons.
|
|
2558
|
+
// Persistence routes through the `device-manager` capability
|
|
2559
|
+
// addon via `ctx.api.deviceManager.*`. On the hub, `ctx.api` is
|
|
2560
|
+
// a broker-routed proxy that resolves the local
|
|
2561
|
+
// `device-manager` Moleculer service in-process — no network
|
|
2562
|
+
// hop. On a forked worker, the same proxy routes through the TCP
|
|
2563
|
+
// transport to the hub's service. Zero custom $hub.* actions needed.
|
|
2564
|
+
const kernelStreamProbe: import("@camstack/types").IKernelStreamProbe = {
|
|
2565
|
+
probe: (url, options) => this.streamProbe.probe(url, options),
|
|
2566
|
+
probeField: (key, value) => this.streamProbe.probeField(key, value),
|
|
2567
|
+
};
|
|
2568
|
+
const deviceManagerApi: import("@camstack/types").DeviceManagerApi =
|
|
2569
|
+
createBrokerDeviceManagerApi({
|
|
2570
|
+
api: this.getBrokerApi(),
|
|
2571
|
+
addonId,
|
|
2572
|
+
nodeId: this.broker.nodeID,
|
|
2573
|
+
logger,
|
|
2574
|
+
eventBus: this.eventBusService,
|
|
2575
|
+
registry: this.deviceRegistry,
|
|
2576
|
+
capabilityRegistry: this.capabilityRegistry,
|
|
2577
|
+
streamProbe: kernelStreamProbe,
|
|
2578
|
+
});
|
|
2579
|
+
|
|
2580
|
+
const registry = this;
|
|
2581
|
+
const rr = this.moleculer.readinessRegistry;
|
|
2582
|
+
const capHandleCache = new Map<string, CapabilityHandle<unknown>>();
|
|
2583
|
+
function getOrCreateHandle<T>(capName: string, scope: ReadinessScope, timeoutMs: number): CapabilityHandle<T> {
|
|
2584
|
+
const key = `${capName}::${scopeKey(scope)}`;
|
|
2585
|
+
const existing = capHandleCache.get(key);
|
|
2586
|
+
if (existing) return existing as CapabilityHandle<T>;
|
|
2587
|
+
const handle = new CapabilityHandle<T>(capName, scope, rr, timeoutMs);
|
|
2588
|
+
capHandleCache.set(key, handle as CapabilityHandle<unknown>);
|
|
2589
|
+
return handle;
|
|
2590
|
+
}
|
|
2591
|
+
const ctx: InternalAddonContext & {
|
|
2592
|
+
integrationRegistry?: unknown;
|
|
2593
|
+
capabilities: import("@camstack/types").CapabilitiesAccess;
|
|
2594
|
+
} = {
|
|
2595
|
+
id: `addon:${addonId}`,
|
|
2596
|
+
logger,
|
|
2597
|
+
eventBus: this.eventBusService,
|
|
2598
|
+
addonConfig: bootstrapConfig,
|
|
2599
|
+
dataDir,
|
|
2600
|
+
get api() {
|
|
2601
|
+
return registry.getBrokerApi();
|
|
2602
|
+
},
|
|
2603
|
+
integrationRegistry: this.integrationRegistry ?? undefined,
|
|
2604
|
+
// Live capability-collection accessor used by addons like stream-broker
|
|
2605
|
+
// (webrtc fan-out), enrichment-engine, and snapshot orchestrator. Reads
|
|
2606
|
+
// through the hub's CapabilityRegistry so the collection reflects
|
|
2607
|
+
// every addon that has declared itself since the consumer initialized.
|
|
2608
|
+
// Without this, those addons silently saw 0 providers and quietly
|
|
2609
|
+
// skipped the fan-out — no errors, just missing streams/wiring.
|
|
2610
|
+
capabilities: {
|
|
2611
|
+
getCollection: <T = unknown>(
|
|
2612
|
+
capName: string,
|
|
2613
|
+
): readonly T[] | undefined => {
|
|
2614
|
+
const items = registry.capabilityRegistry.getCollection<T>(capName);
|
|
2615
|
+
return items ?? undefined;
|
|
2616
|
+
},
|
|
2617
|
+
getCollectionEntries: <T = unknown>(
|
|
2618
|
+
capName: string,
|
|
2619
|
+
): readonly (readonly [string, T])[] | undefined => {
|
|
2620
|
+
const items = registry.capabilityRegistry.getCollectionEntries<T>(capName);
|
|
2621
|
+
return items ?? undefined;
|
|
2622
|
+
},
|
|
2623
|
+
get: <T = unknown>(capName: string): T | undefined => {
|
|
2624
|
+
return registry.capabilityRegistry.getSingleton<T>(capName) ?? undefined;
|
|
2625
|
+
},
|
|
2626
|
+
},
|
|
2627
|
+
deps: new AddonDepsManager(dataDir, logger),
|
|
2628
|
+
kernel: {
|
|
2629
|
+
localNodeId: this.broker.nodeID,
|
|
2630
|
+
storage: storageProvider ?? undefined,
|
|
2631
|
+
deviceRegistry: this.deviceRegistry ?? undefined,
|
|
2632
|
+
devices: deviceManagerApi,
|
|
2633
|
+
cluster: { broker: adaptBrokerToCluster(this.moleculer.broker) },
|
|
2634
|
+
streamProbe: kernelStreamProbe,
|
|
2635
|
+
hwaccel: createKernelHwAccel(),
|
|
2636
|
+
capabilityRegistry: this.capabilityRegistry,
|
|
2637
|
+
readinessRegistry: this.moleculer.readinessRegistry,
|
|
2638
|
+
// D3: handshake-fed native-cap view of the whole cluster. Backed by
|
|
2639
|
+
// `HubNodeRegistry.listNativeCapEntries()` populated by every
|
|
2640
|
+
// `$hub.registerNode` re-handshake. Used by device-manager as the
|
|
2641
|
+
// reliable fallback when push events were lost mid-transport.
|
|
2642
|
+
listClusterNativeCaps: () => this.moleculer.listClusterNativeCaps(),
|
|
2643
|
+
},
|
|
2644
|
+
registerProvider,
|
|
2645
|
+
resolveProvider: <T = unknown>(capName: string): T | null => {
|
|
2646
|
+
return (
|
|
2647
|
+
(registry.capabilityRegistry.getSingleton<T>(capName) as T | null) ??
|
|
2648
|
+
null
|
|
2649
|
+
);
|
|
2650
|
+
},
|
|
2651
|
+
getNativeProvider: <TCap extends CapabilityDefinition>(
|
|
2652
|
+
cap: TCap,
|
|
2653
|
+
deviceId: number,
|
|
2654
|
+
): InferProvider<TCap> => {
|
|
2655
|
+
// Hub-side addons live on the same process as the CapabilityRegistry;
|
|
2656
|
+
// native providers are always resolvable locally.
|
|
2657
|
+
const local = registry.capabilityRegistry.getNativeProvider<
|
|
2658
|
+
InferProvider<TCap>
|
|
2659
|
+
>(cap.name, deviceId);
|
|
2660
|
+
if (!local) {
|
|
2661
|
+
throw new Error(
|
|
2662
|
+
`no native provider for capability '${cap.name}' on device '${deviceId}'`,
|
|
2663
|
+
);
|
|
2664
|
+
}
|
|
2665
|
+
return local;
|
|
2666
|
+
},
|
|
2667
|
+
fetchDevice: async (deviceId: number) => {
|
|
2668
|
+
const api = registry.getBrokerApi();
|
|
2669
|
+
const binding = await api.deviceManager.getBindings.query({ deviceId });
|
|
2670
|
+
return createDeviceProxy(api, binding);
|
|
2671
|
+
},
|
|
2672
|
+
settings: settingsView,
|
|
2673
|
+
useCapability<T = unknown>(capName: string, scope: ReadinessScope = { type: 'global' }) {
|
|
2674
|
+
return getOrCreateHandle<T>(capName, scope, 15_000);
|
|
2675
|
+
},
|
|
2676
|
+
async acquireCapability<T = unknown>(
|
|
2677
|
+
capName: string,
|
|
2678
|
+
scope: ReadinessScope = { type: 'global' },
|
|
2679
|
+
opts: { timeoutMs?: number } = {},
|
|
2680
|
+
) {
|
|
2681
|
+
const timeoutMs = opts.timeoutMs ?? 15_000;
|
|
2682
|
+
const handle = getOrCreateHandle<T>(capName, scope, timeoutMs);
|
|
2683
|
+
return handle;
|
|
2684
|
+
},
|
|
2685
|
+
onCapabilityStateChange(
|
|
2686
|
+
capName: string,
|
|
2687
|
+
scope: ReadinessScope,
|
|
2688
|
+
handler: (state: 'ready' | 'down') => void,
|
|
2689
|
+
) {
|
|
2690
|
+
return rr.onReadyState(capName, scope, (t) => {
|
|
2691
|
+
handler(t.state === 'ready' ? 'ready' : 'down');
|
|
2692
|
+
});
|
|
2693
|
+
},
|
|
2694
|
+
addDisposer(fn: () => void | Promise<void>) {
|
|
2695
|
+
return registry.getOrCreateDisposerChain(addonId).add(fn);
|
|
2696
|
+
},
|
|
2697
|
+
};
|
|
2698
|
+
|
|
2699
|
+
return ctx;
|
|
2700
|
+
}
|
|
2701
|
+
|
|
2702
|
+
/**
|
|
2703
|
+
* Per-addon disposer chain. Created lazily the first time an addon
|
|
2704
|
+
* calls `ctx.addDisposer(...)`. Drained by `restartAddon()` /
|
|
2705
|
+
* `unregisterAddon()` so cleanup callbacks run before the new addon
|
|
2706
|
+
* instance comes up.
|
|
2707
|
+
*/
|
|
2708
|
+
private readonly disposerChains = new Map<string, DisposerChain>();
|
|
2709
|
+
|
|
2710
|
+
private getOrCreateDisposerChain(addonId: string): DisposerChain {
|
|
2711
|
+
let chain = this.disposerChains.get(addonId);
|
|
2712
|
+
if (chain == null) {
|
|
2713
|
+
const log = this.loggingService.createLogger().withTags({ addonId });
|
|
2714
|
+
chain = new DisposerChain({
|
|
2715
|
+
onError: (err, index) => {
|
|
2716
|
+
log.error(`Disposer #${index} threw during teardown`, {
|
|
2717
|
+
meta: { error: errMsg(err) },
|
|
2718
|
+
});
|
|
2719
|
+
},
|
|
2720
|
+
});
|
|
2721
|
+
this.disposerChains.set(addonId, chain);
|
|
2722
|
+
}
|
|
2723
|
+
return chain;
|
|
2724
|
+
}
|
|
2725
|
+
|
|
2726
|
+
/**
|
|
2727
|
+
* Drain (and remove) the disposer chain for an addon. Called by the
|
|
2728
|
+
* addon shutdown / restart flow so resources registered via
|
|
2729
|
+
* `ctx.addDisposer(...)` clean up before the next instance boots.
|
|
2730
|
+
*/
|
|
2731
|
+
private async drainDisposerChain(addonId: string): Promise<void> {
|
|
2732
|
+
const chain = this.disposerChains.get(addonId);
|
|
2733
|
+
if (chain == null) return;
|
|
2734
|
+
this.disposerChains.delete(addonId);
|
|
2735
|
+
await chain.dispose();
|
|
2736
|
+
}
|
|
2737
|
+
|
|
2738
|
+
// ── Group orchestration (Phase G3 — opt-in, not yet wired into boot) ──
|
|
2739
|
+
|
|
2740
|
+
/**
|
|
2741
|
+
* Compute the runner plan for a set of addon ids (base-layer D2).
|
|
2742
|
+
*
|
|
2743
|
+
* The runner contract is one-addon-one-process by default (D2/D9):
|
|
2744
|
+
* - every addon gets its OWN runner, keyed by the addon id via
|
|
2745
|
+
* `resolveRunnerId` — no shipped addon declares `execution.group`
|
|
2746
|
+
* today;
|
|
2747
|
+
* - the group-collapse path (same `group` → shared runner keyed by
|
|
2748
|
+
* group name) is an explicit opt-in mechanism; Phase 5 dissolved
|
|
2749
|
+
* the last real co-location group (`pipeline`) once frames travel
|
|
2750
|
+
* as shared-memory `FrameHandle`s and audio over tRPC — nothing
|
|
2751
|
+
* passes a live JS reference across a process boundary any more.
|
|
2752
|
+
*
|
|
2753
|
+
* Addons with `placement: 'agent-only'` are dropped (they don't run on
|
|
2754
|
+
* the hub). `@camstack/core` builtins are dropped too — they stay
|
|
2755
|
+
* in-process on the hub (every forked runner connects to the hub
|
|
2756
|
+
* broker via static-URL TCP transit, so cap providers hosted on the
|
|
2757
|
+
* hub broker are reachable from any runner with one hop; moving them
|
|
2758
|
+
* into a sibling subprocess would isolate them on a separate Moleculer
|
|
2759
|
+
* node that child brokers cannot reach).
|
|
2760
|
+
*
|
|
2761
|
+
* Read-only. Used by the bootstrap to plan spawns up-front and by
|
|
2762
|
+
* diagnostics that want to render the current topology.
|
|
2763
|
+
*/
|
|
2764
|
+
buildAddonGroupPlan(addonIds: readonly string[]): RunnerPlan {
|
|
2765
|
+
const plan = new Map<string, RunnerAddonPlacement[]>();
|
|
2766
|
+
for (const id of addonIds) {
|
|
2767
|
+
const entry = this.addonEntries.get(id);
|
|
2768
|
+
if (!entry?.declaration || !entry.addonDir) continue;
|
|
2769
|
+
if (entry.packageName === "@camstack/core") continue;
|
|
2770
|
+
const placement = resolveAddonPlacement(entry.declaration);
|
|
2771
|
+
if (placement === "agent-only") continue;
|
|
2772
|
+
const runnerId = resolveRunnerId(entry.declaration, id);
|
|
2773
|
+
const bucket = plan.get(runnerId) ?? [];
|
|
2774
|
+
bucket.push({ addonId: id, addonDir: entry.addonDir });
|
|
2775
|
+
plan.set(runnerId, bucket);
|
|
2776
|
+
}
|
|
2777
|
+
return plan;
|
|
2778
|
+
}
|
|
2779
|
+
|
|
2780
|
+
/**
|
|
2781
|
+
* Spawn ONE runner subprocess that hosts every addon in `addons` and
|
|
2782
|
+
* register their custom-actions catalogs against the shared
|
|
2783
|
+
* CustomActionRegistry. Cap providers register themselves via
|
|
2784
|
+
* `CapabilityBridge.onProviderConnected` — the bridge listens for any
|
|
2785
|
+
* new Moleculer node (the runner's nodeID is `${parent}/${runnerId}`)
|
|
2786
|
+
* and proxies its services into the local `CapabilityRegistry`.
|
|
2787
|
+
*
|
|
2788
|
+
* `runnerId` is the addon id (a solo runner — one addon per process,
|
|
2789
|
+
* D2/D9). It can structurally also be a co-location group name if an
|
|
2790
|
+
* addon ever declared `execution.group`, but no shipped manifest does.
|
|
2791
|
+
*
|
|
2792
|
+
* No-op when the runner is already running (idempotent for tests +
|
|
2793
|
+
* crash respawn paths). Errors propagate so the bootstrap can log the
|
|
2794
|
+
* failed runner and surface every hosted addon as `addon.error`.
|
|
2795
|
+
*/
|
|
2796
|
+
async initializeAddonGroup(
|
|
2797
|
+
runnerId: string,
|
|
2798
|
+
addons: readonly RunnerAddonPlacement[],
|
|
2799
|
+
): Promise<void> {
|
|
2800
|
+
if (addons.length === 0) {
|
|
2801
|
+
throw new Error(`initializeAddonGroup("${runnerId}") requires at least one addon`);
|
|
2802
|
+
}
|
|
2803
|
+
try {
|
|
2804
|
+
await this.broker.call("$process.spawnRunner", {
|
|
2805
|
+
runnerId,
|
|
2806
|
+
addons,
|
|
2807
|
+
});
|
|
2808
|
+
} catch (err: unknown) {
|
|
2809
|
+
throw new Error(
|
|
2810
|
+
`Failed to spawn runner "${runnerId}" (${addons.length} addons): ${errMsg(err)}`,
|
|
2811
|
+
{ cause: err },
|
|
2812
|
+
);
|
|
2813
|
+
}
|
|
2814
|
+
|
|
2815
|
+
// Register custom actions for each addon on the runner. Provider
|
|
2816
|
+
// registration for cap methods is handled by
|
|
2817
|
+
// `CapabilityBridge.onProviderConnected` once the runner's
|
|
2818
|
+
// INFO heartbeat lands.
|
|
2819
|
+
for (const { addonId } of addons) {
|
|
2820
|
+
await this.registerForkedAddonCustomActions(addonId, runnerId);
|
|
2821
|
+
// Mark the entry initialized so the in-process core-builtin boot
|
|
2822
|
+
// passes skip it (those passes only touch `@camstack/core`).
|
|
2823
|
+
const entry = this.addonEntries.get(addonId);
|
|
2824
|
+
if (entry) {
|
|
2825
|
+
entry.initialized = true;
|
|
2826
|
+
this.emitAddonLifecycleEvent("addon.started", addonId);
|
|
2827
|
+
}
|
|
2828
|
+
}
|
|
2829
|
+
|
|
2830
|
+
this.logger.info("Addon runner spawned", {
|
|
2831
|
+
meta: { runnerId, addonCount: addons.length, addonIds: addons.map((a) => a.addonId) },
|
|
2832
|
+
});
|
|
2833
|
+
}
|
|
2834
|
+
|
|
2835
|
+
/**
|
|
2836
|
+
* (Re-)register the custom-action catalog for a forked / group-hosted
|
|
2837
|
+
* addon against the shared `CustomActionRegistry`.
|
|
2838
|
+
*
|
|
2839
|
+
* The catalog (zod `input`/`output` specs) is read STATICALLY from the
|
|
2840
|
+
* addon module's `customActions` named export — the handler dispatches
|
|
2841
|
+
* via `broker.call('<addonId>.custom.<action>')`, so the only divergence
|
|
2842
|
+
* vs an in-process addon is the transport, exactly like cap methods.
|
|
2843
|
+
*
|
|
2844
|
+
* Why a fresh import: `this.addonLoader`'s `module` namespace is
|
|
2845
|
+
* captured once at boot. After a hot-update (`installFromTgz` →
|
|
2846
|
+
* `restartAddon`) the on-disk bundle is newer than that cached module,
|
|
2847
|
+
* so re-reading the boot-time `module` would register a STALE catalog
|
|
2848
|
+
* (or none at all, if the addon was first installed after boot). We
|
|
2849
|
+
* re-`import()` the entry with a cache-busting query so Node's ESM
|
|
2850
|
+
* loader hands back the current bundle.
|
|
2851
|
+
*
|
|
2852
|
+
* Idempotent: drops any prior registration first. No-op (with a debug
|
|
2853
|
+
* log) when the addon exports no `customActions` — most addons don't.
|
|
2854
|
+
*/
|
|
2855
|
+
private async registerForkedAddonCustomActions(
|
|
2856
|
+
addonId: string,
|
|
2857
|
+
runnerId: string,
|
|
2858
|
+
): Promise<void> {
|
|
2859
|
+
// Always clear first so a restart that REMOVES custom actions (or an
|
|
2860
|
+
// addon whose entry no longer exports them) doesn't leave stale
|
|
2861
|
+
// entries resolvable.
|
|
2862
|
+
this.customActionRegistry.unregisterAddon(addonId);
|
|
2863
|
+
|
|
2864
|
+
const entry = this.addonEntries.get(addonId);
|
|
2865
|
+
const addonDir = entry?.addonDir;
|
|
2866
|
+
const declarationEntry = entry?.declaration?.entry;
|
|
2867
|
+
if (!addonDir || !declarationEntry) return;
|
|
2868
|
+
|
|
2869
|
+
// Resolve the built entry the same way `AddonLoader.loadDeclaration`
|
|
2870
|
+
// does: `./src/x.ts` → `dist/x.js`, with index.js fallbacks.
|
|
2871
|
+
const entryFile = declarationEntry
|
|
2872
|
+
.replace(/^\.\//, "")
|
|
2873
|
+
.replace(/^src\//, "dist/")
|
|
2874
|
+
.replace(/\.ts$/, ".js");
|
|
2875
|
+
let entryPath = path.resolve(addonDir, entryFile);
|
|
2876
|
+
if (!fs.existsSync(entryPath)) {
|
|
2877
|
+
const base = entryPath.replace(/\.(js|cjs|mjs)$/, "");
|
|
2878
|
+
const alternatives = [
|
|
2879
|
+
`${base}.cjs`,
|
|
2880
|
+
`${base}.mjs`,
|
|
2881
|
+
path.resolve(addonDir, "dist", "index.js"),
|
|
2882
|
+
path.resolve(addonDir, "dist", "index.cjs"),
|
|
2883
|
+
path.resolve(addonDir, "dist", "index.mjs"),
|
|
2884
|
+
path.resolve(addonDir, declarationEntry),
|
|
2885
|
+
];
|
|
2886
|
+
entryPath = alternatives.find((p) => fs.existsSync(p)) ?? entryPath;
|
|
2887
|
+
}
|
|
2888
|
+
if (!fs.existsSync(entryPath)) return;
|
|
2889
|
+
|
|
2890
|
+
let catalog: unknown;
|
|
2891
|
+
try {
|
|
2892
|
+
// Cache-bust so a hot-updated bundle is re-read instead of served
|
|
2893
|
+
// from Node's ESM module cache.
|
|
2894
|
+
const cacheBustedUrl = `${pathToFileURL(entryPath).href}?t=${Date.now()}`;
|
|
2895
|
+
const modUnknown: unknown = await import(cacheBustedUrl);
|
|
2896
|
+
catalog =
|
|
2897
|
+
modUnknown && typeof modUnknown === "object"
|
|
2898
|
+
? (modUnknown as Record<string, unknown>)["customActions"]
|
|
2899
|
+
: undefined;
|
|
2900
|
+
} catch (err) {
|
|
2901
|
+
this.logger.warn(
|
|
2902
|
+
"Failed to load custom-action catalog for forked addon",
|
|
2903
|
+
{ tags: { addonId }, meta: { error: errMsg(err) } },
|
|
2904
|
+
);
|
|
2905
|
+
return;
|
|
2906
|
+
}
|
|
2907
|
+
|
|
2908
|
+
if (!catalog || typeof catalog !== "object") {
|
|
2909
|
+
this.logger.debug("Forked addon exports no custom actions", {
|
|
2910
|
+
tags: { addonId },
|
|
2911
|
+
meta: { runnerId },
|
|
2912
|
+
});
|
|
2913
|
+
return;
|
|
2914
|
+
}
|
|
2915
|
+
|
|
2916
|
+
this.customActionRegistry.registerAddon(
|
|
2917
|
+
addonId,
|
|
2918
|
+
catalog as import("@camstack/types").CustomActionsSpec,
|
|
2919
|
+
(action, input) => this.broker.call(`${addonId}.custom.${action}`, input),
|
|
2920
|
+
);
|
|
2921
|
+
this.logger.info("Runner addon custom actions registered", {
|
|
2922
|
+
tags: { addonId },
|
|
2923
|
+
meta: { runnerId },
|
|
2924
|
+
});
|
|
2925
|
+
}
|
|
2926
|
+
}
|