@camstack/addon-detection-pipeline 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +638 -0
- package/dist/index.d.ts +638 -0
- package/dist/index.js +5826 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +5801 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +84 -0
- package/python/inference_pool.py +1088 -0
- package/python/postprocessors/__init__.py +24 -0
- package/python/postprocessors/arcface.py +31 -0
- package/python/postprocessors/ctc.py +68 -0
- package/python/postprocessors/saliency.py +44 -0
- package/python/postprocessors/scrfd.py +212 -0
- package/python/postprocessors/softmax.py +43 -0
- package/python/postprocessors/yamnet.py +41 -0
- package/python/postprocessors/yolo.py +278 -0
- package/python/postprocessors/yolo_seg.py +247 -0
- package/python/requirements-coreml.txt +4 -0
- package/python/requirements-onnxruntime.txt +3 -0
- package/python/requirements-openvino.txt +3 -0
- package/python/requirements.txt +9 -0
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,638 @@
|
|
|
1
|
+
import * as _camstack_types from '@camstack/types';
|
|
2
|
+
import { StepDefinition, ModelFormat, IPipelineExecutorProvider, IScopedLogger, IEventBus, ConfigUISchema, PipelineEngineChoice, PipelineSchema, AvailableEngine, PipelineDefaultStep, PipelineConfig, InferenceCapabilities, ModelAvailability, PipelineRunInput, FrameResult, FrameInput, DetectorOutput, PipelineTemplate, PipelineTemplateStep, ConfigUISchemaWithValues, IAddonResolver, BaseAddon, ProviderRegistration, ModelCatalogEntry } from '@camstack/types';
|
|
3
|
+
export { PoolModelConfig, PostprocessorType, StepDefinition } from '@camstack/types';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Pipeline step classes — each step implements IPipelineStep.
|
|
7
|
+
*
|
|
8
|
+
* `definition` holds the static metadata (models, labels, postprocessor, etc.).
|
|
9
|
+
* `getConfigSchema()` declares per-step settings (class filters, thresholds)
|
|
10
|
+
* that the UI renders dynamically and that can be overridden per-run.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/** Compat: flat array of StepDefinition for existing consumers */
|
|
14
|
+
declare const ALL_STEPS: readonly StepDefinition[];
|
|
15
|
+
/**
|
|
16
|
+
* Look up a step definition by ID (compat shortcut).
|
|
17
|
+
* @throws if the step ID is not registered.
|
|
18
|
+
*/
|
|
19
|
+
declare function getStepDefinition(stepId: string): StepDefinition;
|
|
20
|
+
/**
|
|
21
|
+
* Get the default model ID for a step given the current model format.
|
|
22
|
+
* Selects the smallest model that supports the format.
|
|
23
|
+
* Falls back to StepDefinition.defaultModelId if no format-specific model found.
|
|
24
|
+
*/
|
|
25
|
+
declare function getDefaultModelForFormat(stepId: string, format: ModelFormat): string;
|
|
26
|
+
|
|
27
|
+
type BatchMode$1 = 'none' | 'list' | 'window';
|
|
28
|
+
/**
|
|
29
|
+
* Per-runtime pool tuning surfaced through addon-detection-pipeline
|
|
30
|
+
* settings. All fields optional — undefined means "use the runtime
|
|
31
|
+
* default resolved in `engine-factory`".
|
|
32
|
+
*/
|
|
33
|
+
interface PoolTuning {
|
|
34
|
+
readonly batchMode?: BatchMode$1;
|
|
35
|
+
readonly windowMs?: number;
|
|
36
|
+
readonly maxBatchSize?: number;
|
|
37
|
+
readonly numStreams?: number;
|
|
38
|
+
readonly intraOpThreads?: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
declare class DetectionPipelineProvider implements IPipelineExecutorProvider {
|
|
42
|
+
private readonly modelsDir;
|
|
43
|
+
private readonly eventBus;
|
|
44
|
+
/**
|
|
45
|
+
* Closure that returns the addon's config schema. Injected by the addon
|
|
46
|
+
* class at construction so the provider can serve `getDetectionConfigSchema`
|
|
47
|
+
* directly. The addon owns both pieces — no public setter needed.
|
|
48
|
+
*/
|
|
49
|
+
private readonly detectionConfigSchemaSource;
|
|
50
|
+
/**
|
|
51
|
+
* Executor tuning forwarded to the `EngineFactory`. Exposes
|
|
52
|
+
* `concurrency` (Python inference-pool worker threads) plus the
|
|
53
|
+
* per-runtime batch tuning (`tuning`). Undefined fields fall back
|
|
54
|
+
* to runtime defaults — see engine-factory.ts.
|
|
55
|
+
*
|
|
56
|
+
* `pythonPath` is the absolute path to the embedded Python (resolved
|
|
57
|
+
* once by the addon class via `ctx.deps.ensurePython()`); required
|
|
58
|
+
* whenever `runtime: 'python'` is selected. `pythonAddonDir` points
|
|
59
|
+
* to this addon's `python/` dir at runtime — used to locate the
|
|
60
|
+
* `requirements*.txt` files that drive lazy backend installs.
|
|
61
|
+
*/
|
|
62
|
+
private readonly executorOptions;
|
|
63
|
+
private engineFactory;
|
|
64
|
+
private executor;
|
|
65
|
+
private currentSteps;
|
|
66
|
+
private currentEngine;
|
|
67
|
+
private readonly log;
|
|
68
|
+
/** Addon context — ctx.api resolves lazily (direct caller created after boot) */
|
|
69
|
+
private addonCtx;
|
|
70
|
+
/**
|
|
71
|
+
* Per-device {@link DeviceProxy} cache used for zone gating at the
|
|
72
|
+
* runtime path. Reads `state.zones.value` + `state.zoneRules.value`
|
|
73
|
+
* synchronously per frame so detections inside an `exclude` zone
|
|
74
|
+
* never reach the tracker downstream — saves the tracking + event
|
|
75
|
+
* cost on top of the analytics-side filter.
|
|
76
|
+
*/
|
|
77
|
+
private readonly deviceProxies;
|
|
78
|
+
private readonly proxyUnsubs;
|
|
79
|
+
/** Initialization lock — ensures only one init runs at a time; concurrent callers await the same promise */
|
|
80
|
+
private initPromise;
|
|
81
|
+
/**
|
|
82
|
+
* Last logged "step configured" signature — used to dedupe the
|
|
83
|
+
* per-step debug trail so it only fires on actual config CHANGE
|
|
84
|
+
* (not on every frame). `runPipeline` is called once per decoded
|
|
85
|
+
* frame (up to detectionFps × N cameras), so even at debug level
|
|
86
|
+
* the per-step dump produced ~50 lines/sec/camera and drowned the
|
|
87
|
+
* Cluster log viewer.
|
|
88
|
+
*/
|
|
89
|
+
private lastStepsSignature;
|
|
90
|
+
/**
|
|
91
|
+
* True once the engine + models are fully ready for inference. No
|
|
92
|
+
* longer gates runtime dispatch (Phase 4 removed the legacy `runFrame`
|
|
93
|
+
* which read this flag); kept as a diagnostic the admin UI / tests
|
|
94
|
+
* surface via the future `isReady()` helper.
|
|
95
|
+
*/
|
|
96
|
+
private ready;
|
|
97
|
+
/**
|
|
98
|
+
* Warm cache for benchmark engine-override runs.
|
|
99
|
+
*
|
|
100
|
+
* Each override rebuild costs a full Python pool spin-up (~300-500ms)
|
|
101
|
+
* plus the per-model load. Benchmark tabs iterate up to 800× against
|
|
102
|
+
* the same override (e.g. yolov9s / coreml / all) — paying that spin-up
|
|
103
|
+
* each iteration dwarfs actual inference time.
|
|
104
|
+
*
|
|
105
|
+
* When the same override engine is requested within `OVERRIDE_CACHE_TTL_MS`,
|
|
106
|
+
* we reuse the prior transient factory. Hitting a different override
|
|
107
|
+
* or exceeding the TTL disposes the cache and spins a new factory.
|
|
108
|
+
* The prior "main" factory (`priorFactory`) is still restored so
|
|
109
|
+
* runtime dispatch (camera frames) keeps its own resident engine.
|
|
110
|
+
*/
|
|
111
|
+
private overrideCache;
|
|
112
|
+
private overrideCacheTimer;
|
|
113
|
+
private static readonly OVERRIDE_CACHE_TTL_MS;
|
|
114
|
+
/** Read the addon settings store (all keys). */
|
|
115
|
+
private readonly readStore;
|
|
116
|
+
/** Write a patch to the addon settings store. */
|
|
117
|
+
private readonly writeStore;
|
|
118
|
+
/** Read per-device settings. */
|
|
119
|
+
private readonly readDeviceStore;
|
|
120
|
+
constructor(settings: {
|
|
121
|
+
readAddonStore(): Promise<Record<string, unknown>>;
|
|
122
|
+
writeAddonStore(patch: Record<string, unknown>): Promise<void>;
|
|
123
|
+
readDeviceStore?(deviceId: number): Promise<Record<string, unknown>>;
|
|
124
|
+
}, modelsDir: string, logger: IScopedLogger, eventBus?: IEventBus | null,
|
|
125
|
+
/**
|
|
126
|
+
* Closure that returns the addon's config schema. Injected by the addon
|
|
127
|
+
* class at construction so the provider can serve `getDetectionConfigSchema`
|
|
128
|
+
* directly. The addon owns both pieces — no public setter needed.
|
|
129
|
+
*/
|
|
130
|
+
detectionConfigSchemaSource?: (() => ConfigUISchema) | null,
|
|
131
|
+
/**
|
|
132
|
+
* Executor tuning forwarded to the `EngineFactory`. Exposes
|
|
133
|
+
* `concurrency` (Python inference-pool worker threads) plus the
|
|
134
|
+
* per-runtime batch tuning (`tuning`). Undefined fields fall back
|
|
135
|
+
* to runtime defaults — see engine-factory.ts.
|
|
136
|
+
*
|
|
137
|
+
* `pythonPath` is the absolute path to the embedded Python (resolved
|
|
138
|
+
* once by the addon class via `ctx.deps.ensurePython()`); required
|
|
139
|
+
* whenever `runtime: 'python'` is selected. `pythonAddonDir` points
|
|
140
|
+
* to this addon's `python/` dir at runtime — used to locate the
|
|
141
|
+
* `requirements*.txt` files that drive lazy backend installs.
|
|
142
|
+
*/
|
|
143
|
+
executorOptions?: {
|
|
144
|
+
concurrency?: number;
|
|
145
|
+
tuning?: PoolTuning;
|
|
146
|
+
pythonPath?: string;
|
|
147
|
+
pythonAddonDir?: string;
|
|
148
|
+
numWorkers?: number;
|
|
149
|
+
});
|
|
150
|
+
/** Load persisted engine choice. Call after construction. */
|
|
151
|
+
init(): Promise<void>;
|
|
152
|
+
/**
|
|
153
|
+
* Lazy-install the pip requirements file matching `engine.backend` into
|
|
154
|
+
* the embedded Python before the inference pool spawns. No-op for
|
|
155
|
+
* `runtime: 'node'`, for backends with no requirements file on disk,
|
|
156
|
+
* or before the addon context has been wired (warm-pool race window).
|
|
157
|
+
*
|
|
158
|
+
* Idempotent — `installPythonRequirements` short-circuits on a hash
|
|
159
|
+
* marker, so back-to-back EngineFactory rebuilds (benchmark overrides,
|
|
160
|
+
* tuning respawns) skip the pip subprocess after the first install.
|
|
161
|
+
*/
|
|
162
|
+
private ensureBackendDeps;
|
|
163
|
+
/**
|
|
164
|
+
* Warm the Python inference pool up-front so consumers that hit
|
|
165
|
+
* `runPipeline` on the first frame don't race the pool's spawn
|
|
166
|
+
* (`SharedInferencePool: not initialized`). Called from the addon's
|
|
167
|
+
* `onInitialize` so `BaseAddon.autoEmitReadiness` fires `ready`
|
|
168
|
+
* only after the pool is actually callable — consumers that
|
|
169
|
+
* `awaitReady('pipeline-executor', {node:…})` now get a true
|
|
170
|
+
* "callable" signal, not a "process started" signal.
|
|
171
|
+
*
|
|
172
|
+
* Idempotent: delegates to `ensureEngineFactory` which short-circuits
|
|
173
|
+
* on a warmed pool.
|
|
174
|
+
*/
|
|
175
|
+
warmPool(): Promise<void>;
|
|
176
|
+
/** True when the engine + model pool are fully warmed and inference-ready. */
|
|
177
|
+
isReady(): boolean;
|
|
178
|
+
/** Detect the best inference engine for this platform. */
|
|
179
|
+
private static detectBestEngine;
|
|
180
|
+
/** Store the addon context. ctx.api is a lazy getter resolved at call time. */
|
|
181
|
+
setApi(addonCtx: unknown): Promise<void>;
|
|
182
|
+
getSchema(engine?: PipelineEngineChoice): Promise<PipelineSchema>;
|
|
183
|
+
getAvailableEngines(): Promise<PipelineEngineChoice[]>;
|
|
184
|
+
getAvailableEnginesWithDevices(): AvailableEngine[];
|
|
185
|
+
getDefaultSteps(engine: PipelineEngineChoice): Promise<PipelineDefaultStep[]>;
|
|
186
|
+
getGlobalSteps(): Promise<PipelineDefaultStep[] | null>;
|
|
187
|
+
getGlobalPipelineConfig(): Promise<PipelineConfig | null>;
|
|
188
|
+
getSelectedEngine(): Promise<PipelineEngineChoice>;
|
|
189
|
+
getOrchestratorConfigSchema(): Promise<ConfigUISchema>;
|
|
190
|
+
getCapabilities(_forceRefresh?: boolean): Promise<InferenceCapabilities>;
|
|
191
|
+
getAddonModels(input: {
|
|
192
|
+
addonId: string;
|
|
193
|
+
}): Promise<readonly ModelAvailability[]>;
|
|
194
|
+
getDetectionConfigSchema(): Promise<ConfigUISchema | null>;
|
|
195
|
+
downloadModel(input: {
|
|
196
|
+
modelId: string;
|
|
197
|
+
format: ModelFormat;
|
|
198
|
+
addonId: string;
|
|
199
|
+
}): Promise<{
|
|
200
|
+
filePath: string;
|
|
201
|
+
sizeMB: number;
|
|
202
|
+
durationMs: number;
|
|
203
|
+
}>;
|
|
204
|
+
deleteModel(input: {
|
|
205
|
+
modelId: string;
|
|
206
|
+
format: ModelFormat;
|
|
207
|
+
addonId: string;
|
|
208
|
+
}): Promise<{
|
|
209
|
+
success: true;
|
|
210
|
+
}>;
|
|
211
|
+
runPipeline(input: PipelineRunInput, onProgress?: (message: string) => void): Promise<FrameResult>;
|
|
212
|
+
/**
|
|
213
|
+
* Apply a benchmark engine override (mirroring the body of `runPipeline`
|
|
214
|
+
* lines 863-953). Swaps `currentEngine` / `engineFactory` / `executor`
|
|
215
|
+
* to the override's warm factory and returns a restore function the
|
|
216
|
+
* caller MUST invoke in `finally`. When `override` is undefined or
|
|
217
|
+
* matches the current engine, this is a no-op and the restore function
|
|
218
|
+
* does nothing — same shape so callers don't need a conditional path.
|
|
219
|
+
*
|
|
220
|
+
* Used by both `runPipeline` (single-frame benchmark) and
|
|
221
|
+
* `runPipelineBatch` (batched fast path). Without this, the batch
|
|
222
|
+
* path silently ignored `input.engine` and ran the user's override
|
|
223
|
+
* against the addon's saved engine — making the override matrix in
|
|
224
|
+
* the bench UI a no-op for batchSize > 1.
|
|
225
|
+
*/
|
|
226
|
+
private applyEngineOverride;
|
|
227
|
+
/**
|
|
228
|
+
* Schedule eviction of the override cache after `OVERRIDE_CACHE_TTL_MS`
|
|
229
|
+
* of idleness. Rescheduling resets the timer so an active benchmark
|
|
230
|
+
* keeps its warm factory alive until iterations stop arriving.
|
|
231
|
+
*/
|
|
232
|
+
private scheduleOverrideCacheEviction;
|
|
233
|
+
private evictOverrideCache;
|
|
234
|
+
/** Convert PipelineStepInput[] → PipelineDefaultStep[] by enriching from StepDefinitions */
|
|
235
|
+
private inputStepsToPipelineSteps;
|
|
236
|
+
/**
|
|
237
|
+
* Single-flight gate around `ensureModelsForSteps`. Concurrent
|
|
238
|
+
* callers (every camera that fires motion in the same window calls
|
|
239
|
+
* `runPipeline` → `ensureModelsForSteps`) share the same in-flight
|
|
240
|
+
* promise instead of each racing into `loadAdditional`. Without this
|
|
241
|
+
* the pool gets the SAME model loaded N times under N distinct
|
|
242
|
+
* pool indices because every caller saw `isLoadedWithModel === false`
|
|
243
|
+
* before any of them committed via `stepToLoaded.set`.
|
|
244
|
+
*/
|
|
245
|
+
private modelLoadInFlight;
|
|
246
|
+
/** Ensure all models needed by steps are downloaded and loaded in the engine pool. */
|
|
247
|
+
private ensureModelsForSteps;
|
|
248
|
+
/** Download a model with retry + exponential backoff */
|
|
249
|
+
private downloadWithRetry;
|
|
250
|
+
/** Parse JPEG/PNG dimensions without decoding the full image */
|
|
251
|
+
private getJpegDimensions;
|
|
252
|
+
/** Fast JPEG dimension parsing from SOF marker (no external deps) */
|
|
253
|
+
private parseJpegDimensions;
|
|
254
|
+
detect(input: {
|
|
255
|
+
addonId: string;
|
|
256
|
+
frame: FrameInput;
|
|
257
|
+
config?: Record<string, unknown>;
|
|
258
|
+
}): Promise<DetectorOutput>;
|
|
259
|
+
/**
|
|
260
|
+
* Batched run — dispatches N raw frames against the same loaded
|
|
261
|
+
* single-step model in one IPC round-trip via
|
|
262
|
+
* `EngineFactory.batchInferRaw`. Falls back to N parallel
|
|
263
|
+
* `runPipeline` calls when the fast path's preconditions don't hold
|
|
264
|
+
* (multi-step tree, crop children, JPEG-only frames, Node ONNX
|
|
265
|
+
* factory). Returns one minimally-shaped `FrameResult` per input
|
|
266
|
+
* frame in input order.
|
|
267
|
+
*
|
|
268
|
+
* Used by `scripts/bench-scrypted-style.mts` for the
|
|
269
|
+
* Scrypted-equivalent benchmark — Scrypted's `detectObjects(media,
|
|
270
|
+
* {batch})` collapses N requests into one call too, so bypassing
|
|
271
|
+
* the per-call IPC overhead is what makes the comparison fair.
|
|
272
|
+
*/
|
|
273
|
+
runPipelineBatch(input: {
|
|
274
|
+
readonly steps: readonly _camstack_types.PipelineStepInput[];
|
|
275
|
+
readonly engine?: PipelineEngineChoice;
|
|
276
|
+
readonly frames: readonly FrameInput[];
|
|
277
|
+
readonly deviceId?: number;
|
|
278
|
+
readonly sessionId?: string;
|
|
279
|
+
}): Promise<{
|
|
280
|
+
readonly results: readonly FrameResult[];
|
|
281
|
+
}>;
|
|
282
|
+
private runPipelineBatchImpl;
|
|
283
|
+
listReferenceImages(): Promise<Array<{
|
|
284
|
+
id: string;
|
|
285
|
+
filename: string;
|
|
286
|
+
sizeKB: number;
|
|
287
|
+
}>>;
|
|
288
|
+
getReferenceImage(input: {
|
|
289
|
+
filename: string;
|
|
290
|
+
}): Promise<{
|
|
291
|
+
base64: string;
|
|
292
|
+
filename: string;
|
|
293
|
+
} | null>;
|
|
294
|
+
private resolveRefImagesDir;
|
|
295
|
+
listTemplates(): Promise<PipelineTemplate[]>;
|
|
296
|
+
saveTemplate(input: {
|
|
297
|
+
name: string;
|
|
298
|
+
steps: readonly PipelineTemplateStep[];
|
|
299
|
+
engine: PipelineEngineChoice;
|
|
300
|
+
}): Promise<PipelineTemplate>;
|
|
301
|
+
updateTemplate(input: {
|
|
302
|
+
id: string;
|
|
303
|
+
name?: string;
|
|
304
|
+
steps?: readonly PipelineTemplateStep[];
|
|
305
|
+
}): Promise<PipelineTemplate>;
|
|
306
|
+
deleteTemplate(input: {
|
|
307
|
+
id: string;
|
|
308
|
+
}): Promise<void>;
|
|
309
|
+
getDeviceSettingsContribution(_input: {
|
|
310
|
+
deviceId: number;
|
|
311
|
+
}): Promise<ConfigUISchemaWithValues | null>;
|
|
312
|
+
getDeviceLiveContribution(_input: {
|
|
313
|
+
deviceId: number;
|
|
314
|
+
}): Promise<ConfigUISchemaWithValues | null>;
|
|
315
|
+
applyDeviceSettingsPatch(_input: {
|
|
316
|
+
deviceId: number;
|
|
317
|
+
patch: Record<string, unknown>;
|
|
318
|
+
}): Promise<{
|
|
319
|
+
success: true;
|
|
320
|
+
}>;
|
|
321
|
+
getAddonResolver(): Promise<IAddonResolver>;
|
|
322
|
+
shutdown(): Promise<void>;
|
|
323
|
+
/**
|
|
324
|
+
* Resolve and cache a {@link DeviceProxy} for the given camera. Pins
|
|
325
|
+
* the `state.zones` + `state.zoneRules` slice handles so `.value`
|
|
326
|
+
* stays warm via the kernel runtime-state mirror, and runtime gate
|
|
327
|
+
* reads in `runPipeline` are sync (zero per-frame round-trip).
|
|
328
|
+
* Returns null when the addon context isn't available — the gate
|
|
329
|
+
* falls back to "no filtering" instead of blocking inference.
|
|
330
|
+
*/
|
|
331
|
+
private ensureDeviceProxy;
|
|
332
|
+
private releaseDeviceProxy;
|
|
333
|
+
/**
|
|
334
|
+
* Apply detection-stage zone rules to a {@link FrameResult}. Drops
|
|
335
|
+
* first-level detections whose center lies in an `exclude` rule's
|
|
336
|
+
* zones (or, in whitelist mode, isn't in any `include` rule's
|
|
337
|
+
* zones), and any `detail` children whose `parentId` references a
|
|
338
|
+
* dropped first-level. Returns the original result unchanged when
|
|
339
|
+
* the gate has nothing to filter (no zones, no rules, or no
|
|
340
|
+
* detections).
|
|
341
|
+
*/
|
|
342
|
+
private gateDetectionsByZoneRules;
|
|
343
|
+
cacheFrameInPool(input: {
|
|
344
|
+
readonly data: Uint8Array;
|
|
345
|
+
readonly width: number;
|
|
346
|
+
readonly height: number;
|
|
347
|
+
readonly format: 'rgb' | 'bgr' | 'gray';
|
|
348
|
+
}): Promise<{
|
|
349
|
+
frameId: number;
|
|
350
|
+
width: number;
|
|
351
|
+
height: number;
|
|
352
|
+
}>;
|
|
353
|
+
inferCached(input: {
|
|
354
|
+
readonly stepId: string;
|
|
355
|
+
readonly frameId: number;
|
|
356
|
+
}): Promise<Record<string, unknown>>;
|
|
357
|
+
uncacheFrame(input: {
|
|
358
|
+
readonly frameId: number;
|
|
359
|
+
}): Promise<void>;
|
|
360
|
+
getEffectiveTuning(): Promise<{
|
|
361
|
+
batchMode: string;
|
|
362
|
+
windowMs: number;
|
|
363
|
+
maxBatchSize: number;
|
|
364
|
+
concurrency: number;
|
|
365
|
+
}>;
|
|
366
|
+
/**
|
|
367
|
+
* Ensure the engine factory + inference pool is initialized.
|
|
368
|
+
* Does NOT require global steps — used by benchmark/test paths that provide their own steps.
|
|
369
|
+
* If global steps exist, initializes with them. Otherwise, creates an empty pool.
|
|
370
|
+
*/
|
|
371
|
+
private ensureEngineFactory;
|
|
372
|
+
private ensureExecutor;
|
|
373
|
+
/** Actual initialization — download models, create engine, load pool. Called once. */
|
|
374
|
+
private doInitialize;
|
|
375
|
+
/**
|
|
376
|
+
* Phase 2b — resolve the engine from the addon's new schema-backed
|
|
377
|
+
* fields (`engineRuntime`, `engineBackend`, `engineDevice`). Fallback
|
|
378
|
+
* to the legacy `KEY_ENGINE` JSON blob for pre-migration stores.
|
|
379
|
+
* Returns null when neither source has anything; the caller keeps
|
|
380
|
+
* whatever `detectBestEngine()` picked at construction.
|
|
381
|
+
*/
|
|
382
|
+
private loadEngine;
|
|
383
|
+
/**
|
|
384
|
+
* Synchronous availability probe for a Python inference backend. Runs a
|
|
385
|
+
* short `python3 -c "import <mod>"` with a 3s timeout. Used at
|
|
386
|
+
* `loadEngine` time to reject a persisted backend choice the Python
|
|
387
|
+
* interpreter on this host can't actually import — without this the
|
|
388
|
+
* detection-pipeline child process exits with code 1 the moment
|
|
389
|
+
* `inference_pool.py` hits its `from openvino.runtime import Core`.
|
|
390
|
+
*/
|
|
391
|
+
private static isPythonBackendAvailable;
|
|
392
|
+
/**
|
|
393
|
+
* Re-run the platform probe for the inference engine and persist the
|
|
394
|
+
* result into the addon's own global-settings keys. Because those
|
|
395
|
+
* fields are declared `requiresRestart: true`, the
|
|
396
|
+
* `updateGlobalSettings` pathway auto-schedules an addon restart —
|
|
397
|
+
* here we do the write directly via the settings store and trigger
|
|
398
|
+
* restart manually through the addons cap.
|
|
399
|
+
*
|
|
400
|
+
* Returns the chosen engine so the UI can confirm the pick without a
|
|
401
|
+
* second round-trip.
|
|
402
|
+
*/
|
|
403
|
+
/**
|
|
404
|
+
* Phase 2c — flat per-addonId map holding the video-pipeline step
|
|
405
|
+
* config (`modelId` / `settings`). No `enabled` field: whether an
|
|
406
|
+
* addon step runs is a capability-binding concern, not a
|
|
407
|
+
* pipeline-config one. Replaces the orchestrator's per-agent
|
|
408
|
+
* `addonDefaults`. The map is the source of truth; the tree consumed
|
|
409
|
+
* by the executor (`PipelineDefaultStep[]`) is materialised from
|
|
410
|
+
* this map + catalog compat at read time (not done in this commit —
|
|
411
|
+
* phase 2f wires the resolver through).
|
|
412
|
+
*/
|
|
413
|
+
getVideoPipelineSteps(): Promise<Record<string, {
|
|
414
|
+
modelId: string;
|
|
415
|
+
settings: Readonly<Record<string, unknown>>;
|
|
416
|
+
}>>;
|
|
417
|
+
setVideoPipelineSteps(input: {
|
|
418
|
+
readonly steps: Record<string, {
|
|
419
|
+
modelId: string;
|
|
420
|
+
settings: Readonly<Record<string, unknown>>;
|
|
421
|
+
}>;
|
|
422
|
+
}): Promise<{
|
|
423
|
+
success: true;
|
|
424
|
+
}>;
|
|
425
|
+
listLoadedEngines(): Promise<readonly {
|
|
426
|
+
engineKey: string;
|
|
427
|
+
engine: PipelineEngineChoice;
|
|
428
|
+
modelsLoaded: readonly string[];
|
|
429
|
+
inUseByCameras: readonly number[];
|
|
430
|
+
kind: 'runtime' | 'warm-override';
|
|
431
|
+
poolPid: number | null;
|
|
432
|
+
idleMs: number | null;
|
|
433
|
+
idleTtlMs: number | null;
|
|
434
|
+
}[]>;
|
|
435
|
+
spinEngine(input: {
|
|
436
|
+
engine: PipelineEngineChoice;
|
|
437
|
+
}): Promise<{
|
|
438
|
+
success: true;
|
|
439
|
+
}>;
|
|
440
|
+
killEngine(input: {
|
|
441
|
+
engine: PipelineEngineChoice;
|
|
442
|
+
force?: boolean;
|
|
443
|
+
}): Promise<{
|
|
444
|
+
success: boolean;
|
|
445
|
+
reason?: string;
|
|
446
|
+
}>;
|
|
447
|
+
reprobeEngine(): Promise<PipelineEngineChoice>;
|
|
448
|
+
getReferenceAudioFiles(): Promise<readonly {
|
|
449
|
+
filename: string;
|
|
450
|
+
sizeKb: number;
|
|
451
|
+
}[]>;
|
|
452
|
+
getReferenceAudio(input: {
|
|
453
|
+
filename: string;
|
|
454
|
+
}): Promise<{
|
|
455
|
+
base64: string;
|
|
456
|
+
}>;
|
|
457
|
+
getAudioCapabilities(): Promise<{
|
|
458
|
+
activeBackend: string;
|
|
459
|
+
availableBackends: readonly {
|
|
460
|
+
id: string;
|
|
461
|
+
name: string;
|
|
462
|
+
description: string;
|
|
463
|
+
available: boolean;
|
|
464
|
+
rawLabels?: readonly string[];
|
|
465
|
+
}[];
|
|
466
|
+
sampleRate: number;
|
|
467
|
+
chunkDurationMs: number;
|
|
468
|
+
}>;
|
|
469
|
+
runAudioTest(input: {
|
|
470
|
+
addonId: string;
|
|
471
|
+
modelId: string;
|
|
472
|
+
filename?: string;
|
|
473
|
+
settings?: Record<string, unknown>;
|
|
474
|
+
}): Promise<{
|
|
475
|
+
success: boolean;
|
|
476
|
+
error?: string;
|
|
477
|
+
frame?: _camstack_types.AudioResult;
|
|
478
|
+
}>;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* Select the best inference engine for the current platform.
|
|
483
|
+
* Order: CoreML (macOS ARM) > CUDA (NVIDIA) > OpenVINO (Intel) > CPU
|
|
484
|
+
*/
|
|
485
|
+
type EngineRuntime = 'node' | 'python';
|
|
486
|
+
type BatchMode = 'none' | 'list' | 'window';
|
|
487
|
+
interface DetectionPipelineConfig {
|
|
488
|
+
/**
|
|
489
|
+
* Python inference-pool worker concurrency.
|
|
490
|
+
* `0` = runtime-aware auto default (resolved in engine-factory):
|
|
491
|
+
* coreml: 1 — matrix bench shows >1 hurts (CoreML serializes per-context).
|
|
492
|
+
* onnxruntime: 4 — InferenceSession.run is thread-safe; scales with cores.
|
|
493
|
+
* openvino: 1 — OV runtime manages internal infer-request pool.
|
|
494
|
+
* Non-zero values override the auto default.
|
|
495
|
+
*/
|
|
496
|
+
readonly concurrency: number;
|
|
497
|
+
/**
|
|
498
|
+
* Pool batch dispatch strategy. Set per-backend; defaults injected by
|
|
499
|
+
* `getGlobalSettings(overlay)` based on `engineBackend`.
|
|
500
|
+
* - 'none' : one predict_fn per item (no batching)
|
|
501
|
+
* - 'list' : MSG_INFER_BATCH dispatched as `model.predict([{},{}...])`
|
|
502
|
+
* - 'window' : per-model accumulator coalesces concurrent MSG_INFER_RAW
|
|
503
|
+
* calls within `windowMs`, flushes via single batched predict.
|
|
504
|
+
*/
|
|
505
|
+
readonly batchMode: BatchMode | '';
|
|
506
|
+
/** Window-mode accumulator flush window (ms). Only honored when batchMode='window'. 0 = auto. */
|
|
507
|
+
readonly windowMs: number;
|
|
508
|
+
/** Max items per batched predict call. Only honored when batchMode!='none'. 0 = auto. */
|
|
509
|
+
readonly maxBatchSize: number;
|
|
510
|
+
/** OpenVINO num_streams hint. 0 = auto. */
|
|
511
|
+
readonly numStreams: number;
|
|
512
|
+
/** ONNX Runtime intra-op thread count. 0 = auto. */
|
|
513
|
+
readonly intraOpThreads: number;
|
|
514
|
+
/**
|
|
515
|
+
* Number of independent Python subprocess workers in the inference
|
|
516
|
+
* pool. Each worker holds its own MLModel copy + predict context;
|
|
517
|
+
* inference dispatches round-robin across them. 0 = runtime-aware
|
|
518
|
+
* default (CoreML: 2 — required to lift the single-context ANE cap
|
|
519
|
+
* on multi-camera workloads; non-CoreML backends: 1, since they
|
|
520
|
+
* manage their own internal parallelism). Model load/unload
|
|
521
|
+
* propagate to all workers, so RAM cost scales linearly with
|
|
522
|
+
* `numWorkers × modelSize`.
|
|
523
|
+
*/
|
|
524
|
+
readonly numWorkers: number;
|
|
525
|
+
/** Chain level 1 — Python vs Node host runtime. */
|
|
526
|
+
readonly engineRuntime: EngineRuntime;
|
|
527
|
+
/** Chain level 2 — backend within the runtime. */
|
|
528
|
+
readonly engineBackend: string;
|
|
529
|
+
/** Chain level 3 — hardware device within the backend. */
|
|
530
|
+
readonly engineDevice: string;
|
|
531
|
+
/**
|
|
532
|
+
* Readable snapshot of the last-probed platform-best engine. Format:
|
|
533
|
+
* `"runtime/backend/device"`. Set by `reprobeEngine()`; the UI shows
|
|
534
|
+
* it next to the cascade as a hint + provides a "Re-probe" button.
|
|
535
|
+
*/
|
|
536
|
+
readonly probedBestEngine: string;
|
|
537
|
+
}
|
|
538
|
+
/** Derive the model-format from a backend value. Called by the provider. */
|
|
539
|
+
declare function backendToFormat(backend: string): 'onnx' | 'coreml' | 'openvino';
|
|
540
|
+
declare class DetectionPipelineAddon extends BaseAddon<DetectionPipelineConfig> {
|
|
541
|
+
private provider;
|
|
542
|
+
private engineMetricsTimer;
|
|
543
|
+
/** Snapshot-equality cache for engine-metrics emit. Most ticks
|
|
544
|
+
* the engine inventory is unchanged (no model load/unload), so
|
|
545
|
+
* we skip the bus emit and let the heartbeat re-emit at
|
|
546
|
+
* ENGINE_METRICS_HEARTBEAT_MS for liveness. */
|
|
547
|
+
private lastEmittedEngineSnapshot;
|
|
548
|
+
/**
|
|
549
|
+
* Last-applied tuning snapshot, taken at the end of `onInitialize`
|
|
550
|
+
* and refreshed at the end of `onConfigChanged`. Lets the change
|
|
551
|
+
* handler diff the current config against the snapshot and decide
|
|
552
|
+
* whether a pool respawn is necessary — most config edits (engine
|
|
553
|
+
* cascade, audio settings) don't touch the tuning fields and don't
|
|
554
|
+
* deserve a multi-second model reload.
|
|
555
|
+
*/
|
|
556
|
+
private lastAppliedPoolConfig;
|
|
557
|
+
/**
|
|
558
|
+
* Embedded Python path resolved once at boot via
|
|
559
|
+
* `ctx.deps.ensurePython()`. Empty string means the download failed
|
|
560
|
+
* or `runtime: 'node'` was selected — the provider's
|
|
561
|
+
* `ensureBackendDeps` and EngineFactory's `initPythonPool` raise a
|
|
562
|
+
* clear error in that case.
|
|
563
|
+
*/
|
|
564
|
+
private pythonPath;
|
|
565
|
+
/**
|
|
566
|
+
* Absolute path to the addon's bundled `python/` dir (resolved once
|
|
567
|
+
* at boot), passed through to the provider so per-backend
|
|
568
|
+
* `requirements-<runtime>.txt` files can be located at lazy-install
|
|
569
|
+
* time.
|
|
570
|
+
*/
|
|
571
|
+
private pythonAddonDir;
|
|
572
|
+
constructor();
|
|
573
|
+
protected globalSettingsSchema(): _camstack_types.ConfigUISchema;
|
|
574
|
+
/**
|
|
575
|
+
* Override to inject dynamic backend + device options based on the
|
|
576
|
+
* currently-stored runtime + backend. The base schema ships with
|
|
577
|
+
* placeholder lists; here we replace them with the valid subset for
|
|
578
|
+
* whatever the operator has picked. Combined with `immediate: true`
|
|
579
|
+
* on the selects, the UI can refetch after each change and see the
|
|
580
|
+
* next level's options narrow down.
|
|
581
|
+
*/
|
|
582
|
+
getGlobalSettings(overlay?: Record<string, unknown>): Promise<ConfigUISchemaWithValues>;
|
|
583
|
+
/**
|
|
584
|
+
* Ask the platform-probe cap which backends are actually installable
|
|
585
|
+
* on this node (checks for `openvino`, `coremltools`, `onnxruntime`
|
|
586
|
+
* Python modules + hardware presence). Returns a subset of the static
|
|
587
|
+
* catalog; an empty list signals the probe wasn't available so the
|
|
588
|
+
* caller should fall back to the full catalog rather than hiding
|
|
589
|
+
* every option.
|
|
590
|
+
*/
|
|
591
|
+
private resolveAvailableBackends;
|
|
592
|
+
/**
|
|
593
|
+
* Resolve the effective pool tuning for the configured backend.
|
|
594
|
+
*
|
|
595
|
+
* Reads `TUNING_DEFAULTS[backend]` and ignores any persisted override
|
|
596
|
+
* for `concurrency / batchMode / windowMs / maxBatchSize / numStreams /
|
|
597
|
+
* intraOpThreads`. Stored values are quietly discarded so an old
|
|
598
|
+
* suboptimal user-override (saved when the UI exposed these knobs)
|
|
599
|
+
* cannot resurrect itself after a restart.
|
|
600
|
+
*
|
|
601
|
+
* The matrix sweep (`bench-pool-matrix.py` + `bench-coreml-patterns.py`)
|
|
602
|
+
* already chose the best per-backend combo; the operator no longer has
|
|
603
|
+
* a reason to disagree.
|
|
604
|
+
*/
|
|
605
|
+
private resolveBackendTuning;
|
|
606
|
+
protected onInitialize(): Promise<ProviderRegistration[]>;
|
|
607
|
+
/**
|
|
608
|
+
* Emit a single `pipeline.engine-metrics-snapshot` event with the
|
|
609
|
+
* current engine inventory for this node. Source of truth is the
|
|
610
|
+
* provider's `listLoadedEngines` method (same data the cap returns).
|
|
611
|
+
*/
|
|
612
|
+
private emitEngineMetricsSnapshot;
|
|
613
|
+
getModelCatalog(): ModelCatalogEntry[];
|
|
614
|
+
protected onShutdown(): Promise<void>;
|
|
615
|
+
/**
|
|
616
|
+
* Snapshot the pool-bound subset of the EFFECTIVE tuning (post-
|
|
617
|
+
* `resolveBackendTuning`). Stored config values for these fields are
|
|
618
|
+
* ignored, so this snapshot only changes when `engineBackend` flips
|
|
619
|
+
* onto a different `TUNING_DEFAULTS` row.
|
|
620
|
+
*/
|
|
621
|
+
private snapshotPoolConfig;
|
|
622
|
+
private poolConfigChanged;
|
|
623
|
+
/**
|
|
624
|
+
* BaseAddon calls `onConfigChanged` after every settings write.
|
|
625
|
+
* Group-runner addons can't honour the schema's `requiresRestart:
|
|
626
|
+
* true` flag (the restart cap returns "process not found" for
|
|
627
|
+
* processes spawned in a kernel group worker). To make tuning
|
|
628
|
+
* changes actually take effect, watch the pool-bound subset and
|
|
629
|
+
* respawn the provider in place when it flips.
|
|
630
|
+
*
|
|
631
|
+
* Engine cascade (`engineRuntime/Backend/Device`) and audio
|
|
632
|
+
* settings don't require this — those still rely on the addon
|
|
633
|
+
* lifecycle (engineFactory rebuild on next runPipeline).
|
|
634
|
+
*/
|
|
635
|
+
protected onConfigChanged(): Promise<void>;
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
export { ALL_STEPS, DetectionPipelineProvider, backendToFormat, DetectionPipelineAddon as default, getDefaultModelForFormat, getStepDefinition };
|