@camstack/addon-provider-rtsp 1.2.59 → 1.2.60

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/addon.js +391 -24
  2. package/dist/addon.mjs +391 -24
  3. package/package.json +1 -1
package/dist/addon.js CHANGED
@@ -12868,6 +12868,35 @@ var deviceProviderCapability = {
12868
12868
  name: string(),
12869
12869
  type: string()
12870
12870
  }))),
12871
+ /**
12872
+ * Tear down and reconstruct ONE device in place from its persisted rows —
12873
+ * touching no other device this provider owns.
12874
+ *
12875
+ * The primitive `deviceManager.migrateDevice` uses to flush the two
12876
+ * migrated numbers: after `swapIds` the runner's live instance still
12877
+ * carries the PRE-swap numeric id (baked into the object, its native-cap
12878
+ * registrations and its log tags), and a live object cannot be renumbered.
12879
+ * Before this method the only flush was restarting the whole owning addon
12880
+ * — which took every camera the provider owns down with it (28 devices
12881
+ * for one migrated camera, measured 2026-09-04, and the morning of the
12882
+ * same day ~27 devices' native caps did not come back on their own).
12883
+ *
12884
+ * Keyed by `stableId`, deliberately: the numeric id is exactly the thing
12885
+ * that changes. The reply carries the id the device answers on NOW.
12886
+ * Implemented once in `BaseDeviceProvider` — decommission the live
12887
+ * instance (if any), then re-create from the persisted row: the same
12888
+ * teardown/rehydrate pair every graceful shutdown + boot already uses.
12889
+ * An RPC, never an event: a dropped event would leave the runner writing
12890
+ * against the wrong camera (D8).
12891
+ *
12892
+ * Construction can dial hardware, and the migrated source is
12893
+ * characteristically dead — the timeout covers a full activate window
12894
+ * rather than the 60 s default.
12895
+ */
12896
+ reloadDevice: method(object({ stableId: string() }), object({ deviceId: number() }), {
12897
+ kind: "mutation",
12898
+ timeoutMs: 3 * 6e4
12899
+ }),
12871
12900
  supportsDiscovery: method(object({}), boolean()),
12872
12901
  /**
12873
12902
  * Run a network scan. `params` carries optional provider-specific scan
@@ -13195,7 +13224,8 @@ method(object({
13195
13224
  targetId: number()
13196
13225
  }), MigrateDeviceResultSchema, {
13197
13226
  kind: "mutation",
13198
- auth: "admin"
13227
+ auth: "admin",
13228
+ timeoutMs: 12 * 6e4
13199
13229
  }), method(DeviceRegisterPayloadSchema, _void(), { kind: "mutation" }), method(DeviceRemovePayloadSchema, _void(), { kind: "mutation" }), method(DevicePersistConfigPayloadSchema, _void(), { kind: "mutation" }), method(object({ deviceId: number() }), record(string(), unknown())), method(object({ deviceId: number() }), record(string(), unknown())), method(object({ deviceId: number() }), DeviceMetaSchema.nullable()), method(object({
13200
13230
  deviceId: number(),
13201
13231
  name: string()
@@ -32663,6 +32693,147 @@ var BaseDevice = class {
32663
32693
  }
32664
32694
  };
32665
32695
  /**
32696
+ * Delays before retry rounds 1..N — the round count IS the bound.
32697
+ * 10 s catches "the hub was busy for a moment"; the full schedule
32698
+ * (10 + 30 + 90 s of waiting, plus up to one 60 s transport timeout
32699
+ * per attempt) covers a device-manager lock held for minutes — the
32700
+ * 2026-09-04 outage's migration hold was ~3.5 min.
32701
+ */
32702
+ var DEVICE_RESTORE_RETRY_DELAYS_MS = [
32703
+ 1e4,
32704
+ 3e4,
32705
+ 9e4
32706
+ ];
32707
+ /** Abortable sleep — resolves early (never rejects) on abort. */
32708
+ function sleep$1(ms, signal) {
32709
+ return new Promise((resolve) => {
32710
+ if (signal.aborted) {
32711
+ resolve();
32712
+ return;
32713
+ }
32714
+ const onAbort = () => {
32715
+ clearTimeout(timer);
32716
+ resolve();
32717
+ };
32718
+ const timer = setTimeout(() => {
32719
+ signal.removeEventListener("abort", onAbort);
32720
+ resolve();
32721
+ }, ms);
32722
+ timer.unref?.();
32723
+ signal.addEventListener("abort", onAbort, { once: true });
32724
+ });
32725
+ }
32726
+ /** Drain `items` through at most `width` concurrent lanes. `fn` must
32727
+ * not reject (callers wrap their own try/catch). */
32728
+ async function runWithConcurrency(items, width, fn) {
32729
+ const queue = [...items];
32730
+ const laneCount = Math.max(1, Math.min(width, queue.length));
32731
+ const lane = async () => {
32732
+ for (;;) {
32733
+ const item = queue.shift();
32734
+ if (item === void 0) return;
32735
+ await fn(item);
32736
+ }
32737
+ };
32738
+ await Promise.all(Array.from({ length: laneCount }, lane));
32739
+ }
32740
+ var DeviceRestoreRetryScheduler = class {
32741
+ #logger;
32742
+ #attempt;
32743
+ #onPermanentFailure;
32744
+ #delaysMs;
32745
+ #concurrency;
32746
+ #now;
32747
+ #abort = new AbortController();
32748
+ constructor(options) {
32749
+ this.#logger = options.logger;
32750
+ this.#attempt = options.attempt;
32751
+ this.#onPermanentFailure = options.onPermanentFailure;
32752
+ this.#delaysMs = options.delaysMs ?? DEVICE_RESTORE_RETRY_DELAYS_MS;
32753
+ this.#concurrency = options.concurrency ?? 4;
32754
+ this.#now = options.now ?? Date.now;
32755
+ }
32756
+ /** Stop retrying (shutdown). Pending entries are NOT marked
32757
+ * permanently failed — the next boot restores them from disk. */
32758
+ cancel() {
32759
+ this.#abort.abort();
32760
+ }
32761
+ /**
32762
+ * Run the bounded retry rounds. Resolves when every entry has either
32763
+ * restored, been marked permanently failed, or the scheduler was
32764
+ * cancelled. Never rejects.
32765
+ */
32766
+ async run(initialFailures) {
32767
+ let pending = initialFailures.map((failure) => ({
32768
+ saved: failure.saved,
32769
+ lastError: failure.error,
32770
+ attempts: 1
32771
+ }));
32772
+ for (let round = 0; round < this.#delaysMs.length; round += 1) {
32773
+ if (pending.length === 0 || this.#abort.signal.aborted) break;
32774
+ await sleep$1(this.#delaysMs[round] ?? 0, this.#abort.signal);
32775
+ if (this.#abort.signal.aborted) break;
32776
+ pending = await this.#runRound(pending, round);
32777
+ }
32778
+ if (this.#abort.signal.aborted) return [];
32779
+ const terminal = pending.map((entry) => ({
32780
+ deviceId: entry.saved.id,
32781
+ stableId: entry.saved.stableId,
32782
+ type: String(entry.saved.type),
32783
+ attempts: entry.attempts,
32784
+ lastError: entry.lastError,
32785
+ failedAt: this.#now()
32786
+ }));
32787
+ for (const failure of terminal) this.#onPermanentFailure(failure);
32788
+ return terminal;
32789
+ }
32790
+ /** One retry round: parents first (phase 0), then hub-adopted
32791
+ * children (phase 1) — a child's attempt depends on its parent
32792
+ * having landed, exactly like the initial two-pass restore. */
32793
+ async #runRound(pending, round) {
32794
+ const next = [];
32795
+ const parents = pending.filter((entry) => entry.saved.parentDeviceId === null);
32796
+ const children = pending.filter((entry) => entry.saved.parentDeviceId !== null);
32797
+ for (const phase of [parents, children]) await runWithConcurrency(phase, this.#concurrency, async (entry) => {
32798
+ if (this.#abort.signal.aborted) {
32799
+ next.push(entry);
32800
+ return;
32801
+ }
32802
+ const attemptNo = entry.attempts + 1;
32803
+ try {
32804
+ await this.#attempt(entry.saved);
32805
+ this.#logger.info("Device restored on retry", {
32806
+ tags: {
32807
+ deviceId: entry.saved.id,
32808
+ stableId: entry.saved.stableId
32809
+ },
32810
+ meta: { attempt: attemptNo }
32811
+ });
32812
+ } catch (err) {
32813
+ const lastError = err instanceof Error ? err.message : String(err);
32814
+ const remainingRetries = this.#delaysMs.length - (round + 1);
32815
+ this.#logger.warn("Device restore retry failed", {
32816
+ tags: {
32817
+ deviceId: entry.saved.id,
32818
+ stableId: entry.saved.stableId
32819
+ },
32820
+ meta: {
32821
+ attempt: attemptNo,
32822
+ remainingRetries,
32823
+ error: lastError
32824
+ }
32825
+ });
32826
+ next.push({
32827
+ saved: entry.saved,
32828
+ lastError,
32829
+ attempts: attemptNo
32830
+ });
32831
+ }
32832
+ });
32833
+ return next;
32834
+ }
32835
+ };
32836
+ /**
32666
32837
  * Convert an IDevice to the flat DeviceSummary shape expected by the
32667
32838
  * device-provider cap router. Shared across all providers.
32668
32839
  */
@@ -32711,6 +32882,7 @@ var BaseDeviceProvider = class extends BaseAddon {
32711
32882
  }];
32712
32883
  }
32713
32884
  async onShutdown() {
32885
+ this.cancelRestoreRetries();
32714
32886
  const devices = await this.ctx.kernel.devices?.getAll() ?? [];
32715
32887
  for (const device of devices) try {
32716
32888
  await this.ctx.kernel.devices?.decommission(device.id);
@@ -32728,9 +32900,16 @@ var BaseDeviceProvider = class extends BaseAddon {
32728
32900
  async start() {}
32729
32901
  async stop() {}
32730
32902
  async getStatus() {
32903
+ const all = await this.ctx.kernel.devices?.getAll() ?? [];
32904
+ const summary = this.restoreFailureSummary();
32905
+ if (summary === null) return {
32906
+ connected: true,
32907
+ deviceCount: all.length
32908
+ };
32731
32909
  return {
32732
32910
  connected: true,
32733
- deviceCount: (await this.ctx.kernel.devices?.getAll() ?? []).length
32911
+ deviceCount: all.length,
32912
+ error: summary
32734
32913
  };
32735
32914
  }
32736
32915
  async getDevices() {
@@ -32820,8 +32999,137 @@ var BaseDeviceProvider = class extends BaseAddon {
32820
32999
  };
32821
33000
  }
32822
33001
  async restoreDevices(savedDevices) {
32823
- await this.onRestoreDevices(savedDevices);
32824
- if (savedDevices.length > 0) this.ctx.logger.info(`Restored ${savedDevices.length} ${this.providerName} device(s)`);
33002
+ const report = await this.onRestoreDevices(savedDevices);
33003
+ if (savedDevices.length === 0) return;
33004
+ if (report && report.failedCount > 0) {
33005
+ this.ctx.logger.warn(`Restored ${report.restoredCount}/${savedDevices.length} ${this.providerName} device(s) — ${report.failedCount} failed, bounded retry scheduled`);
33006
+ return;
33007
+ }
33008
+ const restoredCount = report ? report.restoredCount : savedDevices.length;
33009
+ this.ctx.logger.info(`Restored ${restoredCount} ${this.providerName} device(s)`);
33010
+ }
33011
+ /** Retry schedule. Overridable (tests use millisecond delays). */
33012
+ restoreRetryDelaysMs = DEVICE_RESTORE_RETRY_DELAYS_MS;
33013
+ /** Retry lane width. See `device-restore-retry.ts` for why retries
33014
+ * never re-stampede full-width while the initial pass does (D167). */
33015
+ restoreRetryConcurrency = 4;
33016
+ _restoreRetryScheduler = null;
33017
+ _restoreRetryCompletion = null;
33018
+ _permanentRestoreFailures = /* @__PURE__ */ new Map();
33019
+ /** Settles when the background retry rounds finish (or `null` when
33020
+ * nothing failed). Exposed for tests and subclass diagnostics —
33021
+ * boot NEVER awaits this: the runner's post-init handshake goes out
33022
+ * with the devices that restored, and a late success is announced
33023
+ * through the `native-cap-change` → `updateCaps` path. */
33024
+ get restoreRetryCompletion() {
33025
+ return this._restoreRetryCompletion;
33026
+ }
33027
+ /** Devices that exhausted the retry bound this process lifetime. */
33028
+ get permanentRestoreFailures() {
33029
+ return [...this._permanentRestoreFailures.values()];
33030
+ }
33031
+ /** One-line operator-facing summary for `getStatus().error`, or
33032
+ * `null` when every device restored. */
33033
+ restoreFailureSummary() {
33034
+ if (this._permanentRestoreFailures.size === 0) return null;
33035
+ const ids = [...this._permanentRestoreFailures.keys()].join(", ");
33036
+ return `${this._permanentRestoreFailures.size} device(s) permanently failed restore (deviceIds: ${ids}) — restart the ${this.providerName} provider to retry`;
33037
+ }
33038
+ cancelRestoreRetries() {
33039
+ this._restoreRetryScheduler?.cancel();
33040
+ this._restoreRetryScheduler = null;
33041
+ }
33042
+ recordPermanentRestoreFailure(failure) {
33043
+ this._permanentRestoreFailures.set(failure.deviceId, failure);
33044
+ this.ctx.logger.error("Device restore permanently failed — its capabilities will not register until the provider restarts", {
33045
+ tags: {
33046
+ deviceId: failure.deviceId,
33047
+ stableId: failure.stableId
33048
+ },
33049
+ meta: {
33050
+ type: failure.type,
33051
+ attempts: failure.attempts,
33052
+ error: failure.lastError
33053
+ }
33054
+ });
33055
+ }
33056
+ scheduleRestoreRetries(failures, attempt) {
33057
+ const scheduler = new DeviceRestoreRetryScheduler({
33058
+ logger: this.ctx.logger,
33059
+ delaysMs: this.restoreRetryDelaysMs,
33060
+ concurrency: this.restoreRetryConcurrency,
33061
+ attempt,
33062
+ onPermanentFailure: (failure) => this.recordPermanentRestoreFailure(failure)
33063
+ });
33064
+ this._restoreRetryScheduler = scheduler;
33065
+ this._restoreRetryCompletion = scheduler.run(failures).then(() => void 0).catch((err) => {
33066
+ this.ctx.logger.error("Restore retry scheduler crashed", { meta: { error: err instanceof Error ? err.message : String(err) } });
33067
+ });
33068
+ }
33069
+ /**
33070
+ * Tear down and reconstruct ONE device from its persisted rows — the
33071
+ * `deviceProvider.reloadDevice` cap method. Persistence is never touched,
33072
+ * and no other device this provider owns is disturbed.
33073
+ *
33074
+ * Keyed by `stableId` because the caller's whole reason to be here is that
33075
+ * the NUMERIC id changed (`deviceManager.migrateDevice` swapped it): the
33076
+ * fresh instance resolves its id through `allocateDeviceId`, which returns
33077
+ * whatever number the row carries NOW. The teardown is `decommission` —
33078
+ * exactly what a graceful shutdown runs per device (fires `removeDevice()`,
33079
+ * unregisters native caps, drops the registry entry) — and the rebuild is
33080
+ * the boot restore's own `create()` path, including its pass 2: first-class
33081
+ * children (hub-adopted cameras under an NVR) are decommissioned with the
33082
+ * parent by the cascade and must be re-created explicitly, because only
33083
+ * accessory children come back through `getAccessoryChildren()`.
33084
+ *
33085
+ * Reloading an accessory child directly is refused (no device class) —
33086
+ * reload its parent instead.
33087
+ */
33088
+ async reloadDevice(input) {
33089
+ const { stableId } = input;
33090
+ const devices = this.ctx.kernel.devices;
33091
+ if (!devices) throw new Error(`${this.providerName}: kernel.devices unavailable — cannot reload`);
33092
+ const live = (await devices.getAll()).find((d) => d.stableId === stableId);
33093
+ if (live) await devices.decommission(live.id);
33094
+ const { id } = await this.ctx.api.deviceManager.allocateDeviceId.mutate({
33095
+ addonId: this.addonId,
33096
+ stableId
33097
+ });
33098
+ const meta = await this.ctx.api.deviceManager.loadMeta.query({ deviceId: id });
33099
+ if (meta === null) throw new Error(`${this.providerName}: no persisted meta for "${stableId}" (id ${id}) — cannot reload`);
33100
+ const deviceType = Object.values(DeviceType).find((t) => t === meta.type);
33101
+ const Class = deviceType !== void 0 ? this.deviceClasses[deviceType] : void 0;
33102
+ if (!Class) throw new Error(`${this.providerName}: no device class for type "${meta.type}" — "${stableId}" is an accessory child; reload its parent instead`);
33103
+ await devices.create(stableId, Class, {}, meta.parentDeviceId ?? null);
33104
+ const rows = await this.ctx.api.deviceManager.listPersistedByAddon.query({ addonId: this.addonId });
33105
+ for (const row of rows) {
33106
+ if (row.parentDeviceId !== id) continue;
33107
+ const childType = Object.values(DeviceType).find((t) => t === row.type);
33108
+ const ChildClass = childType !== void 0 ? this.deviceClasses[childType] : void 0;
33109
+ if (!ChildClass) continue;
33110
+ try {
33111
+ await devices.create(row.stableId, ChildClass, {}, id);
33112
+ } catch (err) {
33113
+ this.ctx.logger.warn("reloadDevice: failed to re-create first-class child", {
33114
+ tags: {
33115
+ deviceId: row.id,
33116
+ stableId: row.stableId
33117
+ },
33118
+ meta: {
33119
+ parentDeviceId: id,
33120
+ error: err instanceof Error ? err.message : String(err)
33121
+ }
33122
+ });
33123
+ }
33124
+ }
33125
+ this.ctx.logger.info("device reloaded in place from persisted rows", {
33126
+ tags: { deviceId: id },
33127
+ meta: {
33128
+ stableId,
33129
+ type: meta.type
33130
+ }
33131
+ });
33132
+ return { deviceId: id };
32825
33133
  }
32826
33134
  /**
32827
33135
  * Restore devices from persisted state. Two-pass:
@@ -32847,55 +33155,108 @@ var BaseDeviceProvider = class extends BaseAddon {
32847
33155
  * accessory-spawn flow handles via the parent's
32848
33156
  * `getAccessoryChildren()`. Override only when the default doesn't
32849
33157
  * fit.
33158
+ *
33159
+ * A row that fails either pass is NOT terminal (D347): it is handed
33160
+ * to a bounded background retry (`DeviceRestoreRetryScheduler`).
33161
+ * Only after the bound is exhausted is the device marked permanently
33162
+ * failed — logged at ERROR with `tags.deviceId` and surfaced via
33163
+ * `getStatus().error`.
32850
33164
  */
32851
33165
  async onRestoreDevices(savedDevices) {
32852
33166
  const restored = /* @__PURE__ */ new Set();
33167
+ const failures = [];
33168
+ const attemptRestore = async (saved) => {
33169
+ if (restored.has(saved.id)) return;
33170
+ const Class = this.deviceClasses[saved.type];
33171
+ if (!Class) throw new Error(`no device class registered for type "${saved.type}"`);
33172
+ if (saved.parentDeviceId !== null && !restored.has(saved.parentDeviceId)) throw new Error(`parent device ${saved.parentDeviceId} not restored`);
33173
+ await this.ctx.kernel.devices.create(saved.stableId, Class, {}, saved.parentDeviceId);
33174
+ restored.add(saved.id);
33175
+ };
32853
33176
  const topLevel = savedDevices.filter((saved) => saved.parentDeviceId === null);
32854
33177
  const restoreOne = async (saved) => {
32855
- const Class = this.deviceClasses[saved.type];
32856
- if (!Class) {
33178
+ if (!this.deviceClasses[saved.type]) {
32857
33179
  this.ctx.logger.warn("No device class registered for restored type — skipping", {
32858
- tags: { stableId: saved.stableId },
33180
+ tags: {
33181
+ deviceId: saved.id,
33182
+ stableId: saved.stableId
33183
+ },
32859
33184
  meta: { type: saved.type }
32860
33185
  });
32861
33186
  return;
32862
33187
  }
32863
33188
  try {
32864
- await this.ctx.kernel.devices.create(saved.stableId, Class, {});
32865
- restored.add(saved.id);
33189
+ await attemptRestore(saved);
32866
33190
  } catch (err) {
32867
- this.ctx.logger.warn("Failed to restore device", {
32868
- tags: { stableId: saved.stableId },
33191
+ const error = err instanceof Error ? err.message : String(err);
33192
+ this.ctx.logger.warn("Failed to restore device — bounded retry scheduled", {
33193
+ tags: {
33194
+ deviceId: saved.id,
33195
+ stableId: saved.stableId
33196
+ },
32869
33197
  meta: {
32870
33198
  type: saved.type,
32871
- error: err instanceof Error ? err.message : String(err)
33199
+ attempt: 1,
33200
+ error
32872
33201
  }
32873
33202
  });
33203
+ failures.push({
33204
+ saved,
33205
+ error
33206
+ });
32874
33207
  }
32875
33208
  };
32876
33209
  await Promise.all(topLevel.map((saved) => restoreOne(saved)));
33210
+ const failedTopLevelIds = new Set(failures.map((failure) => failure.saved.id));
32877
33211
  const childRows = savedDevices.filter((s) => s.parentDeviceId !== null);
32878
33212
  for (const saved of childRows) {
32879
- const Class = this.deviceClasses[saved.type];
32880
- if (!Class) continue;
33213
+ if (!this.deviceClasses[saved.type]) continue;
32881
33214
  if (saved.parentDeviceId === null) continue;
32882
- if (!restored.has(saved.parentDeviceId)) continue;
32883
- try {
32884
- await this.ctx.kernel.devices.create(saved.stableId, Class, {}, saved.parentDeviceId);
32885
- restored.add(saved.id);
32886
- } catch (err) {
32887
- this.ctx.logger.warn("Failed to restore hub-adopted child", {
33215
+ if (restored.has(saved.parentDeviceId)) {
33216
+ try {
33217
+ await attemptRestore(saved);
33218
+ } catch (err) {
33219
+ const error = err instanceof Error ? err.message : String(err);
33220
+ this.ctx.logger.warn("Failed to restore hub-adopted child — bounded retry scheduled", {
33221
+ tags: {
33222
+ deviceId: saved.id,
33223
+ stableId: saved.stableId,
33224
+ parentDeviceId: saved.parentDeviceId
33225
+ },
33226
+ meta: {
33227
+ type: saved.type,
33228
+ attempt: 1,
33229
+ error
33230
+ }
33231
+ });
33232
+ failures.push({
33233
+ saved,
33234
+ error
33235
+ });
33236
+ }
33237
+ continue;
33238
+ }
33239
+ if (failedTopLevelIds.has(saved.parentDeviceId)) {
33240
+ this.ctx.logger.warn("Hub-adopted child deferred — parent failed initial restore", {
32888
33241
  tags: {
33242
+ deviceId: saved.id,
32889
33243
  stableId: saved.stableId,
32890
33244
  parentDeviceId: saved.parentDeviceId
32891
33245
  },
32892
- meta: {
32893
- type: saved.type,
32894
- error: err instanceof Error ? err.message : String(err)
32895
- }
33246
+ meta: { type: saved.type }
32896
33247
  });
33248
+ failures.push({
33249
+ saved,
33250
+ error: `parent device ${saved.parentDeviceId} not restored`
33251
+ });
33252
+ continue;
32897
33253
  }
32898
33254
  }
33255
+ if (failures.length > 0) this.scheduleRestoreRetries(failures, attemptRestore);
33256
+ return {
33257
+ restoredCount: restored.size,
33258
+ failedCount: failures.length
33259
+ };
32899
33260
  }
32900
33261
  /** Convert an IDevice to the flat DeviceSummary for the cap router. */
32901
33262
  toSummary(device) {
@@ -34724,6 +35085,12 @@ Object.freeze({
34724
35085
  addonId: null,
34725
35086
  access: "view"
34726
35087
  },
35088
+ "deviceProvider.reloadDevice": {
35089
+ capName: "device-provider",
35090
+ capScope: "system",
35091
+ addonId: null,
35092
+ access: "create"
35093
+ },
34727
35094
  "deviceProvider.start": {
34728
35095
  capName: "device-provider",
34729
35096
  capScope: "system",
package/dist/addon.mjs CHANGED
@@ -12844,6 +12844,35 @@ var deviceProviderCapability = {
12844
12844
  name: string(),
12845
12845
  type: string()
12846
12846
  }))),
12847
+ /**
12848
+ * Tear down and reconstruct ONE device in place from its persisted rows —
12849
+ * touching no other device this provider owns.
12850
+ *
12851
+ * The primitive `deviceManager.migrateDevice` uses to flush the two
12852
+ * migrated numbers: after `swapIds` the runner's live instance still
12853
+ * carries the PRE-swap numeric id (baked into the object, its native-cap
12854
+ * registrations and its log tags), and a live object cannot be renumbered.
12855
+ * Before this method the only flush was restarting the whole owning addon
12856
+ * — which took every camera the provider owns down with it (28 devices
12857
+ * for one migrated camera, measured 2026-09-04, and the morning of the
12858
+ * same day ~27 devices' native caps did not come back on their own).
12859
+ *
12860
+ * Keyed by `stableId`, deliberately: the numeric id is exactly the thing
12861
+ * that changes. The reply carries the id the device answers on NOW.
12862
+ * Implemented once in `BaseDeviceProvider` — decommission the live
12863
+ * instance (if any), then re-create from the persisted row: the same
12864
+ * teardown/rehydrate pair every graceful shutdown + boot already uses.
12865
+ * An RPC, never an event: a dropped event would leave the runner writing
12866
+ * against the wrong camera (D8).
12867
+ *
12868
+ * Construction can dial hardware, and the migrated source is
12869
+ * characteristically dead — the timeout covers a full activate window
12870
+ * rather than the 60 s default.
12871
+ */
12872
+ reloadDevice: method(object({ stableId: string() }), object({ deviceId: number() }), {
12873
+ kind: "mutation",
12874
+ timeoutMs: 3 * 6e4
12875
+ }),
12847
12876
  supportsDiscovery: method(object({}), boolean()),
12848
12877
  /**
12849
12878
  * Run a network scan. `params` carries optional provider-specific scan
@@ -13171,7 +13200,8 @@ method(object({
13171
13200
  targetId: number()
13172
13201
  }), MigrateDeviceResultSchema, {
13173
13202
  kind: "mutation",
13174
- auth: "admin"
13203
+ auth: "admin",
13204
+ timeoutMs: 12 * 6e4
13175
13205
  }), method(DeviceRegisterPayloadSchema, _void(), { kind: "mutation" }), method(DeviceRemovePayloadSchema, _void(), { kind: "mutation" }), method(DevicePersistConfigPayloadSchema, _void(), { kind: "mutation" }), method(object({ deviceId: number() }), record(string(), unknown())), method(object({ deviceId: number() }), record(string(), unknown())), method(object({ deviceId: number() }), DeviceMetaSchema.nullable()), method(object({
13176
13206
  deviceId: number(),
13177
13207
  name: string()
@@ -32639,6 +32669,147 @@ var BaseDevice = class {
32639
32669
  }
32640
32670
  };
32641
32671
  /**
32672
+ * Delays before retry rounds 1..N — the round count IS the bound.
32673
+ * 10 s catches "the hub was busy for a moment"; the full schedule
32674
+ * (10 + 30 + 90 s of waiting, plus up to one 60 s transport timeout
32675
+ * per attempt) covers a device-manager lock held for minutes — the
32676
+ * 2026-09-04 outage's migration hold was ~3.5 min.
32677
+ */
32678
+ var DEVICE_RESTORE_RETRY_DELAYS_MS = [
32679
+ 1e4,
32680
+ 3e4,
32681
+ 9e4
32682
+ ];
32683
+ /** Abortable sleep — resolves early (never rejects) on abort. */
32684
+ function sleep$1(ms, signal) {
32685
+ return new Promise((resolve) => {
32686
+ if (signal.aborted) {
32687
+ resolve();
32688
+ return;
32689
+ }
32690
+ const onAbort = () => {
32691
+ clearTimeout(timer);
32692
+ resolve();
32693
+ };
32694
+ const timer = setTimeout(() => {
32695
+ signal.removeEventListener("abort", onAbort);
32696
+ resolve();
32697
+ }, ms);
32698
+ timer.unref?.();
32699
+ signal.addEventListener("abort", onAbort, { once: true });
32700
+ });
32701
+ }
32702
+ /** Drain `items` through at most `width` concurrent lanes. `fn` must
32703
+ * not reject (callers wrap their own try/catch). */
32704
+ async function runWithConcurrency(items, width, fn) {
32705
+ const queue = [...items];
32706
+ const laneCount = Math.max(1, Math.min(width, queue.length));
32707
+ const lane = async () => {
32708
+ for (;;) {
32709
+ const item = queue.shift();
32710
+ if (item === void 0) return;
32711
+ await fn(item);
32712
+ }
32713
+ };
32714
+ await Promise.all(Array.from({ length: laneCount }, lane));
32715
+ }
32716
+ var DeviceRestoreRetryScheduler = class {
32717
+ #logger;
32718
+ #attempt;
32719
+ #onPermanentFailure;
32720
+ #delaysMs;
32721
+ #concurrency;
32722
+ #now;
32723
+ #abort = new AbortController();
32724
+ constructor(options) {
32725
+ this.#logger = options.logger;
32726
+ this.#attempt = options.attempt;
32727
+ this.#onPermanentFailure = options.onPermanentFailure;
32728
+ this.#delaysMs = options.delaysMs ?? DEVICE_RESTORE_RETRY_DELAYS_MS;
32729
+ this.#concurrency = options.concurrency ?? 4;
32730
+ this.#now = options.now ?? Date.now;
32731
+ }
32732
+ /** Stop retrying (shutdown). Pending entries are NOT marked
32733
+ * permanently failed — the next boot restores them from disk. */
32734
+ cancel() {
32735
+ this.#abort.abort();
32736
+ }
32737
+ /**
32738
+ * Run the bounded retry rounds. Resolves when every entry has either
32739
+ * restored, been marked permanently failed, or the scheduler was
32740
+ * cancelled. Never rejects.
32741
+ */
32742
+ async run(initialFailures) {
32743
+ let pending = initialFailures.map((failure) => ({
32744
+ saved: failure.saved,
32745
+ lastError: failure.error,
32746
+ attempts: 1
32747
+ }));
32748
+ for (let round = 0; round < this.#delaysMs.length; round += 1) {
32749
+ if (pending.length === 0 || this.#abort.signal.aborted) break;
32750
+ await sleep$1(this.#delaysMs[round] ?? 0, this.#abort.signal);
32751
+ if (this.#abort.signal.aborted) break;
32752
+ pending = await this.#runRound(pending, round);
32753
+ }
32754
+ if (this.#abort.signal.aborted) return [];
32755
+ const terminal = pending.map((entry) => ({
32756
+ deviceId: entry.saved.id,
32757
+ stableId: entry.saved.stableId,
32758
+ type: String(entry.saved.type),
32759
+ attempts: entry.attempts,
32760
+ lastError: entry.lastError,
32761
+ failedAt: this.#now()
32762
+ }));
32763
+ for (const failure of terminal) this.#onPermanentFailure(failure);
32764
+ return terminal;
32765
+ }
32766
+ /** One retry round: parents first (phase 0), then hub-adopted
32767
+ * children (phase 1) — a child's attempt depends on its parent
32768
+ * having landed, exactly like the initial two-pass restore. */
32769
+ async #runRound(pending, round) {
32770
+ const next = [];
32771
+ const parents = pending.filter((entry) => entry.saved.parentDeviceId === null);
32772
+ const children = pending.filter((entry) => entry.saved.parentDeviceId !== null);
32773
+ for (const phase of [parents, children]) await runWithConcurrency(phase, this.#concurrency, async (entry) => {
32774
+ if (this.#abort.signal.aborted) {
32775
+ next.push(entry);
32776
+ return;
32777
+ }
32778
+ const attemptNo = entry.attempts + 1;
32779
+ try {
32780
+ await this.#attempt(entry.saved);
32781
+ this.#logger.info("Device restored on retry", {
32782
+ tags: {
32783
+ deviceId: entry.saved.id,
32784
+ stableId: entry.saved.stableId
32785
+ },
32786
+ meta: { attempt: attemptNo }
32787
+ });
32788
+ } catch (err) {
32789
+ const lastError = err instanceof Error ? err.message : String(err);
32790
+ const remainingRetries = this.#delaysMs.length - (round + 1);
32791
+ this.#logger.warn("Device restore retry failed", {
32792
+ tags: {
32793
+ deviceId: entry.saved.id,
32794
+ stableId: entry.saved.stableId
32795
+ },
32796
+ meta: {
32797
+ attempt: attemptNo,
32798
+ remainingRetries,
32799
+ error: lastError
32800
+ }
32801
+ });
32802
+ next.push({
32803
+ saved: entry.saved,
32804
+ lastError,
32805
+ attempts: attemptNo
32806
+ });
32807
+ }
32808
+ });
32809
+ return next;
32810
+ }
32811
+ };
32812
+ /**
32642
32813
  * Convert an IDevice to the flat DeviceSummary shape expected by the
32643
32814
  * device-provider cap router. Shared across all providers.
32644
32815
  */
@@ -32687,6 +32858,7 @@ var BaseDeviceProvider = class extends BaseAddon {
32687
32858
  }];
32688
32859
  }
32689
32860
  async onShutdown() {
32861
+ this.cancelRestoreRetries();
32690
32862
  const devices = await this.ctx.kernel.devices?.getAll() ?? [];
32691
32863
  for (const device of devices) try {
32692
32864
  await this.ctx.kernel.devices?.decommission(device.id);
@@ -32704,9 +32876,16 @@ var BaseDeviceProvider = class extends BaseAddon {
32704
32876
  async start() {}
32705
32877
  async stop() {}
32706
32878
  async getStatus() {
32879
+ const all = await this.ctx.kernel.devices?.getAll() ?? [];
32880
+ const summary = this.restoreFailureSummary();
32881
+ if (summary === null) return {
32882
+ connected: true,
32883
+ deviceCount: all.length
32884
+ };
32707
32885
  return {
32708
32886
  connected: true,
32709
- deviceCount: (await this.ctx.kernel.devices?.getAll() ?? []).length
32887
+ deviceCount: all.length,
32888
+ error: summary
32710
32889
  };
32711
32890
  }
32712
32891
  async getDevices() {
@@ -32796,8 +32975,137 @@ var BaseDeviceProvider = class extends BaseAddon {
32796
32975
  };
32797
32976
  }
32798
32977
  async restoreDevices(savedDevices) {
32799
- await this.onRestoreDevices(savedDevices);
32800
- if (savedDevices.length > 0) this.ctx.logger.info(`Restored ${savedDevices.length} ${this.providerName} device(s)`);
32978
+ const report = await this.onRestoreDevices(savedDevices);
32979
+ if (savedDevices.length === 0) return;
32980
+ if (report && report.failedCount > 0) {
32981
+ this.ctx.logger.warn(`Restored ${report.restoredCount}/${savedDevices.length} ${this.providerName} device(s) — ${report.failedCount} failed, bounded retry scheduled`);
32982
+ return;
32983
+ }
32984
+ const restoredCount = report ? report.restoredCount : savedDevices.length;
32985
+ this.ctx.logger.info(`Restored ${restoredCount} ${this.providerName} device(s)`);
32986
+ }
32987
+ /** Retry schedule. Overridable (tests use millisecond delays). */
32988
+ restoreRetryDelaysMs = DEVICE_RESTORE_RETRY_DELAYS_MS;
32989
+ /** Retry lane width. See `device-restore-retry.ts` for why retries
32990
+ * never re-stampede full-width while the initial pass does (D167). */
32991
+ restoreRetryConcurrency = 4;
32992
+ _restoreRetryScheduler = null;
32993
+ _restoreRetryCompletion = null;
32994
+ _permanentRestoreFailures = /* @__PURE__ */ new Map();
32995
+ /** Settles when the background retry rounds finish (or `null` when
32996
+ * nothing failed). Exposed for tests and subclass diagnostics —
32997
+ * boot NEVER awaits this: the runner's post-init handshake goes out
32998
+ * with the devices that restored, and a late success is announced
32999
+ * through the `native-cap-change` → `updateCaps` path. */
33000
+ get restoreRetryCompletion() {
33001
+ return this._restoreRetryCompletion;
33002
+ }
33003
+ /** Devices that exhausted the retry bound this process lifetime. */
33004
+ get permanentRestoreFailures() {
33005
+ return [...this._permanentRestoreFailures.values()];
33006
+ }
33007
+ /** One-line operator-facing summary for `getStatus().error`, or
33008
+ * `null` when every device restored. */
33009
+ restoreFailureSummary() {
33010
+ if (this._permanentRestoreFailures.size === 0) return null;
33011
+ const ids = [...this._permanentRestoreFailures.keys()].join(", ");
33012
+ return `${this._permanentRestoreFailures.size} device(s) permanently failed restore (deviceIds: ${ids}) — restart the ${this.providerName} provider to retry`;
33013
+ }
33014
+ cancelRestoreRetries() {
33015
+ this._restoreRetryScheduler?.cancel();
33016
+ this._restoreRetryScheduler = null;
33017
+ }
33018
+ recordPermanentRestoreFailure(failure) {
33019
+ this._permanentRestoreFailures.set(failure.deviceId, failure);
33020
+ this.ctx.logger.error("Device restore permanently failed — its capabilities will not register until the provider restarts", {
33021
+ tags: {
33022
+ deviceId: failure.deviceId,
33023
+ stableId: failure.stableId
33024
+ },
33025
+ meta: {
33026
+ type: failure.type,
33027
+ attempts: failure.attempts,
33028
+ error: failure.lastError
33029
+ }
33030
+ });
33031
+ }
33032
+ scheduleRestoreRetries(failures, attempt) {
33033
+ const scheduler = new DeviceRestoreRetryScheduler({
33034
+ logger: this.ctx.logger,
33035
+ delaysMs: this.restoreRetryDelaysMs,
33036
+ concurrency: this.restoreRetryConcurrency,
33037
+ attempt,
33038
+ onPermanentFailure: (failure) => this.recordPermanentRestoreFailure(failure)
33039
+ });
33040
+ this._restoreRetryScheduler = scheduler;
33041
+ this._restoreRetryCompletion = scheduler.run(failures).then(() => void 0).catch((err) => {
33042
+ this.ctx.logger.error("Restore retry scheduler crashed", { meta: { error: err instanceof Error ? err.message : String(err) } });
33043
+ });
33044
+ }
33045
+ /**
33046
+ * Tear down and reconstruct ONE device from its persisted rows — the
33047
+ * `deviceProvider.reloadDevice` cap method. Persistence is never touched,
33048
+ * and no other device this provider owns is disturbed.
33049
+ *
33050
+ * Keyed by `stableId` because the caller's whole reason to be here is that
33051
+ * the NUMERIC id changed (`deviceManager.migrateDevice` swapped it): the
33052
+ * fresh instance resolves its id through `allocateDeviceId`, which returns
33053
+ * whatever number the row carries NOW. The teardown is `decommission` —
33054
+ * exactly what a graceful shutdown runs per device (fires `removeDevice()`,
33055
+ * unregisters native caps, drops the registry entry) — and the rebuild is
33056
+ * the boot restore's own `create()` path, including its pass 2: first-class
33057
+ * children (hub-adopted cameras under an NVR) are decommissioned with the
33058
+ * parent by the cascade and must be re-created explicitly, because only
33059
+ * accessory children come back through `getAccessoryChildren()`.
33060
+ *
33061
+ * Reloading an accessory child directly is refused (no device class) —
33062
+ * reload its parent instead.
33063
+ */
33064
+ async reloadDevice(input) {
33065
+ const { stableId } = input;
33066
+ const devices = this.ctx.kernel.devices;
33067
+ if (!devices) throw new Error(`${this.providerName}: kernel.devices unavailable — cannot reload`);
33068
+ const live = (await devices.getAll()).find((d) => d.stableId === stableId);
33069
+ if (live) await devices.decommission(live.id);
33070
+ const { id } = await this.ctx.api.deviceManager.allocateDeviceId.mutate({
33071
+ addonId: this.addonId,
33072
+ stableId
33073
+ });
33074
+ const meta = await this.ctx.api.deviceManager.loadMeta.query({ deviceId: id });
33075
+ if (meta === null) throw new Error(`${this.providerName}: no persisted meta for "${stableId}" (id ${id}) — cannot reload`);
33076
+ const deviceType = Object.values(DeviceType).find((t) => t === meta.type);
33077
+ const Class = deviceType !== void 0 ? this.deviceClasses[deviceType] : void 0;
33078
+ if (!Class) throw new Error(`${this.providerName}: no device class for type "${meta.type}" — "${stableId}" is an accessory child; reload its parent instead`);
33079
+ await devices.create(stableId, Class, {}, meta.parentDeviceId ?? null);
33080
+ const rows = await this.ctx.api.deviceManager.listPersistedByAddon.query({ addonId: this.addonId });
33081
+ for (const row of rows) {
33082
+ if (row.parentDeviceId !== id) continue;
33083
+ const childType = Object.values(DeviceType).find((t) => t === row.type);
33084
+ const ChildClass = childType !== void 0 ? this.deviceClasses[childType] : void 0;
33085
+ if (!ChildClass) continue;
33086
+ try {
33087
+ await devices.create(row.stableId, ChildClass, {}, id);
33088
+ } catch (err) {
33089
+ this.ctx.logger.warn("reloadDevice: failed to re-create first-class child", {
33090
+ tags: {
33091
+ deviceId: row.id,
33092
+ stableId: row.stableId
33093
+ },
33094
+ meta: {
33095
+ parentDeviceId: id,
33096
+ error: err instanceof Error ? err.message : String(err)
33097
+ }
33098
+ });
33099
+ }
33100
+ }
33101
+ this.ctx.logger.info("device reloaded in place from persisted rows", {
33102
+ tags: { deviceId: id },
33103
+ meta: {
33104
+ stableId,
33105
+ type: meta.type
33106
+ }
33107
+ });
33108
+ return { deviceId: id };
32801
33109
  }
32802
33110
  /**
32803
33111
  * Restore devices from persisted state. Two-pass:
@@ -32823,55 +33131,108 @@ var BaseDeviceProvider = class extends BaseAddon {
32823
33131
  * accessory-spawn flow handles via the parent's
32824
33132
  * `getAccessoryChildren()`. Override only when the default doesn't
32825
33133
  * fit.
33134
+ *
33135
+ * A row that fails either pass is NOT terminal (D347): it is handed
33136
+ * to a bounded background retry (`DeviceRestoreRetryScheduler`).
33137
+ * Only after the bound is exhausted is the device marked permanently
33138
+ * failed — logged at ERROR with `tags.deviceId` and surfaced via
33139
+ * `getStatus().error`.
32826
33140
  */
32827
33141
  async onRestoreDevices(savedDevices) {
32828
33142
  const restored = /* @__PURE__ */ new Set();
33143
+ const failures = [];
33144
+ const attemptRestore = async (saved) => {
33145
+ if (restored.has(saved.id)) return;
33146
+ const Class = this.deviceClasses[saved.type];
33147
+ if (!Class) throw new Error(`no device class registered for type "${saved.type}"`);
33148
+ if (saved.parentDeviceId !== null && !restored.has(saved.parentDeviceId)) throw new Error(`parent device ${saved.parentDeviceId} not restored`);
33149
+ await this.ctx.kernel.devices.create(saved.stableId, Class, {}, saved.parentDeviceId);
33150
+ restored.add(saved.id);
33151
+ };
32829
33152
  const topLevel = savedDevices.filter((saved) => saved.parentDeviceId === null);
32830
33153
  const restoreOne = async (saved) => {
32831
- const Class = this.deviceClasses[saved.type];
32832
- if (!Class) {
33154
+ if (!this.deviceClasses[saved.type]) {
32833
33155
  this.ctx.logger.warn("No device class registered for restored type — skipping", {
32834
- tags: { stableId: saved.stableId },
33156
+ tags: {
33157
+ deviceId: saved.id,
33158
+ stableId: saved.stableId
33159
+ },
32835
33160
  meta: { type: saved.type }
32836
33161
  });
32837
33162
  return;
32838
33163
  }
32839
33164
  try {
32840
- await this.ctx.kernel.devices.create(saved.stableId, Class, {});
32841
- restored.add(saved.id);
33165
+ await attemptRestore(saved);
32842
33166
  } catch (err) {
32843
- this.ctx.logger.warn("Failed to restore device", {
32844
- tags: { stableId: saved.stableId },
33167
+ const error = err instanceof Error ? err.message : String(err);
33168
+ this.ctx.logger.warn("Failed to restore device — bounded retry scheduled", {
33169
+ tags: {
33170
+ deviceId: saved.id,
33171
+ stableId: saved.stableId
33172
+ },
32845
33173
  meta: {
32846
33174
  type: saved.type,
32847
- error: err instanceof Error ? err.message : String(err)
33175
+ attempt: 1,
33176
+ error
32848
33177
  }
32849
33178
  });
33179
+ failures.push({
33180
+ saved,
33181
+ error
33182
+ });
32850
33183
  }
32851
33184
  };
32852
33185
  await Promise.all(topLevel.map((saved) => restoreOne(saved)));
33186
+ const failedTopLevelIds = new Set(failures.map((failure) => failure.saved.id));
32853
33187
  const childRows = savedDevices.filter((s) => s.parentDeviceId !== null);
32854
33188
  for (const saved of childRows) {
32855
- const Class = this.deviceClasses[saved.type];
32856
- if (!Class) continue;
33189
+ if (!this.deviceClasses[saved.type]) continue;
32857
33190
  if (saved.parentDeviceId === null) continue;
32858
- if (!restored.has(saved.parentDeviceId)) continue;
32859
- try {
32860
- await this.ctx.kernel.devices.create(saved.stableId, Class, {}, saved.parentDeviceId);
32861
- restored.add(saved.id);
32862
- } catch (err) {
32863
- this.ctx.logger.warn("Failed to restore hub-adopted child", {
33191
+ if (restored.has(saved.parentDeviceId)) {
33192
+ try {
33193
+ await attemptRestore(saved);
33194
+ } catch (err) {
33195
+ const error = err instanceof Error ? err.message : String(err);
33196
+ this.ctx.logger.warn("Failed to restore hub-adopted child — bounded retry scheduled", {
33197
+ tags: {
33198
+ deviceId: saved.id,
33199
+ stableId: saved.stableId,
33200
+ parentDeviceId: saved.parentDeviceId
33201
+ },
33202
+ meta: {
33203
+ type: saved.type,
33204
+ attempt: 1,
33205
+ error
33206
+ }
33207
+ });
33208
+ failures.push({
33209
+ saved,
33210
+ error
33211
+ });
33212
+ }
33213
+ continue;
33214
+ }
33215
+ if (failedTopLevelIds.has(saved.parentDeviceId)) {
33216
+ this.ctx.logger.warn("Hub-adopted child deferred — parent failed initial restore", {
32864
33217
  tags: {
33218
+ deviceId: saved.id,
32865
33219
  stableId: saved.stableId,
32866
33220
  parentDeviceId: saved.parentDeviceId
32867
33221
  },
32868
- meta: {
32869
- type: saved.type,
32870
- error: err instanceof Error ? err.message : String(err)
32871
- }
33222
+ meta: { type: saved.type }
32872
33223
  });
33224
+ failures.push({
33225
+ saved,
33226
+ error: `parent device ${saved.parentDeviceId} not restored`
33227
+ });
33228
+ continue;
32873
33229
  }
32874
33230
  }
33231
+ if (failures.length > 0) this.scheduleRestoreRetries(failures, attemptRestore);
33232
+ return {
33233
+ restoredCount: restored.size,
33234
+ failedCount: failures.length
33235
+ };
32875
33236
  }
32876
33237
  /** Convert an IDevice to the flat DeviceSummary for the cap router. */
32877
33238
  toSummary(device) {
@@ -34700,6 +35061,12 @@ Object.freeze({
34700
35061
  addonId: null,
34701
35062
  access: "view"
34702
35063
  },
35064
+ "deviceProvider.reloadDevice": {
35065
+ capName: "device-provider",
35066
+ capScope: "system",
35067
+ addonId: null,
35068
+ access: "create"
35069
+ },
34703
35070
  "deviceProvider.start": {
34704
35071
  capName: "device-provider",
34705
35072
  capScope: "system",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@camstack/addon-provider-rtsp",
3
- "version": "1.2.59",
3
+ "version": "1.2.60",
4
4
  "description": "Generic RTSP camera device provider addon for CamStack",
5
5
  "keywords": [
6
6
  "camstack",