@camstack/addon-provider-amcrest 0.2.60 → 0.2.61

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/addon.js +391 -24
  2. package/dist/addon.mjs +391 -24
  3. package/package.json +1 -1
package/dist/addon.js CHANGED
@@ -12805,6 +12805,35 @@ var deviceProviderCapability = {
12805
12805
  name: string(),
12806
12806
  type: string()
12807
12807
  }))),
12808
+ /**
12809
+ * Tear down and reconstruct ONE device in place from its persisted rows —
12810
+ * touching no other device this provider owns.
12811
+ *
12812
+ * The primitive `deviceManager.migrateDevice` uses to flush the two
12813
+ * migrated numbers: after `swapIds` the runner's live instance still
12814
+ * carries the PRE-swap numeric id (baked into the object, its native-cap
12815
+ * registrations and its log tags), and a live object cannot be renumbered.
12816
+ * Before this method the only flush was restarting the whole owning addon
12817
+ * — which took every camera the provider owns down with it (28 devices
12818
+ * for one migrated camera, measured 2026-09-04, and the morning of the
12819
+ * same day ~27 devices' native caps did not come back on their own).
12820
+ *
12821
+ * Keyed by `stableId`, deliberately: the numeric id is exactly the thing
12822
+ * that changes. The reply carries the id the device answers on NOW.
12823
+ * Implemented once in `BaseDeviceProvider` — decommission the live
12824
+ * instance (if any), then re-create from the persisted row: the same
12825
+ * teardown/rehydrate pair every graceful shutdown + boot already uses.
12826
+ * An RPC, never an event: a dropped event would leave the runner writing
12827
+ * against the wrong camera (D8).
12828
+ *
12829
+ * Construction can dial hardware, and the migrated source is
12830
+ * characteristically dead — the timeout covers a full activate window
12831
+ * rather than the 60 s default.
12832
+ */
12833
+ reloadDevice: method(object({ stableId: string() }), object({ deviceId: number() }), {
12834
+ kind: "mutation",
12835
+ timeoutMs: 3 * 6e4
12836
+ }),
12808
12837
  supportsDiscovery: method(object({}), boolean()),
12809
12838
  /**
12810
12839
  * Run a network scan. `params` carries optional provider-specific scan
@@ -13132,7 +13161,8 @@ method(object({
13132
13161
  targetId: number()
13133
13162
  }), MigrateDeviceResultSchema, {
13134
13163
  kind: "mutation",
13135
- auth: "admin"
13164
+ auth: "admin",
13165
+ timeoutMs: 12 * 6e4
13136
13166
  }), method(DeviceRegisterPayloadSchema, _void(), { kind: "mutation" }), method(DeviceRemovePayloadSchema, _void(), { kind: "mutation" }), method(DevicePersistConfigPayloadSchema, _void(), { kind: "mutation" }), method(object({ deviceId: number() }), record(string(), unknown())), method(object({ deviceId: number() }), record(string(), unknown())), method(object({ deviceId: number() }), DeviceMetaSchema.nullable()), method(object({
13137
13167
  deviceId: number(),
13138
13168
  name: string()
@@ -32878,6 +32908,147 @@ var BaseDevice = class {
32878
32908
  }
32879
32909
  };
32880
32910
  /**
32911
+ * Delays before retry rounds 1..N — the round count IS the bound.
32912
+ * 10 s catches "the hub was busy for a moment"; the full schedule
32913
+ * (10 + 30 + 90 s of waiting, plus up to one 60 s transport timeout
32914
+ * per attempt) covers a device-manager lock held for minutes — the
32915
+ * 2026-09-04 outage's migration hold was ~3.5 min.
32916
+ */
32917
+ var DEVICE_RESTORE_RETRY_DELAYS_MS = [
32918
+ 1e4,
32919
+ 3e4,
32920
+ 9e4
32921
+ ];
32922
+ /** Abortable sleep — resolves early (never rejects) on abort. */
32923
+ function sleep$1(ms, signal) {
32924
+ return new Promise((resolve) => {
32925
+ if (signal.aborted) {
32926
+ resolve();
32927
+ return;
32928
+ }
32929
+ const onAbort = () => {
32930
+ clearTimeout(timer);
32931
+ resolve();
32932
+ };
32933
+ const timer = setTimeout(() => {
32934
+ signal.removeEventListener("abort", onAbort);
32935
+ resolve();
32936
+ }, ms);
32937
+ timer.unref?.();
32938
+ signal.addEventListener("abort", onAbort, { once: true });
32939
+ });
32940
+ }
32941
+ /** Drain `items` through at most `width` concurrent lanes. `fn` must
32942
+ * not reject (callers wrap their own try/catch). */
32943
+ async function runWithConcurrency(items, width, fn) {
32944
+ const queue = [...items];
32945
+ const laneCount = Math.max(1, Math.min(width, queue.length));
32946
+ const lane = async () => {
32947
+ for (;;) {
32948
+ const item = queue.shift();
32949
+ if (item === void 0) return;
32950
+ await fn(item);
32951
+ }
32952
+ };
32953
+ await Promise.all(Array.from({ length: laneCount }, lane));
32954
+ }
32955
+ var DeviceRestoreRetryScheduler = class {
32956
+ #logger;
32957
+ #attempt;
32958
+ #onPermanentFailure;
32959
+ #delaysMs;
32960
+ #concurrency;
32961
+ #now;
32962
+ #abort = new AbortController();
32963
+ constructor(options) {
32964
+ this.#logger = options.logger;
32965
+ this.#attempt = options.attempt;
32966
+ this.#onPermanentFailure = options.onPermanentFailure;
32967
+ this.#delaysMs = options.delaysMs ?? DEVICE_RESTORE_RETRY_DELAYS_MS;
32968
+ this.#concurrency = options.concurrency ?? 4;
32969
+ this.#now = options.now ?? Date.now;
32970
+ }
32971
+ /** Stop retrying (shutdown). Pending entries are NOT marked
32972
+ * permanently failed — the next boot restores them from disk. */
32973
+ cancel() {
32974
+ this.#abort.abort();
32975
+ }
32976
+ /**
32977
+ * Run the bounded retry rounds. Resolves when every entry has either
32978
+ * restored, been marked permanently failed, or the scheduler was
32979
+ * cancelled. Never rejects.
32980
+ */
32981
+ async run(initialFailures) {
32982
+ let pending = initialFailures.map((failure) => ({
32983
+ saved: failure.saved,
32984
+ lastError: failure.error,
32985
+ attempts: 1
32986
+ }));
32987
+ for (let round = 0; round < this.#delaysMs.length; round += 1) {
32988
+ if (pending.length === 0 || this.#abort.signal.aborted) break;
32989
+ await sleep$1(this.#delaysMs[round] ?? 0, this.#abort.signal);
32990
+ if (this.#abort.signal.aborted) break;
32991
+ pending = await this.#runRound(pending, round);
32992
+ }
32993
+ if (this.#abort.signal.aborted) return [];
32994
+ const terminal = pending.map((entry) => ({
32995
+ deviceId: entry.saved.id,
32996
+ stableId: entry.saved.stableId,
32997
+ type: String(entry.saved.type),
32998
+ attempts: entry.attempts,
32999
+ lastError: entry.lastError,
33000
+ failedAt: this.#now()
33001
+ }));
33002
+ for (const failure of terminal) this.#onPermanentFailure(failure);
33003
+ return terminal;
33004
+ }
33005
+ /** One retry round: parents first (phase 0), then hub-adopted
33006
+ * children (phase 1) — a child's attempt depends on its parent
33007
+ * having landed, exactly like the initial two-pass restore. */
33008
+ async #runRound(pending, round) {
33009
+ const next = [];
33010
+ const parents = pending.filter((entry) => entry.saved.parentDeviceId === null);
33011
+ const children = pending.filter((entry) => entry.saved.parentDeviceId !== null);
33012
+ for (const phase of [parents, children]) await runWithConcurrency(phase, this.#concurrency, async (entry) => {
33013
+ if (this.#abort.signal.aborted) {
33014
+ next.push(entry);
33015
+ return;
33016
+ }
33017
+ const attemptNo = entry.attempts + 1;
33018
+ try {
33019
+ await this.#attempt(entry.saved);
33020
+ this.#logger.info("Device restored on retry", {
33021
+ tags: {
33022
+ deviceId: entry.saved.id,
33023
+ stableId: entry.saved.stableId
33024
+ },
33025
+ meta: { attempt: attemptNo }
33026
+ });
33027
+ } catch (err) {
33028
+ const lastError = err instanceof Error ? err.message : String(err);
33029
+ const remainingRetries = this.#delaysMs.length - (round + 1);
33030
+ this.#logger.warn("Device restore retry failed", {
33031
+ tags: {
33032
+ deviceId: entry.saved.id,
33033
+ stableId: entry.saved.stableId
33034
+ },
33035
+ meta: {
33036
+ attempt: attemptNo,
33037
+ remainingRetries,
33038
+ error: lastError
33039
+ }
33040
+ });
33041
+ next.push({
33042
+ saved: entry.saved,
33043
+ lastError,
33044
+ attempts: attemptNo
33045
+ });
33046
+ }
33047
+ });
33048
+ return next;
33049
+ }
33050
+ };
33051
+ /**
32881
33052
  * Convert an IDevice to the flat DeviceSummary shape expected by the
32882
33053
  * device-provider cap router. Shared across all providers.
32883
33054
  */
@@ -32926,6 +33097,7 @@ var BaseDeviceProvider = class extends BaseAddon {
32926
33097
  }];
32927
33098
  }
32928
33099
  async onShutdown() {
33100
+ this.cancelRestoreRetries();
32929
33101
  const devices = await this.ctx.kernel.devices?.getAll() ?? [];
32930
33102
  for (const device of devices) try {
32931
33103
  await this.ctx.kernel.devices?.decommission(device.id);
@@ -32943,9 +33115,16 @@ var BaseDeviceProvider = class extends BaseAddon {
32943
33115
  async start() {}
32944
33116
  async stop() {}
32945
33117
  async getStatus() {
33118
+ const all = await this.ctx.kernel.devices?.getAll() ?? [];
33119
+ const summary = this.restoreFailureSummary();
33120
+ if (summary === null) return {
33121
+ connected: true,
33122
+ deviceCount: all.length
33123
+ };
32946
33124
  return {
32947
33125
  connected: true,
32948
- deviceCount: (await this.ctx.kernel.devices?.getAll() ?? []).length
33126
+ deviceCount: all.length,
33127
+ error: summary
32949
33128
  };
32950
33129
  }
32951
33130
  async getDevices() {
@@ -33035,8 +33214,137 @@ var BaseDeviceProvider = class extends BaseAddon {
33035
33214
  };
33036
33215
  }
33037
33216
  async restoreDevices(savedDevices) {
33038
- await this.onRestoreDevices(savedDevices);
33039
- if (savedDevices.length > 0) this.ctx.logger.info(`Restored ${savedDevices.length} ${this.providerName} device(s)`);
33217
+ const report = await this.onRestoreDevices(savedDevices);
33218
+ if (savedDevices.length === 0) return;
33219
+ if (report && report.failedCount > 0) {
33220
+ this.ctx.logger.warn(`Restored ${report.restoredCount}/${savedDevices.length} ${this.providerName} device(s) — ${report.failedCount} failed, bounded retry scheduled`);
33221
+ return;
33222
+ }
33223
+ const restoredCount = report ? report.restoredCount : savedDevices.length;
33224
+ this.ctx.logger.info(`Restored ${restoredCount} ${this.providerName} device(s)`);
33225
+ }
33226
+ /** Retry schedule. Overridable (tests use millisecond delays). */
33227
+ restoreRetryDelaysMs = DEVICE_RESTORE_RETRY_DELAYS_MS;
33228
+ /** Retry lane width. See `device-restore-retry.ts` for why retries
33229
+ * never re-stampede full-width while the initial pass does (D167). */
33230
+ restoreRetryConcurrency = 4;
33231
+ _restoreRetryScheduler = null;
33232
+ _restoreRetryCompletion = null;
33233
+ _permanentRestoreFailures = /* @__PURE__ */ new Map();
33234
+ /** Settles when the background retry rounds finish (or `null` when
33235
+ * nothing failed). Exposed for tests and subclass diagnostics —
33236
+ * boot NEVER awaits this: the runner's post-init handshake goes out
33237
+ * with the devices that restored, and a late success is announced
33238
+ * through the `native-cap-change` → `updateCaps` path. */
33239
+ get restoreRetryCompletion() {
33240
+ return this._restoreRetryCompletion;
33241
+ }
33242
+ /** Devices that exhausted the retry bound this process lifetime. */
33243
+ get permanentRestoreFailures() {
33244
+ return [...this._permanentRestoreFailures.values()];
33245
+ }
33246
+ /** One-line operator-facing summary for `getStatus().error`, or
33247
+ * `null` when every device restored. */
33248
+ restoreFailureSummary() {
33249
+ if (this._permanentRestoreFailures.size === 0) return null;
33250
+ const ids = [...this._permanentRestoreFailures.keys()].join(", ");
33251
+ return `${this._permanentRestoreFailures.size} device(s) permanently failed restore (deviceIds: ${ids}) — restart the ${this.providerName} provider to retry`;
33252
+ }
33253
+ cancelRestoreRetries() {
33254
+ this._restoreRetryScheduler?.cancel();
33255
+ this._restoreRetryScheduler = null;
33256
+ }
33257
+ recordPermanentRestoreFailure(failure) {
33258
+ this._permanentRestoreFailures.set(failure.deviceId, failure);
33259
+ this.ctx.logger.error("Device restore permanently failed — its capabilities will not register until the provider restarts", {
33260
+ tags: {
33261
+ deviceId: failure.deviceId,
33262
+ stableId: failure.stableId
33263
+ },
33264
+ meta: {
33265
+ type: failure.type,
33266
+ attempts: failure.attempts,
33267
+ error: failure.lastError
33268
+ }
33269
+ });
33270
+ }
33271
+ scheduleRestoreRetries(failures, attempt) {
33272
+ const scheduler = new DeviceRestoreRetryScheduler({
33273
+ logger: this.ctx.logger,
33274
+ delaysMs: this.restoreRetryDelaysMs,
33275
+ concurrency: this.restoreRetryConcurrency,
33276
+ attempt,
33277
+ onPermanentFailure: (failure) => this.recordPermanentRestoreFailure(failure)
33278
+ });
33279
+ this._restoreRetryScheduler = scheduler;
33280
+ this._restoreRetryCompletion = scheduler.run(failures).then(() => void 0).catch((err) => {
33281
+ this.ctx.logger.error("Restore retry scheduler crashed", { meta: { error: err instanceof Error ? err.message : String(err) } });
33282
+ });
33283
+ }
33284
+ /**
33285
+ * Tear down and reconstruct ONE device from its persisted rows — the
33286
+ * `deviceProvider.reloadDevice` cap method. Persistence is never touched,
33287
+ * and no other device this provider owns is disturbed.
33288
+ *
33289
+ * Keyed by `stableId` because the caller's whole reason to be here is that
33290
+ * the NUMERIC id changed (`deviceManager.migrateDevice` swapped it): the
33291
+ * fresh instance resolves its id through `allocateDeviceId`, which returns
33292
+ * whatever number the row carries NOW. The teardown is `decommission` —
33293
+ * exactly what a graceful shutdown runs per device (fires `removeDevice()`,
33294
+ * unregisters native caps, drops the registry entry) — and the rebuild is
33295
+ * the boot restore's own `create()` path, including its pass 2: first-class
33296
+ * children (hub-adopted cameras under an NVR) are decommissioned with the
33297
+ * parent by the cascade and must be re-created explicitly, because only
33298
+ * accessory children come back through `getAccessoryChildren()`.
33299
+ *
33300
+ * Reloading an accessory child directly is refused (no device class) —
33301
+ * reload its parent instead.
33302
+ */
33303
+ async reloadDevice(input) {
33304
+ const { stableId } = input;
33305
+ const devices = this.ctx.kernel.devices;
33306
+ if (!devices) throw new Error(`${this.providerName}: kernel.devices unavailable — cannot reload`);
33307
+ const live = (await devices.getAll()).find((d) => d.stableId === stableId);
33308
+ if (live) await devices.decommission(live.id);
33309
+ const { id } = await this.ctx.api.deviceManager.allocateDeviceId.mutate({
33310
+ addonId: this.addonId,
33311
+ stableId
33312
+ });
33313
+ const meta = await this.ctx.api.deviceManager.loadMeta.query({ deviceId: id });
33314
+ if (meta === null) throw new Error(`${this.providerName}: no persisted meta for "${stableId}" (id ${id}) — cannot reload`);
33315
+ const deviceType = Object.values(DeviceType).find((t) => t === meta.type);
33316
+ const Class = deviceType !== void 0 ? this.deviceClasses[deviceType] : void 0;
33317
+ if (!Class) throw new Error(`${this.providerName}: no device class for type "${meta.type}" — "${stableId}" is an accessory child; reload its parent instead`);
33318
+ await devices.create(stableId, Class, {}, meta.parentDeviceId ?? null);
33319
+ const rows = await this.ctx.api.deviceManager.listPersistedByAddon.query({ addonId: this.addonId });
33320
+ for (const row of rows) {
33321
+ if (row.parentDeviceId !== id) continue;
33322
+ const childType = Object.values(DeviceType).find((t) => t === row.type);
33323
+ const ChildClass = childType !== void 0 ? this.deviceClasses[childType] : void 0;
33324
+ if (!ChildClass) continue;
33325
+ try {
33326
+ await devices.create(row.stableId, ChildClass, {}, id);
33327
+ } catch (err) {
33328
+ this.ctx.logger.warn("reloadDevice: failed to re-create first-class child", {
33329
+ tags: {
33330
+ deviceId: row.id,
33331
+ stableId: row.stableId
33332
+ },
33333
+ meta: {
33334
+ parentDeviceId: id,
33335
+ error: err instanceof Error ? err.message : String(err)
33336
+ }
33337
+ });
33338
+ }
33339
+ }
33340
+ this.ctx.logger.info("device reloaded in place from persisted rows", {
33341
+ tags: { deviceId: id },
33342
+ meta: {
33343
+ stableId,
33344
+ type: meta.type
33345
+ }
33346
+ });
33347
+ return { deviceId: id };
33040
33348
  }
33041
33349
  /**
33042
33350
  * Restore devices from persisted state. Two-pass:
@@ -33062,55 +33370,108 @@ var BaseDeviceProvider = class extends BaseAddon {
33062
33370
  * accessory-spawn flow handles via the parent's
33063
33371
  * `getAccessoryChildren()`. Override only when the default doesn't
33064
33372
  * fit.
33373
+ *
33374
+ * A row that fails either pass is NOT terminal (D347): it is handed
33375
+ * to a bounded background retry (`DeviceRestoreRetryScheduler`).
33376
+ * Only after the bound is exhausted is the device marked permanently
33377
+ * failed — logged at ERROR with `tags.deviceId` and surfaced via
33378
+ * `getStatus().error`.
33065
33379
  */
33066
33380
  async onRestoreDevices(savedDevices) {
33067
33381
  const restored = /* @__PURE__ */ new Set();
33382
+ const failures = [];
33383
+ const attemptRestore = async (saved) => {
33384
+ if (restored.has(saved.id)) return;
33385
+ const Class = this.deviceClasses[saved.type];
33386
+ if (!Class) throw new Error(`no device class registered for type "${saved.type}"`);
33387
+ if (saved.parentDeviceId !== null && !restored.has(saved.parentDeviceId)) throw new Error(`parent device ${saved.parentDeviceId} not restored`);
33388
+ await this.ctx.kernel.devices.create(saved.stableId, Class, {}, saved.parentDeviceId);
33389
+ restored.add(saved.id);
33390
+ };
33068
33391
  const topLevel = savedDevices.filter((saved) => saved.parentDeviceId === null);
33069
33392
  const restoreOne = async (saved) => {
33070
- const Class = this.deviceClasses[saved.type];
33071
- if (!Class) {
33393
+ if (!this.deviceClasses[saved.type]) {
33072
33394
  this.ctx.logger.warn("No device class registered for restored type — skipping", {
33073
- tags: { stableId: saved.stableId },
33395
+ tags: {
33396
+ deviceId: saved.id,
33397
+ stableId: saved.stableId
33398
+ },
33074
33399
  meta: { type: saved.type }
33075
33400
  });
33076
33401
  return;
33077
33402
  }
33078
33403
  try {
33079
- await this.ctx.kernel.devices.create(saved.stableId, Class, {});
33080
- restored.add(saved.id);
33404
+ await attemptRestore(saved);
33081
33405
  } catch (err) {
33082
- this.ctx.logger.warn("Failed to restore device", {
33083
- tags: { stableId: saved.stableId },
33406
+ const error = err instanceof Error ? err.message : String(err);
33407
+ this.ctx.logger.warn("Failed to restore device — bounded retry scheduled", {
33408
+ tags: {
33409
+ deviceId: saved.id,
33410
+ stableId: saved.stableId
33411
+ },
33084
33412
  meta: {
33085
33413
  type: saved.type,
33086
- error: err instanceof Error ? err.message : String(err)
33414
+ attempt: 1,
33415
+ error
33087
33416
  }
33088
33417
  });
33418
+ failures.push({
33419
+ saved,
33420
+ error
33421
+ });
33089
33422
  }
33090
33423
  };
33091
33424
  await Promise.all(topLevel.map((saved) => restoreOne(saved)));
33425
+ const failedTopLevelIds = new Set(failures.map((failure) => failure.saved.id));
33092
33426
  const childRows = savedDevices.filter((s) => s.parentDeviceId !== null);
33093
33427
  for (const saved of childRows) {
33094
- const Class = this.deviceClasses[saved.type];
33095
- if (!Class) continue;
33428
+ if (!this.deviceClasses[saved.type]) continue;
33096
33429
  if (saved.parentDeviceId === null) continue;
33097
- if (!restored.has(saved.parentDeviceId)) continue;
33098
- try {
33099
- await this.ctx.kernel.devices.create(saved.stableId, Class, {}, saved.parentDeviceId);
33100
- restored.add(saved.id);
33101
- } catch (err) {
33102
- this.ctx.logger.warn("Failed to restore hub-adopted child", {
33430
+ if (restored.has(saved.parentDeviceId)) {
33431
+ try {
33432
+ await attemptRestore(saved);
33433
+ } catch (err) {
33434
+ const error = err instanceof Error ? err.message : String(err);
33435
+ this.ctx.logger.warn("Failed to restore hub-adopted child — bounded retry scheduled", {
33436
+ tags: {
33437
+ deviceId: saved.id,
33438
+ stableId: saved.stableId,
33439
+ parentDeviceId: saved.parentDeviceId
33440
+ },
33441
+ meta: {
33442
+ type: saved.type,
33443
+ attempt: 1,
33444
+ error
33445
+ }
33446
+ });
33447
+ failures.push({
33448
+ saved,
33449
+ error
33450
+ });
33451
+ }
33452
+ continue;
33453
+ }
33454
+ if (failedTopLevelIds.has(saved.parentDeviceId)) {
33455
+ this.ctx.logger.warn("Hub-adopted child deferred — parent failed initial restore", {
33103
33456
  tags: {
33457
+ deviceId: saved.id,
33104
33458
  stableId: saved.stableId,
33105
33459
  parentDeviceId: saved.parentDeviceId
33106
33460
  },
33107
- meta: {
33108
- type: saved.type,
33109
- error: err instanceof Error ? err.message : String(err)
33110
- }
33461
+ meta: { type: saved.type }
33111
33462
  });
33463
+ failures.push({
33464
+ saved,
33465
+ error: `parent device ${saved.parentDeviceId} not restored`
33466
+ });
33467
+ continue;
33112
33468
  }
33113
33469
  }
33470
+ if (failures.length > 0) this.scheduleRestoreRetries(failures, attemptRestore);
33471
+ return {
33472
+ restoredCount: restored.size,
33473
+ failedCount: failures.length
33474
+ };
33114
33475
  }
33115
33476
  /** Convert an IDevice to the flat DeviceSummary for the cap router. */
33116
33477
  toSummary(device) {
@@ -35079,6 +35440,12 @@ Object.freeze({
35079
35440
  addonId: null,
35080
35441
  access: "view"
35081
35442
  },
35443
+ "deviceProvider.reloadDevice": {
35444
+ capName: "device-provider",
35445
+ capScope: "system",
35446
+ addonId: null,
35447
+ access: "create"
35448
+ },
35082
35449
  "deviceProvider.start": {
35083
35450
  capName: "device-provider",
35084
35451
  capScope: "system",
package/dist/addon.mjs CHANGED
@@ -12806,6 +12806,35 @@ var deviceProviderCapability = {
12806
12806
  name: string(),
12807
12807
  type: string()
12808
12808
  }))),
12809
+ /**
12810
+ * Tear down and reconstruct ONE device in place from its persisted rows —
12811
+ * touching no other device this provider owns.
12812
+ *
12813
+ * The primitive `deviceManager.migrateDevice` uses to flush the two
12814
+ * migrated numbers: after `swapIds` the runner's live instance still
12815
+ * carries the PRE-swap numeric id (baked into the object, its native-cap
12816
+ * registrations and its log tags), and a live object cannot be renumbered.
12817
+ * Before this method the only flush was restarting the whole owning addon
12818
+ * — which took every camera the provider owns down with it (28 devices
12819
+ * for one migrated camera, measured 2026-09-04, and the morning of the
12820
+ * same day ~27 devices' native caps did not come back on their own).
12821
+ *
12822
+ * Keyed by `stableId`, deliberately: the numeric id is exactly the thing
12823
+ * that changes. The reply carries the id the device answers on NOW.
12824
+ * Implemented once in `BaseDeviceProvider` — decommission the live
12825
+ * instance (if any), then re-create from the persisted row: the same
12826
+ * teardown/rehydrate pair every graceful shutdown + boot already uses.
12827
+ * An RPC, never an event: a dropped event would leave the runner writing
12828
+ * against the wrong camera (D8).
12829
+ *
12830
+ * Construction can dial hardware, and the migrated source is
12831
+ * characteristically dead — the timeout covers a full activate window
12832
+ * rather than the 60 s default.
12833
+ */
12834
+ reloadDevice: method(object({ stableId: string() }), object({ deviceId: number() }), {
12835
+ kind: "mutation",
12836
+ timeoutMs: 3 * 6e4
12837
+ }),
12809
12838
  supportsDiscovery: method(object({}), boolean()),
12810
12839
  /**
12811
12840
  * Run a network scan. `params` carries optional provider-specific scan
@@ -13133,7 +13162,8 @@ method(object({
13133
13162
  targetId: number()
13134
13163
  }), MigrateDeviceResultSchema, {
13135
13164
  kind: "mutation",
13136
- auth: "admin"
13165
+ auth: "admin",
13166
+ timeoutMs: 12 * 6e4
13137
13167
  }), method(DeviceRegisterPayloadSchema, _void(), { kind: "mutation" }), method(DeviceRemovePayloadSchema, _void(), { kind: "mutation" }), method(DevicePersistConfigPayloadSchema, _void(), { kind: "mutation" }), method(object({ deviceId: number() }), record(string(), unknown())), method(object({ deviceId: number() }), record(string(), unknown())), method(object({ deviceId: number() }), DeviceMetaSchema.nullable()), method(object({
13138
13168
  deviceId: number(),
13139
13169
  name: string()
@@ -32879,6 +32909,147 @@ var BaseDevice = class {
32879
32909
  }
32880
32910
  };
32881
32911
  /**
32912
+ * Delays before retry rounds 1..N — the round count IS the bound.
32913
+ * 10 s catches "the hub was busy for a moment"; the full schedule
32914
+ * (10 + 30 + 90 s of waiting, plus up to one 60 s transport timeout
32915
+ * per attempt) covers a device-manager lock held for minutes — the
32916
+ * 2026-09-04 outage's migration hold was ~3.5 min.
32917
+ */
32918
+ var DEVICE_RESTORE_RETRY_DELAYS_MS = [
32919
+ 1e4,
32920
+ 3e4,
32921
+ 9e4
32922
+ ];
32923
+ /** Abortable sleep — resolves early (never rejects) on abort. */
32924
+ function sleep$1(ms, signal) {
32925
+ return new Promise((resolve) => {
32926
+ if (signal.aborted) {
32927
+ resolve();
32928
+ return;
32929
+ }
32930
+ const onAbort = () => {
32931
+ clearTimeout(timer);
32932
+ resolve();
32933
+ };
32934
+ const timer = setTimeout(() => {
32935
+ signal.removeEventListener("abort", onAbort);
32936
+ resolve();
32937
+ }, ms);
32938
+ timer.unref?.();
32939
+ signal.addEventListener("abort", onAbort, { once: true });
32940
+ });
32941
+ }
32942
+ /** Drain `items` through at most `width` concurrent lanes. `fn` must
32943
+ * not reject (callers wrap their own try/catch). */
32944
+ async function runWithConcurrency(items, width, fn) {
32945
+ const queue = [...items];
32946
+ const laneCount = Math.max(1, Math.min(width, queue.length));
32947
+ const lane = async () => {
32948
+ for (;;) {
32949
+ const item = queue.shift();
32950
+ if (item === void 0) return;
32951
+ await fn(item);
32952
+ }
32953
+ };
32954
+ await Promise.all(Array.from({ length: laneCount }, lane));
32955
+ }
32956
+ var DeviceRestoreRetryScheduler = class {
32957
+ #logger;
32958
+ #attempt;
32959
+ #onPermanentFailure;
32960
+ #delaysMs;
32961
+ #concurrency;
32962
+ #now;
32963
+ #abort = new AbortController();
32964
+ constructor(options) {
32965
+ this.#logger = options.logger;
32966
+ this.#attempt = options.attempt;
32967
+ this.#onPermanentFailure = options.onPermanentFailure;
32968
+ this.#delaysMs = options.delaysMs ?? DEVICE_RESTORE_RETRY_DELAYS_MS;
32969
+ this.#concurrency = options.concurrency ?? 4;
32970
+ this.#now = options.now ?? Date.now;
32971
+ }
32972
+ /** Stop retrying (shutdown). Pending entries are NOT marked
32973
+ * permanently failed — the next boot restores them from disk. */
32974
+ cancel() {
32975
+ this.#abort.abort();
32976
+ }
32977
+ /**
32978
+ * Run the bounded retry rounds. Resolves when every entry has either
32979
+ * restored, been marked permanently failed, or the scheduler was
32980
+ * cancelled. Never rejects.
32981
+ */
32982
+ async run(initialFailures) {
32983
+ let pending = initialFailures.map((failure) => ({
32984
+ saved: failure.saved,
32985
+ lastError: failure.error,
32986
+ attempts: 1
32987
+ }));
32988
+ for (let round = 0; round < this.#delaysMs.length; round += 1) {
32989
+ if (pending.length === 0 || this.#abort.signal.aborted) break;
32990
+ await sleep$1(this.#delaysMs[round] ?? 0, this.#abort.signal);
32991
+ if (this.#abort.signal.aborted) break;
32992
+ pending = await this.#runRound(pending, round);
32993
+ }
32994
+ if (this.#abort.signal.aborted) return [];
32995
+ const terminal = pending.map((entry) => ({
32996
+ deviceId: entry.saved.id,
32997
+ stableId: entry.saved.stableId,
32998
+ type: String(entry.saved.type),
32999
+ attempts: entry.attempts,
33000
+ lastError: entry.lastError,
33001
+ failedAt: this.#now()
33002
+ }));
33003
+ for (const failure of terminal) this.#onPermanentFailure(failure);
33004
+ return terminal;
33005
+ }
33006
+ /** One retry round: parents first (phase 0), then hub-adopted
33007
+ * children (phase 1) — a child's attempt depends on its parent
33008
+ * having landed, exactly like the initial two-pass restore. */
33009
+ async #runRound(pending, round) {
33010
+ const next = [];
33011
+ const parents = pending.filter((entry) => entry.saved.parentDeviceId === null);
33012
+ const children = pending.filter((entry) => entry.saved.parentDeviceId !== null);
33013
+ for (const phase of [parents, children]) await runWithConcurrency(phase, this.#concurrency, async (entry) => {
33014
+ if (this.#abort.signal.aborted) {
33015
+ next.push(entry);
33016
+ return;
33017
+ }
33018
+ const attemptNo = entry.attempts + 1;
33019
+ try {
33020
+ await this.#attempt(entry.saved);
33021
+ this.#logger.info("Device restored on retry", {
33022
+ tags: {
33023
+ deviceId: entry.saved.id,
33024
+ stableId: entry.saved.stableId
33025
+ },
33026
+ meta: { attempt: attemptNo }
33027
+ });
33028
+ } catch (err) {
33029
+ const lastError = err instanceof Error ? err.message : String(err);
33030
+ const remainingRetries = this.#delaysMs.length - (round + 1);
33031
+ this.#logger.warn("Device restore retry failed", {
33032
+ tags: {
33033
+ deviceId: entry.saved.id,
33034
+ stableId: entry.saved.stableId
33035
+ },
33036
+ meta: {
33037
+ attempt: attemptNo,
33038
+ remainingRetries,
33039
+ error: lastError
33040
+ }
33041
+ });
33042
+ next.push({
33043
+ saved: entry.saved,
33044
+ lastError,
33045
+ attempts: attemptNo
33046
+ });
33047
+ }
33048
+ });
33049
+ return next;
33050
+ }
33051
+ };
33052
+ /**
32882
33053
  * Convert an IDevice to the flat DeviceSummary shape expected by the
32883
33054
  * device-provider cap router. Shared across all providers.
32884
33055
  */
@@ -32927,6 +33098,7 @@ var BaseDeviceProvider = class extends BaseAddon {
32927
33098
  }];
32928
33099
  }
32929
33100
  async onShutdown() {
33101
+ this.cancelRestoreRetries();
32930
33102
  const devices = await this.ctx.kernel.devices?.getAll() ?? [];
32931
33103
  for (const device of devices) try {
32932
33104
  await this.ctx.kernel.devices?.decommission(device.id);
@@ -32944,9 +33116,16 @@ var BaseDeviceProvider = class extends BaseAddon {
32944
33116
  async start() {}
32945
33117
  async stop() {}
32946
33118
  async getStatus() {
33119
+ const all = await this.ctx.kernel.devices?.getAll() ?? [];
33120
+ const summary = this.restoreFailureSummary();
33121
+ if (summary === null) return {
33122
+ connected: true,
33123
+ deviceCount: all.length
33124
+ };
32947
33125
  return {
32948
33126
  connected: true,
32949
- deviceCount: (await this.ctx.kernel.devices?.getAll() ?? []).length
33127
+ deviceCount: all.length,
33128
+ error: summary
32950
33129
  };
32951
33130
  }
32952
33131
  async getDevices() {
@@ -33036,8 +33215,137 @@ var BaseDeviceProvider = class extends BaseAddon {
33036
33215
  };
33037
33216
  }
33038
33217
  async restoreDevices(savedDevices) {
33039
- await this.onRestoreDevices(savedDevices);
33040
- if (savedDevices.length > 0) this.ctx.logger.info(`Restored ${savedDevices.length} ${this.providerName} device(s)`);
33218
+ const report = await this.onRestoreDevices(savedDevices);
33219
+ if (savedDevices.length === 0) return;
33220
+ if (report && report.failedCount > 0) {
33221
+ this.ctx.logger.warn(`Restored ${report.restoredCount}/${savedDevices.length} ${this.providerName} device(s) — ${report.failedCount} failed, bounded retry scheduled`);
33222
+ return;
33223
+ }
33224
+ const restoredCount = report ? report.restoredCount : savedDevices.length;
33225
+ this.ctx.logger.info(`Restored ${restoredCount} ${this.providerName} device(s)`);
33226
+ }
33227
+ /** Retry schedule. Overridable (tests use millisecond delays). */
33228
+ restoreRetryDelaysMs = DEVICE_RESTORE_RETRY_DELAYS_MS;
33229
+ /** Retry lane width. See `device-restore-retry.ts` for why retries
33230
+ * never re-stampede full-width while the initial pass does (D167). */
33231
+ restoreRetryConcurrency = 4;
33232
+ _restoreRetryScheduler = null;
33233
+ _restoreRetryCompletion = null;
33234
+ _permanentRestoreFailures = /* @__PURE__ */ new Map();
33235
+ /** Settles when the background retry rounds finish (or `null` when
33236
+ * nothing failed). Exposed for tests and subclass diagnostics —
33237
+ * boot NEVER awaits this: the runner's post-init handshake goes out
33238
+ * with the devices that restored, and a late success is announced
33239
+ * through the `native-cap-change` → `updateCaps` path. */
33240
+ get restoreRetryCompletion() {
33241
+ return this._restoreRetryCompletion;
33242
+ }
33243
+ /** Devices that exhausted the retry bound this process lifetime. */
33244
+ get permanentRestoreFailures() {
33245
+ return [...this._permanentRestoreFailures.values()];
33246
+ }
33247
+ /** One-line operator-facing summary for `getStatus().error`, or
33248
+ * `null` when every device restored. */
33249
+ restoreFailureSummary() {
33250
+ if (this._permanentRestoreFailures.size === 0) return null;
33251
+ const ids = [...this._permanentRestoreFailures.keys()].join(", ");
33252
+ return `${this._permanentRestoreFailures.size} device(s) permanently failed restore (deviceIds: ${ids}) — restart the ${this.providerName} provider to retry`;
33253
+ }
33254
+ cancelRestoreRetries() {
33255
+ this._restoreRetryScheduler?.cancel();
33256
+ this._restoreRetryScheduler = null;
33257
+ }
33258
+ recordPermanentRestoreFailure(failure) {
33259
+ this._permanentRestoreFailures.set(failure.deviceId, failure);
33260
+ this.ctx.logger.error("Device restore permanently failed — its capabilities will not register until the provider restarts", {
33261
+ tags: {
33262
+ deviceId: failure.deviceId,
33263
+ stableId: failure.stableId
33264
+ },
33265
+ meta: {
33266
+ type: failure.type,
33267
+ attempts: failure.attempts,
33268
+ error: failure.lastError
33269
+ }
33270
+ });
33271
+ }
33272
+ scheduleRestoreRetries(failures, attempt) {
33273
+ const scheduler = new DeviceRestoreRetryScheduler({
33274
+ logger: this.ctx.logger,
33275
+ delaysMs: this.restoreRetryDelaysMs,
33276
+ concurrency: this.restoreRetryConcurrency,
33277
+ attempt,
33278
+ onPermanentFailure: (failure) => this.recordPermanentRestoreFailure(failure)
33279
+ });
33280
+ this._restoreRetryScheduler = scheduler;
33281
+ this._restoreRetryCompletion = scheduler.run(failures).then(() => void 0).catch((err) => {
33282
+ this.ctx.logger.error("Restore retry scheduler crashed", { meta: { error: err instanceof Error ? err.message : String(err) } });
33283
+ });
33284
+ }
33285
+ /**
33286
+ * Tear down and reconstruct ONE device from its persisted rows — the
33287
+ * `deviceProvider.reloadDevice` cap method. Persistence is never touched,
33288
+ * and no other device this provider owns is disturbed.
33289
+ *
33290
+ * Keyed by `stableId` because the caller's whole reason to be here is that
33291
+ * the NUMERIC id changed (`deviceManager.migrateDevice` swapped it): the
33292
+ * fresh instance resolves its id through `allocateDeviceId`, which returns
33293
+ * whatever number the row carries NOW. The teardown is `decommission` —
33294
+ * exactly what a graceful shutdown runs per device (fires `removeDevice()`,
33295
+ * unregisters native caps, drops the registry entry) — and the rebuild is
33296
+ * the boot restore's own `create()` path, including its pass 2: first-class
33297
+ * children (hub-adopted cameras under an NVR) are decommissioned with the
33298
+ * parent by the cascade and must be re-created explicitly, because only
33299
+ * accessory children come back through `getAccessoryChildren()`.
33300
+ *
33301
+ * Reloading an accessory child directly is refused (no device class) —
33302
+ * reload its parent instead.
33303
+ */
33304
+ async reloadDevice(input) {
33305
+ const { stableId } = input;
33306
+ const devices = this.ctx.kernel.devices;
33307
+ if (!devices) throw new Error(`${this.providerName}: kernel.devices unavailable — cannot reload`);
33308
+ const live = (await devices.getAll()).find((d) => d.stableId === stableId);
33309
+ if (live) await devices.decommission(live.id);
33310
+ const { id } = await this.ctx.api.deviceManager.allocateDeviceId.mutate({
33311
+ addonId: this.addonId,
33312
+ stableId
33313
+ });
33314
+ const meta = await this.ctx.api.deviceManager.loadMeta.query({ deviceId: id });
33315
+ if (meta === null) throw new Error(`${this.providerName}: no persisted meta for "${stableId}" (id ${id}) — cannot reload`);
33316
+ const deviceType = Object.values(DeviceType).find((t) => t === meta.type);
33317
+ const Class = deviceType !== void 0 ? this.deviceClasses[deviceType] : void 0;
33318
+ if (!Class) throw new Error(`${this.providerName}: no device class for type "${meta.type}" — "${stableId}" is an accessory child; reload its parent instead`);
33319
+ await devices.create(stableId, Class, {}, meta.parentDeviceId ?? null);
33320
+ const rows = await this.ctx.api.deviceManager.listPersistedByAddon.query({ addonId: this.addonId });
33321
+ for (const row of rows) {
33322
+ if (row.parentDeviceId !== id) continue;
33323
+ const childType = Object.values(DeviceType).find((t) => t === row.type);
33324
+ const ChildClass = childType !== void 0 ? this.deviceClasses[childType] : void 0;
33325
+ if (!ChildClass) continue;
33326
+ try {
33327
+ await devices.create(row.stableId, ChildClass, {}, id);
33328
+ } catch (err) {
33329
+ this.ctx.logger.warn("reloadDevice: failed to re-create first-class child", {
33330
+ tags: {
33331
+ deviceId: row.id,
33332
+ stableId: row.stableId
33333
+ },
33334
+ meta: {
33335
+ parentDeviceId: id,
33336
+ error: err instanceof Error ? err.message : String(err)
33337
+ }
33338
+ });
33339
+ }
33340
+ }
33341
+ this.ctx.logger.info("device reloaded in place from persisted rows", {
33342
+ tags: { deviceId: id },
33343
+ meta: {
33344
+ stableId,
33345
+ type: meta.type
33346
+ }
33347
+ });
33348
+ return { deviceId: id };
33041
33349
  }
33042
33350
  /**
33043
33351
  * Restore devices from persisted state. Two-pass:
@@ -33063,55 +33371,108 @@ var BaseDeviceProvider = class extends BaseAddon {
33063
33371
  * accessory-spawn flow handles via the parent's
33064
33372
  * `getAccessoryChildren()`. Override only when the default doesn't
33065
33373
  * fit.
33374
+ *
33375
+ * A row that fails either pass is NOT terminal (D347): it is handed
33376
+ * to a bounded background retry (`DeviceRestoreRetryScheduler`).
33377
+ * Only after the bound is exhausted is the device marked permanently
33378
+ * failed — logged at ERROR with `tags.deviceId` and surfaced via
33379
+ * `getStatus().error`.
33066
33380
  */
33067
33381
  async onRestoreDevices(savedDevices) {
33068
33382
  const restored = /* @__PURE__ */ new Set();
33383
+ const failures = [];
33384
+ const attemptRestore = async (saved) => {
33385
+ if (restored.has(saved.id)) return;
33386
+ const Class = this.deviceClasses[saved.type];
33387
+ if (!Class) throw new Error(`no device class registered for type "${saved.type}"`);
33388
+ if (saved.parentDeviceId !== null && !restored.has(saved.parentDeviceId)) throw new Error(`parent device ${saved.parentDeviceId} not restored`);
33389
+ await this.ctx.kernel.devices.create(saved.stableId, Class, {}, saved.parentDeviceId);
33390
+ restored.add(saved.id);
33391
+ };
33069
33392
  const topLevel = savedDevices.filter((saved) => saved.parentDeviceId === null);
33070
33393
  const restoreOne = async (saved) => {
33071
- const Class = this.deviceClasses[saved.type];
33072
- if (!Class) {
33394
+ if (!this.deviceClasses[saved.type]) {
33073
33395
  this.ctx.logger.warn("No device class registered for restored type — skipping", {
33074
- tags: { stableId: saved.stableId },
33396
+ tags: {
33397
+ deviceId: saved.id,
33398
+ stableId: saved.stableId
33399
+ },
33075
33400
  meta: { type: saved.type }
33076
33401
  });
33077
33402
  return;
33078
33403
  }
33079
33404
  try {
33080
- await this.ctx.kernel.devices.create(saved.stableId, Class, {});
33081
- restored.add(saved.id);
33405
+ await attemptRestore(saved);
33082
33406
  } catch (err) {
33083
- this.ctx.logger.warn("Failed to restore device", {
33084
- tags: { stableId: saved.stableId },
33407
+ const error = err instanceof Error ? err.message : String(err);
33408
+ this.ctx.logger.warn("Failed to restore device — bounded retry scheduled", {
33409
+ tags: {
33410
+ deviceId: saved.id,
33411
+ stableId: saved.stableId
33412
+ },
33085
33413
  meta: {
33086
33414
  type: saved.type,
33087
- error: err instanceof Error ? err.message : String(err)
33415
+ attempt: 1,
33416
+ error
33088
33417
  }
33089
33418
  });
33419
+ failures.push({
33420
+ saved,
33421
+ error
33422
+ });
33090
33423
  }
33091
33424
  };
33092
33425
  await Promise.all(topLevel.map((saved) => restoreOne(saved)));
33426
+ const failedTopLevelIds = new Set(failures.map((failure) => failure.saved.id));
33093
33427
  const childRows = savedDevices.filter((s) => s.parentDeviceId !== null);
33094
33428
  for (const saved of childRows) {
33095
- const Class = this.deviceClasses[saved.type];
33096
- if (!Class) continue;
33429
+ if (!this.deviceClasses[saved.type]) continue;
33097
33430
  if (saved.parentDeviceId === null) continue;
33098
- if (!restored.has(saved.parentDeviceId)) continue;
33099
- try {
33100
- await this.ctx.kernel.devices.create(saved.stableId, Class, {}, saved.parentDeviceId);
33101
- restored.add(saved.id);
33102
- } catch (err) {
33103
- this.ctx.logger.warn("Failed to restore hub-adopted child", {
33431
+ if (restored.has(saved.parentDeviceId)) {
33432
+ try {
33433
+ await attemptRestore(saved);
33434
+ } catch (err) {
33435
+ const error = err instanceof Error ? err.message : String(err);
33436
+ this.ctx.logger.warn("Failed to restore hub-adopted child — bounded retry scheduled", {
33437
+ tags: {
33438
+ deviceId: saved.id,
33439
+ stableId: saved.stableId,
33440
+ parentDeviceId: saved.parentDeviceId
33441
+ },
33442
+ meta: {
33443
+ type: saved.type,
33444
+ attempt: 1,
33445
+ error
33446
+ }
33447
+ });
33448
+ failures.push({
33449
+ saved,
33450
+ error
33451
+ });
33452
+ }
33453
+ continue;
33454
+ }
33455
+ if (failedTopLevelIds.has(saved.parentDeviceId)) {
33456
+ this.ctx.logger.warn("Hub-adopted child deferred — parent failed initial restore", {
33104
33457
  tags: {
33458
+ deviceId: saved.id,
33105
33459
  stableId: saved.stableId,
33106
33460
  parentDeviceId: saved.parentDeviceId
33107
33461
  },
33108
- meta: {
33109
- type: saved.type,
33110
- error: err instanceof Error ? err.message : String(err)
33111
- }
33462
+ meta: { type: saved.type }
33112
33463
  });
33464
+ failures.push({
33465
+ saved,
33466
+ error: `parent device ${saved.parentDeviceId} not restored`
33467
+ });
33468
+ continue;
33113
33469
  }
33114
33470
  }
33471
+ if (failures.length > 0) this.scheduleRestoreRetries(failures, attemptRestore);
33472
+ return {
33473
+ restoredCount: restored.size,
33474
+ failedCount: failures.length
33475
+ };
33115
33476
  }
33116
33477
  /** Convert an IDevice to the flat DeviceSummary for the cap router. */
33117
33478
  toSummary(device) {
@@ -35080,6 +35441,12 @@ Object.freeze({
35080
35441
  addonId: null,
35081
35442
  access: "view"
35082
35443
  },
35444
+ "deviceProvider.reloadDevice": {
35445
+ capName: "device-provider",
35446
+ capScope: "system",
35447
+ addonId: null,
35448
+ access: "create"
35449
+ },
35083
35450
  "deviceProvider.start": {
35084
35451
  capName: "device-provider",
35085
35452
  capScope: "system",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@camstack/addon-provider-amcrest",
3
- "version": "0.2.60",
3
+ "version": "0.2.61",
4
4
  "description": "Amcrest/Dahua camera device provider addon for CamStack — Dahua CGI over HTTP(S) with digest auth (snapshot, RTSP catalog, PTZ, image/day-night config)",
5
5
  "keywords": [
6
6
  "camstack",