@camstack/system 1.2.157 → 1.2.160
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +17 -16
- package/dist/addon-runner.mjs +2 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.d.ts +10 -7
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +50 -12
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +50 -12
- package/dist/builtins/backup-orchestrator/write-targets.d.ts +63 -0
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
- package/dist/builtins/device-manager/device-bindings-store.d.ts +56 -10
- package/dist/builtins/device-manager/device-manager.addon.d.ts +11 -0
- package/dist/builtins/device-manager/device-manager.addon.js +216 -81
- package/dist/builtins/device-manager/device-manager.addon.mjs +216 -81
- package/dist/builtins/device-manager/device-meta-actions.d.ts +9 -3
- package/dist/builtins/device-manager/device-meta-store.d.ts +32 -5
- package/dist/builtins/device-manager/device-queries.d.ts +7 -3
- package/dist/builtins/device-manager/write-lock.d.ts +17 -0
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
- package/dist/builtins/sqlite-storage/filter-compiler.d.ts +41 -2
- package/dist/builtins/sqlite-storage/sqlite-settings-backend.d.ts +15 -7
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +0 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +0 -0
- package/dist/builtins/sqlite-storage/vector-index-shared.d.ts +30 -5
- package/dist/builtins/sqlite-storage/vector-index-vec.d.ts +28 -6
- package/dist/builtins/storage-orchestrator/access-guards.d.ts +32 -0
- package/dist/builtins/storage-orchestrator/location-secrets.d.ts +33 -0
- package/dist/builtins/storage-orchestrator/storage-migration-coordinator.d.ts +74 -3
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.d.ts +26 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +552 -15
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +552 -15
- package/dist/builtins/storage-orchestrator/storage-orchestrator.service.d.ts +56 -1
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-CDgIzo82.js → dist-CcvXUhHK.js} +537 -17
- package/dist/{dist-LvKus-mT.mjs → dist-DKs---Vx.mjs} +520 -18
- package/dist/index.js +141 -139
- package/dist/index.mjs +3 -3
- package/dist/kernel/deps/manifest-system-deps.d.ts +15 -0
- package/dist/kernel/index.d.ts +4 -2
- package/dist/kernel/moleculer/cap-usage-registry.d.ts +132 -7
- package/dist/kernel/storage-location-registry.d.ts +16 -1
- package/dist/kernel/transport/local-child-registry.d.ts +57 -0
- package/dist/{manifest-python-deps-CD14Pv0-.mjs → manifest-system-deps-DnoPpLGp.mjs} +466 -83
- package/dist/{manifest-python-deps-DVODn-qc.js → manifest-system-deps-oJMkWSX-.js} +477 -82
- package/dist/{retired-settings-keys-CV4QVdXj.mjs → retired-settings-keys-C3eONTvV.mjs} +1 -1
- package/dist/{retired-settings-keys-BfAzWvPC.js → retired-settings-keys-DXZ2xe7C.js} +1 -1
- package/package.json +2 -1
|
@@ -3,7 +3,7 @@ Object.defineProperties(exports, {
|
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
5
|
const require_chunk = require("../../chunk-Cek0wNdY.js");
|
|
6
|
-
const require_dist = require("../../dist-
|
|
6
|
+
const require_dist = require("../../dist-CcvXUhHK.js");
|
|
7
7
|
let node_crypto = require("node:crypto");
|
|
8
8
|
let node_fs_promises = require("node:fs/promises");
|
|
9
9
|
node_fs_promises = require_chunk.__toESM(node_fs_promises);
|
|
@@ -270,6 +270,8 @@ var BLOCKING_ONLY_CLASSES = ["galleryMedia"];
|
|
|
270
270
|
* link back to this decision.
|
|
271
271
|
*/
|
|
272
272
|
var GALLERY_FORBIDDEN_NEIGHBOUR_TYPES = ["recordings", "recordingsLow"];
|
|
273
|
+
/** The two single-flight mover engines. Lanes run in parallel; within a lane,
|
|
274
|
+
* classes run FIFO because the engine refuses a second concurrent job. */
|
|
273
275
|
var MOVE_LANES = ["media", "footage"];
|
|
274
276
|
var PARTICIPANTS = [
|
|
275
277
|
"pipeline",
|
|
@@ -282,6 +284,18 @@ var REVERSE_PARTICIPANTS = [
|
|
|
282
284
|
"pipeline"
|
|
283
285
|
];
|
|
284
286
|
/**
|
|
287
|
+
* One lane of the seal gate, as an operator reads it.
|
|
288
|
+
*
|
|
289
|
+
* "0" and "unknown" must not print the same, and neither may be silently
|
|
290
|
+
* turned into the other: `{ present: true, rows: null }` is "there are some
|
|
291
|
+
* and the count did not land", which is a refusal with a missing number, not a
|
|
292
|
+
* clean lane.
|
|
293
|
+
*/
|
|
294
|
+
function describeUnstamped(lane) {
|
|
295
|
+
if (!lane.present) return "0";
|
|
296
|
+
return lane.rows === null ? "an unknown number of" : lane.rows.toString();
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
285
299
|
* Durable orchestration layer over the independent recorder and event-media
|
|
286
300
|
* movers. It owns the only public migration state machine; participants expose
|
|
287
301
|
* intentionally narrow lease/move primitives and never alter camera settings.
|
|
@@ -316,7 +330,8 @@ var StorageMigrationCoordinator = class {
|
|
|
316
330
|
toLocationId: target.id,
|
|
317
331
|
moverJobId: null,
|
|
318
332
|
state: null,
|
|
319
|
-
error: null
|
|
333
|
+
error: null,
|
|
334
|
+
progress: null
|
|
320
335
|
});
|
|
321
336
|
if (BLOCKING_ONLY_CLASSES.includes(storageClass)) findings.push({
|
|
322
337
|
code: "blockingOnly",
|
|
@@ -328,12 +343,21 @@ var StorageMigrationCoordinator = class {
|
|
|
328
343
|
if (moves.length === 0) throw new Error("select at least one storage class");
|
|
329
344
|
if (moves.some((move) => move.storageClass === "eventMedia")) {
|
|
330
345
|
const unstamped = await this.deps.participants.analytics.countUnstamped();
|
|
331
|
-
if (unstamped
|
|
332
|
-
if (mode === "nonBlocking") throw new Error(
|
|
346
|
+
if (unstamped === null) {
|
|
347
|
+
if (mode === "nonBlocking") throw new Error("could not determine whether any event-media row still carries no locationId. A non-blocking cutover repoints the default and would silently orphan every such row, so an unmeasured collection is refused exactly like a non-empty one. Retry, or run the blocking mode, which stamps rows as it moves them.");
|
|
333
348
|
findings.push({
|
|
334
349
|
code: "unstampedEventMediaRows",
|
|
335
350
|
storageClass: "eventMedia",
|
|
336
|
-
message:
|
|
351
|
+
message: "the count of event-media rows carrying no locationId could not be taken — treat it as unknown, not as zero. This blocking migration stamps them as it moves them, so it is safe regardless; a non-blocking one would be refused."
|
|
352
|
+
});
|
|
353
|
+
} else if (unstamped.anyPresent) {
|
|
354
|
+
const scale = unstamped.total === null ? "an unknown number of" : unstamped.total.toString();
|
|
355
|
+
const perLane = `${describeUnstamped(unstamped.media)} media, ${describeUnstamped(unstamped.retrainFrames)} retrain frames`;
|
|
356
|
+
if (mode === "nonBlocking") throw new Error(`${scale} event-media row(s) still carry no locationId (${perLane}). A non-blocking cutover would silently orphan them. Run the seal first: pipelineAnalytics.relocateMedia({ mode: "seal", toLocationId: "${this.deps.locations.getDefaultLocation("eventMedia")?.id ?? "eventMedia"}" }).`);
|
|
357
|
+
findings.push({
|
|
358
|
+
code: "unstampedEventMediaRows",
|
|
359
|
+
storageClass: "eventMedia",
|
|
360
|
+
message: `${scale} event-media row(s) carry no locationId (${perLane}). This blocking migration stamps them as it moves them, but a non-blocking one would be refused until a seal pass drives the count to zero.`
|
|
337
361
|
});
|
|
338
362
|
}
|
|
339
363
|
}
|
|
@@ -414,7 +438,8 @@ var StorageMigrationCoordinator = class {
|
|
|
414
438
|
...move,
|
|
415
439
|
moverJobId: null,
|
|
416
440
|
state: null,
|
|
417
|
-
error: null
|
|
441
|
+
error: null,
|
|
442
|
+
progress: null
|
|
418
443
|
})),
|
|
419
444
|
pauseLeaseId: null,
|
|
420
445
|
pausedParticipants: [],
|
|
@@ -448,6 +473,178 @@ var StorageMigrationCoordinator = class {
|
|
|
448
473
|
await Promise.all(job.moves.filter((move) => move.moverJobId !== null).map((move) => this.cancelMove(move)));
|
|
449
474
|
return true;
|
|
450
475
|
}
|
|
476
|
+
/**
|
|
477
|
+
* Every mover running right now, in both lanes — including the ones no
|
|
478
|
+
* migration armed.
|
|
479
|
+
*
|
|
480
|
+
* `status` already carries a migration's own progress (the coordinator folds
|
|
481
|
+
* it onto each move from the poll it is already doing). This exists for the
|
|
482
|
+
* other half: `recording.relocateFootage` and `pipelineAnalytics.relocateMedia`
|
|
483
|
+
* are operator-callable, and until {@link drain} existed that was the only way
|
|
484
|
+
* to run a drain at all. Such a mover has no job to fold into, so without this
|
|
485
|
+
* read a five-hour operation is invisible in the UI.
|
|
486
|
+
*
|
|
487
|
+
* `migrationJobId` is best-effort by construction: the coordinator keeps ONE
|
|
488
|
+
* durable job, so a mover armed by an older, since-overwritten migration
|
|
489
|
+
* reports `null`. That is the honest answer — nothing here can still claim it.
|
|
490
|
+
*/
|
|
491
|
+
async movers() {
|
|
492
|
+
const job = await this.status();
|
|
493
|
+
const owned = /* @__PURE__ */ new Map();
|
|
494
|
+
for (const move of job?.moves ?? []) if (move.moverJobId !== null) owned.set(move.moverJobId, job?.jobId ?? "");
|
|
495
|
+
const [footage, media] = await Promise.all([this.deps.participants.recorder.listMovers(), this.deps.participants.analytics.listMovers()]);
|
|
496
|
+
const observedAt = this.deps.now();
|
|
497
|
+
const label = (lane, jobs) => jobs.map((mover) => ({
|
|
498
|
+
lane,
|
|
499
|
+
job: mover,
|
|
500
|
+
migrationJobId: owned.get(mover.jobId) ?? null,
|
|
501
|
+
observedAt
|
|
502
|
+
}));
|
|
503
|
+
return [...label("footage", footage), ...label("media", media)];
|
|
504
|
+
}
|
|
505
|
+
/**
|
|
506
|
+
* What every class's source STILL holds — the census behind a "drain
|
|
507
|
+
* remaining" action.
|
|
508
|
+
*
|
|
509
|
+
* A class appears here only when something is (or might be) left on a
|
|
510
|
+
* location that is not its default. An empty result therefore means exactly
|
|
511
|
+
* "there is nothing to drain", which is what lets the UI offer the action
|
|
512
|
+
* only when it is true, and what lets {@link drain} refuse rather than start
|
|
513
|
+
* a job that would move nothing and report `done` — the failure mode D295
|
|
514
|
+
* exists to end.
|
|
515
|
+
*
|
|
516
|
+
* `items: null` is "the archive could not be asked" and is still listed. A
|
|
517
|
+
* residue nobody could measure is the case an operator most needs to see;
|
|
518
|
+
* dropping it because the read failed would be the quiet success again.
|
|
519
|
+
*/
|
|
520
|
+
async residue() {
|
|
521
|
+
const out = [];
|
|
522
|
+
for (const storageClass of STORAGE_CLASSES) {
|
|
523
|
+
if (!MOVER_CLASSES.includes(storageClass)) continue;
|
|
524
|
+
const target = this.deps.locations.getDefaultLocation(storageClass);
|
|
525
|
+
if (!target) continue;
|
|
526
|
+
if (laneOf(storageClass) === "media") {
|
|
527
|
+
const count = await unanswerable(this.deps.participants.analytics.residue({
|
|
528
|
+
toLocationId: target.id,
|
|
529
|
+
mode: storageClass === "galleryMedia" ? "gallery" : "move"
|
|
530
|
+
}));
|
|
531
|
+
if (count !== null && count.rows === 0) continue;
|
|
532
|
+
out.push({
|
|
533
|
+
storageClass,
|
|
534
|
+
fromLocationId: "*",
|
|
535
|
+
toLocationId: target.id,
|
|
536
|
+
items: count?.rows ?? null,
|
|
537
|
+
bytes: null
|
|
538
|
+
});
|
|
539
|
+
continue;
|
|
540
|
+
}
|
|
541
|
+
for (const source of this.deps.locations.listLocations({ type: storageClass })) {
|
|
542
|
+
if (source.id === target.id) continue;
|
|
543
|
+
const census = await unanswerable(this.deps.participants.recorder.residue({
|
|
544
|
+
fromLocationId: source.id,
|
|
545
|
+
footageClass: storageClass === "recordingsLow" ? "recordingsLow" : "recordings"
|
|
546
|
+
}));
|
|
547
|
+
if (census !== null && census.segments === 0) continue;
|
|
548
|
+
out.push({
|
|
549
|
+
storageClass,
|
|
550
|
+
fromLocationId: source.id,
|
|
551
|
+
toLocationId: target.id,
|
|
552
|
+
items: census?.segments ?? null,
|
|
553
|
+
bytes: census?.bytes ?? null
|
|
554
|
+
});
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
return out;
|
|
558
|
+
}
|
|
559
|
+
/**
|
|
560
|
+
* Run the DRAIN half alone, against classes whose default has already moved.
|
|
561
|
+
*
|
|
562
|
+
* ## Why this is a second verb rather than a looser `start`
|
|
563
|
+
*
|
|
564
|
+
* `start` refuses a destination that is already the class's default
|
|
565
|
+
* (`"recordingsLow:ssd" is already the "recordingsLow" default`). That refusal
|
|
566
|
+
* is correct and it is load-bearing: there is genuinely nothing left to
|
|
567
|
+
* repoint, and an operator must never be able to re-repoint a migrated class
|
|
568
|
+
* by accident. Making `start` idempotent — "an already-repointed class
|
|
569
|
+
* proceeds straight to draining" — would delete that protection AND make the
|
|
570
|
+
* verb mean two different things depending on state, so the confirmation an
|
|
571
|
+
* operator reads ("pauses the writers…") would be a lie half the time.
|
|
572
|
+
*
|
|
573
|
+
* `drain` instead cannot repoint AT ALL: it never touches
|
|
574
|
+
* `setDefaultLocations`, and its job starts in `draining` with `repointed`
|
|
575
|
+
* already true, so the `repointing` / `refreshing` / `resuming` blocks of
|
|
576
|
+
* {@link run} are behind it and unreachable. The two verbs are disjoint, and
|
|
577
|
+
* `start`'s refusal keeps meaning exactly what it meant.
|
|
578
|
+
*
|
|
579
|
+
* ## Why it re-derives the work instead of resuming the old job
|
|
580
|
+
*
|
|
581
|
+
* The finished job is the audit of what happened; re-opening it destroys
|
|
582
|
+
* that. And a drain is needed in cases where no migration job ever existed
|
|
583
|
+
* (a mover armed by hand, footage stranded on a location an operator added
|
|
584
|
+
* and then un-defaulted). One job = one operation, and the work list comes
|
|
585
|
+
* from {@link residue} — the archive — not from what a previous job believed.
|
|
586
|
+
*/
|
|
587
|
+
async drain(input) {
|
|
588
|
+
if (this.startReserved) throw new Error("storage migration is already active");
|
|
589
|
+
this.startReserved = true;
|
|
590
|
+
try {
|
|
591
|
+
const existing = await this.status();
|
|
592
|
+
if (existing && isTerminal(existing) && existing.pausedParticipants.length > 0) {
|
|
593
|
+
await this.releaseAfterTerminal(existing);
|
|
594
|
+
await this.persist(existing);
|
|
595
|
+
if (existing.pausedParticipants.length > 0) throw new Error(`storage migration ${existing.jobId} still holds maintenance leases`);
|
|
596
|
+
}
|
|
597
|
+
if (existing && !isTerminal(existing)) throw new Error(`storage migration is already active (${existing.jobId})`);
|
|
598
|
+
for (const storageClass of input.classes) {
|
|
599
|
+
if (MOVER_CLASSES.includes(storageClass)) continue;
|
|
600
|
+
throw new Error(`No mover owns "${storageClass}" — there is nothing that can drain it. Move it by hand.`);
|
|
601
|
+
}
|
|
602
|
+
const residue = await this.residue();
|
|
603
|
+
const moves = [];
|
|
604
|
+
const destinations = {};
|
|
605
|
+
for (const storageClass of input.classes) {
|
|
606
|
+
const remaining = residue.filter((entry) => entry.storageClass === storageClass);
|
|
607
|
+
if (remaining.length === 0) throw new Error(`"${storageClass}" has nothing left outside its default location — there is nothing to drain. To CHANGE where it writes, use storageMigration.start with a new destination.`);
|
|
608
|
+
for (const entry of remaining) {
|
|
609
|
+
if (this.deps.locations.getLocationById(entry.toLocationId)?.enabled === false) throw new Error(`The "${storageClass}" drain destination "${entry.toLocationId}" is DISABLED — a disabled location takes no writes, and a drain is the largest write there is. Enable it on the Storage screen, or make an enabled location the "${storageClass}" default first.`);
|
|
610
|
+
moves.push({
|
|
611
|
+
storageClass,
|
|
612
|
+
fromLocationId: entry.fromLocationId,
|
|
613
|
+
toLocationId: entry.toLocationId,
|
|
614
|
+
moverJobId: null,
|
|
615
|
+
state: null,
|
|
616
|
+
error: null,
|
|
617
|
+
progress: null
|
|
618
|
+
});
|
|
619
|
+
destinations[storageClass] = entry.toLocationId;
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
const now = this.deps.now();
|
|
623
|
+
const job = {
|
|
624
|
+
jobId: this.deps.newId(),
|
|
625
|
+
phase: "draining",
|
|
626
|
+
mode: "nonBlocking",
|
|
627
|
+
destinations,
|
|
628
|
+
throttleMbps: input.throttleMbps ?? 40,
|
|
629
|
+
moves,
|
|
630
|
+
pauseLeaseId: null,
|
|
631
|
+
pausedParticipants: [],
|
|
632
|
+
repointed: true,
|
|
633
|
+
cancelRequested: false,
|
|
634
|
+
startedAt: now,
|
|
635
|
+
updatedAt: now,
|
|
636
|
+
finishedAt: null,
|
|
637
|
+
error: null
|
|
638
|
+
};
|
|
639
|
+
await this.persist(job);
|
|
640
|
+
this.active = job;
|
|
641
|
+
this.runPromise = this.run(job);
|
|
642
|
+
this.runPromise;
|
|
643
|
+
return job.jobId;
|
|
644
|
+
} finally {
|
|
645
|
+
this.startReserved = false;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
451
648
|
/** Boot recovery resumes a durable unfinished state. A missing in-memory
|
|
452
649
|
* child mover is recreated from the same copy-if-absent input. */
|
|
453
650
|
async recover() {
|
|
@@ -656,6 +853,14 @@ var StorageMigrationCoordinator = class {
|
|
|
656
853
|
}
|
|
657
854
|
move.state = status.state;
|
|
658
855
|
move.error = status.error;
|
|
856
|
+
move.progress = {
|
|
857
|
+
filesMoved: status.filesMoved,
|
|
858
|
+
filesTotal: status.filesTotal,
|
|
859
|
+
bytesMoved: status.bytesMoved,
|
|
860
|
+
...status.rowsReconciled === void 0 ? {} : { rowsReconciled: status.rowsReconciled },
|
|
861
|
+
startedAt: status.startedAt,
|
|
862
|
+
observedAt: this.deps.now()
|
|
863
|
+
};
|
|
659
864
|
if (status.state === "failed") throw new Error(move.error ?? `${move.storageClass} move failed`);
|
|
660
865
|
if (status.state === "cancelled") {
|
|
661
866
|
job.cancelRequested = true;
|
|
@@ -675,9 +880,10 @@ var StorageMigrationCoordinator = class {
|
|
|
675
880
|
*
|
|
676
881
|
* The gate is the RE-COUNT, not the seal job's terminal state: a seal that
|
|
677
882
|
* failed some rows still finishes, and "finished" is not "there are none
|
|
678
|
-
* left". A
|
|
883
|
+
* left". A remaining row here fails the job while nothing has been paused
|
|
679
884
|
* and nothing has been repointed, which is the cheapest possible place to
|
|
680
|
-
* discover it.
|
|
885
|
+
* discover it. So does a re-count that could not be TAKEN: the gate opens on
|
|
886
|
+
* a measured absence, and only on that.
|
|
681
887
|
*
|
|
682
888
|
* The seal's own mover job id is deliberately NOT durable. It is idempotent
|
|
683
889
|
* and cheap, so a coordinator restart mid-seal simply re-runs the whole
|
|
@@ -687,7 +893,8 @@ var StorageMigrationCoordinator = class {
|
|
|
687
893
|
async sealEventMedia(job) {
|
|
688
894
|
const move = job.moves.find((candidate) => candidate.storageClass === "eventMedia");
|
|
689
895
|
if (move === void 0) return;
|
|
690
|
-
|
|
896
|
+
const before = await this.deps.participants.analytics.countUnstamped();
|
|
897
|
+
if (before === null || before.anyPresent) {
|
|
691
898
|
const started = await this.deps.participants.analytics.startDrain({
|
|
692
899
|
toLocationId: move.fromLocationId,
|
|
693
900
|
mode: "seal"
|
|
@@ -696,7 +903,8 @@ var StorageMigrationCoordinator = class {
|
|
|
696
903
|
if (job.cancelRequested) return;
|
|
697
904
|
}
|
|
698
905
|
const after = await this.deps.participants.analytics.countUnstamped();
|
|
699
|
-
if (after
|
|
906
|
+
if (after === null) throw new Error("event-media seal ran, but whether any row still carries no locationId could not be measured afterwards. Unknown is not zero; nothing has been paused or repointed.");
|
|
907
|
+
if (after.anyPresent) throw new Error(`event-media seal left ${after.total === null ? "an unknown number of" : after.total.toString()} row(s) without a locationId (${describeUnstamped(after.media)} media, ${describeUnstamped(after.retrainFrames)} retrain frames). A non-blocking cutover would silently orphan them; nothing has been paused or repointed.`);
|
|
700
908
|
}
|
|
701
909
|
async waitForSeal(job, moverJobId) {
|
|
702
910
|
const sleep = this.deps.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
@@ -775,6 +983,25 @@ var StorageMigrationCoordinator = class {
|
|
|
775
983
|
await this.deps.state.set(job);
|
|
776
984
|
}
|
|
777
985
|
};
|
|
986
|
+
/**
|
|
987
|
+
* A residue read that FAILED is unanswerable, not zero — and it must not take
|
|
988
|
+
* the other lane down with it.
|
|
989
|
+
*
|
|
990
|
+
* `residue` asks two addons. One of them being mid-restart is an ordinary
|
|
991
|
+
* event, and letting that throw out of the whole call would blank the panel:
|
|
992
|
+
* an operator with 80 GB stranded on a recordings disk would be told nothing,
|
|
993
|
+
* because post-analysis was rebooting. `null` is the same "unknown" the
|
|
994
|
+
* participants themselves return for a read they could not take, so the entry
|
|
995
|
+
* is still LISTED and still labelled unknown — a drain attempted against it
|
|
996
|
+
* then fails loudly with the participant's own message, which is the point.
|
|
997
|
+
*/
|
|
998
|
+
async function unanswerable(read) {
|
|
999
|
+
try {
|
|
1000
|
+
return await read;
|
|
1001
|
+
} catch {
|
|
1002
|
+
return null;
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
778
1005
|
function requireLease(job) {
|
|
779
1006
|
if (job.pauseLeaseId === null) throw new Error("storage migration has no maintenance lease");
|
|
780
1007
|
return job.pauseLeaseId;
|
|
@@ -817,6 +1044,12 @@ function runsUnleased(job) {
|
|
|
817
1044
|
* - **Cardinality conflict throws**: if two addons declare the same `id` with
|
|
818
1045
|
* different `cardinality` values an `Error` is thrown immediately (message
|
|
819
1046
|
* contains the words "cardinality" and the conflicting `id`).
|
|
1047
|
+
* - **Access conflict throws**, for the same reason and more sharply: the
|
|
1048
|
+
* EFFECTIVE access (declared, else `STORAGE_ACCESS_FALLBACK`) must agree.
|
|
1049
|
+
* "First declarer wins" is the right rule for a display name and a
|
|
1050
|
+
* catastrophic one for a safety constraint — it would let a second addon
|
|
1051
|
+
* silently widen a kind that the first one restricted, which is exactly the
|
|
1052
|
+
* accident this constraint exists to make impossible.
|
|
820
1053
|
*
|
|
821
1054
|
* @param perAddon - One entry per addon; each entry is that addon's
|
|
822
1055
|
* `storageLocations` declaration array.
|
|
@@ -827,6 +1060,9 @@ function buildStorageLocationRegistry(perAddon) {
|
|
|
827
1060
|
const existing = map.get(declaration.id);
|
|
828
1061
|
if (existing !== void 0) {
|
|
829
1062
|
if (existing.cardinality !== declaration.cardinality) throw new Error(`Storage location cardinality conflict for id "${declaration.id}": "${existing.cardinality}" (first declarer) vs "${declaration.cardinality}" (later declarer). All addons declaring the same storage location id must agree on cardinality.`);
|
|
1063
|
+
const existingAccess = existing.access ?? "local-path";
|
|
1064
|
+
const declaredAccess = declaration.access ?? "local-path";
|
|
1065
|
+
if (existingAccess !== declaredAccess) throw new Error(`Storage location access conflict for id "${declaration.id}": "${existingAccess}" (first declarer) vs "${declaredAccess}" (later declarer). All addons declaring the same storage location id must agree on access — a later declaration must never widen what an earlier one restricted.`);
|
|
830
1066
|
continue;
|
|
831
1067
|
}
|
|
832
1068
|
map.set(declaration.id, declaration);
|
|
@@ -842,12 +1078,53 @@ function buildStorageLocationRegistry(perAddon) {
|
|
|
842
1078
|
cardinalityOf(id) {
|
|
843
1079
|
return map.get(id)?.cardinality ?? null;
|
|
844
1080
|
},
|
|
1081
|
+
accessOf(id) {
|
|
1082
|
+
const declaration = map.get(id);
|
|
1083
|
+
if (declaration === void 0) return null;
|
|
1084
|
+
return declaration.access ?? "local-path";
|
|
1085
|
+
},
|
|
845
1086
|
list() {
|
|
846
1087
|
return frozen;
|
|
847
1088
|
}
|
|
848
1089
|
};
|
|
849
1090
|
}
|
|
850
1091
|
//#endregion
|
|
1092
|
+
//#region src/builtins/storage-orchestrator/access-guards.ts
|
|
1093
|
+
/**
|
|
1094
|
+
* May `storage.resolve` return a path for this location?
|
|
1095
|
+
*
|
|
1096
|
+
* `resolve` is the only method on the cap whose result LEAVES the storage
|
|
1097
|
+
* abstraction — every caller of it opens the string with `node:fs`. A remote
|
|
1098
|
+
* provider answers with a path on the remote host, which on this node is
|
|
1099
|
+
* either missing or an unrelated local directory of the same name. For the
|
|
1100
|
+
* recorder, writing to the wrong place is not an error the operator sees; it
|
|
1101
|
+
* is a black window in the footage.
|
|
1102
|
+
*
|
|
1103
|
+
* Only a POSITIVE `false` refuses. `undefined` is "not yet classified", not
|
|
1104
|
+
* "remote" — and a conclusion drawn from a read that was never made is how a
|
|
1105
|
+
* doorbell rang 0/3 (D49).
|
|
1106
|
+
*/
|
|
1107
|
+
function resolveRefusalFor(location, locality) {
|
|
1108
|
+
if (locality !== false) return null;
|
|
1109
|
+
return `storage.resolve is a request for a path this node can open with node:fs, and location "${location.id}" is served by the remote provider "${location.providerId}" (nodeLocal: false) — the path it would return exists on the remote host, not here. Read and write this location through the storage cap (read/write, or beginUpload/writeChunk/finalizeUpload) instead.`;
|
|
1110
|
+
}
|
|
1111
|
+
/**
|
|
1112
|
+
* Can this node's filesystem answer "does this location still hold anything?"
|
|
1113
|
+
*
|
|
1114
|
+
* Only for a provider it KNOWS to be node-local. Everything else is `unknown`,
|
|
1115
|
+
* which refuses a delete.
|
|
1116
|
+
*
|
|
1117
|
+
* The direction that matters is the one this closes. A remote location carries
|
|
1118
|
+
* a plain-looking `basePath` (`/backups/camstack`); `fs.readdir` of it on the
|
|
1119
|
+
* hub answers `ENOENT`, and `ENOENT` is deliberately read as `empty` — because
|
|
1120
|
+
* for a LOCAL location an absent root means there is nothing the delete can
|
|
1121
|
+
* strand. Applied to a remote row, that same reasoning turns a full share into
|
|
1122
|
+
* a vacant one and walks D293's delete guard straight past it.
|
|
1123
|
+
*/
|
|
1124
|
+
function canProbeOccupancyLocally(locality) {
|
|
1125
|
+
return locality === true;
|
|
1126
|
+
}
|
|
1127
|
+
//#endregion
|
|
851
1128
|
//#region src/builtins/storage-orchestrator/data-store-dispatch.ts
|
|
852
1129
|
/**
|
|
853
1130
|
* Pick the engine to serve a call.
|
|
@@ -899,6 +1176,106 @@ function createDataStoreDispatch(getEngines) {
|
|
|
899
1176
|
};
|
|
900
1177
|
}
|
|
901
1178
|
//#endregion
|
|
1179
|
+
//#region src/builtins/storage-orchestrator/location-secrets.ts
|
|
1180
|
+
/**
|
|
1181
|
+
* Credential redaction for `StorageLocation.config`.
|
|
1182
|
+
*
|
|
1183
|
+
* ── What was wrong ─────────────────────────────────────────────────────────
|
|
1184
|
+
* `storage.listLocations` carries no `auth` override, so it is `protected` —
|
|
1185
|
+
* every authenticated session reaches it, admin or not. The orchestrator
|
|
1186
|
+
* spread each stored row verbatim. `StorageLocation.config` is
|
|
1187
|
+
* `z.record(z.string(), z.unknown())`, and the SFTP / S3 / WebDAV providers
|
|
1188
|
+
* already declare `password`, `privateKey`, `privateKeyPassphrase`,
|
|
1189
|
+
* `secretAccessKey` and `token` inside it. Nothing was exposed only because
|
|
1190
|
+
* every location on the live hub is `filesystem-storage`, whose config is
|
|
1191
|
+
* `basePath` + `maxUsedGb`. The exposure arrives with the first remote
|
|
1192
|
+
* location, and three providers that collect one already ship.
|
|
1193
|
+
*
|
|
1194
|
+
* ── The mechanism ──────────────────────────────────────────────────────────
|
|
1195
|
+
* Schema-driven, never a hardcoded key list: each provider's
|
|
1196
|
+
* `getProviderInfo().configSchema` already says which of its fields hold a
|
|
1197
|
+
* credential, so it is the one thing that cannot drift from the provider.
|
|
1198
|
+
*
|
|
1199
|
+
* Two directions, and both are needed:
|
|
1200
|
+
* - **out** — {@link redactLocationConfig} replaces every secret value with
|
|
1201
|
+
* `REDACTED_SECRET` on the way to a caller.
|
|
1202
|
+
* - **in** — {@link restoreRedactedSecrets} treats that same sentinel, sent
|
|
1203
|
+
* back on an upsert, as "keep what is stored". Without it the wizard's
|
|
1204
|
+
* edit-and-save round trip would write the sentinel into the config and
|
|
1205
|
+
* lock the operator out of their own share on the first rename.
|
|
1206
|
+
*
|
|
1207
|
+
* ── The trap this deliberately avoids ──────────────────────────────────────
|
|
1208
|
+
* Keying redaction off `type: 'password'` alone MISSES a secret parked in a
|
|
1209
|
+
* `textarea`, and the SFTP provider parks an entire PEM private key there.
|
|
1210
|
+
* Discovery goes through `collectSecretConfigKeys`, whose discriminator is
|
|
1211
|
+
* `type === 'password' || secret === true`. A redaction that quietly misses a
|
|
1212
|
+
* credential is worse than none: it is a leak plus an assurance there is not
|
|
1213
|
+
* one.
|
|
1214
|
+
*/
|
|
1215
|
+
/** Shared empty set for providers that declare no secrets (or are unknown). */
|
|
1216
|
+
var EMPTY_SECRET_KEYS = /* @__PURE__ */ new Set();
|
|
1217
|
+
/** Secret config keys a provider declares, derived from its own form schema. */
|
|
1218
|
+
function secretKeysOfProviderInfo(info) {
|
|
1219
|
+
return require_dist.collectSecretConfigKeys(info.configSchema);
|
|
1220
|
+
}
|
|
1221
|
+
/**
|
|
1222
|
+
* A copy of `config` with every declared secret replaced by the sentinel.
|
|
1223
|
+
* Returns the SAME object when there is nothing to redact, so the common case
|
|
1224
|
+
* (a filesystem location) allocates nothing.
|
|
1225
|
+
*
|
|
1226
|
+
* A key the provider declares secret but that is absent from the stored config
|
|
1227
|
+
* is NOT invented — an absent password must keep reading as absent, or the
|
|
1228
|
+
* wizard would show a credential where none is set and the write-back would
|
|
1229
|
+
* then "keep" a value that never existed.
|
|
1230
|
+
*/
|
|
1231
|
+
function redactLocationConfig(config, secretKeys) {
|
|
1232
|
+
if (secretKeys.size === 0) return config;
|
|
1233
|
+
let touched = false;
|
|
1234
|
+
const out = {};
|
|
1235
|
+
for (const [key, value] of Object.entries(config)) {
|
|
1236
|
+
if (secretKeys.has(key) && value !== void 0 && value !== null && value !== "") {
|
|
1237
|
+
out[key] = require_dist.REDACTED_SECRET;
|
|
1238
|
+
touched = true;
|
|
1239
|
+
continue;
|
|
1240
|
+
}
|
|
1241
|
+
out[key] = value;
|
|
1242
|
+
}
|
|
1243
|
+
return touched ? out : config;
|
|
1244
|
+
}
|
|
1245
|
+
/** The same, applied to a whole location record. */
|
|
1246
|
+
function redactLocation(location, secretKeys) {
|
|
1247
|
+
const config = redactLocationConfig(location.config, secretKeys);
|
|
1248
|
+
if (config === location.config) return location;
|
|
1249
|
+
return {
|
|
1250
|
+
...location,
|
|
1251
|
+
config
|
|
1252
|
+
};
|
|
1253
|
+
}
|
|
1254
|
+
/**
|
|
1255
|
+
* Undo the redaction on the way IN: any secret key whose incoming value is the
|
|
1256
|
+
* sentinel is restored from what is already stored.
|
|
1257
|
+
*
|
|
1258
|
+
* A sentinel with NOTHING stored behind it is dropped rather than persisted —
|
|
1259
|
+
* writing the literal `__camstack_redacted__` as a password is a
|
|
1260
|
+
* lock-yourself-out bug that would only surface at the next connection
|
|
1261
|
+
* attempt, hours later, as an authentication failure with no explanation.
|
|
1262
|
+
*/
|
|
1263
|
+
function restoreRedactedSecrets(incoming, stored, secretKeys) {
|
|
1264
|
+
if (secretKeys.size === 0) return incoming;
|
|
1265
|
+
let touched = false;
|
|
1266
|
+
const out = {};
|
|
1267
|
+
for (const [key, value] of Object.entries(incoming)) {
|
|
1268
|
+
if (value !== "__camstack_redacted__" || !secretKeys.has(key)) {
|
|
1269
|
+
out[key] = value;
|
|
1270
|
+
continue;
|
|
1271
|
+
}
|
|
1272
|
+
touched = true;
|
|
1273
|
+
const previous = stored?.[key];
|
|
1274
|
+
if (previous !== void 0) out[key] = previous;
|
|
1275
|
+
}
|
|
1276
|
+
return touched ? out : incoming;
|
|
1277
|
+
}
|
|
1278
|
+
//#endregion
|
|
902
1279
|
//#region src/builtins/storage-orchestrator/provider-discovery.ts
|
|
903
1280
|
async function collectProviderInfos(providers, onError) {
|
|
904
1281
|
const out = [];
|
|
@@ -1075,6 +1452,34 @@ var StorageOrchestratorService = class {
|
|
|
1075
1452
|
backfilled: backfill.length,
|
|
1076
1453
|
total: this.locations.size
|
|
1077
1454
|
} });
|
|
1455
|
+
this.reportDisabledDefaults();
|
|
1456
|
+
}
|
|
1457
|
+
/**
|
|
1458
|
+
* Shout about a persisted row that is BOTH the type default and disabled.
|
|
1459
|
+
*
|
|
1460
|
+
* `upsertLocation` cannot produce this (`resolveEnabled` force-enables a
|
|
1461
|
+
* default) and neither can `setDefaultLocations` — but hydrate writes rows
|
|
1462
|
+
* straight into the map without either, so a hand-edited store, a partial
|
|
1463
|
+
* migration or a row written by another build can. It matters because every
|
|
1464
|
+
* bare-type ref resolves through {@link getDefaultLocation}: on such a row
|
|
1465
|
+
* `storage.write({ location: 'eventMedia' })` and the migration coordinator's
|
|
1466
|
+
* drain destination both point at a disk the operator turned off. That is the
|
|
1467
|
+
* one thing this model must never do silently.
|
|
1468
|
+
*
|
|
1469
|
+
* Reported, never repaired: flipping an operator's flag back on at boot would
|
|
1470
|
+
* be the system overruling the switch instead of obeying it. The coordinator
|
|
1471
|
+
* refuses the drain (`storage-migration-coordinator.drain`); the bare-ref read
|
|
1472
|
+
* path is deliberately left working, because narrowing a READ is a worse bug
|
|
1473
|
+
* than the one this line reports.
|
|
1474
|
+
*/
|
|
1475
|
+
reportDisabledDefaults() {
|
|
1476
|
+
for (const loc of this.locations.values()) {
|
|
1477
|
+
if (!loc.isDefault || loc.enabled !== false) continue;
|
|
1478
|
+
this.logger.error("storage-orchestrator: the type DEFAULT is disabled — every bare-type ref for this type resolves to a location the operator turned off. Enable it, or make another location the default.", { meta: {
|
|
1479
|
+
id: loc.id,
|
|
1480
|
+
type: loc.type
|
|
1481
|
+
} });
|
|
1482
|
+
}
|
|
1078
1483
|
}
|
|
1079
1484
|
/**
|
|
1080
1485
|
* Inject the declaration-driven cardinality source. Called once by the
|
|
@@ -1146,6 +1551,7 @@ var StorageOrchestratorService = class {
|
|
|
1146
1551
|
loaded: this.locations.size,
|
|
1147
1552
|
isSystemUpgraded: upgraded
|
|
1148
1553
|
} });
|
|
1554
|
+
this.reportDisabledDefaults();
|
|
1149
1555
|
}
|
|
1150
1556
|
/**
|
|
1151
1557
|
* Boot backfill (SP1): stamp `nodeId` on every persisted node-local
|
|
@@ -1255,6 +1661,66 @@ var StorageOrchestratorService = class {
|
|
|
1255
1661
|
* implicitly-demoted siblings are persisted before the in-memory map
|
|
1256
1662
|
* mutation returns. Persistence errors propagate to the caller.
|
|
1257
1663
|
*/
|
|
1664
|
+
/**
|
|
1665
|
+
* Refuse a `(location kind, provider)` pair the kind cannot use.
|
|
1666
|
+
*
|
|
1667
|
+
* The rule, in one line: a `'local-path'` kind requires a provider whose
|
|
1668
|
+
* `getProviderInfo().nodeLocal` is `true`.
|
|
1669
|
+
*
|
|
1670
|
+
* `nodeLocal` is the honest name for "this provider's `resolve` returns a
|
|
1671
|
+
* path on the filesystem of the node that resolved it". The
|
|
1672
|
+
* `storage-provider` cap's own discriminated union already forces every
|
|
1673
|
+
* provider to declare it, and all four remote providers declare `false`, so
|
|
1674
|
+
* no new cap surface is needed to ask the question.
|
|
1675
|
+
*
|
|
1676
|
+
* Three-valued on purpose. Only a POSITIVE `false` refuses:
|
|
1677
|
+
* - `true` → node-local, always fine.
|
|
1678
|
+
* - `false` → the provider is known and known to be remote. REFUSE.
|
|
1679
|
+
* - `undefined` → the provider has not registered yet (early boot, an addon
|
|
1680
|
+
* still loading). That is "unknown", not "remote", and a read that could
|
|
1681
|
+
* not be made must never destroy work (D49) — the system seed runs
|
|
1682
|
+
* through this path before any provider registers. Allowed, and logged,
|
|
1683
|
+
* so the allowance is never silent.
|
|
1684
|
+
*
|
|
1685
|
+
* The DECLARATION side is read the same way. A kind the registry does not
|
|
1686
|
+
* know is `local-path` (fail-closed: an unknown kind is not a permissive
|
|
1687
|
+
* one), but NO REGISTRY AT ALL is a different statement — nothing has been
|
|
1688
|
+
* loaded, so nothing can be concluded about any kind. The addon injects the
|
|
1689
|
+
* registry inside `onInitialize`, before the `storage` cap is mounted, so
|
|
1690
|
+
* there is no window in which an operator upsert sees this branch; a service
|
|
1691
|
+
* constructed without one is the in-memory/early-boot path.
|
|
1692
|
+
*/
|
|
1693
|
+
refuseIncompatibleProvider(input) {
|
|
1694
|
+
const registry = this.registry;
|
|
1695
|
+
if (registry === null) {
|
|
1696
|
+
this.logger.debug("storage-orchestrator: no location declarations loaded — access constraint not evaluated", { meta: {
|
|
1697
|
+
id: input.id,
|
|
1698
|
+
type: input.type,
|
|
1699
|
+
providerId: input.providerId
|
|
1700
|
+
} });
|
|
1701
|
+
return;
|
|
1702
|
+
}
|
|
1703
|
+
const access = registry.accessOf(input.type) ?? "local-path";
|
|
1704
|
+
if (access !== "local-path") return;
|
|
1705
|
+
const nodeLocal = this.nodeLocalResolver?.(input.providerId);
|
|
1706
|
+
if (nodeLocal === true) return;
|
|
1707
|
+
if (nodeLocal === void 0) {
|
|
1708
|
+
this.logger.debug("storage-orchestrator: provider not yet classified — allowing a local-path upsert", { meta: {
|
|
1709
|
+
id: input.id,
|
|
1710
|
+
type: input.type,
|
|
1711
|
+
providerId: input.providerId,
|
|
1712
|
+
access
|
|
1713
|
+
} });
|
|
1714
|
+
return;
|
|
1715
|
+
}
|
|
1716
|
+
this.logger.warn("storage-orchestrator: REFUSED a remote provider for a local-path kind", { meta: {
|
|
1717
|
+
id: input.id,
|
|
1718
|
+
type: input.type,
|
|
1719
|
+
providerId: input.providerId,
|
|
1720
|
+
access
|
|
1721
|
+
} });
|
|
1722
|
+
throw new Error(`Storage kind "${input.type}" is declared access "local-path": the service that owns it reads and writes its bytes with node:fs on the path "storage.resolve" returns, so it can only be backed by a node-local provider. Provider "${input.providerId}" is remote (nodeLocal: false) and its resolved paths do not exist on this node. Refusing location "${input.id}".`);
|
|
1723
|
+
}
|
|
1258
1724
|
upsertLocation(input) {
|
|
1259
1725
|
const now = Date.now();
|
|
1260
1726
|
const existing = this.locations.get(input.id);
|
|
@@ -1264,6 +1730,7 @@ var StorageOrchestratorService = class {
|
|
|
1264
1730
|
if (already) throw new Error(`Storage type "${input.type}" is single — only one location allowed (existing: "${already.id}"). Edit it instead of adding a new one.`);
|
|
1265
1731
|
}
|
|
1266
1732
|
}
|
|
1733
|
+
this.refuseIncompatibleProvider(input);
|
|
1267
1734
|
if (this.nodeLocalResolver?.(input.providerId) === true && !input.nodeId) input = {
|
|
1268
1735
|
...input,
|
|
1269
1736
|
nodeId: "hub"
|
|
@@ -1775,6 +2242,19 @@ var StorageOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
1775
2242
|
*/
|
|
1776
2243
|
nodeLocalByProvider = /* @__PURE__ */ new Map();
|
|
1777
2244
|
/**
|
|
2245
|
+
* Cached `providerId → secret config keys`, derived from each provider's own
|
|
2246
|
+
* `configSchema` in the SAME refresh as `nodeLocalByProvider` — one
|
|
2247
|
+
* `getProviderInfo()` round trip answers both questions.
|
|
2248
|
+
*
|
|
2249
|
+
* An absent providerId yields an EMPTY set, which redacts nothing. That is
|
|
2250
|
+
* the one direction this cache can be wrong in, so it is worth saying why it
|
|
2251
|
+
* is acceptable: the entry is populated at the same moment the provider
|
|
2252
|
+
* becomes resolvable at all, so a location whose provider is unknown here
|
|
2253
|
+
* cannot be dispatched to either — there is no window in which a credential
|
|
2254
|
+
* is readable through a provider the orchestrator can otherwise use.
|
|
2255
|
+
*/
|
|
2256
|
+
secretKeysByProvider = /* @__PURE__ */ new Map();
|
|
2257
|
+
/**
|
|
1778
2258
|
* Disposers run on `onShutdown` — currently the eventBus subscription
|
|
1779
2259
|
* for `capability:provider-registered` events used by the lazy seed
|
|
1780
2260
|
* fallback. Stored separately from the `BaseAddon` disposer chain so
|
|
@@ -1809,13 +2289,24 @@ var StorageOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
1809
2289
|
listLocations: async ({ type }) => {
|
|
1810
2290
|
const rows = type !== void 0 ? service.listLocations({ type }) : service.listLocations();
|
|
1811
2291
|
return Promise.all(rows.map(async (loc) => ({
|
|
1812
|
-
...loc,
|
|
2292
|
+
...this.redacted(loc),
|
|
1813
2293
|
capacity: await this.localCapacityOf(loc)
|
|
1814
2294
|
})));
|
|
1815
2295
|
},
|
|
1816
|
-
getDefaultLocation: async ({ type }) =>
|
|
2296
|
+
getDefaultLocation: async ({ type }) => {
|
|
2297
|
+
const loc = service.getDefaultLocation(type);
|
|
2298
|
+
return loc === null ? null : this.redacted(loc);
|
|
2299
|
+
},
|
|
1817
2300
|
listLocationDeclarations: async () => service.listDeclarations(),
|
|
1818
|
-
upsertLocation: async (input) =>
|
|
2301
|
+
upsertLocation: async (input) => {
|
|
2302
|
+
const stored = service.getLocationById(input.id);
|
|
2303
|
+
const config = restoreRedactedSecrets(input.config, stored?.config, this.secretKeysFor(input.providerId));
|
|
2304
|
+
const saved = service.upsertLocation(config === input.config ? input : {
|
|
2305
|
+
...input,
|
|
2306
|
+
config
|
|
2307
|
+
});
|
|
2308
|
+
return this.redacted(saved);
|
|
2309
|
+
},
|
|
1819
2310
|
deleteLocation: async ({ id, force }) => {
|
|
1820
2311
|
await service.deleteLocation(id, { force: force === true });
|
|
1821
2312
|
},
|
|
@@ -1860,6 +2351,7 @@ var StorageOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
1860
2351
|
},
|
|
1861
2352
|
resolve: async ({ location, relativePath }) => {
|
|
1862
2353
|
const loc = service.resolveRef(location);
|
|
2354
|
+
this.refuseRemoteResolve(loc);
|
|
1863
2355
|
return (await service.getProviderFor(loc)).resolve({
|
|
1864
2356
|
location: loc,
|
|
1865
2357
|
relativePath
|
|
@@ -1982,6 +2474,8 @@ var StorageOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
1982
2474
|
startMove: (input) => this.ctx.api.recording.startStorageMigrationMove.mutate(input),
|
|
1983
2475
|
startDrain: (input) => this.ctx.api.recording.relocateFootage.mutate(input),
|
|
1984
2476
|
getMove: (jobId) => this.ctx.api.recording.getStorageMigrationMoveStatus.query({ jobId }),
|
|
2477
|
+
listMovers: () => this.ctx.api.recording.listRelocateJobs.query({}),
|
|
2478
|
+
residue: (input) => this.ctx.api.recording.getRelocateResidue.query(input),
|
|
1985
2479
|
cancelMove: async (jobId) => (await this.ctx.api.recording.cancelStorageMigrationMove.mutate({ jobId })).cancelled,
|
|
1986
2480
|
refresh: async (leaseId) => {
|
|
1987
2481
|
await this.ctx.api.recording.refreshStorageLocationsForMigration.mutate({ leaseId });
|
|
@@ -1998,6 +2492,8 @@ var StorageOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
1998
2492
|
startDrain: (input) => this.ctx.api.pipelineAnalytics.relocateMedia.mutate(input),
|
|
1999
2493
|
countUnstamped: () => this.ctx.api.pipelineAnalytics.countUnstampedEventMedia.query({}),
|
|
2000
2494
|
getMove: (jobId) => this.ctx.api.pipelineAnalytics.getStorageMigrationMoveStatus.query({ jobId }),
|
|
2495
|
+
listMovers: () => this.ctx.api.pipelineAnalytics.listRelocateMediaJobs.query({}),
|
|
2496
|
+
residue: (input) => this.ctx.api.pipelineAnalytics.countRelocatableMedia.query(input),
|
|
2001
2497
|
cancelMove: async (jobId) => (await this.ctx.api.pipelineAnalytics.cancelStorageMigrationMove.mutate({ jobId })).cancelled,
|
|
2002
2498
|
refresh: async (leaseId) => {
|
|
2003
2499
|
await this.ctx.api.pipelineAnalytics.refreshStorageLocationsForMigration.mutate({ leaseId });
|
|
@@ -2013,7 +2509,10 @@ var StorageOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
2013
2509
|
plan: (input) => migration.plan(input),
|
|
2014
2510
|
start: async (input) => ({ jobId: await migration.start(input) }),
|
|
2015
2511
|
status: ({ jobId }) => migration.status(jobId),
|
|
2016
|
-
cancel: async ({ jobId }) => ({ cancelled: await migration.cancel(jobId) })
|
|
2512
|
+
cancel: async ({ jobId }) => ({ cancelled: await migration.cancel(jobId) }),
|
|
2513
|
+
movers: () => migration.movers(),
|
|
2514
|
+
residue: () => migration.residue(),
|
|
2515
|
+
drain: async (input) => ({ jobId: await migration.drain(input) })
|
|
2017
2516
|
};
|
|
2018
2517
|
await this.seedFromDeclarations();
|
|
2019
2518
|
const eventBus = this.ctx.eventBus;
|
|
@@ -2156,6 +2655,34 @@ var StorageOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
2156
2655
|
}
|
|
2157
2656
|
return out;
|
|
2158
2657
|
}
|
|
2658
|
+
/** Declared secret config keys for a provider; empty when unknown. */
|
|
2659
|
+
secretKeysFor(providerId) {
|
|
2660
|
+
return this.secretKeysByProvider.get(providerId) ?? EMPTY_SECRET_KEYS;
|
|
2661
|
+
}
|
|
2662
|
+
/** A location with its provider's declared secrets replaced by the sentinel. */
|
|
2663
|
+
redacted(location) {
|
|
2664
|
+
return redactLocation(location, this.secretKeysFor(location.providerId));
|
|
2665
|
+
}
|
|
2666
|
+
/**
|
|
2667
|
+
* Is this location backed by a provider that serves a genuine local
|
|
2668
|
+
* filesystem? `true` / `false` / `undefined` — see
|
|
2669
|
+
* `StorageOrchestratorService.refuseIncompatibleProvider` for why the
|
|
2670
|
+
* unknown case is kept distinct rather than folded into either answer.
|
|
2671
|
+
*/
|
|
2672
|
+
providerIsNodeLocal(location) {
|
|
2673
|
+
return this.nodeLocalByProvider.get(location.providerId);
|
|
2674
|
+
}
|
|
2675
|
+
/** See the call site in the `resolve` dispatch, and `access-guards.ts`. */
|
|
2676
|
+
refuseRemoteResolve(location) {
|
|
2677
|
+
const refusal = resolveRefusalFor(location, this.providerIsNodeLocal(location));
|
|
2678
|
+
if (refusal === null) return;
|
|
2679
|
+
this.ctx.logger.warn("storage-orchestrator: REFUSED resolve() on a remote-backed location", { meta: {
|
|
2680
|
+
id: location.id,
|
|
2681
|
+
type: location.type,
|
|
2682
|
+
providerId: location.providerId
|
|
2683
|
+
} });
|
|
2684
|
+
throw new Error(refusal);
|
|
2685
|
+
}
|
|
2159
2686
|
/** statfs capacity of a location's basePath, walking up to the nearest
|
|
2160
2687
|
* existing ancestor. Null for remote-node locations or unstattable paths. */
|
|
2161
2688
|
async localCapacityOf(loc) {
|
|
@@ -2243,6 +2770,13 @@ var StorageOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
2243
2770
|
* errno — `EACCES`, `EIO`, a stale NFS handle — is `unknown` and refuses.
|
|
2244
2771
|
*/
|
|
2245
2772
|
async locationOccupancy(location) {
|
|
2773
|
+
if (!canProbeOccupancyLocally(this.providerIsNodeLocal(location))) {
|
|
2774
|
+
this.ctx.logger.debug("storage-orchestrator: occupancy unknown — location is not backed by a local filesystem", { meta: {
|
|
2775
|
+
id: location.id,
|
|
2776
|
+
providerId: location.providerId
|
|
2777
|
+
} });
|
|
2778
|
+
return "unknown";
|
|
2779
|
+
}
|
|
2246
2780
|
if ((location.nodeId === void 0 || location.nodeId === "" ? HUB_NODE_ID : location.nodeId) !== (this.service?.getLocalNodeId() ?? HUB_NODE_ID)) return "unknown";
|
|
2247
2781
|
const basePath = this.locationBasePath(location.id);
|
|
2248
2782
|
if (basePath === null) return "unknown";
|
|
@@ -2325,7 +2859,10 @@ var StorageOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
2325
2859
|
error: err instanceof Error ? err.message : String(err)
|
|
2326
2860
|
} });
|
|
2327
2861
|
});
|
|
2328
|
-
for (const info of infos)
|
|
2862
|
+
for (const info of infos) {
|
|
2863
|
+
this.nodeLocalByProvider.set(info.providerId, info.nodeLocal);
|
|
2864
|
+
this.secretKeysByProvider.set(info.providerId, secretKeysOfProviderInfo(info));
|
|
2865
|
+
}
|
|
2329
2866
|
}
|
|
2330
2867
|
};
|
|
2331
2868
|
//#endregion
|