@camstack/system 1.2.83 → 1.2.84
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
- package/dist/builtins/device-manager/device-manager.addon.js +1 -1
- package/dist/builtins/device-manager/device-manager.addon.mjs +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +30 -10
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +31 -11
- package/dist/builtins/sqlite-storage/path-guard.d.ts +8 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +1 -1
- package/dist/builtins/storage-orchestrator/index.d.ts +5 -6
- package/dist/builtins/storage-orchestrator/index.mjs +1 -1
- package/dist/builtins/storage-orchestrator/provider-discovery.d.ts +1 -1
- package/dist/builtins/storage-orchestrator/storage-migration-coordinator.d.ts +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.d.ts +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1252 -1191
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1253 -1192
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-EAvNJD24.mjs → dist-BAwlOEUy.mjs} +19 -3
- package/dist/{dist-CqiMPPs0.js → dist-CQy9KjsL.js} +19 -3
- package/dist/index.js +1 -1
- package/dist/index.mjs +2 -2
- package/package.json +1 -1
|
@@ -3,798 +3,579 @@ Object.defineProperties(exports, {
|
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
5
|
const require_chunk = require("../../chunk-Cek0wNdY.js");
|
|
6
|
-
const require_dist = require("../../dist-
|
|
6
|
+
const require_dist = require("../../dist-CQy9KjsL.js");
|
|
7
7
|
let node_crypto = require("node:crypto");
|
|
8
8
|
let node_fs_promises = require("node:fs/promises");
|
|
9
9
|
node_fs_promises = require_chunk.__toESM(node_fs_promises);
|
|
10
10
|
let node_path = require("node:path");
|
|
11
11
|
node_path = require_chunk.__toESM(node_path);
|
|
12
12
|
let _camstack_system = require("@camstack/system");
|
|
13
|
-
//#region src/builtins/storage-orchestrator/
|
|
14
|
-
var
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
13
|
+
//#region src/builtins/storage-orchestrator/location-store.ts
|
|
14
|
+
var STORAGE_LOCATIONS = "storage_locations";
|
|
15
|
+
/**
|
|
16
|
+
* Reserved key under which the cluster `nodeId` (SP1) is persisted INSIDE
|
|
17
|
+
* the `config` JSON blob. Storing it in the blob rather than a dedicated
|
|
18
|
+
* column avoids a destructive `ensureTable` DROP+recreate on deploy (a
|
|
19
|
+
* newly-declared column counts as a missing column → the structured
|
|
20
|
+
* backend drops the table, wiping operator location customizations such
|
|
21
|
+
* as `recordings:default`'s `minFreePercent`). The double-underscore
|
|
22
|
+
* prefix keeps it out of the provider-facing config namespace
|
|
23
|
+
* (`basePath`, `minFreePercent`, …) — the mapper strips it on read so
|
|
24
|
+
* API consumers and providers never see it.
|
|
25
|
+
*/
|
|
26
|
+
var NODE_ID_CONFIG_KEY = "__nodeId";
|
|
27
|
+
/**
|
|
28
|
+
* Reserved key under which the operator opt-in flag (`StorageLocation.enabled`)
|
|
29
|
+
* is persisted INSIDE the `config` JSON blob — same channel, same reason, as
|
|
30
|
+
* {@link NODE_ID_CONFIG_KEY}: a newly-declared COLUMN counts as a missing
|
|
31
|
+
* column, and the structured backend answers that by dropping and recreating
|
|
32
|
+
* the table, wiping every operator customization on it.
|
|
33
|
+
*
|
|
34
|
+
* ABSENT means ACTIVE. A row written before the flag existed carries no key and
|
|
35
|
+
* reads back with `enabled: undefined`, which every consumer treats as enabled —
|
|
36
|
+
* that is why this ships with no migration and no backfill pass.
|
|
37
|
+
*/
|
|
38
|
+
var ENABLED_CONFIG_KEY = "__enabled";
|
|
39
|
+
/**
|
|
40
|
+
* `config` is declared `JSON`, so the store hands the surface an object and
|
|
41
|
+
* gets an object back. There is deliberately NO `node_id` column — `nodeId`
|
|
42
|
+
* rides inside the blob under a reserved key (see {@link NODE_ID_CONFIG_KEY}),
|
|
43
|
+
* because a newly-declared column would put this table through
|
|
44
|
+
* `declareCollection`'s destructive rebuild and wipe operator customizations
|
|
45
|
+
* such as `recordings:default`'s `minFreePercent`.
|
|
46
|
+
*/
|
|
47
|
+
var STORAGE_LOCATIONS_COLUMNS = [
|
|
48
|
+
{
|
|
49
|
+
name: "id",
|
|
50
|
+
type: "TEXT",
|
|
51
|
+
primaryKey: true
|
|
18
52
|
},
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
53
|
+
{
|
|
54
|
+
name: "type",
|
|
55
|
+
type: "TEXT",
|
|
56
|
+
notNull: true
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
name: "display_name",
|
|
60
|
+
type: "TEXT",
|
|
61
|
+
notNull: true
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
name: "provider_id",
|
|
65
|
+
type: "TEXT",
|
|
66
|
+
notNull: true
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
name: "config",
|
|
70
|
+
type: "JSON",
|
|
71
|
+
notNull: true,
|
|
72
|
+
defaultValue: "{}"
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
name: "is_default",
|
|
76
|
+
type: "INTEGER",
|
|
77
|
+
notNull: true,
|
|
78
|
+
defaultValue: 0
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
name: "is_system",
|
|
82
|
+
type: "INTEGER",
|
|
83
|
+
notNull: true,
|
|
84
|
+
defaultValue: 0
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
name: "created_at",
|
|
88
|
+
type: "INTEGER",
|
|
89
|
+
notNull: true
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
name: "updated_at",
|
|
93
|
+
type: "INTEGER",
|
|
94
|
+
notNull: true
|
|
22
95
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
}
|
|
96
|
+
];
|
|
97
|
+
var STORAGE_LOCATIONS_INDEXES = [{
|
|
98
|
+
name: "idx_storage_locations_type",
|
|
99
|
+
columns: ["type"]
|
|
100
|
+
}];
|
|
28
101
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
102
|
+
* `ILocationStore` on the `settings-store` collection surface — the same door
|
|
103
|
+
* every other consumer uses. It holds the `data-store-provider` engine object
|
|
104
|
+
* directly (not the `settings-store` singleton) because since D44 the
|
|
105
|
+
* storage-orchestrator IS that singleton, and asking the registry for it would
|
|
106
|
+
* route this table back through the orchestrator itself.
|
|
33
107
|
*/
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
return bare === "" ? "hub" : bare;
|
|
40
|
-
}
|
|
41
|
-
var StorageOrchestratorService = class {
|
|
42
|
-
logger;
|
|
43
|
-
getProviders;
|
|
44
|
-
locationStore;
|
|
45
|
-
getRemoteProvider;
|
|
46
|
-
locations = /* @__PURE__ */ new Map();
|
|
47
|
-
/**
|
|
48
|
-
* Declaration-driven cardinality source, injected by the addon after
|
|
49
|
-
* the kernel aggregates `storageLocations` declarations. `null` until
|
|
50
|
-
* `setRegistry` runs — `upsertLocation` then treats every type as
|
|
51
|
-
* `'multi'` (no singleton enforcement), the safe default for the
|
|
52
|
-
* in-memory-only / early-boot path.
|
|
53
|
-
*/
|
|
54
|
-
registry = null;
|
|
55
|
-
/**
|
|
56
|
-
* Injected `providerId → nodeLocal` resolver (see {@link NodeLocalResolver}).
|
|
57
|
-
* `null` until {@link setNodeLocalResolver} runs — `upsertLocation` then
|
|
58
|
-
* skips the node-local requirement check entirely (every provider is
|
|
59
|
-
* treated as node-agnostic), the safe default for the in-memory-only /
|
|
60
|
-
* early-boot path that predates the resolver wiring.
|
|
61
|
-
*/
|
|
62
|
-
nodeLocalResolver = null;
|
|
63
|
-
/** `localNodeId` with any forked-child `/addon` suffix stripped. */
|
|
64
|
-
localNode;
|
|
65
|
-
/**
|
|
66
|
-
* `<kind>:<sessionId>` → the dispatch target that OPENED the session.
|
|
67
|
-
* Chunked I/O carries no `StorageLocation` after the first hop, so without
|
|
68
|
-
* this the follow-up chunks fall back to a `providerId`-only lookup — which
|
|
69
|
-
* on a multi-node cluster means the hub's filesystem, not the agent's.
|
|
70
|
-
*/
|
|
71
|
-
sessionTargets = /* @__PURE__ */ new Map();
|
|
72
|
-
constructor(logger, getProviders, locationStore = null, localNodeId = "hub", getRemoteProvider) {
|
|
73
|
-
this.logger = logger;
|
|
74
|
-
this.getProviders = getProviders;
|
|
75
|
-
this.locationStore = locationStore;
|
|
76
|
-
this.getRemoteProvider = getRemoteProvider;
|
|
77
|
-
this.localNode = bareNodeId(localNodeId);
|
|
108
|
+
var SqliteLocationStore = class {
|
|
109
|
+
store;
|
|
110
|
+
declared = false;
|
|
111
|
+
constructor(store) {
|
|
112
|
+
this.store = store;
|
|
78
113
|
}
|
|
79
114
|
/**
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
115
|
+
* Lazy `declareCollection` — keeps the constructor cheap and async-free.
|
|
116
|
+
* Called from every mutation/read path; `declareCollection` is idempotent
|
|
117
|
+
* (and additive on shape changes), so repeated calls collapse to a no-op.
|
|
83
118
|
*/
|
|
84
|
-
|
|
85
|
-
|
|
119
|
+
async ensureDeclared() {
|
|
120
|
+
if (this.declared) return;
|
|
121
|
+
await this.store.declareCollection({
|
|
122
|
+
collection: STORAGE_LOCATIONS,
|
|
123
|
+
columns: STORAGE_LOCATIONS_COLUMNS,
|
|
124
|
+
indexes: STORAGE_LOCATIONS_INDEXES
|
|
125
|
+
});
|
|
126
|
+
this.declared = true;
|
|
86
127
|
}
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
128
|
+
async loadAll() {
|
|
129
|
+
await this.ensureDeclared();
|
|
130
|
+
const rows = await this.store.query({
|
|
131
|
+
collection: STORAGE_LOCATIONS,
|
|
132
|
+
filter: { orderBy: {
|
|
133
|
+
field: "created_at",
|
|
134
|
+
direction: "asc"
|
|
135
|
+
} }
|
|
136
|
+
});
|
|
137
|
+
const out = [];
|
|
138
|
+
for (const row of rows) {
|
|
139
|
+
const mapped = mapRowToLocation(row.data);
|
|
140
|
+
if (mapped) out.push(mapped);
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
90
143
|
}
|
|
91
144
|
/**
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
* in-memory
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
* Reconciles both directions so operator edits survive a reboot:
|
|
98
|
-
* 1. hydrate — DB rows win over any early in-memory seed (restores
|
|
99
|
-
* operator config like `minFreePercent` that the pre-store boot
|
|
100
|
-
* can't see), with the same `isSystem` upgrade as {@link initialize};
|
|
101
|
-
* 2. backfill — in-memory locations the DB doesn't have yet (the
|
|
102
|
-
* pre-store seed defaults on a fresh install) are persisted, so the
|
|
103
|
-
* store becomes the durable source of truth from here on.
|
|
145
|
+
* Replace-or-insert. Implemented as delete + insert rather than a
|
|
146
|
+
* conditional update because the orchestrator serializes every mutation
|
|
147
|
+
* through the in-memory map before dispatching to the store — there's no
|
|
148
|
+
* contention to lose, and delete-then-insert is unconditionally simpler.
|
|
104
149
|
*/
|
|
105
|
-
async
|
|
106
|
-
|
|
107
|
-
this.
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
} : loc;
|
|
115
|
-
this.locations.set(upgraded.id, upgraded);
|
|
116
|
-
if (upgraded !== loc) store.upsert(upgraded).catch(() => {});
|
|
117
|
-
}
|
|
118
|
-
const backfill = [...this.locations.values()].filter((l) => !dbIds.has(l.id));
|
|
119
|
-
for (const loc of backfill) store.upsert(loc).catch((err) => {
|
|
120
|
-
this.logger.warn("storage-orchestrator: attachStore backfill persist failed", { meta: {
|
|
150
|
+
async upsert(loc) {
|
|
151
|
+
await this.ensureDeclared();
|
|
152
|
+
await this.store.deleteWhere({
|
|
153
|
+
collection: STORAGE_LOCATIONS,
|
|
154
|
+
filter: { where: { id: loc.id } }
|
|
155
|
+
});
|
|
156
|
+
await this.store.insert({
|
|
157
|
+
collection: STORAGE_LOCATIONS,
|
|
158
|
+
record: {
|
|
121
159
|
id: loc.id,
|
|
122
|
-
|
|
123
|
-
}
|
|
160
|
+
data: mapLocationToRow(loc)
|
|
161
|
+
}
|
|
124
162
|
});
|
|
125
|
-
this.logger.info("storage-orchestrator: store attached + reconciled", { meta: {
|
|
126
|
-
hydrated: rows.length,
|
|
127
|
-
backfilled: backfill.length,
|
|
128
|
-
total: this.locations.size
|
|
129
|
-
} });
|
|
130
|
-
}
|
|
131
|
-
/**
|
|
132
|
-
* Inject the declaration-driven cardinality source. Called once by the
|
|
133
|
-
* addon after the kernel aggregates every addon's `storageLocations`
|
|
134
|
-
* declarations into a `StorageLocationRegistry`.
|
|
135
|
-
*/
|
|
136
|
-
setRegistry(registry) {
|
|
137
|
-
this.registry = registry;
|
|
138
163
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
setNodeLocalResolver(resolver) {
|
|
146
|
-
this.nodeLocalResolver = resolver;
|
|
164
|
+
async delete(id) {
|
|
165
|
+
await this.ensureDeclared();
|
|
166
|
+
await this.store.deleteWhere({
|
|
167
|
+
collection: STORAGE_LOCATIONS,
|
|
168
|
+
filter: { where: { id } }
|
|
169
|
+
});
|
|
147
170
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
171
|
+
};
|
|
172
|
+
function isPlainObject(value) {
|
|
173
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
174
|
+
}
|
|
175
|
+
function readConfig(raw) {
|
|
176
|
+
if (isPlainObject(raw)) return { ...raw };
|
|
177
|
+
if (typeof raw === "string" && raw.length > 0) return require_dist.parseJsonObject(raw) ?? {};
|
|
178
|
+
return {};
|
|
179
|
+
}
|
|
180
|
+
function mapLocationToRow(loc) {
|
|
181
|
+
const storedConfig = { ...loc.config };
|
|
182
|
+
if (loc.nodeId !== void 0) storedConfig[NODE_ID_CONFIG_KEY] = loc.nodeId;
|
|
183
|
+
if (loc.enabled !== void 0) storedConfig[ENABLED_CONFIG_KEY] = loc.enabled;
|
|
184
|
+
return {
|
|
185
|
+
id: loc.id,
|
|
186
|
+
type: loc.type,
|
|
187
|
+
display_name: loc.displayName,
|
|
188
|
+
provider_id: loc.providerId,
|
|
189
|
+
config: storedConfig,
|
|
190
|
+
is_default: loc.isDefault ? 1 : 0,
|
|
191
|
+
is_system: loc.isSystem ? 1 : 0,
|
|
192
|
+
created_at: loc.createdAt,
|
|
193
|
+
updated_at: loc.updatedAt
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Decode a row read from the collection. Returns `null` when the row's `type`
|
|
198
|
+
* fails the id-format schema or the row has no id — a corrupt row (hand-edited,
|
|
199
|
+
* partial migration) is skipped rather than taking the boot read down with it.
|
|
200
|
+
*/
|
|
201
|
+
function mapRowToLocation(row) {
|
|
202
|
+
const typeRaw = row["type"];
|
|
203
|
+
const parsedType = require_dist.StorageLocationTypeSchema.safeParse(typeRaw);
|
|
204
|
+
if (!parsedType.success) return null;
|
|
205
|
+
const type = parsedType.data;
|
|
206
|
+
const id = String(row["id"] ?? "");
|
|
207
|
+
if (!id) return null;
|
|
208
|
+
const storedConfig = readConfig(row["config"]);
|
|
209
|
+
const nodeIdRaw = storedConfig[NODE_ID_CONFIG_KEY];
|
|
210
|
+
const nodeId = typeof nodeIdRaw === "string" && nodeIdRaw.length > 0 ? nodeIdRaw : void 0;
|
|
211
|
+
const enabledRaw = storedConfig[ENABLED_CONFIG_KEY];
|
|
212
|
+
const enabled = typeof enabledRaw === "boolean" ? enabledRaw : void 0;
|
|
213
|
+
const config = { ...storedConfig };
|
|
214
|
+
delete config[NODE_ID_CONFIG_KEY];
|
|
215
|
+
delete config[ENABLED_CONFIG_KEY];
|
|
216
|
+
return {
|
|
217
|
+
id,
|
|
218
|
+
type,
|
|
219
|
+
displayName: String(row["display_name"] ?? ""),
|
|
220
|
+
providerId: String(row["provider_id"] ?? ""),
|
|
221
|
+
config,
|
|
222
|
+
...nodeId !== void 0 ? { nodeId } : {},
|
|
223
|
+
...enabled !== void 0 ? { enabled } : {},
|
|
224
|
+
isDefault: row["is_default"] === 1 || row["is_default"] === true,
|
|
225
|
+
isSystem: row["is_system"] === 1 || row["is_system"] === true,
|
|
226
|
+
createdAt: Number(row["created_at"] ?? 0),
|
|
227
|
+
updatedAt: Number(row["updated_at"] ?? 0)
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
//#endregion
|
|
231
|
+
//#region src/builtins/storage-orchestrator/storage-migration-coordinator.ts
|
|
232
|
+
var STORAGE_CLASSES = [
|
|
233
|
+
"recordings",
|
|
234
|
+
"recordingsLow",
|
|
235
|
+
"eventMedia"
|
|
236
|
+
];
|
|
237
|
+
var PARTICIPANTS = [
|
|
238
|
+
"pipeline",
|
|
239
|
+
"recorder",
|
|
240
|
+
"analytics"
|
|
241
|
+
];
|
|
242
|
+
var REVERSE_PARTICIPANTS = [
|
|
243
|
+
"analytics",
|
|
244
|
+
"recorder",
|
|
245
|
+
"pipeline"
|
|
246
|
+
];
|
|
247
|
+
/**
|
|
248
|
+
* Durable orchestration layer over the independent recorder and event-media
|
|
249
|
+
* movers. It owns the only public migration state machine; participants expose
|
|
250
|
+
* intentionally narrow lease/move primitives and never alter camera settings.
|
|
251
|
+
*/
|
|
252
|
+
var StorageMigrationCoordinator = class {
|
|
253
|
+
deps;
|
|
254
|
+
active = null;
|
|
255
|
+
runPromise = null;
|
|
256
|
+
startReserved = false;
|
|
257
|
+
constructor(deps) {
|
|
258
|
+
this.deps = deps;
|
|
156
259
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
const upgradedLoc = {
|
|
176
|
-
...loc,
|
|
177
|
-
isSystem: true
|
|
178
|
-
};
|
|
179
|
-
this.locations.set(upgradedLoc.id, upgradedLoc);
|
|
180
|
-
this.locationStore.upsert(upgradedLoc).catch((err) => {
|
|
181
|
-
this.logger.warn("storage-orchestrator: isSystem upgrade persist failed", { meta: {
|
|
182
|
-
id: upgradedLoc.id,
|
|
183
|
-
error: err instanceof Error ? err.message : String(err)
|
|
184
|
-
} });
|
|
260
|
+
async plan(input) {
|
|
261
|
+
const moves = [];
|
|
262
|
+
for (const storageClass of STORAGE_CLASSES) {
|
|
263
|
+
const targetId = input.destinations[storageClass];
|
|
264
|
+
if (targetId === void 0) continue;
|
|
265
|
+
const source = this.deps.locations.getDefaultLocation(storageClass);
|
|
266
|
+
if (!source) throw new Error(`No default storage location for "${storageClass}"`);
|
|
267
|
+
const target = this.deps.locations.getLocationById(targetId);
|
|
268
|
+
if (!target) throw new Error(`Storage location "${targetId}" not found`);
|
|
269
|
+
if (target.type !== storageClass) throw new Error(`Storage location "${targetId}" is type "${target.type}", expected "${storageClass}"`);
|
|
270
|
+
if (source.id === target.id) throw new Error(`Storage location "${targetId}" is already the "${storageClass}" default`);
|
|
271
|
+
moves.push({
|
|
272
|
+
storageClass,
|
|
273
|
+
fromLocationId: source.id,
|
|
274
|
+
toLocationId: target.id,
|
|
275
|
+
moverJobId: null,
|
|
276
|
+
state: null,
|
|
277
|
+
error: null
|
|
185
278
|
});
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
279
|
+
}
|
|
280
|
+
if (moves.length === 0) throw new Error("select at least one storage class");
|
|
281
|
+
return {
|
|
282
|
+
destinations: input.destinations,
|
|
283
|
+
moves: moves.map(({ storageClass, fromLocationId, toLocationId }) => ({
|
|
284
|
+
storageClass,
|
|
285
|
+
fromLocationId,
|
|
286
|
+
toLocationId
|
|
287
|
+
}))
|
|
288
|
+
};
|
|
192
289
|
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
* no-op. A plain local update, not versioned-migration machinery: only
|
|
203
|
-
* the rows missing `nodeId` are mutated + persisted.
|
|
204
|
-
*
|
|
205
|
-
* No-op until the node-local resolver is wired (it classifies which
|
|
206
|
-
* providers are node-local).
|
|
207
|
-
*/
|
|
208
|
-
async backfillNodeIds(hubNodeId) {
|
|
209
|
-
const resolver = this.nodeLocalResolver;
|
|
210
|
-
if (!resolver) return;
|
|
211
|
-
let stamped = 0;
|
|
212
|
-
for (const [id, loc] of Array.from(this.locations)) {
|
|
213
|
-
if (loc.nodeId) continue;
|
|
214
|
-
if (resolver(loc.providerId) !== true) continue;
|
|
215
|
-
const updated = {
|
|
216
|
-
...loc,
|
|
217
|
-
nodeId: hubNodeId,
|
|
218
|
-
updatedAt: Date.now()
|
|
219
|
-
};
|
|
220
|
-
this.locations.set(id, updated);
|
|
221
|
-
stamped++;
|
|
222
|
-
if (this.locationStore) try {
|
|
223
|
-
await this.locationStore.upsert(updated);
|
|
224
|
-
} catch (err) {
|
|
225
|
-
this.logger.error("storage-orchestrator: nodeId backfill persist failed", { meta: {
|
|
226
|
-
id,
|
|
227
|
-
error: err instanceof Error ? err.message : String(err)
|
|
228
|
-
} });
|
|
290
|
+
async start(input) {
|
|
291
|
+
if (this.startReserved) throw new Error("storage migration is already active");
|
|
292
|
+
this.startReserved = true;
|
|
293
|
+
try {
|
|
294
|
+
const existing = await this.status();
|
|
295
|
+
if (existing && isTerminal(existing) && existing.pausedParticipants.length > 0) {
|
|
296
|
+
await this.releaseAfterTerminal(existing);
|
|
297
|
+
await this.persist(existing);
|
|
298
|
+
if (existing.pausedParticipants.length > 0) throw new Error(`storage migration ${existing.jobId} still holds maintenance leases`);
|
|
229
299
|
}
|
|
300
|
+
if (existing && !isTerminal(existing)) throw new Error(`storage migration is already active (${existing.jobId})`);
|
|
301
|
+
const plan = await this.plan(input);
|
|
302
|
+
const now = this.deps.now();
|
|
303
|
+
const job = {
|
|
304
|
+
jobId: this.deps.newId(),
|
|
305
|
+
phase: "planning",
|
|
306
|
+
destinations: input.destinations,
|
|
307
|
+
throttleMbps: input.throttleMbps ?? 40,
|
|
308
|
+
moves: plan.moves.map((move) => ({
|
|
309
|
+
...move,
|
|
310
|
+
moverJobId: null,
|
|
311
|
+
state: null,
|
|
312
|
+
error: null
|
|
313
|
+
})),
|
|
314
|
+
pauseLeaseId: null,
|
|
315
|
+
pausedParticipants: [],
|
|
316
|
+
repointed: false,
|
|
317
|
+
cancelRequested: false,
|
|
318
|
+
startedAt: now,
|
|
319
|
+
updatedAt: now,
|
|
320
|
+
finishedAt: null,
|
|
321
|
+
error: null
|
|
322
|
+
};
|
|
323
|
+
this.active = job;
|
|
324
|
+
await this.persist(job);
|
|
325
|
+
this.runPromise = this.run(job);
|
|
326
|
+
this.runPromise;
|
|
327
|
+
return job.jobId;
|
|
328
|
+
} finally {
|
|
329
|
+
this.startReserved = false;
|
|
230
330
|
}
|
|
231
|
-
if (stamped > 0) this.logger.info("storage-orchestrator: backfilled nodeId on node-local locations", { meta: {
|
|
232
|
-
stamped,
|
|
233
|
-
hubNodeId
|
|
234
|
-
} });
|
|
235
331
|
}
|
|
236
|
-
|
|
237
|
-
const
|
|
238
|
-
|
|
332
|
+
async status(jobId) {
|
|
333
|
+
const job = this.active ?? await this.deps.state.get();
|
|
334
|
+
if (jobId !== void 0 && job?.jobId !== jobId) return null;
|
|
335
|
+
return job;
|
|
239
336
|
}
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
return
|
|
337
|
+
async cancel(jobId) {
|
|
338
|
+
const job = await this.status(jobId);
|
|
339
|
+
if (!job || isTerminal(job) || job.repointed) return false;
|
|
340
|
+
job.cancelRequested = true;
|
|
341
|
+
await this.persist(job);
|
|
342
|
+
await Promise.all(job.moves.filter((move) => move.moverJobId !== null).map((move) => this.cancelMove(move)));
|
|
343
|
+
return true;
|
|
243
344
|
}
|
|
244
|
-
/**
|
|
245
|
-
*
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
if (!target) throw new Error(`Storage location "${targetId}" not found`);
|
|
258
|
-
if (target.type !== type) throw new Error(`Storage location "${targetId}" is type "${target.type}", expected "${type}"`);
|
|
259
|
-
for (const [id, location] of nextById) {
|
|
260
|
-
if (location.type !== type) continue;
|
|
261
|
-
const isDefault = id === targetId;
|
|
262
|
-
if (location.isDefault === isDefault) continue;
|
|
263
|
-
const updated = {
|
|
264
|
-
...location,
|
|
265
|
-
isDefault,
|
|
266
|
-
updatedAt: now
|
|
267
|
-
};
|
|
268
|
-
nextById.set(id, updated);
|
|
269
|
-
touched.set(id, updated);
|
|
270
|
-
}
|
|
345
|
+
/** Boot recovery resumes a durable unfinished state. A missing in-memory
|
|
346
|
+
* child mover is recreated from the same copy-if-absent input. */
|
|
347
|
+
async recover() {
|
|
348
|
+
const job = await this.deps.state.get();
|
|
349
|
+
if (!job) return;
|
|
350
|
+
if (job.cancelRequested) {
|
|
351
|
+
await Promise.all(job.moves.filter((move) => move.moverJobId !== null).map((move) => this.cancelMove(move)));
|
|
352
|
+
await this.waitForMoves(job);
|
|
353
|
+
await this.releaseAfterTerminal(job);
|
|
354
|
+
job.phase = "cancelled";
|
|
355
|
+
job.finishedAt ??= this.deps.now();
|
|
356
|
+
await this.persist(job);
|
|
357
|
+
return;
|
|
271
358
|
}
|
|
272
|
-
if (
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
if (location) previous.set(id, location);
|
|
277
|
-
}
|
|
278
|
-
try {
|
|
279
|
-
for (const location of touched.values()) await this.locationStore.upsert(location);
|
|
280
|
-
} catch (err) {
|
|
281
|
-
for (const location of previous.values()) try {
|
|
282
|
-
await this.locationStore.upsert(location);
|
|
283
|
-
} catch {}
|
|
284
|
-
throw err;
|
|
359
|
+
if (isTerminal(job)) {
|
|
360
|
+
if (job.pausedParticipants.length > 0) {
|
|
361
|
+
await this.releaseAfterTerminal(job);
|
|
362
|
+
await this.persist(job);
|
|
285
363
|
}
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
366
|
+
this.active = job;
|
|
367
|
+
job.pauseLeaseId ??= this.deps.newId();
|
|
368
|
+
if (job.repointed && job.phase === "resuming") job.phase = "refreshing";
|
|
369
|
+
await this.persist(job);
|
|
370
|
+
try {
|
|
371
|
+
await this.pauseParticipants(job, true);
|
|
372
|
+
} catch (err) {
|
|
373
|
+
job.error = err instanceof Error ? err.message : String(err);
|
|
374
|
+
await this.persist(job);
|
|
375
|
+
return;
|
|
376
|
+
}
|
|
377
|
+
if (this.runPromise === null) {
|
|
378
|
+
this.runPromise = this.run(job);
|
|
379
|
+
this.runPromise;
|
|
286
380
|
}
|
|
287
|
-
this.locations.clear();
|
|
288
|
-
for (const [id, location] of nextById) this.locations.set(id, location);
|
|
289
381
|
}
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
*/
|
|
299
|
-
upsertLocation(input) {
|
|
300
|
-
const now = Date.now();
|
|
301
|
-
const existing = this.locations.get(input.id);
|
|
302
|
-
if (!existing) {
|
|
303
|
-
if ((this.registry?.cardinalityOf(input.type) ?? "multi") === "single") {
|
|
304
|
-
const already = [...this.locations.values()].find((l) => l.type === input.type);
|
|
305
|
-
if (already) throw new Error(`Storage type "${input.type}" is single — only one location allowed (existing: "${already.id}"). Edit it instead of adding a new one.`);
|
|
382
|
+
async run(job) {
|
|
383
|
+
try {
|
|
384
|
+
if (job.phase === "planning") await this.setPhase(job, "pausing");
|
|
385
|
+
if (job.phase === "pausing") {
|
|
386
|
+
job.pauseLeaseId ??= job.jobId;
|
|
387
|
+
await this.pauseParticipants(job);
|
|
388
|
+
if (job.cancelRequested) return this.finishCancelled(job);
|
|
389
|
+
await this.setPhase(job, "moving");
|
|
306
390
|
}
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
391
|
+
if (job.phase === "moving") {
|
|
392
|
+
await this.startOrResumeMoves(job);
|
|
393
|
+
await this.waitForMoves(job);
|
|
394
|
+
if (job.cancelRequested) return this.finishCancelled(job);
|
|
395
|
+
await this.setPhase(job, "verifying");
|
|
396
|
+
}
|
|
397
|
+
if (job.phase === "verifying") {
|
|
398
|
+
await this.verifyMoves(job);
|
|
399
|
+
if (job.cancelRequested) return this.finishCancelled(job);
|
|
400
|
+
await this.setPhase(job, "repointing");
|
|
401
|
+
}
|
|
402
|
+
if (job.phase === "repointing") {
|
|
403
|
+
const targets = new Map(job.moves.map((move) => [move.storageClass, move.toLocationId]));
|
|
404
|
+
await this.deps.locations.setDefaultLocations(targets);
|
|
405
|
+
job.repointed = true;
|
|
406
|
+
await this.setPhase(job, "refreshing");
|
|
407
|
+
}
|
|
408
|
+
if (job.phase === "refreshing") {
|
|
409
|
+
const leaseId = requireLease(job);
|
|
410
|
+
if (job.moves.some((move) => move.storageClass !== "eventMedia")) await this.deps.participants.recorder.refresh(leaseId);
|
|
411
|
+
if (job.moves.some((move) => move.storageClass === "eventMedia")) await this.deps.participants.analytics.refresh(leaseId);
|
|
412
|
+
await this.setPhase(job, "resuming");
|
|
413
|
+
}
|
|
414
|
+
if (job.phase === "resuming") {
|
|
415
|
+
await this.resumeParticipants(job);
|
|
416
|
+
await this.setPhase(job, "done");
|
|
417
|
+
}
|
|
418
|
+
} catch (err) {
|
|
419
|
+
job.error = err instanceof Error ? err.message : String(err);
|
|
420
|
+
if (job.repointed) {
|
|
421
|
+
job.phase = "refreshing";
|
|
422
|
+
job.finishedAt = null;
|
|
423
|
+
await this.persist(job);
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
job.phase = "failed";
|
|
427
|
+
job.finishedAt = this.deps.now();
|
|
428
|
+
await this.releaseAfterTerminal(job);
|
|
429
|
+
await this.persist(job);
|
|
430
|
+
} finally {
|
|
431
|
+
if (isTerminal(job)) {
|
|
432
|
+
await this.releaseAfterTerminal(job);
|
|
433
|
+
this.active = job;
|
|
331
434
|
}
|
|
435
|
+
this.runPromise = null;
|
|
332
436
|
}
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
(async () => {
|
|
343
|
-
try {
|
|
344
|
-
for (const d of demoted) await store.upsert(d);
|
|
345
|
-
await store.upsert(next);
|
|
346
|
-
} catch (err) {
|
|
347
|
-
this.logger.error("storage-orchestrator: upsert persistence failed", { meta: {
|
|
348
|
-
id: next.id,
|
|
349
|
-
error: err instanceof Error ? err.message : String(err)
|
|
350
|
-
} });
|
|
351
|
-
}
|
|
352
|
-
})();
|
|
437
|
+
}
|
|
438
|
+
async pauseParticipants(job, reacquire = false) {
|
|
439
|
+
const leaseId = requireLease(job);
|
|
440
|
+
for (const name of PARTICIPANTS) {
|
|
441
|
+
if (job.cancelRequested) return;
|
|
442
|
+
if (!reacquire && job.pausedParticipants.includes(name)) continue;
|
|
443
|
+
await this.participant(name).pause(leaseId);
|
|
444
|
+
if (!job.pausedParticipants.includes(name)) job.pausedParticipants.push(name);
|
|
445
|
+
await this.persist(job);
|
|
353
446
|
}
|
|
354
|
-
return next;
|
|
355
447
|
}
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
* routed to the logger — see `upsertLocation` for the rationale.
|
|
364
|
-
*/
|
|
365
|
-
deleteLocation(id) {
|
|
366
|
-
const loc = this.locations.get(id);
|
|
367
|
-
if (!loc) throw new Error(`Storage location "${id}" not found`);
|
|
368
|
-
if (loc.isSystem) throw new Error(`Storage location "${id}" is system-managed and cannot be deleted. Edit its config (path / providerId) instead.`);
|
|
369
|
-
if (loc.isDefault) {
|
|
370
|
-
if (![...this.locations.values()].find((l) => l.type === loc.type && l.id !== id && l.isDefault)) throw new Error(`Cannot delete default location "${id}" for type "${loc.type}" — promote another location to default first`);
|
|
448
|
+
async resumeParticipants(job) {
|
|
449
|
+
const leaseId = requireLease(job);
|
|
450
|
+
for (const name of REVERSE_PARTICIPANTS) {
|
|
451
|
+
if (!job.pausedParticipants.includes(name)) continue;
|
|
452
|
+
await this.participant(name).resume(leaseId);
|
|
453
|
+
job.pausedParticipants = job.pausedParticipants.filter((participant) => participant !== name);
|
|
454
|
+
await this.persist(job);
|
|
371
455
|
}
|
|
372
|
-
this.locations.delete(id);
|
|
373
|
-
if (this.locationStore) this.locationStore.delete(id).catch((err) => {
|
|
374
|
-
this.logger.error("storage-orchestrator: delete persistence failed", { meta: {
|
|
375
|
-
id,
|
|
376
|
-
error: err instanceof Error ? err.message : String(err)
|
|
377
|
-
} });
|
|
378
|
-
});
|
|
379
456
|
}
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
if (
|
|
395
|
-
|
|
457
|
+
async releaseAfterTerminal(job) {
|
|
458
|
+
if (job.pauseLeaseId === null) return;
|
|
459
|
+
for (const name of REVERSE_PARTICIPANTS) {
|
|
460
|
+
if (!job.pausedParticipants.includes(name)) continue;
|
|
461
|
+
try {
|
|
462
|
+
await this.participant(name).resume(job.pauseLeaseId);
|
|
463
|
+
job.pausedParticipants = job.pausedParticipants.filter((participant) => participant !== name);
|
|
464
|
+
await this.persist(job);
|
|
465
|
+
} catch {}
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
async startOrResumeMoves(job) {
|
|
469
|
+
if (job.cancelRequested) return;
|
|
470
|
+
const mediaMove = job.moves.find((move) => move.storageClass === "eventMedia");
|
|
471
|
+
if (mediaMove?.moverJobId === null) {
|
|
472
|
+
mediaMove.moverJobId = await this.startMove(job, mediaMove);
|
|
473
|
+
await this.persist(job);
|
|
474
|
+
}
|
|
475
|
+
const footageMoves = job.moves.filter((move) => move.storageClass !== "eventMedia");
|
|
476
|
+
for (let index = 0; index < footageMoves.length; index++) {
|
|
477
|
+
const move = footageMoves[index];
|
|
478
|
+
if (move.moverJobId !== null) continue;
|
|
479
|
+
if (!footageMoves.slice(0, index).every((predecessor) => predecessor.state === "done")) return;
|
|
480
|
+
move.moverJobId = await this.startMove(job, move);
|
|
481
|
+
await this.persist(job);
|
|
396
482
|
return;
|
|
397
483
|
}
|
|
398
|
-
for (const [id, loc] of Array.from(this.locations)) {
|
|
399
|
-
if (this.registry.cardinalityOf(loc.type) !== null) continue;
|
|
400
|
-
if (loc.isSystem) {
|
|
401
|
-
this.locations.delete(id);
|
|
402
|
-
if (this.locationStore) this.locationStore.delete(id).catch((err) => {
|
|
403
|
-
this.logger.error("storage-orchestrator: prune persistence failed", { meta: {
|
|
404
|
-
id,
|
|
405
|
-
error: err instanceof Error ? err.message : String(err)
|
|
406
|
-
} });
|
|
407
|
-
});
|
|
408
|
-
this.logger.info("storage-orchestrator: pruned stale system location", { meta: {
|
|
409
|
-
id,
|
|
410
|
-
type: loc.type
|
|
411
|
-
} });
|
|
412
|
-
} else this.logger.warn("storage-orchestrator: location of undeclared type kept (operator-owned)", { meta: {
|
|
413
|
-
id,
|
|
414
|
-
type: loc.type
|
|
415
|
-
} });
|
|
416
|
-
}
|
|
417
484
|
}
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
485
|
+
async waitForMoves(job) {
|
|
486
|
+
const sleep = this.deps.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
|
|
487
|
+
const pollMs = this.deps.pollMs ?? 250;
|
|
488
|
+
for (;;) {
|
|
489
|
+
if (!job.cancelRequested) await this.startOrResumeMoves(job);
|
|
490
|
+
let complete = true;
|
|
491
|
+
for (const [index, move] of job.moves.entries()) {
|
|
492
|
+
if (job.cancelRequested && move.moverJobId === null) continue;
|
|
493
|
+
if (move.storageClass !== "eventMedia" && job.moves.slice(0, index).filter((predecessor) => predecessor.storageClass !== "eventMedia").some((predecessor) => predecessor.state !== "done")) {
|
|
494
|
+
complete = false;
|
|
495
|
+
continue;
|
|
496
|
+
}
|
|
497
|
+
if (move.moverJobId === null) {
|
|
498
|
+
if (!job.cancelRequested) complete = false;
|
|
499
|
+
continue;
|
|
500
|
+
}
|
|
501
|
+
const status = await this.moveStatus(move);
|
|
502
|
+
if (status === null) {
|
|
503
|
+
if (job.cancelRequested) {
|
|
504
|
+
move.state = "cancelled";
|
|
505
|
+
continue;
|
|
506
|
+
}
|
|
507
|
+
move.moverJobId = await this.startMove(job, move);
|
|
508
|
+
move.state = "running";
|
|
509
|
+
await this.persist(job);
|
|
510
|
+
complete = false;
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
move.state = status.state;
|
|
514
|
+
move.error = status.error;
|
|
515
|
+
if (status.state === "failed") throw new Error(move.error ?? `${move.storageClass} move failed`);
|
|
516
|
+
if (status.state === "cancelled") {
|
|
517
|
+
job.cancelRequested = true;
|
|
518
|
+
continue;
|
|
519
|
+
}
|
|
520
|
+
if (status.state !== "done") complete = false;
|
|
521
|
+
}
|
|
522
|
+
await this.persist(job);
|
|
523
|
+
if (complete) return;
|
|
524
|
+
await sleep(pollMs);
|
|
433
525
|
}
|
|
434
|
-
const def = this.getDefaultLocation(ref);
|
|
435
|
-
if (!def) throw new Error(`No default storage location configured for type "${ref}" — register one or pass a fully-qualified id (\`<type>:<slug>\`)`);
|
|
436
|
-
return def;
|
|
437
|
-
}
|
|
438
|
-
/**
|
|
439
|
-
* Find the storage-provider that backs a given location — the entry point
|
|
440
|
-
* every consumer-facing `storage` cap method funnels through.
|
|
441
|
-
*
|
|
442
|
-
* Routing key is `(providerId, nodeId)`, never `providerId` alone: every
|
|
443
|
-
* node's FilesystemStorageProvider registers under the SAME providerId, so
|
|
444
|
-
* a providerId-only match resolves to whichever node asked. That is how a
|
|
445
|
-
* location created on an agent ended up writing to the HUB's filesystem
|
|
446
|
-
* (root-caused 2026-07-29).
|
|
447
|
-
*/
|
|
448
|
-
async getProviderFor(location) {
|
|
449
|
-
return this.getProviderForTarget({
|
|
450
|
-
providerId: location.providerId,
|
|
451
|
-
...location.nodeId !== void 0 && { nodeId: location.nodeId }
|
|
452
|
-
}, `location "${location.id}"`, location.id);
|
|
453
526
|
}
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
*
|
|
457
|
-
* `descriptor` is the operator-facing subject of the error message
|
|
458
|
-
* (`location "recordings:disk"`, `uploadId "u-42"`); `locationId` is the
|
|
459
|
-
* structured tag for the log line when there is a location behind the
|
|
460
|
-
* dispatch (chunked sessions carry only the session id).
|
|
461
|
-
*
|
|
462
|
-
* Both failure branches LOG before throwing, and NEITHER falls back to the
|
|
463
|
-
* local provider — a silent fallback is the bug this method exists to fix.
|
|
464
|
-
*/
|
|
465
|
-
async getProviderForTarget(target, descriptor, locationId) {
|
|
466
|
-
const nodeId = target.nodeId;
|
|
467
|
-
if (nodeId !== void 0 && bareNodeId(nodeId) !== this.localNode) {
|
|
468
|
-
const remote = this.getRemoteProvider?.(nodeId);
|
|
469
|
-
if (remote) return remote;
|
|
470
|
-
this.logger.error("storage-orchestrator: target node has no reachable storage-provider — dispatch refused", { meta: {
|
|
471
|
-
...locationId !== void 0 && { locationId },
|
|
472
|
-
descriptor,
|
|
473
|
-
nodeId,
|
|
474
|
-
providerId: target.providerId,
|
|
475
|
-
localNodeId: this.localNode,
|
|
476
|
-
crossNodeResolverWired: this.getRemoteProvider !== void 0
|
|
477
|
-
} });
|
|
478
|
-
throw new Error(`storage-provider for ${descriptor} lives on node "${nodeId}" which is not reachable`);
|
|
479
|
-
}
|
|
480
|
-
const providers = this.getProviders();
|
|
481
|
-
for (const p of providers) if ((await p.getProviderInfo()).providerId === target.providerId) return p;
|
|
482
|
-
this.logger.error("storage-orchestrator: no local storage-provider for providerId", { meta: {
|
|
483
|
-
...locationId !== void 0 && { locationId },
|
|
484
|
-
descriptor,
|
|
485
|
-
providerId: target.providerId,
|
|
486
|
-
localNodeId: this.localNode,
|
|
487
|
-
registered: providers.length
|
|
488
|
-
} });
|
|
489
|
-
throw new Error(`No storage-provider registered for providerId "${target.providerId}" (${descriptor})`);
|
|
527
|
+
async verifyMoves(job) {
|
|
528
|
+
for (const move of job.moves) if (move.state !== "done") throw new Error(`${move.storageClass} move did not complete verification`);
|
|
490
529
|
}
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
530
|
+
async startMove(job, move) {
|
|
531
|
+
if (move.storageClass === "eventMedia") return (await this.deps.participants.analytics.startMove({
|
|
532
|
+
toLocationId: move.toLocationId,
|
|
533
|
+
throttleMbps: job.throttleMbps,
|
|
534
|
+
leaseId: requireLease(job)
|
|
535
|
+
})).jobId;
|
|
536
|
+
return (await this.deps.participants.recorder.startMove({
|
|
537
|
+
fromLocationId: move.fromLocationId,
|
|
538
|
+
toLocationId: move.toLocationId,
|
|
539
|
+
footageClass: move.storageClass,
|
|
540
|
+
throttleMbps: job.throttleMbps,
|
|
541
|
+
leaseId: requireLease(job)
|
|
542
|
+
})).jobId;
|
|
502
543
|
}
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
this.
|
|
544
|
+
async moveStatus(move) {
|
|
545
|
+
if (move.moverJobId === null) return null;
|
|
546
|
+
return move.storageClass === "eventMedia" ? this.deps.participants.analytics.getMove(move.moverJobId) : this.deps.participants.recorder.getMove(move.moverJobId);
|
|
506
547
|
}
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
this.
|
|
548
|
+
async cancelMove(move) {
|
|
549
|
+
if (move.moverJobId === null) return;
|
|
550
|
+
if (move.storageClass === "eventMedia") await this.deps.participants.analytics.cancelMove(move.moverJobId);
|
|
551
|
+
else await this.deps.participants.recorder.cancelMove(move.moverJobId);
|
|
510
552
|
}
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
* that opened it. Throws when the orchestrator has no record of the open
|
|
514
|
-
* call (provider gone, or a chunk for a session it never issued).
|
|
515
|
-
*/
|
|
516
|
-
async getProviderForSession(kind, sessionId) {
|
|
517
|
-
const vocabulary = SESSION_VOCABULARY[kind];
|
|
518
|
-
const descriptor = `${vocabulary.idLabel} "${sessionId}"`;
|
|
519
|
-
const target = this.sessionTargets.get(sessionKey(kind, sessionId));
|
|
520
|
-
if (!target) {
|
|
521
|
-
this.logger.error("storage-orchestrator: chunk for an unknown session — dispatch refused", { meta: {
|
|
522
|
-
descriptor,
|
|
523
|
-
kind,
|
|
524
|
-
sessionId
|
|
525
|
-
} });
|
|
526
|
-
throw new Error(`Unknown ${vocabulary.idLabel} "${sessionId}" — orchestrator has no record of ${vocabulary.openMethod}`);
|
|
527
|
-
}
|
|
528
|
-
return this.getProviderForTarget(target, descriptor);
|
|
553
|
+
participant(name) {
|
|
554
|
+
return this.deps.participants[name];
|
|
529
555
|
}
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
556
|
+
async setPhase(job, phase) {
|
|
557
|
+
job.phase = phase;
|
|
558
|
+
if (isTerminal(job)) job.finishedAt = this.deps.now();
|
|
559
|
+
await this.persist(job);
|
|
533
560
|
}
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
*
|
|
540
|
-
* Convention: the bare-type ref `'recordings'` resolves via
|
|
541
|
-
* `getDefaultLocation('recordings')` (walks `isDefault === true`); the
|
|
542
|
-
* actual stored `id` is `'recordings:default'` so it satisfies the
|
|
543
|
-
* `^[a-z][a-z0-9-]*:[a-z0-9-]+$` regex on `StorageLocationSchema.id`.
|
|
544
|
-
*
|
|
545
|
-
* `basePath` is the storage root (e.g. `<CAMSTACK_DATA>`). Each
|
|
546
|
-
* location's `config.basePath` is `<basePath>/<id>`, except when the
|
|
547
|
-
* declaration sets `defaultsTo` — then it inherits the resolved root of
|
|
548
|
-
* the referenced location's default instance (sharing the same root
|
|
549
|
-
* directory for derivative slots like `recordingsLow` → `recordings`).
|
|
550
|
-
*/
|
|
551
|
-
seedDefaults(input) {
|
|
552
|
-
let added = 0;
|
|
553
|
-
const declById = new Map(input.declarations.map((d) => [d.id, d]));
|
|
554
|
-
const rootFor = (d) => {
|
|
555
|
-
const dedicated = d.defaultRoot === "media" ? input.mediaBasePath : d.defaultRoot === "backup" ? input.backupBasePath : void 0;
|
|
556
|
-
return node_path.join(dedicated ?? input.basePath, d.id);
|
|
557
|
-
};
|
|
558
|
-
for (const d of input.declarations) {
|
|
559
|
-
if (this.hasAnyLocationOfType(d.id)) continue;
|
|
560
|
-
const base = d.defaultsTo ? this.getLocationById(`${d.defaultsTo}:default`)?.config["basePath"] ?? rootFor(declById.get(d.defaultsTo) ?? d) : rootFor(d);
|
|
561
|
-
this.upsertLocation({
|
|
562
|
-
id: `${d.id}:default`,
|
|
563
|
-
type: d.id,
|
|
564
|
-
displayName: d.displayName,
|
|
565
|
-
providerId: input.providerId,
|
|
566
|
-
config: { basePath: base },
|
|
567
|
-
isDefault: true,
|
|
568
|
-
isSystem: true
|
|
569
|
-
});
|
|
570
|
-
added++;
|
|
571
|
-
}
|
|
572
|
-
if (added > 0) this.logger.info("storage-orchestrator: seeded default locations", { meta: {
|
|
573
|
-
added,
|
|
574
|
-
basePath: input.basePath
|
|
575
|
-
} });
|
|
561
|
+
async finishCancelled(job) {
|
|
562
|
+
job.phase = "cancelled";
|
|
563
|
+
job.finishedAt = this.deps.now();
|
|
564
|
+
await this.releaseAfterTerminal(job);
|
|
565
|
+
await this.persist(job);
|
|
576
566
|
}
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
return false;
|
|
567
|
+
async persist(job) {
|
|
568
|
+
job.updatedAt = this.deps.now();
|
|
569
|
+
await this.deps.state.set(job);
|
|
581
570
|
}
|
|
582
571
|
};
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
* newly-declared column counts as a missing column → the structured
|
|
591
|
-
* backend drops the table, wiping operator location customizations such
|
|
592
|
-
* as `recordings:default`'s `minFreePercent`). The double-underscore
|
|
593
|
-
* prefix keeps it out of the provider-facing config namespace
|
|
594
|
-
* (`basePath`, `minFreePercent`, …) — the mapper strips it on read so
|
|
595
|
-
* API consumers and providers never see it.
|
|
596
|
-
*/
|
|
597
|
-
var NODE_ID_CONFIG_KEY = "__nodeId";
|
|
598
|
-
/**
|
|
599
|
-
* `config` is declared `JSON`, so the store hands the surface an object and
|
|
600
|
-
* gets an object back. There is deliberately NO `node_id` column — `nodeId`
|
|
601
|
-
* rides inside the blob under a reserved key (see {@link NODE_ID_CONFIG_KEY}),
|
|
602
|
-
* because a newly-declared column would put this table through
|
|
603
|
-
* `declareCollection`'s destructive rebuild and wipe operator customizations
|
|
604
|
-
* such as `recordings:default`'s `minFreePercent`.
|
|
605
|
-
*/
|
|
606
|
-
var STORAGE_LOCATIONS_COLUMNS = [
|
|
607
|
-
{
|
|
608
|
-
name: "id",
|
|
609
|
-
type: "TEXT",
|
|
610
|
-
primaryKey: true
|
|
611
|
-
},
|
|
612
|
-
{
|
|
613
|
-
name: "type",
|
|
614
|
-
type: "TEXT",
|
|
615
|
-
notNull: true
|
|
616
|
-
},
|
|
617
|
-
{
|
|
618
|
-
name: "display_name",
|
|
619
|
-
type: "TEXT",
|
|
620
|
-
notNull: true
|
|
621
|
-
},
|
|
622
|
-
{
|
|
623
|
-
name: "provider_id",
|
|
624
|
-
type: "TEXT",
|
|
625
|
-
notNull: true
|
|
626
|
-
},
|
|
627
|
-
{
|
|
628
|
-
name: "config",
|
|
629
|
-
type: "JSON",
|
|
630
|
-
notNull: true,
|
|
631
|
-
defaultValue: "{}"
|
|
632
|
-
},
|
|
633
|
-
{
|
|
634
|
-
name: "is_default",
|
|
635
|
-
type: "INTEGER",
|
|
636
|
-
notNull: true,
|
|
637
|
-
defaultValue: 0
|
|
638
|
-
},
|
|
639
|
-
{
|
|
640
|
-
name: "is_system",
|
|
641
|
-
type: "INTEGER",
|
|
642
|
-
notNull: true,
|
|
643
|
-
defaultValue: 0
|
|
644
|
-
},
|
|
645
|
-
{
|
|
646
|
-
name: "created_at",
|
|
647
|
-
type: "INTEGER",
|
|
648
|
-
notNull: true
|
|
649
|
-
},
|
|
650
|
-
{
|
|
651
|
-
name: "updated_at",
|
|
652
|
-
type: "INTEGER",
|
|
653
|
-
notNull: true
|
|
654
|
-
}
|
|
655
|
-
];
|
|
656
|
-
var STORAGE_LOCATIONS_INDEXES = [{
|
|
657
|
-
name: "idx_storage_locations_type",
|
|
658
|
-
columns: ["type"]
|
|
659
|
-
}];
|
|
660
|
-
/**
|
|
661
|
-
* `ILocationStore` on the `settings-store` collection surface — the same door
|
|
662
|
-
* every other consumer uses. It holds the `data-store-provider` engine object
|
|
663
|
-
* directly (not the `settings-store` singleton) because since D44 the
|
|
664
|
-
* storage-orchestrator IS that singleton, and asking the registry for it would
|
|
665
|
-
* route this table back through the orchestrator itself.
|
|
666
|
-
*/
|
|
667
|
-
var SqliteLocationStore = class {
|
|
668
|
-
store;
|
|
669
|
-
declared = false;
|
|
670
|
-
constructor(store) {
|
|
671
|
-
this.store = store;
|
|
672
|
-
}
|
|
673
|
-
/**
|
|
674
|
-
* Lazy `declareCollection` — keeps the constructor cheap and async-free.
|
|
675
|
-
* Called from every mutation/read path; `declareCollection` is idempotent
|
|
676
|
-
* (and additive on shape changes), so repeated calls collapse to a no-op.
|
|
677
|
-
*/
|
|
678
|
-
async ensureDeclared() {
|
|
679
|
-
if (this.declared) return;
|
|
680
|
-
await this.store.declareCollection({
|
|
681
|
-
collection: STORAGE_LOCATIONS,
|
|
682
|
-
columns: STORAGE_LOCATIONS_COLUMNS,
|
|
683
|
-
indexes: STORAGE_LOCATIONS_INDEXES
|
|
684
|
-
});
|
|
685
|
-
this.declared = true;
|
|
686
|
-
}
|
|
687
|
-
async loadAll() {
|
|
688
|
-
await this.ensureDeclared();
|
|
689
|
-
const rows = await this.store.query({
|
|
690
|
-
collection: STORAGE_LOCATIONS,
|
|
691
|
-
filter: { orderBy: {
|
|
692
|
-
field: "created_at",
|
|
693
|
-
direction: "asc"
|
|
694
|
-
} }
|
|
695
|
-
});
|
|
696
|
-
const out = [];
|
|
697
|
-
for (const row of rows) {
|
|
698
|
-
const mapped = mapRowToLocation(row.data);
|
|
699
|
-
if (mapped) out.push(mapped);
|
|
700
|
-
}
|
|
701
|
-
return out;
|
|
702
|
-
}
|
|
703
|
-
/**
|
|
704
|
-
* Replace-or-insert. Implemented as delete + insert rather than a
|
|
705
|
-
* conditional update because the orchestrator serializes every mutation
|
|
706
|
-
* through the in-memory map before dispatching to the store — there's no
|
|
707
|
-
* contention to lose, and delete-then-insert is unconditionally simpler.
|
|
708
|
-
*/
|
|
709
|
-
async upsert(loc) {
|
|
710
|
-
await this.ensureDeclared();
|
|
711
|
-
await this.store.deleteWhere({
|
|
712
|
-
collection: STORAGE_LOCATIONS,
|
|
713
|
-
filter: { where: { id: loc.id } }
|
|
714
|
-
});
|
|
715
|
-
await this.store.insert({
|
|
716
|
-
collection: STORAGE_LOCATIONS,
|
|
717
|
-
record: {
|
|
718
|
-
id: loc.id,
|
|
719
|
-
data: mapLocationToRow(loc)
|
|
720
|
-
}
|
|
721
|
-
});
|
|
722
|
-
}
|
|
723
|
-
async delete(id) {
|
|
724
|
-
await this.ensureDeclared();
|
|
725
|
-
await this.store.deleteWhere({
|
|
726
|
-
collection: STORAGE_LOCATIONS,
|
|
727
|
-
filter: { where: { id } }
|
|
728
|
-
});
|
|
729
|
-
}
|
|
730
|
-
};
|
|
731
|
-
function isPlainObject(value) {
|
|
732
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
733
|
-
}
|
|
734
|
-
function readConfig(raw) {
|
|
735
|
-
if (isPlainObject(raw)) return { ...raw };
|
|
736
|
-
if (typeof raw === "string" && raw.length > 0) return require_dist.parseJsonObject(raw) ?? {};
|
|
737
|
-
return {};
|
|
738
|
-
}
|
|
739
|
-
function mapLocationToRow(loc) {
|
|
740
|
-
const storedConfig = loc.nodeId !== void 0 ? {
|
|
741
|
-
...loc.config,
|
|
742
|
-
[NODE_ID_CONFIG_KEY]: loc.nodeId
|
|
743
|
-
} : { ...loc.config };
|
|
744
|
-
return {
|
|
745
|
-
id: loc.id,
|
|
746
|
-
type: loc.type,
|
|
747
|
-
display_name: loc.displayName,
|
|
748
|
-
provider_id: loc.providerId,
|
|
749
|
-
config: storedConfig,
|
|
750
|
-
is_default: loc.isDefault ? 1 : 0,
|
|
751
|
-
is_system: loc.isSystem ? 1 : 0,
|
|
752
|
-
created_at: loc.createdAt,
|
|
753
|
-
updated_at: loc.updatedAt
|
|
754
|
-
};
|
|
755
|
-
}
|
|
756
|
-
/**
|
|
757
|
-
* Decode a row read from the collection. Returns `null` when the row's `type`
|
|
758
|
-
* fails the id-format schema or the row has no id — a corrupt row (hand-edited,
|
|
759
|
-
* partial migration) is skipped rather than taking the boot read down with it.
|
|
760
|
-
*/
|
|
761
|
-
function mapRowToLocation(row) {
|
|
762
|
-
const typeRaw = row["type"];
|
|
763
|
-
const parsedType = require_dist.StorageLocationTypeSchema.safeParse(typeRaw);
|
|
764
|
-
if (!parsedType.success) return null;
|
|
765
|
-
const type = parsedType.data;
|
|
766
|
-
const id = String(row["id"] ?? "");
|
|
767
|
-
if (!id) return null;
|
|
768
|
-
const storedConfig = readConfig(row["config"]);
|
|
769
|
-
const nodeIdRaw = storedConfig[NODE_ID_CONFIG_KEY];
|
|
770
|
-
const nodeId = typeof nodeIdRaw === "string" && nodeIdRaw.length > 0 ? nodeIdRaw : void 0;
|
|
771
|
-
const config = { ...storedConfig };
|
|
772
|
-
delete config[NODE_ID_CONFIG_KEY];
|
|
773
|
-
return {
|
|
774
|
-
id,
|
|
775
|
-
type,
|
|
776
|
-
displayName: String(row["display_name"] ?? ""),
|
|
777
|
-
providerId: String(row["provider_id"] ?? ""),
|
|
778
|
-
config,
|
|
779
|
-
...nodeId !== void 0 ? { nodeId } : {},
|
|
780
|
-
isDefault: row["is_default"] === 1 || row["is_default"] === true,
|
|
781
|
-
isSystem: row["is_system"] === 1 || row["is_system"] === true,
|
|
782
|
-
createdAt: Number(row["created_at"] ?? 0),
|
|
783
|
-
updatedAt: Number(row["updated_at"] ?? 0)
|
|
784
|
-
};
|
|
785
|
-
}
|
|
786
|
-
//#endregion
|
|
787
|
-
//#region src/builtins/storage-orchestrator/provider-discovery.ts
|
|
788
|
-
async function collectProviderInfos(providers, onError) {
|
|
789
|
-
const out = [];
|
|
790
|
-
for (const [index, p] of providers.entries()) try {
|
|
791
|
-
const info = await p.getProviderInfo();
|
|
792
|
-
out.push(info);
|
|
793
|
-
} catch (err) {
|
|
794
|
-
onError(p, err, index);
|
|
795
|
-
}
|
|
796
|
-
return out;
|
|
797
|
-
}
|
|
572
|
+
function requireLease(job) {
|
|
573
|
+
if (job.pauseLeaseId === null) throw new Error("storage migration has no maintenance lease");
|
|
574
|
+
return job.pauseLeaseId;
|
|
575
|
+
}
|
|
576
|
+
function isTerminal(job) {
|
|
577
|
+
return job.phase === "done" || job.phase === "failed" || job.phase === "cancelled";
|
|
578
|
+
}
|
|
798
579
|
//#endregion
|
|
799
580
|
//#region src/builtins/storage-orchestrator/data-store-dispatch.ts
|
|
800
581
|
/**
|
|
@@ -845,491 +626,771 @@ function createDataStoreDispatch(getEngines) {
|
|
|
845
626
|
};
|
|
846
627
|
}
|
|
847
628
|
//#endregion
|
|
848
|
-
//#region src/builtins/storage-orchestrator/
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
629
|
+
//#region src/builtins/storage-orchestrator/provider-discovery.ts
|
|
630
|
+
async function collectProviderInfos(providers, onError) {
|
|
631
|
+
const out = [];
|
|
632
|
+
for (const [index, p] of providers.entries()) try {
|
|
633
|
+
const info = await p.getProviderInfo();
|
|
634
|
+
out.push(info);
|
|
635
|
+
} catch (err) {
|
|
636
|
+
onError(p, err, index);
|
|
637
|
+
}
|
|
638
|
+
return out;
|
|
639
|
+
}
|
|
640
|
+
//#endregion
|
|
641
|
+
//#region src/builtins/storage-orchestrator/storage-orchestrator.service.ts
|
|
642
|
+
/**
|
|
643
|
+
* Routing core for the `storage-orchestrator` builtin.
|
|
644
|
+
*
|
|
645
|
+
* Owns the in-memory `locationId → StorageLocation` map and the
|
|
646
|
+
* `(providerId, nodeId) → IStorageProviderImpl` lookup. Every consumer-facing
|
|
647
|
+
* `storage` cap call lands here:
|
|
648
|
+
* 1. `resolveRef(ref)` — bare type or fully-qualified id → concrete location
|
|
649
|
+
* 2. `getProviderFor(location)` — route `(providerId, nodeId)` to either a
|
|
650
|
+
* local `storage-provider` registrant or the provider ON the node the
|
|
651
|
+
* location lives on. `providerId` alone is NOT a routing key: every
|
|
652
|
+
* node's filesystem provider shares it.
|
|
653
|
+
* 3. `getProviderForSession(kind, id)` — the same routing for chunked I/O,
|
|
654
|
+
* which carries no location after the opening call.
|
|
655
|
+
*
|
|
656
|
+
* The provider lookup is injected (`getProviders`) so unit tests can
|
|
657
|
+
* pass fixtures without booting a real CapabilityRegistry. The addon
|
|
658
|
+
* wires the production lookup to
|
|
659
|
+
* `ctx.kernel.capabilityRegistry?.getCollection<IStorageProviderImpl>('storage-provider')`.
|
|
660
|
+
*
|
|
661
|
+
* Task 5 adds first-boot seeding of default locations from addon-declared
|
|
662
|
+
* location types (providers no longer gate by type).
|
|
663
|
+
* Task 6 adds an optional `ILocationStore` for SQLite persistence —
|
|
664
|
+
* `initialize()` hydrates the in-memory map from the store, and every
|
|
665
|
+
* `upsertLocation` / `deleteLocation` mirrors the change to the store.
|
|
666
|
+
* Tests pass `null` (or a fake `ILocationStore`) for in-memory-only
|
|
667
|
+
* operation; production wires `SqliteLocationStore`.
|
|
668
|
+
*/
|
|
669
|
+
var SESSION_VOCABULARY = {
|
|
670
|
+
upload: {
|
|
671
|
+
idLabel: "uploadId",
|
|
672
|
+
openMethod: "beginUpload"
|
|
673
|
+
},
|
|
674
|
+
download: {
|
|
675
|
+
idLabel: "downloadId",
|
|
676
|
+
openMethod: "beginDownload"
|
|
677
|
+
}
|
|
678
|
+
};
|
|
679
|
+
/** Map key for a tracked chunked session — the id spaces never collide. */
|
|
680
|
+
function sessionKey(kind, sessionId) {
|
|
681
|
+
return `${kind}:${sessionId}`;
|
|
682
|
+
}
|
|
683
|
+
/**
|
|
684
|
+
* Bare node id for a raw kernel `localNodeId`. Forked children carry a
|
|
685
|
+
* `<node>/<addon>` suffix; routing is per-NODE, so the addon segment is
|
|
686
|
+
* dropped. (Mirrors `normalizeNodeId` in `@camstack/types/addon`, which is a
|
|
687
|
+
* leaf module not re-exported on any subpath this package can import.)
|
|
688
|
+
*/
|
|
689
|
+
function bareNodeId(raw) {
|
|
690
|
+
if (raw === void 0 || raw === "") return "hub";
|
|
691
|
+
const slash = raw.indexOf("/");
|
|
692
|
+
if (slash < 0) return raw;
|
|
693
|
+
const bare = raw.slice(0, slash);
|
|
694
|
+
return bare === "" ? "hub" : bare;
|
|
695
|
+
}
|
|
696
|
+
var StorageOrchestratorService = class {
|
|
697
|
+
logger;
|
|
698
|
+
getProviders;
|
|
699
|
+
locationStore;
|
|
700
|
+
getRemoteProvider;
|
|
701
|
+
locations = /* @__PURE__ */ new Map();
|
|
702
|
+
/**
|
|
703
|
+
* Declaration-driven cardinality source, injected by the addon after
|
|
704
|
+
* the kernel aggregates `storageLocations` declarations. `null` until
|
|
705
|
+
* `setRegistry` runs — `upsertLocation` then treats every type as
|
|
706
|
+
* `'multi'` (no singleton enforcement), the safe default for the
|
|
707
|
+
* in-memory-only / early-boot path.
|
|
708
|
+
*/
|
|
709
|
+
registry = null;
|
|
710
|
+
/**
|
|
711
|
+
* Injected `providerId → nodeLocal` resolver (see {@link NodeLocalResolver}).
|
|
712
|
+
* `null` until {@link setNodeLocalResolver} runs — `upsertLocation` then
|
|
713
|
+
* skips the node-local requirement check entirely (every provider is
|
|
714
|
+
* treated as node-agnostic), the safe default for the in-memory-only /
|
|
715
|
+
* early-boot path that predates the resolver wiring.
|
|
716
|
+
*/
|
|
717
|
+
nodeLocalResolver = null;
|
|
718
|
+
/** `localNodeId` with any forked-child `/addon` suffix stripped. */
|
|
719
|
+
localNode;
|
|
720
|
+
/**
|
|
721
|
+
* `<kind>:<sessionId>` → the dispatch target that OPENED the session.
|
|
722
|
+
* Chunked I/O carries no `StorageLocation` after the first hop, so without
|
|
723
|
+
* this the follow-up chunks fall back to a `providerId`-only lookup — which
|
|
724
|
+
* on a multi-node cluster means the hub's filesystem, not the agent's.
|
|
725
|
+
*/
|
|
726
|
+
sessionTargets = /* @__PURE__ */ new Map();
|
|
727
|
+
constructor(logger, getProviders, locationStore = null, localNodeId = "hub", getRemoteProvider) {
|
|
728
|
+
this.logger = logger;
|
|
729
|
+
this.getProviders = getProviders;
|
|
730
|
+
this.locationStore = locationStore;
|
|
731
|
+
this.getRemoteProvider = getRemoteProvider;
|
|
732
|
+
this.localNode = bareNodeId(localNodeId);
|
|
733
|
+
}
|
|
734
|
+
/**
|
|
735
|
+
* The node this orchestrator runs on, with any forked-child `/addon`
|
|
736
|
+
* suffix stripped. A location whose `nodeId` differs is NOT this node's
|
|
737
|
+
* to touch — its bytes, and its free space, live elsewhere.
|
|
738
|
+
*/
|
|
739
|
+
getLocalNodeId() {
|
|
740
|
+
return this.localNode;
|
|
741
|
+
}
|
|
742
|
+
/** True once a persistence store is wired (constructor or {@link attachStore}). */
|
|
743
|
+
hasStore() {
|
|
744
|
+
return this.locationStore !== null;
|
|
745
|
+
}
|
|
746
|
+
/**
|
|
747
|
+
* Late-bind the persistence store (production boot: the `settings-store`
|
|
748
|
+
* cap registers after the orchestrator initializes, so the service runs
|
|
749
|
+
* in-memory-only until then). Idempotent — a second call once a store is
|
|
750
|
+
* wired is a no-op.
|
|
751
|
+
*
|
|
752
|
+
* Reconciles both directions so operator edits survive a reboot:
|
|
753
|
+
* 1. hydrate — DB rows win over any early in-memory seed (restores
|
|
754
|
+
* operator config like `minFreePercent` that the pre-store boot
|
|
755
|
+
* can't see), with the same `isSystem` upgrade as {@link initialize};
|
|
756
|
+
* 2. backfill — in-memory locations the DB doesn't have yet (the
|
|
757
|
+
* pre-store seed defaults on a fresh install) are persisted, so the
|
|
758
|
+
* store becomes the durable source of truth from here on.
|
|
759
|
+
*/
|
|
760
|
+
async attachStore(store) {
|
|
761
|
+
if (this.locationStore) return;
|
|
762
|
+
this.locationStore = store;
|
|
763
|
+
const rows = await store.loadAll();
|
|
764
|
+
const dbIds = new Set(rows.map((r) => r.id));
|
|
765
|
+
for (const loc of rows) {
|
|
766
|
+
const upgraded = !loc.isSystem && loc.id === `${loc.type}:default` ? {
|
|
767
|
+
...loc,
|
|
768
|
+
isSystem: true
|
|
769
|
+
} : loc;
|
|
770
|
+
this.locations.set(upgraded.id, upgraded);
|
|
771
|
+
if (upgraded !== loc) store.upsert(upgraded).catch(() => {});
|
|
772
|
+
}
|
|
773
|
+
const backfill = [...this.locations.values()].filter((l) => !dbIds.has(l.id));
|
|
774
|
+
for (const loc of backfill) store.upsert(loc).catch((err) => {
|
|
775
|
+
this.logger.warn("storage-orchestrator: attachStore backfill persist failed", { meta: {
|
|
776
|
+
id: loc.id,
|
|
777
|
+
error: err instanceof Error ? err.message : String(err)
|
|
778
|
+
} });
|
|
779
|
+
});
|
|
780
|
+
this.logger.info("storage-orchestrator: store attached + reconciled", { meta: {
|
|
781
|
+
hydrated: rows.length,
|
|
782
|
+
backfilled: backfill.length,
|
|
783
|
+
total: this.locations.size
|
|
784
|
+
} });
|
|
785
|
+
}
|
|
786
|
+
/**
|
|
787
|
+
* Inject the declaration-driven cardinality source. Called once by the
|
|
788
|
+
* addon after the kernel aggregates every addon's `storageLocations`
|
|
789
|
+
* declarations into a `StorageLocationRegistry`.
|
|
790
|
+
*/
|
|
791
|
+
setRegistry(registry) {
|
|
792
|
+
this.registry = registry;
|
|
793
|
+
}
|
|
794
|
+
/**
|
|
795
|
+
* Inject the synchronous `providerId → nodeLocal` resolver (SP1). Called
|
|
796
|
+
* by the addon from a cached snapshot of every `storage-provider`'s
|
|
797
|
+
* `getProviderInfo().nodeLocal`, refreshed when the provider collection
|
|
798
|
+
* changes. Powers the node-local upsert guard and the boot backfill.
|
|
799
|
+
*/
|
|
800
|
+
setNodeLocalResolver(resolver) {
|
|
801
|
+
this.nodeLocalResolver = resolver;
|
|
802
|
+
}
|
|
803
|
+
/**
|
|
804
|
+
* The full set of addon-declared storage locations, as aggregated by
|
|
805
|
+
* the kernel and injected via {@link setRegistry}. Powers the admin-UI
|
|
806
|
+
* Data screen's per-declaration grouping. Empty until the registry is
|
|
807
|
+
* injected (early-boot / in-memory-only path).
|
|
808
|
+
*/
|
|
809
|
+
listDeclarations() {
|
|
810
|
+
return this.registry ? this.registry.list() : [];
|
|
811
|
+
}
|
|
812
|
+
/**
|
|
813
|
+
* Hydrate the in-memory map from the persistence layer. Called once
|
|
814
|
+
* by the addon during `onInitialize`, before the eager seed pass.
|
|
815
|
+
* Idempotent — running it twice with the same store contents
|
|
816
|
+
* produces the same map. No-op when the service was constructed
|
|
817
|
+
* without a store.
|
|
818
|
+
*
|
|
819
|
+
* Errors loading individual rows are tolerated: the bad row is
|
|
820
|
+
* skipped and logged so a single corrupted record doesn't lock the
|
|
821
|
+
* whole orchestrator out of boot. (The store-side validation pass
|
|
822
|
+
* before insert means only schema-invalid SQL state can produce
|
|
823
|
+
* such a row.)
|
|
824
|
+
*/
|
|
825
|
+
async initialize() {
|
|
826
|
+
if (!this.locationStore) return;
|
|
827
|
+
const rows = await this.locationStore.loadAll();
|
|
828
|
+
let upgraded = 0;
|
|
829
|
+
for (const loc of rows) if (!loc.isSystem && loc.id === `${loc.type}:default`) {
|
|
830
|
+
const upgradedLoc = {
|
|
831
|
+
...loc,
|
|
832
|
+
isSystem: true
|
|
833
|
+
};
|
|
834
|
+
this.locations.set(upgradedLoc.id, upgradedLoc);
|
|
835
|
+
this.locationStore.upsert(upgradedLoc).catch((err) => {
|
|
836
|
+
this.logger.warn("storage-orchestrator: isSystem upgrade persist failed", { meta: {
|
|
837
|
+
id: upgradedLoc.id,
|
|
838
|
+
error: err instanceof Error ? err.message : String(err)
|
|
839
|
+
} });
|
|
840
|
+
});
|
|
841
|
+
upgraded++;
|
|
842
|
+
} else this.locations.set(loc.id, loc);
|
|
843
|
+
this.logger.info("storage-orchestrator: hydrated locations from store", { meta: {
|
|
844
|
+
loaded: this.locations.size,
|
|
845
|
+
isSystemUpgraded: upgraded
|
|
846
|
+
} });
|
|
847
|
+
}
|
|
848
|
+
/**
|
|
849
|
+
* Boot backfill (SP1): stamp `nodeId` on every persisted node-local
|
|
850
|
+
* location that lacks one. Single-node deployments seeded their
|
|
851
|
+
* filesystem locations before `nodeId` existed — they live on `hubNodeId`
|
|
852
|
+
* (the hub). Node-agnostic (remote) locations are left untouched: their
|
|
853
|
+
* absent `nodeId` is correct (reachable from any node).
|
|
854
|
+
*
|
|
855
|
+
* Idempotent — a location that already carries a `nodeId` (any value) is
|
|
856
|
+
* never re-touched, so re-running on a populated map (or a re-boot) is a
|
|
857
|
+
* no-op. A plain local update, not versioned-migration machinery: only
|
|
858
|
+
* the rows missing `nodeId` are mutated + persisted.
|
|
859
|
+
*
|
|
860
|
+
* No-op until the node-local resolver is wired (it classifies which
|
|
861
|
+
* providers are node-local).
|
|
862
|
+
*/
|
|
863
|
+
async backfillNodeIds(hubNodeId) {
|
|
864
|
+
const resolver = this.nodeLocalResolver;
|
|
865
|
+
if (!resolver) return;
|
|
866
|
+
let stamped = 0;
|
|
867
|
+
for (const [id, loc] of Array.from(this.locations)) {
|
|
868
|
+
if (loc.nodeId) continue;
|
|
869
|
+
if (resolver(loc.providerId) !== true) continue;
|
|
870
|
+
const updated = {
|
|
871
|
+
...loc,
|
|
872
|
+
nodeId: hubNodeId,
|
|
873
|
+
updatedAt: Date.now()
|
|
874
|
+
};
|
|
875
|
+
this.locations.set(id, updated);
|
|
876
|
+
stamped++;
|
|
877
|
+
if (this.locationStore) try {
|
|
878
|
+
await this.locationStore.upsert(updated);
|
|
879
|
+
} catch (err) {
|
|
880
|
+
this.logger.error("storage-orchestrator: nodeId backfill persist failed", { meta: {
|
|
881
|
+
id,
|
|
882
|
+
error: err instanceof Error ? err.message : String(err)
|
|
883
|
+
} });
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
if (stamped > 0) this.logger.info("storage-orchestrator: backfilled nodeId on node-local locations", { meta: {
|
|
887
|
+
stamped,
|
|
888
|
+
hubNodeId
|
|
889
|
+
} });
|
|
890
|
+
}
|
|
891
|
+
listLocations(filter) {
|
|
892
|
+
const all = [...this.locations.values()];
|
|
893
|
+
return filter?.type ? all.filter((l) => l.type === filter.type) : all;
|
|
894
|
+
}
|
|
895
|
+
getDefaultLocation(type) {
|
|
896
|
+
for (const loc of this.locations.values()) if (loc.type === type && loc.isDefault) return loc;
|
|
897
|
+
return null;
|
|
898
|
+
}
|
|
899
|
+
/**
|
|
900
|
+
* Repoint several logical defaults as one in-memory commit. The migration
|
|
901
|
+
* coordinator persists its `repointing` phase before calling this method, so
|
|
902
|
+
* a process crash during the backing-store writes is recovered by replaying
|
|
903
|
+
* the same idempotent target set before anyone resumes writers.
|
|
904
|
+
*/
|
|
905
|
+
async setDefaultLocations(targetByType) {
|
|
906
|
+
if (targetByType.size === 0) return;
|
|
907
|
+
const now = Date.now();
|
|
908
|
+
const nextById = new Map(this.locations);
|
|
909
|
+
const touched = /* @__PURE__ */ new Map();
|
|
910
|
+
for (const [type, targetId] of targetByType) {
|
|
911
|
+
const target = nextById.get(targetId);
|
|
912
|
+
if (!target) throw new Error(`Storage location "${targetId}" not found`);
|
|
913
|
+
if (target.type !== type) throw new Error(`Storage location "${targetId}" is type "${target.type}", expected "${type}"`);
|
|
914
|
+
for (const [id, location] of nextById) {
|
|
915
|
+
if (location.type !== type) continue;
|
|
916
|
+
const isDefault = id === targetId;
|
|
917
|
+
const enable = isDefault && location.enabled === false;
|
|
918
|
+
if (location.isDefault === isDefault && !enable) continue;
|
|
919
|
+
const updated = {
|
|
920
|
+
...location,
|
|
921
|
+
isDefault,
|
|
922
|
+
...enable ? { enabled: true } : {},
|
|
923
|
+
updatedAt: now
|
|
924
|
+
};
|
|
925
|
+
nextById.set(id, updated);
|
|
926
|
+
touched.set(id, updated);
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
if (this.locationStore) {
|
|
930
|
+
const previous = /* @__PURE__ */ new Map();
|
|
931
|
+
for (const id of touched.keys()) {
|
|
932
|
+
const location = this.locations.get(id);
|
|
933
|
+
if (location) previous.set(id, location);
|
|
934
|
+
}
|
|
935
|
+
try {
|
|
936
|
+
for (const location of touched.values()) await this.locationStore.upsert(location);
|
|
937
|
+
} catch (err) {
|
|
938
|
+
for (const location of previous.values()) try {
|
|
939
|
+
await this.locationStore.upsert(location);
|
|
940
|
+
} catch {}
|
|
941
|
+
throw err;
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
this.locations.clear();
|
|
945
|
+
for (const [id, location] of nextById) this.locations.set(id, location);
|
|
946
|
+
}
|
|
947
|
+
/**
|
|
948
|
+
* Insert or update a location. If `isDefault: true`, atomically
|
|
949
|
+
* demotes any other default for the same type — operators always see
|
|
950
|
+
* exactly one default per type.
|
|
951
|
+
*
|
|
952
|
+
* When a persistence store is wired (Task 6), the new record AND any
|
|
953
|
+
* implicitly-demoted siblings are persisted before the in-memory map
|
|
954
|
+
* mutation returns. Persistence errors propagate to the caller.
|
|
955
|
+
*/
|
|
956
|
+
upsertLocation(input) {
|
|
957
|
+
const now = Date.now();
|
|
958
|
+
const existing = this.locations.get(input.id);
|
|
959
|
+
if (!existing) {
|
|
960
|
+
if ((this.registry?.cardinalityOf(input.type) ?? "multi") === "single") {
|
|
961
|
+
const already = [...this.locations.values()].find((l) => l.type === input.type);
|
|
962
|
+
if (already) throw new Error(`Storage type "${input.type}" is single — only one location allowed (existing: "${already.id}"). Edit it instead of adding a new one.`);
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
if (this.nodeLocalResolver?.(input.providerId) === true && !input.nodeId) input = {
|
|
966
|
+
...input,
|
|
967
|
+
nodeId: "hub"
|
|
968
|
+
};
|
|
969
|
+
if (existing?.isSystem === true) input = {
|
|
970
|
+
...input,
|
|
971
|
+
isSystem: true
|
|
972
|
+
};
|
|
973
|
+
input = {
|
|
974
|
+
...input,
|
|
975
|
+
...resolveEnabled(input, existing, this.hasAnyLocationOfType(input.type))
|
|
976
|
+
};
|
|
977
|
+
const next = {
|
|
978
|
+
...input,
|
|
979
|
+
createdAt: existing?.createdAt ?? now,
|
|
980
|
+
updatedAt: now
|
|
981
|
+
};
|
|
982
|
+
const demoted = [];
|
|
983
|
+
if (next.isDefault) {
|
|
984
|
+
for (const [otherId, otherLoc] of this.locations) if (otherLoc.type === next.type && otherLoc.id !== next.id && otherLoc.isDefault) {
|
|
985
|
+
const updated = {
|
|
986
|
+
...otherLoc,
|
|
987
|
+
isDefault: false,
|
|
988
|
+
updatedAt: now
|
|
989
|
+
};
|
|
990
|
+
this.locations.set(otherId, updated);
|
|
991
|
+
demoted.push(updated);
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
this.locations.set(next.id, next);
|
|
995
|
+
this.logger.debug("storage-orchestrator: upsertLocation", { meta: {
|
|
996
|
+
id: next.id,
|
|
997
|
+
type: next.type,
|
|
998
|
+
providerId: next.providerId,
|
|
999
|
+
isDefault: next.isDefault
|
|
1000
|
+
} });
|
|
1001
|
+
if (this.locationStore) {
|
|
1002
|
+
const store = this.locationStore;
|
|
1003
|
+
(async () => {
|
|
1004
|
+
try {
|
|
1005
|
+
for (const d of demoted) await store.upsert(d);
|
|
1006
|
+
await store.upsert(next);
|
|
1007
|
+
} catch (err) {
|
|
1008
|
+
this.logger.error("storage-orchestrator: upsert persistence failed", { meta: {
|
|
1009
|
+
id: next.id,
|
|
1010
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1011
|
+
} });
|
|
1012
|
+
}
|
|
1013
|
+
})();
|
|
1014
|
+
}
|
|
1015
|
+
return next;
|
|
1016
|
+
}
|
|
1017
|
+
/**
|
|
1018
|
+
* Remove a location. Refuses to remove the default of a type unless a
|
|
1019
|
+
* sibling default exists (defensive — `upsertLocation` already ensures
|
|
1020
|
+
* at most one default per type, but the logic guards against future
|
|
1021
|
+
* bypass paths e.g. SQLite migration that imports two defaults).
|
|
1022
|
+
*
|
|
1023
|
+
* Persistence (Task 6) mirrors the delete asynchronously, with errors
|
|
1024
|
+
* routed to the logger — see `upsertLocation` for the rationale.
|
|
1025
|
+
*/
|
|
1026
|
+
deleteLocation(id) {
|
|
1027
|
+
const loc = this.locations.get(id);
|
|
1028
|
+
if (!loc) throw new Error(`Storage location "${id}" not found`);
|
|
1029
|
+
if (loc.isSystem) throw new Error(`Storage location "${id}" is system-managed and cannot be deleted. Edit its config (path / providerId) instead.`);
|
|
1030
|
+
if (loc.isDefault) {
|
|
1031
|
+
if (![...this.locations.values()].find((l) => l.type === loc.type && l.id !== id && l.isDefault)) throw new Error(`Cannot delete default location "${id}" for type "${loc.type}" — promote another location to default first`);
|
|
1032
|
+
}
|
|
1033
|
+
this.locations.delete(id);
|
|
1034
|
+
if (this.locationStore) this.locationStore.delete(id).catch((err) => {
|
|
1035
|
+
this.logger.error("storage-orchestrator: delete persistence failed", { meta: {
|
|
1036
|
+
id,
|
|
1037
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1038
|
+
} });
|
|
1039
|
+
});
|
|
855
1040
|
}
|
|
856
|
-
/**
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
this.deps.logger.warn("storage pressure: relief failed for location", { meta: {
|
|
874
|
-
locationId,
|
|
875
|
-
error: err instanceof Error ? err.message : String(err)
|
|
876
|
-
} });
|
|
877
|
-
}
|
|
1041
|
+
/**
|
|
1042
|
+
* Remove system-seeded locations whose type is no longer declared by any
|
|
1043
|
+
* addon (stale defaults from a removed location type). Operator-added
|
|
1044
|
+
* (non-system) locations of an undeclared type are KEPT but warned — the
|
|
1045
|
+
* operator owns them. Requires the registry to be set first.
|
|
1046
|
+
*
|
|
1047
|
+
* FAIL-SAFE: if the registry is EMPTY (no addon declared any location), we
|
|
1048
|
+
* refuse to prune anything. An empty registry almost always means
|
|
1049
|
+
* declarations failed to load (boot ordering, a stale install) — pruning
|
|
1050
|
+
* "everything undeclared" in that state would wipe every system location
|
|
1051
|
+
* (data/logs/recordings/…). Better to keep stale rows than destroy live ones.
|
|
1052
|
+
*/
|
|
1053
|
+
pruneUndeclaredSystemLocations() {
|
|
1054
|
+
if (!this.registry) return;
|
|
1055
|
+
if (this.registry.list().length === 0) {
|
|
1056
|
+
this.logger.warn("storage-orchestrator: skipping prune — no location declarations loaded (fail-safe)");
|
|
1057
|
+
return;
|
|
878
1058
|
}
|
|
879
|
-
const
|
|
880
|
-
|
|
881
|
-
if (
|
|
882
|
-
|
|
883
|
-
if (
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
1059
|
+
for (const [id, loc] of Array.from(this.locations)) {
|
|
1060
|
+
if (this.registry.cardinalityOf(loc.type) !== null) continue;
|
|
1061
|
+
if (loc.isSystem) {
|
|
1062
|
+
this.locations.delete(id);
|
|
1063
|
+
if (this.locationStore) this.locationStore.delete(id).catch((err) => {
|
|
1064
|
+
this.logger.error("storage-orchestrator: prune persistence failed", { meta: {
|
|
1065
|
+
id,
|
|
1066
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1067
|
+
} });
|
|
1068
|
+
});
|
|
1069
|
+
this.logger.info("storage-orchestrator: pruned stale system location", { meta: {
|
|
1070
|
+
id,
|
|
1071
|
+
type: loc.type
|
|
891
1072
|
} });
|
|
892
|
-
}
|
|
1073
|
+
} else this.logger.warn("storage-orchestrator: location of undeclared type kept (operator-owned)", { meta: {
|
|
1074
|
+
id,
|
|
1075
|
+
type: loc.type
|
|
1076
|
+
} });
|
|
893
1077
|
}
|
|
894
1078
|
}
|
|
895
|
-
/**
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1079
|
+
/**
|
|
1080
|
+
* Resolve a `StorageLocationRef` to a concrete `StorageLocation`.
|
|
1081
|
+
* - bare type (e.g. `'backups'`) → default location of that type
|
|
1082
|
+
* - fully-qualified id (e.g. `'backups:nas-01'`) → that exact instance
|
|
1083
|
+
*
|
|
1084
|
+
* Throws an actionable error when nothing matches — every consumer of
|
|
1085
|
+
* the singleton `storage` cap funnels through here, so the error
|
|
1086
|
+
* message is what operators see when the orchestrator's view of the
|
|
1087
|
+
* world doesn't match expectations.
|
|
1088
|
+
*/
|
|
1089
|
+
resolveRef(ref) {
|
|
1090
|
+
if (ref.includes(":")) {
|
|
1091
|
+
const loc = this.locations.get(ref);
|
|
1092
|
+
if (!loc) throw new Error(`Storage location "${ref}" not found`);
|
|
1093
|
+
return loc;
|
|
1094
|
+
}
|
|
1095
|
+
const def = this.getDefaultLocation(ref);
|
|
1096
|
+
if (!def) throw new Error(`No default storage location configured for type "${ref}" — register one or pass a fully-qualified id (\`<type>:<slug>\`)`);
|
|
1097
|
+
return def;
|
|
1098
|
+
}
|
|
1099
|
+
/**
|
|
1100
|
+
* Find the storage-provider that backs a given location — the entry point
|
|
1101
|
+
* every consumer-facing `storage` cap method funnels through.
|
|
1102
|
+
*
|
|
1103
|
+
* Routing key is `(providerId, nodeId)`, never `providerId` alone: every
|
|
1104
|
+
* node's FilesystemStorageProvider registers under the SAME providerId, so
|
|
1105
|
+
* a providerId-only match resolves to whichever node asked. That is how a
|
|
1106
|
+
* location created on an agent ended up writing to the HUB's filesystem
|
|
1107
|
+
* (root-caused 2026-07-29).
|
|
1108
|
+
*/
|
|
1109
|
+
async getProviderFor(location) {
|
|
1110
|
+
return this.getProviderForTarget({
|
|
1111
|
+
providerId: location.providerId,
|
|
1112
|
+
...location.nodeId !== void 0 && { nodeId: location.nodeId }
|
|
1113
|
+
}, `location "${location.id}"`, location.id);
|
|
1114
|
+
}
|
|
1115
|
+
/**
|
|
1116
|
+
* Resolve one dispatch target to the provider that must execute it.
|
|
1117
|
+
*
|
|
1118
|
+
* `descriptor` is the operator-facing subject of the error message
|
|
1119
|
+
* (`location "recordings:disk"`, `uploadId "u-42"`); `locationId` is the
|
|
1120
|
+
* structured tag for the log line when there is a location behind the
|
|
1121
|
+
* dispatch (chunked sessions carry only the session id).
|
|
1122
|
+
*
|
|
1123
|
+
* Both failure branches LOG before throwing, and NEITHER falls back to the
|
|
1124
|
+
* local provider — a silent fallback is the bug this method exists to fix.
|
|
1125
|
+
*/
|
|
1126
|
+
async getProviderForTarget(target, descriptor, locationId) {
|
|
1127
|
+
const nodeId = target.nodeId;
|
|
1128
|
+
if (nodeId !== void 0 && bareNodeId(nodeId) !== this.localNode) {
|
|
1129
|
+
const remote = this.getRemoteProvider?.(nodeId);
|
|
1130
|
+
if (remote) return remote;
|
|
1131
|
+
this.logger.error("storage-orchestrator: target node has no reachable storage-provider — dispatch refused", { meta: {
|
|
1132
|
+
...locationId !== void 0 && { locationId },
|
|
1133
|
+
descriptor,
|
|
1134
|
+
nodeId,
|
|
1135
|
+
providerId: target.providerId,
|
|
1136
|
+
localNodeId: this.localNode,
|
|
1137
|
+
crossNodeResolverWired: this.getRemoteProvider !== void 0
|
|
923
1138
|
} });
|
|
924
|
-
|
|
1139
|
+
throw new Error(`storage-provider for ${descriptor} lives on node "${nodeId}" which is not reachable`);
|
|
925
1140
|
}
|
|
1141
|
+
const providers = this.getProviders();
|
|
1142
|
+
for (const p of providers) if ((await p.getProviderInfo()).providerId === target.providerId) return p;
|
|
1143
|
+
this.logger.error("storage-orchestrator: no local storage-provider for providerId", { meta: {
|
|
1144
|
+
...locationId !== void 0 && { locationId },
|
|
1145
|
+
descriptor,
|
|
1146
|
+
providerId: target.providerId,
|
|
1147
|
+
localNodeId: this.localNode,
|
|
1148
|
+
registered: providers.length
|
|
1149
|
+
} });
|
|
1150
|
+
throw new Error(`No storage-provider registered for providerId "${target.providerId}" (${descriptor})`);
|
|
926
1151
|
}
|
|
927
1152
|
/**
|
|
928
|
-
*
|
|
929
|
-
*
|
|
930
|
-
*
|
|
931
|
-
*
|
|
932
|
-
* sibling alias may re-read a not-yet-settled `statfs`), `false` when nothing
|
|
933
|
-
* was freed (never breached, or breached with no evictable data).
|
|
1153
|
+
* Remember which provider — and on which NODE — opened a chunked session.
|
|
1154
|
+
* `beginUpload` / `beginDownload` carry a `StorageLocation`; every later
|
|
1155
|
+
* chunk carries only the session id, and a session id issued by an agent
|
|
1156
|
+
* means nothing to the hub's provider.
|
|
934
1157
|
*/
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
const share = Math.max(1, Math.ceil(targetBytes * u.bytes / totalEvictable));
|
|
964
|
-
const res = await u.p.evict({
|
|
965
|
-
locationId,
|
|
966
|
-
targetBytes: share
|
|
967
|
-
});
|
|
968
|
-
reclaimed += res.reclaimedBytes;
|
|
969
|
-
}
|
|
970
|
-
if (reclaimed > 0) reclaimedAny = true;
|
|
971
|
-
this.deps.logger.info("storage pressure: relief round", { meta: {
|
|
972
|
-
locationId,
|
|
973
|
-
round,
|
|
974
|
-
freePercent: free,
|
|
975
|
-
threshold,
|
|
976
|
-
targetBytes,
|
|
977
|
-
reclaimedBytes: reclaimed
|
|
1158
|
+
rememberSession(kind, sessionId, location) {
|
|
1159
|
+
this.sessionTargets.set(sessionKey(kind, sessionId), {
|
|
1160
|
+
providerId: location.providerId,
|
|
1161
|
+
...location.nodeId !== void 0 && { nodeId: location.nodeId }
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
/** Drop a session's routing entry (finalize / abort / end). */
|
|
1165
|
+
forgetSession(kind, sessionId) {
|
|
1166
|
+
this.sessionTargets.delete(sessionKey(kind, sessionId));
|
|
1167
|
+
}
|
|
1168
|
+
/** Drop every session entry (addon shutdown). */
|
|
1169
|
+
clearSessions() {
|
|
1170
|
+
this.sessionTargets.clear();
|
|
1171
|
+
}
|
|
1172
|
+
/**
|
|
1173
|
+
* Resolve the provider that owns an in-flight chunked session, on the node
|
|
1174
|
+
* that opened it. Throws when the orchestrator has no record of the open
|
|
1175
|
+
* call (provider gone, or a chunk for a session it never issued).
|
|
1176
|
+
*/
|
|
1177
|
+
async getProviderForSession(kind, sessionId) {
|
|
1178
|
+
const vocabulary = SESSION_VOCABULARY[kind];
|
|
1179
|
+
const descriptor = `${vocabulary.idLabel} "${sessionId}"`;
|
|
1180
|
+
const target = this.sessionTargets.get(sessionKey(kind, sessionId));
|
|
1181
|
+
if (!target) {
|
|
1182
|
+
this.logger.error("storage-orchestrator: chunk for an unknown session — dispatch refused", { meta: {
|
|
1183
|
+
descriptor,
|
|
1184
|
+
kind,
|
|
1185
|
+
sessionId
|
|
978
1186
|
} });
|
|
979
|
-
|
|
1187
|
+
throw new Error(`Unknown ${vocabulary.idLabel} "${sessionId}" — orchestrator has no record of ${vocabulary.openMethod}`);
|
|
980
1188
|
}
|
|
981
|
-
return
|
|
1189
|
+
return this.getProviderForTarget(target, descriptor);
|
|
1190
|
+
}
|
|
1191
|
+
/** Convenience: lookup by id (returns `undefined` if not found). */
|
|
1192
|
+
getLocationById(id) {
|
|
1193
|
+
return this.locations.get(id);
|
|
1194
|
+
}
|
|
1195
|
+
/**
|
|
1196
|
+
* Seed one default `StorageLocation` per addon-declared location that
|
|
1197
|
+
* doesn't already have one. Idempotent — re-running with the same
|
|
1198
|
+
* declarations on a populated map is a no-op (each declared `id`
|
|
1199
|
+
* already resolves to an existing `<id>:default`).
|
|
1200
|
+
*
|
|
1201
|
+
* Convention: the bare-type ref `'recordings'` resolves via
|
|
1202
|
+
* `getDefaultLocation('recordings')` (walks `isDefault === true`); the
|
|
1203
|
+
* actual stored `id` is `'recordings:default'` so it satisfies the
|
|
1204
|
+
* `^[a-z][a-z0-9-]*:[a-z0-9-]+$` regex on `StorageLocationSchema.id`.
|
|
1205
|
+
*
|
|
1206
|
+
* `basePath` is the storage root (e.g. `<CAMSTACK_DATA>`). Each
|
|
1207
|
+
* location's `config.basePath` is `<basePath>/<id>`, except when the
|
|
1208
|
+
* declaration sets `defaultsTo` — then it inherits the resolved root of
|
|
1209
|
+
* the referenced location's default instance (sharing the same root
|
|
1210
|
+
* directory for derivative slots like `recordingsLow` → `recordings`).
|
|
1211
|
+
*/
|
|
1212
|
+
seedDefaults(input) {
|
|
1213
|
+
let added = 0;
|
|
1214
|
+
const declById = new Map(input.declarations.map((d) => [d.id, d]));
|
|
1215
|
+
const rootFor = (d) => {
|
|
1216
|
+
const dedicated = d.defaultRoot === "media" ? input.mediaBasePath : d.defaultRoot === "backup" ? input.backupBasePath : void 0;
|
|
1217
|
+
return node_path.join(dedicated ?? input.basePath, d.id);
|
|
1218
|
+
};
|
|
1219
|
+
for (const d of input.declarations) {
|
|
1220
|
+
if (this.hasAnyLocationOfType(d.id)) continue;
|
|
1221
|
+
const base = d.defaultsTo ? this.getLocationById(`${d.defaultsTo}:default`)?.config["basePath"] ?? rootFor(declById.get(d.defaultsTo) ?? d) : rootFor(d);
|
|
1222
|
+
this.upsertLocation({
|
|
1223
|
+
id: `${d.id}:default`,
|
|
1224
|
+
type: d.id,
|
|
1225
|
+
displayName: d.displayName,
|
|
1226
|
+
providerId: input.providerId,
|
|
1227
|
+
config: { basePath: base },
|
|
1228
|
+
isDefault: true,
|
|
1229
|
+
isSystem: true
|
|
1230
|
+
});
|
|
1231
|
+
added++;
|
|
1232
|
+
}
|
|
1233
|
+
if (added > 0) this.logger.info("storage-orchestrator: seeded default locations", { meta: {
|
|
1234
|
+
added,
|
|
1235
|
+
basePath: input.basePath
|
|
1236
|
+
} });
|
|
1237
|
+
}
|
|
1238
|
+
/** Internal: check whether any location of the given type exists. */
|
|
1239
|
+
hasAnyLocationOfType(type) {
|
|
1240
|
+
for (const loc of this.locations.values()) if (loc.type === type) return true;
|
|
1241
|
+
return false;
|
|
982
1242
|
}
|
|
983
1243
|
};
|
|
984
|
-
//#endregion
|
|
985
|
-
//#region src/builtins/storage-orchestrator/storage-migration-coordinator.ts
|
|
986
|
-
var STORAGE_CLASSES = [
|
|
987
|
-
"recordings",
|
|
988
|
-
"recordingsLow",
|
|
989
|
-
"eventMedia"
|
|
990
|
-
];
|
|
991
|
-
var PARTICIPANTS = [
|
|
992
|
-
"pipeline",
|
|
993
|
-
"recorder",
|
|
994
|
-
"analytics"
|
|
995
|
-
];
|
|
996
|
-
var REVERSE_PARTICIPANTS = [
|
|
997
|
-
"analytics",
|
|
998
|
-
"recorder",
|
|
999
|
-
"pipeline"
|
|
1000
|
-
];
|
|
1001
1244
|
/**
|
|
1002
|
-
*
|
|
1003
|
-
*
|
|
1004
|
-
* intentionally narrow lease/move primitives and never alter camera settings.
|
|
1245
|
+
* Resolve `enabled` for one upsert — see the call site in `upsertLocation` for
|
|
1246
|
+
* the ordered rules. Pure: every input it needs is a parameter.
|
|
1005
1247
|
*/
|
|
1006
|
-
|
|
1248
|
+
function resolveEnabled(input, existing, typeAlreadyHasLocation) {
|
|
1249
|
+
const requested = input.enabled ?? existing?.enabled;
|
|
1250
|
+
if (requested === void 0) {
|
|
1251
|
+
if (existing) return {};
|
|
1252
|
+
return { enabled: input.isDefault || !typeAlreadyHasLocation };
|
|
1253
|
+
}
|
|
1254
|
+
if (input.isDefault && requested === false) return { enabled: true };
|
|
1255
|
+
return { enabled: requested };
|
|
1256
|
+
}
|
|
1257
|
+
//#endregion
|
|
1258
|
+
//#region src/builtins/storage-orchestrator/storage-pressure-manager.ts
|
|
1259
|
+
var DEFAULT_MAX_ROUNDS = 8;
|
|
1260
|
+
var DEFAULT_HYSTERESIS_PERCENT = 1;
|
|
1261
|
+
var StoragePressureManager = class {
|
|
1007
1262
|
deps;
|
|
1008
|
-
active = null;
|
|
1009
|
-
runPromise = null;
|
|
1010
|
-
startReserved = false;
|
|
1011
1263
|
constructor(deps) {
|
|
1012
1264
|
this.deps = deps;
|
|
1013
1265
|
}
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
fromLocationId: source.id,
|
|
1028
|
-
toLocationId: target.id,
|
|
1029
|
-
moverJobId: null,
|
|
1030
|
-
state: null,
|
|
1031
|
-
error: null
|
|
1032
|
-
});
|
|
1033
|
-
}
|
|
1034
|
-
if (moves.length === 0) throw new Error("select at least one storage class");
|
|
1035
|
-
return {
|
|
1036
|
-
destinations: input.destinations,
|
|
1037
|
-
moves: moves.map(({ storageClass, fromLocationId, toLocationId }) => ({
|
|
1038
|
-
storageClass,
|
|
1039
|
-
fromLocationId,
|
|
1040
|
-
toLocationId
|
|
1041
|
-
}))
|
|
1042
|
-
};
|
|
1043
|
-
}
|
|
1044
|
-
async start(input) {
|
|
1045
|
-
if (this.startReserved) throw new Error("storage migration is already active");
|
|
1046
|
-
this.startReserved = true;
|
|
1047
|
-
try {
|
|
1048
|
-
const existing = await this.status();
|
|
1049
|
-
if (existing && isTerminal(existing) && existing.pausedParticipants.length > 0) {
|
|
1050
|
-
await this.releaseAfterTerminal(existing);
|
|
1051
|
-
await this.persist(existing);
|
|
1052
|
-
if (existing.pausedParticipants.length > 0) throw new Error(`storage migration ${existing.jobId} still holds maintenance leases`);
|
|
1053
|
-
}
|
|
1054
|
-
if (existing && !isTerminal(existing)) throw new Error(`storage migration is already active (${existing.jobId})`);
|
|
1055
|
-
const plan = await this.plan(input);
|
|
1056
|
-
const now = this.deps.now();
|
|
1057
|
-
const job = {
|
|
1058
|
-
jobId: this.deps.newId(),
|
|
1059
|
-
phase: "planning",
|
|
1060
|
-
destinations: input.destinations,
|
|
1061
|
-
throttleMbps: input.throttleMbps ?? 40,
|
|
1062
|
-
moves: plan.moves.map((move) => ({
|
|
1063
|
-
...move,
|
|
1064
|
-
moverJobId: null,
|
|
1065
|
-
state: null,
|
|
1066
|
-
error: null
|
|
1067
|
-
})),
|
|
1068
|
-
pauseLeaseId: null,
|
|
1069
|
-
pausedParticipants: [],
|
|
1070
|
-
repointed: false,
|
|
1071
|
-
cancelRequested: false,
|
|
1072
|
-
startedAt: now,
|
|
1073
|
-
updatedAt: now,
|
|
1074
|
-
finishedAt: null,
|
|
1075
|
-
error: null
|
|
1076
|
-
};
|
|
1077
|
-
this.active = job;
|
|
1078
|
-
await this.persist(job);
|
|
1079
|
-
this.runPromise = this.run(job);
|
|
1080
|
-
this.runPromise;
|
|
1081
|
-
return job.jobId;
|
|
1082
|
-
} finally {
|
|
1083
|
-
this.startReserved = false;
|
|
1084
|
-
}
|
|
1085
|
-
}
|
|
1086
|
-
async status(jobId) {
|
|
1087
|
-
const job = this.active ?? await this.deps.state.get();
|
|
1088
|
-
if (jobId !== void 0 && job?.jobId !== jobId) return null;
|
|
1089
|
-
return job;
|
|
1090
|
-
}
|
|
1091
|
-
async cancel(jobId) {
|
|
1092
|
-
const job = await this.status(jobId);
|
|
1093
|
-
if (!job || isTerminal(job) || job.repointed) return false;
|
|
1094
|
-
job.cancelRequested = true;
|
|
1095
|
-
await this.persist(job);
|
|
1096
|
-
await Promise.all(job.moves.filter((move) => move.moverJobId !== null).map((move) => this.cancelMove(move)));
|
|
1097
|
-
return true;
|
|
1098
|
-
}
|
|
1099
|
-
/** Boot recovery resumes a durable unfinished state. A missing in-memory
|
|
1100
|
-
* child mover is recreated from the same copy-if-absent input. */
|
|
1101
|
-
async recover() {
|
|
1102
|
-
const job = await this.deps.state.get();
|
|
1103
|
-
if (!job) return;
|
|
1104
|
-
if (job.cancelRequested) {
|
|
1105
|
-
await Promise.all(job.moves.filter((move) => move.moverJobId !== null).map((move) => this.cancelMove(move)));
|
|
1106
|
-
await this.waitForMoves(job);
|
|
1107
|
-
await this.releaseAfterTerminal(job);
|
|
1108
|
-
job.phase = "cancelled";
|
|
1109
|
-
job.finishedAt ??= this.deps.now();
|
|
1110
|
-
await this.persist(job);
|
|
1111
|
-
return;
|
|
1112
|
-
}
|
|
1113
|
-
if (isTerminal(job)) {
|
|
1114
|
-
if (job.pausedParticipants.length > 0) {
|
|
1115
|
-
await this.releaseAfterTerminal(job);
|
|
1116
|
-
await this.persist(job);
|
|
1117
|
-
}
|
|
1118
|
-
return;
|
|
1119
|
-
}
|
|
1120
|
-
this.active = job;
|
|
1121
|
-
job.pauseLeaseId ??= this.deps.newId();
|
|
1122
|
-
if (job.repointed && job.phase === "resuming") job.phase = "refreshing";
|
|
1123
|
-
await this.persist(job);
|
|
1124
|
-
try {
|
|
1125
|
-
await this.pauseParticipants(job, true);
|
|
1126
|
-
} catch (err) {
|
|
1127
|
-
job.error = err instanceof Error ? err.message : String(err);
|
|
1128
|
-
await this.persist(job);
|
|
1129
|
-
return;
|
|
1130
|
-
}
|
|
1131
|
-
if (this.runPromise === null) {
|
|
1132
|
-
this.runPromise = this.run(job);
|
|
1133
|
-
this.runPromise;
|
|
1134
|
-
}
|
|
1135
|
-
}
|
|
1136
|
-
async run(job) {
|
|
1137
|
-
try {
|
|
1138
|
-
if (job.phase === "planning") await this.setPhase(job, "pausing");
|
|
1139
|
-
if (job.phase === "pausing") {
|
|
1140
|
-
job.pauseLeaseId ??= job.jobId;
|
|
1141
|
-
await this.pauseParticipants(job);
|
|
1142
|
-
if (job.cancelRequested) return this.finishCancelled(job);
|
|
1143
|
-
await this.setPhase(job, "moving");
|
|
1144
|
-
}
|
|
1145
|
-
if (job.phase === "moving") {
|
|
1146
|
-
await this.startOrResumeMoves(job);
|
|
1147
|
-
await this.waitForMoves(job);
|
|
1148
|
-
if (job.cancelRequested) return this.finishCancelled(job);
|
|
1149
|
-
await this.setPhase(job, "verifying");
|
|
1150
|
-
}
|
|
1151
|
-
if (job.phase === "verifying") {
|
|
1152
|
-
await this.verifyMoves(job);
|
|
1153
|
-
if (job.cancelRequested) return this.finishCancelled(job);
|
|
1154
|
-
await this.setPhase(job, "repointing");
|
|
1155
|
-
}
|
|
1156
|
-
if (job.phase === "repointing") {
|
|
1157
|
-
const targets = new Map(job.moves.map((move) => [move.storageClass, move.toLocationId]));
|
|
1158
|
-
await this.deps.locations.setDefaultLocations(targets);
|
|
1159
|
-
job.repointed = true;
|
|
1160
|
-
await this.setPhase(job, "refreshing");
|
|
1161
|
-
}
|
|
1162
|
-
if (job.phase === "refreshing") {
|
|
1163
|
-
const leaseId = requireLease(job);
|
|
1164
|
-
if (job.moves.some((move) => move.storageClass !== "eventMedia")) await this.deps.participants.recorder.refresh(leaseId);
|
|
1165
|
-
if (job.moves.some((move) => move.storageClass === "eventMedia")) await this.deps.participants.analytics.refresh(leaseId);
|
|
1166
|
-
await this.setPhase(job, "resuming");
|
|
1167
|
-
}
|
|
1168
|
-
if (job.phase === "resuming") {
|
|
1169
|
-
await this.resumeParticipants(job);
|
|
1170
|
-
await this.setPhase(job, "done");
|
|
1171
|
-
}
|
|
1172
|
-
} catch (err) {
|
|
1173
|
-
job.error = err instanceof Error ? err.message : String(err);
|
|
1174
|
-
if (job.repointed) {
|
|
1175
|
-
job.phase = "refreshing";
|
|
1176
|
-
job.finishedAt = null;
|
|
1177
|
-
await this.persist(job);
|
|
1178
|
-
return;
|
|
1266
|
+
/** Relieve every managed location currently under its threshold. */
|
|
1267
|
+
async sweep() {
|
|
1268
|
+
const thresholds = await this.deps.getLocationThresholds();
|
|
1269
|
+
const relievedDevices = /* @__PURE__ */ new Set();
|
|
1270
|
+
for (const [locationId, threshold] of thresholds) {
|
|
1271
|
+
if (threshold <= 0) continue;
|
|
1272
|
+
const deviceKey = await this.deps.getLocationDeviceKey?.(locationId) ?? null;
|
|
1273
|
+
if (deviceKey !== null && relievedDevices.has(deviceKey)) {
|
|
1274
|
+
this.deps.logger.debug("storage pressure: skipping location — physical device already relieved this sweep", { meta: {
|
|
1275
|
+
locationId,
|
|
1276
|
+
deviceKey
|
|
1277
|
+
} });
|
|
1278
|
+
continue;
|
|
1179
1279
|
}
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
this.active = job;
|
|
1280
|
+
try {
|
|
1281
|
+
if (await this.relieveLocation(locationId, threshold) && deviceKey !== null) relievedDevices.add(deviceKey);
|
|
1282
|
+
} catch (err) {
|
|
1283
|
+
this.deps.logger.warn("storage pressure: relief failed for location", { meta: {
|
|
1284
|
+
locationId,
|
|
1285
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1286
|
+
} });
|
|
1188
1287
|
}
|
|
1189
|
-
this.runPromise = null;
|
|
1190
|
-
}
|
|
1191
|
-
}
|
|
1192
|
-
async pauseParticipants(job, reacquire = false) {
|
|
1193
|
-
const leaseId = requireLease(job);
|
|
1194
|
-
for (const name of PARTICIPANTS) {
|
|
1195
|
-
if (job.cancelRequested) return;
|
|
1196
|
-
if (!reacquire && job.pausedParticipants.includes(name)) continue;
|
|
1197
|
-
await this.participant(name).pause(leaseId);
|
|
1198
|
-
if (!job.pausedParticipants.includes(name)) job.pausedParticipants.push(name);
|
|
1199
|
-
await this.persist(job);
|
|
1200
|
-
}
|
|
1201
|
-
}
|
|
1202
|
-
async resumeParticipants(job) {
|
|
1203
|
-
const leaseId = requireLease(job);
|
|
1204
|
-
for (const name of REVERSE_PARTICIPANTS) {
|
|
1205
|
-
if (!job.pausedParticipants.includes(name)) continue;
|
|
1206
|
-
await this.participant(name).resume(leaseId);
|
|
1207
|
-
job.pausedParticipants = job.pausedParticipants.filter((participant) => participant !== name);
|
|
1208
|
-
await this.persist(job);
|
|
1209
1288
|
}
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1289
|
+
const caps = await this.deps.getLocationMaxUsedBytes?.() ?? /* @__PURE__ */ new Map();
|
|
1290
|
+
for (const [locationId, maxBytes] of caps) {
|
|
1291
|
+
if (relievedDevices.size > 0) {
|
|
1292
|
+
const deviceKey = await this.deps.getLocationDeviceKey?.(locationId) ?? null;
|
|
1293
|
+
if (deviceKey !== null && relievedDevices.has(deviceKey)) continue;
|
|
1294
|
+
}
|
|
1215
1295
|
try {
|
|
1216
|
-
await this.
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1296
|
+
await this.relieveToUsageCap(locationId, maxBytes);
|
|
1297
|
+
} catch (err) {
|
|
1298
|
+
this.deps.logger.warn("storage pressure: usage-cap relief failed", { meta: {
|
|
1299
|
+
locationId,
|
|
1300
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1301
|
+
} });
|
|
1302
|
+
}
|
|
1220
1303
|
}
|
|
1221
1304
|
}
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
const
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
await
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
const
|
|
1232
|
-
|
|
1233
|
-
if (
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1305
|
+
/** Evict oldest-first until the location's evictable usage fits `maxBytes`. */
|
|
1306
|
+
async relieveToUsageCap(locationId, maxBytes) {
|
|
1307
|
+
const maxRounds = this.deps.maxRounds ?? DEFAULT_MAX_ROUNDS;
|
|
1308
|
+
for (let round = 0; round < maxRounds; round++) {
|
|
1309
|
+
const providers = this.deps.getEvictableProviders();
|
|
1310
|
+
const usages = await Promise.all(providers.map(async (p) => ({
|
|
1311
|
+
p,
|
|
1312
|
+
bytes: (await p.getEvictableUsage({ locationId })).bytes
|
|
1313
|
+
})));
|
|
1314
|
+
const totalUsed = usages.reduce((s, u) => s + u.bytes, 0);
|
|
1315
|
+
const excess = totalUsed - maxBytes;
|
|
1316
|
+
if (excess <= 0) return;
|
|
1317
|
+
let reclaimed = 0;
|
|
1318
|
+
for (const u of usages) {
|
|
1319
|
+
if (u.bytes <= 0) continue;
|
|
1320
|
+
const share = Math.max(1, Math.ceil(excess * u.bytes / totalUsed));
|
|
1321
|
+
const res = await u.p.evict({
|
|
1322
|
+
locationId,
|
|
1323
|
+
targetBytes: share
|
|
1324
|
+
});
|
|
1325
|
+
reclaimed += res.reclaimedBytes;
|
|
1326
|
+
}
|
|
1327
|
+
this.deps.logger.info("storage pressure: usage-cap relief round", { meta: {
|
|
1328
|
+
locationId,
|
|
1329
|
+
round,
|
|
1330
|
+
totalUsed,
|
|
1331
|
+
maxBytes,
|
|
1332
|
+
reclaimedBytes: reclaimed
|
|
1333
|
+
} });
|
|
1334
|
+
if (reclaimed <= 0) return;
|
|
1237
1335
|
}
|
|
1238
1336
|
}
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
job.cancelRequested = true;
|
|
1272
|
-
continue;
|
|
1273
|
-
}
|
|
1274
|
-
if (status.state !== "done") complete = false;
|
|
1337
|
+
/**
|
|
1338
|
+
* Relieve one location until it recovers, nothing is evictable, or a round
|
|
1339
|
+
* makes no progress. Returns `true` when at least one relief round actually
|
|
1340
|
+
* reclaimed bytes (so the caller marks the location's physical device relieved
|
|
1341
|
+
* for this sweep — the trigger for the over-eviction dedup: freed bytes mean a
|
|
1342
|
+
* sibling alias may re-read a not-yet-settled `statfs`), `false` when nothing
|
|
1343
|
+
* was freed (never breached, or breached with no evictable data).
|
|
1344
|
+
*/
|
|
1345
|
+
async relieveLocation(locationId, threshold) {
|
|
1346
|
+
const maxRounds = this.deps.maxRounds ?? DEFAULT_MAX_ROUNDS;
|
|
1347
|
+
const hysteresis = this.deps.hysteresisPercent ?? DEFAULT_HYSTERESIS_PERCENT;
|
|
1348
|
+
let reclaimedAny = false;
|
|
1349
|
+
for (let round = 0; round < maxRounds; round++) {
|
|
1350
|
+
const free = await this.deps.getFreePercent(locationId);
|
|
1351
|
+
if (free >= threshold) return reclaimedAny;
|
|
1352
|
+
const total = await this.deps.getVolumeTotalBytes(locationId);
|
|
1353
|
+
const deficitPercent = threshold - free + hysteresis;
|
|
1354
|
+
const targetBytes = Math.ceil(total * deficitPercent / 100);
|
|
1355
|
+
if (targetBytes <= 0) return reclaimedAny;
|
|
1356
|
+
const providers = this.deps.getEvictableProviders();
|
|
1357
|
+
const usages = await Promise.all(providers.map(async (p) => ({
|
|
1358
|
+
p,
|
|
1359
|
+
bytes: (await p.getEvictableUsage({ locationId })).bytes
|
|
1360
|
+
})));
|
|
1361
|
+
const totalEvictable = usages.reduce((s, u) => s + u.bytes, 0);
|
|
1362
|
+
if (totalEvictable <= 0) {
|
|
1363
|
+
this.deps.logger.debug("storage pressure: breach but no evictable data on location", { meta: {
|
|
1364
|
+
locationId,
|
|
1365
|
+
freePercent: free,
|
|
1366
|
+
threshold
|
|
1367
|
+
} });
|
|
1368
|
+
return reclaimedAny;
|
|
1275
1369
|
}
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1370
|
+
let reclaimed = 0;
|
|
1371
|
+
for (const u of usages) {
|
|
1372
|
+
if (u.bytes <= 0) continue;
|
|
1373
|
+
const share = Math.max(1, Math.ceil(targetBytes * u.bytes / totalEvictable));
|
|
1374
|
+
const res = await u.p.evict({
|
|
1375
|
+
locationId,
|
|
1376
|
+
targetBytes: share
|
|
1377
|
+
});
|
|
1378
|
+
reclaimed += res.reclaimedBytes;
|
|
1379
|
+
}
|
|
1380
|
+
if (reclaimed > 0) reclaimedAny = true;
|
|
1381
|
+
this.deps.logger.info("storage pressure: relief round", { meta: {
|
|
1382
|
+
locationId,
|
|
1383
|
+
round,
|
|
1384
|
+
freePercent: free,
|
|
1385
|
+
threshold,
|
|
1386
|
+
targetBytes,
|
|
1387
|
+
reclaimedBytes: reclaimed
|
|
1388
|
+
} });
|
|
1389
|
+
if (reclaimed <= 0) return reclaimedAny;
|
|
1279
1390
|
}
|
|
1280
|
-
|
|
1281
|
-
async verifyMoves(job) {
|
|
1282
|
-
for (const move of job.moves) if (move.state !== "done") throw new Error(`${move.storageClass} move did not complete verification`);
|
|
1283
|
-
}
|
|
1284
|
-
async startMove(job, move) {
|
|
1285
|
-
if (move.storageClass === "eventMedia") return (await this.deps.participants.analytics.startMove({
|
|
1286
|
-
toLocationId: move.toLocationId,
|
|
1287
|
-
throttleMbps: job.throttleMbps,
|
|
1288
|
-
leaseId: requireLease(job)
|
|
1289
|
-
})).jobId;
|
|
1290
|
-
return (await this.deps.participants.recorder.startMove({
|
|
1291
|
-
fromLocationId: move.fromLocationId,
|
|
1292
|
-
toLocationId: move.toLocationId,
|
|
1293
|
-
footageClass: move.storageClass,
|
|
1294
|
-
throttleMbps: job.throttleMbps,
|
|
1295
|
-
leaseId: requireLease(job)
|
|
1296
|
-
})).jobId;
|
|
1297
|
-
}
|
|
1298
|
-
async moveStatus(move) {
|
|
1299
|
-
if (move.moverJobId === null) return null;
|
|
1300
|
-
return move.storageClass === "eventMedia" ? this.deps.participants.analytics.getMove(move.moverJobId) : this.deps.participants.recorder.getMove(move.moverJobId);
|
|
1301
|
-
}
|
|
1302
|
-
async cancelMove(move) {
|
|
1303
|
-
if (move.moverJobId === null) return;
|
|
1304
|
-
if (move.storageClass === "eventMedia") await this.deps.participants.analytics.cancelMove(move.moverJobId);
|
|
1305
|
-
else await this.deps.participants.recorder.cancelMove(move.moverJobId);
|
|
1306
|
-
}
|
|
1307
|
-
participant(name) {
|
|
1308
|
-
return this.deps.participants[name];
|
|
1309
|
-
}
|
|
1310
|
-
async setPhase(job, phase) {
|
|
1311
|
-
job.phase = phase;
|
|
1312
|
-
if (isTerminal(job)) job.finishedAt = this.deps.now();
|
|
1313
|
-
await this.persist(job);
|
|
1314
|
-
}
|
|
1315
|
-
async finishCancelled(job) {
|
|
1316
|
-
job.phase = "cancelled";
|
|
1317
|
-
job.finishedAt = this.deps.now();
|
|
1318
|
-
await this.releaseAfterTerminal(job);
|
|
1319
|
-
await this.persist(job);
|
|
1320
|
-
}
|
|
1321
|
-
async persist(job) {
|
|
1322
|
-
job.updatedAt = this.deps.now();
|
|
1323
|
-
await this.deps.state.set(job);
|
|
1391
|
+
return reclaimedAny;
|
|
1324
1392
|
}
|
|
1325
1393
|
};
|
|
1326
|
-
function requireLease(job) {
|
|
1327
|
-
if (job.pauseLeaseId === null) throw new Error("storage migration has no maintenance lease");
|
|
1328
|
-
return job.pauseLeaseId;
|
|
1329
|
-
}
|
|
1330
|
-
function isTerminal(job) {
|
|
1331
|
-
return job.phase === "done" || job.phase === "failed" || job.phase === "cancelled";
|
|
1332
|
-
}
|
|
1333
1394
|
//#endregion
|
|
1334
1395
|
//#region src/builtins/storage-orchestrator/storage-orchestrator.addon.ts
|
|
1335
1396
|
/**
|