@camstack/system 1.2.227 → 1.2.228
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtins/storage-orchestrator/location-volume-capacity.d.ts +58 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.d.ts +30 -4
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +187 -41
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +187 -41
- package/package.json +1 -1
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { StorageLocation } from '@camstack/types';
|
|
2
|
+
import { ProviderLocality } from './access-guards.js';
|
|
3
|
+
/** The three `statfs` fields capacity is derived from (`node:fs` StatsFs). */
|
|
4
|
+
export interface StatfsFacts {
|
|
5
|
+
readonly blocks: number;
|
|
6
|
+
readonly bsize: number;
|
|
7
|
+
readonly bavail: number;
|
|
8
|
+
}
|
|
9
|
+
/** The filesystem reads the probe takes, injected so the decision is testable. */
|
|
10
|
+
export interface VolumeProbe {
|
|
11
|
+
readonly access: (target: string) => Promise<void>;
|
|
12
|
+
readonly statfs: (target: string) => Promise<StatfsFacts>;
|
|
13
|
+
}
|
|
14
|
+
export interface VolumeProbeDeps {
|
|
15
|
+
readonly probe: VolumeProbe;
|
|
16
|
+
/** Only ever used to REJECT a relative `basePath` by naming what it would
|
|
17
|
+
* have resolved against — never to resolve one. */
|
|
18
|
+
readonly cwd: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* A measurement, or the reason there is none. `unknown` is never folded into a
|
|
22
|
+
* number: `0` free means "full" to every reader of this figure (D393).
|
|
23
|
+
*/
|
|
24
|
+
export type LocalVolumeCapacity = {
|
|
25
|
+
readonly kind: 'measured';
|
|
26
|
+
/** The path actually stat'd — the location's root, or an ancestor of it. */
|
|
27
|
+
readonly path: string;
|
|
28
|
+
readonly totalBytes: number;
|
|
29
|
+
readonly availableBytes: number;
|
|
30
|
+
} | {
|
|
31
|
+
readonly kind: 'unknown';
|
|
32
|
+
readonly reason: string;
|
|
33
|
+
};
|
|
34
|
+
/** The two figures the cap carries, or `null` — never a zero standing in. */
|
|
35
|
+
export interface MeasuredVolume {
|
|
36
|
+
readonly totalBytes: number;
|
|
37
|
+
readonly availableBytes: number;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Free percent for the pressure guard. An unmeasurable volume answers `100`,
|
|
41
|
+
* which is the value that keeps the guard INERT: a disk nobody measured must
|
|
42
|
+
* never be the reason footage is deleted (D393).
|
|
43
|
+
*/
|
|
44
|
+
export declare function freePercentOf(capacity: MeasuredVolume | null): number;
|
|
45
|
+
/**
|
|
46
|
+
* Volume total for turning a deficit percentage into a byte target. `0` on an
|
|
47
|
+
* unmeasurable volume — the guard's own stop condition reads that as "no
|
|
48
|
+
* target", so nothing is evicted on a reading nobody took.
|
|
49
|
+
*/
|
|
50
|
+
export declare function volumeTotalBytesOf(capacity: MeasuredVolume | null): number;
|
|
51
|
+
/**
|
|
52
|
+
* Free + total bytes on the volume backing `location`, or why there are none.
|
|
53
|
+
*
|
|
54
|
+
* Never throws: an unreachable volume is one degraded ROW, not a failed
|
|
55
|
+
* `listLocations` — a NAS that stops answering must not blank the capacity of
|
|
56
|
+
* every other disk on the node.
|
|
57
|
+
*/
|
|
58
|
+
export declare function readLocalVolumeCapacity(location: StorageLocation, locality: ProviderLocality, localNodeId: string, deps: VolumeProbeDeps): Promise<LocalVolumeCapacity>;
|
|
@@ -177,9 +177,22 @@ export declare class StorageOrchestratorAddon extends BaseAddon<StorageOrchestra
|
|
|
177
177
|
private providerIsNodeLocal;
|
|
178
178
|
/** See the call site in the `resolve` dispatch, and `access-guards.ts`. */
|
|
179
179
|
private refuseRemoteResolve;
|
|
180
|
-
/**
|
|
181
|
-
*
|
|
180
|
+
/**
|
|
181
|
+
* The volume figures for one location, or `null` when this node cannot take
|
|
182
|
+
* a measurement that is genuinely about it.
|
|
183
|
+
*
|
|
184
|
+
* The decision lives in `location-volume-capacity.ts` — including why a
|
|
185
|
+
* remote-provider location (the SMB share that was reporting the hub's own
|
|
186
|
+
* disk) gets nothing rather than a number. Here we only supply the node id,
|
|
187
|
+
* the provider's declared locality and the real filesystem, and log the
|
|
188
|
+
* reason once per distinct fault: `listLocations` is on the admin UI's poll,
|
|
189
|
+
* so a line per call would be a line per second.
|
|
190
|
+
*/
|
|
182
191
|
private localCapacityOf;
|
|
192
|
+
/** locationId → the last unmeasurable reason logged for it. */
|
|
193
|
+
private readonly capacityUnknownLogged;
|
|
194
|
+
private logCapacityUnknownOnce;
|
|
195
|
+
private clearCapacityUnknown;
|
|
183
196
|
/** Per-location cap on EVICTABLE bytes (`config.maxUsedGb`): the operator's
|
|
184
197
|
* "recordings may use at most N GB in total" — enforced by the pressure
|
|
185
198
|
* manager alongside the free-percent guard. */
|
|
@@ -225,9 +238,22 @@ export declare class StorageOrchestratorAddon extends BaseAddon<StorageOrchestra
|
|
|
225
238
|
* errno — `EACCES`, `EIO`, a stale NFS handle — is `unknown` and refuses.
|
|
226
239
|
*/
|
|
227
240
|
private locationOccupancy;
|
|
228
|
-
/**
|
|
241
|
+
/**
|
|
242
|
+
* The pressure guard's two readings come from the SAME probe the UI's
|
|
243
|
+
* capacity does — one capacity truth per location, not two.
|
|
244
|
+
*
|
|
245
|
+
* They used to `statfs` the raw `basePath` themselves, which on this node
|
|
246
|
+
* answers for whatever local path happens to carry that name: a location
|
|
247
|
+
* owned by another node, or served by a remote provider, could decide
|
|
248
|
+
* eviction from the hub's own disk. Both now degrade exactly as they did
|
|
249
|
+
* when the path was unstattable — `100` (guard inert) and `0` (no byte
|
|
250
|
+
* target) — but they do so for every reading that is not ABOUT the location,
|
|
251
|
+
* not only for the ones that threw.
|
|
252
|
+
*/
|
|
253
|
+
private volumeCapacityOf;
|
|
254
|
+
/** Free capacity (%) on a location's volume; 100 (guard inert) when unmeasurable. */
|
|
229
255
|
private locationFreePercent;
|
|
230
|
-
/** Total bytes on a location's volume
|
|
256
|
+
/** Total bytes on a location's volume; 0 when unmeasurable. */
|
|
231
257
|
private locationVolumeTotalBytes;
|
|
232
258
|
/**
|
|
233
259
|
* Seed default storage locations from the addon-declared
|
|
@@ -1725,6 +1725,128 @@ function resolveMinFreePercent(config, providerDefaultPercent, capacity, notice)
|
|
|
1725
1725
|
return Math.min(100, percent);
|
|
1726
1726
|
}
|
|
1727
1727
|
//#endregion
|
|
1728
|
+
//#region src/builtins/storage-orchestrator/location-volume-capacity.ts
|
|
1729
|
+
/**
|
|
1730
|
+
* `StorageLocation.capacity` — the ONE place a location's volume is measured,
|
|
1731
|
+
* and the one place that decides a volume cannot be measured from here.
|
|
1732
|
+
*
|
|
1733
|
+
* ## What it is measuring
|
|
1734
|
+
*
|
|
1735
|
+
* `statfs` answers "how big is the volume, and how much of it is free". It is a
|
|
1736
|
+
* fact about a MOUNT on THIS node. A location only has such a fact when three
|
|
1737
|
+
* things hold at once: the provider serves a genuine local filesystem
|
|
1738
|
+
* (`getProviderInfo().nodeLocal`), the location belongs to this node, and its
|
|
1739
|
+
* `basePath` names an absolute path on it. Miss any of them and the correct
|
|
1740
|
+
* answer is `unknown` — which the cap carries as a null `capacity` and the
|
|
1741
|
+
* admin UI renders as "Capacity unknown", with no bar (D315: an empty bar reads
|
|
1742
|
+
* as plenty of room).
|
|
1743
|
+
*
|
|
1744
|
+
* ## The defect that made this a module
|
|
1745
|
+
*
|
|
1746
|
+
* Measured on the live hub, 2026-09-13. `storage.listLocations` returned two
|
|
1747
|
+
* `backups` locations with byte-identical capacity — `/data/backups` on
|
|
1748
|
+
* `filesystem-storage` and `//192.168.1.89/backups` on `smb-storage`, both
|
|
1749
|
+
* `total 506107977728 / available 193348673536`. `owned` differed correctly
|
|
1750
|
+
* (9.9 GB vs 1.6 GB), so nothing about the rendering was at fault: the probe
|
|
1751
|
+
* simply never looked at the provider. It `path.resolve`d the SMB row's
|
|
1752
|
+
* share-relative `basePath: "camstack"` against the hub process's cwd, found no
|
|
1753
|
+
* such directory, walked up to the nearest existing ancestor — the container
|
|
1754
|
+
* root — and reported the HUB's own disk as the share's. The operator was told
|
|
1755
|
+
* an SMB share was 62% full by a reading nobody had taken of it.
|
|
1756
|
+
*
|
|
1757
|
+
* This is D296's shape in a third place. `access-guards.ts` already refuses
|
|
1758
|
+
* `resolve` and occupancy probes on a remote-backed location for the same
|
|
1759
|
+
* reason; a `statfs` is the same mistake with a number attached instead of a
|
|
1760
|
+
* path. It is also D393: a measurement that failed is `null`, never a figure —
|
|
1761
|
+
* and the reason travels with it so the caller can log which fault it is.
|
|
1762
|
+
*
|
|
1763
|
+
* ## The ancestor walk, and where it stops
|
|
1764
|
+
*
|
|
1765
|
+
* A node-local root that has not been created yet is normal (first boot, a new
|
|
1766
|
+
* location). Measuring the nearest EXISTING ancestor is honest there: it is the
|
|
1767
|
+
* volume a write would land on today. It stops being honest at the filesystem
|
|
1768
|
+
* root — reaching `/` means every named segment is missing, which is what an
|
|
1769
|
+
* unmounted array looks like, and reporting the root disk's size as the array's
|
|
1770
|
+
* is the same lie in a smaller font. That case is `unknown`.
|
|
1771
|
+
*/
|
|
1772
|
+
function unknown(reason) {
|
|
1773
|
+
return {
|
|
1774
|
+
kind: "unknown",
|
|
1775
|
+
reason
|
|
1776
|
+
};
|
|
1777
|
+
}
|
|
1778
|
+
/**
|
|
1779
|
+
* Free percent for the pressure guard. An unmeasurable volume answers `100`,
|
|
1780
|
+
* which is the value that keeps the guard INERT: a disk nobody measured must
|
|
1781
|
+
* never be the reason footage is deleted (D393).
|
|
1782
|
+
*/
|
|
1783
|
+
function freePercentOf(capacity) {
|
|
1784
|
+
if (capacity === null || capacity.totalBytes <= 0) return 100;
|
|
1785
|
+
return capacity.availableBytes / capacity.totalBytes * 100;
|
|
1786
|
+
}
|
|
1787
|
+
/**
|
|
1788
|
+
* Volume total for turning a deficit percentage into a byte target. `0` on an
|
|
1789
|
+
* unmeasurable volume — the guard's own stop condition reads that as "no
|
|
1790
|
+
* target", so nothing is evicted on a reading nobody took.
|
|
1791
|
+
*/
|
|
1792
|
+
function volumeTotalBytesOf(capacity) {
|
|
1793
|
+
return capacity === null ? 0 : capacity.totalBytes;
|
|
1794
|
+
}
|
|
1795
|
+
/**
|
|
1796
|
+
* May this node take a `statfs` that is genuinely ABOUT this location?
|
|
1797
|
+
*
|
|
1798
|
+
* Unlike `resolveRefusalFor`, an unclassified provider (`undefined`) answers
|
|
1799
|
+
* no. That guard protects a WRITE, where refusing on a read nobody made would
|
|
1800
|
+
* break a working location (D49); here the product is a number an operator
|
|
1801
|
+
* reads, and a number justified by an unmade read is exactly what this module
|
|
1802
|
+
* exists to stop.
|
|
1803
|
+
*/
|
|
1804
|
+
function probePathFor(location, locality, localNodeId, cwd) {
|
|
1805
|
+
if (locality !== true) {
|
|
1806
|
+
const why = locality === false ? `is served by the remote provider "${location.providerId}" (nodeLocal: false) — its bytes live on the remote host and no local statfs describes them` : `is served by "${location.providerId}", which this node has not classified yet (no getProviderInfo answer) — a local statfs would be a guess`;
|
|
1807
|
+
return { reason: `location "${location.id}" ${why}` };
|
|
1808
|
+
}
|
|
1809
|
+
if (location.nodeId !== void 0 && location.nodeId !== localNodeId) return { reason: `location "${location.id}" lives on node "${location.nodeId}" and this is "${localNodeId}" — only the owning node can statfs its own volumes` };
|
|
1810
|
+
const basePath = location.config["basePath"];
|
|
1811
|
+
if (typeof basePath !== "string" || basePath.length === 0) return { reason: `location "${location.id}" declares no basePath to measure` };
|
|
1812
|
+
if (!node_path.default.isAbsolute(basePath)) return { reason: `location "${location.id}" declares a RELATIVE basePath ("${basePath}") — that is a path inside its provider's own namespace, not on this node's filesystem, and resolving it against "${cwd}" would measure an unrelated volume` };
|
|
1813
|
+
return { path: node_path.default.resolve(basePath) };
|
|
1814
|
+
}
|
|
1815
|
+
/**
|
|
1816
|
+
* Free + total bytes on the volume backing `location`, or why there are none.
|
|
1817
|
+
*
|
|
1818
|
+
* Never throws: an unreachable volume is one degraded ROW, not a failed
|
|
1819
|
+
* `listLocations` — a NAS that stops answering must not blank the capacity of
|
|
1820
|
+
* every other disk on the node.
|
|
1821
|
+
*/
|
|
1822
|
+
async function readLocalVolumeCapacity(location, locality, localNodeId, deps) {
|
|
1823
|
+
const target = probePathFor(location, locality, localNodeId, deps.cwd);
|
|
1824
|
+
if ("reason" in target) return unknown(target.reason);
|
|
1825
|
+
const root = node_path.default.parse(target.path).root;
|
|
1826
|
+
let probed = target.path;
|
|
1827
|
+
for (;;) try {
|
|
1828
|
+
await deps.probe.access(probed);
|
|
1829
|
+
break;
|
|
1830
|
+
} catch {
|
|
1831
|
+
const parent = node_path.default.dirname(probed);
|
|
1832
|
+
if (parent === probed) return unknown(`nothing on the path "${target.path}" exists on this node — the volume is not mounted, so the root filesystem's size is not an answer about it`);
|
|
1833
|
+
probed = parent;
|
|
1834
|
+
}
|
|
1835
|
+
if (probed === root && target.path !== root) return unknown(`only the filesystem root exists on the path "${target.path}" — the volume is not mounted, so the root filesystem's size is not an answer about it`);
|
|
1836
|
+
try {
|
|
1837
|
+
const st = await deps.probe.statfs(probed);
|
|
1838
|
+
return {
|
|
1839
|
+
kind: "measured",
|
|
1840
|
+
path: probed,
|
|
1841
|
+
totalBytes: st.blocks * st.bsize,
|
|
1842
|
+
availableBytes: st.bavail * st.bsize
|
|
1843
|
+
};
|
|
1844
|
+
} catch (err) {
|
|
1845
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
1846
|
+
return unknown(`statfs("${probed}") failed: ${reason}`);
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
//#endregion
|
|
1728
1850
|
//#region src/builtins/storage-orchestrator/location-secrets.ts
|
|
1729
1851
|
/**
|
|
1730
1852
|
* The config keys the ORCHESTRATOR itself owns, as opposed to the ones a
|
|
@@ -3976,31 +4098,53 @@ var StorageOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
3976
4098
|
} });
|
|
3977
4099
|
throw new Error(refusal);
|
|
3978
4100
|
}
|
|
3979
|
-
/**
|
|
3980
|
-
*
|
|
4101
|
+
/**
|
|
4102
|
+
* The volume figures for one location, or `null` when this node cannot take
|
|
4103
|
+
* a measurement that is genuinely about it.
|
|
4104
|
+
*
|
|
4105
|
+
* The decision lives in `location-volume-capacity.ts` — including why a
|
|
4106
|
+
* remote-provider location (the SMB share that was reporting the hub's own
|
|
4107
|
+
* disk) gets nothing rather than a number. Here we only supply the node id,
|
|
4108
|
+
* the provider's declared locality and the real filesystem, and log the
|
|
4109
|
+
* reason once per distinct fault: `listLocations` is on the admin UI's poll,
|
|
4110
|
+
* so a line per call would be a line per second.
|
|
4111
|
+
*/
|
|
3981
4112
|
async localCapacityOf(loc) {
|
|
3982
|
-
const localNode = this.service?.getLocalNodeId() ??
|
|
3983
|
-
|
|
3984
|
-
|
|
3985
|
-
|
|
3986
|
-
|
|
3987
|
-
|
|
3988
|
-
|
|
3989
|
-
|
|
3990
|
-
|
|
3991
|
-
|
|
3992
|
-
const parent = node_path.dirname(target);
|
|
3993
|
-
if (!parent || parent === target) return null;
|
|
3994
|
-
target = parent;
|
|
3995
|
-
}
|
|
3996
|
-
const st = await node_fs_promises.statfs(target);
|
|
3997
|
-
return {
|
|
3998
|
-
totalBytes: st.blocks * st.bsize,
|
|
3999
|
-
availableBytes: st.bavail * st.bsize
|
|
4000
|
-
};
|
|
4001
|
-
} catch {
|
|
4113
|
+
const localNode = this.service?.getLocalNodeId() ?? HUB_NODE_ID;
|
|
4114
|
+
const result = await readLocalVolumeCapacity(loc, this.providerIsNodeLocal(loc), localNode, {
|
|
4115
|
+
probe: {
|
|
4116
|
+
access: (target) => node_fs_promises.access(target),
|
|
4117
|
+
statfs: async (target) => node_fs_promises.statfs(target)
|
|
4118
|
+
},
|
|
4119
|
+
cwd: node_path.resolve(".")
|
|
4120
|
+
});
|
|
4121
|
+
if (result.kind === "unknown") {
|
|
4122
|
+
this.logCapacityUnknownOnce(loc.id, result.reason);
|
|
4002
4123
|
return null;
|
|
4003
4124
|
}
|
|
4125
|
+
this.clearCapacityUnknown(loc.id, result.path);
|
|
4126
|
+
return {
|
|
4127
|
+
totalBytes: result.totalBytes,
|
|
4128
|
+
availableBytes: result.availableBytes
|
|
4129
|
+
};
|
|
4130
|
+
}
|
|
4131
|
+
/** locationId → the last unmeasurable reason logged for it. */
|
|
4132
|
+
capacityUnknownLogged = /* @__PURE__ */ new Map();
|
|
4133
|
+
logCapacityUnknownOnce(locationId, reason) {
|
|
4134
|
+
if (this.capacityUnknownLogged.get(locationId) === reason) return;
|
|
4135
|
+
this.capacityUnknownLogged.set(locationId, reason);
|
|
4136
|
+
this.ctx.logger.warn("storage-orchestrator: capacity is UNKNOWN for a location — reporting nothing rather than another volume’s figures", { meta: {
|
|
4137
|
+
locationId,
|
|
4138
|
+
reason
|
|
4139
|
+
} });
|
|
4140
|
+
}
|
|
4141
|
+
clearCapacityUnknown(locationId, measuredPath) {
|
|
4142
|
+
if (!this.capacityUnknownLogged.has(locationId)) return;
|
|
4143
|
+
this.capacityUnknownLogged.delete(locationId);
|
|
4144
|
+
this.ctx.logger.info("storage-orchestrator: capacity is measurable again", { meta: {
|
|
4145
|
+
locationId,
|
|
4146
|
+
measuredPath
|
|
4147
|
+
} });
|
|
4004
4148
|
}
|
|
4005
4149
|
/** Per-location cap on EVICTABLE bytes (`config.maxUsedGb`): the operator's
|
|
4006
4150
|
* "recordings may use at most N GB in total" — enforced by the pressure
|
|
@@ -4089,28 +4233,30 @@ var StorageOrchestratorAddon = class extends require_dist.BaseAddon {
|
|
|
4089
4233
|
} });
|
|
4090
4234
|
return occupancy;
|
|
4091
4235
|
}
|
|
4092
|
-
/**
|
|
4236
|
+
/**
|
|
4237
|
+
* The pressure guard's two readings come from the SAME probe the UI's
|
|
4238
|
+
* capacity does — one capacity truth per location, not two.
|
|
4239
|
+
*
|
|
4240
|
+
* They used to `statfs` the raw `basePath` themselves, which on this node
|
|
4241
|
+
* answers for whatever local path happens to carry that name: a location
|
|
4242
|
+
* owned by another node, or served by a remote provider, could decide
|
|
4243
|
+
* eviction from the hub's own disk. Both now degrade exactly as they did
|
|
4244
|
+
* when the path was unstattable — `100` (guard inert) and `0` (no byte
|
|
4245
|
+
* target) — but they do so for every reading that is not ABOUT the location,
|
|
4246
|
+
* not only for the ones that threw.
|
|
4247
|
+
*/
|
|
4248
|
+
async volumeCapacityOf(locationId) {
|
|
4249
|
+
const loc = this.service?.listLocations().find((l) => l.id === locationId);
|
|
4250
|
+
if (!loc) return null;
|
|
4251
|
+
return this.localCapacityOf(loc);
|
|
4252
|
+
}
|
|
4253
|
+
/** Free capacity (%) on a location's volume; 100 (guard inert) when unmeasurable. */
|
|
4093
4254
|
async locationFreePercent(locationId) {
|
|
4094
|
-
|
|
4095
|
-
if (!bp) return 100;
|
|
4096
|
-
try {
|
|
4097
|
-
const st = await node_fs_promises.statfs(bp);
|
|
4098
|
-
if (st.blocks <= 0) return 100;
|
|
4099
|
-
return st.bavail / st.blocks * 100;
|
|
4100
|
-
} catch {
|
|
4101
|
-
return 100;
|
|
4102
|
-
}
|
|
4255
|
+
return freePercentOf(await this.volumeCapacityOf(locationId));
|
|
4103
4256
|
}
|
|
4104
|
-
/** Total bytes on a location's volume
|
|
4257
|
+
/** Total bytes on a location's volume; 0 when unmeasurable. */
|
|
4105
4258
|
async locationVolumeTotalBytes(locationId) {
|
|
4106
|
-
|
|
4107
|
-
if (!bp) return 0;
|
|
4108
|
-
try {
|
|
4109
|
-
const st = await node_fs_promises.statfs(bp);
|
|
4110
|
-
return st.blocks * st.bsize;
|
|
4111
|
-
} catch {
|
|
4112
|
-
return 0;
|
|
4113
|
-
}
|
|
4259
|
+
return volumeTotalBytesOf(await this.volumeCapacityOf(locationId));
|
|
4114
4260
|
}
|
|
4115
4261
|
/**
|
|
4116
4262
|
* Seed default storage locations from the addon-declared
|
|
@@ -1720,6 +1720,128 @@ function resolveMinFreePercent(config, providerDefaultPercent, capacity, notice)
|
|
|
1720
1720
|
return Math.min(100, percent);
|
|
1721
1721
|
}
|
|
1722
1722
|
//#endregion
|
|
1723
|
+
//#region src/builtins/storage-orchestrator/location-volume-capacity.ts
|
|
1724
|
+
/**
|
|
1725
|
+
* `StorageLocation.capacity` — the ONE place a location's volume is measured,
|
|
1726
|
+
* and the one place that decides a volume cannot be measured from here.
|
|
1727
|
+
*
|
|
1728
|
+
* ## What it is measuring
|
|
1729
|
+
*
|
|
1730
|
+
* `statfs` answers "how big is the volume, and how much of it is free". It is a
|
|
1731
|
+
* fact about a MOUNT on THIS node. A location only has such a fact when three
|
|
1732
|
+
* things hold at once: the provider serves a genuine local filesystem
|
|
1733
|
+
* (`getProviderInfo().nodeLocal`), the location belongs to this node, and its
|
|
1734
|
+
* `basePath` names an absolute path on it. Miss any of them and the correct
|
|
1735
|
+
* answer is `unknown` — which the cap carries as a null `capacity` and the
|
|
1736
|
+
* admin UI renders as "Capacity unknown", with no bar (D315: an empty bar reads
|
|
1737
|
+
* as plenty of room).
|
|
1738
|
+
*
|
|
1739
|
+
* ## The defect that made this a module
|
|
1740
|
+
*
|
|
1741
|
+
* Measured on the live hub, 2026-09-13. `storage.listLocations` returned two
|
|
1742
|
+
* `backups` locations with byte-identical capacity — `/data/backups` on
|
|
1743
|
+
* `filesystem-storage` and `//192.168.1.89/backups` on `smb-storage`, both
|
|
1744
|
+
* `total 506107977728 / available 193348673536`. `owned` differed correctly
|
|
1745
|
+
* (9.9 GB vs 1.6 GB), so nothing about the rendering was at fault: the probe
|
|
1746
|
+
* simply never looked at the provider. It `path.resolve`d the SMB row's
|
|
1747
|
+
* share-relative `basePath: "camstack"` against the hub process's cwd, found no
|
|
1748
|
+
* such directory, walked up to the nearest existing ancestor — the container
|
|
1749
|
+
* root — and reported the HUB's own disk as the share's. The operator was told
|
|
1750
|
+
* an SMB share was 62% full by a reading nobody had taken of it.
|
|
1751
|
+
*
|
|
1752
|
+
* This is D296's shape in a third place. `access-guards.ts` already refuses
|
|
1753
|
+
* `resolve` and occupancy probes on a remote-backed location for the same
|
|
1754
|
+
* reason; a `statfs` is the same mistake with a number attached instead of a
|
|
1755
|
+
* path. It is also D393: a measurement that failed is `null`, never a figure —
|
|
1756
|
+
* and the reason travels with it so the caller can log which fault it is.
|
|
1757
|
+
*
|
|
1758
|
+
* ## The ancestor walk, and where it stops
|
|
1759
|
+
*
|
|
1760
|
+
* A node-local root that has not been created yet is normal (first boot, a new
|
|
1761
|
+
* location). Measuring the nearest EXISTING ancestor is honest there: it is the
|
|
1762
|
+
* volume a write would land on today. It stops being honest at the filesystem
|
|
1763
|
+
* root — reaching `/` means every named segment is missing, which is what an
|
|
1764
|
+
* unmounted array looks like, and reporting the root disk's size as the array's
|
|
1765
|
+
* is the same lie in a smaller font. That case is `unknown`.
|
|
1766
|
+
*/
|
|
1767
|
+
function unknown(reason) {
|
|
1768
|
+
return {
|
|
1769
|
+
kind: "unknown",
|
|
1770
|
+
reason
|
|
1771
|
+
};
|
|
1772
|
+
}
|
|
1773
|
+
/**
|
|
1774
|
+
* Free percent for the pressure guard. An unmeasurable volume answers `100`,
|
|
1775
|
+
* which is the value that keeps the guard INERT: a disk nobody measured must
|
|
1776
|
+
* never be the reason footage is deleted (D393).
|
|
1777
|
+
*/
|
|
1778
|
+
function freePercentOf(capacity) {
|
|
1779
|
+
if (capacity === null || capacity.totalBytes <= 0) return 100;
|
|
1780
|
+
return capacity.availableBytes / capacity.totalBytes * 100;
|
|
1781
|
+
}
|
|
1782
|
+
/**
|
|
1783
|
+
* Volume total for turning a deficit percentage into a byte target. `0` on an
|
|
1784
|
+
* unmeasurable volume — the guard's own stop condition reads that as "no
|
|
1785
|
+
* target", so nothing is evicted on a reading nobody took.
|
|
1786
|
+
*/
|
|
1787
|
+
function volumeTotalBytesOf(capacity) {
|
|
1788
|
+
return capacity === null ? 0 : capacity.totalBytes;
|
|
1789
|
+
}
|
|
1790
|
+
/**
|
|
1791
|
+
* May this node take a `statfs` that is genuinely ABOUT this location?
|
|
1792
|
+
*
|
|
1793
|
+
* Unlike `resolveRefusalFor`, an unclassified provider (`undefined`) answers
|
|
1794
|
+
* no. That guard protects a WRITE, where refusing on a read nobody made would
|
|
1795
|
+
* break a working location (D49); here the product is a number an operator
|
|
1796
|
+
* reads, and a number justified by an unmade read is exactly what this module
|
|
1797
|
+
* exists to stop.
|
|
1798
|
+
*/
|
|
1799
|
+
function probePathFor(location, locality, localNodeId, cwd) {
|
|
1800
|
+
if (locality !== true) {
|
|
1801
|
+
const why = locality === false ? `is served by the remote provider "${location.providerId}" (nodeLocal: false) — its bytes live on the remote host and no local statfs describes them` : `is served by "${location.providerId}", which this node has not classified yet (no getProviderInfo answer) — a local statfs would be a guess`;
|
|
1802
|
+
return { reason: `location "${location.id}" ${why}` };
|
|
1803
|
+
}
|
|
1804
|
+
if (location.nodeId !== void 0 && location.nodeId !== localNodeId) return { reason: `location "${location.id}" lives on node "${location.nodeId}" and this is "${localNodeId}" — only the owning node can statfs its own volumes` };
|
|
1805
|
+
const basePath = location.config["basePath"];
|
|
1806
|
+
if (typeof basePath !== "string" || basePath.length === 0) return { reason: `location "${location.id}" declares no basePath to measure` };
|
|
1807
|
+
if (!path.isAbsolute(basePath)) return { reason: `location "${location.id}" declares a RELATIVE basePath ("${basePath}") — that is a path inside its provider's own namespace, not on this node's filesystem, and resolving it against "${cwd}" would measure an unrelated volume` };
|
|
1808
|
+
return { path: path.resolve(basePath) };
|
|
1809
|
+
}
|
|
1810
|
+
/**
|
|
1811
|
+
* Free + total bytes on the volume backing `location`, or why there are none.
|
|
1812
|
+
*
|
|
1813
|
+
* Never throws: an unreachable volume is one degraded ROW, not a failed
|
|
1814
|
+
* `listLocations` — a NAS that stops answering must not blank the capacity of
|
|
1815
|
+
* every other disk on the node.
|
|
1816
|
+
*/
|
|
1817
|
+
async function readLocalVolumeCapacity(location, locality, localNodeId, deps) {
|
|
1818
|
+
const target = probePathFor(location, locality, localNodeId, deps.cwd);
|
|
1819
|
+
if ("reason" in target) return unknown(target.reason);
|
|
1820
|
+
const root = path.parse(target.path).root;
|
|
1821
|
+
let probed = target.path;
|
|
1822
|
+
for (;;) try {
|
|
1823
|
+
await deps.probe.access(probed);
|
|
1824
|
+
break;
|
|
1825
|
+
} catch {
|
|
1826
|
+
const parent = path.dirname(probed);
|
|
1827
|
+
if (parent === probed) return unknown(`nothing on the path "${target.path}" exists on this node — the volume is not mounted, so the root filesystem's size is not an answer about it`);
|
|
1828
|
+
probed = parent;
|
|
1829
|
+
}
|
|
1830
|
+
if (probed === root && target.path !== root) return unknown(`only the filesystem root exists on the path "${target.path}" — the volume is not mounted, so the root filesystem's size is not an answer about it`);
|
|
1831
|
+
try {
|
|
1832
|
+
const st = await deps.probe.statfs(probed);
|
|
1833
|
+
return {
|
|
1834
|
+
kind: "measured",
|
|
1835
|
+
path: probed,
|
|
1836
|
+
totalBytes: st.blocks * st.bsize,
|
|
1837
|
+
availableBytes: st.bavail * st.bsize
|
|
1838
|
+
};
|
|
1839
|
+
} catch (err) {
|
|
1840
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
1841
|
+
return unknown(`statfs("${probed}") failed: ${reason}`);
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
//#endregion
|
|
1723
1845
|
//#region src/builtins/storage-orchestrator/location-secrets.ts
|
|
1724
1846
|
/**
|
|
1725
1847
|
* The config keys the ORCHESTRATOR itself owns, as opposed to the ones a
|
|
@@ -3971,31 +4093,53 @@ var StorageOrchestratorAddon = class extends BaseAddon {
|
|
|
3971
4093
|
} });
|
|
3972
4094
|
throw new Error(refusal);
|
|
3973
4095
|
}
|
|
3974
|
-
/**
|
|
3975
|
-
*
|
|
4096
|
+
/**
|
|
4097
|
+
* The volume figures for one location, or `null` when this node cannot take
|
|
4098
|
+
* a measurement that is genuinely about it.
|
|
4099
|
+
*
|
|
4100
|
+
* The decision lives in `location-volume-capacity.ts` — including why a
|
|
4101
|
+
* remote-provider location (the SMB share that was reporting the hub's own
|
|
4102
|
+
* disk) gets nothing rather than a number. Here we only supply the node id,
|
|
4103
|
+
* the provider's declared locality and the real filesystem, and log the
|
|
4104
|
+
* reason once per distinct fault: `listLocations` is on the admin UI's poll,
|
|
4105
|
+
* so a line per call would be a line per second.
|
|
4106
|
+
*/
|
|
3976
4107
|
async localCapacityOf(loc) {
|
|
3977
|
-
const localNode = this.service?.getLocalNodeId() ??
|
|
3978
|
-
|
|
3979
|
-
|
|
3980
|
-
|
|
3981
|
-
|
|
3982
|
-
|
|
3983
|
-
|
|
3984
|
-
|
|
3985
|
-
|
|
3986
|
-
|
|
3987
|
-
const parent = path$1.dirname(target);
|
|
3988
|
-
if (!parent || parent === target) return null;
|
|
3989
|
-
target = parent;
|
|
3990
|
-
}
|
|
3991
|
-
const st = await fs.statfs(target);
|
|
3992
|
-
return {
|
|
3993
|
-
totalBytes: st.blocks * st.bsize,
|
|
3994
|
-
availableBytes: st.bavail * st.bsize
|
|
3995
|
-
};
|
|
3996
|
-
} catch {
|
|
4108
|
+
const localNode = this.service?.getLocalNodeId() ?? HUB_NODE_ID;
|
|
4109
|
+
const result = await readLocalVolumeCapacity(loc, this.providerIsNodeLocal(loc), localNode, {
|
|
4110
|
+
probe: {
|
|
4111
|
+
access: (target) => fs.access(target),
|
|
4112
|
+
statfs: async (target) => fs.statfs(target)
|
|
4113
|
+
},
|
|
4114
|
+
cwd: path$1.resolve(".")
|
|
4115
|
+
});
|
|
4116
|
+
if (result.kind === "unknown") {
|
|
4117
|
+
this.logCapacityUnknownOnce(loc.id, result.reason);
|
|
3997
4118
|
return null;
|
|
3998
4119
|
}
|
|
4120
|
+
this.clearCapacityUnknown(loc.id, result.path);
|
|
4121
|
+
return {
|
|
4122
|
+
totalBytes: result.totalBytes,
|
|
4123
|
+
availableBytes: result.availableBytes
|
|
4124
|
+
};
|
|
4125
|
+
}
|
|
4126
|
+
/** locationId → the last unmeasurable reason logged for it. */
|
|
4127
|
+
capacityUnknownLogged = /* @__PURE__ */ new Map();
|
|
4128
|
+
logCapacityUnknownOnce(locationId, reason) {
|
|
4129
|
+
if (this.capacityUnknownLogged.get(locationId) === reason) return;
|
|
4130
|
+
this.capacityUnknownLogged.set(locationId, reason);
|
|
4131
|
+
this.ctx.logger.warn("storage-orchestrator: capacity is UNKNOWN for a location — reporting nothing rather than another volume’s figures", { meta: {
|
|
4132
|
+
locationId,
|
|
4133
|
+
reason
|
|
4134
|
+
} });
|
|
4135
|
+
}
|
|
4136
|
+
clearCapacityUnknown(locationId, measuredPath) {
|
|
4137
|
+
if (!this.capacityUnknownLogged.has(locationId)) return;
|
|
4138
|
+
this.capacityUnknownLogged.delete(locationId);
|
|
4139
|
+
this.ctx.logger.info("storage-orchestrator: capacity is measurable again", { meta: {
|
|
4140
|
+
locationId,
|
|
4141
|
+
measuredPath
|
|
4142
|
+
} });
|
|
3999
4143
|
}
|
|
4000
4144
|
/** Per-location cap on EVICTABLE bytes (`config.maxUsedGb`): the operator's
|
|
4001
4145
|
* "recordings may use at most N GB in total" — enforced by the pressure
|
|
@@ -4084,28 +4228,30 @@ var StorageOrchestratorAddon = class extends BaseAddon {
|
|
|
4084
4228
|
} });
|
|
4085
4229
|
return occupancy;
|
|
4086
4230
|
}
|
|
4087
|
-
/**
|
|
4231
|
+
/**
|
|
4232
|
+
* The pressure guard's two readings come from the SAME probe the UI's
|
|
4233
|
+
* capacity does — one capacity truth per location, not two.
|
|
4234
|
+
*
|
|
4235
|
+
* They used to `statfs` the raw `basePath` themselves, which on this node
|
|
4236
|
+
* answers for whatever local path happens to carry that name: a location
|
|
4237
|
+
* owned by another node, or served by a remote provider, could decide
|
|
4238
|
+
* eviction from the hub's own disk. Both now degrade exactly as they did
|
|
4239
|
+
* when the path was unstattable — `100` (guard inert) and `0` (no byte
|
|
4240
|
+
* target) — but they do so for every reading that is not ABOUT the location,
|
|
4241
|
+
* not only for the ones that threw.
|
|
4242
|
+
*/
|
|
4243
|
+
async volumeCapacityOf(locationId) {
|
|
4244
|
+
const loc = this.service?.listLocations().find((l) => l.id === locationId);
|
|
4245
|
+
if (!loc) return null;
|
|
4246
|
+
return this.localCapacityOf(loc);
|
|
4247
|
+
}
|
|
4248
|
+
/** Free capacity (%) on a location's volume; 100 (guard inert) when unmeasurable. */
|
|
4088
4249
|
async locationFreePercent(locationId) {
|
|
4089
|
-
|
|
4090
|
-
if (!bp) return 100;
|
|
4091
|
-
try {
|
|
4092
|
-
const st = await fs.statfs(bp);
|
|
4093
|
-
if (st.blocks <= 0) return 100;
|
|
4094
|
-
return st.bavail / st.blocks * 100;
|
|
4095
|
-
} catch {
|
|
4096
|
-
return 100;
|
|
4097
|
-
}
|
|
4250
|
+
return freePercentOf(await this.volumeCapacityOf(locationId));
|
|
4098
4251
|
}
|
|
4099
|
-
/** Total bytes on a location's volume
|
|
4252
|
+
/** Total bytes on a location's volume; 0 when unmeasurable. */
|
|
4100
4253
|
async locationVolumeTotalBytes(locationId) {
|
|
4101
|
-
|
|
4102
|
-
if (!bp) return 0;
|
|
4103
|
-
try {
|
|
4104
|
-
const st = await fs.statfs(bp);
|
|
4105
|
-
return st.blocks * st.bsize;
|
|
4106
|
-
} catch {
|
|
4107
|
-
return 0;
|
|
4108
|
-
}
|
|
4254
|
+
return volumeTotalBytesOf(await this.volumeCapacityOf(locationId));
|
|
4109
4255
|
}
|
|
4110
4256
|
/**
|
|
4111
4257
|
* Seed default storage locations from the addon-declared
|