@camstack/system 1.2.57 → 1.2.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +1 -1
- package/dist/addon-runner.mjs +1 -1
- package/dist/addon-utils.js +1 -1
- package/dist/addon-utils.mjs +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +2 -2
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +2 -2
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +2 -2
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +2 -2
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +2 -2
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +2 -2
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +2 -2
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +2 -2
- package/dist/builtins/device-manager/device-manager.addon.js +2 -2
- package/dist/builtins/device-manager/device-manager.addon.mjs +2 -2
- package/dist/builtins/doorbell/binding-mirror.d.ts +65 -0
- package/dist/builtins/doorbell/virtual-doorbell.addon.d.ts +56 -11
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +248 -26
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +248 -26
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +2 -2
- package/dist/builtins/snapshot/index.mjs +2 -2
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +2 -2
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +2 -2
- package/dist/builtins/sqlite-storage/index.d.ts +3 -0
- package/dist/builtins/sqlite-storage/index.js +8 -0
- package/dist/builtins/sqlite-storage/index.mjs +2 -2
- package/dist/builtins/sqlite-storage/retired-settings-keys.d.ts +94 -0
- package/dist/builtins/sqlite-storage/sqlite-pragmas.d.ts +85 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.d.ts +1 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +445 -2
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +438 -3
- package/dist/builtins/sqlite-storage/wal-maintenance.d.ts +113 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-DhL_zGRT.mjs → dist-BqJJWCS8.mjs} +994 -34
- package/dist/{dist-BupxnZOi.js → dist-CV6XTApk.js} +1005 -33
- package/dist/index.js +190 -42
- package/dist/index.mjs +191 -43
- package/dist/kernel/addon-installer.d.ts +55 -7
- package/dist/kernel/addon-manifest.d.ts +36 -16
- package/dist/kernel/transport/frame-codec.d.ts +43 -2
- package/dist/{manifest-python-deps-Ck4-9K9m.mjs → manifest-python-deps-CX5tGRqf.mjs} +95 -10
- package/dist/{manifest-python-deps-BqE5j0-O.js → manifest-python-deps-d4acYXVV.js} +95 -10
- package/package.json +1 -1
- package/dist/{model-download-service-hf0ookyy.js → model-download-service-Cpc6SWzV.js} +1 -1
- package/dist/{model-download-service-Cp9f4dk6.mjs → model-download-service-D8B-4ktF.mjs} +1 -1
|
@@ -1,7 +1,218 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Ct as asJsonObject, Pt as parseJsonUnknown, T as decodeVectorBase64, dt as vectorStoreCapability, ft as errMsg, l as RUNTIME_DEFAULTS, pt as BaseAddon, ut as vectorDimFromBase64, w as dataStoreProviderCapability, x as bareAddonId } from "../../dist-BqJJWCS8.mjs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { statSync } from "node:fs";
|
|
4
5
|
import Database from "better-sqlite3";
|
|
6
|
+
//#region src/builtins/sqlite-storage/sqlite-pragmas.ts
|
|
7
|
+
/**
|
|
8
|
+
* Connection tuning for the hub's SQLite database.
|
|
9
|
+
*
|
|
10
|
+
* Every number here is sized against a MEASURED database, not a blog post.
|
|
11
|
+
* On the live hub (2026-08-06), 20 Hz syscall sampling of hub-main put **210
|
|
12
|
+
* of 2,561 samples inside `pread64`** — 161 on `camstack.db` and 49 on
|
|
13
|
+
* `camstack.db-wal`, all 4 KiB page reads, i.e. page-cache misses. That is
|
|
14
|
+
* 8.2% of wall-clock and ~20% during bursts, on the single thread that also
|
|
15
|
+
* serves every tRPC request. fds 79/81 of hub-main were the two files, so the
|
|
16
|
+
* cost is proven in-process rather than inferred.
|
|
17
|
+
*
|
|
18
|
+
* The database was **130 MB** with a **249 MB** WAL high-water mark. Its
|
|
19
|
+
* tenants are not settings: the analytics event/track/media stores and the
|
|
20
|
+
* sqlite-vec index all live in this one file.
|
|
21
|
+
*/
|
|
22
|
+
/**
|
|
23
|
+
* Page cache, in KiB (SQLite reads a NEGATIVE `cache_size` as KiB rather than
|
|
24
|
+
* as a page count, which is what makes it independent of `page_size`).
|
|
25
|
+
*
|
|
26
|
+
* -65536 = **64 MiB**. Sized from the 130 MB file: the hot working set is the
|
|
27
|
+
* B-tree interior pages plus the indexes the analytics queries walk, which is
|
|
28
|
+
* a fraction of the total — 64 MiB covers roughly half the file outright, so a
|
|
29
|
+
* repeated query stops re-reading the same 4 KiB pages from disk. Going to the
|
|
30
|
+
* full 130 MB would buy the tail of a cold table scan for twice the resident
|
|
31
|
+
* cost; going smaller leaves the measured miss rate roughly where it is.
|
|
32
|
+
*
|
|
33
|
+
* Why the build default is not enough: better-sqlite3 compiles with
|
|
34
|
+
* `SQLITE_DEFAULT_CACHE_SIZE=-16000`, so the starting point is ~16 MiB — 8x
|
|
35
|
+
* short of the file, not the ~2 MiB stock SQLite default. This is a 4x
|
|
36
|
+
* increase, not a 32x one.
|
|
37
|
+
*
|
|
38
|
+
* Cost: 64 MiB of malloc'd C memory, ONE connection's worth (better-sqlite3
|
|
39
|
+
* holds a single handle and the vector index shares it). It is off-heap, so it
|
|
40
|
+
* does not move V8's heap ceiling — the thing that produced the hub's previous
|
|
41
|
+
* OOM.
|
|
42
|
+
*/
|
|
43
|
+
var SQLITE_PAGE_CACHE_KIB = -65536;
|
|
44
|
+
/**
|
|
45
|
+
* Memory-mapped I/O window, in bytes. 268435456 = **256 MiB**.
|
|
46
|
+
*
|
|
47
|
+
* Chosen to exceed the 130 MB database with room for it to roughly double, so
|
|
48
|
+
* the whole main file can be mapped. Pages then arrive as minor faults against
|
|
49
|
+
* the OS page cache instead of `pread64` syscalls — which is precisely the
|
|
50
|
+
* 161-sample half of the measurement. It also means a page evicted from
|
|
51
|
+
* SQLite's own cache is usually still resident in the OS cache, so the two
|
|
52
|
+
* knobs compound rather than duplicate.
|
|
53
|
+
*
|
|
54
|
+
* Two honest caveats:
|
|
55
|
+
* - SQLite **never mmaps the WAL**. The 49 WAL samples are not addressed
|
|
56
|
+
* here; they are addressed by keeping the WAL small (see
|
|
57
|
+
* {@link SQLITE_JOURNAL_SIZE_LIMIT_BYTES} and `wal-maintenance.ts`).
|
|
58
|
+
* - With mmap, an I/O error on a mapped page surfaces as SIGBUS rather than a
|
|
59
|
+
* catchable SQLite error. The pragma is asserted in
|
|
60
|
+
* `__tests__/sqlite-pragmas.spec.ts` by reading it back, so a build that
|
|
61
|
+
* ignores it (`SQLITE_MAX_MMAP_SIZE=0`) fails loudly instead of quietly
|
|
62
|
+
* doing nothing.
|
|
63
|
+
*/
|
|
64
|
+
var SQLITE_MMAP_SIZE_BYTES = 268435456;
|
|
65
|
+
/**
|
|
66
|
+
* Upper bound on the WAL file left on disk after a checkpoint resets it, in
|
|
67
|
+
* bytes. 67108864 = **64 MiB**.
|
|
68
|
+
*
|
|
69
|
+
* This is the passive half of the WAL fix and it carries no stall risk at all:
|
|
70
|
+
* it takes no lock and blocks nothing. It only says "when a checkpoint has
|
|
71
|
+
* already succeeded, truncate the file rather than leaving the space
|
|
72
|
+
* allocated". Without it, a WAL that spiked once stays spiked — which is how
|
|
73
|
+
* a 130 MB database ends up with a 249 MB sidecar.
|
|
74
|
+
*
|
|
75
|
+
* 64 MiB is deliberately well above the ~4 MiB `wal_autocheckpoint` threshold
|
|
76
|
+
* (1000 pages), so ordinary operation never touches this ceiling and the file
|
|
77
|
+
* is not being repeatedly truncated and re-grown.
|
|
78
|
+
*/
|
|
79
|
+
var SQLITE_JOURNAL_SIZE_LIMIT_BYTES = 67108864;
|
|
80
|
+
//#endregion
|
|
81
|
+
//#region src/builtins/sqlite-storage/retired-settings-keys.ts
|
|
82
|
+
/**
|
|
83
|
+
* Is THIS node the one whose settings store is the cluster's authority?
|
|
84
|
+
*
|
|
85
|
+
* Mirrors `resolveNodeRole` (server/backend `node-role.ts`) — unset or `'hub'`
|
|
86
|
+
* means hub, which is the launcher's own default. It is duplicated rather than
|
|
87
|
+
* imported because `@camstack/system` is a dependency OF the backend, not the
|
|
88
|
+
* other way round; the parse is four lines and the env var is the contract.
|
|
89
|
+
*
|
|
90
|
+
* Anything else answers `false`. Skipping where the purge should have run is a
|
|
91
|
+
* missed cleanup the next hub boot fixes; running where it should not have is a
|
|
92
|
+
* write to a store this node does not own. Only one of those is recoverable.
|
|
93
|
+
*/
|
|
94
|
+
function settingsStoreIsAuthoritativeHere(env) {
|
|
95
|
+
const raw = (env["CAMSTACK_ROLE"] ?? "").trim().toLowerCase();
|
|
96
|
+
return raw === "" || raw === "hub";
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* The dead keys, per row.
|
|
100
|
+
*
|
|
101
|
+
* `detection-pipeline` / `addon-settings` / `root` — every key in this row is
|
|
102
|
+
* dead; the row itself is kept because the addon writes live keys into it
|
|
103
|
+
* (`pipelineTemplates`, `videoPipelineSteps`).
|
|
104
|
+
*
|
|
105
|
+
* - `pipelineSteps` / `pipelineEngine`: the persisted GLOBAL step + engine
|
|
106
|
+
* seed. Removed with `getGlobalSteps` becoming pure — dispatch builds steps
|
|
107
|
+
* from the orchestrator's `agentSettings` plus per-camera overrides, and
|
|
108
|
+
* never consulted the seed. See `addon-pipeline/src/detection-pipeline/provider.ts`.
|
|
109
|
+
* - `engineRuntime` / `engineBackend` / `engineDevice` / `probedBestEngine`:
|
|
110
|
+
* the per-node engine CASCADE, removed with the operator-facing engine
|
|
111
|
+
* election. Each node now runs a device-array of inference pools balanced
|
|
112
|
+
* per session, and the node-default pool derives live from the node's own
|
|
113
|
+
* hardware (`resolveAutoEngine`). See
|
|
114
|
+
* `addon-pipeline/src/detection-pipeline/engine-store-keys.ts`.
|
|
115
|
+
*/
|
|
116
|
+
var RETIRED_SETTINGS_KEYS = [{
|
|
117
|
+
namespace: "detection-pipeline",
|
|
118
|
+
collection: "addon-settings",
|
|
119
|
+
row: "root",
|
|
120
|
+
keys: ["pipelineSteps", "pipelineEngine"],
|
|
121
|
+
perNodeKeys: [
|
|
122
|
+
"engineRuntime",
|
|
123
|
+
"engineBackend",
|
|
124
|
+
"engineDevice",
|
|
125
|
+
"probedBestEngine"
|
|
126
|
+
],
|
|
127
|
+
reason: "global step/engine seed and the per-node engine cascade — both removed; the live per-camera dispatch path never read either"
|
|
128
|
+
}];
|
|
129
|
+
/**
|
|
130
|
+
* Adapt the settings backend to {@link RetiredKeyStore}.
|
|
131
|
+
*
|
|
132
|
+
* `get` answering a non-object (missing row) is a MISS, not a fault: it maps to
|
|
133
|
+
* `null` and the purge skips the row. A backend throw propagates, which is what
|
|
134
|
+
* routes it to the "read failed, row left untouched" branch.
|
|
135
|
+
*/
|
|
136
|
+
function retiredKeyStoreOf(backend) {
|
|
137
|
+
return {
|
|
138
|
+
async read(spec) {
|
|
139
|
+
return asJsonObject(await backend.get({
|
|
140
|
+
namespace: spec.namespace,
|
|
141
|
+
collection: spec.collection,
|
|
142
|
+
key: spec.row
|
|
143
|
+
}));
|
|
144
|
+
},
|
|
145
|
+
async write(spec, value) {
|
|
146
|
+
await backend.set({
|
|
147
|
+
namespace: spec.namespace,
|
|
148
|
+
collection: spec.collection,
|
|
149
|
+
key: spec.row,
|
|
150
|
+
value
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
/** Keys of `blob` that `spec` retires. Pure. */
|
|
156
|
+
function planRetiredKeyPurge(blob, spec) {
|
|
157
|
+
const exact = new Set(spec.keys);
|
|
158
|
+
const found = [];
|
|
159
|
+
for (const key of Object.keys(blob)) {
|
|
160
|
+
if (exact.has(key)) {
|
|
161
|
+
found.push(key);
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
for (const perNode of spec.perNodeKeys) if (key === perNode || key.startsWith(`${perNode}@`)) {
|
|
165
|
+
found.push(key);
|
|
166
|
+
break;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return found;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Delete every retired key present in the declared rows — on the hub.
|
|
173
|
+
*
|
|
174
|
+
* Off the authoritative node this is a no-op that touches nothing and says
|
|
175
|
+
* nothing (see the module header). Returns one entry per row that actually
|
|
176
|
+
* changed — empty when the store is already clean, which is the steady state
|
|
177
|
+
* after the first boot that ran it.
|
|
178
|
+
*/
|
|
179
|
+
async function purgeRetiredSettingsKeys(store, logger, specs = RETIRED_SETTINGS_KEYS, env = process.env) {
|
|
180
|
+
if (!settingsStoreIsAuthoritativeHere(env)) return [];
|
|
181
|
+
const results = [];
|
|
182
|
+
for (const spec of specs) {
|
|
183
|
+
let blob;
|
|
184
|
+
try {
|
|
185
|
+
blob = await store.read(spec);
|
|
186
|
+
} catch (err) {
|
|
187
|
+
logger.warn("retired settings keys — read failed, row left untouched", { meta: {
|
|
188
|
+
addon: spec.namespace,
|
|
189
|
+
collection: spec.collection,
|
|
190
|
+
row: spec.row,
|
|
191
|
+
error: err instanceof Error ? err.message : String(err)
|
|
192
|
+
} });
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
if (blob === null) continue;
|
|
196
|
+
const doomed = planRetiredKeyPurge(blob, spec);
|
|
197
|
+
if (doomed.length === 0) continue;
|
|
198
|
+
const doomedSet = new Set(doomed);
|
|
199
|
+
const next = Object.fromEntries(Object.entries(blob).filter(([key]) => !doomedSet.has(key)));
|
|
200
|
+
await store.write(spec, next);
|
|
201
|
+
const result = {
|
|
202
|
+
addon: spec.namespace,
|
|
203
|
+
collection: spec.collection,
|
|
204
|
+
row: spec.row,
|
|
205
|
+
keys: doomed
|
|
206
|
+
};
|
|
207
|
+
results.push(result);
|
|
208
|
+
logger.info(`purged ${doomed.length} retired settings keys`, { meta: {
|
|
209
|
+
...result,
|
|
210
|
+
reason: spec.reason
|
|
211
|
+
} });
|
|
212
|
+
}
|
|
213
|
+
return results;
|
|
214
|
+
}
|
|
215
|
+
//#endregion
|
|
5
216
|
//#region src/builtins/sqlite-storage/filter-compiler.ts
|
|
6
217
|
/** Thrown by `mutate` mode. Distinct type so a caller can map it to a 400. */
|
|
7
218
|
var UnsafeFilterError = class extends Error {
|
|
@@ -109,6 +320,9 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
|
|
|
109
320
|
this.db = new Database(this.dbPath);
|
|
110
321
|
this.db.pragma("journal_mode = WAL");
|
|
111
322
|
this.db.pragma("foreign_keys = ON");
|
|
323
|
+
this.db.pragma(`cache_size = ${SQLITE_PAGE_CACHE_KIB}`);
|
|
324
|
+
this.db.pragma(`mmap_size = ${SQLITE_MMAP_SIZE_BYTES}`);
|
|
325
|
+
this.db.pragma(`journal_size_limit = ${SQLITE_JOURNAL_SIZE_LIMIT_BYTES}`);
|
|
112
326
|
for (const collection of SqliteSettingsBackend.CANONICAL_KV_COLLECTIONS) {
|
|
113
327
|
await this.ensureTable(collection, { columns: [{
|
|
114
328
|
name: "id",
|
|
@@ -951,6 +1165,209 @@ function rebuild(row) {
|
|
|
951
1165
|
return out;
|
|
952
1166
|
}
|
|
953
1167
|
//#endregion
|
|
1168
|
+
//#region src/builtins/sqlite-storage/wal-maintenance.ts
|
|
1169
|
+
/**
|
|
1170
|
+
* Size at or above which the WAL is a candidate for a truncating checkpoint,
|
|
1171
|
+
* in bytes. 67108864 = **64 MiB**.
|
|
1172
|
+
*
|
|
1173
|
+
* Matched to `SQLITE_JOURNAL_SIZE_LIMIT_BYTES`: below this, the passive
|
|
1174
|
+
* mechanism (autocheckpoint + `journal_size_limit`) is doing its job and there
|
|
1175
|
+
* is nothing to reclaim. The live hub's WAL had reached 248,955,152 bytes —
|
|
1176
|
+
* roughly 2x the 130 MB database — which is what says the passive path alone
|
|
1177
|
+
* had stopped keeping up.
|
|
1178
|
+
*/
|
|
1179
|
+
var WAL_HIGH_WATER_BYTES = 67108864;
|
|
1180
|
+
/**
|
|
1181
|
+
* How long the WAL must have gone UNWRITTEN before a truncating checkpoint is
|
|
1182
|
+
* allowed, in ms.
|
|
1183
|
+
*
|
|
1184
|
+
* This is the stall guard and the reason this file exists. `wal_checkpoint
|
|
1185
|
+
* (TRUNCATE)` takes the writer lock, waits for readers, and copies the whole
|
|
1186
|
+
* WAL back into the database — synchronously, on the thread that serves every
|
|
1187
|
+
* request. An unconditional periodic one on a busy NVR would be the disease
|
|
1188
|
+
* (hub-main already spends 42 s per 2 minutes inside uninterrupted blocks),
|
|
1189
|
+
* not the cure.
|
|
1190
|
+
*
|
|
1191
|
+
* The idle signal is the WAL file's own mtime. It is one `stat`, and unlike a
|
|
1192
|
+
* write counter maintained in this class it covers EVERY writer on the file —
|
|
1193
|
+
* the settings backend, the sqlite-vec index sharing the handle, and the
|
|
1194
|
+
* short-lived second connection system-backup opens. On a genuinely busy NVR
|
|
1195
|
+
* this gate simply never opens, and `journal_size_limit` remains the only
|
|
1196
|
+
* thing bounding the file. That is the intended failure mode: doing nothing is
|
|
1197
|
+
* strictly better than a 249 MB copy in the middle of a live request.
|
|
1198
|
+
*/
|
|
1199
|
+
var WAL_IDLE_QUIET_MS = 3e4;
|
|
1200
|
+
/**
|
|
1201
|
+
* Ceiling on how long the checkpoint may wait for a lock, in ms.
|
|
1202
|
+
*
|
|
1203
|
+
* better-sqlite3 defaults `busy_timeout` to 5000. A checkpoint invokes the
|
|
1204
|
+
* busy handler while waiting for readers, so without this a "quiet" moment
|
|
1205
|
+
* that turns out not to be quiet could park the main thread for 5 s. 250 ms
|
|
1206
|
+
* means the checkpoint gives up and we try again on the next tick — the WAL
|
|
1207
|
+
* staying big for another minute costs nothing.
|
|
1208
|
+
*/
|
|
1209
|
+
var WAL_CHECKPOINT_BUSY_TIMEOUT_MS = 250;
|
|
1210
|
+
/** How often the gate is evaluated, in ms. Cheap: a `stat` unless it fires. */
|
|
1211
|
+
var WAL_MAINTENANCE_INTERVAL_MS = 6e4;
|
|
1212
|
+
/**
|
|
1213
|
+
* Multiple of the high-water mark past which a persistently un-checkpointable
|
|
1214
|
+
* WAL stops being a debug line and becomes a warning. At 4x (256 MiB) the WAL
|
|
1215
|
+
* is bigger than the database and something is holding a read transaction
|
|
1216
|
+
* open; that is worth seeing in Loki.
|
|
1217
|
+
*/
|
|
1218
|
+
var WAL_ALARM_MULTIPLE = 4;
|
|
1219
|
+
function isCheckpointRow(value) {
|
|
1220
|
+
if (typeof value !== "object" || value === null) return false;
|
|
1221
|
+
if (!("busy" in value) || !("log" in value) || !("checkpointed" in value)) return false;
|
|
1222
|
+
return typeof value.busy === "number" && typeof value.log === "number" && typeof value.checkpointed === "number";
|
|
1223
|
+
}
|
|
1224
|
+
/** Narrow `PRAGMA wal_checkpoint` output without a cast; null when unreadable. */
|
|
1225
|
+
function readCheckpointResult(value) {
|
|
1226
|
+
if (isCheckpointRow(value)) return value;
|
|
1227
|
+
if (Array.isArray(value)) {
|
|
1228
|
+
const [first] = value;
|
|
1229
|
+
if (isCheckpointRow(first)) return first;
|
|
1230
|
+
}
|
|
1231
|
+
return null;
|
|
1232
|
+
}
|
|
1233
|
+
/**
|
|
1234
|
+
* Periodic, heavily gated `wal_checkpoint(TRUNCATE)`.
|
|
1235
|
+
*
|
|
1236
|
+
* The interesting behaviour is all refusal. A tick costs one `stat` in the
|
|
1237
|
+
* common case; it escalates to a lock-taking checkpoint only when the WAL is
|
|
1238
|
+
* over the high-water mark AND nothing has written to it for
|
|
1239
|
+
* {@link WAL_IDLE_QUIET_MS}, and even then the wait is bounded to
|
|
1240
|
+
* {@link WAL_CHECKPOINT_BUSY_TIMEOUT_MS}.
|
|
1241
|
+
*/
|
|
1242
|
+
var WalMaintenance = class {
|
|
1243
|
+
db;
|
|
1244
|
+
walPath;
|
|
1245
|
+
logger;
|
|
1246
|
+
highWaterBytes;
|
|
1247
|
+
idleQuietMs;
|
|
1248
|
+
intervalMs;
|
|
1249
|
+
busyTimeoutMs;
|
|
1250
|
+
timer = null;
|
|
1251
|
+
constructor(options) {
|
|
1252
|
+
this.db = options.db;
|
|
1253
|
+
this.walPath = `${options.dbPath}-wal`;
|
|
1254
|
+
this.logger = options.logger;
|
|
1255
|
+
this.highWaterBytes = options.highWaterBytes ?? 67108864;
|
|
1256
|
+
this.idleQuietMs = options.idleQuietMs ?? 3e4;
|
|
1257
|
+
this.intervalMs = options.intervalMs ?? 6e4;
|
|
1258
|
+
this.busyTimeoutMs = options.busyTimeoutMs ?? 250;
|
|
1259
|
+
}
|
|
1260
|
+
start() {
|
|
1261
|
+
if (this.timer !== null) return;
|
|
1262
|
+
const timer = setInterval(() => {
|
|
1263
|
+
this.runOnce();
|
|
1264
|
+
}, this.intervalMs);
|
|
1265
|
+
timer.unref?.();
|
|
1266
|
+
this.timer = timer;
|
|
1267
|
+
this.logger.info("WAL maintenance armed", { meta: {
|
|
1268
|
+
intervalMs: this.intervalMs,
|
|
1269
|
+
highWaterBytes: this.highWaterBytes,
|
|
1270
|
+
idleQuietMs: this.idleQuietMs
|
|
1271
|
+
} });
|
|
1272
|
+
}
|
|
1273
|
+
stop() {
|
|
1274
|
+
if (this.timer === null) return;
|
|
1275
|
+
clearInterval(this.timer);
|
|
1276
|
+
this.timer = null;
|
|
1277
|
+
}
|
|
1278
|
+
isRunning() {
|
|
1279
|
+
return this.timer !== null;
|
|
1280
|
+
}
|
|
1281
|
+
/** Evaluate the gate once and act. Never throws. */
|
|
1282
|
+
runOnce() {
|
|
1283
|
+
try {
|
|
1284
|
+
return this.evaluate();
|
|
1285
|
+
} catch (err) {
|
|
1286
|
+
const error = errMsg(err);
|
|
1287
|
+
this.logger.warn("WAL maintenance tick failed", { meta: {
|
|
1288
|
+
error,
|
|
1289
|
+
walPath: this.walPath
|
|
1290
|
+
} });
|
|
1291
|
+
return {
|
|
1292
|
+
kind: "failed",
|
|
1293
|
+
error
|
|
1294
|
+
};
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
evaluate() {
|
|
1298
|
+
const stat = this.statWal();
|
|
1299
|
+
if (stat === null) return { kind: "no-wal" };
|
|
1300
|
+
if (stat.size < this.highWaterBytes) return {
|
|
1301
|
+
kind: "below-high-water",
|
|
1302
|
+
walBytes: stat.size
|
|
1303
|
+
};
|
|
1304
|
+
const quietForMs = Date.now() - stat.mtimeMs;
|
|
1305
|
+
if (quietForMs < this.idleQuietMs) {
|
|
1306
|
+
const meta = {
|
|
1307
|
+
walBytes: stat.size,
|
|
1308
|
+
quietForMs,
|
|
1309
|
+
idleQuietMs: this.idleQuietMs
|
|
1310
|
+
};
|
|
1311
|
+
if (stat.size >= this.highWaterBytes * WAL_ALARM_MULTIPLE) this.logger.warn("WAL checkpoint deferred: file far over high-water and still being written", { meta });
|
|
1312
|
+
else this.logger.debug("WAL checkpoint deferred: database is not idle", { meta });
|
|
1313
|
+
return {
|
|
1314
|
+
kind: "deferred-not-idle",
|
|
1315
|
+
walBytes: stat.size,
|
|
1316
|
+
quietForMs
|
|
1317
|
+
};
|
|
1318
|
+
}
|
|
1319
|
+
return this.checkpoint(stat.size);
|
|
1320
|
+
}
|
|
1321
|
+
/**
|
|
1322
|
+
* Take the writer lock and fold the WAL back, with the wait bounded.
|
|
1323
|
+
*
|
|
1324
|
+
* The original `busy_timeout` is restored in a `finally`: this connection is
|
|
1325
|
+
* shared with the settings backend and the vector index, and leaving them on
|
|
1326
|
+
* a 250 ms timeout would turn an ordinary contended write into a spurious
|
|
1327
|
+
* SQLITE_BUSY.
|
|
1328
|
+
*/
|
|
1329
|
+
checkpoint(beforeBytes) {
|
|
1330
|
+
const previousTimeout = this.db.pragma("busy_timeout", { simple: true });
|
|
1331
|
+
this.db.pragma(`busy_timeout = ${this.busyTimeoutMs}`);
|
|
1332
|
+
let raw;
|
|
1333
|
+
try {
|
|
1334
|
+
raw = this.db.pragma("wal_checkpoint(TRUNCATE)");
|
|
1335
|
+
} finally {
|
|
1336
|
+
if (typeof previousTimeout === "number") this.db.pragma(`busy_timeout = ${previousTimeout}`);
|
|
1337
|
+
}
|
|
1338
|
+
const result = readCheckpointResult(raw);
|
|
1339
|
+
const afterBytes = this.statWal()?.size ?? 0;
|
|
1340
|
+
const outcome = {
|
|
1341
|
+
kind: "checkpointed",
|
|
1342
|
+
beforeBytes,
|
|
1343
|
+
afterBytes,
|
|
1344
|
+
busy: result !== null && result.busy !== 0,
|
|
1345
|
+
walFrames: result?.log ?? -1,
|
|
1346
|
+
checkpointedFrames: result?.checkpointed ?? -1
|
|
1347
|
+
};
|
|
1348
|
+
this.logger.info("WAL truncating checkpoint completed", { meta: {
|
|
1349
|
+
beforeBytes,
|
|
1350
|
+
afterBytes,
|
|
1351
|
+
reclaimedBytes: beforeBytes - afterBytes,
|
|
1352
|
+
busy: outcome.busy,
|
|
1353
|
+
walFrames: outcome.walFrames,
|
|
1354
|
+
checkpointedFrames: outcome.checkpointedFrames
|
|
1355
|
+
} });
|
|
1356
|
+
return outcome;
|
|
1357
|
+
}
|
|
1358
|
+
statWal() {
|
|
1359
|
+
try {
|
|
1360
|
+
const s = statSync(this.walPath);
|
|
1361
|
+
return {
|
|
1362
|
+
size: s.size,
|
|
1363
|
+
mtimeMs: s.mtimeMs
|
|
1364
|
+
};
|
|
1365
|
+
} catch {
|
|
1366
|
+
return null;
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
};
|
|
1370
|
+
//#endregion
|
|
954
1371
|
//#region src/builtins/sqlite-storage/sqlite-settings.addon.ts
|
|
955
1372
|
/**
|
|
956
1373
|
* SQLite Settings addon — the relational ENGINE behind the data door.
|
|
@@ -971,11 +1388,12 @@ function rebuild(row) {
|
|
|
971
1388
|
var SqliteSettingsAddon = class extends BaseAddon {
|
|
972
1389
|
backend = null;
|
|
973
1390
|
vectorIndex = null;
|
|
1391
|
+
walMaintenance = null;
|
|
974
1392
|
constructor() {
|
|
975
1393
|
super({});
|
|
976
1394
|
}
|
|
977
1395
|
async onInitialize() {
|
|
978
|
-
const addonId = this.ctx.id
|
|
1396
|
+
const addonId = bareAddonId(this.ctx.id);
|
|
979
1397
|
const path = await import("node:path");
|
|
980
1398
|
const dbPath = this.ctx.dataDir ? path.join(this.ctx.dataDir, "camstack.db") : path.join("camstack-data", "addons-data", addonId, "camstack.db");
|
|
981
1399
|
const pathSource = this.ctx.dataDir ? "dataDir" : "fallback (hardcoded)";
|
|
@@ -994,6 +1412,11 @@ var SqliteSettingsAddon = class extends BaseAddon {
|
|
|
994
1412
|
this.ctx.logger.info("DB file status", { meta: { dbExists } });
|
|
995
1413
|
this.backend = new SqliteSettingsBackend(dbPath, { ...RUNTIME_DEFAULTS });
|
|
996
1414
|
await this.backend.initialize();
|
|
1415
|
+
try {
|
|
1416
|
+
await purgeRetiredSettingsKeys(retiredKeyStoreOf(this.backend), this.ctx.logger.child("RetiredKeys"));
|
|
1417
|
+
} catch (err) {
|
|
1418
|
+
this.ctx.logger.warn("Retired-key purge failed", { meta: { error: errMsg(err) } });
|
|
1419
|
+
}
|
|
997
1420
|
const vectorIndex = this.createVectorIndex(this.backend);
|
|
998
1421
|
await vectorIndex.loadIndexRegistry();
|
|
999
1422
|
this.vectorIndex = vectorIndex;
|
|
@@ -1014,6 +1437,16 @@ var SqliteSettingsAddon = class extends BaseAddon {
|
|
|
1014
1437
|
deleteByFilter: async (input) => ({ deleted: await vectorIndex.deleteByFilter(input.index, input.filter) }),
|
|
1015
1438
|
stats: async (input) => vectorIndex.stats(input.index)
|
|
1016
1439
|
};
|
|
1440
|
+
const db = this.backend.getDatabase();
|
|
1441
|
+
if (db !== null) {
|
|
1442
|
+
const maintenance = new WalMaintenance({
|
|
1443
|
+
db,
|
|
1444
|
+
dbPath,
|
|
1445
|
+
logger: this.ctx.logger.child("WalMaintenance")
|
|
1446
|
+
});
|
|
1447
|
+
maintenance.start();
|
|
1448
|
+
this.walMaintenance = maintenance;
|
|
1449
|
+
}
|
|
1017
1450
|
this.ctx.logger.info("Initialized successfully");
|
|
1018
1451
|
return [{
|
|
1019
1452
|
capability: dataStoreProviderCapability,
|
|
@@ -1024,6 +1457,8 @@ var SqliteSettingsAddon = class extends BaseAddon {
|
|
|
1024
1457
|
}];
|
|
1025
1458
|
}
|
|
1026
1459
|
async onShutdown() {
|
|
1460
|
+
this.walMaintenance?.stop();
|
|
1461
|
+
this.walMaintenance = null;
|
|
1027
1462
|
await this.backend?.shutdown();
|
|
1028
1463
|
}
|
|
1029
1464
|
getBackend() {
|
|
@@ -1061,4 +1496,4 @@ var SqliteSettingsAddon = class extends BaseAddon {
|
|
|
1061
1496
|
}
|
|
1062
1497
|
};
|
|
1063
1498
|
//#endregion
|
|
1064
|
-
export { SqliteSettingsAddon, SqliteSettingsAddon as default, SqliteSettingsBackend as t };
|
|
1499
|
+
export { SqliteSettingsAddon, SqliteSettingsAddon as default, WalMaintenance as a, SQLITE_MMAP_SIZE_BYTES as c, WAL_MAINTENANCE_INTERVAL_MS as i, SQLITE_PAGE_CACHE_KIB as l, WAL_HIGH_WATER_BYTES as n, SqliteSettingsBackend as o, WAL_IDLE_QUIET_MS as r, SQLITE_JOURNAL_SIZE_LIMIT_BYTES as s, WAL_CHECKPOINT_BUSY_TIMEOUT_MS as t };
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { IScopedLogger } from '@camstack/types';
|
|
2
|
+
import { default as Database } from 'better-sqlite3';
|
|
3
|
+
/**
|
|
4
|
+
* Size at or above which the WAL is a candidate for a truncating checkpoint,
|
|
5
|
+
* in bytes. 67108864 = **64 MiB**.
|
|
6
|
+
*
|
|
7
|
+
* Matched to `SQLITE_JOURNAL_SIZE_LIMIT_BYTES`: below this, the passive
|
|
8
|
+
* mechanism (autocheckpoint + `journal_size_limit`) is doing its job and there
|
|
9
|
+
* is nothing to reclaim. The live hub's WAL had reached 248,955,152 bytes —
|
|
10
|
+
* roughly 2x the 130 MB database — which is what says the passive path alone
|
|
11
|
+
* had stopped keeping up.
|
|
12
|
+
*/
|
|
13
|
+
export declare const WAL_HIGH_WATER_BYTES = 67108864;
|
|
14
|
+
/**
|
|
15
|
+
* How long the WAL must have gone UNWRITTEN before a truncating checkpoint is
|
|
16
|
+
* allowed, in ms.
|
|
17
|
+
*
|
|
18
|
+
* This is the stall guard and the reason this file exists. `wal_checkpoint
|
|
19
|
+
* (TRUNCATE)` takes the writer lock, waits for readers, and copies the whole
|
|
20
|
+
* WAL back into the database — synchronously, on the thread that serves every
|
|
21
|
+
* request. An unconditional periodic one on a busy NVR would be the disease
|
|
22
|
+
* (hub-main already spends 42 s per 2 minutes inside uninterrupted blocks),
|
|
23
|
+
* not the cure.
|
|
24
|
+
*
|
|
25
|
+
* The idle signal is the WAL file's own mtime. It is one `stat`, and unlike a
|
|
26
|
+
* write counter maintained in this class it covers EVERY writer on the file —
|
|
27
|
+
* the settings backend, the sqlite-vec index sharing the handle, and the
|
|
28
|
+
* short-lived second connection system-backup opens. On a genuinely busy NVR
|
|
29
|
+
* this gate simply never opens, and `journal_size_limit` remains the only
|
|
30
|
+
* thing bounding the file. That is the intended failure mode: doing nothing is
|
|
31
|
+
* strictly better than a 249 MB copy in the middle of a live request.
|
|
32
|
+
*/
|
|
33
|
+
export declare const WAL_IDLE_QUIET_MS = 30000;
|
|
34
|
+
/**
|
|
35
|
+
* Ceiling on how long the checkpoint may wait for a lock, in ms.
|
|
36
|
+
*
|
|
37
|
+
* better-sqlite3 defaults `busy_timeout` to 5000. A checkpoint invokes the
|
|
38
|
+
* busy handler while waiting for readers, so without this a "quiet" moment
|
|
39
|
+
* that turns out not to be quiet could park the main thread for 5 s. 250 ms
|
|
40
|
+
* means the checkpoint gives up and we try again on the next tick — the WAL
|
|
41
|
+
* staying big for another minute costs nothing.
|
|
42
|
+
*/
|
|
43
|
+
export declare const WAL_CHECKPOINT_BUSY_TIMEOUT_MS = 250;
|
|
44
|
+
/** How often the gate is evaluated, in ms. Cheap: a `stat` unless it fires. */
|
|
45
|
+
export declare const WAL_MAINTENANCE_INTERVAL_MS = 60000;
|
|
46
|
+
/** What a single maintenance tick did. Every branch is named. */
|
|
47
|
+
export type WalMaintenanceOutcome = {
|
|
48
|
+
readonly kind: 'no-wal';
|
|
49
|
+
} | {
|
|
50
|
+
readonly kind: 'below-high-water';
|
|
51
|
+
readonly walBytes: number;
|
|
52
|
+
} | {
|
|
53
|
+
readonly kind: 'deferred-not-idle';
|
|
54
|
+
readonly walBytes: number;
|
|
55
|
+
readonly quietForMs: number;
|
|
56
|
+
} | {
|
|
57
|
+
readonly kind: 'checkpointed';
|
|
58
|
+
readonly beforeBytes: number;
|
|
59
|
+
readonly afterBytes: number;
|
|
60
|
+
readonly busy: boolean;
|
|
61
|
+
readonly walFrames: number;
|
|
62
|
+
readonly checkpointedFrames: number;
|
|
63
|
+
} | {
|
|
64
|
+
readonly kind: 'failed';
|
|
65
|
+
readonly error: string;
|
|
66
|
+
};
|
|
67
|
+
/** Construction inputs for {@link WalMaintenance}. */
|
|
68
|
+
export interface WalMaintenanceOptions {
|
|
69
|
+
readonly db: Database.Database;
|
|
70
|
+
/** Path to the main database file; the WAL is `${dbPath}-wal`. */
|
|
71
|
+
readonly dbPath: string;
|
|
72
|
+
readonly logger: IScopedLogger;
|
|
73
|
+
readonly highWaterBytes?: number;
|
|
74
|
+
readonly idleQuietMs?: number;
|
|
75
|
+
readonly intervalMs?: number;
|
|
76
|
+
readonly busyTimeoutMs?: number;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Periodic, heavily gated `wal_checkpoint(TRUNCATE)`.
|
|
80
|
+
*
|
|
81
|
+
* The interesting behaviour is all refusal. A tick costs one `stat` in the
|
|
82
|
+
* common case; it escalates to a lock-taking checkpoint only when the WAL is
|
|
83
|
+
* over the high-water mark AND nothing has written to it for
|
|
84
|
+
* {@link WAL_IDLE_QUIET_MS}, and even then the wait is bounded to
|
|
85
|
+
* {@link WAL_CHECKPOINT_BUSY_TIMEOUT_MS}.
|
|
86
|
+
*/
|
|
87
|
+
export declare class WalMaintenance {
|
|
88
|
+
private readonly db;
|
|
89
|
+
private readonly walPath;
|
|
90
|
+
private readonly logger;
|
|
91
|
+
private readonly highWaterBytes;
|
|
92
|
+
private readonly idleQuietMs;
|
|
93
|
+
private readonly intervalMs;
|
|
94
|
+
private readonly busyTimeoutMs;
|
|
95
|
+
private timer;
|
|
96
|
+
constructor(options: WalMaintenanceOptions);
|
|
97
|
+
start(): void;
|
|
98
|
+
stop(): void;
|
|
99
|
+
isRunning(): boolean;
|
|
100
|
+
/** Evaluate the gate once and act. Never throws. */
|
|
101
|
+
runOnce(): WalMaintenanceOutcome;
|
|
102
|
+
private evaluate;
|
|
103
|
+
/**
|
|
104
|
+
* Take the writer lock and fold the WAL back, with the wait bounded.
|
|
105
|
+
*
|
|
106
|
+
* The original `busy_timeout` is restored in a `finally`: this connection is
|
|
107
|
+
* shared with the settings backend and the vector index, and leaving them on
|
|
108
|
+
* a 250 ms timeout would turn an ordinary contended write into a spurious
|
|
109
|
+
* SQLITE_BUSY.
|
|
110
|
+
*/
|
|
111
|
+
private checkpoint;
|
|
112
|
+
private statWal;
|
|
113
|
+
}
|
|
@@ -3,7 +3,7 @@ Object.defineProperties(exports, {
|
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
5
|
const require_chunk = require("../../chunk-Cek0wNdY.js");
|
|
6
|
-
const require_dist = require("../../dist-
|
|
6
|
+
const require_dist = require("../../dist-CV6XTApk.js");
|
|
7
7
|
let node_path = require("node:path");
|
|
8
8
|
node_path = require_chunk.__toESM(node_path);
|
|
9
9
|
let node_fs_promises = require("node:fs/promises");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Nt as parseJsonObject, f as StorageLocationTypeSchema, it as storageCapability, nt as settingsStoreCapability, pt as BaseAddon } from "../../dist-BqJJWCS8.mjs";
|
|
2
2
|
import * as path$1 from "node:path";
|
|
3
3
|
import * as fs from "node:fs/promises";
|
|
4
4
|
import { buildStorageLocationRegistry } from "@camstack/system";
|
|
@@ -3,7 +3,7 @@ Object.defineProperties(exports, {
|
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
5
|
require("../../chunk-Cek0wNdY.js");
|
|
6
|
-
const require_dist = require("../../dist-
|
|
6
|
+
const require_dist = require("../../dist-CV6XTApk.js");
|
|
7
7
|
//#region src/builtins/system-config/system-config.addon.ts
|
|
8
8
|
/**
|
|
9
9
|
* Built-in `system-config` addon — Phase 4 of the settings redesign.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { At as hydrateSchema, ft as errMsg, pt as BaseAddon } from "../../dist-BqJJWCS8.mjs";
|
|
2
2
|
//#region src/builtins/system-config/system-config.addon.ts
|
|
3
3
|
/**
|
|
4
4
|
* Built-in `system-config` addon — Phase 4 of the settings redesign.
|
|
@@ -3,7 +3,7 @@ Object.defineProperties(exports, {
|
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
5
|
const require_chunk = require("../../chunk-Cek0wNdY.js");
|
|
6
|
-
const require_dist = require("../../dist-
|
|
6
|
+
const require_dist = require("../../dist-CV6XTApk.js");
|
|
7
7
|
const require_formatter = require("../../formatter-DqAKDlvN.js");
|
|
8
8
|
let node_path = require("node:path");
|
|
9
9
|
node_path = require_chunk.__toESM(node_path);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { G as logDestinationCapability, pt as BaseAddon } from "../../dist-BqJJWCS8.mjs";
|
|
2
2
|
import { t as formatLogLine } from "../../formatter-B7qW8bPJ.mjs";
|
|
3
3
|
import * as path$1 from "node:path";
|
|
4
4
|
import path from "node:path";
|