@camstack/system 1.2.56 → 1.2.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/addon-runner.js +1 -1
- package/dist/addon-runner.mjs +1 -1
- package/dist/addon-utils.js +1 -1
- package/dist/addon-utils.mjs +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +2 -2
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +2 -2
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +2 -2
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +2 -2
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +2 -2
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +2 -2
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +2 -2
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +2 -2
- package/dist/builtins/device-manager/device-manager.addon.js +1 -1
- package/dist/builtins/device-manager/device-manager.addon.mjs +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +2 -2
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +2 -2
- package/dist/builtins/sqlite-storage/index.d.ts +3 -0
- package/dist/builtins/sqlite-storage/index.js +8 -0
- package/dist/builtins/sqlite-storage/index.mjs +2 -2
- package/dist/builtins/sqlite-storage/sqlite-pragmas.d.ts +85 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.d.ts +1 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +304 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +297 -2
- package/dist/builtins/sqlite-storage/wal-maintenance.d.ts +113 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-CblJsHb-.js → dist-Chl2MvMd.js} +692 -25
- package/dist/{dist-BOpRqInW.mjs → dist-UrXyJ9Zm.mjs} +692 -25
- package/dist/index.js +4 -4
- package/dist/index.mjs +5 -5
- package/dist/kernel/transport/frame-codec.d.ts +43 -2
- package/dist/{manifest-python-deps-Ck4-9K9m.mjs → manifest-python-deps-CX5tGRqf.mjs} +95 -10
- package/dist/{manifest-python-deps-BqE5j0-O.js → manifest-python-deps-d4acYXVV.js} +95 -10
- package/package.json +1 -1
- package/dist/{model-download-service-hf0ookyy.js → model-download-service-Cpc6SWzV.js} +1 -1
- package/dist/{model-download-service-Cp9f4dk6.mjs → model-download-service-D8B-4ktF.mjs} +1 -1
|
@@ -3,11 +3,87 @@ Object.defineProperties(exports, {
|
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
5
|
const require_chunk = require("../../chunk-Cek0wNdY.js");
|
|
6
|
-
const require_dist = require("../../dist-
|
|
6
|
+
const require_dist = require("../../dist-Chl2MvMd.js");
|
|
7
7
|
let node_crypto = require("node:crypto");
|
|
8
|
+
let node_fs = require("node:fs");
|
|
8
9
|
let node_module = require("node:module");
|
|
9
10
|
let better_sqlite3 = require("better-sqlite3");
|
|
10
11
|
better_sqlite3 = require_chunk.__toESM(better_sqlite3);
|
|
12
|
+
//#region src/builtins/sqlite-storage/sqlite-pragmas.ts
|
|
13
|
+
/**
|
|
14
|
+
* Connection tuning for the hub's SQLite database.
|
|
15
|
+
*
|
|
16
|
+
* Every number here is sized against a MEASURED database, not a blog post.
|
|
17
|
+
* On the live hub (2026-08-06), 20 Hz syscall sampling of hub-main put **210
|
|
18
|
+
* of 2,561 samples inside `pread64`** — 161 on `camstack.db` and 49 on
|
|
19
|
+
* `camstack.db-wal`, all 4 KiB page reads, i.e. page-cache misses. That is
|
|
20
|
+
* 8.2% of wall-clock and ~20% during bursts, on the single thread that also
|
|
21
|
+
* serves every tRPC request. fds 79/81 of hub-main were the two files, so the
|
|
22
|
+
* cost is proven in-process rather than inferred.
|
|
23
|
+
*
|
|
24
|
+
* The database was **130 MB** with a **249 MB** WAL high-water mark. Its
|
|
25
|
+
* tenants are not settings: the analytics event/track/media stores and the
|
|
26
|
+
* sqlite-vec index all live in this one file.
|
|
27
|
+
*/
|
|
28
|
+
/**
|
|
29
|
+
* Page cache, in KiB (SQLite reads a NEGATIVE `cache_size` as KiB rather than
|
|
30
|
+
* as a page count, which is what makes it independent of `page_size`).
|
|
31
|
+
*
|
|
32
|
+
* -65536 = **64 MiB**. Sized from the 130 MB file: the hot working set is the
|
|
33
|
+
* B-tree interior pages plus the indexes the analytics queries walk, which is
|
|
34
|
+
* a fraction of the total — 64 MiB covers roughly half the file outright, so a
|
|
35
|
+
* repeated query stops re-reading the same 4 KiB pages from disk. Going to the
|
|
36
|
+
* full 130 MB would buy the tail of a cold table scan for twice the resident
|
|
37
|
+
* cost; going smaller leaves the measured miss rate roughly where it is.
|
|
38
|
+
*
|
|
39
|
+
* Why the build default is not enough: better-sqlite3 compiles with
|
|
40
|
+
* `SQLITE_DEFAULT_CACHE_SIZE=-16000`, so the starting point is ~16 MiB — 8x
|
|
41
|
+
* short of the file, not the ~2 MiB stock SQLite default. This is a 4x
|
|
42
|
+
* increase, not a 32x one.
|
|
43
|
+
*
|
|
44
|
+
* Cost: 64 MiB of malloc'd C memory, ONE connection's worth (better-sqlite3
|
|
45
|
+
* holds a single handle and the vector index shares it). It is off-heap, so it
|
|
46
|
+
* does not move V8's heap ceiling — the thing that produced the hub's previous
|
|
47
|
+
* OOM.
|
|
48
|
+
*/
|
|
49
|
+
var SQLITE_PAGE_CACHE_KIB = -65536;
|
|
50
|
+
/**
|
|
51
|
+
* Memory-mapped I/O window, in bytes. 268435456 = **256 MiB**.
|
|
52
|
+
*
|
|
53
|
+
* Chosen to exceed the 130 MB database with room for it to roughly double, so
|
|
54
|
+
* the whole main file can be mapped. Pages then arrive as minor faults against
|
|
55
|
+
* the OS page cache instead of `pread64` syscalls — which is precisely the
|
|
56
|
+
* 161-sample half of the measurement. It also means a page evicted from
|
|
57
|
+
* SQLite's own cache is usually still resident in the OS cache, so the two
|
|
58
|
+
* knobs compound rather than duplicate.
|
|
59
|
+
*
|
|
60
|
+
* Two honest caveats:
|
|
61
|
+
* - SQLite **never mmaps the WAL**. The 49 WAL samples are not addressed
|
|
62
|
+
* here; they are addressed by keeping the WAL small (see
|
|
63
|
+
* {@link SQLITE_JOURNAL_SIZE_LIMIT_BYTES} and `wal-maintenance.ts`).
|
|
64
|
+
* - With mmap, an I/O error on a mapped page surfaces as SIGBUS rather than a
|
|
65
|
+
* catchable SQLite error. The pragma is asserted in
|
|
66
|
+
* `__tests__/sqlite-pragmas.spec.ts` by reading it back, so a build that
|
|
67
|
+
* ignores it (`SQLITE_MAX_MMAP_SIZE=0`) fails loudly instead of quietly
|
|
68
|
+
* doing nothing.
|
|
69
|
+
*/
|
|
70
|
+
var SQLITE_MMAP_SIZE_BYTES = 268435456;
|
|
71
|
+
/**
|
|
72
|
+
* Upper bound on the WAL file left on disk after a checkpoint resets it, in
|
|
73
|
+
* bytes. 67108864 = **64 MiB**.
|
|
74
|
+
*
|
|
75
|
+
* This is the passive half of the WAL fix and it carries no stall risk at all:
|
|
76
|
+
* it takes no lock and blocks nothing. It only says "when a checkpoint has
|
|
77
|
+
* already succeeded, truncate the file rather than leaving the space
|
|
78
|
+
* allocated". Without it, a WAL that spiked once stays spiked — which is how
|
|
79
|
+
* a 130 MB database ends up with a 249 MB sidecar.
|
|
80
|
+
*
|
|
81
|
+
* 64 MiB is deliberately well above the ~4 MiB `wal_autocheckpoint` threshold
|
|
82
|
+
* (1000 pages), so ordinary operation never touches this ceiling and the file
|
|
83
|
+
* is not being repeatedly truncated and re-grown.
|
|
84
|
+
*/
|
|
85
|
+
var SQLITE_JOURNAL_SIZE_LIMIT_BYTES = 67108864;
|
|
86
|
+
//#endregion
|
|
11
87
|
//#region src/builtins/sqlite-storage/filter-compiler.ts
|
|
12
88
|
/** Thrown by `mutate` mode. Distinct type so a caller can map it to a 400. */
|
|
13
89
|
var UnsafeFilterError = class extends Error {
|
|
@@ -115,6 +191,9 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
|
|
|
115
191
|
this.db = new better_sqlite3.default(this.dbPath);
|
|
116
192
|
this.db.pragma("journal_mode = WAL");
|
|
117
193
|
this.db.pragma("foreign_keys = ON");
|
|
194
|
+
this.db.pragma(`cache_size = ${SQLITE_PAGE_CACHE_KIB}`);
|
|
195
|
+
this.db.pragma(`mmap_size = ${SQLITE_MMAP_SIZE_BYTES}`);
|
|
196
|
+
this.db.pragma(`journal_size_limit = ${SQLITE_JOURNAL_SIZE_LIMIT_BYTES}`);
|
|
118
197
|
for (const collection of SqliteSettingsBackend.CANONICAL_KV_COLLECTIONS) {
|
|
119
198
|
await this.ensureTable(collection, { columns: [{
|
|
120
199
|
name: "id",
|
|
@@ -957,6 +1036,209 @@ function rebuild(row) {
|
|
|
957
1036
|
return out;
|
|
958
1037
|
}
|
|
959
1038
|
//#endregion
|
|
1039
|
+
//#region src/builtins/sqlite-storage/wal-maintenance.ts
|
|
1040
|
+
/**
|
|
1041
|
+
* Size at or above which the WAL is a candidate for a truncating checkpoint,
|
|
1042
|
+
* in bytes. 67108864 = **64 MiB**.
|
|
1043
|
+
*
|
|
1044
|
+
* Matched to `SQLITE_JOURNAL_SIZE_LIMIT_BYTES`: below this, the passive
|
|
1045
|
+
* mechanism (autocheckpoint + `journal_size_limit`) is doing its job and there
|
|
1046
|
+
* is nothing to reclaim. The live hub's WAL had reached 248,955,152 bytes —
|
|
1047
|
+
* roughly 2x the 130 MB database — which is what says the passive path alone
|
|
1048
|
+
* had stopped keeping up.
|
|
1049
|
+
*/
|
|
1050
|
+
var WAL_HIGH_WATER_BYTES = 67108864;
|
|
1051
|
+
/**
|
|
1052
|
+
* How long the WAL must have gone UNWRITTEN before a truncating checkpoint is
|
|
1053
|
+
* allowed, in ms.
|
|
1054
|
+
*
|
|
1055
|
+
* This is the stall guard and the reason this file exists. `wal_checkpoint
|
|
1056
|
+
* (TRUNCATE)` takes the writer lock, waits for readers, and copies the whole
|
|
1057
|
+
* WAL back into the database — synchronously, on the thread that serves every
|
|
1058
|
+
* request. An unconditional periodic one on a busy NVR would be the disease
|
|
1059
|
+
* (hub-main already spends 42 s per 2 minutes inside uninterrupted blocks),
|
|
1060
|
+
* not the cure.
|
|
1061
|
+
*
|
|
1062
|
+
* The idle signal is the WAL file's own mtime. It is one `stat`, and unlike a
|
|
1063
|
+
* write counter maintained in this class it covers EVERY writer on the file —
|
|
1064
|
+
* the settings backend, the sqlite-vec index sharing the handle, and the
|
|
1065
|
+
* short-lived second connection system-backup opens. On a genuinely busy NVR
|
|
1066
|
+
* this gate simply never opens, and `journal_size_limit` remains the only
|
|
1067
|
+
* thing bounding the file. That is the intended failure mode: doing nothing is
|
|
1068
|
+
* strictly better than a 249 MB copy in the middle of a live request.
|
|
1069
|
+
*/
|
|
1070
|
+
var WAL_IDLE_QUIET_MS = 3e4;
|
|
1071
|
+
/**
|
|
1072
|
+
* Ceiling on how long the checkpoint may wait for a lock, in ms.
|
|
1073
|
+
*
|
|
1074
|
+
* better-sqlite3 defaults `busy_timeout` to 5000. A checkpoint invokes the
|
|
1075
|
+
* busy handler while waiting for readers, so without this a "quiet" moment
|
|
1076
|
+
* that turns out not to be quiet could park the main thread for 5 s. 250 ms
|
|
1077
|
+
* means the checkpoint gives up and we try again on the next tick — the WAL
|
|
1078
|
+
* staying big for another minute costs nothing.
|
|
1079
|
+
*/
|
|
1080
|
+
var WAL_CHECKPOINT_BUSY_TIMEOUT_MS = 250;
|
|
1081
|
+
/** How often the gate is evaluated, in ms. Cheap: a `stat` unless it fires. */
|
|
1082
|
+
var WAL_MAINTENANCE_INTERVAL_MS = 6e4;
|
|
1083
|
+
/**
|
|
1084
|
+
* Multiple of the high-water mark past which a persistently un-checkpointable
|
|
1085
|
+
* WAL stops being a debug line and becomes a warning. At 4x (256 MiB) the WAL
|
|
1086
|
+
* is bigger than the database and something is holding a read transaction
|
|
1087
|
+
* open; that is worth seeing in Loki.
|
|
1088
|
+
*/
|
|
1089
|
+
var WAL_ALARM_MULTIPLE = 4;
|
|
1090
|
+
function isCheckpointRow(value) {
|
|
1091
|
+
if (typeof value !== "object" || value === null) return false;
|
|
1092
|
+
if (!("busy" in value) || !("log" in value) || !("checkpointed" in value)) return false;
|
|
1093
|
+
return typeof value.busy === "number" && typeof value.log === "number" && typeof value.checkpointed === "number";
|
|
1094
|
+
}
|
|
1095
|
+
/** Narrow `PRAGMA wal_checkpoint` output without a cast; null when unreadable. */
|
|
1096
|
+
function readCheckpointResult(value) {
|
|
1097
|
+
if (isCheckpointRow(value)) return value;
|
|
1098
|
+
if (Array.isArray(value)) {
|
|
1099
|
+
const [first] = value;
|
|
1100
|
+
if (isCheckpointRow(first)) return first;
|
|
1101
|
+
}
|
|
1102
|
+
return null;
|
|
1103
|
+
}
|
|
1104
|
+
/**
|
|
1105
|
+
* Periodic, heavily gated `wal_checkpoint(TRUNCATE)`.
|
|
1106
|
+
*
|
|
1107
|
+
* The interesting behaviour is all refusal. A tick costs one `stat` in the
|
|
1108
|
+
* common case; it escalates to a lock-taking checkpoint only when the WAL is
|
|
1109
|
+
* over the high-water mark AND nothing has written to it for
|
|
1110
|
+
* {@link WAL_IDLE_QUIET_MS}, and even then the wait is bounded to
|
|
1111
|
+
* {@link WAL_CHECKPOINT_BUSY_TIMEOUT_MS}.
|
|
1112
|
+
*/
|
|
1113
|
+
var WalMaintenance = class {
|
|
1114
|
+
db;
|
|
1115
|
+
walPath;
|
|
1116
|
+
logger;
|
|
1117
|
+
highWaterBytes;
|
|
1118
|
+
idleQuietMs;
|
|
1119
|
+
intervalMs;
|
|
1120
|
+
busyTimeoutMs;
|
|
1121
|
+
timer = null;
|
|
1122
|
+
constructor(options) {
|
|
1123
|
+
this.db = options.db;
|
|
1124
|
+
this.walPath = `${options.dbPath}-wal`;
|
|
1125
|
+
this.logger = options.logger;
|
|
1126
|
+
this.highWaterBytes = options.highWaterBytes ?? 67108864;
|
|
1127
|
+
this.idleQuietMs = options.idleQuietMs ?? 3e4;
|
|
1128
|
+
this.intervalMs = options.intervalMs ?? 6e4;
|
|
1129
|
+
this.busyTimeoutMs = options.busyTimeoutMs ?? 250;
|
|
1130
|
+
}
|
|
1131
|
+
start() {
|
|
1132
|
+
if (this.timer !== null) return;
|
|
1133
|
+
const timer = setInterval(() => {
|
|
1134
|
+
this.runOnce();
|
|
1135
|
+
}, this.intervalMs);
|
|
1136
|
+
timer.unref?.();
|
|
1137
|
+
this.timer = timer;
|
|
1138
|
+
this.logger.info("WAL maintenance armed", { meta: {
|
|
1139
|
+
intervalMs: this.intervalMs,
|
|
1140
|
+
highWaterBytes: this.highWaterBytes,
|
|
1141
|
+
idleQuietMs: this.idleQuietMs
|
|
1142
|
+
} });
|
|
1143
|
+
}
|
|
1144
|
+
stop() {
|
|
1145
|
+
if (this.timer === null) return;
|
|
1146
|
+
clearInterval(this.timer);
|
|
1147
|
+
this.timer = null;
|
|
1148
|
+
}
|
|
1149
|
+
isRunning() {
|
|
1150
|
+
return this.timer !== null;
|
|
1151
|
+
}
|
|
1152
|
+
/** Evaluate the gate once and act. Never throws. */
|
|
1153
|
+
runOnce() {
|
|
1154
|
+
try {
|
|
1155
|
+
return this.evaluate();
|
|
1156
|
+
} catch (err) {
|
|
1157
|
+
const error = require_dist.errMsg(err);
|
|
1158
|
+
this.logger.warn("WAL maintenance tick failed", { meta: {
|
|
1159
|
+
error,
|
|
1160
|
+
walPath: this.walPath
|
|
1161
|
+
} });
|
|
1162
|
+
return {
|
|
1163
|
+
kind: "failed",
|
|
1164
|
+
error
|
|
1165
|
+
};
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
evaluate() {
|
|
1169
|
+
const stat = this.statWal();
|
|
1170
|
+
if (stat === null) return { kind: "no-wal" };
|
|
1171
|
+
if (stat.size < this.highWaterBytes) return {
|
|
1172
|
+
kind: "below-high-water",
|
|
1173
|
+
walBytes: stat.size
|
|
1174
|
+
};
|
|
1175
|
+
const quietForMs = Date.now() - stat.mtimeMs;
|
|
1176
|
+
if (quietForMs < this.idleQuietMs) {
|
|
1177
|
+
const meta = {
|
|
1178
|
+
walBytes: stat.size,
|
|
1179
|
+
quietForMs,
|
|
1180
|
+
idleQuietMs: this.idleQuietMs
|
|
1181
|
+
};
|
|
1182
|
+
if (stat.size >= this.highWaterBytes * WAL_ALARM_MULTIPLE) this.logger.warn("WAL checkpoint deferred: file far over high-water and still being written", { meta });
|
|
1183
|
+
else this.logger.debug("WAL checkpoint deferred: database is not idle", { meta });
|
|
1184
|
+
return {
|
|
1185
|
+
kind: "deferred-not-idle",
|
|
1186
|
+
walBytes: stat.size,
|
|
1187
|
+
quietForMs
|
|
1188
|
+
};
|
|
1189
|
+
}
|
|
1190
|
+
return this.checkpoint(stat.size);
|
|
1191
|
+
}
|
|
1192
|
+
/**
|
|
1193
|
+
* Take the writer lock and fold the WAL back, with the wait bounded.
|
|
1194
|
+
*
|
|
1195
|
+
* The original `busy_timeout` is restored in a `finally`: this connection is
|
|
1196
|
+
* shared with the settings backend and the vector index, and leaving them on
|
|
1197
|
+
* a 250 ms timeout would turn an ordinary contended write into a spurious
|
|
1198
|
+
* SQLITE_BUSY.
|
|
1199
|
+
*/
|
|
1200
|
+
checkpoint(beforeBytes) {
|
|
1201
|
+
const previousTimeout = this.db.pragma("busy_timeout", { simple: true });
|
|
1202
|
+
this.db.pragma(`busy_timeout = ${this.busyTimeoutMs}`);
|
|
1203
|
+
let raw;
|
|
1204
|
+
try {
|
|
1205
|
+
raw = this.db.pragma("wal_checkpoint(TRUNCATE)");
|
|
1206
|
+
} finally {
|
|
1207
|
+
if (typeof previousTimeout === "number") this.db.pragma(`busy_timeout = ${previousTimeout}`);
|
|
1208
|
+
}
|
|
1209
|
+
const result = readCheckpointResult(raw);
|
|
1210
|
+
const afterBytes = this.statWal()?.size ?? 0;
|
|
1211
|
+
const outcome = {
|
|
1212
|
+
kind: "checkpointed",
|
|
1213
|
+
beforeBytes,
|
|
1214
|
+
afterBytes,
|
|
1215
|
+
busy: result !== null && result.busy !== 0,
|
|
1216
|
+
walFrames: result?.log ?? -1,
|
|
1217
|
+
checkpointedFrames: result?.checkpointed ?? -1
|
|
1218
|
+
};
|
|
1219
|
+
this.logger.info("WAL truncating checkpoint completed", { meta: {
|
|
1220
|
+
beforeBytes,
|
|
1221
|
+
afterBytes,
|
|
1222
|
+
reclaimedBytes: beforeBytes - afterBytes,
|
|
1223
|
+
busy: outcome.busy,
|
|
1224
|
+
walFrames: outcome.walFrames,
|
|
1225
|
+
checkpointedFrames: outcome.checkpointedFrames
|
|
1226
|
+
} });
|
|
1227
|
+
return outcome;
|
|
1228
|
+
}
|
|
1229
|
+
statWal() {
|
|
1230
|
+
try {
|
|
1231
|
+
const s = (0, node_fs.statSync)(this.walPath);
|
|
1232
|
+
return {
|
|
1233
|
+
size: s.size,
|
|
1234
|
+
mtimeMs: s.mtimeMs
|
|
1235
|
+
};
|
|
1236
|
+
} catch {
|
|
1237
|
+
return null;
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
};
|
|
1241
|
+
//#endregion
|
|
960
1242
|
//#region src/builtins/sqlite-storage/sqlite-settings.addon.ts
|
|
961
1243
|
/**
|
|
962
1244
|
* SQLite Settings addon — the relational ENGINE behind the data door.
|
|
@@ -977,6 +1259,7 @@ function rebuild(row) {
|
|
|
977
1259
|
var SqliteSettingsAddon = class extends require_dist.BaseAddon {
|
|
978
1260
|
backend = null;
|
|
979
1261
|
vectorIndex = null;
|
|
1262
|
+
walMaintenance = null;
|
|
980
1263
|
constructor() {
|
|
981
1264
|
super({});
|
|
982
1265
|
}
|
|
@@ -1020,6 +1303,16 @@ var SqliteSettingsAddon = class extends require_dist.BaseAddon {
|
|
|
1020
1303
|
deleteByFilter: async (input) => ({ deleted: await vectorIndex.deleteByFilter(input.index, input.filter) }),
|
|
1021
1304
|
stats: async (input) => vectorIndex.stats(input.index)
|
|
1022
1305
|
};
|
|
1306
|
+
const db = this.backend.getDatabase();
|
|
1307
|
+
if (db !== null) {
|
|
1308
|
+
const maintenance = new WalMaintenance({
|
|
1309
|
+
db,
|
|
1310
|
+
dbPath,
|
|
1311
|
+
logger: this.ctx.logger.child("WalMaintenance")
|
|
1312
|
+
});
|
|
1313
|
+
maintenance.start();
|
|
1314
|
+
this.walMaintenance = maintenance;
|
|
1315
|
+
}
|
|
1023
1316
|
this.ctx.logger.info("Initialized successfully");
|
|
1024
1317
|
return [{
|
|
1025
1318
|
capability: require_dist.dataStoreProviderCapability,
|
|
@@ -1030,6 +1323,8 @@ var SqliteSettingsAddon = class extends require_dist.BaseAddon {
|
|
|
1030
1323
|
}];
|
|
1031
1324
|
}
|
|
1032
1325
|
async onShutdown() {
|
|
1326
|
+
this.walMaintenance?.stop();
|
|
1327
|
+
this.walMaintenance = null;
|
|
1033
1328
|
await this.backend?.shutdown();
|
|
1034
1329
|
}
|
|
1035
1330
|
getBackend() {
|
|
@@ -1067,6 +1362,14 @@ var SqliteSettingsAddon = class extends require_dist.BaseAddon {
|
|
|
1067
1362
|
}
|
|
1068
1363
|
};
|
|
1069
1364
|
//#endregion
|
|
1365
|
+
exports.SQLITE_JOURNAL_SIZE_LIMIT_BYTES = SQLITE_JOURNAL_SIZE_LIMIT_BYTES;
|
|
1366
|
+
exports.SQLITE_MMAP_SIZE_BYTES = SQLITE_MMAP_SIZE_BYTES;
|
|
1367
|
+
exports.SQLITE_PAGE_CACHE_KIB = SQLITE_PAGE_CACHE_KIB;
|
|
1070
1368
|
exports.SqliteSettingsAddon = SqliteSettingsAddon;
|
|
1071
1369
|
exports.default = SqliteSettingsAddon;
|
|
1072
1370
|
exports.SqliteSettingsBackend = SqliteSettingsBackend;
|
|
1371
|
+
exports.WAL_CHECKPOINT_BUSY_TIMEOUT_MS = WAL_CHECKPOINT_BUSY_TIMEOUT_MS;
|
|
1372
|
+
exports.WAL_HIGH_WATER_BYTES = WAL_HIGH_WATER_BYTES;
|
|
1373
|
+
exports.WAL_IDLE_QUIET_MS = WAL_IDLE_QUIET_MS;
|
|
1374
|
+
exports.WAL_MAINTENANCE_INTERVAL_MS = WAL_MAINTENANCE_INTERVAL_MS;
|
|
1375
|
+
exports.WalMaintenance = WalMaintenance;
|
|
@@ -1,7 +1,83 @@
|
|
|
1
|
-
import { C as dataStoreProviderCapability, Mt as parseJsonUnknown, ct as vectorDimFromBase64, dt as BaseAddon, l as RUNTIME_DEFAULTS, lt as vectorStoreCapability, ut as errMsg, w as decodeVectorBase64, xt as asJsonObject } from "../../dist-
|
|
1
|
+
import { C as dataStoreProviderCapability, Mt as parseJsonUnknown, ct as vectorDimFromBase64, dt as BaseAddon, l as RUNTIME_DEFAULTS, lt as vectorStoreCapability, ut as errMsg, w as decodeVectorBase64, xt as asJsonObject } from "../../dist-UrXyJ9Zm.mjs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { statSync } from "node:fs";
|
|
4
5
|
import Database from "better-sqlite3";
|
|
6
|
+
//#region src/builtins/sqlite-storage/sqlite-pragmas.ts
|
|
7
|
+
/**
|
|
8
|
+
* Connection tuning for the hub's SQLite database.
|
|
9
|
+
*
|
|
10
|
+
* Every number here is sized against a MEASURED database, not a blog post.
|
|
11
|
+
* On the live hub (2026-08-06), 20 Hz syscall sampling of hub-main put **210
|
|
12
|
+
* of 2,561 samples inside `pread64`** — 161 on `camstack.db` and 49 on
|
|
13
|
+
* `camstack.db-wal`, all 4 KiB page reads, i.e. page-cache misses. That is
|
|
14
|
+
* 8.2% of wall-clock and ~20% during bursts, on the single thread that also
|
|
15
|
+
* serves every tRPC request. fds 79/81 of hub-main were the two files, so the
|
|
16
|
+
* cost is proven in-process rather than inferred.
|
|
17
|
+
*
|
|
18
|
+
* The database was **130 MB** with a **249 MB** WAL high-water mark. Its
|
|
19
|
+
* tenants are not settings: the analytics event/track/media stores and the
|
|
20
|
+
* sqlite-vec index all live in this one file.
|
|
21
|
+
*/
|
|
22
|
+
/**
|
|
23
|
+
* Page cache, in KiB (SQLite reads a NEGATIVE `cache_size` as KiB rather than
|
|
24
|
+
* as a page count, which is what makes it independent of `page_size`).
|
|
25
|
+
*
|
|
26
|
+
* -65536 = **64 MiB**. Sized from the 130 MB file: the hot working set is the
|
|
27
|
+
* B-tree interior pages plus the indexes the analytics queries walk, which is
|
|
28
|
+
* a fraction of the total — 64 MiB covers roughly half the file outright, so a
|
|
29
|
+
* repeated query stops re-reading the same 4 KiB pages from disk. Going to the
|
|
30
|
+
* full 130 MB would buy the tail of a cold table scan for twice the resident
|
|
31
|
+
* cost; going smaller leaves the measured miss rate roughly where it is.
|
|
32
|
+
*
|
|
33
|
+
* Why the build default is not enough: better-sqlite3 compiles with
|
|
34
|
+
* `SQLITE_DEFAULT_CACHE_SIZE=-16000`, so the starting point is ~16 MiB — 8x
|
|
35
|
+
* short of the file, not the ~2 MiB stock SQLite default. This is a 4x
|
|
36
|
+
* increase, not a 32x one.
|
|
37
|
+
*
|
|
38
|
+
* Cost: 64 MiB of malloc'd C memory, ONE connection's worth (better-sqlite3
|
|
39
|
+
* holds a single handle and the vector index shares it). It is off-heap, so it
|
|
40
|
+
* does not move V8's heap ceiling — the thing that produced the hub's previous
|
|
41
|
+
* OOM.
|
|
42
|
+
*/
|
|
43
|
+
var SQLITE_PAGE_CACHE_KIB = -65536;
|
|
44
|
+
/**
|
|
45
|
+
* Memory-mapped I/O window, in bytes. 268435456 = **256 MiB**.
|
|
46
|
+
*
|
|
47
|
+
* Chosen to exceed the 130 MB database with room for it to roughly double, so
|
|
48
|
+
* the whole main file can be mapped. Pages then arrive as minor faults against
|
|
49
|
+
* the OS page cache instead of `pread64` syscalls — which is precisely the
|
|
50
|
+
* 161-sample half of the measurement. It also means a page evicted from
|
|
51
|
+
* SQLite's own cache is usually still resident in the OS cache, so the two
|
|
52
|
+
* knobs compound rather than duplicate.
|
|
53
|
+
*
|
|
54
|
+
* Two honest caveats:
|
|
55
|
+
* - SQLite **never mmaps the WAL**. The 49 WAL samples are not addressed
|
|
56
|
+
* here; they are addressed by keeping the WAL small (see
|
|
57
|
+
* {@link SQLITE_JOURNAL_SIZE_LIMIT_BYTES} and `wal-maintenance.ts`).
|
|
58
|
+
* - With mmap, an I/O error on a mapped page surfaces as SIGBUS rather than a
|
|
59
|
+
* catchable SQLite error. The pragma is asserted in
|
|
60
|
+
* `__tests__/sqlite-pragmas.spec.ts` by reading it back, so a build that
|
|
61
|
+
* ignores it (`SQLITE_MAX_MMAP_SIZE=0`) fails loudly instead of quietly
|
|
62
|
+
* doing nothing.
|
|
63
|
+
*/
|
|
64
|
+
var SQLITE_MMAP_SIZE_BYTES = 268435456;
|
|
65
|
+
/**
|
|
66
|
+
* Upper bound on the WAL file left on disk after a checkpoint resets it, in
|
|
67
|
+
* bytes. 67108864 = **64 MiB**.
|
|
68
|
+
*
|
|
69
|
+
* This is the passive half of the WAL fix and it carries no stall risk at all:
|
|
70
|
+
* it takes no lock and blocks nothing. It only says "when a checkpoint has
|
|
71
|
+
* already succeeded, truncate the file rather than leaving the space
|
|
72
|
+
* allocated". Without it, a WAL that spiked once stays spiked — which is how
|
|
73
|
+
* a 130 MB database ends up with a 249 MB sidecar.
|
|
74
|
+
*
|
|
75
|
+
* 64 MiB is deliberately well above the ~4 MiB `wal_autocheckpoint` threshold
|
|
76
|
+
* (1000 pages), so ordinary operation never touches this ceiling and the file
|
|
77
|
+
* is not being repeatedly truncated and re-grown.
|
|
78
|
+
*/
|
|
79
|
+
var SQLITE_JOURNAL_SIZE_LIMIT_BYTES = 67108864;
|
|
80
|
+
//#endregion
|
|
5
81
|
//#region src/builtins/sqlite-storage/filter-compiler.ts
|
|
6
82
|
/** Thrown by `mutate` mode. Distinct type so a caller can map it to a 400. */
|
|
7
83
|
var UnsafeFilterError = class extends Error {
|
|
@@ -109,6 +185,9 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
|
|
|
109
185
|
this.db = new Database(this.dbPath);
|
|
110
186
|
this.db.pragma("journal_mode = WAL");
|
|
111
187
|
this.db.pragma("foreign_keys = ON");
|
|
188
|
+
this.db.pragma(`cache_size = ${SQLITE_PAGE_CACHE_KIB}`);
|
|
189
|
+
this.db.pragma(`mmap_size = ${SQLITE_MMAP_SIZE_BYTES}`);
|
|
190
|
+
this.db.pragma(`journal_size_limit = ${SQLITE_JOURNAL_SIZE_LIMIT_BYTES}`);
|
|
112
191
|
for (const collection of SqliteSettingsBackend.CANONICAL_KV_COLLECTIONS) {
|
|
113
192
|
await this.ensureTable(collection, { columns: [{
|
|
114
193
|
name: "id",
|
|
@@ -951,6 +1030,209 @@ function rebuild(row) {
|
|
|
951
1030
|
return out;
|
|
952
1031
|
}
|
|
953
1032
|
//#endregion
|
|
1033
|
+
//#region src/builtins/sqlite-storage/wal-maintenance.ts
|
|
1034
|
+
/**
|
|
1035
|
+
* Size at or above which the WAL is a candidate for a truncating checkpoint,
|
|
1036
|
+
* in bytes. 67108864 = **64 MiB**.
|
|
1037
|
+
*
|
|
1038
|
+
* Matched to `SQLITE_JOURNAL_SIZE_LIMIT_BYTES`: below this, the passive
|
|
1039
|
+
* mechanism (autocheckpoint + `journal_size_limit`) is doing its job and there
|
|
1040
|
+
* is nothing to reclaim. The live hub's WAL had reached 248,955,152 bytes —
|
|
1041
|
+
* roughly 2x the 130 MB database — which is what says the passive path alone
|
|
1042
|
+
* had stopped keeping up.
|
|
1043
|
+
*/
|
|
1044
|
+
var WAL_HIGH_WATER_BYTES = 67108864;
|
|
1045
|
+
/**
|
|
1046
|
+
* How long the WAL must have gone UNWRITTEN before a truncating checkpoint is
|
|
1047
|
+
* allowed, in ms.
|
|
1048
|
+
*
|
|
1049
|
+
* This is the stall guard and the reason this file exists. `wal_checkpoint
|
|
1050
|
+
* (TRUNCATE)` takes the writer lock, waits for readers, and copies the whole
|
|
1051
|
+
* WAL back into the database — synchronously, on the thread that serves every
|
|
1052
|
+
* request. An unconditional periodic one on a busy NVR would be the disease
|
|
1053
|
+
* (hub-main already spends 42 s per 2 minutes inside uninterrupted blocks),
|
|
1054
|
+
* not the cure.
|
|
1055
|
+
*
|
|
1056
|
+
* The idle signal is the WAL file's own mtime. It is one `stat`, and unlike a
|
|
1057
|
+
* write counter maintained in this class it covers EVERY writer on the file —
|
|
1058
|
+
* the settings backend, the sqlite-vec index sharing the handle, and the
|
|
1059
|
+
* short-lived second connection system-backup opens. On a genuinely busy NVR
|
|
1060
|
+
* this gate simply never opens, and `journal_size_limit` remains the only
|
|
1061
|
+
* thing bounding the file. That is the intended failure mode: doing nothing is
|
|
1062
|
+
* strictly better than a 249 MB copy in the middle of a live request.
|
|
1063
|
+
*/
|
|
1064
|
+
var WAL_IDLE_QUIET_MS = 3e4;
|
|
1065
|
+
/**
|
|
1066
|
+
* Ceiling on how long the checkpoint may wait for a lock, in ms.
|
|
1067
|
+
*
|
|
1068
|
+
* better-sqlite3 defaults `busy_timeout` to 5000. A checkpoint invokes the
|
|
1069
|
+
* busy handler while waiting for readers, so without this a "quiet" moment
|
|
1070
|
+
* that turns out not to be quiet could park the main thread for 5 s. 250 ms
|
|
1071
|
+
* means the checkpoint gives up and we try again on the next tick — the WAL
|
|
1072
|
+
* staying big for another minute costs nothing.
|
|
1073
|
+
*/
|
|
1074
|
+
var WAL_CHECKPOINT_BUSY_TIMEOUT_MS = 250;
|
|
1075
|
+
/** How often the gate is evaluated, in ms. Cheap: a `stat` unless it fires. */
|
|
1076
|
+
var WAL_MAINTENANCE_INTERVAL_MS = 6e4;
|
|
1077
|
+
/**
|
|
1078
|
+
* Multiple of the high-water mark past which a persistently un-checkpointable
|
|
1079
|
+
* WAL stops being a debug line and becomes a warning. At 4x (256 MiB) the WAL
|
|
1080
|
+
* is bigger than the database and something is holding a read transaction
|
|
1081
|
+
* open; that is worth seeing in Loki.
|
|
1082
|
+
*/
|
|
1083
|
+
var WAL_ALARM_MULTIPLE = 4;
|
|
1084
|
+
function isCheckpointRow(value) {
|
|
1085
|
+
if (typeof value !== "object" || value === null) return false;
|
|
1086
|
+
if (!("busy" in value) || !("log" in value) || !("checkpointed" in value)) return false;
|
|
1087
|
+
return typeof value.busy === "number" && typeof value.log === "number" && typeof value.checkpointed === "number";
|
|
1088
|
+
}
|
|
1089
|
+
/** Narrow `PRAGMA wal_checkpoint` output without a cast; null when unreadable. */
|
|
1090
|
+
function readCheckpointResult(value) {
|
|
1091
|
+
if (isCheckpointRow(value)) return value;
|
|
1092
|
+
if (Array.isArray(value)) {
|
|
1093
|
+
const [first] = value;
|
|
1094
|
+
if (isCheckpointRow(first)) return first;
|
|
1095
|
+
}
|
|
1096
|
+
return null;
|
|
1097
|
+
}
|
|
1098
|
+
/**
|
|
1099
|
+
* Periodic, heavily gated `wal_checkpoint(TRUNCATE)`.
|
|
1100
|
+
*
|
|
1101
|
+
* The interesting behaviour is all refusal. A tick costs one `stat` in the
|
|
1102
|
+
* common case; it escalates to a lock-taking checkpoint only when the WAL is
|
|
1103
|
+
* over the high-water mark AND nothing has written to it for
|
|
1104
|
+
* {@link WAL_IDLE_QUIET_MS}, and even then the wait is bounded to
|
|
1105
|
+
* {@link WAL_CHECKPOINT_BUSY_TIMEOUT_MS}.
|
|
1106
|
+
*/
|
|
1107
|
+
var WalMaintenance = class {
|
|
1108
|
+
db;
|
|
1109
|
+
walPath;
|
|
1110
|
+
logger;
|
|
1111
|
+
highWaterBytes;
|
|
1112
|
+
idleQuietMs;
|
|
1113
|
+
intervalMs;
|
|
1114
|
+
busyTimeoutMs;
|
|
1115
|
+
timer = null;
|
|
1116
|
+
constructor(options) {
|
|
1117
|
+
this.db = options.db;
|
|
1118
|
+
this.walPath = `${options.dbPath}-wal`;
|
|
1119
|
+
this.logger = options.logger;
|
|
1120
|
+
this.highWaterBytes = options.highWaterBytes ?? 67108864;
|
|
1121
|
+
this.idleQuietMs = options.idleQuietMs ?? 3e4;
|
|
1122
|
+
this.intervalMs = options.intervalMs ?? 6e4;
|
|
1123
|
+
this.busyTimeoutMs = options.busyTimeoutMs ?? 250;
|
|
1124
|
+
}
|
|
1125
|
+
start() {
|
|
1126
|
+
if (this.timer !== null) return;
|
|
1127
|
+
const timer = setInterval(() => {
|
|
1128
|
+
this.runOnce();
|
|
1129
|
+
}, this.intervalMs);
|
|
1130
|
+
timer.unref?.();
|
|
1131
|
+
this.timer = timer;
|
|
1132
|
+
this.logger.info("WAL maintenance armed", { meta: {
|
|
1133
|
+
intervalMs: this.intervalMs,
|
|
1134
|
+
highWaterBytes: this.highWaterBytes,
|
|
1135
|
+
idleQuietMs: this.idleQuietMs
|
|
1136
|
+
} });
|
|
1137
|
+
}
|
|
1138
|
+
stop() {
|
|
1139
|
+
if (this.timer === null) return;
|
|
1140
|
+
clearInterval(this.timer);
|
|
1141
|
+
this.timer = null;
|
|
1142
|
+
}
|
|
1143
|
+
isRunning() {
|
|
1144
|
+
return this.timer !== null;
|
|
1145
|
+
}
|
|
1146
|
+
/** Evaluate the gate once and act. Never throws. */
|
|
1147
|
+
runOnce() {
|
|
1148
|
+
try {
|
|
1149
|
+
return this.evaluate();
|
|
1150
|
+
} catch (err) {
|
|
1151
|
+
const error = errMsg(err);
|
|
1152
|
+
this.logger.warn("WAL maintenance tick failed", { meta: {
|
|
1153
|
+
error,
|
|
1154
|
+
walPath: this.walPath
|
|
1155
|
+
} });
|
|
1156
|
+
return {
|
|
1157
|
+
kind: "failed",
|
|
1158
|
+
error
|
|
1159
|
+
};
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
evaluate() {
|
|
1163
|
+
const stat = this.statWal();
|
|
1164
|
+
if (stat === null) return { kind: "no-wal" };
|
|
1165
|
+
if (stat.size < this.highWaterBytes) return {
|
|
1166
|
+
kind: "below-high-water",
|
|
1167
|
+
walBytes: stat.size
|
|
1168
|
+
};
|
|
1169
|
+
const quietForMs = Date.now() - stat.mtimeMs;
|
|
1170
|
+
if (quietForMs < this.idleQuietMs) {
|
|
1171
|
+
const meta = {
|
|
1172
|
+
walBytes: stat.size,
|
|
1173
|
+
quietForMs,
|
|
1174
|
+
idleQuietMs: this.idleQuietMs
|
|
1175
|
+
};
|
|
1176
|
+
if (stat.size >= this.highWaterBytes * WAL_ALARM_MULTIPLE) this.logger.warn("WAL checkpoint deferred: file far over high-water and still being written", { meta });
|
|
1177
|
+
else this.logger.debug("WAL checkpoint deferred: database is not idle", { meta });
|
|
1178
|
+
return {
|
|
1179
|
+
kind: "deferred-not-idle",
|
|
1180
|
+
walBytes: stat.size,
|
|
1181
|
+
quietForMs
|
|
1182
|
+
};
|
|
1183
|
+
}
|
|
1184
|
+
return this.checkpoint(stat.size);
|
|
1185
|
+
}
|
|
1186
|
+
/**
|
|
1187
|
+
* Take the writer lock and fold the WAL back, with the wait bounded.
|
|
1188
|
+
*
|
|
1189
|
+
* The original `busy_timeout` is restored in a `finally`: this connection is
|
|
1190
|
+
* shared with the settings backend and the vector index, and leaving them on
|
|
1191
|
+
* a 250 ms timeout would turn an ordinary contended write into a spurious
|
|
1192
|
+
* SQLITE_BUSY.
|
|
1193
|
+
*/
|
|
1194
|
+
checkpoint(beforeBytes) {
|
|
1195
|
+
const previousTimeout = this.db.pragma("busy_timeout", { simple: true });
|
|
1196
|
+
this.db.pragma(`busy_timeout = ${this.busyTimeoutMs}`);
|
|
1197
|
+
let raw;
|
|
1198
|
+
try {
|
|
1199
|
+
raw = this.db.pragma("wal_checkpoint(TRUNCATE)");
|
|
1200
|
+
} finally {
|
|
1201
|
+
if (typeof previousTimeout === "number") this.db.pragma(`busy_timeout = ${previousTimeout}`);
|
|
1202
|
+
}
|
|
1203
|
+
const result = readCheckpointResult(raw);
|
|
1204
|
+
const afterBytes = this.statWal()?.size ?? 0;
|
|
1205
|
+
const outcome = {
|
|
1206
|
+
kind: "checkpointed",
|
|
1207
|
+
beforeBytes,
|
|
1208
|
+
afterBytes,
|
|
1209
|
+
busy: result !== null && result.busy !== 0,
|
|
1210
|
+
walFrames: result?.log ?? -1,
|
|
1211
|
+
checkpointedFrames: result?.checkpointed ?? -1
|
|
1212
|
+
};
|
|
1213
|
+
this.logger.info("WAL truncating checkpoint completed", { meta: {
|
|
1214
|
+
beforeBytes,
|
|
1215
|
+
afterBytes,
|
|
1216
|
+
reclaimedBytes: beforeBytes - afterBytes,
|
|
1217
|
+
busy: outcome.busy,
|
|
1218
|
+
walFrames: outcome.walFrames,
|
|
1219
|
+
checkpointedFrames: outcome.checkpointedFrames
|
|
1220
|
+
} });
|
|
1221
|
+
return outcome;
|
|
1222
|
+
}
|
|
1223
|
+
statWal() {
|
|
1224
|
+
try {
|
|
1225
|
+
const s = statSync(this.walPath);
|
|
1226
|
+
return {
|
|
1227
|
+
size: s.size,
|
|
1228
|
+
mtimeMs: s.mtimeMs
|
|
1229
|
+
};
|
|
1230
|
+
} catch {
|
|
1231
|
+
return null;
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
};
|
|
1235
|
+
//#endregion
|
|
954
1236
|
//#region src/builtins/sqlite-storage/sqlite-settings.addon.ts
|
|
955
1237
|
/**
|
|
956
1238
|
* SQLite Settings addon — the relational ENGINE behind the data door.
|
|
@@ -971,6 +1253,7 @@ function rebuild(row) {
|
|
|
971
1253
|
var SqliteSettingsAddon = class extends BaseAddon {
|
|
972
1254
|
backend = null;
|
|
973
1255
|
vectorIndex = null;
|
|
1256
|
+
walMaintenance = null;
|
|
974
1257
|
constructor() {
|
|
975
1258
|
super({});
|
|
976
1259
|
}
|
|
@@ -1014,6 +1297,16 @@ var SqliteSettingsAddon = class extends BaseAddon {
|
|
|
1014
1297
|
deleteByFilter: async (input) => ({ deleted: await vectorIndex.deleteByFilter(input.index, input.filter) }),
|
|
1015
1298
|
stats: async (input) => vectorIndex.stats(input.index)
|
|
1016
1299
|
};
|
|
1300
|
+
const db = this.backend.getDatabase();
|
|
1301
|
+
if (db !== null) {
|
|
1302
|
+
const maintenance = new WalMaintenance({
|
|
1303
|
+
db,
|
|
1304
|
+
dbPath,
|
|
1305
|
+
logger: this.ctx.logger.child("WalMaintenance")
|
|
1306
|
+
});
|
|
1307
|
+
maintenance.start();
|
|
1308
|
+
this.walMaintenance = maintenance;
|
|
1309
|
+
}
|
|
1017
1310
|
this.ctx.logger.info("Initialized successfully");
|
|
1018
1311
|
return [{
|
|
1019
1312
|
capability: dataStoreProviderCapability,
|
|
@@ -1024,6 +1317,8 @@ var SqliteSettingsAddon = class extends BaseAddon {
|
|
|
1024
1317
|
}];
|
|
1025
1318
|
}
|
|
1026
1319
|
async onShutdown() {
|
|
1320
|
+
this.walMaintenance?.stop();
|
|
1321
|
+
this.walMaintenance = null;
|
|
1027
1322
|
await this.backend?.shutdown();
|
|
1028
1323
|
}
|
|
1029
1324
|
getBackend() {
|
|
@@ -1061,4 +1356,4 @@ var SqliteSettingsAddon = class extends BaseAddon {
|
|
|
1061
1356
|
}
|
|
1062
1357
|
};
|
|
1063
1358
|
//#endregion
|
|
1064
|
-
export { SqliteSettingsAddon, SqliteSettingsAddon as default, SqliteSettingsBackend as t };
|
|
1359
|
+
export { SqliteSettingsAddon, SqliteSettingsAddon as default, WalMaintenance as a, SQLITE_MMAP_SIZE_BYTES as c, WAL_MAINTENANCE_INTERVAL_MS as i, SQLITE_PAGE_CACHE_KIB as l, WAL_HIGH_WATER_BYTES as n, SqliteSettingsBackend as o, WAL_IDLE_QUIET_MS as r, SQLITE_JOURNAL_SIZE_LIMIT_BYTES as s, WAL_CHECKPOINT_BUSY_TIMEOUT_MS as t };
|