@camstack/system 1.2.328 → 1.2.330
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/autotrack/index.js +1 -1
- package/dist/builtins/autotrack/index.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.d.ts +7 -0
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +294 -79
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +294 -79
- package/dist/builtins/camera-grid/index.js +1 -1
- package/dist/builtins/camera-grid/index.mjs +1 -1
- package/dist/builtins/composer/composer.addon.js +2 -2
- package/dist/builtins/composer/composer.addon.mjs +2 -2
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +2 -2
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +2 -2
- package/dist/builtins/device-manager/device-manager.addon.js +2 -2
- package/dist/builtins/device-manager/device-manager.addon.mjs +2 -2
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.d.ts +4 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +0 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +0 -0
- package/dist/builtins/sqlite-storage/sqlite-snapshot-protocol.d.ts +49 -0
- package/dist/builtins/sqlite-storage/sqlite-snapshot-worker.d.ts +33 -0
- package/dist/builtins/sqlite-storage/sqlite-snapshotter.d.ts +98 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
- package/dist/builtins/system-backup/system-backup.service.d.ts +74 -20
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{composition-sources-GkgXF6B9.js → composition-sources-2XxNEet3.js} +1 -1
- package/dist/{composition-sources-I8Rx7woY.mjs → composition-sources-Wj2kWund.mjs} +1 -1
- package/dist/{dist-BUejKKBm.js → dist-Bx4zxM9N.js} +534 -341
- package/dist/{dist-DWq17vC8.mjs → dist-Dp043AI0.mjs} +505 -342
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/{retired-settings-keys-DBPEzGop.js → retired-settings-keys-DR2WMe5d.js} +1 -1
- package/dist/{retired-settings-keys-BfHclEyd.mjs → retired-settings-keys-Dxf9vELs.mjs} +1 -1
- package/dist/sqlite-snapshot-protocol-2QaTouWV.mjs +34 -0
- package/dist/sqlite-snapshot-protocol-B9OndqZv.js +51 -0
- package/dist/sqlite-snapshot-worker.js +185 -0
- package/dist/sqlite-snapshot-worker.mjs +179 -0
- package/package.json +6 -3
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
2
|
const require_chunk = require("./chunk-Cek0wNdY.js");
|
|
3
|
-
const require_dist = require("./dist-
|
|
3
|
+
const require_dist = require("./dist-Bx4zxM9N.js");
|
|
4
4
|
const require_builtins_alerts_alerts_addon = require("./builtins/alerts/alerts.addon.js");
|
|
5
5
|
require("./builtins/alerts/index.js");
|
|
6
6
|
const require_builtins_composer_composer_addon = require("./builtins/composer/composer.addon.js");
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { a as __toCommonJS, i as __require, n as __esmMin, o as __toESM$1, r as __exportAll, t as __commonJSMin$1 } from "./chunk-CNf5ZN-e.mjs";
|
|
2
|
-
import {
|
|
2
|
+
import { $n as DEVICE_SETTINGS_CONTRIBUTION_METHODS, C as LOG_LEVEL_RANK, E as METHOD_ACCESS_MAP, H as addonSettingsCapability, Ht as lifecycleJobSchema, It as isObjectInput, Kt as logLevelAtMost, Nt as isArrayOutputSchema, Ot as extractNestedAddonId, Pt as isCollectionArrayMethod, Qn as DATAPLANE_SECRET_HEADER$1, Rt as isVoidInput, Sr as EventCategory$1, Xn as errMsg$1, _r as readinessKey, ar as asNumber, cr as emitDownForOwnedCaps, er as DEVICE_STATUS_METHOD, fn as resolveMethodAuth, gr as parseJsonUnknown, hn as scopesAllowDeviceCap, hr as parseJsonObject, ir as asJsonObject$1, j as RUNTIME_DEFAULTS, mn as scopesAllowAddon, nn as objectInputDeclaresRequiredDeviceId, nr as ReadinessTimeoutError, on as procedureAuthKey, or as asString$1, qt as looseSchema, sr as createEvent, t as ALL_CAPABILITY_DEFINITIONS, tn as objectInputDeclaresAddonId, tr as ReadinessRegistry, ur as expandCapMethods, vr as resolveCapMount, yr as scopeKey, zt as kebabToCamel } from "./dist-Dp043AI0.mjs";
|
|
3
3
|
import { AlertCenterAddon } from "./builtins/alerts/alerts.addon.mjs";
|
|
4
4
|
import "./builtins/alerts/index.mjs";
|
|
5
5
|
import { ComposerAddon } from "./builtins/composer/composer.addon.mjs";
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
//#region src/builtins/sqlite-storage/sqlite-snapshot-protocol.ts
|
|
2
|
+
/** Thrown inside the worker by a check that refuses the copy. */
|
|
3
|
+
var SnapshotRefusedError = class extends Error {
|
|
4
|
+
reason;
|
|
5
|
+
detail;
|
|
6
|
+
constructor(reason, detail) {
|
|
7
|
+
super(`database snapshot refused (${reason}): ${detail}`);
|
|
8
|
+
this.reason = reason;
|
|
9
|
+
this.detail = detail;
|
|
10
|
+
this.name = "SnapshotRefusedError";
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
function isSnapshotWorkerData(value) {
|
|
14
|
+
if (typeof value !== "object" || value === null) return false;
|
|
15
|
+
if (!("kind" in value) || value.kind !== "sqlite-snapshot") return false;
|
|
16
|
+
if (!("livePath" in value) || !("copyPath" in value) || !("pagesPerStep" in value)) return false;
|
|
17
|
+
return typeof value.livePath === "string" && typeof value.copyPath === "string" && typeof value.pagesPerStep === "number";
|
|
18
|
+
}
|
|
19
|
+
function isTableList(value) {
|
|
20
|
+
return Array.isArray(value) && value.every((t) => typeof t === "object" && t !== null && "name" in t && "rows" in t && typeof t.name === "string" && typeof t.rows === "number");
|
|
21
|
+
}
|
|
22
|
+
function isVerifiedCopy(value) {
|
|
23
|
+
if (typeof value !== "object" || value === null) return false;
|
|
24
|
+
if (!("takenAt" in value) || !("tables" in value) || !("pageCount" in value) || !("pageSize" in value) || !("copyMs" in value) || !("verifyMs" in value)) return false;
|
|
25
|
+
return typeof value.takenAt === "number" && isTableList(value.tables) && typeof value.pageCount === "number" && typeof value.pageSize === "number" && typeof value.copyMs === "number" && typeof value.verifyMs === "number";
|
|
26
|
+
}
|
|
27
|
+
function isSnapshotWorkerMessage(value) {
|
|
28
|
+
if (typeof value !== "object" || value === null || !("kind" in value)) return false;
|
|
29
|
+
if (value.kind === "done") return "copy" in value && isVerifiedCopy(value.copy);
|
|
30
|
+
if (value.kind === "refused") return "reason" in value && (value.reason === "integrity" || value.reason === "row-count" || value.reason === "failed" || value.reason === "no-space") && "detail" in value && typeof value.detail === "string";
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
//#endregion
|
|
34
|
+
export { isSnapshotWorkerData as n, isSnapshotWorkerMessage as r, SnapshotRefusedError as t };
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
//#region src/builtins/sqlite-storage/sqlite-snapshot-protocol.ts
|
|
2
|
+
/** Thrown inside the worker by a check that refuses the copy. */
|
|
3
|
+
var SnapshotRefusedError = class extends Error {
|
|
4
|
+
reason;
|
|
5
|
+
detail;
|
|
6
|
+
constructor(reason, detail) {
|
|
7
|
+
super(`database snapshot refused (${reason}): ${detail}`);
|
|
8
|
+
this.reason = reason;
|
|
9
|
+
this.detail = detail;
|
|
10
|
+
this.name = "SnapshotRefusedError";
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
function isSnapshotWorkerData(value) {
|
|
14
|
+
if (typeof value !== "object" || value === null) return false;
|
|
15
|
+
if (!("kind" in value) || value.kind !== "sqlite-snapshot") return false;
|
|
16
|
+
if (!("livePath" in value) || !("copyPath" in value) || !("pagesPerStep" in value)) return false;
|
|
17
|
+
return typeof value.livePath === "string" && typeof value.copyPath === "string" && typeof value.pagesPerStep === "number";
|
|
18
|
+
}
|
|
19
|
+
function isTableList(value) {
|
|
20
|
+
return Array.isArray(value) && value.every((t) => typeof t === "object" && t !== null && "name" in t && "rows" in t && typeof t.name === "string" && typeof t.rows === "number");
|
|
21
|
+
}
|
|
22
|
+
function isVerifiedCopy(value) {
|
|
23
|
+
if (typeof value !== "object" || value === null) return false;
|
|
24
|
+
if (!("takenAt" in value) || !("tables" in value) || !("pageCount" in value) || !("pageSize" in value) || !("copyMs" in value) || !("verifyMs" in value)) return false;
|
|
25
|
+
return typeof value.takenAt === "number" && isTableList(value.tables) && typeof value.pageCount === "number" && typeof value.pageSize === "number" && typeof value.copyMs === "number" && typeof value.verifyMs === "number";
|
|
26
|
+
}
|
|
27
|
+
function isSnapshotWorkerMessage(value) {
|
|
28
|
+
if (typeof value !== "object" || value === null || !("kind" in value)) return false;
|
|
29
|
+
if (value.kind === "done") return "copy" in value && isVerifiedCopy(value.copy);
|
|
30
|
+
if (value.kind === "refused") return "reason" in value && (value.reason === "integrity" || value.reason === "row-count" || value.reason === "failed" || value.reason === "no-space") && "detail" in value && typeof value.detail === "string";
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
//#endregion
|
|
34
|
+
Object.defineProperty(exports, "SnapshotRefusedError", {
|
|
35
|
+
enumerable: true,
|
|
36
|
+
get: function() {
|
|
37
|
+
return SnapshotRefusedError;
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
Object.defineProperty(exports, "isSnapshotWorkerData", {
|
|
41
|
+
enumerable: true,
|
|
42
|
+
get: function() {
|
|
43
|
+
return isSnapshotWorkerData;
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
Object.defineProperty(exports, "isSnapshotWorkerMessage", {
|
|
47
|
+
enumerable: true,
|
|
48
|
+
get: function() {
|
|
49
|
+
return isSnapshotWorkerMessage;
|
|
50
|
+
}
|
|
51
|
+
});
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
const require_chunk = require("./chunk-Cek0wNdY.js");
|
|
3
|
+
const require_sqlite_snapshot_protocol = require("./sqlite-snapshot-protocol-B9OndqZv.js");
|
|
4
|
+
let better_sqlite3 = require("better-sqlite3");
|
|
5
|
+
better_sqlite3 = require_chunk.__toESM(better_sqlite3);
|
|
6
|
+
let node_worker_threads = require("node:worker_threads");
|
|
7
|
+
//#region src/builtins/sqlite-storage/sqlite-snapshot-worker.ts
|
|
8
|
+
/**
|
|
9
|
+
* The snapshot worker — a thread of the process that OWNS a live SQLite
|
|
10
|
+
* database, whose one job is a consistent, verified copy of it (D693).
|
|
11
|
+
*
|
|
12
|
+
* - Its OWN read-only connection to the live file, in the owner's process: the
|
|
13
|
+
* same footing as the checkpoint worker (D452, D679). No other process opens
|
|
14
|
+
* the database, and no filesystem call here touches the `.db`, `-wal` or
|
|
15
|
+
* `-shm` — the copy is written by SQLite's online backup API.
|
|
16
|
+
* - ONE read transaction for the whole copy. `BEGIN`, then the row count of
|
|
17
|
+
* every b-tree table, then every backup step: they all read the same pinned
|
|
18
|
+
* WAL snapshot. Writes on the settings connection go on (a WAL reader blocks
|
|
19
|
+
* no writer) and do NOT restart the copy, because the backup never opens a
|
|
20
|
+
* read transaction of its own. Without the pin, every write between two
|
|
21
|
+
* steps would restart the copy from page 1 — forever, on a hub that writes
|
|
22
|
+
* every second.
|
|
23
|
+
* - The copy is checked before anyone may call it a snapshot: switched to
|
|
24
|
+
* `journal_mode = DELETE` (one self-contained file), `PRAGMA integrity_check`
|
|
25
|
+
* must answer exactly `ok`, and every table's row count must EQUAL the count
|
|
26
|
+
* read in the pinned transaction. The caller renames it into place only on
|
|
27
|
+
* `done`.
|
|
28
|
+
* - It runs on a thread because both halves are long and synchronous at the
|
|
29
|
+
* scale of the hub's ~800 MB file: on the settings thread they would stall
|
|
30
|
+
* every configuration read in the cluster (the D447 failure).
|
|
31
|
+
*
|
|
32
|
+
* A worker entry is a separate bundle (`dist/sqlite-snapshot-worker.js`, wired
|
|
33
|
+
* in `vite.config.ts` next to `wal-checkpoint-worker`). The functions are
|
|
34
|
+
* exported so a spec can drive them in-process; the thread body runs only when
|
|
35
|
+
* this module is loaded AS the snapshot thread.
|
|
36
|
+
*/
|
|
37
|
+
/** Every table with its own b-tree (a virtual table has none: `rootpage = 0`). */
|
|
38
|
+
var BTREE_TABLES_SQL = "SELECT name FROM sqlite_schema WHERE type = 'table' AND rootpage > 0 ORDER BY name";
|
|
39
|
+
function quoteIdent(name) {
|
|
40
|
+
return `"${name.replace(/"/g, "\"\"")}"`;
|
|
41
|
+
}
|
|
42
|
+
function readNames(db) {
|
|
43
|
+
const out = [];
|
|
44
|
+
for (const row of db.prepare(BTREE_TABLES_SQL).all()) if (typeof row === "object" && row !== null && "name" in row && typeof row.name === "string") out.push(row.name);
|
|
45
|
+
return out;
|
|
46
|
+
}
|
|
47
|
+
function countRows(db, table) {
|
|
48
|
+
const row = db.prepare(`SELECT count(*) AS n FROM ${quoteIdent(table)}`).get();
|
|
49
|
+
if (typeof row === "object" && row !== null && "n" in row && typeof row.n === "number") return row.n;
|
|
50
|
+
throw new Error(`count(*) of ${table} answered no number`);
|
|
51
|
+
}
|
|
52
|
+
function readTables(db) {
|
|
53
|
+
return readNames(db).map((name) => ({
|
|
54
|
+
name,
|
|
55
|
+
rows: countRows(db, name)
|
|
56
|
+
}));
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Copy `source` into `copyPath` from ONE read transaction. `onStep` observes
|
|
60
|
+
* each backup step (a spec interleaves writes there).
|
|
61
|
+
*/
|
|
62
|
+
async function copyPinned(source, copyPath, pagesPerStep, onStep) {
|
|
63
|
+
const started = Date.now();
|
|
64
|
+
source.exec("BEGIN");
|
|
65
|
+
try {
|
|
66
|
+
const tables = readTables(source);
|
|
67
|
+
const takenAt = Date.now();
|
|
68
|
+
await source.backup(copyPath, { progress: () => {
|
|
69
|
+
onStep?.();
|
|
70
|
+
return pagesPerStep;
|
|
71
|
+
} });
|
|
72
|
+
return {
|
|
73
|
+
takenAt,
|
|
74
|
+
tables,
|
|
75
|
+
copyMs: Date.now() - started
|
|
76
|
+
};
|
|
77
|
+
} finally {
|
|
78
|
+
source.exec("COMMIT");
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
function integrityAnswer(db) {
|
|
82
|
+
const out = [];
|
|
83
|
+
for (const row of db.prepare("PRAGMA integrity_check").all()) if (typeof row === "object" && row !== null && "integrity_check" in row) out.push(String(row.integrity_check));
|
|
84
|
+
return out;
|
|
85
|
+
}
|
|
86
|
+
function describeCountMismatch(expected, actual) {
|
|
87
|
+
const got = new Map(actual.map((t) => [t.name, t.rows]));
|
|
88
|
+
const want = new Map(expected.map((t) => [t.name, t.rows]));
|
|
89
|
+
const diffs = [];
|
|
90
|
+
for (const [name, rows] of want) {
|
|
91
|
+
const copyRows = got.get(name);
|
|
92
|
+
if (copyRows === void 0) diffs.push(`${name}: missing from the copy (source ${rows})`);
|
|
93
|
+
else if (copyRows !== rows) diffs.push(`${name}: copy ${copyRows} ≠ source ${rows}`);
|
|
94
|
+
}
|
|
95
|
+
for (const name of got.keys()) if (!want.has(name)) diffs.push(`${name}: in the copy, not in the source`);
|
|
96
|
+
return diffs.length === 0 ? null : diffs.join("; ");
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* The answer of `PRAGMA journal_mode = DELETE` must be `delete`. SQLite answers
|
|
100
|
+
* the mode it is IN, so a switch it could not make comes back as `wal` with no
|
|
101
|
+
* error — and a copy left in WAL needs sidecars that will not travel with it.
|
|
102
|
+
*/
|
|
103
|
+
function requireRollbackJournal(answer) {
|
|
104
|
+
if (answer !== "delete") throw new require_sqlite_snapshot_protocol.SnapshotRefusedError("failed", `the copy stayed in journal mode ${String(answer)} — it would need sidecars to be read`);
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Check a copy against what the pinned source held. Makes the copy one
|
|
108
|
+
* self-contained file first. Throws {@link SnapshotRefusedError}.
|
|
109
|
+
*/
|
|
110
|
+
function verifyCopy(copyPath, expected) {
|
|
111
|
+
const started = Date.now();
|
|
112
|
+
const db = new better_sqlite3.default(copyPath);
|
|
113
|
+
try {
|
|
114
|
+
requireRollbackJournal(db.pragma("journal_mode = DELETE", { simple: true }));
|
|
115
|
+
const answer = integrityAnswer(db);
|
|
116
|
+
if (answer.length !== 1 || answer[0] !== "ok") throw new require_sqlite_snapshot_protocol.SnapshotRefusedError("integrity", answer.slice(0, 5).join(" | "));
|
|
117
|
+
const mismatch = describeCountMismatch(expected, readTables(db));
|
|
118
|
+
if (mismatch !== null) throw new require_sqlite_snapshot_protocol.SnapshotRefusedError("row-count", mismatch);
|
|
119
|
+
const pageCount = db.pragma("page_count", { simple: true });
|
|
120
|
+
const pageSize = db.pragma("page_size", { simple: true });
|
|
121
|
+
if (typeof pageCount !== "number" || typeof pageSize !== "number") throw new Error("page_count / page_size answered no number");
|
|
122
|
+
return {
|
|
123
|
+
pageCount,
|
|
124
|
+
pageSize,
|
|
125
|
+
verifyMs: Date.now() - started
|
|
126
|
+
};
|
|
127
|
+
} finally {
|
|
128
|
+
db.close();
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/** The whole job: open the live database read-only, copy it pinned, check the copy. */
|
|
132
|
+
async function takeVerifiedCopy(data) {
|
|
133
|
+
const source = new better_sqlite3.default(data.livePath, { readonly: true });
|
|
134
|
+
let pinned;
|
|
135
|
+
try {
|
|
136
|
+
source.pragma("cache_size = -2048");
|
|
137
|
+
source.pragma("mmap_size = 0");
|
|
138
|
+
source.pragma("busy_timeout = 5000");
|
|
139
|
+
pinned = await copyPinned(source, data.copyPath, data.pagesPerStep);
|
|
140
|
+
} finally {
|
|
141
|
+
source.close();
|
|
142
|
+
}
|
|
143
|
+
const check = verifyCopy(data.copyPath, pinned.tables);
|
|
144
|
+
return {
|
|
145
|
+
takenAt: pinned.takenAt,
|
|
146
|
+
tables: pinned.tables,
|
|
147
|
+
pageCount: check.pageCount,
|
|
148
|
+
pageSize: check.pageSize,
|
|
149
|
+
copyMs: pinned.copyMs,
|
|
150
|
+
verifyMs: check.verifyMs
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
function toRefusal(err) {
|
|
154
|
+
if (err instanceof require_sqlite_snapshot_protocol.SnapshotRefusedError) return {
|
|
155
|
+
kind: "refused",
|
|
156
|
+
reason: err.reason,
|
|
157
|
+
detail: err.detail
|
|
158
|
+
};
|
|
159
|
+
return {
|
|
160
|
+
kind: "refused",
|
|
161
|
+
reason: "failed",
|
|
162
|
+
detail: err instanceof Error ? err.message : String(err)
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
async function run(data) {
|
|
166
|
+
const port = node_worker_threads.parentPort;
|
|
167
|
+
if (port === null) throw new Error("sqlite-snapshot-worker: no parent port");
|
|
168
|
+
let message;
|
|
169
|
+
try {
|
|
170
|
+
message = {
|
|
171
|
+
kind: "done",
|
|
172
|
+
copy: await takeVerifiedCopy(data)
|
|
173
|
+
};
|
|
174
|
+
} catch (err) {
|
|
175
|
+
message = toRefusal(err);
|
|
176
|
+
}
|
|
177
|
+
port.postMessage(message);
|
|
178
|
+
port.close();
|
|
179
|
+
}
|
|
180
|
+
if (!node_worker_threads.isMainThread && require_sqlite_snapshot_protocol.isSnapshotWorkerData(node_worker_threads.workerData)) run(node_worker_threads.workerData);
|
|
181
|
+
//#endregion
|
|
182
|
+
exports.copyPinned = copyPinned;
|
|
183
|
+
exports.requireRollbackJournal = requireRollbackJournal;
|
|
184
|
+
exports.takeVerifiedCopy = takeVerifiedCopy;
|
|
185
|
+
exports.verifyCopy = verifyCopy;
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { n as isSnapshotWorkerData, t as SnapshotRefusedError } from "./sqlite-snapshot-protocol-2QaTouWV.mjs";
|
|
2
|
+
import Database from "better-sqlite3";
|
|
3
|
+
import { isMainThread, parentPort, workerData } from "node:worker_threads";
|
|
4
|
+
//#region src/builtins/sqlite-storage/sqlite-snapshot-worker.ts
|
|
5
|
+
/**
|
|
6
|
+
* The snapshot worker — a thread of the process that OWNS a live SQLite
|
|
7
|
+
* database, whose one job is a consistent, verified copy of it (D693).
|
|
8
|
+
*
|
|
9
|
+
* - Its OWN read-only connection to the live file, in the owner's process: the
|
|
10
|
+
* same footing as the checkpoint worker (D452, D679). No other process opens
|
|
11
|
+
* the database, and no filesystem call here touches the `.db`, `-wal` or
|
|
12
|
+
* `-shm` — the copy is written by SQLite's online backup API.
|
|
13
|
+
* - ONE read transaction for the whole copy. `BEGIN`, then the row count of
|
|
14
|
+
* every b-tree table, then every backup step: they all read the same pinned
|
|
15
|
+
* WAL snapshot. Writes on the settings connection go on (a WAL reader blocks
|
|
16
|
+
* no writer) and do NOT restart the copy, because the backup never opens a
|
|
17
|
+
* read transaction of its own. Without the pin, every write between two
|
|
18
|
+
* steps would restart the copy from page 1 — forever, on a hub that writes
|
|
19
|
+
* every second.
|
|
20
|
+
* - The copy is checked before anyone may call it a snapshot: switched to
|
|
21
|
+
* `journal_mode = DELETE` (one self-contained file), `PRAGMA integrity_check`
|
|
22
|
+
* must answer exactly `ok`, and every table's row count must EQUAL the count
|
|
23
|
+
* read in the pinned transaction. The caller renames it into place only on
|
|
24
|
+
* `done`.
|
|
25
|
+
* - It runs on a thread because both halves are long and synchronous at the
|
|
26
|
+
* scale of the hub's ~800 MB file: on the settings thread they would stall
|
|
27
|
+
* every configuration read in the cluster (the D447 failure).
|
|
28
|
+
*
|
|
29
|
+
* A worker entry is a separate bundle (`dist/sqlite-snapshot-worker.js`, wired
|
|
30
|
+
* in `vite.config.ts` next to `wal-checkpoint-worker`). The functions are
|
|
31
|
+
* exported so a spec can drive them in-process; the thread body runs only when
|
|
32
|
+
* this module is loaded AS the snapshot thread.
|
|
33
|
+
*/
|
|
34
|
+
/** Every table with its own b-tree (a virtual table has none: `rootpage = 0`). */
|
|
35
|
+
var BTREE_TABLES_SQL = "SELECT name FROM sqlite_schema WHERE type = 'table' AND rootpage > 0 ORDER BY name";
|
|
36
|
+
function quoteIdent(name) {
|
|
37
|
+
return `"${name.replace(/"/g, "\"\"")}"`;
|
|
38
|
+
}
|
|
39
|
+
function readNames(db) {
|
|
40
|
+
const out = [];
|
|
41
|
+
for (const row of db.prepare(BTREE_TABLES_SQL).all()) if (typeof row === "object" && row !== null && "name" in row && typeof row.name === "string") out.push(row.name);
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
function countRows(db, table) {
|
|
45
|
+
const row = db.prepare(`SELECT count(*) AS n FROM ${quoteIdent(table)}`).get();
|
|
46
|
+
if (typeof row === "object" && row !== null && "n" in row && typeof row.n === "number") return row.n;
|
|
47
|
+
throw new Error(`count(*) of ${table} answered no number`);
|
|
48
|
+
}
|
|
49
|
+
function readTables(db) {
|
|
50
|
+
return readNames(db).map((name) => ({
|
|
51
|
+
name,
|
|
52
|
+
rows: countRows(db, name)
|
|
53
|
+
}));
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Copy `source` into `copyPath` from ONE read transaction. `onStep` observes
|
|
57
|
+
* each backup step (a spec interleaves writes there).
|
|
58
|
+
*/
|
|
59
|
+
async function copyPinned(source, copyPath, pagesPerStep, onStep) {
|
|
60
|
+
const started = Date.now();
|
|
61
|
+
source.exec("BEGIN");
|
|
62
|
+
try {
|
|
63
|
+
const tables = readTables(source);
|
|
64
|
+
const takenAt = Date.now();
|
|
65
|
+
await source.backup(copyPath, { progress: () => {
|
|
66
|
+
onStep?.();
|
|
67
|
+
return pagesPerStep;
|
|
68
|
+
} });
|
|
69
|
+
return {
|
|
70
|
+
takenAt,
|
|
71
|
+
tables,
|
|
72
|
+
copyMs: Date.now() - started
|
|
73
|
+
};
|
|
74
|
+
} finally {
|
|
75
|
+
source.exec("COMMIT");
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
function integrityAnswer(db) {
|
|
79
|
+
const out = [];
|
|
80
|
+
for (const row of db.prepare("PRAGMA integrity_check").all()) if (typeof row === "object" && row !== null && "integrity_check" in row) out.push(String(row.integrity_check));
|
|
81
|
+
return out;
|
|
82
|
+
}
|
|
83
|
+
function describeCountMismatch(expected, actual) {
|
|
84
|
+
const got = new Map(actual.map((t) => [t.name, t.rows]));
|
|
85
|
+
const want = new Map(expected.map((t) => [t.name, t.rows]));
|
|
86
|
+
const diffs = [];
|
|
87
|
+
for (const [name, rows] of want) {
|
|
88
|
+
const copyRows = got.get(name);
|
|
89
|
+
if (copyRows === void 0) diffs.push(`${name}: missing from the copy (source ${rows})`);
|
|
90
|
+
else if (copyRows !== rows) diffs.push(`${name}: copy ${copyRows} ≠ source ${rows}`);
|
|
91
|
+
}
|
|
92
|
+
for (const name of got.keys()) if (!want.has(name)) diffs.push(`${name}: in the copy, not in the source`);
|
|
93
|
+
return diffs.length === 0 ? null : diffs.join("; ");
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* The answer of `PRAGMA journal_mode = DELETE` must be `delete`. SQLite answers
|
|
97
|
+
* the mode it is IN, so a switch it could not make comes back as `wal` with no
|
|
98
|
+
* error — and a copy left in WAL needs sidecars that will not travel with it.
|
|
99
|
+
*/
|
|
100
|
+
function requireRollbackJournal(answer) {
|
|
101
|
+
if (answer !== "delete") throw new SnapshotRefusedError("failed", `the copy stayed in journal mode ${String(answer)} — it would need sidecars to be read`);
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Check a copy against what the pinned source held. Makes the copy one
|
|
105
|
+
* self-contained file first. Throws {@link SnapshotRefusedError}.
|
|
106
|
+
*/
|
|
107
|
+
function verifyCopy(copyPath, expected) {
|
|
108
|
+
const started = Date.now();
|
|
109
|
+
const db = new Database(copyPath);
|
|
110
|
+
try {
|
|
111
|
+
requireRollbackJournal(db.pragma("journal_mode = DELETE", { simple: true }));
|
|
112
|
+
const answer = integrityAnswer(db);
|
|
113
|
+
if (answer.length !== 1 || answer[0] !== "ok") throw new SnapshotRefusedError("integrity", answer.slice(0, 5).join(" | "));
|
|
114
|
+
const mismatch = describeCountMismatch(expected, readTables(db));
|
|
115
|
+
if (mismatch !== null) throw new SnapshotRefusedError("row-count", mismatch);
|
|
116
|
+
const pageCount = db.pragma("page_count", { simple: true });
|
|
117
|
+
const pageSize = db.pragma("page_size", { simple: true });
|
|
118
|
+
if (typeof pageCount !== "number" || typeof pageSize !== "number") throw new Error("page_count / page_size answered no number");
|
|
119
|
+
return {
|
|
120
|
+
pageCount,
|
|
121
|
+
pageSize,
|
|
122
|
+
verifyMs: Date.now() - started
|
|
123
|
+
};
|
|
124
|
+
} finally {
|
|
125
|
+
db.close();
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/** The whole job: open the live database read-only, copy it pinned, check the copy. */
|
|
129
|
+
async function takeVerifiedCopy(data) {
|
|
130
|
+
const source = new Database(data.livePath, { readonly: true });
|
|
131
|
+
let pinned;
|
|
132
|
+
try {
|
|
133
|
+
source.pragma("cache_size = -2048");
|
|
134
|
+
source.pragma("mmap_size = 0");
|
|
135
|
+
source.pragma("busy_timeout = 5000");
|
|
136
|
+
pinned = await copyPinned(source, data.copyPath, data.pagesPerStep);
|
|
137
|
+
} finally {
|
|
138
|
+
source.close();
|
|
139
|
+
}
|
|
140
|
+
const check = verifyCopy(data.copyPath, pinned.tables);
|
|
141
|
+
return {
|
|
142
|
+
takenAt: pinned.takenAt,
|
|
143
|
+
tables: pinned.tables,
|
|
144
|
+
pageCount: check.pageCount,
|
|
145
|
+
pageSize: check.pageSize,
|
|
146
|
+
copyMs: pinned.copyMs,
|
|
147
|
+
verifyMs: check.verifyMs
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
function toRefusal(err) {
|
|
151
|
+
if (err instanceof SnapshotRefusedError) return {
|
|
152
|
+
kind: "refused",
|
|
153
|
+
reason: err.reason,
|
|
154
|
+
detail: err.detail
|
|
155
|
+
};
|
|
156
|
+
return {
|
|
157
|
+
kind: "refused",
|
|
158
|
+
reason: "failed",
|
|
159
|
+
detail: err instanceof Error ? err.message : String(err)
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
async function run(data) {
|
|
163
|
+
const port = parentPort;
|
|
164
|
+
if (port === null) throw new Error("sqlite-snapshot-worker: no parent port");
|
|
165
|
+
let message;
|
|
166
|
+
try {
|
|
167
|
+
message = {
|
|
168
|
+
kind: "done",
|
|
169
|
+
copy: await takeVerifiedCopy(data)
|
|
170
|
+
};
|
|
171
|
+
} catch (err) {
|
|
172
|
+
message = toRefusal(err);
|
|
173
|
+
}
|
|
174
|
+
port.postMessage(message);
|
|
175
|
+
port.close();
|
|
176
|
+
}
|
|
177
|
+
if (!isMainThread && isSnapshotWorkerData(workerData)) run(workerData);
|
|
178
|
+
//#endregion
|
|
179
|
+
export { copyPinned, requireRollbackJournal, takeVerifiedCopy, verifyCopy };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@camstack/system",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.330",
|
|
4
4
|
"description": "Core addon for CamStack — builtins, pipeline, process management, auth, logging, events",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"camstack",
|
|
@@ -53,7 +53,7 @@
|
|
|
53
53
|
"category": "system",
|
|
54
54
|
"name": "SQLite Settings",
|
|
55
55
|
"entry": "./dist/builtins/sqlite-storage/sqlite-settings.addon.js",
|
|
56
|
-
"description": "The relational ENGINE behind the data door (SQLite WAL). Registers `data-store-provider`, an internal collection cap; the public `settings-store` door is owned by `storage-orchestrator`, which dispatches here. Callers never reach this addon directly — that is what makes the engine replaceable. It ALSO registers `vector-store` (sqlite-vec) from the same database file and the same open handle. Runs in its OWN process (`execution.isolate`) — `better-sqlite3` `Statement.run` does not belong on the event loop that serves the API ([D181]).",
|
|
56
|
+
"description": "The relational ENGINE behind the data door (SQLite WAL). Registers `data-store-provider`, an internal collection cap; the public `settings-store` door is owned by `storage-orchestrator`, which dispatches here. Callers never reach this addon directly — that is what makes the engine replaceable. It ALSO registers `vector-store` (sqlite-vec) from the same database file and the same open handle, and `database-snapshot`: the verified copy of its database that the nightly backup archives in place of the live files, written by this runner on a thread over its own connection ([D693]). Runs in its OWN process (`execution.isolate`) — `better-sqlite3` `Statement.run` does not belong on the event loop that serves the API ([D181]).",
|
|
57
57
|
"execution": {
|
|
58
58
|
"isolate": true,
|
|
59
59
|
"heapProfile": "heavy",
|
|
@@ -65,6 +65,9 @@
|
|
|
65
65
|
},
|
|
66
66
|
{
|
|
67
67
|
"name": "vector-store"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"name": "database-snapshot"
|
|
68
71
|
}
|
|
69
72
|
]
|
|
70
73
|
},
|
|
@@ -408,7 +411,7 @@
|
|
|
408
411
|
"copy:hook": "node -e \"const fs=require('fs'),p=require('path');const dst=p.join('dist');fs.mkdirSync(dst,{recursive:true});fs.copyFileSync(p.join('src','kernel','moleculer','framework-resolver-hook.mjs'),p.join(dst,'framework-resolver-hook.mjs'));console.log('copied framework-resolver-hook.mjs ->',dst)\"",
|
|
409
412
|
"build": "vite build && npm run copy:hook",
|
|
410
413
|
"dev": "tsup --watch",
|
|
411
|
-
"typecheck": "tsc --noEmit",
|
|
414
|
+
"typecheck": "tsc --noEmit -p tsconfig.spec.json",
|
|
412
415
|
"test": "vitest run",
|
|
413
416
|
"test:watch": "vitest",
|
|
414
417
|
"publish:npm": "npm publish --access public"
|