@camstack/system 1.2.47 → 1.2.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.js +1 -1
- package/dist/builtins/addon-pages-aggregator/addon-pages-aggregator.addon.mjs +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.js +1 -1
- package/dist/builtins/addon-widgets-aggregator/addon-widgets-aggregator.addon.mjs +1 -1
- package/dist/builtins/alerts/alerts.addon.js +1 -1
- package/dist/builtins/alerts/alerts.addon.mjs +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.js +1 -1
- package/dist/builtins/backup-orchestrator/backup-orchestrator.addon.mjs +1 -1
- package/dist/builtins/console-logging/index.js +1 -1
- package/dist/builtins/console-logging/index.mjs +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.js +1 -1
- package/dist/builtins/core-blocks/core-blocks.addon.mjs +1 -1
- package/dist/builtins/device-manager/device-manager.addon.js +1 -1
- package/dist/builtins/device-manager/device-manager.addon.mjs +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.js +1 -1
- package/dist/builtins/doorbell/virtual-doorbell.addon.mjs +1 -1
- package/dist/builtins/hub-forwarder/index.js +1 -1
- package/dist/builtins/hub-forwarder/index.mjs +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.js +1 -1
- package/dist/builtins/liveness-monitor/liveness-monitor.addon.mjs +1 -1
- package/dist/builtins/local-auth/local-auth.addon.js +1 -1
- package/dist/builtins/local-auth/local-auth.addon.mjs +1 -1
- package/dist/builtins/local-network/local-network.addon.js +1 -1
- package/dist/builtins/local-network/local-network.addon.mjs +1 -1
- package/dist/builtins/loki-logging/index.js +1 -1
- package/dist/builtins/loki-logging/index.mjs +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.js +1 -1
- package/dist/builtins/native-metrics/native-metrics.addon.mjs +1 -1
- package/dist/builtins/platform-probe/index.js +1 -1
- package/dist/builtins/platform-probe/index.mjs +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.js +1 -1
- package/dist/builtins/remote-access-orchestrator/remote-access-orchestrator.addon.mjs +1 -1
- package/dist/builtins/snapshot/index.js +1 -1
- package/dist/builtins/snapshot/index.mjs +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage-provider.d.ts +1 -1
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.js +64 -64
- package/dist/builtins/sqlite-storage/filesystem-storage.addon.mjs +64 -64
- package/dist/builtins/sqlite-storage/index.d.ts +5 -6
- package/dist/builtins/sqlite-storage/index.js +20 -20
- package/dist/builtins/sqlite-storage/index.mjs +20 -20
- package/dist/builtins/sqlite-storage/integration-registry.d.ts +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings-backend.d.ts +1 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.d.ts +10 -0
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.js +513 -1
- package/dist/builtins/sqlite-storage/sqlite-settings.addon.mjs +513 -1
- package/dist/builtins/sqlite-storage/vector-index.d.ts +172 -0
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.js +1 -1
- package/dist/builtins/storage-orchestrator/storage-orchestrator.addon.mjs +1 -1
- package/dist/builtins/system-config/system-config.addon.js +1 -1
- package/dist/builtins/system-config/system-config.addon.mjs +1 -1
- package/dist/builtins/winston-logging/index.js +1 -1
- package/dist/builtins/winston-logging/index.mjs +1 -1
- package/dist/{dist-BRBfh1_9.mjs → dist-C7rPTQLR.mjs} +33 -1
- package/dist/{dist-CKjrL1qP.js → dist-CH0DyAwh.js} +56 -0
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/package.json +1 -1
|
@@ -5,26 +5,6 @@ Object.defineProperties(exports, {
|
|
|
5
5
|
require("../../chunk-Cek0wNdY.js");
|
|
6
6
|
const require_builtins_sqlite_storage_filesystem_storage_addon = require("./filesystem-storage.addon.js");
|
|
7
7
|
const require_builtins_sqlite_storage_sqlite_settings_addon = require("./sqlite-settings.addon.js");
|
|
8
|
-
//#region src/builtins/sqlite-storage/device-store.ts
|
|
9
|
-
var DeviceStore = class {
|
|
10
|
-
db;
|
|
11
|
-
constructor(db) {
|
|
12
|
-
this.db = db;
|
|
13
|
-
}
|
|
14
|
-
insert(addonId, device) {
|
|
15
|
-
this.db.prepare(`INSERT INTO devices (addon_id, stable_id, type, name, parent_stable_id) VALUES (?, ?, ?, ?, ?)`).run(addonId, device.stableId, device.type, device.name, device.parentStableId);
|
|
16
|
-
}
|
|
17
|
-
listByAddon(addonId) {
|
|
18
|
-
return this.db.prepare(`SELECT stable_id as stableId, type, name, parent_stable_id as parentStableId, enabled FROM devices WHERE addon_id = ?`).all(addonId);
|
|
19
|
-
}
|
|
20
|
-
listChildren(addonId, parentStableId) {
|
|
21
|
-
return this.db.prepare(`SELECT stable_id as stableId, type, name, parent_stable_id as parentStableId, enabled FROM devices WHERE addon_id = ? AND parent_stable_id = ?`).all(addonId, parentStableId);
|
|
22
|
-
}
|
|
23
|
-
remove(addonId, stableId) {
|
|
24
|
-
this.db.prepare(`DELETE FROM devices WHERE addon_id = ? AND stable_id = ?`).run(addonId, stableId);
|
|
25
|
-
}
|
|
26
|
-
};
|
|
27
|
-
//#endregion
|
|
28
8
|
//#region src/builtins/sqlite-storage/config-store.ts
|
|
29
9
|
var ConfigStore = class {
|
|
30
10
|
db;
|
|
@@ -51,6 +31,26 @@ var ConfigStore = class {
|
|
|
51
31
|
}
|
|
52
32
|
};
|
|
53
33
|
//#endregion
|
|
34
|
+
//#region src/builtins/sqlite-storage/device-store.ts
|
|
35
|
+
var DeviceStore = class {
|
|
36
|
+
db;
|
|
37
|
+
constructor(db) {
|
|
38
|
+
this.db = db;
|
|
39
|
+
}
|
|
40
|
+
insert(addonId, device) {
|
|
41
|
+
this.db.prepare(`INSERT INTO devices (addon_id, stable_id, type, name, parent_stable_id) VALUES (?, ?, ?, ?, ?)`).run(addonId, device.stableId, device.type, device.name, device.parentStableId);
|
|
42
|
+
}
|
|
43
|
+
listByAddon(addonId) {
|
|
44
|
+
return this.db.prepare(`SELECT stable_id as stableId, type, name, parent_stable_id as parentStableId, enabled FROM devices WHERE addon_id = ?`).all(addonId);
|
|
45
|
+
}
|
|
46
|
+
listChildren(addonId, parentStableId) {
|
|
47
|
+
return this.db.prepare(`SELECT stable_id as stableId, type, name, parent_stable_id as parentStableId, enabled FROM devices WHERE addon_id = ? AND parent_stable_id = ?`).all(addonId, parentStableId);
|
|
48
|
+
}
|
|
49
|
+
remove(addonId, stableId) {
|
|
50
|
+
this.db.prepare(`DELETE FROM devices WHERE addon_id = ? AND stable_id = ?`).run(addonId, stableId);
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
//#endregion
|
|
54
54
|
exports.ConfigStore = ConfigStore;
|
|
55
55
|
exports.ConfigStore$1 = ConfigStore;
|
|
56
56
|
exports.DeviceStore = DeviceStore;
|
|
@@ -1,25 +1,5 @@
|
|
|
1
1
|
import { FilesystemStorageAddon, t as FilesystemStorageProvider } from "./filesystem-storage.addon.mjs";
|
|
2
2
|
import { SqliteSettingsAddon, t as SqliteSettingsBackend } from "./sqlite-settings.addon.mjs";
|
|
3
|
-
//#region src/builtins/sqlite-storage/device-store.ts
|
|
4
|
-
var DeviceStore = class {
|
|
5
|
-
db;
|
|
6
|
-
constructor(db) {
|
|
7
|
-
this.db = db;
|
|
8
|
-
}
|
|
9
|
-
insert(addonId, device) {
|
|
10
|
-
this.db.prepare(`INSERT INTO devices (addon_id, stable_id, type, name, parent_stable_id) VALUES (?, ?, ?, ?, ?)`).run(addonId, device.stableId, device.type, device.name, device.parentStableId);
|
|
11
|
-
}
|
|
12
|
-
listByAddon(addonId) {
|
|
13
|
-
return this.db.prepare(`SELECT stable_id as stableId, type, name, parent_stable_id as parentStableId, enabled FROM devices WHERE addon_id = ?`).all(addonId);
|
|
14
|
-
}
|
|
15
|
-
listChildren(addonId, parentStableId) {
|
|
16
|
-
return this.db.prepare(`SELECT stable_id as stableId, type, name, parent_stable_id as parentStableId, enabled FROM devices WHERE addon_id = ? AND parent_stable_id = ?`).all(addonId, parentStableId);
|
|
17
|
-
}
|
|
18
|
-
remove(addonId, stableId) {
|
|
19
|
-
this.db.prepare(`DELETE FROM devices WHERE addon_id = ? AND stable_id = ?`).run(addonId, stableId);
|
|
20
|
-
}
|
|
21
|
-
};
|
|
22
|
-
//#endregion
|
|
23
3
|
//#region src/builtins/sqlite-storage/config-store.ts
|
|
24
4
|
var ConfigStore = class {
|
|
25
5
|
db;
|
|
@@ -46,4 +26,24 @@ var ConfigStore = class {
|
|
|
46
26
|
}
|
|
47
27
|
};
|
|
48
28
|
//#endregion
|
|
29
|
+
//#region src/builtins/sqlite-storage/device-store.ts
|
|
30
|
+
var DeviceStore = class {
|
|
31
|
+
db;
|
|
32
|
+
constructor(db) {
|
|
33
|
+
this.db = db;
|
|
34
|
+
}
|
|
35
|
+
insert(addonId, device) {
|
|
36
|
+
this.db.prepare(`INSERT INTO devices (addon_id, stable_id, type, name, parent_stable_id) VALUES (?, ?, ?, ?, ?)`).run(addonId, device.stableId, device.type, device.name, device.parentStableId);
|
|
37
|
+
}
|
|
38
|
+
listByAddon(addonId) {
|
|
39
|
+
return this.db.prepare(`SELECT stable_id as stableId, type, name, parent_stable_id as parentStableId, enabled FROM devices WHERE addon_id = ?`).all(addonId);
|
|
40
|
+
}
|
|
41
|
+
listChildren(addonId, parentStableId) {
|
|
42
|
+
return this.db.prepare(`SELECT stable_id as stableId, type, name, parent_stable_id as parentStableId, enabled FROM devices WHERE addon_id = ? AND parent_stable_id = ?`).all(addonId, parentStableId);
|
|
43
|
+
}
|
|
44
|
+
remove(addonId, stableId) {
|
|
45
|
+
this.db.prepare(`DELETE FROM devices WHERE addon_id = ? AND stable_id = ?`).run(addonId, stableId);
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
//#endregion
|
|
49
49
|
export { ConfigStore, DeviceStore, FilesystemStorageAddon, FilesystemStorageAddon as default, FilesystemStorageProvider, SqliteSettingsAddon, SqliteSettingsBackend };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { CreateDeviceInput, CreateIntegrationInput, IIntegrationRegistry, Integration, ISettingsStoreProvider, PersistedDevice } from '@camstack/types';
|
|
2
2
|
export declare class IntegrationRegistry implements IIntegrationRegistry {
|
|
3
3
|
private readonly store;
|
|
4
4
|
constructor(backend: ISettingsStoreProvider);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { CollectionColumn, CollectionIndex, DataStoreEngineInfo, HistogramBucket, ISettingsBackend, SettingsCountInput, SettingsDeleteInput, SettingsGetInput, SettingsHistogramInput, SettingsInsertInput, SettingsIsEmptyInput, SettingsQueryInput, SettingsRecord, SettingsSetInput, SettingsUpdateInput } from '@camstack/types';
|
|
1
2
|
import { default as Database } from 'better-sqlite3';
|
|
2
|
-
import { DataStoreEngineInfo, ISettingsBackend, SettingsRecord, SettingsGetInput, SettingsSetInput, SettingsQueryInput, SettingsInsertInput, SettingsUpdateInput, SettingsDeleteInput, SettingsCountInput, SettingsIsEmptyInput, SettingsHistogramInput, HistogramBucket, CollectionColumn, CollectionIndex } from '@camstack/types';
|
|
3
3
|
import { MutationFilterInput } from './filter-compiler.js';
|
|
4
4
|
/** Input for {@link SqliteSettingsBackend.deleteWhere}. */
|
|
5
5
|
interface SettingsDeleteWhereInput {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ProviderRegistration, BaseAddon } from '@camstack/types';
|
|
2
2
|
import { SqliteSettingsBackend } from './sqlite-settings-backend.js';
|
|
3
|
+
import { SqliteVectorIndex } from './vector-index.js';
|
|
3
4
|
/**
|
|
4
5
|
* SQLite Settings addon — the relational ENGINE behind the data door.
|
|
5
6
|
*
|
|
@@ -9,12 +10,21 @@ import { SqliteSettingsBackend } from './sqlite-settings-backend.js';
|
|
|
9
10
|
* ([D44](../../../../../docs/decisions/adr-0044.md)). Callers reach this
|
|
10
11
|
* engine only through the door — which is the point, because it is what
|
|
11
12
|
* makes the engine replaceable.
|
|
13
|
+
*
|
|
14
|
+
* It ALSO serves `vector-store` (singleton, internal), from the same database
|
|
15
|
+
* file and the same open handle. That capability first shipped as its own addon
|
|
16
|
+
* and the separation cost two round trips per write — caller → door → engine —
|
|
17
|
+
* plus a runner process, to reach a database this addon already has open. The
|
|
18
|
+
* capability is what keeps the backend swappable; the package never was.
|
|
12
19
|
*/
|
|
13
20
|
export declare class SqliteSettingsAddon extends BaseAddon {
|
|
14
21
|
private backend;
|
|
22
|
+
private vectorIndex;
|
|
15
23
|
constructor();
|
|
16
24
|
protected onInitialize(): Promise<ProviderRegistration[]>;
|
|
17
25
|
protected onShutdown(): Promise<void>;
|
|
18
26
|
getBackend(): SqliteSettingsBackend | null;
|
|
27
|
+
/** Exposed for tests; null before `onInitialize`. */
|
|
28
|
+
getVectorIndex(): SqliteVectorIndex | null;
|
|
19
29
|
}
|
|
20
30
|
export default SqliteSettingsAddon;
|
|
@@ -3,7 +3,7 @@ Object.defineProperties(exports, {
|
|
|
3
3
|
[Symbol.toStringTag]: { value: "Module" }
|
|
4
4
|
});
|
|
5
5
|
const require_chunk = require("../../chunk-Cek0wNdY.js");
|
|
6
|
-
const require_dist = require("../../dist-
|
|
6
|
+
const require_dist = require("../../dist-CH0DyAwh.js");
|
|
7
7
|
let node_crypto = require("node:crypto");
|
|
8
8
|
let better_sqlite3 = require("better-sqlite3");
|
|
9
9
|
better_sqlite3 = require_chunk.__toESM(better_sqlite3);
|
|
@@ -647,6 +647,481 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
|
|
|
647
647
|
}
|
|
648
648
|
};
|
|
649
649
|
//#endregion
|
|
650
|
+
//#region src/builtins/sqlite-storage/vector-index.ts
|
|
651
|
+
/**
|
|
652
|
+
* SQLite-backed vector index — compact rows, exhaustive cosine ranking.
|
|
653
|
+
*
|
|
654
|
+
* Lives INSIDE the sqlite-settings builtin, against `SqliteSettingsBackend`
|
|
655
|
+
* directly, and that placement is the point. It first shipped as its own addon
|
|
656
|
+
* and that cost two round trips per write — caller → the `settings-store` door
|
|
657
|
+
* → the engine ([D44](../../../../../docs/decisions/adr-0044.md)) — plus a
|
|
658
|
+
* whole runner process, to reach the very database this addon already holds
|
|
659
|
+
* open. Here an upsert is a function call.
|
|
660
|
+
*
|
|
661
|
+
* Replaceability is unaffected: it is the CAPABILITY that makes the backend
|
|
662
|
+
* swappable, not the package. An ANN provider registers `vector-store` from
|
|
663
|
+
* wherever it likes and `setActiveSingleton` picks it.
|
|
664
|
+
*
|
|
665
|
+
* This is the DEFAULT `vector-store` provider, not the final one. It exists to
|
|
666
|
+
* make a row cheap; replacing exhaustive scan with a real ANN index is a
|
|
667
|
+
* different provider registered against the same capability.
|
|
668
|
+
*
|
|
669
|
+
* What it changes versus the per-store JSON columns it replaces:
|
|
670
|
+
*
|
|
671
|
+
* - the vector is a base64 Float32 TEXT column (~2.7 KB for 512 dims) instead
|
|
672
|
+
* of a JSON array of 512 numbers (~11.7 KB measured on the live hub), and
|
|
673
|
+
* decoding is a `readFloatLE` loop instead of `JSON.parse` of 512 tokens;
|
|
674
|
+
* - filterable metadata is promoted to real columns, so `deviceId` /
|
|
675
|
+
* `timestamp` / `className` prefilters are pushed into SQL and indexed
|
|
676
|
+
* rather than applied after loading everything;
|
|
677
|
+
* - a truncated scan is REPORTED (`truncated`), because a search that silently
|
|
678
|
+
* considered a subset reads as "nothing else matched".
|
|
679
|
+
*
|
|
680
|
+
* Metadata that is not one of the promoted columns rides in a JSON blob and is
|
|
681
|
+
* returned verbatim; it cannot be filtered on. That is deliberate — a filter a
|
|
682
|
+
* backend cannot push down is a filter that gets forgotten.
|
|
683
|
+
*/
|
|
684
|
+
/** Metadata keys promoted to real, indexable columns. */
|
|
685
|
+
var PROMOTED_COLUMNS = [
|
|
686
|
+
"deviceId",
|
|
687
|
+
"timestamp",
|
|
688
|
+
"className",
|
|
689
|
+
"modelId"
|
|
690
|
+
];
|
|
691
|
+
/** Ceiling on rows compared in one query, so a pathological index cannot wedge the runner. */
|
|
692
|
+
var MAX_SCAN_ROWS = 2e5;
|
|
693
|
+
var collectionFor = (index) => `vector-store:${index}`;
|
|
694
|
+
/**
|
|
695
|
+
* Where index declarations live.
|
|
696
|
+
*
|
|
697
|
+
* They MUST be durable. The rows survive a restart in SQLite but the `dim` and
|
|
698
|
+
* `metric` that make them queryable were only ever in memory, so after a hub
|
|
699
|
+
* restart `query` and `stats` both threw "index was never declared" until some
|
|
700
|
+
* caller happened to write — which meant semantic search was dead from boot
|
|
701
|
+
* until the next track closed. Callers declare lazily (the dimension comes from
|
|
702
|
+
* the active encoder, unknown at construction), so nothing re-declared on boot.
|
|
703
|
+
*/
|
|
704
|
+
var INDEX_REGISTRY_COLLECTION = "vector-store:__indexes";
|
|
705
|
+
var SqliteVectorIndex = class {
|
|
706
|
+
store;
|
|
707
|
+
logger;
|
|
708
|
+
specs = /* @__PURE__ */ new Map();
|
|
709
|
+
constructor(deps) {
|
|
710
|
+
this.store = deps.store;
|
|
711
|
+
this.logger = deps.logger;
|
|
712
|
+
}
|
|
713
|
+
/**
|
|
714
|
+
* Load persisted index declarations. Call once at startup, BEFORE serving:
|
|
715
|
+
* without it every method on an existing index throws until a write
|
|
716
|
+
* re-declares it.
|
|
717
|
+
*/
|
|
718
|
+
async loadIndexRegistry() {
|
|
719
|
+
try {
|
|
720
|
+
await this.store.declareCollection({
|
|
721
|
+
collection: INDEX_REGISTRY_COLLECTION,
|
|
722
|
+
columns: [
|
|
723
|
+
{
|
|
724
|
+
name: "id",
|
|
725
|
+
type: "TEXT",
|
|
726
|
+
primaryKey: true,
|
|
727
|
+
notNull: true
|
|
728
|
+
},
|
|
729
|
+
{
|
|
730
|
+
name: "dim",
|
|
731
|
+
type: "INTEGER",
|
|
732
|
+
notNull: true
|
|
733
|
+
},
|
|
734
|
+
{
|
|
735
|
+
name: "metric",
|
|
736
|
+
type: "TEXT",
|
|
737
|
+
notNull: true
|
|
738
|
+
}
|
|
739
|
+
]
|
|
740
|
+
});
|
|
741
|
+
const rows = await this.store.query({ collection: INDEX_REGISTRY_COLLECTION });
|
|
742
|
+
for (const row of rows) {
|
|
743
|
+
const dim = row.data["dim"];
|
|
744
|
+
const metric = row.data["metric"];
|
|
745
|
+
if (typeof dim !== "number") continue;
|
|
746
|
+
await this.declareIndex(row.id, dim, isMetric(metric) ? metric : "cosine");
|
|
747
|
+
}
|
|
748
|
+
this.logger.info("vector-store index registry loaded", { meta: { indexes: this.specs.size } });
|
|
749
|
+
} catch (err) {
|
|
750
|
+
this.logger.error("vector-store index registry failed to load", { meta: { error: String(err) } });
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
/**
|
|
754
|
+
* Idempotent. Re-declaring an index with a DIFFERENT dim throws rather than
|
|
755
|
+
* wiping: silently dropping a populated index because a caller shipped a new
|
|
756
|
+
* model would destroy history that cannot be rebuilt.
|
|
757
|
+
*/
|
|
758
|
+
async declareIndex(index, dim, metric) {
|
|
759
|
+
const existing = this.specs.get(index);
|
|
760
|
+
if (existing !== void 0 && existing.dim !== dim) throw new Error(`vector-store: index "${index}" is declared with dim ${existing.dim}; refusing to redeclare as ${dim}`);
|
|
761
|
+
await this.store.declareCollection({
|
|
762
|
+
collection: collectionFor(index),
|
|
763
|
+
columns: [
|
|
764
|
+
{
|
|
765
|
+
name: "id",
|
|
766
|
+
type: "TEXT",
|
|
767
|
+
primaryKey: true,
|
|
768
|
+
notNull: true
|
|
769
|
+
},
|
|
770
|
+
{
|
|
771
|
+
name: "vector",
|
|
772
|
+
type: "TEXT",
|
|
773
|
+
notNull: true
|
|
774
|
+
},
|
|
775
|
+
{
|
|
776
|
+
name: "deviceId",
|
|
777
|
+
type: "INTEGER"
|
|
778
|
+
},
|
|
779
|
+
{
|
|
780
|
+
name: "timestamp",
|
|
781
|
+
type: "INTEGER"
|
|
782
|
+
},
|
|
783
|
+
{
|
|
784
|
+
name: "className",
|
|
785
|
+
type: "TEXT"
|
|
786
|
+
},
|
|
787
|
+
{
|
|
788
|
+
name: "modelId",
|
|
789
|
+
type: "TEXT"
|
|
790
|
+
},
|
|
791
|
+
{
|
|
792
|
+
name: "extra",
|
|
793
|
+
type: "JSON"
|
|
794
|
+
}
|
|
795
|
+
],
|
|
796
|
+
indexes: [
|
|
797
|
+
{
|
|
798
|
+
name: `idx_vec_${index}_dev_ts`,
|
|
799
|
+
columns: ["deviceId", "timestamp"]
|
|
800
|
+
},
|
|
801
|
+
{
|
|
802
|
+
name: `idx_vec_${index}_ts`,
|
|
803
|
+
columns: ["timestamp"]
|
|
804
|
+
},
|
|
805
|
+
{
|
|
806
|
+
name: `idx_vec_${index}_class`,
|
|
807
|
+
columns: ["className"]
|
|
808
|
+
}
|
|
809
|
+
]
|
|
810
|
+
});
|
|
811
|
+
this.specs.set(index, {
|
|
812
|
+
dim,
|
|
813
|
+
metric
|
|
814
|
+
});
|
|
815
|
+
try {
|
|
816
|
+
await this.store.set({
|
|
817
|
+
collection: INDEX_REGISTRY_COLLECTION,
|
|
818
|
+
key: index,
|
|
819
|
+
value: {
|
|
820
|
+
dim,
|
|
821
|
+
metric
|
|
822
|
+
}
|
|
823
|
+
});
|
|
824
|
+
} catch (err) {
|
|
825
|
+
this.logger.warn("vector-store index declaration not persisted", { meta: {
|
|
826
|
+
index,
|
|
827
|
+
dim,
|
|
828
|
+
error: String(err)
|
|
829
|
+
} });
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
/**
|
|
833
|
+
* Resolve an index's spec, recovering it from stored rows when the registry
|
|
834
|
+
* has no entry.
|
|
835
|
+
*
|
|
836
|
+
* The recovery is not belt-and-braces, it is required: indexes created before
|
|
837
|
+
* the registry existed have rows and no declaration, and so would be
|
|
838
|
+
* permanently unqueryable — which is exactly how this was found, with 8,494
|
|
839
|
+
* live vectors that `stats` refused to look at. The dimension is derivable
|
|
840
|
+
* from any stored vector, so derive it and persist it rather than demanding
|
|
841
|
+
* the caller redeclare. The metric is NOT derivable and falls back to
|
|
842
|
+
* `cosine`, which is what every current caller declares.
|
|
843
|
+
*/
|
|
844
|
+
async resolveSpec(index) {
|
|
845
|
+
const known = this.specs.get(index);
|
|
846
|
+
if (known !== void 0) return known;
|
|
847
|
+
const sample = (await this.store.query({
|
|
848
|
+
collection: collectionFor(index),
|
|
849
|
+
filter: { limit: 1 }
|
|
850
|
+
}))[0];
|
|
851
|
+
const vector = sample === void 0 ? void 0 : readRow(sample.data)?.vector;
|
|
852
|
+
if (vector === void 0) throw new Error(`vector-store: index "${index}" was never declared`);
|
|
853
|
+
const dim = require_dist.vectorDimFromBase64(vector);
|
|
854
|
+
this.logger.warn("vector-store recovered an undeclared index from its rows", { meta: {
|
|
855
|
+
index,
|
|
856
|
+
dim,
|
|
857
|
+
metric: "cosine"
|
|
858
|
+
} });
|
|
859
|
+
await this.declareIndex(index, dim, "cosine");
|
|
860
|
+
return {
|
|
861
|
+
dim,
|
|
862
|
+
metric: "cosine"
|
|
863
|
+
};
|
|
864
|
+
}
|
|
865
|
+
/** Upsert items, refusing any whose vector length disagrees with the index. */
|
|
866
|
+
async upsert(index, items) {
|
|
867
|
+
const spec = await this.resolveSpec(index);
|
|
868
|
+
const collection = collectionFor(index);
|
|
869
|
+
let upserted = 0;
|
|
870
|
+
let rejected = 0;
|
|
871
|
+
for (const item of items) {
|
|
872
|
+
if (require_dist.vectorDimFromBase64(item.vector) !== spec.dim) {
|
|
873
|
+
rejected += 1;
|
|
874
|
+
continue;
|
|
875
|
+
}
|
|
876
|
+
const { deviceId, timestamp, className, modelId, extra } = splitMetadata(item.metadata);
|
|
877
|
+
try {
|
|
878
|
+
await this.store.set({
|
|
879
|
+
collection,
|
|
880
|
+
key: item.id,
|
|
881
|
+
value: {
|
|
882
|
+
vector: item.vector,
|
|
883
|
+
deviceId,
|
|
884
|
+
timestamp,
|
|
885
|
+
className,
|
|
886
|
+
modelId,
|
|
887
|
+
extra
|
|
888
|
+
}
|
|
889
|
+
});
|
|
890
|
+
upserted += 1;
|
|
891
|
+
} catch (err) {
|
|
892
|
+
rejected += 1;
|
|
893
|
+
this.logger.warn("vector-store upsert failed", {
|
|
894
|
+
...deviceId !== null ? { tags: { deviceId } } : {},
|
|
895
|
+
meta: {
|
|
896
|
+
index,
|
|
897
|
+
id: item.id,
|
|
898
|
+
error: String(err)
|
|
899
|
+
}
|
|
900
|
+
});
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
if (rejected > 0) this.logger.warn("vector-store rejected items on upsert", { meta: {
|
|
904
|
+
index,
|
|
905
|
+
rejected,
|
|
906
|
+
expectedDim: spec.dim
|
|
907
|
+
} });
|
|
908
|
+
return {
|
|
909
|
+
upserted,
|
|
910
|
+
rejected
|
|
911
|
+
};
|
|
912
|
+
}
|
|
913
|
+
/** Exhaustive ranked search over the rows a prefilter admits. */
|
|
914
|
+
async query(params) {
|
|
915
|
+
const spec = await this.resolveSpec(params.index);
|
|
916
|
+
const queryVec = require_dist.decodeVectorBase64(params.vector);
|
|
917
|
+
if (queryVec.length !== spec.dim) throw new Error(`vector-store: query vector has ${queryVec.length} dims, index "${params.index}" has ${spec.dim}`);
|
|
918
|
+
const rows = await this.store.query({
|
|
919
|
+
collection: collectionFor(params.index),
|
|
920
|
+
filter: {
|
|
921
|
+
...buildSqlFilter(params.filter),
|
|
922
|
+
orderBy: {
|
|
923
|
+
field: "timestamp",
|
|
924
|
+
direction: "desc"
|
|
925
|
+
},
|
|
926
|
+
limit: MAX_SCAN_ROWS
|
|
927
|
+
}
|
|
928
|
+
});
|
|
929
|
+
const minScore = params.minScore ?? Number.NEGATIVE_INFINITY;
|
|
930
|
+
const scored = [];
|
|
931
|
+
for (const row of rows) {
|
|
932
|
+
const stored = readRow(row.data);
|
|
933
|
+
if (stored === null) continue;
|
|
934
|
+
if (!passesUnpushedFilter(stored, params.filter)) continue;
|
|
935
|
+
const vec = require_dist.decodeVectorBase64(stored.vector);
|
|
936
|
+
if (vec.length !== spec.dim) continue;
|
|
937
|
+
const score = scoreFor(spec.metric, queryVec, vec);
|
|
938
|
+
if (score < minScore) continue;
|
|
939
|
+
scored.push({
|
|
940
|
+
id: row.id,
|
|
941
|
+
score,
|
|
942
|
+
metadata: rebuildMetadata(stored)
|
|
943
|
+
});
|
|
944
|
+
}
|
|
945
|
+
scored.sort((a, b) => b.score - a.score);
|
|
946
|
+
return {
|
|
947
|
+
matches: scored.slice(0, params.topK),
|
|
948
|
+
scanned: rows.length,
|
|
949
|
+
truncated: rows.length >= MAX_SCAN_ROWS
|
|
950
|
+
};
|
|
951
|
+
}
|
|
952
|
+
/**
|
|
953
|
+
* Metadata for the given ids, WITHOUT decoding their vectors.
|
|
954
|
+
*
|
|
955
|
+
* The caller is a best-of gate comparing confidences, so the vector is dead
|
|
956
|
+
* weight on that path — reading it would reintroduce exactly the per-row cost
|
|
957
|
+
* this index exists to remove. Missing ids are absent from the result rather
|
|
958
|
+
* than present with an empty metadata object.
|
|
959
|
+
*/
|
|
960
|
+
async getByIds(index, ids) {
|
|
961
|
+
await this.resolveSpec(index);
|
|
962
|
+
if (ids.length === 0) return [];
|
|
963
|
+
const rows = await this.store.query({
|
|
964
|
+
collection: collectionFor(index),
|
|
965
|
+
filter: { limit: MAX_SCAN_ROWS }
|
|
966
|
+
});
|
|
967
|
+
const wanted = new Set(ids);
|
|
968
|
+
const out = [];
|
|
969
|
+
for (const row of rows) {
|
|
970
|
+
if (!wanted.has(row.id)) continue;
|
|
971
|
+
const stored = readRow(row.data);
|
|
972
|
+
if (stored === null) continue;
|
|
973
|
+
out.push({
|
|
974
|
+
id: row.id,
|
|
975
|
+
metadata: rebuildMetadata(stored)
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
return out;
|
|
979
|
+
}
|
|
980
|
+
async deleteByIds(index, ids) {
|
|
981
|
+
const collection = collectionFor(index);
|
|
982
|
+
let deleted = 0;
|
|
983
|
+
for (const id of ids) try {
|
|
984
|
+
await this.store.delete({
|
|
985
|
+
collection,
|
|
986
|
+
key: id
|
|
987
|
+
});
|
|
988
|
+
deleted += 1;
|
|
989
|
+
} catch (err) {
|
|
990
|
+
this.logger.warn("vector-store delete failed", { meta: {
|
|
991
|
+
index,
|
|
992
|
+
id,
|
|
993
|
+
error: String(err)
|
|
994
|
+
} });
|
|
995
|
+
}
|
|
996
|
+
return deleted;
|
|
997
|
+
}
|
|
998
|
+
async deleteByFilter(index, filter) {
|
|
999
|
+
const ids = (await this.store.query({
|
|
1000
|
+
collection: collectionFor(index),
|
|
1001
|
+
filter: {
|
|
1002
|
+
...buildSqlFilter(filter),
|
|
1003
|
+
limit: MAX_SCAN_ROWS
|
|
1004
|
+
}
|
|
1005
|
+
})).filter((r) => {
|
|
1006
|
+
const stored = readRow(r.data);
|
|
1007
|
+
return stored !== null && passesUnpushedFilter(stored, filter);
|
|
1008
|
+
}).map((r) => r.id);
|
|
1009
|
+
return this.deleteByIds(index, ids);
|
|
1010
|
+
}
|
|
1011
|
+
async stats(index) {
|
|
1012
|
+
const spec = await this.resolveSpec(index);
|
|
1013
|
+
return {
|
|
1014
|
+
backend: "sqlite-bruteforce",
|
|
1015
|
+
count: (await this.store.query({
|
|
1016
|
+
collection: collectionFor(index),
|
|
1017
|
+
filter: { limit: MAX_SCAN_ROWS }
|
|
1018
|
+
})).length,
|
|
1019
|
+
dim: spec.dim,
|
|
1020
|
+
metric: spec.metric,
|
|
1021
|
+
exact: true
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
};
|
|
1025
|
+
/** Split caller metadata into promoted columns plus a verbatim remainder. */
|
|
1026
|
+
function splitMetadata(metadata) {
|
|
1027
|
+
const extra = {};
|
|
1028
|
+
for (const [key, value] of Object.entries(metadata)) if (!isPromoted(key)) extra[key] = value;
|
|
1029
|
+
return {
|
|
1030
|
+
deviceId: numberOrNull(metadata["deviceId"]),
|
|
1031
|
+
timestamp: numberOrNull(metadata["timestamp"]),
|
|
1032
|
+
className: stringOrNull(metadata["className"]),
|
|
1033
|
+
modelId: stringOrNull(metadata["modelId"]),
|
|
1034
|
+
extra
|
|
1035
|
+
};
|
|
1036
|
+
}
|
|
1037
|
+
function isPromoted(key) {
|
|
1038
|
+
for (const promoted of PROMOTED_COLUMNS) if (promoted === key) return true;
|
|
1039
|
+
return false;
|
|
1040
|
+
}
|
|
1041
|
+
function numberOrNull(value) {
|
|
1042
|
+
return typeof value === "number" ? value : null;
|
|
1043
|
+
}
|
|
1044
|
+
function stringOrNull(value) {
|
|
1045
|
+
return typeof value === "string" ? value : null;
|
|
1046
|
+
}
|
|
1047
|
+
/** Rebuild the caller's metadata from the promoted columns plus the remainder. */
|
|
1048
|
+
function rebuildMetadata(row) {
|
|
1049
|
+
const out = { ...row.extra };
|
|
1050
|
+
if (row.deviceId !== null) out["deviceId"] = row.deviceId;
|
|
1051
|
+
if (row.timestamp !== null) out["timestamp"] = row.timestamp;
|
|
1052
|
+
if (row.className !== null) out["className"] = row.className;
|
|
1053
|
+
if (row.modelId !== null) out["modelId"] = row.modelId;
|
|
1054
|
+
return out;
|
|
1055
|
+
}
|
|
1056
|
+
/** The part of a filter SQLite can apply: promoted columns only. */
|
|
1057
|
+
function buildSqlFilter(filter) {
|
|
1058
|
+
if (filter === void 0) return {};
|
|
1059
|
+
const where = {};
|
|
1060
|
+
const whereBetween = {};
|
|
1061
|
+
for (const [key, value] of Object.entries(filter.equals ?? {})) if (isPromoted(key)) where[key] = value;
|
|
1062
|
+
for (const [key, range] of Object.entries(filter.between ?? {})) if (isPromoted(key)) whereBetween[key] = range;
|
|
1063
|
+
return {
|
|
1064
|
+
...Object.keys(where).length > 0 ? { where } : {},
|
|
1065
|
+
...Object.keys(whereBetween).length > 0 ? { whereBetween } : {}
|
|
1066
|
+
};
|
|
1067
|
+
}
|
|
1068
|
+
/**
|
|
1069
|
+
* Apply the clauses SQL could not.
|
|
1070
|
+
*
|
|
1071
|
+
* A filter key that is not a promoted column lives inside the `extra` JSON
|
|
1072
|
+
* blob, so it is checked here in JS. It is applied rather than ignored — an
|
|
1073
|
+
* unapplied filter returns rows the caller will treat as matches.
|
|
1074
|
+
*/
|
|
1075
|
+
function passesUnpushedFilter(row, filter) {
|
|
1076
|
+
if (filter === void 0) return true;
|
|
1077
|
+
for (const [key, expected] of Object.entries(filter.equals ?? {})) {
|
|
1078
|
+
if (isPromoted(key)) continue;
|
|
1079
|
+
if (row.extra[key] !== expected) return false;
|
|
1080
|
+
}
|
|
1081
|
+
for (const [key, [lo, hi]] of Object.entries(filter.between ?? {})) {
|
|
1082
|
+
if (isPromoted(key)) continue;
|
|
1083
|
+
const actual = row.extra[key];
|
|
1084
|
+
if (typeof actual !== "number" || actual < lo || actual > hi) return false;
|
|
1085
|
+
}
|
|
1086
|
+
return true;
|
|
1087
|
+
}
|
|
1088
|
+
function readRow(data) {
|
|
1089
|
+
const vector = data["vector"];
|
|
1090
|
+
if (typeof vector !== "string" || vector.length === 0) return null;
|
|
1091
|
+
const extra = data["extra"];
|
|
1092
|
+
return {
|
|
1093
|
+
vector,
|
|
1094
|
+
deviceId: numberOrNull(data["deviceId"]),
|
|
1095
|
+
timestamp: numberOrNull(data["timestamp"]),
|
|
1096
|
+
className: stringOrNull(data["className"]),
|
|
1097
|
+
modelId: stringOrNull(data["modelId"]),
|
|
1098
|
+
extra: isMetadata(extra) ? extra : {}
|
|
1099
|
+
};
|
|
1100
|
+
}
|
|
1101
|
+
function isMetadata(value) {
|
|
1102
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1103
|
+
}
|
|
1104
|
+
/** Narrow a stored metric string back to the enum. */
|
|
1105
|
+
function isMetric(value) {
|
|
1106
|
+
return value === "cosine" || value === "dot" || value === "euclidean";
|
|
1107
|
+
}
|
|
1108
|
+
/** Score two vectors under the index's metric. Higher is nearer in every case. */
|
|
1109
|
+
function scoreFor(metric, a, b) {
|
|
1110
|
+
if (metric === "cosine") return require_dist.cosineSimilarity(a, b);
|
|
1111
|
+
if (metric === "dot") {
|
|
1112
|
+
let sum = 0;
|
|
1113
|
+
for (let i = 0; i < a.length; i += 1) sum += (a[i] ?? 0) * (b[i] ?? 0);
|
|
1114
|
+
return sum;
|
|
1115
|
+
}
|
|
1116
|
+
let sum = 0;
|
|
1117
|
+
for (let i = 0; i < a.length; i += 1) {
|
|
1118
|
+
const d = (a[i] ?? 0) - (b[i] ?? 0);
|
|
1119
|
+
sum += d * d;
|
|
1120
|
+
}
|
|
1121
|
+
const distance = Math.sqrt(sum);
|
|
1122
|
+
return distance === 0 ? 0 : -distance;
|
|
1123
|
+
}
|
|
1124
|
+
//#endregion
|
|
650
1125
|
//#region src/builtins/sqlite-storage/sqlite-settings.addon.ts
|
|
651
1126
|
/**
|
|
652
1127
|
* SQLite Settings addon — the relational ENGINE behind the data door.
|
|
@@ -657,9 +1132,16 @@ var SqliteSettingsBackend = class SqliteSettingsBackend {
|
|
|
657
1132
|
* ([D44](../../../../../docs/decisions/adr-0044.md)). Callers reach this
|
|
658
1133
|
* engine only through the door — which is the point, because it is what
|
|
659
1134
|
* makes the engine replaceable.
|
|
1135
|
+
*
|
|
1136
|
+
* It ALSO serves `vector-store` (singleton, internal), from the same database
|
|
1137
|
+
* file and the same open handle. That capability first shipped as its own addon
|
|
1138
|
+
* and the separation cost two round trips per write — caller → door → engine —
|
|
1139
|
+
* plus a runner process, to reach a database this addon already has open. The
|
|
1140
|
+
* capability is what keeps the backend swappable; the package never was.
|
|
660
1141
|
*/
|
|
661
1142
|
var SqliteSettingsAddon = class extends require_dist.BaseAddon {
|
|
662
1143
|
backend = null;
|
|
1144
|
+
vectorIndex = null;
|
|
663
1145
|
constructor() {
|
|
664
1146
|
super({});
|
|
665
1147
|
}
|
|
@@ -683,10 +1165,36 @@ var SqliteSettingsAddon = class extends require_dist.BaseAddon {
|
|
|
683
1165
|
this.ctx.logger.info("DB file status", { meta: { dbExists } });
|
|
684
1166
|
this.backend = new SqliteSettingsBackend(dbPath, { ...require_dist.RUNTIME_DEFAULTS });
|
|
685
1167
|
await this.backend.initialize();
|
|
1168
|
+
const vectorIndex = new SqliteVectorIndex({
|
|
1169
|
+
store: this.backend,
|
|
1170
|
+
logger: this.ctx.logger.child("VectorIndex")
|
|
1171
|
+
});
|
|
1172
|
+
await vectorIndex.loadIndexRegistry();
|
|
1173
|
+
this.vectorIndex = vectorIndex;
|
|
1174
|
+
const vectorProvider = {
|
|
1175
|
+
declareIndex: async (input) => {
|
|
1176
|
+
await vectorIndex.declareIndex(input.index, input.dim, input.metric);
|
|
1177
|
+
},
|
|
1178
|
+
upsert: async (input) => vectorIndex.upsert(input.index, input.items),
|
|
1179
|
+
query: async (input) => vectorIndex.query({
|
|
1180
|
+
index: input.index,
|
|
1181
|
+
vector: input.vector,
|
|
1182
|
+
topK: input.topK,
|
|
1183
|
+
...input.minScore !== void 0 ? { minScore: input.minScore } : {},
|
|
1184
|
+
...input.filter !== void 0 ? { filter: input.filter } : {}
|
|
1185
|
+
}),
|
|
1186
|
+
getByIds: async (input) => ({ items: await vectorIndex.getByIds(input.index, input.ids) }),
|
|
1187
|
+
deleteByIds: async (input) => ({ deleted: await vectorIndex.deleteByIds(input.index, input.ids) }),
|
|
1188
|
+
deleteByFilter: async (input) => ({ deleted: await vectorIndex.deleteByFilter(input.index, input.filter) }),
|
|
1189
|
+
stats: async (input) => vectorIndex.stats(input.index)
|
|
1190
|
+
};
|
|
686
1191
|
this.ctx.logger.info("Initialized successfully");
|
|
687
1192
|
return [{
|
|
688
1193
|
capability: require_dist.dataStoreProviderCapability,
|
|
689
1194
|
provider: this.backend
|
|
1195
|
+
}, {
|
|
1196
|
+
capability: require_dist.vectorStoreCapability,
|
|
1197
|
+
provider: vectorProvider
|
|
690
1198
|
}];
|
|
691
1199
|
}
|
|
692
1200
|
async onShutdown() {
|
|
@@ -695,6 +1203,10 @@ var SqliteSettingsAddon = class extends require_dist.BaseAddon {
|
|
|
695
1203
|
getBackend() {
|
|
696
1204
|
return this.backend;
|
|
697
1205
|
}
|
|
1206
|
+
/** Exposed for tests; null before `onInitialize`. */
|
|
1207
|
+
getVectorIndex() {
|
|
1208
|
+
return this.vectorIndex;
|
|
1209
|
+
}
|
|
698
1210
|
};
|
|
699
1211
|
//#endregion
|
|
700
1212
|
exports.SqliteSettingsAddon = SqliteSettingsAddon;
|