dsh-data-domain-knowledge 0.1.0-alpha.9 → 0.1.1-alpha.2

package/lib/types/retention.js

@@ -0,0 +1,42 @@
+/** 水位表的表名。它跟着所在的那个库走（审计库与 knowledge.db 各有一份）。 */
+const FLOOR_TABLE = 'retention_floor';
+function ensureFloors(db) {
+    db.exec(`CREATE TABLE IF NOT EXISTS ${FLOOR_TABLE} (name TEXT PRIMARY KEY, floor INTEGER NOT NULL, at TEXT NOT NULL)`);
+}
+function floorOf(db, name, table, key) {
+    const row = db.prepare(`SELECT floor FROM ${FLOOR_TABLE} WHERE name = ?`).get(name);
+    if (row !== undefined)
+        return Number(row.floor ?? 0);
+    const max = Number(db.prepare(`SELECT COALESCE(MAX(${key}), 0) AS m FROM ${table}`).get()?.m ?? 0);
+    db.prepare(`INSERT OR IGNORE INTO ${FLOOR_TABLE}(name, floor, at) VALUES (?, ?, ?)`).run(name, max, new Date().toISOString());
+    return max;
+}
+/**
+ * 裁掉水位以上的超限行，返回删掉的行数。
+ *
+ * 幂等且可反复调用：没有超限时最多花一次索引区间计数。
+ */
+export function applyRetention(db, name, spec) {
+    ensureFloors(db);
+    const floor = floorOf(db, name, spec.table, spec.key);
+    const filter = spec.where === undefined ? '' : ` AND (${spec.where})`;
+    const above = Number(db.prepare(`SELECT COUNT(*) AS n FROM ${spec.table} WHERE ${spec.key} > ?${filter}`).get(floor)?.n ?? 0);
+    if (above <= spec.keep)
+        return 0;
+    if (spec.groupBy === undefined) {
+        const changes = spec.keep <= 0
+            ? db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.key} > ?${filter}`).run(floor)
+            : db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.key} > ? AND ${spec.key} < (
+           SELECT ${spec.key} FROM ${spec.table} WHERE ${spec.key} > ?${filter} ORDER BY ${spec.key} DESC LIMIT 1 OFFSET ?
+         )${filter}`).run(floor, floor, spec.keep - 1);
+        return Number(changes?.changes ?? 0);
+    }
+    const changes = db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.key} IN (
+      SELECT ${spec.key} FROM (
+        SELECT ${spec.key}, ROW_NUMBER() OVER (PARTITION BY ${spec.groupBy} ORDER BY ${spec.key} DESC) AS rn
+        FROM ${spec.table} WHERE ${spec.key} > ?${filter}
+      ) WHERE rn > ?
+    )`).run(floor, spec.keep);
+    return Number(changes?.changes ?? 0);
+}
+//# sourceMappingURL=retention.js.map

package/lib/types/retention.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retention.js","sourceRoot":"","sources":["../../src/retention.ts"],"names":[],"mappings":"AAkCA,iDAAiD;AACjD,MAAM,WAAW,GAAG,iBAAiB,CAAA;AAErC,SAAS,YAAY,CAAC,EAAkB;IACtC,EAAE,CAAC,IAAI,CAAC,8BAA8B,WAAW,oEAAoE,CAAC,CAAA;AACxH,CAAC;AAED,SAAS,OAAO,CAAC,EAAkB,EAAE,IAAY,EAAE,KAAa,EAAE,GAAW;IAC3E,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,qBAAqB,WAAW,iBAAiB,CAAC,CAAC,GAAG,CAAC,IAAI,CAAoC,CAAA;IACtH,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAA;IACpD,MAAM,GAAG,GAAG,MAAM,CAAE,EAAE,CAAC,OAAO,CAAC,uBAAuB,GAAG,mBAAmB,KAAK,EAAE,CAAC,CAAC,GAAG,EAAkC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;IACnI,EAAE,CAAC,OAAO,CAAC,yBAAyB,WAAW,oCAAoC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;IAC7H,OAAO,GAAG,CAAA;AACZ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,EAAkB,EAAE,IAAY,EAAE,IAAmB;IAClF,YAAY,CAAC,EAAE,CAAC,CAAA;IAChB,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAA;IACrD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,KAAK,GAAG,CAAA;IACrE,MAAM,KAAK,GAAG,MAAM,CAAE,EAAE,CAAC,OAAO,CAAC,6BAA6B,IAAI,CAAC,KAAK,UAAU,IAAI,CAAC,GAAG,OAAO,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAiC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;IAC9J,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAA;IAChC,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC;YAC5B,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,KAAK,UAAU,IAAI,CAAC,GAAG,OAAO,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC;YACnF,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,KAAK,UAAU,IAAI,CAAC,GAAG,YAAY,IAAI,CAAC,GAAG;oBAC9D,IAAI,CAAC,GAAG,SAAS,IAAI,CAAC,KAAK,UAAU,IAAI,CAAC,GAAG,OAAO,MAAM,aAAa,IAAI,CAAC,GAAG;YACvF,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAA;QAClD,OAAO,MAAM,CAAE,OAAiC,EAAE,OAAO,IAAI,CAAC,CAAC,CAAA;IACjE,CAAC;IACD,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,KAAK,UAAU,IAAI,CAAC,GAAG;eACzD,IAAI,CAAC,GAAG;iBACN,IAAI,CAAC,GAAG,qCAAqC,IAAI,CAAC,OAAO,aAAa,IAAI,CAAC,GAAG;eAChF,IAAI,CAAC,KAAK,UAAU,IAAI,CAAC,GAAG,OAAO,MAAM;;MAElD,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IAC3B,OAAO,MAAM,CAAE,OAAiC,EAAE,OAAO,IAAI,CAAC,CAAC,CAAA;AACjE,CAAC"}

package/lib/types/retrieval/vector-index.d.ts

@@ -0,0 +1,32 @@
+import type { KnowledgeConfig } from '../types.js';
+import type { DerivedDb } from '../store/derived.js';
+export interface VectorItem {
+    unitId: string;
+    parentUnitId?: string;
+    contentHash: string;
+}
+export interface VectorManifest {
+    version: number;
+    createdAt: string;
+    provider: KnowledgeConfig['embeddingProvider'];
+    model: string;
+    modelFingerprint?: string;
+    /**
+     * 编码实现的指纹（0031）。旧清单没有这个字段 → 视为不兼容，于是修复编码器之后
+     * 一定触发一次真正的重编码，而不是把旧向量按 contentHash 复用回来。
+     */
+    encoder?: string;
+    device: string;
+    dimension: number;
+    quantization: 'binary+hamming+int8';
+    itemCount: number;
+    items: VectorItem[];
+}
+export interface LoadedVectorIndex {
+    manifest: VectorManifest;
+    binary: Buffer;
+    int8: Buffer;
+}
+/** Validate a generation's BLOBs before either reuse or query. */
+export declare function decodeVectorIndex(loaded: ReturnType<DerivedDb['vectorIndex']>, encoder: string): LoadedVectorIndex | null;
+//# sourceMappingURL=vector-index.d.ts.map

package/lib/types/retrieval/vector-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-index.d.ts","sourceRoot":"","sources":["../../../src/retrieval/vector-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAClD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAEpD,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,eAAe,CAAC,mBAAmB,CAAC,CAAA;IAC9C,KAAK,EAAE,MAAM,CAAA;IACb,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,qBAAqB,CAAA;IACnC,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,UAAU,EAAE,CAAA;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,cAAc,CAAA;IACxB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACb;AAED,kEAAkE;AAClE,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,UAAU,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,iBAAiB,GAAG,IAAI,CAUzH"}

package/lib/types/retrieval/vector-index.js

@@ -0,0 +1,15 @@
+/** Validate a generation's BLOBs before either reuse or query. */
+export function decodeVectorIndex(loaded, encoder) {
+    if (!loaded)
+        return null;
+    const manifest = loaded.manifest;
+    if (!manifest || manifest.quantization !== 'binary+hamming+int8' || manifest.encoder !== encoder
+        || !Number.isInteger(manifest.dimension) || manifest.dimension < 8 || manifest.dimension % 8 !== 0
+        || !Number.isInteger(manifest.itemCount) || !Array.isArray(manifest.items)
+        || manifest.itemCount !== manifest.items.length
+        || loaded.binary.length !== manifest.itemCount * Math.ceil(manifest.dimension / 8)
+        || loaded.int8.length !== manifest.itemCount * manifest.dimension)
+        return null;
+    return { ...loaded, manifest };
+}
+//# sourceMappingURL=vector-index.js.map

package/lib/types/retrieval/vector-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-index.js","sourceRoot":"","sources":["../../../src/retrieval/vector-index.ts"],"names":[],"mappings":"AAiCA,kEAAkE;AAClE,MAAM,UAAU,iBAAiB,CAAC,MAA4C,EAAE,OAAe;IAC7F,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAA0B,CAAA;IAClD,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,YAAY,KAAK,qBAAqB,IAAI,QAAQ,CAAC,OAAO,KAAK,OAAO;WAC3F,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,SAAS,GAAG,CAAC,IAAI,QAAQ,CAAC,SAAS,GAAG,CAAC,KAAK,CAAC;WAC/F,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;WACvE,QAAQ,CAAC,SAAS,KAAK,QAAQ,CAAC,KAAK,CAAC,MAAM;WAC5C,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC;WAC/E,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,SAAS,GAAG,QAAQ,CAAC,SAAS;QAAE,OAAO,IAAI,CAAA;IAChF,OAAO,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,CAAA;AAChC,CAAC"}

package/lib/types/retrieval/visibility.d.ts

@@ -0,0 +1,6 @@
+import type { ContextGraph } from '../types.js';
+import type { KnowledgeDb } from '../store/knowledge-db.js';
+export declare function liveGraphKeys(db: KnowledgeDb): Set<string>;
+/** A retained generation must not restore entities that the user deleted. */
+export declare function visibleGraph(graph: ContextGraph, live: Set<string>): ContextGraph;
+//# sourceMappingURL=visibility.d.ts.map

package/lib/types/retrieval/visibility.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"visibility.d.ts","sourceRoot":"","sources":["../../../src/retrieval/visibility.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAA;AAE3D,wBAAgB,aAAa,CAAC,EAAE,EAAE,WAAW,GAAG,GAAG,CAAC,MAAM,CAAC,CAG1D;AAED,6EAA6E;AAC7E,wBAAgB,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,YAAY,CAIjF"}

package/lib/types/retrieval/visibility.js

@@ -0,0 +1,11 @@
+export function liveGraphKeys(db) {
+    const rows = db.raw("SELECT entity_type AS type, entity_id AS id FROM card UNION SELECT 'source' AS type, entity_id AS id FROM source").all();
+    return new Set(rows.map(row => `${row.type}:${row.id}`));
+}
+/** A retained generation must not restore entities that the user deleted. */
+export function visibleGraph(graph, live) {
+    const nodes = graph.nodes.filter(node => live.has(`${node.type}:${node.id}`));
+    const keys = new Set(nodes.map(node => `${node.type}:${node.id}`));
+    return { nodes, edges: graph.edges.filter(edge => keys.has(String(edge.from)) && keys.has(String(edge.to))) };
+}
+//# sourceMappingURL=visibility.js.map

package/lib/types/retrieval/visibility.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"visibility.js","sourceRoot":"","sources":["../../../src/retrieval/visibility.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,aAAa,CAAC,EAAe;IAC3C,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CAAC,kHAAkH,CAAC,CAAC,GAAG,EAAyC,CAAA;IACpL,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;AAC1D,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,YAAY,CAAC,KAAmB,EAAE,IAAiB;IACjE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;IAC7E,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;IAClE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;AAC/G,CAAC"}