dsh-data-domain-knowledge 0.1.0-alpha.9 → 0.1.1-alpha.2

package/src/python/embedding_worker.py

@@ -283,6 +283,45 @@ def encode_multimodal(model_wrapper: Any, items: list[dict[str, Any]], device: s
     return outputs
 
 
+def text_windows(tokenizer, text: str, budget: int, overlap: int = 48) -> list[str]:
+    """Physical windows only. The host retains the complete logical H2 section."""
+    ids = tokenizer.encode(text, add_special_tokens=False)
+    if len(ids) <= budget:
+        return [text]
+    step = max(1, budget - min(overlap, budget // 4))
+    windows = []
+    for start in range(0, len(ids), step):
+        windows.append(tokenizer.decode(ids[start:start + budget], skip_special_tokens=True))
+        if start + budget >= len(ids):
+            break
+    return windows
+
+
+def embedding_tokenizer(model):
+    if has_official_path(model):
+        tokenizer = getattr(model[0].processor, "tokenizer", None)
+    else:
+        tokenizer = getattr(model, "tokenizer", None)
+    if tokenizer is None:
+        raise RuntimeError("EMBEDDING_TOKENIZER_UNAVAILABLE")
+    return tokenizer
+
+
+def split_inputs(request: dict[str, Any]) -> dict[str, Any]:
+    model, _ = cached_model(str(request["model"]), str(request.get("device") or "auto"))
+    tokenizer = embedding_tokenizer(model)
+    limit = min(int(getattr(model, "max_seq_length", 512) or 512), 512)
+    windows = []
+    for item in request.get("inputs", []):
+        prefix = str(item.get("prefix") or "")
+        prefix_size = len(tokenizer.encode(prefix, add_special_tokens=False))
+        budget = limit - prefix_size - 16
+        if budget < 64:
+            raise ValueError("EMBEDDING_SECTION_TITLE_TOO_LONG")
+        windows.append([prefix + "\n\n" + part for part in text_windows(tokenizer, str(item.get("text") or ""), budget)])
+    return {"ok": True, "windows": windows, "tokenLimit": limit}
+
+
 def encode(request: dict[str, Any]) -> dict[str, Any]:
     model_name = str(request.get("model") or "ewin-reg/WeMM-Embedding-2B-Quantized")
     requested_device = str(request.get("device") or "auto")
@@ -334,6 +373,9 @@ def encode(request: dict[str, Any]) -> dict[str, Any]:
                 str(item.get("text") or "") if isinstance(item, dict) else str(item)
                 for item in batch
             ]
+            limit = int(getattr(model, "max_seq_length", 512) or 512)
+            if any(len(model.tokenizer.encode(text, add_special_tokens=True)) > limit for text in generic_batch):
+                raise ValueError("EMBEDDING_INPUT_TOO_LONG: split H2 text or shorten the current subquestion")
             encoded = model.encode(
                 generic_batch,
                 batch_size=batch_size,
@@ -454,9 +496,29 @@ def rerank(request: dict[str, Any]) -> dict[str, Any]:
     if not isinstance(documents, list) or not documents:
         raise ValueError("rerank requires a non-empty documents array")
     model, device, health = cached_reranker(model_name, requested_device)
-    pairs = [(query, str(document)) for document in documents]
+    tokenizer = model.tokenizer
+    limit = min(int(getattr(model, "max_length", 512) or 512), 512)
+    query_size = len(tokenizer.encode(query, add_special_tokens=False))
+    budget = limit - query_size - 16
+    if budget < 64:
+        raise ValueError("RERANK_QUERY_TOO_LONG: shorten the current subquestion")
+    pairs = []
+    owners = []
+    for owner, document in enumerate(documents):
+        title, separator, body = str(document).partition("\n\n")
+        prefix = title + "\n\n" if separator else ""
+        content = body if separator else title
+        document_budget = budget - len(tokenizer.encode(prefix, add_special_tokens=False))
+        if document_budget < 32:
+            raise ValueError("RERANK_SECTION_TITLE_TOO_LONG")
+        for window in text_windows(tokenizer, content, document_budget):
+            pairs.append((query, prefix + window))
+            owners.append(owner)
     with redirect_stdout(sys.stderr):
-        scores = predict_raw(model, pairs)
+        raw_scores = predict_raw(model, pairs)
+    scores = [float("-inf")] * len(documents)
+    for owner, score in zip(owners, raw_scores):
+        scores[owner] = max(scores[owner], float(score))
     return {
         "ok": True,
         "device": device,
@@ -476,6 +538,8 @@ def main() -> None:
             # selects the path and both models are cached per process.
             if str(request.get("task") or "") == "rerank":
                 emit(rerank(request))
+            elif str(request.get("task") or "") == "split":
+                emit(split_inputs(request))
             else:
                 emit(encode(request))
         except Exception as exc:  # return machine-readable errors to the Host

package/src/query-analysis.ts

@@ -1,4 +1,4 @@
-import type { AxisEntity, AxisKind } from './axis-registry.js'
+import type { AxisEntity, AxisKind, AxisRelation } from './axis-registry.js'
 import { normalizeAxisName } from './axis-registry.js'
 import type { CardCoordinate } from './types.js'
 
@@ -20,6 +20,14 @@ export interface QueryCoordinateCandidate {
   matchedAxes: AxisKind[]
 }
 
+export interface QueryRelationCandidate extends QueryCoordinateCandidate {
+  relationId: string
+  relationType: 'related_to'
+  matchedObjectId: string
+  relatedObjectId: string
+  evidence: string
+}
+
 export interface QueryAnalysis {
   query: string
   intent: { id: QueryIntent; confidence: number }
@@ -92,10 +100,11 @@ function coordinateCandidates(
   axes: Record<AxisKind, QueryAxisMatch[]>,
   cards: CoordinateCard[],
 ): QueryCoordinateCandidate[] {
-  const byKind: Record<AxisKind, Map<string, QueryAxisMatch>> = {
-    object: new Map(axes.object.map(item => [item.id, item])),
-    action: new Map(axes.action.map(item => [item.id, item])),
-    scene: new Map(axes.scene.map(item => [item.id, item])),
+  const byKind: Record<AxisKind, Map<string, number>> = {
+    object: new Map(), action: new Map(), scene: new Map(),
+  }
+  for (const kind of ['object', 'action', 'scene'] as const) {
+    for (const match of axes[kind]) byKind[kind].set(match.id, match.confidence)
   }
   const fields: Array<[AxisKind, keyof CardCoordinate]> = [
     ['object', 'objectId'], ['action', 'actionId'], ['scene', 'sceneId'],
@@ -105,10 +114,10 @@ function coordinateCandidates(
     const matchedAxes: AxisKind[] = []
     let score = 0
     for (const [kind, field] of fields) {
-      const match = byKind[kind].get(card.coordinate[field])
-      if (match === undefined) continue
+      const confidence = byKind[kind].get(card.coordinate[field])
+      if (confidence === undefined) continue
       matchedAxes.push(kind)
-      score += match.confidence
+      score += confidence
     }
     if (matchedAxes.length === 0) return []
     // 匹配多个轴的卡片优先，但只匹配一个轴也要可召回，不能硬过滤。
@@ -117,6 +126,57 @@ function coordinateCandidates(
   }).sort((left, right) => right.score - left.score || left.cardKey.localeCompare(right.cardKey))
 }
 
+/**
+ * 一跳 related_to 关系候选。关系只拓展对象轴，不把 scopeId 当层级；
+ * 关联候选进入独立召回路，后续仍由多路融合与重排判断相关性。
+ */
+export function relationCandidates(
+  analysis: QueryAnalysis,
+  relations: AxisRelation[],
+  cards: CoordinateCard[],
+): QueryRelationCandidate[] {
+  const matchedObjects = analysis.axes.object
+  if (matchedObjects.length === 0 || relations.length === 0) return []
+  const matchedById = new Map(matchedObjects.map(item => [item.id, item]))
+  const related = new Map<string, { matchedObjectId: string; score: number; relation: AxisRelation }>()
+  for (const relation of relations) {
+    if (relation.relationType !== 'related_to') continue
+    const left = matchedById.get(relation.fromId)
+    const right = matchedById.get(relation.toId)
+    if (left !== undefined && right !== undefined) continue
+    const matched = left ?? right
+    if (matched === undefined) continue
+    const relatedObjectId = left ? relation.toId : relation.fromId
+    if (matched.id === relatedObjectId) continue
+    const score = matched.confidence * 0.65
+    const previous = related.get(relatedObjectId)
+    if (previous === undefined || score > previous.score) {
+      related.set(relatedObjectId, { matchedObjectId: matched.id, score, relation })
+    }
+  }
+  const byCard = new Map<string, QueryRelationCandidate>()
+  for (const card of cards) {
+    const relatedObjectId = card.coordinate?.objectId
+    if (!relatedObjectId) continue
+    const match = related.get(relatedObjectId)
+    if (match === undefined) continue
+    const candidate: QueryRelationCandidate = {
+      cardKey: card.cardKey,
+      coordinate: card.coordinate!,
+      score: match.score,
+      matchedAxes: ['object'],
+      relationId: match.relation.id,
+      relationType: match.relation.relationType,
+      matchedObjectId: match.matchedObjectId,
+      relatedObjectId,
+      evidence: match.relation.evidence,
+    }
+    const previous = byCard.get(card.cardKey)
+    if (previous === undefined || candidate.score > previous.score) byCard.set(card.cardKey, candidate)
+  }
+  return [...byCard.values()].sort((left, right) => right.score - left.score || left.cardKey.localeCompare(right.cardKey))
+}
+
 export function analyzeQuery(query: string, entities: RegistryEntity[], cards: CoordinateCard[]): QueryAnalysis {
   const axes: Record<AxisKind, QueryAxisMatch[]> = {
     object: matchEntities(query, 'object', entities),

package/src/recall-rank.ts

@@ -34,8 +34,8 @@ export interface RrfOptions {
 /**
  * 名次级融合（Reciprocal Rank Fusion）。
  *
- * **为什么不用分数加权和**：三路的分数是三个不同的量纲 —— 关键词是归一化 BM25、
- * 向量是余弦、图谱是命中分/最大命中分。把它们乘权重相加，等于假设「0.5 的关键词分」
+ * **为什么不用分数加权和**：四路的分数来自不同刻度 —— 关键词是归一化 BM25、
+ * 向量是余弦、坐标与关系路是解析置信分。把它们乘权重相加，等于假设「0.5 的关键词分」
  * 与「0.5 的余弦」一样好，而这个假设没有任何依据。0030 的实际后果就是：向量路输出
  * 常数时，hybrid 的 55% 权重全是噪声，排序退化成 `unitId` 字典序（实测 4 条命中
  * 分数完全相同：`0.2753 = 0.5 × 0.55`）。
@@ -47,7 +47,7 @@ export function rrfFuse(rankings: Partial<Record<RecallRoute, RouteRanking[]>>,
   const { k, weights } = options
   const ranks = new Map<string, Partial<Record<RecallRoute, number>>>()
   const scores = new Map<string, number>()
-  const order: RecallRoute[] = ['coordinate', 'fts', 'vector', 'graph']
+  const order: RecallRoute[] = ['coordinate', 'relation', 'fts', 'vector', 'graph']
   for (const route of order) {
     const list = rankings[route]
     if (!list || list.length === 0) continue
@@ -260,8 +260,8 @@ export function selectContext<T>(items: T[], options: MmrOptions<T>): T[] {
 }
 
 /** `match` 字段由名次推导：多个候选路命中 = hybrid。 */
-export function matchOf(ranks: Partial<Record<RecallRoute, number>>): 'fts' | 'vector' | 'coordinate' | 'graph' | 'hybrid' {
-  const used = (['coordinate', 'fts', 'vector', 'graph'] as RecallRoute[]).filter(route => ranks[route] !== undefined)
+export function matchOf(ranks: Partial<Record<RecallRoute, number>>): 'fts' | 'vector' | 'coordinate' | 'relation' | 'graph' | 'hybrid' {
+  const used = (['coordinate', 'relation', 'fts', 'vector', 'graph'] as RecallRoute[]).filter(route => ranks[route] !== undefined)
   if (used.length >= 2) return 'hybrid'
   return used[0] ?? 'fts'
 }

package/src/retention.ts

@@ -0,0 +1,76 @@
+/**
+ * 保留上限（0070）。
+ *
+ * 文档里写着保留策略（「命名永久 + 最近 100 + 每日末版；机器版本只活到硬上限」），
+ * 而代码里对这几张表一条 `DELETE` 都没有：`revision` / `blob` / `tree` / `operations` /
+ * `pipeline_extractions` / `pipeline_events` 只增不减。这个模块把**新数据**的上限补上。
+ *
+ * ## 为什么是「水位 + 上限」，而不是直接按条数删
+ *
+ * 上限一旦生效，第一次裁剪就会把库里**已有的**历史删掉 —— 那是不可逆的，而用户的决定是
+ * 「只对新数据加上限，不动存量」。所以每张表第一次裁剪时先把当前最大键记成**水位**：
+ * 水位以下的既有行永不删除；水位以上的新行按上限保留。代价是总量变成
+ * 「升级那一刻的存量 + 上限」，而不是「上限」—— 这是「不动存量」能给出的最强的界。
+ *
+ * ## 键列的要求
+ *
+ * `key` 必须是单调递增的整数列（`rowid` 或自增 `seq`）。用时间戳当键不行：同一毫秒里
+ * 两行的顺序不确定，裁剪会随机丢一行。
+ */
+import type { SqliteDatabase } from './fts.js'
+
+export interface RetentionSpec {
+  /** 表名。内插进 SQL，所以调用方只能传常量。 */
+  table: string
+  /** 单调递增的整数键列，例如 `rowid` 或 `seq`。 */
+  key: string
+  /** 水位以上保留多少行；`0` 表示水位以上的匹配行全部删掉（用于回收无引用的 blob）。 */
+  keep: number
+  /** 额外的行条件（可含子查询）。同样只允许常量 SQL。 */
+  where?: string
+  /** 分组表达式：给了就是「每组保留 keep 行」，而不是全表保留。 */
+  groupBy?: string
+}
+
+/** 水位表的表名。它跟着所在的那个库走（审计库与 knowledge.db 各有一份）。 */
+const FLOOR_TABLE = 'retention_floor'
+
+function ensureFloors(db: SqliteDatabase): void {
+  db.exec(`CREATE TABLE IF NOT EXISTS ${FLOOR_TABLE} (name TEXT PRIMARY KEY, floor INTEGER NOT NULL, at TEXT NOT NULL)`)
+}
+
+function floorOf(db: SqliteDatabase, name: string, table: string, key: string): number {
+  const row = db.prepare(`SELECT floor FROM ${FLOOR_TABLE} WHERE name = ?`).get(name) as { floor?: unknown } | undefined
+  if (row !== undefined) return Number(row.floor ?? 0)
+  const max = Number((db.prepare(`SELECT COALESCE(MAX(${key}), 0) AS m FROM ${table}`).get() as { m?: unknown } | undefined)?.m ?? 0)
+  db.prepare(`INSERT OR IGNORE INTO ${FLOOR_TABLE}(name, floor, at) VALUES (?, ?, ?)`).run(name, max, new Date().toISOString())
+  return max
+}
+
+/**
+ * 裁掉水位以上的超限行，返回删掉的行数。
+ *
+ * 幂等且可反复调用：没有超限时最多花一次索引区间计数。
+ */
+export function applyRetention(db: SqliteDatabase, name: string, spec: RetentionSpec): number {
+  ensureFloors(db)
+  const floor = floorOf(db, name, spec.table, spec.key)
+  const filter = spec.where === undefined ? '' : ` AND (${spec.where})`
+  const above = Number((db.prepare(`SELECT COUNT(*) AS n FROM ${spec.table} WHERE ${spec.key} > ?${filter}`).get(floor) as { n?: unknown } | undefined)?.n ?? 0)
+  if (above <= spec.keep) return 0
+  if (spec.groupBy === undefined) {
+    const changes = spec.keep <= 0
+      ? db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.key} > ?${filter}`).run(floor)
+      : db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.key} > ? AND ${spec.key} < (
+           SELECT ${spec.key} FROM ${spec.table} WHERE ${spec.key} > ?${filter} ORDER BY ${spec.key} DESC LIMIT 1 OFFSET ?
+         )${filter}`).run(floor, floor, spec.keep - 1)
+    return Number((changes as { changes?: unknown })?.changes ?? 0)
+  }
+  const changes = db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.key} IN (
+      SELECT ${spec.key} FROM (
+        SELECT ${spec.key}, ROW_NUMBER() OVER (PARTITION BY ${spec.groupBy} ORDER BY ${spec.key} DESC) AS rn
+        FROM ${spec.table} WHERE ${spec.key} > ?${filter}
+      ) WHERE rn > ?
+    )`).run(floor, spec.keep)
+  return Number((changes as { changes?: unknown })?.changes ?? 0)
+}

package/src/retrieval/vector-index.ts

@@ -0,0 +1,45 @@
+import type { KnowledgeConfig } from '../types.js'
+import type { DerivedDb } from '../store/derived.js'
+
+export interface VectorItem {
+  unitId: string
+  parentUnitId?: string
+  contentHash: string
+}
+
+export interface VectorManifest {
+  version: number
+  createdAt: string
+  provider: KnowledgeConfig['embeddingProvider']
+  model: string
+  modelFingerprint?: string
+  /**
+   * 编码实现的指纹（0031）。旧清单没有这个字段 → 视为不兼容，于是修复编码器之后
+   * 一定触发一次真正的重编码，而不是把旧向量按 contentHash 复用回来。
+   */
+  encoder?: string
+  device: string
+  dimension: number
+  quantization: 'binary+hamming+int8'
+  itemCount: number
+  items: VectorItem[]
+}
+
+export interface LoadedVectorIndex {
+  manifest: VectorManifest
+  binary: Buffer
+  int8: Buffer
+}
+
+/** Validate a generation's BLOBs before either reuse or query. */
+export function decodeVectorIndex(loaded: ReturnType<DerivedDb['vectorIndex']>, encoder: string): LoadedVectorIndex | null {
+  if (!loaded) return null
+  const manifest = loaded.manifest as VectorManifest
+  if (!manifest || manifest.quantization !== 'binary+hamming+int8' || manifest.encoder !== encoder
+    || !Number.isInteger(manifest.dimension) || manifest.dimension < 8 || manifest.dimension % 8 !== 0
+    || !Number.isInteger(manifest.itemCount) || !Array.isArray(manifest.items)
+    || manifest.itemCount !== manifest.items.length
+    || loaded.binary.length !== manifest.itemCount * Math.ceil(manifest.dimension / 8)
+    || loaded.int8.length !== manifest.itemCount * manifest.dimension) return null
+  return { ...loaded, manifest }
+}

package/src/retrieval/visibility.ts

@@ -0,0 +1,14 @@
+import type { ContextGraph } from '../types.js'
+import type { KnowledgeDb } from '../store/knowledge-db.js'
+
+export function liveGraphKeys(db: KnowledgeDb): Set<string> {
+  const rows = db.raw("SELECT entity_type AS type, entity_id AS id FROM card UNION SELECT 'source' AS type, entity_id AS id FROM source").all() as Array<{ type: string; id: string }>
+  return new Set(rows.map(row => `${row.type}:${row.id}`))
+}
+
+/** A retained generation must not restore entities that the user deleted. */
+export function visibleGraph(graph: ContextGraph, live: Set<string>): ContextGraph {
+  const nodes = graph.nodes.filter(node => live.has(`${node.type}:${node.id}`))
+  const keys = new Set(nodes.map(node => `${node.type}:${node.id}`))
+  return { nodes, edges: graph.edges.filter(edge => keys.has(String(edge.from)) && keys.has(String(edge.to))) }
+}