@kira905/dsh-memory-toolkit 0.0.0-stage → 0.1.1

package/memory-engine/lib/sync.mjs

@@ -0,0 +1,180 @@
+// 记忆引擎 v2 · 离线层 —— 增量同步（方案 §3.1 主路径）
+//
+// 主路径：memories.updated_at > meta.lastIncrementalSyncAt 增量扫描（只读 memory.db，不写）
+// 批 50；失败保留进度、下轮续跑；换模型 / 维度变化 → 全量重建。
+// ⚠ 兜底（外置，不与本路径共用候选集）见 sweep.mjs。
+import { textHash } from './hashes.mjs';
+import {
+  META_KEYS,
+  getMetaFor,
+  setMetaFor,
+  getMeta,
+  setMeta,
+  getHashes,
+  upsertVectors,
+  countVectors,
+  deleteSource,
+} from './vectors-store.mjs';
+
+/**
+ * @param {object} p
+ * @param {object} p.source   memory-source / pitfall-source（只读）
+ * @param {object} p.store    vectors-store { db }
+ * @param {object} p.embedder 编码器（encode/encodeBatch）
+ * @param {string} p.modelId
+ * @param {number} p.dim
+ * @param {string} [p.sourceName='memory'] 向量来源名（写入 `vectors.source` 与 meta 命名空间）
+ */
+export async function runIncrementalSync({
+  source,
+  store,
+  embedder,
+  modelId,
+  dim,
+  sourceName = 'memory',
+  batchSize = 50,
+  full = false,
+  dryRun = false,
+  log = () => {},
+  now = () => Date.now(),
+  untilEmpty = true,
+}) {
+  const db = store.db;
+  const src = String(sourceName || 'memory');
+  const startedAt = now();
+  // ⚠️ 水位/模型/维度键一律**按来源**读写（共库不串味：方案 §A.6 首要风险 / §1.3-③.1）
+  const metaModel = getMetaFor(db, META_KEYS.modelId, src);
+  const metaDim = getMetaFor(db, META_KEYS.dim, src);
+  const metaHost = getMeta(db, META_KEYS.hostId);
+
+  const reasons = [];
+  if (metaModel && metaModel !== modelId) reasons.push(`模型变更 ${metaModel} → ${modelId}`);
+  if (metaDim && Number(metaDim) !== Number(dim)) reasons.push(`维度变更 ${metaDim} → ${dim}`);
+  if (full) reasons.push('显式 --full');
+
+  let rebuild = reasons.length > 0;
+  let watermark = 0;
+  let cleared = 0;
+
+  if (rebuild) {
+    if (!dryRun) {
+      cleared = deleteSource(db, src);
+      setMetaFor(db, META_KEYS.modelId, src, modelId);
+      setMetaFor(db, META_KEYS.dim, src, String(dim));
+      setMetaFor(db, META_KEYS.lastIncrementalSyncAt, src, '0');
+    }
+    log(`[sync] 全量重建：${reasons.join(' / ')}（清空 ${cleared} 条向量）`);
+  } else {
+    watermark = Number(getMetaFor(db, META_KEYS.lastIncrementalSyncAt, src) || 0);
+  }
+
+  const stats = {
+    source: src,
+    mode: rebuild ? 'rebuild' : 'incremental',
+    reasons,
+    watermarkStart: watermark,
+    watermarkEnd: watermark,
+    scanned: 0,
+    encoded: 0,
+    skippedSameHash: 0,
+    upserted: 0,
+    batches: 0,
+    failures: [],
+    cleared,
+    vectorsBefore: countVectors(db, src),
+    vectorsAfter: 0,
+    elapsedMs: 0,
+    startedAt,
+    finishedAt: 0,
+  };
+
+  let afterId = 0;
+  let maxSeenUpdatedAt = watermark;
+
+  try {
+    for (;;) {
+      const page = source.incrementalPage(watermark, afterId, batchSize);
+      if (!page.length) break;
+      stats.batches += 1;
+      afterId = page[page.length - 1].id;
+      stats.scanned += page.length;
+      for (const r of page) if (r.updated_at > maxSeenUpdatedAt) maxSeenUpdatedAt = r.updated_at;
+
+      const hashes = getHashes(db, src, page.map((r) => r.id));
+      const need = [];
+      for (const r of page) {
+        const h = textHash(r.text);
+        if (hashes.get(String(r.id)) === h) {
+          stats.skippedSameHash += 1;
+          continue;
+        }
+        need.push({ row: r, hash: h });
+      }
+
+      if (need.length) {
+        const vecs = await embedder.encodeBatch(need.map((n) => n.row.text));
+        if (vecs.length !== need.length) {
+          throw new Error(`编码器返回数量不符：期望 ${need.length}，实际 ${vecs.length}`);
+        }
+        if (!dryRun) {
+          upsertVectors(
+            db,
+            need.map((n, i) => ({
+              source: src,
+              refId: n.row.id,
+              model: modelId,
+              dim: vecs[i].length,
+              vec: vecs[i],
+              textHash: n.hash,
+              updatedAt: n.row.updated_at,
+            })),
+          );
+        }
+        stats.encoded += need.length;
+        stats.upserted += need.length;
+      }
+
+      // 进度前移（失败保留进度可续跑）：用「本批已扫到的最大 updated_at」，重扫边界条幂等
+      if (!dryRun) {
+        setMetaFor(db, META_KEYS.modelId, src, modelId);
+        setMetaFor(db, META_KEYS.dim, src, String(dim));
+        if (!metaHost) setMeta(db, META_KEYS.hostId, String(store.hostId || ''));
+        setMetaFor(db, META_KEYS.lastIncrementalSyncAt, src, String(maxSeenUpdatedAt));
+      }
+      log(
+        `[sync] 批 ${stats.batches}：扫 ${page.length} 条（新编码 ${need.length} / 同哈希跳过 ${
+          page.length - need.length
+        }）水位=${maxSeenUpdatedAt}`,
+      );
+      if (!untilEmpty) break;
+    }
+    stats.watermarkEnd = maxSeenUpdatedAt;
+  } catch (e) {
+    stats.failures.push({ stage: 'sync', message: String(e && e.message ? e.message : e), atBatch: stats.batches });
+    stats.watermarkEnd = maxSeenUpdatedAt; // 已成功批次的水位已落盘
+    stats.error = stats.failures[0].message;
+  }
+
+  stats.vectorsAfter = countVectors(db, src);
+  stats.finishedAt = now();
+  stats.elapsedMs = stats.finishedAt - stats.startedAt;
+  if (!dryRun) {
+    setMetaFor(
+      db,
+      META_KEYS.lastSyncStats,
+      src,
+      JSON.stringify({
+        source: src,
+        mode: stats.mode,
+        scanned: stats.scanned,
+        encoded: stats.encoded,
+        skippedSameHash: stats.skippedSameHash,
+        watermarkEnd: stats.watermarkEnd,
+        elapsedMs: stats.elapsedMs,
+        failures: stats.failures.length,
+        at: new Date(stats.finishedAt).toISOString(),
+      }),
+    );
+  }
+  return stats;
+}

package/memory-engine/lib/tick-hook.mjs

@@ -0,0 +1,218 @@
+// 记忆引擎 v2 · 离线层 —— 坑库向量同步「标记 → 消费」钩子（方案 §A.2 第 10 行 · A 点）
+//
+// 要解决的问题：坑库（butler-pitfalls.json）变了之后，谁去触发 `source='pitfall'` 的增量编码？
+//   方案选的是**松耦合信号**：`pitfall` job 末尾**只置一个标记**（不直接编码、不拖慢 job），
+//   消费方（本模块，由 `memory-vectors` 档或手动 CLI 拉起）看到标记才跑一次坑源增量同步。
+//
+// ⚠️ 这是全方案**唯一"有意双写者"**的文件（方案 §A.4 · C5 2026-10-08 回修）：
+//   ① 置标记方 = `pitfall` job  ── 原子写（临时文件 + rename），**必须写 `updatedAt`**
+//   ② 清标记方 = 本模块消费者    ── **原子读改写**，且**只在标记 `updatedAt` 早于本次消费起点时才清**
+//      （否则会把并发新写入的标记一起清掉 = 丢标记、下轮不编码、还无人告警）
+//   两方**不共用同一把锁**（跨 job 共用会把握手串成串行）；靠 `updatedAt` stale 判定 + 原子 rename 兜住。
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { runIncrementalSync } from './sync.mjs';
+
+export const MARKER_SCHEMA = 1;
+/** 标记文件默认名（登记见 `tools/dsh-butler/lib/state-files.mjs`） */
+export const MARKER_FILENAME = 'butler-pitfall-vector-dirty.json';
+
+/** 默认标记文件路径（DSH_HOME 口径；可注入） */
+export function markerPathFor(dshHome = process.env.DSH_HOME || path.join(os.homedir(), '.dsh')) {
+  return path.join(dshHome, MARKER_FILENAME);
+}
+
+function tmpName(p) {
+  return `${p}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
+/**
+ * Windows 上 `rename` 覆盖一个**正被另一个进程读/改名**的目标会概率性抛
+ * `EPERM/EBUSY/EACCES/EEXIST`（无 flock 语义）⇒ 有限次重试（毫秒级，不改变原子性：
+ * 仍是"临时文件 → rename"两步，读者永远看不到半截内容）。
+ */
+function renameWithRetry(from, to, { tries = 50, waitMs = 2 } = {}) {
+  let lastErr = null;
+  for (let i = 0; i < tries; i += 1) {
+    try {
+      fs.renameSync(from, to);
+      return true;
+    } catch (e) {
+      lastErr = e;
+      if (!['EPERM', 'EBUSY', 'EACCES', 'EEXIST'].includes(e && e.code)) throw e;
+      const until = Date.now() + waitMs;
+      while (Date.now() < until) {
+        /* 忙等几毫秒（同步 API，无法 await） */
+      }
+    }
+  }
+  throw lastErr;
+}
+
+/** 原子写 JSON（临时文件 + 同卷 rename；读者永远不会看到半截 JSON） */
+export function atomicWriteJson(p, obj) {
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  const tmp = tmpName(p);
+  fs.writeFileSync(tmp, `${JSON.stringify(obj, null, 2)}\n`, 'utf8');
+  renameWithRetry(tmp, p);
+  return p;
+}
+
+/**
+ * 读标记。**永不抛错**（半截 JSON / 权限问题 ⇒ 返回 error，由调用方决定 fail-open 还是 fail-closed）。
+ * @returns {{present:boolean, marker:object|null, error:string|null}}
+ */
+export function readMarker(p) {
+  let raw;
+  try {
+    raw = fs.readFileSync(p, 'utf8');
+  } catch (e) {
+    if (e && e.code === 'ENOENT') return { present: false, marker: null, error: null };
+    return { present: false, marker: null, error: String(e && e.message ? e.message : e) };
+  }
+  try {
+    return { present: true, marker: JSON.parse(raw), error: null };
+  } catch (e) {
+    return { present: false, marker: null, error: `半截/非法 JSON：${String(e && e.message ? e.message : e)}` };
+  }
+}
+
+/**
+ * 置标记（**置标记方 API**；`pitfall` job 末尾一行调用）。
+ * @param {string} p 标记文件路径
+ * @param {{at?:number, reason?:string, by?:string, counters?:object}} [opts] `at` 必写（= 本次置标记时刻）
+ */
+export function markDirty(p, opts = {}) {
+  const at = Number(opts.at ?? Date.now());
+  atomicWriteJson(p, {
+    schemaVersion: MARKER_SCHEMA,
+    updatedAt: at,
+    at: new Date(at).toISOString(),
+    reason: opts.reason ?? null,
+    by: opts.by ?? null,
+    device: os.hostname(),
+    counters: opts.counters ?? null,
+  });
+  return { ok: true, path: p, updatedAt: at };
+}
+
+/**
+ * 清标记（**消费方 API**）：原子读改写 + stale 判定。
+ *   · 标记缺失 ⇒ 什么都不做
+ *   · `updatedAt >= consumeStart` ⇒ **保留**（并发新标记，不是这次要消费的那份）
+ *   · `updatedAt < consumeStart` ⇒ 摘除（先写 `consumed` 落点，再 `rename` 摘除主文件；同卷 rename 是原子的）
+ *   残余竞态窗：`复读` 与 `rename` 之间仍有极小窗口（Windows 无 flock 的文件级替代）；
+ *   实跑频率差 ≥ 3 个数量级（生产置标记 30 分钟一次 / 消费秒级），实际不构成丢标记。
+ * @returns {{cleared:boolean, reason:string, consumedUpdatedAt?:number, marker?:object}}
+ */
+export function clearMarkerIfStale(p, consumeStart) {
+  const first = readMarker(p);
+  if (!first.present) return { cleared: false, reason: first.error ? 'unreadable' : 'no-marker' };
+  const at = Number(first.marker?.updatedAt || 0);
+  if (!(at < Number(consumeStart))) return { cleared: false, reason: 'fresh-marker-kept', marker: first.marker };
+
+  const consumed = `${p}.consumed`;
+  atomicWriteJson(consumed, {
+    schemaVersion: MARKER_SCHEMA,
+    clearedAt: Date.now(),
+    consumedUpdatedAt: at,
+  });
+
+  const second = readMarker(p);
+  if (!second.present) {
+    try {
+      fs.rmSync(consumed, { force: true });
+    } catch {
+      /* ignore */
+    }
+    return { cleared: false, reason: 'already-gone' };
+  }
+  if (!(Number(second.marker?.updatedAt || 0) < Number(consumeStart))) {
+    try {
+      fs.rmSync(consumed, { force: true });
+    } catch {
+      /* ignore */
+    }
+    return { cleared: false, reason: 'fresh-marker-kept-on-recheck', marker: second.marker };
+  }
+  try {
+    renameWithRetry(p, consumed);
+  } catch (e) {
+    try {
+      fs.rmSync(consumed, { force: true });
+    } catch {
+      /* ignore */
+    }
+    return { cleared: false, reason: `rename-failed:${e && e.code ? e.code : String(e)}` };
+  }
+  try {
+    fs.rmSync(consumed, { force: true });
+  } catch {
+    /* ignore */
+  }
+  return { cleared: true, reason: 'stale-marker-consumed', consumedUpdatedAt: at };
+}
+
+/**
+ * 消费一轮：有标记 ⇒ 跑坑源增量同步 ⇒ 成功后清标记（失败**不清**，下轮重试）。
+ * @param {object} p
+ * @param {string} p.markerPath
+ * @param {object} p.store / p.source / p.embedder / p.modelId / p.dim
+ */
+export async function runPitfallHook({
+  markerPath,
+  store,
+  source,
+  embedder,
+  modelId,
+  dim,
+  batchSize = 50,
+  full = false,
+  dryRun = false,
+  log = () => {},
+  now = () => Date.now(),
+}) {
+  const consumeStart = now();
+  // 🔴 fail-closed 守卫（2026-10-10 实测踩到过）：调用方若把**别的来源**当坑源传进来，
+  //    本函数会一边按 pitfall 命名空间写向量、一边扫记忆行 ⇒ 静默把记忆行错标成坑向量。
+  //    ⇒ 来源名不对就**拒跑**（宁可少跑一轮，绝不污染命名空间）。
+  const srcName = source && source.sourceName ? String(source.sourceName) : null;
+  if (srcName !== 'pitfall') {
+    throw new Error(
+      `runPitfallHook 只接受坑源（sourceName='pitfall'），收到 ${srcName ?? '(未知来源/无 sourceName)'}`
+        + ' ⇒ 拒跑（否则会把该来源的行错标成 pitfall 向量）',
+    );
+  }
+  const r = readMarker(markerPath);
+  if (r.error) {
+    log(`[tick-hook] 标记文件读不动（本轮跳过、不清除）：${r.error}`);
+    return { ok: false, skipped: 'marker-unreadable', error: r.error, consumeStart };
+  }
+  if (!r.present) return { ok: true, skipped: 'no-marker', consumeStart };
+
+  log(`[tick-hook] 标记在位（updatedAt=${r.marker?.updatedAt ?? '?'}，reason=${r.marker?.reason ?? '-'}）⇒ 跑坑源增量同步`);
+  const stats = await runIncrementalSync({
+    source,
+    store,
+    embedder,
+    modelId,
+    dim,
+    sourceName: 'pitfall',
+    batchSize,
+    full,
+    dryRun,
+    log,
+    now,
+  });
+  let cleared = { cleared: false, reason: 'not-attempted' };
+  if (dryRun) cleared = { cleared: false, reason: 'dry-run' };
+  else if (stats.error) {
+    log(`[tick-hook] 同步失败 ⇒ **保留标记**下轮重试：${stats.error}`);
+    cleared = { cleared: false, reason: 'sync-failed-keep-marker' };
+  } else {
+    cleared = clearMarkerIfStale(markerPath, consumeStart);
+    log(`[tick-hook] 清标记：${JSON.stringify(cleared)}`);
+  }
+  return { ok: !stats.error, consumeStart, marker: r.marker, stats, cleared };
+}

package/memory-engine/lib/vectors-store.mjs

@@ -0,0 +1,292 @@
+// 记忆引擎 v2 · 离线层 —— 独立向量库（方案 §3.1）
+//
+// 为什么独立库（D3 已拍）：memory.db 是 DELETE 模式 + busy_timeout=0，
+// 任何第三个写者都可能让主人的 memory_write 失败。向量是可重建的派生物。
+import fs from 'node:fs';
+import path from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { SCHEMA_VERSION } from './config.mjs';
+
+const DDL = `
+CREATE TABLE IF NOT EXISTS vectors (
+  source     TEXT    NOT NULL,
+  ref_id     TEXT    NOT NULL,
+  model      TEXT    NOT NULL,
+  dim        INTEGER NOT NULL,
+  vec        BLOB    NOT NULL,
+  text_hash  TEXT    NOT NULL,
+  updated_at INTEGER NOT NULL,
+  PRIMARY KEY (source, ref_id)
+);
+CREATE TABLE IF NOT EXISTS meta (
+  k TEXT PRIMARY KEY,
+  v TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS vectors_source_model ON vectors (source, model);
+`;
+
+/** meta 键名（方案 §3.1 要求的前 6 项；后两项为兜底轮转的必需运行状态） */
+export const META_KEYS = {
+  schemaVersion: 'schemaVersion',
+  hostId: 'hostId',
+  modelId: 'modelId',
+  dim: 'dim',
+  lastIncrementalSyncAt: 'lastIncrementalSyncAt',
+  lastFullHashSweepAt: 'lastFullHashSweepAt',
+  hashSweepCursor: 'hashSweepCursor',
+  lastSyncStats: 'lastSyncStats',
+};
+
+/**
+ * 按「来源」命名空间的 meta 键前缀（记忆 v2 §3 A.4 / 2026-10-10 落地）。
+ *
+ * 为什么必须分命名空间：坑库（`source='pitfall'`）与记忆（`source='memory'`）**共用一个 vectors.db**，
+ * 两者的水位/游标/模型键若同名会互相覆盖 ⇒ 表现为「记忆增量漏编码且不报错」（方案 §1.3-③.1 / §A.6 首要风险）。
+ * ⇒ 这四个键（+ 两个运行状态键）一律写成 `<key>:<source>`。
+ */
+export const SOURCE_SCOPED_META_KEYS = [
+  'modelId',
+  'dim',
+  'lastIncrementalSyncAt',
+  'hashSweepCursor',
+  'lastFullHashSweepAt',
+  'lastSyncStats',
+];
+
+/** `<baseKey>:<source>`（来源命名空间的 meta 键名） */
+export function metaKeyFor(baseKey, source) {
+  return `${baseKey}:${source}`;
+}
+
+/**
+ * 读 meta（按来源）：先读命名空间键；读不到且来源是 memory ⇒ **兼容读**无后缀旧键
+ * （老库尚未迁移的那一次；语义等价，不改变任何默认行为）。
+ */
+export function getMetaFor(db, baseKey, source) {
+  const v = getMeta(db, metaKeyFor(baseKey, source));
+  if (v != null) return v;
+  if (String(source) === 'memory') return getMeta(db, baseKey);
+  return null;
+}
+
+/**
+ * 写 meta（按来源）+ **冗余写**（方案 §A.4 的 C4 选项①）：
+ * memory 侧同时把无后缀旧键写成同值 ⇒ 回滚到旧代码时读到的水位仍然正确，
+ * 不会走「首次全量」分支把整库 DELETE 重建（626 条 ≈ 102 s 的一次数据面事件）。
+ */
+export function setMetaFor(db, baseKey, source, value) {
+  setMeta(db, metaKeyFor(baseKey, source), value);
+  if (String(source) === 'memory') setMeta(db, baseKey, value);
+}
+
+/**
+ * 一次性命名空间迁移（**幂等**）：把无后缀旧键的现值播种到 `<key>:memory`。
+ * 只在「旧键在 且 新键缺」时写 ⇒ 跑多少次都只迁移一次；旧键**保留不动**（= 冗余写）。
+ * @returns {string[]} 本次实际迁移的键名
+ */
+export function migrateMetaNamespace(db) {
+  const migrated = [];
+  for (const k of SOURCE_SCOPED_META_KEYS) {
+    const old = getMeta(db, k);
+    if (old == null) continue;
+    if (getMeta(db, metaKeyFor(k, 'memory')) != null) continue;
+    setMeta(db, metaKeyFor(k, 'memory'), old);
+    migrated.push(k);
+  }
+  return migrated;
+}
+
+function tableExists(db, name) {
+  const r = db
+    .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?")
+    .get(name);
+  return Boolean(r);
+}
+
+export function ensureSchema(db) {
+  const created = [];
+  if (!tableExists(db, 'vectors')) created.push('vectors');
+  if (!tableExists(db, 'meta')) created.push('meta');
+  db.exec(DDL);
+  return { created, tables: { vectors: tableExists(db, 'vectors'), meta: tableExists(db, 'meta') } };
+}
+
+/**
+ * 打开（必要时创建）向量库。
+ * @param {string} dbPath
+ * @param {{create?: boolean}} [opts]
+ */
+export function openVectorsStore(dbPath, opts = {}) {
+  const create = opts.create !== false;
+  if (create) fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+  if (create === false && !fs.existsSync(dbPath)) {
+    throw new Error(`向量库不存在: ${dbPath}`);
+  }
+  const db = new DatabaseSync(dbPath);
+  db.exec('PRAGMA busy_timeout = 5000;');
+  try {
+    db.exec('PRAGMA journal_mode = WAL;');
+  } catch {
+    /* 只读介质上 WAL 可能不可用，退回默认模式（本机不属此情况） */
+  }
+  const schema = ensureSchema(db);
+  // 【2026-10-10 · 补坑库联动A】meta 命名空间一次性迁移（幂等；旧键保留 = 冗余写，可回滚）
+  const migrated = migrateMetaNamespace(db);
+  const sv = getMeta(db, META_KEYS.schemaVersion);
+  if (sv == null) {
+    setMeta(db, META_KEYS.schemaVersion, SCHEMA_VERSION);
+  } else if (migrated.length && sv !== SCHEMA_VERSION) {
+    // 迁移实际发生 ⇒ 升版本号（表结构未变，只是键位扩了命名空间）
+    setMeta(db, META_KEYS.schemaVersion, SCHEMA_VERSION);
+  }
+  return { db, dbPath, schema, migrated };
+}
+
+export function getMeta(db, k) {
+  const r = db.prepare('SELECT v FROM meta WHERE k = ?').get(k);
+  return r ? r.v : null;
+}
+
+export function setMeta(db, k, v) {
+  db.prepare(
+    'INSERT INTO meta (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v',
+  ).run(String(k), String(v));
+}
+
+export function getMetaAll(db) {
+  const out = {};
+  for (const r of db.prepare('SELECT k, v FROM meta ORDER BY k').all()) out[r.k] = r.v;
+  return out;
+}
+
+export function vecToBlob(vec) {
+  const f = vec instanceof Float32Array ? vec : Float32Array.from(vec);
+  return Buffer.from(f.buffer, f.byteOffset, f.byteLength);
+}
+
+export function blobToVec(blob) {
+  // 注意：sqlite 返回的 Buffer 可能是池化分配（byteOffset 不对齐 4 字节）⇒ 必须切片后重建
+  const ab = blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength);
+  return new Float32Array(ab);
+}
+
+/** 批量 upsert（单事务） */
+export function upsertVectors(db, items) {
+  if (!items.length) return 0;
+  const stmt = db.prepare(`
+    INSERT INTO vectors (source, ref_id, model, dim, vec, text_hash, updated_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+    ON CONFLICT(source, ref_id) DO UPDATE SET
+      model = excluded.model,
+      dim = excluded.dim,
+      vec = excluded.vec,
+      text_hash = excluded.text_hash,
+      updated_at = excluded.updated_at
+  `);
+  db.exec('BEGIN');
+  try {
+    for (const it of items) {
+      stmt.run(
+        it.source,
+        String(it.refId),
+        it.model,
+        it.dim,
+        vecToBlob(it.vec),
+        it.textHash,
+        it.updatedAt,
+      );
+    }
+    db.exec('COMMIT');
+  } catch (e) {
+    try {
+      db.exec('ROLLBACK');
+    } catch {
+      /* ignore */
+    }
+    throw e;
+  }
+  return items.length;
+}
+
+/** 取指定 ref_id 的已有哈希（增量同步的跳过判据） */
+export function getHashes(db, source, refIds) {
+  const out = new Map();
+  if (!refIds.length) return out;
+  const stmt = db.prepare('SELECT text_hash FROM vectors WHERE source = ? AND ref_id = ?');
+  for (const id of refIds) {
+    const r = stmt.get(source, String(id));
+    if (r) out.set(String(id), r.text_hash);
+  }
+  return out;
+}
+
+export function countVectors(db, source) {
+  const r = source
+    ? db.prepare('SELECT COUNT(*) AS n FROM vectors WHERE source = ?').get(source)
+    : db.prepare('SELECT COUNT(*) AS n FROM vectors').get();
+  return r.n;
+}
+
+export function countByModel(db, source, model) {
+  return db
+    .prepare('SELECT COUNT(*) AS n FROM vectors WHERE source = ? AND model = ?')
+    .get(source, model).n;
+}
+
+export function deleteSource(db, source) {
+  const r = db.prepare('DELETE FROM vectors WHERE source = ?').run(source);
+  return Number(r.changes || 0);
+}
+
+/**
+ * 载入整个矩阵（召回用；万级候选纯内存暴力检索，方案 §3.3）。
+ * @returns {{refIds: string[], refIdsNum: number[], dim: number, data: Float32Array, count: number, textHashes: string[], models: string[]}}
+ */
+export function loadMatrix(db, source, model = null) {
+  const rows = model
+    ? db
+        .prepare(
+          'SELECT ref_id, model, dim, vec, text_hash FROM vectors WHERE source = ? AND model = ? ORDER BY CAST(ref_id AS INTEGER)',
+        )
+        .all(source, model)
+    : db
+        .prepare(
+          'SELECT ref_id, model, dim, vec, text_hash FROM vectors WHERE source = ? ORDER BY CAST(ref_id AS INTEGER)',
+        )
+        .all(source);
+  if (!rows.length) {
+    return {
+      refIds: [],
+      refIdsNum: [],
+      dim: 0,
+      data: new Float32Array(0),
+      count: 0,
+      textHashes: [],
+      models: [],
+    };
+  }
+  const dim = rows[0].dim;
+  const data = new Float32Array(rows.length * dim);
+  rows.forEach((r, i) => {
+    if (r.dim !== dim) {
+      throw new Error(
+        `向量库维度不一致：ref_id=${r.ref_id} dim=${r.dim}，首个为 ${dim}（应全量重建）`,
+      );
+    }
+    const v = blobToVec(r.vec);
+    if (v.length !== dim) {
+      throw new Error(`向量 blob 长度异常：ref_id=${r.ref_id} len=${v.length} 期望 ${dim}`);
+    }
+    data.set(v, i * dim);
+  });
+  const models = [...new Set(rows.map((r) => r.model))];
+  return {
+    refIds: rows.map((r) => String(r.ref_id)),
+    refIdsNum: rows.map((r) => Number(r.ref_id)),
+    dim,
+    data,
+    count: rows.length,
+    textHashes: rows.map((r) => r.text_hash),
+    models,
+  };
+}