luckytri 1.1.2 → 1.2.0

package/server/knowledge/manager.js

@@ -67,9 +67,14 @@ function allowedCollection(db, collection, session) {
 }
 
 export class KnowledgeManager {
-  constructor(repo, models) {
+  // `embedding` is her one embedding service. When given, a question is
+  // embedded through it: the vector the recall index just asked for is
+  // reused from its cache, a slow service gives up after a few seconds
+  // instead of holding the reply, and repeated failures pause it for both.
+  constructor(repo, models, { embedding = null } = {}) {
     this.repo = repo;
     this.models = models;
+    this.embedding = embedding;
   }
   collections(session = "") {
     return this.repo.db
@@ -269,8 +274,11 @@ export class KnowledgeManager {
           (!signature ||
             chunk.embedding_profile === signature ||
             (legacy && !chunk.embedding_profile))
-        )
-          score += cosine(vec, unpackVector(chunk.embedding)) * 4;
+        ) {
+          // A vector of another length (an old unsigned chunk) is not comparable.
+          const stored = unpackVector(chunk.embedding);
+          if (stored?.length === vec.length) score += cosine(vec, stored) * 4;
+        }
         const ageDays = (cutoff - (chunk.document_created || 0)) / 86400000;
         if (ageDays > 30) score *= 0.85;
         if (score > (best.get(chunk.id) || 0)) best.set(chunk.id, score);
@@ -308,11 +316,20 @@ export class KnowledgeManager {
       .get();
     if (profile.embedding && queries.length && searchable) {
       try {
-        vectors = await embedBatches(
-          this.models,
-          profile,
-          queries.map((query) => query.slice(0, 2000)),
-        );
+        vectors = this.embedding
+          ? this.embedding.available()
+            ? (
+                await this.embedding.embed(queries, {
+                  purpose: "query",
+                  session,
+                })
+              ).map((vector) => Array.from(vector))
+            : []
+          : await embedBatches(
+              this.models,
+              profile,
+              queries.map((query) => query.slice(0, 2000)),
+            );
       } catch {
         vectors = [];
       }
@@ -322,7 +339,7 @@ export class KnowledgeManager {
   async reembedDocument(id) {
     const rows = this.repo.db
       .prepare(
-        "SELECT id,text FROM core_chunks WHERE document_id=? ORDER BY ordinal",
+        "SELECT id,text,embedding_profile,previous_profile FROM core_chunks WHERE document_id=? ORDER BY ordinal",
       )
       .all(id);
     if (!rows.length) throw Error("文档没有可建立向量的段落");
@@ -331,37 +348,89 @@ export class KnowledgeManager {
       this.models,
     );
     if (!profile.embedding) throw Error("请先启用知识向量模型");
-    const vectors = await embedBatches(
+    const { vectors, ask } = await this.#vectorsFor(rows, signature, (texts) =>
+      embedBatches(this.models, profile, texts),
+    );
+    if (!this.#saveVectors(rows, vectors, signature, id))
+      throw Error("文档已变化，请重新建立向量");
+    return { chunks: rows.length, embedded: ask.length };
+  }
+  // After a model or length change, chunks made under the old profile are
+  // searched by words only. This brings them over a few at a time on the
+  // maintenance tick, through the paced embedding service. Returns how many
+  // chunks moved; 0 when nothing is left or the service is not available.
+  async reembedStale({ limit = 20 } = {}) {
+    if (!this.embedding?.available()) return 0;
+    const { signature, legacy } = resolveEmbeddingProfile(
+      this.repo,
       this.models,
-      profile,
-      rows.map((row) => row.text),
     );
+    if (!signature) return 0;
+    const rows = this.repo.db
+      .prepare(
+        `SELECT c.id,c.text,c.embedding_profile,c.previous_profile FROM core_chunks c
+         JOIN core_documents d ON d.id=c.document_id
+         WHERE d.status='ready' AND c.embedding IS NOT NULL AND c.embedding_profile!=?
+         ${legacy ? "AND c.embedding_profile!=''" : ""}
+         ORDER BY c.created DESC LIMIT ?`,
+      )
+      .all(signature, limit);
+    if (!rows.length) return 0;
+    const { vectors } = await this.#vectorsFor(rows, signature, (texts) =>
+      this.embedding.embed(texts, { purpose: "document" }),
+    );
+    this.#saveVectors(rows, vectors, signature);
+    return rows.length;
+  }
+  // A chunk whose previous vector already belongs to this profile swaps
+  // back; only the rest are asked for.
+  async #vectorsFor(rows, signature, embed) {
+    const ask = rows.filter((row) => row.previous_profile !== signature);
+    const made = ask.length ? await embed(ask.map((row) => row.text)) : [];
+    const vectors = new Map(ask.map((row, i) => [row.id, made[i]]));
+    return { vectors, ask };
+  }
+  // The current vector becomes the previous one, so nothing is thrown away
+  // until a third profile arrives. A trigger re-indexes the chunk.
+  #saveVectors(rows, vectors, signature, documentId = null) {
     const db = this.repo.db;
-    db.exec("SAVEPOINT reembed_document");
+    const fresh = db.prepare(
+      `UPDATE core_chunks SET
+         previous_embedding=CASE WHEN embedding_profile!=? THEN embedding ELSE previous_embedding END,
+         previous_profile=CASE WHEN embedding_profile!=? THEN embedding_profile ELSE previous_profile END,
+         embedding=?,embedding_profile=?
+       WHERE id=? AND text=?`,
+    );
+    const swap = db.prepare(
+      `UPDATE core_chunks SET
+         embedding=previous_embedding,embedding_profile=previous_profile,
+         previous_embedding=embedding,previous_profile=embedding_profile
+       WHERE id=? AND previous_profile=?`,
+    );
+    db.exec("SAVEPOINT reembed_chunks");
     try {
-      const update = db.prepare(
-        "UPDATE core_chunks SET embedding=?,embedding_profile=? WHERE id=? AND document_id=? AND text=?",
-      );
-      rows.forEach((row, index) => {
-        if (
-          !update.run(
-            packVector(vectors[index]),
-            signature,
-            row.id,
-            id,
-            row.text,
-          ).changes
-        )
-          throw Error("文档已变化，请重新建立向量");
-      });
-      db.exec("RELEASE reembed_document");
-    } catch (error) {
-      db.exec("ROLLBACK TO reembed_document");
-      db.exec("RELEASE reembed_document");
-      throw error;
+      for (const row of rows) {
+        const vector = vectors.get(row.id);
+        const changed = vector
+          ? fresh.run(
+              signature,
+              signature,
+              packVector(vector),
+              signature,
+              row.id,
+              row.text,
+            ).changes
+          : swap.run(row.id, signature).changes;
+        if (!changed && documentId) throw Error("changed");
+      }
+      db.exec("RELEASE reembed_chunks");
+    } catch {
+      db.exec("ROLLBACK TO reembed_chunks");
+      db.exec("RELEASE reembed_chunks");
+      return false;
     }
     this.repo.store.revision++;
-    return { chunks: rows.length };
+    return true;
   }
   removeDocument(id) {
     const row = this.repo.db

package/server/knowledge/schema.js

@@ -1,158 +1,197 @@
-import { ftsTokens } from "./retrieval.js";
-
-export function indexMemory(db, id, content) {
-  db.prepare("DELETE FROM core_memory_fts WHERE memory_id=?").run(id);
-  const tokens = ftsTokens(content);
-  if (tokens)
-    db.prepare(
-      "INSERT INTO core_memory_fts(memory_id, tokens) VALUES (?,?)",
-    ).run(id, tokens);
-}
-
-export function indexChunk(db, id, content) {
-  db.prepare("DELETE FROM core_chunk_fts WHERE chunk_id=?").run(id);
-  const tokens = ftsTokens(content);
-  if (tokens)
-    db.prepare("INSERT INTO core_chunk_fts(chunk_id, tokens) VALUES (?,?)").run(
-      id,
-      tokens,
-    );
-}
-
-function mergeLegacyMemories(db) {
-  if (db.prepare("SELECT id FROM core_config WHERE id='memory-merge-v1'").get())
-    return;
-  db.exec("BEGIN IMMEDIATE");
-  try {
-    for (const m of db.prepare("SELECT * FROM memories").all()) {
-      const sessionId =
-        m.scope === "shared"
-          ? "__shared__"
-          : m.scope === "private"
-            ? `__private__:${m.user_id}`
-            : m.scope;
-      const id = "legacy:" + m.id;
-      if (db.prepare("SELECT id FROM core_memories WHERE id=?").get(id))
-        continue;
-      db.prepare(
-        "INSERT INTO core_memories(id,session_id,subject,content,type,confidence,importance,status,locked,sources,created,updated) VALUES (?,?,?,?,'reviewed',1,1,'confirmed',1,'[]',?,?)",
-      ).run(
-        id,
-        sessionId,
-        m.user_id,
-        m.content,
-        m.time || Date.now(),
-        Date.now(),
-      );
-      indexMemory(db, id, m.content);
-    }
-    db.prepare("INSERT INTO core_config(id,value) VALUES (?,?)").run(
-      "memory-merge-v1",
-      JSON.stringify({ time: Date.now() }),
-    );
-    db.exec("COMMIT");
-  } catch (e) {
-    db.exec("ROLLBACK");
-    throw e;
-  }
-}
-
-export const KNOWLEDGE_TRIGGERS = [
-  `CREATE TRIGGER IF NOT EXISTS core_vector_changed AFTER UPDATE OF embedding ON core_chunks BEGIN
-      DELETE FROM core_vector_buckets WHERE chunk_id=NEW.id;
-      UPDATE core_chunks SET vector_indexed=0 WHERE id=NEW.id;
-    END`,
-  `CREATE TRIGGER IF NOT EXISTS core_vector_deleted AFTER DELETE ON core_chunks BEGIN
-      DELETE FROM core_vector_buckets WHERE chunk_id=OLD.id;
-    END`,
-];
-
-export function migrateKnowledge(db) {
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS core_collections (
-      id TEXT PRIMARY KEY,
-      name TEXT NOT NULL,
-      scope TEXT NOT NULL,
-      created INTEGER NOT NULL,
-      updated INTEGER NOT NULL
-    );
-    CREATE TABLE IF NOT EXISTS core_collection_sessions (
-      collection_id TEXT NOT NULL,
-      session_id TEXT NOT NULL,
-      PRIMARY KEY(collection_id, session_id)
-    );
-    CREATE TABLE IF NOT EXISTS core_documents (
-      id TEXT PRIMARY KEY,
-      collection_id TEXT NOT NULL,
-      title TEXT NOT NULL,
-      source TEXT,
-      path TEXT,
-      status TEXT NOT NULL,
-      error TEXT,
-      created INTEGER NOT NULL,
-      updated INTEGER NOT NULL,
-      bytes INTEGER,
-      hash TEXT
-    );
-    CREATE TABLE IF NOT EXISTS core_chunks (
-      id TEXT PRIMARY KEY,
-      document_id TEXT NOT NULL,
-      collection_id TEXT NOT NULL,
-      ordinal INTEGER NOT NULL,
-      heading TEXT,
-      text TEXT NOT NULL,
-      tokens TEXT,
-      embedding BLOB,
-      hash TEXT,
-      created INTEGER NOT NULL
-    );
-  `);
-  db.exec(
-    "CREATE VIRTUAL TABLE IF NOT EXISTS core_memory_fts USING fts5(memory_id UNINDEXED, tokens)",
-  );
-  db.exec(
-    "CREATE VIRTUAL TABLE IF NOT EXISTS core_chunk_fts USING fts5(chunk_id UNINDEXED, tokens)",
-  );
-  if (
-    !db
-      .prepare("PRAGMA table_info(core_chunks)")
-      .all()
-      .some((c) => c.name === "vector_indexed")
-  )
-    db.exec(
-      "ALTER TABLE core_chunks ADD COLUMN vector_indexed INTEGER NOT NULL DEFAULT 0",
-    );
-  if (
-    !db
-      .prepare("PRAGMA table_info(core_chunks)")
-      .all()
-      .some((c) => c.name === "embedding_profile")
-  )
-    db.exec(
-      "ALTER TABLE core_chunks ADD COLUMN embedding_profile TEXT NOT NULL DEFAULT ''",
-    );
-  db.exec(`CREATE INDEX IF NOT EXISTS core_chunks_collection ON core_chunks(collection_id, document_id);
-    CREATE INDEX IF NOT EXISTS core_chunks_unindexed ON core_chunks(vector_indexed) WHERE embedding IS NOT NULL;
-    CREATE TABLE IF NOT EXISTS core_vector_buckets(dimension INTEGER NOT NULL,band INTEGER NOT NULL,bucket INTEGER NOT NULL,chunk_id TEXT NOT NULL,PRIMARY KEY(dimension,band,bucket,chunk_id));
-    CREATE INDEX IF NOT EXISTS core_vector_buckets_chunk ON core_vector_buckets(chunk_id);`);
-  db.exec(KNOWLEDGE_TRIGGERS.join(";") + ";");
-  if (
-    !db
-      .prepare("SELECT id FROM core_collections WHERE id='shared-default'")
-      .get()
-  )
-    db.prepare(
-      "INSERT INTO core_collections(id,name,scope,created,updated) VALUES ('shared-default','共享知识库','shared',?,?)",
-    ).run(Date.now(), Date.now());
-  mergeLegacyMemories(db);
-  if (
-    !db.prepare("SELECT id FROM core_config WHERE id='memory-fts-v1'").get()
-  ) {
-    for (const m of db.prepare("SELECT id,content FROM core_memories").all())
-      indexMemory(db, m.id, m.content);
-    db.prepare("INSERT INTO core_config(id,value) VALUES (?,?)").run(
-      "memory-fts-v1",
-      JSON.stringify({ time: Date.now() }),
-    );
-  }
-}
+import { ftsTokens } from "./retrieval.js";
+
+export function indexMemory(db, id, content) {
+  db.prepare("DELETE FROM core_memory_fts WHERE memory_id=?").run(id);
+  const tokens = ftsTokens(content);
+  if (tokens)
+    db.prepare(
+      "INSERT INTO core_memory_fts(memory_id, tokens) VALUES (?,?)",
+    ).run(id, tokens);
+}
+
+export function indexChunk(db, id, content) {
+  db.prepare("DELETE FROM core_chunk_fts WHERE chunk_id=?").run(id);
+  const tokens = ftsTokens(content);
+  if (tokens)
+    db.prepare("INSERT INTO core_chunk_fts(chunk_id, tokens) VALUES (?,?)").run(
+      id,
+      tokens,
+    );
+}
+
+function mergeLegacyMemories(db) {
+  if (db.prepare("SELECT id FROM core_config WHERE id='memory-merge-v1'").get())
+    return;
+  db.exec("BEGIN IMMEDIATE");
+  try {
+    for (const m of db.prepare("SELECT * FROM memories").all()) {
+      const sessionId =
+        m.scope === "shared"
+          ? "__shared__"
+          : m.scope === "private"
+            ? `__private__:${m.user_id}`
+            : m.scope;
+      const id = "legacy:" + m.id;
+      if (db.prepare("SELECT id FROM core_memories WHERE id=?").get(id))
+        continue;
+      db.prepare(
+        "INSERT INTO core_memories(id,session_id,subject,content,type,confidence,importance,status,locked,sources,created,updated) VALUES (?,?,?,?,'reviewed',1,1,'confirmed',1,'[]',?,?)",
+      ).run(
+        id,
+        sessionId,
+        m.user_id,
+        m.content,
+        m.time || Date.now(),
+        Date.now(),
+      );
+      indexMemory(db, id, m.content);
+    }
+    db.prepare("INSERT INTO core_config(id,value) VALUES (?,?)").run(
+      "memory-merge-v1",
+      JSON.stringify({ time: Date.now() }),
+    );
+    db.exec("COMMIT");
+  } catch (e) {
+    db.exec("ROLLBACK");
+    throw e;
+  }
+}
+
+export const KNOWLEDGE_TRIGGERS = [
+  `CREATE TRIGGER IF NOT EXISTS core_vector_changed AFTER UPDATE OF embedding ON core_chunks BEGIN
+      DELETE FROM core_vector_buckets WHERE chunk_id=NEW.id;
+      UPDATE core_chunks SET vector_indexed=0 WHERE id=NEW.id;
+    END`,
+  `CREATE TRIGGER IF NOT EXISTS core_vector_deleted AFTER DELETE ON core_chunks BEGIN
+      DELETE FROM core_vector_buckets WHERE chunk_id=OLD.id;
+    END`,
+];
+
+export function migrateKnowledge(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS core_collections (
+      id TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      scope TEXT NOT NULL,
+      created INTEGER NOT NULL,
+      updated INTEGER NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS core_collection_sessions (
+      collection_id TEXT NOT NULL,
+      session_id TEXT NOT NULL,
+      PRIMARY KEY(collection_id, session_id)
+    );
+    CREATE TABLE IF NOT EXISTS core_documents (
+      id TEXT PRIMARY KEY,
+      collection_id TEXT NOT NULL,
+      title TEXT NOT NULL,
+      source TEXT,
+      path TEXT,
+      status TEXT NOT NULL,
+      error TEXT,
+      created INTEGER NOT NULL,
+      updated INTEGER NOT NULL,
+      bytes INTEGER,
+      hash TEXT
+    );
+    CREATE TABLE IF NOT EXISTS core_chunks (
+      id TEXT PRIMARY KEY,
+      document_id TEXT NOT NULL,
+      collection_id TEXT NOT NULL,
+      ordinal INTEGER NOT NULL,
+      heading TEXT,
+      text TEXT NOT NULL,
+      tokens TEXT,
+      embedding BLOB,
+      hash TEXT,
+      created INTEGER NOT NULL
+    );
+  `);
+  db.exec(
+    "CREATE VIRTUAL TABLE IF NOT EXISTS core_memory_fts USING fts5(memory_id UNINDEXED, tokens)",
+  );
+  db.exec(
+    "CREATE VIRTUAL TABLE IF NOT EXISTS core_chunk_fts USING fts5(chunk_id UNINDEXED, tokens)",
+  );
+  if (
+    !db
+      .prepare("PRAGMA table_info(core_chunks)")
+      .all()
+      .some((c) => c.name === "vector_indexed")
+  )
+    db.exec(
+      "ALTER TABLE core_chunks ADD COLUMN vector_indexed INTEGER NOT NULL DEFAULT 0",
+    );
+  if (
+    !db
+      .prepare("PRAGMA table_info(core_chunks)")
+      .all()
+      .some((c) => c.name === "embedding_profile")
+  )
+    db.exec(
+      "ALTER TABLE core_chunks ADD COLUMN embedding_profile TEXT NOT NULL DEFAULT ''",
+    );
+  // The vector a chunk had under the profile before the current one, so
+  // switching back costs no requests.
+  for (const [name, type] of [
+    ["previous_embedding", "BLOB"],
+    ["previous_profile", "TEXT NOT NULL DEFAULT ''"],
+  ])
+    if (
+      !db
+        .prepare("PRAGMA table_info(core_chunks)")
+        .all()
+        .some((c) => c.name === name)
+    )
+      db.exec(`ALTER TABLE core_chunks ADD COLUMN ${name} ${type}`);
+  db.exec(`CREATE INDEX IF NOT EXISTS core_chunks_collection ON core_chunks(collection_id, document_id);
+    CREATE INDEX IF NOT EXISTS core_chunks_unindexed ON core_chunks(vector_indexed) WHERE embedding IS NOT NULL;
+    CREATE TABLE IF NOT EXISTS core_vector_buckets(dimension INTEGER NOT NULL,band INTEGER NOT NULL,bucket INTEGER NOT NULL,chunk_id TEXT NOT NULL,PRIMARY KEY(dimension,band,bucket,chunk_id));
+    CREATE INDEX IF NOT EXISTS core_vector_buckets_chunk ON core_vector_buckets(chunk_id);`);
+  db.exec(KNOWLEDGE_TRIGGERS.join(";") + ";");
+  if (
+    !db
+      .prepare("SELECT id FROM core_collections WHERE id='shared-default'")
+      .get()
+  )
+    db.prepare(
+      "INSERT INTO core_collections(id,name,scope,created,updated) VALUES ('shared-default','共享知识库','shared',?,?)",
+    ).run(Date.now(), Date.now());
+  mergeLegacyMemories(db);
+  if (
+    !db.prepare("SELECT id FROM core_config WHERE id='memory-fts-v2'").get()
+  ) {
+    db.exec("BEGIN IMMEDIATE");
+    try {
+      db.exec("DELETE FROM core_memory_fts");
+      for (const m of db.prepare("SELECT id,content FROM core_memories").all())
+        indexMemory(db, m.id, m.content);
+      db.prepare("INSERT INTO core_config(id,value) VALUES (?,?)").run(
+        "memory-fts-v2",
+        JSON.stringify({ time: Date.now() }),
+      );
+      db.exec("COMMIT");
+    } catch (error) {
+      db.exec("ROLLBACK");
+      throw error;
+    }
+  }
+  if (!db.prepare("SELECT id FROM core_config WHERE id='chunk-fts-v1'").get()) {
+    db.exec("BEGIN IMMEDIATE");
+    try {
+      db.exec("DELETE FROM core_chunk_fts");
+      for (const chunk of db
+        .prepare("SELECT id,text FROM core_chunks ORDER BY rowid")
+        .iterate())
+        indexChunk(db, chunk.id, chunk.text);
+      db.prepare("INSERT INTO core_config(id,value) VALUES (?,?)").run(
+        "chunk-fts-v1",
+        JSON.stringify({ time: Date.now() }),
+      );
+      db.exec("COMMIT");
+    } catch (error) {
+      db.exec("ROLLBACK");
+      throw error;
+    }
+  }
+}