luckytri 1.0.0

package/server/knowledge/manager.js

@@ -0,0 +1,386 @@
+import { createHash, randomUUID } from "node:crypto";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { runtimePaths } from "../paths.js";
+import {
+  asPlainDocument,
+  chunkText,
+  cosine,
+  ftsMatchQuery,
+  overlapScore,
+  lexicalTerms,
+  packVector,
+  unpackVector,
+} from "./retrieval.js";
+import { indexChunk } from "./schema.js";
+import { ensureVectorIndex, vectorCandidates } from "./vector-index.js";
+import { resolveEmbeddingProfile, embedBatches } from "./embedding-profile.js";
+import {
+  parseSessionKey,
+  scopedSessionAliases,
+} from "../channels/session-key.js";
+
+function scopesFor(db, session) {
+  try {
+    const parsed = parseSessionKey(session);
+    const aliases = scopedSessionAliases(db, session);
+    return {
+      aliases,
+      kind: parsed.kind,
+      privateKey: `__private__:${parsed.nativeId}`,
+    };
+  } catch {
+    return { aliases: [session], kind: "group", privateKey: "" };
+  }
+}
+
+function saidBy(rows) {
+  const groups = new Map();
+  let anon = 0;
+  for (const row of rows || []) {
+    if (row?.role === "assistant") continue;
+    const id = String(row?.userId || "");
+    const key = id || `\0${anon++}`;
+    const texts = groups.get(key) || [];
+    if (row?.text) texts.push(String(row.text));
+    groups.set(key, texts);
+  }
+  return [...groups.values()]
+    .map((texts) => texts.join(" "))
+    .filter((text) => text.trim());
+}
+
+function allowedCollection(db, collection, session) {
+  if (!session) return true;
+  const { aliases, kind, privateKey } = scopesFor(db, session);
+  if (collection.scope === "shared") return true;
+  if (privateKey && collection.scope === privateKey) return true;
+  if (aliases.includes(collection.scope)) return true;
+  if (String(collection.scope).startsWith("__private__") && kind !== "private")
+    return false;
+  if (!aliases.length) return false;
+  return !!db
+    .prepare(
+      `SELECT 1 FROM core_collection_sessions WHERE collection_id=? AND session_id IN (${aliases.map(() => "?").join(",")})`,
+    )
+    .get(collection.id, ...aliases);
+}
+
+export class KnowledgeManager {
+  constructor(repo, models) {
+    this.repo = repo;
+    this.models = models;
+  }
+  collections(session = "") {
+    return this.repo.db
+      .prepare("SELECT * FROM core_collections ORDER BY updated DESC")
+      .all()
+      .filter((c) => allowedCollection(this.repo.db, c, session));
+  }
+  createCollection({ name, scope = "shared", sessions = [] }) {
+    if (typeof name !== "string" || !name.trim() || name.length > 100)
+      throw Error("知识库名称无效");
+    const id = randomUUID(),
+      now = Date.now();
+    this.repo.db
+      .prepare(
+        "INSERT INTO core_collections(id,name,scope,created,updated) VALUES (?,?,?,?,?)",
+      )
+      .run(id, name.trim(), scope || "shared", now, now);
+    for (const session of sessions)
+      this.repo.db
+        .prepare("INSERT OR IGNORE INTO core_collection_sessions VALUES (?,?)")
+        .run(id, session);
+    this.repo.store.revision++;
+    return this.repo.db
+      .prepare("SELECT * FROM core_collections WHERE id=?")
+      .get(id);
+  }
+  documents(collectionId) {
+    return this.repo.db
+      .prepare(
+        "SELECT * FROM core_documents WHERE collection_id=? ORDER BY updated DESC",
+      )
+      .all(collectionId);
+  }
+  async ingest({ collectionId, title, text, source = "paste", embed = true }) {
+    const collection = this.repo.db
+      .prepare("SELECT * FROM core_collections WHERE id=?")
+      .get(collectionId);
+    if (!collection) throw Error("知识库不存在");
+    if (typeof text !== "string" || !text.trim() || text.length > 400000)
+      throw Error("文档内容应为 1–400000 字");
+    const safeTitle = String(title || "未命名文档").slice(0, 200);
+    const plain = asPlainDocument(text);
+    const chunks = chunkText(plain);
+    if (!chunks.length) throw Error("文档切分后为空");
+    const id = randomUUID(),
+      now = Date.now(),
+      hash = createHash("sha256").update(text).digest("hex");
+    mkdirSync(runtimePaths().knowledge, { recursive: true, mode: 0o700 });
+    const path = join(runtimePaths().knowledge, id + ".md");
+    writeFileSync(path, text);
+    this.repo.db
+      .prepare(
+        "INSERT INTO core_documents(id,collection_id,title,source,path,status,created,updated,bytes,hash) VALUES (?,?,?,?,?,'processing',?,?,?,?)",
+      )
+      .run(
+        id,
+        collectionId,
+        safeTitle,
+        source,
+        path,
+        now,
+        now,
+        Buffer.byteLength(text),
+        hash,
+      );
+    let vectors = [];
+    let embeddingProfile = "";
+    try {
+      if (embed) {
+        const resolved = resolveEmbeddingProfile(this.repo, this.models);
+        vectors = await embedBatches(
+          this.models,
+          resolved.profile,
+          chunks.map((c) => c.text),
+        );
+        if (vectors.length) embeddingProfile = resolved.signature;
+      }
+    } catch (error) {
+      this.repo.db
+        .prepare(
+          "UPDATE core_documents SET status='failed',error=?,updated=? WHERE id=?",
+        )
+        .run(error.message, Date.now(), id);
+      throw error;
+    }
+    this.repo.db.exec("BEGIN IMMEDIATE");
+    try {
+      chunks.forEach((chunk, i) => {
+        const chunkId = randomUUID();
+        this.repo.db
+          .prepare(
+            "INSERT INTO core_chunks(id,document_id,collection_id,ordinal,heading,text,tokens,embedding,hash,created,embedding_profile) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
+          )
+          .run(
+            chunkId,
+            id,
+            collectionId,
+            i,
+            chunk.heading,
+            chunk.text,
+            chunk.text.slice(0, 200),
+            vectors[i] ? packVector(vectors[i]) : null,
+            createHash("sha256").update(chunk.text).digest("hex"),
+            now,
+            embeddingProfile,
+          );
+        indexChunk(this.repo.db, chunkId, chunk.text);
+      });
+      this.repo.db
+        .prepare(
+          "UPDATE core_documents SET status='ready',error=NULL,updated=? WHERE id=?",
+        )
+        .run(Date.now(), id);
+      this.repo.db
+        .prepare("UPDATE core_collections SET updated=? WHERE id=?")
+        .run(Date.now(), collectionId);
+      this.repo.db.exec("COMMIT");
+    } catch (error) {
+      this.repo.db.exec("ROLLBACK");
+      throw error;
+    }
+    this.repo.store.revision++;
+    return this.repo.db
+      .prepare("SELECT * FROM core_documents WHERE id=?")
+      .get(id);
+  }
+  async embedTexts(texts) {
+    const { profile } = resolveEmbeddingProfile(this.repo, this.models);
+    return embedBatches(this.models, profile, texts);
+  }
+  retrieve(
+    session,
+    rows,
+    cutoff = Date.now(),
+    { vectors = [], signature = null, legacy = true } = {},
+  ) {
+    const allowed = new Set(
+      this.repo.db
+        .prepare("SELECT id,scope FROM core_collections")
+        .all()
+        .filter((c) => allowedCollection(this.repo.db, c, session))
+        .map((c) => c.id),
+    );
+    if (!allowed.size) return [];
+    const queries = saidBy(rows).slice(0, 8);
+    if (!queries.length) return [];
+    // Each person's own words. Two people are not added together, and a
+    // line she already said does not count as the room asking. A prepared
+    // vector belongs to that same person, not to the batch mixed together.
+    if (vectors.some((vector) => vector?.length))
+      ensureVectorIndex(this.repo.db);
+    const best = new Map();
+    const selected = new Map();
+    for (let i = 0; i < queries.length; i++) {
+      const query = queries[i];
+      const match = ftsMatchQuery(query);
+      const fts = new Set();
+      if (match)
+        try {
+          for (const hit of this.repo.db
+            .prepare(
+              `SELECT f.chunk_id FROM core_chunk_fts f JOIN core_chunks c ON c.id=f.chunk_id JOIN core_documents d ON d.id=c.document_id WHERE f.tokens MATCH ? AND c.collection_id IN (${[...allowed].map(() => "?").join(",")}) AND d.status='ready' AND d.created<=? ORDER BY f.rank LIMIT 128`,
+            )
+            .all(match, ...allowed, cutoff))
+            fts.add(hit.chunk_id);
+        } catch {
+          /* malformed MATCH */
+        }
+      const queryTerms = lexicalTerms(query);
+      const vec = vectors[i] || null;
+      const ids = [
+        ...new Set([
+          ...fts,
+          ...(vec
+            ? vectorCandidates(this.repo.db, vec, [...allowed], cutoff, {
+                signature,
+                legacy,
+              })
+            : []),
+        ]),
+      ];
+      const chunks = ids.length
+        ? this.repo.db
+            .prepare(
+              `SELECT c.*,d.title,d.created AS document_created FROM core_chunks c JOIN core_documents d ON d.id=c.document_id WHERE c.id IN (${ids.map(() => "?").join(",")})`,
+            )
+            .all(...ids)
+        : [];
+      for (const chunk of chunks) {
+        selected.set(chunk.id, chunk);
+        let score =
+          (fts.has(chunk.id) ? 2 : 0) +
+          overlapScore(chunk.text, queryTerms) * 0.5;
+        if (
+          vec &&
+          chunk.embedding &&
+          (!signature ||
+            chunk.embedding_profile === signature ||
+            (legacy && !chunk.embedding_profile))
+        )
+          score += cosine(vec, unpackVector(chunk.embedding)) * 4;
+        const ageDays = (cutoff - (chunk.document_created || 0)) / 86400000;
+        if (ageDays > 30) score *= 0.85;
+        if (score > (best.get(chunk.id) || 0)) best.set(chunk.id, score);
+      }
+    }
+    return [...best.entries()]
+      .sort((a, b) => b[1] - a[1])
+      .slice(0, 8)
+      .map(([id, score]) => {
+        const chunk = selected.get(id);
+        if (!chunk) return null;
+        return {
+          id: chunk.id,
+          documentId: chunk.document_id,
+          title: chunk.title,
+          heading: chunk.heading,
+          text: chunk.text.slice(0, 800),
+          score,
+          whySelected: score >= 4 ? "semantic+lexical" : "lexical",
+        };
+      })
+      .filter((c) => c?.text);
+  }
+  async retrieveWithEmbed(session, rows, cutoff = Date.now()) {
+    const { profile, signature, legacy } = resolveEmbeddingProfile(
+      this.repo,
+      this.models,
+    );
+    const queries = saidBy(rows).slice(0, 8);
+    let vectors = [];
+    if (profile.embedding && queries.length) {
+      try {
+        vectors = await embedBatches(
+          this.models,
+          profile,
+          queries.map((query) => query.slice(0, 2000)),
+        );
+      } catch {
+        vectors = [];
+      }
+    }
+    return this.retrieve(session, rows, cutoff, { vectors, signature, legacy });
+  }
+  async reembedDocument(id) {
+    const rows = this.repo.db
+      .prepare(
+        "SELECT id,text FROM core_chunks WHERE document_id=? ORDER BY ordinal",
+      )
+      .all(id);
+    if (!rows.length) throw Error("文档没有可建立向量的段落");
+    const { profile, signature } = resolveEmbeddingProfile(
+      this.repo,
+      this.models,
+    );
+    if (!profile.embedding) throw Error("请先启用知识向量模型");
+    const vectors = await embedBatches(
+      this.models,
+      profile,
+      rows.map((row) => row.text),
+    );
+    const db = this.repo.db;
+    db.exec("SAVEPOINT reembed_document");
+    try {
+      const update = db.prepare(
+        "UPDATE core_chunks SET embedding=?,embedding_profile=? WHERE id=? AND document_id=? AND text=?",
+      );
+      rows.forEach((row, index) => {
+        if (
+          !update.run(
+            packVector(vectors[index]),
+            signature,
+            row.id,
+            id,
+            row.text,
+          ).changes
+        )
+          throw Error("文档已变化，请重新建立向量");
+      });
+      db.exec("RELEASE reembed_document");
+    } catch (error) {
+      db.exec("ROLLBACK TO reembed_document");
+      db.exec("RELEASE reembed_document");
+      throw error;
+    }
+    this.repo.store.revision++;
+    return { chunks: rows.length };
+  }
+  removeDocument(id) {
+    const row = this.repo.db
+      .prepare("SELECT * FROM core_documents WHERE id=?")
+      .get(id);
+    if (!row) throw Error("文档不存在");
+    const chunks = this.repo.db
+      .prepare("SELECT id FROM core_chunks WHERE document_id=?")
+      .all(id);
+    this.repo.db.exec("BEGIN IMMEDIATE");
+    try {
+      for (const chunk of chunks)
+        this.repo.db
+          .prepare("DELETE FROM core_chunk_fts WHERE chunk_id=?")
+          .run(chunk.id);
+      this.repo.db
+        .prepare("DELETE FROM core_chunks WHERE document_id=?")
+        .run(id);
+      this.repo.db.prepare("DELETE FROM core_documents WHERE id=?").run(id);
+      this.repo.db.exec("COMMIT");
+    } catch (e) {
+      this.repo.db.exec("ROLLBACK");
+      throw e;
+    }
+    this.repo.store.revision++;
+  }
+}

package/server/knowledge/retrieval.js

@@ -0,0 +1,93 @@
+export function lexicalTerms(s) {
+  return new Set(
+    String(s)
+      .toLowerCase()
+      .match(/[a-z0-9]+|[\u4e00-\u9fff]{1,2}/g) || [],
+  );
+}
+
+export function ftsTokens(s) {
+  return [...lexicalTerms(s)].join(" ");
+}
+
+export function ftsMatchQuery(s) {
+  return [...lexicalTerms(s)]
+    .slice(0, 24)
+    .map((t) => `"${t.replace(/"/g, "")}"`)
+    .filter((t) => t.length > 2)
+    .join(" OR ");
+}
+
+export function overlapScore(text, queryTerms) {
+  if (!queryTerms?.size) return 0;
+  return [...lexicalTerms(text)].filter((t) => queryTerms.has(t)).length;
+}
+
+export function cosine(a, b) {
+  if (!a?.length || !b?.length || a.length !== b.length) return 0;
+  let dot = 0,
+    na = 0,
+    nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  return dot / (Math.sqrt(na) * Math.sqrt(nb) || 1);
+}
+
+export function packVector(values) {
+  return Buffer.from(new Float32Array(values).buffer);
+}
+
+export function unpackVector(blob) {
+  if (!blob) return null;
+  const buf = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
+  return Array.from(
+    new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4),
+  );
+}
+
+export function asPlainDocument(text) {
+  let source = String(text || "").replace(/\r\n/g, "\n");
+  if (/<\/?[a-z][\s\S]*>/i.test(source))
+    source = source
+      .replace(/<script[\s\S]*?<\/script>/gi, "\n")
+      .replace(/<style[\s\S]*?<\/style>/gi, "\n")
+      .replace(/<h([1-3])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, level, title) => {
+        const hashes = "#".repeat(Number(level) || 1);
+        return `\n${hashes} ${title.replace(/<[^>]+>/g, "").trim()}\n`;
+      })
+      .replace(/<[^>]+>/g, " ");
+  return source
+    .replace(/[ \t]+\n/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+export function chunkText(text, { size = 600, overlap = 80 } = {}) {
+  const source = String(text || "")
+    .replace(/\r\n/g, "\n")
+    .trim();
+  if (!source) return [];
+  const parts = source.split(/(?=^#{1,3} )/m);
+  const chunks = [];
+  for (const part of parts) {
+    const heading = part.match(/^#{1,3} ([^\n]+)/)?.[1] || "";
+    const chars = Array.from(part.trim());
+    if (!chars.length) continue;
+    if (chars.length <= size) {
+      chunks.push({ heading, text: part.trim() });
+      continue;
+    }
+    for (let i = 0; i < chars.length; i += Math.max(1, size - overlap)) {
+      const slice = chars
+        .slice(i, i + size)
+        .join("")
+        .trim();
+      if (slice) chunks.push({ heading, text: slice });
+      if (i + size >= chars.length) break;
+    }
+  }
+  return chunks;
+}

package/server/knowledge/schema.js

@@ -0,0 +1,154 @@
+import { ftsTokens } from "./retrieval.js";
+
+export function indexMemory(db, id, content) {
+  db.prepare("DELETE FROM core_memory_fts WHERE memory_id=?").run(id);
+  const tokens = ftsTokens(content);
+  if (tokens)
+    db.prepare(
+      "INSERT INTO core_memory_fts(memory_id, tokens) VALUES (?,?)",
+    ).run(id, tokens);
+}
+
+export function indexChunk(db, id, content) {
+  db.prepare("DELETE FROM core_chunk_fts WHERE chunk_id=?").run(id);
+  const tokens = ftsTokens(content);
+  if (tokens)
+    db.prepare("INSERT INTO core_chunk_fts(chunk_id, tokens) VALUES (?,?)").run(
+      id,
+      tokens,
+    );
+}
+
+function mergeLegacyMemories(db) {
+  if (db.prepare("SELECT id FROM core_config WHERE id='memory-merge-v1'").get())
+    return;
+  db.exec("BEGIN IMMEDIATE");
+  try {
+    for (const m of db.prepare("SELECT * FROM memories").all()) {
+      const sessionId =
+        m.scope === "shared"
+          ? "__shared__"
+          : m.scope === "private"
+            ? `__private__:${m.user_id}`
+            : m.scope;
+      const id = "legacy:" + m.id;
+      if (db.prepare("SELECT id FROM core_memories WHERE id=?").get(id))
+        continue;
+      db.prepare(
+        "INSERT INTO core_memories(id,session_id,subject,content,type,confidence,importance,status,locked,sources,created,updated) VALUES (?,?,?,?,'reviewed',1,1,'confirmed',1,'[]',?,?)",
+      ).run(
+        id,
+        sessionId,
+        m.user_id,
+        m.content,
+        m.time || Date.now(),
+        Date.now(),
+      );
+      indexMemory(db, id, m.content);
+    }
+    db.prepare("INSERT INTO core_config(id,value) VALUES (?,?)").run(
+      "memory-merge-v1",
+      JSON.stringify({ time: Date.now() }),
+    );
+    db.exec("COMMIT");
+  } catch (e) {
+    db.exec("ROLLBACK");
+    throw e;
+  }
+}
+
+export function migrateKnowledge(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS core_collections (
+      id TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      scope TEXT NOT NULL,
+      created INTEGER NOT NULL,
+      updated INTEGER NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS core_collection_sessions (
+      collection_id TEXT NOT NULL,
+      session_id TEXT NOT NULL,
+      PRIMARY KEY(collection_id, session_id)
+    );
+    CREATE TABLE IF NOT EXISTS core_documents (
+      id TEXT PRIMARY KEY,
+      collection_id TEXT NOT NULL,
+      title TEXT NOT NULL,
+      source TEXT,
+      path TEXT,
+      status TEXT NOT NULL,
+      error TEXT,
+      created INTEGER NOT NULL,
+      updated INTEGER NOT NULL,
+      bytes INTEGER,
+      hash TEXT
+    );
+    CREATE TABLE IF NOT EXISTS core_chunks (
+      id TEXT PRIMARY KEY,
+      document_id TEXT NOT NULL,
+      collection_id TEXT NOT NULL,
+      ordinal INTEGER NOT NULL,
+      heading TEXT,
+      text TEXT NOT NULL,
+      tokens TEXT,
+      embedding BLOB,
+      hash TEXT,
+      created INTEGER NOT NULL
+    );
+  `);
+  db.exec(
+    "CREATE VIRTUAL TABLE IF NOT EXISTS core_memory_fts USING fts5(memory_id UNINDEXED, tokens)",
+  );
+  db.exec(
+    "CREATE VIRTUAL TABLE IF NOT EXISTS core_chunk_fts USING fts5(chunk_id UNINDEXED, tokens)",
+  );
+  if (
+    !db
+      .prepare("PRAGMA table_info(core_chunks)")
+      .all()
+      .some((c) => c.name === "vector_indexed")
+  )
+    db.exec(
+      "ALTER TABLE core_chunks ADD COLUMN vector_indexed INTEGER NOT NULL DEFAULT 0",
+    );
+  if (
+    !db
+      .prepare("PRAGMA table_info(core_chunks)")
+      .all()
+      .some((c) => c.name === "embedding_profile")
+  )
+    db.exec(
+      "ALTER TABLE core_chunks ADD COLUMN embedding_profile TEXT NOT NULL DEFAULT ''",
+    );
+  db.exec(`CREATE INDEX IF NOT EXISTS core_chunks_collection ON core_chunks(collection_id, document_id);
+    CREATE INDEX IF NOT EXISTS core_chunks_unindexed ON core_chunks(vector_indexed) WHERE embedding IS NOT NULL;
+    CREATE TABLE IF NOT EXISTS core_vector_buckets(dimension INTEGER NOT NULL,band INTEGER NOT NULL,bucket INTEGER NOT NULL,chunk_id TEXT NOT NULL,PRIMARY KEY(dimension,band,bucket,chunk_id));
+    CREATE INDEX IF NOT EXISTS core_vector_buckets_chunk ON core_vector_buckets(chunk_id);
+    CREATE TRIGGER IF NOT EXISTS core_vector_changed AFTER UPDATE OF embedding ON core_chunks BEGIN
+      DELETE FROM core_vector_buckets WHERE chunk_id=NEW.id;
+      UPDATE core_chunks SET vector_indexed=0 WHERE id=NEW.id;
+    END;
+    CREATE TRIGGER IF NOT EXISTS core_vector_deleted AFTER DELETE ON core_chunks BEGIN
+      DELETE FROM core_vector_buckets WHERE chunk_id=OLD.id;
+    END;`);
+  if (
+    !db
+      .prepare("SELECT id FROM core_collections WHERE id='shared-default'")
+      .get()
+  )
+    db.prepare(
+      "INSERT INTO core_collections(id,name,scope,created,updated) VALUES ('shared-default','共享知识库','shared',?,?)",
+    ).run(Date.now(), Date.now());
+  mergeLegacyMemories(db);
+  if (
+    !db.prepare("SELECT id FROM core_config WHERE id='memory-fts-v1'").get()
+  ) {
+    for (const m of db.prepare("SELECT id,content FROM core_memories").all())
+      indexMemory(db, m.id, m.content);
+    db.prepare("INSERT INTO core_config(id,value) VALUES (?,?)").run(
+      "memory-fts-v1",
+      JSON.stringify({ time: Date.now() }),
+    );
+  }
+}