luckytri 1.0.5 → 1.0.6

package/server/knowledge/schema.js

@@ -1,158 +1,158 @@
-import { ftsTokens } from "./retrieval.js";
-
-export function indexMemory(db, id, content) {
-  db.prepare("DELETE FROM core_memory_fts WHERE memory_id=?").run(id);
-  const tokens = ftsTokens(content);
-  if (tokens)
-    db.prepare(
-      "INSERT INTO core_memory_fts(memory_id, tokens) VALUES (?,?)",
-    ).run(id, tokens);
-}
-
-export function indexChunk(db, id, content) {
-  db.prepare("DELETE FROM core_chunk_fts WHERE chunk_id=?").run(id);
-  const tokens = ftsTokens(content);
-  if (tokens)
-    db.prepare("INSERT INTO core_chunk_fts(chunk_id, tokens) VALUES (?,?)").run(
-      id,
-      tokens,
-    );
-}
-
-function mergeLegacyMemories(db) {
-  if (db.prepare("SELECT id FROM core_config WHERE id='memory-merge-v1'").get())
-    return;
-  db.exec("BEGIN IMMEDIATE");
-  try {
-    for (const m of db.prepare("SELECT * FROM memories").all()) {
-      const sessionId =
-        m.scope === "shared"
-          ? "__shared__"
-          : m.scope === "private"
-            ? `__private__:${m.user_id}`
-            : m.scope;
-      const id = "legacy:" + m.id;
-      if (db.prepare("SELECT id FROM core_memories WHERE id=?").get(id))
-        continue;
-      db.prepare(
-        "INSERT INTO core_memories(id,session_id,subject,content,type,confidence,importance,status,locked,sources,created,updated) VALUES (?,?,?,?,'reviewed',1,1,'confirmed',1,'[]',?,?)",
-      ).run(
-        id,
-        sessionId,
-        m.user_id,
-        m.content,
-        m.time || Date.now(),
-        Date.now(),
-      );
-      indexMemory(db, id, m.content);
-    }
-    db.prepare("INSERT INTO core_config(id,value) VALUES (?,?)").run(
-      "memory-merge-v1",
-      JSON.stringify({ time: Date.now() }),
-    );
-    db.exec("COMMIT");
-  } catch (e) {
-    db.exec("ROLLBACK");
-    throw e;
-  }
-}
-
-export const KNOWLEDGE_TRIGGERS = [
-  `CREATE TRIGGER IF NOT EXISTS core_vector_changed AFTER UPDATE OF embedding ON core_chunks BEGIN
-      DELETE FROM core_vector_buckets WHERE chunk_id=NEW.id;
-      UPDATE core_chunks SET vector_indexed=0 WHERE id=NEW.id;
-    END`,
-  `CREATE TRIGGER IF NOT EXISTS core_vector_deleted AFTER DELETE ON core_chunks BEGIN
-      DELETE FROM core_vector_buckets WHERE chunk_id=OLD.id;
-    END`,
-];
-
-export function migrateKnowledge(db) {
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS core_collections (
-      id TEXT PRIMARY KEY,
-      name TEXT NOT NULL,
-      scope TEXT NOT NULL,
-      created INTEGER NOT NULL,
-      updated INTEGER NOT NULL
-    );
-    CREATE TABLE IF NOT EXISTS core_collection_sessions (
-      collection_id TEXT NOT NULL,
-      session_id TEXT NOT NULL,
-      PRIMARY KEY(collection_id, session_id)
-    );
-    CREATE TABLE IF NOT EXISTS core_documents (
-      id TEXT PRIMARY KEY,
-      collection_id TEXT NOT NULL,
-      title TEXT NOT NULL,
-      source TEXT,
-      path TEXT,
-      status TEXT NOT NULL,
-      error TEXT,
-      created INTEGER NOT NULL,
-      updated INTEGER NOT NULL,
-      bytes INTEGER,
-      hash TEXT
-    );
-    CREATE TABLE IF NOT EXISTS core_chunks (
-      id TEXT PRIMARY KEY,
-      document_id TEXT NOT NULL,
-      collection_id TEXT NOT NULL,
-      ordinal INTEGER NOT NULL,
-      heading TEXT,
-      text TEXT NOT NULL,
-      tokens TEXT,
-      embedding BLOB,
-      hash TEXT,
-      created INTEGER NOT NULL
-    );
-  `);
-  db.exec(
-    "CREATE VIRTUAL TABLE IF NOT EXISTS core_memory_fts USING fts5(memory_id UNINDEXED, tokens)",
-  );
-  db.exec(
-    "CREATE VIRTUAL TABLE IF NOT EXISTS core_chunk_fts USING fts5(chunk_id UNINDEXED, tokens)",
-  );
-  if (
-    !db
-      .prepare("PRAGMA table_info(core_chunks)")
-      .all()
-      .some((c) => c.name === "vector_indexed")
-  )
-    db.exec(
-      "ALTER TABLE core_chunks ADD COLUMN vector_indexed INTEGER NOT NULL DEFAULT 0",
-    );
-  if (
-    !db
-      .prepare("PRAGMA table_info(core_chunks)")
-      .all()
-      .some((c) => c.name === "embedding_profile")
-  )
-    db.exec(
-      "ALTER TABLE core_chunks ADD COLUMN embedding_profile TEXT NOT NULL DEFAULT ''",
-    );
-  db.exec(`CREATE INDEX IF NOT EXISTS core_chunks_collection ON core_chunks(collection_id, document_id);
-    CREATE INDEX IF NOT EXISTS core_chunks_unindexed ON core_chunks(vector_indexed) WHERE embedding IS NOT NULL;
-    CREATE TABLE IF NOT EXISTS core_vector_buckets(dimension INTEGER NOT NULL,band INTEGER NOT NULL,bucket INTEGER NOT NULL,chunk_id TEXT NOT NULL,PRIMARY KEY(dimension,band,bucket,chunk_id));
-    CREATE INDEX IF NOT EXISTS core_vector_buckets_chunk ON core_vector_buckets(chunk_id);`);
-  db.exec(KNOWLEDGE_TRIGGERS.join(";") + ";");
-  if (
-    !db
-      .prepare("SELECT id FROM core_collections WHERE id='shared-default'")
-      .get()
-  )
-    db.prepare(
-      "INSERT INTO core_collections(id,name,scope,created,updated) VALUES ('shared-default','共享知识库','shared',?,?)",
-    ).run(Date.now(), Date.now());
-  mergeLegacyMemories(db);
-  if (
-    !db.prepare("SELECT id FROM core_config WHERE id='memory-fts-v1'").get()
-  ) {
-    for (const m of db.prepare("SELECT id,content FROM core_memories").all())
-      indexMemory(db, m.id, m.content);
-    db.prepare("INSERT INTO core_config(id,value) VALUES (?,?)").run(
-      "memory-fts-v1",
-      JSON.stringify({ time: Date.now() }),
-    );
-  }
-}
+import { ftsTokens } from "./retrieval.js";
+
+export function indexMemory(db, id, content) {
+  db.prepare("DELETE FROM core_memory_fts WHERE memory_id=?").run(id);
+  const tokens = ftsTokens(content);
+  if (tokens)
+    db.prepare(
+      "INSERT INTO core_memory_fts(memory_id, tokens) VALUES (?,?)",
+    ).run(id, tokens);
+}
+
+export function indexChunk(db, id, content) {
+  db.prepare("DELETE FROM core_chunk_fts WHERE chunk_id=?").run(id);
+  const tokens = ftsTokens(content);
+  if (tokens)
+    db.prepare("INSERT INTO core_chunk_fts(chunk_id, tokens) VALUES (?,?)").run(
+      id,
+      tokens,
+    );
+}
+
+function mergeLegacyMemories(db) {
+  if (db.prepare("SELECT id FROM core_config WHERE id='memory-merge-v1'").get())
+    return;
+  db.exec("BEGIN IMMEDIATE");
+  try {
+    for (const m of db.prepare("SELECT * FROM memories").all()) {
+      const sessionId =
+        m.scope === "shared"
+          ? "__shared__"
+          : m.scope === "private"
+            ? `__private__:${m.user_id}`
+            : m.scope;
+      const id = "legacy:" + m.id;
+      if (db.prepare("SELECT id FROM core_memories WHERE id=?").get(id))
+        continue;
+      db.prepare(
+        "INSERT INTO core_memories(id,session_id,subject,content,type,confidence,importance,status,locked,sources,created,updated) VALUES (?,?,?,?,'reviewed',1,1,'confirmed',1,'[]',?,?)",
+      ).run(
+        id,
+        sessionId,
+        m.user_id,
+        m.content,
+        m.time || Date.now(),
+        Date.now(),
+      );
+      indexMemory(db, id, m.content);
+    }
+    db.prepare("INSERT INTO core_config(id,value) VALUES (?,?)").run(
+      "memory-merge-v1",
+      JSON.stringify({ time: Date.now() }),
+    );
+    db.exec("COMMIT");
+  } catch (e) {
+    db.exec("ROLLBACK");
+    throw e;
+  }
+}
+
+export const KNOWLEDGE_TRIGGERS = [
+  `CREATE TRIGGER IF NOT EXISTS core_vector_changed AFTER UPDATE OF embedding ON core_chunks BEGIN
+      DELETE FROM core_vector_buckets WHERE chunk_id=NEW.id;
+      UPDATE core_chunks SET vector_indexed=0 WHERE id=NEW.id;
+    END`,
+  `CREATE TRIGGER IF NOT EXISTS core_vector_deleted AFTER DELETE ON core_chunks BEGIN
+      DELETE FROM core_vector_buckets WHERE chunk_id=OLD.id;
+    END`,
+];
+
+export function migrateKnowledge(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS core_collections (
+      id TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      scope TEXT NOT NULL,
+      created INTEGER NOT NULL,
+      updated INTEGER NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS core_collection_sessions (
+      collection_id TEXT NOT NULL,
+      session_id TEXT NOT NULL,
+      PRIMARY KEY(collection_id, session_id)
+    );
+    CREATE TABLE IF NOT EXISTS core_documents (
+      id TEXT PRIMARY KEY,
+      collection_id TEXT NOT NULL,
+      title TEXT NOT NULL,
+      source TEXT,
+      path TEXT,
+      status TEXT NOT NULL,
+      error TEXT,
+      created INTEGER NOT NULL,
+      updated INTEGER NOT NULL,
+      bytes INTEGER,
+      hash TEXT
+    );
+    CREATE TABLE IF NOT EXISTS core_chunks (
+      id TEXT PRIMARY KEY,
+      document_id TEXT NOT NULL,
+      collection_id TEXT NOT NULL,
+      ordinal INTEGER NOT NULL,
+      heading TEXT,
+      text TEXT NOT NULL,
+      tokens TEXT,
+      embedding BLOB,
+      hash TEXT,
+      created INTEGER NOT NULL
+    );
+  `);
+  db.exec(
+    "CREATE VIRTUAL TABLE IF NOT EXISTS core_memory_fts USING fts5(memory_id UNINDEXED, tokens)",
+  );
+  db.exec(
+    "CREATE VIRTUAL TABLE IF NOT EXISTS core_chunk_fts USING fts5(chunk_id UNINDEXED, tokens)",
+  );
+  if (
+    !db
+      .prepare("PRAGMA table_info(core_chunks)")
+      .all()
+      .some((c) => c.name === "vector_indexed")
+  )
+    db.exec(
+      "ALTER TABLE core_chunks ADD COLUMN vector_indexed INTEGER NOT NULL DEFAULT 0",
+    );
+  if (
+    !db
+      .prepare("PRAGMA table_info(core_chunks)")
+      .all()
+      .some((c) => c.name === "embedding_profile")
+  )
+    db.exec(
+      "ALTER TABLE core_chunks ADD COLUMN embedding_profile TEXT NOT NULL DEFAULT ''",
+    );
+  db.exec(`CREATE INDEX IF NOT EXISTS core_chunks_collection ON core_chunks(collection_id, document_id);
+    CREATE INDEX IF NOT EXISTS core_chunks_unindexed ON core_chunks(vector_indexed) WHERE embedding IS NOT NULL;
+    CREATE TABLE IF NOT EXISTS core_vector_buckets(dimension INTEGER NOT NULL,band INTEGER NOT NULL,bucket INTEGER NOT NULL,chunk_id TEXT NOT NULL,PRIMARY KEY(dimension,band,bucket,chunk_id));
+    CREATE INDEX IF NOT EXISTS core_vector_buckets_chunk ON core_vector_buckets(chunk_id);`);
+  db.exec(KNOWLEDGE_TRIGGERS.join(";") + ";");
+  if (
+    !db
+      .prepare("SELECT id FROM core_collections WHERE id='shared-default'")
+      .get()
+  )
+    db.prepare(
+      "INSERT INTO core_collections(id,name,scope,created,updated) VALUES ('shared-default','共享知识库','shared',?,?)",
+    ).run(Date.now(), Date.now());
+  mergeLegacyMemories(db);
+  if (
+    !db.prepare("SELECT id FROM core_config WHERE id='memory-fts-v1'").get()
+  ) {
+    for (const m of db.prepare("SELECT id,content FROM core_memories").all())
+      indexMemory(db, m.id, m.content);
+    db.prepare("INSERT INTO core_config(id,value) VALUES (?,?)").run(
+      "memory-fts-v1",
+      JSON.stringify({ time: Date.now() }),
+    );
+  }
+}

package/server/knowledge/vector-index.js

@@ -1,90 +1,90 @@
-import { unpackVector } from "./retrieval.js";
-
-const BANDS = 12,
-  BITS = 6;
-export const VECTOR_CANDIDATES = 256;
-
-// Deterministic random hyperplanes: index signatures remain valid across
-// restarts and do not depend on the order documents were imported.
-function signatures(vector) {
-  const result = [];
-  for (let band = 0; band < BANDS; band++) {
-    let bucket = 0;
-    for (let bit = 0; bit < BITS; bit++) {
-      let sum = 0;
-      for (let dimension = 0; dimension < vector.length; dimension++) {
-        let seed =
-          Math.imul(dimension + 1, 0x9e3779b1) ^
-          Math.imul(band * BITS + bit + 1, 0x85ebca6b);
-        seed = Math.imul(seed ^ (seed >>> 16), 0x7feb352d);
-        sum += vector[dimension] * (seed >>> 31 ? 1 : -1);
-      }
-      if (sum >= 0) bucket |= 1 << bit;
-    }
-    result.push({ band, bucket });
-  }
-  return result;
-}
-
-export function ensureVectorIndex(db) {
-  const rows = db.prepare(
-    "SELECT id,embedding FROM core_chunks WHERE vector_indexed=0 AND embedding IS NOT NULL",
-  );
-  const insert = db.prepare(
-    "INSERT OR IGNORE INTO core_vector_buckets(dimension,band,bucket,chunk_id) VALUES (?,?,?,?)",
-  );
-  const mark = db.prepare("UPDATE core_chunks SET vector_indexed=1 WHERE id=?");
-  db.exec("SAVEPOINT vector_index");
-  try {
-    for (const row of rows.iterate()) {
-      const vector = unpackVector(row.embedding);
-      if (
-        vector?.length &&
-        vector.every(Number.isFinite) &&
-        vector.some((value) => value !== 0)
-      )
-        for (const { band, bucket } of signatures(vector))
-          insert.run(vector.length, band, bucket, row.id);
-      mark.run(row.id);
-    }
-    db.exec("RELEASE vector_index");
-  } catch (error) {
-    db.exec("ROLLBACK TO vector_index");
-    db.exec("RELEASE vector_index");
-    throw error;
-  }
-}
-
-export function vectorCandidates(
-  db,
-  vector,
-  collections,
-  cutoff,
-  { signature = null, legacy = true } = {},
-) {
-  if (!vector?.length || !vector.every(Number.isFinite)) return [];
-  const buckets = signatures(vector).flatMap(({ band, bucket }) =>
-    [
-      bucket,
-      ...Array.from({ length: BITS }, (_, bit) => bucket ^ (1 << bit)),
-    ].map((value) => ({ band, bucket: value })),
-  );
-  return db
-    .prepare(
-      `SELECT b.chunk_id FROM core_vector_buckets b
-    JOIN core_chunks c ON c.id=b.chunk_id JOIN core_documents d ON d.id=c.document_id
-    WHERE b.dimension=? AND (${buckets.map(() => "(b.band=? AND b.bucket=?)").join(" OR ")})
-    AND c.collection_id IN (${collections.map(() => "?").join(",")}) AND d.status='ready' AND d.created<=?
-    ${signature ? "AND (c.embedding_profile=?" + (legacy ? " OR c.embedding_profile=''" : "") + ")" : ""}
-    GROUP BY b.chunk_id ORDER BY COUNT(*) DESC, c.created DESC, b.chunk_id LIMIT ?`,
-    )
-    .all(
-      vector.length,
-      ...buckets.flatMap((item) => [item.band, item.bucket]),
-      ...collections,
-      cutoff,
-      ...(signature ? [signature] : []),
-      VECTOR_CANDIDATES,
-    )
-    .map((row) => row.chunk_id);
-}
+import { unpackVector } from "./retrieval.js";
+
+const BANDS = 12,
+  BITS = 6;
+export const VECTOR_CANDIDATES = 256;
+
+// Deterministic random hyperplanes: index signatures remain valid across
+// restarts and do not depend on the order documents were imported.
+function signatures(vector) {
+  const result = [];
+  for (let band = 0; band < BANDS; band++) {
+    let bucket = 0;
+    for (let bit = 0; bit < BITS; bit++) {
+      let sum = 0;
+      for (let dimension = 0; dimension < vector.length; dimension++) {
+        let seed =
+          Math.imul(dimension + 1, 0x9e3779b1) ^
+          Math.imul(band * BITS + bit + 1, 0x85ebca6b);
+        seed = Math.imul(seed ^ (seed >>> 16), 0x7feb352d);
+        sum += vector[dimension] * (seed >>> 31 ? 1 : -1);
+      }
+      if (sum >= 0) bucket |= 1 << bit;
+    }
+    result.push({ band, bucket });
+  }
+  return result;
+}
+
+export function ensureVectorIndex(db) {
+  const rows = db.prepare(
+    "SELECT id,embedding FROM core_chunks WHERE vector_indexed=0 AND embedding IS NOT NULL",
+  );
+  const insert = db.prepare(
+    "INSERT OR IGNORE INTO core_vector_buckets(dimension,band,bucket,chunk_id) VALUES (?,?,?,?)",
+  );
+  const mark = db.prepare("UPDATE core_chunks SET vector_indexed=1 WHERE id=?");
+  db.exec("SAVEPOINT vector_index");
+  try {
+    for (const row of rows.iterate()) {
+      const vector = unpackVector(row.embedding);
+      if (
+        vector?.length &&
+        vector.every(Number.isFinite) &&
+        vector.some((value) => value !== 0)
+      )
+        for (const { band, bucket } of signatures(vector))
+          insert.run(vector.length, band, bucket, row.id);
+      mark.run(row.id);
+    }
+    db.exec("RELEASE vector_index");
+  } catch (error) {
+    db.exec("ROLLBACK TO vector_index");
+    db.exec("RELEASE vector_index");
+    throw error;
+  }
+}
+
+export function vectorCandidates(
+  db,
+  vector,
+  collections,
+  cutoff,
+  { signature = null, legacy = true } = {},
+) {
+  if (!vector?.length || !vector.every(Number.isFinite)) return [];
+  const buckets = signatures(vector).flatMap(({ band, bucket }) =>
+    [
+      bucket,
+      ...Array.from({ length: BITS }, (_, bit) => bucket ^ (1 << bit)),
+    ].map((value) => ({ band, bucket: value })),
+  );
+  return db
+    .prepare(
+      `SELECT b.chunk_id FROM core_vector_buckets b
+    JOIN core_chunks c ON c.id=b.chunk_id JOIN core_documents d ON d.id=c.document_id
+    WHERE b.dimension=? AND (${buckets.map(() => "(b.band=? AND b.bucket=?)").join(" OR ")})
+    AND c.collection_id IN (${collections.map(() => "?").join(",")}) AND d.status='ready' AND d.created<=?
+    ${signature ? "AND (c.embedding_profile=?" + (legacy ? " OR c.embedding_profile=''" : "") + ")" : ""}
+    GROUP BY b.chunk_id ORDER BY COUNT(*) DESC, c.created DESC, b.chunk_id LIMIT ?`,
+    )
+    .all(
+      vector.length,
+      ...buckets.flatMap((item) => [item.band, item.bucket]),
+      ...collections,
+      cutoff,
+      ...(signature ? [signature] : []),
+      VECTOR_CANDIDATES,
+    )
+    .map((row) => row.chunk_id);
+}