@klhapp/skillmux 1.9.3 → 1.11.0

package/src/db-index.ts

@@ -0,0 +1,238 @@
+import { Database } from "bun:sqlite";
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+import type { VaultSkill } from "./vault";
+
+export interface SkillRow {
+  skill_id: string;
+  title: string;
+  description: string;
+  aliases: string;
+  content_sha256: string;
+}
+
+export function openIndex(stateDir: string): Database {
+  mkdirSync(stateDir, { recursive: true });
+  const db = new Database(join(stateDir, "index.sqlite3"), { create: true });
+  db.run("PRAGMA journal_mode = WAL");
+  db.run("PRAGMA busy_timeout = 2000");
+  db.run(`CREATE TABLE IF NOT EXISTS skills (
+    skill_id TEXT PRIMARY KEY,
+    title TEXT NOT NULL,
+    description TEXT NOT NULL,
+    aliases TEXT NOT NULL,
+    content_sha256 TEXT NOT NULL
+  )`);
+  db.run(`CREATE VIRTUAL TABLE IF NOT EXISTS skills_fts USING fts5(
+    skill_id UNINDEXED, title, description, aliases,
+    tokenize = 'unicode61 remove_diacritics 2'
+  )`);
+  db.run(`CREATE TABLE IF NOT EXISTS vectors (
+    skill_id TEXT PRIMARY KEY,
+    content_sha256 TEXT NOT NULL,
+    embedding_fingerprint TEXT NOT NULL DEFAULT '',
+    dim INTEGER NOT NULL,
+    vec BLOB NOT NULL
+  )`);
+  const vectorColumns = db.query("PRAGMA table_info(vectors)").all() as { name: string }[];
+  if (!vectorColumns.some((column) => column.name === "embedding_fingerprint")) {
+    db.run("ALTER TABLE vectors ADD COLUMN embedding_fingerprint TEXT NOT NULL DEFAULT ''");
+  }
+  // Audit rows live in audit.sqlite3; openAudit adopts any left here.
+  db.run(`CREATE TABLE IF NOT EXISTS index_meta (
+    key TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+  )`);
+  return db;
+}
+
+export function upsertSkill(db: Database, skill: VaultSkill): void {
+  const aliases = skill.aliases.join(" ");
+  db.transaction(() => {
+    db.run("DELETE FROM skills WHERE skill_id = ?", [skill.skill_id]);
+    db.run("DELETE FROM skills_fts WHERE skill_id = ?", [skill.skill_id]);
+    db.run(
+      "INSERT INTO skills (skill_id, title, description, aliases, content_sha256) VALUES (?, ?, ?, ?, ?)",
+      [skill.skill_id, skill.title, skill.description, aliases, skill.content_sha256],
+    );
+    db.run(
+      "INSERT INTO skills_fts (skill_id, title, description, aliases) VALUES (?, ?, ?, ?)",
+      [skill.skill_id, skill.title, skill.description, aliases],
+    );
+  })();
+}
+
+export function toSkillRow(skill: VaultSkill): SkillRow {
+  return {
+    skill_id: skill.skill_id,
+    title: skill.title,
+    description: skill.description,
+    aliases: skill.aliases.join(" "),
+    content_sha256: skill.content_sha256,
+  };
+}
+
+/** Replace the whole lexical index with `rows`; drops vectors of removed skills. */
+export function replaceSkills(db: Database, rows: SkillRow[]): void {
+  db.transaction(() => {
+    db.run("DELETE FROM skills");
+    db.run("DELETE FROM skills_fts");
+    for (const row of rows) {
+      db.run(
+        "INSERT INTO skills (skill_id, title, description, aliases, content_sha256) VALUES (?, ?, ?, ?, ?)",
+        [row.skill_id, row.title, row.description, row.aliases, row.content_sha256],
+      );
+      db.run(
+        "INSERT INTO skills_fts (skill_id, title, description, aliases) VALUES (?, ?, ?, ?)",
+        [row.skill_id, row.title, row.description, row.aliases],
+      );
+    }
+    db.run("DELETE FROM vectors WHERE skill_id NOT IN (SELECT skill_id FROM skills)");
+  })();
+}
+
+export function ingestVault(db: Database, skills: VaultSkill[]): void {
+  replaceSkills(db, skills.map(toSkillRow));
+}
+
+export function deleteSkill(db: Database, skillId: string): void {
+  db.transaction(() => {
+    db.run("DELETE FROM skills WHERE skill_id = ?", [skillId]);
+    db.run("DELETE FROM skills_fts WHERE skill_id = ?", [skillId]);
+    db.run("DELETE FROM vectors WHERE skill_id = ?", [skillId]);
+  })();
+}
+
+export function skillCount(db: Database): number {
+  return (db.query("SELECT count(*) AS n FROM skills").get() as { n: number }).n;
+}
+
+export function getSkillRow(db: Database, skillId: string): SkillRow | null {
+  return db.query("SELECT * FROM skills WHERE skill_id = ?").get(skillId) as SkillRow | null;
+}
+
+/**
+ * Sanitize free text into an FTS5 OR-query; returns null when no usable terms
+ * remain. Terms keep any Unicode letters/digits (CJK included) so non-ASCII
+ * queries still get lexical recall — unicode61 tokenizes contiguous CJK runs
+ * as single tokens, so matching works at that granularity.
+ */
+export function toFtsQuery(text: string): string | null {
+  const terms = [
+    ...new Set(
+      text
+        .toLowerCase()
+        .split(/[^\p{L}\p{N}]+/u)
+        .filter((t) => t.length >= 2),
+    ),
+  ];
+  if (terms.length === 0) return null;
+  return terms.map((t) => `"${t}"`).join(" OR ");
+}
+
+export function ftsSearch(db: Database, text: string, k: number): SkillRow[] {
+  const query = toFtsQuery(text);
+  if (query === null) return [];
+  return db
+    .query(
+      `SELECT s.* FROM skills_fts f
+       JOIN skills s ON s.skill_id = f.skill_id
+       WHERE skills_fts MATCH ?
+       ORDER BY bm25(skills_fts) LIMIT ?`,
+    )
+    .all(query, k) as SkillRow[];
+}
+
+export function findExactMatch(db: Database, query: string): SkillRow | null {
+  const cleanQuery = query.trim().toLowerCase();
+  return db
+    .query(
+      `SELECT * FROM skills 
+       WHERE lower(skill_id) = ? 
+          OR lower(title) = ? 
+          OR ' ' || lower(aliases) || ' ' LIKE ?`,
+    )
+    .get(cleanQuery, cleanQuery, `% ${cleanQuery} %`) as SkillRow | null;
+}
+
+export function getIndexMeta(db: Database, key: string): string | null {
+  const row = db
+    .query("SELECT value FROM index_meta WHERE key = ?")
+    .get(key) as { value: string } | null;
+  return row ? row.value : null;
+}
+
+export function setIndexMeta(db: Database, key: string, value: string): void {
+  db.run(
+    `INSERT INTO index_meta (key, value) VALUES (?, ?)
+     ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
+    [key, value],
+  );
+}
+
+export function upsertVector(
+  db: Database,
+  skillId: string,
+  contentSha256: string,
+  embeddingFingerprint: string,
+  vec: Float32Array,
+): void {
+  db.run(
+    `INSERT INTO vectors (skill_id, content_sha256, embedding_fingerprint, dim, vec) VALUES (?, ?, ?, ?, ?)
+     ON CONFLICT(skill_id) DO UPDATE SET
+       content_sha256 = excluded.content_sha256,
+       embedding_fingerprint = excluded.embedding_fingerprint,
+       dim = excluded.dim,
+       vec = excluded.vec`,
+    [skillId, contentSha256, embeddingFingerprint, vec.length, new Uint8Array(vec.buffer, vec.byteOffset, vec.byteLength)],
+  );
+}
+
+/**
+ * Skills with no usable stored vector: none at all, content changed since
+ * embedding, or embedded at a different dimension than currently configured.
+ */
+export function skillsNeedingVectors(db: Database, dimension: number, embeddingFingerprint: string): SkillRow[] {
+  return db
+    .query(
+      `SELECT s.* FROM skills s
+       LEFT JOIN vectors v ON v.skill_id = s.skill_id
+          AND v.content_sha256 = s.content_sha256
+          AND v.dim = ?
+          AND v.embedding_fingerprint = ?
+        WHERE v.skill_id IS NULL`,
+    )
+    .all(dimension, embeddingFingerprint) as SkillRow[];
+}
+
+function cosine(a: Float32Array, b: Float32Array): number {
+  if (a.length !== b.length) return 0;
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i]! * b[i]!;
+    normA += a[i]! * a[i]!;
+    normB += b[i]! * b[i]!;
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+/** Brute-force cosine over every stored vector (vault is ~100 skills; no ANN). */
+export function vectorTopK(db: Database, query: Float32Array, k: number): SkillRow[] {
+  const rows = db
+    .query(
+      `SELECT s.skill_id, s.title, s.description, s.aliases, s.content_sha256, v.vec
+       FROM vectors v JOIN skills s ON s.skill_id = v.skill_id`,
+    )
+    .all() as (SkillRow & { vec: Uint8Array })[];
+  return rows
+    .map(({ vec, ...row }) => ({
+      row,
+      score: cosine(query, new Float32Array(vec.buffer, vec.byteOffset, vec.byteLength / 4)),
+    }))
+    .sort((a, b) => b.score - a.score)
+    .slice(0, k)
+    .map((r) => r.row);
+}