@hasna/knowledge 0.2.3 → 0.2.4

package/src/knowledge-db.ts

@@ -0,0 +1,231 @@
+import { Database } from 'bun:sqlite';
+import { ensureParentDir } from './workspace';
+
+export const CURRENT_SCHEMA_VERSION = 1;
+
+export interface KnowledgeDbStats {
+  schema_version: number;
+  sources: number;
+  source_revisions: number;
+  chunks: number;
+  wiki_pages: number;
+  citations: number;
+  indexes: number;
+  runs: number;
+  run_events: number;
+}
+
+const MIGRATION_1 = `
+PRAGMA journal_mode = WAL;
+PRAGMA foreign_keys = ON;
+
+CREATE TABLE IF NOT EXISTS schema_versions (
+  version INTEGER PRIMARY KEY,
+  applied_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS sources (
+  id TEXT PRIMARY KEY,
+  uri TEXT NOT NULL UNIQUE,
+  kind TEXT NOT NULL,
+  title TEXT,
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  acl_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS source_revisions (
+  id TEXT PRIMARY KEY,
+  source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
+  revision TEXT NOT NULL,
+  hash TEXT,
+  extracted_text_uri TEXT,
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL,
+  UNIQUE(source_id, revision)
+);
+
+CREATE TABLE IF NOT EXISTS chunks (
+  id TEXT PRIMARY KEY,
+  source_revision_id TEXT REFERENCES source_revisions(id) ON DELETE CASCADE,
+  wiki_page_id TEXT,
+  kind TEXT NOT NULL,
+  ordinal INTEGER NOT NULL,
+  text TEXT NOT NULL,
+  token_count INTEGER,
+  start_offset INTEGER,
+  end_offset INTEGER,
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS chunk_embeddings (
+  id TEXT PRIMARY KEY,
+  chunk_id TEXT NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,
+  provider TEXT NOT NULL,
+  model TEXT NOT NULL,
+  dimensions INTEGER NOT NULL,
+  vector_json TEXT NOT NULL,
+  created_at TEXT NOT NULL,
+  UNIQUE(chunk_id, provider, model)
+);
+
+CREATE TABLE IF NOT EXISTS wiki_pages (
+  id TEXT PRIMARY KEY,
+  path TEXT NOT NULL UNIQUE,
+  title TEXT NOT NULL,
+  artifact_uri TEXT,
+  content_hash TEXT,
+  status TEXT NOT NULL DEFAULT 'active',
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS wiki_backlinks (
+  from_page_id TEXT NOT NULL REFERENCES wiki_pages(id) ON DELETE CASCADE,
+  to_page_id TEXT NOT NULL REFERENCES wiki_pages(id) ON DELETE CASCADE,
+  label TEXT,
+  created_at TEXT NOT NULL,
+  PRIMARY KEY(from_page_id, to_page_id)
+);
+
+CREATE TABLE IF NOT EXISTS citations (
+  id TEXT PRIMARY KEY,
+  wiki_page_id TEXT REFERENCES wiki_pages(id) ON DELETE CASCADE,
+  chunk_id TEXT REFERENCES chunks(id) ON DELETE SET NULL,
+  source_uri TEXT NOT NULL,
+  quote TEXT,
+  start_offset INTEGER,
+  end_offset INTEGER,
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS knowledge_indexes (
+  id TEXT PRIMARY KEY,
+  kind TEXT NOT NULL,
+  name TEXT NOT NULL,
+  artifact_uri TEXT,
+  shard_key TEXT,
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  UNIQUE(kind, name, shard_key)
+);
+
+CREATE TABLE IF NOT EXISTS runs (
+  id TEXT PRIMARY KEY,
+  type TEXT NOT NULL,
+  prompt TEXT,
+  status TEXT NOT NULL,
+  provider TEXT,
+  model TEXT,
+  cost_tokens INTEGER NOT NULL DEFAULT 0,
+  cost_usd REAL NOT NULL DEFAULT 0,
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS run_events (
+  id TEXT PRIMARY KEY,
+  run_id TEXT NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
+  level TEXT NOT NULL,
+  event TEXT NOT NULL,
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS provider_usage (
+  id TEXT PRIMARY KEY,
+  run_id TEXT REFERENCES runs(id) ON DELETE SET NULL,
+  provider TEXT NOT NULL,
+  model TEXT NOT NULL,
+  input_tokens INTEGER NOT NULL DEFAULT 0,
+  output_tokens INTEGER NOT NULL DEFAULT 0,
+  cost_usd REAL NOT NULL DEFAULT 0,
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS redaction_findings (
+  id TEXT PRIMARY KEY,
+  source_uri TEXT,
+  run_id TEXT REFERENCES runs(id) ON DELETE SET NULL,
+  severity TEXT NOT NULL,
+  finding_type TEXT NOT NULL,
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS storage_objects (
+  id TEXT PRIMARY KEY,
+  artifact_uri TEXT NOT NULL UNIQUE,
+  kind TEXT NOT NULL,
+  content_type TEXT,
+  hash TEXT,
+  size_bytes INTEGER,
+  metadata_json TEXT NOT NULL DEFAULT '{}',
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+  text,
+  title,
+  source_uri,
+  content='',
+  tokenize='porter unicode61'
+);
+
+INSERT OR IGNORE INTO schema_versions(version, applied_at)
+VALUES (1, datetime('now'));
+`;
+
+export function openKnowledgeDb(path: string): Database {
+  ensureParentDir(path);
+  const db = new Database(path);
+  db.exec('PRAGMA foreign_keys = ON;');
+  return db;
+}
+
+export function migrateKnowledgeDb(path: string): { path: string; schema_version: number } {
+  const db = openKnowledgeDb(path);
+  try {
+    db.exec(MIGRATION_1);
+    return { path, schema_version: getSchemaVersion(db) };
+  } finally {
+    db.close();
+  }
+}
+
+export function getSchemaVersion(db: Database): number {
+  const row = db.query<{ version: number }, []>('SELECT MAX(version) AS version FROM schema_versions').get();
+  return row?.version ?? 0;
+}
+
+function count(db: Database, table: string): number {
+  const row = db.query<{ n: number }, []>(`SELECT COUNT(*) AS n FROM ${table}`).get();
+  return row?.n ?? 0;
+}
+
+export function getKnowledgeDbStats(path: string): KnowledgeDbStats {
+  const db = openKnowledgeDb(path);
+  try {
+    return {
+      schema_version: getSchemaVersion(db),
+      sources: count(db, 'sources'),
+      source_revisions: count(db, 'source_revisions'),
+      chunks: count(db, 'chunks'),
+      wiki_pages: count(db, 'wiki_pages'),
+      citations: count(db, 'citations'),
+      indexes: count(db, 'knowledge_indexes'),
+      runs: count(db, 'runs'),
+      run_events: count(db, 'run_events'),
+    };
+  } finally {
+    db.close();
+  }
+}