chainlesschain 0.37.8 → 0.37.10

package/src/lib/knowledge-importer.js

@@ -0,0 +1,293 @@
+/**
+ * Knowledge importer — parse Markdown, Evernote ENEX, and Notion exports
+ * into the notes table.
+ */
+
+import { readFileSync, readdirSync, statSync, existsSync } from "fs";
+import { join, basename, extname, relative } from "path";
+
+/**
+ * Ensure the notes table exists
+ */
+export function ensureNotesTable(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS notes (
+      id TEXT PRIMARY KEY,
+      title TEXT NOT NULL,
+      content TEXT DEFAULT '',
+      tags TEXT DEFAULT '[]',
+      category TEXT DEFAULT 'general',
+      created_at TEXT DEFAULT (datetime('now')),
+      updated_at TEXT DEFAULT (datetime('now')),
+      deleted_at TEXT DEFAULT NULL
+    )
+  `);
+}
+
+/**
+ * Generate a simple UUID-like ID
+ */
+function generateId() {
+  const hex = () =>
+    Math.floor(Math.random() * 0x10000)
+      .toString(16)
+      .padStart(4, "0");
+  return `${hex()}${hex()}-${hex()}-${hex()}-${hex()}-${hex()}${hex()}${hex()}`;
+}
+
+/**
+ * Insert a note into the database
+ */
+export function insertNote(db, { title, content, tags, category, createdAt }) {
+  const id = generateId();
+  const tagsJson = JSON.stringify(tags || []);
+  const cat = category || "general";
+  const created =
+    createdAt || new Date().toISOString().replace("T", " ").slice(0, 19);
+
+  db.prepare(
+    "INSERT INTO notes (id, title, content, tags, category, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
+  ).run(id, title, content || "", tagsJson, cat, created, created);
+
+  return { id, title, tags: tags || [], category: cat, created_at: created };
+}
+
+// ─── Markdown Import ────────────────────────────────────────────────
+
+/**
+ * Parse a single markdown file into a note object.
+ * Extracts YAML frontmatter if present.
+ */
+export function parseMarkdownFile(filePath) {
+  const raw = readFileSync(filePath, "utf-8");
+  const name = basename(filePath, extname(filePath));
+
+  let title = name;
+  let content = raw;
+  let tags = [];
+  let category = "markdown";
+  let createdAt = null;
+
+  // Try to extract YAML frontmatter
+  const fmMatch = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+  if (fmMatch) {
+    const frontmatter = fmMatch[1];
+    content = fmMatch[2].trim();
+
+    // Parse simple YAML fields
+    const titleMatch = frontmatter.match(/^title:\s*["']?(.+?)["']?\s*$/m);
+    if (titleMatch) title = titleMatch[1];
+
+    const tagsMatch = frontmatter.match(/^tags:\s*\[([^\]]*)\]/m);
+    if (tagsMatch) {
+      tags = tagsMatch[1]
+        .split(",")
+        .map((t) => t.trim().replace(/["']/g, ""))
+        .filter(Boolean);
+    }
+
+    const catMatch = frontmatter.match(/^category:\s*["']?(.+?)["']?\s*$/m);
+    if (catMatch) category = catMatch[1];
+
+    const dateMatch = frontmatter.match(/^date:\s*["']?(.+?)["']?\s*$/m);
+    if (dateMatch) createdAt = dateMatch[1];
+  }
+
+  // Use first H1 as title if no frontmatter title
+  if (title === name && !fmMatch) {
+    const h1Match = content.match(/^#\s+(.+)$/m);
+    if (h1Match) title = h1Match[1];
+  }
+
+  return { title, content, tags, category, createdAt };
+}
+
+/**
+ * Recursively collect all .md files from a directory.
+ */
+export function collectMarkdownFiles(dir) {
+  const results = [];
+
+  function walk(currentDir) {
+    const entries = readdirSync(currentDir);
+    for (const entry of entries) {
+      const fullPath = join(currentDir, entry);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory()) {
+        walk(fullPath);
+      } else if (extname(entry).toLowerCase() === ".md") {
+        results.push(fullPath);
+      }
+    }
+  }
+
+  walk(dir);
+  return results;
+}
+
+/**
+ * Import all markdown files from a directory into the database.
+ */
+export function importMarkdownDir(db, dir) {
+  ensureNotesTable(db);
+  const files = collectMarkdownFiles(dir);
+  const imported = [];
+
+  for (const file of files) {
+    const parsed = parseMarkdownFile(file);
+    const note = insertNote(db, parsed);
+    note.source = relative(dir, file);
+    imported.push(note);
+  }
+
+  return imported;
+}
+
+// ─── Evernote ENEX Import ───────────────────────────────────────────
+
+/**
+ * Parse an ENEX (Evernote Export) XML string into note objects.
+ * ENEX format: <en-export><note><title>...</title><content>...</content><tag>...</tag></note>...</en-export>
+ */
+export function parseEnex(xmlString) {
+  const notes = [];
+  const noteRegex = /<note>([\s\S]*?)<\/note>/gi;
+  let noteMatch;
+
+  while ((noteMatch = noteRegex.exec(xmlString)) !== null) {
+    const noteXml = noteMatch[1];
+
+    const titleMatch = noteXml.match(/<title>([\s\S]*?)<\/title>/i);
+    const title = titleMatch ? titleMatch[1].trim() : "Untitled";
+
+    // Content is wrapped in CDATA inside <content>
+    const contentMatch = noteXml.match(/<content>([\s\S]*?)<\/content>/i);
+    let content = "";
+    if (contentMatch) {
+      let raw = contentMatch[1];
+      // Strip CDATA wrapper
+      const cdataMatch = raw.match(/<!\[CDATA\[([\s\S]*?)\]\]>/);
+      if (cdataMatch) raw = cdataMatch[1];
+      // Strip ENML/HTML tags for plain text
+      content = stripHtml(raw);
+    }
+
+    // Tags
+    const tags = [];
+    const tagRegex = /<tag>([\s\S]*?)<\/tag>/gi;
+    let tagMatch;
+    while ((tagMatch = tagRegex.exec(noteXml)) !== null) {
+      tags.push(tagMatch[1].trim());
+    }
+
+    // Created date
+    const createdMatch = noteXml.match(/<created>([\s\S]*?)<\/created>/i);
+    let createdAt = null;
+    if (createdMatch) {
+      // ENEX dates: 20210315T120000Z → 2021-03-15 12:00:00
+      const d = createdMatch[1].trim();
+      const parsed = d.match(/(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})/);
+      if (parsed) {
+        createdAt = `${parsed[1]}-${parsed[2]}-${parsed[3]} ${parsed[4]}:${parsed[5]}:${parsed[6]}`;
+      }
+    }
+
+    notes.push({ title, content, tags, category: "evernote", createdAt });
+  }
+
+  return notes;
+}
+
+/**
+ * Strip HTML/ENML tags to plain text.
+ */
+export function stripHtml(html) {
+  return html
+    .replace(/<br\s*\/?>/gi, "\n")
+    .replace(/<\/p>/gi, "\n")
+    .replace(/<\/div>/gi, "\n")
+    .replace(/<\/li>/gi, "\n")
+    .replace(/<[^>]+>/g, "")
+    .replace(/&nbsp;/g, " ")
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+/**
+ * Import an Evernote ENEX file into the database.
+ */
+export function importEnexFile(db, filePath) {
+  ensureNotesTable(db);
+  const xmlString = readFileSync(filePath, "utf-8");
+  const parsed = parseEnex(xmlString);
+  const imported = [];
+
+  for (const note of parsed) {
+    const result = insertNote(db, note);
+    imported.push(result);
+  }
+
+  return imported;
+}
+
+// ─── Notion Export Import ───────────────────────────────────────────
+
+/**
+ * Parse a Notion export directory.
+ * Notion exports contain markdown files and may have metadata in filenames
+ * or accompanying JSON/CSV files.
+ */
+export function parseNotionExport(dir) {
+  const notes = [];
+  const files = collectMarkdownFiles(dir);
+
+  for (const file of files) {
+    const raw = readFileSync(file, "utf-8");
+    const fileName = basename(file, ".md");
+
+    // Notion filenames often have a UUID suffix: "My Page abc123def456"
+    // Remove the hex suffix to get the clean title
+    const title = fileName.replace(/\s+[a-f0-9]{32}$/i, "") || fileName;
+
+    // Notion uses # for the first heading, which is usually the page title
+    let content = raw;
+    const h1Match = raw.match(/^#\s+(.+)\n([\s\S]*)$/);
+    if (h1Match) {
+      content = h1Match[2].trim();
+    }
+
+    // Try to detect tags from Notion properties (sometimes at the top)
+    const tags = [];
+    const relPath = relative(dir, file);
+    const parts = relPath.split(/[/\\]/);
+    if (parts.length > 1) {
+      // Use parent folder as a tag
+      tags.push(parts[0]);
+    }
+
+    notes.push({ title, content, tags, category: "notion", createdAt: null });
+  }
+
+  return notes;
+}
+
+/**
+ * Import a Notion export directory into the database.
+ */
+export function importNotionDir(db, dir) {
+  ensureNotesTable(db);
+  const parsed = parseNotionExport(dir);
+  const imported = [];
+
+  for (const note of parsed) {
+    const result = insertNote(db, note);
+    imported.push(result);
+  }
+
+  return imported;
+}

package/src/lib/llm-providers.js

@@ -0,0 +1,325 @@
+/**
+ * LLM Provider registry — supports multiple AI providers with a unified interface.
+ */
+
+/**
+ * Built-in provider definitions.
+ */
+export const BUILT_IN_PROVIDERS = {
+  ollama: {
+    name: "ollama",
+    displayName: "Ollama (Local)",
+    baseUrl: "http://localhost:11434",
+    apiKeyEnv: null,
+    models: ["qwen2:7b", "llama3:8b", "mistral:7b", "codellama:7b"],
+    free: true,
+  },
+  openai: {
+    name: "openai",
+    displayName: "OpenAI",
+    baseUrl: "https://api.openai.com/v1",
+    apiKeyEnv: "OPENAI_API_KEY",
+    models: ["gpt-4o", "gpt-4o-mini", "gpt-4-turbo", "gpt-3.5-turbo", "o1"],
+    free: false,
+  },
+  anthropic: {
+    name: "anthropic",
+    displayName: "Anthropic",
+    baseUrl: "https://api.anthropic.com/v1",
+    apiKeyEnv: "ANTHROPIC_API_KEY",
+    models: [
+      "claude-opus-4-6",
+      "claude-sonnet-4-6",
+      "claude-haiku-4-5-20251001",
+    ],
+    free: false,
+  },
+  deepseek: {
+    name: "deepseek",
+    displayName: "DeepSeek",
+    baseUrl: "https://api.deepseek.com/v1",
+    apiKeyEnv: "DEEPSEEK_API_KEY",
+    models: ["deepseek-chat", "deepseek-coder", "deepseek-reasoner"],
+    free: false,
+  },
+  dashscope: {
+    name: "dashscope",
+    displayName: "DashScope (Alibaba)",
+    baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1",
+    apiKeyEnv: "DASHSCOPE_API_KEY",
+    models: ["qwen-turbo", "qwen-plus", "qwen-max"],
+    free: false,
+  },
+  gemini: {
+    name: "gemini",
+    displayName: "Google Gemini",
+    baseUrl: "https://generativelanguage.googleapis.com/v1beta",
+    apiKeyEnv: "GEMINI_API_KEY",
+    models: ["gemini-2.0-flash", "gemini-2.0-pro", "gemini-1.5-flash"],
+    free: false,
+  },
+  mistral: {
+    name: "mistral",
+    displayName: "Mistral AI",
+    baseUrl: "https://api.mistral.ai/v1",
+    apiKeyEnv: "MISTRAL_API_KEY",
+    models: [
+      "mistral-large-latest",
+      "mistral-medium-latest",
+      "mistral-small-latest",
+    ],
+    free: false,
+  },
+};
+
+/**
+ * Provider registry — manages available providers and active selection.
+ */
+export class LLMProviderRegistry {
+  constructor(db) {
+    this.db = db;
+    this.providers = new Map();
+    this._ensureTable();
+    this._loadBuiltins();
+    this._loadCustom();
+  }
+
+  _ensureTable() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS llm_providers (
+        name TEXT PRIMARY KEY,
+        display_name TEXT NOT NULL,
+        base_url TEXT NOT NULL,
+        api_key_env TEXT,
+        models TEXT DEFAULT '[]',
+        is_active INTEGER DEFAULT 0,
+        custom INTEGER DEFAULT 0,
+        created_at TEXT DEFAULT (datetime('now'))
+      )
+    `);
+  }
+
+  _loadBuiltins() {
+    for (const [name, def] of Object.entries(BUILT_IN_PROVIDERS)) {
+      this.providers.set(name, { ...def, custom: false });
+    }
+  }
+
+  _loadCustom() {
+    const rows = this.db
+      .prepare("SELECT * FROM llm_providers WHERE custom = 1")
+      .all();
+    for (const row of rows) {
+      this.providers.set(row.name, {
+        name: row.name,
+        displayName: row.display_name,
+        baseUrl: row.base_url,
+        apiKeyEnv: row.api_key_env,
+        models: JSON.parse(row.models || "[]"),
+        custom: true,
+        free: false,
+      });
+    }
+  }
+
+  /**
+   * List all providers.
+   */
+  list() {
+    const result = [];
+    for (const [name, provider] of this.providers) {
+      const hasKey = provider.apiKeyEnv
+        ? !!process.env[provider.apiKeyEnv]
+        : true;
+      result.push({
+        name,
+        displayName: provider.displayName,
+        baseUrl: provider.baseUrl,
+        models: provider.models,
+        hasApiKey: hasKey,
+        custom: provider.custom || false,
+        free: provider.free || false,
+      });
+    }
+    return result;
+  }
+
+  /**
+   * Get a specific provider.
+   */
+  get(name) {
+    return this.providers.get(name) || null;
+  }
+
+  /**
+   * Add a custom provider.
+   */
+  addProvider(name, config) {
+    const provider = {
+      name,
+      displayName: config.displayName || name,
+      baseUrl: config.baseUrl,
+      apiKeyEnv: config.apiKeyEnv || null,
+      models: config.models || [],
+      custom: true,
+      free: config.free || false,
+    };
+
+    this.db
+      .prepare(
+        "INSERT OR REPLACE INTO llm_providers (name, display_name, base_url, api_key_env, models, custom) VALUES (?, ?, ?, ?, ?, 1)",
+      )
+      .run(
+        name,
+        provider.displayName,
+        provider.baseUrl,
+        provider.apiKeyEnv,
+        JSON.stringify(provider.models),
+      );
+
+    this.providers.set(name, provider);
+    return provider;
+  }
+
+  /**
+   * Remove a custom provider.
+   */
+  removeProvider(name) {
+    const provider = this.providers.get(name);
+    if (!provider) return false;
+    if (!provider.custom)
+      throw new Error(`Cannot remove built-in provider "${name}"`);
+
+    this.db
+      .prepare("DELETE FROM llm_providers WHERE name = ? AND custom = 1")
+      .run(name);
+    this.providers.delete(name);
+    return true;
+  }
+
+  /**
+   * Get/set the active provider.
+   */
+  getActive() {
+    const row = this.db
+      .prepare("SELECT name FROM llm_providers WHERE is_active = 1")
+      .get();
+    return row ? row.name : "ollama";
+  }
+
+  setActive(name) {
+    if (!this.providers.has(name)) {
+      throw new Error(`Provider "${name}" not found`);
+    }
+    // Reset all
+    this.db.prepare("UPDATE llm_providers SET is_active = 0 WHERE 1=1").run();
+    // Set active
+    this.db
+      .prepare(
+        "INSERT OR REPLACE INTO llm_providers (name, display_name, base_url, api_key_env, models, is_active) VALUES (?, ?, ?, ?, ?, 1)",
+      )
+      .run(
+        name,
+        this.providers.get(name).displayName,
+        this.providers.get(name).baseUrl,
+        this.providers.get(name).apiKeyEnv,
+        JSON.stringify(this.providers.get(name).models),
+      );
+    return this.providers.get(name);
+  }
+
+  /**
+   * Get API key for a provider (from env or config).
+   */
+  getApiKey(name) {
+    const provider = this.providers.get(name);
+    if (!provider) return null;
+    if (!provider.apiKeyEnv) return null;
+    return process.env[provider.apiKeyEnv] || null;
+  }
+
+  /**
+   * Test provider connectivity.
+   */
+  async testProvider(name, model) {
+    const provider = this.providers.get(name);
+    if (!provider) throw new Error(`Provider "${name}" not found`);
+
+    const start = Date.now();
+    const testModel = model || provider.models[0];
+
+    if (name === "ollama") {
+      const res = await fetch(`${provider.baseUrl}/api/generate`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ model: testModel, prompt: "Hi", stream: false }),
+      });
+      if (!res.ok) throw new Error(`HTTP ${res.status}`);
+      const data = await res.json();
+      return {
+        ok: true,
+        elapsed: Date.now() - start,
+        response: data.response?.trim(),
+      };
+    }
+
+    if (name === "gemini") {
+      const key = this.getApiKey(name);
+      if (!key) throw new Error("GEMINI_API_KEY not set");
+      const res = await fetch(
+        `${provider.baseUrl}/models/${testModel}:generateContent?key=${key}`,
+        {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ contents: [{ parts: [{ text: "Hi" }] }] }),
+        },
+      );
+      if (!res.ok) throw new Error(`HTTP ${res.status}`);
+      const data = await res.json();
+      const text = data.candidates?.[0]?.content?.parts?.[0]?.text || "";
+      return { ok: true, elapsed: Date.now() - start, response: text.trim() };
+    }
+
+    if (name === "anthropic") {
+      const key = this.getApiKey(name);
+      if (!key) throw new Error("ANTHROPIC_API_KEY not set");
+      const res = await fetch(`${provider.baseUrl}/messages`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "x-api-key": key,
+          "anthropic-version": "2023-06-01",
+        },
+        body: JSON.stringify({
+          model: testModel,
+          max_tokens: 10,
+          messages: [{ role: "user", content: "Hi" }],
+        }),
+      });
+      if (!res.ok) throw new Error(`HTTP ${res.status}`);
+      const data = await res.json();
+      const text = data.content?.[0]?.text || "";
+      return { ok: true, elapsed: Date.now() - start, response: text.trim() };
+    }
+
+    // OpenAI-compatible (openai, deepseek, dashscope, mistral)
+    const key = this.getApiKey(name);
+    if (!key) throw new Error(`${provider.apiKeyEnv} not set`);
+    const res = await fetch(`${provider.baseUrl}/chat/completions`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${key}`,
+      },
+      body: JSON.stringify({
+        model: testModel,
+        messages: [{ role: "user", content: "Hi" }],
+        max_tokens: 10,
+      }),
+    });
+    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+    const data = await res.json();
+    const text = data.choices?.[0]?.message?.content || "";
+    return { ok: true, elapsed: Date.now() - start, response: text.trim() };
+  }
+}