@disco_trooper/apple-notes-mcp 1.0.0

package/src/db/lancedb.ts

@@ -0,0 +1,263 @@
+import * as lancedb from "@lancedb/lancedb";
+import path from "node:path";
+import os from "node:os";
+import { validateTitle, escapeForFilter } from "./validation.js";
+import type { DBSearchResult as SearchResult } from "../types/index.js";
+import { createDebugLogger } from "../utils/debug.js";
+
+// Schema for stored notes
+export interface NoteRecord {
+  title: string;
+  content: string;
+  vector: number[];
+  folder: string;
+  created: string;      // ISO date
+  modified: string;     // ISO date
+  indexed_at: string;   // ISO date - when embedding was generated
+  [key: string]: unknown; // Index signature for LanceDB compatibility
+}
+
+// SearchResult is imported from ../types/index.js as DBSearchResult
+export type { SearchResult };
+
+// VectorStore interface for future extensibility
+export interface VectorStore {
+  index(records: NoteRecord[]): Promise<void>;
+  update(record: NoteRecord): Promise<void>;
+  delete(title: string): Promise<void>;
+  deleteByFolderAndTitle(folder: string, title: string): Promise<void>;
+  search(queryVector: number[], limit: number): Promise<SearchResult[]>;
+  searchFTS(query: string, limit: number): Promise<SearchResult[]>;
+  getByTitle(title: string): Promise<NoteRecord | null>;
+  getAll(): Promise<NoteRecord[]>;
+  count(): Promise<number>;
+  clear(): Promise<void>;
+}
+
+// Debug logging
+const debug = createDebugLogger("DB");
+
+// LanceDB implementation
+export class LanceDBStore implements VectorStore {
+  private db: lancedb.Connection | null = null;
+  private table: lancedb.Table | null = null;
+  private readonly dbPath: string;
+  private readonly tableName = "notes";
+
+  constructor(dataDir?: string) {
+    this.dbPath = dataDir || path.join(os.homedir(), ".apple-notes-mcp", "data");
+  }
+
+  private async ensureConnection(): Promise<lancedb.Connection> {
+    if (!this.db) {
+      debug(`Connecting to LanceDB at ${this.dbPath}`);
+      this.db = await lancedb.connect(this.dbPath);
+    }
+    return this.db;
+  }
+
+  private async ensureTable(): Promise<lancedb.Table> {
+    if (!this.table) {
+      const db = await this.ensureConnection();
+      try {
+        this.table = await db.openTable(this.tableName);
+        debug(`Opened existing table: ${this.tableName}`);
+      } catch (error) {
+        // Table doesn't exist yet - will be created on first index
+        debug(`Table ${this.tableName} not found, will create on first index. Error:`, error);
+        throw new Error("Index not found. Run index-notes first.");
+      }
+    }
+    return this.table;
+  }
+
+  async index(records: NoteRecord[]): Promise<void> {
+    if (records.length === 0) {
+      debug("No records to index");
+      return;
+    }
+
+    const db = await this.ensureConnection();
+
+    // Drop existing table if exists
+    try {
+      await db.dropTable(this.tableName);
+      debug(`Dropped existing table: ${this.tableName}`);
+    } catch {
+      // Table didn't exist, that's fine
+    }
+
+    // Create new table with records
+    debug(`Creating table with ${records.length} records`);
+    this.table = await db.createTable(this.tableName, records);
+
+    // Create FTS index for hybrid search
+    debug("Creating FTS index on content");
+    await this.table.createIndex("content", {
+      config: lancedb.Index.fts(),
+      replace: true,
+    });
+
+    debug(`Indexed ${records.length} records`);
+  }
+
+  async update(record: NoteRecord): Promise<void> {
+    const table = await this.ensureTable();
+
+    // Add new record first (LanceDB allows duplicates with same title)
+    // This ensures we never lose data - if add fails, old record still exists
+    try {
+      await table.add([record]);
+      debug(`Added new version of record: ${record.title}`);
+    } catch (addError) {
+      // If add fails, old record still exists, throw original error
+      throw addError;
+    }
+
+    // Now delete old record(s) - use indexed_at to identify which is old
+    const validTitle = validateTitle(record.title);
+    const escapedTitle = escapeForFilter(validTitle);
+
+    try {
+      // Delete records with same title but different indexed_at (older versions)
+      const allWithTitle = await table
+        .query()
+        .where(`title = '${escapedTitle}'`)
+        .toArray();
+
+      for (const existing of allWithTitle) {
+        if (existing.indexed_at !== record.indexed_at) {
+          const escapedOldIndexedAt = escapeForFilter(existing.indexed_at as string);
+          await table.delete(`title = '${escapedTitle}' AND indexed_at = '${escapedOldIndexedAt}'`);
+          debug(`Deleted old version: ${record.title} (indexed_at: ${existing.indexed_at})`);
+        }
+      }
+    } catch (deleteError) {
+      // Log but don't fail - we have the new record, old one is just orphaned
+      debug(`Warning: Failed to delete old record versions for: ${record.title}`, deleteError);
+    }
+
+    debug(`Updated record: ${record.title}`);
+  }
+
+  async delete(title: string): Promise<void> {
+    const table = await this.ensureTable();
+    const validTitle = validateTitle(title);
+    const escapedTitle = escapeForFilter(validTitle);
+    await table.delete(`title = '${escapedTitle}'`);
+    debug(`Deleted record: ${title}`);
+  }
+
+  async deleteByFolderAndTitle(folder: string, title: string): Promise<void> {
+    const table = await this.ensureTable();
+    const validTitle = validateTitle(title);
+    const escapedTitle = escapeForFilter(validTitle);
+    const escapedFolder = escapeForFilter(folder);
+    await table.delete(`folder = '${escapedFolder}' AND title = '${escapedTitle}'`);
+    debug(`Deleted record: ${folder}/${title}`);
+  }
+
+  async search(queryVector: number[], limit: number): Promise<SearchResult[]> {
+    const table = await this.ensureTable();
+
+    const results = await table
+      .search(queryVector)
+      .limit(limit)
+      .toArray();
+
+    return results.map((row, index) => ({
+      title: row.title as string,
+      folder: row.folder as string,
+      content: row.content as string,
+      modified: row.modified as string,
+      score: 1 / (1 + index), // Simple rank-based score
+    }));
+  }
+
+  async searchFTS(query: string, limit: number): Promise<SearchResult[]> {
+    const table = await this.ensureTable();
+
+    try {
+      // LanceDB FTS search - use queryType option
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const results = await (table as any)
+        .search(query, { queryType: "fts" })
+        .limit(limit)
+        .toArray();
+
+      return results.map((row: Record<string, unknown>, index: number) => ({
+        title: row.title as string,
+        folder: row.folder as string,
+        content: row.content as string,
+        modified: row.modified as string,
+        score: 1 / (1 + index),
+      }));
+    } catch (error) {
+      // FTS might fail if no index or no matches
+      debug("FTS search failed, returning empty results. Error:", error);
+      return [];
+    }
+  }
+
+  async getByTitle(title: string): Promise<NoteRecord | null> {
+    const table = await this.ensureTable();
+
+    const validTitle = validateTitle(title);
+    const escapedTitle = escapeForFilter(validTitle);
+    const results = await table
+      .query()
+      .where(`title = '${escapedTitle}'`)
+      .limit(1)
+      .toArray();
+
+    if (results.length === 0) return null;
+
+    return results[0] as unknown as NoteRecord;
+  }
+
+  async getAll(): Promise<NoteRecord[]> {
+    const table = await this.ensureTable();
+
+    const results = await table.query().toArray();
+
+    return results.map((row) => ({
+      title: row.title as string,
+      content: row.content as string,
+      vector: row.vector as number[],
+      folder: row.folder as string,
+      created: row.created as string,
+      modified: row.modified as string,
+      indexed_at: row.indexed_at as string,
+    }));
+  }
+
+  async count(): Promise<number> {
+    try {
+      const table = await this.ensureTable();
+      return await table.countRows();
+    } catch {
+      return 0;
+    }
+  }
+
+  async clear(): Promise<void> {
+    const db = await this.ensureConnection();
+    try {
+      await db.dropTable(this.tableName);
+      this.table = null;
+      debug("Cleared table");
+    } catch {
+      // Table didn't exist
+    }
+  }
+}
+
+// Singleton instance
+let storeInstance: LanceDBStore | null = null;
+
+export function getVectorStore(): VectorStore {
+  if (!storeInstance) {
+    storeInstance = new LanceDBStore();
+  }
+  return storeInstance;
+}

package/src/db/validation.test.ts

@@ -0,0 +1,76 @@
+import { describe, it, expect } from "vitest";
+import { validateTitle, escapeForFilter } from "./validation.js";
+
+describe("validateTitle", () => {
+  it("accepts valid titles", () => {
+    expect(validateTitle("My Note")).toBe("My Note");
+    expect(validateTitle("  Trimmed  ")).toBe("Trimmed");
+    expect(validateTitle("Note with numbers 123")).toBe("Note with numbers 123");
+    expect(validateTitle("Unicode: cestina 日本語")).toBe("Unicode: cestina 日本語");
+  });
+
+  it("rejects empty titles", () => {
+    expect(() => validateTitle("")).toThrow("non-empty");
+    expect(() => validateTitle("   ")).toThrow("empty or whitespace");
+  });
+
+  it("rejects very long titles", () => {
+    const longTitle = "a".repeat(501);
+    expect(() => validateTitle(longTitle)).toThrow("maximum length");
+  });
+
+  it("rejects invalid characters", () => {
+    expect(() => validateTitle("Note\x00with\x00null")).toThrow("invalid characters");
+  });
+});
+
+describe("validateTitle - allowed characters", () => {
+  it("allows common punctuation", () => {
+    expect(validateTitle("Note: My Title")).toBe("Note: My Title");
+    expect(validateTitle("Meeting (2024-01-08)")).toBe("Meeting (2024-01-08)");
+    expect(validateTitle("Q&A Session")).toBe("Q&A Session");
+    expect(validateTitle("To-Do List")).toBe("To-Do List");
+    expect(validateTitle("Notes #1")).toBe("Notes #1");
+    expect(validateTitle("50% Complete!")).toBe("50% Complete!");
+  });
+
+  it("allows special characters that appear in real note titles", () => {
+    // Forward slashes (common in paths/dates)
+    expect(validateTitle("Airdrops/zisky")).toBe("Airdrops/zisky");
+    // Apostrophes
+    expect(validateTitle("Disco's Notes")).toBe("Disco's Notes");
+    // Unicode symbols
+    expect(validateTitle("Task ➜ Done")).toBe("Task ➜ Done");
+    // Ellipsis
+    expect(validateTitle("Long title…")).toBe("Long title…");
+    // Backticks, pipes, angle brackets (now allowed)
+    expect(validateTitle("Note `code` here")).toBe("Note `code` here");
+    expect(validateTitle("Option A | Option B")).toBe("Option A | Option B");
+    expect(validateTitle("A > B < C")).toBe("A > B < C");
+  });
+
+  it("rejects control characters", () => {
+    expect(() => validateTitle("Note\x00null")).toThrow("invalid characters");
+    expect(() => validateTitle("Note\x1ftab")).toThrow("invalid characters");
+    expect(() => validateTitle("Note\x7fdel")).toThrow("invalid characters");
+  });
+});
+
+describe("escapeForFilter", () => {
+  it("escapes single quotes", () => {
+    expect(escapeForFilter("O'Brien")).toBe("O''Brien");
+  });
+
+  it("escapes backslashes", () => {
+    expect(escapeForFilter("path\\to\\note")).toBe("path\\\\to\\\\note");
+  });
+
+  it("escapes newlines and tabs", () => {
+    expect(escapeForFilter("line1\nline2")).toBe("line1\\nline2");
+    expect(escapeForFilter("col1\tcol2")).toBe("col1\\tcol2");
+  });
+
+  it("handles combined escapes", () => {
+    expect(escapeForFilter("O'Brien's\nnote")).toBe("O''Brien''s\\nnote");
+  });
+});

package/src/db/validation.ts

@@ -0,0 +1,57 @@
+/**
+ * Input validation for database operations.
+ */
+
+import { MAX_TITLE_LENGTH } from "../config/constants.js";
+
+// Pattern for allowed characters in titles
+// Allows: letters, numbers, whitespace, punctuation, symbols, and most printable chars
+// Security is handled by escapeForFilter(), so we can be permissive here
+// Only reject control characters and null bytes
+const SAFE_TITLE_PATTERN = /^[^\x00-\x1f\x7f]+$/u;
+
+/**
+ * Validate and sanitize a note title for database operations.
+ *
+ * @param title - The title to validate
+ * @throws Error if title is invalid
+ * @returns Sanitized title safe for database queries
+ */
+export function validateTitle(title: string): string {
+  if (!title || typeof title !== "string") {
+    throw new Error("Title must be a non-empty string");
+  }
+
+  const trimmed = title.trim();
+
+  if (trimmed.length === 0) {
+    throw new Error("Title cannot be empty or whitespace only");
+  }
+
+  if (trimmed.length > MAX_TITLE_LENGTH) {
+    throw new Error(`Title exceeds maximum length of ${MAX_TITLE_LENGTH} characters`);
+  }
+
+  // Check for potentially dangerous characters
+  if (!SAFE_TITLE_PATTERN.test(trimmed)) {
+    throw new Error("Title contains invalid characters");
+  }
+
+  return trimmed;
+}
+
+/**
+ * Escape a string for use in LanceDB SQL-like filters.
+ * Uses comprehensive escaping beyond just single quotes.
+ *
+ * @param value - The value to escape
+ * @returns Escaped string safe for filter expressions
+ */
+export function escapeForFilter(value: string): string {
+  return value
+    .replace(/\\/g, "\\\\")  // Escape backslashes first
+    .replace(/'/g, "''")      // Escape single quotes
+    .replace(/\n/g, "\\n")    // Escape newlines
+    .replace(/\r/g, "\\r")    // Escape carriage returns
+    .replace(/\t/g, "\\t");   // Escape tabs
+}

package/src/embeddings/index.test.ts

@@ -0,0 +1,54 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+
+describe("embeddings index", () => {
+  const originalApiKey = process.env.OPENROUTER_API_KEY;
+
+  beforeEach(() => {
+    vi.resetModules();
+    delete process.env.OPENROUTER_API_KEY;
+  });
+
+  afterEach(() => {
+    if (originalApiKey !== undefined) {
+      process.env.OPENROUTER_API_KEY = originalApiKey;
+    } else {
+      delete process.env.OPENROUTER_API_KEY;
+    }
+  });
+
+  describe("detectProvider", () => {
+    it("should detect local provider when no API key", async () => {
+      const { detectProvider } = await import("./index.js");
+      expect(detectProvider()).toBe("local");
+    });
+
+    it("should detect openrouter provider when API key is set", async () => {
+      process.env.OPENROUTER_API_KEY = "test-key";
+      const { detectProvider } = await import("./index.js");
+      expect(detectProvider()).toBe("openrouter");
+    });
+  });
+
+  describe("getProvider", () => {
+    it("should return detected provider", async () => {
+      const { getProvider } = await import("./index.js");
+      expect(getProvider()).toBe("local");
+    });
+  });
+
+  describe("getEmbeddingDimensions", () => {
+    it("should return dimensions for local provider", async () => {
+      const { getEmbeddingDimensions } = await import("./index.js");
+      expect(getEmbeddingDimensions()).toBe(384);
+    });
+  });
+
+  describe("getProviderDescription", () => {
+    it("should return description for local provider", async () => {
+      const { getProviderDescription } = await import("./index.js");
+      const desc = getProviderDescription();
+      expect(desc).toContain("Local");
+      expect(desc).toContain("384");
+    });
+  });
+});

package/src/embeddings/index.ts

@@ -0,0 +1,111 @@
+/**
+ * Embedding provider auto-detection and unified interface.
+ *
+ * Automatically selects between:
+ * - OpenRouter (if OPENROUTER_API_KEY is set)
+ * - Local HuggingFace (fallback)
+ */
+
+import { getOpenRouterEmbedding, getOpenRouterDimensions } from "./openrouter.js";
+import { getLocalEmbedding, getLocalDimensions, getLocalModelName } from "./local.js";
+import { createDebugLogger } from "../utils/debug.js";
+
+// Debug logging
+const debug = createDebugLogger("EMBED");
+
+// Provider type
+export type EmbeddingProvider = "openrouter" | "local";
+
+// Detect which provider to use based on environment
+let detectedProvider: EmbeddingProvider | null = null;
+
+/**
+ * Detect and cache the embedding provider.
+ * Uses OpenRouter if API key is set, otherwise falls back to local.
+ */
+export function detectProvider(): EmbeddingProvider {
+  if (detectedProvider) {
+    return detectedProvider;
+  }
+
+  if (process.env.OPENROUTER_API_KEY) {
+    detectedProvider = "openrouter";
+    debug("Using OpenRouter embeddings (OPENROUTER_API_KEY found)");
+  } else {
+    detectedProvider = "local";
+    debug(`Using local embeddings (${getLocalModelName()})`);
+  }
+
+  return detectedProvider;
+}
+
+/**
+ * Get the current embedding provider.
+ * Call detectProvider() first to ensure detection has occurred.
+ */
+export function getProvider(): EmbeddingProvider {
+  return detectedProvider || detectProvider();
+}
+
+/**
+ * Generate embedding for text using the auto-detected provider.
+ *
+ * @param text - Text to embed
+ * @returns Promise resolving to embedding vector
+ */
+export async function getEmbedding(text: string): Promise<number[]> {
+  const provider = getProvider();
+
+  if (provider === "openrouter") {
+    return getOpenRouterEmbedding(text);
+  } else {
+    return getLocalEmbedding(text);
+  }
+}
+
+/**
+ * Get the embedding dimensions for the current provider.
+ *
+ * @returns Number of dimensions in embedding vectors
+ */
+export function getEmbeddingDimensions(): number {
+  const provider = getProvider();
+
+  if (provider === "openrouter") {
+    return getOpenRouterDimensions();
+  } else {
+    return getLocalDimensions();
+  }
+}
+
+/**
+ * Get a human-readable description of the current provider.
+ *
+ * @returns Provider description string
+ */
+export function getProviderDescription(): string {
+  const provider = getProvider();
+
+  if (provider === "openrouter") {
+    const model = process.env.EMBEDDING_MODEL || "qwen/qwen3-embedding-8b";
+    const dims = getOpenRouterDimensions();
+    return `OpenRouter (${model}, ${dims} dims)`;
+  } else {
+    const model = getLocalModelName();
+    const dims = getLocalDimensions();
+    return `Local (${model}, ${dims} dims)`;
+  }
+}
+
+// Re-export individual providers for direct access if needed
+export {
+  getOpenRouterEmbedding,
+  getOpenRouterDimensions,
+} from "./openrouter.js";
+
+export {
+  getLocalEmbedding,
+  getLocalDimensions,
+  getLocalModelName,
+  isModelLoaded,
+} from "./local.js";

package/src/embeddings/local.test.ts

@@ -0,0 +1,70 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+
+describe("local embeddings", () => {
+  const originalEnv = process.env.EMBEDDING_MODEL;
+
+  beforeEach(() => {
+    vi.resetModules();
+    delete process.env.EMBEDDING_MODEL;
+  });
+
+  afterEach(() => {
+    if (originalEnv !== undefined) {
+      process.env.EMBEDDING_MODEL = originalEnv;
+    } else {
+      delete process.env.EMBEDDING_MODEL;
+    }
+  });
+
+  describe("getLocalDimensions", () => {
+    it("should return 384 for default model", async () => {
+      const { getLocalDimensions } = await import("./local.js");
+      expect(getLocalDimensions()).toBe(384);
+    });
+
+    it("should return 1024 for bge-m3 model", async () => {
+      process.env.EMBEDDING_MODEL = "Xenova/bge-m3";
+      const { getLocalDimensions } = await import("./local.js");
+      expect(getLocalDimensions()).toBe(1024);
+    });
+
+    it("should return default for unknown model", async () => {
+      process.env.EMBEDDING_MODEL = "unknown/model";
+      const { getLocalDimensions } = await import("./local.js");
+      expect(getLocalDimensions()).toBe(384); // DEFAULT_LOCAL_EMBEDDING_DIMS
+    });
+  });
+
+  describe("getLocalModelName", () => {
+    it("should return default model when env not set", async () => {
+      const { getLocalModelName } = await import("./local.js");
+      expect(getLocalModelName()).toBe("Xenova/multilingual-e5-small");
+    });
+
+    it("should return env model when set", async () => {
+      process.env.EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
+      const { getLocalModelName } = await import("./local.js");
+      expect(getLocalModelName()).toBe("Xenova/all-MiniLM-L6-v2");
+    });
+  });
+
+  describe("isModelLoaded", () => {
+    it("should return false before first embedding call", async () => {
+      const { isModelLoaded } = await import("./local.js");
+      expect(isModelLoaded()).toBe(false);
+    });
+  });
+
+  describe("getLocalEmbedding", () => {
+    it("should throw on empty text", async () => {
+      const { getLocalEmbedding } = await import("./local.js");
+      await expect(getLocalEmbedding("")).rejects.toThrow("non-empty string");
+    });
+
+    it("should throw on non-string input", async () => {
+      const { getLocalEmbedding } = await import("./local.js");
+      // @ts-expect-error - testing runtime validation
+      await expect(getLocalEmbedding(null)).rejects.toThrow("non-empty string");
+    });
+  });
+});