@aitytech/agentkits-memory 1.0.1 → 2.0.0

package/README.md

@@ -46,14 +46,13 @@
 | Feature | Benefit |
 |---------|---------|
 | **100% Local** | All data stays on your machine. No cloud, no API keys, no accounts |
-| **Blazing Fast** | SQLite + WASM = instant queries, zero latency |
+| **Blazing Fast** | Native SQLite (better-sqlite3) = instant queries, zero latency |
 | **Zero Config** | Works out of the box. No database setup required |
 | **Cross-Platform** | Windows, macOS, Linux - same code, same speed |
 | **MCP Server** | `memory_save`, `memory_search`, `memory_recall`, `memory_list`, `memory_status` |
 | **Web Viewer** | Browser UI to view, add, edit, delete memories |
 | **Vector Search** | Optional HNSW semantic similarity (no external service) |
 | **Auto-Capture** | Hooks for session context, tool usage, summaries |
-| **Git-Friendly** | Export to markdown for version control |
 
 ---
 
@@ -220,12 +219,63 @@ Memories are stored in `.claude/memory/memory.db` within your project directory.
 ```
 .claude/memory/
 ├── memory.db          # SQLite database
-├── memory.db-wal      # Write-ahead log (temp)
-└── exports/           # Optional markdown exports
+└── memory.db-wal      # Write-ahead log (temp)
 ```
 
 ---
 
+## CJK Language Support
+
+AgentKits Memory has **automatic CJK support** for Chinese, Japanese, and Korean text search.
+
+### Zero Configuration
+
+When `better-sqlite3` is installed (default), CJK search works automatically:
+
+```typescript
+import { ProjectMemoryService } from '@aitytech/agentkits-memory';
+
+const memory = new ProjectMemoryService('.claude/memory');
+await memory.initialize();
+
+// Store CJK content
+await memory.storeEntry({
+  key: 'auth-pattern',
+  content: '認証機能の実装パターン - JWT with refresh tokens',
+  namespace: 'patterns',
+});
+
+// Search in Japanese, Chinese, or Korean - it just works!
+const results = await memory.query({
+  type: 'hybrid',
+  content: '認証機能',
+});
+```
+
+### How It Works
+
+- **Native SQLite**: Uses `better-sqlite3` for maximum performance
+- **Trigram tokenizer**: FTS5 with trigram creates 3-character sequences for CJK matching
+- **Smart fallback**: Short CJK queries (< 3 chars) automatically use LIKE search
+- **BM25 ranking**: Relevance scoring for search results
+
+### Advanced: Japanese Word Segmentation
+
+For advanced Japanese with proper word segmentation, optionally use lindera:
+
+```typescript
+import { createJapaneseOptimizedBackend } from '@aitytech/agentkits-memory';
+
+const backend = createJapaneseOptimizedBackend({
+  databasePath: '.claude/memory/memory.db',
+  linderaPath: './path/to/liblindera_sqlite.dylib',
+});
+```
+
+Requires [lindera-sqlite](https://github.com/lindera/lindera-sqlite) build.
+
+---
+
 ## API Reference
 
 ### ProjectMemoryService
@@ -258,7 +308,6 @@ interface ProjectMemoryConfig {
 | `semanticSearch(content, k)` | Semantic similarity search |
 | `count(namespace?)` | Count entries |
 | `listNamespaces()` | List all namespaces |
-| `exportToMarkdown(namespace)` | Export to markdown |
 | `getStats()` | Get statistics |
 
 ---

package/dist/better-sqlite3-backend.d.ts

@@ -0,0 +1,192 @@
+/**
+ * BetterSqlite3Backend - Native SQLite with FTS5 Trigram for CJK Support
+ *
+ * Production-grade backend using better-sqlite3 (native SQLite).
+ * Provides:
+ * - FTS5 with trigram tokenizer for CJK (Japanese, Chinese, Korean)
+ * - BM25 ranking for relevance scoring
+ * - 10x faster than sql.js for large datasets
+ * - Proper word segmentation for all languages
+ *
+ * Requires:
+ * - Node.js environment (no browser support)
+ * - npm install better-sqlite3
+ *
+ * @module @agentkits/memory/better-sqlite3-backend
+ */
+import { EventEmitter } from 'node:events';
+import type Database from 'better-sqlite3';
+import { IMemoryBackend, MemoryEntry, MemoryEntryUpdate, MemoryQuery, SearchOptions, SearchResult, BackendStats, HealthCheckResult, EmbeddingGenerator } from './types.js';
+/**
+ * Configuration for BetterSqlite3 Backend
+ */
+export interface BetterSqlite3BackendConfig {
+    /** Path to SQLite database file (:memory: for in-memory) */
+    databasePath: string;
+    /** Enable query optimization and WAL mode */
+    optimize: boolean;
+    /** Default namespace */
+    defaultNamespace: string;
+    /** Embedding generator for semantic search */
+    embeddingGenerator?: EmbeddingGenerator;
+    /** Maximum entries before auto-cleanup */
+    maxEntries: number;
+    /** Enable verbose logging */
+    verbose: boolean;
+    /**
+     * FTS5 tokenizer to use
+     * - 'trigram': Best for CJK (Japanese, Chinese, Korean) - works with all languages
+     * - 'unicode61': Standard tokenizer, good for English/Latin
+     * - 'porter': Stemming for English
+     */
+    ftsTokenizer: 'trigram' | 'unicode61' | 'porter';
+    /** Path to custom SQLite extension (e.g., lindera for advanced Japanese) */
+    extensionPath?: string;
+    /** Custom tokenizer name when using extension (e.g., 'lindera_tokenizer') */
+    customTokenizer?: string;
+}
+/**
+ * BetterSqlite3 Backend for Production Memory Storage
+ *
+ * Features:
+ * - Native SQLite performance (10x faster than sql.js)
+ * - FTS5 with trigram tokenizer for CJK language support
+ * - BM25 relevance ranking
+ * - WAL mode for concurrent reads
+ * - Optional extension loading (lindera, ICU, etc.)
+ */
+export declare class BetterSqlite3Backend extends EventEmitter implements IMemoryBackend {
+    private config;
+    private db;
+    private initialized;
+    private ftsAvailable;
+    private stats;
+    constructor(config?: Partial<BetterSqlite3BackendConfig>);
+    /**
+     * Initialize the BetterSqlite3 backend
+     */
+    initialize(): Promise<void>;
+    /**
+     * Create the database schema
+     */
+    private createSchema;
+    /**
+     * Create FTS5 virtual table with appropriate tokenizer
+     */
+    private createFtsTable;
+    /**
+     * Get the active tokenizer being used
+     */
+    getActiveTokenizer(): string;
+    /**
+     * Check if FTS5 is available and CJK optimized
+     */
+    isFtsAvailable(): boolean;
+    /**
+     * Check if CJK is optimally supported (trigram or lindera)
+     */
+    isCjkOptimized(): boolean;
+    /**
+     * Shutdown the backend
+     */
+    shutdown(): Promise<void>;
+    /**
+     * Store a memory entry
+     */
+    store(entry: MemoryEntry): Promise<void>;
+    /**
+     * Retrieve a memory entry by ID
+     */
+    get(id: string): Promise<MemoryEntry | null>;
+    /**
+     * Retrieve a memory entry by key within a namespace
+     */
+    getByKey(namespace: string, key: string): Promise<MemoryEntry | null>;
+    /**
+     * Update a memory entry
+     */
+    update(id: string, update: MemoryEntryUpdate): Promise<MemoryEntry | null>;
+    /**
+     * Delete a memory entry
+     */
+    delete(id: string): Promise<boolean>;
+    /**
+     * Query memory entries
+     */
+    query(query: MemoryQuery): Promise<MemoryEntry[]>;
+    /**
+     * Full-text search using FTS5
+     */
+    searchFts(query: string, options?: {
+        namespace?: string;
+        limit?: number;
+    }): Promise<MemoryEntry[]>;
+    /**
+     * LIKE-based search fallback
+     */
+    private searchLike;
+    /**
+     * Sanitize query for FTS5
+     */
+    private sanitizeFtsQuery;
+    /**
+     * Semantic vector search
+     */
+    search(embedding: Float32Array, options: SearchOptions): Promise<SearchResult[]>;
+    /**
+     * Calculate cosine similarity between two vectors
+     */
+    private cosineSimilarity;
+    /**
+     * Bulk insert entries
+     */
+    bulkInsert(entries: MemoryEntry[]): Promise<void>;
+    /**
+     * Bulk delete entries
+     */
+    bulkDelete(ids: string[]): Promise<number>;
+    /**
+     * Get entry count
+     */
+    count(namespace?: string): Promise<number>;
+    /**
+     * List all namespaces
+     */
+    listNamespaces(): Promise<string[]>;
+    /**
+     * Clear all entries in a namespace
+     */
+    clearNamespace(namespace: string): Promise<number>;
+    /**
+     * Get backend statistics
+     */
+    getStats(): Promise<BackendStats>;
+    /**
+     * Perform health check
+     */
+    healthCheck(): Promise<HealthCheckResult>;
+    /**
+     * Get the underlying database for advanced operations
+     */
+    getDatabase(): Database.Database | null;
+    /**
+     * Rebuild FTS index
+     */
+    rebuildFtsIndex(): Promise<void>;
+    /**
+     * Convert database row to MemoryEntry
+     */
+    private rowToEntry;
+}
+/**
+ * Create a BetterSqlite3 backend with default CJK support
+ */
+export declare function createBetterSqlite3Backend(config?: Partial<BetterSqlite3BackendConfig>): BetterSqlite3Backend;
+/**
+ * Create a BetterSqlite3 backend with lindera extension for advanced Japanese
+ */
+export declare function createJapaneseOptimizedBackend(config: Partial<BetterSqlite3BackendConfig> & {
+    linderaPath: string;
+}): BetterSqlite3Backend;
+export default BetterSqlite3Backend;
+//# sourceMappingURL=better-sqlite3-backend.d.ts.map

package/dist/better-sqlite3-backend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"better-sqlite3-backend.d.ts","sourceRoot":"","sources":["../src/better-sqlite3-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EACL,cAAc,EACd,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,iBAAiB,EAGjB,kBAAkB,EAEnB,MAAM,YAAY,CAAC;AAEpB;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,4DAA4D;IAC5D,YAAY,EAAE,MAAM,CAAC;IAErB,6CAA6C;IAC7C,QAAQ,EAAE,OAAO,CAAC;IAElB,wBAAwB;IACxB,gBAAgB,EAAE,MAAM,CAAC;IAEzB,8CAA8C;IAC9C,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC,0CAA0C;IAC1C,UAAU,EAAE,MAAM,CAAC;IAEnB,6BAA6B;IAC7B,OAAO,EAAE,OAAO,CAAC;IAEjB;;;;;OAKG;IACH,YAAY,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAEjD,4EAA4E;IAC5E,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,6EAA6E;IAC7E,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAqBD;;;;;;;;;GASG;AACH,qBAAa,oBAAqB,SAAQ,YAAa,YAAW,cAAc;IAC9E,OAAO,CAAC,MAAM,CAA6B;IAC3C,OAAO,CAAC,EAAE,CAAkC;IAC5C,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,YAAY,CAAkB;IAGtC,OAAO,CAAC,KAAK,CAKX;gBAEU,MAAM,GAAE,OAAO,CAAC,0BAA0B,CAAM;IAK5D;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAmEjC;;OAEG;IACH,OAAO,CAAC,YAAY;IAoCpB;;OAEG;IACH,OAAO,CAAC,cAAc;IA0EtB;;OAEG;IACH,kBAAkB,IAAI,MAAM;IAO5B;;OAEG;IACH,cAAc,IAAI,OAAO;IAIzB;;OAEG;IACH,cAAc,IAAI,OAAO;IAOzB;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAS/B;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAwC9C;;OAEG;IACG,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAoBlD;;OAEG;IACG,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAkB3E;;OAEG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAiBhF;;OAEG;IACG,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAO1C;;OAEG;IACG,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAoDvD;;OAEG;IACG,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAgD5G;;OAEG;IACH,OAAO,CAAC,UAAU;IA+BlB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;OAEG;IACG,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAoDtF;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAiBxB;;OAEG;IACG,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAyCvD;;OAEG;IACG,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAYhD;;OAEG;IACG,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBhD;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAUzC;;OAEG;IACG,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAUxD;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,YAAY,CAAC;IA0CvC;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC;IA8E/C;;OAEG;IACH,WAAW,IAAI,QAAQ,CAAC,QAAQ,GAAG,IAAI;IAIvC;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAUtC;;OAEG;IACH,OAAO,CAAC,UAAU;CA4BnB;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CACxC,MAAM,CAAC,EAAE,OAAO,CAAC,0BAA0B,CAAC,GAC3C,oBAAoB,CAKtB;AAED;;GAEG;AACH,wBAAgB,8BAA8B,CAC5C,MAAM,EAAE,OAAO,CAAC,0BAA0B,CAAC,GAAG;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,GACpE,oBAAoB,CAMtB;AAED,eAAe,oBAAoB,CAAC"}