@claudelaw/taichu 0.6.0

package/packages/core/src/sqlite-store.js

@@ -0,0 +1,354 @@
+/**
+ * SQLiteStore — 基于 sql.js (WASM) 的持久化存储
+ *
+ * 特点：
+ *   - 纯 JavaScript + WASM，零原生编译依赖
+ *   - 数据持久化到文件，重启不丢失
+ *   - 支持 JSON 字段查询（SQLite JSON1 扩展）
+ *   - 与 MemoryStore 共享同一接口
+ *
+ * sql.js 是 SQLite 编译为 WebAssembly，跨平台，无需 node-gyp。
+ */
+
+import { randomUUID } from 'node:crypto';
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+const SCHEMA_SQL = `
+CREATE TABLE IF NOT EXISTS documents (
+  id           TEXT PRIMARY KEY,
+  type         TEXT NOT NULL,
+  data         TEXT NOT NULL DEFAULT '{}',
+  status       TEXT NOT NULL DEFAULT 'draft',
+  published_at TEXT,
+  tenant_id    TEXT NOT NULL DEFAULT 'default',
+  created_by   TEXT,
+  created_at   TEXT NOT NULL,
+  updated_at   TEXT NOT NULL,
+  meta         TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE INDEX IF NOT EXISTS idx_documents_type ON documents(type);
+CREATE INDEX IF NOT EXISTS idx_documents_status ON documents(status);
+CREATE INDEX IF NOT EXISTS idx_documents_updated_at ON documents(updated_at);
+CREATE INDEX IF NOT EXISTS idx_documents_type_status ON documents(type, status);
+CREATE INDEX IF NOT EXISTS idx_documents_scheduled ON documents(status, published_at);
+CREATE INDEX IF NOT EXISTS idx_documents_tenant ON documents(tenant_id);
+`;
+
+const MIGRATION_SQL = [
+  `ALTER TABLE documents ADD COLUMN published_at TEXT;`,
+  `ALTER TABLE documents ADD COLUMN tenant_id TEXT NOT NULL DEFAULT 'default';`
+];
+
+const INDEX_MIGRATIONS = [
+  'CREATE INDEX IF NOT EXISTS idx_documents_scheduled ON documents(status, published_at)',
+  'CREATE INDEX IF NOT EXISTS idx_documents_tenant ON documents(tenant_id)'
+];
+
+/**
+ * Create a SQLiteStore instance.
+ *
+ * @param {object} config
+ * @param {string} [config.dataDir] — 数据目录路径，默认 `.taichu/data`
+ * @param {string} [config.dbPath] — 数据库文件路径，默认 `{dataDir}/taichu.db`
+ * @returns {Promise<Store>}
+ */
+export async function createSQLiteStore(config = {}) {
+  // Lazy-load sql.js — only when SQLiteStore is actually used
+  const sqlModule = await import('sql.js');
+
+  let SQL = null;
+  let db = null;
+  let dbPath = null;
+  let dirty = false;
+  let flushTimer = null;
+  const FLUSH_INTERVAL = process.env.TAICHU_SQLITE_FLUSH_MS ? parseInt(process.env.TAICHU_SQLITE_FLUSH_MS) : 5000;
+
+  async function init() {
+    SQL = await sqlModule.default();
+    const dataDir = config.dataDir || join(process.cwd(), '.taichu', 'data');
+    dbPath = config.dbPath || join(dataDir, 'taichu.db');
+
+    // Ensure data directory exists
+    if (!existsSync(dataDir)) {
+      await mkdir(dataDir, { recursive: true });
+    }
+
+    // Load existing database or create new one
+    if (existsSync(dbPath)) {
+      const buffer = await readFile(dbPath);
+      db = new SQL.Database(buffer);
+    } else {
+      db = new SQL.Database();
+    }
+
+    // Run schema migration
+    db.run(SCHEMA_SQL);
+
+    // Run column migrations for existing databases
+    for (const mig of MIGRATION_SQL) {
+      try {
+        db.run(mig);
+      } catch (e) {
+        if (!e.message?.includes('duplicate column')) throw e;
+      }
+    }
+
+    // Ensure indices exist (idempotent for new + migrated DBs)
+    for (const idx of INDEX_MIGRATIONS) {
+      try { db.run(idx); } catch (e) { /* ignore */ }
+    }
+
+    await saveToDisk();
+
+    return db;
+  }
+
+  async function saveToDisk() {
+    const data = db.export();
+    const buffer = Buffer.from(data);
+    await writeFile(dbPath, buffer);
+    dirty = false;
+  }
+
+  /** Mark database as dirty and schedule a debounced flush */
+  function markDirty() {
+    dirty = true;
+    if (flushTimer) clearTimeout(flushTimer);
+    flushTimer = setTimeout(async () => {
+      if (dirty) await saveToDisk();
+    }, FLUSH_INTERVAL);
+  }
+
+  // Flush on process exit
+  process.on('beforeExit', async () => {
+    if (dirty && db) await saveToDisk();
+  });
+
+  function rowToDoc(row) {
+    if (!row) return null;
+    return {
+      id: row[0],
+      type: row[1],
+      data: JSON.parse(row[2]),
+      status: row[3],
+      publishedAt: row[4] || null,
+      tenantId: row[5] || 'default',
+      createdBy: row[6],
+      createdAt: row[7],
+      updatedAt: row[8],
+      meta: JSON.parse(row[9])
+    };
+  }
+
+  /**
+   * Escape SQL string value (simple implementation)
+   */
+  function esc(val) {
+    if (val === null || val === undefined) return 'NULL';
+    if (typeof val === 'number') return String(val);
+    return `'${String(val).replace(/'/g, "''")}'`;
+  }
+
+  // Initialize the database
+  await init();
+
+  return {
+    async create(doc) {
+      const now = new Date().toISOString();
+      const id = doc.id || randomUUID();
+      const type = doc.type || 'default';
+      const data = JSON.stringify(doc.data || {});
+      const status = doc.status || 'draft';
+      const publishedAt = doc.publishedAt || null;
+      const tenantId = doc.tenantId || 'default';
+      const createdBy = doc.createdBy || null;
+      const createdAt = doc.createdAt || now;
+      const meta = JSON.stringify(doc.meta || {});
+
+      db.run(
+        `INSERT INTO documents (id, type, data, status, published_at, tenant_id, created_by, created_at, updated_at, meta)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+        [id, type, data, status, publishedAt, tenantId, createdBy, createdAt, now, meta]
+      );
+
+      markDirty();
+
+      return { id, type, data: JSON.parse(data), status, publishedAt, tenantId, createdBy, createdAt, updatedAt: now, meta: JSON.parse(meta) };
+    },
+
+    async get(id) {
+      const stmt = db.prepare('SELECT * FROM documents WHERE id = ?');
+      stmt.bind([id]);
+
+      if (stmt.step()) {
+        const row = stmt.getAsObject();
+        stmt.free();
+        return rowToDocFromObj(row);
+      }
+      stmt.free();
+      return null;
+    },
+
+    async list(options = {}) {
+      const conditions = [];
+      const params = [];
+
+      if (options.type) {
+        conditions.push('type = ?');
+        params.push(options.type);
+      }
+      if (options.status) {
+        conditions.push('status = ?');
+        params.push(options.status);
+      }
+      if (options.tenantId) {
+        conditions.push('tenant_id = ?');
+        params.push(options.tenantId);
+      }
+      if (options.search) {
+        conditions.push("(data LIKE ? OR type LIKE ?)");
+        const q = `%${options.search}%`;
+        params.push(q, q);
+      }
+
+      const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+
+      // Whitelist validation for ORDER BY columns — prevents SQL injection
+      const ALLOWED_ORDER_COLUMNS = new Set(['id', 'type', 'status', 'created_at', 'updated_at']);
+      const orderBy = ALLOWED_ORDER_COLUMNS.has(options.orderBy) ? options.orderBy : 'updated_at';
+      const order = options.order === 'asc' ? 'ASC' : 'DESC';
+      // LIMIT/OFFSET must be integers
+      const limit = Math.min(Math.max(1, parseInt(options.limit) || 50), 1000);
+      const offset = Math.max(0, parseInt(options.offset) || 0);
+
+      const sql = `SELECT * FROM documents ${where} ORDER BY ${orderBy} ${order} LIMIT ? OFFSET ?`;
+      const allParams = [...params, limit, offset];
+      const stmt = db.prepare(sql);
+      stmt.bind(allParams);
+
+      const results = [];
+      while (stmt.step()) {
+        results.push(rowToDocFromObj(stmt.getAsObject()));
+      }
+      stmt.free();
+
+      return results;
+    },
+
+    async update(id, patch) {
+      const existing = await this.get(id);
+      if (!existing) return null;
+
+      const now = new Date().toISOString();
+
+      // Batch all updates in a single transaction
+      db.run('BEGIN TRANSACTION');
+
+      if (patch.data) {
+        const merged = { ...existing.data, ...patch.data };
+        db.run('UPDATE documents SET data = ?, updated_at = ? WHERE id = ?', [
+          JSON.stringify(merged), now, id
+        ]);
+        existing.data = merged;
+      }
+
+      if (patch.status !== undefined) {
+        db.run('UPDATE documents SET status = ?, updated_at = ? WHERE id = ?', [
+          patch.status, now, id
+        ]);
+        existing.status = patch.status;
+      }
+
+      // publishedAt: explicit null clears it, string value sets it
+      if (patch.publishedAt !== undefined) {
+        const val = patch.publishedAt || null;
+        db.run('UPDATE documents SET published_at = ?, updated_at = ? WHERE id = ?', [
+          val, now, id
+        ]);
+        existing.publishedAt = val;
+      }
+
+      if (patch.meta) {
+        const merged = { ...existing.meta, ...patch.meta };
+        db.run('UPDATE documents SET meta = ?, updated_at = ? WHERE id = ?', [
+          JSON.stringify(merged), now, id
+        ]);
+        existing.meta = merged;
+      }
+
+      db.run('UPDATE documents SET updated_at = ? WHERE id = ?', [now, id]);
+      db.run('COMMIT');
+
+      existing.updatedAt = now;
+      markDirty();
+      return existing;
+    },
+
+    async delete(id) {
+      const existing = await this.get(id);
+      if (!existing) return false;
+
+      db.run('DELETE FROM documents WHERE id = ?', [id]);
+      markDirty();
+      return true;
+    },
+
+    async count(options = {}) {
+      const conditions = [];
+      const params = [];
+
+      if (options.type) {
+        conditions.push('type = ?');
+        params.push(options.type);
+      }
+      if (options.status) {
+        conditions.push('status = ?');
+        params.push(options.status);
+      }
+
+      const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+      const stmt = db.prepare(`SELECT COUNT(*) as cnt FROM documents ${where}`);
+      stmt.bind(params);
+
+      let count = 0;
+      if (stmt.step()) {
+        count = stmt.getAsObject().cnt;
+      }
+      stmt.free();
+      return count;
+    },
+
+    /** Close database connection */
+    async close() {
+      if (dirty && db) await saveToDisk();
+      if (db) {
+        db.close();
+        db = null;
+      }
+    },
+
+    /** Get database path */
+    getDbPath() {
+      return dbPath;
+    }
+  };
+}
+
+function rowToDocFromObj(row) {
+  if (!row) return null;
+  return {
+    id: row.id,
+    type: row.type,
+    data: JSON.parse(row.data),
+    status: row.status,
+    publishedAt: row.published_at || null,
+    tenantId: row.tenant_id || 'default',
+    createdBy: row.created_by || null,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+    meta: JSON.parse(row.meta || '{}')
+  };
+}

package/packages/core/src/store.js

@@ -0,0 +1,174 @@
+/**
+ * Store — 存储抽象层
+ *
+ * 提供统一的 CRUD 接口，后端可以置换不同的存储引擎：
+ *   - MemoryStore  — 开发/测试用，内存存储
+ *   - SQLiteStore  — 生产环境，基于 better-sqlite3
+ *   - 未来扩展：PostgresStore, FileStore, etc.
+ *
+ * 所有 Store 实现同一个接口，确保上层代码零修改。
+ */
+
+/**
+ * @typedef {object} Document
+ * @property {string} id — 唯一标识符（UUID）
+ * @property {string} type — 内容类型名称
+ * @property {object} data — 结构化内容数据
+ * @property {string} status — 'draft' | 'scheduled' | 'published' | 'archived'
+ * @property {string|null} publishedAt — 定时发布时间 (ISO 8601)
+ * @property {string} tenantId — 租户 ID (默认 'default')
+ * @property {string} createdAt — ISO 8601
+ * @property {string} updatedAt — ISO 8601
+ * @property {string} [createdBy] — 创建者 ID（人类或 Agent）
+ * @property {object} [meta] — 额外元数据
+ */
+
+/**
+ * @typedef {object} QueryOptions
+ * @property {string} [type] — 按内容类型过滤
+ * @property {string} [status] — 按状态过滤
+ * @property {string} [tenantId] — 按租户 ID 过滤
+ * @property {string} [search] — 全文搜索关键词
+ * @property {number} [limit] — 返回数量上限
+ * @property {number} [offset] — 分页偏移
+ * @property {string} [orderBy] — 排序字段
+ * @property {'asc'|'desc'} [order] — 排序方向
+ */
+
+/**
+ * @typedef {object} Store
+ * @property {function(Document): Promise<Document>} create
+ * @property {function(string): Promise<Document|null>} get
+ * @property {function(QueryOptions): Promise<Document[]>} list
+ * @property {function(string, object): Promise<Document>} update
+ * @property {function(string): Promise<boolean>} delete
+ * @property {function(): Promise<number>} count
+ */
+
+import { randomUUID } from 'node:crypto';
+
+/**
+ * MemoryStore — 内存存储实现
+ * 适合开发、测试、和小规模单机部署
+ */
+export function createMemoryStore() {
+  /** @type {Map<string, Document>} */
+  const docs = new Map();
+
+  return {
+    async create(doc) {
+      const now = new Date().toISOString();
+      const document = {
+        id: doc.id || randomUUID(),
+        type: doc.type || 'default',
+        data: doc.data || {},
+        status: doc.status || 'draft',
+        publishedAt: doc.publishedAt || null,
+        tenantId: doc.tenantId || 'default',
+        createdAt: doc.createdAt || now,
+        updatedAt: now,
+        createdBy: doc.createdBy || null,
+        meta: doc.meta || {}
+      };
+      docs.set(document.id, document);
+      return { ...document, data: { ...document.data } };
+    },
+
+    async get(id) {
+      const doc = docs.get(id);
+      if (!doc) return null;
+      return { ...doc, data: { ...doc.data } };
+    },
+
+    async list(options = {}) {
+      let results = Array.from(docs.values());
+
+      if (options.type) {
+        results = results.filter(d => d.type === options.type);
+      }
+      if (options.status) {
+        results = results.filter(d => d.status === options.status);
+      }
+      if (options.tenantId) {
+        results = results.filter(d => d.tenantId === options.tenantId);
+      }
+      if (options.search) {
+        const q = options.search.toLowerCase();
+        results = results.filter(d =>
+          JSON.stringify(d.data).toLowerCase().includes(q) ||
+          d.type.toLowerCase().includes(q)
+        );
+      }
+
+      // Sort by updatedAt descending by default
+      const orderBy = options.orderBy || 'updatedAt';
+      const order = options.order || 'desc';
+      results.sort((a, b) => {
+        const va = a[orderBy] || '';
+        const vb = b[orderBy] || '';
+        return order === 'desc' ? vb.localeCompare(va) : va.localeCompare(vb);
+      });
+
+      const offset = options.offset || 0;
+      const limit = options.limit || 50;
+      return results.slice(offset, offset + limit).map(d => ({
+        ...d, data: { ...d.data }
+      }));
+    },
+
+    async update(id, patch) {
+      const doc = docs.get(id);
+      if (!doc) return null;
+      const updated = {
+        ...doc,
+        data: patch.data ? { ...doc.data, ...patch.data } : doc.data,
+        status: patch.status !== undefined ? patch.status : doc.status,
+        publishedAt: patch.publishedAt !== undefined ? patch.publishedAt : doc.publishedAt,
+        meta: patch.meta ? { ...doc.meta, ...patch.meta } : doc.meta,
+        updatedAt: new Date().toISOString()
+      };
+      docs.set(id, updated);
+      return { ...updated, data: { ...updated.data } };
+    },
+
+    async delete(id) {
+      return docs.delete(id);
+    },
+
+    async count(options = {}) {
+      let results = Array.from(docs.values());
+      if (options.type) results = results.filter(d => d.type === options.type);
+      if (options.status) results = results.filter(d => d.status === options.status);
+      return results.length;
+    }
+  };
+}
+
+/**
+ * Create a store instance based on the engine configuration.
+ *
+ * Supported engines:
+ *   - 'memory' — in-memory store (default, zero dependencies)
+ *   - 'sqlite'  — SQLite via sql.js WASM (persistent, requires sql.js package)
+ *
+ * @param {object} config
+ * @param {string} [config.engine] — 'memory' | 'sqlite'
+ * @param {string} [config.dataDir] — data directory for file-based stores
+ * @returns {Promise<Store>}
+ */
+export async function createStore(config = {}) {
+  const engine = (config.engine || process.env.TAICHU_STORAGE || 'memory').toLowerCase();
+
+  switch (engine) {
+    case 'memory':
+      return createMemoryStore();
+
+    case 'sqlite': {
+      const { createSQLiteStore } = await import('./sqlite-store.js');
+      return createSQLiteStore(config);
+    }
+
+    default:
+      throw new Error(`Unknown storage engine: "${engine}". Supported: memory, sqlite`);
+  }
+}

package/packages/core/src/tokenizer.js

@@ -0,0 +1,89 @@
+/**
+ * Tokenizer — 可插拔分词器
+ *
+ * 优先级：
+ *   1. nodejieba（中文词级分词，可选安装）
+ *   2. 内置 n-gram（零依赖回退）
+ */
+
+let _cutter = null;
+
+/**
+ * Try to load jieba tokenizer (optional dependency).
+ * Returns null if not installed.
+ */
+export async function tryLoadJieba() {
+  try {
+    const jieba = await import('nodejieba');
+    // nodejieba exports: { cut, tag, extract, ... }
+    const cut = (text) => {
+      return jieba.cut(text);
+    };
+    return cut;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Set a custom tokenizer.
+ */
+export function setTokenizer(fn) {
+  _cutter = fn;
+}
+
+const STOPWORDS = new Set([
+  '的','了','在','是','我','有','和','就','不','人','都','一','一个',
+  '上','也','很','到','说','要','去','你','会','着','没有','看','好',
+  '自己','这','他','她','它','们','那','些','什么','怎么','哪','为什么',
+  '吗','吧','啊','呢','哦','哈','嗯','呀','the','a','an','is','are',
+  'was','were','be','been','in','on','at','to','for','of','with',
+  'by','from','and','or','but','not','this','that','it','as','its'
+]);
+
+/**
+ * Tokenize text. Uses jieba if available, otherwise n-gram fallback.
+ */
+export function tokenize(text, cutter = null) {
+  if (!text || typeof text !== 'string') return [];
+  const cut = cutter || _cutter;
+
+  if (cut) return _jiebaTokens(text, cut);
+  return _ngramTokens(text);
+}
+
+function _jiebaTokens(text, cut) {
+  const cleaned = text
+    .replace(/[^\w\u4e00-\u9fff\s]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+  if (!cleaned) return [];
+  const words = cut(cleaned);
+  return words.filter(w => w.length > 1 && !STOPWORDS.has(w));
+}
+
+function _ngramTokens(text) {
+  const cleaned = text.toLowerCase()
+    .replace(/[，。！？、；：""'（）【】《》\n\r\t]+/g, ' ')
+    .replace(/[^\w\u4e00-\u9fff\s]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+  if (!cleaned) return [];
+
+  const tokens = [];
+  // Split Chinese segments and English words
+  for (let i = 0; i < cleaned.length; i++) {
+    const ch = cleaned[i];
+    if (/[\u4e00-\u9fff]/.test(ch)) {
+      // Chinese: 2-gram
+      if (i + 1 < cleaned.length && /[\u4e00-\u9fff]/.test(cleaned[i + 1])) {
+        tokens.push(cleaned.substring(i, i + 2));
+      }
+    }
+  }
+  // English words
+  const words = cleaned.match(/[a-z]{2,}/g) || [];
+  tokens.push(...words);
+
+  return tokens.filter(t => !STOPWORDS.has(t));
+}