autosnippet 3.2.9 → 3.2.11

package/lib/infrastructure/vector/Chunker.js

@@ -1,177 +0,0 @@
-/**
- * Chunker — 内容分块策略
- * 支持 4 种策略：whole、section（按标题）、fixed（固定大小+重叠）、auto（自适应）
- */
-
-import { estimateTokens } from '../../shared/token-utils.js';
-export { estimateTokens };
-
-const DEFAULT_MAX_CHUNK_TOKENS = 512;
-const DEFAULT_OVERLAP_TOKENS = 50;
-
-/**
- * 将内容分块
- * @param {string} content
- * @param {object} metadata — { type, sourcePath, language, ... }
- * @param {object} options — { strategy, maxChunkTokens, overlapTokens }
- * @returns {Array<{ content: string, metadata: object }>}
- */
-export function chunk(content, metadata = {}, options = {}) {
-  const {
-    strategy = 'auto',
-    maxChunkTokens = DEFAULT_MAX_CHUNK_TOKENS,
-    overlapTokens = DEFAULT_OVERLAP_TOKENS,
-  } = options;
-
-  if (!content || content.trim().length === 0) {
-    return [];
-  }
-
-  const tokens = estimateTokens(content);
-
-  // 选择策略
-  let selectedStrategy = strategy;
-  if (strategy === 'auto') {
-    if (tokens <= maxChunkTokens) {
-      selectedStrategy = 'whole';
-    } else if (content.includes('## ') || content.includes('### ')) {
-      selectedStrategy = 'section';
-    } else {
-      selectedStrategy = 'fixed';
-    }
-  }
-
-  switch (selectedStrategy) {
-    case 'whole':
-      return [{ content, metadata: { ...metadata, chunkIndex: 0, totalChunks: 1 } }];
-    case 'section':
-      return chunkBySection(content, metadata, maxChunkTokens);
-    case 'fixed':
-      return chunkFixed(content, metadata, maxChunkTokens, overlapTokens);
-    default:
-      return [{ content, metadata: { ...metadata, chunkIndex: 0, totalChunks: 1 } }];
-  }
-}
-
-/**
- * 按 Markdown 标题分段
- */
-function chunkBySection(content, metadata, maxChunkTokens) {
-  const sections = [];
-  const lines = content.split('\n');
-  let currentTitle = '';
-  let currentContent = [];
-
-  for (const line of lines) {
-    if (/^#{1,3}\s+/.test(line)) {
-      // 新段落
-      if (currentContent.length > 0) {
-        sections.push({ title: currentTitle, content: currentContent.join('\n') });
-      }
-      currentTitle = line.replace(/^#+\s+/, '').trim();
-      currentContent = [line];
-    } else {
-      currentContent.push(line);
-    }
-  }
-  // 最后一段
-  if (currentContent.length > 0) {
-    sections.push({ title: currentTitle, content: currentContent.join('\n') });
-  }
-
-  // 合并过小段落
-  const merged = [];
-  let buffer = null;
-
-  for (const section of sections) {
-    if (!buffer) {
-      buffer = section;
-      continue;
-    }
-    const combined = `${buffer.content}\n${section.content}`;
-    if (estimateTokens(combined) <= maxChunkTokens) {
-      buffer = { title: buffer.title, content: combined };
-    } else {
-      merged.push(buffer);
-      buffer = section;
-    }
-  }
-  if (buffer) {
-    merged.push(buffer);
-  }
-
-  // 对超大段落做 fixed 分割
-  const results = [];
-  for (let i = 0; i < merged.length; i++) {
-    const section = merged[i];
-    if (estimateTokens(section.content) > maxChunkTokens) {
-      const subChunks = chunkFixed(section.content, metadata, maxChunkTokens, 0);
-      for (const sub of subChunks) {
-        results.push({
-          content: sub.content,
-          metadata: {
-            ...metadata,
-            ...sub.metadata,
-            sectionTitle: section.title,
-            chunkIndex: results.length,
-          },
-        });
-      }
-    } else {
-      results.push({
-        content: section.content,
-        metadata: { ...metadata, sectionTitle: section.title, chunkIndex: results.length },
-      });
-    }
-  }
-
-  // 设置 totalChunks
-  for (const chunk of results) {
-    chunk.metadata.totalChunks = results.length;
-  }
-  return results;
-}
-
-/**
- * 固定大小分块（带重叠）
- */
-function chunkFixed(content, metadata, maxChunkTokens, overlapTokens) {
-  const maxChars = maxChunkTokens * 4;
-  const overlapChars = overlapTokens * 4;
-  const results = [];
-
-  let start = 0;
-  while (start < content.length) {
-    let end = start + maxChars;
-
-    // 尽量在句子边界切割
-    if (end < content.length) {
-      const boundary = content.lastIndexOf('\n', end);
-      if (boundary > start + maxChars * 0.5) {
-        end = boundary + 1;
-      }
-    } else {
-      end = content.length;
-    }
-
-    results.push({
-      content: content.slice(start, end),
-      metadata: { ...metadata, chunkIndex: results.length },
-    });
-
-    // 下一个开始位置（含重叠）
-    const nextStart = end - overlapChars;
-    // 确保至少前进 1 字符，防止 overlap >= maxChars 时无限循环
-    start = nextStart > start ? nextStart : end;
-    if (start >= content.length) {
-      break;
-    }
-  }
-
-  for (const chunk of results) {
-    chunk.metadata.totalChunks = results.length;
-  }
-  return results;
-}
-
-export { DEFAULT_MAX_CHUNK_TOKENS, DEFAULT_OVERLAP_TOKENS };

package/lib/infrastructure/vector/IndexingPipeline.js

@@ -1,209 +0,0 @@
-/**
- * IndexingPipeline — 索引管线
- * scan → chunk → detect incremental changes (sourceHash) → embed via AI → batch upsert
- */
-
-import { createHash } from 'node:crypto';
-import { existsSync, readdirSync, readFileSync } from 'node:fs';
-import { extname, join, relative } from 'node:path';
-import { LanguageService } from '../../shared/LanguageService.js';
-import { chunk } from './Chunker.js';
-
-const SCANNABLE_EXTENSIONS = new Set([
-  '.md',
-  '.markdown',
-  '.txt',
-  '.swift',
-  '.m',
-  '.h',
-  '.js',
-  '.ts',
-  '.jsx',
-  '.tsx',
-  '.py',
-  '.java',
-  '.kt',
-  '.go',
-  '.rs',
-  '.rb',
-]);
-
-export class IndexingPipeline {
-  #vectorStore; // VectorStore 实例
-  #aiProvider; // AiProvider 实例 (可选, 用于 embedding)
-  #scanDirs; // 要扫描的目录
-  #projectRoot;
-
-  constructor(options = {}) {
-    this.#vectorStore = options.vectorStore || null;
-    this.#aiProvider = options.aiProvider || null;
-    this.#scanDirs = options.scanDirs || ['recipes', 'AutoSnippet/recipes'];
-    this.#projectRoot = options.projectRoot || process.cwd();
-  }
-
-  setVectorStore(store) {
-    this.#vectorStore = store;
-  }
-  setAiProvider(provider) {
-    this.#aiProvider = provider;
-  }
-
-  /**
-   * 运行完整索引管线
-   * @param {object} options — { force: boolean, dryRun: boolean }
-   * @returns {{ scanned, chunked, embedded, upserted, skipped, errors }}
-   */
-  async run(options = {}) {
-    const { force = false, dryRun = false } = options;
-    const stats = { scanned: 0, chunked: 0, embedded: 0, upserted: 0, skipped: 0, errors: 0 };
-
-    if (!this.#vectorStore) {
-      throw new Error('VectorStore not set');
-    }
-
-    // 1. 扫描文件
-    const files = this.scan();
-    stats.scanned = files.length;
-
-    // 2. 增量检测 + 分块 + 嵌入 + 写入
-    const existingIds = new Set(await this.#vectorStore.listIds());
-    const batch = [];
-
-    for (const file of files) {
-      try {
-        const content = readFileSync(file.absolutePath, 'utf-8');
-        const hash = this.hashContent(content);
-        const baseId = relative(this.#projectRoot, file.absolutePath).replace(/\//g, '_');
-
-        // 增量检测：hash 未变时跳过
-        if (!force) {
-          const existing = await this.#vectorStore.getById(`${baseId}_0`);
-          if (existing?.metadata?.sourceHash === hash) {
-            stats.skipped++;
-            continue;
-          }
-        }
-
-        // 分块
-        const chunks = chunk(content, {
-          type: file.type,
-          sourcePath: file.relativePath,
-          sourceHash: hash,
-          language: this.#detectLanguage(file.absolutePath),
-        });
-        stats.chunked += chunks.length;
-
-        // 嵌入（如果有 AI Provider）
-        for (let i = 0; i < chunks.length; i++) {
-          const chunkItem = chunks[i];
-          const id = `${baseId}_${i}`;
-          let vector = [];
-
-          if (this.#aiProvider && typeof this.#aiProvider.embed === 'function') {
-            try {
-              vector = await this.#aiProvider.embed(chunkItem.content);
-              stats.embedded++;
-            } catch {
-              /* embed optional */
-            }
-          }
-
-          batch.push({
-            id,
-            content: chunkItem.content,
-            vector,
-            metadata: { ...chunkItem.metadata, chunkIndex: i },
-          });
-        }
-
-        // 清理旧 chunk (如果文件 chunk 数减少)
-        for (const existId of existingIds) {
-          if (existId.startsWith(`${baseId}_`)) {
-            const idx = parseInt(existId.split('_').pop(), 10);
-            if (idx >= chunks.length) {
-              if (!dryRun) {
-                await this.#vectorStore.remove(existId);
-              }
-            }
-          }
-        }
-      } catch (_error) {
-        stats.errors++;
-      }
-    }
-
-    // 3. 批量写入
-    if (!dryRun && batch.length > 0) {
-      await this.#vectorStore.batchUpsert(batch);
-      stats.upserted = batch.length;
-    }
-
-    return stats;
-  }
-
-  /**
-   * 扫描项目中的可索引文件
-   * @returns {Array<{ absolutePath, relativePath, type }>}
-   */
-  scan() {
-    const files = [];
-
-    for (const dir of this.#scanDirs) {
-      const absDir = join(this.#projectRoot, dir);
-      if (!existsSync(absDir)) {
-        continue;
-      }
-      this.#walkDir(absDir, files);
-    }
-
-    // 也扫描根目录的 README
-    const readmePath = join(this.#projectRoot, 'README.md');
-    if (existsSync(readmePath)) {
-      files.push({
-        absolutePath: readmePath,
-        relativePath: 'README.md',
-        type: 'readme',
-      });
-    }
-
-    return files;
-  }
-
-  /**
-   * 计算内容 hash
-   */
-  hashContent(content) {
-    return createHash('sha256').update(content).digest('hex').slice(0, 16);
-  }
-
-  #walkDir(dir, files) {
-    try {
-      const entries = readdirSync(dir, { withFileTypes: true });
-      for (const entry of entries) {
-        const fullPath = join(dir, entry.name);
-        if (entry.isDirectory()) {
-          if (entry.name.startsWith('.') || entry.name === 'node_modules') {
-            continue;
-          }
-          this.#walkDir(fullPath, files);
-        } else if (entry.isFile()) {
-          const ext = extname(entry.name).toLowerCase();
-          if (SCANNABLE_EXTENSIONS.has(ext)) {
-            files.push({
-              absolutePath: fullPath,
-              relativePath: relative(this.#projectRoot, fullPath),
-              type: ext === '.md' || ext === '.markdown' ? 'recipe' : 'code',
-            });
-          }
-        }
-      }
-    } catch {
-      /* skip unreadable dirs */
-    }
-  }
-
-  #detectLanguage(filePath) {
-    const lang = LanguageService.inferLang(filePath);
-    return lang === 'unknown' ? 'text' : lang;
-  }
-}

package/lib/infrastructure/vector/JsonVectorAdapter.js

@@ -1,279 +0,0 @@
-/**
- * JsonVectorAdapter — 基于 JSON 文件的向量存储实现
- * 适用于中小规模（<10K 文档），无外部依赖
- * 支持余弦相似度搜索、混合搜索（向量 70% + 关键词 30%）
- */
-
-import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
-import { dirname, join } from 'node:path';
-import pathGuard from '../../shared/PathGuard.js';
-import { cosineSimilarity } from '../../shared/similarity.js';
-import { VectorStore } from './VectorStore.js';
-
-export class JsonVectorAdapter extends VectorStore {
-  #indexPath;
-  #data; // Map<id, { id, content, vector, metadata }>
-  #dirty;
-
-  constructor(projectRoot, options = {}) {
-    super();
-    const contextDir = options.contextDir || '.autosnippet/context/index';
-    this.#indexPath = options.indexPath || join(projectRoot, contextDir, 'vector_index.json');
-    this.#data = new Map();
-    this.#dirty = false;
-  }
-
-  async init() {
-    this.#load();
-  }
-
-  /**
-   * 同步初始化 — 供 ServiceContainer 懒加载工厂使用
-   * （#load 本身就是同步的 readFileSync，无需 await）
-   */
-  initSync() {
-    this.#load();
-  }
-
-  async upsert(item) {
-    if (!item?.id) {
-      throw new Error('Item must have an id');
-    }
-    this.#data.set(item.id, {
-      id: item.id,
-      content: item.content || '',
-      vector: item.vector || [],
-      metadata: item.metadata || {},
-      updatedAt: Date.now(),
-    });
-    this.#dirty = true;
-    this.#autoSave();
-  }
-
-  async batchUpsert(items) {
-    for (const item of items) {
-      if (!item?.id) {
-        continue;
-      }
-      this.#data.set(item.id, {
-        id: item.id,
-        content: item.content || '',
-        vector: item.vector || [],
-        metadata: item.metadata || {},
-        updatedAt: Date.now(),
-      });
-    }
-    this.#dirty = true;
-    this.#autoSave();
-  }
-
-  async remove(id) {
-    this.#data.delete(id);
-    this.#dirty = true;
-    this.#autoSave();
-  }
-
-  async getById(id) {
-    return this.#data.get(id) || null;
-  }
-
-  /**
-   * 向量相似度搜索（余弦相似度）
-   */
-  async searchVector(queryVector, options = {}) {
-    const { topK = 10, filter = null, minScore = 0 } = options;
-
-    if (!queryVector || queryVector.length === 0) {
-      return [];
-    }
-
-    let candidates = [...this.#data.values()];
-
-    // 应用过滤
-    if (filter) {
-      candidates = this.#applyFilter(candidates, filter);
-    }
-
-    // 计算余弦相似度
-    const scored = candidates
-      .filter((item) => item.vector && item.vector.length > 0)
-      .map((item) => ({
-        item,
-        score: this.#cosineSimilarity(queryVector, item.vector),
-      }))
-      .filter((result) => result.score >= minScore)
-      .sort((a, b) => b.score - a.score)
-      .slice(0, topK);
-
-    return scored;
-  }
-
-  /**
-   * 混合搜索：向量 70% + 关键词 30%
-   */
-  async hybridSearch(queryVector, queryText, options = {}) {
-    const { topK = 10, filter = null } = options;
-
-    let candidates = [...this.#data.values()];
-    if (filter) {
-      candidates = this.#applyFilter(candidates, filter);
-    }
-
-    const scored = candidates
-      .map((item) => {
-        // 向量分数
-        let vectorScore = 0;
-        if (queryVector && queryVector.length > 0 && item.vector && item.vector.length > 0) {
-          vectorScore = this.#cosineSimilarity(queryVector, item.vector);
-        }
-
-        // 关键词分数
-        let keywordScore = 0;
-        if (queryText) {
-          const text = (item.content || '').toLowerCase();
-          const query = queryText.toLowerCase();
-          const words = query.split(/\s+/);
-          const hits = words.filter((w) => text.includes(w)).length;
-          keywordScore = words.length > 0 ? hits / words.length : 0;
-        }
-
-        return {
-          item,
-          score: vectorScore * 0.7 + keywordScore * 0.3,
-          vectorScore,
-          keywordScore,
-        };
-      })
-      .filter((r) => r.score > 0)
-      .sort((a, b) => b.score - a.score)
-      .slice(0, topK);
-
-    return scored;
-  }
-
-  /**
-   * query() — SearchEngine / RetrievalFunnel 使用的向量搜索别名
-   * 接口: query(vector, topK) → Array<{ id, similarity, metadata }>
-   */
-  async query(queryVector, topK = 10) {
-    const results = await this.searchVector(queryVector, { topK });
-    return results.map((r) => ({
-      id: r.item.id,
-      similarity: r.score,
-      score: r.score,
-      content: r.item.content,
-      metadata: r.item.metadata || {},
-    }));
-  }
-
-  async searchByFilter(filter) {
-    return this.#applyFilter([...this.#data.values()], filter);
-  }
-
-  async listIds() {
-    return [...this.#data.keys()];
-  }
-
-  async clear() {
-    this.#data.clear();
-    this.#dirty = true;
-    this.#autoSave();
-  }
-
-  async getStats() {
-    let indexSize = 0;
-    try {
-      if (existsSync(this.#indexPath)) {
-        indexSize = statSync(this.#indexPath).size;
-      }
-    } catch {
-      /* ignore */
-    }
-
-    return {
-      count: this.#data.size,
-      indexSize,
-      indexPath: this.#indexPath,
-      hasVectors: [...this.#data.values()].filter((d) => d.vector?.length > 0).length,
-    };
-  }
-
-  // --- 私有方法 ---
-
-  #applyFilter(items, filter) {
-    return items.filter((item) => {
-      const meta = item.metadata || {};
-      if (filter.type && meta.type !== filter.type) {
-        return false;
-      }
-      if (filter.category && meta.category !== filter.category) {
-        return false;
-      }
-      if (filter.language && meta.language !== filter.language) {
-        return false;
-      }
-      if (filter.sourcePath && !meta.sourcePath?.includes(filter.sourcePath)) {
-        return false;
-      }
-      if (filter.module && meta.module !== filter.module) {
-        return false;
-      }
-      if (filter.tags && Array.isArray(filter.tags)) {
-        const itemTags = meta.tags || [];
-        if (!filter.tags.some((t) => itemTags.includes(t))) {
-          return false;
-        }
-      }
-      if (filter.deprecated === false && meta.deprecated) {
-        return false;
-      }
-      return true;
-    });
-  }
-
-  #cosineSimilarity(a, b) {
-    return cosineSimilarity(a, b);
-  }
-
-  #load() {
-    try {
-      if (!existsSync(this.#indexPath)) {
-        return;
-      }
-      const raw = readFileSync(this.#indexPath, 'utf-8');
-      const items = JSON.parse(raw);
-      if (Array.isArray(items)) {
-        for (const item of items) {
-          if (item?.id) {
-            this.#data.set(item.id, item);
-          }
-        }
-      } else if (typeof items === 'object') {
-        // 兼容旧格式 { id: item }
-        for (const [id, item] of Object.entries(items)) {
-          this.#data.set(id, { ...item, id });
-        }
-      }
-    } catch {
-      /* silent: start empty */
-    }
-  }
-
-  #autoSave() {
-    if (!this.#dirty) {
-      return;
-    }
-    try {
-      const dir = dirname(this.#indexPath);
-      pathGuard.assertProjectWriteSafe(dir);
-      if (!existsSync(dir)) {
-        mkdirSync(dir, { recursive: true });
-      }
-      const items = [...this.#data.values()];
-      writeFileSync(this.#indexPath, JSON.stringify(items, null, 2));
-      this.#dirty = false;
-    } catch {
-      /* silent */
-    }
-  }
-}