autosnippet 3.2.10 → 3.2.12

package/lib/service/recipe/RecipeParser.js

@@ -1,309 +0,0 @@
-/**
- * RecipeParser — Recipe Markdown 解析器
- * 从完整 Recipe MD 提取结构化数据
- */
-
-import fs from 'node:fs';
-import path from 'node:path';
-import { LanguageService } from '../../shared/LanguageService.js';
-
-const FRONTMATTER_RE = /^---\n([\s\S]*?)\n---/;
-const _SNIPPET_HEADING_RE = /^##\s+(?:Snippet|Code|代码)/im;
-const USAGE_HEADING_RE = /^##\s+(?:Usage\s*Guide|用法|使用指南)/im;
-const FENCED_CODE_RE = /```(\w*)\n([\s\S]*?)```/;
-
-export class RecipeParser {
-  /**
-   * 检查文本是否为完整 Recipe MD
-   * 需包含: frontmatter + 代码块 + Usage Guide
-   */
-  isCompleteRecipe(text) {
-    if (!text) {
-      return false;
-    }
-    return FRONTMATTER_RE.test(text) && FENCED_CODE_RE.test(text) && USAGE_HEADING_RE.test(text);
-  }
-
-  /**
-   * 检查是否为「仅介绍」Recipe（有 frontmatter 但无代码块）
-   */
-  isIntroOnly(text) {
-    if (!text) {
-      return false;
-    }
-    return FRONTMATTER_RE.test(text) && !FENCED_CODE_RE.test(text);
-  }
-
-  /**
-   * 解析完整 Recipe MD 为结构化对象
-   * @param {string} text
-   * @returns {object|null}
-   */
-  parse(text) {
-    if (!text) {
-      return null;
-    }
-
-    const frontmatter = this.parseFrontmatter(text);
-    const body = text.replace(FRONTMATTER_RE, '').trim();
-
-    // 提取代码块
-    const codeBlocks = [];
-    let match;
-    const codeRe = /```(\w*)\n([\s\S]*?)```/g;
-    while ((match = codeRe.exec(body)) !== null) {
-      codeBlocks.push({ language: match[1] || 'text', code: match[2].trim() });
-    }
-
-    // 提取 Usage Guide
-    const usageMatch = body.match(USAGE_HEADING_RE);
-    let usageGuide = '';
-    if (usageMatch) {
-      const usageStart = usageMatch.index + usageMatch[0].length;
-      const nextHeading = body.slice(usageStart).search(/^##\s+/m);
-      usageGuide =
-        nextHeading > 0
-          ? body.slice(usageStart, usageStart + nextHeading).trim()
-          : body.slice(usageStart).trim();
-    }
-
-    // 提取标题
-    const titleMatch = body.match(/^#\s+(.+)/m);
-    const title = frontmatter.title || (titleMatch ? titleMatch[1].trim() : '');
-
-    // 提取 headers
-    const headers = this.#extractHeaders(body);
-
-    return {
-      title,
-      summary: frontmatter.summary || frontmatter.description || '',
-      description: frontmatter.description || frontmatter.summary || '',
-      trigger: frontmatter.trigger || this.#generateTrigger(title),
-      category: frontmatter.category || 'general',
-      language:
-        frontmatter.language ||
-        (codeBlocks[0]?.language !== 'text' ? codeBlocks[0]?.language : 'swift'),
-      code: codeBlocks.map((b) => b.code).join('\n\n'),
-      codeBlocks,
-      usageGuide,
-      headers,
-      includeHeaders: headers.length > 0,
-      frontmatter,
-    };
-  }
-
-  /**
-   * 从文本中解析多段 Recipe（按 `---` 分隔）
-   */
-  parseAll(text) {
-    if (!text) {
-      return [];
-    }
-    const segments = text.split(/\n---\n/).filter((s) => s.trim().length > 0);
-    return segments.map((s) => this.parse(s)).filter(Boolean);
-  }
-
-  /**
-   * 解析 frontmatter YAML
-   */
-  parseFrontmatter(text) {
-    const match = text.match(FRONTMATTER_RE);
-    if (!match) {
-      return {};
-    }
-
-    const fm = {};
-    for (const line of match[1].split('\n')) {
-      const colonIdx = line.indexOf(':');
-      if (colonIdx > 0) {
-        const key = line.slice(0, colonIdx).trim();
-        let value = line.slice(colonIdx + 1).trim();
-        if (value.startsWith('[') && value.endsWith(']')) {
-          value = value
-            .slice(1, -1)
-            .split(',')
-            .map((s) => s.trim().replace(/^['"]|['"]$/g, ''));
-        } else if (value === 'true') {
-          value = true;
-        } else if (value === 'false') {
-          value = false;
-        } else if (/^\d+$/.test(value)) {
-          value = parseInt(value, 10);
-        } else {
-          value = value.replace(/^['"]|['"]$/g, '');
-        }
-        fm[key] = value;
-      }
-    }
-    return fm;
-  }
-
-  /**
-   * 从内容提取 trigger
-   */
-  getTrigger(text) {
-    const fm = this.parseFrontmatter(text);
-    return fm.trigger || '';
-  }
-
-  /**
-   * 从文件路径读取并提取 Recipe 候选
-   * @param {string} relativePath - 相对路径
-   * @param {object} opts
-   * @param {string} [opts.projectRoot] - 项目根目录
-   * @returns {{ items: object[], isMarked: boolean }}
-   */
-  async extractFromPath(relativePath, opts = {}) {
-    const projectRoot = opts.projectRoot || process.cwd();
-    const fullPath = path.isAbsolute(relativePath)
-      ? relativePath
-      : path.resolve(projectRoot, relativePath);
-
-    if (!fs.existsSync(fullPath)) {
-      throw new Error(`文件不存在: ${fullPath}`);
-    }
-
-    const content = fs.readFileSync(fullPath, 'utf8');
-    const ext = path.extname(fullPath).toLowerCase();
-    const language = LanguageService.langFromExt(ext);
-
-    // 尝试解析为完整 Recipe Markdown
-    if (this.isCompleteRecipe(content)) {
-      const parsed = this.parse(content);
-      if (parsed) {
-        return { items: [parsed], isMarked: false };
-      }
-    }
-
-    // 尝试多段解析
-    const allRecipes = this.parseAll(content);
-    if (allRecipes.length > 0) {
-      return { items: allRecipes, isMarked: false };
-    }
-
-    // 回退: 将整个文件内容作为代码片段
-    const title = path.basename(fullPath, ext);
-    return {
-      items: [
-        {
-          title,
-          summary: '',
-          description: '',
-          trigger: this.#generateTrigger(title),
-          category: 'Utility',
-          language,
-          code: content,
-          codeBlocks: [{ language, code: content }],
-          usageGuide: '',
-          headers: this.#extractHeaders(content),
-          includeHeaders: false,
-          frontmatter: {},
-        },
-      ],
-      isMarked: false,
-    };
-  }
-
-  /**
-   * 从文本解析 Recipe（优先完整 Markdown 格式）
-   * @param {string} text
-   * @param {object} opts
-   * @param {string} [opts.language]
-   * @param {string} [opts.relativePath]
-   * @returns {object}
-   */
-  async parseFromText(text, opts = {}) {
-    if (!text || text.trim().length === 0) {
-      throw new Error('文本内容为空');
-    }
-
-    // 尝试完整 Recipe 解析
-    if (this.isCompleteRecipe(text)) {
-      const parsed = this.parse(text);
-      if (parsed) {
-        return parsed;
-      }
-    }
-
-    // 尝试批量解析
-    const all = this.parseAll(text);
-    if (all.length > 0) {
-      return all;
-    }
-
-    throw new Error('文本不是有效的 Recipe Markdown 格式');
-  }
-
-  /**
-   * 从文本提取代码片段（兜底方法，不要求 Markdown 格式）
-   * @param {string} text
-   * @param {object} opts
-   * @param {string} [opts.language]
-   * @returns {object}
-   */
-  async extractFromText(text, opts = {}) {
-    if (!text || text.trim().length === 0) {
-      throw new Error('文本内容为空');
-    }
-
-    const language = opts.language || 'unknown';
-
-    // 先尝试标准解析
-    try {
-      const result = await this.parseFromText(text, opts);
-      return result;
-    } catch {
-      /* 继续兜底逻辑 */
-    }
-
-    // 提取代码块
-    const codeBlocks = [];
-    const codeRe = /```(\w*)\n([\s\S]*?)```/g;
-    let match;
-    while ((match = codeRe.exec(text)) !== null) {
-      codeBlocks.push({ language: match[1] || language, code: match[2].trim() });
-    }
-
-    const code = codeBlocks.length > 0 ? codeBlocks.map((b) => b.code).join('\n\n') : text.trim();
-
-    // 简单标题推断
-    const titleLine = text.split('\n').find((l) => l.trim().startsWith('#'));
-    const title = titleLine ? titleLine.replace(/^#+\s*/, '').trim() : 'Untitled Snippet';
-
-    return {
-      title,
-      summary: '',
-      description: '',
-      trigger: this.#generateTrigger(title),
-      category: 'Utility',
-      language,
-      code,
-      codeBlocks: codeBlocks.length > 0 ? codeBlocks : [{ language, code }],
-      usageGuide: '',
-      headers: this.#extractHeaders(code),
-      includeHeaders: false,
-      frontmatter: {},
-    };
-  }
-
-  #extractHeaders(body) {
-    const headers = [];
-    const re = /#import\s+[<"]([^>"]+)[>"]/g;
-    let match;
-    while ((match = re.exec(body)) !== null) {
-      headers.push(match[1]);
-    }
-    return headers;
-  }
-
-  #generateTrigger(title) {
-    if (!title) {
-      return '';
-    }
-    return title
-      .toLowerCase()
-      .replace(/[^a-z0-9]+/g, '_')
-      .replace(/(^_|_$)/g, '')
-      .slice(0, 30);
-  }
-}

package/lib/service/search/CoarseRanker.js

@@ -1,111 +0,0 @@
-/**
- * CoarseRanker — 粗排器
- * 基于 E-E-A-T 标准的 5 维加权排序（BM25 + Semantic + Quality + Freshness + Popularity）
- */
-
-export class CoarseRanker {
-  #weights;
-
-  constructor(options = {}) {
-    this.#weights = {
-      bm25: options.bm25Weight ?? 0.3,
-      semantic: options.semanticWeight ?? 0.3,
-      quality: options.qualityWeight ?? 0.2,
-      freshness: options.freshnessWeight ?? 0.1,
-      popularity: options.popularityWeight ?? 0.1,
-    };
-  }
-
-  /**
-   * 粗排
-   * @param {Array} candidates — 需有 bm25Score、semanticScore 等字段
-   * @returns {Array} — sorted with coarseScore
-   */
-  rank(candidates) {
-    if (!candidates || candidates.length === 0) {
-      return [];
-    }
-
-    return candidates
-      .map((c) => {
-        const bm25 = this.#normalize(c.bm25Score || c.score || 0);
-        const semantic = this.#normalize(c.semanticScore || 0);
-        const quality = this.#computeQuality(c);
-        const freshness = this.#computeFreshness(c);
-        const popularity = this.#computePopularity(c);
-
-        const coarseScore =
-          bm25 * this.#weights.bm25 +
-          semantic * this.#weights.semantic +
-          quality * this.#weights.quality +
-          freshness * this.#weights.freshness +
-          popularity * this.#weights.popularity;
-
-        return {
-          ...c,
-          coarseScore,
-          coarseSignals: { bm25, semantic, quality, freshness, popularity },
-        };
-      })
-      .sort((a, b) => b.coarseScore - a.coarseScore);
-  }
-
-  /**
-   * E-E-A-T 质量评分
-   * - 内容完整性 40%: 有 title + code + description
-   * - 结构质量 30%: 有 category + language + tags
-   * - 代码可读性 30%: 合理长度、有注释
-   */
-  #computeQuality(candidate) {
-    let score = 0;
-
-    // 内容完整性 (40%)
-    const hasTitle = !!candidate.title;
-    const hasCode = !!(candidate.code || candidate.content);
-    const hasDesc = !!(candidate.description || candidate.summary);
-    score += (hasTitle ? 0.15 : 0) + (hasCode ? 0.15 : 0) + (hasDesc ? 0.1 : 0);
-
-    // 结构质量 (30%)
-    const hasCat = !!candidate.category;
-    const hasLang = !!candidate.language;
-    const hasTags = Array.isArray(candidate.tags) && candidate.tags.length > 0;
-    score += (hasCat ? 0.1 : 0) + (hasLang ? 0.1 : 0) + (hasTags ? 0.1 : 0);
-
-    // 代码可读性 (30%)
-    const code = candidate.code || candidate.content || '';
-    const lines = code.split('\n').length;
-    const hasComments = /\/\/|\/\*|#/.test(code);
-    const reasonableLength = lines >= 3 && lines <= 500;
-    score += (hasComments ? 0.15 : 0) + (reasonableLength ? 0.15 : 0);
-
-    return Math.min(score, 1.0);
-  }
-
-  #computeFreshness(candidate) {
-    const updated = candidate.updatedAt || candidate.lastModified || candidate.createdAt;
-    if (!updated) {
-      return 0.5;
-    }
-    // 自动识别秒级/毫秒级 Unix 时间戳 (秒级 ≤ 9999999999 即 2286 年)
-    const ts =
-      typeof updated === 'number' && updated > 0 && updated <= 9999999999
-        ? updated * 1000
-        : typeof updated === 'number'
-          ? updated
-          : new Date(updated).getTime();
-    const ageDays = (Date.now() - ts) / 86400000;
-    if (ageDays < 0) {
-      return 1.0; // 未来时间戳视为最新
-    }
-    return Math.exp((-Math.LN2 * ageDays) / 180); // 半衰期 180 天
-  }
-
-  #computePopularity(candidate) {
-    const usage = candidate.usageCount || 0;
-    return usage > 0 ? Math.min(Math.log10(usage + 1) / 3, 1.0) : 0;
-  }
-
-  #normalize(value) {
-    return Math.min(Math.max(value, 0), 1.0);
-  }
-}

package/lib/service/search/CrossEncoderReranker.js

@@ -1,174 +0,0 @@
-/**
- * CrossEncoderReranker — AI 驱动的语义重排器
- *
- * 替代 Jaccard 相似度，使用 LLM 对 (query, document) 对进行语义相关性评分。
- *
- * 策略:
- *   1. 将候选文档与 query 组成 pairs，批量送入 AI 评分
- *   2. AI 返回每个 pair 的 relevance score (0.0-1.0)
- *   3. 按 score 降序排列
- *
- * 优化:
- *   - 单次 API 调用批量评分（减少延迟和成本）
- *   - 文档截断至 MAX_DOC_LEN 控制 token 消耗
- *   - 候选上限 MAX_CANDIDATES，超出部分保留原始顺序
- *   - AI 不可用时自动降级到 Jaccard
- */
-
-import { jaccardSimilarity } from '../../shared/similarity.js';
-import { tokenize } from './InvertedIndex.js';
-
-const MAX_CANDIDATES = 40; // 超过此数量截断（控制 prompt 大小）
-const MAX_DOC_LEN = 300; // 每个文档最大字符数
-
-export class CrossEncoderReranker {
-  #aiProvider;
-  #logger;
-
-  /**
-   * @param {object} opts
-   * @param {import('../../external/ai/AiProvider.js').AiProvider} opts.aiProvider
-   * @param {object} [opts.logger]
-   */
-  constructor(opts = {}) {
-    this.#aiProvider = opts.aiProvider || null;
-    this.#logger = opts.logger || console;
-  }
-
-  /**
-   * 对候选列表进行语义重排
-   *
-   * @param {string} query — 用户查询
-   * @param {Array<object>} candidates — Layer 1 输出的候选列表
-   * @returns {Promise<Array<object>>} — 附带 semanticScore 的候选列表（降序）
-   */
-  async rerank(query, candidates) {
-    if (!candidates || candidates.length === 0) {
-      return [];
-    }
-    if (!query) {
-      return candidates;
-    }
-
-    // 如果 AI Provider 不可用，降级到 Jaccard
-    if (!this.#aiProvider || typeof this.#aiProvider.chatWithStructuredOutput !== 'function') {
-      return this.#jaccardFallback(query, candidates);
-    }
-
-    // 截取前 MAX_CANDIDATES 个候选，剩余保持原始顺序
-    const head = candidates.slice(0, MAX_CANDIDATES);
-    const tail = candidates.slice(MAX_CANDIDATES);
-
-    try {
-      const scored = await this.#batchScore(query, head);
-      // tail 部分给一个递减的低分以保持稳定排序
-      const minScore =
-        scored.length > 0 ? Math.min(...scored.map((s) => s.semanticScore)) * 0.5 : 0;
-      const tailScored = tail.map((c, i) => ({
-        ...c,
-        semanticScore: Math.max(minScore - (i + 1) * 0.001, 0),
-      }));
-      return [...scored, ...tailScored];
-    } catch (err) {
-      this.#logger.warn?.(
-        `[CrossEncoderReranker] AI scoring failed, falling back to Jaccard: ${err.message}`
-      );
-      return this.#jaccardFallback(query, candidates);
-    }
-  }
-
-  /**
-   * 批量 AI 评分 — 单次 chatWithStructuredOutput 调用
-   */
-  async #batchScore(query, candidates) {
-    const pairs = candidates.map((c, i) => {
-      const doc = this.#extractDocText(c);
-      return `[${i}] ${doc.substring(0, MAX_DOC_LEN)}`;
-    });
-
-    const prompt = `# Task
-Score the relevance of each document to the query. Return ONLY a JSON array.
-
-# Query
-${query}
-
-# Documents
-${pairs.join('\n')}
-
-# Output Format
-Return a JSON array of objects: [{"i": 0, "s": 0.85}, {"i": 1, "s": 0.3}, ...]
-- "i": document index (integer)
-- "s": relevance score (float 0.0-1.0, where 1.0 = perfectly relevant)
-
-Score guidelines:
-- 1.0: exact match or directly answers the query
-- 0.7-0.9: highly relevant, covers the main topic
-- 0.4-0.6: partially relevant, related topic
-- 0.1-0.3: tangentially related
-- 0.0: completely irrelevant
-
-Return ONLY a JSON array, no markdown or explanation.`;
-
-    const result = await this.#aiProvider.chatWithStructuredOutput(prompt, {
-      openChar: '[',
-      closeChar: ']',
-      temperature: 0.1,
-      maxTokens: 2048,
-    });
-
-    if (!Array.isArray(result)) {
-      throw new Error('AI returned non-array result');
-    }
-
-    // 构建 index → score 映射
-    const scoreMap = new Map();
-    for (const item of result) {
-      const idx = item.i ?? item.index;
-      const score = item.s ?? item.score ?? 0;
-      if (typeof idx === 'number' && idx >= 0 && idx < candidates.length) {
-        scoreMap.set(idx, Math.max(0, Math.min(1, score)));
-      }
-    }
-
-    // 合并分数，未评分的给 0
-    return candidates
-      .map((c, i) => ({
-        ...c,
-        semanticScore: scoreMap.get(i) ?? 0,
-      }))
-      .sort((a, b) => b.semanticScore - a.semanticScore);
-  }
-
-  /**
-   * 从候选对象提取用于评分的文本表示
-   */
-  #extractDocText(candidate) {
-    const parts = [
-      candidate.title,
-      candidate.trigger,
-      candidate.description || candidate.summary,
-      candidate.code,
-      candidate.content,
-    ].filter(Boolean);
-    return parts.join(' | ');
-  }
-
-  /**
-   * Jaccard 降级 — 当 AI 不可用时使用
-   */
-  #jaccardFallback(query, candidates) {
-    const queryTokens = new Set(tokenize(query));
-    if (queryTokens.size === 0) {
-      return candidates;
-    }
-
-    return candidates
-      .map((candidate) => {
-        const text = this.#extractDocText(candidate);
-        const docTokens = new Set(tokenize(text));
-        const score = jaccardSimilarity(queryTokens, docTokens);
-        return { ...candidate, semanticScore: score };
-      })
-      .sort((a, b) => b.semanticScore - a.semanticScore);
-  }
-}

package/lib/service/search/InvertedIndex.js

@@ -1,92 +0,0 @@
-/**
- * InvertedIndex — 倒排索引
- * 构建和查询 token → docIndex 映射
- */
-
-// 使用 SearchEngine 的统一分词器（含完整 CJK 单字/bigram 支持）
-// 确保倒排索引与 BM25 搜索使用一致的分词策略，避免中文查询召回率差异
-import { tokenize } from './SearchEngine.js';
-export { tokenize };
-
-/**
- * 构建倒排索引
- * @param {Array<{ id: string, content: string }>} documents
- * @returns {Map<string, Set<number>>}
- */
-export function buildInvertedIndex(documents) {
-  const index = new Map();
-
-  for (let i = 0; i < documents.length; i++) {
-    const doc = documents[i];
-    const text = [doc.title, doc.trigger, doc.content, doc.code, doc.description]
-      .filter(Boolean)
-      .join(' ');
-    const tokens = tokenize(text);
-
-    for (const token of tokens) {
-      if (!index.has(token)) {
-        index.set(token, new Set());
-      }
-      index.get(token).add(i);
-    }
-  }
-
-  return index;
-}
-
-/**
- * 查询倒排索引（OR 语义 — 匹配任一 token）
- * @param {Map<string, Set<number>>} invertedIndex
- * @param {string} query
- * @returns {number[]} — document indices
- */
-export function lookup(invertedIndex, query) {
-  const queryTokens = tokenize(query);
-  if (queryTokens.length === 0) {
-    return [];
-  }
-
-  const resultSet = new Set();
-  for (const token of queryTokens) {
-    const docs = invertedIndex.get(token);
-    if (docs) {
-      for (const docIdx of docs) {
-        resultSet.add(docIdx);
-      }
-    }
-  }
-
-  return [...resultSet];
-}
-
-/**
- * 查询倒排索引（AND 语义 — 匹配所有 token）
- * @param {Map<string, Set<number>>} invertedIndex
- * @param {string} query
- * @returns {number[]}
- */
-export function lookupAll(invertedIndex, query) {
-  const queryTokens = tokenize(query);
-  if (queryTokens.length === 0) {
-    return [];
-  }
-
-  let result = null;
-  for (const token of queryTokens) {
-    const docs = invertedIndex.get(token);
-    if (!docs) {
-      return [];
-    }
-    if (result === null) {
-      result = new Set(docs);
-    } else {
-      for (const idx of result) {
-        if (!docs.has(idx)) {
-          result.delete(idx);
-        }
-      }
-    }
-  }
-
-  return result ? [...result] : [];
-}