@astro-minimax/ai 0.8.2 → 0.9.0

package/dist/search/search-api.js

@@ -0,0 +1,94 @@
+import { scoreDocument, filterLowRelevance, tokenize, pickAnchorTerms, normalizeText } from "./search-utils.js";
+import { buildSearchIndex, getIDFMapForIndex } from "./search-index.js";
+import { hasVectorIndex, rerankWithVectors } from "./vector-reranker.js";
+let articleIndex = null;
+let projectIndex = null;
+const ARTICLE_LIMIT = 10;
+const ARTICLE_LIMIT_BROAD = 20;
+const PROJECT_LIMIT = 5;
+const DEEP_CONTENT_SCORE_THRESHOLD = 8;
+const DEEP_CONTENT_MAX_LENGTH = 1500;
+function initArticleIndex(documents) {
+  articleIndex = buildSearchIndex(documents);
+}
+function initProjectIndex(documents) {
+  projectIndex = buildSearchIndex(documents);
+}
+function searchArticles(query, options = {}) {
+  if (!query.trim() || !articleIndex) return [];
+  const tokens = tokenize(query);
+  if (!tokens.length) return [];
+  const limit = tokens.length <= 2 ? ARTICLE_LIMIT_BROAD : ARTICLE_LIMIT;
+  const rawResults = scoreDocs(articleIndex, tokens, limit * 2);
+  const filtered = applyAnchorFilter(rawResults, query, tokens);
+  const deduplicated = filterLowRelevance(filtered.length > 0 ? filtered : rawResults);
+  const results = deduplicated.slice(0, limit);
+  const topScore = results[0]?.score ?? 0;
+  const secondScore = results[1]?.score ?? 0;
+  const isDeepHit = options.enableDeepContent && topScore >= DEEP_CONTENT_SCORE_THRESHOLD && topScore > secondScore * 1.5;
+  let articles = results.map((result, index) => {
+    const baseUrl = options.siteUrl ?? "";
+    const url = result.url.startsWith("http") ? result.url : `${baseUrl}${result.url}`;
+    const fullContent = isDeepHit && index === 0 && result.content ? result.content.slice(0, DEEP_CONTENT_MAX_LENGTH) : void 0;
+    return {
+      title: result.title,
+      url,
+      summary: result.summary ?? result.excerpt,
+      keyPoints: result.keyPoints,
+      categories: result.categories,
+      dateTime: result.dateTime,
+      fullContent,
+      score: result.score
+    };
+  });
+  if (hasVectorIndex() && articles.length > 1) {
+    articles = rerankWithVectors(query, articles);
+  }
+  return articles;
+}
+function searchProjects(query, options = {}) {
+  if (!query.trim() || !projectIndex) return [];
+  const tokens = tokenize(query);
+  if (!tokens.length) return [];
+  const rawResults = scoreDocs(projectIndex, tokens, PROJECT_LIMIT * 2);
+  if (!rawResults.length) return [];
+  const baseUrl = options.siteUrl ?? "";
+  return rawResults.slice(0, PROJECT_LIMIT).map((r) => ({
+    name: r.title,
+    url: r.url.startsWith("http") ? r.url : `${baseUrl}${r.url}`,
+    description: r.excerpt || r.content.slice(0, 200),
+    score: r.score
+  }));
+}
+function mergeResults(primary, secondary) {
+  const seen = new Set(primary.map((i) => i.url));
+  const merged = [...primary];
+  for (const item of secondary) {
+    if (!seen.has(item.url)) {
+      seen.add(item.url);
+      merged.push(item);
+    }
+  }
+  return merged;
+}
+function scoreDocs(index, tokens, limit) {
+  const idfMap = getIDFMapForIndex();
+  return index.map((doc) => ({ ...doc, score: scoreDocument(tokens, doc, idfMap) })).filter((doc) => doc.score > 0).sort((a, b) => b.score - a.score).slice(0, limit);
+}
+function applyAnchorFilter(results, query, tokens) {
+  if (tokens.length > 2) return results;
+  const anchorTerms = pickAnchorTerms(query, results, 2, 2);
+  if (!anchorTerms.length) return results;
+  const strict = results.filter((r) => {
+    const text = normalizeText([r.title, ...r.keyPoints, ...r.categories].join(" "));
+    return anchorTerms.some((term) => text.includes(term));
+  });
+  return strict.length > 0 ? strict : results;
+}
+export {
+  initArticleIndex,
+  initProjectIndex,
+  mergeResults,
+  searchArticles,
+  searchProjects
+};

package/dist/search/search-index.js

@@ -0,0 +1,32 @@
+import { normalizeText } from "./search-utils.js";
+import { buildIDFMap } from "./idf.js";
+let cachedIDFMap = null;
+function buildSearchIndex(documents) {
+  const indexed = documents.map((doc) => ({
+    ...doc,
+    tokens: buildDocumentTokens(doc)
+  }));
+  if (indexed.length > 0) {
+    cachedIDFMap = buildIDFMap(indexed);
+  }
+  return indexed;
+}
+function getIDFMapForIndex() {
+  return cachedIDFMap;
+}
+function buildDocumentTokens(doc) {
+  const parts = [
+    doc.title,
+    doc.excerpt,
+    doc.content.slice(0, 1e3),
+    ...doc.keyPoints,
+    ...doc.categories,
+    ...doc.tags,
+    doc.summary ?? ""
+  ];
+  return [...new Set(parts.map(normalizeText).join(" ").split(/\s+/).filter(Boolean))];
+}
+export {
+  buildSearchIndex,
+  getIDFMapForIndex
+};

package/dist/search/search-utils.js

@@ -0,0 +1,81 @@
+import { getIDFWeight } from "./idf.js";
+function normalizeText(text) {
+  return text.toLowerCase().replace(/[^\u4e00-\u9fa5\w\s]/g, " ").replace(/\s+/g, " ").trim();
+}
+function tokenize(text) {
+  const normalized = normalizeText(text);
+  const parts = normalized.split(/\s+/).filter(Boolean);
+  return dedupeByContainment(parts);
+}
+function dedupeByContainment(terms) {
+  const unique = [...new Set(terms)];
+  const kept = [];
+  for (const term of unique.sort((a, b) => b.length - a.length)) {
+    if (!kept.some((existing) => existing.includes(term))) {
+      kept.push(term);
+    }
+  }
+  return kept;
+}
+const FIELD_WEIGHTS = {
+  title: 8,
+  keyPoints: 5,
+  categories: 4,
+  tags: 3,
+  excerpt: 3,
+  content: 1
+};
+function scoreDocument(tokens, doc, idfMap) {
+  if (!tokens.length) return 0;
+  let score = 0;
+  const title = normalizeText(doc.title);
+  const excerpt = normalizeText(doc.excerpt);
+  const keyPointsText = normalizeText(doc.keyPoints.join(" "));
+  const categoriesText = normalizeText(doc.categories.join(" "));
+  const tagsText = normalizeText(doc.tags.join(" "));
+  const contentSample = normalizeText(doc.content.slice(0, 500));
+  for (const token of tokens) {
+    if (!token) continue;
+    const idf = getIDFWeight(idfMap ?? null, token);
+    if (title.includes(token)) score += FIELD_WEIGHTS.title * idf;
+    if (keyPointsText.includes(token)) score += FIELD_WEIGHTS.keyPoints * idf;
+    if (categoriesText.includes(token)) score += FIELD_WEIGHTS.categories * idf;
+    if (tagsText.includes(token)) score += FIELD_WEIGHTS.tags * idf;
+    if (excerpt.includes(token)) score += FIELD_WEIGHTS.excerpt * idf;
+    if (contentSample.includes(token)) score += FIELD_WEIGHTS.content * idf;
+  }
+  return score;
+}
+function filterLowRelevance(results, relativeThreshold = 0.35, minAbsoluteScore = 2) {
+  if (results.length <= 3) return results;
+  const topScore = results[0]?.score ?? 0;
+  if (topScore <= 0) return results;
+  const threshold = Math.max(minAbsoluteScore, topScore * relativeThreshold);
+  return results.filter((item, index) => index < 3 || item.score >= threshold);
+}
+function pickAnchorTerms(query, candidates, maxTerms = 2, minTermLength = 2) {
+  const terms = tokenize(query).filter((t) => t.length >= minTermLength);
+  if (terms.length <= maxTerms) return terms.slice(0, maxTerms);
+  if (!candidates.length) return terms.slice(0, maxTerms);
+  const scored = terms.map((term) => {
+    let hitCount = 0;
+    for (const c of candidates) {
+      const text = normalizeText([c.title, ...c.keyPoints, ...c.categories].join(" "));
+      if (text.includes(term)) hitCount++;
+    }
+    if (hitCount <= 0) return { term, score: Number.NEGATIVE_INFINITY };
+    const coverage = hitCount / candidates.length;
+    const specificity = 1 - coverage;
+    const lengthScore = Math.min(term.length, 8) / 8;
+    return { term, score: specificity * 2 + lengthScore };
+  });
+  return scored.filter((s) => Number.isFinite(s.score)).sort((a, b) => b.score - a.score).map((s) => s.term).slice(0, maxTerms);
+}
+export {
+  dedupeByContainment,
+  filterLowRelevance,
+  normalizeText,
+  pickAnchorTerms,
+  scoreDocument,
+  tokenize
+};

package/dist/search/session-cache.js

@@ -0,0 +1,96 @@
+import { MemoryCacheAdapter } from "../cache/memory-adapter.js";
+const SESSION_ID_PATTERN = /^[a-z0-9][a-z0-9_-]{7,63}$/i;
+const SESSION_CACHE_TTL_SECONDS = 600;
+const SESSION_CACHE_TTL_MS = SESSION_CACHE_TTL_SECONDS * 1e3;
+let defaultCache = null;
+function getDefaultCache() {
+  if (!defaultCache) {
+    defaultCache = new MemoryCacheAdapter({
+      defaultTtl: SESSION_CACHE_TTL_SECONDS,
+      maxEntries: 400
+    });
+  }
+  return defaultCache;
+}
+function getSessionCacheKey(req) {
+  const sessionId = req.headers.get("x-session-id")?.trim();
+  if (sessionId && SESSION_ID_PATTERN.test(sessionId)) {
+    return `sid:${sessionId}`;
+  }
+  return null;
+}
+function setCacheAdapter(cache) {
+  defaultCache = cache;
+}
+function getCacheAdapter() {
+  return getDefaultCache();
+}
+async function getCachedContext(key, cache) {
+  const adapter = cache ?? getDefaultCache();
+  const entry = await adapter.get(key);
+  return entry?.value;
+}
+async function setCachedContext(key, ctx, cache) {
+  const adapter = cache ?? getDefaultCache();
+  await adapter.set(key, ctx, { ttl: SESSION_CACHE_TTL_SECONDS });
+}
+async function deleteCachedContext(key, cache) {
+  const adapter = cache ?? getDefaultCache();
+  return adapter.delete(key);
+}
+function cleanupCache(_now) {
+}
+const legacyCache = /* @__PURE__ */ new Map();
+const LEGACY_TTL_MS = 10 * 60 * 1e3;
+const MAX_CACHE_SIZE = 400;
+function getCachedContextSync(key) {
+  const entry = legacyCache.get(key);
+  if (!entry) return void 0;
+  if (Date.now() - entry.updatedAt > LEGACY_TTL_MS) {
+    legacyCache.delete(key);
+    return void 0;
+  }
+  return entry;
+}
+function setCachedContextSync(key, ctx) {
+  legacyCache.set(key, ctx);
+  if (legacyCache.size > MAX_CACHE_SIZE) {
+    const overflow = legacyCache.size - MAX_CACHE_SIZE;
+    const keys = legacyCache.keys();
+    for (let i = 0; i < overflow; i++) {
+      const next = keys.next();
+      if (next.done) break;
+      legacyCache.delete(next.value);
+    }
+  }
+}
+function cleanupCacheLegacy(now) {
+  for (const [key, value] of legacyCache) {
+    if (now - value.updatedAt > LEGACY_TTL_MS) {
+      legacyCache.delete(key);
+    }
+  }
+  if (legacyCache.size > MAX_CACHE_SIZE) {
+    const overflow = legacyCache.size - MAX_CACHE_SIZE;
+    const keys = legacyCache.keys();
+    for (let i = 0; i < overflow; i++) {
+      const next = keys.next();
+      if (next.done) break;
+      legacyCache.delete(next.value);
+    }
+  }
+}
+export {
+  SESSION_CACHE_TTL_MS,
+  SESSION_CACHE_TTL_SECONDS,
+  cleanupCache,
+  cleanupCacheLegacy,
+  deleteCachedContext,
+  getCacheAdapter,
+  getCachedContext,
+  getCachedContextSync,
+  getSessionCacheKey,
+  setCacheAdapter,
+  setCachedContext,
+  setCachedContextSync
+};

package/dist/search/vector-reranker.js

@@ -0,0 +1,103 @@
+let loadedIndex = null;
+let idfCache = null;
+function loadVectorIndex(data) {
+  loadedIndex = data;
+  idfCache = null;
+  if (data?.vocabulary && data.chunks.length > 0) {
+    idfCache = buildIDFFromVocab(data.vocabulary, data.chunks);
+  }
+}
+function clearVectorIndex() {
+  loadedIndex = null;
+  idfCache = null;
+}
+function hasVectorIndex() {
+  return loadedIndex !== null && loadedIndex.chunks.length > 0;
+}
+function rerankWithVectors(query, candidates, alpha = 0.3) {
+  if (!loadedIndex || !idfCache || !loadedIndex.vocabulary) {
+    return candidates;
+  }
+  const queryVector = computeQueryVector(query, loadedIndex.vocabulary, idfCache);
+  if (!queryVector) return candidates;
+  const articleScores = /* @__PURE__ */ new Map();
+  for (const chunk of loadedIndex.chunks) {
+    if (!chunk.vector) continue;
+    const sim = cosineSimilarity(queryVector, chunk.vector);
+    const current = articleScores.get(chunk.postId) ?? 0;
+    if (sim > current) {
+      articleScores.set(chunk.postId, sim);
+    }
+  }
+  if (articleScores.size === 0) return candidates;
+  const maxOriginal = Math.max(...candidates.map((c) => c.score ?? 0), 1);
+  const reranked = candidates.map((article) => {
+    const slug = extractSlugFromUrl(article.url);
+    const vectorScore = articleScores.get(slug) ?? 0;
+    const originalNorm = (article.score ?? 0) / maxOriginal;
+    const blended = originalNorm * (1 - alpha) + vectorScore * alpha;
+    return { ...article, score: blended };
+  });
+  return reranked.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
+}
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length) return 0;
+  let dot = 0, magA = 0, magB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+  const mag = Math.sqrt(magA) * Math.sqrt(magB);
+  return mag === 0 ? 0 : dot / mag;
+}
+function computeQueryVector(query, vocabulary, idf) {
+  const tokens = tokenizeForVector(query);
+  if (!tokens.length) return null;
+  const tf = /* @__PURE__ */ new Map();
+  for (const t of tokens) tf.set(t, (tf.get(t) || 0) + 1);
+  const maxTf = Math.max(...tf.values(), 1);
+  const vector = vocabulary.map((term) => {
+    const termTf = (tf.get(term) || 0) / maxTf;
+    const termIdf = idf.get(term) || 0;
+    return termTf * termIdf;
+  });
+  if (vector.every((v) => v === 0)) return null;
+  return vector;
+}
+function tokenizeForVector(text) {
+  const CJK = /[\u4e00-\u9fff\u3400-\u4dbf]/g;
+  const cjkChars = text.match(CJK) || [];
+  const latin = text.replace(CJK, " ").toLowerCase().split(/\W+/).filter((w) => w.length > 2);
+  return [...cjkChars, ...latin];
+}
+function buildIDFFromVocab(vocabulary, chunks) {
+  const N = chunks.length;
+  const df = /* @__PURE__ */ new Map();
+  for (const chunk of chunks) {
+    const tokens = new Set(tokenizeForVector(chunk.text));
+    for (const term of vocabulary) {
+      if (tokens.has(term)) {
+        df.set(term, (df.get(term) || 0) + 1);
+      }
+    }
+  }
+  const idf = /* @__PURE__ */ new Map();
+  for (const term of vocabulary) {
+    const docCount = df.get(term) || 0;
+    idf.set(term, Math.log(N / (docCount + 1)) + 1);
+  }
+  return idf;
+}
+function extractSlugFromUrl(url) {
+  const path = url.replace(/^https?:\/\/[^/]+/, "");
+  const match = path.match(/^\/([\w-]+)\/posts\/(.+?)\/?$/);
+  if (match) return `${match[1]}/${match[2]}`;
+  return path.replace(/^\/|\/$/g, "");
+}
+export {
+  clearVectorIndex,
+  hasVectorIndex,
+  loadVectorIndex,
+  rerankWithVectors
+};