@absolutejs/rag 0.0.10 → 0.0.11

package/dist/ai/rag/index.js

@@ -13262,6 +13262,7 @@ var validateRAGEmbeddingDimensions = (vector, expectedDimensions, context) => {
 import { readdir, readFile as readFile2 } from "fs/promises";
 import { basename, extname, join, relative, resolve } from "path";
 import { gunzipSync, inflateRawSync } from "zlib";
+var h2IfHttps = (url) => url.startsWith("https://") ? { protocol: "http2" } : {};
 var DEFAULT_MAX_CHUNK_LENGTH = 900;
 var DEFAULT_CHUNK_OVERLAP = 120;
 var DEFAULT_MIN_CHUNK_LENGTH = 80;
@@ -19440,7 +19441,7 @@ var loadRAGDocumentFromURL = async (input) => {
   if (!url) {
     throw new Error("RAG URL is required");
   }
-  const response = await fetch(url);
+  const response = await fetch(url, h2IfHttps(url));
   if (!response.ok) {
     throw new Error(`Failed to fetch RAG URL ${url}: ${response.status} ${response.statusText}`);
   }
@@ -19488,7 +19489,7 @@ var loadRAGDocumentsFromURLs = async (input) => {
     if (!url) {
       throw new Error("RAG URL is required");
     }
-    const response = await fetch(url);
+    const response = await fetch(url, h2IfHttps(url));
     if (!response.ok) {
       throw new Error(`Failed to fetch RAG URL ${url}: ${response.status} ${response.statusText}`);
     }
@@ -30865,6 +30866,94 @@ var ragChat = (config) => {
 // src/ai/rag/htmxConfig.ts
 var createRAGHTMXConfig = (config) => config;
 var createRAGHTMXWorkflowRenderConfig = (config) => config;
+// src/ai/rag/rerankerProviders.ts
+var rowScore = (row) => row.relevance_score ?? row.relevanceScore ?? row.score ?? 0;
+var applyRanking = (candidates, rows, topK) => {
+  const ranked = [];
+  for (const row of rows) {
+    const candidate = candidates[row.index];
+    if (!candidate)
+      continue;
+    ranked.push({ ...candidate, score: rowScore(row) });
+  }
+  ranked.sort((left, right) => right.score - left.score);
+  return ranked.slice(0, topK);
+};
+var limitCandidates = (input) => {
+  const cap = input.candidateTopK ?? input.results.length;
+  return input.results.slice(0, Math.max(0, cap));
+};
+var createHttpCrossEncoderReranker = (options) => {
+  const fetchImpl = options.config.fetch ?? fetch;
+  const defaultModel = options.config.defaultModel ?? options.fallbackModel;
+  return {
+    defaultModel,
+    providerName: options.providerName,
+    rerank: async (input) => {
+      const candidates = limitCandidates(input);
+      if (candidates.length === 0)
+        return [];
+      const model = input.model ?? defaultModel;
+      const documents = candidates.map((candidate) => candidate.chunkText);
+      const topN = Math.min(input.topK, candidates.length);
+      const response = await fetchImpl(options.endpoint, {
+        body: JSON.stringify(options.buildBody({ documents, model, query: input.query, topN })),
+        headers: {
+          Authorization: `Bearer ${options.config.apiKey}`,
+          "Content-Type": "application/json",
+          ...options.config.headers
+        },
+        method: "POST"
+      });
+      if (!response.ok) {
+        throw new Error(`${options.providerName} rerank failed: HTTP ${response.status}`);
+      }
+      const payload = await response.json();
+      const rows = payload.results ?? payload.data ?? [];
+      const ranked = applyRanking(candidates, rows, input.topK);
+      if (input.scoreThreshold !== undefined) {
+        return ranked.filter((result) => result.score >= input.scoreThreshold);
+      }
+      return ranked;
+    }
+  };
+};
+var createCohereRAGReranker = (config) => createHttpCrossEncoderReranker({
+  buildBody: ({ model, query, documents, topN }) => ({
+    documents,
+    model,
+    query,
+    top_n: topN
+  }),
+  config,
+  endpoint: `${config.baseUrl ?? "https://api.cohere.com"}/v2/rerank`,
+  fallbackModel: "rerank-v3.5",
+  providerName: "cohere"
+});
+var createVoyageRAGReranker = (config) => createHttpCrossEncoderReranker({
+  buildBody: ({ model, query, documents, topN }) => ({
+    documents,
+    model,
+    query,
+    top_k: topN
+  }),
+  config,
+  endpoint: `${config.baseUrl ?? "https://api.voyageai.com"}/v1/rerank`,
+  fallbackModel: "rerank-2",
+  providerName: "voyage"
+});
+var createJinaRAGReranker = (config) => createHttpCrossEncoderReranker({
+  buildBody: ({ model, query, documents, topN }) => ({
+    documents,
+    model,
+    query,
+    top_n: topN
+  }),
+  config,
+  endpoint: `${config.baseUrl ?? "https://api.jina.ai"}/v1/rerank`,
+  fallbackModel: "jina-reranker-v2-base-multilingual",
+  providerName: "jina"
+});
 // src/ai/rag/retrievalStrategies.ts
 var tokenize4 = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).filter((token) => token.length > 0);
 var hasAnyToken2 = (tokens, values) => values.some((value) => tokens.includes(value));
@@ -32176,6 +32265,7 @@ var {S3Client } = globalThis.Bun;
 import { createHash } from "crypto";
 import { mkdir as mkdir2, readFile as readFile3, readdir as readdir2 } from "fs/promises";
 import { basename as basename2, dirname as dirname3, extname as extname2, join as join2, relative as relative2, resolve as resolve2 } from "path";
+var h2IfHttps2 = (url) => url.startsWith("https://") ? { protocol: "http2" } : {};
 var toSyncError = (caught) => caught instanceof Error ? caught.message : String(caught);
 var wait = async (delayMs) => {
   if (!(delayMs > 0)) {
@@ -32988,7 +33078,7 @@ var isFeedDocument = (value) => {
   return trimmed.includes("<rss") || trimmed.includes("<channel") || trimmed.includes("<feed") || trimmed.includes("<entry");
 };
 var discoverFeedsFromHTML = async (feed) => {
-  const response = await fetch(feed.url);
+  const response = await fetch(feed.url, h2IfHttps2(feed.url));
   if (!response.ok) {
     return [];
   }
@@ -33034,7 +33124,7 @@ var discoverFeedsFromHTML = async (feed) => {
   }
   const validated = [];
   for (const candidate of discovered.values()) {
-    const candidateResponse = await fetch(candidate.url);
+    const candidateResponse = await fetch(candidate.url, h2IfHttps2(candidate.url));
     if (!candidateResponse.ok) {
       continue;
     }
@@ -33098,7 +33188,7 @@ var parseSitemapEntries = (sitemap, value) => {
 };
 var discoverSitemapsFromRobots = async (sitemap) => {
   const robotsURL = resolveSiblingURL(sitemap.url, "/robots.txt");
-  const response = await fetch(robotsURL);
+  const response = await fetch(robotsURL, h2IfHttps2(robotsURL));
   if (!response.ok) {
     return [];
   }
@@ -33119,7 +33209,7 @@ var discoverSitemapsFromRobots = async (sitemap) => {
 };
 var loadRobotsDisallowRules = async (siteURL) => {
   const robotsURL = resolveSiblingURL(siteURL, "/robots.txt");
-  const response = await fetch(robotsURL);
+  const response = await fetch(robotsURL, h2IfHttps2(robotsURL));
   if (!response.ok) {
     return [];
   }
@@ -33160,7 +33250,7 @@ var discoverRecursiveSitemapURLs = async (input) => {
     }
     seen.add(current.sitemap.url);
     resolved.push(current.sitemap);
-    const response = await fetch(current.sitemap.url);
+    const response = await fetch(current.sitemap.url, h2IfHttps2(current.sitemap.url));
     if (!response.ok) {
       throw new Error(`Failed to load sitemap ${current.sitemap.url}: ${response.status} ${response.statusText}`);
     }
@@ -33311,7 +33401,7 @@ var discoverLinkedPagesFromHTML = async (input) => {
       pruneCounts.robotsBlockedCount += 1;
       continue;
     }
-    const response = await fetch(current.url);
+    const response = await fetch(current.url, h2IfHttps2(current.url));
     if (!response.ok) {
       continue;
     }
@@ -33566,6 +33656,7 @@ var loadDiscoveredGitHubRepositoryFiles = async (input) => {
       repo: input.repo
     });
     const response = await fetch(requestURL, {
+      ...h2IfHttps2(requestURL),
       headers: input.requestHeaders
     });
     if (!response.ok) {
@@ -33729,7 +33820,7 @@ var createRAGFeedSyncSource = (options) => ({
       }
     }
     const discoveredEntries = (await Promise.all([...feedMap.values()].map(async (feed) => {
-      const response = await fetch(feed.url);
+      const response = await fetch(feed.url, h2IfHttps2(feed.url));
       if (!response.ok) {
         throw new Error(`Failed to load feed ${feed.url}: ${response.status} ${response.statusText}`);
       }
@@ -33803,7 +33894,7 @@ var createRAGSitemapSyncSource = (options) => ({
       }
     }
     const discoveredEntries = (await Promise.all([...resolvedSitemapMap.values()].map(async (sitemap) => {
-      const response = await fetch(sitemap.url);
+      const response = await fetch(sitemap.url, h2IfHttps2(sitemap.url));
       if (!response.ok) {
         throw new Error(`Failed to load sitemap ${sitemap.url}: ${response.status} ${response.statusText}`);
       }
@@ -33868,7 +33959,7 @@ var createRAGSiteDiscoverySyncSource = (options) => ({
           }
         }
         const feedEntries = (await Promise.all([...feedMap.values()].map(async (feed) => {
-          const response = await fetch(feed.url);
+          const response = await fetch(feed.url, h2IfHttps2(feed.url));
           if (!response.ok) {
             throw new Error(`Failed to load feed ${feed.url}: ${response.status} ${response.statusText}`);
           }
@@ -33918,7 +34009,7 @@ var createRAGSiteDiscoverySyncSource = (options) => ({
           sitemap
         })))).flat();
         const sitemapEntries = (await Promise.all(resolvedSitemaps.map(async (sitemap) => {
-          const response = await fetch(sitemap.url);
+          const response = await fetch(sitemap.url, h2IfHttps2(sitemap.url));
           if (!response.ok) {
             throw new Error(`Failed to load sitemap ${sitemap.url}: ${response.status} ${response.statusText}`);
           }
@@ -37325,6 +37416,7 @@ export {
   evaluateRAGAnswerGroundingCase,
   evaluateRAGAnswerGrounding,
   deepseekEmbeddings,
+  createVoyageRAGReranker,
   createTextFileExtractor,
   createSQLiteRAGStore,
   createRAGVector,
@@ -37422,12 +37514,14 @@ export {
   createPDFFileExtractor,
   createOfficeDocumentExtractor,
   createLegacyDocumentExtractor,
+  createJinaRAGReranker,
   createInMemoryRAGStore,
   createHeuristicRAGRetrievalStrategy,
   createHeuristicRAGReranker,
   createHeuristicRAGQueryTransform,
   createEmailExtractor,
   createEPUBExtractor,
+  createCohereRAGReranker,
   createBuiltinArchiveExpander,
   compareRAGRetrievalTraceSummaries,
   compareRAGRetrievalStrategies,
@@ -37511,5 +37605,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=A89076338D70690164756E2164756E21
+//# debugId=20E465BEF0B96FD164756E2164756E21
 //# sourceMappingURL=index.js.map