@absolutejs/rag 0.0.10 → 0.0.12

package/dist/ai/rag/index.js

@@ -13262,6 +13262,7 @@ var validateRAGEmbeddingDimensions = (vector, expectedDimensions, context) => {
 import { readdir, readFile as readFile2 } from "fs/promises";
 import { basename, extname, join, relative, resolve } from "path";
 import { gunzipSync, inflateRawSync } from "zlib";
+var h2IfHttps = (url) => url.startsWith("https://") ? { protocol: "http2" } : {};
 var DEFAULT_MAX_CHUNK_LENGTH = 900;
 var DEFAULT_CHUNK_OVERLAP = 120;
 var DEFAULT_MIN_CHUNK_LENGTH = 80;
@@ -19440,7 +19441,7 @@ var loadRAGDocumentFromURL = async (input) => {
   if (!url) {
     throw new Error("RAG URL is required");
   }
-  const response = await fetch(url);
+  const response = await fetch(url, h2IfHttps(url));
   if (!response.ok) {
     throw new Error(`Failed to fetch RAG URL ${url}: ${response.status} ${response.statusText}`);
   }
@@ -19488,7 +19489,7 @@ var loadRAGDocumentsFromURLs = async (input) => {
     if (!url) {
       throw new Error("RAG URL is required");
     }
-    const response = await fetch(url);
+    const response = await fetch(url, h2IfHttps(url));
     if (!response.ok) {
       throw new Error(`Failed to fetch RAG URL ${url}: ${response.status} ${response.statusText}`);
     }
@@ -21713,6 +21714,7 @@ var MAX_INGEST_JOBS = 20;
 var MAX_ADMIN_ACTIONS = 20;
 var MAX_ADMIN_JOBS = 20;
 var DEFAULT_STALE_AFTER_MS = 1000 * 60 * 60 * 24 * 7;
+var WEBSOCKET_OPEN = 1;
 var REQUEST_USER_SUB_HEADER = "x-absolutejs-user-sub";
 var HTML_HEADERS = { "Content-Type": "text/html; charset=utf-8" };
 var defaultParseProvider = (content) => {
@@ -22156,9 +22158,23 @@ var persistSearchTraceIfConfigured = async (input) => {
     await input.onPrune?.(input.retention);
   }
 };
-var buildRAGContextFromQuery = async (config, topK, scoreThreshold, queryText, ragModel, embedding, embeddingModel) => {
+var sendWebSocketError = (ws, error, conversationId, messageId) => {
+  if (typeof ws.readyState === "number" && ws.readyState !== WEBSOCKET_OPEN) {
+    return;
+  }
+  try {
+    ws.send(JSON.stringify({
+      conversationId,
+      message: error instanceof Error ? error.message : String(error),
+      messageId,
+      type: "error"
+    }));
+  } catch {}
+};
+var getProviderDefaultEmbeddingModel = (embedding) => embedding !== undefined && typeof embedding === "object" && typeof embedding.defaultModel === "string" ? embedding.defaultModel : undefined;
+var buildRAGContextFromQuery = async (config, topK, scoreThreshold, queryText, embedding, embeddingModel) => {
   const collection = config.collection ?? (config.ragStore ? createRAGCollection({
-    defaultModel: embeddingModel ?? ragModel,
+    defaultModel: embeddingModel,
     defaultTopK: topK,
     embedding,
     rerank: config.rerank,
@@ -22171,7 +22187,7 @@ var buildRAGContextFromQuery = async (config, topK, scoreThreshold, queryText, r
     };
   }
   const queried = await collection.searchWithTrace({
-    model: embeddingModel ?? ragModel,
+    model: embeddingModel,
     query: queryText,
     scoreThreshold,
     topK
@@ -22194,7 +22210,27 @@ var buildRAGContextFromQuery = async (config, topK, scoreThreshold, queryText, r
     trace: queried.trace
   };
 };
+var assertEmbeddingModelConfigured = (config) => {
+  if (config.collection !== undefined) {
+    return;
+  }
+  if (config.ragStore === undefined) {
+    return;
+  }
+  if (config.embeddingModel !== undefined) {
+    return;
+  }
+  const { embedding } = config;
+  if (embedding === undefined || typeof embedding !== "object") {
+    return;
+  }
+  if (getProviderDefaultEmbeddingModel(embedding) !== undefined) {
+    return;
+  }
+  throw new Error("ragChat: an embedding provider is configured but no embedding model is set. " + "Pass `embeddingModel` to ragChat() or set `defaultModel` on the embedding provider. " + "The chat model cannot be used for embeddings.");
+};
 var ragChat = (config) => {
+  assertEmbeddingModelConfigured(config);
   const path = config.path ?? DEFAULT_PATH;
   const authorizeRAGAction = config.authorizeRAGAction;
   const resolveRAGAccessScope = config.resolveRAGAccessScope;
@@ -22656,53 +22692,57 @@ var ragChat = (config) => {
     }));
   };
   const handleMessage = async (ws, rawContent, rawConversationId, rawAttachments) => {
-    const parsed = parseProvider(rawContent);
-    const { content, providerName } = parsed;
-    const userMessageId = generateId2();
     const assistantMessageId = generateId2();
     const conversationId = rawConversationId ?? generateId2();
-    const conversation = await store.getOrCreate(conversationId);
-    const history = buildHistory(conversation);
-    const model = resolveModel(config, parsed);
-    const ragModel = parsed.model ?? model;
-    appendMessage(conversation, {
-      attachments: rawAttachments,
-      content,
-      conversationId,
-      id: userMessageId,
-      role: "user",
-      timestamp: Date.now()
-    });
-    await store.set(conversationId, conversation);
-    const retrievalStartedAt = Date.now();
-    handleRAGRetrieving(ws, conversationId, assistantMessageId, retrievalStartedAt);
-    const provider = config.provider(providerName);
-    const rag = await buildRAGContextFromQuery(config, topK, scoreThreshold, content, ragModel, config.embedding, config.embeddingModel);
-    const controller = new AbortController;
-    abortControllers.set(conversationId, controller);
-    const { ragContext, sources, trace } = rag;
-    const retrievedAt = Date.now();
-    const retrievalDurationMs = retrievedAt - retrievalStartedAt;
-    handleRAGRetrieved(ws, conversationId, assistantMessageId, sources, retrievalStartedAt, retrievedAt, retrievalDurationMs, trace);
-    await streamAI(ws, conversationId, assistantMessageId, {
-      completeMeta: includeCompleteSources ? { sources } : undefined,
-      maxTurns: config.maxTurns,
-      messages: [
-        ...history,
-        buildUserMessage(content, rawAttachments, ragContext)
-      ],
-      model,
-      provider,
-      signal: controller.signal,
-      systemPrompt: config.systemPrompt,
-      thinking: resolveThinking(config, providerName, model),
-      tools: resolveTools(config, providerName, model),
-      onComplete: async (fullResponse, usage) => {
-        await appendAssistantMessage(conversationId, assistantMessageId, fullResponse, sources, usage, model, retrievalStartedAt, retrievedAt, retrievalDurationMs, trace);
-        abortControllers.delete(conversationId);
-        config.onComplete?.(conversationId, fullResponse, usage, sources);
-      }
-    });
+    try {
+      const parsed = parseProvider(rawContent);
+      const { content, providerName } = parsed;
+      const userMessageId = generateId2();
+      const conversation = await store.getOrCreate(conversationId);
+      const history = buildHistory(conversation);
+      const model = resolveModel(config, parsed);
+      appendMessage(conversation, {
+        attachments: rawAttachments,
+        content,
+        conversationId,
+        id: userMessageId,
+        role: "user",
+        timestamp: Date.now()
+      });
+      await store.set(conversationId, conversation);
+      const retrievalStartedAt = Date.now();
+      handleRAGRetrieving(ws, conversationId, assistantMessageId, retrievalStartedAt);
+      const provider = config.provider(providerName);
+      const rag = await buildRAGContextFromQuery(config, topK, scoreThreshold, content, config.embedding, config.embeddingModel);
+      const controller = new AbortController;
+      abortControllers.set(conversationId, controller);
+      const { ragContext, sources, trace } = rag;
+      const retrievedAt = Date.now();
+      const retrievalDurationMs = retrievedAt - retrievalStartedAt;
+      handleRAGRetrieved(ws, conversationId, assistantMessageId, sources, retrievalStartedAt, retrievedAt, retrievalDurationMs, trace);
+      await streamAI(ws, conversationId, assistantMessageId, {
+        completeMeta: includeCompleteSources ? { sources } : undefined,
+        maxTurns: config.maxTurns,
+        messages: [
+          ...history,
+          buildUserMessage(content, rawAttachments, ragContext)
+        ],
+        model,
+        provider,
+        signal: controller.signal,
+        systemPrompt: config.systemPrompt,
+        thinking: resolveThinking(config, providerName, model),
+        tools: resolveTools(config, providerName, model),
+        onComplete: async (fullResponse, usage) => {
+          await appendAssistantMessage(conversationId, assistantMessageId, fullResponse, sources, usage, model, retrievalStartedAt, retrievedAt, retrievalDurationMs, trace);
+          abortControllers.delete(conversationId);
+          config.onComplete?.(conversationId, fullResponse, usage, sources);
+        }
+      });
+    } catch (error) {
+      abortControllers.delete(conversationId);
+      sendWebSocketError(ws, error, conversationId, assistantMessageId);
+    }
   };
   const resolveCollection = () => config.collection ?? (ragStore ? createRAGCollection({
     defaultModel: config.embeddingModel,
@@ -29421,7 +29461,6 @@ var ragChat = (config) => {
       const parsed = parseProvider(lastMessage.content);
       const { content, providerName } = parsed;
       const model = resolveModel(config, parsed);
-      const ragModel = parsed.model ?? model;
       const assistantMessageId = generateId2();
       const retrievalStartedAt = Date.now();
       yield {
@@ -29433,7 +29472,7 @@ var ragChat = (config) => {
         event: "retrieval"
       };
       const provider = config.provider(providerName);
-      const { ragContext, sources, trace } = await buildRAGContextFromQuery(config, topK, scoreThreshold, content, ragModel, config.embedding, config.embeddingModel);
+      const { ragContext, sources, trace } = await buildRAGContextFromQuery(config, topK, scoreThreshold, content, config.embedding, config.embeddingModel);
       const retrievedAt = Date.now();
       const retrievalDurationMs = retrievedAt - retrievalStartedAt;
       yield {
@@ -29484,16 +29523,20 @@ var ragChat = (config) => {
       if (!message) {
         return;
       }
-      if (message.type === "cancel") {
-        handleCancel(message.conversationId);
-        return;
-      }
-      if (message.type === "branch") {
-        await handleBranch(ws, message.messageId, message.conversationId);
-        return;
-      }
-      if (message.type === "message") {
-        await handleMessage(ws, message.content, message.conversationId, message.attachments);
+      try {
+        if (message.type === "cancel") {
+          handleCancel(message.conversationId);
+          return;
+        }
+        if (message.type === "branch") {
+          await handleBranch(ws, message.messageId, message.conversationId);
+          return;
+        }
+        if (message.type === "message") {
+          await handleMessage(ws, message.content, message.conversationId, message.attachments);
+        }
+      } catch (error) {
+        sendWebSocketError(ws, error, "conversationId" in message ? message.conversationId : undefined);
       }
     }
   }).post(`${path}/search`, async ({ body, request, set }) => {
@@ -30865,6 +30908,94 @@ var ragChat = (config) => {
 // src/ai/rag/htmxConfig.ts
 var createRAGHTMXConfig = (config) => config;
 var createRAGHTMXWorkflowRenderConfig = (config) => config;
+// src/ai/rag/rerankerProviders.ts
+var rowScore = (row) => row.relevance_score ?? row.relevanceScore ?? row.score ?? 0;
+var applyRanking = (candidates, rows, topK) => {
+  const ranked = [];
+  for (const row of rows) {
+    const candidate = candidates[row.index];
+    if (!candidate)
+      continue;
+    ranked.push({ ...candidate, score: rowScore(row) });
+  }
+  ranked.sort((left, right) => right.score - left.score);
+  return ranked.slice(0, topK);
+};
+var limitCandidates = (input) => {
+  const cap = input.candidateTopK ?? input.results.length;
+  return input.results.slice(0, Math.max(0, cap));
+};
+var createHttpCrossEncoderReranker = (options) => {
+  const fetchImpl = options.config.fetch ?? fetch;
+  const defaultModel = options.config.defaultModel ?? options.fallbackModel;
+  return {
+    defaultModel,
+    providerName: options.providerName,
+    rerank: async (input) => {
+      const candidates = limitCandidates(input);
+      if (candidates.length === 0)
+        return [];
+      const model = input.model ?? defaultModel;
+      const documents = candidates.map((candidate) => candidate.chunkText);
+      const topN = Math.min(input.topK, candidates.length);
+      const response = await fetchImpl(options.endpoint, {
+        body: JSON.stringify(options.buildBody({ documents, model, query: input.query, topN })),
+        headers: {
+          Authorization: `Bearer ${options.config.apiKey}`,
+          "Content-Type": "application/json",
+          ...options.config.headers
+        },
+        method: "POST"
+      });
+      if (!response.ok) {
+        throw new Error(`${options.providerName} rerank failed: HTTP ${response.status}`);
+      }
+      const payload = await response.json();
+      const rows = payload.results ?? payload.data ?? [];
+      const ranked = applyRanking(candidates, rows, input.topK);
+      if (input.scoreThreshold !== undefined) {
+        return ranked.filter((result) => result.score >= input.scoreThreshold);
+      }
+      return ranked;
+    }
+  };
+};
+var createCohereRAGReranker = (config) => createHttpCrossEncoderReranker({
+  buildBody: ({ model, query, documents, topN }) => ({
+    documents,
+    model,
+    query,
+    top_n: topN
+  }),
+  config,
+  endpoint: `${config.baseUrl ?? "https://api.cohere.com"}/v2/rerank`,
+  fallbackModel: "rerank-v3.5",
+  providerName: "cohere"
+});
+var createVoyageRAGReranker = (config) => createHttpCrossEncoderReranker({
+  buildBody: ({ model, query, documents, topN }) => ({
+    documents,
+    model,
+    query,
+    top_k: topN
+  }),
+  config,
+  endpoint: `${config.baseUrl ?? "https://api.voyageai.com"}/v1/rerank`,
+  fallbackModel: "rerank-2",
+  providerName: "voyage"
+});
+var createJinaRAGReranker = (config) => createHttpCrossEncoderReranker({
+  buildBody: ({ model, query, documents, topN }) => ({
+    documents,
+    model,
+    query,
+    top_n: topN
+  }),
+  config,
+  endpoint: `${config.baseUrl ?? "https://api.jina.ai"}/v1/rerank`,
+  fallbackModel: "jina-reranker-v2-base-multilingual",
+  providerName: "jina"
+});
 // src/ai/rag/retrievalStrategies.ts
 var tokenize4 = (value) => value.toLowerCase().split(/[^a-z0-9]+/i).map((token) => token.trim()).filter((token) => token.length > 0);
 var hasAnyToken2 = (tokens, values) => values.some((value) => tokens.includes(value));
@@ -31888,7 +32019,9 @@ var toErrorMessage3 = async (response, label) => {
   return new Error(`${label}: ${response.status} ${response.statusText}${detailMessage ? ` (${detailMessage})` : ""}`);
 };
 var normalizeString = (value) => typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined;
-var toUniqueStringArray = (values) => [...new Set(values.filter((value) => typeof value === "string" && value.length > 0))];
+var toUniqueStringArray = (values) => [
+  ...new Set(values.filter((value) => typeof value === "string" && value.length > 0))
+];
 var toContactTitle = (person, fallbackId) => {
   const primaryName = person.names?.find((name) => normalizeString(name.displayName) !== undefined);
   const displayName = normalizeString(primaryName?.displayName);
@@ -32176,6 +32309,7 @@ var {S3Client } = globalThis.Bun;
 import { createHash } from "crypto";
 import { mkdir as mkdir2, readFile as readFile3, readdir as readdir2 } from "fs/promises";
 import { basename as basename2, dirname as dirname3, extname as extname2, join as join2, relative as relative2, resolve as resolve2 } from "path";
+var h2IfHttps2 = (url) => url.startsWith("https://") ? { protocol: "http2" } : {};
 var toSyncError = (caught) => caught instanceof Error ? caught.message : String(caught);
 var wait = async (delayMs) => {
   if (!(delayMs > 0)) {
@@ -32988,7 +33122,7 @@ var isFeedDocument = (value) => {
   return trimmed.includes("<rss") || trimmed.includes("<channel") || trimmed.includes("<feed") || trimmed.includes("<entry");
 };
 var discoverFeedsFromHTML = async (feed) => {
-  const response = await fetch(feed.url);
+  const response = await fetch(feed.url, h2IfHttps2(feed.url));
   if (!response.ok) {
     return [];
   }
@@ -33034,7 +33168,7 @@ var discoverFeedsFromHTML = async (feed) => {
   }
   const validated = [];
   for (const candidate of discovered.values()) {
-    const candidateResponse = await fetch(candidate.url);
+    const candidateResponse = await fetch(candidate.url, h2IfHttps2(candidate.url));
     if (!candidateResponse.ok) {
       continue;
     }
@@ -33098,7 +33232,7 @@ var parseSitemapEntries = (sitemap, value) => {
 };
 var discoverSitemapsFromRobots = async (sitemap) => {
   const robotsURL = resolveSiblingURL(sitemap.url, "/robots.txt");
-  const response = await fetch(robotsURL);
+  const response = await fetch(robotsURL, h2IfHttps2(robotsURL));
   if (!response.ok) {
     return [];
   }
@@ -33119,7 +33253,7 @@ var discoverSitemapsFromRobots = async (sitemap) => {
 };
 var loadRobotsDisallowRules = async (siteURL) => {
   const robotsURL = resolveSiblingURL(siteURL, "/robots.txt");
-  const response = await fetch(robotsURL);
+  const response = await fetch(robotsURL, h2IfHttps2(robotsURL));
   if (!response.ok) {
     return [];
   }
@@ -33160,7 +33294,7 @@ var discoverRecursiveSitemapURLs = async (input) => {
     }
     seen.add(current.sitemap.url);
     resolved.push(current.sitemap);
-    const response = await fetch(current.sitemap.url);
+    const response = await fetch(current.sitemap.url, h2IfHttps2(current.sitemap.url));
     if (!response.ok) {
       throw new Error(`Failed to load sitemap ${current.sitemap.url}: ${response.status} ${response.statusText}`);
     }
@@ -33311,7 +33445,7 @@ var discoverLinkedPagesFromHTML = async (input) => {
       pruneCounts.robotsBlockedCount += 1;
       continue;
     }
-    const response = await fetch(current.url);
+    const response = await fetch(current.url, h2IfHttps2(current.url));
     if (!response.ok) {
       continue;
     }
@@ -33566,6 +33700,7 @@ var loadDiscoveredGitHubRepositoryFiles = async (input) => {
       repo: input.repo
     });
     const response = await fetch(requestURL, {
+      ...h2IfHttps2(requestURL),
       headers: input.requestHeaders
     });
     if (!response.ok) {
@@ -33729,7 +33864,7 @@ var createRAGFeedSyncSource = (options) => ({
       }
     }
     const discoveredEntries = (await Promise.all([...feedMap.values()].map(async (feed) => {
-      const response = await fetch(feed.url);
+      const response = await fetch(feed.url, h2IfHttps2(feed.url));
       if (!response.ok) {
         throw new Error(`Failed to load feed ${feed.url}: ${response.status} ${response.statusText}`);
       }
@@ -33803,7 +33938,7 @@ var createRAGSitemapSyncSource = (options) => ({
       }
     }
     const discoveredEntries = (await Promise.all([...resolvedSitemapMap.values()].map(async (sitemap) => {
-      const response = await fetch(sitemap.url);
+      const response = await fetch(sitemap.url, h2IfHttps2(sitemap.url));
       if (!response.ok) {
         throw new Error(`Failed to load sitemap ${sitemap.url}: ${response.status} ${response.statusText}`);
       }
@@ -33868,7 +34003,7 @@ var createRAGSiteDiscoverySyncSource = (options) => ({
           }
         }
         const feedEntries = (await Promise.all([...feedMap.values()].map(async (feed) => {
-          const response = await fetch(feed.url);
+          const response = await fetch(feed.url, h2IfHttps2(feed.url));
           if (!response.ok) {
             throw new Error(`Failed to load feed ${feed.url}: ${response.status} ${response.statusText}`);
           }
@@ -33918,7 +34053,7 @@ var createRAGSiteDiscoverySyncSource = (options) => ({
           sitemap
         })))).flat();
         const sitemapEntries = (await Promise.all(resolvedSitemaps.map(async (sitemap) => {
-          const response = await fetch(sitemap.url);
+          const response = await fetch(sitemap.url, h2IfHttps2(sitemap.url));
           if (!response.ok) {
             throw new Error(`Failed to load sitemap ${sitemap.url}: ${response.status} ${response.statusText}`);
           }
@@ -37325,6 +37460,7 @@ export {
   evaluateRAGAnswerGroundingCase,
   evaluateRAGAnswerGrounding,
   deepseekEmbeddings,
+  createVoyageRAGReranker,
   createTextFileExtractor,
   createSQLiteRAGStore,
   createRAGVector,
@@ -37422,12 +37558,14 @@ export {
   createPDFFileExtractor,
   createOfficeDocumentExtractor,
   createLegacyDocumentExtractor,
+  createJinaRAGReranker,
   createInMemoryRAGStore,
   createHeuristicRAGRetrievalStrategy,
   createHeuristicRAGReranker,
   createHeuristicRAGQueryTransform,
   createEmailExtractor,
   createEPUBExtractor,
+  createCohereRAGReranker,
   createBuiltinArchiveExpander,
   compareRAGRetrievalTraceSummaries,
   compareRAGRetrievalStrategies,
@@ -37511,5 +37649,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=A89076338D70690164756E2164756E21
+//# debugId=3E3026F16C216BA364756E2164756E21
 //# sourceMappingURL=index.js.map