@cosmocoder/mcp-web-docs 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (240) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +368 -0
  3. package/build/__mocks__/embeddings.d.ts +17 -0
  4. package/build/__mocks__/embeddings.js +66 -0
  5. package/build/__mocks__/embeddings.js.map +1 -0
  6. package/build/config.d.ts +44 -0
  7. package/build/config.js +158 -0
  8. package/build/config.js.map +1 -0
  9. package/build/config.test.d.ts +1 -0
  10. package/build/config.test.js +165 -0
  11. package/build/config.test.js.map +1 -0
  12. package/build/crawler/auth.d.ts +128 -0
  13. package/build/crawler/auth.js +546 -0
  14. package/build/crawler/auth.js.map +1 -0
  15. package/build/crawler/auth.test.d.ts +1 -0
  16. package/build/crawler/auth.test.js +174 -0
  17. package/build/crawler/auth.test.js.map +1 -0
  18. package/build/crawler/base.d.ts +24 -0
  19. package/build/crawler/base.js +149 -0
  20. package/build/crawler/base.js.map +1 -0
  21. package/build/crawler/base.test.d.ts +1 -0
  22. package/build/crawler/base.test.js +234 -0
  23. package/build/crawler/base.test.js.map +1 -0
  24. package/build/crawler/browser-config.d.ts +2 -0
  25. package/build/crawler/browser-config.js +29 -0
  26. package/build/crawler/browser-config.js.map +1 -0
  27. package/build/crawler/browser-config.test.d.ts +1 -0
  28. package/build/crawler/browser-config.test.js +56 -0
  29. package/build/crawler/browser-config.test.js.map +1 -0
  30. package/build/crawler/cheerio.d.ts +11 -0
  31. package/build/crawler/cheerio.js +134 -0
  32. package/build/crawler/cheerio.js.map +1 -0
  33. package/build/crawler/chromium.d.ts +21 -0
  34. package/build/crawler/chromium.js +596 -0
  35. package/build/crawler/chromium.js.map +1 -0
  36. package/build/crawler/content-extractor-types.d.ts +25 -0
  37. package/build/crawler/content-extractor-types.js +2 -0
  38. package/build/crawler/content-extractor-types.js.map +1 -0
  39. package/build/crawler/content-extractors.d.ts +9 -0
  40. package/build/crawler/content-extractors.js +9 -0
  41. package/build/crawler/content-extractors.js.map +1 -0
  42. package/build/crawler/content-utils.d.ts +2 -0
  43. package/build/crawler/content-utils.js +22 -0
  44. package/build/crawler/content-utils.js.map +1 -0
  45. package/build/crawler/content-utils.test.d.ts +1 -0
  46. package/build/crawler/content-utils.test.js +99 -0
  47. package/build/crawler/content-utils.test.js.map +1 -0
  48. package/build/crawler/crawlee-crawler.d.ts +63 -0
  49. package/build/crawler/crawlee-crawler.js +342 -0
  50. package/build/crawler/crawlee-crawler.js.map +1 -0
  51. package/build/crawler/crawlee-crawler.test.d.ts +1 -0
  52. package/build/crawler/crawlee-crawler.test.js +280 -0
  53. package/build/crawler/crawlee-crawler.test.js.map +1 -0
  54. package/build/crawler/default-extractor.d.ts +4 -0
  55. package/build/crawler/default-extractor.js +26 -0
  56. package/build/crawler/default-extractor.js.map +1 -0
  57. package/build/crawler/default-extractor.test.d.ts +1 -0
  58. package/build/crawler/default-extractor.test.js +200 -0
  59. package/build/crawler/default-extractor.test.js.map +1 -0
  60. package/build/crawler/default.d.ts +11 -0
  61. package/build/crawler/default.js +138 -0
  62. package/build/crawler/default.js.map +1 -0
  63. package/build/crawler/docs-crawler.d.ts +26 -0
  64. package/build/crawler/docs-crawler.js +97 -0
  65. package/build/crawler/docs-crawler.js.map +1 -0
  66. package/build/crawler/docs-crawler.test.d.ts +1 -0
  67. package/build/crawler/docs-crawler.test.js +185 -0
  68. package/build/crawler/docs-crawler.test.js.map +1 -0
  69. package/build/crawler/factory.d.ts +6 -0
  70. package/build/crawler/factory.js +83 -0
  71. package/build/crawler/factory.js.map +1 -0
  72. package/build/crawler/github-pages-extractor.d.ts +4 -0
  73. package/build/crawler/github-pages-extractor.js +33 -0
  74. package/build/crawler/github-pages-extractor.js.map +1 -0
  75. package/build/crawler/github-pages-extractor.test.d.ts +1 -0
  76. package/build/crawler/github-pages-extractor.test.js +184 -0
  77. package/build/crawler/github-pages-extractor.test.js.map +1 -0
  78. package/build/crawler/github.d.ts +20 -0
  79. package/build/crawler/github.js +181 -0
  80. package/build/crawler/github.js.map +1 -0
  81. package/build/crawler/github.test.d.ts +1 -0
  82. package/build/crawler/github.test.js +326 -0
  83. package/build/crawler/github.test.js.map +1 -0
  84. package/build/crawler/puppeteer.d.ts +16 -0
  85. package/build/crawler/puppeteer.js +191 -0
  86. package/build/crawler/puppeteer.js.map +1 -0
  87. package/build/crawler/queue-manager.d.ts +43 -0
  88. package/build/crawler/queue-manager.js +169 -0
  89. package/build/crawler/queue-manager.js.map +1 -0
  90. package/build/crawler/queue-manager.test.d.ts +1 -0
  91. package/build/crawler/queue-manager.test.js +509 -0
  92. package/build/crawler/queue-manager.test.js.map +1 -0
  93. package/build/crawler/site-rules.d.ts +11 -0
  94. package/build/crawler/site-rules.js +104 -0
  95. package/build/crawler/site-rules.js.map +1 -0
  96. package/build/crawler/site-rules.test.d.ts +1 -0
  97. package/build/crawler/site-rules.test.js +139 -0
  98. package/build/crawler/site-rules.test.js.map +1 -0
  99. package/build/crawler/storybook-extractor.d.ts +34 -0
  100. package/build/crawler/storybook-extractor.js +767 -0
  101. package/build/crawler/storybook-extractor.js.map +1 -0
  102. package/build/crawler/storybook-extractor.test.d.ts +1 -0
  103. package/build/crawler/storybook-extractor.test.js +491 -0
  104. package/build/crawler/storybook-extractor.test.js.map +1 -0
  105. package/build/embeddings/fastembed.d.ts +25 -0
  106. package/build/embeddings/fastembed.js +188 -0
  107. package/build/embeddings/fastembed.js.map +1 -0
  108. package/build/embeddings/fastembed.test.d.ts +1 -0
  109. package/build/embeddings/fastembed.test.js +307 -0
  110. package/build/embeddings/fastembed.test.js.map +1 -0
  111. package/build/embeddings/openai.d.ts +8 -0
  112. package/build/embeddings/openai.js +56 -0
  113. package/build/embeddings/openai.js.map +1 -0
  114. package/build/embeddings/types.d.ts +4 -0
  115. package/build/embeddings/types.js +2 -0
  116. package/build/embeddings/types.js.map +1 -0
  117. package/build/index.d.ts +2 -0
  118. package/build/index.js +1007 -0
  119. package/build/index.js.map +1 -0
  120. package/build/index.test.d.ts +1 -0
  121. package/build/index.test.js +364 -0
  122. package/build/index.test.js.map +1 -0
  123. package/build/indexing/queue-manager.d.ts +36 -0
  124. package/build/indexing/queue-manager.js +86 -0
  125. package/build/indexing/queue-manager.js.map +1 -0
  126. package/build/indexing/queue-manager.test.d.ts +1 -0
  127. package/build/indexing/queue-manager.test.js +257 -0
  128. package/build/indexing/queue-manager.test.js.map +1 -0
  129. package/build/indexing/status.d.ts +39 -0
  130. package/build/indexing/status.js +207 -0
  131. package/build/indexing/status.js.map +1 -0
  132. package/build/indexing/status.test.d.ts +1 -0
  133. package/build/indexing/status.test.js +246 -0
  134. package/build/indexing/status.test.js.map +1 -0
  135. package/build/processor/content.d.ts +16 -0
  136. package/build/processor/content.js +286 -0
  137. package/build/processor/content.js.map +1 -0
  138. package/build/processor/content.test.d.ts +1 -0
  139. package/build/processor/content.test.js +369 -0
  140. package/build/processor/content.test.js.map +1 -0
  141. package/build/processor/markdown.d.ts +11 -0
  142. package/build/processor/markdown.js +256 -0
  143. package/build/processor/markdown.js.map +1 -0
  144. package/build/processor/markdown.test.d.ts +1 -0
  145. package/build/processor/markdown.test.js +312 -0
  146. package/build/processor/markdown.test.js.map +1 -0
  147. package/build/processor/metadata-parser.d.ts +37 -0
  148. package/build/processor/metadata-parser.js +245 -0
  149. package/build/processor/metadata-parser.js.map +1 -0
  150. package/build/processor/metadata-parser.test.d.ts +1 -0
  151. package/build/processor/metadata-parser.test.js +357 -0
  152. package/build/processor/metadata-parser.test.js.map +1 -0
  153. package/build/processor/processor.d.ts +8 -0
  154. package/build/processor/processor.js +190 -0
  155. package/build/processor/processor.js.map +1 -0
  156. package/build/processor/processor.test.d.ts +1 -0
  157. package/build/processor/processor.test.js +357 -0
  158. package/build/processor/processor.test.js.map +1 -0
  159. package/build/rag/cache.d.ts +10 -0
  160. package/build/rag/cache.js +10 -0
  161. package/build/rag/cache.js.map +1 -0
  162. package/build/rag/code-generator.d.ts +11 -0
  163. package/build/rag/code-generator.js +30 -0
  164. package/build/rag/code-generator.js.map +1 -0
  165. package/build/rag/context-assembler.d.ts +23 -0
  166. package/build/rag/context-assembler.js +113 -0
  167. package/build/rag/context-assembler.js.map +1 -0
  168. package/build/rag/docs-search.d.ts +55 -0
  169. package/build/rag/docs-search.js +380 -0
  170. package/build/rag/docs-search.js.map +1 -0
  171. package/build/rag/pipeline.d.ts +26 -0
  172. package/build/rag/pipeline.js +91 -0
  173. package/build/rag/pipeline.js.map +1 -0
  174. package/build/rag/query-processor.d.ts +14 -0
  175. package/build/rag/query-processor.js +57 -0
  176. package/build/rag/query-processor.js.map +1 -0
  177. package/build/rag/reranker.d.ts +55 -0
  178. package/build/rag/reranker.js +210 -0
  179. package/build/rag/reranker.js.map +1 -0
  180. package/build/rag/response-generator.d.ts +20 -0
  181. package/build/rag/response-generator.js +101 -0
  182. package/build/rag/response-generator.js.map +1 -0
  183. package/build/rag/retriever.d.ts +19 -0
  184. package/build/rag/retriever.js +111 -0
  185. package/build/rag/retriever.js.map +1 -0
  186. package/build/rag/validator.d.ts +22 -0
  187. package/build/rag/validator.js +128 -0
  188. package/build/rag/validator.js.map +1 -0
  189. package/build/rag/version-manager.d.ts +23 -0
  190. package/build/rag/version-manager.js +98 -0
  191. package/build/rag/version-manager.js.map +1 -0
  192. package/build/setupTests.d.ts +4 -0
  193. package/build/setupTests.js +50 -0
  194. package/build/setupTests.js.map +1 -0
  195. package/build/storage/storage.d.ts +38 -0
  196. package/build/storage/storage.js +700 -0
  197. package/build/storage/storage.js.map +1 -0
  198. package/build/storage/storage.test.d.ts +1 -0
  199. package/build/storage/storage.test.js +338 -0
  200. package/build/storage/storage.test.js.map +1 -0
  201. package/build/types/rag.d.ts +27 -0
  202. package/build/types/rag.js +2 -0
  203. package/build/types/rag.js.map +1 -0
  204. package/build/types.d.ts +120 -0
  205. package/build/types.js +2 -0
  206. package/build/types.js.map +1 -0
  207. package/build/util/content-utils.d.ts +31 -0
  208. package/build/util/content-utils.js +120 -0
  209. package/build/util/content-utils.js.map +1 -0
  210. package/build/util/content.d.ts +1 -0
  211. package/build/util/content.js +16 -0
  212. package/build/util/content.js.map +1 -0
  213. package/build/util/docs.d.ts +1 -0
  214. package/build/util/docs.js +26 -0
  215. package/build/util/docs.js.map +1 -0
  216. package/build/util/docs.test.d.ts +1 -0
  217. package/build/util/docs.test.js +49 -0
  218. package/build/util/docs.test.js.map +1 -0
  219. package/build/util/favicon.d.ts +6 -0
  220. package/build/util/favicon.js +88 -0
  221. package/build/util/favicon.js.map +1 -0
  222. package/build/util/favicon.test.d.ts +1 -0
  223. package/build/util/favicon.test.js +140 -0
  224. package/build/util/favicon.test.js.map +1 -0
  225. package/build/util/logger.d.ts +17 -0
  226. package/build/util/logger.js +72 -0
  227. package/build/util/logger.js.map +1 -0
  228. package/build/util/logger.test.d.ts +1 -0
  229. package/build/util/logger.test.js +46 -0
  230. package/build/util/logger.test.js.map +1 -0
  231. package/build/util/security.d.ts +312 -0
  232. package/build/util/security.js +719 -0
  233. package/build/util/security.js.map +1 -0
  234. package/build/util/security.test.d.ts +1 -0
  235. package/build/util/security.test.js +524 -0
  236. package/build/util/security.test.js.map +1 -0
  237. package/build/util/site-detector.d.ts +22 -0
  238. package/build/util/site-detector.js +42 -0
  239. package/build/util/site-detector.js.map +1 -0
  240. package/package.json +112 -0
@@ -0,0 +1,55 @@
1
+ import { SearchResult } from "../types.js";
2
+ /**
3
+ * Reranker interface for document search improvement
4
+ * Responsible for reranking search results to improve relevance
5
+ */
6
+ export interface Reranker {
7
+ /**
8
+ * Rerank chunks based on relevance to the query
9
+ * @param query - The user query
10
+ * @param chunks - The chunks to rerank
11
+ * @param options - Optional reranking settings
12
+ * @returns An array of relevance scores corresponding to each chunk
13
+ */
14
+ rerank(query: string, chunks: SearchResult[], options?: RerankerOptions): Promise<number[]>;
15
+ }
16
+ /**
17
+ * Options for configuring reranker behavior
18
+ */
19
+ export interface RerankerOptions {
20
+ componentNames?: string[];
21
+ propNames?: string[];
22
+ componentBoost?: number;
23
+ propBoost?: number;
24
+ codeBoost?: number;
25
+ semanticWeight?: number;
26
+ exactMatchWeight?: number;
27
+ }
28
+ /**
29
+ * Enhanced simple reranker implementation with smarter heuristics
30
+ * Uses multiple relevance signals and weighted scoring
31
+ */
32
+ export declare class EnhancedSimpleReranker implements Reranker {
33
+ readonly _apiKey: string;
34
+ constructor(_apiKey: string);
35
+ rerank(query: string, chunks: SearchResult[], options?: RerankerOptions): Promise<number[]>;
36
+ private extractPhrases;
37
+ }
38
+ /**
39
+ * Improved OpenAI-based reranker that uses embeddings similarity and additional relevance signals
40
+ */
41
+ export declare class OpenAIReranker implements Reranker {
42
+ private openaiApiKey;
43
+ constructor(openaiApiKey: string);
44
+ rerank(query: string, chunks: SearchResult[], options?: RerankerOptions): Promise<number[]>;
45
+ private cosineSimilarity;
46
+ }
47
+ /**
48
+ * Factory for creating rerankers based on configuration
49
+ */
50
+ export declare class RerankerFactory {
51
+ /**
52
+ * Create a reranker instance based on the specified model
53
+ */
54
+ static create(model: string | undefined, apiKey: string): Reranker;
55
+ }
@@ -0,0 +1,210 @@
1
+ /**
2
+ * Enhanced simple reranker implementation with smarter heuristics
3
+ * Uses multiple relevance signals and weighted scoring
4
+ */
5
+ export class EnhancedSimpleReranker {
6
+ _apiKey;
7
+ constructor(_apiKey) {
8
+ this._apiKey = _apiKey;
9
+ // Store API key but not used directly in this reranker
10
+ void this._apiKey; // No-op to avoid unused variable warning
11
+ }
12
+ async rerank(query, chunks, options = {}) {
13
+ console.debug(`[EnhancedSimpleReranker] Reranking ${chunks.length} chunks`);
14
+ // If we have fewer than 2 chunks, just return score 1 for all
15
+ if (!chunks || chunks.length <= 1) {
16
+ return chunks.map(() => 1);
17
+ }
18
+ try {
19
+ const scores = [];
20
+ const { componentNames = [], propNames = [], componentBoost = 0.2, propBoost = 0.15, codeBoost = 0.1, semanticWeight = 0.6, exactMatchWeight = 0.4 } = options;
21
+ // Normalize query for matching
22
+ const queryLower = query.toLowerCase();
23
+ const queryWords = new Set(queryLower.split(/\s+/).filter(w => w.length > 3));
24
+ // Check if query is code-related
25
+ const isCodeQuery = /\bcode\b|\bexample\b|\bsnippet\b|\bhow to\b|\bimplement\b/i.test(query);
26
+ for (const chunk of chunks) {
27
+ const contentLower = chunk.content.toLowerCase();
28
+ const contentWords = new Set(contentLower.split(/\s+/).filter(w => w.length > 3));
29
+ // --- Calculate base text relevance with Jaccard similarity ---
30
+ const intersection = new Set([...queryWords].filter(x => contentWords.has(x)));
31
+ const union = new Set([...queryWords, ...contentWords]);
32
+ let textRelevance = intersection.size / Math.max(union.size, 1);
33
+ // --- Calculate exact phrase matches ---
34
+ // Split query into 2-3 word phrases and check if they appear in content
35
+ const phrases = this.extractPhrases(queryLower);
36
+ let phraseMatches = 0;
37
+ for (const phrase of phrases) {
38
+ if (contentLower.includes(phrase)) {
39
+ phraseMatches++;
40
+ }
41
+ }
42
+ const phraseScore = phrases.length > 0 ? phraseMatches / phrases.length : 0;
43
+ // --- Apply component name boosting ---
44
+ let componentScore = 0;
45
+ if (componentNames.length > 0) {
46
+ // Check for component name matches (case sensitive as component names are usually PascalCase)
47
+ for (const componentName of componentNames) {
48
+ const regex = new RegExp(`\\b${componentName}\\b`, 'g');
49
+ const matches = (chunk.content.match(regex) || []).length;
50
+ if (matches > 0) {
51
+ componentScore += (matches * componentBoost);
52
+ }
53
+ }
54
+ }
55
+ else {
56
+ // If no specific component names, check for any PascalCase words (potential components)
57
+ const pascalCaseRegex = /\b[A-Z][a-z]+[A-Z]\w*\b/g;
58
+ const componentMatches = chunk.content.match(pascalCaseRegex) || [];
59
+ if (componentMatches.length > 0) {
60
+ componentScore = componentBoost; // Fixed boost for having any component names
61
+ }
62
+ }
63
+ // --- Apply prop name boosting ---
64
+ let propScore = 0;
65
+ if (propNames.length > 0) {
66
+ for (const propName of propNames) {
67
+ const regex = new RegExp(`\\b${propName}\\b`, 'gi');
68
+ const matches = (chunk.content.match(regex) || []).length;
69
+ if (matches > 0) {
70
+ propScore += (matches * propBoost);
71
+ }
72
+ }
73
+ }
74
+ // --- Code example boosting ---
75
+ let codeScore = 0;
76
+ if (isCodeQuery && chunk.metadata.codeBlocks && chunk.metadata.codeBlocks.length > 0) {
77
+ codeScore = codeBoost * Math.min(chunk.metadata.codeBlocks.length, 3); // Cap boost for 3+ code blocks
78
+ }
79
+ // --- Metadata type relevance ---
80
+ let metadataScore = 0;
81
+ if (chunk.metadata.type === 'api' && /\bapi\b|\bprops\b|\bparameters\b|\boptions\b/i.test(query)) {
82
+ metadataScore = 0.15; // Boost API docs for API-related queries
83
+ }
84
+ else if (chunk.metadata.type === 'example' && isCodeQuery) {
85
+ metadataScore = 0.15; // Boost examples for code-related queries
86
+ }
87
+ // --- Combine all relevance signals with weights ---
88
+ // Base score is a weighted combination of text relevance and phrase matching
89
+ const baseScore = (textRelevance * exactMatchWeight) + (phraseScore * (1 - exactMatchWeight));
90
+ // Apply boosts additively to avoid score explosion
91
+ let finalScore = (baseScore * semanticWeight) +
92
+ componentScore +
93
+ propScore +
94
+ codeScore +
95
+ metadataScore;
96
+ // Ensure score is in 0-1 range, with diminishing returns for extreme boosts
97
+ finalScore = Math.min(1, finalScore);
98
+ scores.push(finalScore);
99
+ // Debug logging for first few chunks to help with debugging
100
+ if (scores.length <= 3) {
101
+ console.debug(`[EnhancedSimpleReranker] Chunk ${scores.length} score:`, {
102
+ finalScore,
103
+ baseScore,
104
+ textRelevance,
105
+ phraseScore,
106
+ componentScore,
107
+ propScore,
108
+ codeScore,
109
+ metadataScore
110
+ });
111
+ }
112
+ }
113
+ return scores;
114
+ }
115
+ catch (error) {
116
+ console.error('[EnhancedSimpleReranker] Error in reranking:', error);
117
+ // Return neutral scores if reranking fails
118
+ return chunks.map(() => 0.5);
119
+ }
120
+ }
121
+ // Extract meaningful 2-3 word phrases from query
122
+ extractPhrases(text) {
123
+ const words = text.split(/\s+/).filter(w => w.length > 2);
124
+ const phrases = [];
125
+ // Extract 3-word phrases
126
+ for (let i = 0; i < words.length - 2; i++) {
127
+ phrases.push(`${words[i]} ${words[i + 1]} ${words[i + 2]}`);
128
+ }
129
+ // Extract 2-word phrases
130
+ for (let i = 0; i < words.length - 1; i++) {
131
+ phrases.push(`${words[i]} ${words[i + 1]}`);
132
+ }
133
+ return phrases;
134
+ }
135
+ }
136
+ /**
137
+ * Improved OpenAI-based reranker that uses embeddings similarity and additional relevance signals
138
+ */
139
+ export class OpenAIReranker {
140
+ openaiApiKey;
141
+ constructor(openaiApiKey) {
142
+ this.openaiApiKey = openaiApiKey;
143
+ }
144
+ async rerank(query, chunks, options = {}) {
145
+ console.debug(`[OpenAIReranker] Reranking ${chunks.length} chunks`);
146
+ // If we have fewer than 2 chunks, just return score 1 for all
147
+ if (!chunks || chunks.length <= 1) {
148
+ return chunks.map(() => 1);
149
+ }
150
+ try {
151
+ // Import OpenAI in the method to avoid loading it unnecessarily
152
+ const { default: OpenAI } = await import('openai');
153
+ const openai = new OpenAI({ apiKey: this.openaiApiKey });
154
+ // Get embeddings for the query
155
+ const queryEmbedding = await openai.embeddings.create({
156
+ model: 'text-embedding-ada-002',
157
+ input: query,
158
+ });
159
+ // Get embeddings for all chunks in a single batch request
160
+ const chunkEmbeddings = await openai.embeddings.create({
161
+ model: 'text-embedding-ada-002',
162
+ input: chunks.map(chunk => chunk.content.substring(0, 8000)),
163
+ });
164
+ // Calculate cosine similarity between query and each chunk
165
+ const semanticScores = chunkEmbeddings.data.map((chunkEmbed) => {
166
+ return this.cosineSimilarity(queryEmbedding.data[0].embedding, chunkEmbed.embedding);
167
+ });
168
+ // For enhanced scoring, also incorporate the simple reranker's heuristics
169
+ const simpleReranker = new EnhancedSimpleReranker(this.openaiApiKey);
170
+ const heuristicScores = await simpleReranker.rerank(query, chunks, options);
171
+ // Combine semantic and heuristic scores (weighted average)
172
+ const combinedScores = semanticScores.map((semanticScore, i) => {
173
+ const heuristicScore = heuristicScores[i];
174
+ // Weight semantic score more heavily (70/30 split)
175
+ return (semanticScore * 0.7) + (heuristicScore * 0.3);
176
+ });
177
+ return combinedScores;
178
+ }
179
+ catch (error) {
180
+ console.error('[OpenAIReranker] Error in OpenAI reranking:', error);
181
+ // Fallback to the simpler reranker if OpenAI fails
182
+ const simpleReranker = new EnhancedSimpleReranker(this.openaiApiKey);
183
+ return simpleReranker.rerank(query, chunks, options);
184
+ }
185
+ }
186
+ // Calculate cosine similarity between two vectors
187
+ cosineSimilarity(vecA, vecB) {
188
+ const dotProduct = vecA.reduce((sum, a, i) => sum + a * vecB[i], 0);
189
+ const magA = Math.sqrt(vecA.reduce((sum, a) => sum + a * a, 0));
190
+ const magB = Math.sqrt(vecB.reduce((sum, b) => sum + b * b, 0));
191
+ return dotProduct / (magA * magB);
192
+ }
193
+ }
194
+ /**
195
+ * Factory for creating rerankers based on configuration
196
+ */
197
+ export class RerankerFactory {
198
+ /**
199
+ * Create a reranker instance based on the specified model
200
+ */
201
+ static create(model = 'openai', apiKey) {
202
+ if (model === 'openai') {
203
+ return new OpenAIReranker(apiKey);
204
+ }
205
+ else {
206
+ return new EnhancedSimpleReranker(apiKey);
207
+ }
208
+ }
209
+ }
210
+ //# sourceMappingURL=reranker.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reranker.js","sourceRoot":"","sources":["../../src/rag/reranker.ts"],"names":[],"mappings":"AA8BA;;;GAGG;AACH,MAAM,OAAO,sBAAsB;IACZ;IAArB,YAAqB,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;QAClC,uDAAuD;QACvD,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,yCAAyC;IAC9D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,MAAsB,EAAE,UAA2B,EAAE;QAC/E,OAAO,CAAC,KAAK,CAAC,sCAAsC,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;QAE5E,8DAA8D;QAC9D,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,MAAM,EACJ,cAAc,GAAG,EAAE,EACnB,SAAS,GAAG,EAAE,EACd,cAAc,GAAG,GAAG,EACpB,SAAS,GAAG,IAAI,EAChB,SAAS,GAAG,GAAG,EACf,cAAc,GAAG,GAAG,EACpB,gBAAgB,GAAG,GAAG,EACvB,GAAG,OAAO,CAAC;YAEZ,+BAA+B;YAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAE9E,iCAAiC;YACjC,MAAM,WAAW,GAAG,4DAA4D,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE7F,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBACjD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;gBAElF,gEAAgE;gBAChE,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/E,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC;gBACxD,IAAI,aAAa,GAAG,YAAY,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBAEhE,yCAAyC;gBACzC,wEAAwE;gBACxE,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;gBAChD,IAAI,aAAa,GAAG,CAAC,CAAC;gBACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBAClC,aAAa,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;gBACD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE5E,wCAAwC;gBACxC,IAAI,cAAc,GAAG,CAAC,CAAC;gBACvB,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,8FAA8F;oBAC9F,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;wBAC3C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,aAAa,KAAK,EAAE,GAAG,CAAC,CAAC;wBACxD,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;wBAC1D,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;4BAChB,cAAc,IAAI,CAAC,OAAO,GAAG,cAAc,CAAC,CAAC;wBAC/C,CAAC;oBACH,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,wFAAwF;oBACxF,MAAM,eAAe,GAAG,0BAA0B,CAAC;oBACnD,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;oBACpE,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAChC,cAAc,GAAG,cAAc,CAAC,CAAC,6CAA6C;oBAChF,CAAC;gBACH,CAAC;gBAED,mCAAmC;gBACnC,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;wBACjC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,QAAQ,KAAK,EAAE,IAAI,CAAC,CAAC;wBACpD,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;wBAC1D,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;4BAChB,SAAS,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;wBACrC,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,gCAAgC;gBAChC,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,WAAW,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrF,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,+BAA+B;gBACxG,CAAC;gBAED,kCAAkC;gBAClC,IAAI,aAAa,GAAG,CAAC,CAAC;gBACtB,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,KAAK,IAAI,+CAA+C,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBACjG,aAAa,GAAG,IAAI,CAAC,CAAC,yCAAyC;gBACjE,CAAC;qBAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,WAAW,EAAE,CAAC;oBAC5D,aAAa,GAAG,IAAI,CAAC,CAAC,0CAA0C;gBAClE,CAAC;gBAED,qDAAqD;gBACrD,6EAA6E;gBAC7E,MAAM,SAAS,GAAG,CAAC,aAAa,GAAG,gBAAgB,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC;gBAE9F,mDAAmD;gBACnD,IAAI,UAAU,GAAG,CAAC,SAAS,GAAG,cAAc,CAAC;oBAC5B,cAAc;oBACd,SAAS;oBACT,SAAS;oBACT,aAAa,CAAC;gBAE/B,4EAA4E;gBAC5E,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAErC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAExB,4DAA4D;gBAC5D,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBACvB,OAAO,CAAC,KAAK,CAAC,kCAAkC,MAAM,CAAC,MAAM,SAAS,EAAE;wBACtE,UAAU;wBACV,SAAS;wBACT,aAAa;wBACb,WAAW;wBACX,cAAc;wBACd,SAAS;wBACT,SAAS;wBACT,aAAa;qBACd,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;YACrE,2CAA2C;YAC3C,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,iDAAiD;IACzC,cAAc,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,EAAE,CAAC;QAEnB,yBAAyB;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,yBAAyB;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5C,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,cAAc;IACL;IAApB,YAAoB,YAAoB;QAApB,iBAAY,GAAZ,YAAY,CAAQ;IAAG,CAAC;IAE5C,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,MAAsB,EAAE,UAA2B,EAAE;QAC/E,OAAO,CAAC,KAAK,CAAC,8BAA8B,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;QAEpE,8DAA8D;QAC9D,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC;YACH,gEAAgE;YAChE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;YAEzD,+BAA+B;YAC/B,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACpD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;YAEH,0DAA0D;YAC1D,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACrD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;aAC7D,CAAC,CAAC;YAEH,2DAA2D;YAC3D,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;gBAC7D,OAAO,IAAI,CAAC,gBAAgB,CAC1B,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAChC,UAAU,CAAC,SAAS,CACrB,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,0EAA0E;YAC1E,MAAM,cAAc,GAAG,IAAI,sBAAsB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACrE,MAAM,eAAe,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;YAE5E,2DAA2D;YAC3D,MAAM,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,EAAE,EAAE;gBAC7D,MAAM,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBAC1C,mDAAmD;gBACnD,OAAO,CAAC,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;YAEH,OAAO,cAAc,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,KAAK,CAAC,CAAC;YACpE,mDAAmD;YACnD,MAAM,cAAc,GAAG,IAAI,sBAAsB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACrE,OAAO,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,kDAAkD;IAC1C,gBAAgB,CAAC,IAAc,EAAE,IAAc;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,UAAU,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,eAAe;IAC1B;;OAEG;IACH,MAAM,CAAC,MAAM,CAAC,QAAgB,QAAQ,EAAE,MAAc;QACpD,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvB,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,sBAAsB,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;CACF"}
@@ -0,0 +1,20 @@
1
+ import { AssembledContext } from "./context-assembler.js";
2
+ import { QueryIntent } from "./query-processor.js";
3
+ export interface GeneratedResponse {
4
+ text: string;
5
+ codeExamples?: string[];
6
+ metadata: {
7
+ sources: string[];
8
+ confidence: number;
9
+ generatedAt: Date;
10
+ };
11
+ }
12
+ export declare class ResponseGenerator {
13
+ private openai;
14
+ constructor(apiKey: string);
15
+ generateResponse(context: AssembledContext, queryIntent: QueryIntent): Promise<GeneratedResponse>;
16
+ private prepareContext;
17
+ private getSystemPrompt;
18
+ private isRelevantForIntent;
19
+ private getSources;
20
+ }
@@ -0,0 +1,101 @@
1
+ import OpenAI from "openai";
2
+ export class ResponseGenerator {
3
+ openai;
4
+ constructor(apiKey) {
5
+ this.openai = new OpenAI({ apiKey });
6
+ }
7
+ async generateResponse(context, queryIntent) {
8
+ // Prepare context for the response
9
+ const contextSummary = this.prepareContext(context, queryIntent);
10
+ // Generate response using OpenAI
11
+ const completion = await this.openai.chat.completions.create({
12
+ model: "gpt-3.5-turbo",
13
+ messages: [
14
+ {
15
+ role: "system",
16
+ content: `${this.getSystemPrompt(queryIntent.intent)}
17
+
18
+ Return your response in JSON format:
19
+ {
20
+ "response": "your detailed response text",
21
+ "codeExamples": ["array of relevant code examples"],
22
+ "metadata": {
23
+ "relevance": "number between 0 and 1 indicating response relevance",
24
+ "sources": ["array of source identifiers used"]
25
+ }
26
+ }`
27
+ },
28
+ {
29
+ role: "user",
30
+ content: `Context:\n${contextSummary}\n\nQuery Intent: ${queryIntent.intent}\nEntities: ${queryIntent.entities.join(", ")}\n\nGenerate a comprehensive response.`
31
+ }
32
+ ],
33
+ temperature: 0.7,
34
+ max_tokens: 1000
35
+ });
36
+ const content = completion.choices[0].message.content;
37
+ if (!content) {
38
+ throw new Error("No content returned from OpenAI");
39
+ }
40
+ const result = JSON.parse(content);
41
+ return {
42
+ text: result.response,
43
+ codeExamples: result.codeExamples || [],
44
+ metadata: {
45
+ sources: this.getSources(context),
46
+ confidence: result.metadata.relevance,
47
+ generatedAt: new Date()
48
+ }
49
+ };
50
+ }
51
+ prepareContext(context, queryIntent) {
52
+ const relevantChunks = context.hierarchicalContext
53
+ .filter(chunk => this.isRelevantForIntent(chunk.metadata.type, queryIntent.intent))
54
+ .map(chunk => ({
55
+ content: chunk.content,
56
+ type: chunk.metadata.type,
57
+ reliability: chunk.metadata.sourceReliability
58
+ }))
59
+ .sort((a, b) => b.reliability - a.reliability);
60
+ return relevantChunks
61
+ .map(chunk => `[${chunk.type}]\n${chunk.content}`)
62
+ .join('\n\n');
63
+ }
64
+ getSystemPrompt(intent) {
65
+ const basePrompt = "You are a technical documentation assistant. Generate clear, accurate responses based on the provided context.";
66
+ const intentPrompts = {
67
+ overview: `${basePrompt}\nProvide a comprehensive overview of the topic, covering key features and concepts.`,
68
+ api: `${basePrompt}\nProvide detailed API documentation, including parameters, return types, and examples.`,
69
+ example: `${basePrompt}\nCreate practical code examples that demonstrate proper usage and common patterns.`,
70
+ usage: `${basePrompt}\nFocus on explaining how to use components effectively, including props, configurations, and best practices.`,
71
+ component_usage: `${basePrompt}\nFocus on explaining how to use components effectively, including props, configurations, and best practices.`,
72
+ concept: `${basePrompt}\nExplain technical concepts clearly, using analogies and examples where appropriate.`,
73
+ troubleshooting: `${basePrompt}\nHelp diagnose and solve problems, providing step-by-step solutions and common pitfalls to avoid.`,
74
+ general: basePrompt
75
+ };
76
+ return intentPrompts[intent];
77
+ }
78
+ isRelevantForIntent(chunkType, intent) {
79
+ const relevanceMap = {
80
+ overview: ['overview', 'usage', 'api'],
81
+ api: ['api', 'overview', 'example'],
82
+ example: ['example', 'usage', 'api'],
83
+ usage: ['usage', 'overview', 'example'],
84
+ component_usage: ['usage', 'overview', 'example'],
85
+ concept: ['overview', 'api', 'example'],
86
+ troubleshooting: ['example', 'usage', 'api'],
87
+ general: ['overview', 'usage', 'example', 'api']
88
+ };
89
+ return relevanceMap[intent].includes(chunkType);
90
+ }
91
+ getSources(context) {
92
+ const sources = new Set();
93
+ for (const chunk of context.hierarchicalContext) {
94
+ if (chunk.metadata.citationInfo.version) {
95
+ sources.add(`${chunk.metadata.citationInfo.version}`);
96
+ }
97
+ }
98
+ return Array.from(sources);
99
+ }
100
+ }
101
+ //# sourceMappingURL=response-generator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"response-generator.js","sourceRoot":"","sources":["../../src/rag/response-generator.ts"],"names":[],"mappings":"AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAY5B,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAS;IAEvB,YAAY,MAAc;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAyB,EAAE,WAAwB;QACxE,mCAAmC;QACnC,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAEjE,iCAAiC;QACjC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YAC3D,KAAK,EAAE,eAAe;YACtB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC;;;;;;;;;;YAUlD;iBACH;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,aAAa,cAAc,qBAAqB,WAAW,CAAC,MAAM,eAAe,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,wCAAwC;iBAClK;aACF;YACD,WAAW,EAAE,GAAG;YAChB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACtD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,QAAQ;YACrB,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,EAAE;YACvC,QAAQ,EAAE;gBACR,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;gBACjC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;gBACrC,WAAW,EAAE,IAAI,IAAI,EAAE;aACxB;SACF,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,OAAyB,EAAE,WAAwB;QACxE,MAAM,cAAc,GAAG,OAAO,CAAC,mBAAmB;aAC/C,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;aAClF,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACb,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI;YACzB,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,iBAAiB;SAC9C,CAAC,CAAC;aACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;QAEjD,OAAO,cAAc;aAClB,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;aACjD,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAEO,eAAe,CAAC,MAAuB;QAC7C,MAAM,UAAU,GAAG,gHAAgH,CAAC;QAEpI,MAAM,aAAa,GAAoC;YACrD,QAAQ,EAAE,GAAG,UAAU,sFAAsF;YAC7G,GAAG,EAAE,GAAG,UAAU,yFAAyF;YAC3G,OAAO,EAAE,GAAG,UAAU,qFAAqF;YAC3G,KAAK,EAAE,GAAG,UAAU,+GAA+G;YACnI,eAAe,EAAE,GAAG,UAAU,+GAA+G;YAC7I,OAAO,EAAE,GAAG,UAAU,uFAAuF;YAC7G,eAAe,EAAE,GAAG,UAAU,oGAAoG;YAClI,OAAO,EAAE,UAAU;SACpB,CAAC;QAEF,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,mBAAmB,CAAC,SAAiB,EAAE,MAAuB;QACpE,MAAM,YAAY,GAAsC;YACtD,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC;YACtC,GAAG,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,SAAS,CAAC;YACnC,OAAO,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC;YACpC,KAAK,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC;YACvC,eAAe,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC;YACjD,OAAO,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC;YACvC,eAAe,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC;YAC5C,OAAO,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC;SACjD,CAAC;QAEF,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAClD,CAAC;IAEO,UAAU,CAAC,OAAyB;QAC1C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;YAChD,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;CACF"}
@@ -0,0 +1,19 @@
1
+ import { EnhancedChunk } from "../types/rag.js";
2
+ import { EmbeddingsProvider } from "../embeddings/types.js";
3
+ import { DocumentStore } from "../storage/storage.js";
4
+ import { QueryIntent } from "./query-processor.js";
5
+ export interface RetrievalResult {
6
+ chunks: EnhancedChunk[];
7
+ relevanceScores: number[];
8
+ }
9
+ export interface RetrievalOptions {
10
+ limit?: number;
11
+ minScore?: number;
12
+ filterByIntent?: boolean;
13
+ }
14
+ export declare class ContextRetriever {
15
+ private readonly store;
16
+ private readonly embeddings;
17
+ constructor(store: DocumentStore, embeddings: EmbeddingsProvider);
18
+ retrieveContext(queryIntent: QueryIntent, options?: RetrievalOptions): Promise<RetrievalResult>;
19
+ }
@@ -0,0 +1,111 @@
1
+ import { logger } from "../util/logger.js";
2
+ export class ContextRetriever {
3
+ store;
4
+ embeddings;
5
+ constructor(store, embeddings) {
6
+ this.store = store;
7
+ this.embeddings = embeddings;
8
+ }
9
+ async retrieveContext(queryIntent, options = {}) {
10
+ const { limit = 10, minScore = 0.7, filterByIntent = true } = options;
11
+ logger.debug(`[ContextRetriever] Retrieving context for intent: ${queryIntent.intent}`, {
12
+ embedding: {
13
+ length: queryIntent.embedding.length,
14
+ sample: queryIntent.embedding.slice(0, 5)
15
+ },
16
+ options: { limit, minScore, filterByIntent }
17
+ });
18
+ // Validate the embedding
19
+ if (!queryIntent.embedding || queryIntent.embedding.length === 0) {
20
+ logger.debug('[ContextRetriever] Empty embedding in queryIntent');
21
+ return { chunks: [], relevanceScores: [] };
22
+ }
23
+ // Search for relevant chunks using the original query embedding
24
+ logger.debug(`[ContextRetriever] Searching documents with embedding of length ${queryIntent.embedding.length}`);
25
+ // Cast to any to work around TypeScript errors with the StorageProvider interface
26
+ const searchResults = await this.store.searchDocuments(queryIntent.embedding, // Use the pre-computed query embedding
27
+ {
28
+ limit: limit * 2, // Get more results initially for filtering
29
+ includeVectors: true, // Need vectors for filtering
30
+ filterByType: filterByIntent ? queryIntent.intent : undefined
31
+ });
32
+ logger.debug(`[ContextRetriever] Search returned ${searchResults.length} results`);
33
+ // Filter and process results
34
+ const filteredResults = searchResults
35
+ .filter((result) => {
36
+ // Apply minimum relevance score threshold
37
+ if (result.score < minScore) {
38
+ logger.debug(`[ContextRetriever] Filtering out result with score ${result.score} < ${minScore}`);
39
+ return false;
40
+ }
41
+ if (filterByIntent) {
42
+ // Additional intent-based filtering logic here
43
+ // For example, for API queries, prioritize chunks with API documentation
44
+ // Map intent to metadata type
45
+ const typeMap = {
46
+ 'overview': ['overview'],
47
+ 'api': ['api'],
48
+ 'example': ['example'],
49
+ 'usage': ['usage', 'overview'],
50
+ 'component_usage': ['usage', 'overview'],
51
+ 'concept': ['overview'],
52
+ 'troubleshooting': ['example', 'usage'],
53
+ 'general': ['overview', 'api', 'example', 'usage']
54
+ };
55
+ const allowedTypes = typeMap[queryIntent.intent] || ['overview'];
56
+ const resultType = result.metadata?.type || 'overview';
57
+ const isAllowed = allowedTypes.includes(resultType);
58
+ if (!isAllowed) {
59
+ logger.debug(`[ContextRetriever] Filtering out result with type ${resultType} not in allowed types:`, allowedTypes);
60
+ return false;
61
+ }
62
+ }
63
+ return true;
64
+ })
65
+ .slice(0, limit);
66
+ logger.debug(`[ContextRetriever] After filtering, ${filteredResults.length} results remain`);
67
+ // Convert search results to EnhancedChunks
68
+ const chunks = filteredResults.map((result) => {
69
+ // Log vector information for debugging
70
+ logger.debug(`[ContextRetriever] Processing result:`, {
71
+ id: result.id,
72
+ score: result.score,
73
+ hasVector: !!result.vector,
74
+ vectorLength: result.vector ? result.vector.length : 'N/A'
75
+ });
76
+ // Use the vector from the search result if available, otherwise create a default vector
77
+ const embedding = result.vector && Array.isArray(result.vector) && result.vector.length > 0
78
+ ? result.vector
79
+ : new Array(this.embeddings.dimensions).fill(0);
80
+ return {
81
+ id: result.id,
82
+ content: result.content,
83
+ embedding,
84
+ metadata: {
85
+ sourceReliability: 1.0, // Default reliability score
86
+ lastVerified: new Date(),
87
+ citationInfo: {
88
+ version: "1.0", // Default version
89
+ },
90
+ contextualSummary: result.content.substring(0, 200), // Simple summary
91
+ prerequisites: [],
92
+ validationRules: {
93
+ constraints: [],
94
+ requirements: []
95
+ },
96
+ semanticTags: queryIntent.entities,
97
+ complexity: 'basic', // Default complexity
98
+ type: result.metadata?.type || 'overview', // Use type from search result or default to overview
99
+ framework: result.metadata?.framework,
100
+ language: result.metadata?.language
101
+ },
102
+ relationships: []
103
+ };
104
+ });
105
+ return {
106
+ chunks,
107
+ relevanceScores: filteredResults.map((r) => r.score)
108
+ };
109
+ }
110
+ }
111
+ //# sourceMappingURL=retriever.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retriever.js","sourceRoot":"","sources":["../../src/rag/retriever.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAa3C,MAAM,OAAO,gBAAgB;IAER;IACA;IAFnB,YACmB,KAAoB,EACpB,UAA8B;QAD9B,UAAK,GAAL,KAAK,CAAe;QACpB,eAAU,GAAV,UAAU,CAAoB;IAC9C,CAAC;IAEJ,KAAK,CAAC,eAAe,CACnB,WAAwB,EACxB,UAA4B,EAAE;QAE9B,MAAM,EACJ,KAAK,GAAG,EAAE,EACV,QAAQ,GAAG,GAAG,EACd,cAAc,GAAG,IAAI,EACtB,GAAG,OAAO,CAAC;QAEZ,MAAM,CAAC,KAAK,CAAC,qDAAqD,WAAW,CAAC,MAAM,EAAE,EAAE;YACtF,SAAS,EAAE;gBACT,MAAM,EAAE,WAAW,CAAC,SAAS,CAAC,MAAM;gBACpC,MAAM,EAAE,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;aAC1C;YACD,OAAO,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE;SAC7C,CAAC,CAAC;QAEH,yBAAyB;QACzB,IAAI,CAAC,WAAW,CAAC,SAAS,IAAI,WAAW,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;YAClE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;QAC7C,CAAC;QAED,gEAAgE;QAChE,MAAM,CAAC,KAAK,CAAC,mEAAmE,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;QAChH,kFAAkF;QAClF,MAAM,aAAa,GAAG,MAAO,IAAI,CAAC,KAAa,CAAC,eAAe,CAC7D,WAAW,CAAC,SAAS,EAAE,uCAAuC;QAC9D;YACE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,2CAA2C;YAC7D,cAAc,EAAE,IAAI,EAAE,6BAA6B;YACnD,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;SAC9D,CACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,sCAAsC,aAAa,CAAC,MAAM,UAAU,CAAC,CAAC;QAEnF,6BAA6B;QAC7B,MAAM,eAAe,GAAG,aAAa;aAClC,MAAM,CAAC,CAAC,MAAoB,EAAE,EAAE;YAC/B,0CAA0C;YAC1C,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,EAAE,CAAC;gBAC5B,MAAM,CAAC,KAAK,CAAC,sDAAsD,MAAM,CAAC,KAAK,MAAM,QAAQ,EAAE,CAAC,CAAC;gBACjG,OAAO,KAAK,CAAC;YACf,CAAC;YAED,IAAI,cAAc,EAAE,CAAC;gBACnB,+CAA+C;gBAC/C,yEAAyE;gBACzE,8BAA8B;gBAC9B,MAAM,OAAO,GAAsC;oBACjD,UAAU,EAAE,CAAC,UAAU,CAAC;oBACxB,KAAK,EAAE,CAAC,KAAK,CAAC;oBACd,SAAS,EAAE,CAAC,SAAS,CAAC;oBACtB,OAAO,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC;oBAC9B,iBAAiB,EAAE,CAAC,OAAO,EAAE,UAAU,CAAC;oBACxC,SAAS,EAAE,CAAC,UAAU,CAAC;oBACvB,iBAAiB,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC;oBACvC,SAAS,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC;iBACnD,CAAC;gBAEF,MAAM,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,IAAI,UAAU,CAAC;gBACvD,MAAM,SAAS,GAAG,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;gBAEpD,IAAI,CAAC,SAAS,EAAE,CAAC;oBACf,MAAM,CAAC,KAAK,CAAC,qDAAqD,UAAU,wBAAwB,EAAE,YAAY,CAAC,CAAC;oBACpH,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAEnB,MAAM,CAAC,KAAK,CAAC,uCAAuC,eAAe,CAAC,MAAM,iBAAiB,CAAC,CAAC;QAE7F,2CAA2C;QAC3C,MAAM,MAAM,GAAoB,eAAe,CAAC,GAAG,CAAC,CAAC,MAAoB,EAAE,EAAE;YAC3E,uCAAuC;YACvC,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE;gBACpD,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM;gBAC1B,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;aAC3D,CAAC,CAAC;YAEH,wFAAwF;YACxF,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;gBACzF,CAAC,CAAC,MAAM,CAAC,MAAM;gBACf,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAElD,OAAO;gBACL,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,SAAS;gBACT,QAAQ,EAAE;oBACR,iBAAiB,EAAE,GAAG,EAAE,4BAA4B;oBACpD,YAAY,EAAE,IAAI,IAAI,EAAE;oBACxB,YAAY,EAAE;wBACZ,OAAO,EAAE,KAAK,EAAE,kBAAkB;qBACnC;oBACD,iBAAiB,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,iBAAiB;oBACtE,aAAa,EAAE,EAAE;oBACjB,eAAe,EAAE;wBACf,WAAW,EAAE,EAAE;wBACf,YAAY,EAAE,EAAE;qBACjB;oBACD,YAAY,EAAE,WAAW,CAAC,QAAQ;oBAClC,UAAU,EAAE,OAAO,EAAE,qBAAqB;oBAC1C,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,IAAI,UAAU,EAAE,qDAAqD;oBAChG,SAAS,EAAE,MAAM,CAAC,QAAQ,EAAE,SAAS;oBACrC,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,QAAQ;iBACpC;gBACD,aAAa,EAAE,EAAE;aAClB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,MAAM;YACN,eAAe,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAe,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;SACnE,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,22 @@
1
+ import { GeneratedResponse } from "./response-generator.js";
2
+ import { AssembledContext } from "./context-assembler.js";
3
+ export interface ValidationResult {
4
+ factCheck: boolean;
5
+ codeCheck: boolean;
6
+ consistencyCheck: boolean;
7
+ details: {
8
+ factCheckDetails: string[];
9
+ codeCheckDetails: string[];
10
+ consistencyCheckDetails: string[];
11
+ };
12
+ }
13
+ export declare class ResponseValidator {
14
+ private openai;
15
+ constructor(apiKey: string);
16
+ validateResponse(response: GeneratedResponse, context: AssembledContext): Promise<ValidationResult>;
17
+ private validateFacts;
18
+ private validateCode;
19
+ private validateConsistency;
20
+ private formatContext;
21
+ private checkCommonCodeIssues;
22
+ }