@cosmocoder/mcp-web-docs 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/README.md +132 -10
  2. package/build/crawler/auth.d.ts +17 -6
  3. package/build/crawler/auth.js +166 -31
  4. package/build/crawler/auth.js.map +1 -1
  5. package/build/crawler/auth.test.js +197 -33
  6. package/build/crawler/auth.test.js.map +1 -1
  7. package/build/index.js +68 -16
  8. package/build/index.js.map +1 -1
  9. package/build/index.test.js +134 -0
  10. package/build/index.test.js.map +1 -1
  11. package/build/storage/storage.d.ts +17 -0
  12. package/build/storage/storage.js +102 -5
  13. package/build/storage/storage.js.map +1 -1
  14. package/build/storage/storage.test.js +91 -0
  15. package/build/storage/storage.test.js.map +1 -1
  16. package/build/types.d.ts +4 -0
  17. package/build/util/security.js +8 -0
  18. package/build/util/security.js.map +1 -1
  19. package/build/util/security.test.js +18 -0
  20. package/build/util/security.test.js.map +1 -1
  21. package/package.json +11 -2
  22. package/build/crawler/cheerio.d.ts +0 -11
  23. package/build/crawler/cheerio.js +0 -134
  24. package/build/crawler/cheerio.js.map +0 -1
  25. package/build/crawler/chromium.d.ts +0 -21
  26. package/build/crawler/chromium.js +0 -596
  27. package/build/crawler/chromium.js.map +0 -1
  28. package/build/crawler/default.d.ts +0 -11
  29. package/build/crawler/default.js +0 -138
  30. package/build/crawler/default.js.map +0 -1
  31. package/build/crawler/factory.d.ts +0 -6
  32. package/build/crawler/factory.js +0 -83
  33. package/build/crawler/factory.js.map +0 -1
  34. package/build/crawler/puppeteer.d.ts +0 -16
  35. package/build/crawler/puppeteer.js +0 -191
  36. package/build/crawler/puppeteer.js.map +0 -1
  37. package/build/embeddings/openai.d.ts +0 -8
  38. package/build/embeddings/openai.js +0 -56
  39. package/build/embeddings/openai.js.map +0 -1
  40. package/build/rag/cache.d.ts +0 -10
  41. package/build/rag/cache.js +0 -10
  42. package/build/rag/cache.js.map +0 -1
  43. package/build/rag/code-generator.d.ts +0 -11
  44. package/build/rag/code-generator.js +0 -30
  45. package/build/rag/code-generator.js.map +0 -1
  46. package/build/rag/context-assembler.d.ts +0 -23
  47. package/build/rag/context-assembler.js +0 -113
  48. package/build/rag/context-assembler.js.map +0 -1
  49. package/build/rag/docs-search.d.ts +0 -55
  50. package/build/rag/docs-search.js +0 -380
  51. package/build/rag/docs-search.js.map +0 -1
  52. package/build/rag/pipeline.d.ts +0 -26
  53. package/build/rag/pipeline.js +0 -91
  54. package/build/rag/pipeline.js.map +0 -1
  55. package/build/rag/query-processor.d.ts +0 -14
  56. package/build/rag/query-processor.js +0 -57
  57. package/build/rag/query-processor.js.map +0 -1
  58. package/build/rag/reranker.d.ts +0 -55
  59. package/build/rag/reranker.js +0 -210
  60. package/build/rag/reranker.js.map +0 -1
  61. package/build/rag/response-generator.d.ts +0 -20
  62. package/build/rag/response-generator.js +0 -101
  63. package/build/rag/response-generator.js.map +0 -1
  64. package/build/rag/retriever.d.ts +0 -19
  65. package/build/rag/retriever.js +0 -111
  66. package/build/rag/retriever.js.map +0 -1
  67. package/build/rag/validator.d.ts +0 -22
  68. package/build/rag/validator.js +0 -128
  69. package/build/rag/validator.js.map +0 -1
  70. package/build/rag/version-manager.d.ts +0 -23
  71. package/build/rag/version-manager.js +0 -98
  72. package/build/rag/version-manager.js.map +0 -1
  73. package/build/types/rag.d.ts +0 -27
  74. package/build/types/rag.js +0 -2
  75. package/build/types/rag.js.map +0 -1
  76. package/build/util/content-utils.d.ts +0 -31
  77. package/build/util/content-utils.js +0 -120
  78. package/build/util/content-utils.js.map +0 -1
  79. package/build/util/content.d.ts +0 -1
  80. package/build/util/content.js +0 -16
  81. package/build/util/content.js.map +0 -1
  82. package/build/util/site-detector.d.ts +0 -22
  83. package/build/util/site-detector.js +0 -42
  84. package/build/util/site-detector.js.map +0 -1
@@ -1,91 +0,0 @@
1
- import { QueryProcessor } from "./query-processor.js";
2
- import { DocumentStore } from "../storage/storage.js";
3
- import { OpenAIEmbeddings } from "../embeddings/openai.js";
4
- import { ContextRetriever } from "./retriever.js";
5
- import { ContextAssembler } from "./context-assembler.js";
6
- import { ResponseGenerator } from "./response-generator.js";
7
- import { ResponseValidator } from "./validator.js";
8
- import { VersionManager } from "./version-manager.js";
9
- import { RAGCache } from "./cache.js";
10
- import { logger } from "../util/logger.js";
11
- export class RAGPipeline {
12
- queryProcessor;
13
- contextRetriever;
14
- contextAssembler;
15
- responseGenerator;
16
- responseValidator;
17
- versionManager;
18
- cache;
19
- store;
20
- embeddings;
21
- constructor(config) {
22
- // Initialize embeddings provider
23
- this.embeddings = new OpenAIEmbeddings(config.openaiApiKey);
24
- // Initialize storage
25
- this.store = new DocumentStore(config.dbPath, config.vectorDbPath, this.embeddings, config.maxCacheSize);
26
- // Initialize RAG components
27
- this.queryProcessor = new QueryProcessor(config.openaiApiKey, this.embeddings);
28
- this.contextRetriever = new ContextRetriever(this.store, this.embeddings);
29
- this.contextAssembler = new ContextAssembler();
30
- this.responseGenerator = new ResponseGenerator(config.openaiApiKey);
31
- this.responseValidator = new ResponseValidator(config.openaiApiKey);
32
- this.versionManager = new VersionManager();
33
- this.cache = new RAGCache();
34
- }
35
- async initialize() {
36
- await this.store.initialize();
37
- }
38
- async process(query) {
39
- // Check cache first
40
- const cached = await this.cache.getCachedResponse(query);
41
- if (cached) {
42
- return {
43
- response: cached.response,
44
- validation: {
45
- factCheck: true,
46
- codeCheck: true,
47
- consistencyCheck: true,
48
- details: {
49
- factCheckDetails: [],
50
- codeCheckDetails: [],
51
- consistencyCheckDetails: []
52
- }
53
- }
54
- };
55
- }
56
- try {
57
- // Process query and determine intent
58
- const queryIntent = await this.queryProcessor.processQuery(query);
59
- logger.debug('Query intent classified:', queryIntent);
60
- // Retrieve relevant context
61
- const retrieval = await this.contextRetriever.retrieveContext(queryIntent, {
62
- limit: 10,
63
- minScore: 0.7,
64
- filterByIntent: true
65
- });
66
- logger.debug('Retrieved context chunks:', retrieval.chunks.length);
67
- // Assemble context
68
- const assembled = await this.contextAssembler.assembleContext(retrieval.chunks);
69
- logger.debug('Assembled context with relationships');
70
- // Apply version awareness
71
- await this.versionManager.getVersionedContext(assembled, "1.0");
72
- logger.debug('Applied version context');
73
- // Generate response
74
- const generatedResponse = await this.responseGenerator.generateResponse(assembled, queryIntent);
75
- logger.debug('Generated response');
76
- // Validate response
77
- const validation = await this.responseValidator.validateResponse(generatedResponse, assembled);
78
- logger.debug('Validated response:', validation);
79
- // Cache if validation passed
80
- if (validation.factCheck && validation.codeCheck && validation.consistencyCheck) {
81
- await this.cache.cacheResponse(query, generatedResponse);
82
- }
83
- return { response: generatedResponse, validation };
84
- }
85
- catch (error) {
86
- logger.debug('Error processing RAG pipeline:', error);
87
- throw error;
88
- }
89
- }
90
- }
91
- //# sourceMappingURL=pipeline.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAEtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAqB,MAAM,yBAAyB,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAoB,MAAM,gBAAgB,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAc3C,MAAM,OAAO,WAAW;IACd,cAAc,CAAiB;IAC/B,gBAAgB,CAAmB;IACnC,gBAAgB,CAAmB;IACnC,iBAAiB,CAAoB;IACrC,iBAAiB,CAAoB;IACrC,cAAc,CAAiB;IAC/B,KAAK,CAAW;IAChB,KAAK,CAAgB;IACrB,UAAU,CAAqB;IAEvC,YAAY,MAAiB;QAC3B,iCAAiC;QACjC,IAAI,CAAC,UAAU,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAE5D,qBAAqB;QACrB,IAAI,CAAC,KAAK,GAAG,IAAI,aAAa,CAC5B,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,YAAY,EACnB,IAAI,CAAC,UAAU,EACf,MAAM,CAAC,YAAY,CACpB,CAAC;QAEF,4BAA4B;QAC5B,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/E,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1E,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAC/C,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACpE,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACpE,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,EAAE,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAAa;QACzB,oBAAoB;QACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;QACzD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO;gBACL,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,UAAU,EAAE;oBACV,SAAS,EAAE,IAAI;oBACf,SAAS,EAAE,IAAI;oBACf,gBAAgB,EAAE,IAAI;oBACtB,OAAO,EAAE;wBACP,gBAAgB,EAAE,EAAE;wBACpB,gBAAgB,EAAE,EAAE;wBACpB,uBAAuB,EAAE,EAAE;qBAC5B;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,qCAAqC;YACrC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;YAEtD,4BAA4B;YAC5B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,WAAW,EAAE;gBACzE,KAAK,EAAE,EAAE;gBACT,QAAQ,EAAE,GAAG;gBACb,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEnE,mBAAmB;YACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAChF,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAErD,0BAA0B;YAC1B,MAAM,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAChE,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAExC,oBAAoB;YACpB,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YAChG,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YAEnC,oBAAoB;YACpB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;YAC/F,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,UAAU,CAAC,CAAC;YAEhD,6BAA6B;YAC7B,IAAI,UAAU,CAAC,SAAS,IAAI,UAAU,CAAC,SAAS,IAAI,UAAU,CAAC,gBAAgB,EAAE,CAAC;gBAChF,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;YAC3D,CAAC;YAED,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,CAAC;QACrD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
@@ -1,14 +0,0 @@
1
- import { EmbeddingsProvider } from "../embeddings/types.js";
2
- export type QueryIntentType = 'overview' | 'api' | 'example' | 'usage' | 'component_usage' | 'concept' | 'troubleshooting' | 'general';
3
- export interface QueryIntent {
4
- embedding: number[];
5
- intent: QueryIntentType;
6
- entities: string[];
7
- confidence: number;
8
- }
9
- export declare class QueryProcessor {
10
- private openai;
11
- private embeddings;
12
- constructor(apiKey: string, embeddings: EmbeddingsProvider);
13
- processQuery(query: string): Promise<QueryIntent>;
14
- }
@@ -1,57 +0,0 @@
1
- import OpenAI from "openai";
2
- export class QueryProcessor {
3
- openai;
4
- embeddings;
5
- constructor(apiKey, embeddings) {
6
- this.openai = new OpenAI({ apiKey });
7
- this.embeddings = embeddings;
8
- }
9
- async processQuery(query) {
10
- // Generate embedding for semantic search
11
- const embedding = await this.embeddings.embed(query);
12
- // Classify intent using OpenAI
13
- const completion = await this.openai.chat.completions.create({
14
- model: "gpt-3.5-turbo",
15
- messages: [
16
- {
17
- role: "system",
18
- content: `You are a query intent classifier for a documentation search system.
19
-
20
- Your task is to analyze the query and return a JSON response with the following fields:
21
- {
22
- "intent": "one of: overview, api, example, usage, component_usage, concept, troubleshooting, general",
23
- "entities": ["array of extracted entities like component names, function names, concepts"],
24
- "confidence": "number between 0 and 1"
25
- }
26
-
27
- Intent categories:
28
- - overview: General overview or introduction to a topic
29
- - api: API documentation and reference
30
- - example: Code examples and demonstrations
31
- - usage: How to use features or components
32
- - component_usage: Specific component usage (alternative to usage)
33
- - concept: Conceptual explanations and principles
34
- - troubleshooting: Error resolution and debugging
35
- - general: General queries that don't fit other categories`
36
- },
37
- {
38
- role: "user",
39
- content: query
40
- }
41
- ],
42
- response_format: { type: "json_object" }
43
- });
44
- const content = completion.choices[0].message.content;
45
- if (!content) {
46
- throw new Error("No content returned from OpenAI");
47
- }
48
- const result = JSON.parse(content);
49
- return {
50
- embedding,
51
- intent: result.intent,
52
- entities: result.entities,
53
- confidence: result.confidence
54
- };
55
- }
56
- }
57
- //# sourceMappingURL=query-processor.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"query-processor.js","sourceRoot":"","sources":["../../src/rag/query-processor.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAmB5B,MAAM,OAAO,cAAc;IACjB,MAAM,CAAS;IACf,UAAU,CAAqB;IAEvC,YAAY,MAAc,EAAE,UAA8B;QACxD,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa;QAC9B,yCAAyC;QACzC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAErD,+BAA+B;QAC/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YAC3D,KAAK,EAAE,eAAe;YACtB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE;;;;;;;;;;;;;;;;;qEAiBkD;iBAC5D;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,KAAK;iBACf;aACF;YACD,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;SACzC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACtD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACrD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,OAAO;YACL,SAAS;YACT,MAAM,EAAE,MAAM,CAAC,MAAyB;YACxC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B,CAAC;IACJ,CAAC;CACF"}
@@ -1,55 +0,0 @@
1
- import { SearchResult } from "../types.js";
2
- /**
3
- * Reranker interface for document search improvement
4
- * Responsible for reranking search results to improve relevance
5
- */
6
- export interface Reranker {
7
- /**
8
- * Rerank chunks based on relevance to the query
9
- * @param query - The user query
10
- * @param chunks - The chunks to rerank
11
- * @param options - Optional reranking settings
12
- * @returns An array of relevance scores corresponding to each chunk
13
- */
14
- rerank(query: string, chunks: SearchResult[], options?: RerankerOptions): Promise<number[]>;
15
- }
16
- /**
17
- * Options for configuring reranker behavior
18
- */
19
- export interface RerankerOptions {
20
- componentNames?: string[];
21
- propNames?: string[];
22
- componentBoost?: number;
23
- propBoost?: number;
24
- codeBoost?: number;
25
- semanticWeight?: number;
26
- exactMatchWeight?: number;
27
- }
28
- /**
29
- * Enhanced simple reranker implementation with smarter heuristics
30
- * Uses multiple relevance signals and weighted scoring
31
- */
32
- export declare class EnhancedSimpleReranker implements Reranker {
33
- readonly _apiKey: string;
34
- constructor(_apiKey: string);
35
- rerank(query: string, chunks: SearchResult[], options?: RerankerOptions): Promise<number[]>;
36
- private extractPhrases;
37
- }
38
- /**
39
- * Improved OpenAI-based reranker that uses embeddings similarity and additional relevance signals
40
- */
41
- export declare class OpenAIReranker implements Reranker {
42
- private openaiApiKey;
43
- constructor(openaiApiKey: string);
44
- rerank(query: string, chunks: SearchResult[], options?: RerankerOptions): Promise<number[]>;
45
- private cosineSimilarity;
46
- }
47
- /**
48
- * Factory for creating rerankers based on configuration
49
- */
50
- export declare class RerankerFactory {
51
- /**
52
- * Create a reranker instance based on the specified model
53
- */
54
- static create(model: string | undefined, apiKey: string): Reranker;
55
- }
@@ -1,210 +0,0 @@
1
- /**
2
- * Enhanced simple reranker implementation with smarter heuristics
3
- * Uses multiple relevance signals and weighted scoring
4
- */
5
- export class EnhancedSimpleReranker {
6
- _apiKey;
7
- constructor(_apiKey) {
8
- this._apiKey = _apiKey;
9
- // Store API key but not used directly in this reranker
10
- void this._apiKey; // No-op to avoid unused variable warning
11
- }
12
- async rerank(query, chunks, options = {}) {
13
- console.debug(`[EnhancedSimpleReranker] Reranking ${chunks.length} chunks`);
14
- // If we have fewer than 2 chunks, just return score 1 for all
15
- if (!chunks || chunks.length <= 1) {
16
- return chunks.map(() => 1);
17
- }
18
- try {
19
- const scores = [];
20
- const { componentNames = [], propNames = [], componentBoost = 0.2, propBoost = 0.15, codeBoost = 0.1, semanticWeight = 0.6, exactMatchWeight = 0.4 } = options;
21
- // Normalize query for matching
22
- const queryLower = query.toLowerCase();
23
- const queryWords = new Set(queryLower.split(/\s+/).filter(w => w.length > 3));
24
- // Check if query is code-related
25
- const isCodeQuery = /\bcode\b|\bexample\b|\bsnippet\b|\bhow to\b|\bimplement\b/i.test(query);
26
- for (const chunk of chunks) {
27
- const contentLower = chunk.content.toLowerCase();
28
- const contentWords = new Set(contentLower.split(/\s+/).filter(w => w.length > 3));
29
- // --- Calculate base text relevance with Jaccard similarity ---
30
- const intersection = new Set([...queryWords].filter(x => contentWords.has(x)));
31
- const union = new Set([...queryWords, ...contentWords]);
32
- let textRelevance = intersection.size / Math.max(union.size, 1);
33
- // --- Calculate exact phrase matches ---
34
- // Split query into 2-3 word phrases and check if they appear in content
35
- const phrases = this.extractPhrases(queryLower);
36
- let phraseMatches = 0;
37
- for (const phrase of phrases) {
38
- if (contentLower.includes(phrase)) {
39
- phraseMatches++;
40
- }
41
- }
42
- const phraseScore = phrases.length > 0 ? phraseMatches / phrases.length : 0;
43
- // --- Apply component name boosting ---
44
- let componentScore = 0;
45
- if (componentNames.length > 0) {
46
- // Check for component name matches (case sensitive as component names are usually PascalCase)
47
- for (const componentName of componentNames) {
48
- const regex = new RegExp(`\\b${componentName}\\b`, 'g');
49
- const matches = (chunk.content.match(regex) || []).length;
50
- if (matches > 0) {
51
- componentScore += (matches * componentBoost);
52
- }
53
- }
54
- }
55
- else {
56
- // If no specific component names, check for any PascalCase words (potential components)
57
- const pascalCaseRegex = /\b[A-Z][a-z]+[A-Z]\w*\b/g;
58
- const componentMatches = chunk.content.match(pascalCaseRegex) || [];
59
- if (componentMatches.length > 0) {
60
- componentScore = componentBoost; // Fixed boost for having any component names
61
- }
62
- }
63
- // --- Apply prop name boosting ---
64
- let propScore = 0;
65
- if (propNames.length > 0) {
66
- for (const propName of propNames) {
67
- const regex = new RegExp(`\\b${propName}\\b`, 'gi');
68
- const matches = (chunk.content.match(regex) || []).length;
69
- if (matches > 0) {
70
- propScore += (matches * propBoost);
71
- }
72
- }
73
- }
74
- // --- Code example boosting ---
75
- let codeScore = 0;
76
- if (isCodeQuery && chunk.metadata.codeBlocks && chunk.metadata.codeBlocks.length > 0) {
77
- codeScore = codeBoost * Math.min(chunk.metadata.codeBlocks.length, 3); // Cap boost for 3+ code blocks
78
- }
79
- // --- Metadata type relevance ---
80
- let metadataScore = 0;
81
- if (chunk.metadata.type === 'api' && /\bapi\b|\bprops\b|\bparameters\b|\boptions\b/i.test(query)) {
82
- metadataScore = 0.15; // Boost API docs for API-related queries
83
- }
84
- else if (chunk.metadata.type === 'example' && isCodeQuery) {
85
- metadataScore = 0.15; // Boost examples for code-related queries
86
- }
87
- // --- Combine all relevance signals with weights ---
88
- // Base score is a weighted combination of text relevance and phrase matching
89
- const baseScore = (textRelevance * exactMatchWeight) + (phraseScore * (1 - exactMatchWeight));
90
- // Apply boosts additively to avoid score explosion
91
- let finalScore = (baseScore * semanticWeight) +
92
- componentScore +
93
- propScore +
94
- codeScore +
95
- metadataScore;
96
- // Ensure score is in 0-1 range, with diminishing returns for extreme boosts
97
- finalScore = Math.min(1, finalScore);
98
- scores.push(finalScore);
99
- // Debug logging for first few chunks to help with debugging
100
- if (scores.length <= 3) {
101
- console.debug(`[EnhancedSimpleReranker] Chunk ${scores.length} score:`, {
102
- finalScore,
103
- baseScore,
104
- textRelevance,
105
- phraseScore,
106
- componentScore,
107
- propScore,
108
- codeScore,
109
- metadataScore
110
- });
111
- }
112
- }
113
- return scores;
114
- }
115
- catch (error) {
116
- console.error('[EnhancedSimpleReranker] Error in reranking:', error);
117
- // Return neutral scores if reranking fails
118
- return chunks.map(() => 0.5);
119
- }
120
- }
121
- // Extract meaningful 2-3 word phrases from query
122
- extractPhrases(text) {
123
- const words = text.split(/\s+/).filter(w => w.length > 2);
124
- const phrases = [];
125
- // Extract 3-word phrases
126
- for (let i = 0; i < words.length - 2; i++) {
127
- phrases.push(`${words[i]} ${words[i + 1]} ${words[i + 2]}`);
128
- }
129
- // Extract 2-word phrases
130
- for (let i = 0; i < words.length - 1; i++) {
131
- phrases.push(`${words[i]} ${words[i + 1]}`);
132
- }
133
- return phrases;
134
- }
135
- }
136
- /**
137
- * Improved OpenAI-based reranker that uses embeddings similarity and additional relevance signals
138
- */
139
- export class OpenAIReranker {
140
- openaiApiKey;
141
- constructor(openaiApiKey) {
142
- this.openaiApiKey = openaiApiKey;
143
- }
144
- async rerank(query, chunks, options = {}) {
145
- console.debug(`[OpenAIReranker] Reranking ${chunks.length} chunks`);
146
- // If we have fewer than 2 chunks, just return score 1 for all
147
- if (!chunks || chunks.length <= 1) {
148
- return chunks.map(() => 1);
149
- }
150
- try {
151
- // Import OpenAI in the method to avoid loading it unnecessarily
152
- const { default: OpenAI } = await import('openai');
153
- const openai = new OpenAI({ apiKey: this.openaiApiKey });
154
- // Get embeddings for the query
155
- const queryEmbedding = await openai.embeddings.create({
156
- model: 'text-embedding-ada-002',
157
- input: query,
158
- });
159
- // Get embeddings for all chunks in a single batch request
160
- const chunkEmbeddings = await openai.embeddings.create({
161
- model: 'text-embedding-ada-002',
162
- input: chunks.map(chunk => chunk.content.substring(0, 8000)),
163
- });
164
- // Calculate cosine similarity between query and each chunk
165
- const semanticScores = chunkEmbeddings.data.map((chunkEmbed) => {
166
- return this.cosineSimilarity(queryEmbedding.data[0].embedding, chunkEmbed.embedding);
167
- });
168
- // For enhanced scoring, also incorporate the simple reranker's heuristics
169
- const simpleReranker = new EnhancedSimpleReranker(this.openaiApiKey);
170
- const heuristicScores = await simpleReranker.rerank(query, chunks, options);
171
- // Combine semantic and heuristic scores (weighted average)
172
- const combinedScores = semanticScores.map((semanticScore, i) => {
173
- const heuristicScore = heuristicScores[i];
174
- // Weight semantic score more heavily (70/30 split)
175
- return (semanticScore * 0.7) + (heuristicScore * 0.3);
176
- });
177
- return combinedScores;
178
- }
179
- catch (error) {
180
- console.error('[OpenAIReranker] Error in OpenAI reranking:', error);
181
- // Fallback to the simpler reranker if OpenAI fails
182
- const simpleReranker = new EnhancedSimpleReranker(this.openaiApiKey);
183
- return simpleReranker.rerank(query, chunks, options);
184
- }
185
- }
186
- // Calculate cosine similarity between two vectors
187
- cosineSimilarity(vecA, vecB) {
188
- const dotProduct = vecA.reduce((sum, a, i) => sum + a * vecB[i], 0);
189
- const magA = Math.sqrt(vecA.reduce((sum, a) => sum + a * a, 0));
190
- const magB = Math.sqrt(vecB.reduce((sum, b) => sum + b * b, 0));
191
- return dotProduct / (magA * magB);
192
- }
193
- }
194
- /**
195
- * Factory for creating rerankers based on configuration
196
- */
197
- export class RerankerFactory {
198
- /**
199
- * Create a reranker instance based on the specified model
200
- */
201
- static create(model = 'openai', apiKey) {
202
- if (model === 'openai') {
203
- return new OpenAIReranker(apiKey);
204
- }
205
- else {
206
- return new EnhancedSimpleReranker(apiKey);
207
- }
208
- }
209
- }
210
- //# sourceMappingURL=reranker.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"reranker.js","sourceRoot":"","sources":["../../src/rag/reranker.ts"],"names":[],"mappings":"AA8BA;;;GAGG;AACH,MAAM,OAAO,sBAAsB;IACZ;IAArB,YAAqB,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;QAClC,uDAAuD;QACvD,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,yCAAyC;IAC9D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,MAAsB,EAAE,UAA2B,EAAE;QAC/E,OAAO,CAAC,KAAK,CAAC,sCAAsC,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;QAE5E,8DAA8D;QAC9D,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,MAAM,EACJ,cAAc,GAAG,EAAE,EACnB,SAAS,GAAG,EAAE,EACd,cAAc,GAAG,GAAG,EACpB,SAAS,GAAG,IAAI,EAChB,SAAS,GAAG,GAAG,EACf,cAAc,GAAG,GAAG,EACpB,gBAAgB,GAAG,GAAG,EACvB,GAAG,OAAO,CAAC;YAEZ,+BAA+B;YAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAE9E,iCAAiC;YACjC,MAAM,WAAW,GAAG,4DAA4D,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE7F,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBACjD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;gBAElF,gEAAgE;gBAChE,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/E,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC;gBACxD,IAAI,aAAa,GAAG,YAAY,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBAEhE,yCAAyC;gBACzC,wEAAwE;gBACxE,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;gBAChD,IAAI,aAAa,GAAG,CAAC,CAAC;gBACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBAClC,aAAa,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;gBACD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE5E,wCAAwC;gBACxC,IAAI,cAAc,GAAG,CAAC,CAAC;gBACvB,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,8FAA8F;oBAC9F,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;wBAC3C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,aAAa,KAAK,EAAE,GAAG,CAAC,CAAC;wBACxD,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;wBAC1D,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;4BAChB,cAAc,IAAI,CAAC,OAAO,GAAG,cAAc,CAAC,CAAC;wBAC/C,CAAC;oBACH,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,wFAAwF;oBACxF,MAAM,eAAe,GAAG,0BAA0B,CAAC;oBACnD,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;oBACpE,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAChC,cAAc,GAAG,cAAc,CAAC,CAAC,6CAA6C;oBAChF,CAAC;gBACH,CAAC;gBAED,mCAAmC;gBACnC,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;wBACjC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,QAAQ,KAAK,EAAE,IAAI,CAAC,CAAC;wBACpD,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;wBAC1D,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;4BAChB,SAAS,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;wBACrC,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,gCAAgC;gBAChC,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,WAAW,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrF,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,+BAA+B;gBACxG,CAAC;gBAED,kCAAkC;gBAClC,IAAI,aAAa,GAAG,CAAC,CAAC;gBACtB,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,KAAK,IAAI,+CAA+C,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBACjG,aAAa,GAAG,IAAI,CAAC,CAAC,yCAAyC;gBACjE,CAAC;qBAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,WAAW,EAAE,CAAC;oBAC5D,aAAa,GAAG,IAAI,CAAC,CAAC,0CAA0C;gBAClE,CAAC;gBAED,qDAAqD;gBACrD,6EAA6E;gBAC7E,MAAM,SAAS,GAAG,CAAC,aAAa,GAAG,gBAAgB,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC;gBAE9F,mDAAmD;gBACnD,IAAI,UAAU,GAAG,CAAC,SAAS,GAAG,cAAc,CAAC;oBAC5B,cAAc;oBACd,SAAS;oBACT,SAAS;oBACT,aAAa,CAAC;gBAE/B,4EAA4E;gBAC5E,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAErC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAExB,4DAA4D;gBAC5D,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBACvB,OAAO,CAAC,KAAK,CAAC,kCAAkC,MAAM,CAAC,MAAM,SAAS,EAAE;wBACtE,UAAU;wBACV,SAAS;wBACT,aAAa;wBACb,WAAW;wBACX,cAAc;wBACd,SAAS;wBACT,SAAS;wBACT,aAAa;qBACd,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;YACrE,2CAA2C;YAC3C,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,iDAAiD;IACzC,cAAc,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,EAAE,CAAC;QAEnB,yBAAyB;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,yBAAyB;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5C,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,cAAc;IACL;IAApB,YAAoB,YAAoB;QAApB,iBAAY,GAAZ,YAAY,CAAQ;IAAG,CAAC;IAE5C,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,MAAsB,EAAE,UAA2B,EAAE;QAC/E,OAAO,CAAC,KAAK,CAAC,8BAA8B,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;QAEpE,8DAA8D;QAC9D,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC;YACH,gEAAgE;YAChE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;YAEzD,+BAA+B;YAC/B,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACpD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;YAEH,0DAA0D;YAC1D,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACrD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;aAC7D,CAAC,CAAC;YAEH,2DAA2D;YAC3D,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;gBAC7D,OAAO,IAAI,CAAC,gBAAgB,CAC1B,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAChC,UAAU,CAAC,SAAS,CACrB,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,0EAA0E;YAC1E,MAAM,cAAc,GAAG,IAAI,sBAAsB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACrE,MAAM,eAAe,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;YAE5E,2DAA2D;YAC3D,MAAM,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,EAAE,EAAE;gBAC7D,MAAM,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBAC1C,mDAAmD;gBACnD,OAAO,CAAC,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;YAEH,OAAO,cAAc,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,KAAK,CAAC,CAAC;YACpE,mDAAmD;YACnD,MAAM,cAAc,GAAG,IAAI,sBAAsB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACrE,OAAO,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,kDAAkD;IAC1C,gBAAgB,CAAC,IAAc,EAAE,IAAc;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,UAAU,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,eAAe;IAC1B;;OAEG;IACH,MAAM,CAAC,MAAM,CAAC,QAAgB,QAAQ,EAAE,MAAc;QACpD,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvB,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,sBAAsB,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;CACF"}
@@ -1,20 +0,0 @@
1
- import { AssembledContext } from "./context-assembler.js";
2
- import { QueryIntent } from "./query-processor.js";
3
- export interface GeneratedResponse {
4
- text: string;
5
- codeExamples?: string[];
6
- metadata: {
7
- sources: string[];
8
- confidence: number;
9
- generatedAt: Date;
10
- };
11
- }
12
- export declare class ResponseGenerator {
13
- private openai;
14
- constructor(apiKey: string);
15
- generateResponse(context: AssembledContext, queryIntent: QueryIntent): Promise<GeneratedResponse>;
16
- private prepareContext;
17
- private getSystemPrompt;
18
- private isRelevantForIntent;
19
- private getSources;
20
- }
@@ -1,101 +0,0 @@
1
- import OpenAI from "openai";
2
- export class ResponseGenerator {
3
- openai;
4
- constructor(apiKey) {
5
- this.openai = new OpenAI({ apiKey });
6
- }
7
- async generateResponse(context, queryIntent) {
8
- // Prepare context for the response
9
- const contextSummary = this.prepareContext(context, queryIntent);
10
- // Generate response using OpenAI
11
- const completion = await this.openai.chat.completions.create({
12
- model: "gpt-3.5-turbo",
13
- messages: [
14
- {
15
- role: "system",
16
- content: `${this.getSystemPrompt(queryIntent.intent)}
17
-
18
- Return your response in JSON format:
19
- {
20
- "response": "your detailed response text",
21
- "codeExamples": ["array of relevant code examples"],
22
- "metadata": {
23
- "relevance": "number between 0 and 1 indicating response relevance",
24
- "sources": ["array of source identifiers used"]
25
- }
26
- }`
27
- },
28
- {
29
- role: "user",
30
- content: `Context:\n${contextSummary}\n\nQuery Intent: ${queryIntent.intent}\nEntities: ${queryIntent.entities.join(", ")}\n\nGenerate a comprehensive response.`
31
- }
32
- ],
33
- temperature: 0.7,
34
- max_tokens: 1000
35
- });
36
- const content = completion.choices[0].message.content;
37
- if (!content) {
38
- throw new Error("No content returned from OpenAI");
39
- }
40
- const result = JSON.parse(content);
41
- return {
42
- text: result.response,
43
- codeExamples: result.codeExamples || [],
44
- metadata: {
45
- sources: this.getSources(context),
46
- confidence: result.metadata.relevance,
47
- generatedAt: new Date()
48
- }
49
- };
50
- }
51
- prepareContext(context, queryIntent) {
52
- const relevantChunks = context.hierarchicalContext
53
- .filter(chunk => this.isRelevantForIntent(chunk.metadata.type, queryIntent.intent))
54
- .map(chunk => ({
55
- content: chunk.content,
56
- type: chunk.metadata.type,
57
- reliability: chunk.metadata.sourceReliability
58
- }))
59
- .sort((a, b) => b.reliability - a.reliability);
60
- return relevantChunks
61
- .map(chunk => `[${chunk.type}]\n${chunk.content}`)
62
- .join('\n\n');
63
- }
64
- getSystemPrompt(intent) {
65
- const basePrompt = "You are a technical documentation assistant. Generate clear, accurate responses based on the provided context.";
66
- const intentPrompts = {
67
- overview: `${basePrompt}\nProvide a comprehensive overview of the topic, covering key features and concepts.`,
68
- api: `${basePrompt}\nProvide detailed API documentation, including parameters, return types, and examples.`,
69
- example: `${basePrompt}\nCreate practical code examples that demonstrate proper usage and common patterns.`,
70
- usage: `${basePrompt}\nFocus on explaining how to use components effectively, including props, configurations, and best practices.`,
71
- component_usage: `${basePrompt}\nFocus on explaining how to use components effectively, including props, configurations, and best practices.`,
72
- concept: `${basePrompt}\nExplain technical concepts clearly, using analogies and examples where appropriate.`,
73
- troubleshooting: `${basePrompt}\nHelp diagnose and solve problems, providing step-by-step solutions and common pitfalls to avoid.`,
74
- general: basePrompt
75
- };
76
- return intentPrompts[intent];
77
- }
78
- isRelevantForIntent(chunkType, intent) {
79
- const relevanceMap = {
80
- overview: ['overview', 'usage', 'api'],
81
- api: ['api', 'overview', 'example'],
82
- example: ['example', 'usage', 'api'],
83
- usage: ['usage', 'overview', 'example'],
84
- component_usage: ['usage', 'overview', 'example'],
85
- concept: ['overview', 'api', 'example'],
86
- troubleshooting: ['example', 'usage', 'api'],
87
- general: ['overview', 'usage', 'example', 'api']
88
- };
89
- return relevanceMap[intent].includes(chunkType);
90
- }
91
- getSources(context) {
92
- const sources = new Set();
93
- for (const chunk of context.hierarchicalContext) {
94
- if (chunk.metadata.citationInfo.version) {
95
- sources.add(`${chunk.metadata.citationInfo.version}`);
96
- }
97
- }
98
- return Array.from(sources);
99
- }
100
- }
101
- //# sourceMappingURL=response-generator.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"response-generator.js","sourceRoot":"","sources":["../../src/rag/response-generator.ts"],"names":[],"mappings":"AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAY5B,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAS;IAEvB,YAAY,MAAc;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAyB,EAAE,WAAwB;QACxE,mCAAmC;QACnC,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAEjE,iCAAiC;QACjC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YAC3D,KAAK,EAAE,eAAe;YACtB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC;;;;;;;;;;YAUlD;iBACH;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,aAAa,cAAc,qBAAqB,WAAW,CAAC,MAAM,eAAe,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,wCAAwC;iBAClK;aACF;YACD,WAAW,EAAE,GAAG;YAChB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACtD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,QAAQ;YACrB,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,EAAE;YACvC,QAAQ,EAAE;gBACR,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;gBACjC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;gBACrC,WAAW,EAAE,IAAI,IAAI,EAAE;aACxB;SACF,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,OAAyB,EAAE,WAAwB;QACxE,MAAM,cAAc,GAAG,OAAO,CAAC,mBAAmB;aAC/C,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;aAClF,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACb,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI;YACzB,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,iBAAiB;SAC9C,CAAC,CAAC;aACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;QAEjD,OAAO,cAAc;aAClB,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;aACjD,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAEO,eAAe,CAAC,MAAuB;QAC7C,MAAM,UAAU,GAAG,gHAAgH,CAAC;QAEpI,MAAM,aAAa,GAAoC;YACrD,QAAQ,EAAE,GAAG,UAAU,sFAAsF;YAC7G,GAAG,EAAE,GAAG,UAAU,yFAAyF;YAC3G,OAAO,EAAE,GAAG,UAAU,qFAAqF;YAC3G,KAAK,EAAE,GAAG,UAAU,+GAA+G;YACnI,eAAe,EAAE,GAAG,UAAU,+GAA+G;YAC7I,OAAO,EAAE,GAAG,UAAU,uFAAuF;YAC7G,eAAe,EAAE,GAAG,UAAU,oGAAoG;YAClI,OAAO,EAAE,UAAU;SACpB,CAAC;QAEF,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,mBAAmB,CAAC,SAAiB,EAAE,MAAuB;QACpE,MAAM,YAAY,GAAsC;YACtD,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC;YACtC,GAAG,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,SAAS,CAAC;YACnC,OAAO,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC;YACpC,KAAK,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC;YACvC,eAAe,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC;YACjD,OAAO,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC;YACvC,eAAe,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC;YAC5C,OAAO,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC;SACjD,CAAC;QAEF,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAClD,CAAC;IAEO,UAAU,CAAC,OAAyB;QAC1C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;YAChD,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;CACF"}
@@ -1,19 +0,0 @@
1
- import { EnhancedChunk } from "../types/rag.js";
2
- import { EmbeddingsProvider } from "../embeddings/types.js";
3
- import { DocumentStore } from "../storage/storage.js";
4
- import { QueryIntent } from "./query-processor.js";
5
- export interface RetrievalResult {
6
- chunks: EnhancedChunk[];
7
- relevanceScores: number[];
8
- }
9
- export interface RetrievalOptions {
10
- limit?: number;
11
- minScore?: number;
12
- filterByIntent?: boolean;
13
- }
14
- export declare class ContextRetriever {
15
- private readonly store;
16
- private readonly embeddings;
17
- constructor(store: DocumentStore, embeddings: EmbeddingsProvider);
18
- retrieveContext(queryIntent: QueryIntent, options?: RetrievalOptions): Promise<RetrievalResult>;
19
- }