@cosmocoder/mcp-web-docs 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -10
- package/build/crawler/auth.d.ts +17 -6
- package/build/crawler/auth.js +166 -31
- package/build/crawler/auth.js.map +1 -1
- package/build/crawler/auth.test.js +197 -33
- package/build/crawler/auth.test.js.map +1 -1
- package/build/index.js +68 -16
- package/build/index.js.map +1 -1
- package/build/index.test.js +134 -0
- package/build/index.test.js.map +1 -1
- package/build/storage/storage.d.ts +17 -0
- package/build/storage/storage.js +102 -5
- package/build/storage/storage.js.map +1 -1
- package/build/storage/storage.test.js +91 -0
- package/build/storage/storage.test.js.map +1 -1
- package/build/types.d.ts +4 -0
- package/build/util/security.js +8 -0
- package/build/util/security.js.map +1 -1
- package/build/util/security.test.js +18 -0
- package/build/util/security.test.js.map +1 -1
- package/package.json +11 -2
- package/build/crawler/cheerio.d.ts +0 -11
- package/build/crawler/cheerio.js +0 -134
- package/build/crawler/cheerio.js.map +0 -1
- package/build/crawler/chromium.d.ts +0 -21
- package/build/crawler/chromium.js +0 -596
- package/build/crawler/chromium.js.map +0 -1
- package/build/crawler/default.d.ts +0 -11
- package/build/crawler/default.js +0 -138
- package/build/crawler/default.js.map +0 -1
- package/build/crawler/factory.d.ts +0 -6
- package/build/crawler/factory.js +0 -83
- package/build/crawler/factory.js.map +0 -1
- package/build/crawler/puppeteer.d.ts +0 -16
- package/build/crawler/puppeteer.js +0 -191
- package/build/crawler/puppeteer.js.map +0 -1
- package/build/embeddings/openai.d.ts +0 -8
- package/build/embeddings/openai.js +0 -56
- package/build/embeddings/openai.js.map +0 -1
- package/build/rag/cache.d.ts +0 -10
- package/build/rag/cache.js +0 -10
- package/build/rag/cache.js.map +0 -1
- package/build/rag/code-generator.d.ts +0 -11
- package/build/rag/code-generator.js +0 -30
- package/build/rag/code-generator.js.map +0 -1
- package/build/rag/context-assembler.d.ts +0 -23
- package/build/rag/context-assembler.js +0 -113
- package/build/rag/context-assembler.js.map +0 -1
- package/build/rag/docs-search.d.ts +0 -55
- package/build/rag/docs-search.js +0 -380
- package/build/rag/docs-search.js.map +0 -1
- package/build/rag/pipeline.d.ts +0 -26
- package/build/rag/pipeline.js +0 -91
- package/build/rag/pipeline.js.map +0 -1
- package/build/rag/query-processor.d.ts +0 -14
- package/build/rag/query-processor.js +0 -57
- package/build/rag/query-processor.js.map +0 -1
- package/build/rag/reranker.d.ts +0 -55
- package/build/rag/reranker.js +0 -210
- package/build/rag/reranker.js.map +0 -1
- package/build/rag/response-generator.d.ts +0 -20
- package/build/rag/response-generator.js +0 -101
- package/build/rag/response-generator.js.map +0 -1
- package/build/rag/retriever.d.ts +0 -19
- package/build/rag/retriever.js +0 -111
- package/build/rag/retriever.js.map +0 -1
- package/build/rag/validator.d.ts +0 -22
- package/build/rag/validator.js +0 -128
- package/build/rag/validator.js.map +0 -1
- package/build/rag/version-manager.d.ts +0 -23
- package/build/rag/version-manager.js +0 -98
- package/build/rag/version-manager.js.map +0 -1
- package/build/types/rag.d.ts +0 -27
- package/build/types/rag.js +0 -2
- package/build/types/rag.js.map +0 -1
- package/build/util/content-utils.d.ts +0 -31
- package/build/util/content-utils.js +0 -120
- package/build/util/content-utils.js.map +0 -1
- package/build/util/content.d.ts +0 -1
- package/build/util/content.js +0 -16
- package/build/util/content.js.map +0 -1
- package/build/util/site-detector.d.ts +0 -22
- package/build/util/site-detector.js +0 -42
- package/build/util/site-detector.js.map +0 -1
package/build/rag/pipeline.js
DELETED
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
import { QueryProcessor } from "./query-processor.js";
|
|
2
|
-
import { DocumentStore } from "../storage/storage.js";
|
|
3
|
-
import { OpenAIEmbeddings } from "../embeddings/openai.js";
|
|
4
|
-
import { ContextRetriever } from "./retriever.js";
|
|
5
|
-
import { ContextAssembler } from "./context-assembler.js";
|
|
6
|
-
import { ResponseGenerator } from "./response-generator.js";
|
|
7
|
-
import { ResponseValidator } from "./validator.js";
|
|
8
|
-
import { VersionManager } from "./version-manager.js";
|
|
9
|
-
import { RAGCache } from "./cache.js";
|
|
10
|
-
import { logger } from "../util/logger.js";
|
|
11
|
-
export class RAGPipeline {
|
|
12
|
-
queryProcessor;
|
|
13
|
-
contextRetriever;
|
|
14
|
-
contextAssembler;
|
|
15
|
-
responseGenerator;
|
|
16
|
-
responseValidator;
|
|
17
|
-
versionManager;
|
|
18
|
-
cache;
|
|
19
|
-
store;
|
|
20
|
-
embeddings;
|
|
21
|
-
constructor(config) {
|
|
22
|
-
// Initialize embeddings provider
|
|
23
|
-
this.embeddings = new OpenAIEmbeddings(config.openaiApiKey);
|
|
24
|
-
// Initialize storage
|
|
25
|
-
this.store = new DocumentStore(config.dbPath, config.vectorDbPath, this.embeddings, config.maxCacheSize);
|
|
26
|
-
// Initialize RAG components
|
|
27
|
-
this.queryProcessor = new QueryProcessor(config.openaiApiKey, this.embeddings);
|
|
28
|
-
this.contextRetriever = new ContextRetriever(this.store, this.embeddings);
|
|
29
|
-
this.contextAssembler = new ContextAssembler();
|
|
30
|
-
this.responseGenerator = new ResponseGenerator(config.openaiApiKey);
|
|
31
|
-
this.responseValidator = new ResponseValidator(config.openaiApiKey);
|
|
32
|
-
this.versionManager = new VersionManager();
|
|
33
|
-
this.cache = new RAGCache();
|
|
34
|
-
}
|
|
35
|
-
async initialize() {
|
|
36
|
-
await this.store.initialize();
|
|
37
|
-
}
|
|
38
|
-
async process(query) {
|
|
39
|
-
// Check cache first
|
|
40
|
-
const cached = await this.cache.getCachedResponse(query);
|
|
41
|
-
if (cached) {
|
|
42
|
-
return {
|
|
43
|
-
response: cached.response,
|
|
44
|
-
validation: {
|
|
45
|
-
factCheck: true,
|
|
46
|
-
codeCheck: true,
|
|
47
|
-
consistencyCheck: true,
|
|
48
|
-
details: {
|
|
49
|
-
factCheckDetails: [],
|
|
50
|
-
codeCheckDetails: [],
|
|
51
|
-
consistencyCheckDetails: []
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
try {
|
|
57
|
-
// Process query and determine intent
|
|
58
|
-
const queryIntent = await this.queryProcessor.processQuery(query);
|
|
59
|
-
logger.debug('Query intent classified:', queryIntent);
|
|
60
|
-
// Retrieve relevant context
|
|
61
|
-
const retrieval = await this.contextRetriever.retrieveContext(queryIntent, {
|
|
62
|
-
limit: 10,
|
|
63
|
-
minScore: 0.7,
|
|
64
|
-
filterByIntent: true
|
|
65
|
-
});
|
|
66
|
-
logger.debug('Retrieved context chunks:', retrieval.chunks.length);
|
|
67
|
-
// Assemble context
|
|
68
|
-
const assembled = await this.contextAssembler.assembleContext(retrieval.chunks);
|
|
69
|
-
logger.debug('Assembled context with relationships');
|
|
70
|
-
// Apply version awareness
|
|
71
|
-
await this.versionManager.getVersionedContext(assembled, "1.0");
|
|
72
|
-
logger.debug('Applied version context');
|
|
73
|
-
// Generate response
|
|
74
|
-
const generatedResponse = await this.responseGenerator.generateResponse(assembled, queryIntent);
|
|
75
|
-
logger.debug('Generated response');
|
|
76
|
-
// Validate response
|
|
77
|
-
const validation = await this.responseValidator.validateResponse(generatedResponse, assembled);
|
|
78
|
-
logger.debug('Validated response:', validation);
|
|
79
|
-
// Cache if validation passed
|
|
80
|
-
if (validation.factCheck && validation.codeCheck && validation.consistencyCheck) {
|
|
81
|
-
await this.cache.cacheResponse(query, generatedResponse);
|
|
82
|
-
}
|
|
83
|
-
return { response: generatedResponse, validation };
|
|
84
|
-
}
|
|
85
|
-
catch (error) {
|
|
86
|
-
logger.debug('Error processing RAG pipeline:', error);
|
|
87
|
-
throw error;
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
//# sourceMappingURL=pipeline.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAEtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAqB,MAAM,yBAAyB,CAAC;AAC/E,OAAO,EAAE,iBAAiB,EAAoB,MAAM,gBAAgB,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAc3C,MAAM,OAAO,WAAW;IACd,cAAc,CAAiB;IAC/B,gBAAgB,CAAmB;IACnC,gBAAgB,CAAmB;IACnC,iBAAiB,CAAoB;IACrC,iBAAiB,CAAoB;IACrC,cAAc,CAAiB;IAC/B,KAAK,CAAW;IAChB,KAAK,CAAgB;IACrB,UAAU,CAAqB;IAEvC,YAAY,MAAiB;QAC3B,iCAAiC;QACjC,IAAI,CAAC,UAAU,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAE5D,qBAAqB;QACrB,IAAI,CAAC,KAAK,GAAG,IAAI,aAAa,CAC5B,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,YAAY,EACnB,IAAI,CAAC,UAAU,EACf,MAAM,CAAC,YAAY,CACpB,CAAC;QAEF,4BAA4B;QAC5B,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/E,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC1E,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAC/C,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACpE,IAAI,CAAC,iBAAiB,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACpE,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,EAAE,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAAa;QACzB,oBAAoB;QACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;QACzD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO;gBACL,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,UAAU,EAAE;oBACV,SAAS,EAAE,IAAI;oBACf,SAAS,EAAE,IAAI;oBACf,gBAAgB,EAAE,IAAI;oBACtB,OAAO,EAAE;wBACP,gBAAgB,EAAE,EAAE;wBACpB,gBAAgB,EAAE,EAAE;wBACpB,uBAAuB,EAAE,EAAE;qBAC5B;iBACF;aACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,qCAAqC;YACrC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;YAClE,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;YAEtD,4BAA4B;YAC5B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,WAAW,EAAE;gBACzE,KAAK,EAAE,EAAE;gBACT,QAAQ,EAAE,GAAG;gBACb,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAEnE,mBAAmB;YACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAChF,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAErD,0BAA0B;YAC1B,MAAM,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YAChE,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAExC,oBAAoB;YACpB,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YAChG,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YAEnC,oBAAoB;YACpB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,SAAS,CAAC,CAAC;YAC/F,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,UAAU,CAAC,CAAC;YAEhD,6BAA6B;YAC7B,IAAI,UAAU,CAAC,SAAS,IAAI,UAAU,CAAC,SAAS,IAAI,UAAU,CAAC,gBAAgB,EAAE,CAAC;gBAChF,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;YAC3D,CAAC;YAED,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,CAAC;QACrD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;YACtD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { EmbeddingsProvider } from "../embeddings/types.js";
|
|
2
|
-
export type QueryIntentType = 'overview' | 'api' | 'example' | 'usage' | 'component_usage' | 'concept' | 'troubleshooting' | 'general';
|
|
3
|
-
export interface QueryIntent {
|
|
4
|
-
embedding: number[];
|
|
5
|
-
intent: QueryIntentType;
|
|
6
|
-
entities: string[];
|
|
7
|
-
confidence: number;
|
|
8
|
-
}
|
|
9
|
-
export declare class QueryProcessor {
|
|
10
|
-
private openai;
|
|
11
|
-
private embeddings;
|
|
12
|
-
constructor(apiKey: string, embeddings: EmbeddingsProvider);
|
|
13
|
-
processQuery(query: string): Promise<QueryIntent>;
|
|
14
|
-
}
|
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
import OpenAI from "openai";
|
|
2
|
-
export class QueryProcessor {
|
|
3
|
-
openai;
|
|
4
|
-
embeddings;
|
|
5
|
-
constructor(apiKey, embeddings) {
|
|
6
|
-
this.openai = new OpenAI({ apiKey });
|
|
7
|
-
this.embeddings = embeddings;
|
|
8
|
-
}
|
|
9
|
-
async processQuery(query) {
|
|
10
|
-
// Generate embedding for semantic search
|
|
11
|
-
const embedding = await this.embeddings.embed(query);
|
|
12
|
-
// Classify intent using OpenAI
|
|
13
|
-
const completion = await this.openai.chat.completions.create({
|
|
14
|
-
model: "gpt-3.5-turbo",
|
|
15
|
-
messages: [
|
|
16
|
-
{
|
|
17
|
-
role: "system",
|
|
18
|
-
content: `You are a query intent classifier for a documentation search system.
|
|
19
|
-
|
|
20
|
-
Your task is to analyze the query and return a JSON response with the following fields:
|
|
21
|
-
{
|
|
22
|
-
"intent": "one of: overview, api, example, usage, component_usage, concept, troubleshooting, general",
|
|
23
|
-
"entities": ["array of extracted entities like component names, function names, concepts"],
|
|
24
|
-
"confidence": "number between 0 and 1"
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
Intent categories:
|
|
28
|
-
- overview: General overview or introduction to a topic
|
|
29
|
-
- api: API documentation and reference
|
|
30
|
-
- example: Code examples and demonstrations
|
|
31
|
-
- usage: How to use features or components
|
|
32
|
-
- component_usage: Specific component usage (alternative to usage)
|
|
33
|
-
- concept: Conceptual explanations and principles
|
|
34
|
-
- troubleshooting: Error resolution and debugging
|
|
35
|
-
- general: General queries that don't fit other categories`
|
|
36
|
-
},
|
|
37
|
-
{
|
|
38
|
-
role: "user",
|
|
39
|
-
content: query
|
|
40
|
-
}
|
|
41
|
-
],
|
|
42
|
-
response_format: { type: "json_object" }
|
|
43
|
-
});
|
|
44
|
-
const content = completion.choices[0].message.content;
|
|
45
|
-
if (!content) {
|
|
46
|
-
throw new Error("No content returned from OpenAI");
|
|
47
|
-
}
|
|
48
|
-
const result = JSON.parse(content);
|
|
49
|
-
return {
|
|
50
|
-
embedding,
|
|
51
|
-
intent: result.intent,
|
|
52
|
-
entities: result.entities,
|
|
53
|
-
confidence: result.confidence
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
//# sourceMappingURL=query-processor.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"query-processor.js","sourceRoot":"","sources":["../../src/rag/query-processor.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAmB5B,MAAM,OAAO,cAAc;IACjB,MAAM,CAAS;IACf,UAAU,CAAqB;IAEvC,YAAY,MAAc,EAAE,UAA8B;QACxD,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAa;QAC9B,yCAAyC;QACzC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAErD,+BAA+B;QAC/B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YAC3D,KAAK,EAAE,eAAe;YACtB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE;;;;;;;;;;;;;;;;;qEAiBkD;iBAC5D;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,KAAK;iBACf;aACF;YACD,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;SACzC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACtD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACrD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,OAAO;YACL,SAAS;YACT,MAAM,EAAE,MAAM,CAAC,MAAyB;YACxC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B,CAAC;IACJ,CAAC;CACF"}
|
package/build/rag/reranker.d.ts
DELETED
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
import { SearchResult } from "../types.js";
|
|
2
|
-
/**
|
|
3
|
-
* Reranker interface for document search improvement
|
|
4
|
-
* Responsible for reranking search results to improve relevance
|
|
5
|
-
*/
|
|
6
|
-
export interface Reranker {
|
|
7
|
-
/**
|
|
8
|
-
* Rerank chunks based on relevance to the query
|
|
9
|
-
* @param query - The user query
|
|
10
|
-
* @param chunks - The chunks to rerank
|
|
11
|
-
* @param options - Optional reranking settings
|
|
12
|
-
* @returns An array of relevance scores corresponding to each chunk
|
|
13
|
-
*/
|
|
14
|
-
rerank(query: string, chunks: SearchResult[], options?: RerankerOptions): Promise<number[]>;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Options for configuring reranker behavior
|
|
18
|
-
*/
|
|
19
|
-
export interface RerankerOptions {
|
|
20
|
-
componentNames?: string[];
|
|
21
|
-
propNames?: string[];
|
|
22
|
-
componentBoost?: number;
|
|
23
|
-
propBoost?: number;
|
|
24
|
-
codeBoost?: number;
|
|
25
|
-
semanticWeight?: number;
|
|
26
|
-
exactMatchWeight?: number;
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* Enhanced simple reranker implementation with smarter heuristics
|
|
30
|
-
* Uses multiple relevance signals and weighted scoring
|
|
31
|
-
*/
|
|
32
|
-
export declare class EnhancedSimpleReranker implements Reranker {
|
|
33
|
-
readonly _apiKey: string;
|
|
34
|
-
constructor(_apiKey: string);
|
|
35
|
-
rerank(query: string, chunks: SearchResult[], options?: RerankerOptions): Promise<number[]>;
|
|
36
|
-
private extractPhrases;
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* Improved OpenAI-based reranker that uses embeddings similarity and additional relevance signals
|
|
40
|
-
*/
|
|
41
|
-
export declare class OpenAIReranker implements Reranker {
|
|
42
|
-
private openaiApiKey;
|
|
43
|
-
constructor(openaiApiKey: string);
|
|
44
|
-
rerank(query: string, chunks: SearchResult[], options?: RerankerOptions): Promise<number[]>;
|
|
45
|
-
private cosineSimilarity;
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* Factory for creating rerankers based on configuration
|
|
49
|
-
*/
|
|
50
|
-
export declare class RerankerFactory {
|
|
51
|
-
/**
|
|
52
|
-
* Create a reranker instance based on the specified model
|
|
53
|
-
*/
|
|
54
|
-
static create(model: string | undefined, apiKey: string): Reranker;
|
|
55
|
-
}
|
package/build/rag/reranker.js
DELETED
|
@@ -1,210 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Enhanced simple reranker implementation with smarter heuristics
|
|
3
|
-
* Uses multiple relevance signals and weighted scoring
|
|
4
|
-
*/
|
|
5
|
-
export class EnhancedSimpleReranker {
|
|
6
|
-
_apiKey;
|
|
7
|
-
constructor(_apiKey) {
|
|
8
|
-
this._apiKey = _apiKey;
|
|
9
|
-
// Store API key but not used directly in this reranker
|
|
10
|
-
void this._apiKey; // No-op to avoid unused variable warning
|
|
11
|
-
}
|
|
12
|
-
async rerank(query, chunks, options = {}) {
|
|
13
|
-
console.debug(`[EnhancedSimpleReranker] Reranking ${chunks.length} chunks`);
|
|
14
|
-
// If we have fewer than 2 chunks, just return score 1 for all
|
|
15
|
-
if (!chunks || chunks.length <= 1) {
|
|
16
|
-
return chunks.map(() => 1);
|
|
17
|
-
}
|
|
18
|
-
try {
|
|
19
|
-
const scores = [];
|
|
20
|
-
const { componentNames = [], propNames = [], componentBoost = 0.2, propBoost = 0.15, codeBoost = 0.1, semanticWeight = 0.6, exactMatchWeight = 0.4 } = options;
|
|
21
|
-
// Normalize query for matching
|
|
22
|
-
const queryLower = query.toLowerCase();
|
|
23
|
-
const queryWords = new Set(queryLower.split(/\s+/).filter(w => w.length > 3));
|
|
24
|
-
// Check if query is code-related
|
|
25
|
-
const isCodeQuery = /\bcode\b|\bexample\b|\bsnippet\b|\bhow to\b|\bimplement\b/i.test(query);
|
|
26
|
-
for (const chunk of chunks) {
|
|
27
|
-
const contentLower = chunk.content.toLowerCase();
|
|
28
|
-
const contentWords = new Set(contentLower.split(/\s+/).filter(w => w.length > 3));
|
|
29
|
-
// --- Calculate base text relevance with Jaccard similarity ---
|
|
30
|
-
const intersection = new Set([...queryWords].filter(x => contentWords.has(x)));
|
|
31
|
-
const union = new Set([...queryWords, ...contentWords]);
|
|
32
|
-
let textRelevance = intersection.size / Math.max(union.size, 1);
|
|
33
|
-
// --- Calculate exact phrase matches ---
|
|
34
|
-
// Split query into 2-3 word phrases and check if they appear in content
|
|
35
|
-
const phrases = this.extractPhrases(queryLower);
|
|
36
|
-
let phraseMatches = 0;
|
|
37
|
-
for (const phrase of phrases) {
|
|
38
|
-
if (contentLower.includes(phrase)) {
|
|
39
|
-
phraseMatches++;
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
const phraseScore = phrases.length > 0 ? phraseMatches / phrases.length : 0;
|
|
43
|
-
// --- Apply component name boosting ---
|
|
44
|
-
let componentScore = 0;
|
|
45
|
-
if (componentNames.length > 0) {
|
|
46
|
-
// Check for component name matches (case sensitive as component names are usually PascalCase)
|
|
47
|
-
for (const componentName of componentNames) {
|
|
48
|
-
const regex = new RegExp(`\\b${componentName}\\b`, 'g');
|
|
49
|
-
const matches = (chunk.content.match(regex) || []).length;
|
|
50
|
-
if (matches > 0) {
|
|
51
|
-
componentScore += (matches * componentBoost);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
else {
|
|
56
|
-
// If no specific component names, check for any PascalCase words (potential components)
|
|
57
|
-
const pascalCaseRegex = /\b[A-Z][a-z]+[A-Z]\w*\b/g;
|
|
58
|
-
const componentMatches = chunk.content.match(pascalCaseRegex) || [];
|
|
59
|
-
if (componentMatches.length > 0) {
|
|
60
|
-
componentScore = componentBoost; // Fixed boost for having any component names
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
// --- Apply prop name boosting ---
|
|
64
|
-
let propScore = 0;
|
|
65
|
-
if (propNames.length > 0) {
|
|
66
|
-
for (const propName of propNames) {
|
|
67
|
-
const regex = new RegExp(`\\b${propName}\\b`, 'gi');
|
|
68
|
-
const matches = (chunk.content.match(regex) || []).length;
|
|
69
|
-
if (matches > 0) {
|
|
70
|
-
propScore += (matches * propBoost);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
// --- Code example boosting ---
|
|
75
|
-
let codeScore = 0;
|
|
76
|
-
if (isCodeQuery && chunk.metadata.codeBlocks && chunk.metadata.codeBlocks.length > 0) {
|
|
77
|
-
codeScore = codeBoost * Math.min(chunk.metadata.codeBlocks.length, 3); // Cap boost for 3+ code blocks
|
|
78
|
-
}
|
|
79
|
-
// --- Metadata type relevance ---
|
|
80
|
-
let metadataScore = 0;
|
|
81
|
-
if (chunk.metadata.type === 'api' && /\bapi\b|\bprops\b|\bparameters\b|\boptions\b/i.test(query)) {
|
|
82
|
-
metadataScore = 0.15; // Boost API docs for API-related queries
|
|
83
|
-
}
|
|
84
|
-
else if (chunk.metadata.type === 'example' && isCodeQuery) {
|
|
85
|
-
metadataScore = 0.15; // Boost examples for code-related queries
|
|
86
|
-
}
|
|
87
|
-
// --- Combine all relevance signals with weights ---
|
|
88
|
-
// Base score is a weighted combination of text relevance and phrase matching
|
|
89
|
-
const baseScore = (textRelevance * exactMatchWeight) + (phraseScore * (1 - exactMatchWeight));
|
|
90
|
-
// Apply boosts additively to avoid score explosion
|
|
91
|
-
let finalScore = (baseScore * semanticWeight) +
|
|
92
|
-
componentScore +
|
|
93
|
-
propScore +
|
|
94
|
-
codeScore +
|
|
95
|
-
metadataScore;
|
|
96
|
-
// Ensure score is in 0-1 range, with diminishing returns for extreme boosts
|
|
97
|
-
finalScore = Math.min(1, finalScore);
|
|
98
|
-
scores.push(finalScore);
|
|
99
|
-
// Debug logging for first few chunks to help with debugging
|
|
100
|
-
if (scores.length <= 3) {
|
|
101
|
-
console.debug(`[EnhancedSimpleReranker] Chunk ${scores.length} score:`, {
|
|
102
|
-
finalScore,
|
|
103
|
-
baseScore,
|
|
104
|
-
textRelevance,
|
|
105
|
-
phraseScore,
|
|
106
|
-
componentScore,
|
|
107
|
-
propScore,
|
|
108
|
-
codeScore,
|
|
109
|
-
metadataScore
|
|
110
|
-
});
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
return scores;
|
|
114
|
-
}
|
|
115
|
-
catch (error) {
|
|
116
|
-
console.error('[EnhancedSimpleReranker] Error in reranking:', error);
|
|
117
|
-
// Return neutral scores if reranking fails
|
|
118
|
-
return chunks.map(() => 0.5);
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
// Extract meaningful 2-3 word phrases from query
|
|
122
|
-
extractPhrases(text) {
|
|
123
|
-
const words = text.split(/\s+/).filter(w => w.length > 2);
|
|
124
|
-
const phrases = [];
|
|
125
|
-
// Extract 3-word phrases
|
|
126
|
-
for (let i = 0; i < words.length - 2; i++) {
|
|
127
|
-
phrases.push(`${words[i]} ${words[i + 1]} ${words[i + 2]}`);
|
|
128
|
-
}
|
|
129
|
-
// Extract 2-word phrases
|
|
130
|
-
for (let i = 0; i < words.length - 1; i++) {
|
|
131
|
-
phrases.push(`${words[i]} ${words[i + 1]}`);
|
|
132
|
-
}
|
|
133
|
-
return phrases;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
/**
|
|
137
|
-
* Improved OpenAI-based reranker that uses embeddings similarity and additional relevance signals
|
|
138
|
-
*/
|
|
139
|
-
export class OpenAIReranker {
|
|
140
|
-
openaiApiKey;
|
|
141
|
-
constructor(openaiApiKey) {
|
|
142
|
-
this.openaiApiKey = openaiApiKey;
|
|
143
|
-
}
|
|
144
|
-
async rerank(query, chunks, options = {}) {
|
|
145
|
-
console.debug(`[OpenAIReranker] Reranking ${chunks.length} chunks`);
|
|
146
|
-
// If we have fewer than 2 chunks, just return score 1 for all
|
|
147
|
-
if (!chunks || chunks.length <= 1) {
|
|
148
|
-
return chunks.map(() => 1);
|
|
149
|
-
}
|
|
150
|
-
try {
|
|
151
|
-
// Import OpenAI in the method to avoid loading it unnecessarily
|
|
152
|
-
const { default: OpenAI } = await import('openai');
|
|
153
|
-
const openai = new OpenAI({ apiKey: this.openaiApiKey });
|
|
154
|
-
// Get embeddings for the query
|
|
155
|
-
const queryEmbedding = await openai.embeddings.create({
|
|
156
|
-
model: 'text-embedding-ada-002',
|
|
157
|
-
input: query,
|
|
158
|
-
});
|
|
159
|
-
// Get embeddings for all chunks in a single batch request
|
|
160
|
-
const chunkEmbeddings = await openai.embeddings.create({
|
|
161
|
-
model: 'text-embedding-ada-002',
|
|
162
|
-
input: chunks.map(chunk => chunk.content.substring(0, 8000)),
|
|
163
|
-
});
|
|
164
|
-
// Calculate cosine similarity between query and each chunk
|
|
165
|
-
const semanticScores = chunkEmbeddings.data.map((chunkEmbed) => {
|
|
166
|
-
return this.cosineSimilarity(queryEmbedding.data[0].embedding, chunkEmbed.embedding);
|
|
167
|
-
});
|
|
168
|
-
// For enhanced scoring, also incorporate the simple reranker's heuristics
|
|
169
|
-
const simpleReranker = new EnhancedSimpleReranker(this.openaiApiKey);
|
|
170
|
-
const heuristicScores = await simpleReranker.rerank(query, chunks, options);
|
|
171
|
-
// Combine semantic and heuristic scores (weighted average)
|
|
172
|
-
const combinedScores = semanticScores.map((semanticScore, i) => {
|
|
173
|
-
const heuristicScore = heuristicScores[i];
|
|
174
|
-
// Weight semantic score more heavily (70/30 split)
|
|
175
|
-
return (semanticScore * 0.7) + (heuristicScore * 0.3);
|
|
176
|
-
});
|
|
177
|
-
return combinedScores;
|
|
178
|
-
}
|
|
179
|
-
catch (error) {
|
|
180
|
-
console.error('[OpenAIReranker] Error in OpenAI reranking:', error);
|
|
181
|
-
// Fallback to the simpler reranker if OpenAI fails
|
|
182
|
-
const simpleReranker = new EnhancedSimpleReranker(this.openaiApiKey);
|
|
183
|
-
return simpleReranker.rerank(query, chunks, options);
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
// Calculate cosine similarity between two vectors
|
|
187
|
-
cosineSimilarity(vecA, vecB) {
|
|
188
|
-
const dotProduct = vecA.reduce((sum, a, i) => sum + a * vecB[i], 0);
|
|
189
|
-
const magA = Math.sqrt(vecA.reduce((sum, a) => sum + a * a, 0));
|
|
190
|
-
const magB = Math.sqrt(vecB.reduce((sum, b) => sum + b * b, 0));
|
|
191
|
-
return dotProduct / (magA * magB);
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
/**
|
|
195
|
-
* Factory for creating rerankers based on configuration
|
|
196
|
-
*/
|
|
197
|
-
export class RerankerFactory {
|
|
198
|
-
/**
|
|
199
|
-
* Create a reranker instance based on the specified model
|
|
200
|
-
*/
|
|
201
|
-
static create(model = 'openai', apiKey) {
|
|
202
|
-
if (model === 'openai') {
|
|
203
|
-
return new OpenAIReranker(apiKey);
|
|
204
|
-
}
|
|
205
|
-
else {
|
|
206
|
-
return new EnhancedSimpleReranker(apiKey);
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
//# sourceMappingURL=reranker.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"reranker.js","sourceRoot":"","sources":["../../src/rag/reranker.ts"],"names":[],"mappings":"AA8BA;;;GAGG;AACH,MAAM,OAAO,sBAAsB;IACZ;IAArB,YAAqB,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;QAClC,uDAAuD;QACvD,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,yCAAyC;IAC9D,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,MAAsB,EAAE,UAA2B,EAAE;QAC/E,OAAO,CAAC,KAAK,CAAC,sCAAsC,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;QAE5E,8DAA8D;QAC9D,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,MAAM,EACJ,cAAc,GAAG,EAAE,EACnB,SAAS,GAAG,EAAE,EACd,cAAc,GAAG,GAAG,EACpB,SAAS,GAAG,IAAI,EAChB,SAAS,GAAG,GAAG,EACf,cAAc,GAAG,GAAG,EACpB,gBAAgB,GAAG,GAAG,EACvB,GAAG,OAAO,CAAC;YAEZ,+BAA+B;YAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAE9E,iCAAiC;YACjC,MAAM,WAAW,GAAG,4DAA4D,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE7F,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBACjD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;gBAElF,gEAAgE;gBAChE,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/E,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC;gBACxD,IAAI,aAAa,GAAG,YAAY,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBAEhE,yCAAyC;gBACzC,wEAAwE;gBACxE,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;gBAChD,IAAI,aAAa,GAAG,CAAC,CAAC;gBACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;wBAClC,aAAa,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;gBACD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE5E,wCAAwC;gBACxC,IAAI,cAAc,GAAG,CAAC,CAAC;gBACvB,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,8FAA8F;oBAC9F,KAAK,MAAM,aAAa,IAAI,cAAc,EAAE,CAAC;wBAC3C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,aAAa,KAAK,EAAE,GAAG,CAAC,CAAC;wBACxD,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;wBAC1D,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;4BAChB,cAAc,IAAI,CAAC,OAAO,GAAG,cAAc,CAAC,CAAC;wBAC/C,CAAC;oBACH,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,wFAAwF;oBACxF,MAAM,eAAe,GAAG,0BAA0B,CAAC;oBACnD,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC;oBACpE,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAChC,cAAc,GAAG,cAAc,CAAC,CAAC,6CAA6C;oBAChF,CAAC;gBACH,CAAC;gBAED,mCAAmC;gBACnC,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;wBACjC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,QAAQ,KAAK,EAAE,IAAI,CAAC,CAAC;wBACpD,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;wBAC1D,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;4BAChB,SAAS,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC;wBACrC,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,gCAAgC;gBAChC,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,WAAW,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrF,SAAS,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,+BAA+B;gBACxG,CAAC;gBAED,kCAAkC;gBAClC,IAAI,aAAa,GAAG,CAAC,CAAC;gBACtB,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,KAAK,IAAI,+CAA+C,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBACjG,aAAa,GAAG,IAAI,CAAC,CAAC,yCAAyC;gBACjE,CAAC;qBAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,WAAW,EAAE,CAAC;oBAC5D,aAAa,GAAG,IAAI,CAAC,CAAC,0CAA0C;gBAClE,CAAC;gBAED,qDAAqD;gBACrD,6EAA6E;gBAC7E,MAAM,SAAS,GAAG,CAAC,aAAa,GAAG,gBAAgB,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC,CAAC;gBAE9F,mDAAmD;gBACnD,IAAI,UAAU,GAAG,CAAC,SAAS,GAAG,cAAc,CAAC;oBAC5B,cAAc;oBACd,SAAS;oBACT,SAAS;oBACT,aAAa,CAAC;gBAE/B,4EAA4E;gBAC5E,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAErC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAExB,4DAA4D;gBAC5D,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBACvB,OAAO,CAAC,KAAK,CAAC,kCAAkC,MAAM,CAAC,MAAM,SAAS,EAAE;wBACtE,UAAU;wBACV,SAAS;wBACT,aAAa;wBACb,WAAW;wBACX,cAAc;wBACd,SAAS;wBACT,SAAS;wBACT,aAAa;qBACd,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;YACrE,2CAA2C;YAC3C,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,iDAAiD;IACzC,cAAc,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,EAAE,CAAC;QAEnB,yBAAyB;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,yBAAyB;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,GAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5C,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,cAAc;IACL;IAApB,YAAoB,YAAoB;QAApB,iBAAY,GAAZ,YAAY,CAAQ;IAAG,CAAC;IAE5C,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,MAAsB,EAAE,UAA2B,EAAE;QAC/E,OAAO,CAAC,KAAK,CAAC,8BAA8B,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;QAEpE,8DAA8D;QAC9D,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC;YACH,gEAAgE;YAChE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;YAEzD,+BAA+B;YAC/B,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACpD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;YAEH,0DAA0D;YAC1D,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACrD,KAAK,EAAE,wBAAwB;gBAC/B,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;aAC7D,CAAC,CAAC;YAEH,2DAA2D;YAC3D,MAAM,cAAc,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;gBAC7D,OAAO,IAAI,CAAC,gBAAgB,CAC1B,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAChC,UAAU,CAAC,SAAS,CACrB,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,0EAA0E;YAC1E,MAAM,cAAc,GAAG,IAAI,sBAAsB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACrE,MAAM,eAAe,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;YAE5E,2DAA2D;YAC3D,MAAM,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,EAAE,EAAE;gBAC7D,MAAM,cAAc,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBAC1C,mDAAmD;gBACnD,OAAO,CAAC,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;YAEH,OAAO,cAAc,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,6CAA6C,EAAE,KAAK,CAAC,CAAC;YACpE,mDAAmD;YACnD,MAAM,cAAc,GAAG,IAAI,sBAAsB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACrE,OAAO,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED,kDAAkD;IAC1C,gBAAgB,CAAC,IAAc,EAAE,IAAc;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,OAAO,UAAU,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,eAAe;IAC1B;;OAEG;IACH,MAAM,CAAC,MAAM,CAAC,QAAgB,QAAQ,EAAE,MAAc;QACpD,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YACvB,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,sBAAsB,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import { AssembledContext } from "./context-assembler.js";
|
|
2
|
-
import { QueryIntent } from "./query-processor.js";
|
|
3
|
-
export interface GeneratedResponse {
|
|
4
|
-
text: string;
|
|
5
|
-
codeExamples?: string[];
|
|
6
|
-
metadata: {
|
|
7
|
-
sources: string[];
|
|
8
|
-
confidence: number;
|
|
9
|
-
generatedAt: Date;
|
|
10
|
-
};
|
|
11
|
-
}
|
|
12
|
-
export declare class ResponseGenerator {
|
|
13
|
-
private openai;
|
|
14
|
-
constructor(apiKey: string);
|
|
15
|
-
generateResponse(context: AssembledContext, queryIntent: QueryIntent): Promise<GeneratedResponse>;
|
|
16
|
-
private prepareContext;
|
|
17
|
-
private getSystemPrompt;
|
|
18
|
-
private isRelevantForIntent;
|
|
19
|
-
private getSources;
|
|
20
|
-
}
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
import OpenAI from "openai";
|
|
2
|
-
export class ResponseGenerator {
|
|
3
|
-
openai;
|
|
4
|
-
constructor(apiKey) {
|
|
5
|
-
this.openai = new OpenAI({ apiKey });
|
|
6
|
-
}
|
|
7
|
-
async generateResponse(context, queryIntent) {
|
|
8
|
-
// Prepare context for the response
|
|
9
|
-
const contextSummary = this.prepareContext(context, queryIntent);
|
|
10
|
-
// Generate response using OpenAI
|
|
11
|
-
const completion = await this.openai.chat.completions.create({
|
|
12
|
-
model: "gpt-3.5-turbo",
|
|
13
|
-
messages: [
|
|
14
|
-
{
|
|
15
|
-
role: "system",
|
|
16
|
-
content: `${this.getSystemPrompt(queryIntent.intent)}
|
|
17
|
-
|
|
18
|
-
Return your response in JSON format:
|
|
19
|
-
{
|
|
20
|
-
"response": "your detailed response text",
|
|
21
|
-
"codeExamples": ["array of relevant code examples"],
|
|
22
|
-
"metadata": {
|
|
23
|
-
"relevance": "number between 0 and 1 indicating response relevance",
|
|
24
|
-
"sources": ["array of source identifiers used"]
|
|
25
|
-
}
|
|
26
|
-
}`
|
|
27
|
-
},
|
|
28
|
-
{
|
|
29
|
-
role: "user",
|
|
30
|
-
content: `Context:\n${contextSummary}\n\nQuery Intent: ${queryIntent.intent}\nEntities: ${queryIntent.entities.join(", ")}\n\nGenerate a comprehensive response.`
|
|
31
|
-
}
|
|
32
|
-
],
|
|
33
|
-
temperature: 0.7,
|
|
34
|
-
max_tokens: 1000
|
|
35
|
-
});
|
|
36
|
-
const content = completion.choices[0].message.content;
|
|
37
|
-
if (!content) {
|
|
38
|
-
throw new Error("No content returned from OpenAI");
|
|
39
|
-
}
|
|
40
|
-
const result = JSON.parse(content);
|
|
41
|
-
return {
|
|
42
|
-
text: result.response,
|
|
43
|
-
codeExamples: result.codeExamples || [],
|
|
44
|
-
metadata: {
|
|
45
|
-
sources: this.getSources(context),
|
|
46
|
-
confidence: result.metadata.relevance,
|
|
47
|
-
generatedAt: new Date()
|
|
48
|
-
}
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
prepareContext(context, queryIntent) {
|
|
52
|
-
const relevantChunks = context.hierarchicalContext
|
|
53
|
-
.filter(chunk => this.isRelevantForIntent(chunk.metadata.type, queryIntent.intent))
|
|
54
|
-
.map(chunk => ({
|
|
55
|
-
content: chunk.content,
|
|
56
|
-
type: chunk.metadata.type,
|
|
57
|
-
reliability: chunk.metadata.sourceReliability
|
|
58
|
-
}))
|
|
59
|
-
.sort((a, b) => b.reliability - a.reliability);
|
|
60
|
-
return relevantChunks
|
|
61
|
-
.map(chunk => `[${chunk.type}]\n${chunk.content}`)
|
|
62
|
-
.join('\n\n');
|
|
63
|
-
}
|
|
64
|
-
getSystemPrompt(intent) {
|
|
65
|
-
const basePrompt = "You are a technical documentation assistant. Generate clear, accurate responses based on the provided context.";
|
|
66
|
-
const intentPrompts = {
|
|
67
|
-
overview: `${basePrompt}\nProvide a comprehensive overview of the topic, covering key features and concepts.`,
|
|
68
|
-
api: `${basePrompt}\nProvide detailed API documentation, including parameters, return types, and examples.`,
|
|
69
|
-
example: `${basePrompt}\nCreate practical code examples that demonstrate proper usage and common patterns.`,
|
|
70
|
-
usage: `${basePrompt}\nFocus on explaining how to use components effectively, including props, configurations, and best practices.`,
|
|
71
|
-
component_usage: `${basePrompt}\nFocus on explaining how to use components effectively, including props, configurations, and best practices.`,
|
|
72
|
-
concept: `${basePrompt}\nExplain technical concepts clearly, using analogies and examples where appropriate.`,
|
|
73
|
-
troubleshooting: `${basePrompt}\nHelp diagnose and solve problems, providing step-by-step solutions and common pitfalls to avoid.`,
|
|
74
|
-
general: basePrompt
|
|
75
|
-
};
|
|
76
|
-
return intentPrompts[intent];
|
|
77
|
-
}
|
|
78
|
-
isRelevantForIntent(chunkType, intent) {
|
|
79
|
-
const relevanceMap = {
|
|
80
|
-
overview: ['overview', 'usage', 'api'],
|
|
81
|
-
api: ['api', 'overview', 'example'],
|
|
82
|
-
example: ['example', 'usage', 'api'],
|
|
83
|
-
usage: ['usage', 'overview', 'example'],
|
|
84
|
-
component_usage: ['usage', 'overview', 'example'],
|
|
85
|
-
concept: ['overview', 'api', 'example'],
|
|
86
|
-
troubleshooting: ['example', 'usage', 'api'],
|
|
87
|
-
general: ['overview', 'usage', 'example', 'api']
|
|
88
|
-
};
|
|
89
|
-
return relevanceMap[intent].includes(chunkType);
|
|
90
|
-
}
|
|
91
|
-
getSources(context) {
|
|
92
|
-
const sources = new Set();
|
|
93
|
-
for (const chunk of context.hierarchicalContext) {
|
|
94
|
-
if (chunk.metadata.citationInfo.version) {
|
|
95
|
-
sources.add(`${chunk.metadata.citationInfo.version}`);
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
return Array.from(sources);
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
//# sourceMappingURL=response-generator.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"response-generator.js","sourceRoot":"","sources":["../../src/rag/response-generator.ts"],"names":[],"mappings":"AAEA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAY5B,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAS;IAEvB,YAAY,MAAc;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,OAAyB,EAAE,WAAwB;QACxE,mCAAmC;QACnC,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAEjE,iCAAiC;QACjC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YAC3D,KAAK,EAAE,eAAe;YACtB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC;;;;;;;;;;YAUlD;iBACH;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,aAAa,cAAc,qBAAqB,WAAW,CAAC,MAAM,eAAe,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,wCAAwC;iBAClK;aACF;YACD,WAAW,EAAE,GAAG;YAChB,UAAU,EAAE,IAAI;SACjB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACtD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,QAAQ;YACrB,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,EAAE;YACvC,QAAQ,EAAE;gBACR,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;gBACjC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS;gBACrC,WAAW,EAAE,IAAI,IAAI,EAAE;aACxB;SACF,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,OAAyB,EAAE,WAAwB;QACxE,MAAM,cAAc,GAAG,OAAO,CAAC,mBAAmB;aAC/C,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;aAClF,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACb,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI;YACzB,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,iBAAiB;SAC9C,CAAC,CAAC;aACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;QAEjD,OAAO,cAAc;aAClB,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;aACjD,IAAI,CAAC,MAAM,CAAC,CAAC;IAClB,CAAC;IAEO,eAAe,CAAC,MAAuB;QAC7C,MAAM,UAAU,GAAG,gHAAgH,CAAC;QAEpI,MAAM,aAAa,GAAoC;YACrD,QAAQ,EAAE,GAAG,UAAU,sFAAsF;YAC7G,GAAG,EAAE,GAAG,UAAU,yFAAyF;YAC3G,OAAO,EAAE,GAAG,UAAU,qFAAqF;YAC3G,KAAK,EAAE,GAAG,UAAU,+GAA+G;YACnI,eAAe,EAAE,GAAG,UAAU,+GAA+G;YAC7I,OAAO,EAAE,GAAG,UAAU,uFAAuF;YAC7G,eAAe,EAAE,GAAG,UAAU,oGAAoG;YAClI,OAAO,EAAE,UAAU;SACpB,CAAC;QAEF,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,mBAAmB,CAAC,SAAiB,EAAE,MAAuB;QACpE,MAAM,YAAY,GAAsC;YACtD,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,KAAK,CAAC;YACtC,GAAG,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,SAAS,CAAC;YACnC,OAAO,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC;YACpC,KAAK,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC;YACvC,eAAe,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,SAAS,CAAC;YACjD,OAAO,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC;YACvC,eAAe,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC;YAC5C,OAAO,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC;SACjD,CAAC;QAEF,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAClD,CAAC;IAEO,UAAU,CAAC,OAAyB;QAC1C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;YAChD,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;gBACxC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;CACF"}
|
package/build/rag/retriever.d.ts
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import { EnhancedChunk } from "../types/rag.js";
|
|
2
|
-
import { EmbeddingsProvider } from "../embeddings/types.js";
|
|
3
|
-
import { DocumentStore } from "../storage/storage.js";
|
|
4
|
-
import { QueryIntent } from "./query-processor.js";
|
|
5
|
-
export interface RetrievalResult {
|
|
6
|
-
chunks: EnhancedChunk[];
|
|
7
|
-
relevanceScores: number[];
|
|
8
|
-
}
|
|
9
|
-
export interface RetrievalOptions {
|
|
10
|
-
limit?: number;
|
|
11
|
-
minScore?: number;
|
|
12
|
-
filterByIntent?: boolean;
|
|
13
|
-
}
|
|
14
|
-
export declare class ContextRetriever {
|
|
15
|
-
private readonly store;
|
|
16
|
-
private readonly embeddings;
|
|
17
|
-
constructor(store: DocumentStore, embeddings: EmbeddingsProvider);
|
|
18
|
-
retrieveContext(queryIntent: QueryIntent, options?: RetrievalOptions): Promise<RetrievalResult>;
|
|
19
|
-
}
|