@devflow-tools/knowledge-engine 0.14.4 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bm25.js +1 -1
- package/dist/bm25.js.map +1 -1
- package/dist/context7-fetcher.d.ts +3 -0
- package/dist/context7-fetcher.d.ts.map +1 -1
- package/dist/context7-fetcher.js +109 -8
- package/dist/context7-fetcher.js.map +1 -1
- package/dist/git-fetcher.d.ts +2 -0
- package/dist/git-fetcher.d.ts.map +1 -1
- package/dist/git-fetcher.js +10 -27
- package/dist/git-fetcher.js.map +1 -1
- package/dist/hybrid-search.d.ts +15 -14
- package/dist/hybrid-search.d.ts.map +1 -1
- package/dist/hybrid-search.js +44 -56
- package/dist/hybrid-search.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -3
- package/dist/index.js.map +1 -1
- package/dist/knowledge-engine.d.ts +48 -5
- package/dist/knowledge-engine.d.ts.map +1 -1
- package/dist/knowledge-engine.js +171 -131
- package/dist/knowledge-engine.js.map +1 -1
- package/dist/knowledge-store.d.ts +62 -1
- package/dist/knowledge-store.d.ts.map +1 -1
- package/dist/knowledge-store.js +169 -10
- package/dist/knowledge-store.js.map +1 -1
- package/dist/markdown-converter.d.ts +7 -1
- package/dist/markdown-converter.d.ts.map +1 -1
- package/dist/markdown-converter.js +253 -36
- package/dist/markdown-converter.js.map +1 -1
- package/package.json +5 -5
- package/dist/vector-store.d.ts +0 -12
- package/dist/vector-store.d.ts.map +0 -1
- package/dist/vector-store.js +0 -36
- package/dist/vector-store.js.map +0 -1
package/dist/bm25.js
CHANGED
package/dist/bm25.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../src/bm25.ts"],"names":[],"mappings":"AAAA,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AAef,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,6EAA6E;IAC7E,MAAM,OAAO,GAAG,IAAI;SACjB,WAAW,EAAE;SACb,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE7B,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;IAExB,2DAA2D;IAC3D,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,IAAgB;IAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC;IAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/D,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,YAAY,CAAC,IAAgB;IACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,KAAa,EACb,MAAkB,EAClB,KAAa;IAEb,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhE,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAClE,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEnC,kEAAkE;IAClE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QAChB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;QAC1B,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YAC5C,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,EAAE,GAAG,CAAC,CAAC;YACX,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;gBACpB,IAAI,CAAC,KAAK,IAAI;oBAAE,EAAE,EAAE,CAAC;YACvB,CAAC;YACD,MAAM,SAAS,GAAG,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAChC,MAAM,WAAW,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;YAC7D,OAAO,GAAG,GAAG,GAAG,GAAG,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC;QAC/C,CAAC,EAAE,CAAC,CAAC,CAAC;QAEN,0BAA0B;QAC1B,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAE7E,OAAO;YACL,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,KAAK,CAAC,OAAO
|
|
1
|
+
{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../src/bm25.ts"],"names":[],"mappings":"AAAA,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AAef,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,6EAA6E;IAC7E,MAAM,OAAO,GAAG,IAAI;SACjB,WAAW,EAAE;SACb,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE7B,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;IAExB,2DAA2D;IAC3D,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,IAAgB;IAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC;IAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/D,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,YAAY,CAAC,IAAgB;IACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,KAAa,EACb,MAAkB,EAClB,KAAa;IAEb,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhE,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAClE,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEnC,kEAAkE;IAClE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,OAAO,GAAG,MAAM;SACnB,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QAChB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;QAC1B,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YAC5C,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,EAAE,GAAG,CAAC,CAAC;YACX,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;gBACpB,IAAI,CAAC,KAAK,IAAI;oBAAE,EAAE,EAAE,CAAC;YACvB,CAAC;YACD,MAAM,SAAS,GAAG,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAChC,MAAM,WAAW,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;YAC7D,OAAO,GAAG,GAAG,GAAG,GAAG,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC;QAC/C,CAAC,EAAE,CAAC,CAAC,CAAC;QAEN,0BAA0B;QAC1B,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAE7E,OAAO;YACL,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,GAAG,UAAU;SAC1B,CAAC;IACJ,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;SACxB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAEnB,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -2,7 +2,10 @@ interface Context7Chunk {
|
|
|
2
2
|
title: string;
|
|
3
3
|
content: string;
|
|
4
4
|
sourceUrl: string;
|
|
5
|
+
headingPath?: string;
|
|
6
|
+
codeBlocks?: string;
|
|
5
7
|
}
|
|
8
|
+
export declare function groupContext7Text(text: string, fallbackHeading: string, maxChars?: number): Omit<Context7Chunk, 'sourceUrl'>[];
|
|
6
9
|
export declare function fetchFromContext7(context7Id: string, pluginName: string): Promise<Context7Chunk[]>;
|
|
7
10
|
export {};
|
|
8
11
|
//# sourceMappingURL=context7-fetcher.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context7-fetcher.d.ts","sourceRoot":"","sources":["../src/context7-fetcher.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"context7-fetcher.d.ts","sourceRoot":"","sources":["../src/context7-fetcher.ts"],"names":[],"mappings":"AAIA,UAAU,aAAa;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AA8FD,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,QAAQ,SAAkB,GACzB,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE,CAuCpC;AAED,wBAAsB,iBAAiB,CACrC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GACrC,OAAO,CAAC,aAAa,EAAE,CAAC,CAiC1B"}
|
package/dist/context7-fetcher.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { MAX_CHUNK_CHARS, extractCodeBlocks, sliceByHeadings } from "./markdown-converter.js";
|
|
1
2
|
const CONTEXT7_API = "https://context7.com/api/v2/context";
|
|
2
3
|
const TOPIC_QUERIES = {
|
|
3
4
|
vue: [
|
|
@@ -20,6 +21,110 @@ const TOPIC_QUERIES = {
|
|
|
20
21
|
"server components",
|
|
21
22
|
],
|
|
22
23
|
};
|
|
24
|
+
function contextFenceDelimiter(line) {
|
|
25
|
+
return line.match(/^\s{0,3}(`{3,}|~{3,})/)?.[1];
|
|
26
|
+
}
|
|
27
|
+
function contextParagraphs(text, fallbackHeading) {
|
|
28
|
+
const paragraphs = [];
|
|
29
|
+
const headingStack = [];
|
|
30
|
+
let lines = [];
|
|
31
|
+
let openingFence;
|
|
32
|
+
const flush = () => {
|
|
33
|
+
const content = lines.join('\n').trim();
|
|
34
|
+
lines = [];
|
|
35
|
+
if (content.length < 20)
|
|
36
|
+
return;
|
|
37
|
+
paragraphs.push({
|
|
38
|
+
headingPath: headingStack.join(' > ') || fallbackHeading,
|
|
39
|
+
content,
|
|
40
|
+
});
|
|
41
|
+
};
|
|
42
|
+
for (const line of text.trim().split('\n')) {
|
|
43
|
+
const delimiter = contextFenceDelimiter(line);
|
|
44
|
+
if (delimiter) {
|
|
45
|
+
if (!openingFence)
|
|
46
|
+
openingFence = delimiter;
|
|
47
|
+
else if (delimiter[0] === openingFence[0] && delimiter.length >= openingFence.length) {
|
|
48
|
+
openingFence = undefined;
|
|
49
|
+
}
|
|
50
|
+
lines.push(line);
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
const heading = !openingFence ? line.match(/^(#{1,6})\s+(.+)/) : null;
|
|
54
|
+
if (heading) {
|
|
55
|
+
flush();
|
|
56
|
+
const level = heading[1].length;
|
|
57
|
+
headingStack.splice(level - 1);
|
|
58
|
+
headingStack[level - 1] = heading[2].trim();
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (!openingFence && line.trim().length === 0)
|
|
62
|
+
flush();
|
|
63
|
+
else
|
|
64
|
+
lines.push(line);
|
|
65
|
+
}
|
|
66
|
+
flush();
|
|
67
|
+
return paragraphs;
|
|
68
|
+
}
|
|
69
|
+
function formatContextGroup(group) {
|
|
70
|
+
const paths = [...new Set(group.map(paragraph => paragraph.headingPath))];
|
|
71
|
+
const contentParts = [];
|
|
72
|
+
let currentPath = '';
|
|
73
|
+
for (const paragraph of group) {
|
|
74
|
+
if (paragraph.headingPath !== currentPath) {
|
|
75
|
+
contentParts.push(`## ${paragraph.headingPath}`);
|
|
76
|
+
currentPath = paragraph.headingPath;
|
|
77
|
+
}
|
|
78
|
+
contentParts.push(paragraph.content);
|
|
79
|
+
}
|
|
80
|
+
const content = contentParts.join('\n\n');
|
|
81
|
+
return {
|
|
82
|
+
title: paths.join(' / ').slice(0, 120),
|
|
83
|
+
headingPath: paths.join(' | '),
|
|
84
|
+
content,
|
|
85
|
+
codeBlocks: extractCodeBlocks(content),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
export function groupContext7Text(text, fallbackHeading, maxChars = MAX_CHUNK_CHARS) {
|
|
89
|
+
const paragraphs = contextParagraphs(text, fallbackHeading).flatMap(paragraph => {
|
|
90
|
+
const paragraphLimit = Math.max(1, Math.floor(maxChars / 2) - paragraph.headingPath.length - 8);
|
|
91
|
+
const chunks = sliceByHeadings(paragraph.content, { minChars: 1, maxChars: paragraphLimit });
|
|
92
|
+
return chunks.length > 0
|
|
93
|
+
? chunks.map(chunk => ({ ...paragraph, content: chunk.content }))
|
|
94
|
+
: [paragraph];
|
|
95
|
+
});
|
|
96
|
+
if (paragraphs.length === 0)
|
|
97
|
+
return [];
|
|
98
|
+
const groups = [];
|
|
99
|
+
let current = [];
|
|
100
|
+
let currentLength = 0;
|
|
101
|
+
for (const paragraph of paragraphs) {
|
|
102
|
+
const headingLength = current.at(-1)?.headingPath === paragraph.headingPath
|
|
103
|
+
? 0
|
|
104
|
+
: paragraph.headingPath.length + 5;
|
|
105
|
+
const addedLength = paragraph.content.length + headingLength + (current.length > 0 ? 2 : 0);
|
|
106
|
+
if (current.length > 0 && currentLength + addedLength > maxChars) {
|
|
107
|
+
groups.push(current);
|
|
108
|
+
current = [];
|
|
109
|
+
currentLength = 0;
|
|
110
|
+
}
|
|
111
|
+
current.push(paragraph);
|
|
112
|
+
currentLength += addedLength;
|
|
113
|
+
}
|
|
114
|
+
if (current.length > 0) {
|
|
115
|
+
const previous = groups.at(-1);
|
|
116
|
+
const combinedLength = previous
|
|
117
|
+
? formatContextGroup([...previous, ...current]).content.length
|
|
118
|
+
: Number.POSITIVE_INFINITY;
|
|
119
|
+
if (current.length === 1 && previous && combinedLength <= maxChars) {
|
|
120
|
+
groups[groups.length - 1] = [...previous, ...current];
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
groups.push(current);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return groups.map(formatContextGroup);
|
|
127
|
+
}
|
|
23
128
|
export async function fetchFromContext7(context7Id, pluginName) {
|
|
24
129
|
const topics = TOPIC_QUERIES[pluginName];
|
|
25
130
|
if (!topics || topics.length === 0)
|
|
@@ -35,18 +140,14 @@ export async function fetchFromContext7(context7Id, pluginName) {
|
|
|
35
140
|
const text = await response.text();
|
|
36
141
|
if (!text || text.length < 100)
|
|
37
142
|
continue;
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
const firstLine = para.split('\n')[0].trim();
|
|
42
|
-
const title = firstLine.length > 0 ? firstLine : topic;
|
|
43
|
-
const key = `${pluginName}:${title.slice(0, 60)}`;
|
|
143
|
+
for (const chunk of groupContext7Text(text, topic)) {
|
|
144
|
+
const dedupContent = chunk.content.replace(/^##[^\n]*\n+/, '').replace(/\s+/g, ' ').trim();
|
|
145
|
+
const key = `${pluginName}:${dedupContent.slice(0, 240)}`;
|
|
44
146
|
if (seen.has(key))
|
|
45
147
|
continue;
|
|
46
148
|
seen.add(key);
|
|
47
149
|
results.push({
|
|
48
|
-
|
|
49
|
-
content: para.trim(),
|
|
150
|
+
...chunk,
|
|
50
151
|
sourceUrl: `https://context7.com${context7Id}`,
|
|
51
152
|
});
|
|
52
153
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context7-fetcher.js","sourceRoot":"","sources":["../src/context7-fetcher.ts"],"names":[],"mappings":"AAAA,MAAM,YAAY,GAAG,qCAAqC,CAAC;
|
|
1
|
+
{"version":3,"file":"context7-fetcher.js","sourceRoot":"","sources":["../src/context7-fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE9F,MAAM,YAAY,GAAG,qCAAqC,CAAC;AAU3D,MAAM,aAAa,GAA6B;IAC9C,GAAG,EAAE;QACH,eAAe;QACf,mBAAmB;QACnB,kBAAkB;QAClB,aAAa;QACb,YAAY;QACZ,iBAAiB;QACjB,iBAAiB;QACjB,oBAAoB;QACpB,4BAA4B;QAC5B,OAAO;KACR;IACD,KAAK,EAAE;QACL,qBAAqB;QACrB,qBAAqB;QACrB,kBAAkB;QAClB,aAAa;QACb,mBAAmB;KACpB;CACF,CAAC;AAOF,SAAS,qBAAqB,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,eAAuB;IAC9D,MAAM,UAAU,GAAuB,EAAE,CAAC;IAC1C,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,YAAgC,CAAC;IAErC,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,KAAK,GAAG,EAAE,CAAC;QACX,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;YAAE,OAAO;QAChC,UAAU,CAAC,IAAI,CAAC;YACd,WAAW,EAAE,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,eAAe;YACxD,OAAO;SACR,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,YAAY;gBAAE,YAAY,GAAG,SAAS,CAAC;iBACvC,IAAI,SAAS,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;gBACrF,YAAY,GAAG,SAAS,CAAC;YAC3B,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACtE,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,EAAE,CAAC;YACR,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;YACjC,YAAY,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC/B,YAAY,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;YAC7C,SAAS;QACX,CAAC;QACD,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,EAAE,CAAC;;YAClD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IACD,KAAK,EAAE,CAAC;IACR,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAyB;IACnD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;QAC9B,IAAI,SAAS,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YAC1C,YAAY,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;YACjD,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC;QACtC,CAAC;QACD,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IACD,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QACtC,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;QAC9B,OAAO;QACP,UAAU,EAAE,iBAAiB,CAAC,OAAO,CAAC;KACvC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,IAAY,EACZ,eAAuB,EACvB,QAAQ,GAAG,eAAe;IAE1B,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;QAC9E,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChG,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,CAAC;QAC7F,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC;YACtB,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IACH,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,MAAM,GAAyB,EAAE,CAAC;IACxC,IAAI,OAAO,GAAuB,EAAE,CAAC;IACrC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,KAAK,SAAS,CAAC,WAAW;YACzE,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;QACrC,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,aAAa,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5F,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,aAAa,GAAG,WAAW,GAAG,QAAQ,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,GAAG,EAAE,CAAC;YACb,aAAa,GAAG,CAAC,CAAC;QACpB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxB,aAAa,IAAI,WAAW,CAAC;IAC/B,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,cAAc,GAAG,QAAQ;YAC7B,CAAC,CAAC,kBAAkB,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;YAC9D,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC7B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,IAAI,cAAc,IAAI,QAAQ,EAAE,CAAC;YACnE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC,CAAC;QACxD,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,UAAkB,EAAE,UAAkB;IAEtC,MAAM,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE9C,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,GAAG,YAAY,cAAc,kBAAkB,CAAC,UAAU,CAAC,UAAU,kBAAkB,CAAC,KAAK,CAAC,WAAW,CAAC;YACtH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,SAAS;YAE3B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;gBAAE,SAAS;YAEzC,KAAK,MAAM,KAAK,IAAI,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;gBACnD,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC3F,MAAM,GAAG,GAAG,GAAG,UAAU,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;gBAC1D,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEd,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,KAAK;oBACR,SAAS,EAAE,uBAAuB,UAAU,EAAE;iBAC/C,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/git-fetcher.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"git-fetcher.d.ts","sourceRoot":"","sources":["../src/git-fetcher.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"git-fetcher.d.ts","sourceRoot":"","sources":["../src/git-fetcher.ts"],"names":[],"mappings":"AAKA,UAAU,QAAQ;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAkBD,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,GACxD,OAAO,CAAC,QAAQ,EAAE,CAAC,CAkDrB"}
|
package/dist/git-fetcher.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { execFileSync } from 'child_process';
|
|
2
2
|
import { existsSync, readFileSync, readdirSync, mkdirSync } from 'fs';
|
|
3
3
|
import { join } from 'path';
|
|
4
|
+
import { sliceByHeadings } from './markdown-converter.js';
|
|
4
5
|
function walkFiles(dir, exts) {
|
|
5
6
|
const results = [];
|
|
6
7
|
if (!existsSync(dir))
|
|
@@ -19,28 +20,6 @@ function walkFiles(dir, exts) {
|
|
|
19
20
|
}
|
|
20
21
|
return results;
|
|
21
22
|
}
|
|
22
|
-
function splitByHeadings(markdown) {
|
|
23
|
-
const sections = [];
|
|
24
|
-
const lines = markdown.split('\n');
|
|
25
|
-
let currentHeading = '';
|
|
26
|
-
let currentBody = [];
|
|
27
|
-
for (const line of lines) {
|
|
28
|
-
if (/^##\s/.test(line)) {
|
|
29
|
-
if (currentBody.length > 0) {
|
|
30
|
-
sections.push({ heading: currentHeading || 'Overview', body: currentBody.join('\n').trim() });
|
|
31
|
-
}
|
|
32
|
-
currentHeading = line.replace(/^##\s+/, '').trim();
|
|
33
|
-
currentBody = [];
|
|
34
|
-
}
|
|
35
|
-
else {
|
|
36
|
-
currentBody.push(line);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
if (currentBody.length > 0) {
|
|
40
|
-
sections.push({ heading: currentHeading || 'Overview', body: currentBody.join('\n').trim() });
|
|
41
|
-
}
|
|
42
|
-
return sections;
|
|
43
|
-
}
|
|
44
23
|
export async function fetchFromGitHub(repo, docsPath, tmpDir) {
|
|
45
24
|
const paths = Array.isArray(docsPath) ? docsPath : [docsPath];
|
|
46
25
|
const repoDir = join(tmpDir, repo.replace('/', '_'));
|
|
@@ -75,11 +54,15 @@ export async function fetchFromGitHub(repo, docsPath, tmpDir) {
|
|
|
75
54
|
const content = readFileSync(file, 'utf-8');
|
|
76
55
|
if (content.length < 100)
|
|
77
56
|
continue;
|
|
78
|
-
const
|
|
79
|
-
for (const
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
57
|
+
const chunks = sliceByHeadings(content);
|
|
58
|
+
for (const chunk of chunks) {
|
|
59
|
+
results.push({
|
|
60
|
+
title: chunk.title,
|
|
61
|
+
content: chunk.content,
|
|
62
|
+
path: file.replace(repoDir, ''),
|
|
63
|
+
headingPath: chunk.headingPath,
|
|
64
|
+
codeBlocks: chunk.codeBlocks,
|
|
65
|
+
});
|
|
83
66
|
}
|
|
84
67
|
}
|
|
85
68
|
catch {
|
package/dist/git-fetcher.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"git-fetcher.js","sourceRoot":"","sources":["../src/git-fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AACtE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"git-fetcher.js","sourceRoot":"","sources":["../src/git-fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AACtE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAU1D,SAAS,SAAS,CAAC,GAAW,EAAE,IAAc;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC;IACrC,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc;gBAAE,SAAS;YAC1E,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAC7C,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,IAAY,EAAE,QAA2B,EAAE,MAAc;IAEzD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAErD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,IAAI,CAAC;YACH,YAAY,CAAC,KAAK,EAAE;gBAClB,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,oBAAoB,EAAE,UAAU;gBACzD,sBAAsB,IAAI,MAAM,EAAE,OAAO;aAC1C,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACtC,YAAY,CAAC,KAAK,EAAE,CAAC,iBAAiB,EAAE,KAAK,EAAE,GAAG,KAAK,CAAC,EAAE;gBACxD,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM;aAC5C,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,kCAAkC,IAAI,GAAG,EAAG,GAAa,CAAC,OAAO,CAAC,CAAC;YACjF,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,OAAO,GAAe,EAAE,CAAC;IAE/B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACjC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,SAAS;QACnC,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAC5C,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG;oBAAE,SAAS;gBACnC,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;gBACxC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBAC3B,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;wBACtB,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;wBAC/B,WAAW,EAAE,KAAK,CAAC,WAAW;wBAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;qBAC7B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/hybrid-search.d.ts
CHANGED
|
@@ -1,25 +1,26 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
3
|
-
id: string;
|
|
4
|
-
title: string;
|
|
5
|
-
content: string;
|
|
6
|
-
}
|
|
1
|
+
import type { EmbeddingProvider } from '@devflow-tools/memory-engine';
|
|
2
|
+
import type { KnowledgeChunkRow, KnowledgeStore } from './knowledge-store.js';
|
|
7
3
|
export interface SearchResult {
|
|
8
4
|
id: string;
|
|
9
5
|
title: string;
|
|
10
6
|
content: string;
|
|
11
7
|
score: number;
|
|
12
8
|
}
|
|
9
|
+
export interface KnowledgeHybridSearchResult {
|
|
10
|
+
results: SearchResult[];
|
|
11
|
+
warnings: string[];
|
|
12
|
+
}
|
|
13
13
|
export declare class KnowledgeHybridSearch {
|
|
14
|
-
private
|
|
14
|
+
private store;
|
|
15
15
|
private embedder;
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
constructor(store: KnowledgeStore, embedder: EmbeddingProvider);
|
|
17
|
+
search(query: string, chunks: KnowledgeChunkRow[], bm25Results: Array<{
|
|
18
|
+
id: string;
|
|
19
|
+
rank: number;
|
|
20
|
+
}>, limit: number, sources?: string[]): Promise<SearchResult[]>;
|
|
21
|
+
searchWithStatus(query: string, chunks: KnowledgeChunkRow[], bm25Results: Array<{
|
|
20
22
|
id: string;
|
|
21
|
-
|
|
22
|
-
}>, limit: number): Promise<
|
|
23
|
+
rank: number;
|
|
24
|
+
}>, limit: number, sources?: string[]): Promise<KnowledgeHybridSearchResult>;
|
|
23
25
|
}
|
|
24
|
-
export {};
|
|
25
26
|
//# sourceMappingURL=hybrid-search.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-search.d.ts","sourceRoot":"","sources":["../src/hybrid-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"hybrid-search.d.ts","sourceRoot":"","sources":["../src/hybrid-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE9E,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,2BAA2B;IAC1C,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AA4BD,qBAAa,qBAAqB;IAE9B,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,QAAQ;gBADR,KAAK,EAAE,cAAc,EACrB,QAAQ,EAAE,iBAAiB;IAG/B,MAAM,CACV,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,iBAAiB,EAAE,EAC3B,WAAW,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,EAChD,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,MAAM,EAAE,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;IAIpB,gBAAgB,CACpB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,iBAAiB,EAAE,EAC3B,WAAW,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,EAChD,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,MAAM,EAAE,GACjB,OAAO,CAAC,2BAA2B,CAAC;CAqCxC"}
|
package/dist/hybrid-search.js
CHANGED
|
@@ -1,78 +1,66 @@
|
|
|
1
|
+
const VECTOR_DEGRADED_WARNING = 'Knowledge vector search is not ready; using SQLite BM25 results. Run `devflow doctor` for details.';
|
|
1
2
|
const RRF_K = 60;
|
|
2
|
-
function rrfFusion(
|
|
3
|
-
const
|
|
4
|
-
const
|
|
5
|
-
const
|
|
6
|
-
const vectorWeight = vectorScores.size > 0 ? 0.6 : 0;
|
|
3
|
+
function rrfFusion(bm25Ids, vectorIds) {
|
|
4
|
+
const scores = new Map();
|
|
5
|
+
const bm25Weight = bm25Ids.length > 0 ? 0.4 : 0;
|
|
6
|
+
const vectorWeight = vectorIds.length > 0 ? 0.6 : 0;
|
|
7
7
|
const totalWeight = bm25Weight + vectorWeight;
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
score /= totalWeight;
|
|
18
|
-
fused.set(id, score);
|
|
8
|
+
bm25Ids.forEach((id, index) => {
|
|
9
|
+
scores.set(id, (scores.get(id) ?? 0) + bm25Weight / (RRF_K + index + 1));
|
|
10
|
+
});
|
|
11
|
+
vectorIds.forEach((id, index) => {
|
|
12
|
+
scores.set(id, (scores.get(id) ?? 0) + vectorWeight / (RRF_K + index + 1));
|
|
13
|
+
});
|
|
14
|
+
if (totalWeight > 0) {
|
|
15
|
+
for (const [id, score] of scores)
|
|
16
|
+
scores.set(id, score / totalWeight);
|
|
19
17
|
}
|
|
20
|
-
return
|
|
18
|
+
return scores;
|
|
21
19
|
}
|
|
22
20
|
export class KnowledgeHybridSearch {
|
|
23
|
-
|
|
24
|
-
embedder
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
this.
|
|
21
|
+
store;
|
|
22
|
+
embedder;
|
|
23
|
+
constructor(store, embedder) {
|
|
24
|
+
this.store = store;
|
|
25
|
+
this.embedder = embedder;
|
|
28
26
|
}
|
|
29
|
-
async
|
|
30
|
-
|
|
31
|
-
return;
|
|
32
|
-
try {
|
|
33
|
-
const { pipeline } = await import('@xenova/transformers');
|
|
34
|
-
const pipe = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
|
|
35
|
-
this.embedder = {
|
|
36
|
-
embed: async (text) => {
|
|
37
|
-
const result = await pipe(text, { pooling: 'mean', normalize: true });
|
|
38
|
-
return (Array.isArray(result) ? result[0] : result).data;
|
|
39
|
-
},
|
|
40
|
-
};
|
|
41
|
-
this.embedderReady = true;
|
|
42
|
-
}
|
|
43
|
-
catch {
|
|
44
|
-
// Embedding unavailable — hybrid search degrades to BM25-only
|
|
45
|
-
}
|
|
27
|
+
async search(query, chunks, bm25Results, limit, sources) {
|
|
28
|
+
return (await this.searchWithStatus(query, chunks, bm25Results, limit, sources)).results;
|
|
46
29
|
}
|
|
47
|
-
async
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
30
|
+
async searchWithStatus(query, chunks, bm25Results, limit, sources) {
|
|
31
|
+
let vectorIds = [];
|
|
32
|
+
const warnings = [];
|
|
33
|
+
try {
|
|
34
|
+
const warmed = await this.embedder.warmup();
|
|
35
|
+
if (warmed) {
|
|
53
36
|
const embedding = await this.embedder.embed(query);
|
|
54
|
-
|
|
55
|
-
|
|
37
|
+
vectorIds = this.store
|
|
38
|
+
.searchVector(embedding, limit * 3, sources)
|
|
39
|
+
.map(result => result.id);
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
warnings.push(VECTOR_DEGRADED_WARNING);
|
|
56
43
|
}
|
|
57
|
-
catch { /* vector search failed — fall back to BM25 only */ }
|
|
58
44
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
const
|
|
63
|
-
|
|
45
|
+
catch {
|
|
46
|
+
warnings.push(VECTOR_DEGRADED_WARNING);
|
|
47
|
+
}
|
|
48
|
+
const scores = rrfFusion(bm25Results.map(result => result.id), vectorIds);
|
|
49
|
+
const chunkMap = new Map(chunks.map(chunk => [chunk.id, chunk]));
|
|
50
|
+
const results = [...scores.entries()]
|
|
64
51
|
.map(([id, score]) => {
|
|
65
52
|
const chunk = chunkMap.get(id);
|
|
66
53
|
return {
|
|
67
54
|
id,
|
|
68
|
-
title: chunk?.
|
|
69
|
-
content: chunk?.content
|
|
55
|
+
title: chunk?.heading_path ?? '',
|
|
56
|
+
content: chunk?.content ?? '',
|
|
70
57
|
score: Math.round(score * 10000) / 10000,
|
|
71
58
|
};
|
|
72
59
|
})
|
|
73
|
-
.filter(
|
|
60
|
+
.filter(result => result.content.length > 0)
|
|
74
61
|
.sort((a, b) => b.score - a.score)
|
|
75
62
|
.slice(0, limit);
|
|
63
|
+
return { results, warnings };
|
|
76
64
|
}
|
|
77
65
|
}
|
|
78
66
|
//# sourceMappingURL=hybrid-search.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-search.js","sourceRoot":"","sources":["../src/hybrid-search.ts"],"names":[],"mappings":"AAeA,MAAM,KAAK,GAAG,EAAE,CAAC;AAEjB,SAAS,SAAS,CAChB,
|
|
1
|
+
{"version":3,"file":"hybrid-search.js","sourceRoot":"","sources":["../src/hybrid-search.ts"],"names":[],"mappings":"AAeA,MAAM,uBAAuB,GAC3B,oGAAoG,CAAC;AAEvG,MAAM,KAAK,GAAG,EAAE,CAAC;AAEjB,SAAS,SAAS,CAChB,OAAiB,EACjB,SAAmB;IAEnB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAChD,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,UAAU,GAAG,YAAY,CAAC;IAE9C,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE;QAC5B,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,UAAU,GAAG,CAAC,KAAK,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;IACH,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE;QAC9B,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,GAAG,CAAC,KAAK,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IACH,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,MAAM;YAAE,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,GAAG,WAAW,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,qBAAqB;IAEtB;IACA;IAFV,YACU,KAAqB,EACrB,QAA2B;QAD3B,UAAK,GAAL,KAAK,CAAgB;QACrB,aAAQ,GAAR,QAAQ,CAAmB;IAClC,CAAC;IAEJ,KAAK,CAAC,MAAM,CACV,KAAa,EACb,MAA2B,EAC3B,WAAgD,EAChD,KAAa,EACb,OAAkB;QAElB,OAAO,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3F,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,KAAa,EACb,MAA2B,EAC3B,WAAgD,EAChD,KAAa,EACb,OAAkB;QAElB,IAAI,SAAS,GAAa,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC5C,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACnD,SAAS,GAAG,IAAI,CAAC,KAAK;qBACnB,YAAY,CAAC,SAAS,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,CAAC;qBAC3C,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,MAAM,GAAG,SAAS,CACtB,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,EACpC,SAAS,CACV,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE;YACnB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC/B,OAAO;gBACL,EAAE;gBACF,KAAK,EAAE,KAAK,EAAE,YAAY,IAAI,EAAE;gBAChC,OAAO,EAAE,KAAK,EAAE,OAAO,IAAI,EAAE;gBAC7B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,KAAK;aACzC,CAAC;QACJ,CAAC,CAAC;aACD,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;aACjC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACnB,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
export { KnowledgeEngine } from "./knowledge-engine.js";
|
|
1
|
+
export { KnowledgeEngine, DEFAULT_KNOWLEDGE_TOKEN_BUDGET, estimateKnowledgeTokens, } from "./knowledge-engine.js";
|
|
2
|
+
export type { KnowledgeSearchOptions, KnowledgeSearchStatus, KnowledgeSyncOptions, } from "./knowledge-engine.js";
|
|
2
3
|
export { fetchHtml, fetchHtmlBrowser, fetchHtmlSmart, closeBrowser } from "./fetcher.js";
|
|
3
|
-
export { htmlToMarkdown, sliceByHeadings } from "./markdown-converter.js";
|
|
4
|
+
export { htmlToMarkdown, sliceByHeadings, extractCodeBlocks, MIN_CHUNK_CHARS, MAX_CHUNK_CHARS, } from "./markdown-converter.js";
|
|
5
|
+
export { groupContext7Text } from "./context7-fetcher.js";
|
|
4
6
|
export { bm25Search } from "./bm25.js";
|
|
5
|
-
export {
|
|
7
|
+
export { KnowledgeStore } from "./knowledge-store.js";
|
|
8
|
+
export type { KnowledgeChunkRow, KnowledgeChunkInput, KnowledgeDocInput } from "./knowledge-store.js";
|
|
6
9
|
export { KnowledgeHybridSearch } from "./hybrid-search.js";
|
|
7
10
|
export { BUILTIN_SOURCES } from './types.js';
|
|
8
11
|
export type { KnowledgeSourceConfig } from './types.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,8BAA8B,EAC9B,uBAAuB,GACxB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EACV,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EACL,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,eAAe,GAChB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACtG,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,YAAY,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
export { KnowledgeEngine } from "./knowledge-engine.js";
|
|
1
|
+
export { KnowledgeEngine, DEFAULT_KNOWLEDGE_TOKEN_BUDGET, estimateKnowledgeTokens, } from "./knowledge-engine.js";
|
|
2
2
|
export { fetchHtml, fetchHtmlBrowser, fetchHtmlSmart, closeBrowser } from "./fetcher.js";
|
|
3
|
-
export { htmlToMarkdown, sliceByHeadings } from "./markdown-converter.js";
|
|
3
|
+
export { htmlToMarkdown, sliceByHeadings, extractCodeBlocks, MIN_CHUNK_CHARS, MAX_CHUNK_CHARS, } from "./markdown-converter.js";
|
|
4
|
+
export { groupContext7Text } from "./context7-fetcher.js";
|
|
4
5
|
export { bm25Search } from "./bm25.js";
|
|
5
|
-
export {
|
|
6
|
+
export { KnowledgeStore } from "./knowledge-store.js";
|
|
6
7
|
export { KnowledgeHybridSearch } from "./hybrid-search.js";
|
|
7
8
|
export { BUILTIN_SOURCES } from './types.js';
|
|
8
9
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,8BAA8B,EAC9B,uBAAuB,GACxB,MAAM,uBAAuB,CAAC;AAM/B,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EACL,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,eAAe,GAChB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC"}
|
|
@@ -1,11 +1,26 @@
|
|
|
1
1
|
import type { KnowledgeEntry, KnowledgeSource, SearchOptions, SyncResult, KnowledgeSourceStatus } from "@devflow-tools/sdk";
|
|
2
|
-
import
|
|
2
|
+
import { EmbeddingProvider } from "@devflow-tools/memory-engine";
|
|
3
|
+
export interface KnowledgeSearchOptions extends SearchOptions {
|
|
4
|
+
budgetTokens?: number;
|
|
5
|
+
}
|
|
6
|
+
export interface KnowledgeSearchStatus {
|
|
7
|
+
entries: KnowledgeEntry[];
|
|
8
|
+
warnings: string[];
|
|
9
|
+
budgetTokens: number;
|
|
10
|
+
tokenCount: number;
|
|
11
|
+
truncatedCount: number;
|
|
12
|
+
}
|
|
13
|
+
export interface KnowledgeSyncOptions {
|
|
14
|
+
withHeadings?: boolean;
|
|
15
|
+
deferVectorBackfill?: boolean;
|
|
16
|
+
}
|
|
17
|
+
export declare const DEFAULT_KNOWLEDGE_TOKEN_BUDGET = 3000;
|
|
18
|
+
export declare function estimateKnowledgeTokens(text: string): number;
|
|
3
19
|
export declare class KnowledgeEngine {
|
|
4
20
|
private projectRoot;
|
|
5
|
-
private vectorStore;
|
|
6
21
|
private hybridSearch;
|
|
7
22
|
private store;
|
|
8
|
-
private embedder
|
|
23
|
+
private embedder;
|
|
9
24
|
constructor(opts: {
|
|
10
25
|
projectRoot: string;
|
|
11
26
|
embedder?: EmbeddingProvider;
|
|
@@ -13,9 +28,37 @@ export declare class KnowledgeEngine {
|
|
|
13
28
|
private get knowledgeDir();
|
|
14
29
|
private pluginDir;
|
|
15
30
|
registerSource(source: KnowledgeSource): Promise<void>;
|
|
16
|
-
syncDocs(pluginName?: string): Promise<SyncResult[]>;
|
|
17
|
-
|
|
31
|
+
syncDocs(pluginName?: string, options?: KnowledgeSyncOptions): Promise<SyncResult[]>;
|
|
32
|
+
private clearChunkFiles;
|
|
33
|
+
private replaceDocument;
|
|
34
|
+
search(query: string, options?: KnowledgeSearchOptions): Promise<KnowledgeEntry[]>;
|
|
35
|
+
searchWithStatus(query: string, options?: KnowledgeSearchOptions): Promise<KnowledgeSearchStatus>;
|
|
36
|
+
private packEntries;
|
|
37
|
+
backfillVectors(options?: {
|
|
38
|
+
source?: string;
|
|
39
|
+
batchSize?: number;
|
|
40
|
+
onProgress?: (progress: {
|
|
41
|
+
processed: number;
|
|
42
|
+
remaining: number;
|
|
43
|
+
lastId?: string;
|
|
44
|
+
}) => void;
|
|
45
|
+
}): Promise<{
|
|
46
|
+
processed: number;
|
|
47
|
+
remaining: number;
|
|
48
|
+
}>;
|
|
49
|
+
getVectorCoverage(): {
|
|
50
|
+
chunks: number;
|
|
51
|
+
vectors: number;
|
|
52
|
+
bySource: Array<{
|
|
53
|
+
source: string;
|
|
54
|
+
chunks: number;
|
|
55
|
+
vectors: number;
|
|
56
|
+
}>;
|
|
57
|
+
};
|
|
58
|
+
getScopes(): string[];
|
|
59
|
+
getDistinctSources(): string[];
|
|
18
60
|
lookup(symbol: string, scope?: string): Promise<KnowledgeEntry | null>;
|
|
19
61
|
getSources(): Promise<KnowledgeSourceStatus[]>;
|
|
62
|
+
close(): void;
|
|
20
63
|
}
|
|
21
64
|
//# sourceMappingURL=knowledge-engine.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"knowledge-engine.d.ts","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,UAAU,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"knowledge-engine.d.ts","sourceRoot":"","sources":["../src/knowledge-engine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,UAAU,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAS5H,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAwBjE,MAAM,WAAW,sBAAuB,SAAQ,aAAa;IAC3D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B;AAED,eAAO,MAAM,8BAA8B,OAAO,CAAC;AAEnD,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAG5D;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,YAAY,CAAwB;IAC5C,OAAO,CAAC,KAAK,CAAiB;IAC9B,OAAO,CAAC,QAAQ,CAAoB;gBAExB,IAAI,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,iBAAiB,CAAA;KAAE;IAUvE,OAAO,KAAK,YAAY,GAEvB;IAED,OAAO,CAAC,SAAS;IAIX,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IA8BtD,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IA8M9F,OAAO,CAAC,eAAe;YAMT,eAAe;IAkBvB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAIlF,gBAAgB,CACpB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,qBAAqB,CAAC;IAoDjC,OAAO,CAAC,WAAW;IAuBb,eAAe,CAAC,OAAO,GAAE;QAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,SAAS,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,MAAM,CAAA;SAAE,KAAK,IAAI,CAAC;KACvF,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAmC1D,iBAAiB,IAAI;QACnB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,KAAK,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACtE;IAQD,SAAS,IAAI,MAAM,EAAE;IAIrB,kBAAkB,IAAI,MAAM,EAAE;IAIxB,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAOtE,UAAU,IAAI,OAAO,CAAC,qBAAqB,EAAE,CAAC;IAkCpD,KAAK,IAAI,IAAI;CAGd"}
|