@thoughtflow/codebase 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -3
- package/tsconfig.json +13 -2
- package/dist/contexts/context-decomposer.d.ts +0 -13
- package/dist/contexts/context-decomposer.d.ts.map +0 -1
- package/dist/contexts/context-decomposer.js +0 -97
- package/dist/contexts/context-decomposer.js.map +0 -1
- package/dist/contexts/context-document-manager.d.ts +0 -21
- package/dist/contexts/context-document-manager.d.ts.map +0 -1
- package/dist/contexts/context-document-manager.js +0 -69
- package/dist/contexts/context-document-manager.js.map +0 -1
- package/dist/contexts/context-router.d.ts +0 -13
- package/dist/contexts/context-router.d.ts.map +0 -1
- package/dist/contexts/context-router.js +0 -94
- package/dist/contexts/context-router.js.map +0 -1
- package/dist/index.d.ts +0 -53
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -76
- package/dist/index.js.map +0 -1
- package/dist/libs/codebase/deep-analyzer.d.ts +0 -58
- package/dist/libs/codebase/deep-analyzer.d.ts.map +0 -1
- package/dist/libs/codebase/deep-analyzer.js +0 -239
- package/dist/libs/codebase/deep-analyzer.js.map +0 -1
- package/dist/libs/codebase/graph/dependency-extractor.d.ts +0 -20
- package/dist/libs/codebase/graph/dependency-extractor.d.ts.map +0 -1
- package/dist/libs/codebase/graph/dependency-extractor.js +0 -174
- package/dist/libs/codebase/graph/dependency-extractor.js.map +0 -1
- package/dist/libs/codebase/graph/graph-analysis.d.ts +0 -42
- package/dist/libs/codebase/graph/graph-analysis.d.ts.map +0 -1
- package/dist/libs/codebase/graph/graph-analysis.js +0 -222
- package/dist/libs/codebase/graph/graph-analysis.js.map +0 -1
- package/dist/libs/codebase/graph/graph-visualizer.d.ts +0 -12
- package/dist/libs/codebase/graph/graph-visualizer.d.ts.map +0 -1
- package/dist/libs/codebase/graph/graph-visualizer.js +0 -99
- package/dist/libs/codebase/graph/graph-visualizer.js.map +0 -1
- package/dist/libs/codebase/indexer/batch-processor.d.ts +0 -18
- package/dist/libs/codebase/indexer/batch-processor.d.ts.map +0 -1
- package/dist/libs/codebase/indexer/batch-processor.js +0 -39
- package/dist/libs/codebase/indexer/batch-processor.js.map +0 -1
- package/dist/libs/codebase/indexer/chunker-markdown.d.ts +0 -19
- package/dist/libs/codebase/indexer/chunker-markdown.d.ts.map +0 -1
- package/dist/libs/codebase/indexer/chunker-markdown.js +0 -138
- package/dist/libs/codebase/indexer/chunker-markdown.js.map +0 -1
- package/dist/libs/codebase/indexer/chunker.d.ts +0 -21
- package/dist/libs/codebase/indexer/chunker.d.ts.map +0 -1
- package/dist/libs/codebase/indexer/chunker.js +0 -133
- package/dist/libs/codebase/indexer/chunker.js.map +0 -1
- package/dist/libs/codebase/indexer/explorer.d.ts +0 -29
- package/dist/libs/codebase/indexer/explorer.d.ts.map +0 -1
- package/dist/libs/codebase/indexer/explorer.js +0 -278
- package/dist/libs/codebase/indexer/explorer.js.map +0 -1
- package/dist/libs/codebase/indexer/indexer.d.ts +0 -35
- package/dist/libs/codebase/indexer/indexer.d.ts.map +0 -1
- package/dist/libs/codebase/indexer/indexer.js +0 -172
- package/dist/libs/codebase/indexer/indexer.js.map +0 -1
- package/dist/libs/codebase/knowledge/cross-referencer.d.ts +0 -32
- package/dist/libs/codebase/knowledge/cross-referencer.d.ts.map +0 -1
- package/dist/libs/codebase/knowledge/cross-referencer.js +0 -132
- package/dist/libs/codebase/knowledge/cross-referencer.js.map +0 -1
- package/dist/libs/codebase/knowledge/markdown-extractor.d.ts +0 -34
- package/dist/libs/codebase/knowledge/markdown-extractor.d.ts.map +0 -1
- package/dist/libs/codebase/knowledge/markdown-extractor.js +0 -273
- package/dist/libs/codebase/knowledge/markdown-extractor.js.map +0 -1
- package/dist/libs/codebase/knowledge/summarizer.d.ts +0 -25
- package/dist/libs/codebase/knowledge/summarizer.d.ts.map +0 -1
- package/dist/libs/codebase/knowledge/summarizer.js +0 -61
- package/dist/libs/codebase/knowledge/summarizer.js.map +0 -1
- package/dist/libs/codebase/search/bm25.d.ts +0 -23
- package/dist/libs/codebase/search/bm25.d.ts.map +0 -1
- package/dist/libs/codebase/search/bm25.js +0 -93
- package/dist/libs/codebase/search/bm25.js.map +0 -1
- package/dist/libs/codebase/search/hybrid-search.d.ts +0 -22
- package/dist/libs/codebase/search/hybrid-search.d.ts.map +0 -1
- package/dist/libs/codebase/search/hybrid-search.js +0 -80
- package/dist/libs/codebase/search/hybrid-search.js.map +0 -1
- package/dist/libs/codebase-index.class.d.ts +0 -66
- package/dist/libs/codebase-index.class.d.ts.map +0 -1
- package/dist/libs/codebase-index.class.js +0 -595
- package/dist/libs/codebase-index.class.js.map +0 -1
- package/dist/tools/codebase/codebase-analyze.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-analyze.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-analyze.tool.js +0 -20
- package/dist/tools/codebase/codebase-analyze.tool.js.map +0 -1
- package/dist/tools/codebase/codebase-crossref.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-crossref.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-crossref.tool.js +0 -20
- package/dist/tools/codebase/codebase-crossref.tool.js.map +0 -1
- package/dist/tools/codebase/codebase-flow.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-flow.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-flow.tool.js +0 -20
- package/dist/tools/codebase/codebase-flow.tool.js.map +0 -1
- package/dist/tools/codebase/codebase-graph-circular.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-graph-circular.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-graph-circular.tool.js +0 -17
- package/dist/tools/codebase/codebase-graph-circular.tool.js.map +0 -1
- package/dist/tools/codebase/codebase-graph-query.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-graph-query.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-graph-query.tool.js +0 -21
- package/dist/tools/codebase/codebase-graph-query.tool.js.map +0 -1
- package/dist/tools/codebase/codebase-graph-stats.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-graph-stats.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-graph-stats.tool.js +0 -17
- package/dist/tools/codebase/codebase-graph-stats.tool.js.map +0 -1
- package/dist/tools/codebase/codebase-graph-visualize.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-graph-visualize.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-graph-visualize.tool.js +0 -21
- package/dist/tools/codebase/codebase-graph-visualize.tool.js.map +0 -1
- package/dist/tools/codebase/codebase-impact.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-impact.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-impact.tool.js +0 -21
- package/dist/tools/codebase/codebase-impact.tool.js.map +0 -1
- package/dist/tools/codebase/codebase-index.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-index.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-index.tool.js +0 -21
- package/dist/tools/codebase/codebase-index.tool.js.map +0 -1
- package/dist/tools/codebase/codebase-knowledge.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-knowledge.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-knowledge.tool.js +0 -22
- package/dist/tools/codebase/codebase-knowledge.tool.js.map +0 -1
- package/dist/tools/codebase/codebase-search.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-search.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-search.tool.js +0 -24
- package/dist/tools/codebase/codebase-search.tool.js.map +0 -1
- package/dist/tools/codebase/codebase-status.tool.d.ts +0 -8
- package/dist/tools/codebase/codebase-status.tool.d.ts.map +0 -1
- package/dist/tools/codebase/codebase-status.tool.js +0 -19
- package/dist/tools/codebase/codebase-status.tool.js.map +0 -1
- package/dist/tools/neural/comparative-analysis.tool.d.ts +0 -37
- package/dist/tools/neural/comparative-analysis.tool.d.ts.map +0 -1
- package/dist/tools/neural/comparative-analysis.tool.js +0 -241
- package/dist/tools/neural/comparative-analysis.tool.js.map +0 -1
- package/dist/tools/neural/index.d.ts +0 -12
- package/dist/tools/neural/index.d.ts.map +0 -1
- package/dist/tools/neural/index.js +0 -28
- package/dist/tools/neural/index.js.map +0 -1
- package/dist/tools/neural/neural-deadcode.tool.d.ts +0 -10
- package/dist/tools/neural/neural-deadcode.tool.d.ts.map +0 -1
- package/dist/tools/neural/neural-deadcode.tool.js +0 -133
- package/dist/tools/neural/neural-deadcode.tool.js.map +0 -1
- package/dist/tools/neural/neural-debug.tool.d.ts +0 -11
- package/dist/tools/neural/neural-debug.tool.d.ts.map +0 -1
- package/dist/tools/neural/neural-debug.tool.js +0 -205
- package/dist/tools/neural/neural-debug.tool.js.map +0 -1
- package/dist/tools/neural/neural-explain.tool.d.ts +0 -9
- package/dist/tools/neural/neural-explain.tool.d.ts.map +0 -1
- package/dist/tools/neural/neural-explain.tool.js +0 -100
- package/dist/tools/neural/neural-explain.tool.js.map +0 -1
- package/dist/tools/neural/neural-optimize.tool.d.ts +0 -9
- package/dist/tools/neural/neural-optimize.tool.d.ts.map +0 -1
- package/dist/tools/neural/neural-optimize.tool.js +0 -117
- package/dist/tools/neural/neural-optimize.tool.js.map +0 -1
- package/dist/tools/neural/neural-pattern.tool.d.ts +0 -10
- package/dist/tools/neural/neural-pattern.tool.d.ts.map +0 -1
- package/dist/tools/neural/neural-pattern.tool.js +0 -169
- package/dist/tools/neural/neural-pattern.tool.js.map +0 -1
- package/dist/tools/neural/neural-review.tool.d.ts +0 -11
- package/dist/tools/neural/neural-review.tool.d.ts.map +0 -1
- package/dist/tools/neural/neural-review.tool.js +0 -152
- package/dist/tools/neural/neural-review.tool.js.map +0 -1
- package/dist/tools/neural/neural-security.tool.d.ts +0 -16
- package/dist/tools/neural/neural-security.tool.d.ts.map +0 -1
- package/dist/tools/neural/neural-security.tool.js +0 -256
- package/dist/tools/neural/neural-security.tool.js.map +0 -1
- package/dist/tools/neural/neural-similar.tool.d.ts +0 -9
- package/dist/tools/neural/neural-similar.tool.d.ts.map +0 -1
- package/dist/tools/neural/neural-similar.tool.js +0 -139
- package/dist/tools/neural/neural-similar.tool.js.map +0 -1
- package/dist/tools/neural/neural-smells.tool.d.ts +0 -9
- package/dist/tools/neural/neural-smells.tool.d.ts.map +0 -1
- package/dist/tools/neural/neural-smells.tool.js +0 -123
- package/dist/tools/neural/neural-smells.tool.js.map +0 -1
- package/dist/tools/neural/neural-suggest.tool.d.ts +0 -9
- package/dist/tools/neural/neural-suggest.tool.d.ts.map +0 -1
- package/dist/tools/neural/neural-suggest.tool.js +0 -64
- package/dist/tools/neural/neural-suggest.tool.js.map +0 -1
- package/dist/tools/quality/index.d.ts +0 -8
- package/dist/tools/quality/index.d.ts.map +0 -1
- package/dist/tools/quality/index.js +0 -24
- package/dist/tools/quality/index.js.map +0 -1
- package/dist/tools/quality/quality-complexity.tool.d.ts +0 -11
- package/dist/tools/quality/quality-complexity.tool.d.ts.map +0 -1
- package/dist/tools/quality/quality-complexity.tool.js +0 -152
- package/dist/tools/quality/quality-complexity.tool.js.map +0 -1
- package/dist/tools/quality/quality-coupling.tool.d.ts +0 -9
- package/dist/tools/quality/quality-coupling.tool.d.ts.map +0 -1
- package/dist/tools/quality/quality-coupling.tool.js +0 -51
- package/dist/tools/quality/quality-coupling.tool.js.map +0 -1
- package/dist/tools/quality/quality-coverage.tool.d.ts +0 -9
- package/dist/tools/quality/quality-coverage.tool.d.ts.map +0 -1
- package/dist/tools/quality/quality-coverage.tool.js +0 -104
- package/dist/tools/quality/quality-coverage.tool.js.map +0 -1
- package/dist/tools/quality/quality-deps.tool.d.ts +0 -11
- package/dist/tools/quality/quality-deps.tool.d.ts.map +0 -1
- package/dist/tools/quality/quality-deps.tool.js +0 -119
- package/dist/tools/quality/quality-deps.tool.js.map +0 -1
- package/dist/tools/quality/quality-duplication.tool.d.ts +0 -9
- package/dist/tools/quality/quality-duplication.tool.d.ts.map +0 -1
- package/dist/tools/quality/quality-duplication.tool.js +0 -100
- package/dist/tools/quality/quality-duplication.tool.js.map +0 -1
- package/dist/tools/quality/quality-size.tool.d.ts +0 -9
- package/dist/tools/quality/quality-size.tool.d.ts.map +0 -1
- package/dist/tools/quality/quality-size.tool.js +0 -94
- package/dist/tools/quality/quality-size.tool.js.map +0 -1
- package/dist/tools/quality/quality-types.tool.d.ts +0 -9
- package/dist/tools/quality/quality-types.tool.d.ts.map +0 -1
- package/dist/tools/quality/quality-types.tool.js +0 -95
- package/dist/tools/quality/quality-types.tool.js.map +0 -1
- package/dist/tools/workspace/analyze-project.tool.d.ts +0 -36
- package/dist/tools/workspace/analyze-project.tool.d.ts.map +0 -1
- package/dist/tools/workspace/analyze-project.tool.js +0 -454
- package/dist/tools/workspace/analyze-project.tool.js.map +0 -1
- package/dist/tools/workspace/codebase-research.tool.d.ts +0 -10
- package/dist/tools/workspace/codebase-research.tool.d.ts.map +0 -1
- package/dist/tools/workspace/codebase-research.tool.js +0 -89
- package/dist/tools/workspace/codebase-research.tool.js.map +0 -1
- package/dist/tools/workspace/project-scanner.tool.d.ts +0 -55
- package/dist/tools/workspace/project-scanner.tool.d.ts.map +0 -1
- package/dist/tools/workspace/project-scanner.tool.js +0 -978
- package/dist/tools/workspace/project-scanner.tool.js.map +0 -1
- package/dist/tools/workspace/search-code.tool.d.ts +0 -14
- package/dist/tools/workspace/search-code.tool.d.ts.map +0 -1
- package/dist/tools/workspace/search-code.tool.js +0 -286
- package/dist/tools/workspace/search-code.tool.js.map +0 -1
|
@@ -1,273 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.MarkdownExtractor = void 0;
|
|
37
|
-
const fs = __importStar(require("node:fs"));
|
|
38
|
-
class MarkdownExtractor {
|
|
39
|
-
extract(filePath, content) {
|
|
40
|
-
const text = content ?? this.readFile(filePath);
|
|
41
|
-
const lines = text.split("\n");
|
|
42
|
-
const frontmatter = this.parseFrontmatter(lines);
|
|
43
|
-
const sections = this.parseSections(lines);
|
|
44
|
-
const docType = this.classifyDocType(filePath, frontmatter.data, text);
|
|
45
|
-
const references = this.extractReferences(text);
|
|
46
|
-
const codeRefs = this.extractCodeReferences(text);
|
|
47
|
-
const qualityScore = this.calculateQuality(frontmatter, sections, text, filePath);
|
|
48
|
-
const isStale = this.checkStaleness(frontmatter);
|
|
49
|
-
return {
|
|
50
|
-
id: this.slugify(filePath),
|
|
51
|
-
filePath,
|
|
52
|
-
title: frontmatter.data.title ?? this.extractTitle(lines),
|
|
53
|
-
docType,
|
|
54
|
-
sections,
|
|
55
|
-
frontmatter: frontmatter.data,
|
|
56
|
-
references,
|
|
57
|
-
codeReferences: codeRefs,
|
|
58
|
-
qualityScore,
|
|
59
|
-
isStale,
|
|
60
|
-
};
|
|
61
|
-
}
|
|
62
|
-
extractAll(filePaths, rootPath) {
|
|
63
|
-
return filePaths
|
|
64
|
-
.map((fp) => {
|
|
65
|
-
const fullPath = rootPath + "/" + fp;
|
|
66
|
-
try {
|
|
67
|
-
return this.extract(fullPath);
|
|
68
|
-
}
|
|
69
|
-
catch {
|
|
70
|
-
return null;
|
|
71
|
-
}
|
|
72
|
-
})
|
|
73
|
-
.filter((d) => d !== null);
|
|
74
|
-
}
|
|
75
|
-
readFile(filePath) {
|
|
76
|
-
return fs.readFileSync(filePath, "utf8");
|
|
77
|
-
}
|
|
78
|
-
parseFrontmatter(lines) {
|
|
79
|
-
const data = {};
|
|
80
|
-
let endLine = -1;
|
|
81
|
-
if (lines.length === 0 || lines[0].trim() !== "---") {
|
|
82
|
-
return { data, endLine };
|
|
83
|
-
}
|
|
84
|
-
for (let i = 1; i < lines.length; i++) {
|
|
85
|
-
if (lines[i].trim() === "---") {
|
|
86
|
-
endLine = i;
|
|
87
|
-
break;
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
if (endLine === -1)
|
|
91
|
-
return { data, endLine };
|
|
92
|
-
for (let i = 1; i < endLine; i++) {
|
|
93
|
-
const line = lines[i].trim();
|
|
94
|
-
const colonIdx = line.indexOf(":");
|
|
95
|
-
if (colonIdx === -1)
|
|
96
|
-
continue;
|
|
97
|
-
const key = line.slice(0, colonIdx).trim();
|
|
98
|
-
const value = line.slice(colonIdx + 1).trim();
|
|
99
|
-
if (value.startsWith("[") && value.endsWith("]")) {
|
|
100
|
-
data[key] = value.slice(1, -1).split(",").map((s) => s.trim().replace(/^["']|["']$/g, ""));
|
|
101
|
-
}
|
|
102
|
-
else if (value === "true" || value === "false") {
|
|
103
|
-
data[key] = value === "true";
|
|
104
|
-
}
|
|
105
|
-
else if (/^-?\d+(\.\d+)?$/.test(value)) {
|
|
106
|
-
data[key] = parseFloat(value);
|
|
107
|
-
}
|
|
108
|
-
else {
|
|
109
|
-
data[key] = value.replace(/^["']|["']$/g, "");
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
return { data, endLine };
|
|
113
|
-
}
|
|
114
|
-
parseSections(lines) {
|
|
115
|
-
const sections = [];
|
|
116
|
-
let currentHeading = "";
|
|
117
|
-
let currentLevel = 0;
|
|
118
|
-
let currentLines = [];
|
|
119
|
-
let currentStart = 0;
|
|
120
|
-
for (let i = 0; i < lines.length; i++) {
|
|
121
|
-
const match = lines[i].match(/^(#{1,6})\s+(.+)/);
|
|
122
|
-
if (match) {
|
|
123
|
-
if (currentLines.length > 0) {
|
|
124
|
-
sections.push({
|
|
125
|
-
heading: currentHeading,
|
|
126
|
-
level: currentLevel,
|
|
127
|
-
content: currentLines.join("\n").trim(),
|
|
128
|
-
startLine: currentStart + 1,
|
|
129
|
-
});
|
|
130
|
-
}
|
|
131
|
-
currentHeading = match[2].trim();
|
|
132
|
-
currentLevel = match[1].length;
|
|
133
|
-
currentLines = [];
|
|
134
|
-
currentStart = i + 1;
|
|
135
|
-
}
|
|
136
|
-
else {
|
|
137
|
-
currentLines.push(lines[i]);
|
|
138
|
-
}
|
|
139
|
-
}
|
|
140
|
-
if (currentLines.length > 0 || currentHeading) {
|
|
141
|
-
sections.push({
|
|
142
|
-
heading: currentHeading || "root",
|
|
143
|
-
level: currentLevel,
|
|
144
|
-
content: currentLines.join("\n").trim(),
|
|
145
|
-
startLine: currentStart + 1,
|
|
146
|
-
});
|
|
147
|
-
}
|
|
148
|
-
return sections;
|
|
149
|
-
}
|
|
150
|
-
extractTitle(lines) {
|
|
151
|
-
for (const line of lines) {
|
|
152
|
-
const match = line.match(/^#\s+(.+)/);
|
|
153
|
-
if (match)
|
|
154
|
-
return match[1].trim();
|
|
155
|
-
}
|
|
156
|
-
return "untitled";
|
|
157
|
-
}
|
|
158
|
-
classifyDocType(filePath, frontmatter, content) {
|
|
159
|
-
const fp = filePath.toLowerCase();
|
|
160
|
-
const txt = content.toLowerCase();
|
|
161
|
-
if (fp.includes("architecture") || fp.includes("arch/") || txt.includes("architecture decision"))
|
|
162
|
-
return "architecture";
|
|
163
|
-
if (fp.includes("decision") || fp.includes("adr") || txt.includes("decision record"))
|
|
164
|
-
return "decision";
|
|
165
|
-
if (fp.includes("business") || fp.includes("biznes") || fp.includes("bp-") || fp.includes("business_plan"))
|
|
166
|
-
return "business";
|
|
167
|
-
if (fp.includes("api") || fp.includes("spec") || fp.includes("openapi") || txt.includes("api specification"))
|
|
168
|
-
return "technical";
|
|
169
|
-
if (fp.includes("audit"))
|
|
170
|
-
return "audit";
|
|
171
|
-
if (fp.includes("guideline") || fp.includes("guide") || fp.includes("standard") || fp.includes("convention") || fp.includes("instruction"))
|
|
172
|
-
return "guideline";
|
|
173
|
-
if (fp.includes("readme") || fp.includes("contributing"))
|
|
174
|
-
return "technical";
|
|
175
|
-
const fmType = frontmatter.type;
|
|
176
|
-
if (fmType) {
|
|
177
|
-
const t = fmType.toLowerCase();
|
|
178
|
-
if (["architecture", "decision", "business", "technical", "audit", "guideline"].includes(t)) {
|
|
179
|
-
return t;
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
return "unknown";
|
|
183
|
-
}
|
|
184
|
-
extractReferences(text) {
|
|
185
|
-
const refs = [];
|
|
186
|
-
const linkPattern = /\[([^\]]+)\]\(([^)]+)\)/g;
|
|
187
|
-
let match;
|
|
188
|
-
while ((match = linkPattern.exec(text)) !== null) {
|
|
189
|
-
const target = match[2];
|
|
190
|
-
if (target && !target.startsWith("http") && !target.startsWith("#")) {
|
|
191
|
-
refs.push(target);
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
return [...new Set(refs)];
|
|
195
|
-
}
|
|
196
|
-
extractCodeReferences(text) {
|
|
197
|
-
const refs = [];
|
|
198
|
-
const codeRefPattern = /`([^`]+\.(?:ts|tsx|js|jsx|py|go|rs|java))`/g;
|
|
199
|
-
let match;
|
|
200
|
-
while ((match = codeRefPattern.exec(text)) !== null) {
|
|
201
|
-
refs.push(match[1]);
|
|
202
|
-
}
|
|
203
|
-
const commentRefPattern = /\/\/\s*(?:@feature|@ref|@see|PROJECT-MCP):\s*(\S+)/g;
|
|
204
|
-
while ((match = commentRefPattern.exec(text)) !== null) {
|
|
205
|
-
refs.push(match[1]);
|
|
206
|
-
}
|
|
207
|
-
return [...new Set(refs)];
|
|
208
|
-
}
|
|
209
|
-
calculateQuality(frontmatter, sections, text, filePath) {
|
|
210
|
-
let score = 0;
|
|
211
|
-
if (frontmatter.endLine > -1)
|
|
212
|
-
score += 0.1;
|
|
213
|
-
if (frontmatter.data.title)
|
|
214
|
-
score += 0.1;
|
|
215
|
-
if (frontmatter.data.tags)
|
|
216
|
-
score += 0.05;
|
|
217
|
-
if (frontmatter.data.date || frontmatter.data.updated)
|
|
218
|
-
score += 0.05;
|
|
219
|
-
if (sections.length >= 3)
|
|
220
|
-
score += 0.1;
|
|
221
|
-
if (sections.length >= 5)
|
|
222
|
-
score += 0.05;
|
|
223
|
-
const refs = this.extractReferences(text);
|
|
224
|
-
const codeRefs = this.extractCodeReferences(text);
|
|
225
|
-
if (refs.length > 0)
|
|
226
|
-
score += 0.1;
|
|
227
|
-
if (codeRefs.length > 0)
|
|
228
|
-
score += 0.1;
|
|
229
|
-
const wordCount = text.split(/\s+/).length;
|
|
230
|
-
if (wordCount > 200)
|
|
231
|
-
score += 0.1;
|
|
232
|
-
if (wordCount > 500)
|
|
233
|
-
score += 0.05;
|
|
234
|
-
const autoGenPatterns = [
|
|
235
|
-
/generated by/i, /auto-generated/i, /do not edit/i,
|
|
236
|
-
/this file is automatically/i, /created with/i,
|
|
237
|
-
];
|
|
238
|
-
if (autoGenPatterns.some((p) => p.test(text)))
|
|
239
|
-
score -= 0.2;
|
|
240
|
-
const chatPatterns = [
|
|
241
|
-
/^> /m, /^user:/im, /^assistant:/im, /^system:/im,
|
|
242
|
-
/conversation history/i, /token count/i,
|
|
243
|
-
];
|
|
244
|
-
if (chatPatterns.some((p) => p.test(text)))
|
|
245
|
-
score -= 0.3;
|
|
246
|
-
if (wordCount < 50)
|
|
247
|
-
score -= 0.2;
|
|
248
|
-
return Math.max(0, Math.min(1, score));
|
|
249
|
-
}
|
|
250
|
-
checkStaleness(frontmatter) {
|
|
251
|
-
const dateStr = (frontmatter.data.date ?? frontmatter.data.updated ?? frontmatter.data.lastModified);
|
|
252
|
-
if (!dateStr)
|
|
253
|
-
return false;
|
|
254
|
-
try {
|
|
255
|
-
const date = new Date(dateStr);
|
|
256
|
-
const now = new Date();
|
|
257
|
-
const monthsOld = (now.getTime() - date.getTime()) / (1000 * 60 * 60 * 24 * 30);
|
|
258
|
-
return monthsOld > 6;
|
|
259
|
-
}
|
|
260
|
-
catch {
|
|
261
|
-
return false;
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
slugify(filePath) {
|
|
265
|
-
return filePath
|
|
266
|
-
.replace(/\.md$/, "")
|
|
267
|
-
.replace(/[^a-z0-9]+/gi, "-")
|
|
268
|
-
.replace(/^-+|-+$/g, "")
|
|
269
|
-
.toLowerCase();
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
exports.MarkdownExtractor = MarkdownExtractor;
|
|
273
|
-
//# sourceMappingURL=markdown-extractor.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"markdown-extractor.js","sourceRoot":"","sources":["../../../../src/libs/codebase/knowledge/markdown-extractor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,4CAA8B;AAuB9B,MAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAgB,EAAE,OAAgB;QACxC,MAAM,IAAI,GAAG,OAAO,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACvE,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;QAClF,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QAEjD,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;YAC1B,QAAQ;YACR,KAAK,EAAG,WAAW,CAAC,IAAI,CAAC,KAAgB,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;YACrE,OAAO;YACP,QAAQ;YACR,WAAW,EAAE,WAAW,CAAC,IAAI;YAC7B,UAAU;YACV,cAAc,EAAE,QAAQ;YACxB,YAAY;YACZ,OAAO;SACR,CAAC;IACJ,CAAC;IAED,UAAU,CAAC,SAAmB,EAAE,QAAgB;QAC9C,OAAO,SAAS;aACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACV,MAAM,QAAQ,GAAG,QAAQ,GAAG,GAAG,GAAG,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAA0B,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACvD,CAAC;IAEO,QAAQ,CAAC,QAAgB;QAC/B,OAAO,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC;IAEO,gBAAgB,CAAC,KAAe;QACtC,MAAM,IAAI,GAA4B,EAAE,CAAC;QACzC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;QAEjB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,KAAK,EAAE,CAAC;YACpD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,KAAK,EAAE,CAAC;gBAC9B,OAAO,GAAG,CAAC,CAAC;gBACZ,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,OAAO,KAAK,CAAC,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAE7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,KAAK,CAAC,CAAC;gBAAE,SAAS;YAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9C,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjD,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,CAAC;YAC7F,CAAC;iBAAM,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;gBACjD,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,KAAK,MAAM,CAAC;YAC/B,CAAC;iBAAM,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzC,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;IAEO,aAAa,CAAC,KAAe;QACnC,MAAM,QAAQ,GAAsB,EAAE,CAAC;QACvC,IAAI,cAAc,GAAG,EAAE,CAAC;QACxB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,YAAY,GAAa,EAAE,CAAC;QAChC,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YACjD,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC5B,QAAQ,CAAC,IAAI,CAAC;wBACZ,OAAO,EAAE,cAAc;wBACvB,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;wBACvC,SAAS,EAAE,YAAY,GAAG,CAAC;qBAC5B,CAAC,CAAC;gBACL,CAAC;gBACD,cAAc,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACjC,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gBAC/B,YAAY,GAAG,EAAE,CAAC;gBAClB,YAAY,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC;YAC9C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,cAAc,IAAI,MAAM;gBACjC,KAAK,EAAE,YAAY;gBACnB,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;gBACvC,SAAS,EAAE,YAAY,GAAG,CAAC;aAC5B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,YAAY,CAAC,KAAe;QAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACtC,IAAI,KAAK;gBAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,eAAe,CACrB,QAAgB,EAChB,WAAoC,EACpC,OAAe;QAEf,MAAM,EAAE,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QAElC,IAAI,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,uBAAuB,CAAC;YAAE,OAAO,cAAc,CAAC;QACxH,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC;YAAE,OAAO,UAAU,CAAC;QACxG,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;YAAE,OAAO,UAAU,CAAC;QAC9H,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,mBAAmB,CAAC;YAAE,OAAO,WAAW,CAAC;QACjI,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,OAAO,OAAO,CAAC;QACzC,IAAI,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC;YAAE,OAAO,WAAW,CAAC;QAC/J,IAAI,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;YAAE,OAAO,WAAW,CAAC;QAE7E,MAAM,MAAM,GAAG,WAAW,CAAC,IAA0B,CAAC;QACtD,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5F,OAAO,CAAiC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,iBAAiB,CAAC,IAAY;QACpC,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,WAAW,GAAG,0BAA0B,CAAC;QAC/C,IAAI,KAA6B,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACjD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACpE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5B,CAAC;IAEO,qBAAqB,CAAC,IAAY;QACxC,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,cAAc,GAAG,6CAA6C,CAAC;QACrE,IAAI,KAA6B,CAAC;QAClC,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACpD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;QAED,MAAM,iBAAiB,GAAG,qDAAqD,CAAC;QAChF,OAAO,CAAC,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACvD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5B,CAAC;IAEO,gBAAgB,CACtB,WAA+D,EAC/D,QAA2B,EAC3B,IAAY,EACZ,QAAgB;QAEhB,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,WAAW,CAAC,OAAO,GAAG,CAAC,CAAC;YAAE,KAAK,IAAI,GAAG,CAAC;QAC3C,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK;YAAE,KAAK,IAAI,GAAG,CAAC;QACzC,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI;YAAE,KAAK,IAAI,IAAI,CAAC;QACzC,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO;YAAE,KAAK,IAAI,IAAI,CAAC;QAErE,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC;YAAE,KAAK,IAAI,GAAG,CAAC;QACvC,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC;YAAE,KAAK,IAAI,IAAI,CAAC;QAExC,MAAM,IAAI,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,IAAI,GAAG,CAAC;QAClC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,IAAI,GAAG,CAAC;QAEtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QAC3C,IAAI,SAAS,GAAG,GAAG;YAAE,KAAK,IAAI,GAAG,CAAC;QAClC,IAAI,SAAS,GAAG,GAAG;YAAE,KAAK,IAAI,IAAI,CAAC;QAEnC,MAAM,eAAe,GAAG;YACtB,eAAe,EAAE,iBAAiB,EAAE,cAAc;YAClD,6BAA6B,EAAE,eAAe;SAC/C,CAAC;QACF,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAAE,KAAK,IAAI,GAAG,CAAC;QAE5D,MAAM,YAAY,GAAG;YACnB,MAAM,EAAE,UAAU,EAAE,eAAe,EAAE,YAAY;YACjD,uBAAuB,EAAE,cAAc;SACxC,CAAC;QACF,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAAE,KAAK,IAAI,GAAG,CAAC;QAEzD,IAAI,SAAS,GAAG,EAAE;YAAE,KAAK,IAAI,GAAG,CAAC;QAEjC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,cAAc,CAAC,WAA+D;QACpF,MAAM,OAAO,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,IAAI,WAAW,CAAC,IAAI,CAAC,YAAY,CAAuB,CAAC;QAC3H,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAE3B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YAChF,OAAO,SAAS,GAAG,CAAC,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAEO,OAAO,CAAC,QAAgB;QAC9B,OAAO,QAAQ;aACZ,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;aACpB,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;aAC5B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;aACvB,WAAW,EAAE,CAAC;IACnB,CAAC;CACF;AAtPD,8CAsPC"}
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
export interface SummarizerConfig {
|
|
3
|
-
maxTokensPerChunk?: number;
|
|
4
|
-
concurrency?: number;
|
|
5
|
-
}
|
|
6
|
-
export interface SummarizedChunk {
|
|
7
|
-
chunkId: string;
|
|
8
|
-
summary: string;
|
|
9
|
-
}
|
|
10
|
-
export declare class Summarizer {
|
|
11
|
-
private callLLM;
|
|
12
|
-
private readonly maxTokens;
|
|
13
|
-
private readonly concurrency;
|
|
14
|
-
constructor(callLLM: (prompt: string, schema: z.ZodType) => Promise<Record<string, unknown>>, config?: SummarizerConfig);
|
|
15
|
-
summarizeChunks(chunks: Array<{
|
|
16
|
-
id: string;
|
|
17
|
-
text: string;
|
|
18
|
-
type: string;
|
|
19
|
-
}>): Promise<Map<string, string>>;
|
|
20
|
-
summarizeDocument(title: string, sections: Array<{
|
|
21
|
-
heading: string;
|
|
22
|
-
content: string;
|
|
23
|
-
}>): Promise<string>;
|
|
24
|
-
}
|
|
25
|
-
//# sourceMappingURL=summarizer.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"summarizer.d.ts","sourceRoot":"","sources":["../../../../src/libs/codebase/knowledge/summarizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,WAAW,gBAAgB;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,UAAU;IAKnB,OAAO,CAAC,OAAO;IAJjB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAG3B,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EACxF,MAAM,GAAE,gBAAqB;IAMzB,eAAe,CACnB,MAAM,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,GACxD,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAoCzB,iBAAiB,CACrB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GACpD,OAAO,CAAC,MAAM,CAAC;CAmBnB"}
|
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Summarizer = void 0;
|
|
4
|
-
const zod_1 = require("zod");
|
|
5
|
-
class Summarizer {
|
|
6
|
-
callLLM;
|
|
7
|
-
maxTokens;
|
|
8
|
-
concurrency;
|
|
9
|
-
constructor(callLLM, config = {}) {
|
|
10
|
-
this.callLLM = callLLM;
|
|
11
|
-
this.maxTokens = config.maxTokensPerChunk ?? 200;
|
|
12
|
-
this.concurrency = config.concurrency ?? 3;
|
|
13
|
-
}
|
|
14
|
-
async summarizeChunks(chunks) {
|
|
15
|
-
const summaries = new Map();
|
|
16
|
-
const schema = zod_1.z.object({
|
|
17
|
-
summary: zod_1.z.string().max(this.maxTokens),
|
|
18
|
-
});
|
|
19
|
-
for (let i = 0; i < chunks.length; i += this.concurrency) {
|
|
20
|
-
const batch = chunks.slice(i, i + this.concurrency);
|
|
21
|
-
const results = await Promise.all(batch.map(async (chunk) => {
|
|
22
|
-
const context = chunk.type === "documentation"
|
|
23
|
-
? "Summarize this documentation section in 1-2 sentences. Focus on the key information it conveys."
|
|
24
|
-
: "Summarize this code fragment in 1 sentence. What does it do? Be specific about its purpose.";
|
|
25
|
-
const prompt = `${context}\n\n\`\`\`\n${chunk.text.slice(0, 2000)}\n\`\`\``;
|
|
26
|
-
try {
|
|
27
|
-
const result = await this.callLLM(prompt, schema);
|
|
28
|
-
return { id: chunk.id, summary: result.summary ?? "" };
|
|
29
|
-
}
|
|
30
|
-
catch {
|
|
31
|
-
return { id: chunk.id, summary: "" };
|
|
32
|
-
}
|
|
33
|
-
}));
|
|
34
|
-
for (const result of results) {
|
|
35
|
-
if (result.summary) {
|
|
36
|
-
summaries.set(result.id, result.summary);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
return summaries;
|
|
41
|
-
}
|
|
42
|
-
async summarizeDocument(title, sections) {
|
|
43
|
-
const schema = zod_1.z.object({
|
|
44
|
-
summary: zod_1.z.string().max(500),
|
|
45
|
-
});
|
|
46
|
-
const sectionList = sections
|
|
47
|
-
.slice(0, 10)
|
|
48
|
-
.map((s) => `## ${s.heading}\n${s.content.slice(0, 500)}`)
|
|
49
|
-
.join("\n\n");
|
|
50
|
-
const prompt = `Summarize this document titled "${title}" in 1 paragraph. Cover the main topics and purpose.\n\n${sectionList}`;
|
|
51
|
-
try {
|
|
52
|
-
const result = await this.callLLM(prompt, schema);
|
|
53
|
-
return result.summary ?? "";
|
|
54
|
-
}
|
|
55
|
-
catch {
|
|
56
|
-
return "";
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
exports.Summarizer = Summarizer;
|
|
61
|
-
//# sourceMappingURL=summarizer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"summarizer.js","sourceRoot":"","sources":["../../../../src/libs/codebase/knowledge/summarizer.ts"],"names":[],"mappings":";;;AAAA,6BAAwB;AAYxB,MAAa,UAAU;IAKX;IAJO,SAAS,CAAS;IAClB,WAAW,CAAS;IAErC,YACU,OAAgF,EACxF,SAA2B,EAAE;QADrB,YAAO,GAAP,OAAO,CAAyE;QAGxF,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,iBAAiB,IAAI,GAAG,CAAC;QACjD,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,MAAyD;QAEzD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE5C,MAAM,MAAM,GAAG,OAAC,CAAC,MAAM,CAAC;YACtB,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;SACxC,CAAC,CAAC;QAEH,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACzD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;YACpD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;gBACxB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,KAAK,eAAe;oBAC5C,CAAC,CAAC,iGAAiG;oBACnG,CAAC,CAAC,6FAA6F,CAAC;gBAElG,MAAM,MAAM,GAAG,GAAG,OAAO,eAAe,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC;gBAE5E,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;oBAClD,OAAO,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAG,MAAM,CAAC,OAAkB,IAAI,EAAE,EAAE,CAAC;gBACrE,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;gBACvC,CAAC;YACH,CAAC,CAAC,CACH,CAAC;YAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,KAAa,EACb,QAAqD;QAErD,MAAM,MAAM,GAAG,OAAC,CAAC,MAAM,CAAC;YACtB,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC;SAC7B,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,QAAQ;aACzB,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;aACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;aACzD,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhB,MAAM,MAAM,GAAG,mCAAmC,KAAK,2DAA2D,WAAW,EAAE,CAAC;QAEhI,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAClD,OAAQ,MAAM,CAAC,OAAkB,IAAI,EAAE,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAxED,gCAwEC"}
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
export declare class Bm25Index {
|
|
2
|
-
private docs;
|
|
3
|
-
private docFreq;
|
|
4
|
-
private avgDocLength;
|
|
5
|
-
private totalTokens;
|
|
6
|
-
private readonly k1;
|
|
7
|
-
private readonly b;
|
|
8
|
-
addDocument(id: string, text: string): void;
|
|
9
|
-
addDocuments(docs: Array<{
|
|
10
|
-
id: string;
|
|
11
|
-
text: string;
|
|
12
|
-
}>): void;
|
|
13
|
-
removeDocument(id: string): void;
|
|
14
|
-
search(query: string, limit?: number): Array<{
|
|
15
|
-
id: string;
|
|
16
|
-
score: number;
|
|
17
|
-
}>;
|
|
18
|
-
clear(): void;
|
|
19
|
-
get size(): number;
|
|
20
|
-
private scoreDocument;
|
|
21
|
-
private tokenize;
|
|
22
|
-
}
|
|
23
|
-
//# sourceMappingURL=bm25.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../../../src/libs/codebase/search/bm25.ts"],"names":[],"mappings":"AAMA,qBAAa,SAAS;IACpB,OAAO,CAAC,IAAI,CAAsB;IAClC,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAO;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAQ;IAE1B,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI;IAiB3C,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,IAAI;IAM7D,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAqBhC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAevE,KAAK,IAAI,IAAI;IAOb,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,OAAO,CAAC,aAAa;IAiBrB,OAAO,CAAC,QAAQ;CAOjB"}
|
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Bm25Index = void 0;
|
|
4
|
-
class Bm25Index {
|
|
5
|
-
docs = [];
|
|
6
|
-
docFreq = new Map();
|
|
7
|
-
avgDocLength = 0;
|
|
8
|
-
totalTokens = 0;
|
|
9
|
-
k1 = 1.5;
|
|
10
|
-
b = 0.75;
|
|
11
|
-
addDocument(id, text) {
|
|
12
|
-
const tokens = this.tokenize(text);
|
|
13
|
-
const doc = { id, tokens, text };
|
|
14
|
-
this.docs.push(doc);
|
|
15
|
-
const seenTokens = new Set();
|
|
16
|
-
for (const token of tokens) {
|
|
17
|
-
if (!seenTokens.has(token)) {
|
|
18
|
-
this.docFreq.set(token, (this.docFreq.get(token) ?? 0) + 1);
|
|
19
|
-
seenTokens.add(token);
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
this.totalTokens += tokens.length;
|
|
23
|
-
this.avgDocLength = this.totalTokens / this.docs.length;
|
|
24
|
-
}
|
|
25
|
-
addDocuments(docs) {
|
|
26
|
-
for (const doc of docs) {
|
|
27
|
-
this.addDocument(doc.id, doc.text);
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
removeDocument(id) {
|
|
31
|
-
const idx = this.docs.findIndex((d) => d.id === id);
|
|
32
|
-
if (idx === -1)
|
|
33
|
-
return;
|
|
34
|
-
const doc = this.docs[idx];
|
|
35
|
-
const seenTokens = new Set(doc.tokens);
|
|
36
|
-
for (const token of seenTokens) {
|
|
37
|
-
const freq = (this.docFreq.get(token) ?? 1) - 1;
|
|
38
|
-
if (freq <= 0) {
|
|
39
|
-
this.docFreq.delete(token);
|
|
40
|
-
}
|
|
41
|
-
else {
|
|
42
|
-
this.docFreq.set(token, freq);
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
this.totalTokens -= doc.tokens.length;
|
|
46
|
-
this.docs.splice(idx, 1);
|
|
47
|
-
this.avgDocLength = this.totalTokens / Math.max(1, this.docs.length);
|
|
48
|
-
}
|
|
49
|
-
search(query, limit = 10) {
|
|
50
|
-
const queryTokens = this.tokenize(query);
|
|
51
|
-
const results = [];
|
|
52
|
-
for (const doc of this.docs) {
|
|
53
|
-
const score = this.scoreDocument(doc, queryTokens);
|
|
54
|
-
if (score > 0) {
|
|
55
|
-
results.push({ id: doc.id, score });
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
results.sort((a, b) => b.score - a.score);
|
|
59
|
-
return results.slice(0, limit);
|
|
60
|
-
}
|
|
61
|
-
clear() {
|
|
62
|
-
this.docs = [];
|
|
63
|
-
this.docFreq.clear();
|
|
64
|
-
this.avgDocLength = 0;
|
|
65
|
-
this.totalTokens = 0;
|
|
66
|
-
}
|
|
67
|
-
get size() {
|
|
68
|
-
return this.docs.length;
|
|
69
|
-
}
|
|
70
|
-
scoreDocument(doc, queryTokens) {
|
|
71
|
-
const N = this.docs.length;
|
|
72
|
-
const docLen = doc.tokens.length;
|
|
73
|
-
return queryTokens.reduce((sum, qt) => {
|
|
74
|
-
const tf = doc.tokens.filter((t) => t === qt).length;
|
|
75
|
-
if (tf === 0)
|
|
76
|
-
return sum;
|
|
77
|
-
const df = this.docFreq.get(qt) ?? 1;
|
|
78
|
-
const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
|
|
79
|
-
const numerator = tf * (this.k1 + 1);
|
|
80
|
-
const denominator = tf + this.k1 * (1 - this.b + this.b * (docLen / this.avgDocLength));
|
|
81
|
-
return sum + idf * (numerator / denominator);
|
|
82
|
-
}, 0);
|
|
83
|
-
}
|
|
84
|
-
tokenize(text) {
|
|
85
|
-
return text
|
|
86
|
-
.toLowerCase()
|
|
87
|
-
.replace(/[^a-z0-9_\s]/g, " ")
|
|
88
|
-
.split(/\s+/)
|
|
89
|
-
.filter((t) => t.length >= 2);
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
exports.Bm25Index = Bm25Index;
|
|
93
|
-
//# sourceMappingURL=bm25.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../../../../src/libs/codebase/search/bm25.ts"],"names":[],"mappings":";;;AAMA,MAAa,SAAS;IACZ,IAAI,GAAmB,EAAE,CAAC;IAC1B,OAAO,GAAwB,IAAI,GAAG,EAAE,CAAC;IACzC,YAAY,GAAG,CAAC,CAAC;IACjB,WAAW,GAAG,CAAC,CAAC;IACP,EAAE,GAAG,GAAG,CAAC;IACT,CAAC,GAAG,IAAI,CAAC;IAE1B,WAAW,CAAC,EAAU,EAAE,IAAY;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,GAAG,GAAiB,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC/C,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEpB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC5D,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;QAClC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1D,CAAC;IAED,YAAY,CAAC,IAAyC;QACpD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,cAAc,CAAC,EAAU;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QACpD,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,OAAO;QAEvB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAEvC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAChD,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;gBACd,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,CAAC,KAAa,EAAE,KAAK,GAAG,EAAE;QAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,OAAO,GAAyC,EAAE,CAAC;QAEzD,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;YACnD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC1C,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QACf,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1B,CAAC;IAEO,aAAa,CAAC,GAAiB,EAAE,WAAqB;QAC5D,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC3B,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;QAEjC,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE;YACpC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;YACrD,IAAI,EAAE,KAAK,CAAC;gBAAE,OAAO,GAAG,CAAC;YAEzB,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACtD,MAAM,SAAS,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACrC,MAAM,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YAExF,OAAO,GAAG,GAAG,GAAG,GAAG,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC;QAC/C,CAAC,EAAE,CAAC,CAAC,CAAC;IACR,CAAC;IAEO,QAAQ,CAAC,IAAY;QAC3B,OAAO,IAAI;aACR,WAAW,EAAE;aACb,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC;aAC7B,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IAClC,CAAC;CACF;AAtGD,8BAsGC"}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import type { IVectorStore, IEmbeddingProvider, VectorSearchOptions, VectorSearchResult } from "@thoughtflow/core";
|
|
2
|
-
export interface HybridSearchOptions extends VectorSearchOptions {
|
|
3
|
-
rrfK?: number;
|
|
4
|
-
vectorWeight?: number;
|
|
5
|
-
keywordWeight?: number;
|
|
6
|
-
}
|
|
7
|
-
export declare class HybridSearch {
|
|
8
|
-
private vectorStore;
|
|
9
|
-
private embeddingProvider;
|
|
10
|
-
private bm25;
|
|
11
|
-
private initialized;
|
|
12
|
-
constructor(vectorStore: IVectorStore, embeddingProvider: IEmbeddingProvider);
|
|
13
|
-
indexDocuments(docs: Array<{
|
|
14
|
-
id: string;
|
|
15
|
-
text: string;
|
|
16
|
-
}>): void;
|
|
17
|
-
removeDocument(id: string): void;
|
|
18
|
-
search(query: string, options?: HybridSearchOptions): Promise<VectorSearchResult[]>;
|
|
19
|
-
clear(): void;
|
|
20
|
-
private fuseScore;
|
|
21
|
-
}
|
|
22
|
-
//# sourceMappingURL=hybrid-search.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-search.d.ts","sourceRoot":"","sources":["../../../../src/libs/codebase/search/hybrid-search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,kBAAkB,EAAe,MAAM,mBAAmB,CAAC;AAGhI,MAAM,WAAW,mBAAoB,SAAQ,mBAAmB;IAC9D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,qBAAa,YAAY;IAKrB,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,iBAAiB;IAL3B,OAAO,CAAC,IAAI,CAA8B;IAC1C,OAAO,CAAC,WAAW,CAAS;gBAGlB,WAAW,EAAE,YAAY,EACzB,iBAAiB,EAAE,kBAAkB;IAG/C,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,IAAI;IAK/D,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAI1B,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,kBAAkB,EAAE,CAAC;IA0DhC,KAAK,IAAI,IAAI;IAKb,OAAO,CAAC,SAAS;CAelB"}
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.HybridSearch = void 0;
|
|
4
|
-
const bm25_1 = require("./bm25");
|
|
5
|
-
class HybridSearch {
|
|
6
|
-
vectorStore;
|
|
7
|
-
embeddingProvider;
|
|
8
|
-
bm25 = new bm25_1.Bm25Index();
|
|
9
|
-
initialized = false;
|
|
10
|
-
constructor(vectorStore, embeddingProvider) {
|
|
11
|
-
this.vectorStore = vectorStore;
|
|
12
|
-
this.embeddingProvider = embeddingProvider;
|
|
13
|
-
}
|
|
14
|
-
indexDocuments(docs) {
|
|
15
|
-
this.bm25.addDocuments(docs);
|
|
16
|
-
this.initialized = true;
|
|
17
|
-
}
|
|
18
|
-
removeDocument(id) {
|
|
19
|
-
this.bm25.removeDocument(id);
|
|
20
|
-
}
|
|
21
|
-
async search(query, options) {
|
|
22
|
-
const rrfK = options?.rrfK ?? 60;
|
|
23
|
-
const vectorWeight = options?.vectorWeight ?? 0.5;
|
|
24
|
-
const keywordWeight = options?.keywordWeight ?? 0.5;
|
|
25
|
-
const queryVector = await this.embeddingProvider.embedQuery(query);
|
|
26
|
-
const [vectorResults, keywordResults] = await Promise.all([
|
|
27
|
-
this.vectorStore.search(queryVector, {
|
|
28
|
-
limit: options?.limit ?? 20,
|
|
29
|
-
filter: options?.filter,
|
|
30
|
-
}),
|
|
31
|
-
Promise.resolve(this.bm25.search(query, options?.limit ?? 20)),
|
|
32
|
-
]);
|
|
33
|
-
const fusedScores = new Map();
|
|
34
|
-
for (let i = 0; i < vectorResults.length; i++) {
|
|
35
|
-
fusedScores.set(vectorResults[i].item.id, {
|
|
36
|
-
vectorRank: i + 1,
|
|
37
|
-
keywordRank: Infinity,
|
|
38
|
-
item: vectorResults[i].item,
|
|
39
|
-
});
|
|
40
|
-
}
|
|
41
|
-
const keywordIndex = new Map();
|
|
42
|
-
for (let i = 0; i < keywordResults.length; i++) {
|
|
43
|
-
keywordIndex.set(i + 1, keywordResults[i].id);
|
|
44
|
-
const existing = fusedScores.get(keywordResults[i].id);
|
|
45
|
-
if (existing) {
|
|
46
|
-
existing.keywordRank = i + 1;
|
|
47
|
-
const score = this.fuseScore(rrfK, { vector: existing.vectorRank, keyword: i + 1 }, vectorWeight, keywordWeight);
|
|
48
|
-
keywordIndex.set(i + 1, keywordResults[i].id);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
const fused = [];
|
|
52
|
-
for (const [id, data] of fusedScores) {
|
|
53
|
-
const score = this.fuseScore(rrfK, { vector: data.vectorRank, keyword: data.keywordRank }, vectorWeight, keywordWeight);
|
|
54
|
-
fused.push({ id, score, item: data.item });
|
|
55
|
-
}
|
|
56
|
-
fused.sort((a, b) => b.score - a.score);
|
|
57
|
-
const limit = options?.limit ?? 10;
|
|
58
|
-
const offset = options?.offset ?? 0;
|
|
59
|
-
return fused.slice(offset, offset + limit).map((r) => ({
|
|
60
|
-
item: r.item,
|
|
61
|
-
score: r.score,
|
|
62
|
-
}));
|
|
63
|
-
}
|
|
64
|
-
clear() {
|
|
65
|
-
this.bm25.clear();
|
|
66
|
-
this.initialized = false;
|
|
67
|
-
}
|
|
68
|
-
fuseScore(k, ranks, vectorWeight, keywordWeight) {
|
|
69
|
-
let score = 0;
|
|
70
|
-
if (ranks.vector < Infinity) {
|
|
71
|
-
score += vectorWeight * (1 / (k + ranks.vector));
|
|
72
|
-
}
|
|
73
|
-
if (ranks.keyword < Infinity) {
|
|
74
|
-
score += keywordWeight * (1 / (k + ranks.keyword));
|
|
75
|
-
}
|
|
76
|
-
return score;
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
exports.HybridSearch = HybridSearch;
|
|
80
|
-
//# sourceMappingURL=hybrid-search.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid-search.js","sourceRoot":"","sources":["../../../../src/libs/codebase/search/hybrid-search.ts"],"names":[],"mappings":";;;AACA,iCAAmC;AAQnC,MAAa,YAAY;IAKb;IACA;IALF,IAAI,GAAc,IAAI,gBAAS,EAAE,CAAC;IAClC,WAAW,GAAG,KAAK,CAAC;IAE5B,YACU,WAAyB,EACzB,iBAAqC;QADrC,gBAAW,GAAX,WAAW,CAAc;QACzB,sBAAiB,GAAjB,iBAAiB,CAAoB;IAC5C,CAAC;IAEJ,cAAc,CAAC,IAAyC;QACtD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,cAAc,CAAC,EAAU;QACvB,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAA6B;QAE7B,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;QACjC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,GAAG,CAAC;QAClD,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,GAAG,CAAC;QAEpD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAEnE,MAAM,CAAC,aAAa,EAAE,cAAc,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACxD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE;gBACnC,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE;gBAC3B,MAAM,EAAE,OAAO,EAAE,MAAM;aACxB,CAAC;YACF,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;SAC/D,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,GAAG,EAA0E,CAAC;QAEtG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE;gBACxC,UAAU,EAAE,CAAC,GAAG,CAAC;gBACjB,WAAW,EAAE,QAAQ;gBACrB,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI;aAC5B,CAAC,CAAC;QACL,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACvD,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE,YAAY,EAAE,aAAa,CAAC,CAAC;gBACjH,YAAY,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAA4D,EAAE,CAAC;QAE1E,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAC1B,IAAI,EACJ,EAAE,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,EACtD,YAAY,EACZ,aAAa,CACd,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7C,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC;QAEpC,OAAO,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrD,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;SACf,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC3B,CAAC;IAEO,SAAS,CACf,CAAS,EACT,KAA0C,EAC1C,YAAoB,EACpB,aAAqB;QAErB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC5B,KAAK,IAAI,YAAY,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,KAAK,CAAC,OAAO,GAAG,QAAQ,EAAE,CAAC;YAC7B,KAAK,IAAI,aAAa,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAnGD,oCAmGC"}
|