@appland/appmap 3.172.1 → 3.173.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/built/rpc/explain/SearchContextCollector.js +47 -33
- package/built/rpc/explain/SearchContextCollector.js.map +1 -1
- package/built/{fulltext/withIndex.js → rpc/explain/buildIndex.js} +16 -16
- package/built/rpc/explain/buildIndex.js.map +1 -0
- package/built/rpc/explain/collect-snippets.js +18 -0
- package/built/rpc/explain/collect-snippets.js.map +1 -0
- package/built/rpc/explain/index-files.js +35 -0
- package/built/rpc/explain/index-files.js.map +1 -0
- package/built/rpc/explain/index-snippets.js +11 -0
- package/built/rpc/explain/index-snippets.js.map +1 -0
- package/package.json +4 -3
- package/built/fulltext/FileIndex.js +0 -274
- package/built/fulltext/FileIndex.js.map +0 -1
- package/built/fulltext/SourceIndex.js +0 -132
- package/built/fulltext/SourceIndex.js.map +0 -1
- package/built/fulltext/listGitProjectFIles.js +0 -59
- package/built/fulltext/listGitProjectFIles.js.map +0 -1
- package/built/fulltext/listProjectFiles.js +0 -40
- package/built/fulltext/listProjectFiles.js.map +0 -1
- package/built/fulltext/querySymbols.js +0 -58
- package/built/fulltext/querySymbols.js.map +0 -1
- package/built/fulltext/withIndex.js.map +0 -1
- package/built/rpc/explain/SourceCollector.js +0 -29
- package/built/rpc/explain/SourceCollector.js.map +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
# [@appland/appmap-v3.173.1](https://github.com/getappmap/appmap-js/compare/@appland/appmap-v3.173.0...@appland/appmap-v3.173.1) (2024-11-06)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* Bump @appland/search version ([5a7da7b](https://github.com/getappmap/appmap-js/commit/5a7da7b6b0a15598182b3a509186a7fb35695aa6))
|
|
7
|
+
|
|
8
|
+
# [@appland/appmap-v3.173.0](https://github.com/getappmap/appmap-js/compare/@appland/appmap-v3.172.1...@appland/appmap-v3.173.0) (2024-11-06)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Features
|
|
12
|
+
|
|
13
|
+
* Refactor fulltext search to use @appland/search ([fa52118](https://github.com/getappmap/appmap-js/commit/fa5211812c173657b5c70ce3f4e60e4c996d7495))
|
|
14
|
+
|
|
1
15
|
# [@appland/appmap-v3.172.1](https://github.com/getappmap/appmap-js/compare/@appland/appmap-v3.172.0...@appland/appmap-v3.172.1) (2024-10-31)
|
|
2
16
|
|
|
3
17
|
|
|
@@ -4,13 +4,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
const console_1 = require("console");
|
|
7
|
+
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
|
7
8
|
const navie_1 = require("@appland/navie");
|
|
8
9
|
const AppMapIndex_1 = __importDefault(require("../../fulltext/AppMapIndex"));
|
|
9
10
|
const search_1 = require("../search/search");
|
|
10
|
-
const FileIndex_1 = require("../../fulltext/FileIndex");
|
|
11
|
-
const withIndex_1 = __importDefault(require("../../fulltext/withIndex"));
|
|
12
11
|
const EventCollector_1 = __importDefault(require("./EventCollector"));
|
|
13
|
-
const
|
|
12
|
+
const index_files_1 = __importDefault(require("./index-files"));
|
|
13
|
+
const index_snippets_1 = __importDefault(require("./index-snippets"));
|
|
14
|
+
const collect_snippets_1 = __importDefault(require("./collect-snippets"));
|
|
15
|
+
const buildIndex_1 = __importDefault(require("./buildIndex"));
|
|
14
16
|
class SearchContextCollector {
|
|
15
17
|
constructor(appmapDirectories, sourceDirectories, appmaps, vectorTerms, charLimit) {
|
|
16
18
|
this.appmapDirectories = appmapDirectories;
|
|
@@ -18,7 +20,6 @@ class SearchContextCollector {
|
|
|
18
20
|
this.appmaps = appmaps;
|
|
19
21
|
this.vectorTerms = vectorTerms;
|
|
20
22
|
this.charLimit = charLimit;
|
|
21
|
-
this.query = this.vectorTerms.join(' ');
|
|
22
23
|
}
|
|
23
24
|
async collectContext() {
|
|
24
25
|
let appmapSearchResponse;
|
|
@@ -52,38 +53,51 @@ class SearchContextCollector {
|
|
|
52
53
|
const searchOptions = {
|
|
53
54
|
maxResults: search_1.DEFAULT_MAX_DIAGRAMS,
|
|
54
55
|
};
|
|
55
|
-
appmapSearchResponse = await AppMapIndex_1.default.search(this.appmapDirectories, this.
|
|
56
|
+
appmapSearchResponse = await AppMapIndex_1.default.search(this.appmapDirectories, this.vectorTerms.join(' '), searchOptions);
|
|
56
57
|
}
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
const fileIndex = await (0, buildIndex_1.default)('files', async (indexFile) => {
|
|
59
|
+
const db = new better_sqlite3_1.default(indexFile);
|
|
60
|
+
return await (0, index_files_1.default)(db, this.sourceDirectories, this.includePatterns, this.excludePatterns);
|
|
61
|
+
});
|
|
62
|
+
let fileSearchResults;
|
|
63
|
+
try {
|
|
64
|
+
fileSearchResults = fileIndex.index.search(this.vectorTerms.join(' OR '));
|
|
65
|
+
}
|
|
66
|
+
finally {
|
|
67
|
+
fileIndex.close();
|
|
68
|
+
}
|
|
69
|
+
const snippetIndex = await (0, buildIndex_1.default)('snippets', async (indexFile) => {
|
|
70
|
+
const db = new better_sqlite3_1.default(indexFile);
|
|
71
|
+
return await (0, index_snippets_1.default)(db, fileSearchResults);
|
|
72
|
+
});
|
|
60
73
|
let contextCandidate;
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
(0, console_1.log)(`[search-context]
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
+
try {
|
|
75
|
+
const eventsCollector = new EventCollector_1.default(this.vectorTerms.join(' '), appmapSearchResponse);
|
|
76
|
+
let charCount = 0;
|
|
77
|
+
let maxEventsPerDiagram = 5;
|
|
78
|
+
(0, console_1.log)(`[search-context] Requested char limit: ${this.charLimit}`);
|
|
79
|
+
for (;;) {
|
|
80
|
+
(0, console_1.log)(`[search-context] Collecting context with ${maxEventsPerDiagram} events per diagram.`);
|
|
81
|
+
contextCandidate = await eventsCollector.collectEvents(maxEventsPerDiagram, this.excludePatterns, this.includePatterns, this.includeTypes);
|
|
82
|
+
const codeSnippetCount = contextCandidate.context.filter((item) => item.type === navie_1.ContextV2.ContextItemType.CodeSnippet).length;
|
|
83
|
+
const charLimit = codeSnippetCount === 0 ? this.charLimit : this.charLimit / 4;
|
|
84
|
+
const sourceContext = (0, collect_snippets_1.default)(snippetIndex.index, this.vectorTerms.join(' OR '), charLimit);
|
|
85
|
+
contextCandidate.context = contextCandidate.context.concat(sourceContext);
|
|
86
|
+
const appliedContext = (0, navie_1.applyContext)(contextCandidate.context, this.charLimit);
|
|
87
|
+
const appliedContextSize = appliedContext.reduce((acc, item) => acc + item.content.length, 0);
|
|
88
|
+
contextCandidate.context = appliedContext;
|
|
89
|
+
contextCandidate.contextSize = appliedContextSize;
|
|
90
|
+
(0, console_1.log)(`[search-context] Collected an estimated ${appliedContextSize} characters.`);
|
|
91
|
+
if (appliedContextSize === charCount || appliedContextSize > this.charLimit) {
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
charCount = appliedContextSize;
|
|
95
|
+
maxEventsPerDiagram = Math.ceil(maxEventsPerDiagram * 1.5);
|
|
96
|
+
(0, console_1.log)(`[search-context] Increasing max events per diagram to ${maxEventsPerDiagram}.`);
|
|
74
97
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
contextCandidate.context = appliedContext;
|
|
79
|
-
contextCandidate.contextSize = appliedContextSize;
|
|
80
|
-
(0, console_1.log)(`[search-context] Collected an estimated ${appliedContextSize} characters.`);
|
|
81
|
-
if (appliedContextSize === charCount || appliedContextSize > this.charLimit) {
|
|
82
|
-
break;
|
|
83
|
-
}
|
|
84
|
-
charCount = appliedContextSize;
|
|
85
|
-
maxEventsPerDiagram = Math.ceil(maxEventsPerDiagram * 1.5);
|
|
86
|
-
(0, console_1.log)(`[search-context] Increasing max events per diagram to ${maxEventsPerDiagram}.`);
|
|
98
|
+
}
|
|
99
|
+
finally {
|
|
100
|
+
snippetIndex.close();
|
|
87
101
|
}
|
|
88
102
|
return {
|
|
89
103
|
searchResponse: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SearchContextCollector.js","sourceRoot":"","sources":["../../../src/rpc/explain/SearchContextCollector.ts"],"names":[],"mappings":";;;;;AAAA,qCAA8B;AAC9B,0CAAyD;
|
|
1
|
+
{"version":3,"file":"SearchContextCollector.js","sourceRoot":"","sources":["../../../src/rpc/explain/SearchContextCollector.ts"],"names":[],"mappings":";;;;;AAAA,qCAA8B;AAC9B,oEAAqC;AAErC,0CAAyD;AAIzD,6EAGoC;AACpC,6CAAwD;AACxD,sEAA8C;AAC9C,gEAAuC;AACvC,sEAA6C;AAC7C,0EAAiD;AACjD,8DAAsC;AAEtC,MAAqB,sBAAsB;IAKzC,YACU,iBAA2B,EAC3B,iBAA2B,EAC3B,OAA6B,EAC7B,WAAqB,EACrB,SAAiB;QAJjB,sBAAiB,GAAjB,iBAAiB,CAAU;QAC3B,sBAAiB,GAAjB,iBAAiB,CAAU;QAC3B,YAAO,GAAP,OAAO,CAAsB;QAC7B,gBAAW,GAAX,WAAW,CAAU;QACrB,cAAS,GAAT,SAAS,CAAQ;IACxB,CAAC;IAEJ,KAAK,CAAC,cAAc;QAIlB,IAAI,oBAA0C,CAAC;QAC/C,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO;iBACzB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;gBACd,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC/E,IAAI,CAAC,SAAS;oBAAE,OAAO,SAAS,CAAC;gBAEjC,OAAO;oBACL,MAAM;oBACN,SAAS;oBACT,KAAK,EAAE,CAAC;iBACT,CAAC;YACJ,CAAC,CAAC;iBACD,MAAM,CAAC,OAAO,CAA6B,CAAC;YAC/C,oBAAoB,GAAG;gBACrB,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE;oBACL,GAAG,EAAE,CAAC;oBACN,IAAI,EAAE,CAAC;oBACP,MAAM,EAAE,CAAC;oBACT,MAAM,EAAE,CAAC;iBACV;gBACD,OAAO;gBACP,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;aAChC,CAAC;SACH;aAAM;YACL,iFAAiF;YACjF,MAAM,aAAa,GAAwB;gBACzC,UAAU,EAAE,6BAAoB;aACjC,CAAC;YACF,oBAAoB,GAAG,MAAM,qBAAW,CAAC,MAAM,CAC7C,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAC1B,aAAa,CACd,CAAC;SACH;QAED,MAAM,SAAS,GAAG,MAAM,IAAA,oBAAU,EAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE;YAC9D,MAAM,EAAE,GAAG,IAAI,wBAAO,CAAC,SAAS,CAAC,CAAC;YAClC,OAAO,MAAM,IAAA,qBAAU,EACrB,EAAE,EACF,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,eAAe,CACrB,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,IAAI,iBAAqC,CAAC;QAC1C,IAAI;YACF,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;SAC3E;gBAAS;YACR,SAAS,CAAC,KAAK,EAAE,CAAC;SACnB;QAED,MAAM,YAAY,GAAG,MAAM,IAAA,oBAAU,EAAC,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE;YACpE,MAAM,EAAE,GAAG,IAAI,wBAAO,CAAC,SAAS,CAAC,CAAC;YAClC,OAAO,MAAM,IAAA,wBAAa,EAAC,EAAE,EAAE,iBAAiB,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,IAAI,gBAIH,CAAC;QACF,IAAI;YACF,MAAM,eAAe,GAAG,IAAI,wBAAc,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,oBAAoB,CAAC,CAAC;YAE7F,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,mBAAmB,GAAG,CAAC,CAAC;YAC5B,IAAA,aAAG,EAAC,0CAA0C,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;YAChE,SAAS;gBACP,IAAA,aAAG,EAAC,4CAA4C,mBAAmB,sBAAsB,CAAC,CAAC;gBAE3F,gBAAgB,GAAG,MAAM,eAAe,CAAC,aAAa,CACpD,mBAAmB,EACnB,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,YAAY,CAClB,CAAC;gBAEF,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,OAAO,CAAC,MAAM,CACtD,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,iBAAS,CAAC,eAAe,CAAC,WAAW,CAC9D,CAAC,MAAM,CAAC;gBAET,MAAM,SAAS,GAAG,gBAAgB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;gBAC/E,MAAM,aAAa,GAAG,IAAA,0BAAe,EACnC,YAAY,CAAC,KAAK,EAClB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EAC7B,SAAS,CACV,CAAC;gBACF,gBAAgB,CAAC,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;gBAE1E,MAAM,cAAc,GAAG,IAAA,oBAAY,EAAC,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC9E,MAAM,kBAAkB,GAAG,cAAc,CAAC,MAAM,CAC9C,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EACxC,CAAC,CACF,CAAC;gBACF,gBAAgB,CAAC,OAAO,GAAG,cAAc,CAAC;gBAC1C,gBAAgB,CAAC,WAAW,GAAG,kBAAkB,CAAC;gBAClD,IAAA,aAAG,EAAC,2CAA2C,kBAAkB,cAAc,CAAC,CAAC;gBAEjF,IAAI,kBAAkB,KAAK,SAAS,IAAI,kBAAkB,GAAG,IAAI,CAAC,SAAS,EAAE;oBAC3E,MAAM;iBACP;gBACD,SAAS,GAAG,kBAAkB,CAAC;gBAC/B,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,mBAAmB,GAAG,GAAG,CAAC,CAAC;gBAC3D,IAAA,aAAG,EAAC,yDAAyD,mBAAmB,GAAG,CAAC,CAAC;aACtF;SACF;gBAAS;YACR,YAAY,CAAC,KAAK,EAAE,CAAC;SACtB;QAED,OAAO;YACL,cAAc,EAAE;gBACd,OAAO,EAAE,gBAAgB,CAAC,OAAO;gBACjC,UAAU,EAAE,oBAAoB,CAAC,UAAU;aAC5C;YACD,OAAO,EAAE,gBAAgB,CAAC,OAAO;SAClC,CAAC;IACJ,CAAC;CACF;AAxID,yCAwIC"}
|
|
@@ -2,32 +2,32 @@
|
|
|
2
2
|
// This function is used for a "one-shot" search in which the index us built, used, and discarded.
|
|
3
3
|
// Replace this with a persistent index file that can be used across multiple searches, and is
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
const fs_1 = require("fs");
|
|
5
6
|
const promises_1 = require("fs/promises");
|
|
6
7
|
const os_1 = require("os");
|
|
7
8
|
const path_1 = require("path");
|
|
8
|
-
|
|
9
|
-
async function withIndex(indexName, builder, callback) {
|
|
9
|
+
async function buildIndex(indexName, builder) {
|
|
10
10
|
const tmpDir = await (0, promises_1.mkdtemp)((0, path_1.join)((0, os_1.tmpdir)(), `appmap-${indexName}-${new Date().getTime()}`));
|
|
11
11
|
const indexFile = (0, path_1.join)(tmpDir, 'index.sqlite');
|
|
12
|
-
const
|
|
12
|
+
const index = await builder(indexFile);
|
|
13
|
+
const close = () => {
|
|
13
14
|
try {
|
|
14
15
|
index.close();
|
|
15
16
|
}
|
|
16
17
|
catch (err) {
|
|
17
18
|
console.error(err);
|
|
18
19
|
}
|
|
20
|
+
try {
|
|
21
|
+
(0, fs_1.rmSync)(tmpDir, { recursive: true });
|
|
22
|
+
}
|
|
23
|
+
catch (err) {
|
|
24
|
+
console.error(err);
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
return {
|
|
28
|
+
index,
|
|
29
|
+
close,
|
|
19
30
|
};
|
|
20
|
-
const cleanupDir = async () => await (0, promises_1.rm)(tmpDir, { recursive: true }).catch((err) => console.error(err));
|
|
21
|
-
let index;
|
|
22
|
-
try {
|
|
23
|
-
index = await builder(indexFile);
|
|
24
|
-
return await callback(index);
|
|
25
|
-
}
|
|
26
|
-
finally {
|
|
27
|
-
if (index)
|
|
28
|
-
close(index);
|
|
29
|
-
await cleanupDir();
|
|
30
|
-
}
|
|
31
31
|
}
|
|
32
|
-
exports.default =
|
|
33
|
-
//# sourceMappingURL=
|
|
32
|
+
exports.default = buildIndex;
|
|
33
|
+
//# sourceMappingURL=buildIndex.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"buildIndex.js","sourceRoot":"","sources":["../../../src/rpc/explain/buildIndex.ts"],"names":[],"mappings":";AAAA,kGAAkG;AAClG,8FAA8F;;AAE9F,2BAA4B;AAC5B,0CAAsC;AACtC,2BAA4B;AAC5B,+BAA4B;AAWb,KAAK,UAAU,UAAU,CACtC,SAAiB,EACjB,OAA0C;IAE1C,MAAM,MAAM,GAAG,MAAM,IAAA,kBAAO,EAAC,IAAA,WAAI,EAAC,IAAA,WAAM,GAAE,EAAE,UAAU,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC5F,MAAM,SAAS,GAAG,IAAA,WAAI,EAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAE/C,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC;IAEvC,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI;YACF,KAAK,CAAC,KAAK,EAAE,CAAC;SACf;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;SACpB;QACD,IAAI;YACF,IAAA,WAAM,EAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SACrC;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;SACpB;IACH,CAAC,CAAC;IAEF,OAAO;QACL,KAAK;QACL,KAAK;KACN,CAAC;AACJ,CAAC;AA1BD,6BA0BC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const navie_1 = require("@appland/navie");
|
|
4
|
+
const collectContext_1 = require("./collectContext");
|
|
5
|
+
function collectSnippets(snippetIndex, query, charLimit) {
|
|
6
|
+
const snippets = snippetIndex.searchSnippets(query, Math.round(charLimit / collectContext_1.CHARS_PER_SNIPPET));
|
|
7
|
+
const buildLocation = (result) => {
|
|
8
|
+
return `${result.filePath}:${result.startLine}-${result.endLine}`;
|
|
9
|
+
};
|
|
10
|
+
return snippets.map((snippet) => ({
|
|
11
|
+
directory: snippet.directory,
|
|
12
|
+
type: navie_1.ContextV2.ContextItemType.CodeSnippet,
|
|
13
|
+
content: snippet.content,
|
|
14
|
+
location: buildLocation(snippet),
|
|
15
|
+
}));
|
|
16
|
+
}
|
|
17
|
+
exports.default = collectSnippets;
|
|
18
|
+
//# sourceMappingURL=collect-snippets.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"collect-snippets.js","sourceRoot":"","sources":["../../../src/rpc/explain/collect-snippets.ts"],"names":[],"mappings":";;AAAA,0CAA2C;AAE3C,qDAAqD;AAErD,SAAwB,eAAe,CACrC,YAA0B,EAC1B,KAAa,EACb,SAAiB;IAEjB,MAAM,QAAQ,GAAG,YAAY,CAAC,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,kCAAiB,CAAC,CAAC,CAAC;IAE/F,MAAM,aAAa,GAAG,CAAC,MAA2B,EAAE,EAAE;QACpD,OAAO,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;IACpE,CAAC,CAAC;IAEF,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAChC,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,IAAI,EAAE,iBAAS,CAAC,eAAe,CAAC,WAAW;QAC3C,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,QAAQ,EAAE,aAAa,CAAC,OAAO,CAAC;KACjC,CAAC,CAAC,CAAC;AACN,CAAC;AAjBD,kCAiBC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const debug_1 = __importDefault(require("debug"));
|
|
7
|
+
const search_1 = require("@appland/search");
|
|
8
|
+
const fileNameMatchesFilterPatterns_1 = require("../../fulltext/fileNameMatchesFilterPatterns");
|
|
9
|
+
const debug = (0, debug_1.default)('appmap:rpc:explain:index-files');
|
|
10
|
+
function fileFilter(includePatterns, excludePatterns) {
|
|
11
|
+
return async (path) => {
|
|
12
|
+
debug('Filtering: %s', path);
|
|
13
|
+
if ((0, search_1.isBinaryFile)(path)) {
|
|
14
|
+
debug('Skipping binary file: %s', path);
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
const includeFile = (0, fileNameMatchesFilterPatterns_1.fileNameMatchesFilterPatterns)(path, includePatterns, excludePatterns);
|
|
18
|
+
if (!includeFile)
|
|
19
|
+
return false;
|
|
20
|
+
const isData = (0, search_1.isDataFile)(path);
|
|
21
|
+
if (isData && (await (0, search_1.isLargeFile)(path))) {
|
|
22
|
+
debug('Skipping large data file: %s', path);
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
return true;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
async function indexFiles(db, directories, includePatterns, excludePatterns) {
|
|
29
|
+
const fileIndex = new search_1.FileIndex(db);
|
|
30
|
+
const filter = fileFilter(includePatterns, excludePatterns);
|
|
31
|
+
await (0, search_1.buildFileIndex)(fileIndex, directories, search_1.listProjectFiles, filter, search_1.readFileSafe, search_1.fileTokens);
|
|
32
|
+
return fileIndex;
|
|
33
|
+
}
|
|
34
|
+
exports.default = indexFiles;
|
|
35
|
+
//# sourceMappingURL=index-files.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-files.js","sourceRoot":"","sources":["../../../src/rpc/explain/index-files.ts"],"names":[],"mappings":";;;;;AACA,kDAA8B;AAE9B,4CAUyB;AACzB,gGAA6F;AAE7F,MAAM,KAAK,GAAG,IAAA,eAAS,EAAC,gCAAgC,CAAC,CAAC;AAE1D,SAAS,UAAU,CACjB,eAAqC,EACrC,eAAqC;IAErC,OAAO,KAAK,EAAE,IAAY,EAAE,EAAE;QAC5B,KAAK,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;QAC7B,IAAI,IAAA,qBAAY,EAAC,IAAI,CAAC,EAAE;YACtB,KAAK,CAAC,0BAA0B,EAAE,IAAI,CAAC,CAAC;YACxC,OAAO,KAAK,CAAC;SACd;QAED,MAAM,WAAW,GAAG,IAAA,6DAA6B,EAAC,IAAI,EAAE,eAAe,EAAE,eAAe,CAAC,CAAC;QAC1F,IAAI,CAAC,WAAW;YAAE,OAAO,KAAK,CAAC;QAE/B,MAAM,MAAM,GAAG,IAAA,mBAAU,EAAC,IAAI,CAAC,CAAC;QAChC,IAAI,MAAM,IAAI,CAAC,MAAM,IAAA,oBAAW,EAAC,IAAI,CAAC,CAAC,EAAE;YACvC,KAAK,CAAC,8BAA8B,EAAE,IAAI,CAAC,CAAC;YAC5C,OAAO,KAAK,CAAC;SACd;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;AACJ,CAAC;AAEc,KAAK,UAAU,UAAU,CACtC,EAAoB,EACpB,WAAqB,EACrB,eAAqC,EACrC,eAAqC;IAErC,MAAM,SAAS,GAAG,IAAI,kBAAS,CAAC,EAAE,CAAC,CAAC;IAEpC,MAAM,MAAM,GAAG,UAAU,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;IAC5D,MAAM,IAAA,uBAAc,EAAC,SAAS,EAAE,WAAW,EAAE,yBAAgB,EAAE,MAAM,EAAE,qBAAY,EAAE,mBAAU,CAAC,CAAC;IAEjG,OAAO,SAAS,CAAC;AACnB,CAAC;AAZD,6BAYC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const search_1 = require("@appland/search");
|
|
4
|
+
async function indexSnippets(db, fileSearchResults) {
|
|
5
|
+
const splitter = search_1.langchainSplitter;
|
|
6
|
+
const snippetIndex = new search_1.SnippetIndex(db);
|
|
7
|
+
await (0, search_1.buildSnippetIndex)(snippetIndex, fileSearchResults, search_1.readFileSafe, splitter, search_1.fileTokens);
|
|
8
|
+
return snippetIndex;
|
|
9
|
+
}
|
|
10
|
+
exports.default = indexSnippets;
|
|
11
|
+
//# sourceMappingURL=index-snippets.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-snippets.js","sourceRoot":"","sources":["../../../src/rpc/explain/index-snippets.ts"],"names":[],"mappings":";;AAAA,4CAOyB;AAGV,KAAK,UAAU,aAAa,CACzC,EAAoB,EACpB,iBAAqC;IAErC,MAAM,QAAQ,GAAG,0BAAiB,CAAC;IAEnC,MAAM,YAAY,GAAG,IAAI,qBAAY,CAAC,EAAE,CAAC,CAAC;IAC1C,MAAM,IAAA,0BAAiB,EAAC,YAAY,EAAE,iBAAiB,EAAE,qBAAY,EAAE,QAAQ,EAAE,mBAAU,CAAC,CAAC;IAE7F,OAAO,YAAY,CAAC;AACtB,CAAC;AAVD,gCAUC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@appland/appmap",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.173.1",
|
|
4
4
|
"description": "",
|
|
5
5
|
"bin": "built/cli.js",
|
|
6
6
|
"engines": {
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
"@eslint/js": "~8.57.0",
|
|
39
39
|
"@octokit/types": "^11.1.0",
|
|
40
40
|
"@swc/core": "^1.3.76",
|
|
41
|
-
"@types/better-sqlite3": "^7.6.
|
|
41
|
+
"@types/better-sqlite3": "^7.6.11",
|
|
42
42
|
"@types/cors": "^2.8.17",
|
|
43
43
|
"@types/fs-extra": "^9.0.13",
|
|
44
44
|
"@types/gitconfiglocal": "^2.0.1",
|
|
@@ -94,6 +94,7 @@
|
|
|
94
94
|
"@appland/openapi": "^1.8.0",
|
|
95
95
|
"@appland/rpc": "^1.13.0",
|
|
96
96
|
"@appland/scanner": "^1.86.0",
|
|
97
|
+
"@appland/search": "^1.0.0",
|
|
97
98
|
"@appland/sequence-diagram": "^1.12.0",
|
|
98
99
|
"@octokit/rest": "^20.0.1",
|
|
99
100
|
"@sidvind/better-ajv-errors": "^0.9.1",
|
|
@@ -102,7 +103,7 @@
|
|
|
102
103
|
"applicationinsights": "^2.1.4",
|
|
103
104
|
"async": "^3.2.4",
|
|
104
105
|
"axios": "^0.27.2",
|
|
105
|
-
"better-sqlite3": "^
|
|
106
|
+
"better-sqlite3": "^11.5.0",
|
|
106
107
|
"body-parser": "^1.20.2",
|
|
107
108
|
"boxen": "^5.0.1",
|
|
108
109
|
"chalk": "^4.1.2",
|
|
@@ -1,274 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
|
26
|
-
if (kind === "m") throw new TypeError("Private method is not writable");
|
|
27
|
-
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
|
28
|
-
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
|
29
|
-
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
30
|
-
};
|
|
31
|
-
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
|
32
|
-
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
|
33
|
-
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
34
|
-
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
35
|
-
};
|
|
36
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
37
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
38
|
-
};
|
|
39
|
-
var _FileIndex_insert;
|
|
40
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
41
|
-
exports.filterFiles = exports.buildFileIndex = exports.restoreFileIndex = exports.FileIndex = void 0;
|
|
42
|
-
const promises_1 = require("node:fs/promises");
|
|
43
|
-
const node_path_1 = __importStar(require("node:path"));
|
|
44
|
-
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
|
45
|
-
const assert_1 = __importDefault(require("assert"));
|
|
46
|
-
const debug_1 = __importDefault(require("debug"));
|
|
47
|
-
const fs_1 = require("fs");
|
|
48
|
-
const listProjectFiles_1 = __importDefault(require("./listProjectFiles"));
|
|
49
|
-
const queryKeywords_1 = __importDefault(require("./queryKeywords"));
|
|
50
|
-
const telemetry_1 = require("../telemetry");
|
|
51
|
-
const listGitProjectFIles_1 = __importDefault(require("./listGitProjectFIles"));
|
|
52
|
-
const querySymbols_1 = __importDefault(require("./querySymbols"));
|
|
53
|
-
const fileNameMatchesFilterPatterns_1 = require("./fileNameMatchesFilterPatterns");
|
|
54
|
-
debug_1.default.formatters.a = (v) => {
|
|
55
|
-
return v.join(' ');
|
|
56
|
-
};
|
|
57
|
-
const debug = (0, debug_1.default)('appmap:file-index');
|
|
58
|
-
// enable to see all indexing terms for each file
|
|
59
|
-
const debugTerms = (0, debug_1.default)('appmap:file-index:terms');
|
|
60
|
-
class FileIndex {
|
|
61
|
-
constructor(database) {
|
|
62
|
-
this.database = database;
|
|
63
|
-
this.indexDirectory = this.database.transaction((directory, fileNames, options, offset, limit) => {
|
|
64
|
-
const startIndex = offset !== null && offset !== void 0 ? offset : 0;
|
|
65
|
-
const endIndex = limit ? Math.min(startIndex + limit, fileNames.length) : fileNames.length;
|
|
66
|
-
for (let i = startIndex; i < endIndex; i++) {
|
|
67
|
-
const fileName = fileNames[i];
|
|
68
|
-
this.indexFile(directory, fileName, options);
|
|
69
|
-
}
|
|
70
|
-
});
|
|
71
|
-
_FileIndex_insert.set(this, void 0);
|
|
72
|
-
this.database.exec(`CREATE VIRTUAL TABLE files USING fts5(directory UNINDEXED, file_name UNINDEXED, terms, tokenize = 'porter unicode61')`);
|
|
73
|
-
this.database.pragma('journal_mode = OFF');
|
|
74
|
-
this.database.pragma('synchronous = OFF');
|
|
75
|
-
__classPrivateFieldSet(this, _FileIndex_insert, this.database.prepare('INSERT INTO files (directory, file_name, terms) VALUES (?, ?, ?)'), "f");
|
|
76
|
-
}
|
|
77
|
-
close() {
|
|
78
|
-
this.database.close();
|
|
79
|
-
}
|
|
80
|
-
search(keywords, limit = 10) {
|
|
81
|
-
const query = `SELECT directory, file_name, (rank * -1) score FROM files WHERE files MATCH ? ORDER BY rank LIMIT ?`;
|
|
82
|
-
const searchExpr = (0, queryKeywords_1.default)(keywords).join(' OR ');
|
|
83
|
-
const rows = this.database.prepare(query).all(searchExpr, limit);
|
|
84
|
-
if (debug.enabled)
|
|
85
|
-
for (const row of rows) {
|
|
86
|
-
debug('Found row %s', row.file_name);
|
|
87
|
-
}
|
|
88
|
-
return rows.map((row) => ({
|
|
89
|
-
directory: row.directory,
|
|
90
|
-
fileName: row.file_name,
|
|
91
|
-
score: row.score,
|
|
92
|
-
}));
|
|
93
|
-
}
|
|
94
|
-
async indexDirectories(directories, excludePatterns, includePatterns, batchSize = 100) {
|
|
95
|
-
for (const directory of directories) {
|
|
96
|
-
try {
|
|
97
|
-
const startTime = new Date().getTime();
|
|
98
|
-
const gitState = await telemetry_1.Git.state(directory);
|
|
99
|
-
const fileNames = gitState === telemetry_1.GitState.Ok
|
|
100
|
-
? await (0, listGitProjectFIles_1.default)(directory)
|
|
101
|
-
: await (0, listProjectFiles_1.default)(directory);
|
|
102
|
-
const filteredFileNames = await filterFiles(directory, fileNames, excludePatterns, includePatterns);
|
|
103
|
-
const options = {
|
|
104
|
-
allowGenericParsing: fileNames.length < 15000,
|
|
105
|
-
allowSymbols: fileNames.length < 15000,
|
|
106
|
-
};
|
|
107
|
-
if (options.allowSymbols) {
|
|
108
|
-
debug('Symbol parsing is enabled.');
|
|
109
|
-
debug('Generic symbol parsing is %s', options.allowGenericParsing ? 'enabled.' : 'disabled.');
|
|
110
|
-
}
|
|
111
|
-
else {
|
|
112
|
-
debug('Symbol parsing is disabled.');
|
|
113
|
-
}
|
|
114
|
-
for (let i = 0; i < filteredFileNames.length; i += batchSize) {
|
|
115
|
-
this.indexDirectory(directory, filteredFileNames, options, i, batchSize);
|
|
116
|
-
// yield to the event loop after each chunk
|
|
117
|
-
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
118
|
-
}
|
|
119
|
-
const endTime = new Date().getTime();
|
|
120
|
-
console.log(`Indexed ${fileNames.length} files in ${directory} in ${endTime - startTime}ms`);
|
|
121
|
-
}
|
|
122
|
-
catch (error) {
|
|
123
|
-
console.error(`Error processing directory ${directory}:`, error);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
indexFile(directory, filePath, options = {}) {
|
|
128
|
-
const { allowGenericParsing = true, allowSymbols = true } = options;
|
|
129
|
-
const fileNameTokens = filePath.split(node_path_1.default.sep);
|
|
130
|
-
try {
|
|
131
|
-
let terms = (0, queryKeywords_1.default)(fileNameTokens);
|
|
132
|
-
if (allowSymbols) {
|
|
133
|
-
const symbols = (0, querySymbols_1.default)(node_path_1.default.join(directory, filePath), allowGenericParsing);
|
|
134
|
-
terms = terms.concat((0, queryKeywords_1.default)(symbols).sort());
|
|
135
|
-
}
|
|
136
|
-
debug('Indexing file path %s with %d terms', filePath, terms.length);
|
|
137
|
-
debugTerms('Terms for path %s: %a', filePath, terms);
|
|
138
|
-
__classPrivateFieldGet(this, _FileIndex_insert, "f").run(directory, filePath, terms.join(' '));
|
|
139
|
-
}
|
|
140
|
-
catch (error) {
|
|
141
|
-
console.warn(`Error indexing document ${filePath}`);
|
|
142
|
-
console.warn(error);
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
exports.FileIndex = FileIndex;
|
|
147
|
-
_FileIndex_insert = new WeakMap();
|
|
148
|
-
function restoreFileIndex(indexFileName) {
|
|
149
|
-
(0, assert_1.default)((0, fs_1.existsSync)(indexFileName), `Index file ${indexFileName} does not exist`);
|
|
150
|
-
const database = new better_sqlite3_1.default(indexFileName);
|
|
151
|
-
return new FileIndex(database);
|
|
152
|
-
}
|
|
153
|
-
exports.restoreFileIndex = restoreFileIndex;
|
|
154
|
-
async function buildFileIndex(directories, indexFileName, excludePatterns, includePatterns) {
|
|
155
|
-
(0, assert_1.default)(!(0, fs_1.existsSync)(indexFileName), `Index file ${indexFileName} already exists`);
|
|
156
|
-
const database = new better_sqlite3_1.default(indexFileName);
|
|
157
|
-
const fileIndex = new FileIndex(database);
|
|
158
|
-
await fileIndex.indexDirectories(directories, excludePatterns, includePatterns);
|
|
159
|
-
console.log(`Wrote file index to ${indexFileName}`);
|
|
160
|
-
return fileIndex;
|
|
161
|
-
}
|
|
162
|
-
exports.buildFileIndex = buildFileIndex;
|
|
163
|
-
const BINARY_FILE_EXTENSIONS = [
|
|
164
|
-
'7z',
|
|
165
|
-
'aac',
|
|
166
|
-
'avi',
|
|
167
|
-
'bmp',
|
|
168
|
-
'bz2',
|
|
169
|
-
'class',
|
|
170
|
-
'dll',
|
|
171
|
-
'doc',
|
|
172
|
-
'docx',
|
|
173
|
-
'dylib',
|
|
174
|
-
'ear',
|
|
175
|
-
'exe',
|
|
176
|
-
'eot',
|
|
177
|
-
'flac',
|
|
178
|
-
'flv',
|
|
179
|
-
'gif',
|
|
180
|
-
'gz',
|
|
181
|
-
'ico',
|
|
182
|
-
'jar',
|
|
183
|
-
'jpeg',
|
|
184
|
-
'jpg',
|
|
185
|
-
'js.map',
|
|
186
|
-
'min.js',
|
|
187
|
-
'min.css',
|
|
188
|
-
'mkv',
|
|
189
|
-
'mo',
|
|
190
|
-
'mov',
|
|
191
|
-
'mp3',
|
|
192
|
-
'mp4',
|
|
193
|
-
'mpg',
|
|
194
|
-
'odt',
|
|
195
|
-
'odp',
|
|
196
|
-
'ods',
|
|
197
|
-
'ogg',
|
|
198
|
-
'otf',
|
|
199
|
-
'pdf',
|
|
200
|
-
'po',
|
|
201
|
-
'png',
|
|
202
|
-
'ppt',
|
|
203
|
-
'pptx',
|
|
204
|
-
'pyc',
|
|
205
|
-
'rar',
|
|
206
|
-
'rtf',
|
|
207
|
-
'so',
|
|
208
|
-
'svg',
|
|
209
|
-
'tar',
|
|
210
|
-
'tiff',
|
|
211
|
-
'ttf',
|
|
212
|
-
'wav',
|
|
213
|
-
'webm',
|
|
214
|
-
'webp',
|
|
215
|
-
'woff',
|
|
216
|
-
'woff2',
|
|
217
|
-
'wmv',
|
|
218
|
-
'xls',
|
|
219
|
-
'xlsx',
|
|
220
|
-
'xz',
|
|
221
|
-
'yarn.lock',
|
|
222
|
-
'zip',
|
|
223
|
-
].map((ext) => '.' + ext);
|
|
224
|
-
const DATA_FILE_EXTENSIONS = [
|
|
225
|
-
'csv',
|
|
226
|
-
'dat',
|
|
227
|
-
'log',
|
|
228
|
-
'json',
|
|
229
|
-
'tsv',
|
|
230
|
-
'yaml',
|
|
231
|
-
'yml',
|
|
232
|
-
'xml',
|
|
233
|
-
].map((ext) => '.' + ext);
|
|
234
|
-
const isBinaryFile = (fileName) => {
|
|
235
|
-
return BINARY_FILE_EXTENSIONS.some((ext) => fileName.endsWith(ext));
|
|
236
|
-
};
|
|
237
|
-
const isDataFile = (fileName) => {
|
|
238
|
-
return DATA_FILE_EXTENSIONS.some((ext) => fileName.endsWith(ext));
|
|
239
|
-
};
|
|
240
|
-
async function filterFiles(directory, fileNames, excludePatterns, includePatterns) {
|
|
241
|
-
const result = [];
|
|
242
|
-
for (const fileName of fileNames) {
|
|
243
|
-
if (isBinaryFile(fileName))
|
|
244
|
-
continue;
|
|
245
|
-
const includeFile = (0, fileNameMatchesFilterPatterns_1.fileNameMatchesFilterPatterns)(fileName, includePatterns, excludePatterns);
|
|
246
|
-
if (!includeFile)
|
|
247
|
-
continue;
|
|
248
|
-
let appendFile = false;
|
|
249
|
-
try {
|
|
250
|
-
const stats = await (0, promises_1.stat)((0, node_path_1.join)(directory, fileName));
|
|
251
|
-
if (stats.isFile()) {
|
|
252
|
-
appendFile = true;
|
|
253
|
-
if (stats.size > 50000) {
|
|
254
|
-
if (isDataFile(fileName)) {
|
|
255
|
-
debug('Skipping large data file %s with size %d', fileName, stats.size);
|
|
256
|
-
appendFile = false;
|
|
257
|
-
}
|
|
258
|
-
else {
|
|
259
|
-
debug('WARNING Large file %s with size %d', fileName, stats.size);
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
catch (error) {
|
|
265
|
-
console.warn(`Error checking file ${fileName}`);
|
|
266
|
-
console.warn(error);
|
|
267
|
-
}
|
|
268
|
-
if (appendFile)
|
|
269
|
-
result.push(fileName);
|
|
270
|
-
}
|
|
271
|
-
return result;
|
|
272
|
-
}
|
|
273
|
-
exports.filterFiles = filterFiles;
|
|
274
|
-
//# sourceMappingURL=FileIndex.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"FileIndex.js","sourceRoot":"","sources":["../../src/fulltext/FileIndex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAwC;AACxC,uDAAuC;AAEvC,oEAAqC;AACrC,oDAA4B;AAC5B,kDAA8B;AAC9B,2BAAgC;AAEhC,0EAAkD;AAClD,oEAA4C;AAC5C,4CAA6C;AAC7C,gFAAwD;AACxD,kEAA0C;AAC1C,mFAAgF;AAEhF,eAAS,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE;IAC7B,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACrB,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,IAAA,eAAS,EAAC,mBAAmB,CAAC,CAAC;AAE7C,iDAAiD;AACjD,MAAM,UAAU,GAAG,IAAA,eAAS,EAAC,yBAAyB,CAAC,CAAC;AAaxD,MAAa,SAAS;IACpB,YAAmB,QAA0B;QAA1B,aAAQ,GAAR,QAAQ,CAAkB;QAqFrC,mBAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAChD,CACE,SAAiB,EACjB,SAAmB,EACnB,OAAwB,EACxB,MAAe,EACf,KAAc,EACd,EAAE;YACF,MAAM,UAAU,GAAG,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,CAAC,CAAC;YAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;YAC3F,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE;gBAC1C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9B,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;aAC9C;QACH,CAAC,CACF,CAAC;QAEF,oCAAsC;QArGpC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAChB,uHAAuH,CACxH,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;QAC1C,uBAAA,IAAI,qBAAW,IAAI,CAAC,QAAQ,CAAC,OAAO,CAClC,kEAAkE,CACnE,MAAA,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,MAAM,CAAC,QAAkB,EAAE,KAAK,GAAG,EAAE;QACnC,MAAM,KAAK,GAAG,qGAAqG,CAAC;QAEpH,MAAM,UAAU,GAAG,IAAA,uBAAa,EAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QACjE,IAAI,KAAK,CAAC,OAAO;YACf,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;gBACtB,KAAK,CAAC,cAAc,EAAG,GAA6B,CAAC,SAAS,CAAC,CAAC;aACjE;QACH,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,CAAC;YAC7B,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,QAAQ,EAAE,GAAG,CAAC,SAAS;YACvB,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,WAAqB,EACrB,eAAqC,EACrC,eAAqC,EACrC,SAAS,GAAG,GAAG;QAEf,KAAK,MAAM,SAAS,IAAI,WAAW,EAAE;YACnC,IAAI;gBACF,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBACvC,MAAM,QAAQ,GAAG,MAAM,eAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBAC5C,MAAM,SAAS,GACb,QAAQ,KAAK,oBAAQ,CAAC,EAAE;oBACtB,CAAC,CAAC,MAAM,IAAA,6BAAmB,EAAC,SAAS,CAAC;oBACtC,CAAC,CAAC,MAAM,IAAA,0BAAgB,EAAC,SAAS,CAAC,CAAC;gBAExC,MAAM,iBAAiB,GAAG,MAAM,WAAW,CACzC,SAAS,EACT,SAAS,EACT,eAAe,EACf,eAAe,CAChB,CAAC;gBAEF,MAAM,OAAO,GAAG;oBACd,mBAAmB,EAAE,SAAS,CAAC,MAAM,GAAG,KAAM;oBAC9C,YAAY,EAAE,SAAS,CAAC,MAAM,GAAG,KAAM;iBACxC,CAAC;gBAEF,IAAI,OAAO,CAAC,YAAY,EAAE;oBACxB,KAAK,CAAC,4BAA4B,CAAC,CAAC;oBACpC,KAAK,CACH,8BAA8B,EAC9B,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CACvD,CAAC;iBACH;qBAAM;oBACL,KAAK,CAAC,6BAA6B,CAAC,CAAC;iBACtC;gBAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE;oBAC5D,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;oBAEzE,2CAA2C;oBAC3C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;iBACxD;gBAED,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBACrC,OAAO,CAAC,GAAG,CACT,WAAW,SAAS,CAAC,MAAM,aAAa,SAAS,OAAO,OAAO,GAAG,SAAS,IAAI,CAChF,CAAC;aACH;YAAC,OAAO,KAAK,EAAE;gBACd,OAAO,CAAC,KAAK,CAAC,8BAA8B,SAAS,GAAG,EAAE,KAAK,CAAC,CAAC;aAClE;SACF;IACH,CAAC;IAqBD,SAAS,CAAC,SAAiB,EAAE,QAAgB,EAAE,UAA0B,EAAE;QACzE,MAAM,EAAE,mBAAmB,GAAG,IAAI,EAAE,YAAY,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QACpE,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,mBAAI,CAAC,GAAG,CAAC,CAAC;QAEhD,IAAI;YACF,IAAI,KAAK,GAAG,IAAA,uBAAa,EAAC,cAAc,CAAC,CAAC;YAE1C,IAAI,YAAY,EAAE;gBAChB,MAAM,OAAO,GAAG,IAAA,sBAAY,EAAC,mBAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,mBAAmB,CAAC,CAAC;gBAClF,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,IAAA,uBAAa,EAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;aACrD;YAED,KAAK,CAAC,qCAAqC,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACrE,UAAU,CAAC,uBAAuB,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;YAErD,uBAAA,IAAI,yBAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;SACxD;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,IAAI,CAAC,2BAA2B,QAAQ,EAAE,CAAC,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SACrB;IACH,CAAC;CACF;AA9HD,8BA8HC;;AAED,SAAgB,gBAAgB,CAAC,aAAqB;IACpD,IAAA,gBAAM,EAAC,IAAA,eAAU,EAAC,aAAa,CAAC,EAAE,cAAc,aAAa,iBAAiB,CAAC,CAAC;IAChF,MAAM,QAAQ,GAAG,IAAI,wBAAO,CAAC,aAAa,CAAC,CAAC;IAC5C,OAAO,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC;AACjC,CAAC;AAJD,4CAIC;AAEM,KAAK,UAAU,cAAc,CAClC,WAAqB,EACrB,aAAqB,EACrB,eAA0B,EAC1B,eAA0B;IAE1B,IAAA,gBAAM,EAAC,CAAC,IAAA,eAAU,EAAC,aAAa,CAAC,EAAE,cAAc,aAAa,iBAAiB,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,IAAI,wBAAO,CAAC,aAAa,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC1C,MAAM,SAAS,CAAC,gBAAgB,CAAC,WAAW,EAAE,eAAe,EAAE,eAAe,CAAC,CAAC;IAChF,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,EAAE,CAAC,CAAC;IACpD,OAAO,SAAS,CAAC;AACnB,CAAC;AAZD,wCAYC;AAED,MAAM,sBAAsB,GAAa;IACvC,IAAI;IACJ,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,OAAO;IACP,KAAK;IACL,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,KAAK;IACL,IAAI;IACJ,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,SAAS;IACT,KAAK;IACL,IAAI;IACJ,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,IAAI;IACJ,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,KAAK;IACL,KAAK;IACL,IAAI;IACJ,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,KAAK;IACL,KAAK;IACL,MAAM;IACN,IAAI;IACJ,WAAW;IACX,KAAK;CACN,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;AAE1B,MAAM,oBAAoB,GAAa;IACrC,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,KAAK;IACL,KAAK;CACN,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;AAE1B,MAAM,YAAY,GAAG,CAAC,QAAgB,EAAE,EAAE;IACxC,OAAO,sBAAsB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AACtE,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,QAAgB,EAAE,EAAE;IACtC,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AACpE,CAAC,CAAC;AAEK,KAAK,UAAU,WAAW,CAC/B,SAAiB,EACjB,SAAmB,EACnB,eAA0B,EAC1B,eAA0B;IAE1B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;QAChC,IAAI,YAAY,CAAC,QAAQ,CAAC;YAAE,SAAS;QAErC,MAAM,WAAW,GAAG,IAAA,6DAA6B,EAAC,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,CAAC;QAC9F,IAAI,CAAC,WAAW;YAAE,SAAS;QAE3B,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI;YACF,MAAM,KAAK,GAAG,MAAM,IAAA,eAAI,EAAC,IAAA,gBAAI,EAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;YACpD,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE;gBAClB,UAAU,GAAG,IAAI,CAAC;gBAClB,IAAI,KAAK,CAAC,IAAI,GAAG,KAAM,EAAE;oBACvB,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE;wBACxB,KAAK,CAAC,0CAA0C,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;wBACxE,UAAU,GAAG,KAAK,CAAC;qBACpB;yBAAM;wBACL,KAAK,CAAC,oCAAoC,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;qBACnE;iBACF;aACF;SACF;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,IAAI,CAAC,uBAAuB,QAAQ,EAAE,CAAC,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SACrB;QAED,IAAI,UAAU;YAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KACvC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAnCD,kCAmCC"}
|
|
@@ -1,132 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
|
|
3
|
-
if (kind === "m") throw new TypeError("Private method is not writable");
|
|
4
|
-
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
|
|
5
|
-
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
|
|
6
|
-
return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
|
|
7
|
-
};
|
|
8
|
-
var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
|
9
|
-
if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
|
10
|
-
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
11
|
-
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
12
|
-
};
|
|
13
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
14
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
15
|
-
};
|
|
16
|
-
var _SourceIndex_insert, _SourceIndex_indexChunks;
|
|
17
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
-
exports.buildSourceIndex = exports.SourceIndex = void 0;
|
|
19
|
-
const debug_1 = __importDefault(require("debug"));
|
|
20
|
-
const text_splitter_1 = require("langchain/text_splitter");
|
|
21
|
-
const assert_1 = __importDefault(require("assert"));
|
|
22
|
-
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
|
23
|
-
const fs_1 = require("fs");
|
|
24
|
-
const promises_1 = require("fs/promises");
|
|
25
|
-
const path_1 = require("path");
|
|
26
|
-
const queryKeywords_1 = __importDefault(require("./queryKeywords"));
|
|
27
|
-
const debug = (0, debug_1.default)('appmap:source-index');
|
|
28
|
-
const TEXT_SPLITTER_LANGUAGE_EXTENSIONS = {
|
|
29
|
-
cpp: ['cpp', 'h', 'hpp', 'c', 'cc', 'cxx', 'hxx'],
|
|
30
|
-
go: ['go'],
|
|
31
|
-
java: ['java', 'jsp', 'jspx'],
|
|
32
|
-
js: ['js', 'ts', 'mjs', 'jsx', 'tsx', 'vue', 'svelte'],
|
|
33
|
-
php: ['php'],
|
|
34
|
-
proto: ['proto'],
|
|
35
|
-
python: ['py'],
|
|
36
|
-
rst: ['rst'],
|
|
37
|
-
ruby: ['rb', 'haml', 'erb'],
|
|
38
|
-
rust: ['rs'],
|
|
39
|
-
scala: ['scala'],
|
|
40
|
-
swift: ['swift'],
|
|
41
|
-
markdown: ['md'],
|
|
42
|
-
latex: ['tex'],
|
|
43
|
-
html: ['html'],
|
|
44
|
-
sol: ['sol'],
|
|
45
|
-
};
|
|
46
|
-
class SourceIndex {
|
|
47
|
-
constructor(database) {
|
|
48
|
-
this.database = database;
|
|
49
|
-
_SourceIndex_insert.set(this, void 0);
|
|
50
|
-
_SourceIndex_indexChunks.set(this, this.database.transaction((chunks, directory, fileName) => {
|
|
51
|
-
for (const chunk of chunks) {
|
|
52
|
-
const { from, to } = chunk.metadata.loc.lines;
|
|
53
|
-
try {
|
|
54
|
-
debug(`Indexing document ${fileName} from ${from} to ${to}`);
|
|
55
|
-
const terms = (0, queryKeywords_1.default)([fileName, chunk.pageContent]).join(' ');
|
|
56
|
-
__classPrivateFieldGet(this, _SourceIndex_insert, "f").run(directory, fileName, from, to, chunk.pageContent, terms);
|
|
57
|
-
}
|
|
58
|
-
catch (error) {
|
|
59
|
-
console.warn(`Error indexing document ${fileName} from ${from} to ${to}`);
|
|
60
|
-
console.warn(error);
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
}));
|
|
64
|
-
this.database.exec(`CREATE VIRTUAL TABLE code_snippets USING fts5(directory UNINDEXED, file_name UNINDEXED, from_line UNINDEXED, to_line UNINDEXED, snippet UNINDEXED, terms, tokenize = 'porter unicode61')`);
|
|
65
|
-
this.database.pragma('journal_mode = OFF');
|
|
66
|
-
this.database.pragma('synchronous = OFF');
|
|
67
|
-
__classPrivateFieldSet(this, _SourceIndex_insert, this.database.prepare('INSERT INTO code_snippets (directory, file_name, from_line, to_line, snippet, terms) VALUES (?, ?, ?, ?, ?, ?)'), "f");
|
|
68
|
-
}
|
|
69
|
-
close() {
|
|
70
|
-
this.database.close();
|
|
71
|
-
}
|
|
72
|
-
search(keywords, limit = 10) {
|
|
73
|
-
const query = `SELECT directory, file_name, from_line, to_line, snippet, (rank * -1) score FROM code_snippets WHERE code_snippets MATCH ? ORDER BY rank LIMIT ?`;
|
|
74
|
-
const searchExpr = (0, queryKeywords_1.default)(keywords).join(' OR ');
|
|
75
|
-
debug(`[SourceIndex] Searching for ${searchExpr}`);
|
|
76
|
-
const rows = this.database.prepare(query).all(searchExpr, limit);
|
|
77
|
-
if (debug.enabled)
|
|
78
|
-
rows.forEach((row) => {
|
|
79
|
-
debug(`[SourceIndex] Found row ${row.file_name}`);
|
|
80
|
-
});
|
|
81
|
-
return rows.map((row) => ({
|
|
82
|
-
directory: row.directory,
|
|
83
|
-
fileName: row.file_name,
|
|
84
|
-
from: row.from_line,
|
|
85
|
-
to: row.to_line,
|
|
86
|
-
content: row.snippet,
|
|
87
|
-
score: row.score,
|
|
88
|
-
}));
|
|
89
|
-
}
|
|
90
|
-
async indexFiles(files) {
|
|
91
|
-
for (const file of files) {
|
|
92
|
-
await this.indexFile(file);
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
async indexFile(file) {
|
|
96
|
-
const { directory, fileName } = file;
|
|
97
|
-
const extension = fileName.split('.').pop();
|
|
98
|
-
const language = Object.keys(TEXT_SPLITTER_LANGUAGE_EXTENSIONS).find((language) => TEXT_SPLITTER_LANGUAGE_EXTENSIONS[language].includes(extension));
|
|
99
|
-
let splitter;
|
|
100
|
-
if (language) {
|
|
101
|
-
splitter = text_splitter_1.RecursiveCharacterTextSplitter.fromLanguage(language);
|
|
102
|
-
}
|
|
103
|
-
else {
|
|
104
|
-
debug(`No language found for file: ${fileName}`);
|
|
105
|
-
splitter = new text_splitter_1.RecursiveCharacterTextSplitter();
|
|
106
|
-
}
|
|
107
|
-
const filePath = (0, path_1.join)(directory, fileName);
|
|
108
|
-
let fileContents;
|
|
109
|
-
try {
|
|
110
|
-
fileContents = await (0, promises_1.readFile)(filePath, 'utf-8');
|
|
111
|
-
}
|
|
112
|
-
catch (error) {
|
|
113
|
-
console.warn(`Error reading file ${filePath}`);
|
|
114
|
-
console.warn(error);
|
|
115
|
-
return;
|
|
116
|
-
}
|
|
117
|
-
const chunks = await splitter.createDocuments([fileContents]);
|
|
118
|
-
__classPrivateFieldGet(this, _SourceIndex_indexChunks, "f").call(this, chunks, directory, fileName);
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
exports.SourceIndex = SourceIndex;
|
|
122
|
-
_SourceIndex_insert = new WeakMap(), _SourceIndex_indexChunks = new WeakMap();
|
|
123
|
-
async function buildSourceIndex(indexFileName, files) {
|
|
124
|
-
(0, assert_1.default)(!(0, fs_1.existsSync)(indexFileName), `Index file ${indexFileName} already exists`);
|
|
125
|
-
const database = new better_sqlite3_1.default(indexFileName);
|
|
126
|
-
const sourceIndex = new SourceIndex(database);
|
|
127
|
-
await sourceIndex.indexFiles(files);
|
|
128
|
-
console.log(`Wrote file index to ${indexFileName}`);
|
|
129
|
-
return sourceIndex;
|
|
130
|
-
}
|
|
131
|
-
exports.buildSourceIndex = buildSourceIndex;
|
|
132
|
-
//# sourceMappingURL=SourceIndex.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"SourceIndex.js","sourceRoot":"","sources":["../../src/fulltext/SourceIndex.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AAAA,kDAA8B;AAC9B,2DAGiC;AAEjC,oDAA4B;AAC5B,oEAAqC;AACrC,2BAAgC;AAEhC,0CAAuC;AACvC,+BAA4B;AAC5B,oEAA4C;AAE5C,MAAM,KAAK,GAAG,IAAA,eAAS,EAAC,qBAAqB,CAAC,CAAC;AAE/C,MAAM,iCAAiC,GAAoD;IACzF,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC;IACjD,EAAE,EAAE,CAAC,IAAI,CAAC;IACV,IAAI,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC;IAC7B,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC;IACtD,GAAG,EAAE,CAAC,KAAK,CAAC;IACZ,KAAK,EAAE,CAAC,OAAO,CAAC;IAChB,MAAM,EAAE,CAAC,IAAI,CAAC;IACd,GAAG,EAAE,CAAC,KAAK,CAAC;IACZ,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC;IAC3B,IAAI,EAAE,CAAC,IAAI,CAAC;IACZ,KAAK,EAAE,CAAC,OAAO,CAAC;IAChB,KAAK,EAAE,CAAC,OAAO,CAAC;IAChB,QAAQ,EAAE,CAAC,IAAI,CAAC;IAChB,KAAK,EAAE,CAAC,KAAK,CAAC;IACd,IAAI,EAAE,CAAC,MAAM,CAAC;IACd,GAAG,EAAE,CAAC,KAAK,CAAC;CACb,CAAC;AAgBF,MAAa,WAAW;IACtB,YAAmB,QAA0B;QAA1B,aAAQ,GAAR,QAAQ,CAAkB;QAyC7C,sCAAsC;QA8BtC,mCAAe,IAAI,CAAC,QAAQ,CAAC,WAAW,CACtC,CAAC,MAAkB,EAAE,SAAiB,EAAE,QAAgB,EAAE,EAAE;YAC1D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;gBAC1B,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC;gBAE9C,IAAI;oBACF,KAAK,CAAC,qBAAqB,QAAQ,SAAS,IAAI,OAAO,EAAE,EAAE,CAAC,CAAC;oBAE7D,MAAM,KAAK,GAAG,IAAA,uBAAa,EAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACrE,uBAAA,IAAI,2BAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;iBAC3E;gBAAC,OAAO,KAAK,EAAE;oBACd,OAAO,CAAC,IAAI,CAAC,2BAA2B,QAAQ,SAAS,IAAI,OAAO,EAAE,EAAE,CAAC,CAAC;oBAC1E,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;iBACrB;aACF;QACH,CAAC,CACF,EAAC;QAtFA,IAAI,CAAC,QAAQ,CAAC,IAAI,CAChB,0LAA0L,CAC3L,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;QAC1C,uBAAA,IAAI,uBAAW,IAAI,CAAC,QAAQ,CAAC,OAAO,CAClC,gHAAgH,CACjH,MAAA,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,MAAM,CAAC,QAAkB,EAAE,KAAK,GAAG,EAAE;QACnC,MAAM,KAAK,GAAG,kJAAkJ,CAAC;QAEjK,MAAM,UAAU,GAAG,IAAA,uBAAa,EAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,KAAK,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QACjE,IAAI,KAAK,CAAC,OAAO;YACf,IAAI,CAAC,OAAO,CAAC,CAAC,GAAQ,EAAE,EAAE;gBACxB,KAAK,CAAC,2BAA2B,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;QACL,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,CAAC;YAC7B,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,QAAQ,EAAE,GAAG,CAAC,SAAS;YACvB,IAAI,EAAE,GAAG,CAAC,SAAS;YACnB,EAAE,EAAE,GAAG,CAAC,OAAO;YACf,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAA4B;QAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACxB,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;SAC5B;IACH,CAAC;IAID,KAAK,CAAC,SAAS,CAAC,IAAyB;QACvC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;QACrC,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;QAE5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAChF,iCAAiC,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CACnB,CAAC;QAC/C,IAAI,QAAwC,CAAC;QAC7C,IAAI,QAAQ,EAAE;YACZ,QAAQ,GAAG,8CAA8B,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;SAClE;aAAM;YACL,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;YACjD,QAAQ,GAAG,IAAI,8CAA8B,EAAE,CAAC;SACjD;QACD,MAAM,QAAQ,GAAG,IAAA,WAAI,EAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC3C,IAAI,YAAoB,CAAC;QACzB,IAAI;YACF,YAAY,GAAG,MAAM,IAAA,mBAAQ,EAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;SAClD;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,IAAI,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO;SACR;QAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;QAC9D,uBAAA,IAAI,gCAAa,MAAjB,IAAI,EAAc,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IACjD,CAAC;CAmBF;AAzFD,kCAyFC;;AAEM,KAAK,UAAU,gBAAgB,CACpC,aAAqB,EACrB,KAAuB;IAEvB,IAAA,gBAAM,EAAC,CAAC,IAAA,eAAU,EAAC,aAAa,CAAC,EAAE,cAAc,aAAa,iBAAiB,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,IAAI,wBAAO,CAAC,aAAa,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,EAAE,CAAC,CAAC;IACpD,OAAO,WAAW,CAAC;AACrB,CAAC;AAVD,4CAUC"}
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const debug_1 = __importDefault(require("debug"));
|
|
7
|
-
const utils_1 = require("../utils");
|
|
8
|
-
const node_child_process_1 = require("node:child_process");
|
|
9
|
-
const util_1 = require("util");
|
|
10
|
-
const debug = (0, debug_1.default)('appmap:listGitProjectFiles');
|
|
11
|
-
const exec = (0, util_1.promisify)(node_child_process_1.exec);
|
|
12
|
-
// Run git ls-files and git status to get a list of all git-managed files. By doing it this way,
|
|
13
|
-
// we automatically apply any .gitignore rules.
|
|
14
|
-
async function listGitProjectFiles(directory) {
|
|
15
|
-
const lsFiles = async () => {
|
|
16
|
-
try {
|
|
17
|
-
const { stdout } = await exec('git ls-files', {
|
|
18
|
-
cwd: directory,
|
|
19
|
-
maxBuffer: 1024 ** 2 * 20, // 20 MB
|
|
20
|
-
});
|
|
21
|
-
debug(stdout);
|
|
22
|
-
return stdout.split('\n').filter(Boolean);
|
|
23
|
-
}
|
|
24
|
-
catch (e) {
|
|
25
|
-
if ((0, utils_1.verbose)()) {
|
|
26
|
-
console.error('`git ls-files` failed');
|
|
27
|
-
console.error(e);
|
|
28
|
-
}
|
|
29
|
-
return [];
|
|
30
|
-
}
|
|
31
|
-
};
|
|
32
|
-
const statusFiles = async () => {
|
|
33
|
-
try {
|
|
34
|
-
const { stdout } = await exec('git status --porcelain', {
|
|
35
|
-
cwd: directory,
|
|
36
|
-
maxBuffer: 1024 ** 2 * 20, // 20 MB
|
|
37
|
-
});
|
|
38
|
-
debug(stdout);
|
|
39
|
-
return stdout
|
|
40
|
-
.split('\n')
|
|
41
|
-
.map((line) => {
|
|
42
|
-
// git status --porcelain output starts with 3 characters: staged status, unstaged status,
|
|
43
|
-
// and a space.
|
|
44
|
-
return line.slice(3);
|
|
45
|
-
})
|
|
46
|
-
.filter(Boolean);
|
|
47
|
-
}
|
|
48
|
-
catch (e) {
|
|
49
|
-
if ((0, utils_1.verbose)()) {
|
|
50
|
-
console.error('`git status --porcelain` failed');
|
|
51
|
-
console.error(e);
|
|
52
|
-
}
|
|
53
|
-
return [];
|
|
54
|
-
}
|
|
55
|
-
};
|
|
56
|
-
return Array.from(new Set([...(await lsFiles()), ...(await statusFiles())]));
|
|
57
|
-
}
|
|
58
|
-
exports.default = listGitProjectFiles;
|
|
59
|
-
//# sourceMappingURL=listGitProjectFIles.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"listGitProjectFIles.js","sourceRoot":"","sources":["../../src/fulltext/listGitProjectFIles.ts"],"names":[],"mappings":";;;;;AAAA,kDAA8B;AAE9B,oCAAmC;AACnC,2DAAoD;AACpD,+BAAiC;AAEjC,MAAM,KAAK,GAAG,IAAA,eAAS,EAAC,4BAA4B,CAAC,CAAC;AACtD,MAAM,IAAI,GAAG,IAAA,gBAAS,EAAC,yBAAM,CAAC,CAAC;AAE/B,gGAAgG;AAChG,+CAA+C;AAChC,KAAK,UAAU,mBAAmB,CAAC,SAAiB;IACjE,MAAM,OAAO,GAAG,KAAK,IAAuB,EAAE;QAC5C,IAAI;YACF,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE;gBAC5C,GAAG,EAAE,SAAS;gBACd,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ;aACpC,CAAC,CAAC;YAEH,KAAK,CAAC,MAAM,CAAC,CAAC;YAEd,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SAC3C;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,IAAA,eAAO,GAAE,EAAE;gBACb,OAAO,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;gBACvC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;YACD,OAAO,EAAE,CAAC;SACX;IACH,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,KAAK,IAAuB,EAAE;QAChD,IAAI;YACF,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE;gBACtD,GAAG,EAAE,SAAS;gBACd,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ;aACpC,CAAC,CAAC;YAEH,KAAK,CAAC,MAAM,CAAC,CAAC;YAEd,OAAO,MAAM;iBACV,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACZ,0FAA0F;gBAC1F,eAAe;gBACf,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,CAAC,CAAC;iBACD,MAAM,CAAC,OAAO,CAAC,CAAC;SACpB;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,IAAA,eAAO,GAAE,EAAE;gBACb,OAAO,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACjD,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;YACD,OAAO,EAAE,CAAC;SACX;IACH,CAAC,CAAC;IAEF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,MAAM,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/E,CAAC;AA/CD,sCA+CC"}
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const assert_1 = __importDefault(require("assert"));
|
|
7
|
-
const promises_1 = require("fs/promises");
|
|
8
|
-
const path_1 = require("path");
|
|
9
|
-
const IGNORE_DIRECTORIES = ['node_modules', 'vendor', 'tmp', 'build', 'dist', 'target'];
|
|
10
|
-
const DEFAULT_PROJECT_FILE_LIMIT = 1000;
|
|
11
|
-
// Produce a modest-sized listing of files in the project.
|
|
12
|
-
// Ignore a standard list of binary file extensions and directories that tend to be full of
|
|
13
|
-
// non-source files.
|
|
14
|
-
async function listProjectFiles(directory, fileLimit = DEFAULT_PROJECT_FILE_LIMIT) {
|
|
15
|
-
const files = new Array();
|
|
16
|
-
const ignoreDirectory = (dir) => IGNORE_DIRECTORIES.includes(dir);
|
|
17
|
-
// Perform a breadth-first traversal of a directory, collecting all non-binary files and
|
|
18
|
-
// applying the directory ignore list.
|
|
19
|
-
const processDir = async (dir) => {
|
|
20
|
-
const queue = [dir];
|
|
21
|
-
while (queue.length > 0 && files.length < fileLimit) {
|
|
22
|
-
const currentDir = queue.shift();
|
|
23
|
-
(0, assert_1.default)(currentDir, 'queue should not be empty');
|
|
24
|
-
const entries = await (0, promises_1.readdir)(currentDir, { withFileTypes: true });
|
|
25
|
-
for (const entry of entries) {
|
|
26
|
-
const path = (0, path_1.join)(currentDir, entry.name);
|
|
27
|
-
if (entry.isDirectory()) {
|
|
28
|
-
if (!ignoreDirectory(entry.name))
|
|
29
|
-
queue.push(path);
|
|
30
|
-
}
|
|
31
|
-
else
|
|
32
|
-
files.push((0, path_1.relative)(dir, path));
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
};
|
|
36
|
-
await processDir(directory);
|
|
37
|
-
return files;
|
|
38
|
-
}
|
|
39
|
-
exports.default = listProjectFiles;
|
|
40
|
-
//# sourceMappingURL=listProjectFiles.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"listProjectFiles.js","sourceRoot":"","sources":["../../src/fulltext/listProjectFiles.ts"],"names":[],"mappings":";;;;;AAAA,oDAA4B;AAC5B,0CAAsC;AACtC,+BAAsC;AAEtC,MAAM,kBAAkB,GAAG,CAAC,cAAc,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AAExF,MAAM,0BAA0B,GAAG,IAAI,CAAC;AAExC,0DAA0D;AAC1D,2FAA2F;AAC3F,oBAAoB;AACL,KAAK,UAAU,gBAAgB,CAC5C,SAAiB,EACjB,SAAS,GAAG,0BAA0B;IAEtC,MAAM,KAAK,GAAG,IAAI,KAAK,EAAU,CAAC;IAElC,MAAM,eAAe,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,kBAAkB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAE1E,wFAAwF;IACxF,sCAAsC;IACtC,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE;QACvC,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;QACpB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE;YACnD,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YACjC,IAAA,gBAAM,EAAC,UAAU,EAAE,2BAA2B,CAAC,CAAC;YAEhD,MAAM,OAAO,GAAG,MAAM,IAAA,kBAAO,EAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YACnE,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;gBAC3B,MAAM,IAAI,GAAG,IAAA,WAAI,EAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC1C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE;oBACvB,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC;wBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACpD;;oBAAM,KAAK,CAAC,IAAI,CAAC,IAAA,eAAQ,EAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;aACxC;SACF;IACH,CAAC,CAAC;IAEF,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC;IAE5B,OAAO,KAAK,CAAC;AACf,CAAC;AA7BD,mCA6BC"}
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SymbolRegexes = void 0;
|
|
4
|
-
const fs_1 = require("fs");
|
|
5
|
-
exports.SymbolRegexes = {
|
|
6
|
-
cs: /(((interface|class|enum|struct)\s+(?<symbol1>\w+))|((\s|^)(?!using|try|catch|if|while|do|for|switch)(?<!#define\s+?)(?<symbol2>[\w~$]+)\s*?\([^;)]*?\)[\w\s\d<>[\].:\n]*?{))/g,
|
|
7
|
-
cpp: /(((struct|enum|union|class)\s+(?<symbol1>\w+)\s*?\{)|(}\s*?(?<symbol2>\w+)\s*?;)|((\s|^)(?!try|catch|if|while|do|for|switch)(?<!#define\s+?)(?<symbol3>[\w~$]+)\s*?\([^;)]*?\)[\w\s\d<>[\].:\n]*?{))/g,
|
|
8
|
-
rs: /(struct|enum|union|trait|type|fn)\s+(?<symbol1>[\w\p{L}]+)/gu,
|
|
9
|
-
go: /((type\s+(?<symbol1>[\w\p{L}]+))|(func\s+?(\(.*?\)\s*?)?(?<symbol2>[\w\p{L}]+)\s*?\())/gu,
|
|
10
|
-
rb: /(((class|module)\s+(?<symbol1>\w+))|(def\s+?(?<symbol2>\w+)))/g,
|
|
11
|
-
py: /(class|def)\s+(?<symbol1>\w+)/g,
|
|
12
|
-
java: /(((class|@?interface|enum)\s+(?<symbol1>[\w$]+))|((\s|^)(?!try|catch|if|while|do|for|switch)(?<symbol2>[\w$]+)\s*?\([^;)]*?\)[\w\s\d<>[\].:\n]*?{))/g,
|
|
13
|
-
ts: /(((class|interface|enum|type|function)\s+(?<symbol1>[#$\w\p{L}]+))|((\s|^)(?!using|try|catch|if|while|do|for|switch)(?<symbol2>[#$\w\p{L}]+)\s*?\([^;)]*?\)[\w\s<>[\].:\n]*?\{)|((?<symbol3>[#$\w\p{L}]+)\s*?(=|:)\s*?\(.*?\)\s*?=>))/gu,
|
|
14
|
-
kt: /(((class|typealias)\s+(?<symbol1>[\w_]+))|(fun\s+?(<.+?>\s+)?(.*?\.)?(?<symbol2>\w+)))/g,
|
|
15
|
-
php: /(class|trait|function)\s+(?<symbol1>[\w_$]+)/g,
|
|
16
|
-
};
|
|
17
|
-
const genericSymbolRegex = /(((^|\s)(?!using|try|catch|if|while|do|for|switch)(?<symbol1>[#$\w\p{L}~]+)\s*?\(([^;)])*?\)[\w\s<>[\].:\n]*?\{)|(^(?!.*?(?:#|\/\/|"|')).*?(interface|class|enum|struct|union|trait|type(alias|def)?|fu?nc?(tion)?|module|def)\s+?(?<symbol2>[#$\w\p{L}]+))|((?<symbol3>[#$\w\p{L}~]+)\s*?=\s*?[\w\s<>[\].:\n]*?\{))/gmu;
|
|
18
|
-
// Define aliases for common file extensions
|
|
19
|
-
['js', 'jsx', 'ts', 'tsx', 'vue', 'svelte'].forEach((ext) => {
|
|
20
|
-
exports.SymbolRegexes[ext] = exports.SymbolRegexes.ts;
|
|
21
|
-
});
|
|
22
|
-
['c', 'cc', 'cxx', 'h', 'hpp', 'cpp', 'hxx'].forEach((ext) => {
|
|
23
|
-
exports.SymbolRegexes[ext] = exports.SymbolRegexes.cpp;
|
|
24
|
-
});
|
|
25
|
-
function getMatches(source, regex) {
|
|
26
|
-
var _a, _b;
|
|
27
|
-
const results = [];
|
|
28
|
-
const matches = source.matchAll(regex);
|
|
29
|
-
for (const match of matches) {
|
|
30
|
-
const { groups } = match;
|
|
31
|
-
const symbol = (_b = (_a = groups === null || groups === void 0 ? void 0 : groups.symbol1) !== null && _a !== void 0 ? _a : groups === null || groups === void 0 ? void 0 : groups.symbol2) !== null && _b !== void 0 ? _b : groups === null || groups === void 0 ? void 0 : groups.symbol3;
|
|
32
|
-
if (symbol)
|
|
33
|
-
results.push(symbol);
|
|
34
|
-
}
|
|
35
|
-
return results;
|
|
36
|
-
}
|
|
37
|
-
function querySymbols(path, allowGeneric = true) {
|
|
38
|
-
var _a;
|
|
39
|
-
const fileExtension = (_a = path.split('.').pop()) === null || _a === void 0 ? void 0 : _a.toLowerCase();
|
|
40
|
-
let regex = allowGeneric ? genericSymbolRegex : undefined;
|
|
41
|
-
if (fileExtension && fileExtension in exports.SymbolRegexes) {
|
|
42
|
-
regex = exports.SymbolRegexes[fileExtension];
|
|
43
|
-
}
|
|
44
|
-
if (regex) {
|
|
45
|
-
try {
|
|
46
|
-
// readFileSync performs 2x faster than its async counterpart in this case.
|
|
47
|
-
const content = (0, fs_1.readFileSync)(path, 'utf-8');
|
|
48
|
-
return getMatches(content, regex);
|
|
49
|
-
}
|
|
50
|
-
catch (error) {
|
|
51
|
-
console.warn(`Error reading file ${path}`);
|
|
52
|
-
console.warn(error);
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
return [];
|
|
56
|
-
}
|
|
57
|
-
exports.default = querySymbols;
|
|
58
|
-
//# sourceMappingURL=querySymbols.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"querySymbols.js","sourceRoot":"","sources":["../../src/fulltext/querySymbols.ts"],"names":[],"mappings":";;;AAAA,2BAAkC;AAErB,QAAA,aAAa,GAA2B;IACnD,EAAE,EAAE,+KAA+K;IACnL,GAAG,EAAE,uMAAuM;IAC5M,EAAE,EAAE,8DAA8D;IAClE,EAAE,EAAE,0FAA0F;IAC9F,EAAE,EAAE,gEAAgE;IACpE,EAAE,EAAE,gCAAgC;IACpC,IAAI,EAAE,sJAAsJ;IAC5J,EAAE,EAAE,yOAAyO;IAC7O,EAAE,EAAE,yFAAyF;IAC7F,GAAG,EAAE,+CAA+C;CACrD,CAAC;AAEF,MAAM,kBAAkB,GACtB,yTAAyT,CAAC;AAE5T,4CAA4C;AAC5C,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;IAC1D,qBAAa,CAAC,GAAG,CAAC,GAAG,qBAAa,CAAC,EAAE,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;IAC3D,qBAAa,CAAC,GAAG,CAAC,GAAG,qBAAa,CAAC,GAAG,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,MAAc,EAAE,KAAa;;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEvC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE;QAC3B,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;QACzB,MAAM,MAAM,GAAG,MAAA,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,OAAO,mCAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,OAAO,mCAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,OAAO,CAAC;QAErE,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;KAClC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAwB,YAAY,CAAC,IAAY,EAAE,YAAY,GAAG,IAAI;;IACpE,MAAM,aAAa,GAAG,MAAA,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,0CAAE,WAAW,EAAE,CAAC;IAE3D,IAAI,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1D,IAAI,aAAa,IAAI,aAAa,IAAI,qBAAa,EAAE;QACnD,KAAK,GAAG,qBAAa,CAAC,aAAa,CAAC,CAAC;KACtC;IAED,IAAI,KAAK,EAAE;QACT,IAAI;YACF,2EAA2E;YAC3E,MAAM,OAAO,GAAG,IAAA,iBAAY,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC5C,OAAO,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;SACnC;QAAC,OAAO,KAAK,EAAE;YACd,OAAO,CAAC,IAAI,CAAC,sBAAsB,IAAI,EAAE,CAAC,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;SACrB;KACF;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AApBD,+BAoBC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"withIndex.js","sourceRoot":"","sources":["../../src/fulltext/withIndex.ts"],"names":[],"mappings":";AAAA,kGAAkG;AAClG,8FAA8F;;AAE9F,0CAA0C;AAC1C,2BAA4B;AAC5B,+BAA4B;AAM5B,yCAAyC;AAC1B,KAAK,UAAU,SAAS,CACrC,SAAiB,EACjB,OAA0C,EAC1C,QAAsC;IAEtC,MAAM,MAAM,GAAG,MAAM,IAAA,kBAAO,EAAC,IAAA,WAAI,EAAC,IAAA,WAAM,GAAE,EAAE,UAAU,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC5F,MAAM,SAAS,GAAG,IAAA,WAAI,EAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAE/C,MAAM,KAAK,GAAG,CAAC,KAAQ,EAAE,EAAE;QACzB,IAAI;YACF,KAAK,CAAC,KAAK,EAAE,CAAC;SACf;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;SACpB;IACH,CAAC,CAAC;IACF,MAAM,UAAU,GAAG,KAAK,IAAI,EAAE,CAC5B,MAAM,IAAA,aAAE,EAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAE3E,IAAI,KAAoB,CAAC;IACzB,IAAI;QACF,KAAK,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC;QACjC,OAAO,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;KAC9B;YAAS;QACR,IAAI,KAAK;YAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACxB,MAAM,UAAU,EAAE,CAAC;KACpB;AACH,CAAC;AA1BD,4BA0BC"}
|
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const navie_1 = require("@appland/navie");
|
|
7
|
-
const withIndex_1 = __importDefault(require("../../fulltext/withIndex"));
|
|
8
|
-
const SourceIndex_1 = require("../../fulltext/SourceIndex");
|
|
9
|
-
const collectContext_1 = require("./collectContext");
|
|
10
|
-
class SourceCollector {
|
|
11
|
-
constructor(keywords, fileSearchResponse) {
|
|
12
|
-
this.keywords = keywords;
|
|
13
|
-
this.fileSearchResponse = fileSearchResponse;
|
|
14
|
-
}
|
|
15
|
-
async collectContext(charLimit) {
|
|
16
|
-
const sourceIndexDocuments = await (0, withIndex_1.default)('source', (indexFileName) => (0, SourceIndex_1.buildSourceIndex)(indexFileName, this.fileSearchResponse), (index) => index.search(this.keywords, Math.round(charLimit / collectContext_1.CHARS_PER_SNIPPET)));
|
|
17
|
-
const buildLocation = (doc) => {
|
|
18
|
-
return `${doc.fileName}:${doc.from}-${doc.to}`;
|
|
19
|
-
};
|
|
20
|
-
return sourceIndexDocuments.map((doc) => ({
|
|
21
|
-
directory: doc.directory,
|
|
22
|
-
type: navie_1.ContextV2.ContextItemType.CodeSnippet,
|
|
23
|
-
content: doc.content,
|
|
24
|
-
location: buildLocation(doc),
|
|
25
|
-
}));
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
exports.default = SourceCollector;
|
|
29
|
-
//# sourceMappingURL=SourceCollector.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"SourceCollector.js","sourceRoot":"","sources":["../../../src/rpc/explain/SourceCollector.ts"],"names":[],"mappings":";;;;;AAAA,0CAA2C;AAE3C,yEAAiD;AACjD,4DAAgF;AAChF,qDAAqD;AAErD,MAAqB,eAAe;IAClC,YAAoB,QAAkB,EAAU,kBAAoC;QAAhE,aAAQ,GAAR,QAAQ,CAAU;QAAU,uBAAkB,GAAlB,kBAAkB,CAAkB;IAAG,CAAC;IAExF,KAAK,CAAC,cAAc,CAAC,SAAiB;QACpC,MAAM,oBAAoB,GAAG,MAAM,IAAA,mBAAS,EAC1C,QAAQ,EACR,CAAC,aAAqB,EAAE,EAAE,CAAC,IAAA,8BAAgB,EAAC,aAAa,EAAE,IAAI,CAAC,kBAAkB,CAAC,EACnF,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,kCAAiB,CAAC,CAAC,CAClF,CAAC;QAEF,MAAM,aAAa,GAAG,CAAC,GAAqB,EAAE,EAAE;YAC9C,OAAO,GAAG,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;QACjD,CAAC,CAAC;QAEF,OAAO,oBAAoB,CAAC,GAAG,CAAC,CAAC,GAAqB,EAAE,EAAE,CAAC,CAAC;YAC1D,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,IAAI,EAAE,iBAAS,CAAC,eAAe,CAAC,WAAW;YAC3C,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC;SAC7B,CAAC,CAAC,CAAC;IACN,CAAC;CACF;AArBD,kCAqBC"}
|