clanka 0.26.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -3
- package/dist/Acp.test.js +2 -3
- package/dist/Acp.test.js.map +1 -1
- package/dist/AcpImage.test.js +0 -1
- package/dist/AcpImage.test.js.map +1 -1
- package/dist/Agent.d.ts.map +1 -1
- package/dist/Agent.js +7 -7
- package/dist/Agent.js.map +1 -1
- package/dist/Agent.test.js +7 -7
- package/dist/Agent.test.js.map +1 -1
- package/dist/AgentBashJobs.test.js +7 -7
- package/dist/AgentBashJobs.test.js.map +1 -1
- package/dist/AgentCallerConfig.test.js +0 -1
- package/dist/AgentCallerConfig.test.js.map +1 -1
- package/dist/AgentExecutor.d.ts +2 -3
- package/dist/AgentExecutor.d.ts.map +1 -1
- package/dist/AgentExecutor.js +4 -13
- package/dist/AgentExecutor.js.map +1 -1
- package/dist/AgentExecutor.test.js +20 -20
- package/dist/AgentExecutor.test.js.map +1 -1
- package/dist/AgentImage.test.js +1 -2
- package/dist/AgentImage.test.js.map +1 -1
- package/dist/AgentSkills.test.js +0 -2
- package/dist/AgentSkills.test.js.map +1 -1
- package/dist/AgentSubagents.test.js +57 -29
- package/dist/AgentSubagents.test.js.map +1 -1
- package/dist/AgentTools.d.ts +6 -238
- package/dist/AgentTools.d.ts.map +1 -1
- package/dist/AgentTools.js +11 -39
- package/dist/AgentTools.js.map +1 -1
- package/dist/ClaudeAgent.test.js +0 -1
- package/dist/ClaudeAgent.test.js.map +1 -1
- package/dist/ScriptFormatting.d.ts +5 -0
- package/dist/ScriptFormatting.d.ts.map +1 -1
- package/dist/ScriptFormatting.js +255 -53
- package/dist/ScriptFormatting.js.map +1 -1
- package/dist/ScriptFormatting.test.js +16 -0
- package/dist/ScriptFormatting.test.js.map +1 -1
- package/dist/bin/{Latch-DOdbsGMQ.mjs → Latch-BE6_TYRL.mjs} +1 -1
- package/dist/bin/{McpClient-B-FFG6Pg.mjs → McpClient-DCt4v5W5.mjs} +2 -2
- package/dist/bin/{OutputFormatter-DhhP2i-7.mjs → OutputFormatter-DaFv_C_I.mjs} +2 -3
- package/dist/bin/{Schema-CFdSO6iH.mjs → Schema-B3MiG1aD.mjs} +204 -568
- package/dist/bin/cli.mjs +5106 -2479
- package/dist/bin/{upgrade-C4aOEWJH.mjs → upgrade-WqgIU_21.mjs} +3 -3
- package/dist/cli.js +2 -25
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +0 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -8
- package/dist/index.js.map +1 -1
- package/package.json +1 -7
- package/src/Acp.test.ts +2 -3
- package/src/AcpImage.test.ts +0 -1
- package/src/Agent.test.ts +10 -7
- package/src/Agent.ts +8 -10
- package/src/AgentBashJobs.test.ts +9 -9
- package/src/AgentCallerConfig.test.ts +0 -1
- package/src/AgentExecutor.test.ts +20 -20
- package/src/AgentExecutor.ts +4 -16
- package/src/AgentImage.test.ts +1 -2
- package/src/AgentSkills.test.ts +0 -2
- package/src/AgentSubagents.test.ts +77 -34
- package/src/AgentTools.ts +19 -51
- package/src/ClaudeAgent.test.ts +0 -1
- package/src/ScriptFormatting.test.ts +20 -0
- package/src/ScriptFormatting.ts +234 -47
- package/src/cli.ts +1 -56
- package/src/index.ts +0 -10
- package/dist/ChunkRepo.d.ts +0 -173
- package/dist/ChunkRepo.d.ts.map +0 -1
- package/dist/ChunkRepo.js +0 -168
- package/dist/ChunkRepo.js.map +0 -1
- package/dist/ChunkRepo.test.d.ts +0 -2
- package/dist/ChunkRepo.test.d.ts.map +0 -1
- package/dist/ChunkRepo.test.js +0 -63
- package/dist/ChunkRepo.test.js.map +0 -1
- package/dist/CodeChunker.d.ts +0 -84
- package/dist/CodeChunker.d.ts.map +0 -1
- package/dist/CodeChunker.js +0 -658
- package/dist/CodeChunker.js.map +0 -1
- package/dist/CodeChunker.test.d.ts +0 -2
- package/dist/CodeChunker.test.d.ts.map +0 -1
- package/dist/CodeChunker.test.js +0 -434
- package/dist/CodeChunker.test.js.map +0 -1
- package/dist/SemanticSearch/Service.d.ts +0 -31
- package/dist/SemanticSearch/Service.d.ts.map +0 -1
- package/dist/SemanticSearch/Service.js +0 -29
- package/dist/SemanticSearch/Service.js.map +0 -1
- package/dist/SemanticSearch.d.ts +0 -34
- package/dist/SemanticSearch.d.ts.map +0 -1
- package/dist/SemanticSearch.js +0 -171
- package/dist/SemanticSearch.js.map +0 -1
- package/dist/SemanticSearch.test.d.ts +0 -2
- package/dist/SemanticSearch.test.d.ts.map +0 -1
- package/dist/SemanticSearch.test.js +0 -34
- package/dist/SemanticSearch.test.js.map +0 -1
- package/dist/Sqlite.d.ts +0 -15
- package/dist/Sqlite.d.ts.map +0 -1
- package/dist/Sqlite.js +0 -44
- package/dist/Sqlite.js.map +0 -1
- package/dist/bin/AiError-DBaKz35C.mjs +0 -2723
- package/dist/bin/SemanticSearch-C3a7LAQr.mjs +0 -2558
- package/src/ChunkRepo.test.ts +0 -89
- package/src/ChunkRepo.ts +0 -260
- package/src/CodeChunker.test.ts +0 -493
- package/src/CodeChunker.ts +0 -1018
- package/src/SemanticSearch/Service.ts +0 -50
- package/src/SemanticSearch.test.ts +0 -51
- package/src/SemanticSearch.ts +0 -282
- package/src/Sqlite.ts +0 -60
package/dist/CodeChunker.js
DELETED
|
@@ -1,658 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @since 1.0.0
|
|
3
|
-
*/
|
|
4
|
-
import * as Array from "effect/Array";
|
|
5
|
-
import * as Effect from "effect/Effect";
|
|
6
|
-
import * as FileSystem from "effect/FileSystem";
|
|
7
|
-
import { pipe } from "effect/Function";
|
|
8
|
-
import * as Layer from "effect/Layer";
|
|
9
|
-
import * as Path from "effect/Path";
|
|
10
|
-
import * as Context from "effect/Context";
|
|
11
|
-
import * as Stream from "effect/Stream";
|
|
12
|
-
import * as ChildProcess from "effect/process/ChildProcess";
|
|
13
|
-
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
|
|
14
|
-
import TreeSitter, {} from "tree-sitter";
|
|
15
|
-
import TreeSitterJavaScript from "tree-sitter-javascript";
|
|
16
|
-
import TreeSitterTypeScript from "tree-sitter-typescript";
|
|
17
|
-
/**
|
|
18
|
-
* @since 1.0.0
|
|
19
|
-
* @category Services
|
|
20
|
-
*/
|
|
21
|
-
export class CodeChunker extends Context.Service()("clanka/CodeChunker") {
|
|
22
|
-
}
|
|
23
|
-
const sourceExtensions = new Set([
|
|
24
|
-
"c",
|
|
25
|
-
"cc",
|
|
26
|
-
"cpp",
|
|
27
|
-
"cs",
|
|
28
|
-
"css",
|
|
29
|
-
"cts",
|
|
30
|
-
"cxx",
|
|
31
|
-
"go",
|
|
32
|
-
"gql",
|
|
33
|
-
"graphql",
|
|
34
|
-
"h",
|
|
35
|
-
"hpp",
|
|
36
|
-
"html",
|
|
37
|
-
"ini",
|
|
38
|
-
"java",
|
|
39
|
-
"js",
|
|
40
|
-
"jsx",
|
|
41
|
-
"kt",
|
|
42
|
-
"kts",
|
|
43
|
-
"less",
|
|
44
|
-
"lua",
|
|
45
|
-
"mjs",
|
|
46
|
-
"mts",
|
|
47
|
-
"php",
|
|
48
|
-
"py",
|
|
49
|
-
"rb",
|
|
50
|
-
"rs",
|
|
51
|
-
"sass",
|
|
52
|
-
"scala",
|
|
53
|
-
"scss",
|
|
54
|
-
"sh",
|
|
55
|
-
"sql",
|
|
56
|
-
"svelte",
|
|
57
|
-
"swift",
|
|
58
|
-
"ts",
|
|
59
|
-
"tsx",
|
|
60
|
-
"vue",
|
|
61
|
-
"xml",
|
|
62
|
-
"zsh",
|
|
63
|
-
]);
|
|
64
|
-
const documentationExtensions = new Set([
|
|
65
|
-
"adoc",
|
|
66
|
-
"asciidoc",
|
|
67
|
-
"md",
|
|
68
|
-
"mdx",
|
|
69
|
-
"rst",
|
|
70
|
-
"txt",
|
|
71
|
-
]);
|
|
72
|
-
const ignoredDirectories = new Set([
|
|
73
|
-
".git",
|
|
74
|
-
".next",
|
|
75
|
-
".nuxt",
|
|
76
|
-
".svelte-kit",
|
|
77
|
-
".turbo",
|
|
78
|
-
"build",
|
|
79
|
-
"coverage",
|
|
80
|
-
"dist",
|
|
81
|
-
"node_modules",
|
|
82
|
-
"target",
|
|
83
|
-
]);
|
|
84
|
-
const normalizePath = (path) => path.replace(/\\/g, "/");
|
|
85
|
-
const normalizeText = (content) => content.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
|
|
86
|
-
const meaningfulLinePattern = /[^\s\p{P}]/u;
|
|
87
|
-
const isMeaningfulLine = (line) => meaningfulLinePattern.test(line);
|
|
88
|
-
const languageByExtension = new Map([
|
|
89
|
-
["js", TreeSitterJavaScript],
|
|
90
|
-
["jsx", TreeSitterJavaScript],
|
|
91
|
-
["ts", TreeSitterTypeScript.typescript],
|
|
92
|
-
["tsx", TreeSitterTypeScript.tsx],
|
|
93
|
-
]);
|
|
94
|
-
// const ignoredTopLevelNodeTypes = new Set(["comment", "import_statement"])
|
|
95
|
-
/**
|
|
96
|
-
* @since 1.0.0
|
|
97
|
-
* @category Predicates
|
|
98
|
-
*/
|
|
99
|
-
export const isProbablyMinified = (content) => {
|
|
100
|
-
const normalized = normalizeText(content);
|
|
101
|
-
if (normalized.length < 2_000) {
|
|
102
|
-
return false;
|
|
103
|
-
}
|
|
104
|
-
const lines = normalized.split("\n");
|
|
105
|
-
if (lines.length <= 2) {
|
|
106
|
-
return true;
|
|
107
|
-
}
|
|
108
|
-
let longLines = 0;
|
|
109
|
-
for (const line of lines) {
|
|
110
|
-
if (line.length >= 300) {
|
|
111
|
-
longLines++;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
return lines.length <= 20 && longLines / lines.length >= 0.8;
|
|
115
|
-
};
|
|
116
|
-
/**
|
|
117
|
-
* @since 1.0.0
|
|
118
|
-
* @category Predicates
|
|
119
|
-
*/
|
|
120
|
-
export const isMeaningfulFile = (path) => {
|
|
121
|
-
const normalizedPath = normalizePath(path);
|
|
122
|
-
const lowercasePath = normalizedPath.toLowerCase();
|
|
123
|
-
const parts = lowercasePath.split("/");
|
|
124
|
-
const fileName = parts.at(-1);
|
|
125
|
-
if (fileName === undefined || fileName.length === 0) {
|
|
126
|
-
return false;
|
|
127
|
-
}
|
|
128
|
-
if (parts.some((part) => ignoredDirectories.has(part))) {
|
|
129
|
-
return false;
|
|
130
|
-
}
|
|
131
|
-
if (/\.min\.(?:css|js)$/i.test(fileName)) {
|
|
132
|
-
return false;
|
|
133
|
-
}
|
|
134
|
-
const extensionIndex = fileName.lastIndexOf(".");
|
|
135
|
-
if (extensionIndex === -1) {
|
|
136
|
-
return false;
|
|
137
|
-
}
|
|
138
|
-
const extension = fileName.slice(extensionIndex + 1);
|
|
139
|
-
return (sourceExtensions.has(extension) || documentationExtensions.has(extension));
|
|
140
|
-
};
|
|
141
|
-
const resolveChunkSettings = (options) => {
|
|
142
|
-
const chunkSize = Math.max(1, options.chunkSize);
|
|
143
|
-
const chunkOverlap = Math.max(0, Math.min(chunkSize - 1, options.chunkOverlap));
|
|
144
|
-
const chunkMaxCharacters = options.chunkMaxCharacters !== undefined &&
|
|
145
|
-
Number.isFinite(options.chunkMaxCharacters)
|
|
146
|
-
? Math.max(1, Math.floor(options.chunkMaxCharacters))
|
|
147
|
-
: Number.POSITIVE_INFINITY;
|
|
148
|
-
return {
|
|
149
|
-
chunkSize,
|
|
150
|
-
chunkOverlap,
|
|
151
|
-
chunkMaxCharacters,
|
|
152
|
-
};
|
|
153
|
-
};
|
|
154
|
-
const getPathExtension = (path) => {
|
|
155
|
-
const fileName = path.split("/").at(-1);
|
|
156
|
-
if (fileName === undefined) {
|
|
157
|
-
return undefined;
|
|
158
|
-
}
|
|
159
|
-
const extensionIndex = fileName.lastIndexOf(".");
|
|
160
|
-
if (extensionIndex === -1) {
|
|
161
|
-
return undefined;
|
|
162
|
-
}
|
|
163
|
-
return fileName.slice(extensionIndex + 1).toLowerCase();
|
|
164
|
-
};
|
|
165
|
-
const resolveAstLanguage = (path) => {
|
|
166
|
-
const extension = getPathExtension(path);
|
|
167
|
-
if (extension === undefined) {
|
|
168
|
-
return undefined;
|
|
169
|
-
}
|
|
170
|
-
return languageByExtension.get(extension);
|
|
171
|
-
};
|
|
172
|
-
const lineRangeFromNode = (node) => {
|
|
173
|
-
const startLine = node.startPosition.row + 1;
|
|
174
|
-
const endLine = Math.max(startLine, node.endPosition.row + 1);
|
|
175
|
-
return {
|
|
176
|
-
startLine,
|
|
177
|
-
endLine,
|
|
178
|
-
};
|
|
179
|
-
};
|
|
180
|
-
const hasOnlyWhitespaceLines = (lines, startLine, endLine) => {
|
|
181
|
-
if (startLine > endLine) {
|
|
182
|
-
return true;
|
|
183
|
-
}
|
|
184
|
-
for (let lineIndex = startLine; lineIndex <= endLine; lineIndex++) {
|
|
185
|
-
if ((lines[lineIndex - 1] ?? "").trim().length > 0) {
|
|
186
|
-
return false;
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
return true;
|
|
190
|
-
};
|
|
191
|
-
const lineRangeWithLeadingComments = (node, siblings, nodeIndex, lines) => {
|
|
192
|
-
const baseRange = lineRangeFromNode(node);
|
|
193
|
-
let startLine = baseRange.startLine;
|
|
194
|
-
for (let index = nodeIndex - 1; index >= 0; index--) {
|
|
195
|
-
const sibling = siblings[index];
|
|
196
|
-
if (sibling.type !== "comment") {
|
|
197
|
-
break;
|
|
198
|
-
}
|
|
199
|
-
const commentRange = lineRangeFromNode(sibling);
|
|
200
|
-
if (!hasOnlyWhitespaceLines(lines, commentRange.endLine + 1, startLine - 1)) {
|
|
201
|
-
break;
|
|
202
|
-
}
|
|
203
|
-
startLine = commentRange.startLine;
|
|
204
|
-
}
|
|
205
|
-
return {
|
|
206
|
-
startLine,
|
|
207
|
-
endLine: baseRange.endLine,
|
|
208
|
-
};
|
|
209
|
-
};
|
|
210
|
-
const normalizeLineRange = (range, lineCount) => {
|
|
211
|
-
const startLine = Math.max(1, Math.min(lineCount, range.startLine));
|
|
212
|
-
const endLine = Math.max(1, Math.min(lineCount, range.endLine));
|
|
213
|
-
if (endLine < startLine) {
|
|
214
|
-
return undefined;
|
|
215
|
-
}
|
|
216
|
-
return {
|
|
217
|
-
startLine,
|
|
218
|
-
endLine,
|
|
219
|
-
};
|
|
220
|
-
};
|
|
221
|
-
const lineLengthPrefixSums = (lines) => {
|
|
222
|
-
const sums = [0];
|
|
223
|
-
for (let index = 0; index < lines.length; index++) {
|
|
224
|
-
sums.push(sums[index] + lines[index].length);
|
|
225
|
-
}
|
|
226
|
-
return sums;
|
|
227
|
-
};
|
|
228
|
-
const lineRangeCharacterLength = (prefixSums, range) => prefixSums[range.endLine] -
|
|
229
|
-
prefixSums[range.startLine - 1] +
|
|
230
|
-
(range.endLine - range.startLine);
|
|
231
|
-
const resolveSegmentEndLine = (options) => {
|
|
232
|
-
if (options.settings.chunkMaxCharacters === Number.POSITIVE_INFINITY) {
|
|
233
|
-
return options.maxEndLine;
|
|
234
|
-
}
|
|
235
|
-
let endLine = options.maxEndLine;
|
|
236
|
-
while (endLine > options.startLine &&
|
|
237
|
-
lineRangeCharacterLength(options.prefixSums, {
|
|
238
|
-
startLine: options.startLine,
|
|
239
|
-
endLine,
|
|
240
|
-
}) > options.settings.chunkMaxCharacters) {
|
|
241
|
-
endLine--;
|
|
242
|
-
}
|
|
243
|
-
return endLine;
|
|
244
|
-
};
|
|
245
|
-
const splitRange = (range, settings, prefixSums) => {
|
|
246
|
-
const lineCount = range.endLine - range.startLine + 1;
|
|
247
|
-
if (lineCount <= settings.chunkSize &&
|
|
248
|
-
lineRangeCharacterLength(prefixSums, range) <= settings.chunkMaxCharacters) {
|
|
249
|
-
return [range];
|
|
250
|
-
}
|
|
251
|
-
const out = [];
|
|
252
|
-
for (let startLine = range.startLine; startLine <= range.endLine;) {
|
|
253
|
-
const maxEndLine = Math.min(range.endLine, startLine + settings.chunkSize - 1);
|
|
254
|
-
const endLine = resolveSegmentEndLine({
|
|
255
|
-
startLine,
|
|
256
|
-
maxEndLine,
|
|
257
|
-
settings,
|
|
258
|
-
prefixSums,
|
|
259
|
-
});
|
|
260
|
-
out.push({
|
|
261
|
-
startLine,
|
|
262
|
-
endLine,
|
|
263
|
-
});
|
|
264
|
-
if (endLine >= range.endLine) {
|
|
265
|
-
break;
|
|
266
|
-
}
|
|
267
|
-
startLine = Math.max(startLine + 1, endLine - settings.chunkOverlap + 1);
|
|
268
|
-
}
|
|
269
|
-
return out;
|
|
270
|
-
};
|
|
271
|
-
const nodeText = (node) => {
|
|
272
|
-
if (node === null) {
|
|
273
|
-
return undefined;
|
|
274
|
-
}
|
|
275
|
-
const value = node.text.trim().replace(/\s+/g, " ");
|
|
276
|
-
return value.length === 0 ? undefined : value;
|
|
277
|
-
};
|
|
278
|
-
const nodeFieldText = (node, fieldName) => nodeText(node.childForFieldName(fieldName));
|
|
279
|
-
const isNamespaceNode = (node) => node.type === "internal_module" || node.type === "module";
|
|
280
|
-
const unwrapDeclarationNode = (node) => {
|
|
281
|
-
let current = node;
|
|
282
|
-
while (true) {
|
|
283
|
-
if (current.type === "export_statement") {
|
|
284
|
-
const declaration = current.childForFieldName("declaration") ?? current.namedChildren[0];
|
|
285
|
-
if (declaration === undefined) {
|
|
286
|
-
return current;
|
|
287
|
-
}
|
|
288
|
-
current = declaration;
|
|
289
|
-
continue;
|
|
290
|
-
}
|
|
291
|
-
if (current.type === "ambient_declaration") {
|
|
292
|
-
const declaration = current.namedChildren.find((child) => child.type.endsWith("_declaration") || isNamespaceNode(child));
|
|
293
|
-
if (declaration === undefined) {
|
|
294
|
-
return current;
|
|
295
|
-
}
|
|
296
|
-
current = declaration;
|
|
297
|
-
continue;
|
|
298
|
-
}
|
|
299
|
-
return current;
|
|
300
|
-
}
|
|
301
|
-
};
|
|
302
|
-
const variableDeclarators = (node) => node.namedChildren.filter((child) => child.type === "variable_declarator");
|
|
303
|
-
const variableTypeFromDeclarator = (node) => {
|
|
304
|
-
const value = node.childForFieldName("value");
|
|
305
|
-
if (value !== null && value.type.includes("function")) {
|
|
306
|
-
return "function";
|
|
307
|
-
}
|
|
308
|
-
return "variable";
|
|
309
|
-
};
|
|
310
|
-
const variableTypeFromDeclaration = (node) => {
|
|
311
|
-
const declarators = variableDeclarators(node);
|
|
312
|
-
if (declarators.some((declarator) => variableTypeFromDeclarator(declarator) === "function")) {
|
|
313
|
-
return "function";
|
|
314
|
-
}
|
|
315
|
-
return "variable";
|
|
316
|
-
};
|
|
317
|
-
const chunkTypeFromNode = (node) => {
|
|
318
|
-
switch (node.type) {
|
|
319
|
-
case "class_declaration":
|
|
320
|
-
return "class";
|
|
321
|
-
case "enum_declaration":
|
|
322
|
-
return "enum";
|
|
323
|
-
case "function_declaration":
|
|
324
|
-
case "generator_function_declaration":
|
|
325
|
-
return "function";
|
|
326
|
-
case "internal_module":
|
|
327
|
-
case "module":
|
|
328
|
-
return "namespace";
|
|
329
|
-
case "interface_declaration":
|
|
330
|
-
return "interface";
|
|
331
|
-
case "generator_method_definition":
|
|
332
|
-
case "method_definition":
|
|
333
|
-
return "method";
|
|
334
|
-
case "type_alias_declaration":
|
|
335
|
-
return "type-alias";
|
|
336
|
-
case "lexical_declaration":
|
|
337
|
-
case "variable_declaration":
|
|
338
|
-
return variableTypeFromDeclaration(node);
|
|
339
|
-
case "variable_declarator":
|
|
340
|
-
return variableTypeFromDeclarator(node);
|
|
341
|
-
default:
|
|
342
|
-
return undefined;
|
|
343
|
-
}
|
|
344
|
-
};
|
|
345
|
-
const variableNamesFromDeclaration = (node) => {
|
|
346
|
-
const names = variableDeclarators(node)
|
|
347
|
-
.map((declarator) => nodeFieldText(declarator, "name"))
|
|
348
|
-
.filter((name) => name !== undefined);
|
|
349
|
-
if (names.length === 0) {
|
|
350
|
-
return undefined;
|
|
351
|
-
}
|
|
352
|
-
return names.join(", ");
|
|
353
|
-
};
|
|
354
|
-
const nameFromNode = (node) => {
|
|
355
|
-
switch (node.type) {
|
|
356
|
-
case "class_declaration":
|
|
357
|
-
case "enum_declaration":
|
|
358
|
-
case "function_declaration":
|
|
359
|
-
case "generator_function_declaration":
|
|
360
|
-
case "internal_module":
|
|
361
|
-
case "interface_declaration":
|
|
362
|
-
case "module":
|
|
363
|
-
case "generator_method_definition":
|
|
364
|
-
case "method_definition":
|
|
365
|
-
case "type_alias_declaration":
|
|
366
|
-
case "variable_declarator":
|
|
367
|
-
return nodeFieldText(node, "name");
|
|
368
|
-
case "lexical_declaration":
|
|
369
|
-
case "variable_declaration":
|
|
370
|
-
return variableNamesFromDeclaration(node);
|
|
371
|
-
default:
|
|
372
|
-
return undefined;
|
|
373
|
-
}
|
|
374
|
-
};
|
|
375
|
-
const formatParent = (type, name) => {
|
|
376
|
-
if (type === undefined && name === undefined) {
|
|
377
|
-
return undefined;
|
|
378
|
-
}
|
|
379
|
-
if (type === undefined) {
|
|
380
|
-
return name;
|
|
381
|
-
}
|
|
382
|
-
if (name === undefined) {
|
|
383
|
-
return type;
|
|
384
|
-
}
|
|
385
|
-
return type + " " + name;
|
|
386
|
-
};
|
|
387
|
-
const collectClassMethodRanges = (classNode, parent, lines) => {
|
|
388
|
-
const body = classNode.childForFieldName("body");
|
|
389
|
-
if (body === null) {
|
|
390
|
-
return [];
|
|
391
|
-
}
|
|
392
|
-
const out = [];
|
|
393
|
-
for (let index = 0; index < body.namedChildren.length; index++) {
|
|
394
|
-
const child = body.namedChildren[index];
|
|
395
|
-
if (!child.type.includes("method")) {
|
|
396
|
-
continue;
|
|
397
|
-
}
|
|
398
|
-
out.push({
|
|
399
|
-
...lineRangeWithLeadingComments(child, body.namedChildren, index, lines),
|
|
400
|
-
name: nameFromNode(child),
|
|
401
|
-
type: chunkTypeFromNode(child),
|
|
402
|
-
parent,
|
|
403
|
-
});
|
|
404
|
-
}
|
|
405
|
-
return out;
|
|
406
|
-
};
|
|
407
|
-
const collectAstRanges = (path, content, lines) => {
|
|
408
|
-
const language = resolveAstLanguage(path);
|
|
409
|
-
if (language === undefined) {
|
|
410
|
-
return [];
|
|
411
|
-
}
|
|
412
|
-
try {
|
|
413
|
-
const parser = new TreeSitter();
|
|
414
|
-
parser.setLanguage(language);
|
|
415
|
-
const tree = parser.parse(content, undefined, {
|
|
416
|
-
bufferSize: 1024 * 1024,
|
|
417
|
-
});
|
|
418
|
-
const collectDeclarationRanges = (siblings, parent) => {
|
|
419
|
-
const out = [];
|
|
420
|
-
for (let index = 0; index < siblings.length; index++) {
|
|
421
|
-
const sibling = siblings[index];
|
|
422
|
-
if (sibling.type === "comment" || sibling.type.includes("import")) {
|
|
423
|
-
continue;
|
|
424
|
-
}
|
|
425
|
-
const declarationNode = unwrapDeclarationNode(sibling);
|
|
426
|
-
const type = chunkTypeFromNode(declarationNode);
|
|
427
|
-
const name = nameFromNode(declarationNode);
|
|
428
|
-
if (type === undefined && name === undefined) {
|
|
429
|
-
continue;
|
|
430
|
-
}
|
|
431
|
-
out.push({
|
|
432
|
-
...lineRangeWithLeadingComments(sibling, siblings, index, lines),
|
|
433
|
-
name,
|
|
434
|
-
type,
|
|
435
|
-
parent,
|
|
436
|
-
});
|
|
437
|
-
if (declarationNode.type === "class_declaration") {
|
|
438
|
-
out.push(...collectClassMethodRanges(declarationNode, formatParent(type, name), lines));
|
|
439
|
-
}
|
|
440
|
-
if (isNamespaceNode(declarationNode)) {
|
|
441
|
-
const body = declarationNode.childForFieldName("body");
|
|
442
|
-
if (body !== null) {
|
|
443
|
-
out.push(...collectDeclarationRanges(body.namedChildren, formatParent(type, name)));
|
|
444
|
-
}
|
|
445
|
-
}
|
|
446
|
-
}
|
|
447
|
-
return out;
|
|
448
|
-
};
|
|
449
|
-
return collectDeclarationRanges(tree.rootNode.namedChildren, undefined);
|
|
450
|
-
}
|
|
451
|
-
catch {
|
|
452
|
-
return [];
|
|
453
|
-
}
|
|
454
|
-
};
|
|
455
|
-
const chunksFromRanges = (path, lines, ranges, settings) => {
|
|
456
|
-
const hasMethodChildRange = (classRange) => {
|
|
457
|
-
const parent = formatParent("class", classRange.name);
|
|
458
|
-
return ranges.some((range) => range.type === "method" &&
|
|
459
|
-
range.parent === parent &&
|
|
460
|
-
range.startLine >= classRange.startLine &&
|
|
461
|
-
range.endLine <= classRange.endLine);
|
|
462
|
-
};
|
|
463
|
-
const out = [];
|
|
464
|
-
const seen = new Set();
|
|
465
|
-
const prefixSums = lineLengthPrefixSums(lines);
|
|
466
|
-
for (const range of ranges) {
|
|
467
|
-
const normalizedRange = normalizeLineRange(range, lines.length);
|
|
468
|
-
if (normalizedRange === undefined) {
|
|
469
|
-
continue;
|
|
470
|
-
}
|
|
471
|
-
const allSegments = splitRange(normalizedRange, settings, prefixSums);
|
|
472
|
-
const segments = range.type === "class" &&
|
|
473
|
-
allSegments.length > 1 &&
|
|
474
|
-
hasMethodChildRange({ ...normalizedRange, name: range.name })
|
|
475
|
-
? [allSegments[0]]
|
|
476
|
-
: allSegments;
|
|
477
|
-
for (const segment of segments) {
|
|
478
|
-
const key = String(segment.startLine) +
|
|
479
|
-
":" +
|
|
480
|
-
String(segment.endLine) +
|
|
481
|
-
":" +
|
|
482
|
-
(range.name ?? "") +
|
|
483
|
-
":" +
|
|
484
|
-
(range.type ?? "") +
|
|
485
|
-
":" +
|
|
486
|
-
(range.parent ?? "");
|
|
487
|
-
if (seen.has(key)) {
|
|
488
|
-
continue;
|
|
489
|
-
}
|
|
490
|
-
seen.add(key);
|
|
491
|
-
const chunkLines = lines.slice(segment.startLine - 1, segment.endLine);
|
|
492
|
-
if (!chunkLines.some(isMeaningfulLine)) {
|
|
493
|
-
continue;
|
|
494
|
-
}
|
|
495
|
-
out.push({
|
|
496
|
-
path,
|
|
497
|
-
startLine: segment.startLine,
|
|
498
|
-
endLine: segment.endLine,
|
|
499
|
-
name: range.name,
|
|
500
|
-
type: range.type,
|
|
501
|
-
parent: range.parent,
|
|
502
|
-
content: chunkLines.join("\n"),
|
|
503
|
-
});
|
|
504
|
-
}
|
|
505
|
-
}
|
|
506
|
-
return out.toSorted((left, right) => left.startLine - right.startLine ||
|
|
507
|
-
left.endLine - right.endLine ||
|
|
508
|
-
(left.name ?? "").localeCompare(right.name ?? ""));
|
|
509
|
-
};
|
|
510
|
-
const chunkWithLineWindows = (path, lines, settings) => {
|
|
511
|
-
const out = [];
|
|
512
|
-
const prefixSums = lineLengthPrefixSums(lines);
|
|
513
|
-
for (let index = 0; index < lines.length;) {
|
|
514
|
-
if (!isMeaningfulLine(lines[index])) {
|
|
515
|
-
index++;
|
|
516
|
-
continue;
|
|
517
|
-
}
|
|
518
|
-
const startLine = index + 1;
|
|
519
|
-
const maxEndLine = Math.min(lines.length, startLine + settings.chunkSize - 1);
|
|
520
|
-
const endLine = resolveSegmentEndLine({
|
|
521
|
-
startLine,
|
|
522
|
-
maxEndLine,
|
|
523
|
-
settings,
|
|
524
|
-
prefixSums,
|
|
525
|
-
});
|
|
526
|
-
const chunkLines = lines.slice(startLine - 1, endLine);
|
|
527
|
-
out.push({
|
|
528
|
-
path,
|
|
529
|
-
startLine,
|
|
530
|
-
endLine,
|
|
531
|
-
name: undefined,
|
|
532
|
-
type: undefined,
|
|
533
|
-
parent: undefined,
|
|
534
|
-
content: chunkLines.join("\n"),
|
|
535
|
-
});
|
|
536
|
-
if (endLine >= lines.length) {
|
|
537
|
-
break;
|
|
538
|
-
}
|
|
539
|
-
const nextStartLine = Math.max(startLine + 1, endLine - settings.chunkOverlap + 1);
|
|
540
|
-
index = nextStartLine - 1;
|
|
541
|
-
}
|
|
542
|
-
return out;
|
|
543
|
-
};
|
|
544
|
-
const splitOversizedChunks = (chunks, maxCharacters) => chunks.flatMap((chunk) => {
|
|
545
|
-
if (chunk.content.length <= maxCharacters) {
|
|
546
|
-
return [chunk];
|
|
547
|
-
}
|
|
548
|
-
// Line-based splitting can only exceed the limit for a single long line.
|
|
549
|
-
// Each piece retains that line number and the original AST metadata.
|
|
550
|
-
const pieces = [];
|
|
551
|
-
for (let offset = 0; offset < chunk.content.length; offset += maxCharacters) {
|
|
552
|
-
pieces.push({
|
|
553
|
-
...chunk,
|
|
554
|
-
content: chunk.content.slice(offset, offset + maxCharacters),
|
|
555
|
-
});
|
|
556
|
-
}
|
|
557
|
-
return pieces;
|
|
558
|
-
});
|
|
559
|
-
/**
|
|
560
|
-
* @since 1.0.0
|
|
561
|
-
* @category Constructors
|
|
562
|
-
*/
|
|
563
|
-
export const chunkFileContent = (path, content, options) => {
|
|
564
|
-
if (content.trim().length === 0 || isProbablyMinified(content)) {
|
|
565
|
-
return [];
|
|
566
|
-
}
|
|
567
|
-
const normalizedPath = normalizePath(path);
|
|
568
|
-
const normalizedContent = normalizeText(content);
|
|
569
|
-
const lines = normalizedContent.split("\n");
|
|
570
|
-
if (lines.at(-1) === "") {
|
|
571
|
-
lines.pop();
|
|
572
|
-
}
|
|
573
|
-
if (lines.length === 0) {
|
|
574
|
-
return [];
|
|
575
|
-
}
|
|
576
|
-
const settings = resolveChunkSettings(options);
|
|
577
|
-
const astRanges = collectAstRanges(normalizedPath, normalizedContent, lines);
|
|
578
|
-
if (astRanges.length > 0) {
|
|
579
|
-
const astChunks = chunksFromRanges(normalizedPath, lines, astRanges, settings);
|
|
580
|
-
if (astChunks.length > 0) {
|
|
581
|
-
return splitOversizedChunks(astChunks, settings.chunkMaxCharacters);
|
|
582
|
-
}
|
|
583
|
-
}
|
|
584
|
-
return splitOversizedChunks(chunkWithLineWindows(normalizedPath, lines, settings), settings.chunkMaxCharacters);
|
|
585
|
-
};
|
|
586
|
-
/**
|
|
587
|
-
* @since 1.0.0
|
|
588
|
-
* @category Layers
|
|
589
|
-
*/
|
|
590
|
-
export const layer = Layer.effect(CodeChunker, Effect.gen(function* () {
|
|
591
|
-
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
|
|
592
|
-
const fs = yield* FileSystem.FileSystem;
|
|
593
|
-
const pathService = yield* Path.Path;
|
|
594
|
-
const listFiles = Effect.fn("CodeChunker.listFiles")(function* (options) {
|
|
595
|
-
const root = pathService.resolve(options.root);
|
|
596
|
-
const maxFileSize = options.maxFileSize ?? "1M";
|
|
597
|
-
return yield* pipe(spawner.streamLines(ChildProcess.make("rg", [
|
|
598
|
-
"--files",
|
|
599
|
-
"--hidden",
|
|
600
|
-
"--max-filesize",
|
|
601
|
-
maxFileSize,
|
|
602
|
-
"--glob",
|
|
603
|
-
"!.git",
|
|
604
|
-
], {
|
|
605
|
-
cwd: root,
|
|
606
|
-
stdin: "ignore",
|
|
607
|
-
})), Stream.runCollect, Effect.map(Array.fromIterable), Effect.map((entries) => entries
|
|
608
|
-
.map((entry) => normalizePath(entry.trim()))
|
|
609
|
-
.filter((entry) => entry.length > 0 && isMeaningfulFile(entry))
|
|
610
|
-
.toSorted((left, right) => left.localeCompare(right))), Effect.orDie);
|
|
611
|
-
});
|
|
612
|
-
const chunkFile = Effect.fn("CodeChunker.chunkFile")(function* (options) {
|
|
613
|
-
const root = pathService.resolve(options.root);
|
|
614
|
-
const absolutePath = pathService.resolve(root, options.path);
|
|
615
|
-
const path = normalizePath(pathService.relative(root, absolutePath));
|
|
616
|
-
if (path.length === 0 ||
|
|
617
|
-
path === ".." ||
|
|
618
|
-
path.startsWith("../") ||
|
|
619
|
-
!isMeaningfulFile(path)) {
|
|
620
|
-
return [];
|
|
621
|
-
}
|
|
622
|
-
return yield* pipe(fs.readFileString(absolutePath), Effect.map((content) => chunkFileContent(path, content, options)), Effect.orElseSucceed(() => []));
|
|
623
|
-
});
|
|
624
|
-
const chunkFiles = (options) => Stream.fromArray(options.paths).pipe(Stream.flatMap((path) => pipe(chunkFile({
|
|
625
|
-
root: options.root,
|
|
626
|
-
path,
|
|
627
|
-
chunkSize: options.chunkSize,
|
|
628
|
-
chunkOverlap: options.chunkOverlap,
|
|
629
|
-
...(options.chunkMaxCharacters === undefined
|
|
630
|
-
? {}
|
|
631
|
-
: { chunkMaxCharacters: options.chunkMaxCharacters }),
|
|
632
|
-
}), Stream.fromArrayEffect), { concurrency: 5 }));
|
|
633
|
-
const chunkCodebase = Effect.fnUntraced(function* (options) {
|
|
634
|
-
const root = pathService.resolve(options.root);
|
|
635
|
-
const files = yield* listFiles({
|
|
636
|
-
root,
|
|
637
|
-
...(options.maxFileSize === undefined
|
|
638
|
-
? {}
|
|
639
|
-
: { maxFileSize: options.maxFileSize }),
|
|
640
|
-
});
|
|
641
|
-
return chunkFiles({
|
|
642
|
-
root,
|
|
643
|
-
paths: files,
|
|
644
|
-
chunkSize: options.chunkSize,
|
|
645
|
-
chunkOverlap: options.chunkOverlap,
|
|
646
|
-
...(options.chunkMaxCharacters === undefined
|
|
647
|
-
? {}
|
|
648
|
-
: { chunkMaxCharacters: options.chunkMaxCharacters }),
|
|
649
|
-
});
|
|
650
|
-
}, Stream.unwrap);
|
|
651
|
-
return CodeChunker.of({
|
|
652
|
-
listFiles,
|
|
653
|
-
chunkFile,
|
|
654
|
-
chunkFiles,
|
|
655
|
-
chunkCodebase,
|
|
656
|
-
});
|
|
657
|
-
}));
|
|
658
|
-
//# sourceMappingURL=CodeChunker.js.map
|