clanka 0.26.1 → 0.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +2 -3
  2. package/dist/Acp.test.js +0 -1
  3. package/dist/Acp.test.js.map +1 -1
  4. package/dist/AcpImage.test.js +0 -1
  5. package/dist/AcpImage.test.js.map +1 -1
  6. package/dist/Agent.d.ts.map +1 -1
  7. package/dist/Agent.js +1 -2
  8. package/dist/Agent.js.map +1 -1
  9. package/dist/Agent.test.js +0 -1
  10. package/dist/Agent.test.js.map +1 -1
  11. package/dist/AgentCallerConfig.test.js +0 -1
  12. package/dist/AgentCallerConfig.test.js.map +1 -1
  13. package/dist/AgentExecutor.d.ts +2 -3
  14. package/dist/AgentExecutor.d.ts.map +1 -1
  15. package/dist/AgentExecutor.js +4 -13
  16. package/dist/AgentExecutor.js.map +1 -1
  17. package/dist/AgentImage.test.js +0 -1
  18. package/dist/AgentImage.test.js.map +1 -1
  19. package/dist/AgentSkills.test.js +0 -2
  20. package/dist/AgentSkills.test.js.map +1 -1
  21. package/dist/AgentTools.d.ts +2 -235
  22. package/dist/AgentTools.d.ts.map +1 -1
  23. package/dist/AgentTools.js +2 -32
  24. package/dist/AgentTools.js.map +1 -1
  25. package/dist/ClaudeAgent.test.js +0 -1
  26. package/dist/ClaudeAgent.test.js.map +1 -1
  27. package/dist/McpServers.d.ts +20 -1
  28. package/dist/McpServers.d.ts.map +1 -1
  29. package/dist/McpServers.js +12 -5
  30. package/dist/McpServers.js.map +1 -1
  31. package/dist/McpServers.test.js +115 -0
  32. package/dist/McpServers.test.js.map +1 -1
  33. package/dist/ScriptFormatting.d.ts +5 -0
  34. package/dist/ScriptFormatting.d.ts.map +1 -1
  35. package/dist/ScriptFormatting.js +255 -53
  36. package/dist/ScriptFormatting.js.map +1 -1
  37. package/dist/ScriptFormatting.test.js +16 -0
  38. package/dist/ScriptFormatting.test.js.map +1 -1
  39. package/dist/bin/{Latch-DOdbsGMQ.mjs → Latch-BE6_TYRL.mjs} +1 -1
  40. package/dist/bin/{McpClient-B-FFG6Pg.mjs → McpClient-DCt4v5W5.mjs} +2 -2
  41. package/dist/bin/{OutputFormatter-DhhP2i-7.mjs → OutputFormatter-DaFv_C_I.mjs} +2 -3
  42. package/dist/bin/{Schema-CFdSO6iH.mjs → Schema-B3MiG1aD.mjs} +204 -568
  43. package/dist/bin/cli.mjs +5098 -2472
  44. package/dist/bin/{upgrade-C4aOEWJH.mjs → upgrade-WqgIU_21.mjs} +3 -3
  45. package/dist/cli.js +2 -25
  46. package/dist/cli.js.map +1 -1
  47. package/dist/fixtures/McpFakeServer.d.ts +4 -0
  48. package/dist/fixtures/McpFakeServer.d.ts.map +1 -1
  49. package/dist/fixtures/McpFakeServer.js +13 -0
  50. package/dist/fixtures/McpFakeServer.js.map +1 -1
  51. package/dist/index.d.ts +0 -8
  52. package/dist/index.d.ts.map +1 -1
  53. package/dist/index.js +0 -8
  54. package/dist/index.js.map +1 -1
  55. package/package.json +1 -7
  56. package/src/Acp.test.ts +0 -1
  57. package/src/AcpImage.test.ts +0 -1
  58. package/src/Agent.test.ts +0 -1
  59. package/src/Agent.ts +0 -4
  60. package/src/AgentCallerConfig.test.ts +0 -1
  61. package/src/AgentExecutor.ts +4 -16
  62. package/src/AgentImage.test.ts +0 -1
  63. package/src/AgentSkills.test.ts +0 -2
  64. package/src/AgentTools.ts +4 -41
  65. package/src/ClaudeAgent.test.ts +0 -1
  66. package/src/McpServers.test.ts +198 -0
  67. package/src/McpServers.ts +44 -4
  68. package/src/ScriptFormatting.test.ts +20 -0
  69. package/src/ScriptFormatting.ts +234 -47
  70. package/src/cli.ts +1 -56
  71. package/src/fixtures/McpFakeServer.ts +17 -1
  72. package/src/index.ts +0 -10
  73. package/dist/ChunkRepo.d.ts +0 -173
  74. package/dist/ChunkRepo.d.ts.map +0 -1
  75. package/dist/ChunkRepo.js +0 -168
  76. package/dist/ChunkRepo.js.map +0 -1
  77. package/dist/ChunkRepo.test.d.ts +0 -2
  78. package/dist/ChunkRepo.test.d.ts.map +0 -1
  79. package/dist/ChunkRepo.test.js +0 -63
  80. package/dist/ChunkRepo.test.js.map +0 -1
  81. package/dist/CodeChunker.d.ts +0 -84
  82. package/dist/CodeChunker.d.ts.map +0 -1
  83. package/dist/CodeChunker.js +0 -658
  84. package/dist/CodeChunker.js.map +0 -1
  85. package/dist/CodeChunker.test.d.ts +0 -2
  86. package/dist/CodeChunker.test.d.ts.map +0 -1
  87. package/dist/CodeChunker.test.js +0 -434
  88. package/dist/CodeChunker.test.js.map +0 -1
  89. package/dist/SemanticSearch/Service.d.ts +0 -31
  90. package/dist/SemanticSearch/Service.d.ts.map +0 -1
  91. package/dist/SemanticSearch/Service.js +0 -29
  92. package/dist/SemanticSearch/Service.js.map +0 -1
  93. package/dist/SemanticSearch.d.ts +0 -34
  94. package/dist/SemanticSearch.d.ts.map +0 -1
  95. package/dist/SemanticSearch.js +0 -171
  96. package/dist/SemanticSearch.js.map +0 -1
  97. package/dist/SemanticSearch.test.d.ts +0 -2
  98. package/dist/SemanticSearch.test.d.ts.map +0 -1
  99. package/dist/SemanticSearch.test.js +0 -34
  100. package/dist/SemanticSearch.test.js.map +0 -1
  101. package/dist/Sqlite.d.ts +0 -15
  102. package/dist/Sqlite.d.ts.map +0 -1
  103. package/dist/Sqlite.js +0 -44
  104. package/dist/Sqlite.js.map +0 -1
  105. package/dist/bin/AiError-DBaKz35C.mjs +0 -2723
  106. package/dist/bin/SemanticSearch-C3a7LAQr.mjs +0 -2558
  107. package/src/ChunkRepo.test.ts +0 -89
  108. package/src/ChunkRepo.ts +0 -260
  109. package/src/CodeChunker.test.ts +0 -493
  110. package/src/CodeChunker.ts +0 -1018
  111. package/src/SemanticSearch/Service.ts +0 -50
  112. package/src/SemanticSearch.test.ts +0 -51
  113. package/src/SemanticSearch.ts +0 -282
  114. package/src/Sqlite.ts +0 -60
@@ -1,658 +0,0 @@
1
- /**
2
- * @since 1.0.0
3
- */
4
- import * as Array from "effect/Array";
5
- import * as Effect from "effect/Effect";
6
- import * as FileSystem from "effect/FileSystem";
7
- import { pipe } from "effect/Function";
8
- import * as Layer from "effect/Layer";
9
- import * as Path from "effect/Path";
10
- import * as Context from "effect/Context";
11
- import * as Stream from "effect/Stream";
12
- import * as ChildProcess from "effect/process/ChildProcess";
13
- import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
14
- import TreeSitter, {} from "tree-sitter";
15
- import TreeSitterJavaScript from "tree-sitter-javascript";
16
- import TreeSitterTypeScript from "tree-sitter-typescript";
17
- /**
18
- * @since 1.0.0
19
- * @category Services
20
- */
21
- export class CodeChunker extends Context.Service()("clanka/CodeChunker") {
22
- }
23
- const sourceExtensions = new Set([
24
- "c",
25
- "cc",
26
- "cpp",
27
- "cs",
28
- "css",
29
- "cts",
30
- "cxx",
31
- "go",
32
- "gql",
33
- "graphql",
34
- "h",
35
- "hpp",
36
- "html",
37
- "ini",
38
- "java",
39
- "js",
40
- "jsx",
41
- "kt",
42
- "kts",
43
- "less",
44
- "lua",
45
- "mjs",
46
- "mts",
47
- "php",
48
- "py",
49
- "rb",
50
- "rs",
51
- "sass",
52
- "scala",
53
- "scss",
54
- "sh",
55
- "sql",
56
- "svelte",
57
- "swift",
58
- "ts",
59
- "tsx",
60
- "vue",
61
- "xml",
62
- "zsh",
63
- ]);
64
- const documentationExtensions = new Set([
65
- "adoc",
66
- "asciidoc",
67
- "md",
68
- "mdx",
69
- "rst",
70
- "txt",
71
- ]);
72
- const ignoredDirectories = new Set([
73
- ".git",
74
- ".next",
75
- ".nuxt",
76
- ".svelte-kit",
77
- ".turbo",
78
- "build",
79
- "coverage",
80
- "dist",
81
- "node_modules",
82
- "target",
83
- ]);
84
- const normalizePath = (path) => path.replace(/\\/g, "/");
85
- const normalizeText = (content) => content.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
86
- const meaningfulLinePattern = /[^\s\p{P}]/u;
87
- const isMeaningfulLine = (line) => meaningfulLinePattern.test(line);
88
- const languageByExtension = new Map([
89
- ["js", TreeSitterJavaScript],
90
- ["jsx", TreeSitterJavaScript],
91
- ["ts", TreeSitterTypeScript.typescript],
92
- ["tsx", TreeSitterTypeScript.tsx],
93
- ]);
94
- // const ignoredTopLevelNodeTypes = new Set(["comment", "import_statement"])
95
- /**
96
- * @since 1.0.0
97
- * @category Predicates
98
- */
99
- export const isProbablyMinified = (content) => {
100
- const normalized = normalizeText(content);
101
- if (normalized.length < 2_000) {
102
- return false;
103
- }
104
- const lines = normalized.split("\n");
105
- if (lines.length <= 2) {
106
- return true;
107
- }
108
- let longLines = 0;
109
- for (const line of lines) {
110
- if (line.length >= 300) {
111
- longLines++;
112
- }
113
- }
114
- return lines.length <= 20 && longLines / lines.length >= 0.8;
115
- };
116
- /**
117
- * @since 1.0.0
118
- * @category Predicates
119
- */
120
- export const isMeaningfulFile = (path) => {
121
- const normalizedPath = normalizePath(path);
122
- const lowercasePath = normalizedPath.toLowerCase();
123
- const parts = lowercasePath.split("/");
124
- const fileName = parts.at(-1);
125
- if (fileName === undefined || fileName.length === 0) {
126
- return false;
127
- }
128
- if (parts.some((part) => ignoredDirectories.has(part))) {
129
- return false;
130
- }
131
- if (/\.min\.(?:css|js)$/i.test(fileName)) {
132
- return false;
133
- }
134
- const extensionIndex = fileName.lastIndexOf(".");
135
- if (extensionIndex === -1) {
136
- return false;
137
- }
138
- const extension = fileName.slice(extensionIndex + 1);
139
- return (sourceExtensions.has(extension) || documentationExtensions.has(extension));
140
- };
141
- const resolveChunkSettings = (options) => {
142
- const chunkSize = Math.max(1, options.chunkSize);
143
- const chunkOverlap = Math.max(0, Math.min(chunkSize - 1, options.chunkOverlap));
144
- const chunkMaxCharacters = options.chunkMaxCharacters !== undefined &&
145
- Number.isFinite(options.chunkMaxCharacters)
146
- ? Math.max(1, Math.floor(options.chunkMaxCharacters))
147
- : Number.POSITIVE_INFINITY;
148
- return {
149
- chunkSize,
150
- chunkOverlap,
151
- chunkMaxCharacters,
152
- };
153
- };
154
- const getPathExtension = (path) => {
155
- const fileName = path.split("/").at(-1);
156
- if (fileName === undefined) {
157
- return undefined;
158
- }
159
- const extensionIndex = fileName.lastIndexOf(".");
160
- if (extensionIndex === -1) {
161
- return undefined;
162
- }
163
- return fileName.slice(extensionIndex + 1).toLowerCase();
164
- };
165
- const resolveAstLanguage = (path) => {
166
- const extension = getPathExtension(path);
167
- if (extension === undefined) {
168
- return undefined;
169
- }
170
- return languageByExtension.get(extension);
171
- };
172
- const lineRangeFromNode = (node) => {
173
- const startLine = node.startPosition.row + 1;
174
- const endLine = Math.max(startLine, node.endPosition.row + 1);
175
- return {
176
- startLine,
177
- endLine,
178
- };
179
- };
180
- const hasOnlyWhitespaceLines = (lines, startLine, endLine) => {
181
- if (startLine > endLine) {
182
- return true;
183
- }
184
- for (let lineIndex = startLine; lineIndex <= endLine; lineIndex++) {
185
- if ((lines[lineIndex - 1] ?? "").trim().length > 0) {
186
- return false;
187
- }
188
- }
189
- return true;
190
- };
191
- const lineRangeWithLeadingComments = (node, siblings, nodeIndex, lines) => {
192
- const baseRange = lineRangeFromNode(node);
193
- let startLine = baseRange.startLine;
194
- for (let index = nodeIndex - 1; index >= 0; index--) {
195
- const sibling = siblings[index];
196
- if (sibling.type !== "comment") {
197
- break;
198
- }
199
- const commentRange = lineRangeFromNode(sibling);
200
- if (!hasOnlyWhitespaceLines(lines, commentRange.endLine + 1, startLine - 1)) {
201
- break;
202
- }
203
- startLine = commentRange.startLine;
204
- }
205
- return {
206
- startLine,
207
- endLine: baseRange.endLine,
208
- };
209
- };
210
- const normalizeLineRange = (range, lineCount) => {
211
- const startLine = Math.max(1, Math.min(lineCount, range.startLine));
212
- const endLine = Math.max(1, Math.min(lineCount, range.endLine));
213
- if (endLine < startLine) {
214
- return undefined;
215
- }
216
- return {
217
- startLine,
218
- endLine,
219
- };
220
- };
221
- const lineLengthPrefixSums = (lines) => {
222
- const sums = [0];
223
- for (let index = 0; index < lines.length; index++) {
224
- sums.push(sums[index] + lines[index].length);
225
- }
226
- return sums;
227
- };
228
- const lineRangeCharacterLength = (prefixSums, range) => prefixSums[range.endLine] -
229
- prefixSums[range.startLine - 1] +
230
- (range.endLine - range.startLine);
231
- const resolveSegmentEndLine = (options) => {
232
- if (options.settings.chunkMaxCharacters === Number.POSITIVE_INFINITY) {
233
- return options.maxEndLine;
234
- }
235
- let endLine = options.maxEndLine;
236
- while (endLine > options.startLine &&
237
- lineRangeCharacterLength(options.prefixSums, {
238
- startLine: options.startLine,
239
- endLine,
240
- }) > options.settings.chunkMaxCharacters) {
241
- endLine--;
242
- }
243
- return endLine;
244
- };
245
- const splitRange = (range, settings, prefixSums) => {
246
- const lineCount = range.endLine - range.startLine + 1;
247
- if (lineCount <= settings.chunkSize &&
248
- lineRangeCharacterLength(prefixSums, range) <= settings.chunkMaxCharacters) {
249
- return [range];
250
- }
251
- const out = [];
252
- for (let startLine = range.startLine; startLine <= range.endLine;) {
253
- const maxEndLine = Math.min(range.endLine, startLine + settings.chunkSize - 1);
254
- const endLine = resolveSegmentEndLine({
255
- startLine,
256
- maxEndLine,
257
- settings,
258
- prefixSums,
259
- });
260
- out.push({
261
- startLine,
262
- endLine,
263
- });
264
- if (endLine >= range.endLine) {
265
- break;
266
- }
267
- startLine = Math.max(startLine + 1, endLine - settings.chunkOverlap + 1);
268
- }
269
- return out;
270
- };
271
- const nodeText = (node) => {
272
- if (node === null) {
273
- return undefined;
274
- }
275
- const value = node.text.trim().replace(/\s+/g, " ");
276
- return value.length === 0 ? undefined : value;
277
- };
278
- const nodeFieldText = (node, fieldName) => nodeText(node.childForFieldName(fieldName));
279
- const isNamespaceNode = (node) => node.type === "internal_module" || node.type === "module";
280
- const unwrapDeclarationNode = (node) => {
281
- let current = node;
282
- while (true) {
283
- if (current.type === "export_statement") {
284
- const declaration = current.childForFieldName("declaration") ?? current.namedChildren[0];
285
- if (declaration === undefined) {
286
- return current;
287
- }
288
- current = declaration;
289
- continue;
290
- }
291
- if (current.type === "ambient_declaration") {
292
- const declaration = current.namedChildren.find((child) => child.type.endsWith("_declaration") || isNamespaceNode(child));
293
- if (declaration === undefined) {
294
- return current;
295
- }
296
- current = declaration;
297
- continue;
298
- }
299
- return current;
300
- }
301
- };
302
- const variableDeclarators = (node) => node.namedChildren.filter((child) => child.type === "variable_declarator");
303
- const variableTypeFromDeclarator = (node) => {
304
- const value = node.childForFieldName("value");
305
- if (value !== null && value.type.includes("function")) {
306
- return "function";
307
- }
308
- return "variable";
309
- };
310
- const variableTypeFromDeclaration = (node) => {
311
- const declarators = variableDeclarators(node);
312
- if (declarators.some((declarator) => variableTypeFromDeclarator(declarator) === "function")) {
313
- return "function";
314
- }
315
- return "variable";
316
- };
317
- const chunkTypeFromNode = (node) => {
318
- switch (node.type) {
319
- case "class_declaration":
320
- return "class";
321
- case "enum_declaration":
322
- return "enum";
323
- case "function_declaration":
324
- case "generator_function_declaration":
325
- return "function";
326
- case "internal_module":
327
- case "module":
328
- return "namespace";
329
- case "interface_declaration":
330
- return "interface";
331
- case "generator_method_definition":
332
- case "method_definition":
333
- return "method";
334
- case "type_alias_declaration":
335
- return "type-alias";
336
- case "lexical_declaration":
337
- case "variable_declaration":
338
- return variableTypeFromDeclaration(node);
339
- case "variable_declarator":
340
- return variableTypeFromDeclarator(node);
341
- default:
342
- return undefined;
343
- }
344
- };
345
- const variableNamesFromDeclaration = (node) => {
346
- const names = variableDeclarators(node)
347
- .map((declarator) => nodeFieldText(declarator, "name"))
348
- .filter((name) => name !== undefined);
349
- if (names.length === 0) {
350
- return undefined;
351
- }
352
- return names.join(", ");
353
- };
354
- const nameFromNode = (node) => {
355
- switch (node.type) {
356
- case "class_declaration":
357
- case "enum_declaration":
358
- case "function_declaration":
359
- case "generator_function_declaration":
360
- case "internal_module":
361
- case "interface_declaration":
362
- case "module":
363
- case "generator_method_definition":
364
- case "method_definition":
365
- case "type_alias_declaration":
366
- case "variable_declarator":
367
- return nodeFieldText(node, "name");
368
- case "lexical_declaration":
369
- case "variable_declaration":
370
- return variableNamesFromDeclaration(node);
371
- default:
372
- return undefined;
373
- }
374
- };
375
- const formatParent = (type, name) => {
376
- if (type === undefined && name === undefined) {
377
- return undefined;
378
- }
379
- if (type === undefined) {
380
- return name;
381
- }
382
- if (name === undefined) {
383
- return type;
384
- }
385
- return type + " " + name;
386
- };
387
- const collectClassMethodRanges = (classNode, parent, lines) => {
388
- const body = classNode.childForFieldName("body");
389
- if (body === null) {
390
- return [];
391
- }
392
- const out = [];
393
- for (let index = 0; index < body.namedChildren.length; index++) {
394
- const child = body.namedChildren[index];
395
- if (!child.type.includes("method")) {
396
- continue;
397
- }
398
- out.push({
399
- ...lineRangeWithLeadingComments(child, body.namedChildren, index, lines),
400
- name: nameFromNode(child),
401
- type: chunkTypeFromNode(child),
402
- parent,
403
- });
404
- }
405
- return out;
406
- };
407
- const collectAstRanges = (path, content, lines) => {
408
- const language = resolveAstLanguage(path);
409
- if (language === undefined) {
410
- return [];
411
- }
412
- try {
413
- const parser = new TreeSitter();
414
- parser.setLanguage(language);
415
- const tree = parser.parse(content, undefined, {
416
- bufferSize: 1024 * 1024,
417
- });
418
- const collectDeclarationRanges = (siblings, parent) => {
419
- const out = [];
420
- for (let index = 0; index < siblings.length; index++) {
421
- const sibling = siblings[index];
422
- if (sibling.type === "comment" || sibling.type.includes("import")) {
423
- continue;
424
- }
425
- const declarationNode = unwrapDeclarationNode(sibling);
426
- const type = chunkTypeFromNode(declarationNode);
427
- const name = nameFromNode(declarationNode);
428
- if (type === undefined && name === undefined) {
429
- continue;
430
- }
431
- out.push({
432
- ...lineRangeWithLeadingComments(sibling, siblings, index, lines),
433
- name,
434
- type,
435
- parent,
436
- });
437
- if (declarationNode.type === "class_declaration") {
438
- out.push(...collectClassMethodRanges(declarationNode, formatParent(type, name), lines));
439
- }
440
- if (isNamespaceNode(declarationNode)) {
441
- const body = declarationNode.childForFieldName("body");
442
- if (body !== null) {
443
- out.push(...collectDeclarationRanges(body.namedChildren, formatParent(type, name)));
444
- }
445
- }
446
- }
447
- return out;
448
- };
449
- return collectDeclarationRanges(tree.rootNode.namedChildren, undefined);
450
- }
451
- catch {
452
- return [];
453
- }
454
- };
455
- const chunksFromRanges = (path, lines, ranges, settings) => {
456
- const hasMethodChildRange = (classRange) => {
457
- const parent = formatParent("class", classRange.name);
458
- return ranges.some((range) => range.type === "method" &&
459
- range.parent === parent &&
460
- range.startLine >= classRange.startLine &&
461
- range.endLine <= classRange.endLine);
462
- };
463
- const out = [];
464
- const seen = new Set();
465
- const prefixSums = lineLengthPrefixSums(lines);
466
- for (const range of ranges) {
467
- const normalizedRange = normalizeLineRange(range, lines.length);
468
- if (normalizedRange === undefined) {
469
- continue;
470
- }
471
- const allSegments = splitRange(normalizedRange, settings, prefixSums);
472
- const segments = range.type === "class" &&
473
- allSegments.length > 1 &&
474
- hasMethodChildRange({ ...normalizedRange, name: range.name })
475
- ? [allSegments[0]]
476
- : allSegments;
477
- for (const segment of segments) {
478
- const key = String(segment.startLine) +
479
- ":" +
480
- String(segment.endLine) +
481
- ":" +
482
- (range.name ?? "") +
483
- ":" +
484
- (range.type ?? "") +
485
- ":" +
486
- (range.parent ?? "");
487
- if (seen.has(key)) {
488
- continue;
489
- }
490
- seen.add(key);
491
- const chunkLines = lines.slice(segment.startLine - 1, segment.endLine);
492
- if (!chunkLines.some(isMeaningfulLine)) {
493
- continue;
494
- }
495
- out.push({
496
- path,
497
- startLine: segment.startLine,
498
- endLine: segment.endLine,
499
- name: range.name,
500
- type: range.type,
501
- parent: range.parent,
502
- content: chunkLines.join("\n"),
503
- });
504
- }
505
- }
506
- return out.toSorted((left, right) => left.startLine - right.startLine ||
507
- left.endLine - right.endLine ||
508
- (left.name ?? "").localeCompare(right.name ?? ""));
509
- };
510
- const chunkWithLineWindows = (path, lines, settings) => {
511
- const out = [];
512
- const prefixSums = lineLengthPrefixSums(lines);
513
- for (let index = 0; index < lines.length;) {
514
- if (!isMeaningfulLine(lines[index])) {
515
- index++;
516
- continue;
517
- }
518
- const startLine = index + 1;
519
- const maxEndLine = Math.min(lines.length, startLine + settings.chunkSize - 1);
520
- const endLine = resolveSegmentEndLine({
521
- startLine,
522
- maxEndLine,
523
- settings,
524
- prefixSums,
525
- });
526
- const chunkLines = lines.slice(startLine - 1, endLine);
527
- out.push({
528
- path,
529
- startLine,
530
- endLine,
531
- name: undefined,
532
- type: undefined,
533
- parent: undefined,
534
- content: chunkLines.join("\n"),
535
- });
536
- if (endLine >= lines.length) {
537
- break;
538
- }
539
- const nextStartLine = Math.max(startLine + 1, endLine - settings.chunkOverlap + 1);
540
- index = nextStartLine - 1;
541
- }
542
- return out;
543
- };
544
- const splitOversizedChunks = (chunks, maxCharacters) => chunks.flatMap((chunk) => {
545
- if (chunk.content.length <= maxCharacters) {
546
- return [chunk];
547
- }
548
- // Line-based splitting can only exceed the limit for a single long line.
549
- // Each piece retains that line number and the original AST metadata.
550
- const pieces = [];
551
- for (let offset = 0; offset < chunk.content.length; offset += maxCharacters) {
552
- pieces.push({
553
- ...chunk,
554
- content: chunk.content.slice(offset, offset + maxCharacters),
555
- });
556
- }
557
- return pieces;
558
- });
559
- /**
560
- * @since 1.0.0
561
- * @category Constructors
562
- */
563
- export const chunkFileContent = (path, content, options) => {
564
- if (content.trim().length === 0 || isProbablyMinified(content)) {
565
- return [];
566
- }
567
- const normalizedPath = normalizePath(path);
568
- const normalizedContent = normalizeText(content);
569
- const lines = normalizedContent.split("\n");
570
- if (lines.at(-1) === "") {
571
- lines.pop();
572
- }
573
- if (lines.length === 0) {
574
- return [];
575
- }
576
- const settings = resolveChunkSettings(options);
577
- const astRanges = collectAstRanges(normalizedPath, normalizedContent, lines);
578
- if (astRanges.length > 0) {
579
- const astChunks = chunksFromRanges(normalizedPath, lines, astRanges, settings);
580
- if (astChunks.length > 0) {
581
- return splitOversizedChunks(astChunks, settings.chunkMaxCharacters);
582
- }
583
- }
584
- return splitOversizedChunks(chunkWithLineWindows(normalizedPath, lines, settings), settings.chunkMaxCharacters);
585
- };
586
- /**
587
- * @since 1.0.0
588
- * @category Layers
589
- */
590
- export const layer = Layer.effect(CodeChunker, Effect.gen(function* () {
591
- const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
592
- const fs = yield* FileSystem.FileSystem;
593
- const pathService = yield* Path.Path;
594
- const listFiles = Effect.fn("CodeChunker.listFiles")(function* (options) {
595
- const root = pathService.resolve(options.root);
596
- const maxFileSize = options.maxFileSize ?? "1M";
597
- return yield* pipe(spawner.streamLines(ChildProcess.make("rg", [
598
- "--files",
599
- "--hidden",
600
- "--max-filesize",
601
- maxFileSize,
602
- "--glob",
603
- "!.git",
604
- ], {
605
- cwd: root,
606
- stdin: "ignore",
607
- })), Stream.runCollect, Effect.map(Array.fromIterable), Effect.map((entries) => entries
608
- .map((entry) => normalizePath(entry.trim()))
609
- .filter((entry) => entry.length > 0 && isMeaningfulFile(entry))
610
- .toSorted((left, right) => left.localeCompare(right))), Effect.orDie);
611
- });
612
- const chunkFile = Effect.fn("CodeChunker.chunkFile")(function* (options) {
613
- const root = pathService.resolve(options.root);
614
- const absolutePath = pathService.resolve(root, options.path);
615
- const path = normalizePath(pathService.relative(root, absolutePath));
616
- if (path.length === 0 ||
617
- path === ".." ||
618
- path.startsWith("../") ||
619
- !isMeaningfulFile(path)) {
620
- return [];
621
- }
622
- return yield* pipe(fs.readFileString(absolutePath), Effect.map((content) => chunkFileContent(path, content, options)), Effect.orElseSucceed(() => []));
623
- });
624
- const chunkFiles = (options) => Stream.fromArray(options.paths).pipe(Stream.flatMap((path) => pipe(chunkFile({
625
- root: options.root,
626
- path,
627
- chunkSize: options.chunkSize,
628
- chunkOverlap: options.chunkOverlap,
629
- ...(options.chunkMaxCharacters === undefined
630
- ? {}
631
- : { chunkMaxCharacters: options.chunkMaxCharacters }),
632
- }), Stream.fromArrayEffect), { concurrency: 5 }));
633
- const chunkCodebase = Effect.fnUntraced(function* (options) {
634
- const root = pathService.resolve(options.root);
635
- const files = yield* listFiles({
636
- root,
637
- ...(options.maxFileSize === undefined
638
- ? {}
639
- : { maxFileSize: options.maxFileSize }),
640
- });
641
- return chunkFiles({
642
- root,
643
- paths: files,
644
- chunkSize: options.chunkSize,
645
- chunkOverlap: options.chunkOverlap,
646
- ...(options.chunkMaxCharacters === undefined
647
- ? {}
648
- : { chunkMaxCharacters: options.chunkMaxCharacters }),
649
- });
650
- }, Stream.unwrap);
651
- return CodeChunker.of({
652
- listFiles,
653
- chunkFile,
654
- chunkFiles,
655
- chunkCodebase,
656
- });
657
- }));
658
- //# sourceMappingURL=CodeChunker.js.map