clanka 0.26.1 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.md +2 -3
  2. package/dist/Acp.test.js +0 -1
  3. package/dist/Acp.test.js.map +1 -1
  4. package/dist/AcpImage.test.js +0 -1
  5. package/dist/AcpImage.test.js.map +1 -1
  6. package/dist/Agent.d.ts.map +1 -1
  7. package/dist/Agent.js +1 -2
  8. package/dist/Agent.js.map +1 -1
  9. package/dist/Agent.test.js +0 -1
  10. package/dist/Agent.test.js.map +1 -1
  11. package/dist/AgentCallerConfig.test.js +0 -1
  12. package/dist/AgentCallerConfig.test.js.map +1 -1
  13. package/dist/AgentExecutor.d.ts +2 -3
  14. package/dist/AgentExecutor.d.ts.map +1 -1
  15. package/dist/AgentExecutor.js +4 -13
  16. package/dist/AgentExecutor.js.map +1 -1
  17. package/dist/AgentImage.test.js +0 -1
  18. package/dist/AgentImage.test.js.map +1 -1
  19. package/dist/AgentSkills.test.js +0 -2
  20. package/dist/AgentSkills.test.js.map +1 -1
  21. package/dist/AgentTools.d.ts +2 -235
  22. package/dist/AgentTools.d.ts.map +1 -1
  23. package/dist/AgentTools.js +2 -32
  24. package/dist/AgentTools.js.map +1 -1
  25. package/dist/ClaudeAgent.test.js +0 -1
  26. package/dist/ClaudeAgent.test.js.map +1 -1
  27. package/dist/ScriptFormatting.d.ts +5 -0
  28. package/dist/ScriptFormatting.d.ts.map +1 -1
  29. package/dist/ScriptFormatting.js +255 -53
  30. package/dist/ScriptFormatting.js.map +1 -1
  31. package/dist/ScriptFormatting.test.js +16 -0
  32. package/dist/ScriptFormatting.test.js.map +1 -1
  33. package/dist/bin/{Latch-DOdbsGMQ.mjs → Latch-BE6_TYRL.mjs} +1 -1
  34. package/dist/bin/{McpClient-B-FFG6Pg.mjs → McpClient-DCt4v5W5.mjs} +2 -2
  35. package/dist/bin/{OutputFormatter-DhhP2i-7.mjs → OutputFormatter-DaFv_C_I.mjs} +2 -3
  36. package/dist/bin/{Schema-CFdSO6iH.mjs → Schema-B3MiG1aD.mjs} +204 -568
  37. package/dist/bin/cli.mjs +5093 -2468
  38. package/dist/bin/{upgrade-C4aOEWJH.mjs → upgrade-WqgIU_21.mjs} +3 -3
  39. package/dist/cli.js +2 -25
  40. package/dist/cli.js.map +1 -1
  41. package/dist/index.d.ts +0 -8
  42. package/dist/index.d.ts.map +1 -1
  43. package/dist/index.js +0 -8
  44. package/dist/index.js.map +1 -1
  45. package/package.json +1 -7
  46. package/src/Acp.test.ts +0 -1
  47. package/src/AcpImage.test.ts +0 -1
  48. package/src/Agent.test.ts +0 -1
  49. package/src/Agent.ts +0 -4
  50. package/src/AgentCallerConfig.test.ts +0 -1
  51. package/src/AgentExecutor.ts +4 -16
  52. package/src/AgentImage.test.ts +0 -1
  53. package/src/AgentSkills.test.ts +0 -2
  54. package/src/AgentTools.ts +4 -41
  55. package/src/ClaudeAgent.test.ts +0 -1
  56. package/src/ScriptFormatting.test.ts +20 -0
  57. package/src/ScriptFormatting.ts +234 -47
  58. package/src/cli.ts +1 -56
  59. package/src/index.ts +0 -10
  60. package/dist/ChunkRepo.d.ts +0 -173
  61. package/dist/ChunkRepo.d.ts.map +0 -1
  62. package/dist/ChunkRepo.js +0 -168
  63. package/dist/ChunkRepo.js.map +0 -1
  64. package/dist/ChunkRepo.test.d.ts +0 -2
  65. package/dist/ChunkRepo.test.d.ts.map +0 -1
  66. package/dist/ChunkRepo.test.js +0 -63
  67. package/dist/ChunkRepo.test.js.map +0 -1
  68. package/dist/CodeChunker.d.ts +0 -84
  69. package/dist/CodeChunker.d.ts.map +0 -1
  70. package/dist/CodeChunker.js +0 -658
  71. package/dist/CodeChunker.js.map +0 -1
  72. package/dist/CodeChunker.test.d.ts +0 -2
  73. package/dist/CodeChunker.test.d.ts.map +0 -1
  74. package/dist/CodeChunker.test.js +0 -434
  75. package/dist/CodeChunker.test.js.map +0 -1
  76. package/dist/SemanticSearch/Service.d.ts +0 -31
  77. package/dist/SemanticSearch/Service.d.ts.map +0 -1
  78. package/dist/SemanticSearch/Service.js +0 -29
  79. package/dist/SemanticSearch/Service.js.map +0 -1
  80. package/dist/SemanticSearch.d.ts +0 -34
  81. package/dist/SemanticSearch.d.ts.map +0 -1
  82. package/dist/SemanticSearch.js +0 -171
  83. package/dist/SemanticSearch.js.map +0 -1
  84. package/dist/SemanticSearch.test.d.ts +0 -2
  85. package/dist/SemanticSearch.test.d.ts.map +0 -1
  86. package/dist/SemanticSearch.test.js +0 -34
  87. package/dist/SemanticSearch.test.js.map +0 -1
  88. package/dist/Sqlite.d.ts +0 -15
  89. package/dist/Sqlite.d.ts.map +0 -1
  90. package/dist/Sqlite.js +0 -44
  91. package/dist/Sqlite.js.map +0 -1
  92. package/dist/bin/AiError-DBaKz35C.mjs +0 -2723
  93. package/dist/bin/SemanticSearch-C3a7LAQr.mjs +0 -2558
  94. package/src/ChunkRepo.test.ts +0 -89
  95. package/src/ChunkRepo.ts +0 -260
  96. package/src/CodeChunker.test.ts +0 -493
  97. package/src/CodeChunker.ts +0 -1018
  98. package/src/SemanticSearch/Service.ts +0 -50
  99. package/src/SemanticSearch.test.ts +0 -51
  100. package/src/SemanticSearch.ts +0 -282
  101. package/src/Sqlite.ts +0 -60
@@ -1,658 +0,0 @@
1
- /**
2
- * @since 1.0.0
3
- */
4
- import * as Array from "effect/Array";
5
- import * as Effect from "effect/Effect";
6
- import * as FileSystem from "effect/FileSystem";
7
- import { pipe } from "effect/Function";
8
- import * as Layer from "effect/Layer";
9
- import * as Path from "effect/Path";
10
- import * as Context from "effect/Context";
11
- import * as Stream from "effect/Stream";
12
- import * as ChildProcess from "effect/process/ChildProcess";
13
- import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
14
- import TreeSitter, {} from "tree-sitter";
15
- import TreeSitterJavaScript from "tree-sitter-javascript";
16
- import TreeSitterTypeScript from "tree-sitter-typescript";
17
- /**
18
- * @since 1.0.0
19
- * @category Services
20
- */
21
- export class CodeChunker extends Context.Service()("clanka/CodeChunker") {
22
- }
23
- const sourceExtensions = new Set([
24
- "c",
25
- "cc",
26
- "cpp",
27
- "cs",
28
- "css",
29
- "cts",
30
- "cxx",
31
- "go",
32
- "gql",
33
- "graphql",
34
- "h",
35
- "hpp",
36
- "html",
37
- "ini",
38
- "java",
39
- "js",
40
- "jsx",
41
- "kt",
42
- "kts",
43
- "less",
44
- "lua",
45
- "mjs",
46
- "mts",
47
- "php",
48
- "py",
49
- "rb",
50
- "rs",
51
- "sass",
52
- "scala",
53
- "scss",
54
- "sh",
55
- "sql",
56
- "svelte",
57
- "swift",
58
- "ts",
59
- "tsx",
60
- "vue",
61
- "xml",
62
- "zsh",
63
- ]);
64
- const documentationExtensions = new Set([
65
- "adoc",
66
- "asciidoc",
67
- "md",
68
- "mdx",
69
- "rst",
70
- "txt",
71
- ]);
72
- const ignoredDirectories = new Set([
73
- ".git",
74
- ".next",
75
- ".nuxt",
76
- ".svelte-kit",
77
- ".turbo",
78
- "build",
79
- "coverage",
80
- "dist",
81
- "node_modules",
82
- "target",
83
- ]);
84
- const normalizePath = (path) => path.replace(/\\/g, "/");
85
- const normalizeText = (content) => content.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
86
- const meaningfulLinePattern = /[^\s\p{P}]/u;
87
- const isMeaningfulLine = (line) => meaningfulLinePattern.test(line);
88
- const languageByExtension = new Map([
89
- ["js", TreeSitterJavaScript],
90
- ["jsx", TreeSitterJavaScript],
91
- ["ts", TreeSitterTypeScript.typescript],
92
- ["tsx", TreeSitterTypeScript.tsx],
93
- ]);
94
- // const ignoredTopLevelNodeTypes = new Set(["comment", "import_statement"])
95
- /**
96
- * @since 1.0.0
97
- * @category Predicates
98
- */
99
- export const isProbablyMinified = (content) => {
100
- const normalized = normalizeText(content);
101
- if (normalized.length < 2_000) {
102
- return false;
103
- }
104
- const lines = normalized.split("\n");
105
- if (lines.length <= 2) {
106
- return true;
107
- }
108
- let longLines = 0;
109
- for (const line of lines) {
110
- if (line.length >= 300) {
111
- longLines++;
112
- }
113
- }
114
- return lines.length <= 20 && longLines / lines.length >= 0.8;
115
- };
116
- /**
117
- * @since 1.0.0
118
- * @category Predicates
119
- */
120
- export const isMeaningfulFile = (path) => {
121
- const normalizedPath = normalizePath(path);
122
- const lowercasePath = normalizedPath.toLowerCase();
123
- const parts = lowercasePath.split("/");
124
- const fileName = parts.at(-1);
125
- if (fileName === undefined || fileName.length === 0) {
126
- return false;
127
- }
128
- if (parts.some((part) => ignoredDirectories.has(part))) {
129
- return false;
130
- }
131
- if (/\.min\.(?:css|js)$/i.test(fileName)) {
132
- return false;
133
- }
134
- const extensionIndex = fileName.lastIndexOf(".");
135
- if (extensionIndex === -1) {
136
- return false;
137
- }
138
- const extension = fileName.slice(extensionIndex + 1);
139
- return (sourceExtensions.has(extension) || documentationExtensions.has(extension));
140
- };
141
- const resolveChunkSettings = (options) => {
142
- const chunkSize = Math.max(1, options.chunkSize);
143
- const chunkOverlap = Math.max(0, Math.min(chunkSize - 1, options.chunkOverlap));
144
- const chunkMaxCharacters = options.chunkMaxCharacters !== undefined &&
145
- Number.isFinite(options.chunkMaxCharacters)
146
- ? Math.max(1, Math.floor(options.chunkMaxCharacters))
147
- : Number.POSITIVE_INFINITY;
148
- return {
149
- chunkSize,
150
- chunkOverlap,
151
- chunkMaxCharacters,
152
- };
153
- };
154
- const getPathExtension = (path) => {
155
- const fileName = path.split("/").at(-1);
156
- if (fileName === undefined) {
157
- return undefined;
158
- }
159
- const extensionIndex = fileName.lastIndexOf(".");
160
- if (extensionIndex === -1) {
161
- return undefined;
162
- }
163
- return fileName.slice(extensionIndex + 1).toLowerCase();
164
- };
165
- const resolveAstLanguage = (path) => {
166
- const extension = getPathExtension(path);
167
- if (extension === undefined) {
168
- return undefined;
169
- }
170
- return languageByExtension.get(extension);
171
- };
172
- const lineRangeFromNode = (node) => {
173
- const startLine = node.startPosition.row + 1;
174
- const endLine = Math.max(startLine, node.endPosition.row + 1);
175
- return {
176
- startLine,
177
- endLine,
178
- };
179
- };
180
- const hasOnlyWhitespaceLines = (lines, startLine, endLine) => {
181
- if (startLine > endLine) {
182
- return true;
183
- }
184
- for (let lineIndex = startLine; lineIndex <= endLine; lineIndex++) {
185
- if ((lines[lineIndex - 1] ?? "").trim().length > 0) {
186
- return false;
187
- }
188
- }
189
- return true;
190
- };
191
- const lineRangeWithLeadingComments = (node, siblings, nodeIndex, lines) => {
192
- const baseRange = lineRangeFromNode(node);
193
- let startLine = baseRange.startLine;
194
- for (let index = nodeIndex - 1; index >= 0; index--) {
195
- const sibling = siblings[index];
196
- if (sibling.type !== "comment") {
197
- break;
198
- }
199
- const commentRange = lineRangeFromNode(sibling);
200
- if (!hasOnlyWhitespaceLines(lines, commentRange.endLine + 1, startLine - 1)) {
201
- break;
202
- }
203
- startLine = commentRange.startLine;
204
- }
205
- return {
206
- startLine,
207
- endLine: baseRange.endLine,
208
- };
209
- };
210
- const normalizeLineRange = (range, lineCount) => {
211
- const startLine = Math.max(1, Math.min(lineCount, range.startLine));
212
- const endLine = Math.max(1, Math.min(lineCount, range.endLine));
213
- if (endLine < startLine) {
214
- return undefined;
215
- }
216
- return {
217
- startLine,
218
- endLine,
219
- };
220
- };
221
- const lineLengthPrefixSums = (lines) => {
222
- const sums = [0];
223
- for (let index = 0; index < lines.length; index++) {
224
- sums.push(sums[index] + lines[index].length);
225
- }
226
- return sums;
227
- };
228
- const lineRangeCharacterLength = (prefixSums, range) => prefixSums[range.endLine] -
229
- prefixSums[range.startLine - 1] +
230
- (range.endLine - range.startLine);
231
- const resolveSegmentEndLine = (options) => {
232
- if (options.settings.chunkMaxCharacters === Number.POSITIVE_INFINITY) {
233
- return options.maxEndLine;
234
- }
235
- let endLine = options.maxEndLine;
236
- while (endLine > options.startLine &&
237
- lineRangeCharacterLength(options.prefixSums, {
238
- startLine: options.startLine,
239
- endLine,
240
- }) > options.settings.chunkMaxCharacters) {
241
- endLine--;
242
- }
243
- return endLine;
244
- };
245
- const splitRange = (range, settings, prefixSums) => {
246
- const lineCount = range.endLine - range.startLine + 1;
247
- if (lineCount <= settings.chunkSize &&
248
- lineRangeCharacterLength(prefixSums, range) <= settings.chunkMaxCharacters) {
249
- return [range];
250
- }
251
- const out = [];
252
- for (let startLine = range.startLine; startLine <= range.endLine;) {
253
- const maxEndLine = Math.min(range.endLine, startLine + settings.chunkSize - 1);
254
- const endLine = resolveSegmentEndLine({
255
- startLine,
256
- maxEndLine,
257
- settings,
258
- prefixSums,
259
- });
260
- out.push({
261
- startLine,
262
- endLine,
263
- });
264
- if (endLine >= range.endLine) {
265
- break;
266
- }
267
- startLine = Math.max(startLine + 1, endLine - settings.chunkOverlap + 1);
268
- }
269
- return out;
270
- };
271
- const nodeText = (node) => {
272
- if (node === null) {
273
- return undefined;
274
- }
275
- const value = node.text.trim().replace(/\s+/g, " ");
276
- return value.length === 0 ? undefined : value;
277
- };
278
- const nodeFieldText = (node, fieldName) => nodeText(node.childForFieldName(fieldName));
279
- const isNamespaceNode = (node) => node.type === "internal_module" || node.type === "module";
280
- const unwrapDeclarationNode = (node) => {
281
- let current = node;
282
- while (true) {
283
- if (current.type === "export_statement") {
284
- const declaration = current.childForFieldName("declaration") ?? current.namedChildren[0];
285
- if (declaration === undefined) {
286
- return current;
287
- }
288
- current = declaration;
289
- continue;
290
- }
291
- if (current.type === "ambient_declaration") {
292
- const declaration = current.namedChildren.find((child) => child.type.endsWith("_declaration") || isNamespaceNode(child));
293
- if (declaration === undefined) {
294
- return current;
295
- }
296
- current = declaration;
297
- continue;
298
- }
299
- return current;
300
- }
301
- };
302
- const variableDeclarators = (node) => node.namedChildren.filter((child) => child.type === "variable_declarator");
303
- const variableTypeFromDeclarator = (node) => {
304
- const value = node.childForFieldName("value");
305
- if (value !== null && value.type.includes("function")) {
306
- return "function";
307
- }
308
- return "variable";
309
- };
310
- const variableTypeFromDeclaration = (node) => {
311
- const declarators = variableDeclarators(node);
312
- if (declarators.some((declarator) => variableTypeFromDeclarator(declarator) === "function")) {
313
- return "function";
314
- }
315
- return "variable";
316
- };
317
- const chunkTypeFromNode = (node) => {
318
- switch (node.type) {
319
- case "class_declaration":
320
- return "class";
321
- case "enum_declaration":
322
- return "enum";
323
- case "function_declaration":
324
- case "generator_function_declaration":
325
- return "function";
326
- case "internal_module":
327
- case "module":
328
- return "namespace";
329
- case "interface_declaration":
330
- return "interface";
331
- case "generator_method_definition":
332
- case "method_definition":
333
- return "method";
334
- case "type_alias_declaration":
335
- return "type-alias";
336
- case "lexical_declaration":
337
- case "variable_declaration":
338
- return variableTypeFromDeclaration(node);
339
- case "variable_declarator":
340
- return variableTypeFromDeclarator(node);
341
- default:
342
- return undefined;
343
- }
344
- };
345
- const variableNamesFromDeclaration = (node) => {
346
- const names = variableDeclarators(node)
347
- .map((declarator) => nodeFieldText(declarator, "name"))
348
- .filter((name) => name !== undefined);
349
- if (names.length === 0) {
350
- return undefined;
351
- }
352
- return names.join(", ");
353
- };
354
- const nameFromNode = (node) => {
355
- switch (node.type) {
356
- case "class_declaration":
357
- case "enum_declaration":
358
- case "function_declaration":
359
- case "generator_function_declaration":
360
- case "internal_module":
361
- case "interface_declaration":
362
- case "module":
363
- case "generator_method_definition":
364
- case "method_definition":
365
- case "type_alias_declaration":
366
- case "variable_declarator":
367
- return nodeFieldText(node, "name");
368
- case "lexical_declaration":
369
- case "variable_declaration":
370
- return variableNamesFromDeclaration(node);
371
- default:
372
- return undefined;
373
- }
374
- };
375
- const formatParent = (type, name) => {
376
- if (type === undefined && name === undefined) {
377
- return undefined;
378
- }
379
- if (type === undefined) {
380
- return name;
381
- }
382
- if (name === undefined) {
383
- return type;
384
- }
385
- return type + " " + name;
386
- };
387
- const collectClassMethodRanges = (classNode, parent, lines) => {
388
- const body = classNode.childForFieldName("body");
389
- if (body === null) {
390
- return [];
391
- }
392
- const out = [];
393
- for (let index = 0; index < body.namedChildren.length; index++) {
394
- const child = body.namedChildren[index];
395
- if (!child.type.includes("method")) {
396
- continue;
397
- }
398
- out.push({
399
- ...lineRangeWithLeadingComments(child, body.namedChildren, index, lines),
400
- name: nameFromNode(child),
401
- type: chunkTypeFromNode(child),
402
- parent,
403
- });
404
- }
405
- return out;
406
- };
407
- const collectAstRanges = (path, content, lines) => {
408
- const language = resolveAstLanguage(path);
409
- if (language === undefined) {
410
- return [];
411
- }
412
- try {
413
- const parser = new TreeSitter();
414
- parser.setLanguage(language);
415
- const tree = parser.parse(content, undefined, {
416
- bufferSize: 1024 * 1024,
417
- });
418
- const collectDeclarationRanges = (siblings, parent) => {
419
- const out = [];
420
- for (let index = 0; index < siblings.length; index++) {
421
- const sibling = siblings[index];
422
- if (sibling.type === "comment" || sibling.type.includes("import")) {
423
- continue;
424
- }
425
- const declarationNode = unwrapDeclarationNode(sibling);
426
- const type = chunkTypeFromNode(declarationNode);
427
- const name = nameFromNode(declarationNode);
428
- if (type === undefined && name === undefined) {
429
- continue;
430
- }
431
- out.push({
432
- ...lineRangeWithLeadingComments(sibling, siblings, index, lines),
433
- name,
434
- type,
435
- parent,
436
- });
437
- if (declarationNode.type === "class_declaration") {
438
- out.push(...collectClassMethodRanges(declarationNode, formatParent(type, name), lines));
439
- }
440
- if (isNamespaceNode(declarationNode)) {
441
- const body = declarationNode.childForFieldName("body");
442
- if (body !== null) {
443
- out.push(...collectDeclarationRanges(body.namedChildren, formatParent(type, name)));
444
- }
445
- }
446
- }
447
- return out;
448
- };
449
- return collectDeclarationRanges(tree.rootNode.namedChildren, undefined);
450
- }
451
- catch {
452
- return [];
453
- }
454
- };
455
- const chunksFromRanges = (path, lines, ranges, settings) => {
456
- const hasMethodChildRange = (classRange) => {
457
- const parent = formatParent("class", classRange.name);
458
- return ranges.some((range) => range.type === "method" &&
459
- range.parent === parent &&
460
- range.startLine >= classRange.startLine &&
461
- range.endLine <= classRange.endLine);
462
- };
463
- const out = [];
464
- const seen = new Set();
465
- const prefixSums = lineLengthPrefixSums(lines);
466
- for (const range of ranges) {
467
- const normalizedRange = normalizeLineRange(range, lines.length);
468
- if (normalizedRange === undefined) {
469
- continue;
470
- }
471
- const allSegments = splitRange(normalizedRange, settings, prefixSums);
472
- const segments = range.type === "class" &&
473
- allSegments.length > 1 &&
474
- hasMethodChildRange({ ...normalizedRange, name: range.name })
475
- ? [allSegments[0]]
476
- : allSegments;
477
- for (const segment of segments) {
478
- const key = String(segment.startLine) +
479
- ":" +
480
- String(segment.endLine) +
481
- ":" +
482
- (range.name ?? "") +
483
- ":" +
484
- (range.type ?? "") +
485
- ":" +
486
- (range.parent ?? "");
487
- if (seen.has(key)) {
488
- continue;
489
- }
490
- seen.add(key);
491
- const chunkLines = lines.slice(segment.startLine - 1, segment.endLine);
492
- if (!chunkLines.some(isMeaningfulLine)) {
493
- continue;
494
- }
495
- out.push({
496
- path,
497
- startLine: segment.startLine,
498
- endLine: segment.endLine,
499
- name: range.name,
500
- type: range.type,
501
- parent: range.parent,
502
- content: chunkLines.join("\n"),
503
- });
504
- }
505
- }
506
- return out.toSorted((left, right) => left.startLine - right.startLine ||
507
- left.endLine - right.endLine ||
508
- (left.name ?? "").localeCompare(right.name ?? ""));
509
- };
510
- const chunkWithLineWindows = (path, lines, settings) => {
511
- const out = [];
512
- const prefixSums = lineLengthPrefixSums(lines);
513
- for (let index = 0; index < lines.length;) {
514
- if (!isMeaningfulLine(lines[index])) {
515
- index++;
516
- continue;
517
- }
518
- const startLine = index + 1;
519
- const maxEndLine = Math.min(lines.length, startLine + settings.chunkSize - 1);
520
- const endLine = resolveSegmentEndLine({
521
- startLine,
522
- maxEndLine,
523
- settings,
524
- prefixSums,
525
- });
526
- const chunkLines = lines.slice(startLine - 1, endLine);
527
- out.push({
528
- path,
529
- startLine,
530
- endLine,
531
- name: undefined,
532
- type: undefined,
533
- parent: undefined,
534
- content: chunkLines.join("\n"),
535
- });
536
- if (endLine >= lines.length) {
537
- break;
538
- }
539
- const nextStartLine = Math.max(startLine + 1, endLine - settings.chunkOverlap + 1);
540
- index = nextStartLine - 1;
541
- }
542
- return out;
543
- };
544
- const splitOversizedChunks = (chunks, maxCharacters) => chunks.flatMap((chunk) => {
545
- if (chunk.content.length <= maxCharacters) {
546
- return [chunk];
547
- }
548
- // Line-based splitting can only exceed the limit for a single long line.
549
- // Each piece retains that line number and the original AST metadata.
550
- const pieces = [];
551
- for (let offset = 0; offset < chunk.content.length; offset += maxCharacters) {
552
- pieces.push({
553
- ...chunk,
554
- content: chunk.content.slice(offset, offset + maxCharacters),
555
- });
556
- }
557
- return pieces;
558
- });
559
- /**
560
- * @since 1.0.0
561
- * @category Constructors
562
- */
563
- export const chunkFileContent = (path, content, options) => {
564
- if (content.trim().length === 0 || isProbablyMinified(content)) {
565
- return [];
566
- }
567
- const normalizedPath = normalizePath(path);
568
- const normalizedContent = normalizeText(content);
569
- const lines = normalizedContent.split("\n");
570
- if (lines.at(-1) === "") {
571
- lines.pop();
572
- }
573
- if (lines.length === 0) {
574
- return [];
575
- }
576
- const settings = resolveChunkSettings(options);
577
- const astRanges = collectAstRanges(normalizedPath, normalizedContent, lines);
578
- if (astRanges.length > 0) {
579
- const astChunks = chunksFromRanges(normalizedPath, lines, astRanges, settings);
580
- if (astChunks.length > 0) {
581
- return splitOversizedChunks(astChunks, settings.chunkMaxCharacters);
582
- }
583
- }
584
- return splitOversizedChunks(chunkWithLineWindows(normalizedPath, lines, settings), settings.chunkMaxCharacters);
585
- };
586
- /**
587
- * @since 1.0.0
588
- * @category Layers
589
- */
590
- export const layer = Layer.effect(CodeChunker, Effect.gen(function* () {
591
- const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
592
- const fs = yield* FileSystem.FileSystem;
593
- const pathService = yield* Path.Path;
594
- const listFiles = Effect.fn("CodeChunker.listFiles")(function* (options) {
595
- const root = pathService.resolve(options.root);
596
- const maxFileSize = options.maxFileSize ?? "1M";
597
- return yield* pipe(spawner.streamLines(ChildProcess.make("rg", [
598
- "--files",
599
- "--hidden",
600
- "--max-filesize",
601
- maxFileSize,
602
- "--glob",
603
- "!.git",
604
- ], {
605
- cwd: root,
606
- stdin: "ignore",
607
- })), Stream.runCollect, Effect.map(Array.fromIterable), Effect.map((entries) => entries
608
- .map((entry) => normalizePath(entry.trim()))
609
- .filter((entry) => entry.length > 0 && isMeaningfulFile(entry))
610
- .toSorted((left, right) => left.localeCompare(right))), Effect.orDie);
611
- });
612
- const chunkFile = Effect.fn("CodeChunker.chunkFile")(function* (options) {
613
- const root = pathService.resolve(options.root);
614
- const absolutePath = pathService.resolve(root, options.path);
615
- const path = normalizePath(pathService.relative(root, absolutePath));
616
- if (path.length === 0 ||
617
- path === ".." ||
618
- path.startsWith("../") ||
619
- !isMeaningfulFile(path)) {
620
- return [];
621
- }
622
- return yield* pipe(fs.readFileString(absolutePath), Effect.map((content) => chunkFileContent(path, content, options)), Effect.orElseSucceed(() => []));
623
- });
624
- const chunkFiles = (options) => Stream.fromArray(options.paths).pipe(Stream.flatMap((path) => pipe(chunkFile({
625
- root: options.root,
626
- path,
627
- chunkSize: options.chunkSize,
628
- chunkOverlap: options.chunkOverlap,
629
- ...(options.chunkMaxCharacters === undefined
630
- ? {}
631
- : { chunkMaxCharacters: options.chunkMaxCharacters }),
632
- }), Stream.fromArrayEffect), { concurrency: 5 }));
633
- const chunkCodebase = Effect.fnUntraced(function* (options) {
634
- const root = pathService.resolve(options.root);
635
- const files = yield* listFiles({
636
- root,
637
- ...(options.maxFileSize === undefined
638
- ? {}
639
- : { maxFileSize: options.maxFileSize }),
640
- });
641
- return chunkFiles({
642
- root,
643
- paths: files,
644
- chunkSize: options.chunkSize,
645
- chunkOverlap: options.chunkOverlap,
646
- ...(options.chunkMaxCharacters === undefined
647
- ? {}
648
- : { chunkMaxCharacters: options.chunkMaxCharacters }),
649
- });
650
- }, Stream.unwrap);
651
- return CodeChunker.of({
652
- listFiles,
653
- chunkFile,
654
- chunkFiles,
655
- chunkCodebase,
656
- });
657
- }));
658
- //# sourceMappingURL=CodeChunker.js.map