clanka 0.26.1 → 0.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +2 -3
  2. package/dist/Acp.test.js +0 -1
  3. package/dist/Acp.test.js.map +1 -1
  4. package/dist/AcpImage.test.js +0 -1
  5. package/dist/AcpImage.test.js.map +1 -1
  6. package/dist/Agent.d.ts.map +1 -1
  7. package/dist/Agent.js +1 -2
  8. package/dist/Agent.js.map +1 -1
  9. package/dist/Agent.test.js +0 -1
  10. package/dist/Agent.test.js.map +1 -1
  11. package/dist/AgentCallerConfig.test.js +0 -1
  12. package/dist/AgentCallerConfig.test.js.map +1 -1
  13. package/dist/AgentExecutor.d.ts +2 -3
  14. package/dist/AgentExecutor.d.ts.map +1 -1
  15. package/dist/AgentExecutor.js +4 -13
  16. package/dist/AgentExecutor.js.map +1 -1
  17. package/dist/AgentImage.test.js +0 -1
  18. package/dist/AgentImage.test.js.map +1 -1
  19. package/dist/AgentSkills.test.js +0 -2
  20. package/dist/AgentSkills.test.js.map +1 -1
  21. package/dist/AgentTools.d.ts +2 -235
  22. package/dist/AgentTools.d.ts.map +1 -1
  23. package/dist/AgentTools.js +2 -32
  24. package/dist/AgentTools.js.map +1 -1
  25. package/dist/ClaudeAgent.test.js +0 -1
  26. package/dist/ClaudeAgent.test.js.map +1 -1
  27. package/dist/McpServers.d.ts +20 -1
  28. package/dist/McpServers.d.ts.map +1 -1
  29. package/dist/McpServers.js +12 -5
  30. package/dist/McpServers.js.map +1 -1
  31. package/dist/McpServers.test.js +115 -0
  32. package/dist/McpServers.test.js.map +1 -1
  33. package/dist/ScriptFormatting.d.ts +5 -0
  34. package/dist/ScriptFormatting.d.ts.map +1 -1
  35. package/dist/ScriptFormatting.js +255 -53
  36. package/dist/ScriptFormatting.js.map +1 -1
  37. package/dist/ScriptFormatting.test.js +16 -0
  38. package/dist/ScriptFormatting.test.js.map +1 -1
  39. package/dist/bin/{Latch-DOdbsGMQ.mjs → Latch-BE6_TYRL.mjs} +1 -1
  40. package/dist/bin/{McpClient-B-FFG6Pg.mjs → McpClient-DCt4v5W5.mjs} +2 -2
  41. package/dist/bin/{OutputFormatter-DhhP2i-7.mjs → OutputFormatter-DaFv_C_I.mjs} +2 -3
  42. package/dist/bin/{Schema-CFdSO6iH.mjs → Schema-B3MiG1aD.mjs} +204 -568
  43. package/dist/bin/cli.mjs +5098 -2472
  44. package/dist/bin/{upgrade-C4aOEWJH.mjs → upgrade-WqgIU_21.mjs} +3 -3
  45. package/dist/cli.js +2 -25
  46. package/dist/cli.js.map +1 -1
  47. package/dist/fixtures/McpFakeServer.d.ts +4 -0
  48. package/dist/fixtures/McpFakeServer.d.ts.map +1 -1
  49. package/dist/fixtures/McpFakeServer.js +13 -0
  50. package/dist/fixtures/McpFakeServer.js.map +1 -1
  51. package/dist/index.d.ts +0 -8
  52. package/dist/index.d.ts.map +1 -1
  53. package/dist/index.js +0 -8
  54. package/dist/index.js.map +1 -1
  55. package/package.json +1 -7
  56. package/src/Acp.test.ts +0 -1
  57. package/src/AcpImage.test.ts +0 -1
  58. package/src/Agent.test.ts +0 -1
  59. package/src/Agent.ts +0 -4
  60. package/src/AgentCallerConfig.test.ts +0 -1
  61. package/src/AgentExecutor.ts +4 -16
  62. package/src/AgentImage.test.ts +0 -1
  63. package/src/AgentSkills.test.ts +0 -2
  64. package/src/AgentTools.ts +4 -41
  65. package/src/ClaudeAgent.test.ts +0 -1
  66. package/src/McpServers.test.ts +198 -0
  67. package/src/McpServers.ts +44 -4
  68. package/src/ScriptFormatting.test.ts +20 -0
  69. package/src/ScriptFormatting.ts +234 -47
  70. package/src/cli.ts +1 -56
  71. package/src/fixtures/McpFakeServer.ts +17 -1
  72. package/src/index.ts +0 -10
  73. package/dist/ChunkRepo.d.ts +0 -173
  74. package/dist/ChunkRepo.d.ts.map +0 -1
  75. package/dist/ChunkRepo.js +0 -168
  76. package/dist/ChunkRepo.js.map +0 -1
  77. package/dist/ChunkRepo.test.d.ts +0 -2
  78. package/dist/ChunkRepo.test.d.ts.map +0 -1
  79. package/dist/ChunkRepo.test.js +0 -63
  80. package/dist/ChunkRepo.test.js.map +0 -1
  81. package/dist/CodeChunker.d.ts +0 -84
  82. package/dist/CodeChunker.d.ts.map +0 -1
  83. package/dist/CodeChunker.js +0 -658
  84. package/dist/CodeChunker.js.map +0 -1
  85. package/dist/CodeChunker.test.d.ts +0 -2
  86. package/dist/CodeChunker.test.d.ts.map +0 -1
  87. package/dist/CodeChunker.test.js +0 -434
  88. package/dist/CodeChunker.test.js.map +0 -1
  89. package/dist/SemanticSearch/Service.d.ts +0 -31
  90. package/dist/SemanticSearch/Service.d.ts.map +0 -1
  91. package/dist/SemanticSearch/Service.js +0 -29
  92. package/dist/SemanticSearch/Service.js.map +0 -1
  93. package/dist/SemanticSearch.d.ts +0 -34
  94. package/dist/SemanticSearch.d.ts.map +0 -1
  95. package/dist/SemanticSearch.js +0 -171
  96. package/dist/SemanticSearch.js.map +0 -1
  97. package/dist/SemanticSearch.test.d.ts +0 -2
  98. package/dist/SemanticSearch.test.d.ts.map +0 -1
  99. package/dist/SemanticSearch.test.js +0 -34
  100. package/dist/SemanticSearch.test.js.map +0 -1
  101. package/dist/Sqlite.d.ts +0 -15
  102. package/dist/Sqlite.d.ts.map +0 -1
  103. package/dist/Sqlite.js +0 -44
  104. package/dist/Sqlite.js.map +0 -1
  105. package/dist/bin/AiError-DBaKz35C.mjs +0 -2723
  106. package/dist/bin/SemanticSearch-C3a7LAQr.mjs +0 -2558
  107. package/src/ChunkRepo.test.ts +0 -89
  108. package/src/ChunkRepo.ts +0 -260
  109. package/src/CodeChunker.test.ts +0 -493
  110. package/src/CodeChunker.ts +0 -1018
  111. package/src/SemanticSearch/Service.ts +0 -50
  112. package/src/SemanticSearch.test.ts +0 -51
  113. package/src/SemanticSearch.ts +0 -282
  114. package/src/Sqlite.ts +0 -60
@@ -1,1018 +0,0 @@
1
- /**
2
- * @since 1.0.0
3
- */
4
- import * as Array from "effect/Array"
5
- import * as Effect from "effect/Effect"
6
- import * as FileSystem from "effect/FileSystem"
7
- import { pipe } from "effect/Function"
8
- import * as Layer from "effect/Layer"
9
- import * as Path from "effect/Path"
10
- import * as Context from "effect/Context"
11
- import * as Stream from "effect/Stream"
12
- import * as ChildProcess from "effect/process/ChildProcess"
13
- import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"
14
- import TreeSitter, { type SyntaxNode } from "tree-sitter"
15
- import TreeSitterJavaScript from "tree-sitter-javascript"
16
- import TreeSitterTypeScript from "tree-sitter-typescript"
17
-
18
- /**
19
- * @since 1.0.0
20
- * @category Models
21
- */
22
- export interface CodeChunk {
23
- readonly path: string
24
- readonly startLine: number
25
- readonly endLine: number
26
- readonly name: string | undefined
27
- readonly type: ChunkType | undefined
28
- readonly parent: string | undefined
29
- readonly content: string
30
- }
31
-
32
- /**
33
- * @since 1.0.0
34
- * @category Models
35
- */
36
- export type ChunkType =
37
- | "function"
38
- | "method"
39
- | "class"
40
- | "namespace"
41
- | "interface"
42
- | "type-alias"
43
- | "enum"
44
- | "variable"
45
-
46
- /**
47
- * @since 1.0.0
48
- * @category Services
49
- */
50
- export class CodeChunker extends Context.Service<
51
- CodeChunker,
52
- {
53
- listFiles(options: {
54
- readonly root: string
55
- readonly maxFileSize?: string | undefined
56
- }): Effect.Effect<ReadonlyArray<string>>
57
- chunkFile(options: {
58
- readonly root: string
59
- readonly path: string
60
- readonly chunkSize: number
61
- readonly chunkOverlap: number
62
- readonly chunkMaxCharacters?: number | undefined
63
- }): Effect.Effect<ReadonlyArray<CodeChunk>>
64
- chunkFiles(options: {
65
- readonly root: string
66
- readonly paths: ReadonlyArray<string>
67
- readonly chunkSize: number
68
- readonly chunkOverlap: number
69
- readonly chunkMaxCharacters?: number | undefined
70
- }): Stream.Stream<CodeChunk>
71
- chunkCodebase(options: {
72
- readonly root: string
73
- readonly maxFileSize?: string | undefined
74
- readonly chunkSize: number
75
- readonly chunkOverlap: number
76
- readonly chunkMaxCharacters?: number | undefined
77
- }): Stream.Stream<CodeChunk>
78
- }
79
- >()("clanka/CodeChunker") {}
80
-
81
- const sourceExtensions = new Set([
82
- "c",
83
- "cc",
84
- "cpp",
85
- "cs",
86
- "css",
87
- "cts",
88
- "cxx",
89
- "go",
90
- "gql",
91
- "graphql",
92
- "h",
93
- "hpp",
94
- "html",
95
- "ini",
96
- "java",
97
- "js",
98
- "jsx",
99
- "kt",
100
- "kts",
101
- "less",
102
- "lua",
103
- "mjs",
104
- "mts",
105
- "php",
106
- "py",
107
- "rb",
108
- "rs",
109
- "sass",
110
- "scala",
111
- "scss",
112
- "sh",
113
- "sql",
114
- "svelte",
115
- "swift",
116
- "ts",
117
- "tsx",
118
- "vue",
119
- "xml",
120
- "zsh",
121
- ])
122
-
123
- const documentationExtensions = new Set([
124
- "adoc",
125
- "asciidoc",
126
- "md",
127
- "mdx",
128
- "rst",
129
- "txt",
130
- ])
131
-
132
- const ignoredDirectories = new Set([
133
- ".git",
134
- ".next",
135
- ".nuxt",
136
- ".svelte-kit",
137
- ".turbo",
138
- "build",
139
- "coverage",
140
- "dist",
141
- "node_modules",
142
- "target",
143
- ])
144
-
145
- const normalizePath = (path: string): string => path.replace(/\\/g, "/")
146
-
147
- const normalizeText = (content: string): string =>
148
- content.replace(/\r\n/g, "\n").replace(/\r/g, "\n")
149
-
150
- const meaningfulLinePattern = /[^\s\p{P}]/u
151
-
152
- const isMeaningfulLine = (line: string): boolean =>
153
- meaningfulLinePattern.test(line)
154
-
155
- interface LineRange {
156
- readonly startLine: number
157
- readonly endLine: number
158
- }
159
-
160
- interface ChunkSettings {
161
- readonly chunkSize: number
162
- readonly chunkOverlap: number
163
- readonly chunkMaxCharacters: number
164
- }
165
-
166
- interface ChunkRange extends LineRange {
167
- readonly name: string | undefined
168
- readonly type: ChunkType | undefined
169
- readonly parent: string | undefined
170
- }
171
-
172
- const languageByExtension = new Map<string, unknown>([
173
- ["js", TreeSitterJavaScript],
174
- ["jsx", TreeSitterJavaScript],
175
- ["ts", TreeSitterTypeScript.typescript],
176
- ["tsx", TreeSitterTypeScript.tsx],
177
- ])
178
-
179
- // const ignoredTopLevelNodeTypes = new Set(["comment", "import_statement"])
180
-
181
- /**
182
- * @since 1.0.0
183
- * @category Predicates
184
- */
185
- export const isProbablyMinified = (content: string): boolean => {
186
- const normalized = normalizeText(content)
187
- if (normalized.length < 2_000) {
188
- return false
189
- }
190
-
191
- const lines = normalized.split("\n")
192
- if (lines.length <= 2) {
193
- return true
194
- }
195
-
196
- let longLines = 0
197
- for (const line of lines) {
198
- if (line.length >= 300) {
199
- longLines++
200
- }
201
- }
202
-
203
- return lines.length <= 20 && longLines / lines.length >= 0.8
204
- }
205
-
206
- /**
207
- * @since 1.0.0
208
- * @category Predicates
209
- */
210
- export const isMeaningfulFile = (path: string): boolean => {
211
- const normalizedPath = normalizePath(path)
212
- const lowercasePath = normalizedPath.toLowerCase()
213
- const parts = lowercasePath.split("/")
214
- const fileName = parts.at(-1)
215
- if (fileName === undefined || fileName.length === 0) {
216
- return false
217
- }
218
-
219
- if (parts.some((part) => ignoredDirectories.has(part))) {
220
- return false
221
- }
222
-
223
- if (/\.min\.(?:css|js)$/i.test(fileName)) {
224
- return false
225
- }
226
-
227
- const extensionIndex = fileName.lastIndexOf(".")
228
- if (extensionIndex === -1) {
229
- return false
230
- }
231
-
232
- const extension = fileName.slice(extensionIndex + 1)
233
- return (
234
- sourceExtensions.has(extension) || documentationExtensions.has(extension)
235
- )
236
- }
237
-
238
- const resolveChunkSettings = (options: {
239
- readonly chunkSize: number
240
- readonly chunkOverlap: number
241
- readonly chunkMaxCharacters?: number | undefined
242
- }): ChunkSettings => {
243
- const chunkSize = Math.max(1, options.chunkSize)
244
- const chunkOverlap = Math.max(
245
- 0,
246
- Math.min(chunkSize - 1, options.chunkOverlap),
247
- )
248
- const chunkMaxCharacters =
249
- options.chunkMaxCharacters !== undefined &&
250
- Number.isFinite(options.chunkMaxCharacters)
251
- ? Math.max(1, Math.floor(options.chunkMaxCharacters))
252
- : Number.POSITIVE_INFINITY
253
-
254
- return {
255
- chunkSize,
256
- chunkOverlap,
257
- chunkMaxCharacters,
258
- }
259
- }
260
-
261
- const getPathExtension = (path: string): string | undefined => {
262
- const fileName = path.split("/").at(-1)
263
- if (fileName === undefined) {
264
- return undefined
265
- }
266
-
267
- const extensionIndex = fileName.lastIndexOf(".")
268
- if (extensionIndex === -1) {
269
- return undefined
270
- }
271
-
272
- return fileName.slice(extensionIndex + 1).toLowerCase()
273
- }
274
-
275
- const resolveAstLanguage = (path: string): unknown => {
276
- const extension = getPathExtension(path)
277
- if (extension === undefined) {
278
- return undefined
279
- }
280
-
281
- return languageByExtension.get(extension)
282
- }
283
-
284
- const lineRangeFromNode = (node: SyntaxNode): LineRange => {
285
- const startLine = node.startPosition.row + 1
286
- const endLine = Math.max(startLine, node.endPosition.row + 1)
287
- return {
288
- startLine,
289
- endLine,
290
- }
291
- }
292
-
293
- const hasOnlyWhitespaceLines = (
294
- lines: ReadonlyArray<string>,
295
- startLine: number,
296
- endLine: number,
297
- ): boolean => {
298
- if (startLine > endLine) {
299
- return true
300
- }
301
-
302
- for (let lineIndex = startLine; lineIndex <= endLine; lineIndex++) {
303
- if ((lines[lineIndex - 1] ?? "").trim().length > 0) {
304
- return false
305
- }
306
- }
307
-
308
- return true
309
- }
310
-
311
- const lineRangeWithLeadingComments = (
312
- node: SyntaxNode,
313
- siblings: ReadonlyArray<SyntaxNode>,
314
- nodeIndex: number,
315
- lines: ReadonlyArray<string>,
316
- ): LineRange => {
317
- const baseRange = lineRangeFromNode(node)
318
- let startLine = baseRange.startLine
319
-
320
- for (let index = nodeIndex - 1; index >= 0; index--) {
321
- const sibling = siblings[index]!
322
- if (sibling.type !== "comment") {
323
- break
324
- }
325
-
326
- const commentRange = lineRangeFromNode(sibling)
327
- if (
328
- !hasOnlyWhitespaceLines(lines, commentRange.endLine + 1, startLine - 1)
329
- ) {
330
- break
331
- }
332
-
333
- startLine = commentRange.startLine
334
- }
335
-
336
- return {
337
- startLine,
338
- endLine: baseRange.endLine,
339
- }
340
- }
341
-
342
- const normalizeLineRange = (
343
- range: LineRange,
344
- lineCount: number,
345
- ): LineRange | undefined => {
346
- const startLine = Math.max(1, Math.min(lineCount, range.startLine))
347
- const endLine = Math.max(1, Math.min(lineCount, range.endLine))
348
-
349
- if (endLine < startLine) {
350
- return undefined
351
- }
352
-
353
- return {
354
- startLine,
355
- endLine,
356
- }
357
- }
358
-
359
- const lineLengthPrefixSums = (
360
- lines: ReadonlyArray<string>,
361
- ): ReadonlyArray<number> => {
362
- const sums = [0] as Array<number>
363
-
364
- for (let index = 0; index < lines.length; index++) {
365
- sums.push(sums[index]! + lines[index]!.length)
366
- }
367
-
368
- return sums
369
- }
370
-
371
- const lineRangeCharacterLength = (
372
- prefixSums: ReadonlyArray<number>,
373
- range: LineRange,
374
- ): number =>
375
- prefixSums[range.endLine]! -
376
- prefixSums[range.startLine - 1]! +
377
- (range.endLine - range.startLine)
378
-
379
- const resolveSegmentEndLine = (options: {
380
- readonly startLine: number
381
- readonly maxEndLine: number
382
- readonly settings: ChunkSettings
383
- readonly prefixSums: ReadonlyArray<number>
384
- }): number => {
385
- if (options.settings.chunkMaxCharacters === Number.POSITIVE_INFINITY) {
386
- return options.maxEndLine
387
- }
388
-
389
- let endLine = options.maxEndLine
390
- while (
391
- endLine > options.startLine &&
392
- lineRangeCharacterLength(options.prefixSums, {
393
- startLine: options.startLine,
394
- endLine,
395
- }) > options.settings.chunkMaxCharacters
396
- ) {
397
- endLine--
398
- }
399
-
400
- return endLine
401
- }
402
-
403
- const splitRange = (
404
- range: LineRange,
405
- settings: ChunkSettings,
406
- prefixSums: ReadonlyArray<number>,
407
- ): ReadonlyArray<LineRange> => {
408
- const lineCount = range.endLine - range.startLine + 1
409
- if (
410
- lineCount <= settings.chunkSize &&
411
- lineRangeCharacterLength(prefixSums, range) <= settings.chunkMaxCharacters
412
- ) {
413
- return [range]
414
- }
415
- const out = [] as Array<LineRange>
416
-
417
- for (let startLine = range.startLine; startLine <= range.endLine;) {
418
- const maxEndLine = Math.min(
419
- range.endLine,
420
- startLine + settings.chunkSize - 1,
421
- )
422
- const endLine = resolveSegmentEndLine({
423
- startLine,
424
- maxEndLine,
425
- settings,
426
- prefixSums,
427
- })
428
-
429
- out.push({
430
- startLine,
431
- endLine,
432
- })
433
-
434
- if (endLine >= range.endLine) {
435
- break
436
- }
437
-
438
- startLine = Math.max(startLine + 1, endLine - settings.chunkOverlap + 1)
439
- }
440
-
441
- return out
442
- }
443
-
444
- const nodeText = (node: SyntaxNode | null): string | undefined => {
445
- if (node === null) {
446
- return undefined
447
- }
448
-
449
- const value = node.text.trim().replace(/\s+/g, " ")
450
- return value.length === 0 ? undefined : value
451
- }
452
-
453
- const nodeFieldText = (
454
- node: SyntaxNode,
455
- fieldName: string,
456
- ): string | undefined => nodeText(node.childForFieldName(fieldName))
457
-
458
- const isNamespaceNode = (node: SyntaxNode): boolean =>
459
- node.type === "internal_module" || node.type === "module"
460
-
461
- const unwrapDeclarationNode = (node: SyntaxNode): SyntaxNode => {
462
- let current = node
463
-
464
- while (true) {
465
- if (current.type === "export_statement") {
466
- const declaration =
467
- current.childForFieldName("declaration") ?? current.namedChildren[0]
468
- if (declaration === undefined) {
469
- return current
470
- }
471
- current = declaration
472
- continue
473
- }
474
-
475
- if (current.type === "ambient_declaration") {
476
- const declaration = current.namedChildren.find(
477
- (child) =>
478
- child.type.endsWith("_declaration") || isNamespaceNode(child),
479
- )
480
- if (declaration === undefined) {
481
- return current
482
- }
483
- current = declaration
484
- continue
485
- }
486
-
487
- return current
488
- }
489
- }
490
-
491
- const variableDeclarators = (node: SyntaxNode): ReadonlyArray<SyntaxNode> =>
492
- node.namedChildren.filter((child) => child.type === "variable_declarator")
493
-
494
- const variableTypeFromDeclarator = (node: SyntaxNode): ChunkType => {
495
- const value = node.childForFieldName("value")
496
- if (value !== null && value.type.includes("function")) {
497
- return "function"
498
- }
499
- return "variable"
500
- }
501
-
502
- const variableTypeFromDeclaration = (node: SyntaxNode): ChunkType => {
503
- const declarators = variableDeclarators(node)
504
- if (
505
- declarators.some(
506
- (declarator) => variableTypeFromDeclarator(declarator) === "function",
507
- )
508
- ) {
509
- return "function"
510
- }
511
- return "variable"
512
- }
513
-
514
- const chunkTypeFromNode = (node: SyntaxNode): ChunkType | undefined => {
515
- switch (node.type) {
516
- case "class_declaration":
517
- return "class"
518
- case "enum_declaration":
519
- return "enum"
520
- case "function_declaration":
521
- case "generator_function_declaration":
522
- return "function"
523
- case "internal_module":
524
- case "module":
525
- return "namespace"
526
- case "interface_declaration":
527
- return "interface"
528
- case "generator_method_definition":
529
- case "method_definition":
530
- return "method"
531
- case "type_alias_declaration":
532
- return "type-alias"
533
- case "lexical_declaration":
534
- case "variable_declaration":
535
- return variableTypeFromDeclaration(node)
536
- case "variable_declarator":
537
- return variableTypeFromDeclarator(node)
538
- default:
539
- return undefined
540
- }
541
- }
542
-
543
- const variableNamesFromDeclaration = (node: SyntaxNode): string | undefined => {
544
- const names = variableDeclarators(node)
545
- .map((declarator) => nodeFieldText(declarator, "name"))
546
- .filter((name): name is string => name !== undefined)
547
-
548
- if (names.length === 0) {
549
- return undefined
550
- }
551
-
552
- return names.join(", ")
553
- }
554
-
555
- const nameFromNode = (node: SyntaxNode): string | undefined => {
556
- switch (node.type) {
557
- case "class_declaration":
558
- case "enum_declaration":
559
- case "function_declaration":
560
- case "generator_function_declaration":
561
- case "internal_module":
562
- case "interface_declaration":
563
- case "module":
564
- case "generator_method_definition":
565
- case "method_definition":
566
- case "type_alias_declaration":
567
- case "variable_declarator":
568
- return nodeFieldText(node, "name")
569
- case "lexical_declaration":
570
- case "variable_declaration":
571
- return variableNamesFromDeclaration(node)
572
- default:
573
- return undefined
574
- }
575
- }
576
-
577
- const formatParent = (
578
- type: ChunkType | undefined,
579
- name: string | undefined,
580
- ): string | undefined => {
581
- if (type === undefined && name === undefined) {
582
- return undefined
583
- }
584
- if (type === undefined) {
585
- return name
586
- }
587
- if (name === undefined) {
588
- return type
589
- }
590
- return type + " " + name
591
- }
592
-
593
- const collectClassMethodRanges = (
594
- classNode: SyntaxNode,
595
- parent: string | undefined,
596
- lines: ReadonlyArray<string>,
597
- ): ReadonlyArray<ChunkRange> => {
598
- const body = classNode.childForFieldName("body")
599
- if (body === null) {
600
- return []
601
- }
602
-
603
- const out = [] as Array<ChunkRange>
604
- for (let index = 0; index < body.namedChildren.length; index++) {
605
- const child = body.namedChildren[index]!
606
- if (!child.type.includes("method")) {
607
- continue
608
- }
609
-
610
- out.push({
611
- ...lineRangeWithLeadingComments(child, body.namedChildren, index, lines),
612
- name: nameFromNode(child),
613
- type: chunkTypeFromNode(child),
614
- parent,
615
- })
616
- }
617
-
618
- return out
619
- }
620
-
621
- const collectAstRanges = (
622
- path: string,
623
- content: string,
624
- lines: ReadonlyArray<string>,
625
- ): ReadonlyArray<ChunkRange> => {
626
- const language = resolveAstLanguage(path)
627
- if (language === undefined) {
628
- return []
629
- }
630
-
631
- try {
632
- const parser = new TreeSitter()
633
- parser.setLanguage(language)
634
- const tree = parser.parse(content, undefined, {
635
- bufferSize: 1024 * 1024,
636
- })
637
- const collectDeclarationRanges = (
638
- siblings: ReadonlyArray<SyntaxNode>,
639
- parent: string | undefined,
640
- ): ReadonlyArray<ChunkRange> => {
641
- const out = [] as Array<ChunkRange>
642
-
643
- for (let index = 0; index < siblings.length; index++) {
644
- const sibling = siblings[index]!
645
- if (sibling.type === "comment" || sibling.type.includes("import")) {
646
- continue
647
- }
648
-
649
- const declarationNode = unwrapDeclarationNode(sibling)
650
- const type = chunkTypeFromNode(declarationNode)
651
- const name = nameFromNode(declarationNode)
652
- if (type === undefined && name === undefined) {
653
- continue
654
- }
655
-
656
- out.push({
657
- ...lineRangeWithLeadingComments(sibling, siblings, index, lines),
658
- name,
659
- type,
660
- parent,
661
- })
662
-
663
- if (declarationNode.type === "class_declaration") {
664
- out.push(
665
- ...collectClassMethodRanges(
666
- declarationNode,
667
- formatParent(type, name),
668
- lines,
669
- ),
670
- )
671
- }
672
-
673
- if (isNamespaceNode(declarationNode)) {
674
- const body = declarationNode.childForFieldName("body")
675
- if (body !== null) {
676
- out.push(
677
- ...collectDeclarationRanges(
678
- body.namedChildren,
679
- formatParent(type, name),
680
- ),
681
- )
682
- }
683
- }
684
- }
685
-
686
- return out
687
- }
688
-
689
- return collectDeclarationRanges(tree.rootNode.namedChildren, undefined)
690
- } catch {
691
- return []
692
- }
693
- }
694
-
695
- const chunksFromRanges = (
696
- path: string,
697
- lines: ReadonlyArray<string>,
698
- ranges: ReadonlyArray<ChunkRange>,
699
- settings: ChunkSettings,
700
- ): ReadonlyArray<CodeChunk> => {
701
- const hasMethodChildRange = (
702
- classRange: LineRange & { readonly name: string | undefined },
703
- ) => {
704
- const parent = formatParent("class", classRange.name)
705
- return ranges.some(
706
- (range) =>
707
- range.type === "method" &&
708
- range.parent === parent &&
709
- range.startLine >= classRange.startLine &&
710
- range.endLine <= classRange.endLine,
711
- )
712
- }
713
-
714
- const out = [] as Array<CodeChunk>
715
- const seen = new Set<string>()
716
- const prefixSums = lineLengthPrefixSums(lines)
717
-
718
- for (const range of ranges) {
719
- const normalizedRange = normalizeLineRange(range, lines.length)
720
- if (normalizedRange === undefined) {
721
- continue
722
- }
723
-
724
- const allSegments = splitRange(normalizedRange, settings, prefixSums)
725
- const segments =
726
- range.type === "class" &&
727
- allSegments.length > 1 &&
728
- hasMethodChildRange({ ...normalizedRange, name: range.name })
729
- ? [allSegments[0]!]
730
- : allSegments
731
-
732
- for (const segment of segments) {
733
- const key =
734
- String(segment.startLine) +
735
- ":" +
736
- String(segment.endLine) +
737
- ":" +
738
- (range.name ?? "") +
739
- ":" +
740
- (range.type ?? "") +
741
- ":" +
742
- (range.parent ?? "")
743
- if (seen.has(key)) {
744
- continue
745
- }
746
- seen.add(key)
747
-
748
- const chunkLines = lines.slice(segment.startLine - 1, segment.endLine)
749
- if (!chunkLines.some(isMeaningfulLine)) {
750
- continue
751
- }
752
-
753
- out.push({
754
- path,
755
- startLine: segment.startLine,
756
- endLine: segment.endLine,
757
- name: range.name,
758
- type: range.type,
759
- parent: range.parent,
760
- content: chunkLines.join("\n"),
761
- })
762
- }
763
- }
764
-
765
- return out.toSorted(
766
- (left, right) =>
767
- left.startLine - right.startLine ||
768
- left.endLine - right.endLine ||
769
- (left.name ?? "").localeCompare(right.name ?? ""),
770
- )
771
- }
772
-
773
- const chunkWithLineWindows = (
774
- path: string,
775
- lines: ReadonlyArray<string>,
776
- settings: ChunkSettings,
777
- ): ReadonlyArray<CodeChunk> => {
778
- const out = [] as Array<CodeChunk>
779
- const prefixSums = lineLengthPrefixSums(lines)
780
-
781
- for (let index = 0; index < lines.length;) {
782
- if (!isMeaningfulLine(lines[index]!)) {
783
- index++
784
- continue
785
- }
786
-
787
- const startLine = index + 1
788
- const maxEndLine = Math.min(
789
- lines.length,
790
- startLine + settings.chunkSize - 1,
791
- )
792
- const endLine = resolveSegmentEndLine({
793
- startLine,
794
- maxEndLine,
795
- settings,
796
- prefixSums,
797
- })
798
- const chunkLines = lines.slice(startLine - 1, endLine)
799
-
800
- out.push({
801
- path,
802
- startLine,
803
- endLine,
804
- name: undefined,
805
- type: undefined,
806
- parent: undefined,
807
- content: chunkLines.join("\n"),
808
- })
809
-
810
- if (endLine >= lines.length) {
811
- break
812
- }
813
-
814
- const nextStartLine = Math.max(
815
- startLine + 1,
816
- endLine - settings.chunkOverlap + 1,
817
- )
818
- index = nextStartLine - 1
819
- }
820
-
821
- return out
822
- }
823
-
824
- const splitOversizedChunks = (
825
- chunks: ReadonlyArray<CodeChunk>,
826
- maxCharacters: number,
827
- ): ReadonlyArray<CodeChunk> =>
828
- chunks.flatMap((chunk) => {
829
- if (chunk.content.length <= maxCharacters) {
830
- return [chunk]
831
- }
832
-
833
- // Line-based splitting can only exceed the limit for a single long line.
834
- // Each piece retains that line number and the original AST metadata.
835
- const pieces = [] as Array<CodeChunk>
836
- for (
837
- let offset = 0;
838
- offset < chunk.content.length;
839
- offset += maxCharacters
840
- ) {
841
- pieces.push({
842
- ...chunk,
843
- content: chunk.content.slice(offset, offset + maxCharacters),
844
- })
845
- }
846
- return pieces
847
- })
848
-
849
- /**
850
- * @since 1.0.0
851
- * @category Constructors
852
- */
853
- export const chunkFileContent = (
854
- path: string,
855
- content: string,
856
- options: {
857
- readonly chunkSize: number
858
- readonly chunkOverlap: number
859
- readonly chunkMaxCharacters?: number | undefined
860
- },
861
- ): ReadonlyArray<CodeChunk> => {
862
- if (content.trim().length === 0 || isProbablyMinified(content)) {
863
- return []
864
- }
865
-
866
- const normalizedPath = normalizePath(path)
867
- const normalizedContent = normalizeText(content)
868
- const lines = normalizedContent.split("\n")
869
- if (lines.at(-1) === "") {
870
- lines.pop()
871
- }
872
- if (lines.length === 0) {
873
- return []
874
- }
875
-
876
- const settings = resolveChunkSettings(options)
877
- const astRanges = collectAstRanges(normalizedPath, normalizedContent, lines)
878
- if (astRanges.length > 0) {
879
- const astChunks = chunksFromRanges(
880
- normalizedPath,
881
- lines,
882
- astRanges,
883
- settings,
884
- )
885
- if (astChunks.length > 0) {
886
- return splitOversizedChunks(astChunks, settings.chunkMaxCharacters)
887
- }
888
- }
889
-
890
- return splitOversizedChunks(
891
- chunkWithLineWindows(normalizedPath, lines, settings),
892
- settings.chunkMaxCharacters,
893
- )
894
- }
895
-
896
- /**
897
- * @since 1.0.0
898
- * @category Layers
899
- */
900
- export const layer: Layer.Layer<
901
- CodeChunker,
902
- never,
903
- ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem | Path.Path
904
- > = Layer.effect(
905
- CodeChunker,
906
- Effect.gen(function* () {
907
- const spawner = yield* ChildProcessSpawner.ChildProcessSpawner
908
- const fs = yield* FileSystem.FileSystem
909
- const pathService = yield* Path.Path
910
-
911
- const listFiles: CodeChunker["Service"]["listFiles"] = Effect.fn(
912
- "CodeChunker.listFiles",
913
- )(function* (options): Effect.fn.Return<ReadonlyArray<string>> {
914
- const root = pathService.resolve(options.root)
915
- const maxFileSize = options.maxFileSize ?? "1M"
916
-
917
- return yield* pipe(
918
- spawner.streamLines(
919
- ChildProcess.make(
920
- "rg",
921
- [
922
- "--files",
923
- "--hidden",
924
- "--max-filesize",
925
- maxFileSize,
926
- "--glob",
927
- "!.git",
928
- ],
929
- {
930
- cwd: root,
931
- stdin: "ignore",
932
- },
933
- ),
934
- ),
935
- Stream.runCollect,
936
- Effect.map(Array.fromIterable),
937
- Effect.map((entries) =>
938
- entries
939
- .map((entry) => normalizePath(entry.trim()))
940
- .filter((entry) => entry.length > 0 && isMeaningfulFile(entry))
941
- .toSorted((left, right) => left.localeCompare(right)),
942
- ),
943
- Effect.orDie,
944
- )
945
- })
946
-
947
- const chunkFile: CodeChunker["Service"]["chunkFile"] = Effect.fn(
948
- "CodeChunker.chunkFile",
949
- )(function* (options): Effect.fn.Return<ReadonlyArray<CodeChunk>> {
950
- const root = pathService.resolve(options.root)
951
- const absolutePath = pathService.resolve(root, options.path)
952
- const path = normalizePath(pathService.relative(root, absolutePath))
953
-
954
- if (
955
- path.length === 0 ||
956
- path === ".." ||
957
- path.startsWith("../") ||
958
- !isMeaningfulFile(path)
959
- ) {
960
- return []
961
- }
962
-
963
- return yield* pipe(
964
- fs.readFileString(absolutePath),
965
- Effect.map((content) => chunkFileContent(path, content, options)),
966
- Effect.orElseSucceed(() => []),
967
- )
968
- })
969
-
970
- const chunkFiles: CodeChunker["Service"]["chunkFiles"] = (options) =>
971
- Stream.fromArray(options.paths).pipe(
972
- Stream.flatMap(
973
- (path) =>
974
- pipe(
975
- chunkFile({
976
- root: options.root,
977
- path,
978
- chunkSize: options.chunkSize,
979
- chunkOverlap: options.chunkOverlap,
980
- ...(options.chunkMaxCharacters === undefined
981
- ? {}
982
- : { chunkMaxCharacters: options.chunkMaxCharacters }),
983
- }),
984
- Stream.fromArrayEffect,
985
- ),
986
- { concurrency: 5 },
987
- ),
988
- )
989
-
990
- const chunkCodebase: CodeChunker["Service"]["chunkCodebase"] =
991
- Effect.fnUntraced(function* (options) {
992
- const root = pathService.resolve(options.root)
993
- const files = yield* listFiles({
994
- root,
995
- ...(options.maxFileSize === undefined
996
- ? {}
997
- : { maxFileSize: options.maxFileSize }),
998
- })
999
-
1000
- return chunkFiles({
1001
- root,
1002
- paths: files,
1003
- chunkSize: options.chunkSize,
1004
- chunkOverlap: options.chunkOverlap,
1005
- ...(options.chunkMaxCharacters === undefined
1006
- ? {}
1007
- : { chunkMaxCharacters: options.chunkMaxCharacters }),
1008
- })
1009
- }, Stream.unwrap)
1010
-
1011
- return CodeChunker.of({
1012
- listFiles,
1013
- chunkFile,
1014
- chunkFiles,
1015
- chunkCodebase,
1016
- })
1017
- }),
1018
- )