clanka 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ChunkRepo.d.ts +0 -1
- package/dist/ChunkRepo.d.ts.map +1 -1
- package/dist/ChunkRepo.js +0 -10
- package/dist/ChunkRepo.js.map +1 -1
- package/dist/CodeChunker.d.ts +8 -1
- package/dist/CodeChunker.d.ts.map +1 -1
- package/dist/CodeChunker.js +331 -55
- package/dist/CodeChunker.js.map +1 -1
- package/dist/CodeChunker.test.js +231 -28
- package/dist/CodeChunker.test.js.map +1 -1
- package/dist/SemanticSearch.d.ts +2 -3
- package/dist/SemanticSearch.d.ts.map +1 -1
- package/dist/SemanticSearch.js +36 -15
- package/dist/SemanticSearch.js.map +1 -1
- package/package.json +13 -10
- package/src/ChunkRepo.ts +1 -12
- package/src/CodeChunker.test.ts +253 -32
- package/src/CodeChunker.ts +499 -70
- package/src/SemanticSearch.ts +45 -17
- package/src/fixtures/fiber.txt +255 -0
package/dist/CodeChunker.test.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { describe, expect, it } from "vitest";
|
|
1
|
+
import { assert, describe, expect, it } from "@effect/vitest";
|
|
2
2
|
import { chunkFileContent, isMeaningfulFile, isProbablyMinified, } from "./CodeChunker.js";
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
3
4
|
describe("isMeaningfulFile", () => {
|
|
4
5
|
it("keeps source and documentation files", () => {
|
|
5
6
|
expect(isMeaningfulFile("src/index.ts")).toBe(true);
|
|
@@ -31,7 +32,214 @@ describe("isProbablyMinified", () => {
|
|
|
31
32
|
});
|
|
32
33
|
});
|
|
33
34
|
describe("chunkFileContent", () => {
|
|
34
|
-
it("
|
|
35
|
+
it("splits TypeScript into AST chunks with metadata", () => {
|
|
36
|
+
const content = [
|
|
37
|
+
"// alpha docs",
|
|
38
|
+
"export const alpha = 1",
|
|
39
|
+
"// beta docs",
|
|
40
|
+
"function beta() {",
|
|
41
|
+
" return alpha",
|
|
42
|
+
"}",
|
|
43
|
+
"class Example {",
|
|
44
|
+
" // gamma docs",
|
|
45
|
+
" gamma() {",
|
|
46
|
+
" return beta()",
|
|
47
|
+
" }",
|
|
48
|
+
"}",
|
|
49
|
+
].join("\n");
|
|
50
|
+
const chunks = chunkFileContent("src\\example.ts", content, {
|
|
51
|
+
chunkSize: 10,
|
|
52
|
+
chunkOverlap: 2,
|
|
53
|
+
});
|
|
54
|
+
expect(chunks).toHaveLength(4);
|
|
55
|
+
expect(chunks[0]).toMatchObject({
|
|
56
|
+
path: "src/example.ts",
|
|
57
|
+
startLine: 1,
|
|
58
|
+
endLine: 2,
|
|
59
|
+
name: "alpha",
|
|
60
|
+
type: "variable",
|
|
61
|
+
parent: undefined,
|
|
62
|
+
content: ["// alpha docs", "export const alpha = 1"].join("\n"),
|
|
63
|
+
});
|
|
64
|
+
expect(chunks[1]).toMatchObject({
|
|
65
|
+
startLine: 3,
|
|
66
|
+
endLine: 6,
|
|
67
|
+
name: "beta",
|
|
68
|
+
type: "function",
|
|
69
|
+
parent: undefined,
|
|
70
|
+
content: [
|
|
71
|
+
"// beta docs",
|
|
72
|
+
"function beta() {",
|
|
73
|
+
" return alpha",
|
|
74
|
+
"}",
|
|
75
|
+
].join("\n"),
|
|
76
|
+
});
|
|
77
|
+
expect(chunks[2]).toMatchObject({
|
|
78
|
+
startLine: 7,
|
|
79
|
+
endLine: 12,
|
|
80
|
+
name: "Example",
|
|
81
|
+
type: "class",
|
|
82
|
+
parent: undefined,
|
|
83
|
+
content: [
|
|
84
|
+
"class Example {",
|
|
85
|
+
" // gamma docs",
|
|
86
|
+
" gamma() {",
|
|
87
|
+
" return beta()",
|
|
88
|
+
" }",
|
|
89
|
+
"}",
|
|
90
|
+
].join("\n"),
|
|
91
|
+
});
|
|
92
|
+
expect(chunks[3]).toMatchObject({
|
|
93
|
+
startLine: 8,
|
|
94
|
+
endLine: 11,
|
|
95
|
+
name: "gamma",
|
|
96
|
+
type: "method",
|
|
97
|
+
parent: "class Example",
|
|
98
|
+
content: [
|
|
99
|
+
" // gamma docs",
|
|
100
|
+
" gamma() {",
|
|
101
|
+
" return beta()",
|
|
102
|
+
" }",
|
|
103
|
+
].join("\n"),
|
|
104
|
+
});
|
|
105
|
+
expect(chunkFileContent("src/example.ts", content, {
|
|
106
|
+
chunkSize: 10,
|
|
107
|
+
chunkOverlap: 2,
|
|
108
|
+
})[0]).toEqual(chunks[0]);
|
|
109
|
+
});
|
|
110
|
+
it("splits large AST ranges using chunk settings", () => {
|
|
111
|
+
const content = [
|
|
112
|
+
"export function large() {",
|
|
113
|
+
...Array.from({ length: 60 }, (_, index) => " const line" + String(index + 1) + " = " + String(index + 1)),
|
|
114
|
+
"}",
|
|
115
|
+
].join("\n");
|
|
116
|
+
const chunks = chunkFileContent("src/large.ts", content, {
|
|
117
|
+
chunkSize: 20,
|
|
118
|
+
chunkOverlap: 5,
|
|
119
|
+
});
|
|
120
|
+
expect(chunks).toHaveLength(4);
|
|
121
|
+
expect(chunks[0]).toMatchObject({
|
|
122
|
+
startLine: 1,
|
|
123
|
+
endLine: 20,
|
|
124
|
+
name: "large",
|
|
125
|
+
type: "function",
|
|
126
|
+
parent: undefined,
|
|
127
|
+
});
|
|
128
|
+
expect(chunks[1]).toMatchObject({
|
|
129
|
+
startLine: 16,
|
|
130
|
+
endLine: 35,
|
|
131
|
+
});
|
|
132
|
+
expect(chunks[2]).toMatchObject({
|
|
133
|
+
startLine: 31,
|
|
134
|
+
endLine: 50,
|
|
135
|
+
});
|
|
136
|
+
expect(chunks[3]).toMatchObject({
|
|
137
|
+
startLine: 46,
|
|
138
|
+
endLine: 62,
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
it("keeps only the first oversized class segment when methods are present", () => {
|
|
142
|
+
const content = [
|
|
143
|
+
"class Example {",
|
|
144
|
+
" first() {",
|
|
145
|
+
" const a = 1",
|
|
146
|
+
" const b = 2",
|
|
147
|
+
" return a + b",
|
|
148
|
+
" }",
|
|
149
|
+
"",
|
|
150
|
+
" second() {",
|
|
151
|
+
" const c = 3",
|
|
152
|
+
" const d = 4",
|
|
153
|
+
" return c + d",
|
|
154
|
+
" }",
|
|
155
|
+
"}",
|
|
156
|
+
].join("\n");
|
|
157
|
+
const chunks = chunkFileContent("src/example.ts", content, {
|
|
158
|
+
chunkSize: 6,
|
|
159
|
+
chunkOverlap: 1,
|
|
160
|
+
});
|
|
161
|
+
const classChunks = chunks.filter((chunk) => chunk.type === "class");
|
|
162
|
+
expect(classChunks).toHaveLength(1);
|
|
163
|
+
expect(classChunks[0]).toMatchObject({
|
|
164
|
+
startLine: 1,
|
|
165
|
+
endLine: 6,
|
|
166
|
+
name: "Example",
|
|
167
|
+
type: "class",
|
|
168
|
+
parent: undefined,
|
|
169
|
+
});
|
|
170
|
+
expect(chunks.filter((chunk) => chunk.type === "method")).toMatchObject([
|
|
171
|
+
{
|
|
172
|
+
startLine: 2,
|
|
173
|
+
endLine: 6,
|
|
174
|
+
name: "first",
|
|
175
|
+
parent: "class Example",
|
|
176
|
+
},
|
|
177
|
+
{
|
|
178
|
+
startLine: 8,
|
|
179
|
+
endLine: 12,
|
|
180
|
+
name: "second",
|
|
181
|
+
parent: "class Example",
|
|
182
|
+
},
|
|
183
|
+
]);
|
|
184
|
+
});
|
|
185
|
+
it("includes preceding comments in AST chunks", () => {
|
|
186
|
+
const content = [
|
|
187
|
+
"// alpha docs",
|
|
188
|
+
"export const alpha = 1",
|
|
189
|
+
"/**",
|
|
190
|
+
" * beta docs",
|
|
191
|
+
" */",
|
|
192
|
+
"function beta() {",
|
|
193
|
+
" return alpha",
|
|
194
|
+
"}",
|
|
195
|
+
"class Example {",
|
|
196
|
+
" // gamma docs",
|
|
197
|
+
" gamma() {",
|
|
198
|
+
" return beta()",
|
|
199
|
+
" }",
|
|
200
|
+
"}",
|
|
201
|
+
].join("\n");
|
|
202
|
+
const chunks = chunkFileContent("src/example.ts", content, {
|
|
203
|
+
chunkSize: 20,
|
|
204
|
+
chunkOverlap: 2,
|
|
205
|
+
});
|
|
206
|
+
expect(chunks).toHaveLength(4);
|
|
207
|
+
expect(chunks[0]).toMatchObject({
|
|
208
|
+
startLine: 1,
|
|
209
|
+
endLine: 2,
|
|
210
|
+
name: "alpha",
|
|
211
|
+
type: "variable",
|
|
212
|
+
content: ["// alpha docs", "export const alpha = 1"].join("\n"),
|
|
213
|
+
});
|
|
214
|
+
expect(chunks[1]).toMatchObject({
|
|
215
|
+
startLine: 3,
|
|
216
|
+
endLine: 8,
|
|
217
|
+
name: "beta",
|
|
218
|
+
type: "function",
|
|
219
|
+
content: [
|
|
220
|
+
"/**",
|
|
221
|
+
" * beta docs",
|
|
222
|
+
" */",
|
|
223
|
+
"function beta() {",
|
|
224
|
+
" return alpha",
|
|
225
|
+
"}",
|
|
226
|
+
].join("\n"),
|
|
227
|
+
});
|
|
228
|
+
expect(chunks[3]).toMatchObject({
|
|
229
|
+
startLine: 10,
|
|
230
|
+
endLine: 13,
|
|
231
|
+
name: "gamma",
|
|
232
|
+
type: "method",
|
|
233
|
+
parent: "class Example",
|
|
234
|
+
content: [
|
|
235
|
+
" // gamma docs",
|
|
236
|
+
" gamma() {",
|
|
237
|
+
" return beta()",
|
|
238
|
+
" }",
|
|
239
|
+
].join("\n"),
|
|
240
|
+
});
|
|
241
|
+
});
|
|
242
|
+
it("falls back to line windows for unsupported languages", () => {
|
|
35
243
|
const content = [
|
|
36
244
|
"line 1",
|
|
37
245
|
"line 2",
|
|
@@ -40,51 +248,36 @@ describe("chunkFileContent", () => {
|
|
|
40
248
|
"line 5",
|
|
41
249
|
"line 6",
|
|
42
250
|
].join("\n");
|
|
43
|
-
const chunks = chunkFileContent("
|
|
251
|
+
const chunks = chunkFileContent("docs\\notes.txt", content, {
|
|
44
252
|
chunkSize: 3,
|
|
45
253
|
chunkOverlap: 1,
|
|
46
254
|
});
|
|
47
255
|
expect(chunks).toHaveLength(3);
|
|
48
256
|
expect(chunks[0]).toMatchObject({
|
|
49
|
-
path: "
|
|
257
|
+
path: "docs/notes.txt",
|
|
50
258
|
startLine: 1,
|
|
51
259
|
endLine: 3,
|
|
260
|
+
name: undefined,
|
|
261
|
+
type: undefined,
|
|
262
|
+
parent: undefined,
|
|
52
263
|
content: ["line 1", "line 2", "line 3"].join("\n"),
|
|
53
264
|
});
|
|
54
265
|
expect(chunks[1]).toMatchObject({
|
|
55
266
|
startLine: 3,
|
|
56
267
|
endLine: 5,
|
|
268
|
+
name: undefined,
|
|
269
|
+
type: undefined,
|
|
270
|
+
parent: undefined,
|
|
57
271
|
content: ["line 3", "line 4", "line 5"].join("\n"),
|
|
58
272
|
});
|
|
59
273
|
expect(chunks[2]).toMatchObject({
|
|
60
274
|
startLine: 5,
|
|
61
275
|
endLine: 6,
|
|
276
|
+
name: undefined,
|
|
277
|
+
type: undefined,
|
|
278
|
+
parent: undefined,
|
|
62
279
|
content: ["line 5", "line 6"].join("\n"),
|
|
63
280
|
});
|
|
64
|
-
expect(chunks[0]?.contentHash).toMatch(/^[a-f0-9]{64}$/);
|
|
65
|
-
expect(chunks[0]?.contentHash).toBe(chunkFileContent("src/example.ts", content, {
|
|
66
|
-
chunkSize: 3,
|
|
67
|
-
chunkOverlap: 1,
|
|
68
|
-
})[0]?.contentHash);
|
|
69
|
-
});
|
|
70
|
-
it("skips non-meaningful chunk starts", () => {
|
|
71
|
-
const content = [
|
|
72
|
-
"",
|
|
73
|
-
" ",
|
|
74
|
-
"---",
|
|
75
|
-
"const alpha = 1",
|
|
76
|
-
"const beta = 2",
|
|
77
|
-
].join("\n");
|
|
78
|
-
const chunks = chunkFileContent("src/example.ts", content, {
|
|
79
|
-
chunkSize: 3,
|
|
80
|
-
chunkOverlap: 1,
|
|
81
|
-
});
|
|
82
|
-
expect(chunks).toHaveLength(1);
|
|
83
|
-
expect(chunks[0]).toMatchObject({
|
|
84
|
-
startLine: 4,
|
|
85
|
-
endLine: 5,
|
|
86
|
-
content: ["const alpha = 1", "const beta = 2"].join("\n"),
|
|
87
|
-
});
|
|
88
281
|
});
|
|
89
282
|
it("drops minified-like content", () => {
|
|
90
283
|
const chunks = chunkFileContent("src/bundle.js", "x".repeat(3000), {
|
|
@@ -93,5 +286,15 @@ describe("chunkFileContent", () => {
|
|
|
93
286
|
});
|
|
94
287
|
expect(chunks).toEqual([]);
|
|
95
288
|
});
|
|
289
|
+
it("seperates class methods into their own chunks", () => {
|
|
290
|
+
const fixture = readFileSync("/Volumes/Code/effect/effect-smol/packages/effect/src/internal/effect.ts", "utf-8");
|
|
291
|
+
const chunks = chunkFileContent("src/fiber.ts", fixture, {
|
|
292
|
+
chunkSize: 30,
|
|
293
|
+
chunkOverlap: 0,
|
|
294
|
+
});
|
|
295
|
+
const runLoopChunk = chunks.find((chunk) => chunk.name === "runLoop");
|
|
296
|
+
assert(runLoopChunk, "Expected to find a chunk for the runLoop method");
|
|
297
|
+
assert.strictEqual(runLoopChunk.type, "method");
|
|
298
|
+
});
|
|
96
299
|
});
|
|
97
300
|
//# sourceMappingURL=CodeChunker.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CodeChunker.test.js","sourceRoot":"","sources":["../src/CodeChunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"CodeChunker.test.js","sourceRoot":"","sources":["../src/CodeChunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,gBAAgB,CAAA;AAC7D,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACnD,MAAM,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChD,MAAM,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACvD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACtD,MAAM,CAAC,gBAAgB,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC5D,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC3D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACjE,MAAM,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACrD,MAAM,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC7D,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IAClC,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACzD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,MAAM,GAAG;YACb,oBAAoB;YACpB,oBAAoB;YACpB,oBAAoB;SACrB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACZ,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAChD,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,OAAO,GAAG;YACd,eAAe;YACf,wBAAwB;YACxB,cAAc;YACd,mBAAmB;YACnB,gBAAgB;YAChB,GAAG;YACH,iBAAiB;YACjB,iBAAiB;YACjB,aAAa;YACb,mBAAmB;YACnB,KAAK;YACL,GAAG;SACJ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEZ,MAAM,MAAM,GAAG,gBAAgB,CAAC,iBAAiB,EAAE,OAAO,EAAE;YAC1D,SAAS,EAAE,EAAE;YACb,YAAY,EAAE,CAAC;SAChB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,IAAI,EAAE,gBAAgB;YACtB,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,UAAU;YAChB,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;SAChE,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,UAAU;YAChB,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE;gBACP,cAAc;gBACd,mBAAmB;gBACnB,gBAAgB;gBAChB,GAAG;aACJ,CAAC,IAAI,CAAC,IAAI,CAAC;SACb,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,EAAE;YACX,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE;gBACP,iBAAiB;gBACjB,iBAAiB;gBACjB,aAAa;gBACb,mBAAmB;gBACnB,KAAK;gBACL,GAAG;aACJ,CAAC,IAAI,CAAC,IAAI,CAAC;SACb,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,EAAE;YACX,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,eAAe;YACvB,OAAO,EAAE;gBACP,iBAAiB;gBACjB,aAAa;gBACb,mBAAmB;gBACnB,KAAK;aACN,CAAC,IAAI,CAAC,IAAI,CAAC;SACb,CAAC,CAAA;QAEF,MAAM,CACJ,gBAAgB,CAAC,gBAAgB,EAAE,OAAO,EAAE;YAC1C,SAAS,EAAE,EAAE;YACb,YAAY,EAAE,CAAC;SAChB,CAAC,CAAC,CAAC,CAAC,CACN,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IACtB,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,OAAO,GAAG;YACd,2BAA2B;YAC3B,GAAG,KAAK,CAAC,IAAI,CACX,EAAE,MAAM,EAAE,EAAE,EAAE,EACd,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CACX,cAAc,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CACjE;YACD,GAAG;SACJ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEZ,MAAM,MAAM,GAAG,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE;YACvD,SAAS,EAAE,EAAE;YACb,YAAY,EAAE,CAAC;SAChB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,EAAE;YACX,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,UAAU;YAChB,MAAM,EAAE,SAAS;SAClB,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;SACZ,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;SACZ,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;SACZ,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,uEAAuE,EAAE,GAAG,EAAE;QAC/E,MAAM,OAAO,GAAG;YACd,iBAAiB;YACjB,aAAa;YACb,iBAAiB;YACjB,iBAAiB;YACjB,kBAAkB;YAClB,KAAK;YACL,EAAE;YACF,cAAc;YACd,iBAAiB;YACjB,iBAAiB;YACjB,kBAAkB;YAClB,KAAK;YACL,GAAG;SACJ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEZ,MAAM,MAAM,GAAG,gBAAgB,CAAC,gBAAgB,EAAE,OAAO,EAAE;YACzD,SAAS,EAAE,CAAC;YACZ,YAAY,EAAE,CAAC;SAChB,CAAC,CAAA;QAEF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC,CAAA;QACpE,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACnC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YACnC,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,SAAS;SAClB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC;YACtE;gBACE,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,eAAe;aACxB;YACD;gBACE,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,eAAe;aACxB;SACF,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,OAAO,GAAG;YACd,eAAe;YACf,wBAAwB;YACxB,KAAK;YACL,cAAc;YACd,KAAK;YACL,mBAAmB;YACnB,gBAAgB;YAChB,GAAG;YACH,iBAAiB;YACjB,iBAAiB;YACjB,aAAa;YACb,mBAAmB;YACnB,KAAK;YACL,GAAG;SACJ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEZ,MAAM,MAAM,GAAG,gBAAgB,CAAC,gBAAgB,EAAE,OAAO,EAAE;YACzD,SAAS,EAAE,EAAE;YACb,YAAY,EAAE,CAAC;SAChB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;SAChE,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE;gBACP,KAAK;gBACL,cAAc;gBACd,KAAK;gBACL,mBAAmB;gBACnB,gBAAgB;gBAChB,GAAG;aACJ,CAAC,IAAI,CAAC,IAAI,CAAC;SACb,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,eAAe;YACvB,OAAO,EAAE;gBACP,iBAAiB;gBACjB,aAAa;gBACb,mBAAmB;gBACnB,KAAK;aACN,CAAC,IAAI,CAAC,IAAI,CAAC;SACb,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,OAAO,GAAG;YACd,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,QAAQ;SACT,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEZ,MAAM,MAAM,GAAG,gBAAgB,CAAC,iBAAiB,EAAE,OAAO,EAAE;YAC1D,SAAS,EAAE,CAAC;YACZ,YAAY,EAAE,CAAC;SAChB,CAAC,CAAA;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QAC9B,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,IAAI,EAAE,gBAAgB;YACtB,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;SACnD,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;SACnD,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;YAC9B,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;YACV,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS;YACf,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;SACzC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,MAAM,GAAG,gBAAgB,CAAC,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YACjE,SAAS,EAAE,EAAE;YACb,YAAY,EAAE,CAAC;SAChB,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAC5B,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,OAAO,GAAG,YAAY,CAC1B,yEAAyE,EACzE,OAAO,CACR,CAAA;QACD,MAAM,MAAM,GAAG,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE;YACvD,SAAS,EAAE,EAAE;YACb,YAAY,EAAE,CAAC;SAChB,CAAC,CAAA;QACF,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAA;QACrE,MAAM,CAAC,YAAY,EAAE,iDAAiD,CAAC,CAAA;QACvE,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IACjD,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
|
package/dist/SemanticSearch.d.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @since 1.0.0
|
|
3
|
-
*/
|
|
4
1
|
import * as Effect from "effect/Effect";
|
|
2
|
+
import * as CodeChunker from "./CodeChunker.ts";
|
|
5
3
|
import * as Layer from "effect/Layer";
|
|
6
4
|
import * as EmbeddingModel from "effect/unstable/ai/EmbeddingModel";
|
|
7
5
|
import * as Path from "effect/Path";
|
|
@@ -30,6 +28,7 @@ export declare const makeEmbeddingResolver: (resolver: EmbeddingModel.Service["r
|
|
|
30
28
|
readonly embeddingBatchSize?: number | undefined;
|
|
31
29
|
readonly embeddingRequestDelay?: Duration.Input | undefined;
|
|
32
30
|
}) => EmbeddingModel.Service["resolver"];
|
|
31
|
+
export declare const chunkEmbeddingInput: (chunk: CodeChunker.CodeChunk) => string;
|
|
33
32
|
/**
|
|
34
33
|
* @since 1.0.0
|
|
35
34
|
* @category Layers
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SemanticSearch.d.ts","sourceRoot":"","sources":["../src/SemanticSearch.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"SemanticSearch.d.ts","sourceRoot":"","sources":["../src/SemanticSearch.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,KAAK,WAAW,MAAM,kBAAkB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AAGrC,OAAO,KAAK,cAAc,MAAM,mCAAmC,CAAA;AAGnE,OAAO,KAAK,IAAI,MAAM,aAAa,CAAA;AACnC,OAAO,KAAK,UAAU,MAAM,mBAAmB,CAAA;AAE/C,OAAO,KAAK,QAAQ,MAAM,iBAAiB,CAAA;AAG3C,OAAO,KAAK,KAAK,QAAQ,MAAM,8BAA8B,CAAA;AAC7D,OAAO,KAAK,KAAK,cAAc,MAAM,wCAAwC,CAAA;AAC7E,OAAO,KAAK,KAAK,aAAa,MAAM,sBAAsB,CAAA;AAC1D,OAAO,KAAK,KAAK,mBAAmB,MAAM,6CAA6C,CAAA;AACvF,OAAO,KAAK,KAAK,UAAU,MAAM,mBAAmB,CAAA;;oBAUhC;QACd,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;QACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KACvB,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;qBACR,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;qBAC5B,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;;AAZjD;;;GAGG;AACH,qBAAa,cAAe,SAAQ,mBAUO;CAAG;AAS9C,eAAO,MAAM,qBAAqB,GAChC,UAAU,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC,EAC5C,SAAS;IACP,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAChD,QAAQ,CAAC,qBAAqB,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAA;CAC5D,KACA,cAAc,CAAC,OAAO,CAAC,UAAU,CAMjC,CAAA;AAEH,eAAO,MAAM,mBAAmB,GAAI,OAAO,WAAW,CAAC,SAAS,KAAG,MAwBlE,CAAA;AAKD;;;GAGG;AACH,eAAO,MAAM,KAAK,GAAI,SAAS;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACtC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAChD,QAAQ,CAAC,qBAAqB,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,SAAS,CAAA;IAC3D,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAC1C,KAAG,KAAK,CAAC,KAAK,CACb,cAAc,EACZ,QAAQ,CAAC,QAAQ,GACjB,cAAc,CAAC,cAAc,GAC7B,aAAa,CAAC,aAAa,EAC3B,cAAc,CAAC,cAAc,GAC7B,IAAI,CAAC,IAAI,GACT,mBAAmB,CAAC,mBAAmB,GACvC,UAAU,CAAC,UAAU,GACrB,cAAc,CAAC,UAAU,CAqL1B,CAAA;AAEH;;;GAGG;AACH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,KAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAQ9D,CAAA;AAEH;;;GAGG;AACH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,KAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAQ9D,CAAA"}
|
package/dist/SemanticSearch.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @since 1.0.0
|
|
3
3
|
*/
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
4
5
|
import * as Effect from "effect/Effect";
|
|
5
6
|
import * as ChunkRepo from "./ChunkRepo.js";
|
|
6
7
|
import * as CodeChunker from "./CodeChunker.js";
|
|
@@ -17,18 +18,41 @@ import * as Duration from "effect/Duration";
|
|
|
17
18
|
import * as FiberHandle from "effect/FiberHandle";
|
|
18
19
|
import { SqliteLayer } from "./Sqlite.js";
|
|
19
20
|
import * as Console from "effect/Console";
|
|
20
|
-
const normalizePath = (path) => path.replace(/\\/g, "/");
|
|
21
|
-
const chunkConfig = {
|
|
22
|
-
chunkSize: 20,
|
|
23
|
-
chunkOverlap: 5,
|
|
24
|
-
};
|
|
25
21
|
/**
|
|
26
22
|
* @since 1.0.0
|
|
27
23
|
* @category Services
|
|
28
24
|
*/
|
|
29
25
|
export class SemanticSearch extends ServiceMap.Service()("clanka/SemanticSearch/SemanticSearch") {
|
|
30
26
|
}
|
|
27
|
+
const normalizePath = (path) => path.replace(/\\/g, "/");
|
|
28
|
+
const chunkConfig = {
|
|
29
|
+
chunkSize: 30,
|
|
30
|
+
chunkOverlap: 0,
|
|
31
|
+
};
|
|
31
32
|
export const makeEmbeddingResolver = (resolver, options) => resolver.pipe(RequestResolver.setDelay(options.embeddingRequestDelay ?? Duration.millis(50)), RequestResolver.batchN(options.embeddingBatchSize ?? 500));
|
|
33
|
+
export const chunkEmbeddingInput = (chunk) => {
|
|
34
|
+
const headerLines = ["---", "file: " + chunk.path];
|
|
35
|
+
if (chunk.name !== undefined) {
|
|
36
|
+
headerLines.push("name: " + chunk.name);
|
|
37
|
+
}
|
|
38
|
+
if (chunk.type !== undefined) {
|
|
39
|
+
headerLines.push("type: " + chunk.type);
|
|
40
|
+
}
|
|
41
|
+
if (chunk.parent !== undefined) {
|
|
42
|
+
headerLines.push("parent: " + chunk.parent);
|
|
43
|
+
}
|
|
44
|
+
headerLines.push("---");
|
|
45
|
+
const contentLines = chunk.content.split("\n");
|
|
46
|
+
let contentWithLines = "";
|
|
47
|
+
for (let i = 0; i < contentLines.length; i++) {
|
|
48
|
+
if (i > 0) {
|
|
49
|
+
contentWithLines += "\n";
|
|
50
|
+
}
|
|
51
|
+
contentWithLines += `${chunk.startLine + i}: ${contentLines[i]}`;
|
|
52
|
+
}
|
|
53
|
+
return headerLines.join("\n") + "\n\n" + contentWithLines;
|
|
54
|
+
};
|
|
55
|
+
const hashChunkInput = (input) => createHash("sha256").update(input).digest("hex");
|
|
32
56
|
/**
|
|
33
57
|
* @since 1.0.0
|
|
34
58
|
* @category Layers
|
|
@@ -54,30 +78,27 @@ export const layer = (options) => Layer.effect(SemanticSearch, Effect.gen(functi
|
|
|
54
78
|
return Option.some(relativePath);
|
|
55
79
|
};
|
|
56
80
|
const processChunk = Effect.fnUntraced(function* (options) {
|
|
81
|
+
const input = chunkEmbeddingInput(options.chunk);
|
|
82
|
+
const hash = hashChunkInput(input);
|
|
57
83
|
if (options.checkExisting) {
|
|
58
84
|
const id = yield* repo.exists({
|
|
59
85
|
path: options.chunk.path,
|
|
60
86
|
startLine: options.chunk.startLine,
|
|
61
|
-
hash
|
|
87
|
+
hash,
|
|
62
88
|
});
|
|
63
89
|
if (Option.isSome(id)) {
|
|
64
90
|
yield* repo.setSyncId(id.value, options.syncId);
|
|
65
91
|
return;
|
|
66
92
|
}
|
|
67
93
|
}
|
|
68
|
-
const result = yield* Effect.request(new EmbeddingModel.EmbeddingRequest({
|
|
69
|
-
input: `File: ${options.chunk.path}
|
|
70
|
-
Lines: ${options.chunk.startLine}-${options.chunk.endLine}
|
|
71
|
-
|
|
72
|
-
${options.chunk.content}`,
|
|
73
|
-
}), resolver);
|
|
94
|
+
const result = yield* Effect.request(new EmbeddingModel.EmbeddingRequest({ input }), resolver);
|
|
74
95
|
const vector = new Float32Array(result.vector);
|
|
75
96
|
yield* repo.insert(ChunkRepo.Chunk.insert.makeUnsafe({
|
|
76
97
|
path: options.chunk.path,
|
|
77
98
|
startLine: options.chunk.startLine,
|
|
78
99
|
endLine: options.chunk.endLine,
|
|
79
|
-
hash
|
|
80
|
-
content:
|
|
100
|
+
hash,
|
|
101
|
+
content: input,
|
|
81
102
|
vector,
|
|
82
103
|
syncId: options.syncId,
|
|
83
104
|
}));
|
|
@@ -115,7 +136,7 @@ ${options.chunk.content}`,
|
|
|
115
136
|
vector: new Float32Array(vector),
|
|
116
137
|
limit: options.limit,
|
|
117
138
|
});
|
|
118
|
-
return results.map((r) => r.
|
|
139
|
+
return results.map((r) => r.content).join("\n\n");
|
|
119
140
|
}, Effect.orDie),
|
|
120
141
|
updateFile: Effect.fn("SemanticSearch.updateFile")(function* (path) {
|
|
121
142
|
yield* Fiber.join(initialIndex);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SemanticSearch.js","sourceRoot":"","sources":["../src/SemanticSearch.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,SAAS,MAAM,gBAAgB,CAAA;AAC3C,OAAO,KAAK,WAAW,MAAM,kBAAkB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AACtC,OAAO,KAAK,cAAc,MAAM,mCAAmC,CAAA;AACnE,OAAO,KAAK,eAAe,MAAM,wBAAwB,CAAA;AACzD,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,IAAI,MAAM,aAAa,CAAA;AACnC,OAAO,KAAK,UAAU,MAAM,mBAAmB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,QAAQ,MAAM,iBAAiB,CAAA;AAC3C,OAAO,KAAK,WAAW,MAAM,oBAAoB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAMzC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AAEzC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;AAEhE,MAAM,WAAW,GAAG;IAClB,SAAS,EAAE,EAAE;IACb,YAAY,EAAE,CAAC;CACP,CAAA;AAEV
|
|
1
|
+
{"version":3,"file":"SemanticSearch.js","sourceRoot":"","sources":["../src/SemanticSearch.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,SAAS,MAAM,gBAAgB,CAAA;AAC3C,OAAO,KAAK,WAAW,MAAM,kBAAkB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AACtC,OAAO,KAAK,cAAc,MAAM,mCAAmC,CAAA;AACnE,OAAO,KAAK,eAAe,MAAM,wBAAwB,CAAA;AACzD,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,IAAI,MAAM,aAAa,CAAA;AACnC,OAAO,KAAK,UAAU,MAAM,mBAAmB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,QAAQ,MAAM,iBAAiB,CAAA;AAC3C,OAAO,KAAK,WAAW,MAAM,oBAAoB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAMzC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AAEzC;;;GAGG;AACH,MAAM,OAAO,cAAe,SAAQ,UAAU,CAAC,OAAO,EAUnD,CAAC,sCAAsC,CAAC;CAAG;AAE9C,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;AAEhE,MAAM,WAAW,GAAG;IAClB,SAAS,EAAE,EAAE;IACb,YAAY,EAAE,CAAC;CACP,CAAA;AAEV,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,QAA4C,EAC5C,OAGC,EACmC,EAAE,CACtC,QAAQ,CAAC,IAAI,CACX,eAAe,CAAC,QAAQ,CACtB,OAAO,CAAC,qBAAqB,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CACrD,EACD,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,IAAI,GAAG,CAAC,CAC1D,CAAA;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,KAA4B,EAAU,EAAE;IAC1E,MAAM,WAAW,GAAG,CAAC,KAAK,EAAE,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAA;IAElD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,WAAW,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAA;IACzC,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,WAAW,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAA;IACzC,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,WAAW,CAAC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAA;IAC7C,CAAC;IACD,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAEvB,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC9C,IAAI,gBAAgB,GAAG,EAAE,CAAA;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACV,gBAAgB,IAAI,IAAI,CAAA;QAC1B,CAAC;QACD,gBAAgB,IAAI,GAAG,KAAK,CAAC,SAAS,GAAG,CAAC,KAAK,YAAY,CAAC,CAAC,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,gBAAgB,CAAA;AAC3D,CAAC,CAAA;AAED,MAAM,cAAc,GAAG,CAAC,KAAa,EAAU,EAAE,CAC/C,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAElD;;;GAGG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,OAMrB,EAUC,EAAE,CACF,KAAK,CAAC,MAAM,CACV,cAAc,EACd,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,WAAW,CAAA;IAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,SAAS,CAAA;IACvC,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,cAAc,CAAC,cAAc,CAAA;IACvD,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAA;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,qBAAqB,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IACpE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAA;IAC/C,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAA;IAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAA;IAEtC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAyB,EAAE;QACjE,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACpD,MAAM,YAAY,GAAG,aAAa,CAChC,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,CACzC,CAAA;QACD,IACE,YAAY,CAAC,MAAM,KAAK,CAAC;YACzB,YAAY,KAAK,IAAI;YACrB,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,EAC9B,CAAC;YACD,OAAO,MAAM,CAAC,IAAI,EAAE,CAAA;QACtB,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAClC,CAAC,CAAA;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CACpC,QAAQ,CAAC,EAAE,OAIV;QACC,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAChD,MAAM,IAAI,GAAG,cAAc,CAAC,KAAK,CAAC,CAAA;QAElC,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YAC1B,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC5B,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI;gBACxB,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS;gBAClC,IAAI;aACL,CAAC,CAAA;YACF,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;gBACtB,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;gBAC/C,OAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAClC,IAAI,cAAc,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,CAAC,EAC9C,QAAQ,CACT,CAAA;QACD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QAC9C,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAChB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC;YAChC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI;YACxB,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS;YAClC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,OAAO;YAC9B,IAAI;YACJ,OAAO,EAAE,KAAK;YACd,MAAM;YACN,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CACH,CAAA;IACH,CAAC,EACD,MAAM,CAAC,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM;QACX,OAAO,EAAE,uCAAuC;KACjD,CAAC,EACF,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAClB,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;QAC1B,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE;KAC1D,CAAC,CACL,CAAA;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAChC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/D,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAA;QAEtD,KAAK,CAAC,CAAC,IAAI,CACT,OAAO,CAAC,aAAa,CAAC;YACpB,IAAI;YACJ,GAAG,WAAW;SACf,CAAC,EACF,MAAM,CAAC,GAAG,CACR,CAAC,KAAK,EAAE,EAAE,CACR,YAAY,CAAC;YACX,KAAK;YACL,MAAM;YACN,aAAa,EAAE,IAAI;SACpB,CAAC,EACJ,EAAE,WAAW,EAAE,CAChB,EACD,MAAM,CAAC,QAAQ,CAChB,CAAA;QAED,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAA;QAEnC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAA;IACxD,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EACvC,MAAM,CAAC,WAAW,CAAC,sBAAsB,CAAC,EAC1C,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAChD,CAAA;IAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE;QACnD,aAAa,EAAE,IAAI;KACpB,CAAC,CAAA;IAEF,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAA;IACpC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAClB,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EACjC,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,UAAU,CAClB,CAAA;IAED,OAAO,cAAc,CAAC,EAAE,CAAC;QACvB,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,uBAAuB,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAO;YAC3D,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/B,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAA;YAC1C,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YACzD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;gBACjC,MAAM,EAAE,IAAI,YAAY,CAAC,MAAM,CAAC;gBAChC,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAA;YACF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACnD,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC,2BAA2B,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI;YAChE,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/B,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YAC5C,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC/B,OAAM;YACR,CAAC;YAED,KAAK,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;YAE3C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;gBACtC,IAAI;gBACJ,IAAI,EAAE,WAAW,CAAC,KAAK;gBACvB,GAAG,WAAW;aACf,CAAC,CAAA;YACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAM;YACR,CAAC;YAED,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;YAE/D,KAAK,CAAC,CAAC,IAAI,CACT,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EACxB,MAAM,CAAC,GAAG,CACR,CAAC,KAAK,EAAE,EAAE,CACR,YAAY,CAAC;gBACX,KAAK;gBACL,MAAM;gBACN,aAAa,EAAE,KAAK;aACrB,CAAC,EACJ,EAAE,WAAW,EAAE,CAChB,EACD,MAAM,CAAC,QAAQ,CAChB,CAAA;QACH,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC,2BAA2B,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI;YAChE,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/B,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YAC5C,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC/B,OAAM;YACR,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QAC7C,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;KACjB,CAAC,CAAA;AACJ,CAAC,CAAC,CACH,CAAC,IAAI,CACJ,KAAK,CAAC,OAAO,CAAC;IACZ,WAAW,CAAC,KAAK;IACjB,SAAS,CAAC,KAAK,CAAC,IAAI,CAClB,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,IAAI,uBAAuB,CAAC,CAAC,CACxE;CACF,CAAC,CACH,CAAA;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAuB,EAAE,CACnE,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,IAAI,CACvC,MAAM,CAAC,OAAO,CACZ,MAAM,CAAC,KAAK,CAAC;IACX,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI;IACzB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;CAC9C,CAAC,CACH,CACF,CAAA;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAuB,EAAE,CACnE,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,IAAI,CACvC,MAAM,CAAC,OAAO,CACZ,MAAM,CAAC,KAAK,CAAC;IACX,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI;IACzB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;CAC9C,CAAC,CACH,CACF,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "clanka",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.2.
|
|
4
|
+
"version": "0.2.7",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
@@ -22,11 +22,14 @@
|
|
|
22
22
|
"url": "https://github.com/Effectful-Tech/clanka.git"
|
|
23
23
|
},
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@effect/sql-sqlite-node": "4.0.0-beta.
|
|
25
|
+
"@effect/sql-sqlite-node": "4.0.0-beta.35",
|
|
26
26
|
"@modelcontextprotocol/sdk": "^1.27.1",
|
|
27
27
|
"@vscode/ripgrep": "^1.17.1",
|
|
28
28
|
"chalk": "^5.6.2",
|
|
29
29
|
"glob": "^13.0.6",
|
|
30
|
+
"tree-sitter": "^0.21.1",
|
|
31
|
+
"tree-sitter-javascript": "^0.23.1",
|
|
32
|
+
"tree-sitter-typescript": "^0.23.2",
|
|
30
33
|
"turndown": "^7.2.2"
|
|
31
34
|
},
|
|
32
35
|
"optionalDependencies": {
|
|
@@ -39,22 +42,22 @@
|
|
|
39
42
|
"@sqliteai/sqlite-vector-win32-x86_64": "0.9.93"
|
|
40
43
|
},
|
|
41
44
|
"peerDependencies": {
|
|
42
|
-
"@effect/ai-openai": "4.0.0-beta.
|
|
43
|
-
"@effect/ai-openai-compat": "4.0.0-beta.
|
|
44
|
-
"effect": "4.0.0-beta.
|
|
45
|
+
"@effect/ai-openai": "4.0.0-beta.35",
|
|
46
|
+
"@effect/ai-openai-compat": "4.0.0-beta.35",
|
|
47
|
+
"effect": "4.0.0-beta.35"
|
|
45
48
|
},
|
|
46
49
|
"devDependencies": {
|
|
47
50
|
"@changesets/changelog-github": "^0.6.0",
|
|
48
51
|
"@changesets/cli": "^2.29.8",
|
|
49
|
-
"@effect/ai-openai": "4.0.0-beta.
|
|
50
|
-
"@effect/ai-openai-compat": "4.0.0-beta.
|
|
52
|
+
"@effect/ai-openai": "4.0.0-beta.35",
|
|
53
|
+
"@effect/ai-openai-compat": "4.0.0-beta.35",
|
|
51
54
|
"@effect/language-service": "^0.80.0",
|
|
52
|
-
"@effect/platform-node": "4.0.0-beta.
|
|
53
|
-
"@effect/vitest": "4.0.0-beta.
|
|
55
|
+
"@effect/platform-node": "4.0.0-beta.35",
|
|
56
|
+
"@effect/vitest": "4.0.0-beta.35",
|
|
54
57
|
"@types/node": "^25.5.0",
|
|
55
58
|
"@types/turndown": "^5.0.6",
|
|
56
59
|
"@typescript/native-preview": "7.0.0-dev.20260315.1",
|
|
57
|
-
"effect": "4.0.0-beta.
|
|
60
|
+
"effect": "4.0.0-beta.35",
|
|
58
61
|
"husky": "^9.1.7",
|
|
59
62
|
"lint-staged": "^16.4.0",
|
|
60
63
|
"oxlint": "^1.55.0",
|
package/src/ChunkRepo.ts
CHANGED
|
@@ -81,18 +81,7 @@ export class Chunk extends Model.Class<Chunk>("Chunk")({
|
|
|
81
81
|
hash: Schema.String,
|
|
82
82
|
vector: Float32ArrayField,
|
|
83
83
|
syncId: SyncId,
|
|
84
|
-
}) {
|
|
85
|
-
format(): string {
|
|
86
|
-
const lines = this.content.split("\n")
|
|
87
|
-
let out = ""
|
|
88
|
-
for (let i = 0; i < lines.length; i++) {
|
|
89
|
-
out += `${this.startLine + i}: ${lines[i]}\n`
|
|
90
|
-
}
|
|
91
|
-
return `File: ${this.path}
|
|
92
|
-
|
|
93
|
-
${out}`
|
|
94
|
-
}
|
|
95
|
-
}
|
|
84
|
+
}) {}
|
|
96
85
|
|
|
97
86
|
/**
|
|
98
87
|
* @since 1.0.0
|