clanka 0.26.1 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -3
- package/dist/Acp.test.js +0 -1
- package/dist/Acp.test.js.map +1 -1
- package/dist/AcpImage.test.js +0 -1
- package/dist/AcpImage.test.js.map +1 -1
- package/dist/Agent.d.ts.map +1 -1
- package/dist/Agent.js +1 -2
- package/dist/Agent.js.map +1 -1
- package/dist/Agent.test.js +0 -1
- package/dist/Agent.test.js.map +1 -1
- package/dist/AgentCallerConfig.test.js +0 -1
- package/dist/AgentCallerConfig.test.js.map +1 -1
- package/dist/AgentExecutor.d.ts +2 -3
- package/dist/AgentExecutor.d.ts.map +1 -1
- package/dist/AgentExecutor.js +4 -13
- package/dist/AgentExecutor.js.map +1 -1
- package/dist/AgentImage.test.js +0 -1
- package/dist/AgentImage.test.js.map +1 -1
- package/dist/AgentSkills.test.js +0 -2
- package/dist/AgentSkills.test.js.map +1 -1
- package/dist/AgentTools.d.ts +2 -235
- package/dist/AgentTools.d.ts.map +1 -1
- package/dist/AgentTools.js +2 -32
- package/dist/AgentTools.js.map +1 -1
- package/dist/ClaudeAgent.test.js +0 -1
- package/dist/ClaudeAgent.test.js.map +1 -1
- package/dist/McpServers.d.ts +20 -1
- package/dist/McpServers.d.ts.map +1 -1
- package/dist/McpServers.js +12 -5
- package/dist/McpServers.js.map +1 -1
- package/dist/McpServers.test.js +115 -0
- package/dist/McpServers.test.js.map +1 -1
- package/dist/ScriptFormatting.d.ts +5 -0
- package/dist/ScriptFormatting.d.ts.map +1 -1
- package/dist/ScriptFormatting.js +255 -53
- package/dist/ScriptFormatting.js.map +1 -1
- package/dist/ScriptFormatting.test.js +16 -0
- package/dist/ScriptFormatting.test.js.map +1 -1
- package/dist/bin/{Latch-DOdbsGMQ.mjs → Latch-BE6_TYRL.mjs} +1 -1
- package/dist/bin/{McpClient-B-FFG6Pg.mjs → McpClient-DCt4v5W5.mjs} +2 -2
- package/dist/bin/{OutputFormatter-DhhP2i-7.mjs → OutputFormatter-DaFv_C_I.mjs} +2 -3
- package/dist/bin/{Schema-CFdSO6iH.mjs → Schema-B3MiG1aD.mjs} +204 -568
- package/dist/bin/cli.mjs +5098 -2472
- package/dist/bin/{upgrade-C4aOEWJH.mjs → upgrade-WqgIU_21.mjs} +3 -3
- package/dist/cli.js +2 -25
- package/dist/cli.js.map +1 -1
- package/dist/fixtures/McpFakeServer.d.ts +4 -0
- package/dist/fixtures/McpFakeServer.d.ts.map +1 -1
- package/dist/fixtures/McpFakeServer.js +13 -0
- package/dist/fixtures/McpFakeServer.js.map +1 -1
- package/dist/index.d.ts +0 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -8
- package/dist/index.js.map +1 -1
- package/package.json +1 -7
- package/src/Acp.test.ts +0 -1
- package/src/AcpImage.test.ts +0 -1
- package/src/Agent.test.ts +0 -1
- package/src/Agent.ts +0 -4
- package/src/AgentCallerConfig.test.ts +0 -1
- package/src/AgentExecutor.ts +4 -16
- package/src/AgentImage.test.ts +0 -1
- package/src/AgentSkills.test.ts +0 -2
- package/src/AgentTools.ts +4 -41
- package/src/ClaudeAgent.test.ts +0 -1
- package/src/McpServers.test.ts +198 -0
- package/src/McpServers.ts +44 -4
- package/src/ScriptFormatting.test.ts +20 -0
- package/src/ScriptFormatting.ts +234 -47
- package/src/cli.ts +1 -56
- package/src/fixtures/McpFakeServer.ts +17 -1
- package/src/index.ts +0 -10
- package/dist/ChunkRepo.d.ts +0 -173
- package/dist/ChunkRepo.d.ts.map +0 -1
- package/dist/ChunkRepo.js +0 -168
- package/dist/ChunkRepo.js.map +0 -1
- package/dist/ChunkRepo.test.d.ts +0 -2
- package/dist/ChunkRepo.test.d.ts.map +0 -1
- package/dist/ChunkRepo.test.js +0 -63
- package/dist/ChunkRepo.test.js.map +0 -1
- package/dist/CodeChunker.d.ts +0 -84
- package/dist/CodeChunker.d.ts.map +0 -1
- package/dist/CodeChunker.js +0 -658
- package/dist/CodeChunker.js.map +0 -1
- package/dist/CodeChunker.test.d.ts +0 -2
- package/dist/CodeChunker.test.d.ts.map +0 -1
- package/dist/CodeChunker.test.js +0 -434
- package/dist/CodeChunker.test.js.map +0 -1
- package/dist/SemanticSearch/Service.d.ts +0 -31
- package/dist/SemanticSearch/Service.d.ts.map +0 -1
- package/dist/SemanticSearch/Service.js +0 -29
- package/dist/SemanticSearch/Service.js.map +0 -1
- package/dist/SemanticSearch.d.ts +0 -34
- package/dist/SemanticSearch.d.ts.map +0 -1
- package/dist/SemanticSearch.js +0 -171
- package/dist/SemanticSearch.js.map +0 -1
- package/dist/SemanticSearch.test.d.ts +0 -2
- package/dist/SemanticSearch.test.d.ts.map +0 -1
- package/dist/SemanticSearch.test.js +0 -34
- package/dist/SemanticSearch.test.js.map +0 -1
- package/dist/Sqlite.d.ts +0 -15
- package/dist/Sqlite.d.ts.map +0 -1
- package/dist/Sqlite.js +0 -44
- package/dist/Sqlite.js.map +0 -1
- package/dist/bin/AiError-DBaKz35C.mjs +0 -2723
- package/dist/bin/SemanticSearch-C3a7LAQr.mjs +0 -2558
- package/src/ChunkRepo.test.ts +0 -89
- package/src/ChunkRepo.ts +0 -260
- package/src/CodeChunker.test.ts +0 -493
- package/src/CodeChunker.ts +0 -1018
- package/src/SemanticSearch/Service.ts +0 -50
- package/src/SemanticSearch.test.ts +0 -51
- package/src/SemanticSearch.ts +0 -282
- package/src/Sqlite.ts +0 -60
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @since 1.0.0
|
|
3
|
-
*/
|
|
4
|
-
import * as Effect from "effect/Effect"
|
|
5
|
-
import * as Option from "effect/Option"
|
|
6
|
-
import * as Context from "effect/Context"
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* @since 1.0.0
|
|
10
|
-
* @category Services
|
|
11
|
-
*/
|
|
12
|
-
export class SemanticSearch extends Context.Service<
|
|
13
|
-
SemanticSearch,
|
|
14
|
-
{
|
|
15
|
-
search(options: {
|
|
16
|
-
readonly query: string
|
|
17
|
-
readonly limit: number
|
|
18
|
-
}): Effect.Effect<string>
|
|
19
|
-
updateFile(path: string): Effect.Effect<void>
|
|
20
|
-
removeFile(path: string): Effect.Effect<void>
|
|
21
|
-
}
|
|
22
|
-
>()("clanka/SemanticSearch/SemanticSearch") {}
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* @since 1.0.0
|
|
26
|
-
* @category Utils
|
|
27
|
-
*/
|
|
28
|
-
export const maybeUpdateFile = (path: string): Effect.Effect<void> =>
|
|
29
|
-
Effect.serviceOption(SemanticSearch).pipe(
|
|
30
|
-
Effect.flatMap(
|
|
31
|
-
Option.match({
|
|
32
|
-
onNone: () => Effect.void,
|
|
33
|
-
onSome: (service) => service.updateFile(path),
|
|
34
|
-
}),
|
|
35
|
-
),
|
|
36
|
-
)
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* @since 1.0.0
|
|
40
|
-
* @category Utils
|
|
41
|
-
*/
|
|
42
|
-
export const maybeRemoveFile = (path: string): Effect.Effect<void> =>
|
|
43
|
-
Effect.serviceOption(SemanticSearch).pipe(
|
|
44
|
-
Effect.flatMap(
|
|
45
|
-
Option.match({
|
|
46
|
-
onNone: () => Effect.void,
|
|
47
|
-
onSome: (service) => service.removeFile(path),
|
|
48
|
-
}),
|
|
49
|
-
),
|
|
50
|
-
)
|
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
import { assert, describe, it } from "@effect/vitest"
|
|
2
|
-
import * as Duration from "effect/Duration"
|
|
3
|
-
import * as Effect from "effect/Effect"
|
|
4
|
-
import * as Fiber from "effect/Fiber"
|
|
5
|
-
import * as RequestResolver from "effect/RequestResolver"
|
|
6
|
-
import { TestClock } from "effect/testing"
|
|
7
|
-
import type * as EmbeddingModel from "effect/ai/EmbeddingModel"
|
|
8
|
-
import { makeEmbeddingResolver } from "./SemanticSearch.ts"
|
|
9
|
-
|
|
10
|
-
const baseResolver = RequestResolver.make<EmbeddingModel.EmbeddingRequest>(
|
|
11
|
-
() => Effect.void,
|
|
12
|
-
)
|
|
13
|
-
|
|
14
|
-
describe("SemanticSearch.makeEmbeddingResolver", () => {
|
|
15
|
-
it.effect("uses embeddingRequestDelay instead of embeddingBatchSize", () =>
|
|
16
|
-
Effect.gen(function* () {
|
|
17
|
-
const resolver = makeEmbeddingResolver(baseResolver, {
|
|
18
|
-
embeddingBatchSize: 1,
|
|
19
|
-
embeddingRequestDelay: Duration.millis(200),
|
|
20
|
-
})
|
|
21
|
-
|
|
22
|
-
const fiber = yield* Effect.forkChild(resolver.delay)
|
|
23
|
-
|
|
24
|
-
yield* TestClock.adjust(Duration.millis(199))
|
|
25
|
-
assert.strictEqual(fiber.pollUnsafe(), undefined)
|
|
26
|
-
|
|
27
|
-
yield* TestClock.adjust(Duration.millis(1))
|
|
28
|
-
assert.notStrictEqual(fiber.pollUnsafe(), undefined)
|
|
29
|
-
|
|
30
|
-
yield* Fiber.join(fiber)
|
|
31
|
-
}),
|
|
32
|
-
)
|
|
33
|
-
|
|
34
|
-
it.effect("defaults delay to 50ms regardless of batch size", () =>
|
|
35
|
-
Effect.gen(function* () {
|
|
36
|
-
const resolver = makeEmbeddingResolver(baseResolver, {
|
|
37
|
-
embeddingBatchSize: 999,
|
|
38
|
-
})
|
|
39
|
-
|
|
40
|
-
const fiber = yield* Effect.forkChild(resolver.delay)
|
|
41
|
-
|
|
42
|
-
yield* TestClock.adjust(Duration.millis(49))
|
|
43
|
-
assert.strictEqual(fiber.pollUnsafe(), undefined)
|
|
44
|
-
|
|
45
|
-
yield* TestClock.adjust(Duration.millis(1))
|
|
46
|
-
assert.notStrictEqual(fiber.pollUnsafe(), undefined)
|
|
47
|
-
|
|
48
|
-
yield* Fiber.join(fiber)
|
|
49
|
-
}),
|
|
50
|
-
)
|
|
51
|
-
})
|
package/src/SemanticSearch.ts
DELETED
|
@@ -1,282 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @since 1.0.0
|
|
3
|
-
*/
|
|
4
|
-
import { createHash } from "node:crypto"
|
|
5
|
-
import * as Effect from "effect/Effect"
|
|
6
|
-
import * as ChunkRepo from "./ChunkRepo.ts"
|
|
7
|
-
import * as CodeChunker from "./CodeChunker.ts"
|
|
8
|
-
import * as Layer from "effect/Layer"
|
|
9
|
-
import * as Stream from "effect/Stream"
|
|
10
|
-
import { pipe } from "effect/Function"
|
|
11
|
-
import * as EmbeddingModel from "effect/ai/EmbeddingModel"
|
|
12
|
-
import * as RequestResolver from "effect/RequestResolver"
|
|
13
|
-
import * as Option from "effect/Option"
|
|
14
|
-
import * as Path from "effect/Path"
|
|
15
|
-
import * as Fiber from "effect/Fiber"
|
|
16
|
-
import * as Duration from "effect/Duration"
|
|
17
|
-
import * as FiberHandle from "effect/FiberHandle"
|
|
18
|
-
import { SqliteLayer } from "./Sqlite.ts"
|
|
19
|
-
import type * as SqlError from "effect/sql/SqlError"
|
|
20
|
-
import type * as SqliteMigrator from "@effect/sql-sqlite-node/SqliteMigrator"
|
|
21
|
-
import type * as PlatformError from "effect/PlatformError"
|
|
22
|
-
import type * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"
|
|
23
|
-
import type * as FileSystem from "effect/FileSystem"
|
|
24
|
-
import * as Console from "effect/Console"
|
|
25
|
-
import { SemanticSearch } from "./SemanticSearch/Service.ts"
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
* @since 1.0.0
|
|
29
|
-
* @category Services
|
|
30
|
-
*/
|
|
31
|
-
export * from "./SemanticSearch/Service.ts"
|
|
32
|
-
|
|
33
|
-
const normalizePath = (path: string) => path.replace(/\\/g, "/")
|
|
34
|
-
|
|
35
|
-
const resolveChunkConfig = (options: {
|
|
36
|
-
readonly chunkMaxCharacters?: number | undefined
|
|
37
|
-
}) => ({
|
|
38
|
-
chunkSize: 30,
|
|
39
|
-
chunkOverlap: 0,
|
|
40
|
-
chunkMaxCharacters: options.chunkMaxCharacters ?? 15_000,
|
|
41
|
-
})
|
|
42
|
-
|
|
43
|
-
export const makeEmbeddingResolver = (
|
|
44
|
-
resolver: EmbeddingModel.EmbeddingModel["resolver"],
|
|
45
|
-
options: {
|
|
46
|
-
readonly embeddingBatchSize?: number | undefined
|
|
47
|
-
readonly embeddingRequestDelay?: Duration.Input | undefined
|
|
48
|
-
},
|
|
49
|
-
): EmbeddingModel.EmbeddingModel["resolver"] =>
|
|
50
|
-
resolver.pipe(
|
|
51
|
-
RequestResolver.setDelay(
|
|
52
|
-
options.embeddingRequestDelay ?? Duration.millis(50),
|
|
53
|
-
),
|
|
54
|
-
RequestResolver.batchN(options.embeddingBatchSize ?? 100),
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
export const chunkEmbeddingInput = (chunk: CodeChunker.CodeChunk): string => {
|
|
58
|
-
const headerLines = ["---", "file: " + chunk.path]
|
|
59
|
-
|
|
60
|
-
if (chunk.name !== undefined) {
|
|
61
|
-
headerLines.push("name: " + chunk.name)
|
|
62
|
-
}
|
|
63
|
-
if (chunk.type !== undefined) {
|
|
64
|
-
headerLines.push("type: " + chunk.type)
|
|
65
|
-
}
|
|
66
|
-
if (chunk.parent !== undefined) {
|
|
67
|
-
headerLines.push("parent: " + chunk.parent)
|
|
68
|
-
}
|
|
69
|
-
headerLines.push("---")
|
|
70
|
-
|
|
71
|
-
const contentLines = chunk.content.split("\n")
|
|
72
|
-
let contentWithLines = ""
|
|
73
|
-
for (let i = 0; i < contentLines.length; i++) {
|
|
74
|
-
if (i > 0) {
|
|
75
|
-
contentWithLines += "\n"
|
|
76
|
-
}
|
|
77
|
-
contentWithLines += `${chunk.startLine + i}: ${contentLines[i]}`
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
return headerLines.join("\n") + "\n\n" + contentWithLines
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
const hashChunkInput = (input: string): string =>
|
|
84
|
-
createHash("sha256").update(input).digest("hex")
|
|
85
|
-
|
|
86
|
-
/**
|
|
87
|
-
* @since 1.0.0
|
|
88
|
-
* @category Layers
|
|
89
|
-
*/
|
|
90
|
-
export const layer = (options: {
|
|
91
|
-
readonly directory: string
|
|
92
|
-
readonly database?: string | undefined
|
|
93
|
-
readonly embeddingBatchSize?: number | undefined
|
|
94
|
-
readonly embeddingRequestDelay?: Duration.Input | undefined
|
|
95
|
-
readonly concurrency?: number | undefined
|
|
96
|
-
readonly chunkMaxCharacters?: number | undefined
|
|
97
|
-
}): Layer.Layer<
|
|
98
|
-
SemanticSearch,
|
|
99
|
-
| SqlError.SqlError
|
|
100
|
-
| SqliteMigrator.MigrationError
|
|
101
|
-
| PlatformError.PlatformError,
|
|
102
|
-
| EmbeddingModel.EmbeddingModel
|
|
103
|
-
| Path.Path
|
|
104
|
-
| ChildProcessSpawner.ChildProcessSpawner
|
|
105
|
-
| FileSystem.FileSystem
|
|
106
|
-
| EmbeddingModel.Dimensions
|
|
107
|
-
> =>
|
|
108
|
-
Layer.effect(
|
|
109
|
-
SemanticSearch,
|
|
110
|
-
Effect.gen(function* () {
|
|
111
|
-
const chunker = yield* CodeChunker.CodeChunker
|
|
112
|
-
const repo = yield* ChunkRepo.ChunkRepo
|
|
113
|
-
const embeddings = yield* EmbeddingModel.EmbeddingModel
|
|
114
|
-
const pathService = yield* Path.Path
|
|
115
|
-
const root = pathService.resolve(options.directory)
|
|
116
|
-
const resolver = makeEmbeddingResolver(embeddings.resolver, options)
|
|
117
|
-
const concurrency = options.concurrency ?? 2000
|
|
118
|
-
const chunkConfig = resolveChunkConfig(options)
|
|
119
|
-
const indexHandle = yield* FiberHandle.make()
|
|
120
|
-
const console = yield* Console.Console
|
|
121
|
-
|
|
122
|
-
const resolveIndexedPath = (path: string): Option.Option<string> => {
|
|
123
|
-
const absolutePath = pathService.resolve(root, path)
|
|
124
|
-
const relativePath = normalizePath(
|
|
125
|
-
pathService.relative(root, absolutePath),
|
|
126
|
-
)
|
|
127
|
-
if (
|
|
128
|
-
relativePath.length === 0 ||
|
|
129
|
-
relativePath === ".." ||
|
|
130
|
-
relativePath.startsWith("../")
|
|
131
|
-
) {
|
|
132
|
-
return Option.none()
|
|
133
|
-
}
|
|
134
|
-
return Option.some(relativePath)
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
const processChunk = Effect.fnUntraced(
|
|
138
|
-
function* (options: {
|
|
139
|
-
readonly chunk: CodeChunker.CodeChunk
|
|
140
|
-
readonly syncId: ChunkRepo.SyncId
|
|
141
|
-
readonly checkExisting: boolean
|
|
142
|
-
}) {
|
|
143
|
-
const input = chunkEmbeddingInput(options.chunk)
|
|
144
|
-
const hash = hashChunkInput(input)
|
|
145
|
-
|
|
146
|
-
if (options.checkExisting) {
|
|
147
|
-
const id = yield* repo.exists(hash)
|
|
148
|
-
if (Option.isSome(id)) {
|
|
149
|
-
yield* repo.setSyncId(id.value, options.syncId)
|
|
150
|
-
return
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
const result = yield* Effect.request(
|
|
155
|
-
new EmbeddingModel.EmbeddingRequest({ input }),
|
|
156
|
-
resolver,
|
|
157
|
-
)
|
|
158
|
-
const vector = new Float32Array(result.vector)
|
|
159
|
-
yield* repo.insert(
|
|
160
|
-
ChunkRepo.Chunk.insert.make({
|
|
161
|
-
path: options.chunk.path,
|
|
162
|
-
hash,
|
|
163
|
-
content: input,
|
|
164
|
-
vector,
|
|
165
|
-
syncId: options.syncId,
|
|
166
|
-
}),
|
|
167
|
-
)
|
|
168
|
-
},
|
|
169
|
-
Effect.ignore({
|
|
170
|
-
log: "Warn",
|
|
171
|
-
message: "Failed to process chunk for embedding",
|
|
172
|
-
}),
|
|
173
|
-
(effect, options) =>
|
|
174
|
-
Effect.annotateLogs(effect, {
|
|
175
|
-
chunk: `${options.chunk.path}/${options.chunk.startLine}`,
|
|
176
|
-
}),
|
|
177
|
-
)
|
|
178
|
-
|
|
179
|
-
const index = Effect.gen(function* () {
|
|
180
|
-
const syncId = ChunkRepo.SyncId.make(crypto.randomUUID())
|
|
181
|
-
yield* Effect.logInfo("Starting SemanticSearch index")
|
|
182
|
-
|
|
183
|
-
yield* pipe(
|
|
184
|
-
chunker.chunkCodebase({
|
|
185
|
-
root,
|
|
186
|
-
...chunkConfig,
|
|
187
|
-
}),
|
|
188
|
-
Stream.tap(
|
|
189
|
-
(chunk) =>
|
|
190
|
-
processChunk({
|
|
191
|
-
chunk,
|
|
192
|
-
syncId,
|
|
193
|
-
checkExisting: true,
|
|
194
|
-
}),
|
|
195
|
-
{ concurrency },
|
|
196
|
-
),
|
|
197
|
-
Stream.runDrain,
|
|
198
|
-
)
|
|
199
|
-
|
|
200
|
-
yield* repo.deleteForSyncId(syncId)
|
|
201
|
-
|
|
202
|
-
yield* Effect.logInfo("Finished SemanticSearch index")
|
|
203
|
-
}).pipe(
|
|
204
|
-
Effect.withSpan("SemanticSearch.index"),
|
|
205
|
-
Effect.withLogSpan("SemanticSearch.index"),
|
|
206
|
-
Effect.provideService(Console.Console, console),
|
|
207
|
-
)
|
|
208
|
-
|
|
209
|
-
const runIndex = FiberHandle.run(indexHandle, index, {
|
|
210
|
-
onlyIfMissing: true,
|
|
211
|
-
})
|
|
212
|
-
|
|
213
|
-
const initialIndex = yield* runIndex
|
|
214
|
-
yield* runIndex.pipe(
|
|
215
|
-
Effect.delay(Duration.minutes(3)),
|
|
216
|
-
Effect.forever,
|
|
217
|
-
Effect.forkScoped,
|
|
218
|
-
)
|
|
219
|
-
|
|
220
|
-
return SemanticSearch.of({
|
|
221
|
-
search: Effect.fn("SemanticSearch.search")(function* (options) {
|
|
222
|
-
yield* Fiber.join(initialIndex)
|
|
223
|
-
yield* Effect.annotateCurrentSpan(options)
|
|
224
|
-
const { vector } = yield* embeddings.embed(options.query)
|
|
225
|
-
const results = yield* repo.search({
|
|
226
|
-
vector: new Float32Array(vector),
|
|
227
|
-
limit: options.limit,
|
|
228
|
-
})
|
|
229
|
-
return results.map((r) => r.content).join("\n\n")
|
|
230
|
-
}, Effect.orDie),
|
|
231
|
-
updateFile: Effect.fn("SemanticSearch.updateFile")(function* (path) {
|
|
232
|
-
yield* Fiber.join(initialIndex)
|
|
233
|
-
const indexedPath = resolveIndexedPath(path)
|
|
234
|
-
if (Option.isNone(indexedPath)) {
|
|
235
|
-
return
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
yield* repo.deleteByPath(indexedPath.value)
|
|
239
|
-
|
|
240
|
-
const chunks = yield* chunker.chunkFile({
|
|
241
|
-
root,
|
|
242
|
-
path: indexedPath.value,
|
|
243
|
-
...chunkConfig,
|
|
244
|
-
})
|
|
245
|
-
if (chunks.length === 0) {
|
|
246
|
-
return
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
const syncId = ChunkRepo.SyncId.make(crypto.randomUUID())
|
|
250
|
-
|
|
251
|
-
yield* pipe(
|
|
252
|
-
Stream.fromArray(chunks),
|
|
253
|
-
Stream.tap(
|
|
254
|
-
(chunk) =>
|
|
255
|
-
processChunk({
|
|
256
|
-
chunk,
|
|
257
|
-
syncId,
|
|
258
|
-
checkExisting: false,
|
|
259
|
-
}),
|
|
260
|
-
{ concurrency },
|
|
261
|
-
),
|
|
262
|
-
Stream.runDrain,
|
|
263
|
-
)
|
|
264
|
-
}, Effect.orDie),
|
|
265
|
-
removeFile: Effect.fn("SemanticSearch.removeFile")(function* (path) {
|
|
266
|
-
yield* Fiber.join(initialIndex)
|
|
267
|
-
const indexedPath = resolveIndexedPath(path)
|
|
268
|
-
if (Option.isNone(indexedPath)) {
|
|
269
|
-
return
|
|
270
|
-
}
|
|
271
|
-
yield* repo.deleteByPath(indexedPath.value)
|
|
272
|
-
}, Effect.orDie),
|
|
273
|
-
})
|
|
274
|
-
}),
|
|
275
|
-
).pipe(
|
|
276
|
-
Layer.provide([
|
|
277
|
-
CodeChunker.layer,
|
|
278
|
-
ChunkRepo.layer.pipe(
|
|
279
|
-
Layer.provide(SqliteLayer(options.database ?? ".clanka/search.sqlite")),
|
|
280
|
-
),
|
|
281
|
-
]),
|
|
282
|
-
)
|
package/src/Sqlite.ts
DELETED
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @since 1.0.0
|
|
3
|
-
*/
|
|
4
|
-
import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient"
|
|
5
|
-
import * as SqliteMigrator from "@effect/sql-sqlite-node/SqliteMigrator"
|
|
6
|
-
import * as Effect from "effect/Effect"
|
|
7
|
-
import * as Layer from "effect/Layer"
|
|
8
|
-
import * as SqlClient from "effect/sql/SqlClient"
|
|
9
|
-
import * as FileSystem from "effect/FileSystem"
|
|
10
|
-
import * as Path from "effect/Path"
|
|
11
|
-
import { getLoadablePath } from "sqlite-vec"
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* @since 1.0.0
|
|
15
|
-
* @category Layers
|
|
16
|
-
*/
|
|
17
|
-
export const SqliteLayer = (database: string) =>
|
|
18
|
-
SqliteMigrator.layer({
|
|
19
|
-
loader: SqliteMigrator.fromRecord({
|
|
20
|
-
"0001_create_chunks": Effect.gen(function* () {
|
|
21
|
-
const sql = yield* SqlClient.SqlClient
|
|
22
|
-
|
|
23
|
-
yield* sql`CREATE TABLE IF NOT EXISTS chunks (
|
|
24
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
25
|
-
path TEXT NOT NULL,
|
|
26
|
-
content TEXT NOT NULL,
|
|
27
|
-
hash TEXT NOT NULL,
|
|
28
|
-
vector BLOB NOT NULL,
|
|
29
|
-
syncId TEXT NOT NULL
|
|
30
|
-
)`
|
|
31
|
-
|
|
32
|
-
yield* sql`CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks (hash)`
|
|
33
|
-
}),
|
|
34
|
-
}),
|
|
35
|
-
}).pipe(
|
|
36
|
-
Layer.provide(
|
|
37
|
-
Layer.effectDiscard(
|
|
38
|
-
Effect.gen(function* () {
|
|
39
|
-
const client = yield* SqliteClient.SqliteClient
|
|
40
|
-
yield* client.loadExtension(getLoadablePath())
|
|
41
|
-
}),
|
|
42
|
-
),
|
|
43
|
-
),
|
|
44
|
-
Layer.provideMerge(
|
|
45
|
-
SqliteClient.layer({
|
|
46
|
-
filename: database,
|
|
47
|
-
}),
|
|
48
|
-
),
|
|
49
|
-
Layer.provide(
|
|
50
|
-
Layer.effectDiscard(
|
|
51
|
-
Effect.gen(function* () {
|
|
52
|
-
const fs = yield* FileSystem.FileSystem
|
|
53
|
-
const path = yield* Path.Path
|
|
54
|
-
const directory = path.dirname(database)
|
|
55
|
-
if (directory === ".") return
|
|
56
|
-
yield* fs.makeDirectory(directory, { recursive: true })
|
|
57
|
-
}),
|
|
58
|
-
),
|
|
59
|
-
),
|
|
60
|
-
)
|