clanka 0.26.1 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -3
- package/dist/Acp.test.js +0 -1
- package/dist/Acp.test.js.map +1 -1
- package/dist/AcpImage.test.js +0 -1
- package/dist/AcpImage.test.js.map +1 -1
- package/dist/Agent.d.ts.map +1 -1
- package/dist/Agent.js +1 -2
- package/dist/Agent.js.map +1 -1
- package/dist/Agent.test.js +0 -1
- package/dist/Agent.test.js.map +1 -1
- package/dist/AgentCallerConfig.test.js +0 -1
- package/dist/AgentCallerConfig.test.js.map +1 -1
- package/dist/AgentExecutor.d.ts +2 -3
- package/dist/AgentExecutor.d.ts.map +1 -1
- package/dist/AgentExecutor.js +4 -13
- package/dist/AgentExecutor.js.map +1 -1
- package/dist/AgentImage.test.js +0 -1
- package/dist/AgentImage.test.js.map +1 -1
- package/dist/AgentSkills.test.js +0 -2
- package/dist/AgentSkills.test.js.map +1 -1
- package/dist/AgentTools.d.ts +2 -235
- package/dist/AgentTools.d.ts.map +1 -1
- package/dist/AgentTools.js +2 -32
- package/dist/AgentTools.js.map +1 -1
- package/dist/ClaudeAgent.test.js +0 -1
- package/dist/ClaudeAgent.test.js.map +1 -1
- package/dist/McpServers.d.ts +20 -1
- package/dist/McpServers.d.ts.map +1 -1
- package/dist/McpServers.js +12 -5
- package/dist/McpServers.js.map +1 -1
- package/dist/McpServers.test.js +115 -0
- package/dist/McpServers.test.js.map +1 -1
- package/dist/ScriptFormatting.d.ts +5 -0
- package/dist/ScriptFormatting.d.ts.map +1 -1
- package/dist/ScriptFormatting.js +255 -53
- package/dist/ScriptFormatting.js.map +1 -1
- package/dist/ScriptFormatting.test.js +16 -0
- package/dist/ScriptFormatting.test.js.map +1 -1
- package/dist/bin/{Latch-DOdbsGMQ.mjs → Latch-BE6_TYRL.mjs} +1 -1
- package/dist/bin/{McpClient-B-FFG6Pg.mjs → McpClient-DCt4v5W5.mjs} +2 -2
- package/dist/bin/{OutputFormatter-DhhP2i-7.mjs → OutputFormatter-DaFv_C_I.mjs} +2 -3
- package/dist/bin/{Schema-CFdSO6iH.mjs → Schema-B3MiG1aD.mjs} +204 -568
- package/dist/bin/cli.mjs +5098 -2472
- package/dist/bin/{upgrade-C4aOEWJH.mjs → upgrade-WqgIU_21.mjs} +3 -3
- package/dist/cli.js +2 -25
- package/dist/cli.js.map +1 -1
- package/dist/fixtures/McpFakeServer.d.ts +4 -0
- package/dist/fixtures/McpFakeServer.d.ts.map +1 -1
- package/dist/fixtures/McpFakeServer.js +13 -0
- package/dist/fixtures/McpFakeServer.js.map +1 -1
- package/dist/index.d.ts +0 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -8
- package/dist/index.js.map +1 -1
- package/package.json +1 -7
- package/src/Acp.test.ts +0 -1
- package/src/AcpImage.test.ts +0 -1
- package/src/Agent.test.ts +0 -1
- package/src/Agent.ts +0 -4
- package/src/AgentCallerConfig.test.ts +0 -1
- package/src/AgentExecutor.ts +4 -16
- package/src/AgentImage.test.ts +0 -1
- package/src/AgentSkills.test.ts +0 -2
- package/src/AgentTools.ts +4 -41
- package/src/ClaudeAgent.test.ts +0 -1
- package/src/McpServers.test.ts +198 -0
- package/src/McpServers.ts +44 -4
- package/src/ScriptFormatting.test.ts +20 -0
- package/src/ScriptFormatting.ts +234 -47
- package/src/cli.ts +1 -56
- package/src/fixtures/McpFakeServer.ts +17 -1
- package/src/index.ts +0 -10
- package/dist/ChunkRepo.d.ts +0 -173
- package/dist/ChunkRepo.d.ts.map +0 -1
- package/dist/ChunkRepo.js +0 -168
- package/dist/ChunkRepo.js.map +0 -1
- package/dist/ChunkRepo.test.d.ts +0 -2
- package/dist/ChunkRepo.test.d.ts.map +0 -1
- package/dist/ChunkRepo.test.js +0 -63
- package/dist/ChunkRepo.test.js.map +0 -1
- package/dist/CodeChunker.d.ts +0 -84
- package/dist/CodeChunker.d.ts.map +0 -1
- package/dist/CodeChunker.js +0 -658
- package/dist/CodeChunker.js.map +0 -1
- package/dist/CodeChunker.test.d.ts +0 -2
- package/dist/CodeChunker.test.d.ts.map +0 -1
- package/dist/CodeChunker.test.js +0 -434
- package/dist/CodeChunker.test.js.map +0 -1
- package/dist/SemanticSearch/Service.d.ts +0 -31
- package/dist/SemanticSearch/Service.d.ts.map +0 -1
- package/dist/SemanticSearch/Service.js +0 -29
- package/dist/SemanticSearch/Service.js.map +0 -1
- package/dist/SemanticSearch.d.ts +0 -34
- package/dist/SemanticSearch.d.ts.map +0 -1
- package/dist/SemanticSearch.js +0 -171
- package/dist/SemanticSearch.js.map +0 -1
- package/dist/SemanticSearch.test.d.ts +0 -2
- package/dist/SemanticSearch.test.d.ts.map +0 -1
- package/dist/SemanticSearch.test.js +0 -34
- package/dist/SemanticSearch.test.js.map +0 -1
- package/dist/Sqlite.d.ts +0 -15
- package/dist/Sqlite.d.ts.map +0 -1
- package/dist/Sqlite.js +0 -44
- package/dist/Sqlite.js.map +0 -1
- package/dist/bin/AiError-DBaKz35C.mjs +0 -2723
- package/dist/bin/SemanticSearch-C3a7LAQr.mjs +0 -2558
- package/src/ChunkRepo.test.ts +0 -89
- package/src/ChunkRepo.ts +0 -260
- package/src/CodeChunker.test.ts +0 -493
- package/src/CodeChunker.ts +0 -1018
- package/src/SemanticSearch/Service.ts +0 -50
- package/src/SemanticSearch.test.ts +0 -51
- package/src/SemanticSearch.ts +0 -282
- package/src/Sqlite.ts +0 -60
package/src/cli.ts
CHANGED
|
@@ -29,7 +29,6 @@ import * as Config from "effect/Config"
|
|
|
29
29
|
import * as KeyValueStore from "effect/persistence/KeyValueStore"
|
|
30
30
|
import * as Option from "effect/Option"
|
|
31
31
|
import type * as Redacted from "effect/Redacted"
|
|
32
|
-
import { OpenAiClient, OpenAiEmbeddingModel } from "@effect/ai-openai"
|
|
33
32
|
import * as DeviceCodeHandler from "./DeviceCodeHandler.ts"
|
|
34
33
|
import packageJson from "../package.json" with { type: "json" }
|
|
35
34
|
|
|
@@ -242,14 +241,6 @@ const fast = Flag.Boolean("fast").pipe(
|
|
|
242
241
|
),
|
|
243
242
|
)
|
|
244
243
|
|
|
245
|
-
const semantic = Flag.Directory("search").pipe(
|
|
246
|
-
Flag.withDescription(
|
|
247
|
-
"Directory for semantic search data (uses OPENAI_API_KEY env var)",
|
|
248
|
-
),
|
|
249
|
-
Flag.withAlias("s"),
|
|
250
|
-
Flag.optional,
|
|
251
|
-
)
|
|
252
|
-
|
|
253
244
|
const prompt = Flag.String("prompt").pipe(
|
|
254
245
|
Flag.withDescription("Pass a prompt in non-interactive mode"),
|
|
255
246
|
Flag.optional,
|
|
@@ -365,42 +356,6 @@ const AcpSessions = Layer.unwrap(
|
|
|
365
356
|
}),
|
|
366
357
|
).pipe(Layer.provide(NodeServices.layer))
|
|
367
358
|
|
|
368
|
-
const Search = (directory: string) =>
|
|
369
|
-
Layer.unwrap(
|
|
370
|
-
Effect.gen(function* () {
|
|
371
|
-
const apiKey = yield* Config.Redacted("OPENAI_API_KEY").pipe(
|
|
372
|
-
Config.option,
|
|
373
|
-
)
|
|
374
|
-
|
|
375
|
-
if (Option.isNone(apiKey)) {
|
|
376
|
-
yield* Effect.logWarning("OPENAI_API_KEY is not set")
|
|
377
|
-
return Layer.empty
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
const path = yield* Path.Path
|
|
381
|
-
|
|
382
|
-
const SemanticSearch = yield* Effect.promise(
|
|
383
|
-
() => import("./SemanticSearch.ts"),
|
|
384
|
-
)
|
|
385
|
-
|
|
386
|
-
return SemanticSearch.layer({
|
|
387
|
-
directory: process.cwd(),
|
|
388
|
-
database: path.join(directory, "search.sqlite"),
|
|
389
|
-
}).pipe(
|
|
390
|
-
Layer.provide(
|
|
391
|
-
OpenAiEmbeddingModel.model("text-embedding-3-small", {
|
|
392
|
-
dimensions: 1536,
|
|
393
|
-
}),
|
|
394
|
-
),
|
|
395
|
-
Layer.provide(
|
|
396
|
-
OpenAiClient.layer({
|
|
397
|
-
apiKey: apiKey.value,
|
|
398
|
-
}),
|
|
399
|
-
),
|
|
400
|
-
)
|
|
401
|
-
}),
|
|
402
|
-
).pipe(Layer.provide([NodeServices.layer, HttpClientLive]))
|
|
403
|
-
|
|
404
359
|
const acpModel = Flag.String("model").pipe(
|
|
405
360
|
Flag.withAlias("m"),
|
|
406
361
|
Flag.withDescription(
|
|
@@ -502,7 +457,6 @@ Command.make("clanka", {
|
|
|
502
457
|
fetch,
|
|
503
458
|
model,
|
|
504
459
|
fast,
|
|
505
|
-
semantic,
|
|
506
460
|
prompt,
|
|
507
461
|
compaction,
|
|
508
462
|
compactionThreshold,
|
|
@@ -518,7 +472,6 @@ Command.make("clanka", {
|
|
|
518
472
|
provider,
|
|
519
473
|
model: modelRaw,
|
|
520
474
|
fast,
|
|
521
|
-
semantic,
|
|
522
475
|
prompt: nonInteractivePrompt,
|
|
523
476
|
apiKey,
|
|
524
477
|
apiUrl,
|
|
@@ -587,15 +540,7 @@ Command.make("clanka", {
|
|
|
587
540
|
Agent.layerLocal({
|
|
588
541
|
directory: process.cwd(),
|
|
589
542
|
excludeTools: flags.fetch ? [] : ["fetch"],
|
|
590
|
-
}).pipe(
|
|
591
|
-
Layer.provide(
|
|
592
|
-
Option.match(semantic, {
|
|
593
|
-
onNone: () => Layer.empty,
|
|
594
|
-
onSome: Search,
|
|
595
|
-
}),
|
|
596
|
-
),
|
|
597
|
-
Layer.provideMerge(McpServers.layer(mcpServers)),
|
|
598
|
-
),
|
|
543
|
+
}).pipe(Layer.provideMerge(McpServers.layer(mcpServers))),
|
|
599
544
|
Model,
|
|
600
545
|
]),
|
|
601
546
|
)
|
|
@@ -98,11 +98,17 @@ export const makeFakeServer = (options: {
|
|
|
98
98
|
* - `endSession` answers every later request with 404, as a server does for
|
|
99
99
|
* a session it has ended. `afterEnd` lists those requests as
|
|
100
100
|
* `"<HTTP method> <JSON-RPC method>"`.
|
|
101
|
+
* - `authorize`, when given, answers 401 to a request whose `Authorization`
|
|
102
|
+
* header it rejects, and 404 to any other path, so OAuth discovery finds no
|
|
103
|
+
* metadata.
|
|
101
104
|
*/
|
|
102
105
|
export const listenHttp = async (
|
|
103
106
|
server: Server,
|
|
104
107
|
path = "/mcp",
|
|
105
|
-
options: {
|
|
108
|
+
options: {
|
|
109
|
+
readonly resumable?: boolean
|
|
110
|
+
readonly authorize?: (authorization: string | undefined) => boolean
|
|
111
|
+
} = {},
|
|
106
112
|
): Promise<{
|
|
107
113
|
readonly url: string
|
|
108
114
|
readonly authorization: ReadonlyArray<string | undefined>
|
|
@@ -123,6 +129,16 @@ export const listenHttp = async (
|
|
|
123
129
|
let ended = false
|
|
124
130
|
const http = Http.createServer((request, response) => {
|
|
125
131
|
authorization.push(request.headers.authorization)
|
|
132
|
+
if (options.authorize !== undefined) {
|
|
133
|
+
if (new URL(request.url ?? "/", "http://localhost").pathname !== path) {
|
|
134
|
+
response.writeHead(404).end()
|
|
135
|
+
return
|
|
136
|
+
}
|
|
137
|
+
if (!options.authorize(request.headers.authorization)) {
|
|
138
|
+
response.writeHead(401, { "www-authenticate": "Bearer" }).end()
|
|
139
|
+
return
|
|
140
|
+
}
|
|
141
|
+
}
|
|
126
142
|
if (!ended) {
|
|
127
143
|
void transport.handleRequest(request, response)
|
|
128
144
|
return
|
package/src/index.ts
CHANGED
|
@@ -44,11 +44,6 @@ export * as Codex from "./Codex.ts"
|
|
|
44
44
|
*/
|
|
45
45
|
export * as ModelsRepo from "./ModelsRepo.ts"
|
|
46
46
|
|
|
47
|
-
/**
|
|
48
|
-
* @since 1.0.0
|
|
49
|
-
*/
|
|
50
|
-
export * as CodeChunker from "./CodeChunker.ts"
|
|
51
|
-
|
|
52
47
|
/**
|
|
53
48
|
* @since 1.0.0
|
|
54
49
|
*/
|
|
@@ -84,11 +79,6 @@ export * as DeviceCodeHandler from "./DeviceCodeHandler.ts"
|
|
|
84
79
|
*/
|
|
85
80
|
export * as OutputFormatter from "./OutputFormatter.ts"
|
|
86
81
|
|
|
87
|
-
/**
|
|
88
|
-
* @since 1.0.0
|
|
89
|
-
*/
|
|
90
|
-
export * as SemanticSearch from "./SemanticSearch.ts"
|
|
91
|
-
|
|
92
82
|
/**
|
|
93
83
|
* @since 1.0.0
|
|
94
84
|
*/
|
package/dist/ChunkRepo.d.ts
DELETED
|
@@ -1,173 +0,0 @@
|
|
|
1
|
-
/** @effect-diagnostics schemaNumber:off */
|
|
2
|
-
import * as Effect from "effect/Effect";
|
|
3
|
-
import * as Layer from "effect/Layer";
|
|
4
|
-
import * as Schema from "effect/Schema";
|
|
5
|
-
import * as Context from "effect/Context";
|
|
6
|
-
import * as SqlClient from "effect/sql/SqlClient";
|
|
7
|
-
import * as SqlError from "effect/sql/SqlError";
|
|
8
|
-
import type * as Cause from "effect/Cause";
|
|
9
|
-
import type * as Option from "effect/Option";
|
|
10
|
-
import * as EmbeddingModel from "effect/ai/EmbeddingModel";
|
|
11
|
-
/**
|
|
12
|
-
* @since 1.0.0
|
|
13
|
-
* @category Models
|
|
14
|
-
*/
|
|
15
|
-
export declare const ChunkId: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
16
|
-
/**
|
|
17
|
-
* @since 1.0.0
|
|
18
|
-
* @category Models
|
|
19
|
-
*/
|
|
20
|
-
export type ChunkId = typeof ChunkId.Type;
|
|
21
|
-
/**
|
|
22
|
-
* @since 1.0.0
|
|
23
|
-
* @category Models
|
|
24
|
-
*/
|
|
25
|
-
export declare const SyncId: Schema.brand<Schema.String, "ChunkRepo/SyncId">;
|
|
26
|
-
/**
|
|
27
|
-
* @since 1.0.0
|
|
28
|
-
* @category Models
|
|
29
|
-
*/
|
|
30
|
-
export type SyncId = typeof SyncId.Type;
|
|
31
|
-
/**
|
|
32
|
-
* @since 1.0.0
|
|
33
|
-
* @category Models
|
|
34
|
-
*/
|
|
35
|
-
export declare const Float32ArraySchema: Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>;
|
|
36
|
-
/**
|
|
37
|
-
* @since 1.0.0
|
|
38
|
-
* @category Models
|
|
39
|
-
*/
|
|
40
|
-
export declare const Float32ArrayFromArray: Schema.decodeTo<Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>, Schema.$Array<Schema.Number>, never, never>;
|
|
41
|
-
/**
|
|
42
|
-
* @since 1.0.0
|
|
43
|
-
* @category Models
|
|
44
|
-
*/
|
|
45
|
-
export declare const Float32ArrayField: import("effect/schema/VariantSchema").Field<{
|
|
46
|
-
readonly insert: Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>;
|
|
47
|
-
readonly update: Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>;
|
|
48
|
-
readonly jsonCreate: Schema.decodeTo<Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>, Schema.$Array<Schema.Number>, never, never>;
|
|
49
|
-
readonly jsonUpdate: Schema.decodeTo<Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>, Schema.$Array<Schema.Number>, never, never>;
|
|
50
|
-
}>;
|
|
51
|
-
declare const Chunk_base: import("effect/schema/VariantSchema").Class<Chunk, {
|
|
52
|
-
readonly id: import("effect/schema/VariantSchema").Field<{
|
|
53
|
-
readonly json: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
54
|
-
readonly jsonUpdate: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
55
|
-
readonly select: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
56
|
-
readonly update: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
57
|
-
}>;
|
|
58
|
-
readonly path: Schema.String;
|
|
59
|
-
readonly content: Schema.String;
|
|
60
|
-
readonly hash: Schema.String;
|
|
61
|
-
readonly vector: import("effect/schema/VariantSchema").Field<{
|
|
62
|
-
readonly insert: Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>;
|
|
63
|
-
readonly update: Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>;
|
|
64
|
-
readonly jsonCreate: Schema.decodeTo<Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>, Schema.$Array<Schema.Number>, never, never>;
|
|
65
|
-
readonly jsonUpdate: Schema.decodeTo<Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>, Schema.$Array<Schema.Number>, never, never>;
|
|
66
|
-
}>;
|
|
67
|
-
readonly syncId: Schema.brand<Schema.String, "ChunkRepo/SyncId">;
|
|
68
|
-
}, Schema.Struct<import("effect/schema/VariantSchema").ExtractFields<"select", {
|
|
69
|
-
readonly id: import("effect/schema/VariantSchema").Field<{
|
|
70
|
-
readonly json: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
71
|
-
readonly jsonUpdate: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
72
|
-
readonly select: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
73
|
-
readonly update: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
74
|
-
}>;
|
|
75
|
-
readonly path: Schema.String;
|
|
76
|
-
readonly content: Schema.String;
|
|
77
|
-
readonly hash: Schema.String;
|
|
78
|
-
readonly vector: import("effect/schema/VariantSchema").Field<{
|
|
79
|
-
readonly insert: Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>;
|
|
80
|
-
readonly update: Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>;
|
|
81
|
-
readonly jsonCreate: Schema.decodeTo<Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>, Schema.$Array<Schema.Number>, never, never>;
|
|
82
|
-
readonly jsonUpdate: Schema.decodeTo<Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>, Schema.$Array<Schema.Number>, never, never>;
|
|
83
|
-
}>;
|
|
84
|
-
readonly syncId: Schema.brand<Schema.String, "ChunkRepo/SyncId">;
|
|
85
|
-
}, true>>, "insert" | "json" | "jsonCreate" | "jsonUpdate" | "select" | "update", "select", {}> & {
|
|
86
|
-
readonly insert: Schema.Struct<{
|
|
87
|
-
readonly path: Schema.String;
|
|
88
|
-
readonly content: Schema.String;
|
|
89
|
-
readonly hash: Schema.String;
|
|
90
|
-
readonly vector: Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>;
|
|
91
|
-
readonly syncId: Schema.brand<Schema.String, "ChunkRepo/SyncId">;
|
|
92
|
-
}>;
|
|
93
|
-
readonly json: Schema.Struct<{
|
|
94
|
-
readonly id: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
95
|
-
readonly path: Schema.String;
|
|
96
|
-
readonly content: Schema.String;
|
|
97
|
-
readonly hash: Schema.String;
|
|
98
|
-
readonly syncId: Schema.brand<Schema.String, "ChunkRepo/SyncId">;
|
|
99
|
-
}>;
|
|
100
|
-
readonly jsonCreate: Schema.Struct<{
|
|
101
|
-
readonly path: Schema.String;
|
|
102
|
-
readonly content: Schema.String;
|
|
103
|
-
readonly hash: Schema.String;
|
|
104
|
-
readonly vector: Schema.decodeTo<Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>, Schema.$Array<Schema.Number>, never, never>;
|
|
105
|
-
readonly syncId: Schema.brand<Schema.String, "ChunkRepo/SyncId">;
|
|
106
|
-
}>;
|
|
107
|
-
readonly jsonUpdate: Schema.Struct<{
|
|
108
|
-
readonly id: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
109
|
-
readonly path: Schema.String;
|
|
110
|
-
readonly content: Schema.String;
|
|
111
|
-
readonly hash: Schema.String;
|
|
112
|
-
readonly vector: Schema.decodeTo<Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>, Schema.$Array<Schema.Number>, never, never>;
|
|
113
|
-
readonly syncId: Schema.brand<Schema.String, "ChunkRepo/SyncId">;
|
|
114
|
-
}>;
|
|
115
|
-
readonly select: Schema.Struct<{
|
|
116
|
-
readonly id: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
117
|
-
readonly path: Schema.String;
|
|
118
|
-
readonly content: Schema.String;
|
|
119
|
-
readonly hash: Schema.String;
|
|
120
|
-
readonly syncId: Schema.brand<Schema.String, "ChunkRepo/SyncId">;
|
|
121
|
-
}>;
|
|
122
|
-
readonly update: Schema.Struct<{
|
|
123
|
-
readonly id: Schema.brand<Schema.Number, "ChunkRepo/ChunkId">;
|
|
124
|
-
readonly path: Schema.String;
|
|
125
|
-
readonly content: Schema.String;
|
|
126
|
-
readonly hash: Schema.String;
|
|
127
|
-
readonly vector: Schema.instanceOf<Float32Array<ArrayBuffer>, Float32Array<ArrayBufferLike>>;
|
|
128
|
-
readonly syncId: Schema.brand<Schema.String, "ChunkRepo/SyncId">;
|
|
129
|
-
}>;
|
|
130
|
-
};
|
|
131
|
-
/**
|
|
132
|
-
* @since 1.0.0
|
|
133
|
-
* @category Models
|
|
134
|
-
*/
|
|
135
|
-
export declare class Chunk extends Chunk_base {
|
|
136
|
-
}
|
|
137
|
-
declare const ChunkRepo_base: Context.ServiceClass<ChunkRepo, "clanka/ChunkRepo", {
|
|
138
|
-
insert(chunk: typeof Chunk.insert.Type): Effect.Effect<Chunk, ChunkRepoError>;
|
|
139
|
-
findById(id: ChunkId): Effect.Effect<Chunk, ChunkRepoError | Cause.NoSuchElementError>;
|
|
140
|
-
exists(hash: string): Effect.Effect<Option.Option<ChunkId>, ChunkRepoError>;
|
|
141
|
-
search(options: {
|
|
142
|
-
readonly vector: Float32Array;
|
|
143
|
-
readonly limit: number;
|
|
144
|
-
}): Effect.Effect<Array<Chunk>, ChunkRepoError>;
|
|
145
|
-
quantize: Effect.Effect<void, ChunkRepoError>;
|
|
146
|
-
setSyncId(chunkId: ChunkId, syncId: SyncId): Effect.Effect<void, ChunkRepoError>;
|
|
147
|
-
deleteByPath(path: string): Effect.Effect<void, ChunkRepoError>;
|
|
148
|
-
deleteForSyncId(syncId: SyncId): Effect.Effect<void, ChunkRepoError>;
|
|
149
|
-
}>;
|
|
150
|
-
/**
|
|
151
|
-
* @since 1.0.0
|
|
152
|
-
* @category Services
|
|
153
|
-
*/
|
|
154
|
-
export declare class ChunkRepo extends ChunkRepo_base {
|
|
155
|
-
}
|
|
156
|
-
declare const ChunkRepoError_base: Schema.Class<ChunkRepoError, Schema.TaggedStruct<"ChunkRepoError", {
|
|
157
|
-
readonly reason: Schema.Union<readonly [typeof SqlError.SqlError]>;
|
|
158
|
-
}>, Cause.YieldableError>;
|
|
159
|
-
/**
|
|
160
|
-
* @since 1.0.0
|
|
161
|
-
* @category Errors
|
|
162
|
-
*/
|
|
163
|
-
export declare class ChunkRepoError extends ChunkRepoError_base {
|
|
164
|
-
readonly cause: SqlError.SqlError;
|
|
165
|
-
readonly message: string;
|
|
166
|
-
}
|
|
167
|
-
/**
|
|
168
|
-
* @since 1.0.0
|
|
169
|
-
* @category Layers
|
|
170
|
-
*/
|
|
171
|
-
export declare const layer: Layer.Layer<ChunkRepo, SqlError.SqlError, EmbeddingModel.Dimensions | SqlClient.SqlClient>;
|
|
172
|
-
export {};
|
|
173
|
-
//# sourceMappingURL=ChunkRepo.d.ts.map
|
package/dist/ChunkRepo.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ChunkRepo.d.ts","sourceRoot":"","sources":["../src/ChunkRepo.ts"],"names":[],"mappings":"AAAA,2CAA2C;AAC3C,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AAEzC,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAA;AACjD,OAAO,KAAK,QAAQ,MAAM,qBAAqB,CAAA;AAG/C,OAAO,KAAK,KAAK,KAAK,MAAM,cAAc,CAAA;AAC1C,OAAO,KAAK,KAAK,MAAM,MAAM,eAAe,CAAA;AAC5C,OAAO,KAAK,cAAc,MAAM,0BAA0B,CAAA;AAI1D;;;GAGG;AACH,eAAO,MAAM,OAAO,kDAAwD,CAAA;AAC5E;;;GAGG;AACH,MAAM,MAAM,OAAO,GAAG,OAAO,OAAO,CAAC,IAAI,CAAA;AAEzC;;;GAGG;AACH,eAAO,MAAM,MAAM,iDAAuD,CAAA;AAC1E;;;GAGG;AACH,MAAM,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,IAAI,CAAA;AAEvC;;;GAGG;AACH,eAAO,MAAM,kBAAkB,6EAGhB,CAAA;AAEf;;;GAGG;AACH,eAAO,MAAM,qBAAqB,0IAQjC,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,iBAAiB;;;;;EAK5B,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEF;;;GAGG;AACH,qBAAa,KAAM,SAAQ,UAOzB;CAAG;;kBAUQ,OAAO,KAAK,CAAC,MAAM,CAAC,IAAI,GAC9B,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,cAAc,CAAC;iBAGjC,OAAO,GACV,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,cAAc,GAAG,KAAK,CAAC,kBAAkB,CAAC;iBAErD,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC;oBAE3D;QACd,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAA;QAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KACvB,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC;cAErC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;uBAGlC,OAAO,UACR,MAAM,GACb,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;uBACnB,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;4BACvC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC;;AA7BxE;;;GAGG;AACH,qBAAa,SAAU,SAAQ,cA2BR;CAAG;;;;AAE1B;;;GAGG;AACH,qBAAa,cAAe,SAAQ,mBAKnC;IACC,QAAQ,CAAC,KAAK,oBAAc;IAC5B,QAAQ,CAAC,OAAO,SAAsB;CACvC;AAED;;;GAGG;AACH,eAAO,MAAM,KAAK,4FA0HjB,CAAA"}
|
package/dist/ChunkRepo.js
DELETED
|
@@ -1,168 +0,0 @@
|
|
|
1
|
-
/** @effect-diagnostics schemaNumber:off */
|
|
2
|
-
import * as Effect from "effect/Effect";
|
|
3
|
-
import * as Layer from "effect/Layer";
|
|
4
|
-
import * as Schema from "effect/Schema";
|
|
5
|
-
import * as SchemaTransformation from "effect/SchemaTransformation";
|
|
6
|
-
import * as Context from "effect/Context";
|
|
7
|
-
import * as Model from "effect/schema/Model";
|
|
8
|
-
import * as SqlClient from "effect/sql/SqlClient";
|
|
9
|
-
import * as SqlError from "effect/sql/SqlError";
|
|
10
|
-
import * as SqlModel from "effect/sql/SqlModel";
|
|
11
|
-
import * as SqlSchema from "effect/sql/SqlSchema";
|
|
12
|
-
import * as EmbeddingModel from "effect/ai/EmbeddingModel";
|
|
13
|
-
import * as SqlResolver from "effect/sql/SqlResolver";
|
|
14
|
-
import * as RequestResolver from "effect/RequestResolver";
|
|
15
|
-
/**
|
|
16
|
-
* @since 1.0.0
|
|
17
|
-
* @category Models
|
|
18
|
-
*/
|
|
19
|
-
export const ChunkId = Schema.Number.pipe(Schema.brand("ChunkRepo/ChunkId"));
|
|
20
|
-
/**
|
|
21
|
-
* @since 1.0.0
|
|
22
|
-
* @category Models
|
|
23
|
-
*/
|
|
24
|
-
export const SyncId = Schema.String.pipe(Schema.brand("ChunkRepo/SyncId"));
|
|
25
|
-
/**
|
|
26
|
-
* @since 1.0.0
|
|
27
|
-
* @category Models
|
|
28
|
-
*/
|
|
29
|
-
export const Float32ArraySchema = Schema.instanceOf(Float32Array);
|
|
30
|
-
/**
|
|
31
|
-
* @since 1.0.0
|
|
32
|
-
* @category Models
|
|
33
|
-
*/
|
|
34
|
-
export const Float32ArrayFromArray = Schema.Array(Schema.Number).pipe(Schema.decodeTo(Float32ArraySchema, SchemaTransformation.transform({
|
|
35
|
-
decode: (arr) => new Float32Array(arr),
|
|
36
|
-
encode: (array) => Array.from(array),
|
|
37
|
-
})));
|
|
38
|
-
/**
|
|
39
|
-
* @since 1.0.0
|
|
40
|
-
* @category Models
|
|
41
|
-
*/
|
|
42
|
-
export const Float32ArrayField = Model.Field({
|
|
43
|
-
insert: Float32ArraySchema,
|
|
44
|
-
update: Float32ArraySchema,
|
|
45
|
-
jsonCreate: Float32ArrayFromArray,
|
|
46
|
-
jsonUpdate: Float32ArrayFromArray,
|
|
47
|
-
});
|
|
48
|
-
/**
|
|
49
|
-
* @since 1.0.0
|
|
50
|
-
* @category Models
|
|
51
|
-
*/
|
|
52
|
-
export class Chunk extends Model.Class("Chunk")({
|
|
53
|
-
id: ChunkId.pipe(Model.FieldExcept(["insert", "jsonCreate"])),
|
|
54
|
-
path: Schema.String,
|
|
55
|
-
content: Schema.String,
|
|
56
|
-
hash: Schema.String,
|
|
57
|
-
vector: Float32ArrayField,
|
|
58
|
-
syncId: SyncId,
|
|
59
|
-
}) {
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* @since 1.0.0
|
|
63
|
-
* @category Services
|
|
64
|
-
*/
|
|
65
|
-
export class ChunkRepo extends Context.Service()("clanka/ChunkRepo") {
|
|
66
|
-
}
|
|
67
|
-
/**
|
|
68
|
-
* @since 1.0.0
|
|
69
|
-
* @category Errors
|
|
70
|
-
*/
|
|
71
|
-
export class ChunkRepoError extends Schema.TaggedError()("ChunkRepoError", {
|
|
72
|
-
reason: Schema.Union([SqlError.SqlError]),
|
|
73
|
-
}) {
|
|
74
|
-
cause = this.reason;
|
|
75
|
-
message = this.reason.message;
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* @since 1.0.0
|
|
79
|
-
* @category Layers
|
|
80
|
-
*/
|
|
81
|
-
export const layer = Layer.effect(ChunkRepo, Effect.gen(function* () {
|
|
82
|
-
const sql = yield* SqlClient.SqlClient;
|
|
83
|
-
const dimensions = yield* EmbeddingModel.Dimensions;
|
|
84
|
-
yield* Effect.gen(function* () {
|
|
85
|
-
yield* sql `CREATE VIRTUAL TABLE IF NOT EXISTS chunk_vectors USING vec0(
|
|
86
|
-
vector float[${sql.literal(String(dimensions))}]
|
|
87
|
-
)`;
|
|
88
|
-
yield* sql `CREATE TRIGGER IF NOT EXISTS chunks_vector_insert
|
|
89
|
-
AFTER INSERT ON chunks
|
|
90
|
-
BEGIN
|
|
91
|
-
INSERT INTO chunk_vectors(rowid, vector) VALUES (new.id, new.vector);
|
|
92
|
-
END`;
|
|
93
|
-
yield* sql `CREATE TRIGGER IF NOT EXISTS chunks_vector_update
|
|
94
|
-
AFTER UPDATE OF vector ON chunks
|
|
95
|
-
BEGIN
|
|
96
|
-
DELETE FROM chunk_vectors WHERE rowid = old.id;
|
|
97
|
-
INSERT INTO chunk_vectors(rowid, vector) VALUES (new.id, new.vector);
|
|
98
|
-
END`;
|
|
99
|
-
yield* sql `CREATE TRIGGER IF NOT EXISTS chunks_vector_delete
|
|
100
|
-
AFTER DELETE ON chunks
|
|
101
|
-
BEGIN
|
|
102
|
-
DELETE FROM chunk_vectors WHERE rowid = old.id;
|
|
103
|
-
END`;
|
|
104
|
-
yield* sql `DELETE FROM chunk_vectors
|
|
105
|
-
WHERE rowid NOT IN (SELECT id FROM chunks)`;
|
|
106
|
-
yield* sql `INSERT INTO chunk_vectors(rowid, vector)
|
|
107
|
-
SELECT id, vector FROM chunks
|
|
108
|
-
WHERE id NOT IN (SELECT rowid FROM chunk_vectors)`;
|
|
109
|
-
}).pipe(sql.withTransaction);
|
|
110
|
-
const loaders = yield* SqlModel.makeResolvers(Chunk, {
|
|
111
|
-
tableName: "chunks",
|
|
112
|
-
idColumn: "id",
|
|
113
|
-
spanPrefix: "ChunkRepo",
|
|
114
|
-
});
|
|
115
|
-
const search = SqlSchema.findAll({
|
|
116
|
-
Request: Schema.Struct({
|
|
117
|
-
vector: Float32ArraySchema,
|
|
118
|
-
limit: Schema.Number,
|
|
119
|
-
}),
|
|
120
|
-
Result: Chunk,
|
|
121
|
-
execute: ({ vector, limit }) => sql `
|
|
122
|
-
select chunks.id, chunks.path, chunks.content, chunks.hash, chunks.syncId
|
|
123
|
-
from chunk_vectors
|
|
124
|
-
JOIN chunks ON chunks.id = chunk_vectors.rowid
|
|
125
|
-
WHERE chunk_vectors.vector MATCH ${vector}
|
|
126
|
-
AND k = CAST(${limit} AS INTEGER)
|
|
127
|
-
ORDER BY chunk_vectors.distance
|
|
128
|
-
`,
|
|
129
|
-
});
|
|
130
|
-
const exists = SqlResolver.findById({
|
|
131
|
-
Id: Schema.String, // hash
|
|
132
|
-
Result: Schema.Struct({
|
|
133
|
-
id: ChunkId,
|
|
134
|
-
hash: Schema.String,
|
|
135
|
-
}),
|
|
136
|
-
ResultId(result) {
|
|
137
|
-
return result.hash;
|
|
138
|
-
},
|
|
139
|
-
execute: (hashes) => sql `select id, hash from chunks where ${sql.in("hash", hashes)}`,
|
|
140
|
-
}).pipe(RequestResolver.setDelay(5));
|
|
141
|
-
const insertResolver = loaders.insert.pipe(RequestResolver.setDelay(5));
|
|
142
|
-
const findByIdResolver = loaders.findById.pipe(RequestResolver.setDelay(5));
|
|
143
|
-
return ChunkRepo.of({
|
|
144
|
-
insert: (insert) => SqlResolver.request(insert, insertResolver).pipe(Effect.catchTags({
|
|
145
|
-
SchemaError: Effect.die,
|
|
146
|
-
ResultLengthMismatch: Effect.die,
|
|
147
|
-
}, (reason) => Effect.fail(new ChunkRepoError({ reason })))),
|
|
148
|
-
findById: (id) => SqlResolver.request(id, findByIdResolver).pipe(Effect.catchTags({
|
|
149
|
-
SchemaError: Effect.die,
|
|
150
|
-
SqlError: (reason) => Effect.fail(new ChunkRepoError({ reason })),
|
|
151
|
-
})),
|
|
152
|
-
exists: (hash) => SqlResolver.request(hash, exists).pipe(Effect.map((result) => result.id), Effect.catchNoSuchElement, Effect.catchTags({
|
|
153
|
-
SqlError: (reason) => Effect.fail(new ChunkRepoError({ reason })),
|
|
154
|
-
SchemaError: Effect.die,
|
|
155
|
-
})),
|
|
156
|
-
search: Effect.fn("ChunkRepo.search")(function* (options) {
|
|
157
|
-
return yield* search(options).pipe(Effect.catchTags({
|
|
158
|
-
SqlError: (reason) => Effect.fail(new ChunkRepoError({ reason })),
|
|
159
|
-
SchemaError: Effect.die,
|
|
160
|
-
}));
|
|
161
|
-
}),
|
|
162
|
-
quantize: Effect.void,
|
|
163
|
-
setSyncId: (chunkId, syncId) => sql `update chunks set syncId = ${syncId} where id = ${chunkId}`.pipe(Effect.mapError((reason) => new ChunkRepoError({ reason }))),
|
|
164
|
-
deleteByPath: (path) => sql `delete from chunks where path = ${path}`.pipe(Effect.mapError((reason) => new ChunkRepoError({ reason }))),
|
|
165
|
-
deleteForSyncId: (syncId) => sql `delete from chunks where syncId != ${syncId}`.pipe(Effect.mapError((reason) => new ChunkRepoError({ reason }))),
|
|
166
|
-
});
|
|
167
|
-
}));
|
|
168
|
-
//# sourceMappingURL=ChunkRepo.js.map
|
package/dist/ChunkRepo.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ChunkRepo.js","sourceRoot":"","sources":["../src/ChunkRepo.ts"],"names":[],"mappings":"AAAA,2CAA2C;AAC3C,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,oBAAoB,MAAM,6BAA6B,CAAA;AACnE,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAA;AAC5C,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAA;AACjD,OAAO,KAAK,QAAQ,MAAM,qBAAqB,CAAA;AAC/C,OAAO,KAAK,QAAQ,MAAM,qBAAqB,CAAA;AAC/C,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAA;AAGjD,OAAO,KAAK,cAAc,MAAM,0BAA0B,CAAA;AAC1D,OAAO,KAAK,WAAW,MAAM,wBAAwB,CAAA;AACrD,OAAO,KAAK,eAAe,MAAM,wBAAwB,CAAA;AAEzD;;;GAGG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAA;AAO5E;;;GAGG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAA;AAO1E;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,CAGjD,YAAY,CAAC,CAAA;AAEf;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CACnE,MAAM,CAAC,QAAQ,CACb,kBAAkB,EAClB,oBAAoB,CAAC,SAAS,CAAC;IAC7B,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC;IACtC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;CACrC,CAAC,CACH,CACF,CAAA;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,CAAC,KAAK,CAAC;IAC3C,MAAM,EAAE,kBAAkB;IAC1B,MAAM,EAAE,kBAAkB;IAC1B,UAAU,EAAE,qBAAqB;IACjC,UAAU,EAAE,qBAAqB;CAClC,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,OAAO,KAAM,SAAQ,KAAK,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAC;IACrD,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;IAC7D,IAAI,EAAE,MAAM,CAAC,MAAM;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM;IACtB,IAAI,EAAE,MAAM,CAAC,MAAM;IACnB,MAAM,EAAE,iBAAiB;IACzB,MAAM,EAAE,MAAM;CACf,CAAC;CAAG;AAEL;;;GAGG;AACH,MAAM,OAAO,SAAU,SAAQ,OAAO,CAAC,OAAO,EA2B3C,CAAC,kBAAkB,CAAC;CAAG;AAE1B;;;GAGG;AACH,MAAM,OAAO,cAAe,SAAQ,MAAM,CAAC,WAAW,EAAkB,CACtE,gBAAgB,EAChB;IACE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;CAC1C,CACF;IACU,KAAK,GAAG,IAAI,CAAC,MAAM,CAAA;IACnB,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAA;CACvC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAC/B,SAAS,EACT,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,SAAS,CAAA;IACtC,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,cAAc,CAAC,UAAU,CAAA;IAEnD,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QACzB,KAAK,CAAC,CAAC,GAAG,CAAA;uBACO,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC9C,CAAA;QACF,KAAK,CAAC,CAAC,GAAG,CAAA;;;;YAIJ,CAAA;QACN,KAAK,CAAC,CAAC,GAAG,CAAA;;;;;YAKJ,CAAA;QACN,KAAK,CAAC,CAAC,GAAG,CAAA;;;;YAIJ,CAAA;QACN,KAAK,CAAC,CAAC,GAAG,CAAA;mDACmC,CAAA;QAC7C,KAAK,CAAC,CAAC,GAAG,CAAA;;0DAE0C,CAAA;IACtD,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IAE5B,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,EAAE;QACnD,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,IAAI;QACd,UAAU,EAAE,WAAW;KACxB,CAAC,CAAA;IAEF,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC;QAC/B,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;YACrB,MAAM,EAAE,kBAAkB;YAC1B,KAAK,EAAE,MAAM,CAAC,MAAM;SACrB,CAAC;QACF,MAAM,EAAE,KAAK;QACb,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,CAC7B,GAAG,CAAA;;;;6CAIkC,MAAM;yBAC1B,KAAK;;SAErB;KACJ,CAAC,CAAA;IAEF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC;QAClC,EAAE,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO;QAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;YACpB,EAAE,EAAE,OAAO;YACX,IAAI,EAAE,MAAM,CAAC,MAAM;SACpB,CAAC;QACF,QAAQ,CAAC,MAAM;YACb,OAAO,MAAM,CAAC,IAAI,CAAA;QACpB,CAAC;QACD,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAClB,GAAG,CAAA,qCAAqC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE;KACnE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;IAEpC,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;IACvE,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;IAE3E,OAAO,SAAS,CAAC,EAAE,CAAC;QAClB,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,CACjB,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,IAAI,CAC9C,MAAM,CAAC,SAAS,CACd;YACE,WAAW,EAAE,MAAM,CAAC,GAAG;YACvB,oBAAoB,EAAE,MAAM,CAAC,GAAG;SACjC,EACD,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CACxD,CACF;QACH,QAAQ,EAAE,CAAC,EAAE,EAAE,EAAE,CACf,WAAW,CAAC,OAAO,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC,IAAI,CAC5C,MAAM,CAAC,SAAS,CAAC;YACf,WAAW,EAAE,MAAM,CAAC,GAAG;YACvB,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAClE,CAAC,CACH;QACH,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CACf,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,CACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,EACjC,MAAM,CAAC,kBAAkB,EACzB,MAAM,CAAC,SAAS,CAAC;YACf,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,WAAW,EAAE,MAAM,CAAC,GAAG;SACxB,CAAC,CACH;QACH,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAO;YACtD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,OAAc,CAAC,CAAC,IAAI,CACvC,MAAM,CAAC,SAAS,CAAC;gBACf,QAAQ,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;gBACjE,WAAW,EAAE,MAAM,CAAC,GAAG;aACxB,CAAC,CACH,CAAA;QACH,CAAC,CAAC;QACF,QAAQ,EAAE,MAAM,CAAC,IAAI;QACrB,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAC7B,GAAG,CAAA,8BAA8B,MAAM,eAAe,OAAO,EAAE,CAAC,IAAI,CAClE,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAC5D;QACH,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CACrB,GAAG,CAAA,mCAAmC,IAAI,EAAE,CAAC,IAAI,CAC/C,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAC5D;QACH,eAAe,EAAE,CAAC,MAAM,EAAE,EAAE,CAC1B,GAAG,CAAA,sCAAsC,MAAM,EAAE,CAAC,IAAI,CACpD,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAC5D;KACJ,CAAC,CAAA;AACJ,CAAC,CAAC,CACH,CAAA"}
|
package/dist/ChunkRepo.test.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ChunkRepo.test.d.ts","sourceRoot":"","sources":["../src/ChunkRepo.test.ts"],"names":[],"mappings":""}
|
package/dist/ChunkRepo.test.js
DELETED
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
import { mkdtemp, rm } from "node:fs/promises";
|
|
2
|
-
import { tmpdir } from "node:os";
|
|
3
|
-
import { join } from "node:path";
|
|
4
|
-
import * as NodeServices from "@effect/platform-node/NodeServices";
|
|
5
|
-
import { assert, describe, it } from "@effect/vitest";
|
|
6
|
-
import * as Effect from "effect/Effect";
|
|
7
|
-
import * as Layer from "effect/Layer";
|
|
8
|
-
import * as EmbeddingModel from "effect/ai/EmbeddingModel";
|
|
9
|
-
import * as SqlClient from "effect/sql/SqlClient";
|
|
10
|
-
import { Chunk, ChunkRepo, layer, SyncId } from "./ChunkRepo.js";
|
|
11
|
-
import { SqliteLayer } from "./Sqlite.js";
|
|
12
|
-
const withDatabase = Effect.acquireRelease(Effect.promise(() => mkdtemp(join(tmpdir(), "clanka-"))), (directory) => Effect.promise(() => rm(directory, { recursive: true })));
|
|
13
|
-
const sqliteLayer = (database) => SqliteLayer(database).pipe(Layer.provide(NodeServices.layer));
|
|
14
|
-
describe("ChunkRepo", () => {
|
|
15
|
-
it.live("backfills and synchronizes the vec0 index", () => Effect.gen(function* () {
|
|
16
|
-
const directory = yield* withDatabase;
|
|
17
|
-
const database = join(directory, "search.sqlite");
|
|
18
|
-
const existingVector = new Float32Array([0, 1, 0]);
|
|
19
|
-
yield* Effect.gen(function* () {
|
|
20
|
-
const sql = yield* SqlClient.SqlClient;
|
|
21
|
-
yield* sql `INSERT INTO chunks(path, content, hash, vector, syncId)
|
|
22
|
-
VALUES ('existing.ts', 'existing', 'existing', ${existingVector}, 'sync')`;
|
|
23
|
-
}).pipe(Effect.provide(sqliteLayer(database)), Effect.scoped);
|
|
24
|
-
const databaseLayer = sqliteLayer(database);
|
|
25
|
-
const repoLayer = Layer.merge(layer.pipe(Layer.provide(databaseLayer)), databaseLayer).pipe(Layer.provide(Layer.succeed(EmbeddingModel.Dimensions, 3)));
|
|
26
|
-
yield* Effect.gen(function* () {
|
|
27
|
-
const repo = yield* ChunkRepo;
|
|
28
|
-
const sql = yield* SqlClient.SqlClient;
|
|
29
|
-
const backfilled = yield* repo.search({
|
|
30
|
-
vector: existingVector,
|
|
31
|
-
limit: 1,
|
|
32
|
-
});
|
|
33
|
-
assert.strictEqual(backfilled[0]?.path, "existing.ts");
|
|
34
|
-
const inserted = yield* repo.insert(Chunk.insert.make({
|
|
35
|
-
path: "inserted.ts",
|
|
36
|
-
content: "inserted",
|
|
37
|
-
hash: "inserted",
|
|
38
|
-
vector: new Float32Array([1, 0, 0]),
|
|
39
|
-
syncId: SyncId.make("sync"),
|
|
40
|
-
}));
|
|
41
|
-
const insertedResult = yield* repo.search({
|
|
42
|
-
vector: new Float32Array([1, 0, 0]),
|
|
43
|
-
limit: 1,
|
|
44
|
-
});
|
|
45
|
-
assert.strictEqual(insertedResult[0]?.id, inserted.id);
|
|
46
|
-
yield* sql `UPDATE chunks
|
|
47
|
-
SET vector = ${new Float32Array([0, 0, 1])}
|
|
48
|
-
WHERE id = ${inserted.id}`;
|
|
49
|
-
const updatedResult = yield* repo.search({
|
|
50
|
-
vector: new Float32Array([0, 0, 1]),
|
|
51
|
-
limit: 1,
|
|
52
|
-
});
|
|
53
|
-
assert.strictEqual(updatedResult[0]?.id, inserted.id);
|
|
54
|
-
yield* repo.deleteByPath("inserted.ts");
|
|
55
|
-
const deletedResult = yield* repo.search({
|
|
56
|
-
vector: new Float32Array([0, 0, 1]),
|
|
57
|
-
limit: 2,
|
|
58
|
-
});
|
|
59
|
-
assert.deepStrictEqual(deletedResult.map((chunk) => chunk.path), ["existing.ts"]);
|
|
60
|
-
}).pipe(Effect.provide(repoLayer), Effect.scoped);
|
|
61
|
-
}));
|
|
62
|
-
});
|
|
63
|
-
//# sourceMappingURL=ChunkRepo.test.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ChunkRepo.test.js","sourceRoot":"","sources":["../src/ChunkRepo.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,KAAK,YAAY,MAAM,oCAAoC,CAAA;AAClE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,gBAAgB,CAAA;AACrD,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,cAAc,MAAM,0BAA0B,CAAA;AAC1D,OAAO,KAAK,SAAS,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,MAAM,YAAY,GAAG,MAAM,CAAC,cAAc,CACxC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC,EACxD,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CACxE,CAAA;AAED,MAAM,WAAW,GAAG,CAAC,QAAgB,EAAE,EAAE,CACvC,WAAW,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;AAE/D,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,IAAI,CAAC,2CAA2C,EAAE,GAAG,EAAE,CACxD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAClB,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,YAAY,CAAA;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;QACjD,MAAM,cAAc,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAElD,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YACzB,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,SAAS,CAAA;YACtC,KAAK,CAAC,CAAC,GAAG,CAAA;2DACyC,cAAc,WAAW,CAAA;QAC9E,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAE7D,MAAM,aAAa,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAA;QAC3C,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAC3B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,EACxC,aAAa,CACd,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAElE,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YACzB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,SAAS,CAAA;YAC7B,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,SAAS,CAAA;YAEtC,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;gBACpC,MAAM,EAAE,cAAc;gBACtB,KAAK,EAAE,CAAC;aACT,CAAC,CAAA;YACF,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,CAAC,CAAA;YAEtD,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CACjC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,aAAa;gBACnB,OAAO,EAAE,UAAU;gBACnB,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBACnC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;aAC5B,CAAC,CACH,CAAA;YAED,MAAM,cAAc,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;gBACxC,MAAM,EAAE,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBACnC,KAAK,EAAE,CAAC;aACT,CAAC,CAAA;YACF,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAA;YAEtD,KAAK,CAAC,CAAC,GAAG,CAAA;yBACO,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;uBAC7B,QAAQ,CAAC,EAAE,EAAE,CAAA;YAE5B,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;gBACvC,MAAM,EAAE,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBACnC,KAAK,EAAE,CAAC;aACT,CAAC,CAAA;YACF,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAA;YAErD,KAAK,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,CAAA;YAEvC,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;gBACvC,MAAM,EAAE,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBACnC,KAAK,EAAE,CAAC;aACT,CAAC,CAAA;YACF,MAAM,CAAC,eAAe,CACpB,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EACxC,CAAC,aAAa,CAAC,CAChB,CAAA;QACH,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;IACnD,CAAC,CAAC,CACH,CAAA;AACH,CAAC,CAAC,CAAA"}
|