clanka 0.1.22 → 0.2.1

package/dist/SemanticSearch.js

@@ -10,12 +10,18 @@ import { pipe } from "effect/Function";
 import * as EmbeddingModel from "effect/unstable/ai/EmbeddingModel";
 import * as RequestResolver from "effect/RequestResolver";
 import * as Option from "effect/Option";
+import * as Path from "effect/Path";
 import * as ServiceMap from "effect/ServiceMap";
 import * as Fiber from "effect/Fiber";
 import * as Duration from "effect/Duration";
 import * as FiberHandle from "effect/FiberHandle";
 import { SqliteLayer } from "./Sqlite.js";
 import * as Console from "effect/Console";
+const normalizePath = (path) => path.replace(/\\/g, "/");
+const chunkConfig = {
+    chunkSize: 20,
+    chunkOverlap: 5,
+};
 /**
  * @since 1.0.0
  * @category Services
@@ -30,48 +36,68 @@ export const layer = (options) => Layer.effect(SemanticSearch, Effect.gen(functi
     const chunker = yield* CodeChunker.CodeChunker;
     const repo = yield* ChunkRepo.ChunkRepo;
     const embeddings = yield* EmbeddingModel.EmbeddingModel;
+    const pathService = yield* Path.Path;
+    const root = pathService.resolve(options.directory);
     const resolver = embeddings.resolver.pipe(RequestResolver.setDelay(options.embeddingBatchSize ?? Duration.millis(50)), RequestResolver.batchN(options.embeddingBatchSize ?? 500));
+    const concurrency = options.concurrency ?? 2000;
     const indexHandle = yield* FiberHandle.make();
     const console = yield* Console.Console;
-    const index = Effect.gen(function* () {
-        const syncId = ChunkRepo.SyncId.makeUnsafe(crypto.randomUUID());
-        yield* Effect.logInfo("Starting SemanticSearch index");
-        yield* pipe(chunker.chunkCodebase({
-            root: options.directory,
-            chunkSize: 20,
-            chunkOverlap: 5,
-        }), Stream.tap(Effect.fnUntraced(function* (chunk) {
+    const resolveIndexedPath = (path) => {
+        const absolutePath = pathService.resolve(root, path);
+        const relativePath = normalizePath(pathService.relative(root, absolutePath));
+        if (relativePath.length === 0 ||
+            relativePath === ".." ||
+            relativePath.startsWith("../")) {
+            return Option.none();
+        }
+        return Option.some(relativePath);
+    };
+    const processChunk = Effect.fnUntraced(function* (options) {
+        if (options.checkExisting) {
             const id = yield* repo.exists({
-                path: chunk.path,
-                startLine: chunk.startLine,
-                hash: chunk.contentHash,
+                path: options.chunk.path,
+                startLine: options.chunk.startLine,
+                hash: options.chunk.contentHash,
             });
             if (Option.isSome(id)) {
-                yield* repo.setSyncId(id.value, syncId);
+                yield* repo.setSyncId(id.value, options.syncId);
                 return;
             }
-            const result = yield* Effect.request(new EmbeddingModel.EmbeddingRequest({
-                input: `File: ${chunk.path}
-Lines: ${chunk.startLine}-${chunk.endLine}
+        }
+        const result = yield* Effect.request(new EmbeddingModel.EmbeddingRequest({
+            input: `File: ${options.chunk.path}
+Lines: ${options.chunk.startLine}-${options.chunk.endLine}
 
-${chunk.content}`,
-            }), resolver);
-            const vector = new Float32Array(result.vector);
-            yield* repo.insert(ChunkRepo.Chunk.insert.makeUnsafe({
-                path: chunk.path,
-                startLine: chunk.startLine,
-                endLine: chunk.endLine,
-                hash: chunk.contentHash,
-                content: chunk.content,
-                vector,
-                syncId,
-            }));
-        }, Effect.ignore({
-            log: "Warn",
-            message: "Failed to process chunk for embedding",
-        }), (effect, chunk) => Effect.annotateLogs(effect, {
-            chunk: `${chunk.path}/${chunk.startLine}`,
-        })), { concurrency: options.concurrency ?? 2000 }), Stream.runDrain);
+${options.chunk.content}`,
+        }), resolver);
+        const vector = new Float32Array(result.vector);
+        yield* repo.insert(ChunkRepo.Chunk.insert.makeUnsafe({
+            path: options.chunk.path,
+            startLine: options.chunk.startLine,
+            endLine: options.chunk.endLine,
+            hash: options.chunk.contentHash,
+            content: options.chunk.content,
+            vector,
+            syncId: options.syncId,
+        }));
+    }, Effect.ignore({
+        log: "Warn",
+        message: "Failed to process chunk for embedding",
+    }), (effect, options) => Effect.annotateLogs(effect, {
+        chunk: `${options.chunk.path}/${options.chunk.startLine}`,
+    }));
+    const index = Effect.gen(function* () {
+        const syncId = ChunkRepo.SyncId.makeUnsafe(crypto.randomUUID());
+        yield* Effect.logInfo("Starting SemanticSearch index");
+        yield* pipe(chunker.chunkCodebase({
+            root,
+            ...chunkConfig,
+        }), Stream.tap((chunk) => processChunk({
+            chunk,
+            syncId,
+            checkExisting: true,
+        }), { concurrency }), Stream.runDrain);
+        yield* repo.deleteForSyncId(syncId);
         yield* Effect.logInfo("Finished SemanticSearch index");
     }).pipe(Effect.withSpan("SemanticSearch.index"), Effect.withLogSpan("SemanticSearch.index"), Effect.provideService(Console.Console, console));
     const runIndex = FiberHandle.run(indexHandle, index, {
@@ -90,7 +116,36 @@ ${chunk.content}`,
             });
             return results.map((r) => r.format()).join("\n\n");
         }, Effect.orDie),
-        reindex: Effect.asVoid(runIndex),
+        updateFile: Effect.fn("SemanticSearch.updateFile")(function* (path) {
+            yield* Fiber.join(initialIndex);
+            const indexedPath = resolveIndexedPath(path);
+            if (Option.isNone(indexedPath)) {
+                return;
+            }
+            yield* repo.deleteByPath(indexedPath.value);
+            const chunks = yield* chunker.chunkFile({
+                root,
+                path: indexedPath.value,
+                ...chunkConfig,
+            });
+            if (chunks.length === 0) {
+                return;
+            }
+            const syncId = ChunkRepo.SyncId.makeUnsafe(crypto.randomUUID());
+            yield* pipe(Stream.fromArray(chunks), Stream.tap((chunk) => processChunk({
+                chunk,
+                syncId,
+                checkExisting: false,
+            }), { concurrency }), Stream.runDrain);
+        }, Effect.orDie),
+        removeFile: Effect.fn("SemanticSearch.removeFile")(function* (path) {
+            yield* Fiber.join(initialIndex);
+            const indexedPath = resolveIndexedPath(path);
+            if (Option.isNone(indexedPath)) {
+                return;
+            }
+            yield* repo.deleteByPath(indexedPath.value);
+        }, Effect.orDie),
     });
 })).pipe(Layer.provide([
     CodeChunker.layer,
@@ -100,8 +155,16 @@ ${chunk.content}`,
  * @since 1.0.0
  * @category Utils
  */
-export const maybeReindex = Effect.serviceOption(SemanticSearch).pipe(Effect.flatMap(Option.match({
+export const maybeUpdateFile = (path) => Effect.serviceOption(SemanticSearch).pipe(Effect.flatMap(Option.match({
+    onNone: () => Effect.void,
+    onSome: (service) => service.updateFile(path),
+})));
+/**
+ * @since 1.0.0
+ * @category Utils
+ */
+export const maybeRemoveFile = (path) => Effect.serviceOption(SemanticSearch).pipe(Effect.flatMap(Option.match({
     onNone: () => Effect.void,
-    onSome: (service) => service.reindex,
+    onSome: (service) => service.removeFile(path),
 })));
 //# sourceMappingURL=SemanticSearch.js.map

package/dist/SemanticSearch.js.map

@@ -1 +1 @@
-{"version":3,"file":"SemanticSearch.js","sourceRoot":"","sources":["../src/SemanticSearch.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,SAAS,MAAM,gBAAgB,CAAA;AAC3C,OAAO,KAAK,WAAW,MAAM,kBAAkB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AACtC,OAAO,KAAK,cAAc,MAAM,mCAAmC,CAAA;AACnE,OAAO,KAAK,eAAe,MAAM,wBAAwB,CAAA;AACzD,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAEvC,OAAO,KAAK,UAAU,MAAM,mBAAmB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,QAAQ,MAAM,iBAAiB,CAAA;AAC3C,OAAO,KAAK,WAAW,MAAM,oBAAoB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAMzC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AAEzC;;;GAGG;AACH,MAAM,OAAO,cAAe,SAAQ,UAAU,CAAC,OAAO,EASnD,CAAC,sCAAsC,CAAC;CAAG;AAE9C;;;GAGG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,OAMrB,EAUC,EAAE,CACF,KAAK,CAAC,MAAM,CACV,cAAc,EACd,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,WAAW,CAAA;IAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,SAAS,CAAA;IACvC,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,cAAc,CAAC,cAAc,CAAA;IACvD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,CACvC,eAAe,CAAC,QAAQ,CACtB,OAAO,CAAC,kBAAkB,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAClD,EACD,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,IAAI,GAAG,CAAC,CAC1D,CAAA;IACD,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAA;IAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAA;IAEtC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAChC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/D,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAA;QAEtD,KAAK,CAAC,CAAC,IAAI,CACT,OAAO,CAAC,aAAa,CAAC;YACpB,IAAI,EAAE,OAAO,CAAC,SAAS;YACvB,SAAS,EAAE,EAAE;YACb,YAAY,EAAE,CAAC;SAChB,CAAC,EACF,MAAM,CAAC,GAAG,CACR,MAAM,CAAC,UAAU,CACf,QAAQ,CAAC,EAAE,KAAK;YACd,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC5B,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,IAAI,EAAE,KAAK,CAAC,WAAW;aACxB,CAAC,CAAA;YACF,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;gBACtB,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;gBACvC,OAAM;YACR,CAAC;YACD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAClC,IAAI,cAAc,CAAC,gBAAgB,CAAC;gBAClC,KAAK,EAAE,SAAS,KAAK,CAAC,IAAI;SACrC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,OAAO;;EAEvC,KAAK,CAAC,OAAO,EAAE;aACE,CAAC,EACF,QAAQ,CACT,CAAA;YACD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAC9C,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAChB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC;gBAChC,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,IAAI,EAAE,KAAK,CAAC,WAAW;gBACvB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,MAAM;gBACN,MAAM;aACP,CAAC,CACH,CAAA;QACH,CAAC,EACD,MAAM,CAAC,MAAM,CAAC;YACZ,GAAG,EAAE,MAAM;YACX,OAAO,EAAE,uCAAuC;SACjD,CAAC,EACF,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAChB,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;YAC1B,KAAK,EAAE,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,SAAS,EAAE;SAC1C,CAAC,CACL,EACD,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI,EAAE,CAC7C,EACD,MAAM,CAAC,QAAQ,CAChB,CAAA;QAED,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAA;IACxD,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EACvC,MAAM,CAAC,WAAW,CAAC,sBAAsB,CAAC,EAC1C,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAChD,CAAA;IAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE;QACnD,aAAa,EAAE,IAAI;KACpB,CAAC,CAAA;IAEF,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAA;IACpC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAClB,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EACjC,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,UAAU,CAClB,CAAA;IAED,OAAO,cAAc,CAAC,EAAE,CAAC;QACvB,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,uBAAuB,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAO;YAC3D,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/B,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAA;YAC1C,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YACzD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;gBACjC,MAAM,EAAE,IAAI,YAAY,CAAC,MAAM,CAAC;gBAChC,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAA;YACF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACpD,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC;KACjC,CAAC,CAAA;AACJ,CAAC,CAAC,CACH,CAAC,IAAI,CACJ,KAAK,CAAC,OAAO,CAAC;IACZ,WAAW,CAAC,KAAK;IACjB,SAAS,CAAC,KAAK,CAAC,IAAI,CAClB,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,IAAI,uBAAuB,CAAC,CAAC,CACxE;CACF,CAAC,CACH,CAAA;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAwB,MAAM,CAAC,aAAa,CACnE,cAAc,CACf,CAAC,IAAI,CACJ,MAAM,CAAC,OAAO,CACZ,MAAM,CAAC,KAAK,CAAC;IACX,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI;IACzB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO;CACrC,CAAC,CACH,CACF,CAAA"}
+{"version":3,"file":"SemanticSearch.js","sourceRoot":"","sources":["../src/SemanticSearch.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,SAAS,MAAM,gBAAgB,CAAA;AAC3C,OAAO,KAAK,WAAW,MAAM,kBAAkB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AACtC,OAAO,KAAK,cAAc,MAAM,mCAAmC,CAAA;AACnE,OAAO,KAAK,eAAe,MAAM,wBAAwB,CAAA;AACzD,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,IAAI,MAAM,aAAa,CAAA;AACnC,OAAO,KAAK,UAAU,MAAM,mBAAmB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,QAAQ,MAAM,iBAAiB,CAAA;AAC3C,OAAO,KAAK,WAAW,MAAM,oBAAoB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAMzC,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AAEzC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;AAEhE,MAAM,WAAW,GAAG;IAClB,SAAS,EAAE,EAAE;IACb,YAAY,EAAE,CAAC;CACP,CAAA;AAEV;;;GAGG;AACH,MAAM,OAAO,cAAe,SAAQ,UAAU,CAAC,OAAO,EAUnD,CAAC,sCAAsC,CAAC;CAAG;AAE9C;;;GAGG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,OAMrB,EAUC,EAAE,CACF,KAAK,CAAC,MAAM,CACV,cAAc,EACd,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,WAAW,CAAA;IAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,SAAS,CAAC,SAAS,CAAA;IACvC,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,cAAc,CAAC,cAAc,CAAA;IACvD,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAA;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,CACvC,eAAe,CAAC,QAAQ,CACtB,OAAO,CAAC,kBAAkB,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAClD,EACD,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,IAAI,GAAG,CAAC,CAC1D,CAAA;IACD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAA;IAC/C,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAA;IAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,CAAA;IAEtC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAyB,EAAE;QACjE,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACpD,MAAM,YAAY,GAAG,aAAa,CAChC,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,CACzC,CAAA;QACD,IACE,YAAY,CAAC,MAAM,KAAK,CAAC;YACzB,YAAY,KAAK,IAAI;YACrB,YAAY,CAAC,UAAU,CAAC,KAAK,CAAC,EAC9B,CAAC;YACD,OAAO,MAAM,CAAC,IAAI,EAAE,CAAA;QACtB,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAClC,CAAC,CAAA;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CACpC,QAAQ,CAAC,EAAE,OAIV;QACC,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YAC1B,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC5B,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI;gBACxB,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS;gBAClC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,WAAW;aAChC,CAAC,CAAA;YACF,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;gBACtB,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;gBAC/C,OAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAClC,IAAI,cAAc,CAAC,gBAAgB,CAAC;YAClC,KAAK,EAAE,SAAS,OAAO,CAAC,KAAK,CAAC,IAAI;SACvC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO;;EAEvD,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE;SACZ,CAAC,EACF,QAAQ,CACT,CAAA;QACD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QAC9C,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAChB,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC;YAChC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI;YACxB,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS;YAClC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,OAAO;YAC9B,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,WAAW;YAC/B,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,OAAO;YAC9B,MAAM;YACN,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CACH,CAAA;IACH,CAAC,EACD,MAAM,CAAC,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM;QACX,OAAO,EAAE,uCAAuC;KACjD,CAAC,EACF,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAClB,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;QAC1B,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE;KAC1D,CAAC,CACL,CAAA;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAChC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/D,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAA;QAEtD,KAAK,CAAC,CAAC,IAAI,CACT,OAAO,CAAC,aAAa,CAAC;YACpB,IAAI;YACJ,GAAG,WAAW;SACf,CAAC,EACF,MAAM,CAAC,GAAG,CACR,CAAC,KAAK,EAAE,EAAE,CACR,YAAY,CAAC;YACX,KAAK;YACL,MAAM;YACN,aAAa,EAAE,IAAI;SACpB,CAAC,EACJ,EAAE,WAAW,EAAE,CAChB,EACD,MAAM,CAAC,QAAQ,CAChB,CAAA;QAED,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAA;QAEnC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAA;IACxD,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EACvC,MAAM,CAAC,WAAW,CAAC,sBAAsB,CAAC,EAC1C,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAChD,CAAA;IAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,EAAE;QACnD,aAAa,EAAE,IAAI;KACpB,CAAC,CAAA;IAEF,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,QAAQ,CAAA;IACpC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAClB,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EACjC,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,UAAU,CAClB,CAAA;IAED,OAAO,cAAc,CAAC,EAAE,CAAC;QACvB,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,uBAAuB,CAAC,CAAC,QAAQ,CAAC,EAAE,OAAO;YAC3D,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/B,KAAK,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAA;YAC1C,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YACzD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;gBACjC,MAAM,EAAE,IAAI,YAAY,CAAC,MAAM,CAAC;gBAChC,KAAK,EAAE,OAAO,CAAC,KAAK;aACrB,CAAC,CAAA;YACF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACpD,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC,2BAA2B,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI;YAChE,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/B,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YAC5C,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC/B,OAAM;YACR,CAAC;YAED,KAAK,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;YAE3C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC;gBACtC,IAAI;gBACJ,IAAI,EAAE,WAAW,CAAC,KAAK;gBACvB,GAAG,WAAW;aACf,CAAC,CAAA;YACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAM;YACR,CAAC;YAED,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAA;YAE/D,KAAK,CAAC,CAAC,IAAI,CACT,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EACxB,MAAM,CAAC,GAAG,CACR,CAAC,KAAK,EAAE,EAAE,CACR,YAAY,CAAC;gBACX,KAAK;gBACL,MAAM;gBACN,aAAa,EAAE,KAAK;aACrB,CAAC,EACJ,EAAE,WAAW,EAAE,CAChB,EACD,MAAM,CAAC,QAAQ,CAChB,CAAA;QACH,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC,2BAA2B,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI;YAChE,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/B,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YAC5C,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC/B,OAAM;YACR,CAAC;YACD,KAAK,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QAC7C,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;KACjB,CAAC,CAAA;AACJ,CAAC,CAAC,CACH,CAAC,IAAI,CACJ,KAAK,CAAC,OAAO,CAAC;IACZ,WAAW,CAAC,KAAK;IACjB,SAAS,CAAC,KAAK,CAAC,IAAI,CAClB,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,IAAI,uBAAuB,CAAC,CAAC,CACxE;CACF,CAAC,CACH,CAAA;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAuB,EAAE,CACnE,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,IAAI,CACvC,MAAM,CAAC,OAAO,CACZ,MAAM,CAAC,KAAK,CAAC;IACX,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI;IACzB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;CAC9C,CAAC,CACH,CACF,CAAA;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAuB,EAAE,CACnE,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,IAAI,CACvC,MAAM,CAAC,OAAO,CACZ,MAAM,CAAC,KAAK,CAAC;IACX,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI;IACzB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC;CAC9C,CAAC,CACH,CACF,CAAA"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "clanka",
   "type": "module",
-  "version": "0.1.22",
+  "version": "0.2.1",
   "publishConfig": {
     "access": "public"
   },

package/src/Agent.ts

@@ -166,6 +166,7 @@ ${content}
     let finalSummary = Option.none<string>()
 
     const output = yield* Queue.make<Output, AgentFinished | AiError.AiError>()
+    let inputTokens = 0
     let outputTokens = 0
     const prompt = opts.disableHistory ? MutableRef.make(Prompt.empty) : history
 
@@ -376,10 +377,12 @@ ${content}
                     outputTokens += usage.outputTokens.total
                   }
                   if (usage.inputTokens.total !== undefined) {
+                    inputTokens += usage.inputTokens.total
                     maybeSend({
                       agentId,
                       part: new Usage({
-                        inputTokens: usage.inputTokens.total,
+                        contextTokens: usage.inputTokens.total,
+                        inputTokens,
                         outputTokens,
                       }),
                     })
@@ -704,6 +707,7 @@ export class ScriptEnd extends Schema.TaggedClass<ScriptEnd>()(
  * @category Output
  */
 export class Usage extends Schema.TaggedClass<Usage>()("Usage", {
+  contextTokens: Schema.Number,
   inputTokens: Schema.Number,
   outputTokens: Schema.Number,
 }) {}

package/src/AgentTools.ts

@@ -212,7 +212,9 @@ const SearchTool = Toolkit.make(
   Tool.make("search", {
     description: "Semantic code search",
     parameters: Schema.Struct({
-      query: Schema.String,
+      query: Schema.String.annotate({
+        documentation: "Describe what you are looking for",
+      }),
       limit: Schema.optional(Schema.Finite).annotate({
         documentation: "Number of results (defaults to 5, max 10)",
       }),
@@ -295,14 +297,18 @@ export const AgentToolHandlersNoDeps = AgentToolsWithSearch.toLayer(
           recursive: true,
         })
         yield* fs.writeFileString(path, options.content)
-        yield* SemanticSearch.maybeReindex
+        yield* SemanticSearch.maybeUpdateFile(pathService.relative(cwd, path))
       }, Effect.orDie),
       removeFile: Effect.fn("AgentTools.removeFile")(function* (path) {
         yield* Effect.logInfo(`Calling "removeFile"`).pipe(
           Effect.annotateLogs({ path }),
         )
         const cwd = yield* CurrentDirectory
-        return yield* fs.remove(pathService.resolve(cwd, path), { force: true })
+        const absolutePath = pathService.resolve(cwd, path)
+        yield* fs.remove(absolutePath, { force: true })
+        yield* SemanticSearch.maybeRemoveFile(
+          pathService.relative(cwd, absolutePath),
+        )
       }, Effect.orDie),
       renameFile: Effect.fn("AgentTools.renameFile")(function* (options) {
         yield* Effect.logInfo(`Calling "renameFile"`).pipe(
@@ -314,7 +320,9 @@ export const AgentToolHandlersNoDeps = AgentToolsWithSearch.toLayer(
         yield* fs.makeDirectory(pathService.dirname(to), {
           recursive: true,
         })
-        return yield* fs.rename(from, to)
+        yield* fs.rename(from, to)
+        yield* SemanticSearch.maybeRemoveFile(pathService.relative(cwd, from))
+        yield* SemanticSearch.maybeUpdateFile(pathService.relative(cwd, to))
       }, Effect.orDie),
       mkdir: Effect.fn("AgentTools.mkdir")(function* (path) {
         yield* Effect.logInfo(`Calling "mkdir"`).pipe(
@@ -458,8 +466,7 @@ export const AgentToolHandlersNoDeps = AgentToolsWithSearch.toLayer(
             }
         >
         const out = [] as Array<string>
-        const rel = (path: string) =>
-          pathService.relative(cwd, path).replaceAll("\\", "/")
+        const rel = (path: string) => pathService.relative(cwd, path)
         const load = Effect.fn("AgentTools.applyPatch.load")(function* (
           path: string,
           reason: "delete" | "update",
@@ -545,6 +552,7 @@ export const AgentToolHandlersNoDeps = AgentToolsWithSearch.toLayer(
                 recursive: true,
               })
               yield* fs.writeFileString(step.path, step.next)
+              yield* SemanticSearch.maybeUpdateFile(rel(step.path))
               break
             }
             case "move": {
@@ -553,17 +561,18 @@ export const AgentToolHandlersNoDeps = AgentToolsWithSearch.toLayer(
               })
               yield* fs.writeFileString(step.movePath, step.next)
               yield* fs.remove(step.path)
+              yield* SemanticSearch.maybeRemoveFile(rel(step.path))
+              yield* SemanticSearch.maybeUpdateFile(rel(step.movePath))
               break
             }
             case "delete": {
               yield* fs.remove(step.path)
+              yield* SemanticSearch.maybeRemoveFile(rel(step.path))
               break
             }
           }
         }
 
-        yield* SemanticSearch.maybeReindex
-
         return `Success. Updated the following files:\n${out.join("\n")}`
       }, Effect.orDie),
       delegate: Effect.fn("AgentTools.delegate")(function* (prompt) {

package/src/ChunkRepo.ts

@@ -126,6 +126,7 @@ export class ChunkRepo extends ServiceMap.Service<
       chunkId: ChunkId,
       syncId: SyncId,
     ): Effect.Effect<void, ChunkRepoError>
+    deleteByPath(path: string): Effect.Effect<void, ChunkRepoError>
     deleteForSyncId(syncId: SyncId): Effect.Effect<void, ChunkRepoError>
   }
 >()("clanka/ChunkRepo") {}
@@ -223,6 +224,10 @@ export const layer = Layer.effect(
         sql`update chunks set syncId = ${syncId} where id = ${chunkId}`.pipe(
           Effect.mapError((reason) => new ChunkRepoError({ reason })),
         ),
+      deleteByPath: (path) =>
+        sql`delete from chunks where path = ${path}`.pipe(
+          Effect.mapError((reason) => new ChunkRepoError({ reason })),
+        ),
       deleteForSyncId: (syncId) =>
         sql`delete from chunks where syncId != ${syncId}`.pipe(
           Effect.mapError((reason) => new ChunkRepoError({ reason })),

package/src/CodeChunker.ts

@@ -36,6 +36,18 @@ export class CodeChunker extends ServiceMap.Service<
       readonly root: string
       readonly maxFileSize?: string | undefined
     }): Effect.Effect<ReadonlyArray<string>>
+    chunkFile(options: {
+      readonly root: string
+      readonly path: string
+      readonly chunkSize: number
+      readonly chunkOverlap: number
+    }): Effect.Effect<ReadonlyArray<CodeChunk>>
+    chunkFiles(options: {
+      readonly root: string
+      readonly paths: ReadonlyArray<string>
+      readonly chunkSize: number
+      readonly chunkOverlap: number
+    }): Stream.Stream<CodeChunk>
     chunkCodebase(options: {
       readonly root: string
       readonly maxFileSize?: string | undefined
@@ -346,6 +358,46 @@ export const layer: Layer.Layer<
       )
     })
 
+    const chunkFile: CodeChunker["Service"]["chunkFile"] = Effect.fn(
+      "CodeChunker.chunkFile",
+    )(function* (options): Effect.fn.Return<ReadonlyArray<CodeChunk>> {
+      const root = pathService.resolve(options.root)
+      const absolutePath = pathService.resolve(root, options.path)
+      const path = normalizePath(pathService.relative(root, absolutePath))
+
+      if (
+        path.length === 0 ||
+        path === ".." ||
+        path.startsWith("../") ||
+        !isMeaningfulFile(path)
+      ) {
+        return []
+      }
+
+      return yield* pipe(
+        fs.readFileString(absolutePath),
+        Effect.map((content) => chunkFileContent(path, content, options)),
+        Effect.catch(() => Effect.succeed([])),
+      )
+    })
+
+    const chunkFiles: CodeChunker["Service"]["chunkFiles"] = (options) =>
+      Stream.fromArray(options.paths).pipe(
+        Stream.flatMap(
+          (path) =>
+            pipe(
+              chunkFile({
+                root: options.root,
+                path,
+                chunkSize: options.chunkSize,
+                chunkOverlap: options.chunkOverlap,
+              }),
+              Stream.fromArrayEffect,
+            ),
+          { concurrency: 5 },
+        ),
+      )
+
     const chunkCodebase: CodeChunker["Service"]["chunkCodebase"] =
       Effect.fnUntraced(function* (options) {
         const root = pathService.resolve(options.root)
@@ -356,26 +408,18 @@ export const layer: Layer.Layer<
             : { maxFileSize: options.maxFileSize }),
         })
 
-        return Stream.fromArray(files).pipe(
-          Stream.flatMap(
-            (path) => {
-              const absolutePath = pathService.resolve(root, path)
-              return pipe(
-                fs.readFileString(absolutePath),
-                Effect.map((content) =>
-                  chunkFileContent(path, content, options),
-                ),
-                Effect.catch(() => Effect.succeed([])),
-                Stream.fromArrayEffect,
-              )
-            },
-            { concurrency: 5 },
-          ),
-        )
+        return chunkFiles({
+          root,
+          paths: files,
+          chunkSize: options.chunkSize,
+          chunkOverlap: options.chunkOverlap,
+        })
       }, Stream.unwrap)
 
     return CodeChunker.of({
       listFiles,
+      chunkFile,
+      chunkFiles,
       chunkCodebase,
     })
   }),

package/src/OutputFormatter.ts

@@ -78,7 +78,7 @@ ${output.summary}\n\n`
           return `${prefix}${chalk.red(`Error: ${output.error.reason._tag}. Retrying...`)}\n\n${chalk.dim(Cause.pretty(Cause.fail(output.error)))}\n\n`
         }
         case "Usage": {
-          return `${prefix}${chalkInfoHeading(`${infoIcon} Usage:`)} ${numberFormat.format(output.inputTokens)} input / ${numberFormat.format(output.outputTokens)} output\n\n`
+          return `${prefix}${chalkInfoHeading(`${infoIcon} Usage:`)} ${numberFormat.format(output.inputTokens)} context / ${numberFormat.format(output.inputTokens)} input / ${numberFormat.format(output.outputTokens)} output\n\n`
         }
       }
     }),