clanka 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/CodeChunker.d.ts.map +1 -1
- package/dist/CodeChunker.js +17 -2
- package/dist/CodeChunker.js.map +1 -1
- package/dist/CodeChunker.test.js +49 -2
- package/dist/CodeChunker.test.js.map +1 -1
- package/dist/Compaction.d.ts +2 -1
- package/dist/Compaction.d.ts.map +1 -1
- package/dist/Compaction.js +30 -3
- package/dist/Compaction.js.map +1 -1
- package/dist/CompactionTokens.test.d.ts +2 -0
- package/dist/CompactionTokens.test.d.ts.map +1 -0
- package/dist/CompactionTokens.test.js +152 -0
- package/dist/CompactionTokens.test.js.map +1 -0
- package/dist/ExaSearch.d.ts.map +1 -1
- package/dist/ExaSearch.js +13 -5
- package/dist/ExaSearch.js.map +1 -1
- package/dist/Image.d.ts.map +1 -1
- package/dist/Image.js +10 -18
- package/dist/Image.js.map +1 -1
- package/dist/OutputFormatter.js +1 -1
- package/dist/OutputFormatter.js.map +1 -1
- package/dist/SemanticSearch.js +2 -2
- package/dist/SemanticSearch.js.map +1 -1
- package/dist/bin/McpClient-D6vFD-wP.mjs +16337 -0
- package/dist/bin/McpClient-D6vFD-wP.mjs.map +1 -0
- package/dist/bin/Schema-D6y3R-At.mjs +28728 -0
- package/dist/bin/Schema-D6y3R-At.mjs.map +1 -0
- package/dist/bin/SemanticSearch-D9iZ_Mi9.mjs +3372 -0
- package/dist/bin/SemanticSearch-D9iZ_Mi9.mjs.map +1 -0
- package/dist/bin/Undici-ChB4UBDa.mjs +28 -0
- package/dist/bin/Undici-ChB4UBDa.mjs.map +1 -0
- package/dist/bin/cli.mjs +57559 -0
- package/dist/bin/cli.mjs.map +1 -0
- package/dist/cli.js +1 -1
- package/dist/cli.js.map +1 -1
- package/package.json +7 -4
- package/src/CodeChunker.test.ts +62 -2
- package/src/CodeChunker.ts +30 -2
- package/src/Compaction.ts +32 -4
- package/src/CompactionTokens.test.ts +214 -0
- package/src/ExaSearch.ts +18 -5
- package/src/Image.ts +12 -3
- package/src/OutputFormatter.ts +1 -1
- package/src/SemanticSearch.ts +2 -2
- package/src/cli.ts +1 -1
package/dist/cli.js
CHANGED
|
@@ -26,7 +26,7 @@ import * as Config from "effect/Config";
|
|
|
26
26
|
import * as KeyValueStore from "effect/unstable/persistence/KeyValueStore";
|
|
27
27
|
import * as Option from "effect/Option";
|
|
28
28
|
import { OpenAiClient, OpenAiEmbeddingModel } from "@effect/ai-openai";
|
|
29
|
-
import
|
|
29
|
+
import * as DeviceCodeHandler from "./DeviceCodeHandler.js";
|
|
30
30
|
import packageJson from "../package.json" with { type: "json" };
|
|
31
31
|
const version = packageJson.version;
|
|
32
32
|
const providers = ["openai", "copilot", "xai"];
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAA;AACpD,OAAO,KAAK,cAAc,MAAM,sCAAsC,CAAA;AACtE,OAAO,KAAK,OAAO,MAAM,6BAA6B,CAAA;AACtD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAA;AAChD,OAAO,KAAK,YAAY,MAAM,oCAAoC,CAAA;AAClE,OAAO,KAAK,WAAW,MAAM,mCAAmC,CAAA;AAChE,OAAO,KAAK,UAAU,MAAM,kCAAkC,CAAA;AAC9D,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAC/B,OAAO,KAAK,aAAa,MAAM,oBAAoB,CAAA;AACnD,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AACvC,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAC/B,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,KAAK,UAAU,MAAM,iBAAiB,CAAA;AAC7C,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,eAAe,MAAM,sBAAsB,CAAA;AACvD,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AACtC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,IAAI,MAAM,aAAa,CAAA;AACnC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAGvC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,aAAa,MAAM,2CAA2C,CAAA;AAC1E,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAA;AACtE,OAAO,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,MAAM,MAAM,4BAA4B,CAAA;AACpD,OAAO,KAAK,cAAc,MAAM,sCAAsC,CAAA;AACtE,OAAO,KAAK,OAAO,MAAM,6BAA6B,CAAA;AACtD,OAAO,KAAK,IAAI,MAAM,0BAA0B,CAAA;AAChD,OAAO,KAAK,YAAY,MAAM,oCAAoC,CAAA;AAClE,OAAO,KAAK,WAAW,MAAM,mCAAmC,CAAA;AAChE,OAAO,KAAK,UAAU,MAAM,kCAAkC,CAAA;AAC9D,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAC/B,OAAO,KAAK,aAAa,MAAM,oBAAoB,CAAA;AACnD,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AACvC,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAC/B,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,KAAK,UAAU,MAAM,iBAAiB,CAAA;AAC7C,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,eAAe,MAAM,sBAAsB,CAAA;AACvD,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AACtC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,IAAI,MAAM,aAAa,CAAA;AACnC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAGvC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,aAAa,MAAM,2CAA2C,CAAA;AAC1E,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAA;AACtE,OAAO,KAAK,iBAAiB,MAAM,wBAAwB,CAAA;AAC3D,OAAO,WAAW,MAAM,iBAAiB,CAAC,OAAO,IAAI,EAAE,MAAM,EAAE,CAAA;AAE/D,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAA;AAInC,MAAM,SAAS,GAA4B,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;AAEvE,MAAM,iBAAiB,GAAG,CACxB,KAIC,EACD,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAA;AAExD,MAAM,UAAU,GAAG,CAAC,QAAkB,EAAE,KAAa,EAAE,MAAc,EAAE,EAAE,CACvE,QAAQ,KAAK,QAAQ;IACnB,CAAC,CAAC,iBAAiB,CACf,KAAK,CAAC,cAAc,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,MAAa,EAAE,EAAE,CAAC,CACtE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC1C,CAAC,CAAC,QAAQ,KAAK,KAAK;QAClB,CAAC,CAAC,iBAAiB,CACf,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,MAAa,EAAE,EAAE,CAAC,CAC3D,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACxC,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CACrE,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CACnC,CAAA;AAET,MAAM,YAAY,GAAG,CAAC,OAAe,EAAE,EAAE;IACvC,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC7D,OAAO,QAAQ,KAAK,SAAS;QAC3B,SAAS,CAAC,QAAQ,CAAC,QAAoB,CAAC;QACxC,KAAK,KAAK,SAAS;QACnB,MAAM,KAAK,SAAS;QACpB,IAAI,CAAC,MAAM,KAAK,CAAC;QACjB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,QAAoB,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAChE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;AACnB,CAAC,CAAA;AAED,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAChC,MAAM,CAAC,KAAK,CACV,MAAM,CAAC,UAAU,CACf,CAAC,OAAe,EAAE,EAAE,CAClB,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,kBAAkB,OAAO,yCAAyC,CACrE,CACF,CACF,CAAA;AAED,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC,IAAI,CACxD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EACnB,IAAI,CAAC,kBAAkB,CACrB,MAAM,CAAC,MAAM,CAAC;IACZ,OAAO,EAAE,mBAAmB;IAC5B,OAAO,EAAE;QACP;YACE,KAAK,EAAE,QAAQ;YACf,KAAK,EAAE,QAAQ;YACf,QAAQ,EAAE,IAAI;SACf;QACD;YACE,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE,SAAS;SACjB;QACD;YACE,KAAK,EAAE,KAAK;YACZ,KAAK,EAAE,KAAK;SACb;KACF;CACF,CAAC,CACH,CACF,CAAA;AAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;AAE3E,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,IAAI,CAC5C,IAAI,CAAC,eAAe,CAClB,kEAAkE,CACnE,EACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EACnB,IAAI,CAAC,QAAQ,CACd,CAAA;AAED,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CACvC,IAAI,CAAC,eAAe,CAAC,uCAAuC,CAAC,EAC7D,IAAI,CAAC,QAAQ,CACd,CAAA;AAED,6DAA6D;AAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,IAAI,CAChD,IAAI,CAAC,eAAe,CAClB,mKAAmK,CACpK,EACD,IAAI,CAAC,kBAAkB,CACrB,MAAM,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CACnE,CACF,CAAA;AAED,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CACtB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAA;IAE7B,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC,IAAI,CACrE,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CACjB,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,CAChC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CACjD,CACF,CACF,CAAA;IACD,OAAO,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAA;AACvE,CAAC,CAAC,CACH,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;AAEzC,MAAM,MAAM,GAAG,CAAC,SAAiB,EAAE,EAAE,CACnC,KAAK,CAAC,MAAM,CACV,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAC1D,MAAM,CAAC,MAAM,CACd,CAAA;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAA;QACrD,OAAO,KAAK,CAAC,KAAK,CAAA;IACpB,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAA;IAE7B,MAAM,cAAc,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAC1C,GAAG,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CACpC,CAAA;IAED,OAAO,cAAc,CAAC,KAAK,CAAC;QAC1B,SAAS,EAAE,OAAO,CAAC,GAAG,EAAE;QACxB,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC;KAChD,CAAC,CAAC,IAAI,CACL,KAAK,CAAC,OAAO,CACX,oBAAoB,CAAC,KAAK,CAAC,wBAAwB,EAAE;QACnD,UAAU,EAAE,IAAI;KACjB,CAAC,CACH,EACD,KAAK,CAAC,OAAO,CACX,YAAY,CAAC,KAAK,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC,KAAK;KACrB,CAAC,CACH,CACF,CAAA;AACH,CAAC,CAAC,CACH,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,KAAK,EAAE,cAAc,CAAC,WAAW,CAAC,CAAC,CAAC,CAAA;AAEzE,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CACxC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EACnB,IAAI,CAAC,eAAe,CAClB,8EAA8E,CAC/E,EACD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EACxB,IAAI,CAAC,WAAW,CAAC,2BAA2B,CAAC,CAC9C,CAAA;AAED,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC,IAAI,CACnE,OAAO,CAAC,eAAe,CACrB,yDAAyD,CAC1D,EACD,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAChC,GAAG,CAAC,QAAQ,CAAC;IACX,OAAO;IACP,YAAY,EAAE,KAAK;IACnB,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE,CACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CACb,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAC7D;IACH,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CACrB,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAChE,UAAU,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CACpC;CACJ,CAAC,CACH,EACD,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,CACjC,KAAK,CAAC,QAAQ,CACZ,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,EAClC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,EACvC,iBAAiB,CAAC,QAAQ,EAC1B,UAAU,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAC3D,CACF,CACF,CAAA;AAED,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE;IACrB,QAAQ;IACR,KAAK;IACL,QAAQ;IACR,MAAM;IACN,UAAU;CACX,CAAC,CAAC,IAAI,CACL,OAAO,CAAC,WAAW,CACjB,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,EAC3B,QAAQ,EACR,KAAK,EAAE,QAAQ,EACf,QAAQ,EACR,MAAM,EAAE,oBAAoB,GAC7B;IACC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAA;IAChC,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE;QAClD,MAAM,EAAE,MAAM,CAAC,OAAO;QACtB,MAAM,EAAE,GAAG,EAAE,CACX,MAAM,CAAC,MAAM,CAAC;YACZ,OAAO,EAAE,eAAe;YACxB,OAAO,EACL,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,oBAAoB;YAC7D,QAAQ,CAAC,KAAK;gBACZ,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC;oBAClC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;oBACvB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YAClC,CAAC;SACF,CAAC;KACL,CAAC,CAAA;IACF,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,GAAG,CAAqB,CAAA;IACvE,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CAAA;IAEpD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;QAChC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,KAAK,CAAA;QAEhC,IAAI,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,EAAE,CAAC;YACxC,OAAO,KAAK,CAAC,CAAC,IAAI,CAChB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,oBAAoB,CAAC,KAAK,EAAE,CAAC,EAClD,MAAM,CAAC,MAAM,EACb,eAAe,CAAC,MAAM,EAAE,EACxB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAC3B,CAAA;QACH,CAAC;QAED,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;gBAClC,OAAO,EAAE,GAAG;aACb,CAAC,CAAA;YAEF,KAAK,CAAC,CAAC,IAAI,CACT,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EACtB,MAAM,CAAC,MAAM,EACb,eAAe,CAAC,MAAM,CAAC,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAAC,EAChD,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAC3B,CAAA;YAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACjB,CAAC;IACH,CAAC,CAAC,CAAC,IAAI,CACL,MAAM,CAAC,OAAO,CAAC;QACb,KAAK,CAAC,UAAU,CAAC;YACf,SAAS,EAAE,OAAO,CAAC,GAAG,EAAE;SACzB,CAAC,CAAC,IAAI,CACL,KAAK,CAAC,OAAO,CACX,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE;YACrB,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK;YACzB,MAAM,EAAE,MAAM;SACf,CAAC,CACH,CACF;QACD,KAAK;KACN,CAAC,CACH,CAAA;AACH,CAAC,CAAC,CACH,EACD,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,EAAE,CACzC,KAAK,CAAC,QAAQ,CACZ,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EACnD,UAAU,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAC3D,CACF,EACD,OAAO,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,EAC9B,OAAO,CAAC,GAAG,CAAC;IACV,OAAO;CACR,CAAC,EACF,MAAM,CAAC,OAAO,CAAC;IACb,YAAY,CAAC,KAAK;IAClB,GAAG;IACH,cAAc,CAAC,WAAW;IAC1B,UAAU,CAAC,2BAA2B;IACtC,iBAAiB,CAAC,YAAY;CAC/B,CAAC,EACF,WAAW,CAAC,OAAO,CACpB,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "clanka",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.7.
|
|
4
|
+
"version": "0.7.2",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
7
7
|
},
|
|
8
8
|
"bin": {
|
|
9
|
-
"clanka": "dist/cli.
|
|
9
|
+
"clanka": "dist/bin/cli.mjs"
|
|
10
10
|
},
|
|
11
11
|
"exports": {
|
|
12
12
|
".": "./dist/index.js",
|
|
@@ -34,7 +34,9 @@
|
|
|
34
34
|
"tree-sitter": "^0.21.1",
|
|
35
35
|
"tree-sitter-javascript": "^0.23.1",
|
|
36
36
|
"tree-sitter-typescript": "^0.23.2",
|
|
37
|
-
"turndown": "^7.2.4"
|
|
37
|
+
"turndown": "^7.2.4",
|
|
38
|
+
"undici": "^8.10.2",
|
|
39
|
+
"ws": "^8.21.3"
|
|
38
40
|
},
|
|
39
41
|
"peerDependencies": {
|
|
40
42
|
"@effect/ai-openai": ">=4.0.0-rc.115 <4.0.1",
|
|
@@ -59,6 +61,7 @@
|
|
|
59
61
|
"lint-staged": "^17.5.1",
|
|
60
62
|
"oxlint": "^1.83.0",
|
|
61
63
|
"prettier": "^3.9.7",
|
|
64
|
+
"tsdown": "^0.23.0",
|
|
62
65
|
"typescript": "^7.0.2",
|
|
63
66
|
"vitest": "^5.0.1"
|
|
64
67
|
},
|
|
@@ -74,6 +77,6 @@
|
|
|
74
77
|
"scripts": {
|
|
75
78
|
"check": "oxlint -c .oxlintrc.json --fix && pnpm tsc -b tsconfig.json",
|
|
76
79
|
"test": "vitest run",
|
|
77
|
-
"build": "tsc -b tsconfig.json"
|
|
80
|
+
"build": "tsc -b tsconfig.json && tsdown"
|
|
78
81
|
}
|
|
79
82
|
}
|
package/src/CodeChunker.test.ts
CHANGED
|
@@ -177,14 +177,22 @@ describe("chunkFileContent", () => {
|
|
|
177
177
|
chunkMaxCharacters: 21,
|
|
178
178
|
})
|
|
179
179
|
|
|
180
|
-
expect(chunks).
|
|
180
|
+
expect(chunks.every((chunk) => chunk.content.length <= 21)).toBe(true)
|
|
181
|
+
expect(chunks).toHaveLength(5)
|
|
181
182
|
expect(chunks).toMatchObject([
|
|
182
183
|
{
|
|
183
184
|
startLine: 1,
|
|
184
185
|
endLine: 1,
|
|
185
186
|
name: "demo",
|
|
186
187
|
type: "function",
|
|
187
|
-
content: "export function demo(
|
|
188
|
+
content: "export function demo(",
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
startLine: 1,
|
|
192
|
+
endLine: 1,
|
|
193
|
+
name: "demo",
|
|
194
|
+
type: "function",
|
|
195
|
+
content: ") {",
|
|
188
196
|
},
|
|
189
197
|
{
|
|
190
198
|
startLine: 2,
|
|
@@ -239,6 +247,58 @@ describe("chunkFileContent", () => {
|
|
|
239
247
|
])
|
|
240
248
|
})
|
|
241
249
|
|
|
250
|
+
it.each(["src/long.ts", "docs/long.txt"])(
|
|
251
|
+
"splits oversized lines in %s without losing content or line metadata",
|
|
252
|
+
(path) => {
|
|
253
|
+
const longLine = ` const value = "${"x".repeat(25_000)}"`
|
|
254
|
+
const content = [
|
|
255
|
+
"function demo() {",
|
|
256
|
+
longLine,
|
|
257
|
+
" return value",
|
|
258
|
+
"}",
|
|
259
|
+
].join("\n")
|
|
260
|
+
const chunks = chunkFileContent(path, content, {
|
|
261
|
+
chunkSize: 30,
|
|
262
|
+
chunkOverlap: 1,
|
|
263
|
+
chunkMaxCharacters: 10_000,
|
|
264
|
+
})
|
|
265
|
+
|
|
266
|
+
expect(chunks.length).toBeGreaterThan(0)
|
|
267
|
+
expect(chunks.every((chunk) => chunk.content.length <= 10_000)).toBe(true)
|
|
268
|
+
const pieces = chunks.filter((chunk) => chunk.startLine === 2)
|
|
269
|
+
expect(pieces).toHaveLength(3)
|
|
270
|
+
expect(pieces.map((chunk) => chunk.content).join("")).toBe(longLine)
|
|
271
|
+
for (const piece of pieces) {
|
|
272
|
+
expect(piece).toMatchObject({ path, startLine: 2, endLine: 2 })
|
|
273
|
+
if (path.endsWith(".ts")) {
|
|
274
|
+
expect(piece).toMatchObject({ name: "demo", type: "function" })
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
expect(
|
|
278
|
+
chunks.some((chunk) => chunk.content.includes("return value")),
|
|
279
|
+
).toBe(true)
|
|
280
|
+
},
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
it("preserves a line exactly at the character limit", () => {
|
|
284
|
+
const line = "x".repeat(10_000)
|
|
285
|
+
const chunks = chunkFileContent(
|
|
286
|
+
"docs/limit.txt",
|
|
287
|
+
`before\n${line}\nafter`,
|
|
288
|
+
{
|
|
289
|
+
chunkSize: 30,
|
|
290
|
+
chunkOverlap: 0,
|
|
291
|
+
chunkMaxCharacters: 10_000,
|
|
292
|
+
},
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
expect(chunks.map((chunk) => chunk.content)).toEqual([
|
|
296
|
+
"before",
|
|
297
|
+
line,
|
|
298
|
+
"after",
|
|
299
|
+
])
|
|
300
|
+
})
|
|
301
|
+
|
|
242
302
|
it("keeps only the first oversized class segment when methods are present", () => {
|
|
243
303
|
const content = [
|
|
244
304
|
"class Example {",
|
package/src/CodeChunker.ts
CHANGED
|
@@ -821,6 +821,31 @@ const chunkWithLineWindows = (
|
|
|
821
821
|
return out
|
|
822
822
|
}
|
|
823
823
|
|
|
824
|
+
const splitOversizedChunks = (
|
|
825
|
+
chunks: ReadonlyArray<CodeChunk>,
|
|
826
|
+
maxCharacters: number,
|
|
827
|
+
): ReadonlyArray<CodeChunk> =>
|
|
828
|
+
chunks.flatMap((chunk) => {
|
|
829
|
+
if (chunk.content.length <= maxCharacters) {
|
|
830
|
+
return [chunk]
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
// Line-based splitting can only exceed the limit for a single long line.
|
|
834
|
+
// Each piece retains that line number and the original AST metadata.
|
|
835
|
+
const pieces = [] as Array<CodeChunk>
|
|
836
|
+
for (
|
|
837
|
+
let offset = 0;
|
|
838
|
+
offset < chunk.content.length;
|
|
839
|
+
offset += maxCharacters
|
|
840
|
+
) {
|
|
841
|
+
pieces.push({
|
|
842
|
+
...chunk,
|
|
843
|
+
content: chunk.content.slice(offset, offset + maxCharacters),
|
|
844
|
+
})
|
|
845
|
+
}
|
|
846
|
+
return pieces
|
|
847
|
+
})
|
|
848
|
+
|
|
824
849
|
/**
|
|
825
850
|
* @since 1.0.0
|
|
826
851
|
* @category Constructors
|
|
@@ -858,11 +883,14 @@ export const chunkFileContent = (
|
|
|
858
883
|
settings,
|
|
859
884
|
)
|
|
860
885
|
if (astChunks.length > 0) {
|
|
861
|
-
return astChunks
|
|
886
|
+
return splitOversizedChunks(astChunks, settings.chunkMaxCharacters)
|
|
862
887
|
}
|
|
863
888
|
}
|
|
864
889
|
|
|
865
|
-
return
|
|
890
|
+
return splitOversizedChunks(
|
|
891
|
+
chunkWithLineWindows(normalizedPath, lines, settings),
|
|
892
|
+
settings.chunkMaxCharacters,
|
|
893
|
+
)
|
|
866
894
|
}
|
|
867
895
|
|
|
868
896
|
/**
|
package/src/Compaction.ts
CHANGED
|
@@ -205,16 +205,44 @@ export const capOutput = (
|
|
|
205
205
|
// Token accounting
|
|
206
206
|
// =============================================================================
|
|
207
207
|
|
|
208
|
+
// Provider image costs vary with dimensions and detail settings. Use a fixed
|
|
209
|
+
// fallback until real usage arrives, never the encoded payload size.
|
|
210
|
+
const imageTokenEstimate = 1_500
|
|
211
|
+
|
|
212
|
+
const estimateMessageChars = (message: Prompt.Message): number => {
|
|
213
|
+
if (message.role !== "user" && message.role !== "assistant") {
|
|
214
|
+
return JSON.stringify(message).length
|
|
215
|
+
}
|
|
216
|
+
let imageTokens = 0
|
|
217
|
+
const content = message.content.map((part) => {
|
|
218
|
+
if (part.type !== "file" || !part.mediaType.startsWith("image/")) {
|
|
219
|
+
return part
|
|
220
|
+
}
|
|
221
|
+
imageTokens += imageTokenEstimate
|
|
222
|
+
// Replace the part before stringifying so binary data, base64 and URLs
|
|
223
|
+
// are all opaque, and the original prompt stays untouched.
|
|
224
|
+
return { type: part.type, mediaType: part.mediaType }
|
|
225
|
+
})
|
|
226
|
+
return JSON.stringify({ ...message, content }).length + imageTokens * 4
|
|
227
|
+
}
|
|
228
|
+
|
|
208
229
|
/**
|
|
209
|
-
* Cheap token estimate
|
|
230
|
+
* Cheap token estimate: JSON length / 4, with a fixed cost per image instead
|
|
231
|
+
* of serializing image payloads. Used when no
|
|
210
232
|
* `contextTokens` usage has been observed yet (first turn, ACP session load)
|
|
211
233
|
* and for sizing the kept tail.
|
|
212
234
|
*
|
|
213
235
|
* @since 1.0.0
|
|
214
236
|
* @category Tokens
|
|
215
237
|
*/
|
|
216
|
-
export const estimateTokens = (prompt: Prompt.Prompt): number =>
|
|
217
|
-
|
|
238
|
+
export const estimateTokens = (prompt: Prompt.Prompt): number => {
|
|
239
|
+
// Include the array brackets and commas, rounding only once as before.
|
|
240
|
+
let chars = 2 + Math.max(0, prompt.content.length - 1)
|
|
241
|
+
for (const message of prompt.content) {
|
|
242
|
+
chars += estimateMessageChars(message)
|
|
243
|
+
}
|
|
244
|
+
return Math.ceil(chars / 4)
|
|
245
|
+
}
|
|
218
246
|
|
|
219
247
|
/**
|
|
220
248
|
* Token estimate for a single message. Same measure as `estimateTokens`.
|
|
@@ -223,7 +251,7 @@ export const estimateTokens = (prompt: Prompt.Prompt): number =>
|
|
|
223
251
|
* @category Tokens
|
|
224
252
|
*/
|
|
225
253
|
export const estimateMessageTokens = (message: Prompt.Message): number =>
|
|
226
|
-
Math.ceil(
|
|
254
|
+
Math.ceil(estimateMessageChars(message) / 4)
|
|
227
255
|
|
|
228
256
|
/**
|
|
229
257
|
* Whether the threshold trigger fires for the next model call.
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { assert, describe, it } from "@effect/vitest"
|
|
2
|
+
import * as Encoding from "effect/Encoding"
|
|
3
|
+
import * as Option from "effect/Option"
|
|
4
|
+
import * as Schema from "effect/Schema"
|
|
5
|
+
import * as Prompt from "effect/unstable/ai/Prompt"
|
|
6
|
+
import * as Compaction from "./Compaction.ts"
|
|
7
|
+
|
|
8
|
+
const text = (value: string) => Prompt.makePart("text", { text: value })
|
|
9
|
+
const user = (value: string) =>
|
|
10
|
+
Prompt.makeMessage("user", { content: [text(value)] })
|
|
11
|
+
|
|
12
|
+
// Estimation treats image payloads as opaque; no image decoder is involved.
|
|
13
|
+
const imageMessage = (size: number) =>
|
|
14
|
+
Prompt.makeMessage("user", {
|
|
15
|
+
content: [
|
|
16
|
+
text("Describe this screenshot"),
|
|
17
|
+
Prompt.makePart("file", {
|
|
18
|
+
mediaType: "image/png",
|
|
19
|
+
fileName: "screenshot.png",
|
|
20
|
+
data: new Uint8Array(size).fill(137),
|
|
21
|
+
}),
|
|
22
|
+
],
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
// ACP persists history through this same JSON codec in its schema store.
|
|
26
|
+
const historyCodec = Schema.toCodecJson(Prompt.Prompt)
|
|
27
|
+
const reload = (prompt: Prompt.Prompt) =>
|
|
28
|
+
Schema.decodeSync(historyCodec)(Schema.encodeSync(historyCodec)(prompt))
|
|
29
|
+
|
|
30
|
+
const toolCall = Prompt.makeMessage("assistant", {
|
|
31
|
+
content: [
|
|
32
|
+
Prompt.makePart("reasoning", { text: "Inspect the screenshot first." }),
|
|
33
|
+
Prompt.makePart("tool-call", {
|
|
34
|
+
id: "c1",
|
|
35
|
+
name: "execute",
|
|
36
|
+
params: { script: 'console.log(await readFile({ path: "README.md" }))' },
|
|
37
|
+
providerExecuted: false,
|
|
38
|
+
}),
|
|
39
|
+
],
|
|
40
|
+
})
|
|
41
|
+
const toolResult = (result: string) =>
|
|
42
|
+
Prompt.makeMessage("tool", {
|
|
43
|
+
content: [
|
|
44
|
+
Prompt.makePart("tool-result", {
|
|
45
|
+
id: "c1",
|
|
46
|
+
name: "execute",
|
|
47
|
+
result,
|
|
48
|
+
isFailure: false,
|
|
49
|
+
providerExecuted: false,
|
|
50
|
+
}),
|
|
51
|
+
],
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
describe("Compaction image token estimates", () => {
|
|
55
|
+
for (const representation of ["binary", "reloaded base64"] as const) {
|
|
56
|
+
const history = (size: number) => {
|
|
57
|
+
const prompt = Prompt.fromMessages([imageMessage(size)])
|
|
58
|
+
return representation === "binary" ? prompt : reload(prompt)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
it(`${representation}: prompt estimate does not scale with image bytes`, () => {
|
|
62
|
+
assert.strictEqual(
|
|
63
|
+
Compaction.estimateTokens(history(1024 * 1024)),
|
|
64
|
+
Compaction.estimateTokens(history(32)),
|
|
65
|
+
)
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
it(`${representation}: message estimate does not scale with image bytes`, () => {
|
|
69
|
+
assert.strictEqual(
|
|
70
|
+
Compaction.estimateMessageTokens(history(1024 * 1024).content[0]!),
|
|
71
|
+
Compaction.estimateMessageTokens(history(32).content[0]!),
|
|
72
|
+
)
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
it(`${representation}: one screenshot does not trigger compaction without usage`, () => {
|
|
76
|
+
assert.isFalse(
|
|
77
|
+
Compaction.shouldCompact({
|
|
78
|
+
prompt: history(1024 * 1024),
|
|
79
|
+
contextTokens: undefined,
|
|
80
|
+
config: Compaction.defaultConfig,
|
|
81
|
+
}),
|
|
82
|
+
)
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
it(`${representation}: a recent screenshot does not force an unnecessary cut`, () => {
|
|
86
|
+
const prompt = Prompt.fromMessages([
|
|
87
|
+
user("Earlier question"),
|
|
88
|
+
...history(1024 * 1024).content,
|
|
89
|
+
user("What changed?"),
|
|
90
|
+
])
|
|
91
|
+
assert.isTrue(
|
|
92
|
+
Option.isNone(
|
|
93
|
+
Compaction.split(prompt, Compaction.defaultConfig.keepRecentTokens),
|
|
94
|
+
),
|
|
95
|
+
)
|
|
96
|
+
})
|
|
97
|
+
|
|
98
|
+
it(`${representation}: a screenshot does not cause reasoning or tool output to be trimmed`, () => {
|
|
99
|
+
const kept = [
|
|
100
|
+
toolCall,
|
|
101
|
+
toolResult("output ".repeat(1000)),
|
|
102
|
+
...history(1024 * 1024).content,
|
|
103
|
+
]
|
|
104
|
+
assert.isTrue(
|
|
105
|
+
Compaction.trimKept(kept, Compaction.defaultConfig.keepRecentTokens) ===
|
|
106
|
+
kept,
|
|
107
|
+
)
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
it(`${representation}: still counts text alongside an image`, () => {
|
|
111
|
+
const original = history(32).content[0]!
|
|
112
|
+
assert.strictEqual(original.role, "user")
|
|
113
|
+
if (original.role !== "user") return
|
|
114
|
+
const extended = Prompt.makeMessage("user", {
|
|
115
|
+
content: [
|
|
116
|
+
text("Describe this screenshot" + "x".repeat(4000)),
|
|
117
|
+
original.content[1]!,
|
|
118
|
+
],
|
|
119
|
+
})
|
|
120
|
+
assert.strictEqual(
|
|
121
|
+
Compaction.estimateMessageTokens(extended) -
|
|
122
|
+
Compaction.estimateMessageTokens(original),
|
|
123
|
+
1000,
|
|
124
|
+
)
|
|
125
|
+
assert.strictEqual(
|
|
126
|
+
Compaction.estimateTokens(Prompt.fromMessages([extended])) -
|
|
127
|
+
Compaction.estimateTokens(Prompt.fromMessages([original])),
|
|
128
|
+
1000,
|
|
129
|
+
)
|
|
130
|
+
})
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
it("preserves estimates across the binary-to-base64 persistence round trip", () => {
|
|
134
|
+
const prompt = Prompt.fromMessages([imageMessage(1024)])
|
|
135
|
+
const restored = reload(prompt)
|
|
136
|
+
const message = restored.content[0]!
|
|
137
|
+
assert.strictEqual(message.role, "user")
|
|
138
|
+
if (message.role !== "user") return
|
|
139
|
+
const image = message.content[1]!
|
|
140
|
+
assert.strictEqual(image.type, "file")
|
|
141
|
+
if (image.type !== "file") return
|
|
142
|
+
assert.strictEqual(
|
|
143
|
+
image.data,
|
|
144
|
+
Encoding.encodeBase64(new Uint8Array(1024).fill(137)),
|
|
145
|
+
)
|
|
146
|
+
assert.strictEqual(
|
|
147
|
+
Compaction.estimateTokens(restored),
|
|
148
|
+
Compaction.estimateTokens(prompt),
|
|
149
|
+
)
|
|
150
|
+
assert.strictEqual(
|
|
151
|
+
Compaction.estimateMessageTokens(message),
|
|
152
|
+
Compaction.estimateMessageTokens(prompt.content[0]!),
|
|
153
|
+
)
|
|
154
|
+
})
|
|
155
|
+
|
|
156
|
+
it("charges for each image rather than ignoring all file parts", () => {
|
|
157
|
+
const one = imageMessage(32)
|
|
158
|
+
const two = Prompt.makeMessage("user", {
|
|
159
|
+
content: [...one.content, one.content[1]!],
|
|
160
|
+
})
|
|
161
|
+
for (const message of [one, two]) {
|
|
162
|
+
assert.isAbove(
|
|
163
|
+
Compaction.estimateMessageTokens(message),
|
|
164
|
+
Compaction.estimateMessageTokens(user("Describe this screenshot")),
|
|
165
|
+
)
|
|
166
|
+
}
|
|
167
|
+
assert.isAbove(
|
|
168
|
+
Compaction.estimateMessageTokens(two),
|
|
169
|
+
Compaction.estimateMessageTokens(one),
|
|
170
|
+
)
|
|
171
|
+
assert.isAbove(
|
|
172
|
+
Compaction.estimateTokens(Prompt.fromMessages([two])),
|
|
173
|
+
Compaction.estimateTokens(Prompt.fromMessages([one])),
|
|
174
|
+
)
|
|
175
|
+
})
|
|
176
|
+
})
|
|
177
|
+
|
|
178
|
+
describe("Compaction non-image token estimates", () => {
|
|
179
|
+
const messages = [
|
|
180
|
+
Prompt.makeMessage("system", { content: "You are a test." }),
|
|
181
|
+
user('Text with "quotes", a newline\n and Unicode: 日本語'),
|
|
182
|
+
toolCall,
|
|
183
|
+
toolResult("output ".repeat(1000)),
|
|
184
|
+
]
|
|
185
|
+
|
|
186
|
+
it("preserves JSON-length accounting for text, reasoning, tool calls and results", () => {
|
|
187
|
+
for (const message of messages) {
|
|
188
|
+
assert.strictEqual(
|
|
189
|
+
Compaction.estimateMessageTokens(message),
|
|
190
|
+
Math.ceil(JSON.stringify(message).length / 4),
|
|
191
|
+
)
|
|
192
|
+
}
|
|
193
|
+
const prompt = Prompt.fromMessages(messages)
|
|
194
|
+
assert.strictEqual(
|
|
195
|
+
Compaction.estimateTokens(prompt),
|
|
196
|
+
Math.ceil(JSON.stringify(prompt.content).length / 4),
|
|
197
|
+
)
|
|
198
|
+
})
|
|
199
|
+
|
|
200
|
+
it("still triggers compaction for large text and tool output", () => {
|
|
201
|
+
for (const message of [
|
|
202
|
+
user("x".repeat(900_000)),
|
|
203
|
+
toolResult("x".repeat(900_000)),
|
|
204
|
+
]) {
|
|
205
|
+
assert.isTrue(
|
|
206
|
+
Compaction.shouldCompact({
|
|
207
|
+
prompt: Prompt.fromMessages([message]),
|
|
208
|
+
contextTokens: undefined,
|
|
209
|
+
config: Compaction.defaultConfig,
|
|
210
|
+
}),
|
|
211
|
+
)
|
|
212
|
+
}
|
|
213
|
+
})
|
|
214
|
+
})
|
package/src/ExaSearch.ts
CHANGED
|
@@ -8,7 +8,6 @@ import { pipe } from "effect/Function"
|
|
|
8
8
|
import * as Layer from "effect/Layer"
|
|
9
9
|
import * as Schema from "effect/Schema"
|
|
10
10
|
import * as Context from "effect/Context"
|
|
11
|
-
import * as McpClient from "./McpClient.ts"
|
|
12
11
|
|
|
13
12
|
/**
|
|
14
13
|
* @since 1.0.0
|
|
@@ -55,10 +54,24 @@ export class ExaError extends Schema.TaggedError<ExaError>()("ExaError", {
|
|
|
55
54
|
export const layer = Layer.effect(
|
|
56
55
|
ExaSearch,
|
|
57
56
|
Effect.gen(function* () {
|
|
58
|
-
const
|
|
57
|
+
const scope = yield* Effect.scope
|
|
58
|
+
const getClient = yield* Effect.cachedWithTTL(
|
|
59
|
+
Effect.gen(function* () {
|
|
60
|
+
const McpClient = yield* Effect.tryPromise(
|
|
61
|
+
() => import("./McpClient.ts"),
|
|
62
|
+
)
|
|
63
|
+
const context = yield* Layer.buildWithScope(McpClient.layer, scope)
|
|
64
|
+
return Context.get(context, McpClient.McpClient)
|
|
65
|
+
}),
|
|
66
|
+
(exit) => (Exit.isSuccess(exit) ? Duration.infinity : Duration.zero),
|
|
67
|
+
)
|
|
59
68
|
|
|
60
69
|
const connect = yield* Effect.cachedWithTTL(
|
|
61
|
-
|
|
70
|
+
Effect.gen(function* () {
|
|
71
|
+
const client = yield* getClient
|
|
72
|
+
yield* client.connect({ url: "https://mcp.exa.ai/mcp" })
|
|
73
|
+
return client
|
|
74
|
+
}),
|
|
62
75
|
(exit) => (Exit.isSuccess(exit) ? Duration.infinity : Duration.zero),
|
|
63
76
|
)
|
|
64
77
|
|
|
@@ -69,7 +82,7 @@ export const layer = Layer.effect(
|
|
|
69
82
|
return ExaSearch.of({
|
|
70
83
|
search: Effect.fn("ExaSearch.search")(
|
|
71
84
|
function* (options) {
|
|
72
|
-
yield* connect
|
|
85
|
+
const client = yield* connect
|
|
73
86
|
const results = yield* pipe(
|
|
74
87
|
client.toolCall({
|
|
75
88
|
name: "web_search_exa",
|
|
@@ -86,4 +99,4 @@ export const layer = Layer.effect(
|
|
|
86
99
|
),
|
|
87
100
|
})
|
|
88
101
|
}),
|
|
89
|
-
)
|
|
102
|
+
)
|
package/src/Image.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
*
|
|
13
13
|
* @since 1.0.0
|
|
14
14
|
*/
|
|
15
|
-
import * as Photon from "@silvia-odwyer/photon-node"
|
|
15
|
+
import type * as Photon from "@silvia-odwyer/photon-node"
|
|
16
16
|
import * as Effect from "effect/Effect"
|
|
17
17
|
import * as Encoding from "effect/Encoding"
|
|
18
18
|
import type * as FileSystem from "effect/FileSystem"
|
|
@@ -451,11 +451,12 @@ const fits = (bytes: Uint8Array, limits: Limits) =>
|
|
|
451
451
|
base64Length(bytes.length) <= limits.maxBytes
|
|
452
452
|
|
|
453
453
|
const decode = (
|
|
454
|
+
photon: typeof Photon,
|
|
454
455
|
data: Uint8Array,
|
|
455
456
|
): Effect.Effect<Photon.PhotonImage, ImageError> =>
|
|
456
457
|
Effect.try({
|
|
457
458
|
try: () => {
|
|
458
|
-
const image =
|
|
459
|
+
const image = photon.PhotonImage.new_from_byteslice(data)
|
|
459
460
|
// Photon returns an empty image rather than throwing for some inputs.
|
|
460
461
|
if (image.get_width() === 0 || image.get_height() === 0) {
|
|
461
462
|
image.free()
|
|
@@ -527,7 +528,15 @@ export const prepare = Effect.fnUntraced(function* (options: {
|
|
|
527
528
|
message: `Image exceeds the ${maxInputPixels} pixel input limit`,
|
|
528
529
|
})
|
|
529
530
|
}
|
|
530
|
-
const
|
|
531
|
+
const Photon = yield* Effect.tryPromise({
|
|
532
|
+
try: () => import("@silvia-odwyer/photon-node"),
|
|
533
|
+
catch: (cause) =>
|
|
534
|
+
new ImageError({
|
|
535
|
+
reason: "Decode",
|
|
536
|
+
message: `Could not load image decoder: ${cause instanceof Error ? cause.message : String(cause)}`,
|
|
537
|
+
}),
|
|
538
|
+
})
|
|
539
|
+
const image = yield* decode(Photon, options.data)
|
|
531
540
|
let current = image
|
|
532
541
|
try {
|
|
533
542
|
const width = image.get_width()
|
package/src/OutputFormatter.ts
CHANGED
|
@@ -12,7 +12,7 @@ import chalk from "chalk"
|
|
|
12
12
|
import type * as Prompt from "effect/unstable/ai/Prompt"
|
|
13
13
|
import * as Cause from "effect/Cause"
|
|
14
14
|
import { identity } from "effect/Function"
|
|
15
|
-
import
|
|
15
|
+
import * as Predicate from "effect/Predicate"
|
|
16
16
|
|
|
17
17
|
/**
|
|
18
18
|
* @since 1.0.0
|
package/src/SemanticSearch.ts
CHANGED
|
@@ -37,7 +37,7 @@ const resolveChunkConfig = (options: {
|
|
|
37
37
|
}) => ({
|
|
38
38
|
chunkSize: 30,
|
|
39
39
|
chunkOverlap: 0,
|
|
40
|
-
chunkMaxCharacters: options.chunkMaxCharacters ??
|
|
40
|
+
chunkMaxCharacters: options.chunkMaxCharacters ?? 15_000,
|
|
41
41
|
})
|
|
42
42
|
|
|
43
43
|
export const makeEmbeddingResolver = (
|
|
@@ -51,7 +51,7 @@ export const makeEmbeddingResolver = (
|
|
|
51
51
|
RequestResolver.setDelay(
|
|
52
52
|
options.embeddingRequestDelay ?? Duration.millis(50),
|
|
53
53
|
),
|
|
54
|
-
RequestResolver.batchN(options.embeddingBatchSize ??
|
|
54
|
+
RequestResolver.batchN(options.embeddingBatchSize ?? 32),
|
|
55
55
|
)
|
|
56
56
|
|
|
57
57
|
export const chunkEmbeddingInput = (chunk: CodeChunker.CodeChunk): string => {
|
package/src/cli.ts
CHANGED
|
@@ -28,7 +28,7 @@ import * as Config from "effect/Config"
|
|
|
28
28
|
import * as KeyValueStore from "effect/unstable/persistence/KeyValueStore"
|
|
29
29
|
import * as Option from "effect/Option"
|
|
30
30
|
import { OpenAiClient, OpenAiEmbeddingModel } from "@effect/ai-openai"
|
|
31
|
-
import
|
|
31
|
+
import * as DeviceCodeHandler from "./DeviceCodeHandler.ts"
|
|
32
32
|
import packageJson from "../package.json" with { type: "json" }
|
|
33
33
|
|
|
34
34
|
const version = packageJson.version
|