@sourcepress/ai 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omc/state/last-tool-error.json +7 -0
- package/.turbo/turbo-build.log +4 -0
- package/.turbo/turbo-test.log +24 -0
- package/LICENSE +21 -0
- package/dist/__tests__/budget.test.d.ts +2 -0
- package/dist/__tests__/budget.test.d.ts.map +1 -0
- package/dist/__tests__/budget.test.js +96 -0
- package/dist/__tests__/budget.test.js.map +1 -0
- package/dist/__tests__/classify.test.d.ts +2 -0
- package/dist/__tests__/classify.test.d.ts.map +1 -0
- package/dist/__tests__/classify.test.js +72 -0
- package/dist/__tests__/classify.test.js.map +1 -0
- package/dist/__tests__/eval-runner.test.d.ts +2 -0
- package/dist/__tests__/eval-runner.test.d.ts.map +1 -0
- package/dist/__tests__/eval-runner.test.js +171 -0
- package/dist/__tests__/eval-runner.test.js.map +1 -0
- package/dist/__tests__/extract.test.d.ts +2 -0
- package/dist/__tests__/extract.test.d.ts.map +1 -0
- package/dist/__tests__/extract.test.js +79 -0
- package/dist/__tests__/extract.test.js.map +1 -0
- package/dist/__tests__/find-gaps.test.d.ts +2 -0
- package/dist/__tests__/find-gaps.test.d.ts.map +1 -0
- package/dist/__tests__/find-gaps.test.js +82 -0
- package/dist/__tests__/find-gaps.test.js.map +1 -0
- package/dist/__tests__/generate.test.d.ts +2 -0
- package/dist/__tests__/generate.test.d.ts.map +1 -0
- package/dist/__tests__/generate.test.js +68 -0
- package/dist/__tests__/generate.test.js.map +1 -0
- package/dist/__tests__/improve-prompt.test.d.ts +2 -0
- package/dist/__tests__/improve-prompt.test.d.ts.map +1 -0
- package/dist/__tests__/improve-prompt.test.js +32 -0
- package/dist/__tests__/improve-prompt.test.js.map +1 -0
- package/dist/__tests__/intent-impact.test.d.ts +2 -0
- package/dist/__tests__/intent-impact.test.d.ts.map +1 -0
- package/dist/__tests__/intent-impact.test.js +51 -0
- package/dist/__tests__/intent-impact.test.js.map +1 -0
- package/dist/__tests__/judge.test.d.ts +2 -0
- package/dist/__tests__/judge.test.d.ts.map +1 -0
- package/dist/__tests__/judge.test.js +61 -0
- package/dist/__tests__/judge.test.js.map +1 -0
- package/dist/__tests__/score.test.d.ts +2 -0
- package/dist/__tests__/score.test.d.ts.map +1 -0
- package/dist/__tests__/score.test.js +50 -0
- package/dist/__tests__/score.test.js.map +1 -0
- package/dist/__tests__/staleness.test.d.ts +2 -0
- package/dist/__tests__/staleness.test.d.ts.map +1 -0
- package/dist/__tests__/staleness.test.js +66 -0
- package/dist/__tests__/staleness.test.js.map +1 -0
- package/dist/budget.d.ts +13 -0
- package/dist/budget.d.ts.map +1 -0
- package/dist/budget.js +40 -0
- package/dist/budget.js.map +1 -0
- package/dist/eval/runner.d.ts +34 -0
- package/dist/eval/runner.d.ts.map +1 -0
- package/dist/eval/runner.js +128 -0
- package/dist/eval/runner.js.map +1 -0
- package/dist/functions/classify.d.ts +5 -0
- package/dist/functions/classify.d.ts.map +1 -0
- package/dist/functions/classify.js +43 -0
- package/dist/functions/classify.js.map +1 -0
- package/dist/functions/extract.d.ts +5 -0
- package/dist/functions/extract.d.ts.map +1 -0
- package/dist/functions/extract.js +57 -0
- package/dist/functions/extract.js.map +1 -0
- package/dist/functions/find-gaps.d.ts +5 -0
- package/dist/functions/find-gaps.d.ts.map +1 -0
- package/dist/functions/find-gaps.js +51 -0
- package/dist/functions/find-gaps.js.map +1 -0
- package/dist/functions/generate.d.ts +5 -0
- package/dist/functions/generate.d.ts.map +1 -0
- package/dist/functions/generate.js +39 -0
- package/dist/functions/generate.js.map +1 -0
- package/dist/functions/improve-prompt.d.ts +5 -0
- package/dist/functions/improve-prompt.d.ts.map +1 -0
- package/dist/functions/improve-prompt.js +38 -0
- package/dist/functions/improve-prompt.js.map +1 -0
- package/dist/functions/index.d.ts +11 -0
- package/dist/functions/index.d.ts.map +1 -0
- package/dist/functions/index.js +11 -0
- package/dist/functions/index.js.map +1 -0
- package/dist/functions/intent-impact.d.ts +5 -0
- package/dist/functions/intent-impact.d.ts.map +1 -0
- package/dist/functions/intent-impact.js +45 -0
- package/dist/functions/intent-impact.js.map +1 -0
- package/dist/functions/judge.d.ts +5 -0
- package/dist/functions/judge.d.ts.map +1 -0
- package/dist/functions/judge.js +32 -0
- package/dist/functions/judge.js.map +1 -0
- package/dist/functions/model-factory.d.ts +4 -0
- package/dist/functions/model-factory.d.ts.map +1 -0
- package/dist/functions/model-factory.js +52 -0
- package/dist/functions/model-factory.js.map +1 -0
- package/dist/functions/score.d.ts +5 -0
- package/dist/functions/score.d.ts.map +1 -0
- package/dist/functions/score.js +47 -0
- package/dist/functions/score.js.map +1 -0
- package/dist/functions/staleness.d.ts +5 -0
- package/dist/functions/staleness.d.ts.map +1 -0
- package/dist/functions/staleness.js +45 -0
- package/dist/functions/staleness.js.map +1 -0
- package/dist/functions/usage.d.ts +8 -0
- package/dist/functions/usage.d.ts.map +1 -0
- package/dist/functions/usage.js +13 -0
- package/dist/functions/usage.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -0
- package/dist/provider.d.ts +10 -0
- package/dist/provider.d.ts.map +1 -0
- package/dist/provider.js +32 -0
- package/dist/provider.js.map +1 -0
- package/dist/types.d.ts +207 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +41 -0
- package/src/__tests__/budget.test.ts +103 -0
- package/src/__tests__/classify.test.ts +90 -0
- package/src/__tests__/eval-runner.test.ts +199 -0
- package/src/__tests__/extract.test.ts +92 -0
- package/src/__tests__/find-gaps.test.ts +93 -0
- package/src/__tests__/generate.test.ts +92 -0
- package/src/__tests__/improve-prompt.test.ts +42 -0
- package/src/__tests__/intent-impact.test.ts +62 -0
- package/src/__tests__/judge.test.ts +78 -0
- package/src/__tests__/score.test.ts +61 -0
- package/src/__tests__/staleness.test.ts +77 -0
- package/src/budget.ts +47 -0
- package/src/eval/runner.ts +163 -0
- package/src/functions/classify.ts +54 -0
- package/src/functions/extract.ts +72 -0
- package/src/functions/find-gaps.ts +65 -0
- package/src/functions/generate.ts +51 -0
- package/src/functions/improve-prompt.ts +48 -0
- package/src/functions/index.ts +10 -0
- package/src/functions/intent-impact.ts +56 -0
- package/src/functions/judge.ts +41 -0
- package/src/functions/model-factory.ts +60 -0
- package/src/functions/score.ts +56 -0
- package/src/functions/staleness.ts +54 -0
- package/src/functions/usage.ts +25 -0
- package/src/index.ts +47 -0
- package/src/provider.ts +41 -0
- package/src/types.ts +225 -0
- package/tsconfig.json +5 -0
- package/vitest.config.ts +2 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"tool_name": "Bash",
|
|
3
|
+
"tool_input_preview": "{\"command\":\"find packages/ai/src -name \\\"*.ts\\\" -not -path \\\"*__tests__*\\\" -not -path \\\"*node_modules*\\\" | while read f; do\\n sed -i '' 's/usage?.promptTokens/usage?.inputTokens/g; s/usage\\\\.promptTo...",
|
|
4
|
+
"error": "Exit code 1\nfind: packages/ai/src: No such file or directory\nFixed\n(eval):cd:4: no such file or directory: packages/ai",
|
|
5
|
+
"timestamp": "2026-04-04T21:54:17.369Z",
|
|
6
|
+
"retry_count": 2
|
|
7
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
|
|
2
|
+
> @sourcepress/ai@0.1.0 test /Users/fabianvontiedemann/Developer/org/sourcepress/packages/ai
|
|
3
|
+
> vitest run
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
RUN v3.2.4 /Users/fabianvontiedemann/Developer/org/sourcepress/packages/ai
|
|
7
|
+
|
|
8
|
+
✓ src/__tests__/classify.test.ts (4 tests) 12ms
|
|
9
|
+
✓ src/__tests__/score.test.ts (2 tests) 27ms
|
|
10
|
+
✓ src/__tests__/staleness.test.ts (2 tests) 15ms
|
|
11
|
+
✓ src/__tests__/budget.test.ts (7 tests) 2ms
|
|
12
|
+
✓ src/__tests__/improve-prompt.test.ts (1 test) 16ms
|
|
13
|
+
✓ src/__tests__/find-gaps.test.ts (2 tests) 7ms
|
|
14
|
+
✓ src/__tests__/intent-impact.test.ts (2 tests) 15ms
|
|
15
|
+
✓ src/__tests__/judge.test.ts (3 tests) 7ms
|
|
16
|
+
✓ src/__tests__/extract.test.ts (3 tests) 61ms
|
|
17
|
+
✓ src/__tests__/generate.test.ts (3 tests) 72ms
|
|
18
|
+
✓ src/__tests__/eval-runner.test.ts (5 tests) 162ms
|
|
19
|
+
|
|
20
|
+
Test Files 11 passed (11)
|
|
21
|
+
Tests 34 passed (34)
|
|
22
|
+
Start at 13:45:48
|
|
23
|
+
Duration 1.43s (transform 377ms, setup 0ms, collect 1.14s, tests 397ms, environment 1ms, prepare 1.84s)
|
|
24
|
+
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 SourcePress Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"budget.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/budget.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { beforeEach, describe, expect, it } from "vitest";
|
|
2
|
+
import { BudgetTracker } from "../budget.js";
|
|
3
|
+
describe("BudgetTracker", () => {
|
|
4
|
+
let tracker;
|
|
5
|
+
beforeEach(() => {
|
|
6
|
+
tracker = new BudgetTracker({ daily_limit_usd: 5.0, warn_at_usd: 3.0 });
|
|
7
|
+
});
|
|
8
|
+
it("starts with zero spent", () => {
|
|
9
|
+
const status = tracker.getStatus();
|
|
10
|
+
expect(status.spent_today_usd).toBe(0);
|
|
11
|
+
expect(status.remaining_usd).toBe(5.0);
|
|
12
|
+
expect(status.is_over_limit).toBe(false);
|
|
13
|
+
expect(status.is_warned).toBe(false);
|
|
14
|
+
});
|
|
15
|
+
it("tracks token usage", () => {
|
|
16
|
+
tracker.record({
|
|
17
|
+
input_tokens: 1000,
|
|
18
|
+
output_tokens: 500,
|
|
19
|
+
estimated_cost_usd: 0.01,
|
|
20
|
+
function_name: "classify",
|
|
21
|
+
timestamp: new Date().toISOString(),
|
|
22
|
+
});
|
|
23
|
+
const status = tracker.getStatus();
|
|
24
|
+
expect(status.spent_today_usd).toBe(0.01);
|
|
25
|
+
expect(status.remaining_usd).toBe(4.99);
|
|
26
|
+
});
|
|
27
|
+
it("warns when approaching limit", () => {
|
|
28
|
+
tracker.record({
|
|
29
|
+
input_tokens: 100000,
|
|
30
|
+
output_tokens: 50000,
|
|
31
|
+
estimated_cost_usd: 3.5,
|
|
32
|
+
function_name: "extract",
|
|
33
|
+
timestamp: new Date().toISOString(),
|
|
34
|
+
});
|
|
35
|
+
const status = tracker.getStatus();
|
|
36
|
+
expect(status.is_warned).toBe(true);
|
|
37
|
+
expect(status.is_over_limit).toBe(false);
|
|
38
|
+
});
|
|
39
|
+
it("detects over-limit", () => {
|
|
40
|
+
tracker.record({
|
|
41
|
+
input_tokens: 200000,
|
|
42
|
+
output_tokens: 100000,
|
|
43
|
+
estimated_cost_usd: 5.5,
|
|
44
|
+
function_name: "judge",
|
|
45
|
+
timestamp: new Date().toISOString(),
|
|
46
|
+
});
|
|
47
|
+
expect(tracker.getStatus().is_over_limit).toBe(true);
|
|
48
|
+
});
|
|
49
|
+
it("can check if operation is allowed", () => {
|
|
50
|
+
expect(tracker.canSpend(4.0)).toBe(true);
|
|
51
|
+
expect(tracker.canSpend(6.0)).toBe(false);
|
|
52
|
+
tracker.record({
|
|
53
|
+
input_tokens: 100000,
|
|
54
|
+
output_tokens: 50000,
|
|
55
|
+
estimated_cost_usd: 4.0,
|
|
56
|
+
function_name: "score",
|
|
57
|
+
timestamp: new Date().toISOString(),
|
|
58
|
+
});
|
|
59
|
+
expect(tracker.canSpend(0.5)).toBe(true);
|
|
60
|
+
expect(tracker.canSpend(2.0)).toBe(false);
|
|
61
|
+
});
|
|
62
|
+
it("returns usage history", () => {
|
|
63
|
+
tracker.record({
|
|
64
|
+
input_tokens: 1000,
|
|
65
|
+
output_tokens: 500,
|
|
66
|
+
estimated_cost_usd: 0.01,
|
|
67
|
+
function_name: "classify",
|
|
68
|
+
timestamp: new Date().toISOString(),
|
|
69
|
+
});
|
|
70
|
+
tracker.record({
|
|
71
|
+
input_tokens: 2000,
|
|
72
|
+
output_tokens: 1000,
|
|
73
|
+
estimated_cost_usd: 0.02,
|
|
74
|
+
function_name: "extract",
|
|
75
|
+
timestamp: new Date().toISOString(),
|
|
76
|
+
});
|
|
77
|
+
const history = tracker.getHistory();
|
|
78
|
+
expect(history).toHaveLength(2);
|
|
79
|
+
expect(history[0].function_name).toBe("classify");
|
|
80
|
+
expect(history[1].function_name).toBe("extract");
|
|
81
|
+
});
|
|
82
|
+
it("resets daily spending", () => {
|
|
83
|
+
tracker.record({
|
|
84
|
+
input_tokens: 100000,
|
|
85
|
+
output_tokens: 50000,
|
|
86
|
+
estimated_cost_usd: 3.0,
|
|
87
|
+
function_name: "judge",
|
|
88
|
+
timestamp: new Date().toISOString(),
|
|
89
|
+
});
|
|
90
|
+
tracker.resetDaily();
|
|
91
|
+
const status = tracker.getStatus();
|
|
92
|
+
expect(status.spent_today_usd).toBe(0);
|
|
93
|
+
expect(status.remaining_usd).toBe(5.0);
|
|
94
|
+
});
|
|
95
|
+
});
|
|
96
|
+
//# sourceMappingURL=budget.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"budget.test.js","sourceRoot":"","sources":["../../src/__tests__/budget.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,IAAI,OAAsB,CAAC;IAC3B,UAAU,CAAC,GAAG,EAAE;QACf,OAAO,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAC7B,OAAO,CAAC,MAAM,CAAC;YACd,YAAY,EAAE,IAAI;YAClB,aAAa,EAAE,GAAG;YAClB,kBAAkB,EAAE,IAAI;YACxB,aAAa,EAAE,UAAU;YACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACnC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;QACvC,OAAO,CAAC,MAAM,CAAC;YACd,YAAY,EAAE,MAAM;YACpB,aAAa,EAAE,KAAK;YACpB,kBAAkB,EAAE,GAAG;YACvB,aAAa,EAAE,SAAS;YACxB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACnC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAC7B,OAAO,CAAC,MAAM,CAAC;YACd,YAAY,EAAE,MAAM;YACpB,aAAa,EAAE,MAAM;YACrB,kBAAkB,EAAE,GAAG;YACvB,aAAa,EAAE,OAAO;YACtB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACnC,CAAC,CAAC;QACH,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC5C,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,OAAO,CAAC,MAAM,CAAC;YACd,YAAY,EAAE,MAAM;YACpB,aAAa,EAAE,KAAK;YACpB,kBAAkB,EAAE,GAAG;YACvB,aAAa,EAAE,OAAO;YACtB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACnC,CAAC,CAAC;QACH,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAChC,OAAO,CAAC,MAAM,CAAC;YACd,YAAY,EAAE,IAAI;YAClB,aAAa,EAAE,GAAG;YAClB,kBAAkB,EAAE,IAAI;YACxB,aAAa,EAAE,UAAU;YACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACnC,CAAC,CAAC;QACH,OAAO,CAAC,MAAM,CAAC;YACd,YAAY,EAAE,IAAI;YAClB,aAAa,EAAE,IAAI;YACnB,kBAAkB,EAAE,IAAI;YACxB,aAAa,EAAE,SAAS;YACxB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACnC,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAClD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAChC,OAAO,CAAC,MAAM,CAAC;YACd,YAAY,EAAE,MAAM;YACpB,aAAa,EAAE,KAAK;YACpB,kBAAkB,EAAE,GAAG;YACvB,aAAa,EAAE,OAAO;YACtB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACnC,CAAC,CAAC;QACH,OAAO,CAAC,UAAU,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"classify.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/classify.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from "vitest";
|
|
2
|
+
import { BudgetTracker } from "../budget.js";
|
|
3
|
+
import { classify } from "../functions/classify.js";
|
|
4
|
+
vi.mock("ai", () => ({ generateObject: vi.fn() }));
|
|
5
|
+
import { generateObject } from "ai";
|
|
6
|
+
const mockProvider = {
|
|
7
|
+
provider: "anthropic",
|
|
8
|
+
model: "claude-sonnet-4-5-20250514",
|
|
9
|
+
};
|
|
10
|
+
describe("classify", () => {
|
|
11
|
+
it("classifies structured text correctly", async () => {
|
|
12
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
13
|
+
object: {
|
|
14
|
+
quality: "structured",
|
|
15
|
+
quality_score: 8,
|
|
16
|
+
type: "project-notes",
|
|
17
|
+
reasoning: "Well-organized document with clear sections",
|
|
18
|
+
},
|
|
19
|
+
usage: { promptTokens: 500, completionTokens: 100 },
|
|
20
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
21
|
+
});
|
|
22
|
+
const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
|
|
23
|
+
const result = await classify({ text: "Meeting with Acme Corp 2026-04-01. Discussed Next.js migration timeline." }, mockProvider, budget);
|
|
24
|
+
expect(result.quality).toBe("structured");
|
|
25
|
+
expect(result.quality_score).toBe(8);
|
|
26
|
+
expect(result.type).toBe("project-notes");
|
|
27
|
+
expect(result.usage.function_name).toBe("classify");
|
|
28
|
+
});
|
|
29
|
+
it("classifies draft-quality text", async () => {
|
|
30
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
31
|
+
object: { quality: "draft", quality_score: 5, type: "brainstorm", reasoning: "Rough notes" },
|
|
32
|
+
usage: { promptTokens: 300, completionTokens: 80 },
|
|
33
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
34
|
+
});
|
|
35
|
+
const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
|
|
36
|
+
const result = await classify({ text: "maybe we should try... react? or vue?" }, mockProvider, budget);
|
|
37
|
+
expect(result.quality).toBe("draft");
|
|
38
|
+
expect(result.quality_score).toBeLessThanOrEqual(6);
|
|
39
|
+
});
|
|
40
|
+
it("respects available_types constraint", async () => {
|
|
41
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
42
|
+
object: {
|
|
43
|
+
quality: "structured",
|
|
44
|
+
quality_score: 7,
|
|
45
|
+
type: "meeting-notes",
|
|
46
|
+
reasoning: "Typed as meeting-notes",
|
|
47
|
+
},
|
|
48
|
+
usage: { promptTokens: 400, completionTokens: 90 },
|
|
49
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
50
|
+
});
|
|
51
|
+
const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
|
|
52
|
+
const result = await classify({
|
|
53
|
+
text: "Team standup 2026-04-04",
|
|
54
|
+
available_types: ["meeting-notes", "project-notes", "transcript"],
|
|
55
|
+
}, mockProvider, budget);
|
|
56
|
+
expect(result.type).toBe("meeting-notes");
|
|
57
|
+
});
|
|
58
|
+
it("records budget usage", async () => {
|
|
59
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
60
|
+
object: { quality: "structured", quality_score: 8, type: "notes", reasoning: "OK" },
|
|
61
|
+
usage: { promptTokens: 500, completionTokens: 100 },
|
|
62
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
63
|
+
});
|
|
64
|
+
const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
|
|
65
|
+
await classify({ text: "test" }, mockProvider, budget);
|
|
66
|
+
const history = budget.getHistory();
|
|
67
|
+
expect(history).toHaveLength(1);
|
|
68
|
+
expect(history[0].function_name).toBe("classify");
|
|
69
|
+
expect(history[0].input_tokens).toBe(500);
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
//# sourceMappingURL=classify.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"classify.test.js","sourceRoot":"","sources":["../../src/__tests__/classify.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AAGpD,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AAEpC,MAAM,YAAY,GAAqB;IACtC,QAAQ,EAAE,WAAW;IACrB,KAAK,EAAE,4BAA4B;CACnC,CAAC;AAEF,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACrD,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE;gBACP,OAAO,EAAE,YAAY;gBACrB,aAAa,EAAE,CAAC;gBAChB,IAAI,EAAE,eAAe;gBACrB,SAAS,EAAE,6CAA6C;aACxD;YACD,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACnD,yFAAyF;SAClF,CAAC,CAAC;QACV,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAC5B,EAAE,IAAI,EAAE,0EAA0E,EAAE,EACpF,YAAY,EACZ,MAAM,CACN,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC9C,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,aAAa,EAAE;YAC5F,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE;YAClD,yFAAyF;SAClF,CAAC,CAAC;QACV,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAC5B,EAAE,IAAI,EAAE,uCAAuC,EAAE,EACjD,YAAY,EACZ,MAAM,CACN,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;QACpD,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE;gBACP,OAAO,EAAE,YAAY;gBACrB,aAAa,EAAE,CAAC;gBAChB,IAAI,EAAE,eAAe;gBACrB,SAAS,EAAE,wBAAwB;aACnC;YACD,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE;YAClD,yFAAyF;SAClF,CAAC,CAAC;QACV,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAC5B;YACC,IAAI,EAAE,yBAAyB;YAC/B,eAAe,EAAE,CAAC,eAAe,EAAE,eAAe,EAAE,YAAY,CAAC;SACjE,EACD,YAAY,EACZ,MAAM,CACN,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QACrC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE;YACnF,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACnD,yFAAyF;SAClF,CAAC,CAAC;QACV,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3D,MAAM,QAAQ,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAClD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3C,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-runner.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/eval-runner.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from "vitest";
|
|
2
|
+
import { BudgetTracker } from "../budget.js";
|
|
3
|
+
import { EvalRunner } from "../eval/runner.js";
|
|
4
|
+
vi.mock("ai", () => ({ generateObject: vi.fn() }));
|
|
5
|
+
import { generateObject } from "ai";
|
|
6
|
+
const mockProvider = {
|
|
7
|
+
provider: "anthropic",
|
|
8
|
+
model: "claude-sonnet-4-5-20250514",
|
|
9
|
+
};
|
|
10
|
+
const baseConfig = {
|
|
11
|
+
content_type: "case-study",
|
|
12
|
+
knowledge_context: "Acme Corp migrated 12 microservices to Next.js in 8 weeks. Load time improved by 40%.",
|
|
13
|
+
gold_standard: "# Perfect Case Study\n\nDetailed migration with 12 services, 8 weeks, 40% improvement.",
|
|
14
|
+
judge_prompt: "Score this content 0-100 against the gold standard.",
|
|
15
|
+
generation_prompt: "Write a compelling case study.",
|
|
16
|
+
threshold: 70,
|
|
17
|
+
max_iterations: 3,
|
|
18
|
+
};
|
|
19
|
+
describe("EvalRunner", () => {
|
|
20
|
+
it("keeps content when score meets threshold on first iteration", async () => {
|
|
21
|
+
// Generate
|
|
22
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
23
|
+
object: {
|
|
24
|
+
frontmatter: { title: "Acme Corp Case Study" },
|
|
25
|
+
body: "# Acme Corp\n\nWe migrated 12 services in 8 weeks, improving load time by 40%.",
|
|
26
|
+
},
|
|
27
|
+
usage: { promptTokens: 3000, completionTokens: 500 },
|
|
28
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
29
|
+
});
|
|
30
|
+
// Judge — score above threshold
|
|
31
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
32
|
+
object: { score: 85, reasoning: "Excellent match. Specific metrics included." },
|
|
33
|
+
usage: { promptTokens: 2000, completionTokens: 200 },
|
|
34
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
35
|
+
});
|
|
36
|
+
const budget = new BudgetTracker({ daily_limit_usd: 10.0 });
|
|
37
|
+
const runner = new EvalRunner(mockProvider, budget);
|
|
38
|
+
const result = await runner.run(baseConfig);
|
|
39
|
+
expect(result.final_status).toBe("keep");
|
|
40
|
+
expect(result.final_score).toBe(85);
|
|
41
|
+
expect(result.iterations).toHaveLength(1);
|
|
42
|
+
expect(result.final_content).toBeTruthy();
|
|
43
|
+
expect(result.prompt_improved).toBe(false);
|
|
44
|
+
});
|
|
45
|
+
it("improves prompt and retries when score is below threshold", async () => {
|
|
46
|
+
// Iteration 1: Generate
|
|
47
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
48
|
+
object: {
|
|
49
|
+
frontmatter: { title: "Acme" },
|
|
50
|
+
body: "We helped Acme.",
|
|
51
|
+
},
|
|
52
|
+
usage: { promptTokens: 2000, completionTokens: 200 },
|
|
53
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
54
|
+
});
|
|
55
|
+
// Iteration 1: Judge — below threshold
|
|
56
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
57
|
+
object: { score: 40, reasoning: "Too generic, no metrics." },
|
|
58
|
+
usage: { promptTokens: 1500, completionTokens: 150 },
|
|
59
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
60
|
+
});
|
|
61
|
+
// Iteration 1: Improve prompt
|
|
62
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
63
|
+
object: {
|
|
64
|
+
improved_prompt: "Write a case study with specific metrics and client quotes.",
|
|
65
|
+
changes_summary: "Added metrics requirement.",
|
|
66
|
+
},
|
|
67
|
+
usage: { promptTokens: 1000, completionTokens: 200 },
|
|
68
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
69
|
+
});
|
|
70
|
+
// Iteration 2: Generate with improved prompt
|
|
71
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
72
|
+
object: {
|
|
73
|
+
frontmatter: { title: "Acme Corp — Migration to Next.js" },
|
|
74
|
+
body: "# Acme Corp\n\n12 services migrated in 8 weeks. 40% load time improvement.",
|
|
75
|
+
},
|
|
76
|
+
usage: { promptTokens: 2500, completionTokens: 400 },
|
|
77
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
78
|
+
});
|
|
79
|
+
// Iteration 2: Judge — above threshold
|
|
80
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
81
|
+
object: {
|
|
82
|
+
score: 78,
|
|
83
|
+
reasoning: "Good metrics, clear structure. Matches gold standard well.",
|
|
84
|
+
},
|
|
85
|
+
usage: { promptTokens: 2000, completionTokens: 180 },
|
|
86
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
87
|
+
});
|
|
88
|
+
const budget = new BudgetTracker({ daily_limit_usd: 10.0 });
|
|
89
|
+
const runner = new EvalRunner(mockProvider, budget);
|
|
90
|
+
const result = await runner.run(baseConfig);
|
|
91
|
+
expect(result.final_status).toBe("keep");
|
|
92
|
+
expect(result.final_score).toBe(78);
|
|
93
|
+
expect(result.iterations).toHaveLength(2);
|
|
94
|
+
expect(result.prompt_improved).toBe(true);
|
|
95
|
+
expect(result.final_prompt).toContain("metrics");
|
|
96
|
+
});
|
|
97
|
+
it("discards after max iterations if threshold never met", async () => {
|
|
98
|
+
// Mock 3 iterations, all below threshold
|
|
99
|
+
for (let i = 0; i < 3; i++) {
|
|
100
|
+
// Generate
|
|
101
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
102
|
+
object: {
|
|
103
|
+
frontmatter: { title: "Attempt" },
|
|
104
|
+
body: "Generic content.",
|
|
105
|
+
},
|
|
106
|
+
usage: { promptTokens: 2000, completionTokens: 200 },
|
|
107
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
108
|
+
});
|
|
109
|
+
// Judge — below threshold
|
|
110
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
111
|
+
object: { score: 30 + i * 10, reasoning: "Still not good enough." },
|
|
112
|
+
usage: { promptTokens: 1500, completionTokens: 150 },
|
|
113
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
114
|
+
});
|
|
115
|
+
// Improve prompt (not on last iteration)
|
|
116
|
+
if (i < 2) {
|
|
117
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
118
|
+
object: {
|
|
119
|
+
improved_prompt: `Improved prompt v${i + 2}.`,
|
|
120
|
+
changes_summary: "Tweaked instructions.",
|
|
121
|
+
},
|
|
122
|
+
usage: { promptTokens: 1000, completionTokens: 200 },
|
|
123
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const budget = new BudgetTracker({ daily_limit_usd: 10.0 });
|
|
128
|
+
const runner = new EvalRunner(mockProvider, budget);
|
|
129
|
+
const result = await runner.run(baseConfig);
|
|
130
|
+
expect(result.final_status).toBe("discard");
|
|
131
|
+
expect(result.iterations).toHaveLength(3);
|
|
132
|
+
expect(result.final_content).toBeUndefined();
|
|
133
|
+
});
|
|
134
|
+
it("judgeOnly scores existing content without loop", async () => {
|
|
135
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
136
|
+
object: { score: 72, reasoning: "Decent quality, matches intent." },
|
|
137
|
+
usage: { promptTokens: 1500, completionTokens: 100 },
|
|
138
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
139
|
+
});
|
|
140
|
+
const budget = new BudgetTracker({ daily_limit_usd: 10.0 });
|
|
141
|
+
const runner = new EvalRunner(mockProvider, budget);
|
|
142
|
+
const result = await runner.judgeOnly({
|
|
143
|
+
draft: "# Existing content\n\nSome published text.",
|
|
144
|
+
gold_standard: "# Perfect example.",
|
|
145
|
+
judge_prompt: "Score 0-100.",
|
|
146
|
+
});
|
|
147
|
+
expect(result.score).toBe(72);
|
|
148
|
+
expect(result.reasoning).toBeTruthy();
|
|
149
|
+
});
|
|
150
|
+
it("accumulates total usage across iterations", async () => {
|
|
151
|
+
// Generate
|
|
152
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
153
|
+
object: { frontmatter: { title: "T" }, body: "B" },
|
|
154
|
+
usage: { promptTokens: 1000, completionTokens: 100 },
|
|
155
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
156
|
+
});
|
|
157
|
+
// Judge — keep
|
|
158
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
159
|
+
object: { score: 90, reasoning: "Great." },
|
|
160
|
+
usage: { promptTokens: 500, completionTokens: 50 },
|
|
161
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
162
|
+
});
|
|
163
|
+
const budget = new BudgetTracker({ daily_limit_usd: 10.0 });
|
|
164
|
+
const runner = new EvalRunner(mockProvider, budget);
|
|
165
|
+
const result = await runner.run(baseConfig);
|
|
166
|
+
expect(result.total_usage.input_tokens).toBe(1500);
|
|
167
|
+
expect(result.total_usage.output_tokens).toBe(150);
|
|
168
|
+
expect(result.total_usage.estimated_cost_usd).toBeGreaterThan(0);
|
|
169
|
+
});
|
|
170
|
+
});
|
|
171
|
+
//# sourceMappingURL=eval-runner.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-runner.test.js","sourceRoot":"","sources":["../../src/__tests__/eval-runner.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AAEpC,MAAM,YAAY,GAAqB;IACtC,QAAQ,EAAE,WAAW;IACrB,KAAK,EAAE,4BAA4B;CACnC,CAAC;AAEF,MAAM,UAAU,GAAG;IAClB,YAAY,EAAE,YAAY;IAC1B,iBAAiB,EAChB,uFAAuF;IACxF,aAAa,EACZ,wFAAwF;IACzF,YAAY,EAAE,qDAAqD;IACnE,iBAAiB,EAAE,gCAAgC;IACnD,SAAS,EAAE,EAAE;IACb,cAAc,EAAE,CAAC;CACjB,CAAC;AAEF,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;QAC5E,WAAW;QACX,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE;gBACP,WAAW,EAAE,EAAE,KAAK,EAAE,sBAAsB,EAAE;gBAC9C,IAAI,EAAE,gFAAgF;aACtF;YACD,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACpD,yFAAyF;SAClF,CAAC,CAAC;QAEV,gCAAgC;QAChC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,6CAA6C,EAAE;YAC/E,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACpD,yFAAyF;SAClF,CAAC,CAAC;QAEV,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE5C,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,UAAU,EAAE,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QAC1E,wBAAwB;QACxB,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE;gBACP,WAAW,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE;gBAC9B,IAAI,EAAE,iBAAiB;aACvB;YACD,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACpD,yFAAyF;SAClF,CAAC,CAAC;QAEV,uCAAuC;QACvC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,0BAA0B,EAAE;YAC5D,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACpD,yFAAyF;SAClF,CAAC,CAAC;QAEV,8BAA8B;QAC9B,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE;gBACP,eAAe,EAAE,6DAA6D;gBAC9E,eAAe,EAAE,4BAA4B;aAC7C;YACD,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACpD,yFAAyF;SAClF,CAAC,CAAC;QAEV,6CAA6C;QAC7C,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE;gBACP,WAAW,EAAE,EAAE,KAAK,EAAE,kCAAkC,EAAE;gBAC1D,IAAI,EAAE,4EAA4E;aAClF;YACD,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACpD,yFAAyF;SAClF,CAAC,CAAC;QAEV,uCAAuC;QACvC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE;gBACP,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,4DAA4D;aACvE;YACD,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACpD,yFAAyF;SAClF,CAAC,CAAC;QAEV,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE5C,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACrE,yCAAyC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,WAAW;YACX,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;gBAC/C,MAAM,EAAE;oBACP,WAAW,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE;oBACjC,IAAI,EAAE,kBAAkB;iBACxB;gBACD,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;gBACpD,yFAAyF;aAClF,CAAC,CAAC;YAEV,0BAA0B;YAC1B,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;gBAC/C,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,GAAG,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,wBAAwB,EAAE;gBACnE,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;gBACpD,yFAAyF;aAClF,CAAC,CAAC;YAEV,yCAAyC;YACzC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACX,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;oBAC/C,MAAM,EAAE;wBACP,eAAe,EAAE,oBAAoB,CAAC,GAAG,CAAC,GAAG;wBAC7C,eAAe,EAAE,uBAAuB;qBACxC;oBACD,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;oBACpD,yFAAyF;iBAClF,CAAC,CAAC;YACX,CAAC;QACF,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE5C,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,aAAa,EAAE,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC/D,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,iCAAiC,EAAE;YACnE,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACpD,yFAAyF;SAClF,CAAC,CAAC;QAEV,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC;YACrC,KAAK,EAAE,4CAA4C;YACnD,aAAa,EAAE,oBAAoB;YACnC,YAAY,EAAE,cAAc;SAC5B,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,UAAU,EAAE,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QAC1D,WAAW;QACX,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;YAClD,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACpD,yFAAyF;SAClF,CAAC,CAAC;QAEV,eAAe;QACf,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;YAC1C,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE;YAClD,yFAAyF;SAClF,CAAC,CAAC;QAEV,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAE5C,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,kBAAkB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/extract.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from "vitest";
|
|
2
|
+
import { BudgetTracker } from "../budget.js";
|
|
3
|
+
import { extract } from "../functions/extract.js";
|
|
4
|
+
vi.mock("ai", () => ({ generateObject: vi.fn() }));
|
|
5
|
+
import { generateObject } from "ai";
|
|
6
|
+
const mockProvider = {
|
|
7
|
+
provider: "anthropic",
|
|
8
|
+
model: "claude-sonnet-4-5-20250514",
|
|
9
|
+
};
|
|
10
|
+
describe("extract", () => {
|
|
11
|
+
it("extracts entities and relations from text", async () => {
|
|
12
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
13
|
+
object: {
|
|
14
|
+
entities: [
|
|
15
|
+
{ type: "client", name: "Acme Corp", aliases: ["Acme", "ACME"], confidence: 0.95 },
|
|
16
|
+
{ type: "technology", name: "Next.js", aliases: ["NextJS"], confidence: 0.99 },
|
|
17
|
+
],
|
|
18
|
+
relations: [
|
|
19
|
+
{
|
|
20
|
+
from_entity: "Acme Corp",
|
|
21
|
+
to_entity: "Next.js",
|
|
22
|
+
relation_type: "uses",
|
|
23
|
+
confidence: 0.9,
|
|
24
|
+
evidence: "Acme Corp is migrating to Next.js",
|
|
25
|
+
},
|
|
26
|
+
],
|
|
27
|
+
},
|
|
28
|
+
usage: { promptTokens: 800, completionTokens: 200 },
|
|
29
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
30
|
+
});
|
|
31
|
+
const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
|
|
32
|
+
const result = await extract({
|
|
33
|
+
text: "Meeting with Acme Corp. They want to migrate to Next.js.",
|
|
34
|
+
file_path: "knowledge/clients/acme.md",
|
|
35
|
+
}, mockProvider, budget);
|
|
36
|
+
expect(result.entities).toHaveLength(2);
|
|
37
|
+
expect(result.entities[0].name).toBe("Acme Corp");
|
|
38
|
+
expect(result.relations).toHaveLength(1);
|
|
39
|
+
expect(result.usage.function_name).toBe("extract");
|
|
40
|
+
});
|
|
41
|
+
it("includes existing entities as context", async () => {
|
|
42
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
43
|
+
object: {
|
|
44
|
+
entities: [{ type: "technology", name: "React", aliases: [], confidence: 0.95 }],
|
|
45
|
+
relations: [
|
|
46
|
+
{
|
|
47
|
+
from_entity: "React",
|
|
48
|
+
to_entity: "Next.js",
|
|
49
|
+
relation_type: "used_by",
|
|
50
|
+
confidence: 0.98,
|
|
51
|
+
evidence: "React is the foundation of Next.js",
|
|
52
|
+
},
|
|
53
|
+
],
|
|
54
|
+
},
|
|
55
|
+
usage: { promptTokens: 1000, completionTokens: 150 },
|
|
56
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
57
|
+
});
|
|
58
|
+
const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
|
|
59
|
+
const result = await extract({
|
|
60
|
+
text: "React is great for building UIs.",
|
|
61
|
+
file_path: "knowledge/tech/react.md",
|
|
62
|
+
existing_entities: [{ type: "technology", name: "Next.js" }],
|
|
63
|
+
}, mockProvider, budget);
|
|
64
|
+
expect(result.entities).toHaveLength(1);
|
|
65
|
+
expect(result.relations).toHaveLength(1);
|
|
66
|
+
});
|
|
67
|
+
it("records budget usage", async () => {
|
|
68
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
69
|
+
object: { entities: [], relations: [] },
|
|
70
|
+
usage: { promptTokens: 400, completionTokens: 50 },
|
|
71
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
72
|
+
});
|
|
73
|
+
const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
|
|
74
|
+
await extract({ text: "Empty text", file_path: "test.md" }, mockProvider, budget);
|
|
75
|
+
expect(budget.getHistory()).toHaveLength(1);
|
|
76
|
+
expect(budget.getHistory()[0].function_name).toBe("extract");
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
//# sourceMappingURL=extract.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract.test.js","sourceRoot":"","sources":["../../src/__tests__/extract.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AAEpC,MAAM,YAAY,GAAqB;IACtC,QAAQ,EAAE,WAAW;IACrB,KAAK,EAAE,4BAA4B;CACnC,CAAC;AAEF,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;IACxB,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QAC1D,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE;gBACP,QAAQ,EAAE;oBACT,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE;oBAClF,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE;iBAC9E;gBACD,SAAS,EAAE;oBACV;wBACC,WAAW,EAAE,WAAW;wBACxB,SAAS,EAAE,SAAS;wBACpB,aAAa,EAAE,MAAM;wBACrB,UAAU,EAAE,GAAG;wBACf,QAAQ,EAAE,mCAAmC;qBAC7C;iBACD;aACD;YACD,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACnD,yFAAyF;SAClF,CAAC,CAAC;QACV,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,MAAM,OAAO,CAC3B;YACC,IAAI,EAAE,0DAA0D;YAChE,SAAS,EAAE,2BAA2B;SACtC,EACD,YAAY,EACZ,MAAM,CACN,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACtD,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE;gBACP,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;gBAChF,SAAS,EAAE;oBACV;wBACC,WAAW,EAAE,OAAO;wBACpB,SAAS,EAAE,SAAS;wBACpB,aAAa,EAAE,SAAS;wBACxB,UAAU,EAAE,IAAI;wBAChB,QAAQ,EAAE,oCAAoC;qBAC9C;iBACD;aACD;YACD,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE;YACpD,yFAAyF;SAClF,CAAC,CAAC;QACV,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3D,MAAM,MAAM,GAAG,MAAM,OAAO,CAC3B;YACC,IAAI,EAAE,kCAAkC;YACxC,SAAS,EAAE,yBAAyB;YACpC,iBAAiB,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;SAC5D,EACD,YAAY,EACZ,MAAM,CACN,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QACrC,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,qBAAqB,CAAC;YAC/C,MAAM,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;YACvC,KAAK,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE;YAClD,yFAAyF;SAClF,CAAC,CAAC;QACV,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3D,MAAM,OAAO,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QAClF,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"find-gaps.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/find-gaps.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from "vitest";
|
|
2
|
+
import { BudgetTracker } from "../budget.js";
|
|
3
|
+
import { findGaps } from "../functions/find-gaps.js";
|
|
4
|
+
vi.mock("ai", () => ({ generateObject: vi.fn() }));
|
|
5
|
+
import { generateObject } from "ai";
|
|
6
|
+
const mockProvider = {
|
|
7
|
+
provider: "anthropic",
|
|
8
|
+
model: "claude-sonnet-4-5-20250514",
|
|
9
|
+
};
|
|
10
|
+
describe("findGaps", () => {
|
|
11
|
+
it("identifies knowledge without corresponding content", async () => {
|
|
12
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
13
|
+
object: {
|
|
14
|
+
gaps: [
|
|
15
|
+
{
|
|
16
|
+
entity_name: "React",
|
|
17
|
+
entity_type: "technology",
|
|
18
|
+
suggested_content_type: "blog-post",
|
|
19
|
+
priority: "high",
|
|
20
|
+
reason: "Core tech with no content page.",
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
entity_name: "TypeScript",
|
|
24
|
+
entity_type: "technology",
|
|
25
|
+
suggested_content_type: "service-page",
|
|
26
|
+
priority: "medium",
|
|
27
|
+
reason: "Appears in projects but no content.",
|
|
28
|
+
},
|
|
29
|
+
],
|
|
30
|
+
},
|
|
31
|
+
usage: { promptTokens: 2500, completionTokens: 300 },
|
|
32
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
33
|
+
});
|
|
34
|
+
const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
|
|
35
|
+
const result = await findGaps({
|
|
36
|
+
entities: [
|
|
37
|
+
{ type: "technology", name: "React" },
|
|
38
|
+
{ type: "technology", name: "TypeScript" },
|
|
39
|
+
{ type: "technology", name: "Next.js" },
|
|
40
|
+
],
|
|
41
|
+
relations: [{ from: "Next.js", to: "React", type: "built_on" }],
|
|
42
|
+
existing_content: [
|
|
43
|
+
{
|
|
44
|
+
path: "content/cases/acme.mdx",
|
|
45
|
+
title: "Acme Corp — Next.js Migration",
|
|
46
|
+
summary: "Case study about Next.js",
|
|
47
|
+
},
|
|
48
|
+
],
|
|
49
|
+
}, mockProvider, budget);
|
|
50
|
+
expect(result.gaps).toHaveLength(2);
|
|
51
|
+
expect(result.gaps[0].entity_name).toBe("React");
|
|
52
|
+
expect(result.gaps[0].priority).toBe("high");
|
|
53
|
+
expect(result.usage.function_name).toBe("findGaps");
|
|
54
|
+
});
|
|
55
|
+
it("uses business context for prioritization", async () => {
|
|
56
|
+
vi.mocked(generateObject).mockResolvedValueOnce({
|
|
57
|
+
object: {
|
|
58
|
+
gaps: [
|
|
59
|
+
{
|
|
60
|
+
entity_name: "AI Strategy",
|
|
61
|
+
entity_type: "service",
|
|
62
|
+
suggested_content_type: "service-page",
|
|
63
|
+
priority: "high",
|
|
64
|
+
reason: "Core business offering.",
|
|
65
|
+
},
|
|
66
|
+
],
|
|
67
|
+
},
|
|
68
|
+
usage: { promptTokens: 2000, completionTokens: 150 },
|
|
69
|
+
// biome-ignore lint/suspicious/noExplicitAny: partial mock of generateObject return type
|
|
70
|
+
});
|
|
71
|
+
const budget = new BudgetTracker({ daily_limit_usd: 5.0 });
|
|
72
|
+
const result = await findGaps({
|
|
73
|
+
entities: [{ type: "service", name: "AI Strategy" }],
|
|
74
|
+
relations: [],
|
|
75
|
+
existing_content: [],
|
|
76
|
+
business_context: "We are a digital agency focused on AI strategy consulting.",
|
|
77
|
+
}, mockProvider, budget);
|
|
78
|
+
expect(result.gaps).toHaveLength(1);
|
|
79
|
+
expect(result.gaps[0].priority).toBe("high");
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
//# sourceMappingURL=find-gaps.test.js.map
|