@inspectr/mcplab-core 1.21.0 → 1.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent.d.ts +1 -0
- package/dist/agent.d.ts.map +1 -1
- package/dist/agent.js +1 -0
- package/dist/agent.js.map +1 -1
- package/dist/config-values.d.ts +7 -0
- package/dist/config-values.d.ts.map +1 -0
- package/dist/config-values.js +15 -0
- package/dist/config-values.js.map +1 -0
- package/dist/config.d.ts +4 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -1
- package/dist/config.js.map +1 -1
- package/dist/eval.d.ts +6 -0
- package/dist/eval.d.ts.map +1 -1
- package/dist/eval.js +43 -1
- package/dist/eval.js.map +1 -1
- package/dist/evaluation-config-store.d.ts +12 -5
- package/dist/evaluation-config-store.d.ts.map +1 -1
- package/dist/evaluation-config-store.js +45 -13
- package/dist/evaluation-config-store.js.map +1 -1
- package/dist/index.d.ts +26 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -1
- package/dist/index.js.map +1 -1
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +7 -15
- package/dist/mcp.js.map +1 -1
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +5 -3
- package/dist/runner.js.map +1 -1
- package/dist/scenario-builder.d.ts +20 -0
- package/dist/scenario-builder.d.ts.map +1 -0
- package/dist/scenario-builder.js +82 -0
- package/dist/scenario-builder.js.map +1 -0
- package/dist/test-case-store.d.ts +19 -5
- package/dist/test-case-store.d.ts.map +1 -1
- package/dist/test-case-store.js +106 -37
- package/dist/test-case-store.js.map +1 -1
- package/dist/tool-definition-token-estimates.d.ts +23 -0
- package/dist/tool-definition-token-estimates.d.ts.map +1 -0
- package/dist/tool-definition-token-estimates.js +25 -0
- package/dist/tool-definition-token-estimates.js.map +1 -0
- package/dist/trace-token-estimates.d.ts +8 -1
- package/dist/trace-token-estimates.d.ts.map +1 -1
- package/dist/trace-token-estimates.js +3 -3
- package/dist/trace-token-estimates.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,24 +1,38 @@
|
|
|
1
|
-
import type { SourceScenario } from './types.js';
|
|
1
|
+
import type { EvalRules, SourceScenario } from './types.js';
|
|
2
2
|
export type TestCaseCreateInput = {
|
|
3
3
|
id: string;
|
|
4
4
|
name?: string;
|
|
5
5
|
servers: string[];
|
|
6
6
|
prompt: string;
|
|
7
7
|
requiredTools?: string[];
|
|
8
|
+
forbiddenTools?: string[];
|
|
9
|
+
toolSequence?: string[];
|
|
8
10
|
responseRegexPatterns?: string[];
|
|
11
|
+
eval?: EvalRules;
|
|
12
|
+
extractRules?: Array<{
|
|
13
|
+
name: string;
|
|
14
|
+
regex: string;
|
|
15
|
+
}>;
|
|
9
16
|
};
|
|
10
17
|
export type CreatedTestCaseFile = {
|
|
11
18
|
id: string;
|
|
12
19
|
path: string;
|
|
13
20
|
testCase: SourceScenario;
|
|
14
21
|
};
|
|
15
|
-
|
|
16
|
-
* Creates one canonical Test Case. Both the app API and MCP tool call this
|
|
17
|
-
* function so they share input validation and file persistence semantics.
|
|
18
|
-
*/
|
|
22
|
+
export type UpdatedTestCaseFile = CreatedTestCaseFile;
|
|
19
23
|
export declare function createTestCaseFile(params: {
|
|
20
24
|
librariesDir: string;
|
|
21
25
|
knownServerIds: Iterable<string>;
|
|
22
26
|
testCase: TestCaseCreateInput;
|
|
23
27
|
}): CreatedTestCaseFile;
|
|
28
|
+
export declare function readTestCaseFile(params: {
|
|
29
|
+
librariesDir: string;
|
|
30
|
+
filePath: string;
|
|
31
|
+
}): CreatedTestCaseFile;
|
|
32
|
+
export declare function updateTestCaseFile(params: {
|
|
33
|
+
librariesDir: string;
|
|
34
|
+
knownServerIds: Iterable<string>;
|
|
35
|
+
filePath: string;
|
|
36
|
+
testCase: TestCaseCreateInput;
|
|
37
|
+
}): UpdatedTestCaseFile;
|
|
24
38
|
//# sourceMappingURL=test-case-store.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-case-store.d.ts","sourceRoot":"","sources":["../src/test-case-store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"test-case-store.d.ts","sourceRoot":"","sources":["../src/test-case-store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAG5D,MAAM,MAAM,mBAAmB,GAAG;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,YAAY,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvD,CAAC;AACF,MAAM,MAAM,mBAAmB,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,cAAc,CAAA;CAAE,CAAC;AACzF,MAAM,MAAM,mBAAmB,GAAG,mBAAmB,CAAC;AAEtD,wBAAgB,kBAAkB,CAAC,MAAM,EAAE;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjC,QAAQ,EAAE,mBAAmB,CAAC;CAC/B,GAAG,mBAAmB,CA0CtB;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE;IACvC,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,mBAAmB,CAatB;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,mBAAmB,CAAC;CAC/B,GAAG,mBAAmB,CAiCtB"}
|
package/dist/test-case-store.js
CHANGED
|
@@ -1,61 +1,130 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { resolve, sep } from 'node:path';
|
|
3
|
-
import { stringify as stringifyYaml } from 'yaml';
|
|
4
|
-
|
|
5
|
-
* Creates one canonical Test Case. Both the app API and MCP tool call this
|
|
6
|
-
* function so they share input validation and file persistence semantics.
|
|
7
|
-
*/
|
|
3
|
+
import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
|
|
4
|
+
import { buildScenarioEntry } from './scenario-builder.js';
|
|
8
5
|
export function createTestCaseFile(params) {
|
|
9
|
-
const
|
|
10
|
-
|
|
6
|
+
const input = params.testCase;
|
|
7
|
+
const id = input.id.trim();
|
|
8
|
+
if (!/^[a-zA-Z0-9_-]+$/.test(id))
|
|
11
9
|
throw new Error('Test Case id must use letters, numbers, hyphens, or underscores.');
|
|
12
|
-
|
|
13
|
-
const prompt = params.testCase.prompt.trim();
|
|
10
|
+
const prompt = input.prompt.trim();
|
|
14
11
|
if (!prompt)
|
|
15
12
|
throw new Error('A Test Case needs a prompt.');
|
|
16
13
|
if (prompt.length > 4000)
|
|
17
14
|
throw new Error('Test Case prompt must be 4000 characters or fewer.');
|
|
18
|
-
const servers =
|
|
19
|
-
if (servers.length
|
|
15
|
+
const servers = unique(input.servers);
|
|
16
|
+
if (!servers.length)
|
|
20
17
|
throw new Error('A Test Case needs at least one MCP server.');
|
|
21
|
-
const
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
throw new Error(`Unknown MCP server(s): ${missingServers.join(', ')}`);
|
|
18
|
+
const missing = servers.filter((server) => !new Set(params.knownServerIds).has(server));
|
|
19
|
+
if (missing.length)
|
|
20
|
+
throw new Error(`Unknown MCP server(s): ${missing.join(', ')}`);
|
|
25
21
|
const librariesDir = resolve(params.librariesDir);
|
|
26
22
|
const testCasesDir = resolve(librariesDir, 'test-cases');
|
|
27
|
-
if (!testCasesDir.startsWith(`${librariesDir}${sep}`))
|
|
23
|
+
if (!testCasesDir.startsWith(`${librariesDir}${sep}`))
|
|
28
24
|
throw new Error('Test Case path is outside the configured libraries directory.');
|
|
29
|
-
}
|
|
30
25
|
mkdirSync(testCasesDir, { recursive: true });
|
|
31
26
|
const path = resolve(testCasesDir, `${id}.yaml`);
|
|
32
|
-
if (!path.startsWith(`${testCasesDir}${sep}`))
|
|
27
|
+
if (!path.startsWith(`${testCasesDir}${sep}`))
|
|
33
28
|
throw new Error('Test Case path is outside the canonical test-cases directory.');
|
|
34
|
-
}
|
|
35
29
|
if (existsSync(path))
|
|
36
30
|
throw new Error(`Test Case '${id}' already exists.`);
|
|
37
|
-
const
|
|
38
|
-
const responseRegexPatterns = uniqueNonEmptyStrings(params.testCase.responseRegexPatterns ?? []);
|
|
39
|
-
const testCase = {
|
|
31
|
+
const testCase = buildScenarioEntry({
|
|
40
32
|
id,
|
|
41
|
-
name:
|
|
33
|
+
name: input.name?.trim() || id,
|
|
42
34
|
servers,
|
|
43
35
|
prompt,
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
36
|
+
required_tools: input.requiredTools,
|
|
37
|
+
forbidden_tools: input.forbiddenTools,
|
|
38
|
+
tool_sequence: input.toolSequence,
|
|
39
|
+
response_regex_patterns: input.responseRegexPatterns,
|
|
40
|
+
eval: input.eval,
|
|
41
|
+
extract_rules: input.extractRules
|
|
42
|
+
});
|
|
43
|
+
try {
|
|
44
|
+
writeFileSync(path, `${stringifyYaml(testCase)}\n`, { encoding: 'utf8', flag: 'wx' });
|
|
45
|
+
}
|
|
46
|
+
catch (error) {
|
|
47
|
+
if (error.code === 'EEXIST') {
|
|
48
|
+
throw new Error(`Test Case '${id}' already exists.`);
|
|
49
|
+
}
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
56
52
|
return { id, path, testCase };
|
|
57
53
|
}
|
|
58
|
-
function
|
|
54
|
+
export function readTestCaseFile(params) {
|
|
55
|
+
const librariesDir = resolve(params.librariesDir);
|
|
56
|
+
const testCasesDir = resolve(librariesDir, 'test-cases');
|
|
57
|
+
const path = resolve(testCasesDir, params.filePath);
|
|
58
|
+
if (!path.startsWith(`${testCasesDir}${sep}`) || !/\.ya?ml$/i.test(path))
|
|
59
|
+
throw new Error('Test Case path is outside the canonical test-cases directory.');
|
|
60
|
+
if (!existsSync(path))
|
|
61
|
+
throw new Error(`Test Case not found: ${params.filePath}`);
|
|
62
|
+
const testCase = parseYaml(readFileSync(path, 'utf8'));
|
|
63
|
+
return {
|
|
64
|
+
id: String(testCase.id ?? '').trim(),
|
|
65
|
+
path,
|
|
66
|
+
testCase
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
export function updateTestCaseFile(params) {
|
|
70
|
+
const existing = readTestCaseFile({
|
|
71
|
+
librariesDir: params.librariesDir,
|
|
72
|
+
filePath: params.filePath
|
|
73
|
+
});
|
|
74
|
+
const input = params.testCase;
|
|
75
|
+
const id = input.id.trim();
|
|
76
|
+
if (!/^[a-zA-Z0-9_-]+$/.test(id))
|
|
77
|
+
throw new Error('Test Case id must use letters, numbers, hyphens, or underscores.');
|
|
78
|
+
const prompt = input.prompt.trim();
|
|
79
|
+
if (!prompt)
|
|
80
|
+
throw new Error('A Test Case needs a prompt.');
|
|
81
|
+
const servers = unique(input.servers);
|
|
82
|
+
if (!servers.length)
|
|
83
|
+
throw new Error('A Test Case needs at least one MCP server.');
|
|
84
|
+
const known = new Set(params.knownServerIds);
|
|
85
|
+
const missing = servers.filter((server) => !known.has(server));
|
|
86
|
+
if (missing.length)
|
|
87
|
+
throw new Error(`Unknown MCP server(s): ${missing.join(', ')}`);
|
|
88
|
+
const testCase = buildScenarioEntry({
|
|
89
|
+
id,
|
|
90
|
+
name: input.name?.trim() || id,
|
|
91
|
+
servers,
|
|
92
|
+
prompt,
|
|
93
|
+
required_tools: input.requiredTools,
|
|
94
|
+
forbidden_tools: input.forbiddenTools,
|
|
95
|
+
tool_sequence: input.toolSequence,
|
|
96
|
+
response_regex_patterns: input.responseRegexPatterns,
|
|
97
|
+
eval: input.eval,
|
|
98
|
+
extract_rules: input.extractRules
|
|
99
|
+
});
|
|
100
|
+
const librariesDir = resolve(params.librariesDir);
|
|
101
|
+
const testCasesDir = resolve(librariesDir, 'test-cases');
|
|
102
|
+
assertUniqueTestCaseId(testCasesDir, id, existing.path);
|
|
103
|
+
writeFileSync(existing.path, `${stringifyYaml(testCase)}\n`, { encoding: 'utf8' });
|
|
104
|
+
return { id, path: existing.path, testCase };
|
|
105
|
+
}
|
|
106
|
+
function assertUniqueTestCaseId(testCasesDir, id, excludingPath) {
|
|
107
|
+
if (!existsSync(testCasesDir))
|
|
108
|
+
return;
|
|
109
|
+
for (const fileName of readdirSync(testCasesDir)) {
|
|
110
|
+
if (!/\.ya?ml$/i.test(fileName))
|
|
111
|
+
continue;
|
|
112
|
+
const candidatePath = resolve(testCasesDir, fileName);
|
|
113
|
+
if (excludingPath && candidatePath === excludingPath)
|
|
114
|
+
continue;
|
|
115
|
+
let parsed;
|
|
116
|
+
try {
|
|
117
|
+
parsed = parseYaml(readFileSync(candidatePath, 'utf8'));
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
if (typeof parsed?.id === 'string' && parsed.id.trim() === id) {
|
|
123
|
+
throw new Error(`Test Case id '${id}' is already used by another Test Case: ${fileName}`);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
function unique(values) {
|
|
59
128
|
return [...new Set(values.map((value) => value.trim()).filter(Boolean))];
|
|
60
129
|
}
|
|
61
130
|
//# sourceMappingURL=test-case-store.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-case-store.js","sourceRoot":"","sources":["../src/test-case-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"test-case-store.js","sourceRoot":"","sources":["../src/test-case-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC1F,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,SAAS,IAAI,aAAa,EAAE,MAAM,MAAM,CAAC;AAEtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAiB3D,MAAM,UAAU,kBAAkB,CAAC,MAIlC;IACC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC5D,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAChG,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACxF,IAAI,OAAO,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpF,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IACzD,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,YAAY,GAAG,GAAG,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACjD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,YAAY,GAAG,GAAG,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,IAAI,UAAU,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;IAC3E,MAAM,QAAQ,GAAG,kBAAkB,CAAC;QAClC,EAAE;QACF,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE;QAC9B,OAAO;QACP,MAAM;QACN,cAAc,EAAE,KAAK,CAAC,aAAa;QACnC,eAAe,EAAE,KAAK,CAAC,cAAc;QACrC,aAAa,EAAE,KAAK,CAAC,YAAY;QACjC,uBAAuB,EAAE,KAAK,CAAC,qBAAqB;QACpD,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,aAAa,EAAE,KAAK,CAAC,YAAY;KAClC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,aAAa,CAAC,IAAI,EAAE,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACxF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CAAC,cAAc,EAAE,mBAAmB,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAGhC;IACC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IACpD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,YAAY,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACnF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,GAAG,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAmB,CAAC;IACzE,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QACpC,IAAI;QACJ,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,MAKlC;IACC,MAAM,QAAQ,GAAG,gBAAgB,CAAC;QAChC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC9B,MAAM,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3B,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/D,IAAI,OAAO,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpF,MAAM,QAAQ,GAAG,kBAAkB,CAAC;QAClC,EAAE;QACF,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE;QAC9B,OAAO;QACP,MAAM;QACN,cAAc,EAAE,KAAK,CAAC,aAAa;QACnC,eAAe,EAAE,KAAK,CAAC,cAAc;QACrC,aAAa,EAAE,KAAK,CAAC,YAAY;QACjC,uBAAuB,EAAE,KAAK,CAAC,qBAAqB;QACpD,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,aAAa,EAAE,KAAK,CAAC,YAAY;KAClC,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IACzD,sBAAsB,CAAC,YAAY,EAAE,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACxD,aAAa,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IACnF,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;AAC/C,CAAC;AAED,SAAS,sBAAsB,CAAC,YAAoB,EAAE,EAAU,EAAE,aAAsB;IACtF,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO;IACtC,KAAK,MAAM,QAAQ,IAAI,WAAW,CAAC,YAAY,CAAC,EAAE,CAAC;QACjD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,SAAS;QAC1C,MAAM,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QACtD,IAAI,aAAa,IAAI,aAAa,KAAK,aAAa;YAAE,SAAS;QAC/D,IAAI,MAAsC,CAAC;QAC3C,IAAI,CAAC;YACH,MAAM,GAAG,SAAS,CAAC,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,CAAmC,CAAC;QAC5F,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,OAAO,MAAM,EAAE,EAAE,KAAK,QAAQ,IAAI,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,iBAAiB,EAAE,2CAA2C,QAAQ,EAAE,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB;IAC9B,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC3E,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { EstimatedTokens } from './types.js';
|
|
2
|
+
export interface ToolDefinitionForTokenEstimate {
|
|
3
|
+
name: string;
|
|
4
|
+
title?: string;
|
|
5
|
+
description?: string;
|
|
6
|
+
inputSchema?: unknown;
|
|
7
|
+
outputSchema?: unknown;
|
|
8
|
+
}
|
|
9
|
+
export interface ToolDefinitionTokenEstimate {
|
|
10
|
+
name: string;
|
|
11
|
+
total: number;
|
|
12
|
+
}
|
|
13
|
+
export interface ToolDefinitionTokenEstimates {
|
|
14
|
+
toolCount: number;
|
|
15
|
+
total: number;
|
|
16
|
+
tools: ToolDefinitionTokenEstimate[];
|
|
17
|
+
method: EstimatedTokens['method'];
|
|
18
|
+
}
|
|
19
|
+
export declare function estimateToolDefinitionTokens(
|
|
20
|
+
tools: ToolDefinitionForTokenEstimate[],
|
|
21
|
+
model: string
|
|
22
|
+
): ToolDefinitionTokenEstimates;
|
|
23
|
+
//# sourceMappingURL=tool-definition-token-estimates.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-definition-token-estimates.d.ts","sourceRoot":"","sources":["../src/tool-definition-token-estimates.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAGlD,MAAM,WAAW,8BAA8B;IAC7C,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,4BAA4B;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,2BAA2B,EAAE,CAAC;IACrC,MAAM,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;CACnC;AAYD,wBAAgB,4BAA4B,CAC1C,KAAK,EAAE,8BAA8B,EAAE,EACvC,KAAK,EAAE,MAAM,GACZ,4BAA4B,CAa9B"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { estimateTextTokens } from './trace-token-estimates.js';
|
|
2
|
+
function serializeToolDefinition(tool) {
|
|
3
|
+
return JSON.stringify({
|
|
4
|
+
name: tool.name,
|
|
5
|
+
...(tool.title !== undefined ? { title: tool.title } : {}),
|
|
6
|
+
...(tool.description !== undefined ? { description: tool.description } : {}),
|
|
7
|
+
...(tool.inputSchema !== undefined ? { inputSchema: tool.inputSchema } : {}),
|
|
8
|
+
...(tool.outputSchema !== undefined ? { outputSchema: tool.outputSchema } : {})
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
export function estimateToolDefinitionTokens(tools, model) {
|
|
12
|
+
const estimates = tools.map((tool) => {
|
|
13
|
+
const estimate = estimateTextTokens(serializeToolDefinition(tool), model);
|
|
14
|
+
return { name: tool.name, total: estimate.count, method: estimate.method };
|
|
15
|
+
});
|
|
16
|
+
return {
|
|
17
|
+
toolCount: estimates.length,
|
|
18
|
+
total: estimates.reduce((total, estimate) => total + estimate.total, 0),
|
|
19
|
+
tools: estimates.map(({ name, total }) => ({ name, total })),
|
|
20
|
+
method: estimates.some((estimate) => estimate.method === 'js_tiktoken_fallback')
|
|
21
|
+
? 'js_tiktoken_fallback'
|
|
22
|
+
: 'js_tiktoken_estimate'
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=tool-definition-token-estimates.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-definition-token-estimates.js","sourceRoot":"","sources":["../src/tool-definition-token-estimates.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAsBhE,SAAS,uBAAuB,CAAC,IAAoC;IACnE,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,GAAG,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,GAAG,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,GAAG,CAAC,IAAI,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChF,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,4BAA4B,CAC1C,KAAuC,EACvC,KAAa;IAEb,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACnC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;QAC1E,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC7E,CAAC,CAAC,CAAC;IACH,OAAO;QACL,SAAS,EAAE,SAAS,CAAC,MAAM;QAC3B,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QACvE,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5D,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,KAAK,sBAAsB,CAAC;YAC9E,CAAC,CAAC,sBAAsB;YACxB,CAAC,CAAC,sBAAsB;KAC3B,CAAC;AACJ,CAAC"}
|
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
import type { TraceMessage } from './types.js';
|
|
1
|
+
import type { EstimatedTokens, TraceMessage } from './types.js';
|
|
2
|
+
export declare function estimateTextTokens(
|
|
3
|
+
text: string,
|
|
4
|
+
model: string
|
|
5
|
+
): {
|
|
6
|
+
count: number;
|
|
7
|
+
method: EstimatedTokens['method'];
|
|
8
|
+
};
|
|
2
9
|
export declare function enrichTraceMessagesWithEstimatedTokens(
|
|
3
10
|
messages: TraceMessage[],
|
|
4
11
|
model: string
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"trace-token-estimates.d.ts","sourceRoot":"","sources":["../src/trace-token-estimates.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"trace-token-estimates.d.ts","sourceRoot":"","sources":["../src/trace-token-estimates.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAUhE,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GACZ;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAA;CAAE,CAQtD;AAgBD,wBAAgB,sCAAsC,CACpD,QAAQ,EAAE,YAAY,EAAE,EACxB,KAAK,EAAE,MAAM,GACZ,YAAY,EAAE,CA8ChB"}
|
|
@@ -7,7 +7,7 @@ function normalizeJson(value) {
|
|
|
7
7
|
return String(value ?? '');
|
|
8
8
|
}
|
|
9
9
|
}
|
|
10
|
-
function
|
|
10
|
+
export function estimateTextTokens(text, model) {
|
|
11
11
|
try {
|
|
12
12
|
const encoding = encodingForModel(model);
|
|
13
13
|
return { count: encoding.encode(text).length, method: 'js_tiktoken_estimate' };
|
|
@@ -18,8 +18,8 @@ function estimateTokenCount(text, model) {
|
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
20
|
function estimateForBlock(inputText, outputText, model) {
|
|
21
|
-
const input =
|
|
22
|
-
const output =
|
|
21
|
+
const input = estimateTextTokens(inputText, model);
|
|
22
|
+
const output = estimateTextTokens(outputText, model);
|
|
23
23
|
return {
|
|
24
24
|
input: input.count,
|
|
25
25
|
output: output.count,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"trace-token-estimates.js","sourceRoot":"","sources":["../src/trace-token-estimates.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG5D,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,
|
|
1
|
+
{"version":3,"file":"trace-token-estimates.js","sourceRoot":"","sources":["../src/trace-token-estimates.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG5D,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,IAAY,EACZ,KAAa;IAEb,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAA+C,CAAC,CAAC;QACnF,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC;IACjF,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,QAAQ,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC;QAC5C,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,sBAAsB,EAAE,CAAC;IACjF,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAiB,EAAE,UAAkB,EAAE,KAAa;IAC5E,MAAM,KAAK,GAAG,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,kBAAkB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IACrD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,MAAM,EAAE,MAAM,CAAC,KAAK;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK;QACjC,MAAM,EACJ,KAAK,CAAC,MAAM,KAAK,sBAAsB,IAAI,MAAM,CAAC,MAAM,KAAK,sBAAsB;YACjF,CAAC,CAAC,sBAAsB;YACxB,CAAC,CAAC,sBAAsB;KAC7B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CACpD,QAAwB,EACxB,KAAa;IAEb,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAEvE,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC1C,GAAG,OAAO;QACV,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;KACxD,CAAC,CAAC,CAAC;IAEJ,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW;YAAE,SAAS;QAE3C,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CACrC,CAAC,KAAK,EAA2E,EAAE,CACjF,KAAK,CAAC,IAAI,KAAK,UAAU,CAC5B,CAAC;QACF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEpC,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACxC,MAAM,cAAc,GAClB,WAAW,EAAE,IAAI,KAAK,MAAM;YAC1B,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CACxB,CAAC,KAAK,EAA8E,EAAE,CACpF,KAAK,CAAC,IAAI,KAAK,aAAa,CAC/B;YACH,CAAC,CAAC,EAAE,CAAC;QACT,MAAM,aAAa,GAAG,IAAI,GAAG,CAC3B,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAU,CAAC,CACnE,CAAC;QAEF,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,YAAY,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACpD,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzC,MAAM,UAAU,GAAG,MAAM;gBACvB,CAAC,CAAC,MAAM,CAAC,OAAO;qBACX,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;qBACtC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;qBACxB,IAAI,CAAC,IAAI,CAAC;gBACf,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,QAAQ,GAAG,gBAAgB,CAAC,YAAY,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YACnE,GAAG,CAAC,gBAAgB,GAAG,QAAQ,CAAC;YAChC,IAAI,MAAM;gBAAE,MAAM,CAAC,gBAAgB,GAAG,QAAQ,CAAC;QACjD,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|