@haikeilabs/agentware 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -0
- package/dist/evals/cases/file_search.d.ts +4 -0
- package/dist/evals/cases/file_search.d.ts.map +1 -0
- package/dist/evals/cases/file_search.js +99 -0
- package/dist/evals/cases/general.d.ts +4 -0
- package/dist/evals/cases/general.d.ts.map +1 -0
- package/dist/evals/cases/general.js +113 -0
- package/dist/evals/index.d.ts +5 -0
- package/dist/evals/index.d.ts.map +1 -0
- package/dist/evals/index.js +4 -0
- package/dist/evals/main.d.ts +2 -0
- package/dist/evals/main.d.ts.map +1 -0
- package/dist/evals/main.js +109 -0
- package/dist/evals/models.d.ts +38 -0
- package/dist/evals/models.d.ts.map +1 -0
- package/dist/evals/models.js +69 -0
- package/dist/evals/runner.d.ts +40 -0
- package/dist/evals/runner.d.ts.map +1 -0
- package/dist/evals/runner.js +131 -0
- package/dist/executor/executor.d.ts +42 -0
- package/dist/executor/executor.d.ts.map +1 -0
- package/dist/executor/executor.js +63 -0
- package/dist/executor/index.d.ts +2 -0
- package/dist/executor/index.d.ts.map +1 -0
- package/dist/executor/index.js +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +9 -0
- package/dist/jobs/index.d.ts +3 -0
- package/dist/jobs/index.d.ts.map +1 -0
- package/dist/jobs/index.js +2 -0
- package/dist/jobs/job.d.ts +18 -0
- package/dist/jobs/job.d.ts.map +1 -0
- package/dist/jobs/job.js +20 -0
- package/dist/jobs/manager.d.ts +23 -0
- package/dist/jobs/manager.d.ts.map +1 -0
- package/dist/jobs/manager.js +63 -0
- package/dist/llm/backend.d.ts +9 -0
- package/dist/llm/backend.d.ts.map +1 -0
- package/dist/llm/backend.js +1 -0
- package/dist/llm/index.d.ts +4 -0
- package/dist/llm/index.d.ts.map +1 -0
- package/dist/llm/index.js +1 -0
- package/dist/llm/request.d.ts +28 -0
- package/dist/llm/request.d.ts.map +1 -0
- package/dist/llm/request.js +7 -0
- package/dist/llm/response.d.ts +17 -0
- package/dist/llm/response.d.ts.map +1 -0
- package/dist/llm/response.js +1 -0
- package/dist/llmcontext/context_window.d.ts +36 -0
- package/dist/llmcontext/context_window.d.ts.map +1 -0
- package/dist/llmcontext/context_window.js +115 -0
- package/dist/llmcontext/index.d.ts +4 -0
- package/dist/llmcontext/index.d.ts.map +1 -0
- package/dist/llmcontext/index.js +3 -0
- package/dist/llmcontext/manager.d.ts +27 -0
- package/dist/llmcontext/manager.d.ts.map +1 -0
- package/dist/llmcontext/manager.js +27 -0
- package/dist/llmcontext/strategies.d.ts +21 -0
- package/dist/llmcontext/strategies.d.ts.map +1 -0
- package/dist/llmcontext/strategies.js +180 -0
- package/dist/memory/client.d.ts +107 -0
- package/dist/memory/client.d.ts.map +1 -0
- package/dist/memory/client.js +210 -0
- package/dist/memory/index.d.ts +5 -0
- package/dist/memory/index.d.ts.map +1 -0
- package/dist/memory/index.js +2 -0
- package/dist/memory/tools.d.ts +22 -0
- package/dist/memory/tools.d.ts.map +1 -0
- package/dist/memory/tools.js +57 -0
- package/dist/middleware/audit.d.ts +27 -0
- package/dist/middleware/audit.d.ts.map +1 -0
- package/dist/middleware/audit.js +26 -0
- package/dist/middleware/guardrails/error_tracker.d.ts +34 -0
- package/dist/middleware/guardrails/error_tracker.d.ts.map +1 -0
- package/dist/middleware/guardrails/error_tracker.js +70 -0
- package/dist/middleware/guardrails/nudge.d.ts +17 -0
- package/dist/middleware/guardrails/nudge.d.ts.map +1 -0
- package/dist/middleware/guardrails/nudge.js +67 -0
- package/dist/middleware/guardrails/response_validator.d.ts +29 -0
- package/dist/middleware/guardrails/response_validator.d.ts.map +1 -0
- package/dist/middleware/guardrails/response_validator.js +145 -0
- package/dist/middleware/guardrails/step_enforcer.d.ts +20 -0
- package/dist/middleware/guardrails/step_enforcer.d.ts.map +1 -0
- package/dist/middleware/guardrails/step_enforcer.js +65 -0
- package/dist/middleware/index.d.ts +5 -0
- package/dist/middleware/index.d.ts.map +1 -0
- package/dist/middleware/index.js +4 -0
- package/dist/middleware/inference.d.ts +28 -0
- package/dist/middleware/inference.d.ts.map +1 -0
- package/dist/middleware/inference.js +141 -0
- package/dist/middleware/middleware.d.ts +22 -0
- package/dist/middleware/middleware.d.ts.map +1 -0
- package/dist/middleware/middleware.js +51 -0
- package/dist/middleware/policy.d.ts +46 -0
- package/dist/middleware/policy.d.ts.map +1 -0
- package/dist/middleware/policy.js +122 -0
- package/dist/middleware/types.d.ts +42 -0
- package/dist/middleware/types.d.ts.map +1 -0
- package/dist/middleware/types.js +36 -0
- package/dist/prompts/generator.d.ts +10 -0
- package/dist/prompts/generator.d.ts.map +1 -0
- package/dist/prompts/generator.js +24 -0
- package/dist/prompts/index.d.ts +2 -0
- package/dist/prompts/index.d.ts.map +1 -0
- package/dist/prompts/index.js +1 -0
- package/dist/toolformat/formatter.d.ts +20 -0
- package/dist/toolformat/formatter.d.ts.map +1 -0
- package/dist/toolformat/formatter.js +22 -0
- package/dist/toolformat/index.d.ts +2 -0
- package/dist/toolformat/index.d.ts.map +1 -0
- package/dist/toolformat/index.js +1 -0
- package/dist/tools/async.d.ts +42 -0
- package/dist/tools/async.d.ts.map +1 -0
- package/dist/tools/async.js +89 -0
- package/dist/tools/index.d.ts +5 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +3 -0
- package/dist/tools/registry.d.ts +11 -0
- package/dist/tools/registry.d.ts.map +1 -0
- package/dist/tools/registry.js +27 -0
- package/dist/tools/tool.d.ts +29 -0
- package/dist/tools/tool.d.ts.map +1 -0
- package/dist/tools/tool.js +31 -0
- package/package.json +49 -0
package/README.md
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# @haikeilabs/agentware
|
|
2
|
+
|
|
3
|
+
`@haikeilabs/agentware` provides tool and agent-runtime primitives for applications
|
|
4
|
+
that already own their agent loop, channel integration, identities, and tool
|
|
5
|
+
handlers.
|
|
6
|
+
|
|
7
|
+
## Async tool wrapper
|
|
8
|
+
|
|
9
|
+
Use `wrapTool` to make an existing tool callable through a consistent async
|
|
10
|
+
boundary. It preserves the host's input, output, and trusted context types;
|
|
11
|
+
Agentware does not infer identity or privileges from tool arguments.
|
|
12
|
+
|
|
13
|
+
```ts
|
|
14
|
+
import { wrapTool } from "@haikeilabs/agentware";
|
|
15
|
+
|
|
16
|
+
type Selection = { queryId: string };
|
|
17
|
+
type TrustedContext = { principalId: string; verifiedGrant: string };
|
|
18
|
+
|
|
19
|
+
const queryModel = wrapTool(
|
|
20
|
+
async (selection: Selection, context: TrustedContext, signal: AbortSignal) => {
|
|
21
|
+
// Keep the application's existing authorization and provider guard here.
|
|
22
|
+
return runExistingTool(selection, context.verifiedGrant, signal);
|
|
23
|
+
},
|
|
24
|
+
{ name: "semantic-model-query", timeoutMs: 5_000 },
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
const result = await queryModel(
|
|
28
|
+
{ queryId: "commission-summary" },
|
|
29
|
+
trustedContext,
|
|
30
|
+
{ signal: request.signal },
|
|
31
|
+
);
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The wrapper propagates host cancellation and deadlines to the handler's
|
|
35
|
+
`AbortSignal`. It raises `ToolAbortedError` when the host cancels and
|
|
36
|
+
`ToolTimeoutError` when its configured deadline expires. Lifecycle hooks expose
|
|
37
|
+
only tool name, outcome, and duration; they intentionally exclude raw inputs
|
|
38
|
+
and host context.
|
|
39
|
+
|
|
40
|
+
## Integration boundary
|
|
41
|
+
|
|
42
|
+
Register only the wrapped handler with an agent framework or tool registry.
|
|
43
|
+
Keep channel ingress, identity verification, delegated credentials, and
|
|
44
|
+
domain-specific authorization in the host application. Agentware is not a
|
|
45
|
+
policy engine; a policy gate can wrap this same boundary later.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file_search.d.ts","sourceRoot":"","sources":["../../../src/evals/cases/file_search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAExD,eAAO,MAAM,eAAe,EAAE,cAAc,EA4D3C,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,QAAQ,EAqCrC,CAAC"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
export const FileSearchTools = [
|
|
2
|
+
{
|
|
3
|
+
type: "function",
|
|
4
|
+
function: {
|
|
5
|
+
name: "glob",
|
|
6
|
+
description: "Find files matching a glob pattern in a directory",
|
|
7
|
+
parameters: {
|
|
8
|
+
type: "object",
|
|
9
|
+
properties: {
|
|
10
|
+
pattern: {
|
|
11
|
+
type: "string",
|
|
12
|
+
description: "Glob pattern to match files",
|
|
13
|
+
},
|
|
14
|
+
directory: {
|
|
15
|
+
type: "string",
|
|
16
|
+
description: "Directory to search in",
|
|
17
|
+
},
|
|
18
|
+
},
|
|
19
|
+
required: ["pattern"],
|
|
20
|
+
},
|
|
21
|
+
},
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
type: "function",
|
|
25
|
+
function: {
|
|
26
|
+
name: "read_file",
|
|
27
|
+
description: "Read contents of a file",
|
|
28
|
+
parameters: {
|
|
29
|
+
type: "object",
|
|
30
|
+
properties: {
|
|
31
|
+
path: {
|
|
32
|
+
type: "string",
|
|
33
|
+
description: "Path to the file to read",
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
required: ["path"],
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
type: "function",
|
|
42
|
+
function: {
|
|
43
|
+
name: "search_files",
|
|
44
|
+
description: "Search for text content in files",
|
|
45
|
+
parameters: {
|
|
46
|
+
type: "object",
|
|
47
|
+
properties: {
|
|
48
|
+
query: {
|
|
49
|
+
type: "string",
|
|
50
|
+
description: "Text to search for",
|
|
51
|
+
},
|
|
52
|
+
path: {
|
|
53
|
+
type: "string",
|
|
54
|
+
description: "Directory path to search in",
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
required: ["query"],
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
];
|
|
62
|
+
export const FileSearchCases = [
|
|
63
|
+
{
|
|
64
|
+
name: "glob_python_files",
|
|
65
|
+
description: "Find all Python files in the current directory",
|
|
66
|
+
systemPrompt: "You are a helpful assistant with file system access. Use the provided tools to help the user.",
|
|
67
|
+
userMessage: "Find all Python files in the current directory",
|
|
68
|
+
tools: FileSearchTools,
|
|
69
|
+
expectedTool: "glob",
|
|
70
|
+
maxTurns: 10,
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
name: "glob_md_files",
|
|
74
|
+
description: "Find all Markdown files",
|
|
75
|
+
systemPrompt: "You are a helpful assistant with file system access. Use the provided tools to help the user.",
|
|
76
|
+
userMessage: "List all markdown files (*.md) in this directory",
|
|
77
|
+
tools: FileSearchTools,
|
|
78
|
+
expectedTool: "glob",
|
|
79
|
+
maxTurns: 10,
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
name: "search_code",
|
|
83
|
+
description: "Search for specific code pattern",
|
|
84
|
+
systemPrompt: "You are a helpful assistant with file system access. Use the provided tools to help the user.",
|
|
85
|
+
userMessage: "Search for all files containing 'func main'",
|
|
86
|
+
tools: FileSearchTools,
|
|
87
|
+
expectedTool: "search_files",
|
|
88
|
+
maxTurns: 10,
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
name: "read_config",
|
|
92
|
+
description: "Read a configuration file",
|
|
93
|
+
systemPrompt: "You are a helpful assistant with file system access. Use the provided tools to help the user.",
|
|
94
|
+
userMessage: "Read the contents of go.mod",
|
|
95
|
+
tools: FileSearchTools,
|
|
96
|
+
expectedTool: "read_file",
|
|
97
|
+
maxTurns: 10,
|
|
98
|
+
},
|
|
99
|
+
];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"general.d.ts","sourceRoot":"","sources":["../../../src/evals/cases/general.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAExD,eAAO,MAAM,YAAY,EAAE,cAAc,EAiExC,CAAC;AAEF,eAAO,MAAM,YAAY,EAAE,QAAQ,EA8ClC,CAAC"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
export const GeneralTools = [
|
|
2
|
+
{
|
|
3
|
+
type: "function",
|
|
4
|
+
function: {
|
|
5
|
+
name: "calculator",
|
|
6
|
+
description: "Perform mathematical calculations",
|
|
7
|
+
parameters: {
|
|
8
|
+
type: "object",
|
|
9
|
+
properties: {
|
|
10
|
+
expression: {
|
|
11
|
+
type: "string",
|
|
12
|
+
description: "Mathematical expression to evaluate",
|
|
13
|
+
},
|
|
14
|
+
},
|
|
15
|
+
required: ["expression"],
|
|
16
|
+
},
|
|
17
|
+
},
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
type: "function",
|
|
21
|
+
function: {
|
|
22
|
+
name: "get_weather",
|
|
23
|
+
description: "Get weather information for a location",
|
|
24
|
+
parameters: {
|
|
25
|
+
type: "object",
|
|
26
|
+
properties: {
|
|
27
|
+
location: {
|
|
28
|
+
type: "string",
|
|
29
|
+
description: "City name",
|
|
30
|
+
},
|
|
31
|
+
units: {
|
|
32
|
+
type: "string",
|
|
33
|
+
enum: ["celsius", "fahrenheit"],
|
|
34
|
+
description: "Temperature units",
|
|
35
|
+
},
|
|
36
|
+
},
|
|
37
|
+
required: ["location"],
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
type: "function",
|
|
43
|
+
function: {
|
|
44
|
+
name: "translate",
|
|
45
|
+
description: "Translate text between languages",
|
|
46
|
+
parameters: {
|
|
47
|
+
type: "object",
|
|
48
|
+
properties: {
|
|
49
|
+
text: {
|
|
50
|
+
type: "string",
|
|
51
|
+
description: "Text to translate",
|
|
52
|
+
},
|
|
53
|
+
target_lang: {
|
|
54
|
+
type: "string",
|
|
55
|
+
description: "Target language code",
|
|
56
|
+
},
|
|
57
|
+
source_lang: {
|
|
58
|
+
type: "string",
|
|
59
|
+
description: "Source language code (optional, auto-detect if not provided)",
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
required: ["text", "target_lang"],
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
},
|
|
66
|
+
];
|
|
67
|
+
export const GeneralCases = [
|
|
68
|
+
{
|
|
69
|
+
name: "calculator_add",
|
|
70
|
+
description: "Simple addition calculation",
|
|
71
|
+
systemPrompt: "You are a helpful assistant with access to tools. Use them when needed.",
|
|
72
|
+
userMessage: "What is 123 + 456?",
|
|
73
|
+
tools: GeneralTools,
|
|
74
|
+
expectedTool: "calculator",
|
|
75
|
+
maxTurns: 10,
|
|
76
|
+
},
|
|
77
|
+
{
|
|
78
|
+
name: "calculator_complex",
|
|
79
|
+
description: "Complex mathematical expression",
|
|
80
|
+
systemPrompt: "You are a helpful assistant with access to tools. Use them when needed.",
|
|
81
|
+
userMessage: "Calculate (15 * 8) + (100 / 4) - 50",
|
|
82
|
+
tools: GeneralTools,
|
|
83
|
+
expectedTool: "calculator",
|
|
84
|
+
maxTurns: 10,
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
name: "get_weather",
|
|
88
|
+
description: "Get weather for a city",
|
|
89
|
+
systemPrompt: "You are a helpful assistant with access to tools. Use them when needed.",
|
|
90
|
+
userMessage: "What's the weather like in Tokyo?",
|
|
91
|
+
tools: GeneralTools,
|
|
92
|
+
expectedTool: "get_weather",
|
|
93
|
+
maxTurns: 10,
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
name: "translate_english_to_spanish",
|
|
97
|
+
description: "Translate text to Spanish",
|
|
98
|
+
systemPrompt: "You are a helpful assistant with access to tools. Use them when needed.",
|
|
99
|
+
userMessage: "Translate 'Hello, how are you?' to Spanish",
|
|
100
|
+
tools: GeneralTools,
|
|
101
|
+
expectedTool: "translate",
|
|
102
|
+
maxTurns: 10,
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
name: "translate_with_source",
|
|
106
|
+
description: "Translate with specified source language",
|
|
107
|
+
systemPrompt: "You are a helpful assistant with access to tools. Use them when needed.",
|
|
108
|
+
userMessage: "Translate 'Bonjour' from French to English",
|
|
109
|
+
tools: GeneralTools,
|
|
110
|
+
expectedTool: "translate",
|
|
111
|
+
maxTurns: 10,
|
|
112
|
+
},
|
|
113
|
+
];
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { ModelClient, ToolCall, Usage, ModelResult, ChatMessage, ToolDefinition, ToolFunc } from "./models";
|
|
2
|
+
export { EvalRunner, EvalCase, EvalResult, EvalReport } from "./runner";
|
|
3
|
+
export { GeneralTools, GeneralCases } from "./cases/general";
|
|
4
|
+
export { FileSearchTools, FileSearchCases } from "./cases/file_search";
|
|
5
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAC5G,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/evals/main.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as path from "path";
|
|
3
|
+
import { EvalRunner } from "./runner.js";
|
|
4
|
+
import { FileSearchCases } from "./cases/file_search.js";
|
|
5
|
+
import { GeneralCases } from "./cases/general.js";
|
|
6
|
+
async function mockToolExecutor(toolName, args) {
|
|
7
|
+
switch (toolName) {
|
|
8
|
+
case "glob": {
|
|
9
|
+
const pattern = args.pattern || "";
|
|
10
|
+
const directory = args.directory || ".";
|
|
11
|
+
const fullPattern = path.join(directory, pattern);
|
|
12
|
+
const files = globSyncSimple(fullPattern).slice(0, 10);
|
|
13
|
+
return JSON.stringify(files);
|
|
14
|
+
}
|
|
15
|
+
case "read_file": {
|
|
16
|
+
const filePath = args.path || "";
|
|
17
|
+
try {
|
|
18
|
+
const content = fs.readFileSync(filePath, "utf-8");
|
|
19
|
+
return content.slice(0, 1000);
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return `File not found: ${filePath}`;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
case "search_files":
|
|
26
|
+
return JSON.stringify(["match in file1.go", "match in file2.go"]);
|
|
27
|
+
case "calculator": {
|
|
28
|
+
const expression = args.expression || "0";
|
|
29
|
+
try {
|
|
30
|
+
const result = safeEval(expression);
|
|
31
|
+
return String(result);
|
|
32
|
+
}
|
|
33
|
+
catch (e) {
|
|
34
|
+
return `Error: ${e}`;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
case "get_weather": {
|
|
38
|
+
const location = args.location || "";
|
|
39
|
+
return JSON.stringify({ location, temp: 72, condition: "sunny" });
|
|
40
|
+
}
|
|
41
|
+
case "translate": {
|
|
42
|
+
const text = args.text || "";
|
|
43
|
+
const targetLang = args.target_lang || "";
|
|
44
|
+
return JSON.stringify({
|
|
45
|
+
original: text,
|
|
46
|
+
translated: `[translated: ${text}]`,
|
|
47
|
+
target: targetLang,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
default:
|
|
51
|
+
return `Mock result for ${toolName}`;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
function safeEval(expr) {
|
|
55
|
+
const sanitized = expr.replace(/[^0-9+\-*/().\s]/g, "");
|
|
56
|
+
const result = Function(`"use strict"; return (${sanitized})`)();
|
|
57
|
+
return Number(result);
|
|
58
|
+
}
|
|
59
|
+
function globSyncSimple(_pattern) {
|
|
60
|
+
return [];
|
|
61
|
+
}
|
|
62
|
+
async function main() {
|
|
63
|
+
const args = process.argv.slice(2);
|
|
64
|
+
let fileSearch = false;
|
|
65
|
+
let general = false;
|
|
66
|
+
let models = "qwen3.6-27b-mtp";
|
|
67
|
+
let baseURL = "http://pedrogpt:8000";
|
|
68
|
+
let maxTurns = 10;
|
|
69
|
+
for (let i = 0; i < args.length; i++) {
|
|
70
|
+
if (args[i] === "--file-search")
|
|
71
|
+
fileSearch = true;
|
|
72
|
+
else if (args[i] === "--general")
|
|
73
|
+
general = true;
|
|
74
|
+
else if (args[i] === "--models" && args[i + 1])
|
|
75
|
+
models = args[++i];
|
|
76
|
+
else if (args[i] === "--base-url" && args[i + 1])
|
|
77
|
+
baseURL = args[++i];
|
|
78
|
+
else if (args[i] === "--max-turns" && args[i + 1])
|
|
79
|
+
maxTurns = parseInt(args[++i], 10);
|
|
80
|
+
}
|
|
81
|
+
let cases;
|
|
82
|
+
let outputFile;
|
|
83
|
+
if (fileSearch) {
|
|
84
|
+
cases = FileSearchCases;
|
|
85
|
+
outputFile = "file_search_results.json";
|
|
86
|
+
}
|
|
87
|
+
else if (general) {
|
|
88
|
+
cases = GeneralCases;
|
|
89
|
+
outputFile = "general_results.json";
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
cases = [...FileSearchCases, ...GeneralCases];
|
|
93
|
+
outputFile = "results.json";
|
|
94
|
+
}
|
|
95
|
+
const modelList = models.split(",").map((m) => m.trim()).filter(Boolean);
|
|
96
|
+
const runner = new EvalRunner(baseURL, maxTurns);
|
|
97
|
+
const report = await runner.runEvals(cases, modelList, mockToolExecutor);
|
|
98
|
+
const outputDir = path.join("python", "src", "evals", "output");
|
|
99
|
+
fs.mkdirSync(outputDir, { recursive: true });
|
|
100
|
+
const outputPath = path.join(outputDir, outputFile);
|
|
101
|
+
runner.saveReport(report, outputPath);
|
|
102
|
+
console.log("\n=== Summary ===");
|
|
103
|
+
for (const model of modelList) {
|
|
104
|
+
const passRate = runner.passRate(model) * 100;
|
|
105
|
+
console.log(`${model}: ${passRate.toFixed(1)}% pass rate`);
|
|
106
|
+
}
|
|
107
|
+
console.log(`\nResults saved to: ${outputPath}`);
|
|
108
|
+
}
|
|
109
|
+
main().catch(console.error);
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
export interface ToolCall {
|
|
2
|
+
id: string;
|
|
3
|
+
name: string;
|
|
4
|
+
arguments: string;
|
|
5
|
+
}
|
|
6
|
+
export interface Usage {
|
|
7
|
+
prompt_tokens: number;
|
|
8
|
+
completion_tokens: number;
|
|
9
|
+
total_tokens: number;
|
|
10
|
+
}
|
|
11
|
+
export interface ModelResult {
|
|
12
|
+
content: string;
|
|
13
|
+
tool_calls: ToolCall[];
|
|
14
|
+
finish_reason: string;
|
|
15
|
+
usage: Usage;
|
|
16
|
+
}
|
|
17
|
+
export interface ChatMessage {
|
|
18
|
+
role: string;
|
|
19
|
+
content?: string;
|
|
20
|
+
}
|
|
21
|
+
export interface ToolDefinition {
|
|
22
|
+
type: string;
|
|
23
|
+
function: ToolFunc;
|
|
24
|
+
}
|
|
25
|
+
export interface ToolFunc {
|
|
26
|
+
name: string;
|
|
27
|
+
description: string;
|
|
28
|
+
parameters: Record<string, unknown>;
|
|
29
|
+
}
|
|
30
|
+
export declare class ModelClient {
|
|
31
|
+
private baseURL;
|
|
32
|
+
private model;
|
|
33
|
+
private apiKey;
|
|
34
|
+
private timeout;
|
|
35
|
+
constructor(baseURL: string, model: string, apiKey?: string, timeout?: number);
|
|
36
|
+
complete(messages: ChatMessage[], tools?: ToolDefinition[], temperature?: number, maxTokens?: number): Promise<ModelResult>;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=models.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../../src/evals/models.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,QAAQ;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,KAAK;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,QAAQ,EAAE,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,KAAK,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,GAAE,MAAW,EAAE,OAAO,GAAE,MAAc;IAOlF,QAAQ,CACZ,QAAQ,EAAE,WAAW,EAAE,EACvB,KAAK,CAAC,EAAE,cAAc,EAAE,EACxB,WAAW,GAAE,MAAY,EACzB,SAAS,GAAE,MAAa,GACvB,OAAO,CAAC,WAAW,CAAC;CA8ExB"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
export class ModelClient {
|
|
2
|
+
baseURL;
|
|
3
|
+
model;
|
|
4
|
+
apiKey;
|
|
5
|
+
timeout;
|
|
6
|
+
constructor(baseURL, model, apiKey = "", timeout = 60000) {
|
|
7
|
+
this.baseURL = baseURL;
|
|
8
|
+
this.model = model;
|
|
9
|
+
this.apiKey = apiKey;
|
|
10
|
+
this.timeout = timeout;
|
|
11
|
+
}
|
|
12
|
+
async complete(messages, tools, temperature = 0.0, maxTokens = 2048) {
|
|
13
|
+
const payload = {
|
|
14
|
+
model: this.model,
|
|
15
|
+
messages: messages,
|
|
16
|
+
temperature: temperature,
|
|
17
|
+
max_tokens: maxTokens,
|
|
18
|
+
};
|
|
19
|
+
if (tools) {
|
|
20
|
+
payload.tools = tools;
|
|
21
|
+
}
|
|
22
|
+
const headers = {
|
|
23
|
+
"Content-Type": "application/json",
|
|
24
|
+
};
|
|
25
|
+
if (this.apiKey) {
|
|
26
|
+
headers["Authorization"] = `Bearer ${this.apiKey}`;
|
|
27
|
+
}
|
|
28
|
+
const controller = new AbortController();
|
|
29
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
30
|
+
try {
|
|
31
|
+
const response = await fetch(`${this.baseURL}/chat/completions`, {
|
|
32
|
+
method: "POST",
|
|
33
|
+
headers: headers,
|
|
34
|
+
body: JSON.stringify(payload),
|
|
35
|
+
signal: controller.signal,
|
|
36
|
+
});
|
|
37
|
+
clearTimeout(timeoutId);
|
|
38
|
+
if (!response.ok) {
|
|
39
|
+
const errorText = await response.text();
|
|
40
|
+
throw new Error(`HTTP ${response.status}: ${errorText}`);
|
|
41
|
+
}
|
|
42
|
+
const data = await response.json();
|
|
43
|
+
const choice = data.choices[0];
|
|
44
|
+
const toolCalls = [];
|
|
45
|
+
if (choice.message.tool_calls) {
|
|
46
|
+
for (const tc of choice.message.tool_calls) {
|
|
47
|
+
toolCalls.push({
|
|
48
|
+
id: tc.id,
|
|
49
|
+
name: tc.function.name,
|
|
50
|
+
arguments: tc.function.arguments,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return {
|
|
55
|
+
content: choice.message.content || "",
|
|
56
|
+
tool_calls: toolCalls,
|
|
57
|
+
finish_reason: "",
|
|
58
|
+
usage: data.usage,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
clearTimeout(timeoutId);
|
|
63
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
64
|
+
throw new Error(`Request timeout after ${this.timeout}ms`);
|
|
65
|
+
}
|
|
66
|
+
throw error;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { ToolDefinition } from "./models.js";
|
|
2
|
+
export { ToolDefinition };
|
|
3
|
+
export interface EvalCase {
|
|
4
|
+
name: string;
|
|
5
|
+
description: string;
|
|
6
|
+
systemPrompt: string;
|
|
7
|
+
userMessage: string;
|
|
8
|
+
tools: ToolDefinition[];
|
|
9
|
+
expectedTool: string;
|
|
10
|
+
maxTurns: number;
|
|
11
|
+
}
|
|
12
|
+
export interface EvalResult {
|
|
13
|
+
case_name: string;
|
|
14
|
+
model_name: string;
|
|
15
|
+
success: boolean;
|
|
16
|
+
turns: number;
|
|
17
|
+
tool_calls: Array<{
|
|
18
|
+
turn: number;
|
|
19
|
+
name: string;
|
|
20
|
+
arguments: string;
|
|
21
|
+
}>;
|
|
22
|
+
error: string;
|
|
23
|
+
duration_ms: number;
|
|
24
|
+
}
|
|
25
|
+
export interface EvalReport {
|
|
26
|
+
timestamp: string;
|
|
27
|
+
models: string[];
|
|
28
|
+
results: EvalResult[];
|
|
29
|
+
}
|
|
30
|
+
export declare class EvalRunner {
|
|
31
|
+
private baseURL;
|
|
32
|
+
private maxTurns;
|
|
33
|
+
private results;
|
|
34
|
+
constructor(baseURL: string, maxTurns?: number);
|
|
35
|
+
passRate(model: string): number;
|
|
36
|
+
runCase(case_: EvalCase, model: string, toolExecutor: (toolName: string, args: Record<string, unknown>) => Promise<string>): Promise<EvalResult>;
|
|
37
|
+
runEvals(cases: EvalCase[], models: string[], toolExecutor: (toolName: string, args: Record<string, unknown>) => Promise<string>): Promise<EvalReport>;
|
|
38
|
+
saveReport(report: EvalReport, outputPath: string): void;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAA4B,cAAc,EAAE,MAAM,aAAa,CAAC;AAEvE,OAAO,EAAE,cAAc,EAAE,CAAC;AAE1B,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrE,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,OAAO,CAAoB;gBAEvB,OAAO,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAW;IAKlD,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAOzB,OAAO,CACX,KAAK,EAAE,QAAQ,EACf,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,GACjF,OAAO,CAAC,UAAU,CAAC;IAiFhB,QAAQ,CACZ,KAAK,EAAE,QAAQ,EAAE,EACjB,MAAM,EAAE,MAAM,EAAE,EAChB,YAAY,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,GACjF,OAAO,CAAC,UAAU,CAAC;IA2BtB,UAAU,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;CAoBzD"}
|