opc-agent 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +75 -75
- package/README.md +235 -358
- package/README.zh-CN.md +415 -415
- package/dist/channels/web.js +256 -256
- package/dist/core/knowledge.d.ts +5 -0
- package/dist/core/knowledge.js +39 -2
- package/dist/deploy/hermes.js +22 -22
- package/dist/deploy/openclaw.js +31 -31
- package/dist/index.d.ts +0 -4
- package/dist/index.js +1 -7
- package/dist/providers/index.d.ts +1 -1
- package/dist/providers/index.js +158 -14
- package/dist/schema/oad.d.ts +3 -3
- package/dist/templates/code-reviewer.js +5 -5
- package/dist/templates/customer-service.js +2 -2
- package/dist/templates/data-analyst.js +5 -5
- package/dist/templates/knowledge-base.js +2 -2
- package/dist/templates/sales-assistant.js +4 -4
- package/dist/templates/teacher.js +6 -6
- package/docs/.vitepress/config.ts +103 -103
- package/docs/api/cli.md +48 -48
- package/docs/api/oad-schema.md +64 -64
- package/docs/api/sdk.md +80 -80
- package/docs/guide/concepts.md +51 -51
- package/docs/guide/configuration.md +79 -79
- package/docs/guide/deployment.md +42 -42
- package/docs/guide/getting-started.md +44 -44
- package/docs/guide/templates.md +28 -28
- package/docs/guide/testing.md +84 -84
- package/docs/index.md +27 -27
- package/docs/zh/api/cli.md +54 -54
- package/docs/zh/api/oad-schema.md +87 -87
- package/docs/zh/api/sdk.md +102 -102
- package/docs/zh/guide/concepts.md +104 -104
- package/docs/zh/guide/configuration.md +135 -135
- package/docs/zh/guide/deployment.md +81 -81
- package/docs/zh/guide/getting-started.md +82 -82
- package/docs/zh/guide/templates.md +84 -84
- package/docs/zh/guide/testing.md +88 -88
- package/docs/zh/index.md +27 -27
- package/examples/customer-service-demo/README.md +90 -90
- package/examples/customer-service-demo/oad.yaml +107 -107
- package/package.json +50 -50
- package/src/analytics/index.ts +66 -66
- package/src/channels/discord.ts +192 -192
- package/src/channels/email.ts +177 -177
- package/src/channels/feishu.ts +236 -236
- package/src/channels/index.ts +15 -15
- package/src/channels/slack.ts +160 -160
- package/src/channels/telegram.ts +90 -90
- package/src/channels/voice.ts +106 -106
- package/src/channels/webhook.ts +199 -199
- package/src/channels/websocket.ts +87 -87
- package/src/channels/wechat.ts +149 -149
- package/src/cli.ts +119 -1
- package/src/core/a2a.ts +143 -143
- package/src/core/agent.ts +152 -152
- package/src/core/analytics-engine.ts +186 -186
- package/src/core/auth.ts +57 -57
- package/src/core/cache.ts +141 -141
- package/src/core/compose.ts +77 -77
- package/src/core/config.ts +14 -14
- package/src/core/errors.ts +148 -148
- package/src/core/hitl.ts +138 -138
- package/src/core/logger.ts +57 -57
- package/src/core/orchestrator.ts +215 -215
- package/src/core/performance.ts +187 -187
- package/src/core/rate-limiter.ts +128 -128
- package/src/core/room.ts +109 -109
- package/src/core/runtime.ts +152 -152
- package/src/core/sandbox.ts +101 -101
- package/src/core/security.ts +171 -171
- package/src/core/types.ts +68 -68
- package/src/core/versioning.ts +106 -106
- package/src/core/watch.ts +178 -178
- package/src/core/workflow.ts +235 -235
- package/src/deploy/hermes.ts +156 -156
- package/src/deploy/openclaw.ts +200 -200
- package/src/i18n/index.ts +216 -216
- package/src/index.ts +6 -2
- package/src/memory/deepbrain.ts +108 -108
- package/src/memory/index.ts +34 -34
- package/src/plugins/index.ts +208 -208
- package/src/schema/oad.ts +154 -155
- package/src/skills/base.ts +16 -16
- package/src/skills/document.ts +100 -100
- package/src/skills/http.ts +35 -35
- package/src/skills/index.ts +27 -27
- package/src/skills/scheduler.ts +80 -80
- package/src/skills/webhook-trigger.ts +59 -59
- package/src/templates/code-reviewer.ts +30 -34
- package/src/templates/customer-service.ts +76 -80
- package/src/templates/data-analyst.ts +66 -70
- package/src/templates/executive-assistant.ts +71 -71
- package/src/templates/financial-advisor.ts +60 -60
- package/src/templates/knowledge-base.ts +27 -31
- package/src/templates/legal-assistant.ts +71 -71
- package/src/templates/sales-assistant.ts +75 -79
- package/src/templates/teacher.ts +75 -79
- package/src/testing/index.ts +181 -181
- package/src/tools/calculator.ts +73 -73
- package/src/tools/datetime.ts +149 -149
- package/src/tools/json-transform.ts +187 -187
- package/src/tools/mcp.ts +76 -76
- package/src/tools/text-analysis.ts +116 -116
- package/src/traces/index.ts +132 -0
- package/templates/Dockerfile +15 -15
- package/templates/code-reviewer/README.md +27 -27
- package/templates/code-reviewer/oad.yaml +41 -41
- package/templates/customer-service/README.md +22 -22
- package/templates/customer-service/oad.yaml +36 -36
- package/templates/docker-compose.yml +21 -21
- package/templates/ecommerce-assistant/README.md +45 -45
- package/templates/ecommerce-assistant/oad.yaml +47 -47
- package/templates/knowledge-base/README.md +28 -28
- package/templates/knowledge-base/oad.yaml +38 -38
- package/templates/sales-assistant/README.md +26 -26
- package/templates/sales-assistant/oad.yaml +43 -43
- package/templates/tech-support/README.md +43 -43
- package/templates/tech-support/oad.yaml +45 -45
- package/tests/a2a.test.ts +66 -66
- package/tests/agent.test.ts +72 -72
- package/tests/analytics.test.ts +50 -50
- package/tests/channel.test.ts +39 -39
- package/tests/e2e.test.ts +134 -134
- package/tests/errors.test.ts +83 -83
- package/tests/hitl.test.ts +71 -71
- package/tests/i18n.test.ts +41 -41
- package/tests/mcp.test.ts +54 -54
- package/tests/oad.test.ts +68 -68
- package/tests/performance.test.ts +115 -115
- package/tests/plugin.test.ts +74 -74
- package/tests/room.test.ts +106 -106
- package/tests/runtime.test.ts +42 -42
- package/tests/sandbox.test.ts +46 -46
- package/tests/security.test.ts +60 -60
- package/tests/templates.test.ts +77 -77
- package/tests/v070.test.ts +76 -76
- package/tests/versioning.test.ts +75 -75
- package/tests/voice.test.ts +61 -61
- package/tests/webhook.test.ts +29 -29
- package/tests/workflow.test.ts +143 -143
- package/tsconfig.json +19 -19
- package/vitest.config.ts +9 -9
- package/dist/core/streaming.d.ts +0 -56
- package/dist/core/streaming.js +0 -160
- package/dist/tools/gateway.d.ts +0 -28
- package/dist/tools/gateway.js +0 -177
- package/src/dtv/data.ts +0 -29
- package/src/dtv/trust.ts +0 -43
- package/src/dtv/value.ts +0 -47
- package/src/marketplace/index.ts +0 -223
package/src/templates/teacher.ts
CHANGED
|
@@ -1,79 +1,75 @@
|
|
|
1
|
-
import { BaseSkill } from '../skills/base';
|
|
2
|
-
import type { AgentContext, Message, SkillResult } from '../core/types';
|
|
3
|
-
|
|
4
|
-
export class LessonPlanSkill extends BaseSkill {
|
|
5
|
-
name = 'lesson-plan';
|
|
6
|
-
description = 'Create and manage lesson plans';
|
|
7
|
-
|
|
8
|
-
async execute(_context: AgentContext, message: Message): Promise<SkillResult> {
|
|
9
|
-
const lower = message.content.toLowerCase();
|
|
10
|
-
if (lower.includes('lesson') || lower.includes('plan') || lower.includes('curriculum') || lower.includes('syllabus')) {
|
|
11
|
-
return this.match('I can help create a lesson plan. What subject, grade level, and learning objectives should I focus on?', 0.85);
|
|
12
|
-
}
|
|
13
|
-
return this.noMatch();
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
export class QuizSkill extends BaseSkill {
|
|
18
|
-
name = 'quiz-generator';
|
|
19
|
-
description = 'Generate quizzes and assessments';
|
|
20
|
-
|
|
21
|
-
async execute(_context: AgentContext, message: Message): Promise<SkillResult> {
|
|
22
|
-
const lower = message.content.toLowerCase();
|
|
23
|
-
if (lower.includes('quiz') || lower.includes('test') || lower.includes('assessment') || lower.includes('exam') || lower.includes('question')) {
|
|
24
|
-
return this.match('I\'ll create a quiz for you. What topic, difficulty level, and number of questions would you like?', 0.85);
|
|
25
|
-
}
|
|
26
|
-
return this.noMatch();
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export class ExplainSkill extends BaseSkill {
|
|
31
|
-
name = 'concept-explainer';
|
|
32
|
-
description = 'Explain concepts at appropriate level';
|
|
33
|
-
|
|
34
|
-
async execute(_context: AgentContext, message: Message): Promise<SkillResult> {
|
|
35
|
-
const lower = message.content.toLowerCase();
|
|
36
|
-
if (lower.includes('explain') || lower.includes('what is') || lower.includes('how does') || lower.includes('why')) {
|
|
37
|
-
return this.match('Let me explain that concept. What\'s your current level of understanding so I can tailor my explanation?', 0.75);
|
|
38
|
-
}
|
|
39
|
-
return this.noMatch();
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
export const TEACHER_SYSTEM_PROMPT = `You are a patient and encouraging teacher assistant. Your goals:
|
|
44
|
-
1. Create engaging lesson plans tailored to student level
|
|
45
|
-
2. Generate quizzes and assessments with answer keys
|
|
46
|
-
3. Explain complex concepts using analogies and examples
|
|
47
|
-
4. Provide constructive feedback and encouragement
|
|
48
|
-
5. Adapt teaching style to different learning preferences
|
|
49
|
-
Be patient, use clear language, and always check for understanding. Use the Socratic method when appropriate.`;
|
|
50
|
-
|
|
51
|
-
export function createTeacherConfig() {
|
|
52
|
-
return {
|
|
53
|
-
apiVersion: 'opc/v1' as const,
|
|
54
|
-
kind: 'Agent' as const,
|
|
55
|
-
metadata: {
|
|
56
|
-
name: 'teacher',
|
|
57
|
-
version: '1.0.0',
|
|
58
|
-
description: 'AI teacher assistant with lesson planning, quiz generation, and concept explanation',
|
|
59
|
-
author: 'OPC Agent',
|
|
60
|
-
license: 'Apache-2.0',
|
|
61
|
-
},
|
|
62
|
-
spec: {
|
|
63
|
-
provider: { default: 'openai', allowed: ['openai', 'deepseek', 'qwen'] },
|
|
64
|
-
model: 'gpt-4o-mini',
|
|
65
|
-
systemPrompt: TEACHER_SYSTEM_PROMPT,
|
|
66
|
-
skills: [
|
|
67
|
-
{ name: 'lesson-plan', description: 'Create lesson plans' },
|
|
68
|
-
{ name: 'quiz-generator', description: 'Generate quizzes' },
|
|
69
|
-
{ name: 'concept-explainer', description: 'Explain concepts' },
|
|
70
|
-
],
|
|
71
|
-
channels: [{ type: 'web' as const, port: 3000 }],
|
|
72
|
-
memory: { shortTerm: true, longTerm: true },
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
},
|
|
77
|
-
},
|
|
78
|
-
};
|
|
79
|
-
}
|
|
1
|
+
import { BaseSkill } from '../skills/base';
|
|
2
|
+
import type { AgentContext, Message, SkillResult } from '../core/types';
|
|
3
|
+
|
|
4
|
+
export class LessonPlanSkill extends BaseSkill {
|
|
5
|
+
name = 'lesson-plan';
|
|
6
|
+
description = 'Create and manage lesson plans';
|
|
7
|
+
|
|
8
|
+
async execute(_context: AgentContext, message: Message): Promise<SkillResult> {
|
|
9
|
+
const lower = message.content.toLowerCase();
|
|
10
|
+
if (lower.includes('lesson') || lower.includes('plan') || lower.includes('curriculum') || lower.includes('syllabus')) {
|
|
11
|
+
return this.match('I can help create a lesson plan. What subject, grade level, and learning objectives should I focus on?', 0.85);
|
|
12
|
+
}
|
|
13
|
+
return this.noMatch();
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export class QuizSkill extends BaseSkill {
|
|
18
|
+
name = 'quiz-generator';
|
|
19
|
+
description = 'Generate quizzes and assessments';
|
|
20
|
+
|
|
21
|
+
async execute(_context: AgentContext, message: Message): Promise<SkillResult> {
|
|
22
|
+
const lower = message.content.toLowerCase();
|
|
23
|
+
if (lower.includes('quiz') || lower.includes('test') || lower.includes('assessment') || lower.includes('exam') || lower.includes('question')) {
|
|
24
|
+
return this.match('I\'ll create a quiz for you. What topic, difficulty level, and number of questions would you like?', 0.85);
|
|
25
|
+
}
|
|
26
|
+
return this.noMatch();
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export class ExplainSkill extends BaseSkill {
|
|
31
|
+
name = 'concept-explainer';
|
|
32
|
+
description = 'Explain concepts at appropriate level';
|
|
33
|
+
|
|
34
|
+
async execute(_context: AgentContext, message: Message): Promise<SkillResult> {
|
|
35
|
+
const lower = message.content.toLowerCase();
|
|
36
|
+
if (lower.includes('explain') || lower.includes('what is') || lower.includes('how does') || lower.includes('why')) {
|
|
37
|
+
return this.match('Let me explain that concept. What\'s your current level of understanding so I can tailor my explanation?', 0.75);
|
|
38
|
+
}
|
|
39
|
+
return this.noMatch();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export const TEACHER_SYSTEM_PROMPT = `You are a patient and encouraging teacher assistant. Your goals:
|
|
44
|
+
1. Create engaging lesson plans tailored to student level
|
|
45
|
+
2. Generate quizzes and assessments with answer keys
|
|
46
|
+
3. Explain complex concepts using analogies and examples
|
|
47
|
+
4. Provide constructive feedback and encouragement
|
|
48
|
+
5. Adapt teaching style to different learning preferences
|
|
49
|
+
Be patient, use clear language, and always check for understanding. Use the Socratic method when appropriate.`;
|
|
50
|
+
|
|
51
|
+
export function createTeacherConfig() {
|
|
52
|
+
return {
|
|
53
|
+
apiVersion: 'opc/v1' as const,
|
|
54
|
+
kind: 'Agent' as const,
|
|
55
|
+
metadata: {
|
|
56
|
+
name: 'teacher',
|
|
57
|
+
version: '1.0.0',
|
|
58
|
+
description: 'AI teacher assistant with lesson planning, quiz generation, and concept explanation',
|
|
59
|
+
author: 'OPC Agent',
|
|
60
|
+
license: 'Apache-2.0',
|
|
61
|
+
},
|
|
62
|
+
spec: {
|
|
63
|
+
provider: { default: 'openai', allowed: ['openai', 'deepseek', 'qwen'] },
|
|
64
|
+
model: 'gpt-4o-mini',
|
|
65
|
+
systemPrompt: TEACHER_SYSTEM_PROMPT,
|
|
66
|
+
skills: [
|
|
67
|
+
{ name: 'lesson-plan', description: 'Create lesson plans' },
|
|
68
|
+
{ name: 'quiz-generator', description: 'Generate quizzes' },
|
|
69
|
+
{ name: 'concept-explainer', description: 'Explain concepts' },
|
|
70
|
+
],
|
|
71
|
+
channels: [{ type: 'web' as const, port: 3000 }],
|
|
72
|
+
memory: { shortTerm: true, longTerm: true },
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
}
|
package/src/testing/index.ts
CHANGED
|
@@ -1,181 +1,181 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Agent Testing Framework - Define test cases in OAD, run with `opc test`.
|
|
3
|
-
* Supports assertions on response content, tool calls, and latency.
|
|
4
|
-
*/
|
|
5
|
-
import * as fs from 'fs';
|
|
6
|
-
import * as path from 'path';
|
|
7
|
-
import * as yaml from 'js-yaml';
|
|
8
|
-
import { AgentRuntime } from '../core/runtime';
|
|
9
|
-
|
|
10
|
-
export interface TestCase {
|
|
11
|
-
name: string;
|
|
12
|
-
input: string;
|
|
13
|
-
expect?: {
|
|
14
|
-
contains?: string[];
|
|
15
|
-
notContains?: string[];
|
|
16
|
-
toolCalled?: string[];
|
|
17
|
-
maxLatencyMs?: number;
|
|
18
|
-
};
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export interface TestResult {
|
|
22
|
-
name: string;
|
|
23
|
-
passed: boolean;
|
|
24
|
-
durationMs: number;
|
|
25
|
-
response?: string;
|
|
26
|
-
failures: string[];
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
export interface TestReport {
|
|
30
|
-
total: number;
|
|
31
|
-
passed: number;
|
|
32
|
-
failed: number;
|
|
33
|
-
duration: number;
|
|
34
|
-
results: TestResult[];
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Load test cases from OAD spec.testing or a separate test file.
|
|
39
|
-
*/
|
|
40
|
-
export function loadTestCases(oadPath: string): TestCase[] {
|
|
41
|
-
const raw = fs.readFileSync(oadPath, 'utf-8');
|
|
42
|
-
const config = yaml.load(raw) as any;
|
|
43
|
-
|
|
44
|
-
// Check spec.testing.cases
|
|
45
|
-
if (config?.spec?.testing?.cases) {
|
|
46
|
-
return config.spec.testing.cases;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
// Check for companion test file
|
|
50
|
-
const dir = path.dirname(oadPath);
|
|
51
|
-
const testFile = path.join(dir, 'tests.yaml');
|
|
52
|
-
if (fs.existsSync(testFile)) {
|
|
53
|
-
const testRaw = fs.readFileSync(testFile, 'utf-8');
|
|
54
|
-
const testConfig = yaml.load(testRaw) as any;
|
|
55
|
-
return testConfig?.cases ?? testConfig ?? [];
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
return [];
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
/**
|
|
62
|
-
* Run all test cases against an agent.
|
|
63
|
-
*/
|
|
64
|
-
export async function runTests(oadPath: string): Promise<TestReport> {
|
|
65
|
-
const cases = loadTestCases(oadPath);
|
|
66
|
-
const results: TestResult[] = [];
|
|
67
|
-
const startTime = Date.now();
|
|
68
|
-
|
|
69
|
-
if (cases.length === 0) {
|
|
70
|
-
// Generate default smoke test
|
|
71
|
-
cases.push({
|
|
72
|
-
name: 'smoke-test',
|
|
73
|
-
input: 'Hello! What can you help me with?',
|
|
74
|
-
expect: { maxLatencyMs: 30000 },
|
|
75
|
-
});
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
const runtime = new AgentRuntime();
|
|
79
|
-
await runtime.loadConfig(oadPath);
|
|
80
|
-
const agent = await runtime.initialize();
|
|
81
|
-
|
|
82
|
-
for (const tc of cases) {
|
|
83
|
-
const result: TestResult = {
|
|
84
|
-
name: tc.name,
|
|
85
|
-
passed: true,
|
|
86
|
-
durationMs: 0,
|
|
87
|
-
failures: [],
|
|
88
|
-
};
|
|
89
|
-
|
|
90
|
-
const t0 = Date.now();
|
|
91
|
-
try {
|
|
92
|
-
const response = await agent.handleMessage({
|
|
93
|
-
id: `test_${Date.now()}`,
|
|
94
|
-
role: 'user',
|
|
95
|
-
content: tc.input,
|
|
96
|
-
timestamp: Date.now(),
|
|
97
|
-
});
|
|
98
|
-
result.durationMs = Date.now() - t0;
|
|
99
|
-
result.response = response.content;
|
|
100
|
-
|
|
101
|
-
if (tc.expect) {
|
|
102
|
-
// Check contains
|
|
103
|
-
if (tc.expect.contains) {
|
|
104
|
-
for (const s of tc.expect.contains) {
|
|
105
|
-
if (!response.content.toLowerCase().includes(s.toLowerCase())) {
|
|
106
|
-
result.failures.push(`Expected response to contain "${s}"`);
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
// Check notContains
|
|
111
|
-
if (tc.expect.notContains) {
|
|
112
|
-
for (const s of tc.expect.notContains) {
|
|
113
|
-
if (response.content.toLowerCase().includes(s.toLowerCase())) {
|
|
114
|
-
result.failures.push(`Expected response NOT to contain "${s}"`);
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
// Check latency
|
|
119
|
-
if (tc.expect.maxLatencyMs && result.durationMs > tc.expect.maxLatencyMs) {
|
|
120
|
-
result.failures.push(`Latency ${result.durationMs}ms exceeded max ${tc.expect.maxLatencyMs}ms`);
|
|
121
|
-
}
|
|
122
|
-
// Check tool calls (from metadata if available)
|
|
123
|
-
if (tc.expect.toolCalled && (response as any).toolsCalled) {
|
|
124
|
-
for (const tool of tc.expect.toolCalled) {
|
|
125
|
-
if (!(response as any).toolsCalled.includes(tool)) {
|
|
126
|
-
result.failures.push(`Expected tool "${tool}" to be called`);
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
result.passed = result.failures.length === 0;
|
|
133
|
-
} catch (err) {
|
|
134
|
-
result.durationMs = Date.now() - t0;
|
|
135
|
-
result.passed = false;
|
|
136
|
-
result.failures.push(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
results.push(result);
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
const totalDuration = Date.now() - startTime;
|
|
143
|
-
const passed = results.filter(r => r.passed).length;
|
|
144
|
-
|
|
145
|
-
return {
|
|
146
|
-
total: results.length,
|
|
147
|
-
passed,
|
|
148
|
-
failed: results.length - passed,
|
|
149
|
-
duration: totalDuration,
|
|
150
|
-
results,
|
|
151
|
-
};
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
/**
|
|
155
|
-
* Format test report for console output.
|
|
156
|
-
*/
|
|
157
|
-
export function formatReport(report: TestReport): string {
|
|
158
|
-
const lines: string[] = [];
|
|
159
|
-
lines.push('');
|
|
160
|
-
lines.push('═══════════════════════════════════════════');
|
|
161
|
-
lines.push(' OPC Agent Test Report');
|
|
162
|
-
lines.push('═══════════════════════════════════════════');
|
|
163
|
-
lines.push('');
|
|
164
|
-
|
|
165
|
-
for (const r of report.results) {
|
|
166
|
-
const icon = r.passed ? '✔' : '✘';
|
|
167
|
-
const status = r.passed ? 'PASS' : 'FAIL';
|
|
168
|
-
lines.push(` ${icon} [${status}] ${r.name} (${r.durationMs}ms)`);
|
|
169
|
-
for (const f of r.failures) {
|
|
170
|
-
lines.push(` → ${f}`);
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
lines.push('');
|
|
175
|
-
lines.push('───────────────────────────────────────────');
|
|
176
|
-
lines.push(` Total: ${report.total} Passed: ${report.passed} Failed: ${report.failed} Duration: ${report.duration}ms`);
|
|
177
|
-
lines.push('───────────────────────────────────────────');
|
|
178
|
-
lines.push('');
|
|
179
|
-
|
|
180
|
-
return lines.join('\n');
|
|
181
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Agent Testing Framework - Define test cases in OAD, run with `opc test`.
|
|
3
|
+
* Supports assertions on response content, tool calls, and latency.
|
|
4
|
+
*/
|
|
5
|
+
import * as fs from 'fs';
|
|
6
|
+
import * as path from 'path';
|
|
7
|
+
import * as yaml from 'js-yaml';
|
|
8
|
+
import { AgentRuntime } from '../core/runtime';
|
|
9
|
+
|
|
10
|
+
export interface TestCase {
|
|
11
|
+
name: string;
|
|
12
|
+
input: string;
|
|
13
|
+
expect?: {
|
|
14
|
+
contains?: string[];
|
|
15
|
+
notContains?: string[];
|
|
16
|
+
toolCalled?: string[];
|
|
17
|
+
maxLatencyMs?: number;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface TestResult {
|
|
22
|
+
name: string;
|
|
23
|
+
passed: boolean;
|
|
24
|
+
durationMs: number;
|
|
25
|
+
response?: string;
|
|
26
|
+
failures: string[];
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface TestReport {
|
|
30
|
+
total: number;
|
|
31
|
+
passed: number;
|
|
32
|
+
failed: number;
|
|
33
|
+
duration: number;
|
|
34
|
+
results: TestResult[];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Load test cases from OAD spec.testing or a separate test file.
|
|
39
|
+
*/
|
|
40
|
+
export function loadTestCases(oadPath: string): TestCase[] {
|
|
41
|
+
const raw = fs.readFileSync(oadPath, 'utf-8');
|
|
42
|
+
const config = yaml.load(raw) as any;
|
|
43
|
+
|
|
44
|
+
// Check spec.testing.cases
|
|
45
|
+
if (config?.spec?.testing?.cases) {
|
|
46
|
+
return config.spec.testing.cases;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Check for companion test file
|
|
50
|
+
const dir = path.dirname(oadPath);
|
|
51
|
+
const testFile = path.join(dir, 'tests.yaml');
|
|
52
|
+
if (fs.existsSync(testFile)) {
|
|
53
|
+
const testRaw = fs.readFileSync(testFile, 'utf-8');
|
|
54
|
+
const testConfig = yaml.load(testRaw) as any;
|
|
55
|
+
return testConfig?.cases ?? testConfig ?? [];
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return [];
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Run all test cases against an agent.
|
|
63
|
+
*/
|
|
64
|
+
export async function runTests(oadPath: string): Promise<TestReport> {
|
|
65
|
+
const cases = loadTestCases(oadPath);
|
|
66
|
+
const results: TestResult[] = [];
|
|
67
|
+
const startTime = Date.now();
|
|
68
|
+
|
|
69
|
+
if (cases.length === 0) {
|
|
70
|
+
// Generate default smoke test
|
|
71
|
+
cases.push({
|
|
72
|
+
name: 'smoke-test',
|
|
73
|
+
input: 'Hello! What can you help me with?',
|
|
74
|
+
expect: { maxLatencyMs: 30000 },
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const runtime = new AgentRuntime();
|
|
79
|
+
await runtime.loadConfig(oadPath);
|
|
80
|
+
const agent = await runtime.initialize();
|
|
81
|
+
|
|
82
|
+
for (const tc of cases) {
|
|
83
|
+
const result: TestResult = {
|
|
84
|
+
name: tc.name,
|
|
85
|
+
passed: true,
|
|
86
|
+
durationMs: 0,
|
|
87
|
+
failures: [],
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
const t0 = Date.now();
|
|
91
|
+
try {
|
|
92
|
+
const response = await agent.handleMessage({
|
|
93
|
+
id: `test_${Date.now()}`,
|
|
94
|
+
role: 'user',
|
|
95
|
+
content: tc.input,
|
|
96
|
+
timestamp: Date.now(),
|
|
97
|
+
});
|
|
98
|
+
result.durationMs = Date.now() - t0;
|
|
99
|
+
result.response = response.content;
|
|
100
|
+
|
|
101
|
+
if (tc.expect) {
|
|
102
|
+
// Check contains
|
|
103
|
+
if (tc.expect.contains) {
|
|
104
|
+
for (const s of tc.expect.contains) {
|
|
105
|
+
if (!response.content.toLowerCase().includes(s.toLowerCase())) {
|
|
106
|
+
result.failures.push(`Expected response to contain "${s}"`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
// Check notContains
|
|
111
|
+
if (tc.expect.notContains) {
|
|
112
|
+
for (const s of tc.expect.notContains) {
|
|
113
|
+
if (response.content.toLowerCase().includes(s.toLowerCase())) {
|
|
114
|
+
result.failures.push(`Expected response NOT to contain "${s}"`);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
// Check latency
|
|
119
|
+
if (tc.expect.maxLatencyMs && result.durationMs > tc.expect.maxLatencyMs) {
|
|
120
|
+
result.failures.push(`Latency ${result.durationMs}ms exceeded max ${tc.expect.maxLatencyMs}ms`);
|
|
121
|
+
}
|
|
122
|
+
// Check tool calls (from metadata if available)
|
|
123
|
+
if (tc.expect.toolCalled && (response as any).toolsCalled) {
|
|
124
|
+
for (const tool of tc.expect.toolCalled) {
|
|
125
|
+
if (!(response as any).toolsCalled.includes(tool)) {
|
|
126
|
+
result.failures.push(`Expected tool "${tool}" to be called`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
result.passed = result.failures.length === 0;
|
|
133
|
+
} catch (err) {
|
|
134
|
+
result.durationMs = Date.now() - t0;
|
|
135
|
+
result.passed = false;
|
|
136
|
+
result.failures.push(`Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
results.push(result);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const totalDuration = Date.now() - startTime;
|
|
143
|
+
const passed = results.filter(r => r.passed).length;
|
|
144
|
+
|
|
145
|
+
return {
|
|
146
|
+
total: results.length,
|
|
147
|
+
passed,
|
|
148
|
+
failed: results.length - passed,
|
|
149
|
+
duration: totalDuration,
|
|
150
|
+
results,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Format test report for console output.
|
|
156
|
+
*/
|
|
157
|
+
export function formatReport(report: TestReport): string {
|
|
158
|
+
const lines: string[] = [];
|
|
159
|
+
lines.push('');
|
|
160
|
+
lines.push('═══════════════════════════════════════════');
|
|
161
|
+
lines.push(' OPC Agent Test Report');
|
|
162
|
+
lines.push('═══════════════════════════════════════════');
|
|
163
|
+
lines.push('');
|
|
164
|
+
|
|
165
|
+
for (const r of report.results) {
|
|
166
|
+
const icon = r.passed ? '✔' : '✘';
|
|
167
|
+
const status = r.passed ? 'PASS' : 'FAIL';
|
|
168
|
+
lines.push(` ${icon} [${status}] ${r.name} (${r.durationMs}ms)`);
|
|
169
|
+
for (const f of r.failures) {
|
|
170
|
+
lines.push(` → ${f}`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
lines.push('');
|
|
175
|
+
lines.push('───────────────────────────────────────────');
|
|
176
|
+
lines.push(` Total: ${report.total} Passed: ${report.passed} Failed: ${report.failed} Duration: ${report.duration}ms`);
|
|
177
|
+
lines.push('───────────────────────────────────────────');
|
|
178
|
+
lines.push('');
|
|
179
|
+
|
|
180
|
+
return lines.join('\n');
|
|
181
|
+
}
|
package/src/tools/calculator.ts
CHANGED
|
@@ -1,73 +1,73 @@
|
|
|
1
|
-
import type { MCPTool, MCPToolResult } from './mcp';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Calculator Tool — v0.8.0
|
|
5
|
-
* Safe math expression evaluation as an LLM function tool.
|
|
6
|
-
*/
|
|
7
|
-
export const CalculatorTool: MCPTool = {
|
|
8
|
-
name: 'calculator',
|
|
9
|
-
description: 'Evaluate a mathematical expression. Supports basic arithmetic, powers, sqrt, abs, min, max, round, ceil, floor, PI, E.',
|
|
10
|
-
inputSchema: {
|
|
11
|
-
type: 'object',
|
|
12
|
-
properties: {
|
|
13
|
-
expression: {
|
|
14
|
-
type: 'string',
|
|
15
|
-
description: 'Mathematical expression to evaluate, e.g. "2 + 3 * 4" or "sqrt(144) + PI"',
|
|
16
|
-
},
|
|
17
|
-
},
|
|
18
|
-
required: ['expression'],
|
|
19
|
-
},
|
|
20
|
-
|
|
21
|
-
async execute(input: Record<string, unknown>): Promise<MCPToolResult> {
|
|
22
|
-
const expr = String(input.expression ?? '');
|
|
23
|
-
try {
|
|
24
|
-
const result = safeEval(expr);
|
|
25
|
-
return { content: String(result) };
|
|
26
|
-
} catch (err) {
|
|
27
|
-
return { content: `Error: ${(err as Error).message}`, isError: true };
|
|
28
|
-
}
|
|
29
|
-
},
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
/** Safe math evaluator — no eval(), no arbitrary code */
|
|
33
|
-
function safeEval(expr: string): number {
|
|
34
|
-
// Whitelist: digits, operators, parens, dots, commas, spaces, and known functions
|
|
35
|
-
const sanitized = expr.replace(/\s+/g, '');
|
|
36
|
-
const allowed = /^[0-9+\-*/().,%^a-zA-Z_]+$/;
|
|
37
|
-
if (!allowed.test(sanitized)) {
|
|
38
|
-
throw new Error('Invalid characters in expression');
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
// Replace known math functions/constants
|
|
42
|
-
const prepared = sanitized
|
|
43
|
-
.replace(/\bPI\b/gi, String(Math.PI))
|
|
44
|
-
.replace(/\bE\b/g, String(Math.E))
|
|
45
|
-
.replace(/\bsqrt\b/gi, 'Math.sqrt')
|
|
46
|
-
.replace(/\babs\b/gi, 'Math.abs')
|
|
47
|
-
.replace(/\bmin\b/gi, 'Math.min')
|
|
48
|
-
.replace(/\bmax\b/gi, 'Math.max')
|
|
49
|
-
.replace(/\bround\b/gi, 'Math.round')
|
|
50
|
-
.replace(/\bceil\b/gi, 'Math.ceil')
|
|
51
|
-
.replace(/\bfloor\b/gi, 'Math.floor')
|
|
52
|
-
.replace(/\bpow\b/gi, 'Math.pow')
|
|
53
|
-
.replace(/\blog\b/gi, 'Math.log')
|
|
54
|
-
.replace(/\blog10\b/gi, 'Math.log10')
|
|
55
|
-
.replace(/\bsin\b/gi, 'Math.sin')
|
|
56
|
-
.replace(/\bcos\b/gi, 'Math.cos')
|
|
57
|
-
.replace(/\btan\b/gi, 'Math.tan')
|
|
58
|
-
.replace(/\^/g, '**');
|
|
59
|
-
|
|
60
|
-
// Block anything that isn't math
|
|
61
|
-
if (/[a-zA-Z_]/.test(prepared.replace(/Math\.\w+/g, ''))) {
|
|
62
|
-
throw new Error('Unsupported function or variable in expression');
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
// Use Function constructor with restricted scope
|
|
66
|
-
const fn = new Function('Math', `"use strict"; return (${prepared});`);
|
|
67
|
-
const result = fn(Math);
|
|
68
|
-
|
|
69
|
-
if (typeof result !== 'number' || !isFinite(result)) {
|
|
70
|
-
throw new Error('Expression did not evaluate to a finite number');
|
|
71
|
-
}
|
|
72
|
-
return result;
|
|
73
|
-
}
|
|
1
|
+
import type { MCPTool, MCPToolResult } from './mcp';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Calculator Tool — v0.8.0
|
|
5
|
+
* Safe math expression evaluation as an LLM function tool.
|
|
6
|
+
*/
|
|
7
|
+
export const CalculatorTool: MCPTool = {
|
|
8
|
+
name: 'calculator',
|
|
9
|
+
description: 'Evaluate a mathematical expression. Supports basic arithmetic, powers, sqrt, abs, min, max, round, ceil, floor, PI, E.',
|
|
10
|
+
inputSchema: {
|
|
11
|
+
type: 'object',
|
|
12
|
+
properties: {
|
|
13
|
+
expression: {
|
|
14
|
+
type: 'string',
|
|
15
|
+
description: 'Mathematical expression to evaluate, e.g. "2 + 3 * 4" or "sqrt(144) + PI"',
|
|
16
|
+
},
|
|
17
|
+
},
|
|
18
|
+
required: ['expression'],
|
|
19
|
+
},
|
|
20
|
+
|
|
21
|
+
async execute(input: Record<string, unknown>): Promise<MCPToolResult> {
|
|
22
|
+
const expr = String(input.expression ?? '');
|
|
23
|
+
try {
|
|
24
|
+
const result = safeEval(expr);
|
|
25
|
+
return { content: String(result) };
|
|
26
|
+
} catch (err) {
|
|
27
|
+
return { content: `Error: ${(err as Error).message}`, isError: true };
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
/** Safe math evaluator — no eval(), no arbitrary code */
|
|
33
|
+
function safeEval(expr: string): number {
|
|
34
|
+
// Whitelist: digits, operators, parens, dots, commas, spaces, and known functions
|
|
35
|
+
const sanitized = expr.replace(/\s+/g, '');
|
|
36
|
+
const allowed = /^[0-9+\-*/().,%^a-zA-Z_]+$/;
|
|
37
|
+
if (!allowed.test(sanitized)) {
|
|
38
|
+
throw new Error('Invalid characters in expression');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Replace known math functions/constants
|
|
42
|
+
const prepared = sanitized
|
|
43
|
+
.replace(/\bPI\b/gi, String(Math.PI))
|
|
44
|
+
.replace(/\bE\b/g, String(Math.E))
|
|
45
|
+
.replace(/\bsqrt\b/gi, 'Math.sqrt')
|
|
46
|
+
.replace(/\babs\b/gi, 'Math.abs')
|
|
47
|
+
.replace(/\bmin\b/gi, 'Math.min')
|
|
48
|
+
.replace(/\bmax\b/gi, 'Math.max')
|
|
49
|
+
.replace(/\bround\b/gi, 'Math.round')
|
|
50
|
+
.replace(/\bceil\b/gi, 'Math.ceil')
|
|
51
|
+
.replace(/\bfloor\b/gi, 'Math.floor')
|
|
52
|
+
.replace(/\bpow\b/gi, 'Math.pow')
|
|
53
|
+
.replace(/\blog\b/gi, 'Math.log')
|
|
54
|
+
.replace(/\blog10\b/gi, 'Math.log10')
|
|
55
|
+
.replace(/\bsin\b/gi, 'Math.sin')
|
|
56
|
+
.replace(/\bcos\b/gi, 'Math.cos')
|
|
57
|
+
.replace(/\btan\b/gi, 'Math.tan')
|
|
58
|
+
.replace(/\^/g, '**');
|
|
59
|
+
|
|
60
|
+
// Block anything that isn't math
|
|
61
|
+
if (/[a-zA-Z_]/.test(prepared.replace(/Math\.\w+/g, ''))) {
|
|
62
|
+
throw new Error('Unsupported function or variable in expression');
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Use Function constructor with restricted scope
|
|
66
|
+
const fn = new Function('Math', `"use strict"; return (${prepared});`);
|
|
67
|
+
const result = fn(Math);
|
|
68
|
+
|
|
69
|
+
if (typeof result !== 'number' || !isFinite(result)) {
|
|
70
|
+
throw new Error('Expression did not evaluate to a finite number');
|
|
71
|
+
}
|
|
72
|
+
return result;
|
|
73
|
+
}
|