opc-agent 4.0.44 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/ISSUE_TEMPLATE/bug_report.md +20 -20
- package/.github/ISSUE_TEMPLATE/feature_request.md +14 -14
- package/.github/PULL_REQUEST_TEMPLATE.md +13 -13
- package/CHANGELOG.md +48 -48
- package/CONTRIBUTING.md +36 -36
- package/README.zh-CN.md +497 -497
- package/dist/channels/wechat.js +6 -6
- package/dist/cli.js +2 -2
- package/dist/core/runtime.js +18 -0
- package/dist/deploy/index.js +56 -56
- package/dist/providers/index.js +39 -13
- package/dist/studio/server.js +211 -20
- package/dist/studio-ui/index.html +279 -24
- package/dist/ui/components.js +105 -105
- package/examples/README.md +22 -22
- package/examples/basic-agent.ts +90 -90
- package/examples/brain-integration.ts +71 -71
- package/examples/multi-channel.ts +74 -74
- package/fix-sidebar.mjs +188 -188
- package/install.ps1 +154 -154
- package/install.sh +164 -164
- package/package.json +1 -1
- package/scripts/install.ps1 +31 -31
- package/scripts/install.sh +40 -40
- package/serve-studio.js +13 -13
- package/serve-test.js +25 -25
- package/src/channels/dingtalk.ts +46 -46
- package/src/channels/email.ts +351 -351
- package/src/channels/feishu.ts +349 -349
- package/src/channels/googlechat.ts +42 -42
- package/src/channels/imessage.ts +31 -31
- package/src/channels/irc.ts +82 -82
- package/src/channels/line.ts +32 -32
- package/src/channels/matrix.ts +33 -33
- package/src/channels/mattermost.ts +57 -57
- package/src/channels/msteams.ts +32 -32
- package/src/channels/nostr.ts +32 -32
- package/src/channels/qq.ts +33 -33
- package/src/channels/signal.ts +32 -32
- package/src/channels/sms.ts +33 -33
- package/src/channels/telegram.ts +616 -616
- package/src/channels/twitch.ts +65 -65
- package/src/channels/voice-call.ts +100 -100
- package/src/channels/websocket.ts +399 -399
- package/src/channels/wechat.ts +329 -329
- package/src/channels/whatsapp.ts +32 -32
- package/src/cli/chat.ts +99 -99
- package/src/cli/setup.ts +314 -314
- package/src/cli.ts +2 -2
- package/src/core/agent.ts +476 -476
- package/src/core/api-server.ts +277 -277
- package/src/core/audio.ts +98 -98
- package/src/core/collaboration.ts +275 -275
- package/src/core/context-discovery.ts +85 -85
- package/src/core/context-refs.ts +140 -140
- package/src/core/gateway.ts +106 -106
- package/src/core/heartbeat.ts +51 -51
- package/src/core/hooks.ts +105 -105
- package/src/core/ide-bridge.ts +133 -133
- package/src/core/node-network.ts +86 -86
- package/src/core/profiles.ts +122 -122
- package/src/core/runtime.ts +18 -0
- package/src/core/scheduler.ts +187 -187
- package/src/core/session-manager.ts +137 -137
- package/src/core/subagent.ts +98 -98
- package/src/core/vision.ts +180 -180
- package/src/core/workflow-graph.ts +365 -365
- package/src/daemon.ts +96 -96
- package/src/deploy/index.ts +255 -255
- package/src/doctor.ts +156 -156
- package/src/eval/index.ts +211 -211
- package/src/eval/suites/basic.json +16 -16
- package/src/eval/suites/memory.json +12 -12
- package/src/eval/suites/safety.json +14 -14
- package/src/hub/brain-seed.ts +54 -54
- package/src/hub/client.ts +60 -60
- package/src/mcp/servers/calculator-mcp.ts +65 -65
- package/src/mcp/servers/crypto-mcp.ts +73 -73
- package/src/mcp/servers/database-mcp.ts +72 -72
- package/src/mcp/servers/datetime-mcp.ts +69 -69
- package/src/mcp/servers/filesystem.ts +66 -66
- package/src/mcp/servers/github-mcp.ts +58 -58
- package/src/mcp/servers/index.ts +63 -63
- package/src/mcp/servers/json-mcp.ts +102 -102
- package/src/mcp/servers/memory-mcp.ts +56 -56
- package/src/mcp/servers/regex-mcp.ts +53 -53
- package/src/mcp/servers/web-mcp.ts +49 -49
- package/src/memory/context-compressor.ts +189 -189
- package/src/memory/seed-loader.ts +212 -212
- package/src/memory/user-profiler.ts +215 -215
- package/src/plugins/content-filter.ts +23 -23
- package/src/plugins/logger.ts +18 -18
- package/src/plugins/rate-limiter.ts +38 -38
- package/src/protocols/a2a/client.ts +132 -132
- package/src/protocols/a2a/index.ts +8 -8
- package/src/protocols/a2a/server.ts +333 -333
- package/src/protocols/a2a/types.ts +88 -88
- package/src/protocols/a2a/utils.ts +50 -50
- package/src/protocols/agui/client.ts +83 -83
- package/src/protocols/agui/index.ts +4 -4
- package/src/protocols/agui/server.ts +218 -218
- package/src/protocols/agui/types.ts +153 -153
- package/src/protocols/index.ts +2 -2
- package/src/protocols/mcp/agent-tools.ts +134 -134
- package/src/protocols/mcp/index.ts +8 -8
- package/src/protocols/mcp/server.ts +262 -262
- package/src/protocols/mcp/types.ts +69 -69
- package/src/providers/index.ts +632 -608
- package/src/publish/index.ts +376 -376
- package/src/scheduler/cron-engine.ts +191 -191
- package/src/scheduler/index.ts +2 -2
- package/src/schema/oad.ts +217 -217
- package/src/security/approval.ts +131 -131
- package/src/security/approvals.ts +143 -143
- package/src/security/elevated.ts +105 -105
- package/src/security/guardrails.ts +248 -248
- package/src/security/index.ts +9 -9
- package/src/security/keys.ts +87 -87
- package/src/security/secrets.ts +129 -129
- package/src/skills/builtin/index.ts +408 -408
- package/src/skills/marketplace.ts +113 -113
- package/src/skills/types.ts +42 -42
- package/src/studio/server.ts +209 -22
- package/src/studio/templates-data.ts +178 -178
- package/src/studio-ui/index.html +279 -24
- package/src/telemetry/index.ts +324 -324
- package/src/tools/builtin/browser.ts +299 -299
- package/src/tools/builtin/datetime.ts +41 -41
- package/src/tools/builtin/file.ts +107 -107
- package/src/tools/builtin/home-assistant.ts +116 -116
- package/src/tools/builtin/rl-tools.ts +243 -243
- package/src/tools/builtin/shell.ts +43 -43
- package/src/tools/builtin/vision.ts +64 -64
- package/src/tools/builtin/web-search.ts +126 -126
- package/src/tools/builtin/web.ts +35 -35
- package/src/tools/document-processor.ts +213 -213
- package/src/tools/image-generator.ts +150 -150
- package/src/tools/integrations/calendar.ts +73 -73
- package/src/tools/integrations/code-exec.ts +39 -39
- package/src/tools/integrations/csv-analyzer.ts +92 -92
- package/src/tools/integrations/database.ts +44 -44
- package/src/tools/integrations/email-send.ts +76 -76
- package/src/tools/integrations/git-tool.ts +42 -42
- package/src/tools/integrations/github-tool.ts +76 -76
- package/src/tools/integrations/image-gen.ts +56 -56
- package/src/tools/integrations/index.ts +92 -92
- package/src/tools/integrations/jira.ts +83 -83
- package/src/tools/integrations/notion.ts +71 -71
- package/src/tools/integrations/npm-tool.ts +48 -48
- package/src/tools/integrations/pdf-reader.ts +58 -58
- package/src/tools/integrations/slack.ts +65 -65
- package/src/tools/integrations/summarizer.ts +49 -49
- package/src/tools/integrations/translator.ts +48 -48
- package/src/tools/integrations/trello.ts +60 -60
- package/src/tools/integrations/vector-search.ts +42 -42
- package/src/tools/integrations/web-scraper.ts +47 -47
- package/src/tools/integrations/web-search.ts +58 -58
- package/src/tools/integrations/webhook.ts +38 -38
- package/src/tools/mcp-client.ts +131 -131
- package/src/tools/web-scraper.ts +179 -179
- package/src/tools/web-search.ts +180 -180
- package/src/ui/components.ts +127 -127
- package/srv-out.txt +1 -1
- package/templates/ecommerce-assistant/README.md +45 -45
- package/templates/ecommerce-assistant/oad.yaml +47 -47
- package/templates/tech-support/README.md +43 -43
- package/templates/tech-support/oad.yaml +45 -45
- package/test-agent/Dockerfile +9 -9
- package/test-agent/README.md +50 -50
- package/test-agent/agent.yaml +23 -23
- package/test-agent/docker-compose.yml +11 -11
- package/test-agent/oad.yaml +31 -31
- package/test-agent/package-lock.json +1492 -1492
- package/test-agent/package.json +17 -17
- package/test-agent/src/index.ts +24 -24
- package/test-agent/src/skills/echo.ts +15 -15
- package/test-agent/tsconfig.json +24 -24
- package/test-full.js +43 -43
- package/test-sidebar.js +22 -22
- package/test-studio3.js +75 -75
- package/test-studio4.js +41 -41
- package/tests/a2a-protocol.test.ts +285 -285
- package/tests/agui-protocol.test.ts +246 -246
- package/tests/api-server.test.ts +148 -148
- package/tests/approvals.test.ts +89 -89
- package/tests/audio.test.ts +40 -40
- package/tests/brain-seed-extended.test.ts +490 -490
- package/tests/brain-seed.test.ts +239 -239
- package/tests/browser.test.ts +179 -179
- package/tests/channels/discord.test.ts +79 -79
- package/tests/channels/email.test.ts +148 -148
- package/tests/channels/feishu.test.ts +123 -123
- package/tests/channels/telegram.test.ts +129 -129
- package/tests/channels/websocket.test.ts +53 -53
- package/tests/channels/wechat.test.ts +170 -170
- package/tests/channels-extra.test.ts +45 -45
- package/tests/chat-cli.test.ts +160 -160
- package/tests/cli.test.ts +46 -46
- package/tests/context-compressor.test.ts +172 -172
- package/tests/context-refs.test.ts +121 -121
- package/tests/cron-engine.test.ts +101 -101
- package/tests/daemon.test.ts +135 -135
- package/tests/deepbrain-wire.test.ts +234 -234
- package/tests/deploy-and-dag.test.ts +196 -196
- package/tests/doctor.test.ts +38 -38
- package/tests/document-processor.test.ts +69 -69
- package/tests/e2e-nocode.test.ts +442 -442
- package/tests/elevated.test.ts +69 -69
- package/tests/eval.test.ts +173 -173
- package/tests/gateway.test.ts +63 -63
- package/tests/guardrails.test.ts +177 -177
- package/tests/home-assistant.test.ts +40 -40
- package/tests/hooks.test.ts +79 -79
- package/tests/ide-bridge.test.ts +38 -38
- package/tests/image-generator.test.ts +84 -84
- package/tests/init-role.test.ts +124 -124
- package/tests/integrations.test.ts +249 -249
- package/tests/mcp-client.test.ts +92 -92
- package/tests/mcp-server.test.ts +178 -178
- package/tests/mcp-servers.test.ts +260 -260
- package/tests/node-network.test.ts +74 -74
- package/tests/plugin-a2a-enhanced.test.ts +230 -230
- package/tests/profiles.test.ts +61 -61
- package/tests/publish.test.ts +231 -231
- package/tests/rl-tools.test.ts +93 -93
- package/tests/sandbox-manager.test.ts +46 -46
- package/tests/scheduler.test.ts +200 -200
- package/tests/secrets.test.ts +107 -107
- package/tests/security-enhanced.test.ts +233 -233
- package/tests/settings-api.test.ts +148 -148
- package/tests/setup.test.ts +73 -73
- package/tests/subagent.test.ts +193 -193
- package/tests/telegram-discord.test.ts +60 -60
- package/tests/telemetry.test.ts +186 -186
- package/tests/user-profiler.test.ts +169 -169
- package/tests/v090-features.test.ts +254 -254
- package/tests/vision.test.ts +61 -61
- package/tests/voice-call.test.ts +47 -47
- package/tests/voice-enhanced.test.ts +169 -169
- package/tests/voice-interaction.test.ts +38 -38
- package/tests/web-search.test.ts +155 -155
- package/tests/workflow-graph.test.ts +279 -279
- package/tutorial/customer-service-agent/README.md +612 -612
- package/tutorial/customer-service-agent/SOUL.md +26 -26
- package/tutorial/customer-service-agent/agent.yaml +63 -63
- package/tutorial/customer-service-agent/package.json +19 -19
- package/tutorial/customer-service-agent/src/index.ts +69 -69
- package/tutorial/customer-service-agent/src/skills/faq.ts +27 -27
- package/tutorial/customer-service-agent/src/skills/ticket.ts +22 -22
- package/tutorial/customer-service-agent/tsconfig.json +14 -14
package/tests/web-search.test.ts
CHANGED
|
@@ -1,155 +1,155 @@
|
|
|
1
|
-
import { describe, it, expect } from 'vitest';
|
|
2
|
-
import { parseDuckDuckGoHTML, DEFAULT_SEARCH_CONFIG, webSearch } from '../src/tools/web-search';
|
|
3
|
-
import { extractReadableContent } from '../src/tools/web-scraper';
|
|
4
|
-
|
|
5
|
-
describe('Web Search - DuckDuckGo HTML Parser', () => {
|
|
6
|
-
const mockDDGHTML = `
|
|
7
|
-
<html><body>
|
|
8
|
-
<div class="result__body">
|
|
9
|
-
<a class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fpage1">Example Page One</a>
|
|
10
|
-
<a class="result__snippet">This is the first result snippet about example.</a>
|
|
11
|
-
</div>
|
|
12
|
-
<div class="result__body">
|
|
13
|
-
<a class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.org%2Fpage2">Example & Page Two</a>
|
|
14
|
-
<a class="result__snippet">Second result with <b>bold</b> text.</a>
|
|
15
|
-
</div>
|
|
16
|
-
<div class="result__body">
|
|
17
|
-
<a class="result__a" href="https://direct-url.com/page3">Direct URL Page</a>
|
|
18
|
-
<a class="result__snippet">Third result snippet.</a>
|
|
19
|
-
</div>
|
|
20
|
-
</body></html>
|
|
21
|
-
`;
|
|
22
|
-
|
|
23
|
-
it('should parse search results from DuckDuckGo HTML', () => {
|
|
24
|
-
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
25
|
-
expect(results.length).toBe(3);
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
it('should extract title and URL correctly', () => {
|
|
29
|
-
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
30
|
-
expect(results[0].title).toBe('Example Page One');
|
|
31
|
-
expect(results[0].url).toBe('https://example.com/page1');
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
it('should decode uddg redirect URLs', () => {
|
|
35
|
-
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
36
|
-
expect(results[0].url).toBe('https://example.com/page1');
|
|
37
|
-
expect(results[1].url).toBe('https://example.org/page2');
|
|
38
|
-
});
|
|
39
|
-
|
|
40
|
-
it('should handle direct URLs without uddg', () => {
|
|
41
|
-
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
42
|
-
expect(results[2].url).toBe('https://direct-url.com/page3');
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
it('should extract snippets and strip HTML', () => {
|
|
46
|
-
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
47
|
-
expect(results[0].snippet).toBe('This is the first result snippet about example.');
|
|
48
|
-
expect(results[1].snippet).toBe('Second result with bold text.');
|
|
49
|
-
});
|
|
50
|
-
|
|
51
|
-
it('should decode HTML entities in titles', () => {
|
|
52
|
-
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
53
|
-
expect(results[1].title).toBe('Example & Page Two');
|
|
54
|
-
});
|
|
55
|
-
|
|
56
|
-
it('should return empty array for empty HTML', () => {
|
|
57
|
-
expect(parseDuckDuckGoHTML('')).toEqual([]);
|
|
58
|
-
expect(parseDuckDuckGoHTML('<html><body></body></html>')).toEqual([]);
|
|
59
|
-
});
|
|
60
|
-
|
|
61
|
-
it('should have correct default config', () => {
|
|
62
|
-
expect(DEFAULT_SEARCH_CONFIG.defaultEngine).toBe('duckduckgo');
|
|
63
|
-
expect(DEFAULT_SEARCH_CONFIG.enabled).toBe(true);
|
|
64
|
-
});
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
describe('Web Scraper - Content Extraction', () => {
|
|
68
|
-
it('should extract title from HTML', () => {
|
|
69
|
-
const html = '<html><head><title>Test Page</title></head><body><p>Hello world</p></body></html>';
|
|
70
|
-
const result = extractReadableContent(html, 'https://example.com');
|
|
71
|
-
expect(result.title).toBe('Test Page');
|
|
72
|
-
});
|
|
73
|
-
|
|
74
|
-
it('should extract content and convert to markdown', () => {
|
|
75
|
-
const html = `
|
|
76
|
-
<html><head><title>Test</title></head><body>
|
|
77
|
-
<article>
|
|
78
|
-
<h1>Main Title</h1>
|
|
79
|
-
<p>This is a <strong>bold</strong> paragraph.</p>
|
|
80
|
-
<p>Second paragraph with <a href="https://link.com">a link</a>.</p>
|
|
81
|
-
</article>
|
|
82
|
-
</body></html>
|
|
83
|
-
`;
|
|
84
|
-
const result = extractReadableContent(html, 'https://example.com');
|
|
85
|
-
expect(result.content).toContain('# Main Title');
|
|
86
|
-
expect(result.content).toContain('**bold**');
|
|
87
|
-
expect(result.content).toContain('[a link](https://link.com)');
|
|
88
|
-
});
|
|
89
|
-
|
|
90
|
-
it('should remove script and style tags', () => {
|
|
91
|
-
const html = `
|
|
92
|
-
<html><head><title>Test</title></head><body>
|
|
93
|
-
<script>alert('xss')</script>
|
|
94
|
-
<style>.hidden { display: none; }</style>
|
|
95
|
-
<p>Visible content</p>
|
|
96
|
-
</body></html>
|
|
97
|
-
`;
|
|
98
|
-
const result = extractReadableContent(html, 'https://example.com');
|
|
99
|
-
expect(result.content).not.toContain('alert');
|
|
100
|
-
expect(result.content).not.toContain('.hidden');
|
|
101
|
-
expect(result.content).toContain('Visible content');
|
|
102
|
-
});
|
|
103
|
-
|
|
104
|
-
it('should remove nav and footer', () => {
|
|
105
|
-
const html = `
|
|
106
|
-
<html><head><title>Test</title></head><body>
|
|
107
|
-
<nav><a href="/">Home</a><a href="/about">About</a></nav>
|
|
108
|
-
<main><p>Main content here</p></main>
|
|
109
|
-
<footer>Copyright 2024</footer>
|
|
110
|
-
</body></html>
|
|
111
|
-
`;
|
|
112
|
-
const result = extractReadableContent(html, 'https://example.com');
|
|
113
|
-
expect(result.content).toContain('Main content here');
|
|
114
|
-
});
|
|
115
|
-
|
|
116
|
-
it('should truncate content at maxLength', () => {
|
|
117
|
-
const longContent = '<html><head><title>Test</title></head><body><p>' + 'a'.repeat(10000) + '</p></body></html>';
|
|
118
|
-
const result = extractReadableContent(longContent, 'https://example.com', 100);
|
|
119
|
-
expect(result.content.length).toBeLessThanOrEqual(120); // 100 + truncation message
|
|
120
|
-
expect(result.content).toContain('[truncated]');
|
|
121
|
-
});
|
|
122
|
-
|
|
123
|
-
it('should track word count', () => {
|
|
124
|
-
const html = '<html><head><title>Test</title></head><body><p>One two three four five</p></body></html>';
|
|
125
|
-
const result = extractReadableContent(html, 'https://example.com');
|
|
126
|
-
expect(result.wordCount).toBeGreaterThan(0);
|
|
127
|
-
});
|
|
128
|
-
|
|
129
|
-
it('should use URL as title when no title tag', () => {
|
|
130
|
-
const html = '<html><body><p>No title page</p></body></html>';
|
|
131
|
-
const result = extractReadableContent(html, 'https://example.com');
|
|
132
|
-
expect(result.title).toBe('https://example.com');
|
|
133
|
-
});
|
|
134
|
-
|
|
135
|
-
it('should decode HTML entities', () => {
|
|
136
|
-
const html = '<html><head><title>Test & Page</title></head><body><p>Content with <brackets> and "quotes"</p></body></html>';
|
|
137
|
-
const result = extractReadableContent(html, 'https://example.com');
|
|
138
|
-
expect(result.title).toBe('Test & Page');
|
|
139
|
-
expect(result.content).toContain('<brackets>');
|
|
140
|
-
});
|
|
141
|
-
});
|
|
142
|
-
|
|
143
|
-
describe('Web Search Config', () => {
|
|
144
|
-
it('should have sensible defaults', () => {
|
|
145
|
-
expect(DEFAULT_SEARCH_CONFIG.enabled).toBe(true);
|
|
146
|
-
expect(DEFAULT_SEARCH_CONFIG.defaultEngine).toBe('duckduckgo');
|
|
147
|
-
expect(DEFAULT_SEARCH_CONFIG.engines.duckduckgo?.enabled).toBe(true);
|
|
148
|
-
});
|
|
149
|
-
|
|
150
|
-
it('should return empty when disabled', async () => {
|
|
151
|
-
const config = { ...DEFAULT_SEARCH_CONFIG, enabled: false };
|
|
152
|
-
const results = await webSearch('test', config);
|
|
153
|
-
expect(results).toEqual([]);
|
|
154
|
-
});
|
|
155
|
-
});
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { parseDuckDuckGoHTML, DEFAULT_SEARCH_CONFIG, webSearch } from '../src/tools/web-search';
|
|
3
|
+
import { extractReadableContent } from '../src/tools/web-scraper';
|
|
4
|
+
|
|
5
|
+
describe('Web Search - DuckDuckGo HTML Parser', () => {
|
|
6
|
+
const mockDDGHTML = `
|
|
7
|
+
<html><body>
|
|
8
|
+
<div class="result__body">
|
|
9
|
+
<a class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fpage1">Example Page One</a>
|
|
10
|
+
<a class="result__snippet">This is the first result snippet about example.</a>
|
|
11
|
+
</div>
|
|
12
|
+
<div class="result__body">
|
|
13
|
+
<a class="result__a" href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.org%2Fpage2">Example & Page Two</a>
|
|
14
|
+
<a class="result__snippet">Second result with <b>bold</b> text.</a>
|
|
15
|
+
</div>
|
|
16
|
+
<div class="result__body">
|
|
17
|
+
<a class="result__a" href="https://direct-url.com/page3">Direct URL Page</a>
|
|
18
|
+
<a class="result__snippet">Third result snippet.</a>
|
|
19
|
+
</div>
|
|
20
|
+
</body></html>
|
|
21
|
+
`;
|
|
22
|
+
|
|
23
|
+
it('should parse search results from DuckDuckGo HTML', () => {
|
|
24
|
+
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
25
|
+
expect(results.length).toBe(3);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('should extract title and URL correctly', () => {
|
|
29
|
+
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
30
|
+
expect(results[0].title).toBe('Example Page One');
|
|
31
|
+
expect(results[0].url).toBe('https://example.com/page1');
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it('should decode uddg redirect URLs', () => {
|
|
35
|
+
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
36
|
+
expect(results[0].url).toBe('https://example.com/page1');
|
|
37
|
+
expect(results[1].url).toBe('https://example.org/page2');
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it('should handle direct URLs without uddg', () => {
|
|
41
|
+
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
42
|
+
expect(results[2].url).toBe('https://direct-url.com/page3');
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it('should extract snippets and strip HTML', () => {
|
|
46
|
+
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
47
|
+
expect(results[0].snippet).toBe('This is the first result snippet about example.');
|
|
48
|
+
expect(results[1].snippet).toBe('Second result with bold text.');
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it('should decode HTML entities in titles', () => {
|
|
52
|
+
const results = parseDuckDuckGoHTML(mockDDGHTML);
|
|
53
|
+
expect(results[1].title).toBe('Example & Page Two');
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it('should return empty array for empty HTML', () => {
|
|
57
|
+
expect(parseDuckDuckGoHTML('')).toEqual([]);
|
|
58
|
+
expect(parseDuckDuckGoHTML('<html><body></body></html>')).toEqual([]);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('should have correct default config', () => {
|
|
62
|
+
expect(DEFAULT_SEARCH_CONFIG.defaultEngine).toBe('duckduckgo');
|
|
63
|
+
expect(DEFAULT_SEARCH_CONFIG.enabled).toBe(true);
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
describe('Web Scraper - Content Extraction', () => {
|
|
68
|
+
it('should extract title from HTML', () => {
|
|
69
|
+
const html = '<html><head><title>Test Page</title></head><body><p>Hello world</p></body></html>';
|
|
70
|
+
const result = extractReadableContent(html, 'https://example.com');
|
|
71
|
+
expect(result.title).toBe('Test Page');
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('should extract content and convert to markdown', () => {
|
|
75
|
+
const html = `
|
|
76
|
+
<html><head><title>Test</title></head><body>
|
|
77
|
+
<article>
|
|
78
|
+
<h1>Main Title</h1>
|
|
79
|
+
<p>This is a <strong>bold</strong> paragraph.</p>
|
|
80
|
+
<p>Second paragraph with <a href="https://link.com">a link</a>.</p>
|
|
81
|
+
</article>
|
|
82
|
+
</body></html>
|
|
83
|
+
`;
|
|
84
|
+
const result = extractReadableContent(html, 'https://example.com');
|
|
85
|
+
expect(result.content).toContain('# Main Title');
|
|
86
|
+
expect(result.content).toContain('**bold**');
|
|
87
|
+
expect(result.content).toContain('[a link](https://link.com)');
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('should remove script and style tags', () => {
|
|
91
|
+
const html = `
|
|
92
|
+
<html><head><title>Test</title></head><body>
|
|
93
|
+
<script>alert('xss')</script>
|
|
94
|
+
<style>.hidden { display: none; }</style>
|
|
95
|
+
<p>Visible content</p>
|
|
96
|
+
</body></html>
|
|
97
|
+
`;
|
|
98
|
+
const result = extractReadableContent(html, 'https://example.com');
|
|
99
|
+
expect(result.content).not.toContain('alert');
|
|
100
|
+
expect(result.content).not.toContain('.hidden');
|
|
101
|
+
expect(result.content).toContain('Visible content');
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it('should remove nav and footer', () => {
|
|
105
|
+
const html = `
|
|
106
|
+
<html><head><title>Test</title></head><body>
|
|
107
|
+
<nav><a href="/">Home</a><a href="/about">About</a></nav>
|
|
108
|
+
<main><p>Main content here</p></main>
|
|
109
|
+
<footer>Copyright 2024</footer>
|
|
110
|
+
</body></html>
|
|
111
|
+
`;
|
|
112
|
+
const result = extractReadableContent(html, 'https://example.com');
|
|
113
|
+
expect(result.content).toContain('Main content here');
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('should truncate content at maxLength', () => {
|
|
117
|
+
const longContent = '<html><head><title>Test</title></head><body><p>' + 'a'.repeat(10000) + '</p></body></html>';
|
|
118
|
+
const result = extractReadableContent(longContent, 'https://example.com', 100);
|
|
119
|
+
expect(result.content.length).toBeLessThanOrEqual(120); // 100 + truncation message
|
|
120
|
+
expect(result.content).toContain('[truncated]');
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('should track word count', () => {
|
|
124
|
+
const html = '<html><head><title>Test</title></head><body><p>One two three four five</p></body></html>';
|
|
125
|
+
const result = extractReadableContent(html, 'https://example.com');
|
|
126
|
+
expect(result.wordCount).toBeGreaterThan(0);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it('should use URL as title when no title tag', () => {
|
|
130
|
+
const html = '<html><body><p>No title page</p></body></html>';
|
|
131
|
+
const result = extractReadableContent(html, 'https://example.com');
|
|
132
|
+
expect(result.title).toBe('https://example.com');
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it('should decode HTML entities', () => {
|
|
136
|
+
const html = '<html><head><title>Test & Page</title></head><body><p>Content with <brackets> and "quotes"</p></body></html>';
|
|
137
|
+
const result = extractReadableContent(html, 'https://example.com');
|
|
138
|
+
expect(result.title).toBe('Test & Page');
|
|
139
|
+
expect(result.content).toContain('<brackets>');
|
|
140
|
+
});
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
describe('Web Search Config', () => {
|
|
144
|
+
it('should have sensible defaults', () => {
|
|
145
|
+
expect(DEFAULT_SEARCH_CONFIG.enabled).toBe(true);
|
|
146
|
+
expect(DEFAULT_SEARCH_CONFIG.defaultEngine).toBe('duckduckgo');
|
|
147
|
+
expect(DEFAULT_SEARCH_CONFIG.engines.duckduckgo?.enabled).toBe(true);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
it('should return empty when disabled', async () => {
|
|
151
|
+
const config = { ...DEFAULT_SEARCH_CONFIG, enabled: false };
|
|
152
|
+
const results = await webSearch('test', config);
|
|
153
|
+
expect(results).toEqual([]);
|
|
154
|
+
});
|
|
155
|
+
});
|