opc-agent 4.1.0 → 4.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. package/.github/ISSUE_TEMPLATE/bug_report.md +20 -20
  2. package/.github/ISSUE_TEMPLATE/feature_request.md +14 -14
  3. package/.github/PULL_REQUEST_TEMPLATE.md +13 -13
  4. package/CHANGELOG.md +48 -48
  5. package/CONTRIBUTING.md +36 -36
  6. package/README.zh-CN.md +497 -497
  7. package/USABILITY-ISSUES.md +73 -0
  8. package/dist/channels/web.js +8 -2
  9. package/dist/channels/wechat.js +6 -6
  10. package/dist/cli.js +200 -85
  11. package/dist/core/runtime.js +37 -15
  12. package/dist/deploy/index.js +56 -56
  13. package/dist/doctor.d.ts +1 -0
  14. package/dist/doctor.js +105 -10
  15. package/dist/memory/deepbrain.d.ts +1 -1
  16. package/dist/memory/deepbrain.js +95 -4
  17. package/dist/scheduler/cron-engine.js +3 -36
  18. package/dist/studio/server.js +30 -1
  19. package/dist/studio-ui/index.html +230 -10
  20. package/dist/ui/components.js +105 -105
  21. package/examples/README.md +22 -22
  22. package/examples/basic-agent.ts +90 -90
  23. package/examples/brain-integration.ts +71 -71
  24. package/examples/multi-channel.ts +74 -74
  25. package/fix-sidebar.mjs +188 -188
  26. package/install.ps1 +154 -154
  27. package/install.sh +164 -164
  28. package/package.json +1 -1
  29. package/scripts/install.ps1 +31 -31
  30. package/scripts/install.sh +40 -40
  31. package/serve-studio.js +13 -13
  32. package/serve-test.js +25 -25
  33. package/src/channels/dingtalk.ts +46 -46
  34. package/src/channels/email.ts +351 -351
  35. package/src/channels/feishu.ts +349 -349
  36. package/src/channels/googlechat.ts +42 -42
  37. package/src/channels/imessage.ts +31 -31
  38. package/src/channels/irc.ts +82 -82
  39. package/src/channels/line.ts +32 -32
  40. package/src/channels/matrix.ts +33 -33
  41. package/src/channels/mattermost.ts +57 -57
  42. package/src/channels/msteams.ts +32 -32
  43. package/src/channels/nostr.ts +32 -32
  44. package/src/channels/qq.ts +33 -33
  45. package/src/channels/signal.ts +32 -32
  46. package/src/channels/sms.ts +33 -33
  47. package/src/channels/telegram.ts +616 -616
  48. package/src/channels/twitch.ts +65 -65
  49. package/src/channels/voice-call.ts +100 -100
  50. package/src/channels/web.ts +8 -2
  51. package/src/channels/websocket.ts +399 -399
  52. package/src/channels/wechat.ts +329 -329
  53. package/src/channels/whatsapp.ts +32 -32
  54. package/src/cli/chat.ts +99 -99
  55. package/src/cli/setup.ts +314 -314
  56. package/src/cli.ts +195 -92
  57. package/src/core/agent.ts +476 -476
  58. package/src/core/api-server.ts +277 -277
  59. package/src/core/audio.ts +98 -98
  60. package/src/core/collaboration.ts +275 -275
  61. package/src/core/context-discovery.ts +85 -85
  62. package/src/core/context-refs.ts +140 -140
  63. package/src/core/gateway.ts +106 -106
  64. package/src/core/heartbeat.ts +51 -51
  65. package/src/core/hooks.ts +105 -105
  66. package/src/core/ide-bridge.ts +133 -133
  67. package/src/core/node-network.ts +86 -86
  68. package/src/core/profiles.ts +122 -122
  69. package/src/core/runtime.ts +25 -0
  70. package/src/core/scheduler.ts +187 -187
  71. package/src/core/session-manager.ts +137 -137
  72. package/src/core/subagent.ts +98 -98
  73. package/src/core/vision.ts +180 -180
  74. package/src/core/workflow-graph.ts +365 -365
  75. package/src/daemon.ts +96 -96
  76. package/src/deploy/index.ts +255 -255
  77. package/src/doctor.ts +98 -11
  78. package/src/eval/index.ts +211 -211
  79. package/src/eval/suites/basic.json +16 -16
  80. package/src/eval/suites/memory.json +12 -12
  81. package/src/eval/suites/safety.json +14 -14
  82. package/src/hub/brain-seed.ts +54 -54
  83. package/src/hub/client.ts +60 -60
  84. package/src/mcp/servers/calculator-mcp.ts +65 -65
  85. package/src/mcp/servers/crypto-mcp.ts +73 -73
  86. package/src/mcp/servers/database-mcp.ts +72 -72
  87. package/src/mcp/servers/datetime-mcp.ts +69 -69
  88. package/src/mcp/servers/filesystem.ts +66 -66
  89. package/src/mcp/servers/github-mcp.ts +58 -58
  90. package/src/mcp/servers/index.ts +63 -63
  91. package/src/mcp/servers/json-mcp.ts +102 -102
  92. package/src/mcp/servers/memory-mcp.ts +56 -56
  93. package/src/mcp/servers/regex-mcp.ts +53 -53
  94. package/src/mcp/servers/web-mcp.ts +49 -49
  95. package/src/memory/context-compressor.ts +189 -189
  96. package/src/memory/deepbrain.ts +99 -5
  97. package/src/memory/seed-loader.ts +212 -212
  98. package/src/memory/user-profiler.ts +215 -215
  99. package/src/plugins/content-filter.ts +23 -23
  100. package/src/plugins/logger.ts +18 -18
  101. package/src/plugins/rate-limiter.ts +38 -38
  102. package/src/protocols/a2a/client.ts +132 -132
  103. package/src/protocols/a2a/index.ts +8 -8
  104. package/src/protocols/a2a/server.ts +333 -333
  105. package/src/protocols/a2a/types.ts +88 -88
  106. package/src/protocols/a2a/utils.ts +50 -50
  107. package/src/protocols/agui/client.ts +83 -83
  108. package/src/protocols/agui/index.ts +4 -4
  109. package/src/protocols/agui/server.ts +218 -218
  110. package/src/protocols/agui/types.ts +153 -153
  111. package/src/protocols/index.ts +2 -2
  112. package/src/protocols/mcp/agent-tools.ts +134 -134
  113. package/src/protocols/mcp/index.ts +8 -8
  114. package/src/protocols/mcp/server.ts +262 -262
  115. package/src/protocols/mcp/types.ts +69 -69
  116. package/src/providers/index.ts +632 -632
  117. package/src/publish/index.ts +376 -376
  118. package/src/scheduler/cron-engine.ts +191 -191
  119. package/src/scheduler/index.ts +2 -2
  120. package/src/schema/oad.ts +217 -217
  121. package/src/security/approval.ts +131 -131
  122. package/src/security/approvals.ts +143 -143
  123. package/src/security/elevated.ts +105 -105
  124. package/src/security/guardrails.ts +248 -248
  125. package/src/security/index.ts +9 -9
  126. package/src/security/keys.ts +87 -87
  127. package/src/security/secrets.ts +129 -129
  128. package/src/skills/builtin/index.ts +408 -408
  129. package/src/skills/marketplace.ts +113 -113
  130. package/src/skills/types.ts +42 -42
  131. package/src/studio/server.ts +31 -1
  132. package/src/studio/templates-data.ts +178 -178
  133. package/src/studio-ui/index.html +230 -10
  134. package/src/telemetry/index.ts +324 -324
  135. package/src/tools/builtin/browser.ts +299 -299
  136. package/src/tools/builtin/datetime.ts +41 -41
  137. package/src/tools/builtin/file.ts +107 -107
  138. package/src/tools/builtin/home-assistant.ts +116 -116
  139. package/src/tools/builtin/rl-tools.ts +243 -243
  140. package/src/tools/builtin/shell.ts +43 -43
  141. package/src/tools/builtin/vision.ts +64 -64
  142. package/src/tools/builtin/web-search.ts +126 -126
  143. package/src/tools/builtin/web.ts +35 -35
  144. package/src/tools/document-processor.ts +213 -213
  145. package/src/tools/image-generator.ts +150 -150
  146. package/src/tools/integrations/calendar.ts +73 -73
  147. package/src/tools/integrations/code-exec.ts +39 -39
  148. package/src/tools/integrations/csv-analyzer.ts +92 -92
  149. package/src/tools/integrations/database.ts +44 -44
  150. package/src/tools/integrations/email-send.ts +76 -76
  151. package/src/tools/integrations/git-tool.ts +42 -42
  152. package/src/tools/integrations/github-tool.ts +76 -76
  153. package/src/tools/integrations/image-gen.ts +56 -56
  154. package/src/tools/integrations/index.ts +92 -92
  155. package/src/tools/integrations/jira.ts +83 -83
  156. package/src/tools/integrations/notion.ts +71 -71
  157. package/src/tools/integrations/npm-tool.ts +48 -48
  158. package/src/tools/integrations/pdf-reader.ts +58 -58
  159. package/src/tools/integrations/slack.ts +65 -65
  160. package/src/tools/integrations/summarizer.ts +49 -49
  161. package/src/tools/integrations/translator.ts +48 -48
  162. package/src/tools/integrations/trello.ts +60 -60
  163. package/src/tools/integrations/vector-search.ts +42 -42
  164. package/src/tools/integrations/web-scraper.ts +47 -47
  165. package/src/tools/integrations/web-search.ts +58 -58
  166. package/src/tools/integrations/webhook.ts +38 -38
  167. package/src/tools/mcp-client.ts +131 -131
  168. package/src/tools/web-scraper.ts +179 -179
  169. package/src/tools/web-search.ts +180 -180
  170. package/src/ui/components.ts +127 -127
  171. package/srv-out.txt +1 -1
  172. package/templates/ecommerce-assistant/README.md +45 -45
  173. package/templates/ecommerce-assistant/oad.yaml +47 -47
  174. package/templates/tech-support/README.md +43 -43
  175. package/templates/tech-support/oad.yaml +45 -45
  176. package/test-agent/Dockerfile +9 -9
  177. package/test-agent/README.md +50 -50
  178. package/test-agent/agent.yaml +23 -23
  179. package/test-agent/docker-compose.yml +11 -11
  180. package/test-agent/oad.yaml +31 -31
  181. package/test-agent/package-lock.json +1492 -1492
  182. package/test-agent/package.json +17 -17
  183. package/test-agent/src/index.ts +24 -24
  184. package/test-agent/src/skills/echo.ts +15 -15
  185. package/test-agent/tsconfig.json +24 -24
  186. package/test-full.js +43 -43
  187. package/test-sidebar.js +22 -22
  188. package/test-studio3.js +75 -75
  189. package/test-studio4.js +41 -41
  190. package/tests/a2a-protocol.test.ts +285 -285
  191. package/tests/agui-protocol.test.ts +246 -246
  192. package/tests/api-server.test.ts +148 -148
  193. package/tests/approvals.test.ts +89 -89
  194. package/tests/audio.test.ts +40 -40
  195. package/tests/brain-seed-extended.test.ts +490 -490
  196. package/tests/brain-seed.test.ts +239 -239
  197. package/tests/browser.test.ts +179 -179
  198. package/tests/channels/discord.test.ts +79 -79
  199. package/tests/channels/email.test.ts +148 -148
  200. package/tests/channels/feishu.test.ts +123 -123
  201. package/tests/channels/telegram.test.ts +129 -129
  202. package/tests/channels/websocket.test.ts +53 -53
  203. package/tests/channels/wechat.test.ts +170 -170
  204. package/tests/channels-extra.test.ts +45 -45
  205. package/tests/chat-cli.test.ts +160 -160
  206. package/tests/cli.test.ts +46 -46
  207. package/tests/context-compressor.test.ts +172 -172
  208. package/tests/context-refs.test.ts +121 -121
  209. package/tests/cron-engine.test.ts +101 -101
  210. package/tests/daemon.test.ts +135 -135
  211. package/tests/deepbrain-wire.test.ts +234 -234
  212. package/tests/deploy-and-dag.test.ts +196 -196
  213. package/tests/doctor.test.ts +38 -38
  214. package/tests/document-processor.test.ts +69 -69
  215. package/tests/e2e-nocode.test.ts +442 -442
  216. package/tests/elevated.test.ts +69 -69
  217. package/tests/eval.test.ts +173 -173
  218. package/tests/gateway.test.ts +63 -63
  219. package/tests/guardrails.test.ts +177 -177
  220. package/tests/home-assistant.test.ts +40 -40
  221. package/tests/hooks.test.ts +79 -79
  222. package/tests/ide-bridge.test.ts +38 -38
  223. package/tests/image-generator.test.ts +84 -84
  224. package/tests/init-role.test.ts +124 -124
  225. package/tests/integrations.test.ts +249 -249
  226. package/tests/mcp-client.test.ts +92 -92
  227. package/tests/mcp-server.test.ts +178 -178
  228. package/tests/mcp-servers.test.ts +260 -260
  229. package/tests/node-network.test.ts +74 -74
  230. package/tests/plugin-a2a-enhanced.test.ts +230 -230
  231. package/tests/profiles.test.ts +61 -61
  232. package/tests/publish.test.ts +231 -231
  233. package/tests/rl-tools.test.ts +93 -93
  234. package/tests/sandbox-manager.test.ts +46 -46
  235. package/tests/scheduler.test.ts +200 -200
  236. package/tests/secrets.test.ts +107 -107
  237. package/tests/security-enhanced.test.ts +233 -233
  238. package/tests/settings-api.test.ts +148 -148
  239. package/tests/setup.test.ts +73 -73
  240. package/tests/subagent.test.ts +193 -193
  241. package/tests/telegram-discord.test.ts +60 -60
  242. package/tests/telemetry.test.ts +186 -186
  243. package/tests/user-profiler.test.ts +169 -169
  244. package/tests/v090-features.test.ts +254 -254
  245. package/tests/vision.test.ts +61 -61
  246. package/tests/voice-call.test.ts +47 -47
  247. package/tests/voice-enhanced.test.ts +169 -169
  248. package/tests/voice-interaction.test.ts +38 -38
  249. package/tests/web-search.test.ts +155 -155
  250. package/tests/workflow-graph.test.ts +279 -279
  251. package/tutorial/customer-service-agent/README.md +612 -612
  252. package/tutorial/customer-service-agent/SOUL.md +26 -26
  253. package/tutorial/customer-service-agent/agent.yaml +63 -63
  254. package/tutorial/customer-service-agent/package.json +19 -19
  255. package/tutorial/customer-service-agent/src/index.ts +69 -69
  256. package/tutorial/customer-service-agent/src/skills/faq.ts +27 -27
  257. package/tutorial/customer-service-agent/src/skills/ticket.ts +22 -22
  258. package/tutorial/customer-service-agent/tsconfig.json +14 -14
@@ -1,131 +1,131 @@
1
- import { spawn, type ChildProcess } from 'child_process';
2
- import type { MCPToolDefinition, MCPToolResult } from './mcp';
3
-
4
- export interface MCPServerConfig {
5
- name: string;
6
- command: string;
7
- args?: string[];
8
- env?: Record<string, string>;
9
- }
10
-
11
- export class MCPClient {
12
- private process: ChildProcess | null = null;
13
- private config: MCPServerConfig | null = null;
14
- private nextId = 1;
15
- private pending = new Map<number, { resolve: (v: any) => void; reject: (e: Error) => void }>();
16
- private buffer = '';
17
- private connected = false;
18
-
19
- async connect(config: MCPServerConfig): Promise<void> {
20
- this.config = config;
21
- this.process = spawn(config.command, config.args ?? [], {
22
- stdio: ['pipe', 'pipe', 'pipe'],
23
- env: { ...process.env, ...config.env },
24
- });
25
-
26
- this.process.stdout!.on('data', (data: Buffer) => {
27
- this.buffer += data.toString();
28
- this.processBuffer();
29
- });
30
-
31
- this.process.on('error', (err) => {
32
- for (const [, p] of this.pending) p.reject(err);
33
- this.pending.clear();
34
- });
35
-
36
- this.process.on('exit', () => {
37
- this.connected = false;
38
- for (const [, p] of this.pending) p.reject(new Error('MCP server exited'));
39
- this.pending.clear();
40
- });
41
-
42
- // Send initialize
43
- await this.sendRequest('initialize', {
44
- protocolVersion: '2024-11-05',
45
- capabilities: {},
46
- clientInfo: { name: 'opc-agent', version: '0.7.0' },
47
- });
48
-
49
- // Send initialized notification
50
- this.sendNotification('notifications/initialized', {});
51
- this.connected = true;
52
- }
53
-
54
- private processBuffer(): void {
55
- const lines = this.buffer.split('\n');
56
- this.buffer = lines.pop() ?? '';
57
- for (const line of lines) {
58
- const trimmed = line.trim();
59
- if (!trimmed) continue;
60
- try {
61
- const msg = JSON.parse(trimmed);
62
- if (msg.id !== undefined && this.pending.has(msg.id)) {
63
- const p = this.pending.get(msg.id)!;
64
- this.pending.delete(msg.id);
65
- if (msg.error) {
66
- p.reject(new Error(msg.error.message || JSON.stringify(msg.error)));
67
- } else {
68
- p.resolve(msg.result);
69
- }
70
- }
71
- } catch { /* skip non-JSON lines */ }
72
- }
73
- }
74
-
75
- private sendRequest(method: string, params?: Record<string, unknown>): Promise<any> {
76
- return new Promise((resolve, reject) => {
77
- if (!this.process?.stdin?.writable) {
78
- reject(new Error('MCP server not connected'));
79
- return;
80
- }
81
- const id = this.nextId++;
82
- this.pending.set(id, { resolve, reject });
83
- const msg = JSON.stringify({ jsonrpc: '2.0', method, params: params ?? {}, id });
84
- this.process.stdin.write(msg + '\n');
85
-
86
- // Timeout after 30s
87
- setTimeout(() => {
88
- if (this.pending.has(id)) {
89
- this.pending.delete(id);
90
- reject(new Error(`MCP request timed out: ${method}`));
91
- }
92
- }, 30000);
93
- });
94
- }
95
-
96
- private sendNotification(method: string, params: Record<string, unknown>): void {
97
- if (!this.process?.stdin?.writable) return;
98
- const msg = JSON.stringify({ jsonrpc: '2.0', method, params });
99
- this.process.stdin.write(msg + '\n');
100
- }
101
-
102
- async listTools(): Promise<MCPToolDefinition[]> {
103
- const result = await this.sendRequest('tools/list');
104
- return (result.tools ?? []).map((t: any) => ({
105
- name: t.name,
106
- description: t.description ?? '',
107
- inputSchema: t.inputSchema ?? {},
108
- }));
109
- }
110
-
111
- async callTool(name: string, input: Record<string, unknown>): Promise<MCPToolResult> {
112
- const result = await this.sendRequest('tools/call', { name, arguments: input });
113
- const content = (result.content ?? [])
114
- .map((c: any) => c.text ?? JSON.stringify(c))
115
- .join('\n');
116
- return { content, isError: result.isError ?? false };
117
- }
118
-
119
- async disconnect(): Promise<void> {
120
- if (this.process) {
121
- this.process.kill();
122
- this.process = null;
123
- }
124
- this.connected = false;
125
- this.pending.clear();
126
- }
127
-
128
- isConnected(): boolean {
129
- return this.connected;
130
- }
131
- }
1
+ import { spawn, type ChildProcess } from 'child_process';
2
+ import type { MCPToolDefinition, MCPToolResult } from './mcp';
3
+
4
+ export interface MCPServerConfig {
5
+ name: string;
6
+ command: string;
7
+ args?: string[];
8
+ env?: Record<string, string>;
9
+ }
10
+
11
+ export class MCPClient {
12
+ private process: ChildProcess | null = null;
13
+ private config: MCPServerConfig | null = null;
14
+ private nextId = 1;
15
+ private pending = new Map<number, { resolve: (v: any) => void; reject: (e: Error) => void }>();
16
+ private buffer = '';
17
+ private connected = false;
18
+
19
+ async connect(config: MCPServerConfig): Promise<void> {
20
+ this.config = config;
21
+ this.process = spawn(config.command, config.args ?? [], {
22
+ stdio: ['pipe', 'pipe', 'pipe'],
23
+ env: { ...process.env, ...config.env },
24
+ });
25
+
26
+ this.process.stdout!.on('data', (data: Buffer) => {
27
+ this.buffer += data.toString();
28
+ this.processBuffer();
29
+ });
30
+
31
+ this.process.on('error', (err) => {
32
+ for (const [, p] of this.pending) p.reject(err);
33
+ this.pending.clear();
34
+ });
35
+
36
+ this.process.on('exit', () => {
37
+ this.connected = false;
38
+ for (const [, p] of this.pending) p.reject(new Error('MCP server exited'));
39
+ this.pending.clear();
40
+ });
41
+
42
+ // Send initialize
43
+ await this.sendRequest('initialize', {
44
+ protocolVersion: '2024-11-05',
45
+ capabilities: {},
46
+ clientInfo: { name: 'opc-agent', version: '0.7.0' },
47
+ });
48
+
49
+ // Send initialized notification
50
+ this.sendNotification('notifications/initialized', {});
51
+ this.connected = true;
52
+ }
53
+
54
+ private processBuffer(): void {
55
+ const lines = this.buffer.split('\n');
56
+ this.buffer = lines.pop() ?? '';
57
+ for (const line of lines) {
58
+ const trimmed = line.trim();
59
+ if (!trimmed) continue;
60
+ try {
61
+ const msg = JSON.parse(trimmed);
62
+ if (msg.id !== undefined && this.pending.has(msg.id)) {
63
+ const p = this.pending.get(msg.id)!;
64
+ this.pending.delete(msg.id);
65
+ if (msg.error) {
66
+ p.reject(new Error(msg.error.message || JSON.stringify(msg.error)));
67
+ } else {
68
+ p.resolve(msg.result);
69
+ }
70
+ }
71
+ } catch { /* skip non-JSON lines */ }
72
+ }
73
+ }
74
+
75
+ private sendRequest(method: string, params?: Record<string, unknown>): Promise<any> {
76
+ return new Promise((resolve, reject) => {
77
+ if (!this.process?.stdin?.writable) {
78
+ reject(new Error('MCP server not connected'));
79
+ return;
80
+ }
81
+ const id = this.nextId++;
82
+ this.pending.set(id, { resolve, reject });
83
+ const msg = JSON.stringify({ jsonrpc: '2.0', method, params: params ?? {}, id });
84
+ this.process.stdin.write(msg + '\n');
85
+
86
+ // Timeout after 30s
87
+ setTimeout(() => {
88
+ if (this.pending.has(id)) {
89
+ this.pending.delete(id);
90
+ reject(new Error(`MCP request timed out: ${method}`));
91
+ }
92
+ }, 30000);
93
+ });
94
+ }
95
+
96
+ private sendNotification(method: string, params: Record<string, unknown>): void {
97
+ if (!this.process?.stdin?.writable) return;
98
+ const msg = JSON.stringify({ jsonrpc: '2.0', method, params });
99
+ this.process.stdin.write(msg + '\n');
100
+ }
101
+
102
+ async listTools(): Promise<MCPToolDefinition[]> {
103
+ const result = await this.sendRequest('tools/list');
104
+ return (result.tools ?? []).map((t: any) => ({
105
+ name: t.name,
106
+ description: t.description ?? '',
107
+ inputSchema: t.inputSchema ?? {},
108
+ }));
109
+ }
110
+
111
+ async callTool(name: string, input: Record<string, unknown>): Promise<MCPToolResult> {
112
+ const result = await this.sendRequest('tools/call', { name, arguments: input });
113
+ const content = (result.content ?? [])
114
+ .map((c: any) => c.text ?? JSON.stringify(c))
115
+ .join('\n');
116
+ return { content, isError: result.isError ?? false };
117
+ }
118
+
119
+ async disconnect(): Promise<void> {
120
+ if (this.process) {
121
+ this.process.kill();
122
+ this.process = null;
123
+ }
124
+ this.connected = false;
125
+ this.pending.clear();
126
+ }
127
+
128
+ isConnected(): boolean {
129
+ return this.connected;
130
+ }
131
+ }
@@ -1,179 +1,179 @@
1
- /**
2
- * Web Scraper - v0.10.0
3
- * Fetch URL content and extract readable text in markdown format.
4
- * Uses a simple readability-style extraction (no external dependencies).
5
- */
6
-
7
- export interface ScrapedContent {
8
- title: string;
9
- content: string; // markdown
10
- url: string;
11
- wordCount: number;
12
- }
13
-
14
- const MAX_CONTENT_LENGTH = 5000;
15
-
16
- /**
17
- * Fetch a URL and extract readable content as markdown.
18
- */
19
- export async function scrapeUrl(url: string, maxLength = MAX_CONTENT_LENGTH): Promise<ScrapedContent> {
20
- const response = await fetch(url, {
21
- headers: {
22
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
23
- Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
24
- },
25
- signal: AbortSignal.timeout(15000),
26
- redirect: 'follow',
27
- });
28
-
29
- const contentType = response.headers.get('content-type') || '';
30
- const text = await response.text();
31
-
32
- // If not HTML, return raw text
33
- if (!contentType.includes('html')) {
34
- const truncated = text.slice(0, maxLength);
35
- return {
36
- title: url,
37
- content: truncated,
38
- url,
39
- wordCount: truncated.split(/\s+/).length,
40
- };
41
- }
42
-
43
- return extractReadableContent(text, url, maxLength);
44
- }
45
-
46
- /**
47
- * Extract readable content from HTML using simple heuristics.
48
- */
49
- export function extractReadableContent(html: string, url: string, maxLength = MAX_CONTENT_LENGTH): ScrapedContent {
50
- // Extract title
51
- const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
52
- const title = titleMatch ? decodeEntities(titleMatch[1]).trim() : url;
53
-
54
- // Remove non-content elements
55
- let content = html;
56
-
57
- // Remove script, style, nav, header, footer, aside, iframe
58
- const removePatterns = [
59
- /<script[\s\S]*?<\/script>/gi,
60
- /<style[\s\S]*?<\/style>/gi,
61
- /<nav[\s\S]*?<\/nav>/gi,
62
- /<footer[\s\S]*?<\/footer>/gi,
63
- /<aside[\s\S]*?<\/aside>/gi,
64
- /<iframe[\s\S]*?<\/iframe>/gi,
65
- /<noscript[\s\S]*?<\/noscript>/gi,
66
- /<!--[\s\S]*?-->/g,
67
- ];
68
-
69
- for (const pattern of removePatterns) {
70
- content = content.replace(pattern, '');
71
- }
72
-
73
- // Try to find main content area
74
- const mainContent = findMainContent(content);
75
- content = mainContent || content;
76
-
77
- // Convert to markdown-ish text
78
- content = htmlToMarkdown(content);
79
-
80
- // Clean up whitespace
81
- content = content
82
- .replace(/\n{3,}/g, '\n\n')
83
- .replace(/[ \t]+/g, ' ')
84
- .trim();
85
-
86
- // Truncate
87
- if (content.length > maxLength) {
88
- content = content.slice(0, maxLength) + '\n\n...[truncated]';
89
- }
90
-
91
- return {
92
- title,
93
- content,
94
- url,
95
- wordCount: content.split(/\s+/).filter(Boolean).length,
96
- };
97
- }
98
-
99
- /**
100
- * Try to find the main content area of the page.
101
- */
102
- function findMainContent(html: string): string | null {
103
- // Try common content selectors
104
- const patterns = [
105
- /<article[^>]*>([\s\S]*?)<\/article>/i,
106
- /<main[^>]*>([\s\S]*?)<\/main>/i,
107
- /<div[^>]*class="[^"]*(?:content|article|post|entry|main)[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
108
- /<div[^>]*id="[^"]*(?:content|article|post|entry|main)[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
109
- ];
110
-
111
- for (const pattern of patterns) {
112
- const match = html.match(pattern);
113
- if (match && match[1] && match[1].length > 200) {
114
- return match[1];
115
- }
116
- }
117
-
118
- // Fallback: find body content
119
- const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
120
- return bodyMatch ? bodyMatch[1] : null;
121
- }
122
-
123
- /**
124
- * Simple HTML to Markdown conversion.
125
- */
126
- function htmlToMarkdown(html: string): string {
127
- let md = html;
128
-
129
- // Headers
130
- md = md.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, '\n# $1\n');
131
- md = md.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, '\n## $1\n');
132
- md = md.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, '\n### $1\n');
133
- md = md.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, '\n#### $1\n');
134
- md = md.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, '\n##### $1\n');
135
- md = md.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, '\n###### $1\n');
136
-
137
- // Paragraphs and line breaks
138
- md = md.replace(/<p[^>]*>/gi, '\n');
139
- md = md.replace(/<\/p>/gi, '\n');
140
- md = md.replace(/<br\s*\/?>/gi, '\n');
141
-
142
- // Links
143
- md = md.replace(/<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi, '[$2]($1)');
144
-
145
- // Bold and italic
146
- md = md.replace(/<(?:strong|b)[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi, '**$1**');
147
- md = md.replace(/<(?:em|i)[^>]*>([\s\S]*?)<\/(?:em|i)>/gi, '*$1*');
148
-
149
- // Code
150
- md = md.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, '`$1`');
151
- md = md.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, '\n```\n$1\n```\n');
152
-
153
- // Lists
154
- md = md.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n');
155
-
156
- // Blockquote
157
- md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, '\n> $1\n');
158
-
159
- // Remove remaining HTML tags
160
- md = md.replace(/<[^>]+>/g, '');
161
-
162
- // Decode entities
163
- md = decodeEntities(md);
164
-
165
- return md;
166
- }
167
-
168
- function decodeEntities(text: string): string {
169
- return text
170
- .replace(/&amp;/g, '&')
171
- .replace(/&lt;/g, '<')
172
- .replace(/&gt;/g, '>')
173
- .replace(/&quot;/g, '"')
174
- .replace(/&#x27;/g, "'")
175
- .replace(/&#39;/g, "'")
176
- .replace(/&nbsp;/g, ' ')
177
- .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n)))
178
- .replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCharCode(parseInt(n, 16)));
179
- }
1
+ /**
2
+ * Web Scraper - v0.10.0
3
+ * Fetch URL content and extract readable text in markdown format.
4
+ * Uses a simple readability-style extraction (no external dependencies).
5
+ */
6
+
7
+ export interface ScrapedContent {
8
+ title: string;
9
+ content: string; // markdown
10
+ url: string;
11
+ wordCount: number;
12
+ }
13
+
14
+ const MAX_CONTENT_LENGTH = 5000;
15
+
16
+ /**
17
+ * Fetch a URL and extract readable content as markdown.
18
+ */
19
+ export async function scrapeUrl(url: string, maxLength = MAX_CONTENT_LENGTH): Promise<ScrapedContent> {
20
+ const response = await fetch(url, {
21
+ headers: {
22
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
23
+ Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
24
+ },
25
+ signal: AbortSignal.timeout(15000),
26
+ redirect: 'follow',
27
+ });
28
+
29
+ const contentType = response.headers.get('content-type') || '';
30
+ const text = await response.text();
31
+
32
+ // If not HTML, return raw text
33
+ if (!contentType.includes('html')) {
34
+ const truncated = text.slice(0, maxLength);
35
+ return {
36
+ title: url,
37
+ content: truncated,
38
+ url,
39
+ wordCount: truncated.split(/\s+/).length,
40
+ };
41
+ }
42
+
43
+ return extractReadableContent(text, url, maxLength);
44
+ }
45
+
46
+ /**
47
+ * Extract readable content from HTML using simple heuristics.
48
+ */
49
+ export function extractReadableContent(html: string, url: string, maxLength = MAX_CONTENT_LENGTH): ScrapedContent {
50
+ // Extract title
51
+ const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
52
+ const title = titleMatch ? decodeEntities(titleMatch[1]).trim() : url;
53
+
54
+ // Remove non-content elements
55
+ let content = html;
56
+
57
+ // Remove script, style, nav, header, footer, aside, iframe
58
+ const removePatterns = [
59
+ /<script[\s\S]*?<\/script>/gi,
60
+ /<style[\s\S]*?<\/style>/gi,
61
+ /<nav[\s\S]*?<\/nav>/gi,
62
+ /<footer[\s\S]*?<\/footer>/gi,
63
+ /<aside[\s\S]*?<\/aside>/gi,
64
+ /<iframe[\s\S]*?<\/iframe>/gi,
65
+ /<noscript[\s\S]*?<\/noscript>/gi,
66
+ /<!--[\s\S]*?-->/g,
67
+ ];
68
+
69
+ for (const pattern of removePatterns) {
70
+ content = content.replace(pattern, '');
71
+ }
72
+
73
+ // Try to find main content area
74
+ const mainContent = findMainContent(content);
75
+ content = mainContent || content;
76
+
77
+ // Convert to markdown-ish text
78
+ content = htmlToMarkdown(content);
79
+
80
+ // Clean up whitespace
81
+ content = content
82
+ .replace(/\n{3,}/g, '\n\n')
83
+ .replace(/[ \t]+/g, ' ')
84
+ .trim();
85
+
86
+ // Truncate
87
+ if (content.length > maxLength) {
88
+ content = content.slice(0, maxLength) + '\n\n...[truncated]';
89
+ }
90
+
91
+ return {
92
+ title,
93
+ content,
94
+ url,
95
+ wordCount: content.split(/\s+/).filter(Boolean).length,
96
+ };
97
+ }
98
+
99
+ /**
100
+ * Try to find the main content area of the page.
101
+ */
102
+ function findMainContent(html: string): string | null {
103
+ // Try common content selectors
104
+ const patterns = [
105
+ /<article[^>]*>([\s\S]*?)<\/article>/i,
106
+ /<main[^>]*>([\s\S]*?)<\/main>/i,
107
+ /<div[^>]*class="[^"]*(?:content|article|post|entry|main)[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
108
+ /<div[^>]*id="[^"]*(?:content|article|post|entry|main)[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
109
+ ];
110
+
111
+ for (const pattern of patterns) {
112
+ const match = html.match(pattern);
113
+ if (match && match[1] && match[1].length > 200) {
114
+ return match[1];
115
+ }
116
+ }
117
+
118
+ // Fallback: find body content
119
+ const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
120
+ return bodyMatch ? bodyMatch[1] : null;
121
+ }
122
+
123
+ /**
124
+ * Simple HTML to Markdown conversion.
125
+ */
126
+ function htmlToMarkdown(html: string): string {
127
+ let md = html;
128
+
129
+ // Headers
130
+ md = md.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, '\n# $1\n');
131
+ md = md.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, '\n## $1\n');
132
+ md = md.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, '\n### $1\n');
133
+ md = md.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, '\n#### $1\n');
134
+ md = md.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, '\n##### $1\n');
135
+ md = md.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, '\n###### $1\n');
136
+
137
+ // Paragraphs and line breaks
138
+ md = md.replace(/<p[^>]*>/gi, '\n');
139
+ md = md.replace(/<\/p>/gi, '\n');
140
+ md = md.replace(/<br\s*\/?>/gi, '\n');
141
+
142
+ // Links
143
+ md = md.replace(/<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi, '[$2]($1)');
144
+
145
+ // Bold and italic
146
+ md = md.replace(/<(?:strong|b)[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi, '**$1**');
147
+ md = md.replace(/<(?:em|i)[^>]*>([\s\S]*?)<\/(?:em|i)>/gi, '*$1*');
148
+
149
+ // Code
150
+ md = md.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, '`$1`');
151
+ md = md.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, '\n```\n$1\n```\n');
152
+
153
+ // Lists
154
+ md = md.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n');
155
+
156
+ // Blockquote
157
+ md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, '\n> $1\n');
158
+
159
+ // Remove remaining HTML tags
160
+ md = md.replace(/<[^>]+>/g, '');
161
+
162
+ // Decode entities
163
+ md = decodeEntities(md);
164
+
165
+ return md;
166
+ }
167
+
168
+ function decodeEntities(text: string): string {
169
+ return text
170
+ .replace(/&amp;/g, '&')
171
+ .replace(/&lt;/g, '<')
172
+ .replace(/&gt;/g, '>')
173
+ .replace(/&quot;/g, '"')
174
+ .replace(/&#x27;/g, "'")
175
+ .replace(/&#39;/g, "'")
176
+ .replace(/&nbsp;/g, ' ')
177
+ .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n)))
178
+ .replace(/&#x([0-9a-fA-F]+);/g, (_, n) => String.fromCharCode(parseInt(n, 16)));
179
+ }