0xkobold 0.0.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HEARTBEAT.md +66 -32
- package/README.md +220 -1
- package/dist/package.json +3 -2
- package/dist/src/agent/bootstrap-loader.js +138 -0
- package/dist/src/agent/bootstrap-loader.js.map +1 -0
- package/dist/src/agent/context-pruning.js +279 -0
- package/dist/src/agent/context-pruning.js.map +1 -0
- package/dist/src/agent/dynamic-personality.js +211 -0
- package/dist/src/agent/dynamic-personality.js.map +1 -0
- package/dist/src/agent/embedded-runner.js +79 -0
- package/dist/src/agent/embedded-runner.js.map +1 -0
- package/dist/src/agent/index.js +16 -0
- package/dist/src/agent/index.js.map +1 -0
- package/dist/src/agent/system-prompt.js +84 -0
- package/dist/src/agent/system-prompt.js.map +1 -0
- package/dist/src/agent/task-router.js +194 -0
- package/dist/src/agent/task-router.js.map +1 -0
- package/dist/src/agent/tools/index.js +2 -0
- package/dist/src/agent/tools/index.js.map +1 -0
- package/dist/src/agent/tools/spawn-agent.js +200 -0
- package/dist/src/agent/tools/spawn-agent.js.map +1 -0
- package/dist/src/agent/types/definitions.js +317 -0
- package/dist/src/agent/types/definitions.js.map +1 -0
- package/dist/src/agent/types/index.js +2 -0
- package/dist/src/agent/types/index.js.map +1 -0
- package/dist/src/agent/user-profile.js +300 -0
- package/dist/src/agent/user-profile.js.map +1 -0
- package/dist/src/agents/task-router.js +194 -0
- package/dist/src/agents/task-router.js.map +1 -0
- package/dist/src/agents/tools/index.js +2 -0
- package/dist/src/agents/tools/index.js.map +1 -0
- package/dist/src/agents/tools/spawn-agent.js +200 -0
- package/dist/src/agents/tools/spawn-agent.js.map +1 -0
- package/dist/src/agents/types/definitions.js +317 -0
- package/dist/src/agents/types/definitions.js.map +1 -0
- package/dist/src/agents/types/index.js +2 -0
- package/dist/src/agents/types/index.js.map +1 -0
- package/dist/src/cli/commands/embedded.js +36 -0
- package/dist/src/cli/commands/embedded.js.map +1 -0
- package/dist/src/cli/commands/gateway.js +127 -166
- package/dist/src/cli/commands/gateway.js.map +1 -1
- package/dist/src/cli/program-fixed.js +1 -0
- package/dist/src/cli/program-fixed.js.map +1 -0
- package/dist/src/cli/program.js +10 -29
- package/dist/src/cli/program.js.map +1 -1
- package/dist/src/extensions/core/agent-lifecycle-extension.js +323 -0
- package/dist/src/extensions/core/agent-lifecycle-extension.js.map +1 -0
- package/dist/src/extensions/core/agent-orchestrator-extension.js +642 -0
- package/dist/src/extensions/core/agent-orchestrator-extension.js.map +1 -0
- package/dist/src/extensions/core/agent-workspace-extension.js +174 -0
- package/dist/src/extensions/core/agent-workspace-extension.js.map +1 -0
- package/dist/src/extensions/core/autonomous-executor-extension.js +145 -0
- package/dist/src/extensions/core/autonomous-executor-extension.js.map +1 -0
- package/dist/src/extensions/core/autonomous-subagent-extension.js +333 -0
- package/dist/src/extensions/core/autonomous-subagent-extension.js.map +1 -0
- package/dist/src/extensions/core/deprecated/agent-lifecycle-extension.js +323 -0
- package/dist/src/extensions/core/deprecated/agent-lifecycle-extension.js.map +1 -0
- package/dist/src/extensions/core/deprecated/autonomous-subagent-extension.js +333 -0
- package/dist/src/extensions/core/deprecated/autonomous-subagent-extension.js.map +1 -0
- package/dist/src/extensions/core/deprecated/subagent-extension.js +647 -0
- package/dist/src/extensions/core/deprecated/subagent-extension.js.map +1 -0
- package/dist/src/extensions/core/mode-manager-extension.js +11 -0
- package/dist/src/extensions/core/mode-manager-extension.js.map +1 -1
- package/dist/src/extensions/core/perennial-memory-extension.js +10 -2
- package/dist/src/extensions/core/perennial-memory-extension.js.map +1 -1
- package/dist/src/extensions/core/persona-loader-extension.js +61 -129
- package/dist/src/extensions/core/persona-loader-extension.js.map +1 -1
- package/dist/src/extensions/core/websearch-enhanced-extension.js +705 -0
- package/dist/src/extensions/core/websearch-enhanced-extension.js.map +1 -0
- package/dist/src/extensions/core/websearch-v2-extension.js +303 -0
- package/dist/src/extensions/core/websearch-v2-extension.js.map +1 -0
- package/dist/src/gateway/discord-bot.js +185 -0
- package/dist/src/gateway/discord-bot.js.map +1 -0
- package/dist/src/gateway/index.js +10 -350
- package/dist/src/gateway/index.js.map +1 -1
- package/dist/src/gateway/server.js +325 -0
- package/dist/src/gateway/server.js.map +1 -0
- package/dist/src/gateway/websocket-server.js +142 -0
- package/dist/src/gateway/websocket-server.js.map +1 -0
- package/dist/src/heartbeat/checkin.js +185 -0
- package/dist/src/heartbeat/checkin.js.map +1 -0
- package/dist/src/heartbeat/index.js +5 -0
- package/dist/src/heartbeat/index.js.map +1 -0
- package/dist/src/heartbeat/notifications.js +216 -0
- package/dist/src/heartbeat/notifications.js.map +1 -0
- package/dist/src/heartbeat/scheduler.js +284 -0
- package/dist/src/heartbeat/scheduler.js.map +1 -0
- package/dist/src/index.js +21 -3
- package/dist/src/index.js.map +1 -1
- package/dist/src/mode/auto-detector.js +211 -0
- package/dist/src/mode/auto-detector.js.map +1 -0
- package/dist/src/mode/index.js +3 -0
- package/dist/src/mode/index.js.map +1 -0
- package/dist/src/mode/natural-switcher.js +123 -0
- package/dist/src/mode/natural-switcher.js.map +1 -0
- package/dist/src/skills/builtin/api-worker.js +88 -0
- package/dist/src/skills/builtin/api-worker.js.map +1 -0
- package/dist/src/skills/builtin/nextjs-worker.js +55 -0
- package/dist/src/skills/builtin/nextjs-worker.js.map +1 -0
- package/dist/src/skills/builtin/real-workers.js +166 -0
- package/dist/src/skills/builtin/real-workers.js.map +1 -0
- package/dist/src/skills/builtin/sql-worker.js +61 -0
- package/dist/src/skills/builtin/sql-worker.js.map +1 -0
- package/dist/src/skills/builtin/test-worker.js +79 -0
- package/dist/src/skills/builtin/test-worker.js.map +1 -0
- package/dist/src/skills/builtin/web-research.js +77 -0
- package/dist/src/skills/builtin/web-research.js.map +1 -0
- package/dist/src/skills/framework.js +250 -0
- package/dist/src/skills/framework.js.map +1 -0
- package/dist/src/skills/index.js +10 -10
- package/dist/src/skills/index.js.map +1 -1
- package/dist/src/streaming/block-streamer.js +172 -0
- package/dist/src/streaming/block-streamer.js.map +1 -0
- package/dist/src/streaming/index.js +2 -0
- package/dist/src/streaming/index.js.map +1 -0
- package/dist/src/workspace/index.js +2 -0
- package/dist/src/workspace/index.js.map +1 -0
- package/dist/src/workspace/manager.js +181 -0
- package/dist/src/workspace/manager.js.map +1 -0
- package/package.json +3 -2
|
@@ -0,0 +1,705 @@
|
|
|
1
|
+
import { $ } from "bun";
|
|
2
|
+
import { parseArgs } from "../command-args.js";
|
|
3
|
+
// ═══════════════════════════════════════════════════════════════
|
|
4
|
+
// CONTENT EXTRACTION METHODS (Cascade)
|
|
5
|
+
// ═══════════════════════════════════════════════════════════════
|
|
6
|
+
/**
|
|
7
|
+
* Method 1: Fast fetch for simple HTML sites
|
|
8
|
+
*/
|
|
9
|
+
async function fastFetch(url, maxLength) {
|
|
10
|
+
try {
|
|
11
|
+
const controller = new AbortController();
|
|
12
|
+
const timeout = setTimeout(() => controller.abort(), 10000);
|
|
13
|
+
const response = await fetch(url, {
|
|
14
|
+
headers: {
|
|
15
|
+
'User-Agent': 'Mozilla/5.0 (compatible; Bot/1.0)'
|
|
16
|
+
},
|
|
17
|
+
signal: controller.signal
|
|
18
|
+
});
|
|
19
|
+
clearTimeout(timeout);
|
|
20
|
+
if (!response.ok)
|
|
21
|
+
return null;
|
|
22
|
+
const html = await response.text();
|
|
23
|
+
const title = html.match(/<title[^>]*>(.*?)<\/title>/i)?.[1]?.trim() || "Untitled";
|
|
24
|
+
// Basic content extraction
|
|
25
|
+
const content = html
|
|
26
|
+
.replace(/<script[^>]*>.*?<\/script>/gi, '')
|
|
27
|
+
.replace(/<style[^>]*>.*?<\/style>/gi, '')
|
|
28
|
+
.replace(/<[^>]*>/g, ' ')
|
|
29
|
+
.replace(/\s+/g, ' ')
|
|
30
|
+
.trim()
|
|
31
|
+
.slice(0, maxLength);
|
|
32
|
+
// Check if content is substantial
|
|
33
|
+
if (content.length < 200)
|
|
34
|
+
return null;
|
|
35
|
+
return { content, title, method: 'fast', url };
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Browser Manager - Singleton pattern for browser reuse
|
|
43
|
+
*/
|
|
44
|
+
class BrowserManager {
|
|
45
|
+
browser = null;
|
|
46
|
+
context = null;
|
|
47
|
+
lastUsed = 0;
|
|
48
|
+
POOL_TTL_MS = 120000; // 2 minutes
|
|
49
|
+
async getBrowser() {
|
|
50
|
+
const { chromium } = await import('playwright');
|
|
51
|
+
// Check if existing browser is still alive
|
|
52
|
+
if (this.browser && Date.now() - this.lastUsed < this.POOL_TTL_MS) {
|
|
53
|
+
try {
|
|
54
|
+
// Test if browser is still responsive
|
|
55
|
+
await this.browser.contexts();
|
|
56
|
+
this.lastUsed = Date.now();
|
|
57
|
+
return { browser: this.browser, context: this.context, newBrowser: false };
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
// Browser died, recreate
|
|
61
|
+
await this.close();
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
// Create new browser
|
|
65
|
+
console.log("[WebSearch] Creating new browser instance");
|
|
66
|
+
this.browser = await chromium.launch({
|
|
67
|
+
headless: true,
|
|
68
|
+
args: [
|
|
69
|
+
'--no-sandbox',
|
|
70
|
+
'--disable-setuid-sandbox',
|
|
71
|
+
'--disable-dev-shm-usage',
|
|
72
|
+
'--disable-gpu',
|
|
73
|
+
'--no-first-run',
|
|
74
|
+
'--disable-background-networking'
|
|
75
|
+
]
|
|
76
|
+
});
|
|
77
|
+
this.context = await this.browser.newContext({
|
|
78
|
+
viewport: { width: 1280, height: 720 },
|
|
79
|
+
userAgent: 'Mozilla/5.0 (compatible; 0xKobold/0.2)'
|
|
80
|
+
});
|
|
81
|
+
this.lastUsed = Date.now();
|
|
82
|
+
return { browser: this.browser, context: this.context, newBrowser: true };
|
|
83
|
+
}
|
|
84
|
+
async close() {
|
|
85
|
+
if (this.browser) {
|
|
86
|
+
try {
|
|
87
|
+
await this.browser.close();
|
|
88
|
+
}
|
|
89
|
+
catch { }
|
|
90
|
+
this.browser = null;
|
|
91
|
+
this.context = null;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const browserManager = new BrowserManager();
|
|
96
|
+
const requestQueue = [];
|
|
97
|
+
let isProcessing = false;
|
|
98
|
+
const MAX_CONCURRENT = 2;
|
|
99
|
+
const MAX_RETRIES = 3;
|
|
100
|
+
/**
|
|
101
|
+
* Process queue with limited concurrency
|
|
102
|
+
*/
|
|
103
|
+
async function processQueue() {
|
|
104
|
+
if (isProcessing)
|
|
105
|
+
return;
|
|
106
|
+
isProcessing = true;
|
|
107
|
+
while (requestQueue.length > 0) {
|
|
108
|
+
// Take up to MAX_CONCURRENT requests
|
|
109
|
+
const batch = requestQueue.splice(0, MAX_CONCURRENT);
|
|
110
|
+
// Process batch in parallel
|
|
111
|
+
await Promise.all(batch.map(req => processRequest(req)));
|
|
112
|
+
// Small delay between batches to avoid rate limiting
|
|
113
|
+
if (requestQueue.length > 0) {
|
|
114
|
+
await new Promise(r => setTimeout(r, 500));
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
isProcessing = false;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Process a single request with retries
|
|
121
|
+
*/
|
|
122
|
+
async function processRequest(req, attempt = 1) {
|
|
123
|
+
try {
|
|
124
|
+
const result = await playwrightFetchWithTimeout(req.url, req.maxLength, req.timeoutMs);
|
|
125
|
+
req.resolve(result);
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
if (attempt < MAX_RETRIES) {
|
|
129
|
+
// Exponential backoff: 2s, 4s, 8s
|
|
130
|
+
const delay = Math.min(2000 * Math.pow(2, attempt - 1), 30000);
|
|
131
|
+
console.log(`[WebFetch] Retry ${attempt + 1}/${MAX_RETRIES} for ${req.url} after ${delay}ms`);
|
|
132
|
+
await new Promise(r => setTimeout(r, delay));
|
|
133
|
+
await processRequest(req, attempt + 1);
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
console.log(`[WebFetch] Failed after ${MAX_RETRIES} attempts: ${req.url}`);
|
|
137
|
+
req.resolve(null);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Playwright fetch with strict timeout enforcement
|
|
143
|
+
*/
|
|
144
|
+
async function playwrightFetchWithTimeout(url, maxLength, timeoutMs = 15000) {
|
|
145
|
+
const startTime = Date.now();
|
|
146
|
+
// Create abort controller for this request
|
|
147
|
+
const controller = new AbortController();
|
|
148
|
+
const timeoutId = setTimeout(() => controller.abort(), timeoutMs + 2000); // +2s buffer
|
|
149
|
+
try {
|
|
150
|
+
const { context, newBrowser } = await browserManager.getBrowser();
|
|
151
|
+
// Create page with request timeout
|
|
152
|
+
const page = await context.newPage();
|
|
153
|
+
// Set very aggressive timeouts
|
|
154
|
+
page.setDefaultTimeout(Math.min(timeoutMs, 10000));
|
|
155
|
+
page.setDefaultNavigationTimeout(Math.min(timeoutMs, 10000));
|
|
156
|
+
// Watch for abort signal
|
|
157
|
+
controller.signal.addEventListener('abort', () => {
|
|
158
|
+
page.close().catch(() => { });
|
|
159
|
+
});
|
|
160
|
+
try {
|
|
161
|
+
// Quick navigation - no waiting for full load
|
|
162
|
+
const response = await page.goto(url, {
|
|
163
|
+
waitUntil: 'commit', // Fastest: just wait for headers
|
|
164
|
+
timeout: Math.min(timeoutMs, 10000)
|
|
165
|
+
});
|
|
166
|
+
if (!response) {
|
|
167
|
+
throw new Error("No response");
|
|
168
|
+
}
|
|
169
|
+
// Give it a moment to start rendering
|
|
170
|
+
await page.waitForTimeout(500);
|
|
171
|
+
// Extract whatever we have
|
|
172
|
+
const extracted = await page.evaluate((maxLen) => {
|
|
173
|
+
const doc = globalThis.document;
|
|
174
|
+
// Try main content areas quickly
|
|
175
|
+
const main = doc.querySelector('main, article, .content, [role="main"]');
|
|
176
|
+
if (main?.innerText?.trim().length > 100) {
|
|
177
|
+
return main.innerText.slice(0, maxLen);
|
|
178
|
+
}
|
|
179
|
+
// Fallback: body text
|
|
180
|
+
const bodyText = doc.body?.innerText || '';
|
|
181
|
+
return bodyText.slice(0, maxLen);
|
|
182
|
+
}, maxLength);
|
|
183
|
+
const title = await page.title().catch(() => 'Untitled');
|
|
184
|
+
if (!extracted || extracted.length < 50) {
|
|
185
|
+
throw new Error("Insufficient content");
|
|
186
|
+
}
|
|
187
|
+
return {
|
|
188
|
+
content: extracted,
|
|
189
|
+
title: title,
|
|
190
|
+
url,
|
|
191
|
+
method: newBrowser ? 'playwright-new' : 'playwright-pooled'
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
finally {
|
|
195
|
+
await page.close();
|
|
196
|
+
clearTimeout(timeoutId);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
catch (error) {
|
|
200
|
+
console.log(`[WebFetch] Error: ${error}`);
|
|
201
|
+
throw error; // Will trigger retry
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Queue-based Playwright fetch
|
|
206
|
+
*/
|
|
207
|
+
async function playwrightFetch(url, maxLength, timeoutMs = 15000) {
|
|
208
|
+
return new Promise((resolve, reject) => {
|
|
209
|
+
requestQueue.push({
|
|
210
|
+
url,
|
|
211
|
+
maxLength,
|
|
212
|
+
timeoutMs,
|
|
213
|
+
resolve,
|
|
214
|
+
reject,
|
|
215
|
+
startTime: Date.now()
|
|
216
|
+
});
|
|
217
|
+
// Start processing
|
|
218
|
+
processQueue();
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Method 3: Readability-style extraction using Bun/regex
|
|
223
|
+
*/
|
|
224
|
+
async function readabilityFetch(url, maxLength) {
|
|
225
|
+
try {
|
|
226
|
+
const response = await fetch(url, {
|
|
227
|
+
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; Bot/1.0)' }
|
|
228
|
+
});
|
|
229
|
+
if (!response.ok)
|
|
230
|
+
return null;
|
|
231
|
+
const html = await response.text();
|
|
232
|
+
// Try to find article content
|
|
233
|
+
const articleMatch = html.match(/<article[^>]*>([\s\S]*?)<\/article>/i);
|
|
234
|
+
const mainMatch = html.match(/<main[^>]*>([\s\S]*?)<\/main>/i);
|
|
235
|
+
const contentDiv = html.match(/<div[^>]*class="[^"]*(?:content|article|post)[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
|
|
236
|
+
const rawContent = articleMatch?.[1] || mainMatch?.[1] || contentDiv?.[1];
|
|
237
|
+
if (!rawContent)
|
|
238
|
+
return null;
|
|
239
|
+
// Clean the content
|
|
240
|
+
const content = rawContent
|
|
241
|
+
.replace(/<script[^>]*>.*?<\/script>/gi, '')
|
|
242
|
+
.replace(/<style[^>]*>.*?<\/style>/gi, '')
|
|
243
|
+
.replace(/<[^>]*>/g, ' ')
|
|
244
|
+
.replace(/\s+/g, ' ')
|
|
245
|
+
.trim()
|
|
246
|
+
.slice(0, maxLength);
|
|
247
|
+
const title = html.match(/<title[^>]*>(.*?)<\/title>/i)?.[1]?.trim() || "Untitled";
|
|
248
|
+
if (content.length < 200)
|
|
249
|
+
return null;
|
|
250
|
+
return { content, title, method: 'readability', url };
|
|
251
|
+
}
|
|
252
|
+
catch {
|
|
253
|
+
return null;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* CASCADE: Try all methods in order of speed → quality
|
|
258
|
+
*/
|
|
259
|
+
async function cascadeFetch(url, maxLength = 5000, usePlaywright = false, timeoutMs = 15000) {
|
|
260
|
+
console.log(`[WebSearch] Starting cascade fetch for ${url}`);
|
|
261
|
+
// Level 1: Fast HTML fetch (skip if forcing Playwright)
|
|
262
|
+
if (!usePlaywright) {
|
|
263
|
+
console.log("[WebSearch] Trying fast fetch...");
|
|
264
|
+
const fast = await fastFetch(url, maxLength);
|
|
265
|
+
if (fast && fast.content.length > 1000) {
|
|
266
|
+
console.log("[WebSearch] Fast fetch succeeded");
|
|
267
|
+
return fast;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
// Level 2: Readability extraction
|
|
271
|
+
if (!usePlaywright) {
|
|
272
|
+
console.log("[WebSearch] Trying readability extraction...");
|
|
273
|
+
const readability = await readabilityFetch(url, maxLength);
|
|
274
|
+
if (readability) {
|
|
275
|
+
console.log("[WebSearch] Readability extraction succeeded");
|
|
276
|
+
return readability;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
// Level 3: JavaScript rendering with Playwright
|
|
280
|
+
console.log(`[WebSearch] Trying Playwright (timeout: ${timeoutMs}ms)...`);
|
|
281
|
+
const playwright = await playwrightFetch(url, maxLength, timeoutMs);
|
|
282
|
+
if (playwright) {
|
|
283
|
+
console.log("[WebSearch] Playwright succeeded");
|
|
284
|
+
return playwright;
|
|
285
|
+
}
|
|
286
|
+
// Fallback: Return whatever we got from fast fetch
|
|
287
|
+
return null;
|
|
288
|
+
}
|
|
289
|
+
// ═══════════════════════════════════════════════════════════════
|
|
290
|
+
// SEARCH METHODS
|
|
291
|
+
// ═══════════════════════════════════════════════════════════════
|
|
292
|
+
async function searchDuckDuckGo(query, limit) {
|
|
293
|
+
const results = [];
|
|
294
|
+
try {
|
|
295
|
+
// Use Lite/DuckDuckGo HTML version
|
|
296
|
+
const searchUrl = `https://duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
297
|
+
const html = await $ `curl -s -A "Mozilla/5.0 (compatible; SearchBot/1.0)" --max-time 10 ${searchUrl}`.text();
|
|
298
|
+
// Parse results
|
|
299
|
+
const resultRegex = /<a[^>]*class="result__a"[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gi;
|
|
300
|
+
let match;
|
|
301
|
+
while ((match = resultRegex.exec(html)) && results.length < limit) {
|
|
302
|
+
const url = match[1].replace(/^\/l\/\?kh=-\d+&u=/, ''); // DuckDuckGo redirect
|
|
303
|
+
const decodedUrl = decodeURIComponent(url);
|
|
304
|
+
const title = match[2].replace(/<[^>]*>/g, '');
|
|
305
|
+
if (decodedUrl && title && !decodedUrl.includes('duckduckgo.com')) {
|
|
306
|
+
results.push({ title, url: decodedUrl, snippet: "" });
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
catch (error) {
|
|
311
|
+
console.log("[WebSearch] DuckDuckGo search failed:", error);
|
|
312
|
+
}
|
|
313
|
+
return results;
|
|
314
|
+
}
|
|
315
|
+
async function searchSearX(query, limit, instance) {
|
|
316
|
+
const results = [];
|
|
317
|
+
const searxInstances = instance ? [instance] : [
|
|
318
|
+
"https://searx.be",
|
|
319
|
+
"https://searx.tiekoetter.com",
|
|
320
|
+
"https://search.sapti.me"
|
|
321
|
+
];
|
|
322
|
+
for (const baseUrl of searxInstances) {
|
|
323
|
+
try {
|
|
324
|
+
const searchUrl = `${baseUrl}/search?q=${encodeURIComponent(query)}&format=json`;
|
|
325
|
+
const response = await fetch(searchUrl, {
|
|
326
|
+
headers: { 'User-Agent': 'Mozilla/5.0' },
|
|
327
|
+
signal: AbortSignal.timeout(10000)
|
|
328
|
+
});
|
|
329
|
+
if (!response.ok)
|
|
330
|
+
continue;
|
|
331
|
+
const data = await response.json();
|
|
332
|
+
if (data.results) {
|
|
333
|
+
for (const r of data.results.slice(0, limit)) {
|
|
334
|
+
results.push({
|
|
335
|
+
title: r.title || "Untitled",
|
|
336
|
+
url: r.url,
|
|
337
|
+
snippet: r.content || r.abstract || ""
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
if (results.length >= limit)
|
|
341
|
+
break;
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
catch {
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return results;
|
|
349
|
+
}
|
|
350
|
+
// ═══════════════════════════════════════════════════════════════
|
|
351
|
+
// ARGUMENT EXTRACTION
|
|
352
|
+
// ═══════════════════════════════════════════════════════════════
|
|
353
|
+
function extractArg(args, key, defaultVal) {
|
|
354
|
+
if (!args)
|
|
355
|
+
return defaultVal;
|
|
356
|
+
if (args[key] !== undefined)
|
|
357
|
+
return args[key];
|
|
358
|
+
if (args.parameters?.[key] !== undefined)
|
|
359
|
+
return args.parameters[key];
|
|
360
|
+
if (Array.isArray(args) && args[0]?.[key] !== undefined)
|
|
361
|
+
return args[0][key];
|
|
362
|
+
if (args.input?.[key] !== undefined)
|
|
363
|
+
return args.input[key];
|
|
364
|
+
if (args.arguments?.[key] !== undefined)
|
|
365
|
+
return args.arguments[key];
|
|
366
|
+
return defaultVal;
|
|
367
|
+
}
|
|
368
|
+
// ═══════════════════════════════════════════════════════════════
|
|
369
|
+
// MAIN EXTENSION
|
|
370
|
+
// ═══════════════════════════════════════════════════════════════
|
|
371
|
+
export default function enhancedWebSearchExtension(pi) {
|
|
372
|
+
// ═══════════════════════════════════════════════════════════════
|
|
373
|
+
// COMMANDS
|
|
374
|
+
// ═══════════════════════════════════════════════════════════════
|
|
375
|
+
pi.registerCommand("deep-fetch", {
|
|
376
|
+
description: "Fetch JavaScript-rendered content from a URL using Playwright",
|
|
377
|
+
handler: async (args, ctx) => {
|
|
378
|
+
const parsed = parseArgs(args, [
|
|
379
|
+
{ name: "url", description: "URL to fetch", required: true },
|
|
380
|
+
{ name: "max", description: "Max characters", required: false }
|
|
381
|
+
]);
|
|
382
|
+
const url = parsed.url;
|
|
383
|
+
const max = parseInt(String(parsed.max)) || 8000;
|
|
384
|
+
if (!url?.startsWith("http")) {
|
|
385
|
+
ctx.ui?.notify?.("❌ URL must start with http:// or https://", "error");
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
ctx.ui?.notify?.(`🔍 Deep fetching: ${url}`, "info");
|
|
389
|
+
const result = await cascadeFetch(url, max);
|
|
390
|
+
if (!result) {
|
|
391
|
+
ctx.ui?.notify?.("❌ Failed to fetch content", "error");
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
const lines = [
|
|
395
|
+
`📄 ${result.title}`,
|
|
396
|
+
`Method: ${result.method} | Source: ${result.url}`,
|
|
397
|
+
"─────────────────────────────────────────",
|
|
398
|
+
"",
|
|
399
|
+
result.content.slice(0, max),
|
|
400
|
+
result.content.length > max
|
|
401
|
+
? `\n... (${result.content.length - max} more chars)`
|
|
402
|
+
: "",
|
|
403
|
+
"",
|
|
404
|
+
"─────────────────────────────────────────"
|
|
405
|
+
];
|
|
406
|
+
ctx.ui?.notify?.(lines.join("\n"), "info");
|
|
407
|
+
}
|
|
408
|
+
});
|
|
409
|
+
pi.registerCommand("web-search-deep", {
|
|
410
|
+
description: "Search web + fetch content from top results",
|
|
411
|
+
handler: async (args, ctx) => {
|
|
412
|
+
const parsed = parseArgs(args, [
|
|
413
|
+
{ name: "query", description: "Search query", required: true },
|
|
414
|
+
{ name: "results", description: "Number of results", required: false }
|
|
415
|
+
]);
|
|
416
|
+
const query = parsed.query;
|
|
417
|
+
const numResults = parseInt(String(parsed.results)) || 3;
|
|
418
|
+
ctx.ui?.notify?.(`🔍 Searching + fetching: "${query}"`, "info");
|
|
419
|
+
// Search
|
|
420
|
+
let results = await searchDuckDuckGo(query, numResults * 2);
|
|
421
|
+
if (results.length < numResults) {
|
|
422
|
+
const searxResults = await searchSearX(query, numResults * 2);
|
|
423
|
+
results = [...results, ...searxResults].slice(0, numResults * 2);
|
|
424
|
+
}
|
|
425
|
+
if (results.length === 0) {
|
|
426
|
+
ctx.ui?.notify?.("❌ No search results found", "error");
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
// Fetch content from top results
|
|
430
|
+
ctx.ui?.notify?.(`📥 Fetching content from ${Math.min(numResults, results.length)} sources...`, "info");
|
|
431
|
+
const fetched = [];
|
|
432
|
+
for (let i = 0; i < Math.min(numResults, results.length); i++) {
|
|
433
|
+
const r = results[i];
|
|
434
|
+
ctx.ui?.notify?.(` Fetching ${i + 1}/${numResults}: ${r.title.slice(0, 50)}...`, "info");
|
|
435
|
+
const content = await cascadeFetch(r.url, 3000);
|
|
436
|
+
if (content)
|
|
437
|
+
fetched.push(content);
|
|
438
|
+
}
|
|
439
|
+
// Format output
|
|
440
|
+
const lines = [
|
|
441
|
+
`🔍 Research Results: "${query}"`,
|
|
442
|
+
`Sources: ${fetched.length} / ${results.length} found`,
|
|
443
|
+
"═══════════════════════════════════════════",
|
|
444
|
+
""
|
|
445
|
+
];
|
|
446
|
+
for (let i = 0; i < fetched.length; i++) {
|
|
447
|
+
const f = fetched[i];
|
|
448
|
+
lines.push(`## ${i + 1}. ${f.title}`);
|
|
449
|
+
lines.push(`Source: ${f.url} | Method: ${f.method}`);
|
|
450
|
+
lines.push("");
|
|
451
|
+
lines.push(f.content.slice(0, 2500));
|
|
452
|
+
lines.push("");
|
|
453
|
+
lines.push("─────────────────────────────────────────");
|
|
454
|
+
lines.push("");
|
|
455
|
+
}
|
|
456
|
+
ctx.ui?.notify?.(lines.join("\n"), "info");
|
|
457
|
+
}
|
|
458
|
+
});
|
|
459
|
+
// ═══════════════════════════════════════════════════════════════
|
|
460
|
+
// ENHANCED TOOLS
|
|
461
|
+
// ═══════════════════════════════════════════════════════════════
|
|
462
|
+
// Replace existing web_fetch with enhanced version
|
|
463
|
+
pi.registerTool({
|
|
464
|
+
// @ts-ignore Tool
|
|
465
|
+
name: "web_fetch",
|
|
466
|
+
label: "Web Fetch",
|
|
467
|
+
description: "Fetch and extract content from a web page. Uses cascade: fast HTML → readability → Playwright for JS sites. Now with configurable timeout.",
|
|
468
|
+
// @ts-ignore TSchema
|
|
469
|
+
parameters: {
|
|
470
|
+
type: "object",
|
|
471
|
+
properties: {
|
|
472
|
+
url: {
|
|
473
|
+
type: "string",
|
|
474
|
+
description: "Full URL to fetch (must include http:// or https://)"
|
|
475
|
+
},
|
|
476
|
+
max_length: {
|
|
477
|
+
type: "number",
|
|
478
|
+
default: 5000,
|
|
479
|
+
description: "Maximum characters to retrieve"
|
|
480
|
+
},
|
|
481
|
+
use_playwright: {
|
|
482
|
+
type: "boolean",
|
|
483
|
+
default: false,
|
|
484
|
+
description: "Force use of Playwright for JavaScript-rendered content"
|
|
485
|
+
},
|
|
486
|
+
timeout_ms: {
|
|
487
|
+
type: "number",
|
|
488
|
+
default: 15000,
|
|
489
|
+
description: "Timeout in milliseconds (default: 15000, max: 60000)"
|
|
490
|
+
}
|
|
491
|
+
},
|
|
492
|
+
required: ["url"],
|
|
493
|
+
},
|
|
494
|
+
async execute(toolCallId, args, signal, onUpdate, ctx) {
|
|
495
|
+
const url = extractArg(args, 'url');
|
|
496
|
+
const max_length = extractArg(args, 'max_length', 5000);
|
|
497
|
+
const use_playwright = extractArg(args, 'use_playwright', false);
|
|
498
|
+
const timeout_ms = Math.min(extractArg(args, 'timeout_ms', 15000), 60000);
|
|
499
|
+
if (!url || typeof url !== 'string') {
|
|
500
|
+
return {
|
|
501
|
+
content: [{ type: "text", text: "Invalid URL provided" }],
|
|
502
|
+
details: { error: "invalid_url" },
|
|
503
|
+
};
|
|
504
|
+
}
|
|
505
|
+
if (!url.startsWith("http")) {
|
|
506
|
+
return {
|
|
507
|
+
content: [{ type: "text", text: "URL must start with http:// or https://" }],
|
|
508
|
+
details: { error: "invalid_url_scheme" },
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
let result;
|
|
512
|
+
if (use_playwright) {
|
|
513
|
+
result = await playwrightFetch(url, max_length, timeout_ms);
|
|
514
|
+
}
|
|
515
|
+
else {
|
|
516
|
+
result = await cascadeFetch(url, max_length, use_playwright, timeout_ms);
|
|
517
|
+
}
|
|
518
|
+
if (!result) {
|
|
519
|
+
return {
|
|
520
|
+
content: [{ type: "text", text: `Failed to fetch content from ${url}` }],
|
|
521
|
+
details: { error: "fetch_failed", url },
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
return {
|
|
525
|
+
content: [
|
|
526
|
+
{
|
|
527
|
+
type: "text",
|
|
528
|
+
text: `# ${result.title}\n\n${result.content}\n\n[Source: ${result.url} | Method: ${result.method}]`
|
|
529
|
+
},
|
|
530
|
+
],
|
|
531
|
+
details: {
|
|
532
|
+
url,
|
|
533
|
+
title: result.title,
|
|
534
|
+
method: result.method,
|
|
535
|
+
length: result.content.length,
|
|
536
|
+
},
|
|
537
|
+
};
|
|
538
|
+
},
|
|
539
|
+
});
|
|
540
|
+
// Enhanced web search with content fetching
|
|
541
|
+
pi.registerTool({
|
|
542
|
+
// @ts-ignore Tool
|
|
543
|
+
name: "web_search",
|
|
544
|
+
label: "Web Search",
|
|
545
|
+
description: "Search web and optionally fetch content from results. Use for research, documentation lookup, finding code examples.",
|
|
546
|
+
// @ts-ignore Tool parameters schema
|
|
547
|
+
parameters: {
|
|
548
|
+
type: "object",
|
|
549
|
+
properties: {
|
|
550
|
+
query: {
|
|
551
|
+
type: "string",
|
|
552
|
+
description: "Search query - be specific"
|
|
553
|
+
},
|
|
554
|
+
limit: {
|
|
555
|
+
type: "number",
|
|
556
|
+
default: 5,
|
|
557
|
+
description: "Number of results (1-10)"
|
|
558
|
+
},
|
|
559
|
+
fetch_content: {
|
|
560
|
+
type: "boolean",
|
|
561
|
+
default: false,
|
|
562
|
+
description: "Whether to fetch full content from top results"
|
|
563
|
+
},
|
|
564
|
+
fetch_sources: {
|
|
565
|
+
type: "number",
|
|
566
|
+
default: 3,
|
|
567
|
+
description: "How many sources to fetch content from (if fetch_content is true)"
|
|
568
|
+
}
|
|
569
|
+
},
|
|
570
|
+
required: ["query"],
|
|
571
|
+
},
|
|
572
|
+
async execute(toolCallId, args, signal, onUpdate, ctx) {
|
|
573
|
+
const query = extractArg(args, 'query');
|
|
574
|
+
const limit = extractArg(args, 'limit', 5);
|
|
575
|
+
const fetch_content = extractArg(args, 'fetch_content', false);
|
|
576
|
+
const fetch_sources = extractArg(args, 'fetch_sources', 3);
|
|
577
|
+
if (!query || typeof query !== 'string') {
|
|
578
|
+
return {
|
|
579
|
+
content: [{ type: "text", text: "Invalid search query" }],
|
|
580
|
+
details: { error: "invalid_query" },
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
// Search
|
|
584
|
+
let results = await searchDuckDuckGo(query, Math.min(limit, 10));
|
|
585
|
+
if (results.length < limit) {
|
|
586
|
+
const searxResults = await searchSearX(query, Math.min(limit, 10));
|
|
587
|
+
results = [...results, ...searxResults].slice(0, limit);
|
|
588
|
+
}
|
|
589
|
+
if (results.length === 0) {
|
|
590
|
+
return {
|
|
591
|
+
content: [{ type: "text", text: "No search results found" }],
|
|
592
|
+
details: { query, error: "no_results" },
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
// Format basic results
|
|
596
|
+
const basicFormatted = results
|
|
597
|
+
.map((r, i) => `${i + 1}. ${r.title}\n ${r.url}${r.snippet ? "\n " + r.snippet : ""}`)
|
|
598
|
+
.join("\n\n");
|
|
599
|
+
// Optionally fetch content
|
|
600
|
+
if (fetch_content) {
|
|
601
|
+
const fetchedContent = [];
|
|
602
|
+
for (let i = 0; i < Math.min(fetch_sources, results.length); i++) {
|
|
603
|
+
const result = await cascadeFetch(results[i].url, 3000);
|
|
604
|
+
if (result) {
|
|
605
|
+
fetchedContent.push(`## ${result.title}\n${result.content.slice(0, 2500)}...\n[Source: ${result.url}]`);
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
return {
|
|
609
|
+
content: [
|
|
610
|
+
{ type: "text", text: `Search results for "${query}":\n\n${basicFormatted}` },
|
|
611
|
+
{ type: "text", text: `\n\nDetailed content from ${fetchedContent.length} sources:\n\n${fetchedContent.join("\n\n---\n\n")}` }
|
|
612
|
+
],
|
|
613
|
+
details: {
|
|
614
|
+
query,
|
|
615
|
+
results: results.length,
|
|
616
|
+
fetched: fetchedContent.length,
|
|
617
|
+
urls: results.map(r => r.url),
|
|
618
|
+
},
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
return {
|
|
622
|
+
content: [
|
|
623
|
+
{ type: "text", text: `Search results for "${query}":\n\n${basicFormatted}` }
|
|
624
|
+
],
|
|
625
|
+
details: {
|
|
626
|
+
query,
|
|
627
|
+
results: results.length,
|
|
628
|
+
urls: results.map(r => r.url),
|
|
629
|
+
},
|
|
630
|
+
};
|
|
631
|
+
},
|
|
632
|
+
});
|
|
633
|
+
// Deep research tool
|
|
634
|
+
pi.registerTool({
|
|
635
|
+
// @ts-ignore Tool
|
|
636
|
+
name: "web_research",
|
|
637
|
+
label: "Web Research",
|
|
638
|
+
description: "Deep research: search + fetch content from multiple sources with synthesis. Best for comprehensive answers.",
|
|
639
|
+
// @ts-ignore Tool parameters schema
|
|
640
|
+
parameters: {
|
|
641
|
+
type: "object",
|
|
642
|
+
properties: {
|
|
643
|
+
question: {
|
|
644
|
+
type: "string",
|
|
645
|
+
description: "The research question"
|
|
646
|
+
},
|
|
647
|
+
sources: {
|
|
648
|
+
type: "number",
|
|
649
|
+
default: 5,
|
|
650
|
+
description: "Number of sources to analyze (1-10)"
|
|
651
|
+
}
|
|
652
|
+
},
|
|
653
|
+
required: ["question"],
|
|
654
|
+
},
|
|
655
|
+
async execute(toolCallId, args, signal, onUpdate, ctx) {
|
|
656
|
+
const question = extractArg(args, 'question');
|
|
657
|
+
const sources = Math.min(extractArg(args, 'sources', 5), 10);
|
|
658
|
+
if (!question) {
|
|
659
|
+
return {
|
|
660
|
+
content: [{ type: "text", text: "Invalid question provided" }],
|
|
661
|
+
details: { error: "invalid_question" },
|
|
662
|
+
};
|
|
663
|
+
}
|
|
664
|
+
// Search
|
|
665
|
+
let searchResults = await searchDuckDuckGo(question, sources * 2);
|
|
666
|
+
if (searchResults.length < sources) {
|
|
667
|
+
const searxResults = await searchSearX(question, sources * 2);
|
|
668
|
+
searchResults = [...searchResults, ...searxResults].slice(0, sources * 2);
|
|
669
|
+
}
|
|
670
|
+
if (searchResults.length === 0) {
|
|
671
|
+
return {
|
|
672
|
+
content: [{ type: "text", text: `Could not find information about: "${question}"` }],
|
|
673
|
+
details: { question, error: "no_results" },
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
// Fetch content from sources
|
|
677
|
+
const fetched = [];
|
|
678
|
+
for (let i = 0; i < Math.min(sources, searchResults.length); i++) {
|
|
679
|
+
const result = await cascadeFetch(searchResults[i].url, 3000);
|
|
680
|
+
if (result)
|
|
681
|
+
fetched.push(result);
|
|
682
|
+
}
|
|
683
|
+
const summary = fetched.length > 0
|
|
684
|
+
? `Research on: "${question}"\n\n` +
|
|
685
|
+
`Found ${fetched.length} relevant sources:\n\n` +
|
|
686
|
+
fetched.map((c, i) => `## ${i + 1}. ${c.title}\n${c.content.slice(0, 2000)}...\n(Source: ${c.url} | Method: ${c.method})`).join("\n\n---\n\n")
|
|
687
|
+
: `Found ${searchResults.length} search results but could not fetch detailed content:\n\n` +
|
|
688
|
+
searchResults.slice(0, sources).map(r => `- ${r.title}: ${r.url}`).join("\n");
|
|
689
|
+
return {
|
|
690
|
+
content: [{ type: "text", text: summary }],
|
|
691
|
+
details: {
|
|
692
|
+
question,
|
|
693
|
+
sources_found: searchResults.length,
|
|
694
|
+
sources_fetched: fetched.length,
|
|
695
|
+
fetched,
|
|
696
|
+
},
|
|
697
|
+
};
|
|
698
|
+
},
|
|
699
|
+
});
|
|
700
|
+
console.log("[WebSearch Enhanced] Extension loaded");
|
|
701
|
+
console.log("[WebSearch Enhanced] Tools: web_search, web_fetch, web_research");
|
|
702
|
+
console.log("[WebSearch Enhanced] Commands: /deep-fetch, /web-search-deep");
|
|
703
|
+
console.log("[WebSearch Enhanced] Features: Cascade extraction (fast → readability → Playwright)");
|
|
704
|
+
}
|
|
705
|
+
//# sourceMappingURL=websearch-enhanced-extension.js.map
|