@jcheesepkg/nanobot 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web.d.mts","names":[],"sources":["../../../src/agent/tools/web.ts"],"mappings":";;;;cA0Ea,aAAA,SAAsB,IAAA;EAAA,SACxB,IAAA;EAAA,SACA,WAAA;EAAA,SACA,UAAA;;;;;;;;;;;;;;;;UAcD,MAAA;EAAA,QACA,UAAA;cAEI,MAAA;IAAW,MAAA;IAAiB,UAAA;EAAA;EAMlC,OAAA,CAAQ,IAAA,EAAM,MAAA,oBAA0B,OAAA;AAAA;;
|
|
1
|
+
{"version":3,"file":"web.d.mts","names":[],"sources":["../../../src/agent/tools/web.ts"],"mappings":";;;;cA0Ea,aAAA,SAAsB,IAAA;EAAA,SACxB,IAAA;EAAA,SACA,WAAA;EAAA,SACA,UAAA;;;;;;;;;;;;;;;;UAcD,MAAA;EAAA,QACA,UAAA;cAEI,MAAA;IAAW,MAAA;IAAiB,UAAA;EAAA;EAMlC,OAAA,CAAQ,IAAA,EAAM,MAAA,oBAA0B,OAAA;AAAA;;cAsDnC,YAAA,SAAqB,IAAA;EAAA,SACvB,IAAA;EAAA,SACA,WAAA;EAAA,SAEA,UAAA;;;;;;;;;;;;;;;;;;;UAcD,eAAA;cAEI,MAAA;IAAW,QAAA;EAAA;EAKjB,OAAA,CAAQ,IAAA,EAAM,MAAA,oBAA0B,OAAA;AAAA"}
|
package/dist/agent/tools/web.mjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web.mjs","names":[],"sources":["../../../src/agent/tools/web.ts"],"sourcesContent":["import { Tool } from \"./base.js\";\n\nconst USER_AGENT =\n \"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36\";\n\n/** Strip HTML tags and decode entities. */\nfunction stripTags(text: string): string {\n let result = text;\n result = result.replace(/<script[\\s\\S]*?<\\/script>/gi, \"\");\n result = result.replace(/<style[\\s\\S]*?<\\/style>/gi, \"\");\n result = result.replace(/<[^>]+>/g, \"\");\n // Decode common HTML entities\n result = result\n .replace(/&/g, \"&\")\n .replace(/</g, \"<\")\n .replace(/>/g, \">\")\n .replace(/"/g, '\"')\n .replace(/'/g, \"'\")\n .replace(/ /g, \" \");\n return result.trim();\n}\n\n/** Normalize whitespace. */\nfunction normalize(text: string): string {\n return text\n .replace(/[ \\t]+/g, \" \")\n .replace(/\\n{3,}/g, \"\\n\\n\")\n .trim();\n}\n\n/** Validate URL. */\nfunction validateUrl(url: string): { valid: boolean; error: string } {\n try {\n const parsed = new URL(url);\n if (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\") {\n return {\n valid: false,\n error: `Only http/https allowed, got '${parsed.protocol}'`,\n };\n }\n if (!parsed.hostname) {\n return { valid: false, error: \"Missing domain\" };\n }\n return { valid: true, error: \"\" };\n } catch (err) {\n return { valid: false, error: String(err) };\n }\n}\n\n/** Convert HTML to basic markdown. */\nfunction htmlToMarkdown(html: string): string {\n let text = html;\n // Links\n text = text.replace(\n /<a\\s+[^>]*href=[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi,\n (_m, url, inner) => `[${stripTags(inner)}](${url})`,\n );\n // Headings\n text = text.replace(\n /<h([1-6])[^>]*>([\\s\\S]*?)<\\/h\\1>/gi,\n (_m, level, inner) => `\\n${\"#\".repeat(Number(level))} ${stripTags(inner)}\\n`,\n );\n // List items\n text = text.replace(\n /<li[^>]*>([\\s\\S]*?)<\\/li>/gi,\n (_m, inner) => `\\n- ${stripTags(inner)}`,\n );\n // Block elements\n text = text.replace(/<\\/(p|div|section|article)>/gi, \"\\n\\n\");\n text = text.replace(/<(br|hr)\\s*\\/?>/gi, \"\\n\");\n return normalize(stripTags(text));\n}\n\n/** Search the web using Brave Search API. */\nexport class WebSearchTool extends Tool {\n readonly name = \"web_search\";\n readonly description = \"Search the web. Returns titles, URLs, and snippets.\";\n readonly parameters = {\n type: \"object\",\n properties: {\n query: { type: \"string\", description: \"Search query\" },\n count: {\n type: \"integer\",\n description: \"Results (1-10)\",\n minimum: 1,\n maximum: 10,\n },\n },\n required: [\"query\"],\n };\n\n private apiKey: string;\n private maxResults: number;\n\n constructor(params?: { apiKey?: string; maxResults?: number }) {\n super();\n this.apiKey = params?.apiKey ?? process.env.BRAVE_API_KEY ?? \"\";\n this.maxResults = params?.maxResults ?? 5;\n }\n\n async execute(args: Record<string, unknown>): Promise<string> {\n const query = String(args.query);\n const count = Math.min(\n Math.max(args.count ? Number(args.count) : this.maxResults, 1),\n 10,\n );\n\n if (!this.apiKey) {\n return \"Error: BRAVE_API_KEY not configured\";\n }\n\n try {\n const url = new URL(\"https://api.search.brave.com/res/v1/web/search\");\n url.searchParams.set(\"q\", query);\n url.searchParams.set(\"count\", String(count));\n\n const resp = await fetch(url.toString(), {\n headers: {\n Accept: \"application/json\",\n \"X-Subscription-Token\": this.apiKey,\n },\n signal: AbortSignal.timeout(10000),\n });\n\n if (!resp.ok) {\n return `Error: Search API returned ${resp.status}`;\n }\n\n const data = (await resp.json()) as {\n web?: { results?: Array<{ title?: string; url?: string; description?: string }> };\n };\n const results = data.web?.results ?? [];\n\n if (results.length === 0) {\n return `No results for: ${query}`;\n }\n\n const lines = [`Results for: ${query}\\n`];\n for (let i = 0; i < Math.min(results.length, count); i++) {\n const item = results[i];\n lines.push(`${i + 1}. ${item.title ?? \"\"}\\n ${item.url ?? \"\"}`);\n if (item.description) {\n lines.push(` ${item.description}`);\n }\n }\n return lines.join(\"\\n\");\n } catch (err) {\n return `Error: ${err instanceof Error ? err.message : err}`;\n }\n }\n}\n\n/** Fetch and extract content from a URL. */\nexport class WebFetchTool extends Tool {\n readonly name = \"web_fetch\";\n readonly description =\n \"Fetch URL and extract readable content (HTML -> markdown/text).\";\n readonly parameters = {\n type: \"object\",\n properties: {\n url: { type: \"string\", description: \"URL to fetch\" },\n extractMode: {\n type: \"string\",\n enum: [\"markdown\", \"text\"],\n description: \"Extract mode\",\n },\n maxChars: { type: \"integer\", minimum: 100 },\n },\n required: [\"url\"],\n };\n\n private defaultMaxChars: number;\n\n constructor(params?: { maxChars?: number }) {\n super();\n this.defaultMaxChars = params?.maxChars ?? 50000;\n }\n\n async execute(args: Record<string, unknown>): Promise<string> {\n const url = String(args.url);\n const extractMode = String(args.extractMode ?? \"markdown\");\n const maxChars = args.maxChars\n ? Number(args.maxChars)\n : this.defaultMaxChars;\n\n const { valid, error: validationError } = validateUrl(url);\n if (!valid) {\n return JSON.stringify({\n error: `URL validation failed: ${validationError}`,\n url,\n });\n }\n\n try {\n const resp = await fetch(url, {\n headers: { \"User-Agent\": USER_AGENT },\n redirect: \"follow\",\n signal: AbortSignal.timeout(30000),\n });\n\n if (!resp.ok) {\n return JSON.stringify({\n error: `HTTP ${resp.status}`,\n url,\n });\n }\n\n const contentType = resp.headers.get(\"content-type\") ?? \"\";\n const body = await resp.text();\n let text: string;\n let extractor: string;\n\n if (contentType.includes(\"application/json\")) {\n try {\n text = JSON.stringify(JSON.parse(body), null, 2);\n } catch {\n text = body;\n }\n extractor = \"json\";\n } else if (\n contentType.includes(\"text/html\") ||\n body.slice(0, 256).toLowerCase().startsWith(\"<!doctype\") ||\n body.slice(0, 256).toLowerCase().startsWith(\"<html\")\n ) {\n // Extract readable content from HTML\n text =\n extractMode === \"markdown\"\n ? htmlToMarkdown(body)\n : stripTags(body);\n extractor = \"html\";\n } else {\n text = body;\n extractor = \"raw\";\n }\n\n const truncated = text.length > maxChars;\n if (truncated) {\n text = text.slice(0, maxChars);\n }\n\n return JSON.stringify({\n url,\n finalUrl: resp.url,\n status: resp.status,\n extractor,\n truncated,\n length: text.length,\n text,\n });\n } catch (err) {\n return JSON.stringify({\n error: err instanceof Error ? err.message : String(err),\n url,\n });\n }\n }\n}\n"],"mappings":";;;AAEA,MAAM,aACJ;;AAGF,SAAS,UAAU,MAAsB;CACvC,IAAI,SAAS;AACb,UAAS,OAAO,QAAQ,+BAA+B,GAAG;AAC1D,UAAS,OAAO,QAAQ,6BAA6B,GAAG;AACxD,UAAS,OAAO,QAAQ,YAAY,GAAG;AAEvC,UAAS,OACN,QAAQ,UAAU,IAAI,CACtB,QAAQ,SAAS,IAAI,CACrB,QAAQ,SAAS,IAAI,CACrB,QAAQ,WAAW,KAAI,CACvB,QAAQ,UAAU,IAAI,CACtB,QAAQ,WAAW,IAAI;AAC1B,QAAO,OAAO,MAAM;;;AAItB,SAAS,UAAU,MAAsB;AACvC,QAAO,KACJ,QAAQ,WAAW,IAAI,CACvB,QAAQ,WAAW,OAAO,CAC1B,MAAM;;;AAIX,SAAS,YAAY,KAAgD;AACnE,KAAI;EACF,MAAM,SAAS,IAAI,IAAI,IAAI;AAC3B,MAAI,OAAO,aAAa,WAAW,OAAO,aAAa,SACrD,QAAO;GACL,OAAO;GACP,OAAO,iCAAiC,OAAO,SAAS;GACzD;AAEH,MAAI,CAAC,OAAO,SACV,QAAO;GAAE,OAAO;GAAO,OAAO;GAAkB;AAElD,SAAO;GAAE,OAAO;GAAM,OAAO;GAAI;UAC1B,KAAK;AACZ,SAAO;GAAE,OAAO;GAAO,OAAO,OAAO,IAAI;GAAE;;;;AAK/C,SAAS,eAAe,MAAsB;CAC5C,IAAI,OAAO;AAEX,QAAO,KAAK,QACV,2DACC,IAAI,KAAK,UAAU,IAAI,UAAU,MAAM,CAAC,IAAI,IAAI,GAClD;AAED,QAAO,KAAK,QACV,uCACC,IAAI,OAAO,UAAU,KAAK,IAAI,OAAO,OAAO,MAAM,CAAC,CAAC,GAAG,UAAU,MAAM,CAAC,IAC1E;AAED,QAAO,KAAK,QACV,gCACC,IAAI,UAAU,OAAO,UAAU,MAAM,GACvC;AAED,QAAO,KAAK,QAAQ,iCAAiC,OAAO;AAC5D,QAAO,KAAK,QAAQ,qBAAqB,KAAK;AAC9C,QAAO,UAAU,UAAU,KAAK,CAAC;;;AAInC,IAAa,gBAAb,cAAmC,KAAK;CACtC,AAAS,OAAO;CAChB,AAAS,cAAc;CACvB,AAAS,aAAa;EACpB,MAAM;EACN,YAAY;GACV,OAAO;IAAE,MAAM;IAAU,aAAa;IAAgB;GACtD,OAAO;IACL,MAAM;IACN,aAAa;IACb,SAAS;IACT,SAAS;IACV;GACF;EACD,UAAU,CAAC,QAAQ;EACpB;CAED,AAAQ;CACR,AAAQ;CAER,YAAY,QAAmD;AAC7D,SAAO;AACP,OAAK,SAAS,QAAQ,UAAU,QAAQ,IAAI,iBAAiB;AAC7D,OAAK,aAAa,QAAQ,cAAc;;CAG1C,MAAM,QAAQ,MAAgD;EAC5D,MAAM,QAAQ,OAAO,KAAK,MAAM;EAChC,MAAM,QAAQ,KAAK,IACjB,KAAK,IAAI,KAAK,QAAQ,OAAO,KAAK,MAAM,GAAG,KAAK,YAAY,EAAE,EAC9D,GACD;AAED,MAAI,CAAC,KAAK,OACR,QAAO;AAGT,MAAI;GACF,MAAM,MAAM,IAAI,IAAI,iDAAiD;AACrE,OAAI,aAAa,IAAI,KAAK,MAAM;AAChC,OAAI,aAAa,IAAI,SAAS,OAAO,MAAM,CAAC;GAE5C,MAAM,OAAO,MAAM,MAAM,IAAI,UAAU,EAAE;IACvC,SAAS;KACP,QAAQ;KACR,wBAAwB,KAAK;KAC9B;IACD,QAAQ,YAAY,QAAQ,IAAM;IACnC,CAAC;AAEF,OAAI,CAAC,KAAK,GACR,QAAO,8BAA8B,KAAK;GAM5C,MAAM,WAHQ,MAAM,KAAK,MAAM,EAGV,KAAK,WAAW,EAAE;AAEvC,OAAI,QAAQ,WAAW,EACrB,QAAO,mBAAmB;GAG5B,MAAM,QAAQ,CAAC,gBAAgB,MAAM,IAAI;AACzC,QAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IAAI,QAAQ,QAAQ,MAAM,EAAE,KAAK;IACxD,MAAM,OAAO,QAAQ;AACrB,UAAM,KAAK,GAAG,IAAI,EAAE,IAAI,KAAK,SAAS,GAAG,OAAO,KAAK,OAAO,KAAK;AACjE,QAAI,KAAK,YACP,OAAM,KAAK,MAAM,KAAK,cAAc;;AAGxC,UAAO,MAAM,KAAK,KAAK;WAChB,KAAK;AACZ,UAAO,UAAU,eAAe,QAAQ,IAAI,UAAU;;;;;AAM5D,IAAa,eAAb,cAAkC,KAAK;CACrC,AAAS,OAAO;CAChB,AAAS,cACP;CACF,AAAS,aAAa;EACpB,MAAM;EACN,YAAY;GACV,KAAK;IAAE,MAAM;IAAU,aAAa;IAAgB;GACpD,aAAa;IACX,MAAM;IACN,MAAM,CAAC,YAAY,OAAO;IAC1B,aAAa;IACd;GACD,UAAU;IAAE,MAAM;IAAW,SAAS;IAAK;GAC5C;EACD,UAAU,CAAC,MAAM;EAClB;CAED,AAAQ;CAER,YAAY,QAAgC;AAC1C,SAAO;AACP,OAAK,kBAAkB,QAAQ,YAAY;;CAG7C,MAAM,QAAQ,MAAgD;EAC5D,MAAM,MAAM,OAAO,KAAK,IAAI;EAC5B,MAAM,cAAc,OAAO,KAAK,eAAe,WAAW;EAC1D,MAAM,WAAW,KAAK,WAClB,OAAO,KAAK,SAAS,GACrB,KAAK;EAET,MAAM,EAAE,OAAO,OAAO,oBAAoB,YAAY,IAAI;AAC1D,MAAI,CAAC,MACH,QAAO,KAAK,UAAU;GACpB,OAAO,0BAA0B;GACjC;GACD,CAAC;AAGJ,MAAI;GACF,MAAM,OAAO,MAAM,MAAM,KAAK;IAC5B,SAAS,EAAE,cAAc,YAAY;IACrC,UAAU;IACV,QAAQ,YAAY,QAAQ,IAAM;IACnC,CAAC;AAEF,OAAI,CAAC,KAAK,GACR,QAAO,KAAK,UAAU;IACpB,OAAO,QAAQ,KAAK;IACpB;IACD,CAAC;GAGJ,MAAM,cAAc,KAAK,QAAQ,IAAI,eAAe,IAAI;GACxD,MAAM,OAAO,MAAM,KAAK,MAAM;GAC9B,IAAI;GACJ,IAAI;AAEJ,OAAI,YAAY,SAAS,mBAAmB,EAAE;AAC5C,QAAI;AACF,YAAO,KAAK,UAAU,KAAK,MAAM,KAAK,EAAE,MAAM,EAAE;YAC1C;AACN,YAAO;;AAET,gBAAY;cAEZ,YAAY,SAAS,YAAY,IACjC,KAAK,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,YAAY,IACxD,KAAK,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,QAAQ,EACpD;AAEA,WACE,gBAAgB,aACZ,eAAe,KAAK,GACpB,UAAU,KAAK;AACrB,gBAAY;UACP;AACL,WAAO;AACP,gBAAY;;GAGd,MAAM,YAAY,KAAK,SAAS;AAChC,OAAI,UACF,QAAO,KAAK,MAAM,GAAG,SAAS;AAGhC,UAAO,KAAK,UAAU;IACpB;IACA,UAAU,KAAK;IACf,QAAQ,KAAK;IACb;IACA;IACA,QAAQ,KAAK;IACb;IACD,CAAC;WACK,KAAK;AACZ,UAAO,KAAK,UAAU;IACpB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;IACvD;IACD,CAAC"}
|
|
1
|
+
{"version":3,"file":"web.mjs","names":[],"sources":["../../../src/agent/tools/web.ts"],"sourcesContent":["import { Tool } from \"./base.js\";\n\nconst USER_AGENT =\n \"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36\";\n\n/** Strip HTML tags and decode entities. */\nfunction stripTags(text: string): string {\n let result = text;\n result = result.replace(/<script[\\s\\S]*?<\\/script>/gi, \"\");\n result = result.replace(/<style[\\s\\S]*?<\\/style>/gi, \"\");\n result = result.replace(/<[^>]+>/g, \"\");\n // Decode common HTML entities\n result = result\n .replace(/&/g, \"&\")\n .replace(/</g, \"<\")\n .replace(/>/g, \">\")\n .replace(/"/g, '\"')\n .replace(/'/g, \"'\")\n .replace(/ /g, \" \");\n return result.trim();\n}\n\n/** Normalize whitespace. */\nfunction normalize(text: string): string {\n return text\n .replace(/[ \\t]+/g, \" \")\n .replace(/\\n{3,}/g, \"\\n\\n\")\n .trim();\n}\n\n/** Validate URL. */\nfunction validateUrl(url: string): { valid: boolean; error: string } {\n try {\n const parsed = new URL(url);\n if (parsed.protocol !== \"http:\" && parsed.protocol !== \"https:\") {\n return {\n valid: false,\n error: `Only http/https allowed, got '${parsed.protocol}'`,\n };\n }\n if (!parsed.hostname) {\n return { valid: false, error: \"Missing domain\" };\n }\n return { valid: true, error: \"\" };\n } catch (err) {\n return { valid: false, error: String(err) };\n }\n}\n\n/** Convert HTML to basic markdown. */\nfunction htmlToMarkdown(html: string): string {\n let text = html;\n // Links\n text = text.replace(\n /<a\\s+[^>]*href=[\"']([^\"']+)[\"'][^>]*>([\\s\\S]*?)<\\/a>/gi,\n (_m, url, inner) => `[${stripTags(inner)}](${url})`,\n );\n // Headings\n text = text.replace(\n /<h([1-6])[^>]*>([\\s\\S]*?)<\\/h\\1>/gi,\n (_m, level, inner) => `\\n${\"#\".repeat(Number(level))} ${stripTags(inner)}\\n`,\n );\n // List items\n text = text.replace(\n /<li[^>]*>([\\s\\S]*?)<\\/li>/gi,\n (_m, inner) => `\\n- ${stripTags(inner)}`,\n );\n // Block elements\n text = text.replace(/<\\/(p|div|section|article)>/gi, \"\\n\\n\");\n text = text.replace(/<(br|hr)\\s*\\/?>/gi, \"\\n\");\n return normalize(stripTags(text));\n}\n\n/** Search the web using Brave Search API. */\nexport class WebSearchTool extends Tool {\n readonly name = \"web_search\";\n readonly description = \"Search the web. Returns titles, URLs, and snippets.\";\n readonly parameters = {\n type: \"object\",\n properties: {\n query: { type: \"string\", description: \"Search query\" },\n count: {\n type: \"integer\",\n description: \"Results (1-10)\",\n minimum: 1,\n maximum: 10,\n },\n },\n required: [\"query\"],\n };\n\n private apiKey: string;\n private maxResults: number;\n\n constructor(params?: { apiKey?: string; maxResults?: number }) {\n super();\n this.apiKey = params?.apiKey ?? process.env.BRAVE_API_KEY ?? \"\";\n this.maxResults = params?.maxResults ?? 5;\n }\n\n async execute(args: Record<string, unknown>): Promise<string> {\n const query = String(args.query);\n const count = Math.min(\n Math.max(args.count ? Number(args.count) : this.maxResults, 1),\n 10,\n );\n\n if (!this.apiKey) {\n return \"Error: BRAVE_API_KEY not configured\";\n }\n\n try {\n const url = new URL(\"https://api.search.brave.com/res/v1/web/search\");\n url.searchParams.set(\"q\", query);\n url.searchParams.set(\"count\", String(count));\n\n const resp = await fetch(url.toString(), {\n headers: {\n Accept: \"application/json\",\n \"Accept-Encoding\": \"gzip\",\n \"X-Subscription-Token\": this.apiKey,\n },\n signal: AbortSignal.timeout(10000),\n });\n\n if (!resp.ok) {\n return `Error: Search API returned ${resp.status}`;\n }\n\n const data = (await resp.json()) as {\n web?: { results?: Array<{ title?: string; url?: string; description?: string }> };\n };\n const results = data.web?.results ?? [];\n\n if (results.length === 0) {\n return `No results for: ${query}`;\n }\n\n const lines = [`Results for: ${query}\\n`];\n for (let i = 0; i < Math.min(results.length, count); i++) {\n const item = results[i];\n lines.push(`${i + 1}. ${item.title ?? \"\"}\\n ${item.url ?? \"\"}`);\n if (item.description) {\n lines.push(` ${item.description}`);\n }\n }\n return lines.join(\"\\n\");\n } catch (err) {\n return `Error: ${err instanceof Error ? err.message : err}`;\n }\n }\n}\n\n/** Fetch and extract content from a URL. */\nexport class WebFetchTool extends Tool {\n readonly name = \"web_fetch\";\n readonly description =\n \"Fetch URL and extract readable content (HTML -> markdown/text).\";\n readonly parameters = {\n type: \"object\",\n properties: {\n url: { type: \"string\", description: \"URL to fetch\" },\n extractMode: {\n type: \"string\",\n enum: [\"markdown\", \"text\"],\n description: \"Extract mode\",\n },\n maxChars: { type: \"integer\", minimum: 100 },\n },\n required: [\"url\"],\n };\n\n private defaultMaxChars: number;\n\n constructor(params?: { maxChars?: number }) {\n super();\n this.defaultMaxChars = params?.maxChars ?? 50000;\n }\n\n async execute(args: Record<string, unknown>): Promise<string> {\n const url = String(args.url);\n const extractMode = String(args.extractMode ?? \"markdown\");\n const maxChars = args.maxChars\n ? Number(args.maxChars)\n : this.defaultMaxChars;\n\n const { valid, error: validationError } = validateUrl(url);\n if (!valid) {\n return JSON.stringify({\n error: `URL validation failed: ${validationError}`,\n url,\n });\n }\n\n try {\n const resp = await fetch(url, {\n headers: { \"User-Agent\": USER_AGENT },\n redirect: \"follow\",\n signal: AbortSignal.timeout(30000),\n });\n\n if (!resp.ok) {\n return JSON.stringify({\n error: `HTTP ${resp.status}`,\n url,\n });\n }\n\n const contentType = resp.headers.get(\"content-type\") ?? \"\";\n const body = await resp.text();\n let text: string;\n let extractor: string;\n\n if (contentType.includes(\"application/json\")) {\n try {\n text = JSON.stringify(JSON.parse(body), null, 2);\n } catch {\n text = body;\n }\n extractor = \"json\";\n } else if (\n contentType.includes(\"text/html\") ||\n body.slice(0, 256).toLowerCase().startsWith(\"<!doctype\") ||\n body.slice(0, 256).toLowerCase().startsWith(\"<html\")\n ) {\n // Extract readable content from HTML\n text =\n extractMode === \"markdown\"\n ? htmlToMarkdown(body)\n : stripTags(body);\n extractor = \"html\";\n } else {\n text = body;\n extractor = \"raw\";\n }\n\n const truncated = text.length > maxChars;\n if (truncated) {\n text = text.slice(0, maxChars);\n }\n\n return JSON.stringify({\n url,\n finalUrl: resp.url,\n status: resp.status,\n extractor,\n truncated,\n length: text.length,\n text,\n });\n } catch (err) {\n return JSON.stringify({\n error: err instanceof Error ? err.message : String(err),\n url,\n });\n }\n }\n}\n"],"mappings":";;;AAEA,MAAM,aACJ;;AAGF,SAAS,UAAU,MAAsB;CACvC,IAAI,SAAS;AACb,UAAS,OAAO,QAAQ,+BAA+B,GAAG;AAC1D,UAAS,OAAO,QAAQ,6BAA6B,GAAG;AACxD,UAAS,OAAO,QAAQ,YAAY,GAAG;AAEvC,UAAS,OACN,QAAQ,UAAU,IAAI,CACtB,QAAQ,SAAS,IAAI,CACrB,QAAQ,SAAS,IAAI,CACrB,QAAQ,WAAW,KAAI,CACvB,QAAQ,UAAU,IAAI,CACtB,QAAQ,WAAW,IAAI;AAC1B,QAAO,OAAO,MAAM;;;AAItB,SAAS,UAAU,MAAsB;AACvC,QAAO,KACJ,QAAQ,WAAW,IAAI,CACvB,QAAQ,WAAW,OAAO,CAC1B,MAAM;;;AAIX,SAAS,YAAY,KAAgD;AACnE,KAAI;EACF,MAAM,SAAS,IAAI,IAAI,IAAI;AAC3B,MAAI,OAAO,aAAa,WAAW,OAAO,aAAa,SACrD,QAAO;GACL,OAAO;GACP,OAAO,iCAAiC,OAAO,SAAS;GACzD;AAEH,MAAI,CAAC,OAAO,SACV,QAAO;GAAE,OAAO;GAAO,OAAO;GAAkB;AAElD,SAAO;GAAE,OAAO;GAAM,OAAO;GAAI;UAC1B,KAAK;AACZ,SAAO;GAAE,OAAO;GAAO,OAAO,OAAO,IAAI;GAAE;;;;AAK/C,SAAS,eAAe,MAAsB;CAC5C,IAAI,OAAO;AAEX,QAAO,KAAK,QACV,2DACC,IAAI,KAAK,UAAU,IAAI,UAAU,MAAM,CAAC,IAAI,IAAI,GAClD;AAED,QAAO,KAAK,QACV,uCACC,IAAI,OAAO,UAAU,KAAK,IAAI,OAAO,OAAO,MAAM,CAAC,CAAC,GAAG,UAAU,MAAM,CAAC,IAC1E;AAED,QAAO,KAAK,QACV,gCACC,IAAI,UAAU,OAAO,UAAU,MAAM,GACvC;AAED,QAAO,KAAK,QAAQ,iCAAiC,OAAO;AAC5D,QAAO,KAAK,QAAQ,qBAAqB,KAAK;AAC9C,QAAO,UAAU,UAAU,KAAK,CAAC;;;AAInC,IAAa,gBAAb,cAAmC,KAAK;CACtC,AAAS,OAAO;CAChB,AAAS,cAAc;CACvB,AAAS,aAAa;EACpB,MAAM;EACN,YAAY;GACV,OAAO;IAAE,MAAM;IAAU,aAAa;IAAgB;GACtD,OAAO;IACL,MAAM;IACN,aAAa;IACb,SAAS;IACT,SAAS;IACV;GACF;EACD,UAAU,CAAC,QAAQ;EACpB;CAED,AAAQ;CACR,AAAQ;CAER,YAAY,QAAmD;AAC7D,SAAO;AACP,OAAK,SAAS,QAAQ,UAAU,QAAQ,IAAI,iBAAiB;AAC7D,OAAK,aAAa,QAAQ,cAAc;;CAG1C,MAAM,QAAQ,MAAgD;EAC5D,MAAM,QAAQ,OAAO,KAAK,MAAM;EAChC,MAAM,QAAQ,KAAK,IACjB,KAAK,IAAI,KAAK,QAAQ,OAAO,KAAK,MAAM,GAAG,KAAK,YAAY,EAAE,EAC9D,GACD;AAED,MAAI,CAAC,KAAK,OACR,QAAO;AAGT,MAAI;GACF,MAAM,MAAM,IAAI,IAAI,iDAAiD;AACrE,OAAI,aAAa,IAAI,KAAK,MAAM;AAChC,OAAI,aAAa,IAAI,SAAS,OAAO,MAAM,CAAC;GAE5C,MAAM,OAAO,MAAM,MAAM,IAAI,UAAU,EAAE;IACvC,SAAS;KACP,QAAQ;KACR,mBAAmB;KACnB,wBAAwB,KAAK;KAC9B;IACD,QAAQ,YAAY,QAAQ,IAAM;IACnC,CAAC;AAEF,OAAI,CAAC,KAAK,GACR,QAAO,8BAA8B,KAAK;GAM5C,MAAM,WAHQ,MAAM,KAAK,MAAM,EAGV,KAAK,WAAW,EAAE;AAEvC,OAAI,QAAQ,WAAW,EACrB,QAAO,mBAAmB;GAG5B,MAAM,QAAQ,CAAC,gBAAgB,MAAM,IAAI;AACzC,QAAK,IAAI,IAAI,GAAG,IAAI,KAAK,IAAI,QAAQ,QAAQ,MAAM,EAAE,KAAK;IACxD,MAAM,OAAO,QAAQ;AACrB,UAAM,KAAK,GAAG,IAAI,EAAE,IAAI,KAAK,SAAS,GAAG,OAAO,KAAK,OAAO,KAAK;AACjE,QAAI,KAAK,YACP,OAAM,KAAK,MAAM,KAAK,cAAc;;AAGxC,UAAO,MAAM,KAAK,KAAK;WAChB,KAAK;AACZ,UAAO,UAAU,eAAe,QAAQ,IAAI,UAAU;;;;;AAM5D,IAAa,eAAb,cAAkC,KAAK;CACrC,AAAS,OAAO;CAChB,AAAS,cACP;CACF,AAAS,aAAa;EACpB,MAAM;EACN,YAAY;GACV,KAAK;IAAE,MAAM;IAAU,aAAa;IAAgB;GACpD,aAAa;IACX,MAAM;IACN,MAAM,CAAC,YAAY,OAAO;IAC1B,aAAa;IACd;GACD,UAAU;IAAE,MAAM;IAAW,SAAS;IAAK;GAC5C;EACD,UAAU,CAAC,MAAM;EAClB;CAED,AAAQ;CAER,YAAY,QAAgC;AAC1C,SAAO;AACP,OAAK,kBAAkB,QAAQ,YAAY;;CAG7C,MAAM,QAAQ,MAAgD;EAC5D,MAAM,MAAM,OAAO,KAAK,IAAI;EAC5B,MAAM,cAAc,OAAO,KAAK,eAAe,WAAW;EAC1D,MAAM,WAAW,KAAK,WAClB,OAAO,KAAK,SAAS,GACrB,KAAK;EAET,MAAM,EAAE,OAAO,OAAO,oBAAoB,YAAY,IAAI;AAC1D,MAAI,CAAC,MACH,QAAO,KAAK,UAAU;GACpB,OAAO,0BAA0B;GACjC;GACD,CAAC;AAGJ,MAAI;GACF,MAAM,OAAO,MAAM,MAAM,KAAK;IAC5B,SAAS,EAAE,cAAc,YAAY;IACrC,UAAU;IACV,QAAQ,YAAY,QAAQ,IAAM;IACnC,CAAC;AAEF,OAAI,CAAC,KAAK,GACR,QAAO,KAAK,UAAU;IACpB,OAAO,QAAQ,KAAK;IACpB;IACD,CAAC;GAGJ,MAAM,cAAc,KAAK,QAAQ,IAAI,eAAe,IAAI;GACxD,MAAM,OAAO,MAAM,KAAK,MAAM;GAC9B,IAAI;GACJ,IAAI;AAEJ,OAAI,YAAY,SAAS,mBAAmB,EAAE;AAC5C,QAAI;AACF,YAAO,KAAK,UAAU,KAAK,MAAM,KAAK,EAAE,MAAM,EAAE;YAC1C;AACN,YAAO;;AAET,gBAAY;cAEZ,YAAY,SAAS,YAAY,IACjC,KAAK,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,YAAY,IACxD,KAAK,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,QAAQ,EACpD;AAEA,WACE,gBAAgB,aACZ,eAAe,KAAK,GACpB,UAAU,KAAK;AACrB,gBAAY;UACP;AACL,WAAO;AACP,gBAAY;;GAGd,MAAM,YAAY,KAAK,SAAS;AAChC,OAAI,UACF,QAAO,KAAK,MAAM,GAAG,SAAS;AAGhC,UAAO,KAAK,UAAU;IACpB;IACA,UAAU,KAAK;IACf,QAAQ,KAAK;IACb;IACA;IACA,QAAQ,KAAK;IACb;IACD,CAAC;WACK,KAAK;AACZ,UAAO,KAAK,UAAU;IACpB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,IAAI;IACvD;IACD,CAAC"}
|