@fettstorch/clai 0.1.7 → 0.1.9

package/dist/cli.js

@@ -28997,22 +28997,22 @@ var require_diagnostics = __commonJS((exports, module) => {
     const debuglog = fetchDebuglog.enabled ? fetchDebuglog : undiciDebugLog;
     diagnosticsChannel.channel("undici:client:beforeConnect").subscribe((evt) => {
       const {
-        connectParams: { version, protocol, port, host }
+        connectParams: { version: version2, protocol, port, host }
       } = evt;
-      debuglog("connecting to %s using %s%s", `${host}${port ? `:${port}` : ""}`, protocol, version);
+      debuglog("connecting to %s using %s%s", `${host}${port ? `:${port}` : ""}`, protocol, version2);
     });
     diagnosticsChannel.channel("undici:client:connected").subscribe((evt) => {
       const {
-        connectParams: { version, protocol, port, host }
+        connectParams: { version: version2, protocol, port, host }
       } = evt;
-      debuglog("connected to %s using %s%s", `${host}${port ? `:${port}` : ""}`, protocol, version);
+      debuglog("connected to %s using %s%s", `${host}${port ? `:${port}` : ""}`, protocol, version2);
     });
     diagnosticsChannel.channel("undici:client:connectError").subscribe((evt) => {
       const {
-        connectParams: { version, protocol, port, host },
+        connectParams: { version: version2, protocol, port, host },
         error
       } = evt;
-      debuglog("connection to %s using %s%s errored - %s", `${host}${port ? `:${port}` : ""}`, protocol, version, error.message);
+      debuglog("connection to %s using %s%s errored - %s", `${host}${port ? `:${port}` : ""}`, protocol, version2, error.message);
     });
     diagnosticsChannel.channel("undici:client:sendHeaders").subscribe((evt) => {
       const {
@@ -29047,22 +29047,22 @@ var require_diagnostics = __commonJS((exports, module) => {
       const debuglog = undiciDebugLog.enabled ? undiciDebugLog : websocketDebuglog;
       diagnosticsChannel.channel("undici:client:beforeConnect").subscribe((evt) => {
         const {
-          connectParams: { version, protocol, port, host }
+          connectParams: { version: version2, protocol, port, host }
         } = evt;
-        debuglog("connecting to %s%s using %s%s", host, port ? `:${port}` : "", protocol, version);
+        debuglog("connecting to %s%s using %s%s", host, port ? `:${port}` : "", protocol, version2);
       });
       diagnosticsChannel.channel("undici:client:connected").subscribe((evt) => {
         const {
-          connectParams: { version, protocol, port, host }
+          connectParams: { version: version2, protocol, port, host }
         } = evt;
-        debuglog("connected to %s%s using %s%s", host, port ? `:${port}` : "", protocol, version);
+        debuglog("connected to %s%s using %s%s", host, port ? `:${port}` : "", protocol, version2);
       });
       diagnosticsChannel.channel("undici:client:connectError").subscribe((evt) => {
         const {
-          connectParams: { version, protocol, port, host },
+          connectParams: { version: version2, protocol, port, host },
           error
         } = evt;
-        debuglog("connection to %s%s using %s%s errored - %s", host, port ? `:${port}` : "", protocol, version, error.message);
+        debuglog("connection to %s%s using %s%s errored - %s", host, port ? `:${port}` : "", protocol, version2, error.message);
       });
       diagnosticsChannel.channel("undici:client:sendHeaders").subscribe((evt) => {
         const {
@@ -54255,9 +54255,10 @@ function ora(options) {
   return new Ora(options);
 }
 // package.json
+var version = "0.1.9";
 var package_default = {
   name: "@fettstorch/clai",
-  version: "0.1.7",
+  version,
   main: "dist/index.js",
   bin: {
     clai: "dist/cli.js"
@@ -68031,8 +68032,7 @@ async function scrape(input) {
     }));
     return results.filter((result) => result !== null);
   } catch (error) {
-    console.error("Error during scraping:", error);
-    throw error;
+    return [];
   }
 }
 function isValidUrl(input) {
@@ -68048,55 +68048,31 @@ function normalizeUrl(url) {
   return url;
 }
 async function getSearchResults(query) {
-  try {
-    return await getSearXResults(query);
-  } catch (_2) {
-    console.log("Trying Google search...");
+  const searchEngines = [
+    { name: "SearX", fn: getSearXResults },
+    { name: "Google", fn: getGoogleResults },
+    { name: "DuckDuckGo", fn: getDuckDuckGoResults },
+    { name: "Wikipedia", fn: getWikipediaResults }
+  ];
+  for (const engine of searchEngines) {
     try {
-      return await getGoogleResults(query);
-    } catch (_3) {
-      console.log("Trying DuckDuckGo search...");
-      try {
-        return await getDuckDuckGoResults(query);
-      } catch (_4) {
-        console.log("Using emergency fallback...");
-        return getEmergencyResults(query);
-      }
-    }
-  }
-}
-function getEmergencyResults(query) {
-  const results = [];
-  const cleanQuery = query.toLowerCase().replace(/[^a-z0-9\s]/g, "").trim();
-  const words = cleanQuery.split(/\s+/).filter((word) => word.length > 2);
-  if (words.length > 0) {
-    const mainWord = words[0];
-    results.push(`https://en.wikipedia.org/wiki/${encodeURIComponent(query.replace(/\s+/g, "_"))}`);
-    if (mainWord.length > 3) {
-      results.push(`https://${mainWord}.com`);
-      results.push(`https://www.${mainWord}.org`);
+      const result = await engine.fn(query);
+      console.log(`[${engine.name}]::✅`);
+      return result;
+    } catch (_2) {
+      console.log(`[${engine.name}]::❌`);
     }
-    results.push(`https://www.reddit.com/search/?q=${encodeURIComponent(query)}`);
   }
-  console.log("Emergency fallback returning:", results.join(", "));
-  return results.length > 0 ? results.slice(0, 3) : [
-    `https://en.wikipedia.org/wiki/${encodeURIComponent(query.replace(/\s+/g, "_"))}`
-  ];
+  console.log("All search engines failed - no URLs to scrape");
+  throw new Error("No search results available");
 }
 async function getSearXResults(query) {
-  const searxInstances = [
-    "https://searx.be",
-    "https://search.sapti.me",
-    "https://searx.tiekoetter.com",
-    "https://searx.prvcy.eu"
-  ];
+  const searxInstances = ["https://searx.be", "https://search.sapti.me"];
   for (const instance of searxInstances) {
     try {
       const searchUrl = `${instance}/search?q=${encodeURIComponent(query)}&format=json&categories=general`;
-      console.log("Trying SearX search...");
-      const response = await fetch(searchUrl, {
+      const response = await enhancedFetch(searchUrl, {
         headers: {
-          "User-Agent": getRandomUserAgent(),
           Accept: "application/json"
         }
       });
@@ -68113,7 +68089,6 @@ async function getSearXResults(query) {
         }
       }
       if (urls.length > 0) {
-        console.log(`✓ SearX found ${urls.length} results`);
         return urls.slice(0, 3);
       }
     } catch (error) {
@@ -68121,9 +68096,31 @@ async function getSearXResults(query) {
   }
   throw new Error("All SearX instances failed");
 }
+async function getWikipediaResults(query) {
+  const searchUrl = `https://en.wikipedia.org/w/api.php?action=opensearch&search=${encodeURIComponent(query)}&limit=3&format=json&origin=*`;
+  const response = await enhancedFetch(searchUrl, {
+    headers: {
+      Accept: "application/json"
+    }
+  });
+  if (!response.ok) {
+    throw new Error(`Wikipedia API error: ${response.status}`);
+  }
+  const data2 = await response.json();
+  if (Array.isArray(data2) && data2.length >= 4 && Array.isArray(data2[3])) {
+    const urls = data2[3]?.filter((url) => url?.startsWith("https://"));
+    if (urls?.length > 0) {
+      return urls;
+    }
+  }
+  throw new Error("No Wikipedia results found");
+}
 async function getGoogleResults(query) {
   const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&num=10`;
   const html3 = await fetchHtml(searchUrl);
+  if (html3.includes("If you're having trouble accessing Google Search") || html3.includes("unusual traffic from your computer network")) {
+    throw new Error("Google blocked request - detected as bot");
+  }
   const cheerioDoc = load2(html3);
   const urls = [];
   cheerioDoc('a[href^="/url?q="]').each((_2, element) => {
@@ -68151,13 +68148,18 @@ async function getGoogleResults(query) {
   if (uniqueUrls.length === 0) {
     throw new Error("No search results found in Google response");
   }
-  console.log(`✓ Google found ${uniqueUrls.length} results`);
   return uniqueUrls;
 }
 async function getDuckDuckGoResults(query) {
   const searchUrl = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1&skip_disambig=1`;
-  const response = await fetch(searchUrl);
+  const response = await enhancedFetch(searchUrl);
+  if (!response.ok) {
+    throw new Error(`DuckDuckGo API error: ${response.status}`);
+  }
   const data2 = await response.json();
+  if (data2.Abstract?.includes("redirects users to a non-JavaScript site") || data2.Abstract?.includes("DuckDuckGo redirects users") || data2.AbstractText?.includes("redirects users to a non-JavaScript site") || data2.AbstractText?.includes("DuckDuckGo redirects users")) {
+    throw new Error("DuckDuckGo blocked request - JavaScript disabled redirect");
+  }
   const urls = [];
   if (data2.AbstractURL) {
     urls.push(data2.AbstractURL);
@@ -68169,29 +68171,39 @@ async function getDuckDuckGoResults(query) {
       }
     }
   }
+  if (urls.length === 0 && data2.DefinitionURL) {
+    urls.push(data2.DefinitionURL);
+  }
   if (urls.length === 0) {
     throw new Error("No search results found in DuckDuckGo response");
   }
-  console.log(`✓ DuckDuckGo found ${urls.length} results`);
   return urls;
 }
-async function fetchHtml(url) {
-  const response = await fetch(url, {
-    headers: {
-      "User-Agent": getRandomUserAgent(),
-      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
-      "Accept-Language": "en-US,en;q=0.9",
-      "Accept-Encoding": "gzip, deflate, br",
-      DNT: "1",
-      Connection: "keep-alive",
-      "Upgrade-Insecure-Requests": "1",
-      "Sec-Fetch-Dest": "document",
-      "Sec-Fetch-Mode": "navigate",
-      "Sec-Fetch-Site": "none",
-      "Sec-Fetch-User": "?1",
-      "Cache-Control": "max-age=0"
-    }
+async function enhancedFetch(url, options = {}) {
+  const headers = {
+    "User-Agent": getRandomUserAgent(),
+    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "sec-ch-ua": '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"macOS"',
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "cross-site",
+    "Sec-Fetch-User": "?1",
+    "Upgrade-Insecure-Requests": "1",
+    "Cache-Control": "no-cache",
+    Pragma: "no-cache",
+    ...options.headers
+  };
+  return fetch(url, {
+    ...options,
+    headers
   });
+}
+async function fetchHtml(url) {
+  const response = await enhancedFetch(url);
   return response.text();
 }
 function getRandomUserAgent() {
@@ -73255,25 +73267,83 @@ async function summarizeWebPage(content, openAIApiKey) {
   });
   return result;
 }
+async function summarizeQuery(query, openAIApiKey) {
+  const openai = openaiClient(openAIApiKey);
+  const prompt2 = `You are an expert educator and researcher. Answer the following query with accurate, helpful information:
+
+"${query}"
+
+Guidelines:
+1. Provide a comprehensive but concise answer
+2. Use bullet points, lists, and tables when appropriate
+3. Include relevant examples or step-by-step instructions if applicable
+4. Format your response in valid markdown
+5. Be factual and cite general knowledge sources when relevant
+6. If you suggest external resources, format them as links in the response
+7. Mark proper nouns as bold e.g. **OpenAI**
+8. Use appropriate headings (##, ###) to structure your response
+9. If the query is about current events beyond your knowledge cutoff, mention that limitation
+
+Provide a thorough, educational response that directly addresses the user's query.`;
+  const schema = {
+    textual: {
+      type: "string",
+      description: "Comprehensive answer to the user query"
+    },
+    links: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          name: {
+            type: "string",
+            description: "Descriptive name of the recommended resource"
+          },
+          url: {
+            type: "string",
+            description: "URL to the recommended resource"
+          }
+        },
+        required: ["name", "url"]
+      }
+    }
+  };
+  const result = await openai.completeStructured(prompt2, {
+    temperature: 0.7,
+    responseSchema: schema
+  });
+  return result;
+}
 
 // src/index.ts
 async function clai(input, openAIKey) {
   const scrapedData = await scrape(input);
-  const combinedContent = scrapedData.map((data2) => `Content from ${data2.url}:
+  const usefulData = scrapedData.filter((data2) => data2.content.length > 200 && !data2.content.includes("Wikipedia does not have an article") && !data2.content.includes("page not found") && !data2.content.includes("404") && !data2.content.includes("error"));
+  if (usefulData.length > 0) {
+    const combinedContent = usefulData.map((data2) => `Content from ${data2.url}:
 ${data2.content}`).join(`
 
 `);
-  const result = await summarizeWebPage(combinedContent, openAIKey);
+    const result2 = await summarizeWebPage(combinedContent, openAIKey);
+    return {
+      summary: result2.textual.trim(),
+      links: result2.links,
+      sources: usefulData.map((data2) => data2.url)
+    };
+  }
+  console.log("No scraped data available - using OpenAI directly for query...");
+  const result = await summarizeQuery(input, openAIKey);
   return {
     summary: result.textual.trim(),
     links: result.links,
-    sources: scrapedData.map((data2) => data2.url)
+    sources: ["OpenAI Knowledge Base"]
   };
 }
 
 // src/cli.ts
 var program2 = new Command;
 async function main2() {
+  console.log(`[clAi]::${source_default.cyan(version)}`);
   try {
     program2.name("clai").description("AI-powered web scraping tool").version(package_default.version).argument("[input...]", "URL or search terms to analyze").action(async (inputs) => {
       const openAIKey = process.env.OPENAI_API_KEY;

package/dist/index.js

@@ -41059,8 +41059,7 @@ async function scrape(input) {
     }));
     return results.filter((result) => result !== null);
   } catch (error) {
-    console.error("Error during scraping:", error);
-    throw error;
+    return [];
   }
 }
 function isValidUrl(input) {
@@ -41076,55 +41075,31 @@ function normalizeUrl(url) {
   return url;
 }
 async function getSearchResults(query) {
-  try {
-    return await getSearXResults(query);
-  } catch (_) {
-    console.log("Trying Google search...");
+  const searchEngines = [
+    { name: "SearX", fn: getSearXResults },
+    { name: "Google", fn: getGoogleResults },
+    { name: "DuckDuckGo", fn: getDuckDuckGoResults },
+    { name: "Wikipedia", fn: getWikipediaResults }
+  ];
+  for (const engine of searchEngines) {
     try {
-      return await getGoogleResults(query);
-    } catch (_2) {
-      console.log("Trying DuckDuckGo search...");
-      try {
-        return await getDuckDuckGoResults(query);
-      } catch (_3) {
-        console.log("Using emergency fallback...");
-        return getEmergencyResults(query);
-      }
-    }
-  }
-}
-function getEmergencyResults(query) {
-  const results = [];
-  const cleanQuery = query.toLowerCase().replace(/[^a-z0-9\s]/g, "").trim();
-  const words = cleanQuery.split(/\s+/).filter((word) => word.length > 2);
-  if (words.length > 0) {
-    const mainWord = words[0];
-    results.push(`https://en.wikipedia.org/wiki/${encodeURIComponent(query.replace(/\s+/g, "_"))}`);
-    if (mainWord.length > 3) {
-      results.push(`https://${mainWord}.com`);
-      results.push(`https://www.${mainWord}.org`);
+      const result = await engine.fn(query);
+      console.log(`[${engine.name}]::✅`);
+      return result;
+    } catch (_) {
+      console.log(`[${engine.name}]::❌`);
     }
-    results.push(`https://www.reddit.com/search/?q=${encodeURIComponent(query)}`);
   }
-  console.log("Emergency fallback returning:", results.join(", "));
-  return results.length > 0 ? results.slice(0, 3) : [
-    `https://en.wikipedia.org/wiki/${encodeURIComponent(query.replace(/\s+/g, "_"))}`
-  ];
+  console.log("All search engines failed - no URLs to scrape");
+  throw new Error("No search results available");
 }
 async function getSearXResults(query) {
-  const searxInstances = [
-    "https://searx.be",
-    "https://search.sapti.me",
-    "https://searx.tiekoetter.com",
-    "https://searx.prvcy.eu"
-  ];
+  const searxInstances = ["https://searx.be", "https://search.sapti.me"];
   for (const instance of searxInstances) {
     try {
       const searchUrl = `${instance}/search?q=${encodeURIComponent(query)}&format=json&categories=general`;
-      console.log("Trying SearX search...");
-      const response = await fetch(searchUrl, {
+      const response = await enhancedFetch(searchUrl, {
         headers: {
-          "User-Agent": getRandomUserAgent(),
           Accept: "application/json"
         }
       });
@@ -41141,7 +41116,6 @@ async function getSearXResults(query) {
         }
       }
       if (urls.length > 0) {
-        console.log(`✓ SearX found ${urls.length} results`);
         return urls.slice(0, 3);
       }
     } catch (error) {
@@ -41149,9 +41123,31 @@ async function getSearXResults(query) {
   }
   throw new Error("All SearX instances failed");
 }
+async function getWikipediaResults(query) {
+  const searchUrl = `https://en.wikipedia.org/w/api.php?action=opensearch&search=${encodeURIComponent(query)}&limit=3&format=json&origin=*`;
+  const response = await enhancedFetch(searchUrl, {
+    headers: {
+      Accept: "application/json"
+    }
+  });
+  if (!response.ok) {
+    throw new Error(`Wikipedia API error: ${response.status}`);
+  }
+  const data2 = await response.json();
+  if (Array.isArray(data2) && data2.length >= 4 && Array.isArray(data2[3])) {
+    const urls = data2[3]?.filter((url) => url?.startsWith("https://"));
+    if (urls?.length > 0) {
+      return urls;
+    }
+  }
+  throw new Error("No Wikipedia results found");
+}
 async function getGoogleResults(query) {
   const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&num=10`;
   const html3 = await fetchHtml(searchUrl);
+  if (html3.includes("If you're having trouble accessing Google Search") || html3.includes("unusual traffic from your computer network")) {
+    throw new Error("Google blocked request - detected as bot");
+  }
   const cheerioDoc = load(html3);
   const urls = [];
   cheerioDoc('a[href^="/url?q="]').each((_, element) => {
@@ -41179,13 +41175,18 @@ async function getGoogleResults(query) {
   if (uniqueUrls.length === 0) {
     throw new Error("No search results found in Google response");
   }
-  console.log(`✓ Google found ${uniqueUrls.length} results`);
   return uniqueUrls;
 }
 async function getDuckDuckGoResults(query) {
   const searchUrl = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1&skip_disambig=1`;
-  const response = await fetch(searchUrl);
+  const response = await enhancedFetch(searchUrl);
+  if (!response.ok) {
+    throw new Error(`DuckDuckGo API error: ${response.status}`);
+  }
   const data2 = await response.json();
+  if (data2.Abstract?.includes("redirects users to a non-JavaScript site") || data2.Abstract?.includes("DuckDuckGo redirects users") || data2.AbstractText?.includes("redirects users to a non-JavaScript site") || data2.AbstractText?.includes("DuckDuckGo redirects users")) {
+    throw new Error("DuckDuckGo blocked request - JavaScript disabled redirect");
+  }
   const urls = [];
   if (data2.AbstractURL) {
     urls.push(data2.AbstractURL);
@@ -41197,29 +41198,39 @@ async function getDuckDuckGoResults(query) {
       }
     }
   }
+  if (urls.length === 0 && data2.DefinitionURL) {
+    urls.push(data2.DefinitionURL);
+  }
   if (urls.length === 0) {
     throw new Error("No search results found in DuckDuckGo response");
   }
-  console.log(`✓ DuckDuckGo found ${urls.length} results`);
   return urls;
 }
-async function fetchHtml(url) {
-  const response = await fetch(url, {
-    headers: {
-      "User-Agent": getRandomUserAgent(),
-      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
-      "Accept-Language": "en-US,en;q=0.9",
-      "Accept-Encoding": "gzip, deflate, br",
-      DNT: "1",
-      Connection: "keep-alive",
-      "Upgrade-Insecure-Requests": "1",
-      "Sec-Fetch-Dest": "document",
-      "Sec-Fetch-Mode": "navigate",
-      "Sec-Fetch-Site": "none",
-      "Sec-Fetch-User": "?1",
-      "Cache-Control": "max-age=0"
-    }
+async function enhancedFetch(url, options = {}) {
+  const headers = {
+    "User-Agent": getRandomUserAgent(),
+    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "sec-ch-ua": '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"macOS"',
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "cross-site",
+    "Sec-Fetch-User": "?1",
+    "Upgrade-Insecure-Requests": "1",
+    "Cache-Control": "no-cache",
+    Pragma: "no-cache",
+    ...options.headers
+  };
+  return fetch(url, {
+    ...options,
+    headers
   });
+}
+async function fetchHtml(url) {
+  const response = await enhancedFetch(url);
   return response.text();
 }
 function getRandomUserAgent() {
@@ -46353,19 +46364,76 @@ async function summarizeWebPage(content, openAIApiKey) {
   });
   return result;
 }
+async function summarizeQuery(query, openAIApiKey) {
+  const openai = openaiClient(openAIApiKey);
+  const prompt = `You are an expert educator and researcher. Answer the following query with accurate, helpful information:
+
+"${query}"
+
+Guidelines:
+1. Provide a comprehensive but concise answer
+2. Use bullet points, lists, and tables when appropriate
+3. Include relevant examples or step-by-step instructions if applicable
+4. Format your response in valid markdown
+5. Be factual and cite general knowledge sources when relevant
+6. If you suggest external resources, format them as links in the response
+7. Mark proper nouns as bold e.g. **OpenAI**
+8. Use appropriate headings (##, ###) to structure your response
+9. If the query is about current events beyond your knowledge cutoff, mention that limitation
+
+Provide a thorough, educational response that directly addresses the user's query.`;
+  const schema = {
+    textual: {
+      type: "string",
+      description: "Comprehensive answer to the user query"
+    },
+    links: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          name: {
+            type: "string",
+            description: "Descriptive name of the recommended resource"
+          },
+          url: {
+            type: "string",
+            description: "URL to the recommended resource"
+          }
+        },
+        required: ["name", "url"]
+      }
+    }
+  };
+  const result = await openai.completeStructured(prompt, {
+    temperature: 0.7,
+    responseSchema: schema
+  });
+  return result;
+}
 
 // src/index.ts
 async function clai(input, openAIKey) {
   const scrapedData = await scrape(input);
-  const combinedContent = scrapedData.map((data2) => `Content from ${data2.url}:
+  const usefulData = scrapedData.filter((data2) => data2.content.length > 200 && !data2.content.includes("Wikipedia does not have an article") && !data2.content.includes("page not found") && !data2.content.includes("404") && !data2.content.includes("error"));
+  if (usefulData.length > 0) {
+    const combinedContent = usefulData.map((data2) => `Content from ${data2.url}:
 ${data2.content}`).join(`
 
 `);
-  const result = await summarizeWebPage(combinedContent, openAIKey);
+    const result2 = await summarizeWebPage(combinedContent, openAIKey);
+    return {
+      summary: result2.textual.trim(),
+      links: result2.links,
+      sources: usefulData.map((data2) => data2.url)
+    };
+  }
+  console.log("No scraped data available - using OpenAI directly for query...");
+  const result = await summarizeQuery(input, openAIKey);
   return {
     summary: result.textual.trim(),
     links: result.links,
-    sources: scrapedData.map((data2) => data2.url)
+    sources: ["OpenAI Knowledge Base"]
   };
 }
 var src_default = clai;