@fettstorch/clai 0.1.6 → 0.1.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fettstorch/clai",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "main": "dist/index.js",
   "bin": {
     "clai": "dist/cli.js"
@@ -18,14 +18,14 @@
   "license": "ISC",
   "description": "AI-powered webpage summarizer",
   "dependencies": {
+    "@biomejs/biome": "^1.9.4",
     "@fettstorch/jule": "^0.5.3",
     "chalk": "^5.3.0",
     "cheerio": "^1.0.0-rc.12",
     "commander": "^12.1.0",
     "inquirer": "^12.1.0",
     "openai": "^4.73.0",
-    "ora": "^8.1.1",
-    "googleapis": "^126.0.1"
+    "ora": "^8.1.1"
   },
   "devDependencies": {
     "@types/inquirer": "^9.0.7",

package/src/cli.ts

@@ -1,39 +1,43 @@
 #!/usr/bin/env bun
-import { Command } from 'commander';
-import inquirer from 'inquirer';
-import chalk from 'chalk';
-import ora from 'ora';
-import { clai } from './index';
-import { when } from '@fettstorch/jule';
-import pkg from '../package.json' assert { type: 'json' };
+import { when } from "@fettstorch/jule";
+import chalk from "chalk";
+import { Command } from "commander";
+import inquirer from "inquirer";
+import ora from "ora";
+import pkg from "../package.json" assert { type: "json" };
+import { clai } from "./index";
+import { version } from "../package.json";
 
 const program = new Command();
 
 async function main() {
+  console.log(`[clAi]::${chalk.cyan(version)}`);
   try {
     program
-      .name('clai')
-      .description('AI-powered web scraping tool')
+      .name("clai")
+      .description("AI-powered web scraping tool")
       .version(pkg.version)
-      .argument('[input...]', 'URL or search terms to analyze')
+      .argument("[input...]", "URL or search terms to analyze")
       .action(async (inputs: string[]) => {
         const openAIKey = process.env.OPENAI_API_KEY;
-        
+
         if (!openAIKey) {
-          console.error(chalk.red('❌ OPENAI_API_KEY environment variable is not set'));
+          console.error(
+            chalk.red("❌ OPENAI_API_KEY environment variable is not set")
+          );
           process.exit(1);
         }
 
-        let input = inputs?.join(' ');
-        
+        let input = inputs?.join(" ");
+
         if (!input) {
           const answers = await inquirer.prompt([
             {
-              type: 'input',
-              name: 'input',
-              message: 'Enter a URL or search query:',
-              validate: (input) => input.length > 0
-            }
+              type: "input",
+              name: "input",
+              message: "Enter a URL or search query:",
+              validate: (input) => input.length > 0,
+            },
           ]);
           input = answers.input;
         }
@@ -44,27 +48,27 @@ async function main() {
 
     await program.parseAsync();
   } catch (error) {
-    console.error(chalk.red('Fatal error:'), error);
+    console.error(chalk.red("Fatal error:"), error);
     process.exit(1);
   }
 }
 
 async function animateText(text: string, delay = 25) {
   let shouldComplete = false;
-  
+
   // Setup keypress listener
   const keypressHandler = (str: string, key: { name: string }) => {
-    if (key.name === 'return') {
+    if (key.name === "return") {
       shouldComplete = true;
     }
   };
-  
-  process.stdin.on('keypress', keypressHandler);
-  
+
+  process.stdin.on("keypress", keypressHandler);
+
   // Enable raw mode to get keypress events
   process.stdin.setRawMode(true);
   process.stdin.resume();
-  
+
   let currentIndex = 0;
   while (currentIndex < text.length) {
     if (shouldComplete) {
@@ -72,84 +76,92 @@ async function animateText(text: string, delay = 25) {
       process.stdout.write(text.slice(currentIndex));
       break;
     }
-    
+
     process.stdout.write(text[currentIndex]);
     currentIndex++;
-    
+
     if (!shouldComplete) {
-      await new Promise(resolve => setTimeout(resolve, delay));
+      await new Promise((resolve) => setTimeout(resolve, delay));
     }
   }
-  
+
   // Cleanup
   process.stdin.setRawMode(false);
   process.stdin.pause();
-  process.stdin.removeListener('keypress', keypressHandler);
-  
-  process.stdout.write('\n');
+  process.stdin.removeListener("keypress", keypressHandler);
+
+  process.stdout.write("\n");
 }
 
 function formatMarkdownForTerminal(text: string): string {
   // Handle headings first
-  const headingHandled = text.replace(/^(#{1,3})\s+(.*?)$/gm, (_, hashes, content) => when(hashes.length)({
-    1: () => `\n${chalk.yellow.bold('═══ ')}${chalk.yellow.bold(content)}${chalk.yellow.bold(' ═══')}`,
-    2: () => chalk.yellowBright.bold(content),
-    3: () => chalk.yellow(content),
-    else: () => content
-  }));
-  
+  const headingHandled = text.replace(
+    /^(#{1,3})\s+(.*?)$/gm,
+    (_, hashes, content) =>
+      when(hashes.length)({
+        1: () =>
+          `\n${chalk.yellow.bold("═══ ")}${chalk.yellow.bold(
+            content
+          )}${chalk.yellow.bold(" ═══")}`,
+        2: () => chalk.yellowBright.bold(content),
+        3: () => chalk.yellow(content),
+        else: () => content,
+      })
+  );
+
   // Handle regular bold text after headings
-  const boldHandled = headingHandled.replace(/\*\*(.*?)\*\*/g, (_, content) => chalk.bold(content));
-  
+  const boldHandled = headingHandled.replace(/\*\*(.*?)\*\*/g, (_, content) =>
+    chalk.bold(content)
+  );
+
   return boldHandled;
 }
 
 async function analyzeInput(input: string, openAIKey: string) {
-  const spinner = ora('Thinking...').start();
-  
+  const spinner = ora("Thinking...").start();
+
   try {
     const result = await clai(input, openAIKey);
-    spinner.succeed('AHA!');
-    
-    console.log(chalk.green.bold('\n📝 ═══ Summary ═══ :'));
+    spinner.succeed("AHA!");
+
+    console.log(chalk.green.bold("\n📝 ═══ Summary ═══ :"));
     const formattedContent = formatMarkdownForTerminal(result.summary);
     await animateText(formattedContent);
-    
+
     // Prompt user to select a link
     const { selectedLink } = await inquirer.prompt([
       {
-        type: 'list',
-        name: 'selectedLink',
-        message: '\n\nWhat now?:',
+        type: "list",
+        name: "selectedLink",
+        message: "\n\nWhat now?:",
         choices: [
-          { name: chalk.yellow('🔍 New search'), value: 'new' },
-          ...result.links.map(link => ({
+          { name: chalk.yellow("🔍 New search"), value: "new" },
+          ...result.links.map((link) => ({
             name: `${chalk.bold(link.name)}: ${chalk.cyan(link.url)}`,
-            value: link.url
+            value: link.url,
           })),
-          { name: 'Exit', value: 'exit' }
-        ]
-      }
+          { name: "Exit", value: "exit" },
+        ],
+      },
     ]);
 
-    if (selectedLink === 'new') {
+    if (selectedLink === "new") {
       const { input: newInput } = await inquirer.prompt([
         {
-          type: 'input',
-          name: 'input',
-          message: 'Enter a URL or search query:',
-          validate: (input) => input.length > 0
-        }
+          type: "input",
+          name: "input",
+          message: "Enter a URL or search query:",
+          validate: (input) => input.length > 0,
+        },
       ]);
       await analyzeInput(newInput, openAIKey);
-    } else if (selectedLink && selectedLink !== 'exit') {
+    } else if (selectedLink && selectedLink !== "exit") {
       await analyzeInput(selectedLink, openAIKey);
     }
-
   } catch (error) {
-    spinner?.fail('Analysis failed');
-    console.error(chalk.red('Error:'), error);
+    spinner?.fail("Analysis failed");
+    console.error(chalk.red("Error:"), error);
   }
 }
 
-main(); 
+main();

package/src/index.ts

@@ -1,5 +1,5 @@
-import { scrape } from './scraper';
-import { summarizeWebPage as summarize } from './summarizer';
+import { scrape } from "./scraper";
+import { summarizeWebPage as summarize, summarizeQuery } from "./summarizer";
 
 export interface SummaryOutput {
   summary: string;
@@ -15,7 +15,7 @@ export interface SummaryOutput {
  * @param input - URL or search query to analyze
  * @param openAIKey - OpenAI API key
  * @returns Promise with summary, extracted links, and source URLs
- * 
+ *
  * @example
  * ```ts
  * const result = await clai('https://example.com', 'your-openai-key')
@@ -24,22 +24,48 @@ export interface SummaryOutput {
  * console.log(result.sources) // Source URLs
  * ```
  */
-export async function clai(input: string, openAIKey: string): Promise<SummaryOutput> {
+export async function clai(
+  input: string,
+  openAIKey: string
+): Promise<SummaryOutput> {
   const scrapedData = await scrape(input);
-  
-  // Combine all content with source attribution
-  const combinedContent = scrapedData
-    .map(data => `Content from ${data.url}:\n${data.content}`)
-    .join('\n\n');
-  
-  const result = await summarize(combinedContent, openAIKey);
-  
+
+  // Check if we have useful scraped data (not just error pages)
+  const usefulData = scrapedData.filter(
+    (data) =>
+      data.content.length > 200 &&
+      !data.content.includes("Wikipedia does not have an article") &&
+      !data.content.includes("page not found") &&
+      !data.content.includes("404") &&
+      !data.content.includes("error")
+  );
+
+  // If we have useful scraped data, use it
+  if (usefulData.length > 0) {
+    // Combine all useful content with source attribution
+    const combinedContent = usefulData
+      .map((data) => `Content from ${data.url}:\n${data.content}`)
+      .join("\n\n");
+
+    const result = await summarize(combinedContent, openAIKey);
+
+    return {
+      summary: result.textual.trim(),
+      links: result.links,
+      sources: usefulData.map((data) => data.url),
+    };
+  }
+
+  // If no scraped data available, use OpenAI directly with the query
+  console.log("No scraped data available - using OpenAI directly for query...");
+  const result = await summarizeQuery(input, openAIKey);
+
   return {
     summary: result.textual.trim(),
     links: result.links,
-    sources: scrapedData.map(data => data.url)
+    sources: ["OpenAI Knowledge Base"],
   };
 }
 
 // Default export for easier importing
-export default clai; 
+export default clai;

package/src/openai.ts

@@ -1,5 +1,5 @@
-import { once } from '@fettstorch/jule';
-import OpenAI from 'openai';
+import { once } from "@fettstorch/jule";
+import OpenAI from "openai";
 
 const MAX_INPUT_TOKENS = 10000;
 
@@ -9,12 +9,6 @@ function truncateContent(content: string): string {
   return content.slice(0, maxChars);
 }
 
-export interface StructuredResponse<T> {
-  function_call: {
-    arguments: string;
-  };
-}
-
 class OpenAIWrapper {
   private client: OpenAI;
 
@@ -30,16 +24,16 @@ class OpenAIWrapper {
     } = {}
   ): Promise<string> {
     const truncatedPrompt = truncateContent(prompt);
-    const { model = 'gpt-4o-turbo', temperature = 0.6 } = options;
+    const { model = "gpt-4o", temperature = 0.6 } = options;
 
     const response = await this.client.chat.completions.create({
       model,
-      messages: [{ role: 'user', content: truncatedPrompt }],
+      messages: [{ role: "user", content: truncatedPrompt }],
       temperature,
-      max_tokens: 2000
+      max_tokens: 2000,
     });
 
-    return response.choices[0]?.message?.content ?? '';
+    return response.choices[0]?.message?.content ?? "";
   }
 
   async completeStructured<T>(
@@ -52,41 +46,45 @@ class OpenAIWrapper {
     }
   ): Promise<T> {
     const truncatedPrompt = truncateContent(prompt);
-    const { 
-      model = 'gpt-4o-mini', 
+    const {
+      model = "gpt-4o",
       temperature = 1.6,
-      functionName = 'generate_response',
-      responseSchema 
+      functionName = "generate_response",
+      responseSchema,
     } = options;
 
     const response = await this.client.chat.completions.create({
       model,
-      messages: [{ role: 'user', content: truncatedPrompt }],
+      messages: [{ role: "user", content: truncatedPrompt }],
       temperature,
       max_tokens: 2000,
-      functions: [{
-        name: functionName,
-        parameters: {
-          type: 'object',
-          properties: responseSchema,
-          required: Object.keys(responseSchema)
-        }
-      }],
-      function_call: { name: functionName }
+      functions: [
+        {
+          name: functionName,
+          parameters: {
+            type: "object",
+            properties: responseSchema,
+            required: Object.keys(responseSchema),
+          },
+        },
+      ],
+      function_call: { name: functionName },
     });
 
     const functionCall = response.choices[0]?.message?.function_call;
     if (!functionCall?.arguments) {
-      throw new Error('No function call arguments received');
+      throw new Error("No function call arguments received");
     }
 
     return JSON.parse(functionCall.arguments) as T;
   }
 }
 
-export const openaiClient: (apiKey?: string) => OpenAIWrapper = once((apiKey?: string) => {
+export const openaiClient: (apiKey?: string) => OpenAIWrapper = once(
+  (apiKey?: string) => {
     if (!apiKey) {
-        throw new Error('OPENAI_API_KEY is not set')
+      throw new Error("OPENAI_API_KEY is not set");
     }
-    return new OpenAIWrapper(apiKey)
-}); 
+    return new OpenAIWrapper(apiKey);
+  }
+);