@fettstorch/clai 0.1.0

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@fettstorch/clai",
+  "version": "0.1.0",
+  "main": "dist/index.js",
+  "bin": {
+    "clai": "dist/cli.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/schnullerpip/clai.git"
+  },
+  "scripts": {
+    "start": "bun run src/cli.ts",
+    "build": "bun build ./src/index.ts --outdir dist --target node && bun build ./src/cli.ts --outdir dist --target node",
+    "dev": "bun --watch src/cli.ts"
+  },
+  "author": "schnullerpip (https://github.com/schnullerpip)",
+  "license": "ISC",
+  "description": "AI-powered webpage summarizer",
+  "dependencies": {
+    "@fettstorch/jule": "^0.5.3",
+    "chalk": "^5.3.0",
+    "cheerio": "^1.0.0-rc.12",
+    "commander": "^12.1.0",
+    "inquirer": "^12.1.0",
+    "openai": "^4.73.0",
+    "ora": "^8.1.1",
+    "googleapis": "^126.0.1"
+  },
+  "devDependencies": {
+    "@types/inquirer": "^9.0.7",
+    "@types/node": "^20.11.19",
+    "bun-types": "latest"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/cli.ts

@@ -0,0 +1,137 @@
+#!/usr/bin/env bun
+import { Command } from 'commander';
+import inquirer from 'inquirer';
+import chalk from 'chalk';
+import ora from 'ora';
+import { clai } from './index';
+
+const program = new Command();
+
+async function main() {
+  try {
+    program
+      .name('clai')
+      .description('AI-powered web scraping tool')
+      .version('1.0.0')
+      .argument('[input...]', 'URL or search terms to analyze')
+      .action(async (inputs: string[]) => {
+        const openAIKey = process.env.OPENAI_API_KEY;
+        
+        if (!openAIKey) {
+          console.error(chalk.red('❌ OPENAI_API_KEY environment variable is not set'));
+          process.exit(1);
+        }
+
+        let input = inputs?.join(' ');
+        
+        if (!input) {
+          const answers = await inquirer.prompt([
+            {
+              type: 'input',
+              name: 'input',
+              message: 'Enter a URL or search query:',
+              validate: (input) => input.length > 0
+            }
+          ]);
+          input = answers.input;
+        }
+
+        await analyzeInput(input, openAIKey);
+        process.exit(0);
+      });
+
+    await program.parseAsync();
+  } catch (error) {
+    console.error(chalk.red('Fatal error:'), error);
+    process.exit(1);
+  }
+}
+
+async function animateText(text: string, delay = 25) {
+  let shouldComplete = false;
+  
+  // Setup keypress listener
+  const keypressHandler = (str: string, key: { name: string }) => {
+    if (key.name === 'return') {
+      shouldComplete = true;
+    }
+  };
+  
+  process.stdin.on('keypress', keypressHandler);
+  
+  // Enable raw mode to get keypress events
+  process.stdin.setRawMode(true);
+  process.stdin.resume();
+  
+  let currentIndex = 0;
+  while (currentIndex < text.length) {
+    if (shouldComplete) {
+      // Show remaining text immediately
+      process.stdout.write(text.slice(currentIndex));
+      break;
+    }
+    
+    process.stdout.write(text[currentIndex]);
+    currentIndex++;
+    
+    if (!shouldComplete) {
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+  
+  // Cleanup
+  process.stdin.setRawMode(false);
+  process.stdin.pause();
+  process.stdin.removeListener('keypress', keypressHandler);
+  
+  process.stdout.write('\n');
+}
+
+async function analyzeInput(input: string, openAIKey: string) {
+  const spinner = ora('Analyzing content...').start();
+  
+  try {
+    const result = await clai(input, openAIKey);
+    spinner.succeed('Analysis complete');
+    
+    console.log(chalk.green.bold('\n📝 Summary:'));
+    await animateText(result.summary);
+    
+    // Prompt user to select a link
+    const { selectedLink } = await inquirer.prompt([
+      {
+        type: 'list',
+        name: 'selectedLink',
+        message: '\n\nWhat now?:',
+        choices: [
+          { name: chalk.yellow('🔍 New search'), value: 'new' },
+          ...result.links.map(link => ({
+            name: `${chalk.bold(link.name)}: ${chalk.cyan(link.url)}`,
+            value: link.url
+          })),
+          { name: 'Exit', value: 'exit' }
+        ]
+      }
+    ]);
+
+    if (selectedLink === 'new') {
+      const { input: newInput } = await inquirer.prompt([
+        {
+          type: 'input',
+          name: 'input',
+          message: 'Enter a URL or search query:',
+          validate: (input) => input.length > 0
+        }
+      ]);
+      await analyzeInput(newInput, openAIKey);
+    } else if (selectedLink && selectedLink !== 'exit') {
+      await analyzeInput(selectedLink, openAIKey);
+    }
+
+  } catch (error) {
+    spinner?.fail('Analysis failed');
+    console.error(chalk.red('Error:'), error);
+  }
+}
+
+main(); 

package/src/index.ts

@@ -0,0 +1,45 @@
+import { scrape } from './scraper';
+import { summarizeWebPage as summarize } from './summarizer';
+
+export interface SummaryOutput {
+  summary: string;
+  links: ReadonlyArray<{
+    name: string;
+    url: string;
+  }>;
+  sources: string[];
+}
+
+/**
+ * Scrapes and analyzes webpages using AI
+ * @param input - URL or search query to analyze
+ * @param openAIKey - OpenAI API key
+ * @returns Promise with summary, extracted links, and source URLs
+ * 
+ * @example
+ * ```ts
+ * const result = await clai('https://example.com', 'your-openai-key')
+ * console.log(result.summary) // AI generated summary
+ * console.log(result.links) // Extracted links
+ * console.log(result.sources) // Source URLs
+ * ```
+ */
+export async function clai(input: string, openAIKey: string): Promise<SummaryOutput> {
+  const scrapedData = await scrape(input);
+  
+  // Combine all content with source attribution
+  const combinedContent = scrapedData
+    .map(data => `Content from ${data.url}:\n${data.content}`)
+    .join('\n\n');
+  
+  const result = await summarize(combinedContent, openAIKey);
+  
+  return {
+    summary: result.textual.trim(),
+    links: result.links,
+    sources: scrapedData.map(data => data.url)
+  };
+}
+
+// Default export for easier importing
+export default clai; 

package/src/openai.ts

@@ -0,0 +1,92 @@
+import { once } from '@fettstorch/jule';
+import OpenAI from 'openai';
+
+const MAX_INPUT_TOKENS = 10000;
+
+function truncateContent(content: string): string {
+  const maxChars = MAX_INPUT_TOKENS * 4;
+  if (content.length <= maxChars) return content;
+  return content.slice(0, maxChars) + '... (content truncated)';
+}
+
+export interface StructuredResponse<T> {
+  function_call: {
+    arguments: string;
+  };
+}
+
+class OpenAIWrapper {
+  private client: OpenAI;
+
+  constructor(apiKey: string) {
+    this.client = new OpenAI({ apiKey });
+  }
+
+  async complete(
+    prompt: string,
+    options: {
+      model?: string;
+      temperature?: number;
+    } = {}
+  ): Promise<string> {
+    const truncatedPrompt = truncateContent(prompt);
+    const { model = 'gpt-4o-turbo', temperature = 0.6 } = options;
+
+    const response = await this.client.chat.completions.create({
+      model,
+      messages: [{ role: 'user', content: truncatedPrompt }],
+      temperature,
+      max_tokens: 2000
+    });
+
+    return response.choices[0]?.message?.content ?? '';
+  }
+
+  async completeStructured<T>(
+    prompt: string,
+    options: {
+      model?: string;
+      temperature?: number;
+      functionName?: string;
+      responseSchema: Record<string, unknown>;
+    }
+  ): Promise<T> {
+    const truncatedPrompt = truncateContent(prompt);
+    const { 
+      model = 'gpt-3.5-turbo', 
+      temperature = 0.6,
+      functionName = 'generate_response',
+      responseSchema 
+    } = options;
+
+    const response = await this.client.chat.completions.create({
+      model,
+      messages: [{ role: 'user', content: truncatedPrompt }],
+      temperature,
+      max_tokens: 2000,
+      functions: [{
+        name: functionName,
+        parameters: {
+          type: 'object',
+          properties: responseSchema,
+          required: Object.keys(responseSchema)
+        }
+      }],
+      function_call: { name: functionName }
+    });
+
+    const functionCall = response.choices[0]?.message?.function_call;
+    if (!functionCall?.arguments) {
+      throw new Error('No function call arguments received');
+    }
+
+    return JSON.parse(functionCall.arguments) as T;
+  }
+}
+
+export const openaiClient: (apiKey?: string) => OpenAIWrapper = once((apiKey?: string) => {
+    if (!apiKey) {
+        throw new Error('OPENAI_API_KEY is not set')
+    }
+    return new OpenAIWrapper(apiKey)
+}); 

package/src/scraper.ts

@@ -0,0 +1,102 @@
+import * as Cheerio from 'cheerio';
+
+export interface ScrapedData {
+  title: string;
+  content: string;
+  url: string;
+}
+
+export async function scrape(input: string): Promise<ScrapedData[]> {
+  try {
+    let urls: string[];
+    
+    if (isValidUrl(input)) {
+      urls = [normalizeUrl(input)];
+    } else {
+      urls = await getGoogleResults(input);
+    }
+
+    // Fetch all URLs in parallel
+    const results = await Promise.all(
+      urls.map(async (url) => {
+        try {
+          const html = await fetchHtml(url);
+          const data = extractDataFromHtml(html);
+          return { ...data, url };
+        } catch (error) {
+          console.error(`Error scraping ${url}:`, error);
+          return null;
+        }
+      })
+    );
+
+    // Filter out failed scrapes
+    return results.filter((result): result is ScrapedData => result !== null);
+  } catch (error) {
+    console.error('Error during scraping:', error);
+    throw error;
+  }
+}
+
+// --- module private
+
+function isValidUrl(input: string): boolean {
+  return !input.includes(' ');
+}
+
+function normalizeUrl(url: string): string {
+  if (!url.startsWith('http://') && !url.startsWith('https://')) {
+    return `https://${url}`;
+  }
+  return url;
+}
+
+async function getGoogleResults(query: string): Promise<string[]> {
+  const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}`;
+  const html = await fetchHtml(searchUrl);
+  
+  const urlPattern = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/gi;
+  const urls = html.match(urlPattern) || [];
+  
+  const queryWords = query
+    .toLowerCase()
+    .split(/\s+/)
+    .filter(word => word.length > 2);
+  
+  const filteredUrls = new Set(
+    urls.filter(url => {
+      const urlLower = url.toLowerCase();
+      return !urlLower.includes('www.google') && 
+             !urlLower.includes('gstatic.com') && 
+             !urlLower.includes('googleapis.com') &&
+             !urlLower.includes('googleadservices') &&
+             queryWords.some(word => urlLower.includes(word));
+    })
+  );
+
+  const results = [...filteredUrls].slice(0, 3);
+  
+  if (results.length === 0) {
+    throw new Error('No search results found');
+  }
+  
+  return results;
+}
+
+async function fetchHtml(url: string): Promise<string> {
+  const response = await fetch(url, {
+    headers: {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+    }
+  });
+  return response.text();
+}
+
+function extractDataFromHtml(html: string): ScrapedData {
+  const cheerioDoc = Cheerio.load(html);
+  return {
+    title: cheerioDoc('title').text(),
+    content: cheerioDoc('body').text(),
+    url: cheerioDoc('link[rel="canonical"]').attr('href') || ''
+  };
+}

package/src/summarizer.ts

@@ -0,0 +1,77 @@
+import { openaiClient } from './openai';
+
+export type SummaryResult = Readonly<{
+  textual: string;
+  links: ReadonlyArray<{
+    name: string;
+    url: string;
+  }>;
+}>
+
+/**
+ * Summarizes content and extracts relevant links using OpenAI
+ * @param content - The text content to analyze and summarize
+ * @param maxLength - Maximum length of the summary in words
+ * @returns Promise containing the summary text and extracted links
+ * @throws Will throw an error if OpenAI API call fails
+ * 
+ * @example
+ * ```ts
+ * const result = await summarizeContent(longText, 100)
+ * console.log(result.textual) // Summary text
+ * console.log(result.links) // Array of extracted links
+ * ```
+ */
+
+export async function summarizeWebPage(content: string, openAIApiKey: string): Promise<SummaryResult> {
+  const openai = openaiClient(openAIApiKey);
+  
+  const prompt = `Your are an expert educator. Analyze the following text and create a
+  concise summary with the following guidelines:
+   1. Always use bullet points, lists and tables over paragraphs.
+   2. Produce valid markdown output
+   3. Use the articles titles and headings as a guide
+   4. Try to present the most relevant information
+   5. Extract all meaningful links from the text
+   6. Don't narrate the content e.g. 'The text says that the earth is round' but rather use the content itself e.g. 'The earth is round'
+   7. Don't use the word 'text' or 'content' in your summary
+   8. Don't try to emulate emotions or tone of the original content, always be neutral and objective
+   9. If the content is instructional repeat the instructions step by step e.g.:
+    - Step 1: Do this
+    - Step 2: Do that
+    - Step 3: Done
+   10. Mark proper nouns as bold e.g. **Harry Potter**
+  
+  Don't just summarize, cite the key information.
+  
+  Text to analyze:\n"${content}\n"`;
+
+  const schema = {
+    textual: {
+      type: 'string',
+      description: 'Concise summary of the text'
+    },
+    links: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          name: {
+            type: 'string',
+            description: 'Descriptive name or title of the link'
+          },
+          url: {
+            type: 'string',
+            description: 'The URL of the link'
+          }
+        },
+        required: ['name', 'url']
+      }
+    }
+  };
+
+  return openai.completeStructured<SummaryResult>(prompt, {
+    temperature: 0.3,
+    responseSchema: schema
+  });
+} 

package/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "esnext",
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "types": ["bun-types"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["src/**/*"]
+}