@fettstorch/clai 0.1.7 ā†’ 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/dist/cli.js +148 -78
  2. package/dist/index.js +133 -65
  3. package/package.json +36 -36
  4. package/src/cli.ts +152 -148
  5. package/src/index.ts +45 -22
  6. package/src/openai.ts +0 -6
  7. package/src/scraper.ts +207 -93
  8. package/src/summarizer.ts +101 -40
package/package.json CHANGED
@@ -1,38 +1,38 @@
1
1
  {
2
- "name": "@fettstorch/clai",
3
- "version": "0.1.7",
4
- "main": "dist/index.js",
5
- "bin": {
6
- "clai": "dist/cli.js"
7
- },
8
- "repository": {
9
- "type": "git",
10
- "url": "git+https://github.com/schnullerpip/clai.git"
11
- },
12
- "scripts": {
13
- "start": "bun run src/cli.ts",
14
- "build": "bun build ./src/index.ts --outdir dist --target node && bun build ./src/cli.ts --outdir dist --target node",
15
- "dev": "bun --watch src/cli.ts"
16
- },
17
- "author": "schnullerpip (https://github.com/schnullerpip)",
18
- "license": "ISC",
19
- "description": "AI-powered webpage summarizer",
20
- "dependencies": {
21
- "@biomejs/biome": "^1.9.4",
22
- "@fettstorch/jule": "^0.5.3",
23
- "chalk": "^5.3.0",
24
- "cheerio": "^1.0.0-rc.12",
25
- "commander": "^12.1.0",
26
- "inquirer": "^12.1.0",
27
- "openai": "^4.73.0",
28
- "ora": "^8.1.1"
29
- },
30
- "devDependencies": {
31
- "@types/inquirer": "^9.0.7",
32
- "@types/node": "^20.11.19",
33
- "bun-types": "latest"
34
- },
35
- "publishConfig": {
36
- "access": "public"
37
- }
2
+ "name": "@fettstorch/clai",
3
+ "version": "0.1.9",
4
+ "main": "dist/index.js",
5
+ "bin": {
6
+ "clai": "dist/cli.js"
7
+ },
8
+ "repository": {
9
+ "type": "git",
10
+ "url": "git+https://github.com/schnullerpip/clai.git"
11
+ },
12
+ "scripts": {
13
+ "start": "bun run src/cli.ts",
14
+ "build": "bun build ./src/index.ts --outdir dist --target node && bun build ./src/cli.ts --outdir dist --target node",
15
+ "dev": "bun --watch src/cli.ts"
16
+ },
17
+ "author": "schnullerpip (https://github.com/schnullerpip)",
18
+ "license": "ISC",
19
+ "description": "AI-powered webpage summarizer",
20
+ "dependencies": {
21
+ "@biomejs/biome": "^1.9.4",
22
+ "@fettstorch/jule": "^0.5.3",
23
+ "chalk": "^5.3.0",
24
+ "cheerio": "^1.0.0-rc.12",
25
+ "commander": "^12.1.0",
26
+ "inquirer": "^12.1.0",
27
+ "openai": "^4.73.0",
28
+ "ora": "^8.1.1"
29
+ },
30
+ "devDependencies": {
31
+ "@types/inquirer": "^9.0.7",
32
+ "@types/node": "^20.11.19",
33
+ "bun-types": "latest"
34
+ },
35
+ "publishConfig": {
36
+ "access": "public"
37
+ }
38
38
  }
package/src/cli.ts CHANGED
@@ -1,163 +1,167 @@
1
1
  #!/usr/bin/env bun
2
- import { when } from '@fettstorch/jule'
3
- import chalk from 'chalk'
4
- import { Command } from 'commander'
5
- import inquirer from 'inquirer'
6
- import ora from 'ora'
7
- import pkg from '../package.json' assert { type: 'json' }
8
- import { clai } from './index'
2
+ import { when } from "@fettstorch/jule";
3
+ import chalk from "chalk";
4
+ import { Command } from "commander";
5
+ import inquirer from "inquirer";
6
+ import ora from "ora";
7
+ import pkg from "../package.json" assert { type: "json" };
8
+ import { clai } from "./index";
9
+ import { version } from "../package.json";
9
10
 
10
- const program = new Command()
11
+ const program = new Command();
11
12
 
12
13
  async function main() {
13
- try {
14
- program
15
- .name('clai')
16
- .description('AI-powered web scraping tool')
17
- .version(pkg.version)
18
- .argument('[input...]', 'URL or search terms to analyze')
19
- .action(async (inputs: string[]) => {
20
- const openAIKey = process.env.OPENAI_API_KEY
21
-
22
- if (!openAIKey) {
23
- console.error(
24
- chalk.red('āŒ OPENAI_API_KEY environment variable is not set'),
25
- )
26
- process.exit(1)
27
- }
28
-
29
- let input = inputs?.join(' ')
30
-
31
- if (!input) {
32
- const answers = await inquirer.prompt([
33
- {
34
- type: 'input',
35
- name: 'input',
36
- message: 'Enter a URL or search query:',
37
- validate: (input) => input.length > 0,
38
- },
39
- ])
40
- input = answers.input
41
- }
42
-
43
- await analyzeInput(input, openAIKey)
44
- process.exit(0)
45
- })
46
-
47
- await program.parseAsync()
48
- } catch (error) {
49
- console.error(chalk.red('Fatal error:'), error)
50
- process.exit(1)
51
- }
14
+ console.log(`[clAi]::${chalk.cyan(version)}`);
15
+ try {
16
+ program
17
+ .name("clai")
18
+ .description("AI-powered web scraping tool")
19
+ .version(pkg.version)
20
+ .argument("[input...]", "URL or search terms to analyze")
21
+ .action(async (inputs: string[]) => {
22
+ const openAIKey = process.env.OPENAI_API_KEY;
23
+
24
+ if (!openAIKey) {
25
+ console.error(
26
+ chalk.red("āŒ OPENAI_API_KEY environment variable is not set")
27
+ );
28
+ process.exit(1);
29
+ }
30
+
31
+ let input = inputs?.join(" ");
32
+
33
+ if (!input) {
34
+ const answers = await inquirer.prompt([
35
+ {
36
+ type: "input",
37
+ name: "input",
38
+ message: "Enter a URL or search query:",
39
+ validate: (input) => input.length > 0,
40
+ },
41
+ ]);
42
+ input = answers.input;
43
+ }
44
+
45
+ await analyzeInput(input, openAIKey);
46
+ process.exit(0);
47
+ });
48
+
49
+ await program.parseAsync();
50
+ } catch (error) {
51
+ console.error(chalk.red("Fatal error:"), error);
52
+ process.exit(1);
53
+ }
52
54
  }
53
55
 
54
56
  async function animateText(text: string, delay = 25) {
55
- let shouldComplete = false
56
-
57
- // Setup keypress listener
58
- const keypressHandler = (str: string, key: { name: string }) => {
59
- if (key.name === 'return') {
60
- shouldComplete = true
61
- }
62
- }
63
-
64
- process.stdin.on('keypress', keypressHandler)
65
-
66
- // Enable raw mode to get keypress events
67
- process.stdin.setRawMode(true)
68
- process.stdin.resume()
69
-
70
- let currentIndex = 0
71
- while (currentIndex < text.length) {
72
- if (shouldComplete) {
73
- // Show remaining text immediately
74
- process.stdout.write(text.slice(currentIndex))
75
- break
76
- }
77
-
78
- process.stdout.write(text[currentIndex])
79
- currentIndex++
80
-
81
- if (!shouldComplete) {
82
- await new Promise((resolve) => setTimeout(resolve, delay))
83
- }
84
- }
85
-
86
- // Cleanup
87
- process.stdin.setRawMode(false)
88
- process.stdin.pause()
89
- process.stdin.removeListener('keypress', keypressHandler)
90
-
91
- process.stdout.write('\n')
57
+ let shouldComplete = false;
58
+
59
+ // Setup keypress listener
60
+ const keypressHandler = (str: string, key: { name: string }) => {
61
+ if (key.name === "return") {
62
+ shouldComplete = true;
63
+ }
64
+ };
65
+
66
+ process.stdin.on("keypress", keypressHandler);
67
+
68
+ // Enable raw mode to get keypress events
69
+ process.stdin.setRawMode(true);
70
+ process.stdin.resume();
71
+
72
+ let currentIndex = 0;
73
+ while (currentIndex < text.length) {
74
+ if (shouldComplete) {
75
+ // Show remaining text immediately
76
+ process.stdout.write(text.slice(currentIndex));
77
+ break;
78
+ }
79
+
80
+ process.stdout.write(text[currentIndex]);
81
+ currentIndex++;
82
+
83
+ if (!shouldComplete) {
84
+ await new Promise((resolve) => setTimeout(resolve, delay));
85
+ }
86
+ }
87
+
88
+ // Cleanup
89
+ process.stdin.setRawMode(false);
90
+ process.stdin.pause();
91
+ process.stdin.removeListener("keypress", keypressHandler);
92
+
93
+ process.stdout.write("\n");
92
94
  }
93
95
 
94
96
  function formatMarkdownForTerminal(text: string): string {
95
- // Handle headings first
96
- const headingHandled = text.replace(
97
- /^(#{1,3})\s+(.*?)$/gm,
98
- (_, hashes, content) =>
99
- when(hashes.length)({
100
- 1: () =>
101
- `\n${chalk.yellow.bold('ā•ā•ā• ')}${chalk.yellow.bold(content)}${chalk.yellow.bold(' ā•ā•ā•')}`,
102
- 2: () => chalk.yellowBright.bold(content),
103
- 3: () => chalk.yellow(content),
104
- else: () => content,
105
- }),
106
- )
107
-
108
- // Handle regular bold text after headings
109
- const boldHandled = headingHandled.replace(/\*\*(.*?)\*\*/g, (_, content) =>
110
- chalk.bold(content),
111
- )
112
-
113
- return boldHandled
97
+ // Handle headings first
98
+ const headingHandled = text.replace(
99
+ /^(#{1,3})\s+(.*?)$/gm,
100
+ (_, hashes, content) =>
101
+ when(hashes.length)({
102
+ 1: () =>
103
+ `\n${chalk.yellow.bold("ā•ā•ā• ")}${chalk.yellow.bold(
104
+ content
105
+ )}${chalk.yellow.bold(" ā•ā•ā•")}`,
106
+ 2: () => chalk.yellowBright.bold(content),
107
+ 3: () => chalk.yellow(content),
108
+ else: () => content,
109
+ })
110
+ );
111
+
112
+ // Handle regular bold text after headings
113
+ const boldHandled = headingHandled.replace(/\*\*(.*?)\*\*/g, (_, content) =>
114
+ chalk.bold(content)
115
+ );
116
+
117
+ return boldHandled;
114
118
  }
115
119
 
116
120
  async function analyzeInput(input: string, openAIKey: string) {
117
- const spinner = ora('Thinking...').start()
118
-
119
- try {
120
- const result = await clai(input, openAIKey)
121
- spinner.succeed('AHA!')
122
-
123
- console.log(chalk.green.bold('\nšŸ“ ā•ā•ā• Summary ā•ā•ā• :'))
124
- const formattedContent = formatMarkdownForTerminal(result.summary)
125
- await animateText(formattedContent)
126
-
127
- // Prompt user to select a link
128
- const { selectedLink } = await inquirer.prompt([
129
- {
130
- type: 'list',
131
- name: 'selectedLink',
132
- message: '\n\nWhat now?:',
133
- choices: [
134
- { name: chalk.yellow('šŸ” New search'), value: 'new' },
135
- ...result.links.map((link) => ({
136
- name: `${chalk.bold(link.name)}: ${chalk.cyan(link.url)}`,
137
- value: link.url,
138
- })),
139
- { name: 'Exit', value: 'exit' },
140
- ],
141
- },
142
- ])
143
-
144
- if (selectedLink === 'new') {
145
- const { input: newInput } = await inquirer.prompt([
146
- {
147
- type: 'input',
148
- name: 'input',
149
- message: 'Enter a URL or search query:',
150
- validate: (input) => input.length > 0,
151
- },
152
- ])
153
- await analyzeInput(newInput, openAIKey)
154
- } else if (selectedLink && selectedLink !== 'exit') {
155
- await analyzeInput(selectedLink, openAIKey)
156
- }
157
- } catch (error) {
158
- spinner?.fail('Analysis failed')
159
- console.error(chalk.red('Error:'), error)
160
- }
121
+ const spinner = ora("Thinking...").start();
122
+
123
+ try {
124
+ const result = await clai(input, openAIKey);
125
+ spinner.succeed("AHA!");
126
+
127
+ console.log(chalk.green.bold("\nšŸ“ ā•ā•ā• Summary ā•ā•ā• :"));
128
+ const formattedContent = formatMarkdownForTerminal(result.summary);
129
+ await animateText(formattedContent);
130
+
131
+ // Prompt user to select a link
132
+ const { selectedLink } = await inquirer.prompt([
133
+ {
134
+ type: "list",
135
+ name: "selectedLink",
136
+ message: "\n\nWhat now?:",
137
+ choices: [
138
+ { name: chalk.yellow("šŸ” New search"), value: "new" },
139
+ ...result.links.map((link) => ({
140
+ name: `${chalk.bold(link.name)}: ${chalk.cyan(link.url)}`,
141
+ value: link.url,
142
+ })),
143
+ { name: "Exit", value: "exit" },
144
+ ],
145
+ },
146
+ ]);
147
+
148
+ if (selectedLink === "new") {
149
+ const { input: newInput } = await inquirer.prompt([
150
+ {
151
+ type: "input",
152
+ name: "input",
153
+ message: "Enter a URL or search query:",
154
+ validate: (input) => input.length > 0,
155
+ },
156
+ ]);
157
+ await analyzeInput(newInput, openAIKey);
158
+ } else if (selectedLink && selectedLink !== "exit") {
159
+ await analyzeInput(selectedLink, openAIKey);
160
+ }
161
+ } catch (error) {
162
+ spinner?.fail("Analysis failed");
163
+ console.error(chalk.red("Error:"), error);
164
+ }
161
165
  }
162
166
 
163
- main()
167
+ main();
package/src/index.ts CHANGED
@@ -1,13 +1,13 @@
1
- import { scrape } from './scraper'
2
- import { summarizeWebPage as summarize } from './summarizer'
1
+ import { scrape } from "./scraper";
2
+ import { summarizeWebPage as summarize, summarizeQuery } from "./summarizer";
3
3
 
4
4
  export interface SummaryOutput {
5
- summary: string
6
- links: ReadonlyArray<{
7
- name: string
8
- url: string
9
- }>
10
- sources: string[]
5
+ summary: string;
6
+ links: ReadonlyArray<{
7
+ name: string;
8
+ url: string;
9
+ }>;
10
+ sources: string[];
11
11
  }
12
12
 
13
13
  /**
@@ -25,24 +25,47 @@ export interface SummaryOutput {
25
25
  * ```
26
26
  */
27
27
  export async function clai(
28
- input: string,
29
- openAIKey: string,
28
+ input: string,
29
+ openAIKey: string
30
30
  ): Promise<SummaryOutput> {
31
- const scrapedData = await scrape(input)
31
+ const scrapedData = await scrape(input);
32
32
 
33
- // Combine all content with source attribution
34
- const combinedContent = scrapedData
35
- .map((data) => `Content from ${data.url}:\n${data.content}`)
36
- .join('\n\n')
33
+ // Check if we have useful scraped data (not just error pages)
34
+ const usefulData = scrapedData.filter(
35
+ (data) =>
36
+ data.content.length > 200 &&
37
+ !data.content.includes("Wikipedia does not have an article") &&
38
+ !data.content.includes("page not found") &&
39
+ !data.content.includes("404") &&
40
+ !data.content.includes("error")
41
+ );
37
42
 
38
- const result = await summarize(combinedContent, openAIKey)
43
+ // If we have useful scraped data, use it
44
+ if (usefulData.length > 0) {
45
+ // Combine all useful content with source attribution
46
+ const combinedContent = usefulData
47
+ .map((data) => `Content from ${data.url}:\n${data.content}`)
48
+ .join("\n\n");
39
49
 
40
- return {
41
- summary: result.textual.trim(),
42
- links: result.links,
43
- sources: scrapedData.map((data) => data.url),
44
- }
50
+ const result = await summarize(combinedContent, openAIKey);
51
+
52
+ return {
53
+ summary: result.textual.trim(),
54
+ links: result.links,
55
+ sources: usefulData.map((data) => data.url),
56
+ };
57
+ }
58
+
59
+ // If no scraped data available, use OpenAI directly with the query
60
+ console.log("No scraped data available - using OpenAI directly for query...");
61
+ const result = await summarizeQuery(input, openAIKey);
62
+
63
+ return {
64
+ summary: result.textual.trim(),
65
+ links: result.links,
66
+ sources: ["OpenAI Knowledge Base"],
67
+ };
45
68
  }
46
69
 
47
70
  // Default export for easier importing
48
- export default clai
71
+ export default clai;
package/src/openai.ts CHANGED
@@ -9,12 +9,6 @@ function truncateContent(content: string): string {
9
9
  return content.slice(0, maxChars);
10
10
  }
11
11
 
12
- export interface StructuredResponse<T> {
13
- function_call: {
14
- arguments: string;
15
- };
16
- }
17
-
18
12
  class OpenAIWrapper {
19
13
  private client: OpenAI;
20
14