@involvex/bun-crawl 0.0.0-stage → 0.1.0

package/eslint.config.mjs

@@ -0,0 +1,37 @@
+import js from '@eslint/js'
+import tseslint from 'typescript-eslint'
+
+export default tseslint.config(
+	{
+		ignores: [
+			'dist/',
+			'node_modules/',
+			'*.exe',
+			'*.md',
+			'bun.lock',
+			'IMPLEMENTATION_PLAN.md',
+		],
+	},
+	js.configs.recommended,
+	...tseslint.configs.recommended,
+	{
+		files: ['src/**/*.ts'],
+		languageOptions: {
+			ecmaVersion: 'latest',
+			sourceType: 'module',
+			parserOptions: {
+				project: './tsconfig.json',
+				tsconfigRootDir: import.meta.dirname,
+			},
+		},
+		rules: {
+			'@typescript-eslint/no-explicit-any': 'warn',
+			'@typescript-eslint/no-unused-vars': ['warn', {argsIgnorePattern: '^_'}],
+			'@typescript-eslint/consistent-type-imports': 'error',
+			'@typescript-eslint/no-floating-promises': 'warn',
+			'@typescript-eslint/await-thenable': 'error',
+			'preserve-caught-error': 'off',
+			'no-console': ['warn', {allow: ['warn', 'error']}],
+		},
+	},
+)

package/package.json

@@ -1,6 +1,41 @@
 {
-  "name": "@involvex/bun-crawl",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+	"name": "@involvex/bun-crawl",
+	"version": "0.1.0",
+	"author": "involvex",
+	"type": "module",
+	"main": "src/index.ts",
+	"bin": {
+		"bun-crawl": "./src/cli.ts",
+		"buncrawl": "./dist/bun-crawl.js"
+	},
+	"scripts": {
+		"prebuild": "bun run format && bun run lint:fix && bun run typecheck",
+		"build": "bun build --target node --minify ./src/index.ts --outfile dist/bun-crawl.js",
+		"dev": "bun run src/cli.ts",
+		"format": "prettier . --write",
+		"lint": "eslint .",
+		"lint:fix": "eslint . --fix",
+		"test": "bun test",
+		"typecheck": "tsc --noEmit"
+	},
+	"prettier": "@involvex/prettier-config",
+	"dependencies": {
+		"@mozilla/readability": "^0.6.0",
+		"citty": "^0.2.2",
+		"linkedom": "^0.18.13",
+		"turndown": "^7.2.4"
+	},
+	"devDependencies": {
+		"@eslint/js": "^10.0.1",
+		"@involvex/prettier-config": "^0.0.6",
+		"@types/bun": "latest",
+		"@types/mozilla-readability": "^0.2.1",
+		"@types/turndown": "^5.0.6",
+		"@typescript-eslint/eslint-plugin": "^8.70.0",
+		"@typescript-eslint/parser": "^8.70.0",
+		"eslint": "^10.10.0",
+		"prettier": "^3.9.6",
+		"typescript": "~5.6.0",
+		"typescript-eslint": "^8.70.0"
+	}
+}

package/src/ai/interface.ts

@@ -0,0 +1,7 @@
+export interface AIExtractor {
+	extract<T = string>(markdown: string, schemaPrompt: string): Promise<T>
+}
+
+export interface AIExtractorConfig {
+	enabled?: boolean
+}

package/src/ai/null-ai.ts

@@ -0,0 +1,10 @@
+import type {AIExtractor} from './interface'
+
+export class NullAIExtractor implements AIExtractor {
+	async extract<T = string>(
+		markdown: string,
+		_schemaPrompt: string,
+	): Promise<T> {
+		return markdown as T
+	}
+}

package/src/ai/zen-ai.ts

@@ -0,0 +1,57 @@
+// TODO: Implement when OpenCode Zen API is available
+// This is a template for future OpenAI-compatible Chat Completions integration
+
+/*
+export interface ZenAIConfig {
+  endpoint: string;        // e.g. "https://api.opencode.ai/v1/chat/completions"
+  apiKey: string;
+  model?: string;          // e.g. "gpt-4o", "claude-3.5-sonnet"
+  temperature?: number;
+  maxTokens?: number;
+}
+
+export class ZenAIExtractor implements AIExtractor {
+  constructor(private config: ZenAIConfig) {}
+  
+  async extract<T = string>(markdown: string, schemaPrompt: string): Promise<T> {
+    const response = await fetch(this.config.endpoint, {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${this.config.apiKey}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        model: this.config.model || 'gpt-4o',
+        messages: [
+          { role: 'system', content: schemaPrompt },
+          { role: 'user', content: markdown }
+        ],
+        temperature: this.config.temperature ?? 0,
+        max_tokens: this.config.maxTokens ?? 4000,
+      }),
+    });
+    
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`Zen AI API error: ${response.status} - ${error}`);
+    }
+    
+    const data = await response.json();
+    const content = data.choices[0]?.message?.content;
+    
+    if (!content) {
+      throw new Error('No content in AI response');
+    }
+    
+    return content as T;
+  }
+}
+*/
+
+export class ZenAIExtractor {
+	constructor() {
+		throw new Error(
+			'ZenAIExtractor not yet implemented. See src/ai/zen-ai.ts for template.',
+		)
+	}
+}

package/src/cli.ts

@@ -0,0 +1,309 @@
+#!/usr/bin/env bun
+
+import {writeFile} from 'fs/promises'
+import {parseArgs} from 'node:util'
+import {dirname, join} from 'path'
+import {NullAIExtractor} from './ai/null-ai'
+import {Cleaner} from './core/cleaner'
+import {Crawler} from './core/crawler'
+import {Fetcher} from './core/fetcher'
+import {MarkdownConverter} from './core/markdown'
+import type {CrawlOptions, ScrapeResult} from './types'
+
+const pkg = await import('../package.json', {with: {type: 'json'}}).then(
+	m => m.default,
+)
+
+function createFetcher(options: {
+	timeout?: number
+	userAgent?: string
+}): Fetcher {
+	return new Fetcher({
+		timeout: options.timeout,
+		userAgent: options.userAgent,
+	})
+}
+
+function createCleaner(): Cleaner {
+	return new Cleaner()
+}
+
+function createConverter(): MarkdownConverter {
+	return new MarkdownConverter()
+}
+
+function createCrawler(
+	fetcher: Fetcher,
+	cleaner: Cleaner,
+	converter: MarkdownConverter,
+	options: CrawlOptions,
+): Crawler {
+	return new Crawler(fetcher, cleaner, converter, options)
+}
+
+import {mkdir} from 'fs/promises'
+
+async function writeOutput(output: string, filePath?: string): Promise<void> {
+	if (filePath) {
+		const dir = dirname(filePath)
+		if (dir !== '.') {
+			await mkdir(dir, {recursive: true})
+		}
+		await writeFile(filePath, output, 'utf-8')
+		console.error(`Written to ${filePath}`)
+	} else {
+		// eslint-disable-next-line no-console
+		console.log(output)
+	}
+}
+
+function showHelp(): void {
+	// eslint-disable-next-line no-console
+	console.log(`
+bun-crawl v${pkg.version} - Open-source Firecrawl alternative built with Bun
+
+USAGE:
+  bun-crawl <command> <url> [options]
+
+COMMANDS:
+  scrape    Scrape a single URL and output markdown
+  crawl     Crawl a website recursively
+
+SCRAPE OPTIONS:
+  -o, --output <file>     Output file path (stdout if omitted)
+  -f, --format <format>   Output format: markdown or json (default: markdown)
+  -t, --timeout <ms>      Request timeout in milliseconds (default: 30000)
+  -u, --user-agent <ua>   Custom User-Agent string (default: bun-crawl/0.1.0)
+  --json                  Output machine-readable JSON (alias for --format json)
+
+CRAWL OPTIONS:
+  -d, --depth <num>       Maximum crawl depth (default: 2)
+  -l, --limit <num>       Maximum pages to crawl (default: 20)
+  -O, --out-dir <dir>     Output directory for markdown files (default: ./crawl-output)
+  --same-domain           Stay on the same domain (default: true)
+  -c, --concurrency <num> Concurrent requests (default: 5)
+  -t, --timeout <ms>      Request timeout in milliseconds (default: 30000)
+  -u, --user-agent <ua>   Custom User-Agent string (default: bun-crawl/0.1.0)
+  --json                  Output machine-readable JSON summary
+
+EXAMPLES:
+  bun-crawl scrape https://example.com -o page.md
+  bun-crawl scrape https://example.com --format json
+  bun-crawl crawl https://example.com -d 3 -l 50 -O ./output
+  bun-crawl crawl https://example.com --json
+`)
+}
+
+function parseCliArgs(rawArgs: string[]): {
+	command: string
+	url: string
+	options: Record<string, unknown>
+} {
+	if (rawArgs.length === 0 || rawArgs[0] === '--help' || rawArgs[0] === '-h') {
+		showHelp()
+		process.exit(0)
+	}
+
+	if (rawArgs.length < 2) {
+		console.error('Error: Missing command and URL')
+		showHelp()
+		process.exit(1)
+	}
+
+	const command = rawArgs[0]
+	const url = rawArgs[1]
+
+	if (!['scrape', 'crawl'].includes(command)) {
+		console.error(
+			`Error: Unknown command "${command}". Use 'scrape' or 'crawl'`,
+		)
+		showHelp()
+		process.exit(1)
+	}
+
+	const optionArgs = rawArgs.slice(2)
+
+	const {values} = parseArgs({
+		args: optionArgs,
+		options: {
+			// Scrape options
+			output: {type: 'string', short: 'o'},
+			format: {type: 'string', short: 'f', default: 'markdown'},
+			// Crawl options
+			depth: {type: 'string', short: 'd', default: '2'},
+			limit: {type: 'string', short: 'l', default: '20'},
+			outDir: {type: 'string', short: 'O', default: './crawl-output'},
+			sameDomain: {type: 'boolean', default: true},
+			concurrency: {type: 'string', short: 'c', default: '5'},
+			// Common options
+			timeout: {type: 'string', short: 't', default: '30000'},
+			userAgent: {type: 'string', short: 'u', default: 'bun-crawl/0.1.0'},
+			json: {type: 'boolean'},
+		},
+		allowPositionals: true,
+		strict: false,
+	})
+
+	return {command, url, options: values}
+}
+
+function getNumberOption(
+	options: Record<string, unknown>,
+	name: string,
+	defaultValue: number,
+): number {
+	const val = options[name]
+	if (val === undefined || val === '') return defaultValue
+	const parsed = Number(val)
+	return isNaN(parsed) ? defaultValue : parsed
+}
+
+function getBooleanOption(
+	options: Record<string, unknown>,
+	name: string,
+	defaultValue: boolean,
+): boolean {
+	const val = options[name]
+	if (val === undefined) return defaultValue
+	return val === true || val === 'true'
+}
+
+function getStringOption(
+	options: Record<string, unknown>,
+	name: string,
+	defaultValue: string,
+): string {
+	const val = options[name]
+	if (val === undefined || val === '') return defaultValue
+	return String(val)
+}
+
+async function runScrape(
+	url: string,
+	options: Record<string, unknown>,
+): Promise<void> {
+	const timeout = getNumberOption(options, 'timeout', 30000)
+	const userAgent = getStringOption(options, 'userAgent', 'bun-crawl/0.1.0')
+
+	const fetcher = createFetcher({timeout, userAgent})
+	const cleaner = createCleaner()
+	const converter = createConverter()
+	const ai = new NullAIExtractor()
+
+	const {html, url: finalUrl, status} = await fetcher.fetch(url)
+
+	if (status >= 400) {
+		throw new Error(`HTTP ${status}: ${finalUrl}`)
+	}
+
+	const {content, title, metadata} = cleaner.clean(html, finalUrl)
+	let markdown = converter.convert(content)
+	markdown = await ai.extract(markdown, '')
+
+	const result: ScrapeResult = {
+		url: finalUrl,
+		title,
+		markdown,
+		html: content,
+		metadata,
+	}
+
+	const outputFormat = getBooleanOption(options, 'json', false)
+		? 'json'
+		: getStringOption(options, 'format', 'markdown')
+
+	if (outputFormat === 'json') {
+		await writeOutput(
+			JSON.stringify(result, null, 2),
+			getStringOption(options, 'output', ''),
+		)
+	} else {
+		await writeOutput(markdown, getStringOption(options, 'output', ''))
+	}
+}
+
+async function runCrawl(
+	url: string,
+	options: Record<string, unknown>,
+): Promise<void> {
+	const depth = getNumberOption(options, 'depth', 2)
+	const limit = getNumberOption(options, 'limit', 20)
+	const outDir = getStringOption(options, 'outDir', './crawl-output')
+	const sameDomain = getBooleanOption(options, 'sameDomain', true)
+	const concurrency = getNumberOption(options, 'concurrency', 5)
+	const timeout = getNumberOption(options, 'timeout', 30000)
+	const userAgent = getStringOption(options, 'userAgent', 'bun-crawl/0.1.0')
+	const outputJson = getBooleanOption(options, 'json', false)
+
+	const fetcher = createFetcher({timeout, userAgent})
+	const cleaner = createCleaner()
+	const converter = createConverter()
+
+	const crawlOptions: CrawlOptions = {
+		depth,
+		limit,
+		sameDomain,
+		concurrency,
+	}
+
+	const crawler = createCrawler(fetcher, cleaner, converter, crawlOptions)
+
+	console.error(`Starting crawl: ${url}`)
+	console.error(`Depth: ${depth}, Limit: ${limit}, Concurrency: ${concurrency}`)
+
+	const result = await crawler.crawl(url)
+
+	console.error(
+		`Crawl complete: ${result.stats.succeeded} succeeded, ${result.stats.failed} failed`,
+	)
+
+	if (outputJson) {
+		const summary = {
+			stats: result.stats,
+			pages: result.pages.map(p => ({
+				url: p.url,
+				title: p.title,
+			})),
+			errors: result.errors,
+		}
+		await writeOutput(JSON.stringify(summary, null, 2), undefined)
+	} else {
+		for (const page of result.pages) {
+			const urlObj = new URL(page.url)
+			const pathName =
+				urlObj.pathname === '/'
+					? 'index'
+					: urlObj.pathname.replace(/^\//, '').replace(/\//g, '_')
+			const fileName = `${pathName || 'index'}.md`
+			const filePath = join(outDir, fileName)
+
+			await writeOutput(page.markdown, filePath)
+		}
+
+		if (result.errors.length > 0) {
+			console.error('\nErrors:')
+			for (const err of result.errors) {
+				console.error(`  ${err.url}: ${err.error}`)
+			}
+		}
+	}
+}
+
+async function main() {
+	const {command, url, options} = parseCliArgs(process.argv.slice(2))
+
+	try {
+		if (command === 'scrape') {
+			await runScrape(url, options)
+		} else if (command === 'crawl') {
+			await runCrawl(url, options)
+		}
+	} catch (error) {
+		console.error(
+			`Error: ${error instanceof Error ? error.message : String(error)}`,
+		)
+		process.exit(1)
+	}
+}
+
+await main()

package/src/core/cleaner.ts

@@ -0,0 +1,47 @@
+import {Readability} from '@mozilla/readability'
+import {parseHTML} from 'linkedom'
+import type {ScrapeResult} from '../types'
+
+export interface CleanerResult {
+	content: string
+	title: string
+	metadata: Record<string, unknown>
+}
+
+export class Cleaner {
+	clean(html: string, url: string): CleanerResult {
+		const {document} = parseHTML(html, {url})
+		const reader = new Readability(document)
+		const article = reader.parse()
+
+		if (!article) {
+			return {
+				content: '',
+				title: document.title || '',
+				metadata: {},
+			}
+		}
+
+		const metadata: Record<string, unknown> = {}
+		if (article.excerpt) metadata.description = article.excerpt
+		if (article.byline) metadata.author = article.byline
+		if (article.publishedTime) metadata.publishedTime = article.publishedTime
+
+		return {
+			content: article.content ?? '',
+			title: article.title || document.title || '',
+			metadata,
+		}
+	}
+
+	createScrapeResult(url: string, html: string): ScrapeResult {
+		const {content, title, metadata} = this.clean(html, url)
+		return {
+			url,
+			title,
+			markdown: '',
+			html: content,
+			metadata,
+		}
+	}
+}

package/src/core/crawler.ts

@@ -0,0 +1,146 @@
+import {URL} from 'url'
+import type {CrawlOptions, CrawlResult, ScrapeResult} from '../types'
+import type {Cleaner} from './cleaner'
+import type {Fetcher} from './fetcher'
+import type {MarkdownConverter} from './markdown'
+
+export class Crawler {
+	private readonly fetcher: Fetcher
+	private readonly cleaner: Cleaner
+	private readonly converter: MarkdownConverter
+	private readonly options: CrawlOptions
+	private readonly visited: Set<string> = new Set()
+	private readonly queue: Array<{url: string; depth: number}> = []
+
+	constructor(
+		fetcher: Fetcher,
+		cleaner: Cleaner,
+		converter: MarkdownConverter,
+		options: CrawlOptions,
+	) {
+		this.fetcher = fetcher
+		this.cleaner = cleaner
+		this.converter = converter
+		this.options = options
+	}
+
+	async crawl(startUrl: string): Promise<CrawlResult> {
+		const baseUrl = new URL(startUrl)
+		const results: ScrapeResult[] = []
+		const errors: Array<{url: string; error: string}> = []
+
+		this.queue.push({url: startUrl, depth: 0})
+
+		while (this.queue.length > 0 && results.length < this.options.limit) {
+			const batch = this.queue.splice(0, this.options.concurrency)
+			const promises = batch.map(async ({url, depth}) => {
+				if (this.visited.has(url)) return null
+				if (depth > this.options.depth) return null
+
+				this.visited.add(url)
+
+				try {
+					const result = await this.scrapePage(url, baseUrl, depth)
+					if (result) {
+						results.push(result)
+					}
+				} catch (error) {
+					errors.push({
+						url,
+						error: error instanceof Error ? error.message : String(error),
+					})
+				}
+			})
+
+			await Promise.all(promises)
+		}
+
+		return {
+			pages: results,
+			errors,
+			stats: {
+				total: results.length + errors.length,
+				succeeded: results.length,
+				failed: errors.length,
+			},
+		}
+	}
+
+	private async scrapePage(
+		url: string,
+		baseUrl: URL,
+		depth: number,
+	): Promise<ScrapeResult | null> {
+		const {html, url: finalUrl, status} = await this.fetcher.fetch(url)
+
+		if (status >= 400) {
+			throw new Error(`HTTP ${status}: ${finalUrl}`)
+		}
+
+		const {content, title, metadata} = this.cleaner.clean(html, finalUrl)
+		const markdown = this.converter.convert(content)
+
+		const result: ScrapeResult = {
+			url: finalUrl,
+			title,
+			markdown,
+			html: content,
+			metadata,
+		}
+
+		if (depth < this.options.depth) {
+			const links = await this.extractLinks(html, finalUrl)
+			for (const link of links) {
+				if (!this.visited.has(link) && this.shouldCrawl(link, baseUrl)) {
+					this.queue.push({url: link, depth: depth + 1})
+				}
+			}
+		}
+
+		return result
+	}
+
+	private async extractLinks(html: string, baseUrl: string): Promise<string[]> {
+		const links: string[] = []
+
+		const rewriter = new HTMLRewriter().on('a', {
+			element(element) {
+				const href = element.getAttribute('href')
+				if (href) links.push(href)
+			},
+		})
+
+		try {
+			const response = rewriter.transform(new Response(html))
+			await response.text()
+		} catch {
+			// Ignore parsing errors
+		}
+
+		return links
+			.map(href => this.resolveUrl(href, baseUrl))
+			.filter((url): url is string => url !== null)
+	}
+
+	private resolveUrl(href: string, baseUrl: string): string | null {
+		try {
+			const url = new URL(href, baseUrl)
+			url.hash = ''
+			return url.toString()
+		} catch {
+			return null
+		}
+	}
+
+	private shouldCrawl(url: string, baseUrl: URL): boolean {
+		try {
+			const targetUrl = new URL(url)
+			if (this.options.sameDomain && targetUrl.host !== baseUrl.host) {
+				return false
+			}
+			return targetUrl.protocol === 'http:' || targetUrl.protocol === 'https:'
+		} catch {
+			return false
+		}
+	}
+}