@involvex/bun-crawl 0.0.0-stage → 0.1.0

package/src/core/fetcher.ts

@@ -0,0 +1,64 @@
+import type {FetcherOptions} from '../types'
+
+export class Fetcher {
+	private readonly defaultOptions: Required<FetcherOptions>
+
+	constructor(options: FetcherOptions = {}) {
+		this.defaultOptions = {
+			timeout: options.timeout ?? 30000,
+			userAgent: options.userAgent ?? 'bun-crawl/0.1.0',
+			headers: options.headers ?? {},
+			followRedirects: options.followRedirects ?? true,
+		}
+	}
+
+	async fetch(
+		url: string,
+	): Promise<{html: string; url: string; status: number}> {
+		const controller = new AbortController()
+		const timeoutId = setTimeout(
+			() => controller.abort(),
+			this.defaultOptions.timeout,
+		)
+
+		try {
+			const response = await fetch(url, {
+				method: 'GET',
+				headers: {
+					'User-Agent': this.defaultOptions.userAgent,
+					Accept:
+						'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+					'Accept-Language': 'en-US,en;q=0.5',
+					...this.defaultOptions.headers,
+				},
+				redirect: this.defaultOptions.followRedirects ? 'follow' : 'manual',
+				signal: controller.signal,
+			})
+
+			clearTimeout(timeoutId)
+
+			const html = await response.text()
+
+			return {
+				html,
+				url: response.url,
+				status: response.status,
+			}
+		} catch (error) {
+			clearTimeout(timeoutId)
+
+			if (error instanceof Error && error.name === 'AbortError') {
+				throw new Error(
+					`Request timeout after ${this.defaultOptions.timeout}ms: ${url}`,
+					{cause: error},
+				)
+			}
+
+			const caughtError =
+				error instanceof Error ? error : new Error(String(error))
+			throw new Error(`Failed to fetch ${url}: ${caughtError.message}`, {
+				cause: caughtError,
+			})
+		}
+	}
+}

package/src/core/markdown.ts

@@ -0,0 +1,108 @@
+import TurndownService from 'turndown'
+
+export interface MarkdownOptions {
+	headingStyle?: 'setext' | 'atx'
+	codeBlockStyle?: 'fenced' | 'indented'
+	bulletListMarker?: '-' | '*' | '+'
+	emDelimiter?: '_' | '*'
+	strongDelimiter?: '__' | '**'
+	linkStyle?: 'inlined' | 'referenced'
+	linkReferenceStyle?: 'full' | 'collapsed' | 'shortcut'
+}
+
+export class MarkdownConverter {
+	private readonly turndown: TurndownService
+
+	constructor(options: MarkdownOptions = {}) {
+		this.turndown = new TurndownService({
+			headingStyle: options.headingStyle ?? 'atx',
+			codeBlockStyle: options.codeBlockStyle ?? 'fenced',
+			bulletListMarker: options.bulletListMarker ?? '-',
+			emDelimiter: options.emDelimiter ?? '_',
+			strongDelimiter: options.strongDelimiter ?? '**',
+			linkStyle: options.linkStyle ?? 'inlined',
+			linkReferenceStyle: options.linkReferenceStyle ?? 'full',
+		})
+
+		this.configureRules()
+	}
+
+	private configureRules(): void {
+		this.turndown.addRule('strikeThrough', {
+			filter: ['del', 's'],
+			replacement: content => `~~${content}~~`,
+		})
+
+		this.turndown.addRule('underline', {
+			filter: 'u',
+			replacement: content => `<u>${content}</u>`,
+		})
+
+		this.turndown.addRule('subscript', {
+			filter: 'sub',
+			replacement: content => `~${content}~`,
+		})
+
+		this.turndown.addRule('superscript', {
+			filter: 'sup',
+			replacement: content => `^${content}^`,
+		})
+
+		this.turndown.addRule('details', {
+			filter: 'details',
+			replacement: (content, node) => {
+				const summary = node.querySelector('summary')?.textContent || 'Details'
+				return `\n<details>\n<summary>${summary}</summary>\n${content}\n</details>\n`
+			},
+		})
+
+		this.turndown.addRule('taskListItems', {
+			filter: node =>
+				node.nodeName === 'INPUT' && node.getAttribute('type') === 'checkbox',
+			replacement: (_, node) => {
+				const checked = node.hasAttribute('checked') ? 'x' : ' '
+				return `- [${checked}] `
+			},
+		})
+
+		this.turndown.addRule('table', {
+			filter: 'table',
+			replacement: (content, _node) => {
+				return `\n${content}\n`
+			},
+		})
+
+		this.turndown.addRule('tableRow', {
+			filter: 'tr',
+			replacement: (content, node) => {
+				const isHeader = node.parentElement?.nodeName === 'THEAD'
+				const cells = node.querySelectorAll('th, td')
+				const cellContents = Array.from(cells).map(
+					(cell: Element) =>
+						cell.textContent?.replace(/\|/g, '\\|').trim() || '',
+				)
+				const separator = isHeader
+					? cellContents.map(() => '---').join('|')
+					: ''
+				return `| ${cellContents.join(' | ')} |\n${separator ? `| ${separator} |\n` : ''}`
+			},
+		})
+
+		this.turndown.addRule('pre', {
+			filter: 'pre',
+			replacement: (content, node) => {
+				const code = node.querySelector('code')
+				const language = code?.className?.match(/language-(\w+)/)?.[1] || ''
+				return `\n\`\`\`${language}\n${content}\n\`\`\`\n`
+			},
+		})
+	}
+
+	convert(html: string): string {
+		return this.turndown.turndown(html).trim()
+	}
+
+	getTurndownService(): TurndownService {
+		return this.turndown
+	}
+}

package/src/index.ts

@@ -0,0 +1,13 @@
+export type {AIExtractor, AIExtractorConfig} from './ai/interface'
+export {NullAIExtractor} from './ai/null-ai'
+export {ZenAIExtractor} from './ai/zen-ai'
+export {Cleaner, type CleanerResult} from './core/cleaner'
+export {Crawler} from './core/crawler'
+export {Fetcher} from './core/fetcher'
+export {MarkdownConverter, type MarkdownOptions} from './core/markdown'
+export type {
+	CrawlOptions,
+	CrawlResult,
+	FetcherOptions,
+	ScrapeResult,
+} from './types'

package/src/types.ts

@@ -0,0 +1,54 @@
+export interface ScrapeResult {
+	url: string
+	title: string
+	markdown: string
+	html: string
+	metadata: {
+		description?: string
+		author?: string
+		publishedTime?: string
+		[key: string]: unknown
+	}
+}
+
+export interface CrawlOptions {
+	depth: number
+	limit: number
+	sameDomain: boolean
+	concurrency: number
+}
+
+export interface CrawlResult {
+	pages: ScrapeResult[]
+	errors: Array<{url: string; error: string}>
+	stats: {
+		total: number
+		succeeded: number
+		failed: number
+	}
+}
+
+export interface FetcherOptions {
+	timeout?: number
+	userAgent?: string
+	headers?: Record<string, string>
+	followRedirects?: boolean
+}
+
+export interface AIExtractorConfig {
+	enabled?: boolean
+}
+
+export interface CliOptions {
+	[key: string]: string | boolean | undefined
+	output?: string | boolean
+	format?: string | boolean
+	depth?: string | boolean
+	limit?: string | boolean
+	outDir?: string | boolean
+	sameDomain?: boolean
+	concurrency?: string | boolean
+	timeout?: string | boolean
+	userAgent?: string | boolean
+	json?: boolean
+}

package/tsconfig.json

@@ -0,0 +1,22 @@
+{
+	"compilerOptions": {
+		"target": "ES2022",
+		"module": "ESNext",
+		"moduleResolution": "bundler",
+		"lib": ["ES2022", "DOM"],
+		"strict": true,
+		"esModuleInterop": true,
+		"skipLibCheck": true,
+		"forceConsistentCasingInFileNames": true,
+		"resolveJsonModule": true,
+		"declaration": true,
+		"outDir": "./dist",
+		"noEmit": true,
+		"allowImportingTsExtensions": true,
+		"verbatimModuleSyntax": true,
+		"moduleDetection": "force",
+		"types": ["bun"]
+	},
+	"include": ["src/**/*"],
+	"exclude": ["node_modules", "dist"]
+}