@kibhq/core 0.1.0

package/src/ingest/extractors/github.ts

@@ -0,0 +1,135 @@
+import type { ExtractOptions, Extractor, ExtractResult } from "./interface.js";
+
+export function createGithubExtractor(): Extractor {
+	return {
+		type: "github",
+
+		async extract(url: string, options?: ExtractOptions): Promise<ExtractResult> {
+			const parsed = parseGithubUrl(url);
+			if (!parsed) {
+				throw new Error(`Could not parse GitHub URL: ${url}`);
+			}
+
+			const { owner, repo, branch } = parsed;
+			const apiBase = `https://api.github.com/repos/${owner}/${repo}`;
+
+			const headers: Record<string, string> = {
+				Accept: "application/vnd.github.v3+json",
+				"User-Agent": "kib/0.1",
+			};
+
+			// Use GITHUB_TOKEN if available for higher rate limits
+			if (process.env.GITHUB_TOKEN) {
+				headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}`;
+			}
+
+			// Fetch repo metadata
+			const repoResponse = await fetch(apiBase, { headers });
+			if (!repoResponse.ok) {
+				throw new Error(
+					`Failed to fetch repo info: ${repoResponse.status} ${repoResponse.statusText}`,
+				);
+			}
+			const repoData = (await repoResponse.json()) as any;
+
+			// Fetch README
+			let readme = "";
+			try {
+				const readmeResponse = await fetch(`${apiBase}/readme`, {
+					headers: { ...headers, Accept: "application/vnd.github.v3.raw" },
+				});
+				if (readmeResponse.ok) {
+					readme = await readmeResponse.text();
+				}
+			} catch {
+				// No README
+			}
+
+			// Fetch file tree (top level only)
+			let fileTree = "";
+			try {
+				const ref = branch ?? repoData.default_branch ?? "main";
+				const treeResponse = await fetch(`${apiBase}/git/trees/${ref}`, { headers });
+				if (treeResponse.ok) {
+					const treeData = (await treeResponse.json()) as any;
+					const files = (treeData.tree ?? [])
+						.map((f: any) => `${f.type === "tree" ? "📁" : "📄"} ${f.path}`)
+						.slice(0, 50); // Cap at 50 entries
+					fileTree = files.join("\n");
+				}
+			} catch {
+				// No tree
+			}
+
+			const title = options?.title ?? `${owner}/${repo}`;
+			const description = repoData.description ?? "";
+			const stars = repoData.stargazers_count ?? 0;
+			const language = repoData.language ?? "Unknown";
+			const topics = repoData.topics ?? [];
+
+			const sections: string[] = [
+				`# ${title}`,
+				"",
+				description ? `> ${description}` : "",
+				"",
+				`**Language:** ${language} | **Stars:** ${stars.toLocaleString()} | **URL:** ${url}`,
+			];
+
+			if (topics.length > 0) {
+				sections.push(`**Topics:** ${topics.join(", ")}`);
+			}
+
+			if (fileTree) {
+				sections.push("", "## File Structure", "", "```", fileTree, "```");
+			}
+
+			if (readme) {
+				sections.push("", "## README", "", readme);
+			}
+
+			return {
+				title,
+				content: sections.filter((s) => s !== undefined).join("\n"),
+				metadata: {
+					owner,
+					repo,
+					stars,
+					language,
+					topics,
+					url,
+				},
+			};
+		},
+	};
+}
+
+interface ParsedGithubUrl {
+	owner: string;
+	repo: string;
+	branch?: string;
+}
+
+export function parseGithubUrl(url: string): ParsedGithubUrl | null {
+	try {
+		const parsed = new URL(url.trim());
+		if (parsed.hostname !== "github.com" && parsed.hostname !== "www.github.com") {
+			return null;
+		}
+
+		const parts = parsed.pathname.split("/").filter(Boolean);
+		if (parts.length < 2) return null;
+
+		const owner = parts[0]!;
+		const repo = parts[1]!;
+
+		// Check for /tree/branch pattern
+		let branch: string | undefined;
+		if (parts[2] === "tree" && parts[3]) {
+			branch = parts[3];
+		}
+
+		return { owner, repo, branch };
+	} catch {
+		return null;
+	}
+}

package/src/ingest/extractors/interface.ts

@@ -0,0 +1,26 @@
+import type { SourceType } from "../../types.js";
+
+export interface ExtractOptions {
+	/** Override the detected title */
+	title?: string;
+	/** Additional tags to attach */
+	tags?: string[];
+	/** Whether to download images referenced in the content */
+	downloadImages?: boolean;
+}
+
+export interface ExtractResult {
+	/** Extracted/detected title */
+	title: string;
+	/** Cleaned markdown content */
+	content: string;
+	/** Source-specific metadata */
+	metadata: Record<string, unknown>;
+}
+
+export interface Extractor {
+	/** Which source type this extractor handles */
+	type: SourceType;
+	/** Extract content from the given input (URL or file path) */
+	extract(input: string, options?: ExtractOptions): Promise<ExtractResult>;
+}

package/src/ingest/extractors/pdf.ts

@@ -0,0 +1,130 @@
+import { readFile } from "node:fs/promises";
+import { basename, extname } from "node:path";
+import type { ExtractOptions, Extractor, ExtractResult } from "./interface.js";
+
+// Lazy-load pdf-parse (it's heavy)
+let pdfParse: any = null;
+
+async function getPdfParse() {
+	if (!pdfParse) {
+		const mod = await import("pdf-parse");
+		// pdf-parse exports default as the function in some builds
+		pdfParse = mod.default ?? mod;
+	}
+	return pdfParse;
+}
+
+export function createPdfExtractor(): Extractor {
+	return {
+		type: "pdf",
+
+		async extract(input: string, options?: ExtractOptions): Promise<ExtractResult> {
+			const parse = await getPdfParse();
+
+			let buffer: Buffer;
+			if (input.startsWith("http://") || input.startsWith("https://")) {
+				// Fetch PDF from URL
+				const response = await fetch(input, {
+					headers: {
+						"User-Agent": "Mozilla/5.0 (compatible; kib/0.1)",
+					},
+					redirect: "follow",
+				});
+				if (!response.ok) {
+					throw new Error(
+						`Failed to fetch PDF from ${input}: ${response.status} ${response.statusText}`,
+					);
+				}
+				buffer = Buffer.from(await response.arrayBuffer());
+			} else {
+				// Read local PDF file
+				buffer = await readFile(input);
+			}
+
+			const data = await parse(buffer);
+
+			const title =
+				options?.title ??
+				data.info?.Title ??
+				extractTitleFromText(data.text) ??
+				formatFilename(input);
+
+			const author = data.info?.Author ?? undefined;
+			const date = data.info?.CreationDate ? parsePdfDate(data.info.CreationDate) : undefined;
+
+			// Clean up the extracted text into readable markdown
+			const content = formatPdfText(data.text, title);
+
+			return {
+				title,
+				content,
+				metadata: {
+					author,
+					date,
+					pageCount: data.numpages,
+					fileType: ".pdf",
+				},
+			};
+		},
+	};
+}
+
+/**
+ * Try to extract a title from the first few lines of PDF text.
+ * Academic papers often have the title as the first prominent line.
+ */
+function extractTitleFromText(text: string): string | undefined {
+	const lines = text
+		.split("\n")
+		.map((l) => l.trim())
+		.filter(Boolean);
+	// First non-empty line that's between 10-200 chars and doesn't look like metadata
+	for (const line of lines.slice(0, 5)) {
+		if (
+			line.length >= 10 &&
+			line.length <= 200 &&
+			!line.match(/^(abstract|introduction|page|copyright|\d)/i)
+		) {
+			return line;
+		}
+	}
+	return undefined;
+}
+
+function formatFilename(input: string): string {
+	if (input.startsWith("http")) {
+		const url = new URL(input);
+		const parts = url.pathname.split("/");
+		const last = parts[parts.length - 1] ?? "document";
+		return last.replace(/\.pdf$/i, "").replace(/[-_]/g, " ");
+	}
+	const name = basename(input, extname(input));
+	return name.replace(/[-_]/g, " ");
+}
+
+function formatPdfText(text: string, title: string): string {
+	// Split into paragraphs (double newline or more)
+	const cleaned = text
+		// Normalize whitespace
+		.replace(/\r\n/g, "\n")
+		// Remove form feeds and other control chars
+		.replace(/[\f\v]/g, "\n")
+		// Collapse 3+ newlines to 2
+		.replace(/\n{3,}/g, "\n\n")
+		// Remove lines that are just page numbers
+		.replace(/^\s*\d+\s*$/gm, "")
+		.trim();
+
+	return `# ${title}\n\n${cleaned}`;
+}
+
+/**
+ * Parse PDF date format (D:20240315120000+00'00') to ISO date string.
+ */
+function parsePdfDate(dateStr: string): string | undefined {
+	const match = dateStr.match(/D:(\d{4})(\d{2})(\d{2})/);
+	if (match) {
+		return `${match[1]}-${match[2]}-${match[3]}`;
+	}
+	return undefined;
+}

package/src/ingest/extractors/web.test.ts

@@ -0,0 +1,242 @@
+import { describe, expect, test } from "bun:test";
+import { extractFromHtml } from "./web.js";
+
+describe("web extractor", () => {
+	describe("extractFromHtml", () => {
+		test("extracts title from <title> tag", () => {
+			const html = `
+				<html>
+					<head><title>Test Article - Example Blog</title></head>
+					<body><article><p>Content here.</p></article></body>
+				</html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.title).toBe("Test Article");
+		});
+
+		test("prefers og:title over <title>", () => {
+			const html = `
+				<html>
+					<head>
+						<title>Fallback Title</title>
+						<meta property="og:title" content="OG Title">
+					</head>
+					<body><article><p>Some long content that makes this valid content for extraction.</p></article></body>
+				</html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.title).toBe("OG Title");
+		});
+
+		test("uses custom title from options", () => {
+			const html = `<html><head><title>Original</title></head><body><p>Content.</p></body></html>`;
+			const result = extractFromHtml(html, "https://example.com", {
+				title: "Custom Title",
+			});
+			expect(result.title).toBe("Custom Title");
+		});
+
+		test("extracts content from <article> tag", () => {
+			const html = `
+				<html><body>
+					<nav>Navigation stuff that should be removed</nav>
+					<article>
+						<h1>Main Article</h1>
+						<p>This is the important article content that we want to extract from the page.</p>
+					</article>
+					<footer>Footer stuff that should be removed</footer>
+				</body></html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.content).toContain("Main Article");
+			expect(result.content).toContain("important article content");
+			expect(result.content).not.toContain("Navigation stuff");
+			expect(result.content).not.toContain("Footer stuff");
+		});
+
+		test("falls back to <main> when no <article>", () => {
+			const html = `
+				<html><body>
+					<nav>Skip this</nav>
+					<main>
+						<h1>Main Content</h1>
+						<p>This is the main content area with enough text to pass the threshold for extraction.</p>
+					</main>
+				</body></html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.content).toContain("Main Content");
+			expect(result.content).not.toContain("Skip this");
+		});
+
+		test("falls back to body when no semantic elements", () => {
+			const html = `
+				<html><body>
+					<div>
+						<h1>Simple Page</h1>
+						<p>Just a simple page with some content.</p>
+					</div>
+				</body></html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.content).toContain("Simple Page");
+		});
+
+		test("removes script and style tags", () => {
+			const html = `
+				<html><body>
+					<script>alert('xss')</script>
+					<style>.foo { color: red; }</style>
+					<article><p>Clean content that should be extracted without any script or style artifacts.</p></article>
+				</body></html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.content).not.toContain("alert");
+			expect(result.content).not.toContain("color: red");
+			expect(result.content).toContain("Clean content");
+		});
+
+		test("converts HTML headings to markdown", () => {
+			const html = `
+				<html><body><article>
+					<h1>Heading 1</h1>
+					<h2>Heading 2</h2>
+					<p>Paragraph text underneath the headings with enough length to be valid content for extraction.</p>
+				</article></body></html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.content).toContain("# Heading 1");
+			expect(result.content).toContain("## Heading 2");
+		});
+
+		test("converts HTML lists to markdown", () => {
+			const html = `
+				<html><body><article>
+					<ul>
+						<li>Item one</li>
+						<li>Item two</li>
+						<li>Item three</li>
+					</ul>
+					<p>Some additional content to make this a valid extraction with enough length.</p>
+				</article></body></html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.content).toContain("Item one");
+			expect(result.content).toContain("Item two");
+			// Turndown uses "-" bullet markers
+			expect(result.content).toMatch(/-\s+Item one/);
+		});
+
+		test("converts code blocks to fenced markdown", () => {
+			const html = `
+				<html><body><article>
+					<p>Here is some code that demonstrates the concept with enough context for extraction:</p>
+					<pre><code>const x = 1;
+const y = 2;</code></pre>
+				</article></body></html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.content).toContain("```");
+			expect(result.content).toContain("const x = 1;");
+		});
+
+		test("replaces images with alt text description", () => {
+			const html = `
+				<html><body><article>
+					<p>Content about diagrams and explanations with enough text for extraction threshold.</p>
+					<img src="diagram.png" alt="Architecture diagram showing the system layout">
+				</article></body></html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.content).toContain("[Image: Architecture diagram");
+			expect(result.content).not.toContain("diagram.png");
+		});
+
+		test("extracts author metadata", () => {
+			const html = `
+				<html>
+					<head><meta name="author" content="Jane Doe"></head>
+					<body><article><p>Article content with enough words for extraction threshold to be met.</p></article></body>
+				</html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.metadata.author).toBe("Jane Doe");
+		});
+
+		test("extracts date metadata", () => {
+			const html = `
+				<html>
+					<head><meta property="article:published_time" content="2024-03-15"></head>
+					<body><article><p>Article content with enough words for extraction.</p></article></body>
+				</html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.metadata.date).toBe("2024-03-15");
+		});
+
+		test("extracts date from time element", () => {
+			const html = `
+				<html><body><article>
+					<time datetime="2024-06-01">June 1, 2024</time>
+					<p>Article content with enough words for extraction threshold to be met by the extractor.</p>
+				</article></body></html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.metadata.date).toBe("2024-06-01");
+		});
+
+		test("preserves URL in metadata", () => {
+			const result = extractFromHtml(
+				"<html><body><p>Content.</p></body></html>",
+				"https://example.com/article",
+			);
+			expect(result.metadata.url).toBe("https://example.com/article");
+		});
+
+		test("strips common title suffixes", () => {
+			const html = `<html><head><title>Article Title | My Blog</title></head><body><p>C</p></body></html>`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.title).toBe("Article Title");
+		});
+
+		test("strips title suffix with dash separator", () => {
+			const html = `<html><head><title>Great Post - The Newsletter</title></head><body><p>C</p></body></html>`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.title).toBe("Great Post");
+		});
+
+		test("handles title that is only a suffix pattern", () => {
+			const html = `<html><head><title></title></head><body><p>Content.</p></body></html>`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.title).toBe("Untitled");
+		});
+
+		test("removes ad-related elements", () => {
+			const html = `
+				<html><body>
+					<article>
+						<p>Real content that we want to keep in the extraction with enough words to matter.</p>
+						<div class="ad">Buy our product!</div>
+						<div class="advertisement">Sponsor content</div>
+					</article>
+				</body></html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.content).toContain("Real content");
+			expect(result.content).not.toContain("Buy our product");
+			expect(result.content).not.toContain("Sponsor content");
+		});
+
+		test("removes cookie banners", () => {
+			const html = `
+				<html><body>
+					<div class="cookie-banner">Accept cookies?</div>
+					<article><p>The actual interesting content of the page that we want to extract properly.</p></article>
+				</body></html>
+			`;
+			const result = extractFromHtml(html, "https://example.com");
+			expect(result.content).not.toContain("cookie");
+			expect(result.content).toContain("actual interesting content");
+		});
+	});
+});

package/src/ingest/extractors/web.ts

@@ -0,0 +1,163 @@
+import * as cheerio from "cheerio";
+import TurndownService from "turndown";
+import type { ExtractOptions, Extractor, ExtractResult } from "./interface.js";
+
+const REMOVE_SELECTORS = [
+	"script",
+	"style",
+	"noscript",
+	"iframe",
+	"nav",
+	"footer",
+	"header:not(article header)",
+	"aside",
+	".sidebar",
+	".nav",
+	".navigation",
+	".menu",
+	".footer",
+	".header",
+	".ad",
+	".ads",
+	".advertisement",
+	".cookie-banner",
+	".popup",
+	".modal",
+	".comments",
+	".comment-section",
+	"[role='navigation']",
+	"[role='banner']",
+	"[role='contentinfo']",
+	"[role='complementary']",
+].join(", ");
+
+const CONTENT_SELECTORS = [
+	"article",
+	"[role='main']",
+	"main",
+	".post-content",
+	".article-content",
+	".entry-content",
+	".content",
+	"#content",
+	".post",
+	".article",
+	".blog-post",
+];
+
+export function createWebExtractor(): Extractor {
+	return {
+		type: "web",
+
+		async extract(url: string, options?: ExtractOptions): Promise<ExtractResult> {
+			const response = await fetch(url, {
+				headers: {
+					"User-Agent": "Mozilla/5.0 (compatible; kib/0.1; +https://github.com/kib-cli/kib)",
+					Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+				},
+				redirect: "follow",
+			});
+
+			if (!response.ok) {
+				throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+			}
+
+			const html = await response.text();
+			return extractFromHtml(html, url, options);
+		},
+	};
+}
+
+/**
+ * Extract content from raw HTML. Exported for testing without network.
+ */
+export function extractFromHtml(
+	html: string,
+	url: string,
+	options?: ExtractOptions,
+): ExtractResult {
+	const $ = cheerio.load(html);
+
+	// Extract metadata before removing elements
+	const title =
+		options?.title ??
+		$('meta[property="og:title"]').attr("content") ??
+		$("title").first().text().trim() ??
+		$("h1").first().text().trim() ??
+		"Untitled";
+
+	const author =
+		$('meta[name="author"]').attr("content") ??
+		$('meta[property="article:author"]').attr("content") ??
+		($('[rel="author"]').first().text().trim() || undefined);
+
+	const date =
+		$('meta[property="article:published_time"]').attr("content") ??
+		$("time[datetime]").first().attr("datetime") ??
+		$('meta[name="date"]').attr("content") ??
+		undefined;
+
+	const description =
+		$('meta[property="og:description"]').attr("content") ??
+		$('meta[name="description"]').attr("content") ??
+		undefined;
+
+	// Remove unwanted elements
+	$(REMOVE_SELECTORS).remove();
+
+	// Find main content
+	let contentHtml = "";
+	for (const selector of CONTENT_SELECTORS) {
+		const el = $(selector).first();
+		if (el.length && el.text().trim().length > 100) {
+			contentHtml = el.html() ?? "";
+			break;
+		}
+	}
+
+	// Fallback to body
+	if (!contentHtml) {
+		contentHtml = $("body").html() ?? $.html();
+	}
+
+	// Convert to markdown
+	const turndown = new TurndownService({
+		headingStyle: "atx",
+		codeBlockStyle: "fenced",
+		bulletListMarker: "-",
+	});
+
+	// Remove image tags by default (they'd be broken links)
+	turndown.addRule("images", {
+		filter: "img",
+		replacement: (_content, node) => {
+			const alt = (node as HTMLElement).getAttribute("alt");
+			if (alt && alt.length > 5) {
+				return `[Image: ${alt}]`;
+			}
+			return "";
+		},
+	});
+
+	const markdown = turndown.turndown(contentHtml);
+
+	return {
+		title: cleanTitle(title),
+		content: markdown,
+		metadata: {
+			author,
+			date,
+			description,
+			url,
+		},
+	};
+}
+
+function cleanTitle(title: string): string {
+	return (
+		title
+			// Remove common suffixes like "| Site Name" or "- Blog Name"
+			.replace(/\s*[|–—-]\s*[^|–—-]+$/, "")
+			.trim() || "Untitled"
+	);
+}