@kibhq/core 0.1.0

package/src/ingest/extractors/youtube.test.ts

@@ -0,0 +1,44 @@
+import { describe, expect, test } from "bun:test";
+import { extractVideoId } from "./youtube.js";
+
+describe("youtube extractor", () => {
+	describe("extractVideoId", () => {
+		test("extracts from standard watch URL", () => {
+			expect(extractVideoId("https://www.youtube.com/watch?v=dQw4w9WgXcQ")).toBe("dQw4w9WgXcQ");
+		});
+
+		test("extracts from short URL", () => {
+			expect(extractVideoId("https://youtu.be/dQw4w9WgXcQ")).toBe("dQw4w9WgXcQ");
+		});
+
+		test("extracts from embed URL", () => {
+			expect(extractVideoId("https://www.youtube.com/embed/dQw4w9WgXcQ")).toBe("dQw4w9WgXcQ");
+		});
+
+		test("extracts from URL with extra params", () => {
+			expect(extractVideoId("https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=120")).toBe(
+				"dQw4w9WgXcQ",
+			);
+		});
+
+		test("extracts from URL without www", () => {
+			expect(extractVideoId("https://youtube.com/watch?v=dQw4w9WgXcQ")).toBe("dQw4w9WgXcQ");
+		});
+
+		test("extracts from mobile URL", () => {
+			expect(extractVideoId("https://m.youtube.com/watch?v=dQw4w9WgXcQ")).toBe("dQw4w9WgXcQ");
+		});
+
+		test("returns null for invalid URL", () => {
+			expect(extractVideoId("https://example.com")).toBeNull();
+		});
+
+		test("returns null for YouTube URL without video ID", () => {
+			expect(extractVideoId("https://www.youtube.com/channel/UCxyz")).toBeNull();
+		});
+
+		test("handles whitespace", () => {
+			expect(extractVideoId("  https://youtu.be/dQw4w9WgXcQ  ")).toBe("dQw4w9WgXcQ");
+		});
+	});
+});

package/src/ingest/extractors/youtube.ts

@@ -0,0 +1,166 @@
+import type { ExtractOptions, Extractor, ExtractResult } from "./interface.js";
+
+export function createYoutubeExtractor(): Extractor {
+	return {
+		type: "youtube",
+
+		async extract(url: string, options?: ExtractOptions): Promise<ExtractResult> {
+			const videoId = extractVideoId(url);
+			if (!videoId) {
+				throw new Error(`Could not extract video ID from URL: ${url}`);
+			}
+
+			// Fetch video page to get title and metadata
+			const pageData = await fetchVideoPage(videoId);
+
+			// Attempt to fetch transcript
+			let transcript: string | null = null;
+			try {
+				transcript = await fetchTranscript(videoId);
+			} catch {
+				// Transcript not available — fall back to description
+			}
+
+			const title = options?.title ?? pageData.title ?? `YouTube Video ${videoId}`;
+
+			let content: string;
+			if (transcript) {
+				content = `# ${title}\n\n**Source:** https://www.youtube.com/watch?v=${videoId}\n\n## Transcript\n\n${transcript}`;
+			} else if (pageData.description) {
+				content = `# ${title}\n\n**Source:** https://www.youtube.com/watch?v=${videoId}\n\n## Description\n\n${pageData.description}\n\n*Note: Transcript was not available for this video.*`;
+			} else {
+				content = `# ${title}\n\n**Source:** https://www.youtube.com/watch?v=${videoId}\n\n*No transcript or description available.*`;
+			}
+
+			return {
+				title,
+				content,
+				metadata: {
+					videoId,
+					channelName: pageData.channelName,
+					url: `https://www.youtube.com/watch?v=${videoId}`,
+					hasTranscript: transcript !== null,
+				},
+			};
+		},
+	};
+}
+
+export function extractVideoId(url: string): string | null {
+	const trimmed = url.trim();
+
+	// youtu.be/VIDEO_ID
+	const shortMatch = trimmed.match(/youtu\.be\/([a-zA-Z0-9_-]{11})/);
+	if (shortMatch) return shortMatch[1]!;
+
+	// youtube.com/watch?v=VIDEO_ID
+	try {
+		const parsed = new URL(trimmed);
+		const v = parsed.searchParams.get("v");
+		if (v && v.length === 11) return v;
+	} catch {
+		// Not a valid URL
+	}
+
+	// youtube.com/embed/VIDEO_ID
+	const embedMatch = trimmed.match(/youtube\.com\/embed\/([a-zA-Z0-9_-]{11})/);
+	if (embedMatch) return embedMatch[1]!;
+
+	return null;
+}
+
+interface VideoPageData {
+	title: string | null;
+	description: string | null;
+	channelName: string | null;
+}
+
+async function fetchVideoPage(videoId: string): Promise<VideoPageData> {
+	// Use oembed API — no auth needed, returns JSON
+	try {
+		const response = await fetch(
+			`https://www.youtube.com/oembed?url=https://www.youtube.com/watch?v=${videoId}&format=json`,
+		);
+		if (response.ok) {
+			const data = (await response.json()) as any;
+			return {
+				title: data.title ?? null,
+				description: null, // oembed doesn't include description
+				channelName: data.author_name ?? null,
+			};
+		}
+	} catch {
+		// Fallback
+	}
+
+	return { title: null, description: null, channelName: null };
+}
+
+async function fetchTranscript(videoId: string): Promise<string> {
+	// Fetch the video page to get the captions track URL
+	const response = await fetch(`https://www.youtube.com/watch?v=${videoId}`, {
+		headers: {
+			"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+			"Accept-Language": "en-US,en;q=0.9",
+		},
+	});
+
+	if (!response.ok) {
+		throw new Error(`Failed to fetch video page: ${response.status}`);
+	}
+
+	const html = await response.text();
+
+	// Extract captions data from the page
+	const captionMatch = html.match(/"captionTracks":\[(\{.*?\})\]/);
+	if (!captionMatch) {
+		throw new Error("No captions available");
+	}
+
+	// Parse the first caption track URL
+	const trackData = JSON.parse(`[${captionMatch[1]}]`);
+	const track = trackData[0];
+	if (!track?.baseUrl) {
+		throw new Error("No caption track URL found");
+	}
+
+	// Fetch the transcript XML
+	const transcriptResponse = await fetch(track.baseUrl);
+	if (!transcriptResponse.ok) {
+		throw new Error("Failed to fetch transcript");
+	}
+
+	const xml = await transcriptResponse.text();
+
+	// Parse XML transcript into plain text
+	return parseTranscriptXml(xml);
+}
+
+function parseTranscriptXml(xml: string): string {
+	const lines: string[] = [];
+	const textRegex = /<text[^>]*>([\s\S]*?)<\/text>/g;
+	let match: RegExpExecArray | null;
+
+	while ((match = textRegex.exec(xml)) !== null) {
+		const text = match[1]!
+			.replace(/&amp;/g, "&")
+			.replace(/&lt;/g, "<")
+			.replace(/&gt;/g, ">")
+			.replace(/&quot;/g, '"')
+			.replace(/&#39;/g, "'")
+			.replace(/<[^>]+>/g, "") // strip any HTML tags
+			.trim();
+
+		if (text) {
+			lines.push(text);
+		}
+	}
+
+	// Join into paragraphs — group ~5 lines together
+	const paragraphs: string[] = [];
+	for (let i = 0; i < lines.length; i += 5) {
+		paragraphs.push(lines.slice(i, i + 5).join(" "));
+	}
+
+	return paragraphs.join("\n\n");
+}

package/src/ingest/ingest.test.ts

@@ -0,0 +1,187 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { existsSync } from "node:fs";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { initVault, loadManifest } from "../vault.js";
+import { ingestSource } from "./ingest.js";
+
+let tempDir: string;
+
+afterEach(async () => {
+	if (tempDir) await rm(tempDir, { recursive: true, force: true });
+});
+
+async function makeTempVault() {
+	tempDir = await mkdtemp(join(tmpdir(), "kib-ingest-test-"));
+	await initVault(tempDir, { name: "test" });
+	return tempDir;
+}
+
+describe("ingestSource", () => {
+	test("ingests a local markdown file", async () => {
+		const root = await makeTempVault();
+
+		// Create a test file
+		const testFile = join(root, "test-article.md");
+		await writeFile(testFile, "# Test Article\n\nThis is some test content for ingestion.");
+
+		const result = await ingestSource(root, testFile);
+
+		expect(result.skipped).toBe(false);
+		expect(result.sourceType).toBe("file");
+		expect(result.title).toBe("Test Article");
+		expect(result.wordCount).toBeGreaterThan(0);
+		expect(result.path).toMatch(/^raw\/articles\//);
+
+		// Verify file was written
+		const rawPath = join(root, result.path);
+		expect(existsSync(rawPath)).toBe(true);
+
+		// Verify manifest was updated
+		const manifest = await loadManifest(root);
+		expect(manifest.stats.totalSources).toBe(1);
+		expect(Object.keys(manifest.sources)).toHaveLength(1);
+
+		const source = Object.values(manifest.sources)[0]!;
+		expect(source.sourceType).toBe("file");
+		expect(source.lastCompiled).toBeNull();
+		expect(source.metadata.title).toBe("Test Article");
+	});
+
+	test("ingests a local text file", async () => {
+		const root = await makeTempVault();
+
+		const testFile = join(root, "notes.txt");
+		await writeFile(testFile, "Some plain text notes about a topic.");
+
+		const result = await ingestSource(root, testFile);
+
+		expect(result.skipped).toBe(false);
+		expect(result.sourceType).toBe("file");
+	});
+
+	test("deduplicates identical content", async () => {
+		const root = await makeTempVault();
+
+		const testFile = join(root, "article.md");
+		await writeFile(testFile, "# Unique Content\n\nThis exact content should only appear once.");
+
+		const result1 = await ingestSource(root, testFile);
+		expect(result1.skipped).toBe(false);
+
+		const result2 = await ingestSource(root, testFile);
+		expect(result2.skipped).toBe(true);
+		expect(result2.skipReason).toContain("Duplicate");
+
+		// Manifest should still have only 1 source
+		const manifest = await loadManifest(root);
+		expect(manifest.stats.totalSources).toBe(1);
+	});
+
+	test("allows different content even from same path", async () => {
+		const root = await makeTempVault();
+
+		const testFile = join(root, "article.md");
+		await writeFile(testFile, "# Version 1\n\nOriginal content.");
+		const result1 = await ingestSource(root, testFile);
+		expect(result1.skipped).toBe(false);
+
+		await writeFile(testFile, "# Version 2\n\nUpdated content that is different.");
+		const result2 = await ingestSource(root, testFile);
+		expect(result2.skipped).toBe(false);
+
+		const manifest = await loadManifest(root);
+		expect(manifest.stats.totalSources).toBe(2);
+	});
+
+	test("uses custom category when specified", async () => {
+		const root = await makeTempVault();
+
+		const testFile = join(root, "notes.md");
+		await writeFile(testFile, "# Notes\n\nContent.");
+
+		const result = await ingestSource(root, testFile, { category: "papers" });
+		expect(result.path).toMatch(/^raw\/papers\//);
+	});
+
+	test("uses custom title when specified", async () => {
+		const root = await makeTempVault();
+
+		const testFile = join(root, "data.md");
+		await writeFile(testFile, "Some data.");
+
+		const result = await ingestSource(root, testFile, { title: "My Custom Title" });
+		expect(result.title).toBe("My Custom Title");
+	});
+
+	test("routes PDF files to papers category", async () => {
+		const root = await makeTempVault();
+
+		// We can't easily test actual PDF extraction without a real PDF,
+		// but we can verify the source type detection routes correctly
+		// by using a .md file with forced sourceType
+		const testFile = join(root, "test.md");
+		await writeFile(testFile, "# PDF Content\n\nExtracted from a PDF.");
+
+		const result = await ingestSource(root, testFile, { sourceType: "file", category: "papers" });
+		expect(result.path).toMatch(/^raw\/papers\//);
+	});
+
+	test("ingests multiple sources and tracks them all", async () => {
+		const root = await makeTempVault();
+
+		const file1 = join(root, "first.md");
+		const file2 = join(root, "second.md");
+		const file3 = join(root, "third.md");
+		await writeFile(file1, "# First Article\n\nContent one.");
+		await writeFile(file2, "# Second Article\n\nContent two.");
+		await writeFile(file3, "# Third Article\n\nContent three.");
+
+		await ingestSource(root, file1);
+		await ingestSource(root, file2);
+		await ingestSource(root, file3);
+
+		const manifest = await loadManifest(root);
+		expect(manifest.stats.totalSources).toBe(3);
+		expect(Object.keys(manifest.sources)).toHaveLength(3);
+
+		// All sources should be pending compilation
+		const pending = Object.values(manifest.sources).filter((s) => s.lastCompiled === null);
+		expect(pending).toHaveLength(3);
+	});
+
+	test("ingests code files wrapped in code blocks", async () => {
+		const root = await makeTempVault();
+
+		const testFile = join(root, "example.ts");
+		await writeFile(testFile, "const greeting = 'hello world';\nconsole.log(greeting);");
+
+		const result = await ingestSource(root, testFile);
+
+		expect(result.skipped).toBe(false);
+		expect(result.sourceType).toBe("file");
+
+		// Read the raw file and verify it contains code block
+		const { readRaw } = await import("../vault.js");
+		// The path is raw/articles/example.md, we need to strip "raw/" prefix
+		const rawContent = await readRaw(root, result.path.replace(/^raw\//, ""));
+		expect(rawContent).toContain("```typescript");
+	});
+
+	test("normalized content includes frontmatter", async () => {
+		const root = await makeTempVault();
+
+		const testFile = join(root, "article.md");
+		await writeFile(testFile, "# My Great Article\n\nAmazing content here.");
+
+		const result = await ingestSource(root, testFile);
+
+		const { readRaw } = await import("../vault.js");
+		const rawContent = await readRaw(root, result.path.replace(/^raw\//, ""));
+		expect(rawContent).toContain("---");
+		expect(rawContent).toContain('title: "My Great Article"');
+		expect(rawContent).toContain("source_type: file");
+		expect(rawContent).toContain("word_count:");
+	});
+});

package/src/ingest/ingest.ts

@@ -0,0 +1,179 @@
+import { basename, extname, resolve } from "node:path";
+import { hash } from "../hash.js";
+import type { IngestResult, Manifest, SourceEntry, SourceType } from "../types.js";
+import { loadManifest, saveManifest, writeRaw } from "../vault.js";
+import type { Extractor } from "./extractors/interface.js";
+import { countWords, normalizeSource, slugify } from "./normalize.js";
+import { detectSourceType } from "./router.js";
+
+interface IngestOptions {
+	/** Override the detected source type */
+	sourceType?: SourceType;
+	/** Override category (raw/ subdirectory) */
+	category?: string;
+	/** Additional tags */
+	tags?: string[];
+	/** Custom title */
+	title?: string;
+}
+
+/**
+ * Ingest a single source into the vault.
+ *
+ * 1. Detect source type
+ * 2. Route to the correct extractor
+ * 3. Extract content
+ * 4. Hash content (dedup check)
+ * 5. Normalize with frontmatter
+ * 6. Write to raw/
+ * 7. Update manifest
+ */
+export async function ingestSource(
+	root: string,
+	uri: string,
+	options: IngestOptions = {},
+): Promise<IngestResult> {
+	const sourceType = options.sourceType ?? detectSourceType(uri);
+
+	// Get the extractor for this source type
+	const extractor = await getExtractor(sourceType);
+
+	// Extract content
+	const extracted = await extractor.extract(uri, { title: options.title, tags: options.tags });
+
+	// Hash the extracted content for dedup
+	const contentHash = await hash(extracted.content);
+
+	// Load manifest and check for duplicates
+	const manifest = await loadManifest(root);
+
+	// Check if we already have this exact content
+	const existingSource = findExistingSource(manifest, uri, contentHash);
+	if (existingSource) {
+		return {
+			sourceId: existingSource.id,
+			path: existingSource.path,
+			sourceType,
+			title: extracted.title,
+			wordCount: countWords(extracted.content),
+			skipped: true,
+			skipReason: "Duplicate content (same hash already ingested)",
+		};
+	}
+
+	// Normalize content with frontmatter
+	const normalizedContent = normalizeSource({
+		title: extracted.title,
+		content: extracted.content,
+		sourceType,
+		originalUrl: isUrl(uri) ? uri : undefined,
+		metadata: extracted.metadata,
+	});
+
+	// Determine file path within raw/
+	const category = options.category ?? categoryForType(sourceType);
+	const slug = slugify(extracted.title);
+	const relativePath = `${category}/${slug}.md`;
+
+	// Write to raw/
+	await writeRaw(root, relativePath, normalizedContent);
+
+	// Generate a source ID
+	const sourceId = `src_${contentHash.slice(0, 12)}`;
+
+	// Update manifest
+	const now = new Date().toISOString();
+	const wordCount = countWords(extracted.content);
+
+	const sourceEntry: SourceEntry = {
+		hash: contentHash,
+		ingestedAt: now,
+		lastCompiled: null,
+		sourceType,
+		originalUrl: isUrl(uri) ? uri : undefined,
+		producedArticles: [],
+		metadata: {
+			title: extracted.title,
+			author: extracted.metadata.author as string | undefined,
+			date: extracted.metadata.date as string | undefined,
+			wordCount,
+		},
+	};
+
+	manifest.sources[sourceId] = sourceEntry;
+	manifest.stats.totalSources = Object.keys(manifest.sources).length;
+
+	await saveManifest(root, manifest);
+
+	return {
+		sourceId,
+		path: `raw/${relativePath}`,
+		sourceType,
+		title: extracted.title,
+		wordCount,
+		skipped: false,
+	};
+}
+
+async function getExtractor(sourceType: SourceType): Promise<Extractor> {
+	switch (sourceType) {
+		case "web": {
+			const { createWebExtractor } = await import("./extractors/web.js");
+			return createWebExtractor();
+		}
+		case "pdf": {
+			const { createPdfExtractor } = await import("./extractors/pdf.js");
+			return createPdfExtractor();
+		}
+		case "youtube": {
+			const { createYoutubeExtractor } = await import("./extractors/youtube.js");
+			return createYoutubeExtractor();
+		}
+		case "github": {
+			const { createGithubExtractor } = await import("./extractors/github.js");
+			return createGithubExtractor();
+		}
+		case "file": {
+			const { createFileExtractor } = await import("./extractors/file.js");
+			return createFileExtractor();
+		}
+		case "image":
+			throw new Error("Image extraction requires vision model support (coming soon)");
+		default:
+			throw new Error(`Unsupported source type: ${sourceType}`);
+	}
+}
+
+function categoryForType(sourceType: SourceType): string {
+	switch (sourceType) {
+		case "pdf":
+			return "papers";
+		case "youtube":
+			return "transcripts";
+		case "github":
+			return "repos";
+		case "image":
+			return "images";
+		default:
+			return "articles";
+	}
+}
+
+function findExistingSource(
+	manifest: Manifest,
+	uri: string,
+	contentHash: string,
+): { id: string; path: string } | null {
+	for (const [id, source] of Object.entries(manifest.sources)) {
+		// Same content hash = same content regardless of URL
+		if (source.hash === contentHash) {
+			return { id, path: source.producedArticles[0] ?? "" };
+		}
+		// Same URL but different hash = content changed, allow re-ingest
+	}
+	return null;
+}
+
+function isUrl(str: string): boolean {
+	return str.startsWith("http://") || str.startsWith("https://");
+}

package/src/ingest/normalize.test.ts

@@ -0,0 +1,120 @@
+import { describe, expect, test } from "bun:test";
+import { countWords, normalizeSource, slugify } from "./normalize.js";
+
+describe("slugify", () => {
+	test("converts title to kebab-case", () => {
+		expect(slugify("Transformer Architecture")).toBe("transformer-architecture");
+	});
+
+	test("strips special characters", () => {
+		expect(slugify("What's New in React 19?")).toBe("whats-new-in-react-19");
+	});
+
+	test("collapses multiple dashes", () => {
+		expect(slugify("foo  --  bar")).toBe("foo-bar");
+	});
+
+	test("strips leading/trailing dashes", () => {
+		expect(slugify("--hello--")).toBe("hello");
+	});
+
+	test("truncates at 80 chars", () => {
+		const long = "a".repeat(100);
+		expect(slugify(long).length).toBeLessThanOrEqual(80);
+	});
+
+	test("handles empty string", () => {
+		expect(slugify("")).toBe("");
+	});
+
+	test("handles unicode by stripping", () => {
+		expect(slugify("Vaswani et al. (2017)")).toBe("vaswani-et-al-2017");
+	});
+});
+
+describe("countWords", () => {
+	test("counts plain text words", () => {
+		expect(countWords("hello world foo bar")).toBe(4);
+	});
+
+	test("ignores code blocks", () => {
+		const text = "before\n```\nconst x = 1;\n```\nafter";
+		expect(countWords(text)).toBe(2); // "before" and "after"
+	});
+
+	test("ignores inline code", () => {
+		expect(countWords("use `const` to declare")).toBe(3);
+	});
+
+	test("strips markdown syntax", () => {
+		expect(countWords("# Hello **World**")).toBe(2);
+	});
+
+	test("handles empty string", () => {
+		expect(countWords("")).toBe(0);
+	});
+
+	test("handles whitespace only", () => {
+		expect(countWords("   \n\n  ")).toBe(0);
+	});
+});
+
+describe("normalizeSource", () => {
+	test("generates frontmatter with required fields", () => {
+		const result = normalizeSource({
+			title: "Test Article",
+			content: "# Test\n\nSome content here with words.",
+			sourceType: "web",
+			originalUrl: "https://example.com",
+		});
+
+		expect(result).toContain('title: "Test Article"');
+		expect(result).toContain("source_type: web");
+		expect(result).toContain('url: "https://example.com"');
+		expect(result).toContain("word_count:");
+		expect(result).toContain("ingested:");
+	});
+
+	test("includes author and date when present", () => {
+		const result = normalizeSource({
+			title: "Paper",
+			content: "Content.",
+			sourceType: "pdf",
+			metadata: { author: "John Doe", date: "2024-01-01" },
+		});
+
+		expect(result).toContain('author: "John Doe"');
+		expect(result).toContain('date: "2024-01-01"');
+	});
+
+	test("escapes quotes in title", () => {
+		const result = normalizeSource({
+			title: 'He said "hello"',
+			content: "Content.",
+			sourceType: "file",
+		});
+
+		expect(result).toContain('title: "He said \\"hello\\""');
+	});
+
+	test("removes excessive blank lines", () => {
+		const result = normalizeSource({
+			title: "Test",
+			content: "Line 1\n\n\n\n\nLine 2",
+			sourceType: "file",
+		});
+
+		expect(result).not.toContain("\n\n\n");
+		expect(result).toContain("Line 1\n\nLine 2");
+	});
+
+	test("normalizes CRLF to LF", () => {
+		const result = normalizeSource({
+			title: "Test",
+			content: "Line 1\r\nLine 2",
+			sourceType: "file",
+		});
+
+		expect(result).not.toContain("\r");
+	});
+});