@sourcepress/knowledge 0.1.0

package/src/index.ts

@@ -0,0 +1,27 @@
+export { InMemoryKnowledgeStore } from "./knowledge-store.js";
+export { JsonFileStore } from "./json-file-store.js";
+export { KnowledgeEngine } from "./knowledge-engine.js";
+export { GraphBuilder, type GraphBuilderOptions } from "./graph-builder.js";
+export { GraphOps, type GraphStats } from "./graph-ops.js";
+export type {
+	ExtractionResult,
+	ExtractedEntity,
+	ExtractedRelation,
+	EntityCluster,
+	KnowledgeGraph,
+	GraphQueryResult,
+	KnowledgeGap,
+	StaleContent,
+	KnowledgeStoreBackend,
+	KnowledgeFileFilter,
+} from "./types.js";
+export { Scraper } from "./ingestion/scraper.js";
+export { SitemapParser } from "./ingestion/sitemap-parser.js";
+export type {
+	Fetcher,
+	ScrapeResult,
+	SitemapSection,
+	SitemapResult,
+	SitemapRunOptions,
+	BatchProgress,
+} from "./ingestion/types.js";

package/src/ingestion/index.ts

@@ -0,0 +1,10 @@
+export { Scraper } from "./scraper.js";
+export { SitemapParser } from "./sitemap-parser.js";
+export type {
+	Fetcher,
+	ScrapeResult,
+	SitemapSection,
+	SitemapResult,
+	SitemapRunOptions,
+	BatchProgress,
+} from "./types.js";

package/src/ingestion/scraper.ts

@@ -0,0 +1,137 @@
+import { Readability } from "@mozilla/readability";
+import { parseHTML } from "linkedom";
+import type { Fetcher, ScrapeResult } from "./types.js";
+
+const PRIVATE_IP_PATTERNS = [
+	/^127\./,
+	/^10\./,
+	/^192\.168\./,
+	/^172\.(1[6-9]|2\d|3[01])\./,
+	/^169\.254\./,
+	/^::1$/,
+	/^fc00:/i,
+	/^fe80:/i,
+];
+
+/**
+ * Validate a URL against SSRF risks.
+ * Throws if the URL targets a private/loopback address or non-http(s) scheme.
+ */
+export function validateUrl(rawUrl: string): URL {
+	let parsed: URL;
+	try {
+		parsed = new URL(rawUrl);
+	} catch {
+		throw new Error(`Invalid URL: ${rawUrl}`);
+	}
+
+	if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+		throw new Error(
+			`URL scheme "${parsed.protocol}" is not allowed. Only http and https are permitted.`,
+		);
+	}
+
+	const hostname = parsed.hostname.toLowerCase();
+	for (const pattern of PRIVATE_IP_PATTERNS) {
+		if (pattern.test(hostname)) {
+			throw new Error(
+				`URL hostname "${hostname}" resolves to a private or loopback address, which is not allowed.`,
+			);
+		}
+	}
+
+	// Block "localhost" explicitly
+	if (hostname === "localhost" || hostname === "0.0.0.0") {
+		throw new Error(`URL hostname "${hostname}" is not allowed.`);
+	}
+
+	return parsed;
+}
+
+/**
+ * Scrapes a URL into readable content.
+ * Uses linkedom for DOM parsing (works in all runtimes) + Mozilla Readability for extraction.
+ */
+export class Scraper {
+	private fetcher: Fetcher;
+
+	constructor(fetcher?: Fetcher) {
+		this.fetcher = fetcher ?? globalThis.fetch.bind(globalThis);
+	}
+
+	async scrape(url: string): Promise<ScrapeResult> {
+		validateUrl(url);
+
+		const response = await this.fetcher(url, undefined);
+		if (!response.ok) {
+			throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+		}
+
+		const html = await response.text();
+		return this.extractFromHtml(url, html);
+	}
+
+	extractFromHtml(url: string, html: string): ScrapeResult {
+		const { document } = parseHTML(html);
+		// biome-ignore lint/suspicious/noExplicitAny: linkedom document is not typed as DOM Document
+		const reader = new Readability(document as any);
+		const article = reader.parse();
+
+		const text = article?.textContent?.trim() ?? "";
+		if (!article || text.length < 50) {
+			throw new Error(`No readable content found at ${url}`);
+		}
+
+		const markdown = this.htmlToMarkdown(article.content);
+
+		return {
+			url,
+			title: article.title || new URL(url).pathname,
+			content: article.textContent,
+			markdown,
+			byline: article.byline || undefined,
+			excerpt: article.excerpt || undefined,
+			length: article.textContent.length,
+			scraped_at: new Date().toISOString(),
+		};
+	}
+
+	/**
+	 * Lightweight HTML to markdown conversion.
+	 * Handles common elements: headings, paragraphs, links, lists, bold, italic, code.
+	 */
+	private htmlToMarkdown(html: string): string {
+		let md = html;
+		// Headings
+		md = md.replace(/<h1[^>]*>(.*?)<\/h1>/gi, "# $1\n\n");
+		md = md.replace(/<h2[^>]*>(.*?)<\/h2>/gi, "## $1\n\n");
+		md = md.replace(/<h3[^>]*>(.*?)<\/h3>/gi, "### $1\n\n");
+		md = md.replace(/<h4[^>]*>(.*?)<\/h4>/gi, "#### $1\n\n");
+		// Bold and italic
+		md = md.replace(/<(strong|b)>(.*?)<\/\1>/gi, "**$2**");
+		md = md.replace(/<(em|i)>(.*?)<\/\1>/gi, "*$2*");
+		// Code
+		md = md.replace(/<code>(.*?)<\/code>/gi, "`$1`");
+		md = md.replace(/<pre[^>]*>(.*?)<\/pre>/gis, "```\n$1\n```\n\n");
+		// Links
+		md = md.replace(/<a[^>]+href="([^"]*)"[^>]*>(.*?)<\/a>/gi, "[$2]($1)");
+		// Lists
+		md = md.replace(/<li[^>]*>(.*?)<\/li>/gi, "- $1\n");
+		md = md.replace(/<\/?[uo]l[^>]*>/gi, "\n");
+		// Paragraphs and line breaks
+		md = md.replace(/<br\s*\/?>/gi, "\n");
+		md = md.replace(/<p[^>]*>(.*?)<\/p>/gis, "$1\n\n");
+		// Strip remaining HTML tags
+		md = md.replace(/<[^>]+>/g, "");
+		// Decode basic HTML entities
+		md = md.replace(/&amp;/g, "&");
+		md = md.replace(/&lt;/g, "<");
+		md = md.replace(/&gt;/g, ">");
+		md = md.replace(/&quot;/g, '"');
+		md = md.replace(/&#39;/g, "'");
+		md = md.replace(/&nbsp;/g, " ");
+		// Clean up whitespace
+		md = md.replace(/\n{3,}/g, "\n\n").trim();
+		return md;
+	}
+}

package/src/ingestion/sitemap-parser.ts

@@ -0,0 +1,119 @@
+import { validateUrl } from "./scraper.js";
+import type { Fetcher, SitemapResult, SitemapRunOptions, SitemapSection } from "./types.js";
+
+/**
+ * Parses XML sitemaps and groups URLs by path pattern for interactive selection.
+ */
+export class SitemapParser {
+	private fetcher: Fetcher;
+
+	constructor(fetcher?: Fetcher) {
+		this.fetcher = fetcher ?? globalThis.fetch.bind(globalThis);
+	}
+
+	/**
+	 * Fetch and parse a sitemap, returning URLs grouped by path section.
+	 */
+	async parse(sitemapUrl: string): Promise<SitemapResult> {
+		validateUrl(sitemapUrl);
+		const response = await this.fetcher(sitemapUrl);
+		if (!response.ok) {
+			throw new Error(`Failed to fetch sitemap ${sitemapUrl}: ${response.status}`);
+		}
+
+		const xml = await response.text();
+		const urls = this.extractUrls(xml);
+
+		if (urls.length === 0) {
+			throw new Error(`No URLs found in sitemap ${sitemapUrl}`);
+		}
+
+		const sections = this.groupByPattern(urls);
+
+		return {
+			sitemap_url: sitemapUrl,
+			sections,
+			total_urls: urls.length,
+		};
+	}
+
+	/**
+	 * Filter URLs based on include/exclude patterns.
+	 * Patterns use simple glob matching: "/blog/*" matches "/blog/anything".
+	 */
+	filterUrls(result: SitemapResult, options: SitemapRunOptions): string[] {
+		let urls = result.sections.flatMap((s) => s.urls);
+
+		if (options.include && options.include.length > 0) {
+			urls = urls.filter((url) => {
+				const path = new URL(url).pathname;
+				return options.include?.some((pattern) => this.matchPattern(path, pattern));
+			});
+		}
+
+		if (options.exclude && options.exclude.length > 0) {
+			urls = urls.filter((url) => {
+				const path = new URL(url).pathname;
+				return !options.exclude?.some((pattern) => this.matchPattern(path, pattern));
+			});
+		}
+
+		return urls;
+	}
+
+	/**
+	 * Extract <loc> URLs from sitemap XML.
+	 * Handles both regular sitemaps and sitemap indexes.
+	 */
+	private extractUrls(xml: string): string[] {
+		const urls: string[] = [];
+		const locRegex = /<loc>\s*(.*?)\s*<\/loc>/gi;
+		for (const match of xml.matchAll(locRegex)) {
+			const url = match[1].trim();
+			if (url) urls.push(url);
+		}
+		return urls;
+	}
+
+	/**
+	 * Group URLs by their first path segment to create browsable sections.
+	 */
+	private groupByPattern(urls: string[]): SitemapSection[] {
+		const groups = new Map<string, string[]>();
+
+		for (const url of urls) {
+			try {
+				const pathname = new URL(url).pathname;
+				const segments = pathname.split("/").filter(Boolean);
+				const section = segments.length > 0 ? `/${segments[0]}` : "/";
+				const existing = groups.get(section) ?? [];
+				existing.push(url);
+				groups.set(section, existing);
+			} catch {
+				// Skip invalid URLs
+			}
+		}
+
+		return Array.from(groups.entries())
+			.map(([pattern, sectionUrls]) => ({
+				name:
+					pattern === "/" ? "Root" : pattern.slice(1).charAt(0).toUpperCase() + pattern.slice(2),
+				pattern: `${pattern}/*`,
+				urls: sectionUrls,
+				count: sectionUrls.length,
+			}))
+			.sort((a, b) => b.count - a.count);
+	}
+
+	/**
+	 * Simple glob pattern matching.
+	 * Supports trailing wildcard: "/blog/*" matches "/blog/my-post".
+	 */
+	private matchPattern(path: string, pattern: string): boolean {
+		if (pattern.endsWith("/*")) {
+			const prefix = pattern.slice(0, -2);
+			return path === prefix || path.startsWith(`${prefix}/`);
+		}
+		return path === pattern;
+	}
+}

package/src/ingestion/types.ts

@@ -0,0 +1,57 @@
+/**
+ * Injectable fetch function for testability.
+ * Matches the standard fetch signature (subset).
+ */
+export type Fetcher = (url: string, init?: RequestInit) => Promise<Response>;
+
+/**
+ * Result of scraping a single URL.
+ */
+export interface ScrapeResult {
+	url: string;
+	title: string;
+	content: string; // Readable text extracted from HTML
+	markdown: string; // Content converted to markdown
+	byline?: string;
+	excerpt?: string;
+	length: number; // Character count of content
+	scraped_at: string;
+}
+
+/**
+ * Sitemap section — a group of URLs sharing a path pattern.
+ */
+export interface SitemapSection {
+	name: string;
+	pattern: string;
+	urls: string[];
+	count: number;
+}
+
+/**
+ * Parsed sitemap result — sections grouped by URL pattern.
+ */
+export interface SitemapResult {
+	sitemap_url: string;
+	sections: SitemapSection[];
+	total_urls: number;
+}
+
+/**
+ * Options for running a sitemap import.
+ */
+export interface SitemapRunOptions {
+	sitemap_url: string;
+	include?: string[];
+	exclude?: string[];
+}
+
+/**
+ * Progress info for batch operations.
+ */
+export interface BatchProgress {
+	completed: number;
+	total: number;
+	failed: number;
+	results: Array<{ url: string; path: string; success: boolean; error?: string }>;
+}

package/src/json-file-store.ts

@@ -0,0 +1,127 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { KnowledgeFile } from "@sourcepress/core";
+import type {
+	ExtractedEntity,
+	KnowledgeFileFilter,
+	KnowledgeGraph,
+	KnowledgeStoreBackend,
+} from "./types.js";
+
+interface SerializedGraph {
+	entities: Array<[string, ExtractedEntity]>;
+	relations: KnowledgeGraph["relations"];
+	clusters: KnowledgeGraph["clusters"];
+	built_at: string;
+	file_count: number;
+}
+
+export class JsonFileStore implements KnowledgeStoreBackend {
+	private readonly knowledgePath: string;
+	private readonly graphPath: string;
+
+	private files: Map<string, KnowledgeFile> | null = null;
+	private graph: KnowledgeGraph | null | undefined = undefined;
+
+	constructor(dir: string) {
+		this.knowledgePath = join(dir, "knowledge.json");
+		this.graphPath = join(dir, "graph.json");
+
+		if (!existsSync(dir)) {
+			mkdirSync(dir, { recursive: true });
+		}
+	}
+
+	async store(file: KnowledgeFile): Promise<void> {
+		const files = this.loadFiles();
+		files.set(file.path, { ...file });
+		this.writeFiles(files);
+	}
+
+	async retrieve(path: string): Promise<KnowledgeFile | null> {
+		return this.loadFiles().get(path) ?? null;
+	}
+
+	async list(filter?: KnowledgeFileFilter): Promise<KnowledgeFile[]> {
+		let results = Array.from(this.loadFiles().values());
+		if (filter?.type) {
+			results = results.filter((f) => f.type === filter.type);
+		}
+		if (filter?.quality) {
+			results = results.filter((f) => f.quality === filter.quality);
+		}
+		if (filter?.source) {
+			results = results.filter((f) => f.source === filter.source);
+		}
+		if (filter?.since) {
+			const since = filter.since;
+			results = results.filter((f) => f.ingested_at >= since);
+		}
+		return results;
+	}
+
+	async delete(path: string): Promise<boolean> {
+		const files = this.loadFiles();
+		const deleted = files.delete(path);
+		if (deleted) {
+			this.writeFiles(files);
+		}
+		return deleted;
+	}
+
+	async count(): Promise<number> {
+		return this.loadFiles().size;
+	}
+
+	async saveGraph(graph: KnowledgeGraph): Promise<void> {
+		this.graph = graph;
+		const serialized: SerializedGraph = {
+			entities: Array.from(graph.entities.entries()),
+			relations: graph.relations,
+			clusters: graph.clusters,
+			built_at: graph.built_at,
+			file_count: graph.file_count,
+		};
+		writeFileSync(this.graphPath, JSON.stringify(serialized, null, "\t"), "utf-8");
+	}
+
+	async loadGraph(): Promise<KnowledgeGraph | null> {
+		if (this.graph !== undefined) {
+			return this.graph;
+		}
+		if (!existsSync(this.graphPath)) {
+			this.graph = null;
+			return null;
+		}
+		const raw = readFileSync(this.graphPath, "utf-8");
+		const data = JSON.parse(raw) as SerializedGraph;
+		this.graph = {
+			entities: new Map(data.entities),
+			relations: data.relations,
+			clusters: data.clusters,
+			built_at: data.built_at,
+			file_count: data.file_count,
+		};
+		return this.graph;
+	}
+
+	private loadFiles(): Map<string, KnowledgeFile> {
+		if (this.files !== null) {
+			return this.files;
+		}
+		if (!existsSync(this.knowledgePath)) {
+			this.files = new Map();
+			return this.files;
+		}
+		const raw = readFileSync(this.knowledgePath, "utf-8");
+		const entries = JSON.parse(raw) as Array<[string, KnowledgeFile]>;
+		this.files = new Map(entries);
+		return this.files;
+	}
+
+	private writeFiles(files: Map<string, KnowledgeFile>): void {
+		this.files = files;
+		const entries = Array.from(files.entries());
+		writeFileSync(this.knowledgePath, JSON.stringify(entries, null, "\t"), "utf-8");
+	}
+}

package/src/knowledge-engine.ts

@@ -0,0 +1,217 @@
+import type { BudgetTracker, ResolvedProvider } from "@sourcepress/ai";
+import { classify, extract } from "@sourcepress/ai";
+import type { ContentFile, KnowledgeFile } from "@sourcepress/core";
+import { GraphBuilder } from "./graph-builder.js";
+import { GraphOps } from "./graph-ops.js";
+import { Scraper } from "./ingestion/scraper.js";
+import { SitemapParser } from "./ingestion/sitemap-parser.js";
+import type {
+	BatchProgress,
+	Fetcher,
+	SitemapResult,
+	SitemapRunOptions,
+} from "./ingestion/types.js";
+import type {
+	ExtractedEntity,
+	GraphQueryResult,
+	KnowledgeGap,
+	KnowledgeGraph,
+	KnowledgeStoreBackend,
+	StaleContent,
+} from "./types.js";
+
+export class KnowledgeEngine {
+	private store: KnowledgeStoreBackend;
+	private provider: ResolvedProvider;
+	private budget: BudgetTracker;
+	private graphOps: GraphOps | null = null;
+	private currentGraph: KnowledgeGraph | null = null;
+	private scraper: Scraper;
+	private sitemapParser: SitemapParser;
+
+	constructor(
+		store: KnowledgeStoreBackend,
+		provider: ResolvedProvider,
+		budget: BudgetTracker,
+		fetcher?: Fetcher,
+	) {
+		this.store = store;
+		this.provider = provider;
+		this.budget = budget;
+		this.scraper = new Scraper(fetcher);
+		this.sitemapParser = new SitemapParser(fetcher);
+	}
+
+	async initialize(): Promise<void> {
+		const graph = await this.store.loadGraph();
+		if (graph) {
+			this.currentGraph = graph;
+			this.graphOps = new GraphOps(graph);
+		}
+	}
+
+	async ingest(
+		path: string,
+		body: string,
+		source: KnowledgeFile["source"],
+		sourceUrl?: string,
+	): Promise<KnowledgeFile> {
+		const classification = await classify({ text: body }, this.provider, this.budget);
+		const extraction = await extract(
+			{ text: body, file_path: path, existing_entities: this.getExistingEntities() },
+			this.provider,
+			this.budget,
+		);
+
+		const knowledgeFile: KnowledgeFile = {
+			path,
+			type: classification.type,
+			quality: classification.quality,
+			quality_score: classification.quality_score,
+			entities: extraction.entities.map((e) => ({ type: e.type, name: e.name })),
+			ingested_at: new Date().toISOString(),
+			source,
+			source_url: sourceUrl,
+			body,
+		};
+
+		await this.store.store(knowledgeFile);
+		return knowledgeFile;
+	}
+
+	async buildGraph(): Promise<KnowledgeGraph> {
+		const files = await this.store.list();
+		const builder = new GraphBuilder();
+
+		for (const file of files) {
+			const extraction = await extract(
+				{
+					text: file.body,
+					file_path: file.path,
+					existing_entities: Array.from(builder.getEntities().values()),
+				},
+				this.provider,
+				this.budget,
+			);
+
+			const entities: ExtractedEntity[] = extraction.entities.map((e) => ({
+				type: e.type,
+				name: e.name,
+				aliases: e.aliases ?? [],
+				confidence: e.confidence ?? 0.5,
+				source_file: file.path,
+			}));
+
+			builder.addEntities(entities);
+			builder.addRelations(extraction.relations.map((r) => ({ ...r, source_file: file.path })));
+		}
+
+		const graph = builder.build();
+		this.currentGraph = graph;
+		this.graphOps = new GraphOps(graph);
+		await this.store.saveGraph(graph);
+		return graph;
+	}
+
+	query(nameOrAlias: string): GraphQueryResult | null {
+		if (!this.graphOps) return null;
+		return this.graphOps.query(nameOrAlias);
+	}
+
+	findGaps(contentFiles: ContentFile[]): KnowledgeGap[] {
+		if (!this.graphOps) return [];
+		return this.graphOps.findGaps(contentFiles);
+	}
+
+	findStale(
+		contentFiles: ContentFile[],
+		knowledgeTimestamps: Record<string, string>,
+	): StaleContent[] {
+		if (!this.graphOps) return [];
+		return this.graphOps.findStale(contentFiles, knowledgeTimestamps);
+	}
+
+	getGraph(): KnowledgeGraph | null {
+		return this.currentGraph;
+	}
+	getStore(): KnowledgeStoreBackend {
+		return this.store;
+	}
+
+	/**
+	 * Import a single URL: scrape → classify → extract entities → store.
+	 */
+	async importUrl(url: string): Promise<KnowledgeFile> {
+		const scraped = await this.scraper.scrape(url);
+		const path = this.urlToPath(url);
+		return this.ingest(path, scraped.markdown, "url", url);
+	}
+
+	/**
+	 * Parse a sitemap and return grouped sections for interactive selection.
+	 */
+	async parseSitemap(sitemapUrl: string): Promise<SitemapResult> {
+		return this.sitemapParser.parse(sitemapUrl);
+	}
+
+	/**
+	 * Filter a parsed sitemap result by include/exclude patterns.
+	 */
+	filterSitemapUrls(result: SitemapResult, options: SitemapRunOptions): string[] {
+		return this.sitemapParser.filterUrls(result, options);
+	}
+
+	/**
+	 * Import a batch of URLs. Calls progressCallback for each completed URL.
+	 */
+	async importBatch(
+		urls: string[],
+		progressCallback?: (completed: number, total: number, failed: number) => void,
+	): Promise<BatchProgress> {
+		const progress: BatchProgress = {
+			completed: 0,
+			total: urls.length,
+			failed: 0,
+			results: [],
+		};
+
+		for (const url of urls) {
+			try {
+				const file = await this.importUrl(url);
+				progress.completed++;
+				progress.results.push({ url, path: file.path, success: true });
+			} catch (error) {
+				progress.failed++;
+				progress.completed++;
+				progress.results.push({
+					url,
+					path: "",
+					success: false,
+					error: error instanceof Error ? error.message : String(error),
+				});
+			}
+			progressCallback?.(progress.completed, progress.total, progress.failed);
+		}
+
+		return progress;
+	}
+
+	/**
+	 * Convert a URL to a knowledge file path.
+	 * https://example.com/blog/post-1 → knowledge/example.com/blog/post-1.md
+	 */
+	private urlToPath(url: string): string {
+		const parsed = new URL(url);
+		const pathname = parsed.pathname.replace(/\/$/, "") || "/index";
+		const clean = pathname.replace(/\.[^.]+$/, ""); // Remove file extensions
+		return `knowledge/${parsed.hostname}${clean}.md`;
+	}
+
+	private getExistingEntities(): Array<{ type: string; name: string }> {
+		if (!this.currentGraph) return [];
+		return Array.from(this.currentGraph.entities.values()).map((e) => ({
+			type: e.type,
+			name: e.name,
+		}));
+	}
+}