getfilepress 0.0.0 → 0.1.0

package/packages/core/src/lib/theme.ts

@@ -0,0 +1,5 @@
+// Self-hosted latin/latin-ext fonts + the Essay theme (no third-party requests).
+// A site imports this once from its root layout:
+//   import '@filepress/core/theme';
+import './styles/fonts.css';
+import './styles/theme.css';

package/packages/import/package.json

@@ -0,0 +1,27 @@
+{
+	"name": "@filepress/import",
+	"version": "0.1.0",
+	"private": true,
+	"type": "module",
+	"description": "Crawl an existing site and scaffold a filepress content-only sibling (optional Ollama restyle).",
+	"bin": {
+		"filepress-import": "./src/cli.ts"
+	},
+	"scripts": {
+		"import": "tsx src/cli.ts",
+		"test": "vitest run",
+		"test:watch": "vitest"
+	},
+	"dependencies": {
+		"fast-xml-parser": "^5.2.5",
+		"linkedom": "^0.18.12",
+		"turndown": "^7.2.1"
+	},
+	"devDependencies": {
+		"@types/node": "^26.1.0",
+		"@types/turndown": "^5.0.5",
+		"tsx": "^4.20.5",
+		"typescript": "^6.0.3",
+		"vitest": "^4.1.9"
+	}
+}

package/packages/import/src/cli.ts

@@ -0,0 +1,346 @@
+#!/usr/bin/env node
+/**
+ * filepress import — crawl a site, extract content, scaffold a sibling filepress site.
+ *
+ * Usage:
+ *   pnpm --filter @filepress/import import -- --source https://example.com
+ *   filepress import --source https://example.com --inspire https://www.catalystforge.com
+ */
+import { createInterface } from 'node:readline/promises';
+import { isAbsolute, resolve } from 'node:path';
+import { stdin as input, stdout as output } from 'node:process';
+import { discoverSite } from './discover.ts';
+import { extractSite } from './extract.ts';
+import { fetchText } from './fetch.ts';
+import type { DesignBrief, ImportOptions } from './ir.ts';
+import {
+	briefFromInspiration,
+	extractInspirationSignals,
+	type InspirationSignals
+} from './inspire.ts';
+import {
+	generateDesignBrief,
+	ollamaAvailable,
+	ollamaSetupHint,
+	summarizeHtmlForBrief
+} from './ollama.ts';
+import { harvestImagesFromPage, planImages, type ImagePlan } from './images.ts';
+import { formatAttributionMarkdown, planStockCovers } from './stock.ts';
+import { DEFAULT_BRIEF, themeCssFromBrief, tokensFromSourceCss } from './theme.ts';
+import {
+	defaultEngineRoot,
+	defaultOutPath,
+	siteNameFromUrl,
+	writeSite
+} from './write-site.ts';
+
+/** pnpm sets INIT_CWD to the directory where the user invoked the command. */
+function resolveUserPath(p: string): string {
+	if (isAbsolute(p)) return p;
+	const base = process.env.INIT_CWD?.trim() || process.cwd();
+	return resolve(base, p);
+}
+
+function fail(msg: string): never {
+	console.error(`filepress import: ${msg}`);
+	process.exit(1);
+}
+
+function parseArgs(argv: string[]) {
+	const out: {
+		_: string[];
+		source?: string;
+		inspire: string[];
+		out?: string;
+		name?: string;
+		title?: string;
+		author?: string;
+		url?: string;
+		ollama: string;
+		model: string;
+		noLlm: boolean;
+		dryRun: boolean;
+		force: boolean;
+		yes: boolean;
+		fetchImages: boolean;
+	} = {
+		_: [],
+		inspire: [],
+		ollama: process.env.OLLAMA_HOST?.trim() || 'http://127.0.0.1:11434',
+		model: process.env.FILEPRESS_OLLAMA_MODEL?.trim() || 'gemma4:12b',
+		noLlm: false,
+		dryRun: false,
+		force: false,
+		yes: false,
+		fetchImages: false
+	};
+
+	for (let i = 0; i < argv.length; i++) {
+		const a = argv[i];
+		const next = () => {
+			const v = argv[++i];
+			if (!v) fail(`${a} requires a value`);
+			return v;
+		};
+		if (a === '--source') out.source = next();
+		else if (a === '--inspire') out.inspire.push(next());
+		else if (a === '--out') out.out = next();
+		else if (a === '--name') out.name = next();
+		else if (a === '--title') out.title = next();
+		else if (a === '--author') out.author = next();
+		else if (a === '--url') out.url = next();
+		else if (a === '--ollama') out.ollama = next();
+		else if (a === '--model') out.model = next();
+		else if (a === '--no-llm') out.noLlm = true;
+		else if (a === '--dry-run') out.dryRun = true;
+		else if (a === '--force') out.force = true;
+		else if (a === '--yes' || a === '-y') out.yes = true;
+		else if (a === '--fetch-images') out.fetchImages = true;
+		else if (a === '--help' || a === '-h') out._.push('help');
+		else out._.push(a);
+	}
+	return out;
+}
+
+async function prompt(rl: ReturnType<typeof createInterface>, q: string, def?: string) {
+	const hint = def ? ` [${def}]` : '';
+	const ans = (await rl.question(`${q}${hint}: `)).trim();
+	return ans || def || '';
+}
+
+async function main() {
+	const args = parseArgs(process.argv.slice(2));
+	if (args._.includes('help')) {
+		console.log(`Usage: filepress import --source <url> [options]
+
+Options:
+  --source <url>       Site to import (required)
+  --inspire <url>      Inspiration site (repeatable, up to 3; blended into one theme)
+  --out <path>         Sibling output directory
+  --name <slug>        Site package name (lowercase)
+  --title / --author / --url
+  --ollama <host>      Default http://127.0.0.1:11434
+  --model <name>       Default gemma4:12b
+  --no-llm             Skip Ollama; token theme from source CSS / defaults
+  --dry-run            Crawl + report only (no write)
+  --force              Overwrite generated content in --out
+  --fetch-images       Download Openverse CC covers + source portrait/logo into static/images/
+  --yes                Skip confirmation prompt
+`);
+		return;
+	}
+
+	const interactive = Boolean(process.stdin.isTTY) && !args.yes;
+	const rl = interactive ? createInterface({ input, output }) : null;
+
+	try {
+		let source = args.source || '';
+		if (!source && rl) source = await prompt(rl, 'Source site URL');
+		if (!source) fail('`--source` is required');
+
+		let inspire = [...args.inspire];
+		if (!inspire.length && rl) {
+			for (let n = 1; n <= 3; n++) {
+				const q =
+					n === 1
+						? 'Inspiration URL 1/3 (optional, blank to skip)'
+						: `Inspiration URL ${n}/3 (optional, blank to stop)`;
+				const one = await prompt(rl, q);
+				if (!one) break;
+				inspire.push(one);
+			}
+		}
+		if (inspire.length > 3) {
+			console.warn('import: using first 3 --inspire URLs (max 3)');
+			inspire = inspire.slice(0, 3);
+		}
+
+		const engineRoot = defaultEngineRoot();
+		const siteName = (args.name || siteNameFromUrl(source)).replace(/[^a-z0-9-]/g, '');
+		if (!siteName) fail('could not derive a site name; pass --name');
+
+		let out = args.out
+			? resolveUserPath(args.out)
+			: defaultOutPath(engineRoot, siteName);
+		if (rl && !args.out) {
+			const picked = await prompt(rl, 'Output directory', out);
+			if (picked) out = resolveUserPath(picked);
+		}
+
+		const opts: ImportOptions = {
+			source,
+			inspire,
+			out,
+			siteName,
+			title: args.title,
+			author: args.author,
+			canonicalUrl: args.url,
+			ollamaHost: args.ollama,
+			ollamaModel: args.model,
+			noLlm: args.noLlm,
+			dryRun: args.dryRun,
+			force: args.force,
+			fetchImages: args.fetchImages,
+			engineRoot
+		};
+
+		console.log(`\nimport: discovering ${source} …`);
+		const discovered = await discoverSite(source);
+		console.log(
+			`import: found ${discovered.urls.length} URLs, ${discovered.rss.length} RSS items`
+		);
+
+		console.log('import: extracting content …');
+		const ir = await extractSite(discovered);
+		if (opts.title) ir.identity.title = opts.title;
+		if (opts.author) ir.identity.author = opts.author;
+		if (opts.canonicalUrl) ir.identity.canonicalUrl = opts.canonicalUrl.replace(/\/+$/, '');
+
+		console.log(
+			`import: ${ir.posts.length} posts, ${ir.pages.length} pages, title="${ir.identity.title}"`
+		);
+
+		let brief: DesignBrief | null = null;
+		const homeHtml = await fetchText(discovered.origin)
+			.then((r) => r.text)
+			.catch(() => '');
+
+		// Inspiration drives the look. Source site only fills gaps when no --inspire.
+		const inspireSignals: InspirationSignals[] = [];
+		const inspireSummaries: string[] = [];
+		for (const u of inspire) {
+			try {
+				console.log(`import: sampling inspiration ${u} …`);
+				const signals = await extractInspirationSignals(u);
+				inspireSignals.push(signals);
+				const { text } = await fetchText(u);
+				inspireSummaries.push(`${u}: ${summarizeHtmlForBrief(text)}`);
+				console.log(`  → ${signals.paletteMode} · ${signals.notes.join(' · ')}`);
+			} catch (e) {
+				console.warn(
+					`import: inspiration fetch failed ${u}: ${e instanceof Error ? e.message : e}`
+				);
+			}
+		}
+
+		const seed =
+			inspireSignals.length > 0
+				? briefFromInspiration(inspireSignals)
+				: {
+						...DEFAULT_BRIEF,
+						tokens: { ...DEFAULT_BRIEF.tokens, ...tokensFromSourceCss(homeHtml) }
+					};
+
+		if (!opts.noLlm) {
+			const up = await ollamaAvailable(opts.ollamaHost);
+			if (!up) {
+				console.warn(`import: ${ollamaSetupHint(opts.ollamaHost)}`);
+				console.warn('import: continuing with extracted inspiration brief (--no-llm path)');
+				brief = seed;
+			} else {
+				console.log(`import: refining design brief with ${opts.ollamaModel} …`);
+				brief = await generateDesignBrief({
+					host: opts.ollamaHost,
+					model: opts.ollamaModel,
+					ir,
+					inspireSummaries,
+					inspireSignals,
+					seed
+				});
+			}
+		} else {
+			brief = seed;
+		}
+
+		console.log('import: harvesting image candidates …');
+		const harvested = [];
+		for (const u of [source, ...inspire]) {
+			try {
+				harvested.push(...(await harvestImagesFromPage(u, u)));
+			} catch (e) {
+				console.warn(
+					`import: image harvest failed ${u}: ${e instanceof Error ? e.message : e}`
+				);
+			}
+		}
+		let imagePlan: ImagePlan = planImages(ir, harvested, brief);
+		ir.notes.push(...imagePlan.notes);
+		console.log(
+			`import: ${harvested.length} image candidates · hero=${imagePlan.chosen.hero ? 'yes' : 'no'}`
+		);
+
+		if (opts.dryRun) {
+			console.log('\n--- dry-run SiteIR summary ---');
+			console.log(
+				JSON.stringify(
+					{
+						identity: ir.identity,
+						posts: ir.posts.map((p) => ({
+							slug: p.slug,
+							date: p.date,
+							title: p.title,
+							tags: p.tags
+						})),
+						pages: ir.pages.map((p) => ({ slug: p.slug, title: p.title })),
+						nav: ir.nav,
+						topics: ir.topics,
+						lede: ir.lede,
+						notes: ir.notes,
+						images: imagePlan.chosen,
+						unsplashQueries: imagePlan.unsplashQueries
+					},
+					null,
+					2
+				)
+			);
+			if (brief) {
+				console.log('\n--- design brief ---');
+				console.log(JSON.stringify(brief, null, 2));
+				console.log('\n--- theme.css preview (first 60 lines) ---');
+				console.log(themeCssFromBrief(brief).split('\n').slice(0, 60).join('\n'));
+			}
+			console.log(`\nimport: dry-run complete (would write to ${out})`);
+			return;
+		}
+
+		if (rl && !args.yes) {
+			const ok = await prompt(rl, `Write site to ${out}?`, 'y');
+			if (!/^y(es)?$/i.test(ok)) {
+				console.log('import: aborted');
+				return;
+			}
+		}
+
+		if (opts.fetchImages) {
+			console.log('import: searching Openverse for free stock covers …');
+			const stock = await planStockCovers(brief, {
+				author: ir.identity.author,
+				includeHeader: true,
+				includeHero: false
+			});
+			ir.notes.push(...stock.notes);
+			const attr = formatAttributionMarkdown(stock.hits);
+			if (attr) ir.notes.push('Stock attribution:', attr);
+			// Covers from stock; portrait/logo from source harvest only.
+			brief = {
+				...brief,
+				images: {
+					...imagePlan.chosen,
+					...stock.images,
+					portrait: imagePlan.chosen.portrait,
+					logo: imagePlan.chosen.logo
+				}
+			};
+		}
+
+		console.log(`import: writing ${out} …`);
+		const { reportPath } = await writeSite(ir, opts, brief);
+		console.log(`import: done.\n  report: ${reportPath}\n  next: cd ${out} && pnpm install && pnpm filepress dev`);
+	} finally {
+		rl?.close();
+	}
+}
+
+main().catch((e) => {
+	fail(e instanceof Error ? e.message : String(e));
+});

package/packages/import/src/discover.ts

@@ -0,0 +1,190 @@
+import { XMLParser } from 'fast-xml-parser';
+import { fetchText, originOf, resolveUrl, sameOrigin } from './fetch.ts';
+
+export type DiscoveredUrl = {
+	url: string;
+	kind: 'home' | 'post' | 'page' | 'tag' | 'listing' | 'other';
+};
+
+const xmlParser = new XMLParser({
+	ignoreAttributes: false,
+	attributeNamePrefix: '@_'
+});
+
+function asArray<T>(v: T | T[] | undefined | null): T[] {
+	if (v === undefined || v === null) return [];
+	return Array.isArray(v) ? v : [v];
+}
+
+function classify(url: string, origin: string): DiscoveredUrl['kind'] {
+	const u = new URL(url);
+	if (u.origin !== new URL(origin).origin) return 'other';
+	const path = u.pathname.replace(/\/+$/, '') || '/';
+	if (path === '/') return 'home';
+	if (/\/tags?\//i.test(path) || /\/topics?\//i.test(path)) return 'tag';
+	if (/\/(writing|essays|posts|blog|articles)\/[^/]+/i.test(path)) return 'post';
+	if (/\/(writing|essays|posts|blog|articles)\/?$/i.test(path)) return 'listing';
+	if (/\/(about|contact|speaking|now|colophon|privacy|resume|cv)\/?$/i.test(path)) return 'page';
+	// Single-segment paths are likely pages; deeper unknown paths → other
+	const segs = path.split('/').filter(Boolean);
+	if (segs.length === 1) return 'page';
+	if (segs.length >= 2) return 'post';
+	return 'other';
+}
+
+async function readRobotsSitemaps(origin: string): Promise<string[]> {
+	try {
+		const { status, text } = await fetchText(`${origin}/robots.txt`);
+		if (status >= 400) return [`${origin}/sitemap.xml`, `${origin}/sitemap-0.xml`];
+		const maps: string[] = [];
+		for (const line of text.split(/\r?\n/)) {
+			const m = line.match(/^\s*sitemap:\s*(.+)$/i);
+			if (m) maps.push(m[1].trim());
+		}
+		if (maps.length) return maps;
+	} catch {
+		/* fall through */
+	}
+	return [`${origin}/sitemap.xml`, `${origin}/sitemap-0.xml`];
+}
+
+async function urlsFromSitemap(sitemapUrl: string, origin: string): Promise<string[]> {
+	const { status, text, contentType } = await fetchText(sitemapUrl);
+	if (status >= 400) return [];
+	if (!/xml|text\/plain/i.test(contentType) && !text.trimStart().startsWith('<?xml')) {
+		return [];
+	}
+
+	let doc: unknown;
+	try {
+		doc = xmlParser.parse(text);
+	} catch {
+		return [];
+	}
+
+	const root = doc as Record<string, unknown>;
+	// sitemap index
+	const index = root.sitemapindex as { sitemap?: { loc?: string } | { loc?: string }[] } | undefined;
+	if (index?.sitemap) {
+		const nested = asArray(index.sitemap);
+		const out: string[] = [];
+		for (const s of nested) {
+			if (s.loc) out.push(...(await urlsFromSitemap(String(s.loc), origin)));
+		}
+		return out;
+	}
+
+	const urlset = root.urlset as { url?: { loc?: string } | { loc?: string }[] } | undefined;
+	if (!urlset?.url) return [];
+	return asArray(urlset.url)
+		.map((u) => (u.loc ? String(u.loc) : ''))
+		.filter((u) => u && sameOrigin(u, origin));
+}
+
+export type RssItem = {
+	title: string;
+	link: string;
+	description: string | null;
+	pubDate: string | null;
+};
+
+async function readRss(origin: string): Promise<{ channelTitle: string | null; items: RssItem[] }> {
+	const candidates = [`${origin}/rss.xml`, `${origin}/feed.xml`, `${origin}/atom.xml`, `${origin}/feed`];
+	for (const feedUrl of candidates) {
+		try {
+			const { status, text } = await fetchText(feedUrl);
+			if (status >= 400) continue;
+			if (!/<rss|<feed/i.test(text)) continue;
+			const doc = xmlParser.parse(text) as Record<string, unknown>;
+			if (doc.rss) {
+				const channel = (doc.rss as { channel?: Record<string, unknown> }).channel ?? {};
+				const items = asArray(channel.item as RssItem | RssItem[] | undefined).map((it) => ({
+					title: String((it as { title?: string }).title ?? '').trim(),
+					link: String((it as { link?: string }).link ?? '').trim(),
+					description: (it as { description?: string }).description
+						? String((it as { description?: string }).description).trim()
+						: null,
+					pubDate: (it as { pubDate?: string }).pubDate
+						? String((it as { pubDate?: string }).pubDate).trim()
+						: null
+				}));
+				return {
+					channelTitle: channel.title ? String(channel.title) : null,
+					items: items.filter((i) => i.link && i.title)
+				};
+			}
+		} catch {
+			/* try next */
+		}
+	}
+	return { channelTitle: null, items: [] };
+}
+
+export type DiscoverResult = {
+	origin: string;
+	urls: DiscoveredUrl[];
+	rss: RssItem[];
+	rssTitle: string | null;
+};
+
+/** Discover URLs via robots/sitemap + RSS. Deterministic. */
+export async function discoverSite(sourceUrl: string): Promise<DiscoverResult> {
+	const origin = originOf(sourceUrl.endsWith('/') ? sourceUrl : `${sourceUrl}/`);
+	const sitemapUrls = await readRobotsSitemaps(origin);
+	const found = new Set<string>();
+	found.add(`${origin}/`);
+
+	for (const sm of sitemapUrls) {
+		for (const u of await urlsFromSitemap(sm, origin)) {
+			found.add(u.endsWith('/') || u.includes('.') ? u : `${u}/`);
+			// normalize without forcing trailing slash for later fetch
+			found.add(u.replace(/\/+$/, '') || `${origin}/`);
+		}
+	}
+
+	const { channelTitle, items } = await readRss(origin);
+	for (const item of items) {
+		const abs = resolveUrl(origin, item.link);
+		if (abs && sameOrigin(abs, origin)) found.add(abs);
+	}
+
+	// Always probe common pages
+	for (const p of ['/about', '/contact', '/writing', '/essays', '/blog', '/posts']) {
+		found.add(`${origin}${p}`);
+	}
+
+	const urls: DiscoveredUrl[] = [...found]
+		.map((url) => {
+			try {
+				const clean = new URL(url).href;
+				return { url: clean, kind: classify(clean, origin) };
+			} catch {
+				return null;
+			}
+		})
+		.filter((x): x is DiscoveredUrl => Boolean(x));
+
+	// Deduplicate by pathname
+	const byPath = new Map<string, DiscoveredUrl>();
+	for (const u of urls) {
+		const key = new URL(u.url).pathname.replace(/\/+$/, '') || '/';
+		const prev = byPath.get(key);
+		if (!prev || (prev.kind === 'other' && u.kind !== 'other')) byPath.set(key, u);
+		else if (!prev) byPath.set(key, u);
+	}
+
+	// Prefer RSS classification for posts
+	for (const item of items) {
+		const abs = resolveUrl(origin, item.link);
+		if (!abs) continue;
+		const key = new URL(abs).pathname.replace(/\/+$/, '') || '/';
+		byPath.set(key, { url: abs, kind: 'post' });
+	}
+
+	return {
+		origin,
+		urls: [...byPath.values()],
+		rss: items,
+		rssTitle: channelTitle
+	};
+}