@flow-like/widget-bundler 0.1.1

package/src/pack.ts

@@ -0,0 +1,502 @@
+import { createHash } from "node:crypto";
+import {
+	existsSync,
+	mkdirSync,
+	readFileSync,
+	readdirSync,
+	statSync,
+	writeFileSync,
+} from "node:fs";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import { type ZipOptions, type Zippable, gzipSync, zipSync } from "fflate";
+import { parse as parseToml } from "smol-toml";
+import {
+	BUNDLE_FORMAT_VERSION,
+	BUNDLE_MANIFEST_PATH,
+	type BundleSharedEntry,
+	type BundleWidgetEntry,
+	type WidgetBundleManifest,
+	manifestToJson,
+	unsafeArchiveEntryPaths,
+} from "./bundle-format";
+import {
+	WIDGET_PROTOCOL,
+	contractToJson,
+	isValidPackageId,
+} from "./contract-types";
+import { buildCsp, injectCspMeta } from "./csp";
+import { extractContract } from "./extract";
+import { injectContractScript, inlineHtml } from "./inline";
+
+export const CONNECT_HOSTS_REMOVED =
+	'The connectHosts pack option was removed: declare network sources per widget in widget.config.ts "csp"';
+
+export interface PackOptions {
+	out?: string;
+	/** Extra bundle asset source for the pack-time meta CSP */
+	servingPrefix?: string | null;
+	createdAt?: string;
+	quiet?: boolean;
+}
+
+export interface PackResult {
+	outPath: string;
+	bytes: Uint8Array;
+	/** Whole-file sha256 hex (the manifest `widget_bundle_hash`) */
+	hash: string;
+	manifest: WidgetBundleManifest;
+	warnings: string[];
+	report: string;
+}
+
+export interface PackageInfo {
+	id: string;
+	version: string;
+}
+
+/** Fixed ZIP entry mtime (1980-01-01) for deterministic archives */
+export const ZIP_ENTRY_MTIME_MS = 315532800000;
+
+export function sha256Hex(data: Uint8Array): string {
+	return createHash("sha256").update(data).digest("hex");
+}
+
+export function entryHash(data: Uint8Array): string {
+	return `sha256:${sha256Hex(data)}`;
+}
+
+/**
+ * Read the package `id`/`version` from `flow-like.toml`; accepts the manifest
+ * form (top-level keys) and the template form (`[package]` table).
+ */
+export function readPackageInfo(projectDir: string): PackageInfo {
+	const tomlPath = join(projectDir, "flow-like.toml");
+	if (!existsSync(tomlPath)) {
+		throw new Error(`No flow-like.toml found in ${resolve(projectDir)}`);
+	}
+	const doc = parseToml(readFileSync(tomlPath, "utf8")) as Record<
+		string,
+		unknown
+	>;
+	const pkg =
+		typeof doc.package === "object" && doc.package !== null
+			? (doc.package as Record<string, unknown>)
+			: {};
+	const id = typeof doc.id === "string" ? doc.id : pkg.id;
+	const version = typeof doc.version === "string" ? doc.version : pkg.version;
+	if (typeof id !== "string" || id.length === 0) {
+		throw new Error(
+			`${tomlPath} is missing the package 'id' (top-level or [package].id)`,
+		);
+	}
+	if (!isValidPackageId(id)) {
+		throw new Error(
+			`Invalid package id ${JSON.stringify(id)} in ${tomlPath}: use only letters, digits, '.', '_' and '-'`,
+		);
+	}
+	if (typeof version !== "string" || version.length === 0) {
+		throw new Error(
+			`${tomlPath} is missing the package 'version' (top-level or [package].version)`,
+		);
+	}
+	return { id, version };
+}
+
+/** Archive path as case-insensitive filesystems that strip trailing dots and spaces resolve it */
+function foldedArchivePath(path: string): string | null {
+	const segments: string[] = [];
+	for (const segment of path.split("/")) {
+		const folded = segment
+			.toUpperCase()
+			.toLowerCase()
+			.replace(/[. ]+$/, "");
+		if (folded.length === 0) return null;
+		segments.push(folded);
+	}
+	return segments.join("/");
+}
+
+/**
+ * Archive names that would overwrite each other when unpacked. Mirrors
+ * `archive_name_collisions` in packages/wasm/schema/src/widget_bundle.rs.
+ */
+export function archiveNameCollisions(names: readonly string[]): string[] {
+	const errors: string[] = [];
+	const entries: [string, string][] = [];
+	for (const name of names) {
+		if (name.endsWith("/")) continue;
+		const folded = foldedArchivePath(name);
+		if (folded === null) {
+			errors.push(
+				`Widget bundle entry '${name}' has a path segment made only of dots or spaces`,
+			);
+			continue;
+		}
+		entries.push([name, folded]);
+	}
+	const files = new Map<string, string>();
+	const directories = new Map<string, string>();
+	for (const [name, folded] of entries) {
+		const existing = files.get(folded);
+		if (existing !== undefined) {
+			errors.push(
+				`Widget bundle entries '${existing}' and '${name}' collide on case-insensitive filesystems`,
+			);
+		}
+		files.set(folded, name);
+		for (
+			let cut = folded.lastIndexOf("/");
+			cut !== -1;
+			cut = folded.lastIndexOf("/", cut - 1)
+		) {
+			const parent = folded.slice(0, cut);
+			if (!directories.has(parent)) directories.set(parent, name);
+		}
+	}
+	for (const [name, folded] of entries) {
+		const other = directories.get(folded);
+		if (other !== undefined) {
+			errors.push(
+				`Widget bundle entry '${name}' collides with directory of '${other}' on case-insensitive filesystems`,
+			);
+		}
+	}
+	return errors;
+}
+
+export interface FrameworkGroup {
+	name: string;
+	dir: string;
+	framework: string;
+}
+
+const FRAMEWORK_DEPENDENCIES: [string, string][] = [
+	["svelte", "svelte"],
+	["vue", "vue"],
+	["solid-js", "solid"],
+	["preact", "preact"],
+	["lit", "lit"],
+	["react", "react"],
+];
+
+export function discoverGroups(projectDir: string): FrameworkGroup[] {
+	const widgetsDir = join(projectDir, "widgets");
+	if (!existsSync(widgetsDir)) return [];
+	const groups: FrameworkGroup[] = [];
+	for (const name of readdirSync(widgetsDir).sort()) {
+		const dir = join(widgetsDir, name);
+		if (!statSync(dir).isDirectory()) continue;
+		const packageJsonPath = join(dir, "package.json");
+		if (!existsSync(packageJsonPath)) continue;
+		groups.push({
+			name,
+			dir,
+			framework: detectFramework(packageJsonPath, name),
+		});
+	}
+	return groups;
+}
+
+function detectFramework(packageJsonPath: string, fallback: string): string {
+	try {
+		const pkg = JSON.parse(readFileSync(packageJsonPath, "utf8")) as {
+			dependencies?: Record<string, string>;
+			devDependencies?: Record<string, string>;
+		};
+		const deps = { ...pkg.dependencies, ...pkg.devDependencies };
+		for (const [dep, framework] of FRAMEWORK_DEPENDENCIES) {
+			if (deps[dep]) return framework;
+		}
+	} catch {
+		// informational only; fall through to the directory name
+	}
+	return fallback;
+}
+
+export function discoverGroupWidgets(groupDir: string): string[] {
+	const widgetsDir = join(groupDir, "src", "widgets");
+	if (!existsSync(widgetsDir)) return [];
+	return readdirSync(widgetsDir)
+		.sort()
+		.filter(
+			(name) =>
+				statSync(join(widgetsDir, name)).isDirectory() &&
+				existsSync(join(widgetsDir, name, "widget.config.ts")),
+		);
+}
+
+function findBuiltDocument(
+	groupDir: string,
+	widgetId: string,
+	groupWidgetCount: number,
+): string {
+	const candidates = [
+		join(groupDir, "dist", "widgets", widgetId, "index.html"),
+		join(groupDir, "dist", "src", "widgets", widgetId, "index.html"),
+	];
+	if (groupWidgetCount === 1) {
+		candidates.push(join(groupDir, "dist", "index.html"));
+	}
+	for (const candidate of candidates) {
+		if (existsSync(candidate)) return candidate;
+	}
+	throw new Error(
+		`No built document found for widget '${widgetId}'. Tried:\n${candidates
+			.map((c) => `  - ${c}`)
+			.join(
+				"\n",
+			)}\nRun \`bun run build\` in ${groupDir} first (the group's build script must emit dist/).`,
+	);
+}
+
+function collectSharedFiles(distDir: string): Map<string, Uint8Array> {
+	const sharedDir = join(distDir, "shared");
+	const files = new Map<string, Uint8Array>();
+	if (!existsSync(sharedDir)) return files;
+	const walk = (dir: string) => {
+		for (const name of readdirSync(dir).sort()) {
+			const full = join(dir, name);
+			if (statSync(full).isDirectory()) {
+				walk(full);
+			} else {
+				const rel = relative(sharedDir, full).split(sep).join("/");
+				files.set(`shared/${rel}`, new Uint8Array(readFileSync(full)));
+			}
+		}
+	};
+	walk(sharedDir);
+	return files;
+}
+
+function formatSize(bytes: number): string {
+	if (bytes < 1024) return `${bytes} B`;
+	if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+	return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
+}
+
+function resolveCreatedAt(explicit: string | undefined): string | undefined {
+	if (explicit !== undefined) return explicit;
+	const epoch = process.env.SOURCE_DATE_EPOCH;
+	if (!epoch) return undefined;
+	const seconds = Number(epoch);
+	if (!Number.isFinite(seconds)) return undefined;
+	return new Date(seconds * 1000).toISOString();
+}
+
+interface PackedWidget {
+	id: string;
+	name: string;
+	description: string;
+	framework: string;
+	html: Uint8Array;
+	contractJson: Uint8Array;
+	assets: string[];
+	gzipSize: number;
+}
+
+/** Build the `.flwb` widget bundle for a scaffolded package project. */
+export async function pack(
+	projectDir: string,
+	opts: PackOptions = {},
+): Promise<PackResult> {
+	if ("connectHosts" in opts) {
+		throw new Error(CONNECT_HOSTS_REMOVED);
+	}
+	const project = resolve(projectDir);
+	const info = readPackageInfo(project);
+	const groups = discoverGroups(project);
+	if (groups.length === 0) {
+		throw new Error(
+			`No framework groups found under ${join(project, "widgets")} (expected widgets/<group>/package.json)`,
+		);
+	}
+
+	const warnings: string[] = [];
+	const widgets: PackedWidget[] = [];
+	const shared = new Map<string, Uint8Array>();
+	const seenIds = new Set<string>();
+
+	for (const group of groups) {
+		const widgetIds = discoverGroupWidgets(group.dir);
+		if (widgetIds.length === 0) continue;
+		const distDir = resolve(group.dir, "dist");
+		const groupShared = collectSharedFiles(distDir);
+
+		for (const widgetId of widgetIds) {
+			const configPath = join(
+				group.dir,
+				"src",
+				"widgets",
+				widgetId,
+				"widget.config.ts",
+			);
+			const extracted = extractContract(configPath);
+			const csp = buildCsp(
+				opts.servingPrefix ?? null,
+				extracted.contract.capabilities,
+			);
+			warnings.push(...extracted.warnings);
+			if (extracted.config.id !== widgetId) {
+				throw new Error(
+					`Widget id '${extracted.config.id}' in ${configPath} does not match its directory name '${widgetId}'`,
+				);
+			}
+			if (seenIds.has(widgetId)) {
+				throw new Error(
+					`Duplicate widget id across framework groups: ${widgetId}`,
+				);
+			}
+			seenIds.add(widgetId);
+
+			const htmlPath = findBuiltDocument(group.dir, widgetId, widgetIds.length);
+			const htmlDir = dirname(htmlPath);
+			const resolveAsset = (relPath: string): Uint8Array | null => {
+				const full = resolve(htmlDir, relPath);
+				if (full !== distDir && !full.startsWith(distDir + sep)) return null;
+				if (!existsSync(full) || !statSync(full).isFile()) return null;
+				return new Uint8Array(readFileSync(full));
+			};
+
+			let inlineResult: ReturnType<typeof inlineHtml>;
+			try {
+				inlineResult = inlineHtml(readFileSync(htmlPath, "utf8"), resolveAsset);
+			} catch (e) {
+				throw new Error(
+					`Failed to inline ${htmlPath}: ${e instanceof Error ? e.message : e}`,
+				);
+			}
+
+			for (const asset of inlineResult.external) {
+				const chunk = groupShared.get(asset);
+				if (!chunk) {
+					throw new Error(
+						`Widget '${widgetId}' references '${asset}' but ${join(distDir, asset)} does not exist`,
+					);
+				}
+				const existing = shared.get(asset);
+				if (existing && entryHash(existing) !== entryHash(chunk)) {
+					throw new Error(
+						`Shared chunk collision: '${asset}' has different content across framework groups`,
+					);
+				}
+			}
+			for (const [path, data] of groupShared) {
+				const existing = shared.get(path);
+				if (existing && entryHash(existing) !== entryHash(data)) {
+					throw new Error(
+						`Shared chunk collision: '${path}' has different content across framework groups`,
+					);
+				}
+				shared.set(path, data);
+			}
+
+			let html = injectContractScript(inlineResult.html, extracted.contract);
+			html = injectCspMeta(html, csp);
+			const htmlBytes = new TextEncoder().encode(html);
+
+			widgets.push({
+				id: widgetId,
+				name: extracted.config.name,
+				description: extracted.config.description,
+				framework: group.framework,
+				html: htmlBytes,
+				contractJson: new TextEncoder().encode(
+					contractToJson(extracted.contract),
+				),
+				assets: inlineResult.external,
+				gzipSize: gzipSync(htmlBytes, { level: 6, mtime: 0 }).length,
+			});
+		}
+	}
+
+	if (widgets.length === 0) {
+		throw new Error(
+			`No widgets found (expected widgets/<group>/src/widgets/<id>/widget.config.ts under ${project})`,
+		);
+	}
+	widgets.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+
+	const sharedEntries: BundleSharedEntry[] = [...shared.keys()]
+		.sort()
+		.map((path) => ({ path, hash: entryHash(shared.get(path) as Uint8Array) }));
+
+	const widgetEntries: BundleWidgetEntry[] = widgets.map((widget) => ({
+		id: widget.id,
+		name: widget.name,
+		description: widget.description,
+		entry: `widgets/${widget.id}/index.html`,
+		contract: `widgets/${widget.id}/contract.json`,
+		entryHash: entryHash(widget.html),
+		assets: widget.assets,
+		framework: widget.framework,
+		sizeHint: { raw: widget.html.length, gzip: widget.gzipSize },
+	}));
+
+	const createdAt = resolveCreatedAt(opts.createdAt);
+	const manifest: WidgetBundleManifest = {
+		formatVersion: BUNDLE_FORMAT_VERSION,
+		packageId: info.id,
+		packageVersion: info.version,
+		protocol: WIDGET_PROTOCOL,
+		...(createdAt !== undefined && { createdAt }),
+		shared: sharedEntries,
+		widgets: widgetEntries,
+	};
+
+	const entries = new Map<string, Uint8Array>();
+	entries.set(
+		BUNDLE_MANIFEST_PATH,
+		new TextEncoder().encode(manifestToJson(manifest)),
+	);
+	for (const [path, data] of shared) entries.set(path, data);
+	for (const widget of widgets) {
+		entries.set(`widgets/${widget.id}/index.html`, widget.html);
+		entries.set(`widgets/${widget.id}/contract.json`, widget.contractJson);
+	}
+	const names = [...entries.keys()];
+	const archiveErrors = [
+		...unsafeArchiveEntryPaths(names),
+		...archiveNameCollisions(names),
+	];
+	if (archiveErrors.length > 0) {
+		throw new Error(archiveErrors.join("; "));
+	}
+
+	const zipOptions: ZipOptions = {
+		level: 6,
+		mtime: new Date(ZIP_ENTRY_MTIME_MS),
+	};
+	const zippable: Zippable = {};
+	for (const path of [...entries.keys()].sort()) {
+		zippable[path] = [entries.get(path) as Uint8Array, zipOptions];
+	}
+	const bytes = zipSync(zippable);
+	const hash = sha256Hex(bytes);
+
+	const outPath = resolve(opts.out ?? join(project, "widgets.flwb"));
+	mkdirSync(dirname(outPath), { recursive: true });
+	writeFileSync(outPath, bytes);
+
+	const lines: string[] = [];
+	for (const widget of widgets) {
+		lines.push(
+			`widget ${widget.id.padEnd(24)} raw ${formatSize(widget.html.length).padStart(9)}  gzip ${formatSize(widget.gzipSize).padStart(9)}`,
+		);
+	}
+	for (const [path, data] of [...shared.entries()].sort(([a], [b]) =>
+		a < b ? -1 : 1,
+	)) {
+		lines.push(
+			`shared ${path.padEnd(24)} raw ${formatSize(data.length).padStart(9)}`,
+		);
+	}
+	lines.push(`bundle ${outPath}`);
+	lines.push(
+		`total ${formatSize(bytes.length)} (${widgets.length} widget${widgets.length === 1 ? "" : "s"}, ${shared.size} shared chunk${shared.size === 1 ? "" : "s"})`,
+	);
+	lines.push(`sha256 ${hash}`);
+	const report = lines.join("\n");
+	if (!opts.quiet) console.log(report);
+
+	return { outPath, bytes, hash, manifest, warnings, report };
+}

package/src/psl.ts

@@ -0,0 +1,145 @@
+// Mirror of the ICANN parts of packages/wasm/schema/src/widget_sources
+// (index.rs, rules.rs), built from the generated PSL rules. Driven by
+// packages/wasm/schema/tests/fixtures/widget_source_classification.json.
+
+import { PUBLIC_SUFFIX_RULES } from "./generated/widget-source-data";
+
+export const WILDCARD_PUBLIC_SUFFIX_MESSAGE =
+	"wildcard base is a public suffix or spans public suffixes";
+
+interface IcannIndex {
+	normal: Set<string>;
+	wildcard: Set<string>;
+	exception: Set<string>;
+	tlds: Set<string>;
+	below: Set<string>;
+}
+
+let index: IcannIndex | null = null;
+
+function labelStarts(host: string): number[] {
+	const starts = [0];
+	for (let position = 0; position < host.length; position++) {
+		if (host[position] === ".") starts.push(position + 1);
+	}
+	return starts;
+}
+
+function lastLabel(domain: string): string {
+	return domain.slice(domain.lastIndexOf(".") + 1);
+}
+
+function strictAncestors(domain: string): string[] {
+	return labelStarts(domain)
+		.slice(1)
+		.map((start) => domain.slice(start));
+}
+
+/** Label count of the ICANN public suffix of `host`, implicit `*` included. */
+function icannSuffixLabels(rules: IcannIndex, host: string): number {
+	const starts = labelStarts(host);
+	const count = starts.length;
+	const exception = starts.findIndex((start) =>
+		rules.exception.has(host.slice(start)),
+	);
+	if (exception >= 0) return count - exception - 1;
+	const matched = starts.findIndex((start, position) => {
+		const next = starts[position + 1];
+		return (
+			rules.normal.has(host.slice(start)) ||
+			(next !== undefined && rules.wildcard.has(host.slice(next)))
+		);
+	});
+	return matched >= 0 ? count - matched : 1;
+}
+
+function isSuffixIn(rules: IcannIndex, host: string): boolean {
+	return icannSuffixLabels(rules, host) >= labelStarts(host).length;
+}
+
+function buildIndex(): IcannIndex {
+	const rules: IcannIndex = {
+		normal: new Set(),
+		wildcard: new Set(),
+		exception: new Set(),
+		tlds: new Set(),
+		below: new Set(),
+	};
+	const wildcardDomains: string[] = [];
+	const normalDomains: string[] = [];
+	for (const line of PUBLIC_SUFFIX_RULES.split("\n")) {
+		if (!line.startsWith("i ")) continue;
+		const rule = line.slice(2);
+		if (rule.startsWith("*.")) {
+			const domain = rule.slice(2);
+			rules.wildcard.add(domain);
+			wildcardDomains.push(domain);
+			rules.tlds.add(lastLabel(domain));
+		} else if (rule.startsWith("!")) {
+			const domain = rule.slice(1);
+			rules.exception.add(domain);
+			rules.tlds.add(lastLabel(domain));
+		} else {
+			rules.normal.add(rule);
+			normalDomains.push(rule);
+			rules.tlds.add(lastLabel(rule));
+		}
+	}
+	const candidates = [
+		...wildcardDomains.flatMap((domain) => [
+			domain,
+			...strictAncestors(domain),
+		]),
+		...normalDomains.flatMap(strictAncestors),
+	];
+	for (const candidate of candidates) {
+		if (!isSuffixIn(rules, candidate)) rules.below.add(candidate);
+	}
+	return rules;
+}
+
+function icann(): IcannIndex {
+	index ??= buildIndex();
+	return index;
+}
+
+/** Whether `host` is itself an ICANN public suffix (`co.uk`, `foo.ck`). */
+export function isIcannSuffix(host: string): boolean {
+	return isSuffixIn(icann(), host);
+}
+
+/** Whether `host` is a strict ancestor of an ICANN rule without being one (`kawasaki.jp`). */
+export function isIcannBelow(host: string): boolean {
+	return icann().below.has(host);
+}
+
+/** Whether `label` is the last label of an ICANN rule. */
+export function isIcannTld(label: string): boolean {
+	return icann().tlds.has(label);
+}
+
+/** Base of a leading-label wildcard source (`https://*.b.com` → `b.com`). */
+export function wildcardSourceBase(source: string): string | null {
+	const separator = source.indexOf("://");
+	if (separator < 0) return null;
+	const host = source.slice(separator + 3).split("/")[0] ?? "";
+	return host.startsWith("*.") ? host.slice(2) : null;
+}
+
+/**
+ * Mirrors `validate_wildcard_bases`: a wildcard whose base is an ICANN public
+ * suffix or spans one is rejected (`wildcard-public-suffix`). Other sources
+ * are ignored; the structural grammar reports their errors.
+ */
+export function validateWildcardBases(sources: Iterable<string>): string[] {
+	const errors: string[] = [];
+	for (const source of sources) {
+		const base = wildcardSourceBase(source);
+		if (base !== null && (isIcannSuffix(base) || isIcannBelow(base))) {
+			errors.push(
+				`Invalid csp source "${source}": ${WILDCARD_PUBLIC_SUFFIX_MESSAGE}`,
+			);
+		}
+	}
+	return errors;
+}