@nijaru/pi-search 0.0.7 → 0.0.8

package/src/x-browser.ts

@@ -0,0 +1,181 @@
+import { chmod, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { SafeFetchError } from "./fetch-errors";
+import type { Lookup } from "./ssrf";
+import type { XReadMode } from "./contracts";
+import type { XUrl } from "./x-public";
+import { browserEndpoint, XBrowserConnection } from "./x-cdp";
+import { readXBrowserPage, type XBrowserDocument } from "./x-browser-page";
+export type { XBrowserDocument } from "./x-browser-page";
+
+export type XBrowserName = "brave" | "chrome" | "chromium";
+interface Preferences { mode: Exclude<XReadMode, "browser">; browser?: XBrowserName }
+export interface XBrowserSelection {
+	readonly mode: XReadMode;
+	read(target: XUrl, signal: AbortSignal, maxBytes: number, lookup?: Lookup): Promise<XBrowserDocument | undefined>;
+}
+export interface XBrowserReader {
+	select(signal: AbortSignal): Promise<XBrowserSelection>;
+}
+export function browserDirectories(): Record<XBrowserName, string> {
+	const home = homedir();
+	if (process.platform === "darwin") {
+		const root = join(home, "Library", "Application Support");
+		return { brave: join(root, "BraveSoftware", "Brave-Browser"), chrome: join(root, "Google", "Chrome"), chromium: join(root, "Chromium") };
+	}
+	if (process.platform === "win32") {
+		const root = process.env.LOCALAPPDATA ?? join(home, "AppData", "Local");
+		return { brave: join(root, "BraveSoftware", "Brave-Browser", "User Data"), chrome: join(root, "Google", "Chrome", "User Data"), chromium: join(root, "Chromium", "User Data") };
+	}
+	const root = process.env.XDG_CONFIG_HOME ?? join(home, ".config");
+	return { brave: join(root, "BraveSoftware", "Brave-Browser"), chrome: join(root, "google-chrome"), chromium: join(root, "chromium") };
+}
+function canceled(): SafeFetchError { return new SafeFetchError({ kind: "canceled", message: "X browser operation canceled" }); }
+function abortable<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
+	if (signal.aborted) return Promise.reject(canceled());
+	return new Promise((resolve, reject) => {
+		const abort = (): void => reject(canceled());
+		signal.addEventListener("abort", abort, { once: true });
+		promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
+	});
+}
+
+/** Explicit authorization for an existing browser. Owns only its connection and temporary X tabs. */
+export class XBrowserAccess implements XBrowserReader {
+	private queue: Promise<unknown> = Promise.resolve();
+	private connection?: XBrowserConnection;
+	private connectedBrowser?: XBrowserName;
+	private active?: AbortController;
+	private stopped = false;
+	private revision = 0;
+	constructor(private readonly directory: string, private readonly browsers = browserDirectories()) {}
+
+	private async preferences(): Promise<Preferences> {
+		let value: unknown;
+		try { value = JSON.parse(await readFile(join(this.directory, "preferences.json"), "utf8")); }
+		catch (error) {
+			if ((error as NodeJS.ErrnoException).code === "ENOENT") return { mode: "auto" };
+			throw new SafeFetchError({ kind: "extraction", message: "Could not read X access preferences", cause: error });
+		}
+		const p = value as Partial<Preferences> | null;
+		if (!p || !["auto", "public", "direct"].includes(p.mode ?? "") || p.browser !== undefined && !["brave", "chrome", "chromium"].includes(p.browser)) throw new SafeFetchError({ kind: "extraction", message: "Invalid X access preferences" });
+		return { mode: p.mode!, ...(p.browser ? { browser: p.browser } : {}) };
+	}
+	private async save(preferences: Preferences, signal?: AbortSignal): Promise<void> {
+		await mkdir(this.directory, { recursive: true, mode: 0o700 });
+		await chmod(this.directory, 0o700);
+		const temporary = join(this.directory, `preferences-${randomUUID()}.tmp`);
+		try {
+			await writeFile(temporary, JSON.stringify(preferences), { mode: 0o600, flag: "wx" });
+			if (signal?.aborted) throw canceled();
+			await rename(temporary, join(this.directory, "preferences.json"));
+		} finally { await rm(temporary, { force: true }); }
+	}
+	select(signal: AbortSignal): Promise<XBrowserSelection> {
+		const revision = this.revision;
+		return this.serialized(async () => {
+			const preferences = await this.preferences();
+			return {
+				mode: preferences.mode,
+				// Bind the read to this policy, so revocation between routing and reading
+				// cannot turn an authorized request into a public fallback.
+				read: (target, signal, maxBytes, lookup) => this.read(preferences, revision, target, signal, maxBytes, lookup),
+			};
+		}, signal, revision);
+	}
+	async status(): Promise<string> {
+		const p = await this.preferences();
+		return `X access: ${p.mode}; ${p.browser ? `${p.browser} authorized${this.connection?.connected ? " and connected" : " (browser approval may be needed)"}` : "browser not authorized"}. ${p.mode === "direct" ? "Only direct X HTTP is allowed." : "Public extraction uses free FxTwitter."} No paid retrieval or separate browser profile.`;
+	}
+	private serialized<T>(task: () => Promise<T>, signal: AbortSignal, revision = this.revision): Promise<T> {
+		const check = (): void => { if (signal.aborted || this.stopped || revision !== this.revision) throw canceled(); };
+		const pending = this.queue.catch(() => undefined).then(async () => {
+			check();
+			const result = await task();
+			check();
+			return result;
+		});
+		this.queue = pending;
+		return abortable(pending, signal);
+	}
+	private async attach(browser: XBrowserName, signal: AbortSignal): Promise<XBrowserConnection> {
+		if (this.connection?.connected && this.connectedBrowser === browser) return this.connection;
+		this.connection?.close();
+		const endpoint = await browserEndpoint(this.browsers[browser]);
+		this.connection = await XBrowserConnection.open(endpoint, signal);
+		this.connectedBrowser = browser;
+		return this.connection;
+	}
+	async connect(confirm: () => Promise<boolean>, browser: XBrowserName = "brave"): Promise<boolean> {
+		const controller = new AbortController();
+		const timeout = setTimeout(() => controller.abort(), 300_000);
+		try {
+			return await this.serialized(async () => {
+				this.active = controller;
+				let authorized = false;
+				try {
+					if (!(await abortable(confirm(), controller.signal))) return false;
+					await this.attach(browser, controller.signal);
+					await this.save({ mode: "auto", browser }, controller.signal);
+					authorized = true;
+					return true;
+				} finally {
+					if (!authorized) this.detach();
+					this.active = undefined;
+				}
+			}, controller.signal);
+		} finally { clearTimeout(timeout); }
+	}
+	private async read(preferences: Preferences, revision: number, target: XUrl, signal: AbortSignal, maxBytes: number, lookup?: Lookup): Promise<XBrowserDocument | undefined> {
+		return this.serialized(async () => {
+			const controller = new AbortController();
+			this.active = controller;
+			const abort = (): void => controller.abort();
+			signal.addEventListener("abort", abort, { once: true });
+			if (signal.aborted) abort();
+			try {
+				if (controller.signal.aborted) throw canceled();
+				if (!preferences.browser) return undefined;
+				if (preferences.mode === "direct") throw canceled();
+				const connection = await this.attach(preferences.browser, controller.signal);
+				const document = await readXBrowserPage(connection, target, controller.signal, maxBytes, lookup);
+				if (controller.signal.aborted) throw canceled();
+				return document;
+			} catch (error) {
+				if (controller.signal.aborted) throw canceled();
+				if (error instanceof SafeFetchError) throw error;
+				throw new SafeFetchError({ kind: "extraction", message: "Authenticated X read failed. Check that X is signed in in your connected browser. Use public mode only if you intend to disclose this post ID to FxTwitter", cause: error });
+			} finally { signal.removeEventListener("abort", abort); this.active = undefined; }
+		}, signal, revision);
+	}
+	private change(task: () => Promise<void>): Promise<void> {
+		this.revision++;
+		this.active?.abort();
+		const pending = this.queue.catch(() => undefined).then(task);
+		this.queue = pending;
+		return pending;
+	}
+	private detach(): void {
+		this.connection?.close();
+		this.connection = undefined;
+		this.connectedBrowser = undefined;
+	}
+	async setMode(mode: Preferences["mode"]): Promise<void> {
+		await this.change(async () => {
+			if (mode !== "auto") this.detach();
+			await this.save({ ...await this.preferences(), mode });
+		});
+	}
+	async disconnect(): Promise<void> {
+		await this.change(async () => {
+			this.detach();
+			await this.save({ mode: (await this.preferences()).mode });
+		});
+	}
+	async shutdown(): Promise<void> {
+		this.stopped = true;
+		await this.change(async () => this.detach());
+	}
+}

package/src/x-cdp.ts

@@ -0,0 +1,101 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { SafeFetchError } from "./fetch-errors";
+
+/** Native discovery only. Never scans ports, launches a browser, or reads profile secrets. */
+export async function browserEndpoint(directory: string): Promise<string> {
+	let text: string;
+	try { text = await readFile(join(directory, "DevToolsActivePort"), "utf8"); }
+	catch { throw new SafeFetchError({ kind: "extraction", message: "No browser connection. Enable remote debugging in your running browser's inspect page, then run /search-x connect" }); }
+	const [port, path, ...rest] = text.trim().split(/\r?\n/);
+	if (text.length > 1024 || rest.length || !/^\d{1,5}$/.test(port ?? "") || Number(port) < 1 || Number(port) > 65535 || !/^\/devtools\/browser\/[A-Za-z0-9-]+$/.test(path ?? "")) {
+		throw new SafeFetchError({ kind: "extraction", message: "Invalid browser connection metadata" });
+	}
+	return `ws://127.0.0.1:${Number(port)}${path}`;
+}
+
+type EventHandler = (params: any, sessionId?: string) => void;
+interface CommandOptions<T> { timeoutMs?: number; onLateResult?: (value: T) => void }
+interface Pending { method: string; resolve(value: any): void; reject(error: Error): void; timer: ReturnType<typeof setTimeout>; expired?: boolean; onLateResult?: (value: any) => void }
+
+/** Target-scoped CDP: unlike general browser drivers, never auto-attaches to existing tabs. */
+export class XBrowserConnection {
+	private nextId = 0;
+	private pending = new Map<number, Pending>();
+	private handlers = new Map<string, Set<EventHandler>>();
+	private constructor(private readonly socket: WebSocket) {
+		socket.addEventListener("message", (event) => {
+			if (typeof event.data !== "string" || event.data.length > 8_000_000) { this.close(); return; }
+			let message: any;
+			try { message = JSON.parse(event.data); } catch { this.close(); return; }
+			if (!message || typeof message !== "object") { this.close(); return; }
+			const pending = this.pending.get(message.id);
+			if (pending) {
+				this.pending.delete(message.id);
+				clearTimeout(pending.timer);
+				// Protocol errors can contain session details. Never echo raw messages or endpoints.
+				if (message.error) pending.reject(new Error(`Browser command ${pending.method} failed`));
+				else if (pending.expired) pending.onLateResult?.(message.result);
+				else pending.resolve(message.result);
+			} else if (typeof message.method === "string") {
+				for (const handler of this.handlers.get(message.method) ?? []) handler(message.params, message.sessionId);
+			}
+		});
+		socket.addEventListener("close", () => this.rejectPending());
+		socket.addEventListener("error", () => this.rejectPending());
+	}
+	static async open(endpoint: string, signal: AbortSignal): Promise<XBrowserConnection> {
+		if (signal.aborted) throw new SafeFetchError({ kind: "canceled", message: "Browser connection canceled" });
+		const socket = new WebSocket(endpoint);
+		const connection = new XBrowserConnection(socket);
+		await new Promise<void>((resolve, reject) => {
+			const cleanup = (): void => {
+				socket.removeEventListener("open", opened);
+				socket.removeEventListener("error", failed);
+				socket.removeEventListener("close", failed);
+				signal.removeEventListener("abort", abort);
+			};
+			const opened = (): void => { cleanup(); resolve(); };
+			const failed = (): void => { cleanup(); connection.close(); reject(new SafeFetchError({ kind: "extraction", message: "Browser connection refused or denied. Enable remote debugging and approve the browser's connection prompt" })); };
+			const abort = (): void => { cleanup(); connection.close(); reject(new SafeFetchError({ kind: "canceled", message: "Browser connection canceled" })); };
+			socket.addEventListener("open", opened, { once: true });
+			socket.addEventListener("error", failed, { once: true });
+			socket.addEventListener("close", failed, { once: true });
+			signal.addEventListener("abort", abort, { once: true });
+			if (signal.aborted) abort();
+		});
+		return connection;
+	}
+	get connected(): boolean { return this.socket.readyState === WebSocket.OPEN; }
+	send<T = Record<string, never>>(method: string, params: object = {}, sessionId?: string, options: CommandOptions<T> = {}): Promise<T> {
+		if (!this.connected) return Promise.reject(new Error("Browser disconnected"));
+		const id = ++this.nextId;
+		return new Promise<T>((resolve, reject) => {
+			const timer = setTimeout(() => {
+				const pending = this.pending.get(id);
+				// Resource creation may complete after a timeout. Retain only its cleanup callback.
+				if (pending?.onLateResult) pending.expired = true;
+				else this.pending.delete(id);
+				reject(new Error(`Browser command ${method} timed out`));
+			}, options.timeoutMs ?? 10_000);
+			this.pending.set(id, { method, resolve, reject, timer, onLateResult: options.onLateResult });
+			try { this.socket.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) })); }
+			catch { this.pending.delete(id); clearTimeout(timer); reject(new Error("Browser disconnected")); }
+		});
+	}
+	on(method: string, handler: EventHandler): () => void {
+		let handlers = this.handlers.get(method);
+		if (!handlers) { handlers = new Set(); this.handlers.set(method, handlers); }
+		handlers.add(handler);
+		return () => { handlers.delete(handler); if (!handlers.size) this.handlers.delete(method); };
+	}
+	private rejectPending(): void {
+		for (const { reject, timer } of this.pending.values()) { clearTimeout(timer); reject(new Error("Browser disconnected")); }
+		this.pending.clear();
+	}
+	close(): void {
+		// Closing the transport is NOT Browser.close or Target.closeTarget on a user tab.
+		this.socket.close();
+		this.rejectPending();
+	}
+}

package/src/x-dom.d.mts

@@ -0,0 +1,7 @@
+export interface XDomDocument {
+	readonly html: string;
+	readonly title: string;
+	readonly postCount: number;
+	readonly collapsedPostIds?: readonly string[];
+}
+export function extractXDom(document: unknown, target: { readonly url: string; readonly postId?: string }, maxCharacters?: number): XDomDocument | null;

package/src/x-dom.mjs

@@ -0,0 +1,77 @@
+/** Fixed DOM reader shared by local HTML extraction and the opt-in browser. */
+export function extractXDom(document, target, maxCharacters = 1_000_000) {
+	const clean = (element) => {
+		const clone = element.cloneNode(true);
+		clone.querySelectorAll("script,style,iframe,object,embed,form,button,video,audio").forEach((el) => el.remove());
+		// X's rich-text headings contain layout divs; preserve inline markup without
+		// turning a heading into an empty Markdown heading followed by a paragraph.
+		for (const block of clone.querySelectorAll('h1 div,h1 p,h2 div,h2 p,h3 div,h3 p,h4 div,h4 p,h5 div,h5 p,h6 div,h6 p')) block.replaceWith(...block.childNodes);
+		for (const el of [clone, ...clone.querySelectorAll("*")]) {
+			for (const attr of [...el.attributes]) {
+				if (!['href', 'src', 'alt', 'title', 'class'].includes(attr.name)) el.removeAttribute(attr.name);
+			}
+			for (const attr of ['href', 'src']) {
+				const value = el.getAttribute(attr);
+				if (!value) continue;
+				try {
+					const url = new URL(value, target.url);
+					if (!['https:', 'http:'].includes(url.protocol) || url.username || url.password) el.removeAttribute(attr);
+					else el.setAttribute(attr, url.href);
+				} catch { el.removeAttribute(attr); }
+			}
+		}
+		return clone.outerHTML;
+	};
+	const bodies = document.querySelectorAll('.x-article-body, [data-testid="twitterArticleRichTextView"]');
+	if (/\/article\//.test(new URL(target.url).pathname) && bodies.length === 1 && bodies[0].textContent.trim()) {
+		const html = clean(bodies[0]);
+		if (html.length > maxCharacters) throw new Error('X article exceeds the extraction limit');
+		return { html, title: ((document.querySelector('[data-testid="twitter-article-title"]') || document.querySelector('h1'))?.textContent?.trim() || document.title || 'X Article').slice(0, 512), postCount: 1 };
+	}
+	if (!target.postId) return null;
+	const posts = [];
+	const collapsedPostIds = [];
+	let author;
+	let found = false;
+	for (const element of document.querySelectorAll('article[data-testid="tweet"]')) {
+		const stamp = element.querySelector('time');
+		const href = stamp?.closest('a')?.getAttribute('href');
+		if (!href) continue;
+		const permalink = new URL(href, target.url);
+		if (!['x.com', 'www.x.com', 'twitter.com', 'www.twitter.com'].includes(permalink.hostname) || permalink.username || permalink.password || permalink.port) continue;
+		const match = permalink.pathname.match(/^\/([A-Za-z0-9_]{1,15})\/status\/(\d{2,20})(?:\/|$)/);
+		if (!match) continue;
+		if (!found) {
+			if (match[2] !== target.postId) continue;
+			found = true;
+			author = match[1].toLowerCase();
+		} else if (match[1].toLowerCase() !== author) break;
+		const text = element.querySelector('[data-testid="tweetText"]');
+		const photos = element.querySelectorAll('[data-testid="tweetPhoto"] img');
+		const video = element.querySelector('video, [data-testid="videoPlayer"]');
+		if (!text?.textContent?.trim() && !photos.length && !video) {
+			if (!posts.length) return null;
+			break;
+		}
+		if (element.querySelector('[data-testid="tweet-text-show-more-link"]')) collapsedPostIds.push(match[2]);
+		const url = `https://x.com/${match[1]}/status/${match[2]}`;
+		const part = document.createElement('section');
+		const source = document.createElement('a');
+		source.setAttribute('href', url);
+		source.textContent = `@${match[1]} · ${stamp.getAttribute('datetime') || match[2]}`;
+		part.appendChild(source);
+		if (text) part.appendChild(text.cloneNode(true));
+		for (const photo of photos) part.appendChild(photo.cloneNode(true));
+		if (video) {
+			const notice = document.createElement('p');
+			notice.textContent = 'Video attached; media was not downloaded or transcribed.';
+			part.appendChild(notice);
+		}
+		posts.push(clean(part));
+		if (posts.length >= 100) break;
+	}
+	if (!posts.length) return null;
+	const html = posts.join('\n<hr>\n');
+	if (html.length > maxCharacters) throw new Error('X thread exceeds the extraction limit');
+	return { html, title: `@${author} on X`, postCount: posts.length, ...(collapsedPostIds.length ? { collapsedPostIds } : {}) };
+}