@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/peel-tls.js

@@ -0,0 +1,220 @@
+/**
+ * PeelTLS — WebPeel's TLS Fingerprint Spoofing Engine
+ * BSD-licensed replacement for CycleTLS (GPL-3).
+ *
+ * Manages a singleton Go binary process that provides TLS fingerprint spoofing
+ * via uTLS (utls library). The binary exposes a local HTTP API that this module
+ * communicates with.
+ */
+import { existsSync } from 'fs';
+import { resolve as pathResolve } from 'path';
+import { spawn } from 'child_process';
+import { randomBytes } from 'crypto';
+import { platform, arch } from 'os';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+// Chrome 133 user agent (matches Go binary's default fingerprint)
+const CHROME_133_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36';
+const DEFAULT_HEADERS = {
+    'Sec-Ch-Ua': '"Chromium";v="133", "Google Chrome";v="133"',
+    'Sec-Ch-Ua-Mobile': '?0',
+    'Sec-Ch-Ua-Platform': '"macOS"',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9',
+};
+// ─── Singleton state ──────────────────────────────────────────────────────────
+let binaryProcess = null;
+let binaryPort = null;
+let binaryToken = null;
+let startPromise = null;
+// ─── Binary discovery ─────────────────────────────────────────────────────────
+/** Get the binary path for the current platform/arch. */
+function getBinaryPath() {
+    const plat = platform(); // 'darwin' | 'linux' | 'win32'
+    const ar = arch(); // 'x64' | 'arm64'
+    const platMap = {
+        darwin: 'darwin',
+        linux: 'linux',
+        win32: 'windows',
+    };
+    const archMap = {
+        x64: 'x64',
+        arm64: 'arm64',
+    };
+    const suffix = plat === 'win32' ? '.exe' : '';
+    const name = `peeltls-${platMap[plat] ?? plat}-${archMap[ar] ?? ar}${suffix}`;
+    // Resolve __dirname for both ESM and CJS
+    let thisDir;
+    try {
+        thisDir = dirname(fileURLToPath(import.meta.url));
+    }
+    catch {
+        thisDir = __dirname ?? process.cwd();
+    }
+    const candidates = [
+        // Relative to this file: src/core/ -> ../../peeltls/dist/
+        pathResolve(thisDir, '..', '..', 'peeltls', 'dist', name),
+        // From project root (cwd)
+        pathResolve(process.cwd(), 'peeltls', 'dist', name),
+        // Fallback: just 'peeltls' in dist (for development)
+        pathResolve(process.cwd(), 'peeltls', 'dist', 'peeltls'),
+    ];
+    for (const candidate of candidates) {
+        if (existsSync(candidate))
+            return candidate;
+    }
+    throw new Error(`PeelTLS binary not found for ${plat}/${ar}.\nExpected: ${candidates[0]}\nBuild it with: cd peeltls && bash build.sh`);
+}
+/** Check if the PeelTLS binary is available on this platform. */
+export function isPeelTLSAvailable() {
+    try {
+        getBinaryPath();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// ─── Process management ───────────────────────────────────────────────────────
+/** Start the PeelTLS Go binary (singleton — only spawned once). */
+async function ensureRunning() {
+    // Already running
+    if (binaryPort !== null && binaryToken !== null && binaryProcess !== null && !binaryProcess.killed) {
+        return { port: binaryPort, token: binaryToken };
+    }
+    // Another call already started it
+    if (startPromise !== null) {
+        await startPromise;
+        return { port: binaryPort, token: binaryToken };
+    }
+    startPromise = new Promise((resolve, reject) => {
+        const binPath = getBinaryPath();
+        const token = randomBytes(32).toString('hex');
+        const proc = spawn(binPath, ['--port', '0', '--token', token], {
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        let stdoutBuf = '';
+        const startupTimeout = setTimeout(() => {
+            proc.kill();
+            reject(new Error('PeelTLS binary failed to start within 10s'));
+        }, 10_000);
+        proc.stdout.on('data', (chunk) => {
+            stdoutBuf += chunk.toString();
+            const newlineIdx = stdoutBuf.indexOf('\n');
+            if (newlineIdx !== -1) {
+                clearTimeout(startupTimeout);
+                try {
+                    const info = JSON.parse(stdoutBuf.slice(0, newlineIdx));
+                    binaryPort = info.port;
+                    binaryToken = token;
+                    binaryProcess = proc;
+                    resolve();
+                }
+                catch {
+                    proc.kill();
+                    reject(new Error(`PeelTLS binary output invalid JSON: ${stdoutBuf}`));
+                }
+            }
+        });
+        proc.stderr.on('data', (chunk) => {
+            if (process.env['DEBUG']) {
+                process.stderr.write(`[peeltls] ${chunk.toString().trim()}\n`);
+            }
+        });
+        proc.on('error', (err) => {
+            clearTimeout(startupTimeout);
+            reject(new Error(`PeelTLS binary failed to spawn: ${err.message}`));
+        });
+        proc.on('exit', (code) => {
+            binaryProcess = null;
+            binaryPort = null;
+            binaryToken = null;
+            startPromise = null;
+            if (code !== 0 && code !== null) {
+                clearTimeout(startupTimeout);
+                // Don't reject here if already resolved — just clean state
+            }
+        });
+    });
+    await startPromise;
+    startPromise = null; // Reset so it can restart if the process exits later
+    return { port: binaryPort, token: binaryToken };
+}
+// ─── Public API ───────────────────────────────────────────────────────────────
+/** Fetch a URL using PeelTLS TLS fingerprint spoofing. */
+export async function peelTLSFetch(url, options) {
+    const { port, token } = await ensureRunning();
+    const mergedHeaders = {
+        'User-Agent': CHROME_133_UA,
+        ...DEFAULT_HEADERS,
+        ...(options?.headers ?? {}),
+    };
+    const requestBody = JSON.stringify({
+        url,
+        method: 'GET',
+        headers: mergedHeaders,
+        fingerprint: options?.fingerprint ?? 'chrome-133',
+        proxy: options?.proxy ?? '',
+        timeout: Math.round((options?.timeout ?? 30_000) / 1000),
+        followRedirects: true,
+        maxRedirects: 10,
+    });
+    if (process.env['DEBUG']) {
+        process.stderr.write(`[peeltls] fetch: ${url}\n`);
+    }
+    const timeoutMs = options?.timeout ?? 30_000;
+    const response = await fetch(`http://127.0.0.1:${port}/fetch`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${token}`,
+        },
+        body: requestBody,
+        signal: AbortSignal.timeout(timeoutMs + 5_000), // extra buffer over Go's own timeout
+    });
+    const result = (await response.json());
+    if (result.error) {
+        throw new Error(`PeelTLS fetch failed: ${result.error}`);
+    }
+    return {
+        html: result.body ?? '',
+        url: result.finalUrl ?? url,
+        statusCode: result.status ?? 0,
+        contentType: result.headers?.['content-type'] ?? 'text/html',
+        method: 'peeltls',
+    };
+}
+/** Gracefully shut down the PeelTLS binary process. */
+export async function shutdownPeelTLS() {
+    if (binaryPort === null || binaryToken === null || binaryProcess === null)
+        return;
+    try {
+        await fetch(`http://127.0.0.1:${binaryPort}/shutdown`, {
+            method: 'POST',
+            headers: { Authorization: `Bearer ${binaryToken}` },
+            signal: AbortSignal.timeout(5_000),
+        });
+    }
+    catch {
+        // Force kill if graceful shutdown fails or times out
+        binaryProcess?.kill('SIGTERM');
+    }
+    binaryProcess = null;
+    binaryPort = null;
+    binaryToken = null;
+}
+// ─── Process exit cleanup ─────────────────────────────────────────────────────
+process.on('exit', () => {
+    binaryProcess?.kill('SIGTERM');
+});
+process.on('SIGINT', () => {
+    binaryProcess?.kill('SIGTERM');
+    process.exit(0);
+});
+process.on('SIGTERM', () => {
+    binaryProcess?.kill('SIGTERM');
+    process.exit(0);
+});

package/dist/core/pipeline.d.ts

@@ -0,0 +1,132 @@
+/**
+ * WebPeel pipeline stages
+ *
+ * Each stage is an exported async function that reads from / writes to the
+ * mutable PipelineContext.  The stages are called in order by peel().
+ */
+import { type AutoScrollOptions } from './actions.js';
+import { type DomainExtractResult } from '../ee/domain-extractors.js';
+import { type ReadabilityResult } from './readability.js';
+import { type QuickAnswerResult } from './quick-answer.js';
+import { Timer } from './timing.js';
+import type { PeelOptions, PeelResult, ImageInfo } from '../types.js';
+import type { DomainVerification } from './domain-verify.js';
+import type { BrandingProfile } from './branding.js';
+import type { ChangeResult } from './change-tracking.js';
+import type { DesignAnalysis } from './design-analysis.js';
+import type { SafeBrowsingResult } from './safe-browsing.js';
+/** Mutable context threaded through pipeline stages */
+export interface PipelineContext {
+    url: string;
+    options: PeelOptions;
+    timer: Timer;
+    startTime: number;
+    render: boolean;
+    stealth: boolean;
+    wait: number;
+    format: 'markdown' | 'text' | 'html' | 'clean';
+    timeout: number;
+    userAgent?: string;
+    screenshot: boolean;
+    screenshotFullPage: boolean;
+    selector?: string;
+    exclude?: string[];
+    includeTags?: string[];
+    excludeTags?: string[];
+    headers?: Record<string, string>;
+    cookies?: string[];
+    raw: boolean;
+    actions?: any[];
+    extract?: any;
+    maxTokens?: number;
+    extractImagesFlag: boolean;
+    profileDir?: string;
+    headed: boolean;
+    storageState?: any;
+    proxy?: string;
+    fullPage: boolean;
+    autoScrollOpts?: AutoScrollOptions;
+    fetchResult?: any;
+    contentType: 'document' | 'html' | 'json' | 'xml' | 'text' | 'image';
+    content: string;
+    title: string;
+    metadata: any;
+    links: string[];
+    quality: number;
+    prunedPercent?: number;
+    linkCount: number;
+    freshness?: {
+        lastModified?: string;
+        etag?: string;
+        fetchedAt: string;
+        cacheControl?: string;
+    };
+    jsonLdType?: string;
+    budgetFallback?: boolean;
+    readabilityResult?: ReadabilityResult;
+    imagesList?: ImageInfo[];
+    extracted?: Record<string, any>;
+    domainData?: DomainExtractResult;
+    quickAnswerResult?: QuickAnswerResult;
+    brandingProfile?: BrandingProfile;
+    designAnalysisResult?: DesignAnalysis;
+    changeResult?: ChangeResult;
+    summaryText?: string;
+    screenshotBase64?: string;
+    /** True when domain API extraction handled the content (skip redundant extraction) */
+    domainApiHandled?: boolean;
+    /** True when server returned pre-rendered markdown (Content-Type: text/markdown) */
+    serverMarkdown?: boolean;
+    /** True when HTTP fetch completed in < 500ms — enables fast path (skip challenge detection) */
+    fastPath?: boolean;
+    /** Non-fatal warnings accumulated during the pipeline run */
+    warnings: string[];
+    /** True when an auth wall was detected on this page */
+    authRequired?: boolean;
+    /** Raw HTML size in characters (measured from fetched content before any conversion) */
+    rawHtmlSize?: number;
+    /** Safe Browsing check result (set early in pipeline, before fetch) */
+    safeBrowsingResult?: SafeBrowsingResult;
+    /** Active domain verification result (TLS + DNS + headers) */
+    domainVerification?: DomainVerification | null;
+}
+/** Create the initial PipelineContext with defaults */
+export declare function createContext(url: string, options: PeelOptions): PipelineContext;
+/**
+ * Resolve all PeelOptions values into flat context fields with defaults applied.
+ * Force render=true when screenshot/stealth/actions/branding/autoScroll requested.
+ * Parse the autoScroll option.
+ */
+export declare function normalizeOptions(ctx: PipelineContext): void;
+/**
+ * If the URL is a YouTube URL, attempt transcript extraction.
+ * Returns a PeelResult on success, or null to fall through to normal pipeline.
+ */
+export declare function handleYouTube(ctx: PipelineContext): Promise<PeelResult | null>;
+/**
+ * Fetch the URL via smartFetch, handle autoScroll, and store result in ctx.fetchResult.
+ */
+export declare function fetchContent(ctx: PipelineContext): Promise<void>;
+/**
+ * Detect and set ctx.contentType based on response headers and content.
+ */
+export declare function detectContentType(ctx: PipelineContext): void;
+/**
+ * Parse content from fetchResult based on the detected contentType.
+ * Sets ctx.content, ctx.title, ctx.metadata, ctx.links, ctx.quality, ctx.prunedPercent.
+ */
+export declare function parseContent(ctx: PipelineContext): Promise<void>;
+/**
+ * Run all post-processing in sequence:
+ * readability, image extraction, structured extraction,
+ * maxTokens truncation, budget distillation, domain extractors, quick answer.
+ */
+export declare function postProcess(ctx: PipelineContext): Promise<void>;
+/**
+ * Screenshot base64 conversion, branding extraction (needs page), change tracking, AI summary.
+ */
+export declare function finalize(ctx: PipelineContext): Promise<void>;
+/**
+ * Assemble the final PeelResult from the pipeline context.
+ */
+export declare function buildResult(ctx: PipelineContext): PeelResult;