@humbletoes/google-search 1.0.0

package/dist/search.js

@@ -0,0 +1,1080 @@
+import { chromium, devices } from "playwright";
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+import logger from "./logger.js";
+import { SecureBrowserConfig, GOOGLE_DOMAINS, getRandomDelay, } from "./browser-config.js";
+/**
+ * Validate and resolve state file path for security
+ * @param userPath - User-provided path or undefined for default
+ * @returns Resolved, validated path
+ */
+function resolveStateFilePath(userPath) {
+    const homeDir = os.homedir();
+    const tmpDir = os.tmpdir();
+    // Default to home directory if no path provided
+    if (!userPath) {
+        return path.join(homeDir, ".google-search-browser-state.json");
+    }
+    // Resolve the path
+    const resolvedPath = path.resolve(userPath);
+    // Prevent directory traversal attacks
+    // Ensure the path is within allowed directories (home, tmp, or cwd)
+    const cwd = process.cwd();
+    const isInHome = resolvedPath.startsWith(path.normalize(homeDir + path.sep));
+    const isInTmp = resolvedPath.startsWith(path.normalize(tmpDir + path.sep));
+    const isInCwd = resolvedPath.startsWith(path.normalize(cwd + path.sep));
+    const isAbsoluteInAllowed = path.isAbsolute(resolvedPath) && (isInHome || isInTmp);
+    const isRelativePath = !path.isAbsolute(userPath); // Relative paths resolve from cwd
+    if (!isInCwd && !isAbsoluteInAllowed && !isRelativePath) {
+        throw new Error(`Invalid state file path: ${userPath}. Path must be within home directory, tmp directory, or relative to current working directory.`);
+    }
+    return resolvedPath;
+}
+/**
+ * Get the actual configuration of the host machine
+ * @param userLocale User-specified locale (if any)
+ * @returns Fingerprint configuration based on host machine
+ */
+function getHostMachineConfig(userLocale) {
+    // Force English locale for consistent English results
+    const systemLocale = userLocale || "en-US";
+    // Force US Eastern timezone to ensure English date formats
+    const timezoneId = "America/New_York";
+    // Detect system color scheme
+    // Node.js cannot directly get system color scheme, use reasonable defaults
+    // Can infer from time: use dark mode at night, light mode during the day
+    const hour = new Date().getHours();
+    const colorScheme = hour >= 19 || hour < 7 ? "dark" : "light";
+    // Use reasonable defaults for other settings
+    const reducedMotion = "no-preference"; // Most users won't enable reduced animations
+    const forcedColors = "none"; // Most users won't enable forced colors
+    return {
+        deviceName: "Desktop Chrome",
+        locale: systemLocale,
+        timezoneId,
+        colorScheme,
+        reducedMotion,
+        forcedColors,
+    };
+}
+/**
+ * Common browser setup logic shared between search functions
+ * @param config Browser setup configuration
+ * @returns Browser setup data
+ */
+function setupBrowserEnvironment(config) {
+    // Check if state file exists
+    let storageState = undefined;
+    let savedState = {};
+    // Fingerprint configuration file path
+    const fingerprintFile = config.stateFile.replace(".json", "-fingerprint.json");
+    if (fs.existsSync(config.stateFile)) {
+        logger.info({ stateFile: config.stateFile }, "Found browser state file, will use saved browser state to avoid anti-bot detection");
+        storageState = config.stateFile;
+        // Try to load saved fingerprint configuration
+        if (fs.existsSync(fingerprintFile)) {
+            try {
+                const fingerprintData = fs.readFileSync(fingerprintFile, "utf8");
+                savedState = JSON.parse(fingerprintData);
+                logger.info("Loaded saved browser fingerprint configuration");
+            }
+            catch (e) {
+                logger.warn({ error: e }, "Unable to load fingerprint configuration file, will create new fingerprint");
+            }
+        }
+    }
+    else {
+        logger.info({ stateFile: config.stateFile }, "Browser state file not found, will create new browser session and fingerprint");
+    }
+    // Get random device configuration or use saved configuration
+    const getRandomDeviceConfig = () => {
+        const deviceList = [
+            "Desktop Chrome",
+            "Desktop Edge",
+            "Desktop Firefox",
+            "Desktop Safari",
+        ];
+        if (savedState.fingerprint?.deviceName &&
+            devices[savedState.fingerprint.deviceName]) {
+            // Use saved device configuration
+            return [
+                savedState.fingerprint.deviceName,
+                devices[savedState.fingerprint.deviceName],
+            ];
+        }
+        else {
+            // Randomly select a device
+            const randomDevice = deviceList[Math.floor(Math.random() * deviceList.length)];
+            return [randomDevice, devices[randomDevice]];
+        }
+    };
+    return {
+        storageState,
+        savedState,
+        fingerprintFile,
+        googleDomains: GOOGLE_DOMAINS,
+        getRandomDeviceConfig,
+        getRandomDelay,
+    };
+}
+/**
+ * Execute Google search and return results
+ * @param query Search keywords
+ * @param options Search options
+ * @returns Search results
+ */
+export async function googleSearch(query, options = {}, existingBrowser) {
+    // Set default options with secure state file path
+    const { limit = 20, timeout = 60000, stateFile: userStateFile, noSaveState = false, locale = "en-US", // Default to English
+     } = options;
+    // Resolve and validate state file path
+    const stateFile = resolveStateFilePath(userStateFile);
+    // Ignore the passed headless parameter, always start in headless mode
+    const useHeadless = true;
+    logger.info({ options: { ...options, stateFile } }, "Initializing browser...");
+    // Setup browser environment
+    const { storageState, savedState, fingerprintFile, googleDomains, getRandomDeviceConfig, getRandomDelay, } = setupBrowserEnvironment({
+        stateFile,
+        timeout,
+        locale,
+        noSaveState,
+    });
+    // Define a function to perform search, can be reused for headless and headed modes
+    async function performSearch(headless) {
+        let browser;
+        let browserWasProvided = false;
+        if (existingBrowser) {
+            browser = existingBrowser;
+            browserWasProvided = true;
+            logger.info("Using existing browser instance");
+        }
+        else {
+            logger.info({ headless }, `Preparing to start browser in ${headless ? "headless" : "headed"} mode...`);
+            // Initialize browser with secure arguments from shared config
+            browser = await chromium.launch({
+                headless,
+                timeout: timeout * 2, // Increase browser startup timeout
+                args: SecureBrowserConfig.getDefaultSearchArgs(),
+                ignoreDefaultArgs: ["--enable-automation"],
+            });
+            logger.info("Browser started successfully!");
+        }
+        // Get device configuration - use saved or randomly generated
+        const [deviceName, deviceConfig] = getRandomDeviceConfig();
+        // Create browser context options
+        let contextOptions = {
+            ...deviceConfig,
+        };
+        // If there is saved fingerprint configuration, use it; otherwise use the actual settings of the host machine
+        if (savedState.fingerprint) {
+            contextOptions = {
+                ...contextOptions,
+                locale: savedState.fingerprint.locale,
+                timezoneId: savedState.fingerprint.timezoneId,
+                colorScheme: savedState.fingerprint.colorScheme,
+                reducedMotion: savedState.fingerprint.reducedMotion,
+                forcedColors: savedState.fingerprint.forcedColors,
+            };
+            logger.info("Using saved browser fingerprint configuration");
+        }
+        else {
+            // Get the actual settings of the host machine
+            const hostConfig = getHostMachineConfig(locale);
+            // If need to use different device type, re-get device configuration
+            if (hostConfig.deviceName !== deviceName) {
+                logger.info({ deviceType: hostConfig.deviceName }, "Use device type based on host machine settings");
+                // Use new device configuration
+                contextOptions = { ...devices[hostConfig.deviceName] };
+            }
+            contextOptions = {
+                ...contextOptions,
+                locale: hostConfig.locale,
+                timezoneId: hostConfig.timezoneId,
+                colorScheme: hostConfig.colorScheme,
+                reducedMotion: hostConfig.reducedMotion,
+                forcedColors: hostConfig.forcedColors,
+            };
+            // Save newly generated fingerprint configuration
+            savedState.fingerprint = hostConfig;
+            logger.info({
+                locale: hostConfig.locale,
+                timezone: hostConfig.timezoneId,
+                colorScheme: hostConfig.colorScheme,
+                deviceType: hostConfig.deviceName,
+            }, "New browser fingerprint configuration generated based on host machine");
+        }
+        // Add general options - ensure desktop configuration is used
+        contextOptions = {
+            ...contextOptions,
+            permissions: ["geolocation", "notifications"],
+            acceptDownloads: true,
+            isMobile: false, // Force desktop mode
+            hasTouch: false, // Disable touch functionality
+            javaScriptEnabled: true,
+        };
+        if (storageState) {
+            logger.info("Loading saved browser state...");
+        }
+        const context = await browser.newContext(storageState ? { ...contextOptions, storageState } : contextOptions);
+        // Set additional browser properties to avoid detection
+        await context.addInitScript(() => {
+            // Override navigator properties
+            Object.defineProperty(navigator, "webdriver", { get: () => false });
+            Object.defineProperty(navigator, "plugins", {
+                get: () => [1, 2, 3, 4, 5],
+            });
+            Object.defineProperty(navigator, "languages", {
+                get: () => ["en-US", "en"],
+            });
+            // Override window properties
+            // @ts-expect-error - Ignore error that chrome property does not exist
+            window.chrome = {
+                runtime: {},
+                loadTimes: function () { },
+                csi: function () { },
+                app: {},
+            };
+            // Add WebGL fingerprint randomization
+            if (typeof WebGLRenderingContext !== "undefined") {
+                const getParameter = WebGLRenderingContext.prototype.getParameter;
+                WebGLRenderingContext.prototype.getParameter = function (parameter) {
+                    // Randomize UNMASKED_VENDOR_WEBGL and UNMASKED_RENDERER_WEBGL
+                    if (parameter === 37445) {
+                        return "Intel Inc.";
+                    }
+                    if (parameter === 37446) {
+                        return "Intel Iris OpenGL Engine";
+                    }
+                    return getParameter.call(this, parameter);
+                };
+            }
+        });
+        const page = await context.newPage();
+        // Set additional page properties
+        await page.addInitScript(() => {
+            // Simulate real screen size and color depth
+            Object.defineProperty(window.screen, "width", { get: () => 1920 });
+            Object.defineProperty(window.screen, "height", { get: () => 1080 });
+            Object.defineProperty(window.screen, "colorDepth", { get: () => 24 });
+            Object.defineProperty(window.screen, "pixelDepth", { get: () => 24 });
+        });
+        try {
+            // Use saved Google domain or randomly select one
+            let selectedDomain;
+            if (savedState.googleDomain) {
+                selectedDomain = savedState.googleDomain;
+                logger.info({ domain: selectedDomain }, "Using saved Google domain");
+            }
+            else {
+                const domains = [...googleDomains];
+                selectedDomain = domains[Math.floor(Math.random() * domains.length)];
+                // Save selected domain
+                savedState.googleDomain = selectedDomain;
+                logger.info({ domain: selectedDomain }, "Randomly select Google domain");
+            }
+            logger.info("Accessing Google search page...");
+            // Visit Google search page
+            const response = await page.goto(selectedDomain, {
+                timeout,
+                waitUntil: "load",
+            });
+            // Check if redirected to human-machine verification page
+            const currentUrl = page.url();
+            const sorryPatterns = [
+                "google.com/sorry/index",
+                "google.com/sorry",
+                "recaptcha",
+                "captcha",
+                "unusual traffic",
+            ];
+            const isBlockedPage = sorryPatterns.some((pattern) => currentUrl.includes(pattern) ||
+                (response && response.url().toString().includes(pattern)));
+            if (isBlockedPage) {
+                if (headless) {
+                    logger.warn("Detected human-machine verification page, restarting browser in headed mode...");
+                    // Close current page and context
+                    await page.close();
+                    await context.close();
+                    // If it's an externally provided browser, don't close it, but create a new browser instance
+                    if (browserWasProvided) {
+                        logger.info("Encountered human-machine verification when using external browser instance, creating new browser instance...");
+                        // Create a new browser instance, no longer using the externally provided instance
+                        const newBrowser = await chromium.launch({
+                            headless: false, // Use headed mode
+                            timeout: timeout * 2,
+                            args: SecureBrowserConfig.getDefaultSearchArgs(),
+                            ignoreDefaultArgs: ["--enable-automation"],
+                        });
+                        // Execute search using new browser instance
+                        try {
+                            const tempContext = await newBrowser.newContext(contextOptions);
+                            const tempPage = await tempContext.newPage();
+                            // Code to handle human-machine verification can be added here
+                            // ...
+                            // Close temporary browser after completion
+                            await newBrowser.close();
+                            // Re-execute search
+                            return performSearch(false);
+                        }
+                        catch (error) {
+                            await newBrowser.close();
+                            throw error;
+                        }
+                    }
+                    else {
+                        // If not externally provided browser, close directly and re-execute search
+                        await browser.close();
+                        return performSearch(false); // Re-execute search in headed mode
+                    }
+                }
+                else {
+                    logger.warn("Detected human-machine verification page, please complete verification in browser...");
+                    // Wait for user to complete verification and redirect back to search page
+                    await page.waitForNavigation({
+                        timeout: timeout * 2,
+                        url: (url) => {
+                            const urlStr = url.toString();
+                            return sorryPatterns.every((pattern) => !urlStr.includes(pattern));
+                        },
+                    });
+                    logger.info("Human-machine verification completed, continuing search...");
+                }
+            }
+            logger.info({ query }, "Entering search keywords");
+            // Wait for search box to appear - try multiple possible selectors
+            const searchInputSelectors = [
+                "textarea[name='q']",
+                "input[name='q']",
+                "textarea[title='Search']",
+                "input[title='Search']",
+                "textarea[aria-label='Search']",
+                "input[aria-label='Search']",
+                "textarea",
+            ];
+            let searchInput = null;
+            for (const selector of searchInputSelectors) {
+                searchInput = await page.$(selector);
+                if (searchInput) {
+                    logger.info({ selector }, "Found search box");
+                    break;
+                }
+            }
+            if (!searchInput) {
+                logger.error("Unable to find search box");
+                throw new Error("Unable to find search box");
+            }
+            // Click search box directly, reduce delay
+            await searchInput.click();
+            // Input entire query string directly, instead of typing character by character
+            await page.keyboard.type(query, { delay: getRandomDelay(10, 30) });
+            // Reduce delay before pressing enter
+            await page.waitForTimeout(getRandomDelay(100, 300));
+            await page.keyboard.press("Enter");
+            logger.info("Waiting for page to finish loading...");
+            // Wait for page to finish loading
+            await page.waitForLoadState("load", { timeout });
+            // Check if URL after search is redirected to human-machine verification page
+            const searchUrl = page.url();
+            const isBlockedAfterSearch = sorryPatterns.some((pattern) => searchUrl.includes(pattern));
+            if (isBlockedAfterSearch) {
+                if (headless) {
+                    logger.warn("Detected human-machine verification page after search, restarting browser in headed mode...");
+                    // Close current page and context
+                    await page.close();
+                    await context.close();
+                    // If it's an externally provided browser, don't close it, but create a new browser instance
+                    if (browserWasProvided) {
+                        logger.info("Encountered human-machine verification after search when using external browser instance, creating new browser instance...");
+                        // Create a new browser instance, no longer using the externally provided instance
+                        const newBrowser = await chromium.launch({
+                            headless: false, // Use headed mode
+                            timeout: timeout * 2,
+                            args: SecureBrowserConfig.getDefaultSearchArgs(),
+                            ignoreDefaultArgs: ["--enable-automation"],
+                        });
+                        // Execute search using new browser instance
+                        try {
+                            const tempContext = await newBrowser.newContext(contextOptions);
+                            const tempPage = await tempContext.newPage();
+                            // Code to handle human-machine verification can be added here
+                            // ...
+                            // Close temporary browser after completion
+                            await newBrowser.close();
+                            // Re-execute search
+                            return performSearch(false);
+                        }
+                        catch (error) {
+                            await newBrowser.close();
+                            throw error;
+                        }
+                    }
+                    else {
+                        // If not externally provided browser, close directly and re-execute search
+                        await browser.close();
+                        return performSearch(false); // Re-execute search in headed mode
+                    }
+                }
+                else {
+                    logger.warn("Detected human-machine verification page after search, please complete verification in browser...");
+                    // Wait for user to complete verification and redirect back to search page
+                    await page.waitForNavigation({
+                        timeout: timeout * 2,
+                        url: (url) => {
+                            const urlStr = url.toString();
+                            return sorryPatterns.every((pattern) => !urlStr.includes(pattern));
+                        },
+                    });
+                    logger.info("Human-machine verification completed, continuing search...");
+                    // Wait for page to reload
+                    await page.waitForLoadState("load", { timeout });
+                }
+            }
+            logger.info({ url: page.url() }, "Waiting for search results to load...");
+            // Try multiple possible search result selectors
+            const searchResultSelectors = [
+                "#search",
+                "#rso",
+                ".g",
+                "[data-sokoban-container]",
+                "div[role='main']",
+            ];
+            let resultsFound = false;
+            for (const selector of searchResultSelectors) {
+                try {
+                    await page.waitForSelector(selector, { timeout: timeout / 2 });
+                    logger.info({ selector }, "Found search results");
+                    resultsFound = true;
+                    break;
+                }
+                catch {
+                    // Continue trying next selector
+                }
+            }
+            if (!resultsFound) {
+                // If search results cannot be found, check if redirected to human-machine verification page
+                const currentUrl = page.url();
+                const isBlockedDuringResults = sorryPatterns.some((pattern) => currentUrl.includes(pattern));
+                if (isBlockedDuringResults) {
+                    if (headless) {
+                        logger.warn("Detected human-machine verification page while waiting for search results, restarting browser in headed mode...");
+                        // Close current page and context
+                        await page.close();
+                        await context.close();
+                        // If it's an externally provided browser, don't close it, but create a new browser instance
+                        if (browserWasProvided) {
+                            logger.info("Encountered human-machine verification while waiting for search results when using external browser instance, creating new browser instance...");
+                            // Create a new browser instance, no longer using the externally provided instance
+                            const newBrowser = await chromium.launch({
+                                headless: false, // Use headed mode
+                                timeout: timeout * 2,
+                                args: SecureBrowserConfig.getDefaultSearchArgs(),
+                                ignoreDefaultArgs: ["--enable-automation"],
+                            });
+                            // Execute search using new browser instance
+                            try {
+                                const tempContext = await newBrowser.newContext(contextOptions);
+                                const tempPage = await tempContext.newPage();
+                                // Code to handle human-machine verification can be added here
+                                // ...
+                                // Close temporary browser after completion
+                                await newBrowser.close();
+                                // Re-execute search
+                                return performSearch(false);
+                            }
+                            catch (error) {
+                                await newBrowser.close();
+                                throw error;
+                            }
+                        }
+                        else {
+                            // If not externally provided browser, close directly and re-execute search
+                            await browser.close();
+                            return performSearch(false); // Re-execute search in headed mode
+                        }
+                    }
+                    else {
+                        logger.warn("Detected human-machine verification page while waiting for search results, please complete verification in browser...");
+                        // Wait for user to complete verification and redirect back to search page
+                        await page.waitForNavigation({
+                            timeout: timeout * 2,
+                            url: (url) => {
+                                const urlStr = url.toString();
+                                return sorryPatterns.every((pattern) => !urlStr.includes(pattern));
+                            },
+                        });
+                        logger.info("Human-machine verification completed, continuing search...");
+                        // Try waiting for search results again
+                        for (const selector of searchResultSelectors) {
+                            try {
+                                await page.waitForSelector(selector, { timeout: timeout / 2 });
+                                logger.info({ selector }, "Found search results after verification");
+                                resultsFound = true;
+                                break;
+                            }
+                            catch {
+                                // Continue trying next selector
+                            }
+                        }
+                        if (!resultsFound) {
+                            logger.error("Unable to find search result elements");
+                            throw new Error("Unable to find search result elements");
+                        }
+                    }
+                }
+                else {
+                    // If not a human-machine verification issue, throw error
+                    logger.error("Unable to find search result elements");
+                    throw new Error("Unable to find search result elements");
+                }
+            }
+            // Reduce waiting time
+            await page.waitForTimeout(getRandomDelay(200, 500));
+            logger.info("Extracting search results...");
+            // Track search start time for performance metrics
+            const searchStartTime = Date.now();
+            // Extract search results
+            const results = await page.evaluate((maxResults) => {
+                const results = [];
+                const seenUrls = new Set(); // For deduplication
+                // Helper function to extract domain from URL
+                const extractDomain = (url) => {
+                    try {
+                        const urlObj = new URL(url);
+                        return urlObj.hostname.replace(/^www\./, "");
+                    }
+                    catch {
+                        return "";
+                    }
+                };
+                // Helper function to detect content type from URL
+                const detectContentType = (url) => {
+                    const lowerUrl = url.toLowerCase();
+                    if (lowerUrl.endsWith(".pdf") || lowerUrl.includes(".pdf?"))
+                        return "pdf";
+                    if (lowerUrl.endsWith(".doc") ||
+                        lowerUrl.endsWith(".docx") ||
+                        lowerUrl.endsWith(".txt"))
+                        return "doc";
+                    if (lowerUrl.includes("youtube.com") ||
+                        lowerUrl.includes("vimeo.com") ||
+                        lowerUrl.endsWith(".mp4") ||
+                        lowerUrl.endsWith(".avi") ||
+                        lowerUrl.includes("/video/"))
+                        return "video";
+                    if (lowerUrl.endsWith(".jpg") ||
+                        lowerUrl.endsWith(".jpeg") ||
+                        lowerUrl.endsWith(".png") ||
+                        lowerUrl.endsWith(".gif") ||
+                        lowerUrl.endsWith(".webp") ||
+                        lowerUrl.includes("/image/"))
+                        return "image";
+                    return "html";
+                };
+                // Helper function to detect if URL is secure
+                const isSecureUrl = (url) => {
+                    return url.startsWith("https://");
+                };
+                // Helper function to count words in text
+                const countWords = (text) => {
+                    return text
+                        .trim()
+                        .split(/\s+/)
+                        .filter((word) => word.length > 0).length;
+                };
+                // Define multiple selector sets, sorted by priority
+                const selectorSets = [
+                    {
+                        container: "#search div[data-hveid]",
+                        title: "h3",
+                        snippet: ".VwiC3b",
+                    },
+                    {
+                        container: "#rso div[data-hveid]",
+                        title: "h3",
+                        snippet: '[data-sncf="1"]',
+                    },
+                    {
+                        container: ".g",
+                        title: "h3",
+                        snippet: 'div[style*="webkit-line-clamp"]',
+                    },
+                    {
+                        container: "div[jscontroller][data-hveid]",
+                        title: "h3",
+                        snippet: 'div[role="text"]',
+                    },
+                ];
+                // Alternative snippet selectors
+                const alternativeSnippetSelectors = [
+                    ".VwiC3b",
+                    '[data-sncf="1"]',
+                    'div[style*="webkit-line-clamp"]',
+                    'div[role="text"]',
+                ];
+                // Try each selector set
+                for (const selectors of selectorSets) {
+                    if (results.length >= maxResults)
+                        break; // Stop if quantity limit reached
+                    const containers = document.querySelectorAll(selectors.container);
+                    for (const container of containers) {
+                        if (results.length >= maxResults)
+                            break;
+                        const titleElement = container.querySelector(selectors.title);
+                        if (!titleElement)
+                            continue;
+                        const title = (titleElement.textContent || "").trim();
+                        // Find link
+                        let link = "";
+                        const linkInTitle = titleElement.querySelector("a");
+                        if (linkInTitle) {
+                            link = linkInTitle.href;
+                        }
+                        else {
+                            let current = titleElement;
+                            while (current && current.tagName !== "A") {
+                                current = current.parentElement;
+                            }
+                            if (current && current instanceof HTMLAnchorElement) {
+                                link = current.href;
+                            }
+                            else {
+                                const containerLink = container.querySelector("a");
+                                if (containerLink) {
+                                    link = containerLink.href;
+                                }
+                            }
+                        }
+                        // Filter invalid or duplicate links
+                        if (!link || !link.startsWith("http") || seenUrls.has(link))
+                            continue;
+                        // Find snippet
+                        let snippet = "";
+                        const snippetElement = container.querySelector(selectors.snippet);
+                        if (snippetElement) {
+                            snippet = (snippetElement.textContent || "").trim();
+                        }
+                        else {
+                            // Try other snippet selectors
+                            for (const altSelector of alternativeSnippetSelectors) {
+                                const element = container.querySelector(altSelector);
+                                if (element) {
+                                    snippet = (element.textContent || "").trim();
+                                    break;
+                                }
+                            }
+                            // If still no snippet found, try general method
+                            if (!snippet) {
+                                const textNodes = Array.from(container.querySelectorAll("div")).filter((el) => !el.querySelector("h3") &&
+                                    (el.textContent || "").trim().length > 20);
+                                if (textNodes.length > 0) {
+                                    snippet = (textNodes[0].textContent || "").trim();
+                                }
+                            }
+                        }
+                        // Only add results with title and link
+                        if (title && link) {
+                            // Check for rich snippet indicators
+                            const hasRichSnippet = !!(container.querySelector(".kno-rdesc") || // Knowledge panel
+                                container.querySelector("[data-attrid]") || // Structured data
+                                container.querySelector(".commercial-unit-desktop-top") || // Featured snippets
+                                container.querySelector('[role="heading"]') // FAQ/structured content
+                            );
+                            const contentType = detectContentType(link);
+                            const wordCount = countWords(title + " " + snippet);
+                            results.push({
+                                title,
+                                link,
+                                snippet,
+                                domain: extractDomain(link),
+                                position: results.length + 1,
+                                snippetLength: snippet.length,
+                                hasRichSnippet,
+                                contentType,
+                                isSecure: isSecureUrl(link),
+                                hasImages: container.querySelector("img") !== null,
+                                wordCount,
+                                language: "en", // Default to English
+                            });
+                            seenUrls.add(link); // Record processed URL
+                        }
+                    }
+                }
+                // If main selectors didn't find enough results, try more general method (as supplement)
+                if (results.length < maxResults) {
+                    const anchorElements = Array.from(document.querySelectorAll("a[href^='http']"));
+                    for (const el of anchorElements) {
+                        if (results.length >= maxResults)
+                            break;
+                        // Check if el is HTMLAnchorElement
+                        if (!(el instanceof HTMLAnchorElement)) {
+                            continue;
+                        }
+                        const link = el.href;
+                        // Filter out navigation links, image links, existing links, etc.
+                        if (!link ||
+                            seenUrls.has(link) ||
+                            link.includes("google.com/") ||
+                            link.includes("accounts.google") ||
+                            link.includes("support.google")) {
+                            continue;
+                        }
+                        const title = (el.textContent || "").trim();
+                        if (!title)
+                            continue; // Skip links without text content
+                        // Try to get surrounding text as snippet
+                        let snippet = "";
+                        let parent = el.parentElement;
+                        for (let i = 0; i < 3 && parent; i++) {
+                            const text = (parent.textContent || "").trim();
+                            // Ensure snippet text differs from title and has certain length
+                            if (text.length > 20 && text !== title) {
+                                snippet = text;
+                                break; // Stop upward search when suitable snippet found
+                            }
+                            parent = parent.parentElement;
+                        }
+                        const contentType = detectContentType(link);
+                        const wordCount = countWords(title + " " + snippet);
+                        results.push({
+                            title,
+                            link,
+                            snippet,
+                            domain: extractDomain(link),
+                            position: results.length + 1,
+                            snippetLength: snippet.length,
+                            hasRichSnippet: false,
+                            contentType,
+                            isSecure: isSecureUrl(link),
+                            hasImages: false,
+                            wordCount,
+                            language: "en",
+                        });
+                        seenUrls.add(link);
+                    }
+                }
+                return results.slice(0, maxResults); // Ensure not exceeding limit
+            }, limit); // Pass limit to evaluate function
+            const searchTime = Date.now() - searchStartTime;
+            logger.info({ count: results.length, searchTimeMs: searchTime }, "Successfully obtained search results");
+            try {
+                // Save browser state (unless user specifies not to save)
+                if (!noSaveState) {
+                    logger.info({ stateFile }, "Saving browser state...");
+                    // Ensure directory exists
+                    const stateDir = path.dirname(stateFile);
+                    if (!fs.existsSync(stateDir)) {
+                        fs.mkdirSync(stateDir, { recursive: true });
+                    }
+                    // Save state
+                    await context.storageState({ path: stateFile });
+                    logger.info("Browser state saved successfully!");
+                    // Save fingerprint configuration
+                    try {
+                        fs.writeFileSync(fingerprintFile, JSON.stringify(savedState, null, 2), "utf8");
+                        logger.info({ fingerprintFile }, "Fingerprint configuration saved");
+                    }
+                    catch (fingerprintError) {
+                        logger.error({ error: fingerprintError }, "Error saving fingerprint configuration");
+                    }
+                }
+                else {
+                    logger.info("Not saving browser state according to user settings");
+                }
+            }
+            catch (error) {
+                logger.error({ error }, "Error saving browser state");
+            }
+            // Only close browser if it's not externally provided
+            if (!browserWasProvided) {
+                logger.info("Closing browser...");
+                await browser.close();
+            }
+            else {
+                logger.info("Keeping browser instance open");
+            }
+            // Return search results with metadata
+            const resultSummary = results.length > 0
+                ? `Found ${results.length} results${results[0].hasRichSnippet ? " (including rich snippets)" : ""}`
+                : "No results found";
+            return {
+                query,
+                results,
+                totalResults: results.length,
+                searchTime,
+                timestamp: new Date().toISOString(),
+                resultSummary,
+            };
+        }
+        catch (error) {
+            logger.error({ error }, "Error occurred during search");
+            try {
+                // Try to save browser state, even if error occurred
+                if (!noSaveState) {
+                    logger.info({ stateFile }, "Saving browser state...");
+                    const stateDir = path.dirname(stateFile);
+                    if (!fs.existsSync(stateDir)) {
+                        fs.mkdirSync(stateDir, { recursive: true });
+                    }
+                    await context.storageState({ path: stateFile });
+                    // Save fingerprint configuration
+                    try {
+                        fs.writeFileSync(fingerprintFile, JSON.stringify(savedState, null, 2), "utf8");
+                        logger.info({ fingerprintFile }, "Fingerprint configuration saved");
+                    }
+                    catch (fingerprintError) {
+                        logger.error({ error: fingerprintError }, "Error saving fingerprint configuration");
+                    }
+                }
+            }
+            catch (stateError) {
+                logger.error({ error: stateError }, "Error saving browser state");
+            }
+            // Only close browser if it's not externally provided
+            if (!browserWasProvided) {
+                logger.info("Closing browser...");
+                await browser.close();
+            }
+            else {
+                logger.info("Keeping browser instance open");
+            }
+            // Re-throw the error instead of returning error results
+            throw error;
+        }
+        // Remove finally block, as resource cleanup is already handled in try and catch blocks
+    }
+    // First try to execute search in headless mode
+    return performSearch(useHeadless);
+}
+/**
+ * Get the raw HTML of Google search result page
+ * @param query Search keywords
+ * @param options Search options
+ * @param saveToFile Whether to save HTML to file (optional)
+ * @param outputPath HTML output file path (optional, defaults to './google-search-html/[query]-[timestamp].html')
+ * @returns Response object containing HTML content
+ */
+export async function getGoogleSearchPageHtml(query, options = {}, saveToFile = false, outputPath) {
+    // Set default options, consistent with googleSearch
+    const { timeout = 60000, stateFile: userStateFile, noSaveState = false, locale = "en-US", // Default to English
+     } = options;
+    // Resolve and validate state file path
+    const stateFile = resolveStateFilePath(userStateFile);
+    // Ignore passed headless parameter, always start in headless mode
+    const useHeadless = true;
+    logger.info({ options: { ...options, stateFile } }, "Initializing browser to get search page HTML...");
+    // Setup browser environment
+    const { storageState, savedState, fingerprintFile, googleDomains, getRandomDeviceConfig, getRandomDelay, } = setupBrowserEnvironment({
+        stateFile,
+        timeout,
+        locale,
+        noSaveState,
+    });
+    // Define a dedicated function to get HTML
+    async function performSearchAndGetHtml(headless) {
+        let browser;
+        // Initialize browser with secure arguments from shared config
+        browser = await chromium.launch({
+            headless,
+            timeout: timeout * 2, // Increase browser startup timeout
+            args: SecureBrowserConfig.getDefaultSearchArgs(),
+            ignoreDefaultArgs: ["--enable-automation"],
+        });
+        // Get device configuration - use saved or randomly generated
+        const [deviceName, deviceConfig] = getRandomDeviceConfig();
+        // Create browser context options
+        let contextOptions = {
+            ...deviceConfig,
+        };
+        // If there is saved fingerprint configuration, use it; otherwise use the actual settings of the host machine
+        if (savedState.fingerprint) {
+            contextOptions = {
+                ...contextOptions,
+                locale: savedState.fingerprint.locale,
+                timezoneId: savedState.fingerprint.timezoneId,
+                colorScheme: savedState.fingerprint.colorScheme,
+                reducedMotion: savedState.fingerprint.reducedMotion,
+                forcedColors: savedState.fingerprint.forcedColors,
+            };
+            logger.info("Using saved browser fingerprint configuration");
+        }
+        else {
+            // Get the actual settings of the host machine
+            const hostConfig = getHostMachineConfig(locale);
+            contextOptions = {
+                ...contextOptions,
+                locale: hostConfig.locale,
+                timezoneId: hostConfig.timezoneId,
+                colorScheme: hostConfig.colorScheme,
+                reducedMotion: hostConfig.reducedMotion,
+                forcedColors: hostConfig.forcedColors,
+            };
+            // Save newly generated fingerprint configuration
+            savedState.fingerprint = hostConfig;
+            logger.info({
+                locale: hostConfig.locale,
+                timezone: hostConfig.timezoneId,
+                colorScheme: hostConfig.colorScheme,
+                deviceType: hostConfig.deviceName,
+            }, "New browser fingerprint configuration generated based on host machine");
+        }
+        // Add general options - ensure desktop configuration is used
+        contextOptions = {
+            ...contextOptions,
+            permissions: ["geolocation", "notifications"],
+            acceptDownloads: true,
+            isMobile: false, // Force desktop mode
+            hasTouch: false, // Disable touch functionality
+            javaScriptEnabled: true,
+        };
+        const context = await browser.newContext(storageState ? { ...contextOptions, storageState } : contextOptions);
+        // Set additional browser properties to avoid detection
+        await context.addInitScript(() => {
+            // Override navigator properties
+            Object.defineProperty(navigator, "webdriver", { get: () => false });
+            Object.defineProperty(navigator, "plugins", {
+                get: () => [1, 2, 3, 4, 5],
+            });
+            Object.defineProperty(navigator, "languages", {
+                get: () => ["en-US", "en"],
+            });
+            // Override window properties
+            // @ts-expect-error - Ignore error that chrome property does not exist
+            window.chrome = {
+                runtime: {},
+                loadTimes: function () { },
+                csi: function () { },
+                app: {},
+            };
+        });
+        const page = await context.newPage();
+        // Set additional page properties
+        await page.addInitScript(() => {
+            // Simulate real screen size and color depth
+            Object.defineProperty(window.screen, "width", { get: () => 1920 });
+            Object.defineProperty(window.screen, "height", { get: () => 1080 });
+            Object.defineProperty(window.screen, "colorDepth", { get: () => 24 });
+            Object.defineProperty(window.screen, "pixelDepth", { get: () => 24 });
+        });
+        try {
+            // Use saved Google domain or randomly select one
+            let selectedDomain;
+            if (savedState.googleDomain) {
+                selectedDomain = savedState.googleDomain;
+                logger.info({ domain: selectedDomain }, "Using saved Google domain");
+            }
+            else {
+                const domains = [...googleDomains];
+                selectedDomain = domains[Math.floor(Math.random() * domains.length)];
+                // Save selected domain
+                savedState.googleDomain = selectedDomain;
+                logger.info({ domain: selectedDomain }, "Randomly select Google domain");
+            }
+            logger.info("Accessing Google search page...");
+            // Visit Google search page
+            await page.goto(selectedDomain, {
+                timeout,
+                waitUntil: "load",
+            });
+            logger.info({ query }, "Entering search keywords");
+            // Wait for search box to appear - try multiple possible selectors
+            const searchInputSelectors = [
+                "textarea[name='q']",
+                "input[name='q']",
+                "textarea[title='Search']",
+                "input[title='Search']",
+                "textarea[aria-label='Search']",
+                "input[aria-label='Search']",
+                "textarea",
+            ];
+            let searchInput = null;
+            for (const selector of searchInputSelectors) {
+                searchInput = await page.$(selector);
+                if (searchInput) {
+                    logger.info({ selector }, "Found search box");
+                    break;
+                }
+            }
+            if (!searchInput) {
+                logger.error("Unable to find search box");
+                throw new Error("Unable to find search box");
+            }
+            // Click search box directly, reduce delay
+            await searchInput.click();
+            // Input entire query string directly, instead of typing character by character
+            await page.keyboard.type(query, { delay: getRandomDelay(10, 30) });
+            // Reduce delay before pressing enter
+            await page.waitForTimeout(getRandomDelay(100, 300));
+            await page.keyboard.press("Enter");
+            logger.info("Waiting for page to finish loading...");
+            // Wait for page to finish loading
+            await page.waitForLoadState("load", { timeout });
+            logger.info({ url: page.url() }, "Search page loaded successfully");
+            // Get page HTML content
+            const html = await page.content();
+            // Take screenshot if requested
+            let screenshotPath;
+            const timestamp = Date.now();
+            const sanitizedQuery = query.replace(/[^a-zA-Z0-9]/g, "_").slice(0, 50);
+            // Determine output path
+            let finalOutputPath = outputPath;
+            if (saveToFile && !finalOutputPath) {
+                // Default output directory
+                const outputDir = path.join(process.cwd(), "google-search-html");
+                if (!fs.existsSync(outputDir)) {
+                    fs.mkdirSync(outputDir, { recursive: true });
+                }
+                finalOutputPath = path.join(outputDir, `${sanitizedQuery}-${timestamp}.html`);
+            }
+            // Save HTML file if requested
+            if (saveToFile && finalOutputPath) {
+                fs.writeFileSync(finalOutputPath, html, "utf8");
+                logger.info({ path: finalOutputPath }, "HTML saved to file");
+            }
+            // Take screenshot
+            const screenshotDir = path.join(process.cwd(), "google-search-screenshots");
+            if (!fs.existsSync(screenshotDir)) {
+                fs.mkdirSync(screenshotDir, { recursive: true });
+            }
+            screenshotPath = path.join(screenshotDir, `${sanitizedQuery}-${timestamp}.png`);
+            await page.screenshot({ path: screenshotPath, fullPage: true });
+            logger.info({ path: screenshotPath }, "Screenshot saved");
+            // Close browser
+            await browser.close();
+            // Save browser state (unless user specifies not to save)
+            if (!noSaveState) {
+                try {
+                    const stateDir = path.dirname(stateFile);
+                    if (!fs.existsSync(stateDir)) {
+                        fs.mkdirSync(stateDir, { recursive: true });
+                    }
+                    await context.storageState({ path: stateFile });
+                    fs.writeFileSync(fingerprintFile, JSON.stringify(savedState, null, 2), "utf8");
+                }
+                catch (saveError) {
+                    logger.error({ error: saveError }, "Error saving browser state");
+                }
+            }
+            // Clean up HTML (remove script and style tags for cleaner output)
+            const cleanedHtml = html
+                .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
+                .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "");
+            return {
+                query,
+                html: cleanedHtml,
+                url: page.url(),
+                savedPath: finalOutputPath,
+                screenshotPath,
+                originalHtmlLength: html.length,
+            };
+        }
+        catch (error) {
+            logger.error({ error }, "Error occurred during HTML retrieval");
+            await browser.close();
+            throw error;
+        }
+    }
+    // Execute in headless mode
+    return performSearchAndGetHtml(useHeadless);
+}
+//# sourceMappingURL=search.js.map