@isdk/web-searcher 0.1.1

package/dist/index.d.mts

@@ -0,0 +1,321 @@
+import * as _isdk_web_fetcher from '@isdk/web-fetcher';
+import { FetcherOptions, FetchSession } from '@isdk/web-fetcher';
+import { IBaseFactoryOptions } from 'custom-factory';
+
+/**
+ * Interface representing a standardized search result item.
+ * This ensures consistency across different search engines.
+ */
+interface StandardSearchResult {
+    /** The title of the search result. */
+    title: string;
+    /** The URL of the search result. */
+    url: string;
+    /** A brief snippet or description of the result. */
+    snippet?: string;
+    /** An optional image URL associated with the result. */
+    image?: string;
+    /** Allows for engine-specific extra fields (e.g., rank, author, date). */
+    [key: string]: any;
+}
+/**
+ * Configuration for pagination strategies.
+ * Defines how the searcher should navigate to the next page of results.
+ */
+interface PaginationConfig {
+    /**
+     * The type of pagination mechanism:
+     * - 'url-param': Pagination is handled by modifying URL parameters (e.g., `?page=2` or `?start=10`).
+     * - 'click-next': Pagination is handled by clicking a "Next" button on the page (only works in 'browser' mode).
+     */
+    type: 'url-param' | 'click-next';
+    /**
+     * The name of the URL parameter used for pagination.
+     * Required if type is 'url-param'.
+     * @example 'start' for Google, 'page' or 'p' for others.
+     */
+    paramName?: string;
+    /**
+     * The starting value for the pagination parameter.
+     * @default 0
+     */
+    startValue?: number;
+    /**
+     * The increment step for each page.
+     * - If the parameter represents an item offset (like Google's 'start'), this might be 10.
+     * - If the parameter represents a page number, this is usually 1.
+     * @default 1
+     */
+    increment?: number;
+    /**
+     * The CSS selector for the "Next" page button.
+     * Required if type is 'click-next'.
+     */
+    nextButtonSelector?: string;
+}
+/**
+ * Context object passed to the transform function.
+ */
+interface SearchContext {
+    /** The original search query. */
+    query: string;
+    /** The current page index (0-based). */
+    page: number;
+    /** The requested limit of results. */
+    limit?: number;
+}
+type SearchTimeRangePreset = 'all' | 'day' | 'week' | 'month' | 'year';
+interface CustomTimeRange {
+    /** Start date (Date object or string like 'YYYY-MM-DD'). */
+    from: Date | string;
+    /** End date (Date object or string like 'YYYY-MM-DD'). Defaults to current date if omitted. */
+    to?: Date | string;
+}
+type SearchTimeRange = SearchTimeRangePreset | CustomTimeRange;
+type SearchCategory = 'all' | 'images' | 'videos' | 'news';
+type SafeSearchLevel = 'off' | 'moderate' | 'strict';
+/**
+ * Options provided when executing a search.
+ */
+interface SearchOptions {
+    /** The maximum number of results to retrieve. */
+    limit?: number;
+    /**
+     * Date range for the search results.
+     * Default: 'all'
+     */
+    timeRange?: SearchTimeRange;
+    /**
+     * The category of results to return.
+     * Default: 'all' (web search)
+     */
+    category?: SearchCategory;
+    /**
+     * Region code (ISO 3166-1 alpha-2) to bias results (e.g., 'US', 'CN', 'JP').
+     */
+    region?: string;
+    /**
+     * Language code (ISO 639-1) for the interface or results (e.g., 'en', 'zh-CN').
+     */
+    language?: string;
+    /**
+     * Safe search filtering level.
+     * Default: engine dependent (usually 'moderate' or 'strict' by default).
+     */
+    safeSearch?: SafeSearchLevel;
+    /**
+     * A custom transform function to filter or modify results at runtime.
+     * This runs AFTER the engine-level transform.
+     */
+    transform?: (results: StandardSearchResult[], context: SearchContext) => Promise<StandardSearchResult[]> | StandardSearchResult[];
+    /** Any other custom variables to be injected into the template. */
+    [key: string]: any;
+}
+
+/**
+ * Constructor definition for Searcher subclasses.
+ */
+type SearcherConstructor = new (options?: FetcherOptions) => WebSearcher;
+/**
+ * The abstract base class for all search engines.
+ *
+ * It extends `FetchSession`, meaning each `WebSearcher` instance is an active session
+ * capable of maintaining state (e.g., cookies, local storage) across multiple search queries.
+ *
+ * Developers should extend this class to create specific search engine implementations
+ * (e.g., Google, Bing, DuckDuckGo).
+ *
+ * @example
+ * ```typescript
+ * class MySearcher extends WebSearcher {
+ *   get template() {
+ *     return { url: '...' };
+ *   }
+ * }
+ * WebSearcher.register(MySearcher);
+ * ```
+ */
+declare abstract class WebSearcher extends FetchSession {
+    static _isFactory: boolean;
+    /**
+     * Custom engine name. If not provided, it is derived from the class name.
+     * For example, `GoogleSearcher` becomes `Google`.
+     */
+    static name?: string;
+    /**
+     * Engine alias(es). Can be a single string or an array of strings.
+     * Useful for registering shorthand names (e.g., 'g' for 'Google').
+     */
+    static alias?: string | string[];
+    /**
+     * Registers a search engine class.
+     *
+     * @param ctor - The search engine class to register.
+     * @param options - Registration options. If a string is provided, it is used as the registered name.
+     * @returns `true` if registration was successful.
+     */
+    static register: (ctor: typeof WebSearcher, options?: IBaseFactoryOptions | string) => boolean;
+    /**
+     * Unregisters a search engine.
+     *
+     * @param name - The name or class to unregister.
+     */
+    static unregister: (name?: string | typeof WebSearcher) => void;
+    /**
+     * Retrieves a registered search engine class by name.
+     *
+     * @param name - The name of the engine (e.g., 'Google').
+     * @returns The search engine class constructor.
+     */
+    static get: (name: string) => typeof WebSearcher;
+    /**
+     * Creates an instance of the registered search engine.
+     *
+     * @param name - The name of the engine.
+     * @param args - Arguments to pass to the constructor.
+     * @returns An instance of the search engine.
+     */
+    static createObject: (name: string, ...args: any[]) => WebSearcher;
+    /**
+     * Iterates over all registered engines.
+     *
+     * @param cb - Callback function to invoke for each registered engine.
+     */
+    static forEach: (cb: (ctor: typeof WebSearcher, name: string) => void) => void;
+    /**
+     * Sets aliases for a registered engine.
+     *
+     * @param ctor - The search engine class.
+     * @param aliases - Aliases to add.
+     */
+    static setAliases: (ctor: typeof WebSearcher, ...aliases: string[]) => void;
+    /**
+     * Static helper to execute a one-off search.
+     *
+     * It creates an instance of the specified engine, executes the search, and then
+     * automatically disposes of the session.
+     *
+     * @param engineName - The name of the engine to use (e.g., 'Google').
+     * @param query - The search query string.
+     * @param options - Combined search options and fetcher options.
+     * @returns A promise resolving to an array of standardized search results.
+     */
+    static search(engineName: string, query: string, options?: SearchOptions & FetcherOptions): Promise<StandardSearchResult[]>;
+    /**
+     * The declarative template for the fetch options.
+     *
+     * Subclasses **must** implement this getter to provide the engine configuration,
+     * including the base URL, search parameters pattern, and extraction rules.
+     *
+     * Supports variable injection using syntax like `${query}`, `${offset}`, etc.
+     *
+     * @example
+     * ```typescript
+     * get template() {
+     *   return {
+     *     url: 'https://example.com/search?q=${query}',
+     *     actions: [ ... ]
+     *   };
+     * }
+     * ```
+     */
+    abstract get template(): FetcherOptions;
+    /**
+     * Optional pagination configuration.
+     * Defines how the searcher navigates to subsequent pages.
+     *
+     * If undefined, the searcher will only fetch the first page.
+     */
+    get pagination(): PaginationConfig | undefined;
+    protected createContext(options?: FetcherOptions): _isdk_web_fetcher.FetchContext;
+    /**
+     * Executes a search query.
+     *
+     * This method handles the pagination loop, variable injection, fetching,
+     * and result transformation.
+     *
+     * @param query - The search query string.
+     * @param options - Optional search parameters (e.g., limit, timeRange).
+     * @returns A promise resolving to an array of standardized search results.
+     */
+    search(query: string, options?: SearchOptions): Promise<StandardSearchResult[]>;
+    /**
+     * Transform and clean the raw extracted results.
+     *
+     * Subclasses should override this method to provide engine-specific cleaning,
+     * normalization, or post-processing of the data extracted by the fetcher.
+     *
+     * @param outputs - The complete outputs object from the fetch actions.
+     * @param context - The search context (query, page, etc.).
+     * @returns A promise resolving to an array of standardized search results.
+     */
+    protected transform(outputs: Record<string, any>, context: SearchContext): Promise<StandardSearchResult[]>;
+    /**
+     * Transforms standard options into engine-specific template variables.
+     *
+     * Subclasses should override this to map standard options like 'timeRange',
+     * 'category', 'region' into the specific URL parameters required by the engine
+     * (e.g., mapping `timeRange: 'day'` to `tbs: 'qdr:d'` for Google).
+     *
+     * @param options - The search options provided by the user.
+     * @returns A dictionary of variables to be injected into the template.
+     */
+    protected formatOptions(options: SearchOptions): Record<string, any>;
+}
+
+/**
+ * A sample implementation of a Google Search scraper.
+ *
+ * @remarks
+ * **⚠️ DEMO ONLY ⚠️**
+ *
+ * This class serves as a **reference implementation** to demonstrate how to extend
+ * the `WebSearcher` base class. It is **NOT intended for production use**.
+ *
+ * Google frequently changes its HTML structure and employs sophisticated anti-bot measures.
+ * A production-grade Google scraper would require robust proxy rotation, CAPTCHA solving,
+ * and constant maintenance of selectors, or usage of an official API.
+ *
+ * Use this class to understand:
+ * 1. How to define a fetch template with variable injection.
+ * 2. How to map standard options (like time range) to engine-specific URL parameters.
+ * 3. How to handle pagination.
+ * 4. How to transform and clean raw extracted data.
+ */
+declare class GoogleSearcher extends WebSearcher {
+    static alias: string[];
+    /**
+     * Defines the fetch template for Google Search.
+     *
+     * @returns The fetcher configuration including the URL pattern and extraction rules.
+     */
+    get template(): FetcherOptions;
+    /**
+     * Configures pagination for Google Search results.
+     * Uses the 'start' URL parameter, incrementing by 10 for each page.
+     */
+    get pagination(): PaginationConfig;
+    /**
+     * Maps standard `SearchOptions` to Google's specific URL parameters.
+     *
+     * - `timeRange` -> `tbs` (e.g., 'qdr:d' for day)
+     * - `category` -> `tbm` (e.g., 'isch' for images)
+     * - `region` -> `gl`
+     * - `language` -> `hl`
+     * - `safeSearch` -> `safe`
+     *
+     * @param options - The user-provided search options.
+     * @returns A map of variables to inject into the URL template.
+     */
+    protected formatOptions(options: SearchOptions): Record<string, any>;
+    /**
+     * Cleans and normalizes the extracted results.
+     * Specifically, it unwraps Google's redirect URLs (starting with `/url?q=`).
+     *
+     * @param outputs - The raw outputs from the fetcher.
+     * @returns An array of cleaned search results.
+     */
+    protected transform(outputs: Record<string, any>): Promise<any[]>;
+}
+
+export { type CustomTimeRange, GoogleSearcher, type PaginationConfig, type SafeSearchLevel, type SearchCategory, type SearchContext, type SearchOptions, type SearchTimeRange, type SearchTimeRangePreset, type SearcherConstructor, type StandardSearchResult, WebSearcher };

package/dist/index.d.ts

@@ -0,0 +1,321 @@
+import * as _isdk_web_fetcher from '@isdk/web-fetcher';
+import { FetcherOptions, FetchSession } from '@isdk/web-fetcher';
+import { IBaseFactoryOptions } from 'custom-factory';
+
+/**
+ * Interface representing a standardized search result item.
+ * This ensures consistency across different search engines.
+ */
+interface StandardSearchResult {
+    /** The title of the search result. */
+    title: string;
+    /** The URL of the search result. */
+    url: string;
+    /** A brief snippet or description of the result. */
+    snippet?: string;
+    /** An optional image URL associated with the result. */
+    image?: string;
+    /** Allows for engine-specific extra fields (e.g., rank, author, date). */
+    [key: string]: any;
+}
+/**
+ * Configuration for pagination strategies.
+ * Defines how the searcher should navigate to the next page of results.
+ */
+interface PaginationConfig {
+    /**
+     * The type of pagination mechanism:
+     * - 'url-param': Pagination is handled by modifying URL parameters (e.g., `?page=2` or `?start=10`).
+     * - 'click-next': Pagination is handled by clicking a "Next" button on the page (only works in 'browser' mode).
+     */
+    type: 'url-param' | 'click-next';
+    /**
+     * The name of the URL parameter used for pagination.
+     * Required if type is 'url-param'.
+     * @example 'start' for Google, 'page' or 'p' for others.
+     */
+    paramName?: string;
+    /**
+     * The starting value for the pagination parameter.
+     * @default 0
+     */
+    startValue?: number;
+    /**
+     * The increment step for each page.
+     * - If the parameter represents an item offset (like Google's 'start'), this might be 10.
+     * - If the parameter represents a page number, this is usually 1.
+     * @default 1
+     */
+    increment?: number;
+    /**
+     * The CSS selector for the "Next" page button.
+     * Required if type is 'click-next'.
+     */
+    nextButtonSelector?: string;
+}
+/**
+ * Context object passed to the transform function.
+ */
+interface SearchContext {
+    /** The original search query. */
+    query: string;
+    /** The current page index (0-based). */
+    page: number;
+    /** The requested limit of results. */
+    limit?: number;
+}
+type SearchTimeRangePreset = 'all' | 'day' | 'week' | 'month' | 'year';
+interface CustomTimeRange {
+    /** Start date (Date object or string like 'YYYY-MM-DD'). */
+    from: Date | string;
+    /** End date (Date object or string like 'YYYY-MM-DD'). Defaults to current date if omitted. */
+    to?: Date | string;
+}
+type SearchTimeRange = SearchTimeRangePreset | CustomTimeRange;
+type SearchCategory = 'all' | 'images' | 'videos' | 'news';
+type SafeSearchLevel = 'off' | 'moderate' | 'strict';
+/**
+ * Options provided when executing a search.
+ */
+interface SearchOptions {
+    /** The maximum number of results to retrieve. */
+    limit?: number;
+    /**
+     * Date range for the search results.
+     * Default: 'all'
+     */
+    timeRange?: SearchTimeRange;
+    /**
+     * The category of results to return.
+     * Default: 'all' (web search)
+     */
+    category?: SearchCategory;
+    /**
+     * Region code (ISO 3166-1 alpha-2) to bias results (e.g., 'US', 'CN', 'JP').
+     */
+    region?: string;
+    /**
+     * Language code (ISO 639-1) for the interface or results (e.g., 'en', 'zh-CN').
+     */
+    language?: string;
+    /**
+     * Safe search filtering level.
+     * Default: engine dependent (usually 'moderate' or 'strict' by default).
+     */
+    safeSearch?: SafeSearchLevel;
+    /**
+     * A custom transform function to filter or modify results at runtime.
+     * This runs AFTER the engine-level transform.
+     */
+    transform?: (results: StandardSearchResult[], context: SearchContext) => Promise<StandardSearchResult[]> | StandardSearchResult[];
+    /** Any other custom variables to be injected into the template. */
+    [key: string]: any;
+}
+
+/**
+ * Constructor definition for Searcher subclasses.
+ */
+type SearcherConstructor = new (options?: FetcherOptions) => WebSearcher;
+/**
+ * The abstract base class for all search engines.
+ *
+ * It extends `FetchSession`, meaning each `WebSearcher` instance is an active session
+ * capable of maintaining state (e.g., cookies, local storage) across multiple search queries.
+ *
+ * Developers should extend this class to create specific search engine implementations
+ * (e.g., Google, Bing, DuckDuckGo).
+ *
+ * @example
+ * ```typescript
+ * class MySearcher extends WebSearcher {
+ *   get template() {
+ *     return { url: '...' };
+ *   }
+ * }
+ * WebSearcher.register(MySearcher);
+ * ```
+ */
+declare abstract class WebSearcher extends FetchSession {
+    static _isFactory: boolean;
+    /**
+     * Custom engine name. If not provided, it is derived from the class name.
+     * For example, `GoogleSearcher` becomes `Google`.
+     */
+    static name?: string;
+    /**
+     * Engine alias(es). Can be a single string or an array of strings.
+     * Useful for registering shorthand names (e.g., 'g' for 'Google').
+     */
+    static alias?: string | string[];
+    /**
+     * Registers a search engine class.
+     *
+     * @param ctor - The search engine class to register.
+     * @param options - Registration options. If a string is provided, it is used as the registered name.
+     * @returns `true` if registration was successful.
+     */
+    static register: (ctor: typeof WebSearcher, options?: IBaseFactoryOptions | string) => boolean;
+    /**
+     * Unregisters a search engine.
+     *
+     * @param name - The name or class to unregister.
+     */
+    static unregister: (name?: string | typeof WebSearcher) => void;
+    /**
+     * Retrieves a registered search engine class by name.
+     *
+     * @param name - The name of the engine (e.g., 'Google').
+     * @returns The search engine class constructor.
+     */
+    static get: (name: string) => typeof WebSearcher;
+    /**
+     * Creates an instance of the registered search engine.
+     *
+     * @param name - The name of the engine.
+     * @param args - Arguments to pass to the constructor.
+     * @returns An instance of the search engine.
+     */
+    static createObject: (name: string, ...args: any[]) => WebSearcher;
+    /**
+     * Iterates over all registered engines.
+     *
+     * @param cb - Callback function to invoke for each registered engine.
+     */
+    static forEach: (cb: (ctor: typeof WebSearcher, name: string) => void) => void;
+    /**
+     * Sets aliases for a registered engine.
+     *
+     * @param ctor - The search engine class.
+     * @param aliases - Aliases to add.
+     */
+    static setAliases: (ctor: typeof WebSearcher, ...aliases: string[]) => void;
+    /**
+     * Static helper to execute a one-off search.
+     *
+     * It creates an instance of the specified engine, executes the search, and then
+     * automatically disposes of the session.
+     *
+     * @param engineName - The name of the engine to use (e.g., 'Google').
+     * @param query - The search query string.
+     * @param options - Combined search options and fetcher options.
+     * @returns A promise resolving to an array of standardized search results.
+     */
+    static search(engineName: string, query: string, options?: SearchOptions & FetcherOptions): Promise<StandardSearchResult[]>;
+    /**
+     * The declarative template for the fetch options.
+     *
+     * Subclasses **must** implement this getter to provide the engine configuration,
+     * including the base URL, search parameters pattern, and extraction rules.
+     *
+     * Supports variable injection using syntax like `${query}`, `${offset}`, etc.
+     *
+     * @example
+     * ```typescript
+     * get template() {
+     *   return {
+     *     url: 'https://example.com/search?q=${query}',
+     *     actions: [ ... ]
+     *   };
+     * }
+     * ```
+     */
+    abstract get template(): FetcherOptions;
+    /**
+     * Optional pagination configuration.
+     * Defines how the searcher navigates to subsequent pages.
+     *
+     * If undefined, the searcher will only fetch the first page.
+     */
+    get pagination(): PaginationConfig | undefined;
+    protected createContext(options?: FetcherOptions): _isdk_web_fetcher.FetchContext;
+    /**
+     * Executes a search query.
+     *
+     * This method handles the pagination loop, variable injection, fetching,
+     * and result transformation.
+     *
+     * @param query - The search query string.
+     * @param options - Optional search parameters (e.g., limit, timeRange).
+     * @returns A promise resolving to an array of standardized search results.
+     */
+    search(query: string, options?: SearchOptions): Promise<StandardSearchResult[]>;
+    /**
+     * Transform and clean the raw extracted results.
+     *
+     * Subclasses should override this method to provide engine-specific cleaning,
+     * normalization, or post-processing of the data extracted by the fetcher.
+     *
+     * @param outputs - The complete outputs object from the fetch actions.
+     * @param context - The search context (query, page, etc.).
+     * @returns A promise resolving to an array of standardized search results.
+     */
+    protected transform(outputs: Record<string, any>, context: SearchContext): Promise<StandardSearchResult[]>;
+    /**
+     * Transforms standard options into engine-specific template variables.
+     *
+     * Subclasses should override this to map standard options like 'timeRange',
+     * 'category', 'region' into the specific URL parameters required by the engine
+     * (e.g., mapping `timeRange: 'day'` to `tbs: 'qdr:d'` for Google).
+     *
+     * @param options - The search options provided by the user.
+     * @returns A dictionary of variables to be injected into the template.
+     */
+    protected formatOptions(options: SearchOptions): Record<string, any>;
+}
+
+/**
+ * A sample implementation of a Google Search scraper.
+ *
+ * @remarks
+ * **⚠️ DEMO ONLY ⚠️**
+ *
+ * This class serves as a **reference implementation** to demonstrate how to extend
+ * the `WebSearcher` base class. It is **NOT intended for production use**.
+ *
+ * Google frequently changes its HTML structure and employs sophisticated anti-bot measures.
+ * A production-grade Google scraper would require robust proxy rotation, CAPTCHA solving,
+ * and constant maintenance of selectors, or usage of an official API.
+ *
+ * Use this class to understand:
+ * 1. How to define a fetch template with variable injection.
+ * 2. How to map standard options (like time range) to engine-specific URL parameters.
+ * 3. How to handle pagination.
+ * 4. How to transform and clean raw extracted data.
+ */
+declare class GoogleSearcher extends WebSearcher {
+    static alias: string[];
+    /**
+     * Defines the fetch template for Google Search.
+     *
+     * @returns The fetcher configuration including the URL pattern and extraction rules.
+     */
+    get template(): FetcherOptions;
+    /**
+     * Configures pagination for Google Search results.
+     * Uses the 'start' URL parameter, incrementing by 10 for each page.
+     */
+    get pagination(): PaginationConfig;
+    /**
+     * Maps standard `SearchOptions` to Google's specific URL parameters.
+     *
+     * - `timeRange` -> `tbs` (e.g., 'qdr:d' for day)
+     * - `category` -> `tbm` (e.g., 'isch' for images)
+     * - `region` -> `gl`
+     * - `language` -> `hl`
+     * - `safeSearch` -> `safe`
+     *
+     * @param options - The user-provided search options.
+     * @returns A map of variables to inject into the URL template.
+     */
+    protected formatOptions(options: SearchOptions): Record<string, any>;
+    /**
+     * Cleans and normalizes the extracted results.
+     * Specifically, it unwraps Google's redirect URLs (starting with `/url?q=`).
+     *
+     * @param outputs - The raw outputs from the fetcher.
+     * @returns An array of cleaned search results.
+     */
+    protected transform(outputs: Record<string, any>): Promise<any[]>;
+}
+
+export { type CustomTimeRange, GoogleSearcher, type PaginationConfig, type SafeSearchLevel, type SearchCategory, type SearchContext, type SearchOptions, type SearchTimeRange, type SearchTimeRangePreset, type SearcherConstructor, type StandardSearchResult, WebSearcher };

package/dist/index.js

@@ -0,0 +1 @@
+"use strict";var t,e=Object.defineProperty,r=Object.getOwnPropertyDescriptor,s=Object.getOwnPropertyNames,a=Object.prototype.hasOwnProperty,i={};((t,r)=>{for(var s in r)e(t,s,{get:r[s],enumerable:!0})})(i,{GoogleSearcher:()=>h,WebSearcher:()=>f}),module.exports=(t=i,((t,i,n,o)=>{if(i&&"object"==typeof i||"function"==typeof i)for(let c of s(i))a.call(t,c)||c===n||e(t,c,{get:()=>i[c],enumerable:!(o=r(i,c))||o.enumerable});return t})(e({},"__esModule",{value:!0}),t));var n=require("@isdk/web-fetcher"),o=require("custom-factory"),c=require("lodash-es");function l(t,e){if("string"==typeof t)return t.replace(/\$\{(.*?)\}/g,(t,r)=>{const s=e[r.trim()];return void 0!==s?String(s):""});if(Array.isArray(t))return t.map(t=>l(t,e));if((0,c.isPlainObject)(t)){const r={};for(const s in t)Object.prototype.hasOwnProperty.call(t,s)&&(r[s]=l(t[s],e));return r}return t}var u=require("lodash-es"),f=class extends n.FetchSession{static async search(t,e,r={}){const s=this.createObject(t,r);if(!s)throw new Error(`Search engine not found: ${t}`);try{return await s.search(e,r)}finally{await s.dispose()}}get pagination(){}createContext(t=this.options){const e=this.template,r=(0,u.defaultsDeep)({},e,t);return e.engine&&"auto"!==e.engine||!t.engine||(r.engine=t.engine),super.createContext(r)}async search(t,e={}){const r=e.limit||10,s=[];let a=0;const i=this.pagination?.startValue??0,n=this.pagination?.increment??1;for(;s.length<r;){const o=this.formatOptions(e),c=i+a*n,f={...e,...o,query:t,page:a+i,offset:c,limit:r},h=l(this.template,f),m=(0,u.defaultsDeep)({},h,e),d=[];if(0===a||"url-param"===this.pagination?.type?m.url&&d.push({id:"goto",params:{url:m.url}}):"click-next"===this.pagination?.type&&this.pagination.nextButtonSelector&&(d.push({id:"click",params:{selector:this.pagination.nextButtonSelector}}),d.push({id:"waitFor",params:{networkIdle:!0,ms:500}})),m.actions){const t=m.actions.filter(t=>!(d.length>0&&"goto"===d[0].id&&"goto"===t.id));d.push(...t)}m.engine&&this.context.engine!==m.engine&&m.engine;const{outputs:g}=await this.executeAll(d),p={query:t,page:a,limit:e.limit};let w=[];if(w=await this.transform(g,p),e.transform&&(w=await e.transform(w,p)),!w||0===w.length)break;if(s.push(...w),s.length>=r||!this.pagination)break;if(a++,a>10)break}return s.slice(0,r)}async transform(t,e){return t.results||[]}formatOptions(t){return{...t}}};f._isFactory=!1,(0,o.addBaseFactoryAbility)(f),f.prototype.name="Searcher";var h=class extends f{get template(){return{engine:"browser",browser:{headless:!1},url:"https://www.google.com/search?q=${query}&start=${offset}&tbs=${tbs}&tbm=${tbm}&gl=${gl}&hl=${hl}&safe=${safe}",actions:[{id:"extract",storeAs:"results",params:{type:"array",selector:"#main #search",items:{url:{selector:"a:has(h3)",attribute:"href",required:!0},title:{selector:"a:has(h3) h3",required:!0,mode:"innerText"},snippet:{selector:"div[style*='-webkit-line-clamp']",type:"html"}}}}]}}get pagination(){return{type:"url-param",paramName:"start",startValue:0,increment:10}}formatOptions(t){const e={};if(t.timeRange)if("string"==typeof t.timeRange){const r={day:"qdr:d",week:"qdr:w",month:"qdr:m",year:"qdr:y"};r[t.timeRange]&&(e.tbs=r[t.timeRange])}else{const r=new Date(t.timeRange.from),s=t.timeRange.to?new Date(t.timeRange.to):new Date;if(!isNaN(r.getTime())&&!isNaN(s.getTime())){const t=t=>`${t.getMonth()+1}/${t.getDate()}/${t.getFullYear()}`;e.tbs=`cdr:1,cd_min:${t(r)},cd_max:${t(s)}`}}if(t.category){const r={images:"isch",videos:"vid",news:"nws"};r[t.category]&&(e.tbm=r[t.category])}return t.region&&(e.gl=t.region),t.language&&(e.hl=t.language),t.safeSearch&&("strict"===t.safeSearch?e.safe="active":"off"===t.safeSearch&&(e.safe="images")),e}async transform(t){const e=t.results||[];return Array.isArray(e)?e.map(t=>{if(t.url&&t.url.startsWith("/url?q="))try{const e=new URL(t.url,"https://www.google.com").searchParams.get("q");e&&(t.url=e)}catch(t){}return t}):[]}};h.alias=["google"];

package/dist/index.mjs

@@ -0,0 +1 @@
+import{FetchSession as t}from"@isdk/web-fetcher";import{addBaseFactoryAbility as r}from"custom-factory";import{isPlainObject as e}from"lodash-es";function s(t,r){if("string"==typeof t)return t.replace(/\$\{(.*?)\}/g,(t,e)=>{const s=r[e.trim()];return void 0!==s?String(s):""});if(Array.isArray(t))return t.map(t=>s(t,r));if(e(t)){const e={};for(const a in t)Object.prototype.hasOwnProperty.call(t,a)&&(e[a]=s(t[a],r));return e}return t}import{defaultsDeep as a}from"lodash-es";var i=class extends t{static async search(t,r,e={}){const s=this.createObject(t,e);if(!s)throw new Error(`Search engine not found: ${t}`);try{return await s.search(r,e)}finally{await s.dispose()}}get pagination(){}createContext(t=this.options){const r=this.template,e=a({},r,t);return r.engine&&"auto"!==r.engine||!t.engine||(e.engine=t.engine),super.createContext(e)}async search(t,r={}){const e=r.limit||10,i=[];let o=0;const n=this.pagination?.startValue??0,c=this.pagination?.increment??1;for(;i.length<e;){const l=this.formatOptions(r),m=n+o*c,h={...r,...l,query:t,page:o+n,offset:m,limit:e},f=s(this.template,h),u=a({},f,r),p=[];if(0===o||"url-param"===this.pagination?.type?u.url&&p.push({id:"goto",params:{url:u.url}}):"click-next"===this.pagination?.type&&this.pagination.nextButtonSelector&&(p.push({id:"click",params:{selector:this.pagination.nextButtonSelector}}),p.push({id:"waitFor",params:{networkIdle:!0,ms:500}})),u.actions){const t=u.actions.filter(t=>!(p.length>0&&"goto"===p[0].id&&"goto"===t.id));p.push(...t)}u.engine&&this.context.engine!==u.engine&&u.engine;const{outputs:d}=await this.executeAll(p),w={query:t,page:o,limit:r.limit};let g=[];if(g=await this.transform(d,w),r.transform&&(g=await r.transform(g,w)),!g||0===g.length)break;if(i.push(...g),i.length>=e||!this.pagination)break;if(o++,o>10)break}return i.slice(0,e)}async transform(t,r){return t.results||[]}formatOptions(t){return{...t}}};i._isFactory=!1,r(i),i.prototype.name="Searcher";var o=class extends i{get template(){return{engine:"browser",browser:{headless:!1},url:"https://www.google.com/search?q=${query}&start=${offset}&tbs=${tbs}&tbm=${tbm}&gl=${gl}&hl=${hl}&safe=${safe}",actions:[{id:"extract",storeAs:"results",params:{type:"array",selector:"#main #search",items:{url:{selector:"a:has(h3)",attribute:"href",required:!0},title:{selector:"a:has(h3) h3",required:!0,mode:"innerText"},snippet:{selector:"div[style*='-webkit-line-clamp']",type:"html"}}}}]}}get pagination(){return{type:"url-param",paramName:"start",startValue:0,increment:10}}formatOptions(t){const r={};if(t.timeRange)if("string"==typeof t.timeRange){const e={day:"qdr:d",week:"qdr:w",month:"qdr:m",year:"qdr:y"};e[t.timeRange]&&(r.tbs=e[t.timeRange])}else{const e=new Date(t.timeRange.from),s=t.timeRange.to?new Date(t.timeRange.to):new Date;if(!isNaN(e.getTime())&&!isNaN(s.getTime())){const t=t=>`${t.getMonth()+1}/${t.getDate()}/${t.getFullYear()}`;r.tbs=`cdr:1,cd_min:${t(e)},cd_max:${t(s)}`}}if(t.category){const e={images:"isch",videos:"vid",news:"nws"};e[t.category]&&(r.tbm=e[t.category])}return t.region&&(r.gl=t.region),t.language&&(r.hl=t.language),t.safeSearch&&("strict"===t.safeSearch?r.safe="active":"off"===t.safeSearch&&(r.safe="images")),r}async transform(t){const r=t.results||[];return Array.isArray(r)?r.map(t=>{if(t.url&&t.url.startsWith("/url?q="))try{const r=new URL(t.url,"https://www.google.com").searchParams.get("q");r&&(t.url=r)}catch(t){}return t}):[]}};o.alias=["google"];export{o as GoogleSearcher,i as WebSearcher};