@firekid/scraper 1.0.1

package/dist/index.d.mts

@@ -0,0 +1,366 @@
+import { Page, Browser, BrowserContext } from 'playwright';
+import winston from 'winston';
+
+interface FirekidConfig {
+    headless?: boolean;
+    bypassCloudflare?: boolean;
+    maxWorkers?: number;
+    timeout?: number;
+    dataDir?: string;
+    logLevel?: 'error' | 'warn' | 'info' | 'debug';
+}
+interface ScrapingResult {
+    success: boolean;
+    data: any;
+    errors: string[];
+    timestamp: number;
+}
+type CmdAction = 'GOTO' | 'BACK' | 'FORWARD' | 'REFRESH' | 'CLICK' | 'TYPE' | 'PRESS' | 'SELECT' | 'CHECK' | 'UPLOAD' | 'WAIT' | 'WAITLOAD' | 'SCROLL' | 'SCROLLDOWN' | 'SCAN' | 'EXTRACT' | 'SCREENSHOT' | 'PAGINATE' | 'INFINITESCROLL' | 'FETCH' | 'DOWNLOAD' | 'REFERER' | 'BYPASS_CLOUDFLARE' | 'REPEAT' | 'IF' | 'LOOP';
+interface CmdStep {
+    action: CmdAction;
+    args: string[];
+    children?: CmdStep[];
+    line: number;
+}
+interface CmdFile {
+    site: string;
+    steps: CmdStep[];
+    raw: string;
+}
+interface Seed {
+    id: string;
+    chromeVersion: string;
+    screenWidth: number;
+    screenHeight: number;
+    language: string;
+    timezone: string;
+    canvasNoise: number;
+    webglVendor: string;
+    webglRenderer: string;
+    audioNoise: number;
+    fonts: string[];
+}
+interface GhostOptions {
+    siteHost?: string;
+    fresh?: boolean;
+    seed?: Seed;
+}
+interface CFTokens {
+    cfClearance?: string;
+    cfBm?: string;
+    headers?: Record<string, string>;
+}
+interface FetchOptions {
+    url: string;
+    referer?: string;
+    autoReferer?: boolean;
+    method?: 'GET' | 'POST';
+    headers?: Record<string, string>;
+    cookies?: Record<string, string>;
+    body?: any;
+    followRedirects?: boolean;
+    timeout?: number;
+}
+interface FetchResponse {
+    status: number;
+    headers: Record<string, string>;
+    data: any;
+}
+interface SitePattern {
+    type: string;
+    selectors: Record<string, string>;
+    flow: string[];
+    successRate: number;
+}
+interface RecordedAction {
+    type: string;
+    selectors: SelectorSet;
+    value?: string;
+    timestamp: number;
+    element?: any;
+    text?: string;
+    fieldType?: string;
+}
+interface SelectorSet {
+    primary: string;
+    fallbacks: string[];
+}
+type Mode = 'downloader' | 'scrape' | 'navigator' | 'ssr' | 'api-hunter' | 'auto';
+interface FormField {
+    selector: string;
+    fallbackSelectors: string[];
+    type: FieldType;
+    label: string | null;
+    placeholder: string;
+    required: boolean;
+    validation: ValidationRule[];
+}
+type FieldType = 'email' | 'password' | 'username' | 'phone' | 'name' | 'text';
+interface ValidationRule {
+    type: 'regex' | 'minlength' | 'maxlength';
+    value?: any;
+    pattern?: string;
+}
+interface FormTemplate {
+    fields: FormField[];
+    submitButton: string;
+    formType: FormType;
+}
+type FormType = 'LOGIN' | 'REGISTRATION' | 'SEARCH' | 'CONTACT' | 'GENERIC';
+interface BehaviorProfile {
+    typingSpeed: {
+        min: number;
+        max: number;
+    };
+    mouseMovements: Array<{
+        x: number;
+        y: number;
+        timestamp: number;
+    }>;
+    scrollPatterns: Array<{
+        y: number;
+        timestamp: number;
+    }>;
+    pauseDistribution: number[];
+    clickTiming: Array<{
+        timestamp: number;
+    }>;
+}
+
+declare class FirekidScraper {
+    private config;
+    private browser;
+    private context;
+    private page;
+    private cfManager;
+    constructor(userConfig?: FirekidConfig);
+    init(): Promise<void>;
+    goto(url: string): Promise<void>;
+    extract(url: string, selectors: Record<string, string>): Promise<any>;
+    auto(url: string): Promise<ScrapingResult>;
+    runCommandFile(filePath: string): Promise<ScrapingResult>;
+    close(): Promise<void>;
+    getPage(): Page | null;
+    getBrowser(): Browser | null;
+    getContext(): BrowserContext | null;
+}
+
+declare class CommandParser {
+    private variables;
+    setVariable(key: string, value: string): void;
+    private resolve;
+    private parseLine;
+    parse(content: string, filePath?: string): CmdFile;
+    load(filePath: string): CmdFile;
+    findAll(dir?: string): string[];
+}
+
+interface CmdResult {
+    success: boolean;
+    skipped: string[];
+    extracted: Record<string, unknown>[];
+    errors: {
+        line: number;
+        action: string;
+        error: string;
+    }[];
+}
+declare class CommandExecutor {
+    private page;
+    private url;
+    private result;
+    private smartFetch;
+    private cfManager;
+    private variables;
+    constructor(page: Page, url: string);
+    execute(cmd: CmdFile): Promise<CmdResult>;
+    private runStep;
+    private goto;
+    private click;
+    private type;
+    private press;
+    private select;
+    private check;
+    private upload;
+    private wait;
+    private scroll;
+    private scrollDown;
+    private scan;
+    private extract;
+    private screenshot;
+    private paginate;
+    private infiniteScroll;
+    private fetch;
+    private download;
+    private setReferer;
+    private bypassCloudflare;
+    private repeat;
+    private conditional;
+    private loop;
+}
+
+declare class HumanBehavior {
+    private seed;
+    private profile;
+    constructor(seed: Seed);
+    private generateProfile;
+    randomDelay(min?: number, max?: number): Promise<void>;
+    humanClick(page: Page, selector: string): Promise<void>;
+    humanType(page: Page, selector: string, text: string): Promise<void>;
+    randomScroll(page: Page): Promise<void>;
+    randomMouseMovement(page: Page): Promise<void>;
+    getProfile(): BehaviorProfile;
+}
+
+declare function getNewSeed(): Seed;
+declare function getSeedForSite(siteHost: string): Seed;
+
+declare function applyGhost(context: BrowserContext, options?: GhostOptions): Promise<HumanBehavior>;
+
+declare class CloudflareManager {
+    detect(page: Page): Promise<boolean>;
+    waitForClearance(page: Page, timeoutMs?: number): Promise<boolean>;
+    extractTokens(context: BrowserContext): Promise<CFTokens>;
+    detectWAF(page: Page): Promise<string | null>;
+    handleCloudflare(page: Page, url: string): Promise<boolean>;
+    detectTurnstile(page: Page): Promise<boolean>;
+    handleTurnstile(page: Page, url: string): Promise<boolean>;
+    isTurnstileSolved(page: Page): Promise<boolean>;
+    waitForTurnstileSolved(page: Page): Promise<void>;
+}
+
+declare class SmartFetch {
+    private pageContext;
+    private lastReferer;
+    setPageContext(page: Page): void;
+    fetch(options: FetchOptions): Promise<FetchResponse>;
+    download(url: string, outputPath: string, referer?: string): Promise<void>;
+    getLastReferer(): string;
+}
+
+declare class ActionRecorder {
+    private browser;
+    private page;
+    private actions;
+    private isRecording;
+    private selectorGen;
+    private patternDetector;
+    private cmdGenerator;
+    private startUrl;
+    constructor();
+    startRecording(url: string): Promise<void>;
+    private attachListeners;
+    stopRecording(): Promise<void>;
+    getActions(): RecordedAction[];
+}
+
+declare class PatternCache {
+    private db;
+    constructor();
+    private initialize;
+    save(site: string, pattern: SitePattern): void;
+    load(site: string): SitePattern | null;
+    updateSuccessRate(site: string, success: boolean): void;
+    listAll(): Array<{
+        site: string;
+        pattern: SitePattern;
+    }>;
+    clear(): void;
+    close(): void;
+}
+
+declare const config: {
+    browser: {
+        headless: boolean;
+        timeout: number;
+        maxWorkers: number;
+    };
+    cloudflare: {
+        bypass: string;
+        turnstileSolver: string;
+    };
+    captcha: {
+        apiKey: string;
+    };
+    server: {
+        enabled: boolean;
+        port: number;
+        apiKey: string;
+    };
+    proxy: {
+        enabled: boolean;
+        url: string;
+    };
+    storage: {
+        dataDir: string;
+        patternsDb: string;
+        sessionsDb: string;
+    };
+    logging: {
+        level: "error" | "warn" | "info" | "debug";
+    };
+    recording: {
+        autoHideAfterSolve: boolean;
+        recordScreenshots: boolean;
+    };
+    rateLimit: {
+        enabled: boolean;
+        max: number;
+        window: number;
+    };
+    advanced: {
+        enableTelemetry: boolean;
+        enableAnalytics: boolean;
+    };
+};
+declare function getConfig(): {
+    browser: {
+        headless: boolean;
+        timeout: number;
+        maxWorkers: number;
+    };
+    cloudflare: {
+        bypass: string;
+        turnstileSolver: string;
+    };
+    captcha: {
+        apiKey: string;
+    };
+    server: {
+        enabled: boolean;
+        port: number;
+        apiKey: string;
+    };
+    proxy: {
+        enabled: boolean;
+        url: string;
+    };
+    storage: {
+        dataDir: string;
+        patternsDb: string;
+        sessionsDb: string;
+    };
+    logging: {
+        level: "error" | "warn" | "info" | "debug";
+    };
+    recording: {
+        autoHideAfterSolve: boolean;
+        recordScreenshots: boolean;
+    };
+    rateLimit: {
+        enabled: boolean;
+        max: number;
+        window: number;
+    };
+    advanced: {
+        enableTelemetry: boolean;
+        enableAnalytics: boolean;
+    };
+};
+declare function updateConfig(updates: Partial<typeof config>): void;
+
+declare const logger: winston.Logger;
+
+declare function setLogLevel(level: 'error' | 'warn' | 'info' | 'debug'): void;
+
+export { ActionRecorder, type BehaviorProfile, CloudflareManager, type CmdFile, type CmdStep, CommandExecutor, CommandParser, type FetchOptions, type FetchResponse, type FirekidConfig, FirekidScraper, type FormTemplate, type GhostOptions, type Mode, PatternCache, type ScrapingResult, type Seed, type SitePattern, SmartFetch, applyGhost, config, getConfig, getNewSeed, getSeedForSite, logger, setLogLevel, updateConfig };