@lobu/connector-sdk 6.1.1 → 7.0.0

package/dist/base.js

@@ -1,122 +0,0 @@
-import { Value } from '@sinclair/typebox/value';
-import { sdkLogger } from './logger.js';
-export class RateLimitError extends Error {
-    retryAfterMs;
-    constructor(message, retryAfterMs) {
-        super(message);
-        this.name = 'RateLimitError';
-        this.retryAfterMs = retryAfterMs;
-    }
-}
-/**
- * Base feed implementation with common functionality
- * All platform-specific feeds should extend this class
- */
-export class BaseFeed {
-    authSchema = { methods: [{ type: 'none' }] };
-    getParentFeedDefinitions(_options) {
-        return [];
-    }
-    /**
-     * Validate options using TypeBox schema
-     * Subclasses can call this for schema validation before adding custom business logic
-     */
-    validateWithSchema(options) {
-        try {
-            const errors = [...Value.Errors(this.optionsSchema, options)];
-            if (errors.length > 0) {
-                // Format first error for user-friendly message
-                const firstError = errors[0];
-                const field = firstError.path.replace(/^\//, '');
-                return `Invalid option ${field ? `"${field}"` : ''}: ${firstError.message}`;
-            }
-            return null;
-        }
-        catch (error) {
-            sdkLogger.error({ error }, '[BaseFeed] Schema validation error:');
-            return 'Invalid feed options format';
-        }
-    }
-    /**
-     * Get rate limit information for this platform
-     * Override this method in platform-specific feeds
-     * Default is conservative: 10 requests per minute
-     */
-    getRateLimit() {
-        return {
-            requests_per_minute: 10,
-            recommended_interval_ms: 6000, // 6 seconds
-        };
-    }
-    /**
-     * Helper to check if content is newer than checkpoint
-     */
-    isNewerThan(contentDate, checkpoint) {
-        if (!checkpoint || !checkpoint.last_timestamp)
-            return true;
-        return contentDate > checkpoint.last_timestamp;
-    }
-    /**
-     * Calculate lookback date from options
-     * @param options - Feed options with optional lookback_days
-     * @param defaultDays - Default lookback period (default: 365)
-     */
-    getLookbackDate(options, defaultDays = 365) {
-        const lookbackDays = options.lookback_days || defaultDays;
-        return new Date(Date.now() - lookbackDays * 24 * 60 * 60 * 1000);
-    }
-    /**
-     * Sleep for specified milliseconds (for rate limiting)
-     */
-    sleep(ms) {
-        return new Promise((resolve) => setTimeout(resolve, ms));
-    }
-    /**
-     * Check if feed is in incremental mode
-     */
-    isIncrementalMode(checkpoint, paginationToken) {
-        return !!checkpoint?.last_timestamp && !paginationToken;
-    }
-    /**
-     * Helper to deduplicate content by origin_id
-     */
-    deduplicate(contents, seenIds) {
-        return contents.filter((c) => {
-            if (!c.origin_id)
-                return false;
-            if (seenIds.has(c.origin_id))
-                return false;
-            seenIds.add(c.origin_id);
-            return true;
-        });
-    }
-    /**
-     * Handle HTTP errors with structured logging and platform-specific messages
-     */
-    handleHTTPError(status, context, platformName) {
-        const platform = platformName || this.displayName;
-        sdkLogger.error({
-            status,
-            context,
-            platform: this.type,
-            timestamp: new Date().toISOString(),
-        }, `[${platform}Feed] HTTP ${status} error:`);
-        const errorMessages = {
-            400: `Bad request to ${platform}: ${context}. Check your feed options.`,
-            401: `Authentication failed for ${platform}. Check your API credentials.`,
-            403: `Access forbidden to ${platform} resource: ${context}. The resource may be private or require authentication.`,
-            404: `Resource not found on ${platform}: ${context}. Verify the resource exists.`,
-            422: `Invalid request to ${platform}: ${context}. Check your parameters.`,
-            429: `${platform} rate limit exceeded. Please wait before retrying.`,
-            500: `${platform} server error (${status}). This is temporary, please retry later.`,
-            502: `${platform} bad gateway (${status}). This is temporary, please retry later.`,
-            503: `${platform} service unavailable (${status}). This is temporary, please retry later.`,
-        };
-        const message = errorMessages[status] || `${platform} API error: ${status}`;
-        if (status === 429) {
-            throw new RateLimitError(message);
-        }
-        throw new Error(message);
-    }
-}
-//# sourceMappingURL=base.js.map

package/dist/base.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"base.js","sourceRoot":"","sources":["../src/base.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAaxC,MAAM,OAAO,cAAe,SAAQ,KAAK;IAC9B,YAAY,CAAU;IAE/B,YAAY,OAAe,EAAE,YAAqB;QAChD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;QAC7B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,OAAgB,QAAQ;IAQnB,UAAU,GAAmB,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAgBtE,wBAAwB,CAAC,QAAqB;QAC5C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;;OAGG;IACO,kBAAkB,CAAC,OAAoB;QAC/C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC;YAC9D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,+CAA+C;gBAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC7B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACjD,OAAO,kBAAkB,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,UAAU,CAAC,OAAO,EAAE,CAAC;YAC9E,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,qCAAqC,CAAC,CAAC;YAClE,OAAO,6BAA6B,CAAC;QACvC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,YAAY;QACV,OAAO;YACL,mBAAmB,EAAE,EAAE;YACvB,uBAAuB,EAAE,IAAI,EAAE,YAAY;SAC5C,CAAC;IACJ,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,WAAiB,EAAE,UAA6B;QACpE,IAAI,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAC3D,OAAO,WAAW,GAAG,UAAU,CAAC,cAAc,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACO,eAAe,CAAC,OAAoB,EAAE,cAAsB,GAAG;QACvE,MAAM,YAAY,GAAG,OAAO,CAAC,aAAa,IAAI,WAAW,CAAC;QAC1D,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IACnE,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,EAAU;QACxB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACO,iBAAiB,CACzB,UAA6B,EAC7B,eAA+B;QAE/B,OAAO,CAAC,CAAC,UAAU,EAAE,cAAc,IAAI,CAAC,eAAe,CAAC;IAC1D,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,QAAmB,EAAE,OAAoB;QAC7D,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3B,IAAI,CAAC,CAAC,CAAC,SAAS;gBAAE,OAAO,KAAK,CAAC;YAC/B,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC3C,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACzB,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,MAAc,EAAE,OAAe,EAAE,YAAqB;QAC9E,MAAM,QAAQ,GAAG,YAAY,IAAI,IAAI,CAAC,WAAW,CAAC;QAElD,SAAS,CAAC,KAAK,CACb;YACE,MAAM;YACN,OAAO;YACP,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,EACD,IAAI,QAAQ,cAAc,MAAM,SAAS,CAC1C,CAAC;QAEF,MAAM,aAAa,GAA2B;YAC5C,GAAG,EAAE,kBAAkB,QAAQ,KAAK,OAAO,4BAA4B;YACvE,GAAG,EAAE,6BAA6B,QAAQ,+BAA+B;YACzE,GAAG,EAAE,uBAAuB,QAAQ,cAAc,OAAO,0DAA0D;YACnH,GAAG,EAAE,yBAAyB,QAAQ,KAAK,OAAO,+BAA+B;YACjF,GAAG,EAAE,sBAAsB,QAAQ,KAAK,OAAO,0BAA0B;YACzE,GAAG,EAAE,GAAG,QAAQ,oDAAoD;YACpE,GAAG,EAAE,GAAG,QAAQ,kBAAkB,MAAM,2CAA2C;YACnF,GAAG,EAAE,GAAG,QAAQ,iBAAiB,MAAM,2CAA2C;YAClF,GAAG,EAAE,GAAG,QAAQ,yBAAyB,MAAM,2CAA2C;SAC3F,CAAC;QAEF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,GAAG,QAAQ,eAAe,MAAM,EAAE,CAAC;QAC5E,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;CACF"}

package/dist/browser-paginated.d.ts

@@ -1,141 +0,0 @@
-/**
- * Browser Paginated Feed Base Class
- *
- * Extends PaginatedFeed for browser-based feeds (Playwright).
- */
-import type { Cookie, Page } from 'playwright';
-import type { PageFetchResult, PaginatedCheckpoint, PaginationConfig } from './paginated.js';
-import { PaginatedFeed } from './paginated.js';
-import type { Checkpoint, Env, FeedOptions, FeedSyncResult, SessionState } from './types.js';
-/**
- * Browser session state - Playwright-compatible format
- */
-export interface BrowserSessionState extends SessionState {
-    /** Playwright-compatible cookies */
-    cookies?: Cookie[];
-    /** localStorage key-value pairs (applied via page.evaluate) */
-    localStorage?: Record<string, string>;
-    /** Extra HTTP headers to set */
-    headers?: Record<string, string>;
-}
-/**
- * Cookie consent configuration for handling cookie banners
- */
-export interface CookieConsentConfig {
-    /** CSS selectors to detect cookie consent banner */
-    bannerSelectors: string[];
-    /** CSS selectors for accept button */
-    acceptSelectors: string[];
-    /** Timeout in ms to wait for banner (default: 2000) */
-    timeout?: number;
-}
-/**
- * CAPTCHA detection configuration
- */
-export interface CaptchaConfig {
-    /** Enable CAPTCHA detection (default: false) */
-    enabled: boolean;
-    /** CSS selectors that indicate CAPTCHA presence */
-    selectors?: string[];
-    /** Text patterns in page content that indicate CAPTCHA */
-    textPatterns?: string[];
-}
-/**
- * Browser configuration for the feed
- */
-export interface BrowserFeedConfig {
-    /** Enable stealth mode to avoid bot detection (default: true) */
-    stealth: boolean;
-    /** Custom viewport dimensions (default: browser default) */
-    viewport?: {
-        width: number;
-        height: number;
-    };
-    /** Custom user agent (default: browser default) */
-    userAgent?: string;
-    /** Navigation waitUntil option */
-    waitUntil: 'load' | 'domcontentloaded' | 'networkidle';
-    /** Navigation timeout in ms (default: 60000) */
-    navigationTimeout: number;
-    /** Cookie consent handling configuration */
-    cookieConsent?: CookieConsentConfig;
-    /** CAPTCHA detection configuration */
-    captcha?: CaptchaConfig;
-}
-/**
- * Browser-specific pagination configuration
- */
-export interface BrowserPaginationConfig extends PaginationConfig {
-    /** Number of pages to fetch per sync run (default: 1, G2 uses 5) */
-    pagesPerRun: number;
-    /** Delay between page navigations in ms (default: 2000) */
-    pageDelayMs: number;
-}
-/**
- * Base class for browser-based feeds with pagination
- */
-export declare abstract class BrowserPaginatedFeed<TItem, TCheckpoint extends PaginatedCheckpoint = PaginatedCheckpoint> extends PaginatedFeed<TItem, TCheckpoint> {
-    readonly apiType: "browser";
-    private _browser;
-    private _page;
-    private _screenshotDir;
-    private _isFirstPage;
-    private _currentPageNumber;
-    /**
-     * ABSTRACT: Get browser configuration for this feed
-     */
-    protected abstract getBrowserConfig(): BrowserFeedConfig;
-    /**
-     * ABSTRACT: Get base URL from feed options
-     */
-    protected abstract getBaseUrl(options: FeedOptions): string;
-    /**
-     * ABSTRACT: Build URL for a specific page number
-     */
-    protected abstract buildPageUrl(baseUrl: string, pageNumber: number): string;
-    /**
-     * ABSTRACT: Wait for page content to be ready after navigation
-     */
-    protected abstract waitForContent(page: Page): Promise<void>;
-    /**
-     * ABSTRACT: Extract items from the current page DOM
-     */
-    protected abstract extractItems(page: Page): Promise<TItem[]>;
-    /**
-     * Browser-specific pagination configuration
-     */
-    protected getBrowserPaginationConfig(): BrowserPaginationConfig;
-    /**
-     * Override pull() to manage browser lifecycle
-     */
-    pull(options: FeedOptions, checkpoint: TCheckpoint | null, env: Env, sessionState?: SessionState | null, updateCheckpointFn?: (checkpoint: Checkpoint) => Promise<void>): Promise<FeedSyncResult>;
-    /**
-     * Apply session state to browser page before navigation
-     */
-    protected applySessionState(page: Page, sessionState: BrowserSessionState): Promise<void>;
-    /**
-     * Capture browser session state after sync for persistence
-     */
-    protected captureSessionState(page: Page): Promise<BrowserSessionState>;
-    /**
-     * fetchPage() implementation for browser-based pagination
-     */
-    protected fetchPage(cursor: string | null, options: FeedOptions, _env: Env): Promise<PageFetchResult<TItem>>;
-    /**
-     * Handle cookie consent banner
-     */
-    protected handleCookieConsent(page: Page, config: CookieConsentConfig): Promise<void>;
-    /**
-     * Check for CAPTCHA and throw if detected
-     */
-    protected checkForCaptcha(page: Page, config: CaptchaConfig): Promise<void>;
-    /**
-     * Get current page number
-     */
-    protected getCurrentPageNumber(): number;
-    /**
-     * Get the Playwright page instance
-     */
-    protected getPage(): Page | null;
-}
-//# sourceMappingURL=browser-paginated.d.ts.map

package/dist/browser-paginated.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"browser-paginated.d.ts","sourceRoot":"","sources":["../src/browser-paginated.ts"],"names":[],"mappings":"AACA;;;;GAIG;AAEH,OAAO,KAAK,EAAW,MAAM,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGxD,OAAO,KAAK,EAAE,eAAe,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAC7F,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE7F;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,YAAY;IACvD,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,+DAA+D;IAC/D,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,gCAAgC;IAChC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,oDAAoD;IACpD,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,sCAAsC;IACtC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,uDAAuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,gDAAgD;IAChD,OAAO,EAAE,OAAO,CAAC;IACjB,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,iEAAiE;IACjE,OAAO,EAAE,OAAO,CAAC;IACjB,4DAA4D;IAC5D,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kCAAkC;IAClC,SAAS,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;IACvD,gDAAgD;IAChD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,4CAA4C;IAC5C,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,sCAAsC;IACtC,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,gBAAgB;IAC/D,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,8BAAsB,oBAAoB,CACxC,KAAK,EACL,WAAW,SAAS,mBAAmB,GAAG,mBAAmB,CAC7D,SAAQ,aAAa,CAAC,KAAK,EAAE,WAAW,CAAC;IACzC,QAAQ,CAAC,OAAO,EAAG,SAAS,CAAU;IAEtC,OAAO,CAAC,QAAQ,CAAwB;IACxC,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,cAAc,CAAc;IACpC,OAAO,CAAC,YAAY,CAAiB;IACrC,OAAO,CAAC,kBAAkB,CAAa;IAEvC;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,gBAAgB,IAAI,iBAAiB;IAExD;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM;IAE3D;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM;IAE5E;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAE5D;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAE7D;;OAEG;IACH,SAAS,CAAC,0BAA0B,IAAI,uBAAuB;IAQ/D;;OAEG;IACG,IAAI,CACR,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,WAAW,GAAG,IAAI,EAC9B,GAAG,EAAE,GAAG,EACR,YAAY,CAAC,EAAE,YAAY,GAAG,IAAI,EAClC,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAC7D,OAAO,CAAC,cAAc,CAAC;IA2D1B;;OAEG;cACa,iBAAiB,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAiC/F;;OAEG;cACa,mBAAmB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAsC7E;;OAEG;cACa,SAAS,CACvB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,WAAW,EACpB,IAAI,EAAE,GAAG,GACR,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAsDlC;;OAEG;cACa,mBAAmB,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IA0C3F;;OAEG;cACa,eAAe,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IA0BjF;;OAEG;IACH,SAAS,CAAC,oBAAoB,IAAI,MAAM;IAIxC;;OAEG;IACH,SAAS,CAAC,OAAO,IAAI,IAAI,GAAG,IAAI;CAGjC"}

package/dist/browser-paginated.js

@@ -1,269 +0,0 @@
-/// <reference lib="dom" />
-/**
- * Browser Paginated Feed Base Class
- *
- * Extends PaginatedFeed for browser-based feeds (Playwright).
- */
-import { captureErrorArtifacts, launchBrowser } from './browser/launcher.js';
-import { sdkLogger } from './logger.js';
-import { PaginatedFeed } from './paginated.js';
-/**
- * Base class for browser-based feeds with pagination
- */
-export class BrowserPaginatedFeed extends PaginatedFeed {
-    apiType = 'browser';
-    _browser = null;
-    _page = null;
-    _screenshotDir = '';
-    _isFirstPage = true;
-    _currentPageNumber = 0;
-    /**
-     * Browser-specific pagination configuration
-     */
-    getBrowserPaginationConfig() {
-        return {
-            ...this.getPaginationConfig(),
-            pagesPerRun: 1,
-            pageDelayMs: 2000,
-        };
-    }
-    /**
-     * Override pull() to manage browser lifecycle
-     */
-    async pull(options, checkpoint, env, sessionState, updateCheckpointFn) {
-        const config = this.getBrowserConfig();
-        const browserSessionState = sessionState;
-        const { browser, screenshotDir } = await launchBrowser(env, { stealth: config.stealth });
-        this._browser = browser;
-        this._page = (await browser.newPage());
-        this._screenshotDir = screenshotDir;
-        this._isFirstPage = true;
-        this._currentPageNumber = 0;
-        if (config.viewport) {
-            await this._page.setViewportSize(config.viewport);
-        }
-        if (config.userAgent) {
-            await this._page.setExtraHTTPHeaders({
-                'User-Agent': config.userAgent,
-            });
-        }
-        if (browserSessionState) {
-            await this.applySessionState(this._page, browserSessionState);
-        }
-        try {
-            const result = await this.paginate(options, checkpoint, env, updateCheckpointFn);
-            let parentMapRecord;
-            if (result.parentMap && result.parentMap.size > 0) {
-                parentMapRecord = Object.fromEntries(result.parentMap);
-            }
-            const capturedSessionState = await this.captureSessionState(this._page);
-            return {
-                contents: result.contents,
-                checkpoint: result.checkpoint,
-                metadata: {
-                    items_found: result.contents.length,
-                    items_skipped: 0,
-                    parent_map: parentMapRecord,
-                    next_sync_recommended_at: result.nextSyncRecommendedAt,
-                },
-                auth_update: capturedSessionState,
-            };
-        }
-        catch (error) {
-            if (this._page) {
-                await captureErrorArtifacts(this._page, error, this.type, this._screenshotDir);
-            }
-            throw error;
-        }
-        finally {
-            if (this._browser) {
-                await this._browser.close();
-                this._browser = null;
-                this._page = null;
-            }
-        }
-    }
-    /**
-     * Apply session state to browser page before navigation
-     */
-    async applySessionState(page, sessionState) {
-        if (sessionState.cookies && sessionState.cookies.length > 0) {
-            try {
-                await page.context().addCookies(sessionState.cookies);
-                sdkLogger.info({ cookieCount: sessionState.cookies.length }, `[${this.type}] Applied session cookies`);
-            }
-            catch (error) {
-                sdkLogger.warn({ error }, `[${this.type}] Failed to apply session cookies`);
-            }
-        }
-        if (sessionState.headers && Object.keys(sessionState.headers).length > 0) {
-            try {
-                await page.setExtraHTTPHeaders(sessionState.headers);
-                sdkLogger.info({ headerCount: Object.keys(sessionState.headers).length }, `[${this.type}] Applied session headers`);
-            }
-            catch (error) {
-                sdkLogger.warn({ error }, `[${this.type}] Failed to apply session headers`);
-            }
-        }
-        if (sessionState.localStorage && Object.keys(sessionState.localStorage).length > 0) {
-            sdkLogger.debug({ keyCount: Object.keys(sessionState.localStorage).length }, `[${this.type}] localStorage will be applied after first navigation`);
-        }
-    }
-    /**
-     * Capture browser session state after sync for persistence
-     */
-    async captureSessionState(page) {
-        const sessionState = {};
-        try {
-            const cookies = await page.context().cookies();
-            if (cookies.length > 0) {
-                sessionState.cookies = cookies;
-                sdkLogger.debug({ cookieCount: cookies.length }, `[${this.type}] Captured session cookies`);
-            }
-        }
-        catch (error) {
-            sdkLogger.warn({ error }, `[${this.type}] Failed to capture session cookies`);
-        }
-        try {
-            const localStorage = await page.evaluate(() => {
-                const storage = {};
-                for (let i = 0; i < window.localStorage.length; i++) {
-                    const key = window.localStorage.key(i);
-                    if (key) {
-                        storage[key] = window.localStorage.getItem(key) || '';
-                    }
-                }
-                return storage;
-            });
-            if (Object.keys(localStorage).length > 0) {
-                sessionState.localStorage = localStorage;
-                sdkLogger.debug({ keyCount: Object.keys(localStorage).length }, `[${this.type}] Captured localStorage`);
-            }
-        }
-        catch (error) {
-            sdkLogger.debug({ error }, `[${this.type}] Could not capture localStorage (may be expected)`);
-        }
-        return sessionState;
-    }
-    /**
-     * fetchPage() implementation for browser-based pagination
-     */
-    async fetchPage(cursor, options, _env) {
-        const pageNumber = cursor ? parseInt(cursor, 10) : 1;
-        this._currentPageNumber = pageNumber;
-        const baseUrl = this.getBaseUrl(options);
-        const pageUrl = this.buildPageUrl(baseUrl, pageNumber);
-        const config = this.getBrowserConfig();
-        const paginationConfig = this.getBrowserPaginationConfig();
-        const page = this._page;
-        sdkLogger.info({ pageUrl, pageNumber, maxPages: paginationConfig.pagesPerRun }, `[${this.type}] Fetching page`);
-        await page.goto(pageUrl, {
-            waitUntil: config.waitUntil,
-            timeout: config.navigationTimeout,
-        });
-        if (this._isFirstPage && config.cookieConsent) {
-            await this.handleCookieConsent(page, config.cookieConsent);
-            this._isFirstPage = false;
-        }
-        if (config.captcha?.enabled) {
-            await this.checkForCaptcha(page, config.captcha);
-        }
-        await this.waitForContent(page);
-        const items = await this.extractItems(page);
-        sdkLogger.info({ itemCount: items.length, pageNumber }, `[${this.type}] Extracted items from page`);
-        const hasNext = items.length > 0 &&
-            items.length >= paginationConfig.pageSize &&
-            pageNumber < paginationConfig.pagesPerRun;
-        if (hasNext && paginationConfig.pageDelayMs > 0) {
-            await this.sleep(paginationConfig.pageDelayMs);
-        }
-        return {
-            items,
-            nextToken: hasNext ? String(pageNumber + 1) : null,
-            rawCount: items.length,
-        };
-    }
-    /**
-     * Handle cookie consent banner
-     */
-    async handleCookieConsent(page, config) {
-        try {
-            const timeout = config.timeout ?? 2000;
-            for (const selector of config.bannerSelectors) {
-                try {
-                    const banner = await page.$(selector);
-                    if (banner) {
-                        for (const acceptSelector of config.acceptSelectors) {
-                            try {
-                                const acceptButton = await page.$(acceptSelector);
-                                if (acceptButton) {
-                                    await acceptButton.click();
-                                    await this.sleep(1000);
-                                    sdkLogger.debug(`[${this.type}] Cookie consent accepted`);
-                                    return;
-                                }
-                            }
-                            catch (_e) {
-                                // Try next accept selector
-                            }
-                        }
-                    }
-                }
-                catch (_e) {
-                    // Try next banner selector
-                }
-            }
-            const bannerSelector = config.bannerSelectors.join(', ');
-            try {
-                await page.waitForSelector(bannerSelector, { timeout });
-                const acceptSelector = config.acceptSelectors.join(', ');
-                await page.click(acceptSelector);
-                await this.sleep(1000);
-                sdkLogger.debug(`[${this.type}] Cookie consent accepted (waited)`);
-            }
-            catch (_e) {
-                // No cookie banner or already dismissed
-            }
-        }
-        catch (_e) {
-            sdkLogger.debug(`[${this.type}] No cookie banner found or already handled`);
-        }
-    }
-    /**
-     * Check for CAPTCHA and throw if detected
-     */
-    async checkForCaptcha(page, config) {
-        const hasCaptcha = await page.evaluate((cfg) => {
-            if (cfg.selectors) {
-                for (const selector of cfg.selectors) {
-                    if (document.querySelector(selector))
-                        return true;
-                }
-            }
-            if (cfg.textPatterns) {
-                const bodyText = document.body.textContent || '';
-                for (const pattern of cfg.textPatterns) {
-                    if (bodyText.includes(pattern))
-                        return true;
-                }
-            }
-            return false;
-        }, { selectors: config.selectors, textPatterns: config.textPatterns });
-        if (hasCaptcha) {
-            throw new Error(`CAPTCHA detected - ${this.displayName} blocking access`);
-        }
-    }
-    /**
-     * Get current page number
-     */
-    getCurrentPageNumber() {
-        return this._currentPageNumber;
-    }
-    /**
-     * Get the Playwright page instance
-     */
-    getPage() {
-        return this._page;
-    }
-}
-//# sourceMappingURL=browser-paginated.js.map

package/dist/browser-paginated.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"browser-paginated.js","sourceRoot":"","sources":["../src/browser-paginated.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B;;;;GAIG;AAGH,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAqE/C;;GAEG;AACH,MAAM,OAAgB,oBAGpB,SAAQ,aAAiC;IAChC,OAAO,GAAG,SAAkB,CAAC;IAE9B,QAAQ,GAAmB,IAAI,CAAC;IAChC,KAAK,GAAgB,IAAI,CAAC;IAC1B,cAAc,GAAW,EAAE,CAAC;IAC5B,YAAY,GAAY,IAAI,CAAC;IAC7B,kBAAkB,GAAW,CAAC,CAAC;IA2BvC;;OAEG;IACO,0BAA0B;QAClC,OAAO;YACL,GAAG,IAAI,CAAC,mBAAmB,EAAE;YAC7B,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,IAAI;SAClB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CACR,OAAoB,EACpB,UAA8B,EAC9B,GAAQ,EACR,YAAkC,EAClC,kBAA8D;QAE9D,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACvC,MAAM,mBAAmB,GAAG,YAAsD,CAAC;QAEnF,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,QAAQ,GAAG,OAAkB,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,EAAE,CAAS,CAAC;QAC/C,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC;QAE5B,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC;gBACnC,YAAY,EAAE,MAAM,CAAC,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,IAAI,mBAAmB,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,kBAAkB,CAAC,CAAC;YAEjF,IAAI,eAAmD,CAAC;YACxD,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAClD,eAAe,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACzD,CAAC;YAED,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAExE,OAAO;gBACL,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,QAAQ,EAAE;oBACR,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;oBACnC,aAAa,EAAE,CAAC;oBAChB,UAAU,EAAE,eAAe;oBAC3B,wBAAwB,EAAE,MAAM,CAAC,qBAAqB;iBACvD;gBACD,WAAW,EAAE,oBAAoB;aAClC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,MAAM,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;YACjF,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACrB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,iBAAiB,CAAC,IAAU,EAAE,YAAiC;QAC7E,IAAI,YAAY,CAAC,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5D,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;gBACtD,SAAS,CAAC,IAAI,CACZ,EAAE,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,EAC5C,IAAI,IAAI,CAAC,IAAI,2BAA2B,CACzC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,mCAAmC,CAAC,CAAC;YAC9E,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;gBACrD,SAAS,CAAC,IAAI,CACZ,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EACzD,IAAI,IAAI,CAAC,IAAI,2BAA2B,CACzC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,mCAAmC,CAAC,CAAC;YAC9E,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnF,SAAS,CAAC,KAAK,CACb,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,MAAM,EAAE,EAC3D,IAAI,IAAI,CAAC,IAAI,uDAAuD,CACrE,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,IAAU;QAC5C,MAAM,YAAY,GAAwB,EAAE,CAAC;QAE7C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;YAC/C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,YAAY,CAAC,OAAO,GAAG,OAAO,CAAC;gBAC/B,SAAS,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,4BAA4B,CAAC,CAAC;YAC9F,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,qCAAqC,CAAC,CAAC;QAChF,CAAC;QAED,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBAC5C,MAAM,OAAO,GAA2B,EAAE,CAAC;gBAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACpD,MAAM,GAAG,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBACvC,IAAI,GAAG,EAAE,CAAC;wBACR,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;oBACxD,CAAC;gBACH,CAAC;gBACD,OAAO,OAAO,CAAC;YACjB,CAAC,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzC,YAAY,CAAC,YAAY,GAAG,YAAY,CAAC;gBACzC,SAAS,CAAC,KAAK,CACb,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,EAAE,EAC9C,IAAI,IAAI,CAAC,IAAI,yBAAyB,CACvC,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,oDAAoD,CAAC,CAAC;QAChG,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,SAAS,CACvB,MAAqB,EACrB,OAAoB,EACpB,IAAS;QAET,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC;QAErC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACvC,MAAM,gBAAgB,GAAG,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAM,CAAC;QAEzB,SAAS,CAAC,IAAI,CACZ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,CAAC,WAAW,EAAE,EAC/D,IAAI,IAAI,CAAC,IAAI,iBAAiB,CAC/B,CAAC;QAEF,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACvB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,MAAM,CAAC,iBAAiB;SAClC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YAC9C,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;YAC3D,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC5B,CAAC;QAED,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEhC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAE5C,SAAS,CAAC,IAAI,CACZ,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,EACvC,IAAI,IAAI,CAAC,IAAI,6BAA6B,CAC3C,CAAC;QAEF,MAAM,OAAO,GACX,KAAK,CAAC,MAAM,GAAG,CAAC;YAChB,KAAK,CAAC,MAAM,IAAI,gBAAgB,CAAC,QAAQ;YACzC,UAAU,GAAG,gBAAgB,CAAC,WAAW,CAAC;QAE5C,IAAI,OAAO,IAAI,gBAAgB,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YAChD,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;QACjD,CAAC;QAED,OAAO;YACL,KAAK;YACL,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;YAClD,QAAQ,EAAE,KAAK,CAAC,MAAM;SACvB,CAAC;IACJ,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,IAAU,EAAE,MAA2B;QACzE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC;YAEvC,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;gBAC9C,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;oBACtC,IAAI,MAAM,EAAE,CAAC;wBACX,KAAK,MAAM,cAAc,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;4BACpD,IAAI,CAAC;gCACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;gCAClD,IAAI,YAAY,EAAE,CAAC;oCACjB,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC;oCAC3B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oCACvB,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,2BAA2B,CAAC,CAAC;oCAC1D,OAAO;gCACT,CAAC;4BACH,CAAC;4BAAC,OAAO,EAAE,EAAE,CAAC;gCACZ,2BAA2B;4BAC7B,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACZ,2BAA2B;gBAC7B,CAAC;YACH,CAAC;YAED,MAAM,cAAc,GAAG,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzD,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;gBACxD,MAAM,cAAc,GAAG,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzD,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;gBACjC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvB,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,oCAAoC,CAAC,CAAC;YACrE,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,wCAAwC;YAC1C,CAAC;QACH,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACZ,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,6CAA6C,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,eAAe,CAAC,IAAU,EAAE,MAAqB;QAC/D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CACpC,CAAC,GAAsD,EAAE,EAAE;YACzD,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;gBAClB,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;oBACrC,IAAI,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC;wBAAE,OAAO,IAAI,CAAC;gBACpD,CAAC;YACH,CAAC;YAED,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;gBACjD,KAAK,MAAM,OAAO,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;oBACvC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;wBAAE,OAAO,IAAI,CAAC;gBAC9C,CAAC;YACH,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC,EACD,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CACnE,CAAC;QAEF,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,CAAC,WAAW,kBAAkB,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED;;OAEG;IACO,oBAAoB;QAC5B,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IAED;;OAEG;IACO,OAAO;QACf,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;CACF"}

package/dist/http.d.ts

@@ -1,18 +0,0 @@
-import { type KyInstance, type Options } from 'ky';
-/**
- * Create a configured ky instance with custom options
- */
-export declare function createHttpClient(options?: Options): KyInstance;
-/**
- * Default HTTP client for feeds with standard User-Agent
- */
-export declare const httpClient: KyInstance;
-/**
- * Create an HTTP client with authentication headers
- */
-export declare function createAuthenticatedClient(authHeader: string, additionalHeaders?: Record<string, string>): KyInstance;
-/**
- * HTTP client for JSON APIs
- */
-export declare const jsonHttpClient: KyInstance;
-//# sourceMappingURL=http.d.ts.map

package/dist/http.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,OAAW,EAAE,KAAK,UAAU,EAAE,KAAK,OAAO,EAAE,MAAM,IAAI,CAAC;AA8BvD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,UAAU,CAY9D;AAED;;GAEG;AACH,eAAO,MAAM,UAAU,YAIrB,CAAC;AAEH;;GAEG;AACH,wBAAgB,yBAAyB,CACvC,UAAU,EAAE,MAAM,EAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACzC,UAAU,CAQZ;AAED;;GAEG;AACH,eAAO,MAAM,cAAc,YAMzB,CAAC"}

package/dist/http.js

@@ -1,74 +0,0 @@
-import ky from 'ky';
-/**
- * Shared HTTP client configuration for all feeds
- */
-/**
- * Default retry configuration
- * - Max 2 retries (3 total attempts)
- * - Only retry transient errors (429, 5xx)
- * - Exponential backoff up to 5 seconds
- * - 30s timeout per request
- */
-const defaultRetryConfig = {
-    retry: {
-        limit: 2, // Max 2 retries (3 total attempts)
-        methods: ['get', 'post'],
-        statusCodes: [
-            408, // Request Timeout
-            429, // Too Many Requests (rate limit)
-            500, // Internal Server Error
-            502, // Bad Gateway
-            503, // Service Unavailable
-            504, // Gateway Timeout
-        ],
-        backoffLimit: 5000, // Max 5 seconds delay between retries
-    },
-    timeout: 30000, // 30 second timeout per request
-};
-/**
- * Create a configured ky instance with custom options
- */
-export function createHttpClient(options) {
-    return ky.create({
-        ...defaultRetryConfig,
-        ...options,
-        // Merge retry config if provided
-        retry: options?.retry
-            ? {
-                ...defaultRetryConfig.retry,
-                ...(typeof options.retry === 'number' ? { limit: options.retry } : options.retry),
-            }
-            : defaultRetryConfig.retry,
-    });
-}
-/**
- * Default HTTP client for feeds with standard User-Agent
- */
-export const httpClient = createHttpClient({
-    headers: {
-        'User-Agent': 'UserResearchBot/1.0',
-    },
-});
-/**
- * Create an HTTP client with authentication headers
- */
-export function createAuthenticatedClient(authHeader, additionalHeaders) {
-    return createHttpClient({
-        headers: {
-            'User-Agent': 'UserResearchBot/1.0',
-            Authorization: authHeader,
-            ...additionalHeaders,
-        },
-    });
-}
-/**
- * HTTP client for JSON APIs
- */
-export const jsonHttpClient = createHttpClient({
-    headers: {
-        'User-Agent': 'UserResearchBot/1.0',
-        Accept: 'application/json',
-        'Content-Type': 'application/json',
-    },
-});
-//# sourceMappingURL=http.js.map

package/dist/http.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqC,MAAM,IAAI,CAAC;AAEvD;;GAEG;AAEH;;;;;;GAMG;AACH,MAAM,kBAAkB,GAAG;IACzB,KAAK,EAAE;QACL,KAAK,EAAE,CAAC,EAAE,mCAAmC;QAC7C,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE;YACX,GAAG,EAAE,kBAAkB;YACvB,GAAG,EAAE,iCAAiC;YACtC,GAAG,EAAE,wBAAwB;YAC7B,GAAG,EAAE,cAAc;YACnB,GAAG,EAAE,sBAAsB;YAC3B,GAAG,EAAE,kBAAkB;SACxB;QACD,YAAY,EAAE,IAAI,EAAE,sCAAsC;KAC3D;IACD,OAAO,EAAE,KAAK,EAAE,gCAAgC;CACjD,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAiB;IAChD,OAAO,EAAE,CAAC,MAAM,CAAC;QACf,GAAG,kBAAkB;QACrB,GAAG,OAAO;QACV,iCAAiC;QACjC,KAAK,EAAE,OAAO,EAAE,KAAK;YACnB,CAAC,CAAC;gBACE,GAAG,kBAAkB,CAAC,KAAK;gBAC3B,GAAG,CAAC,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;aAClF;YACH,CAAC,CAAC,kBAAkB,CAAC,KAAK;KAC7B,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;IACzC,OAAO,EAAE;QACP,YAAY,EAAE,qBAAqB;KACpC;CACF,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,UAAU,yBAAyB,CACvC,UAAkB,EAClB,iBAA0C;IAE1C,OAAO,gBAAgB,CAAC;QACtB,OAAO,EAAE;YACP,YAAY,EAAE,qBAAqB;YACnC,aAAa,EAAE,UAAU;YACzB,GAAG,iBAAiB;SACrB;KACF,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,gBAAgB,CAAC;IAC7C,OAAO,EAAE;QACP,YAAY,EAAE,qBAAqB;QACnC,MAAM,EAAE,kBAAkB;QAC1B,cAAc,EAAE,kBAAkB;KACnC;CACF,CAAC,CAAC"}