@lobu/connector-sdk 6.0.1 → 7.0.0

package/dist/browser-paginated.js

@@ -1,269 +0,0 @@
-/// <reference lib="dom" />
-/**
- * Browser Paginated Feed Base Class
- *
- * Extends PaginatedFeed for browser-based feeds (Playwright).
- */
-import { captureErrorArtifacts, launchBrowser } from './browser/launcher.js';
-import { sdkLogger } from './logger.js';
-import { PaginatedFeed } from './paginated.js';
-/**
- * Base class for browser-based feeds with pagination
- */
-export class BrowserPaginatedFeed extends PaginatedFeed {
-    apiType = 'browser';
-    _browser = null;
-    _page = null;
-    _screenshotDir = '';
-    _isFirstPage = true;
-    _currentPageNumber = 0;
-    /**
-     * Browser-specific pagination configuration
-     */
-    getBrowserPaginationConfig() {
-        return {
-            ...this.getPaginationConfig(),
-            pagesPerRun: 1,
-            pageDelayMs: 2000,
-        };
-    }
-    /**
-     * Override pull() to manage browser lifecycle
-     */
-    async pull(options, checkpoint, env, sessionState, updateCheckpointFn) {
-        const config = this.getBrowserConfig();
-        const browserSessionState = sessionState;
-        const { browser, screenshotDir } = await launchBrowser(env, { stealth: config.stealth });
-        this._browser = browser;
-        this._page = (await browser.newPage());
-        this._screenshotDir = screenshotDir;
-        this._isFirstPage = true;
-        this._currentPageNumber = 0;
-        if (config.viewport) {
-            await this._page.setViewportSize(config.viewport);
-        }
-        if (config.userAgent) {
-            await this._page.setExtraHTTPHeaders({
-                'User-Agent': config.userAgent,
-            });
-        }
-        if (browserSessionState) {
-            await this.applySessionState(this._page, browserSessionState);
-        }
-        try {
-            const result = await this.paginate(options, checkpoint, env, updateCheckpointFn);
-            let parentMapRecord;
-            if (result.parentMap && result.parentMap.size > 0) {
-                parentMapRecord = Object.fromEntries(result.parentMap);
-            }
-            const capturedSessionState = await this.captureSessionState(this._page);
-            return {
-                contents: result.contents,
-                checkpoint: result.checkpoint,
-                metadata: {
-                    items_found: result.contents.length,
-                    items_skipped: 0,
-                    parent_map: parentMapRecord,
-                    next_sync_recommended_at: result.nextSyncRecommendedAt,
-                },
-                auth_update: capturedSessionState,
-            };
-        }
-        catch (error) {
-            if (this._page) {
-                await captureErrorArtifacts(this._page, error, this.type, this._screenshotDir);
-            }
-            throw error;
-        }
-        finally {
-            if (this._browser) {
-                await this._browser.close();
-                this._browser = null;
-                this._page = null;
-            }
-        }
-    }
-    /**
-     * Apply session state to browser page before navigation
-     */
-    async applySessionState(page, sessionState) {
-        if (sessionState.cookies && sessionState.cookies.length > 0) {
-            try {
-                await page.context().addCookies(sessionState.cookies);
-                sdkLogger.info({ cookieCount: sessionState.cookies.length }, `[${this.type}] Applied session cookies`);
-            }
-            catch (error) {
-                sdkLogger.warn({ error }, `[${this.type}] Failed to apply session cookies`);
-            }
-        }
-        if (sessionState.headers && Object.keys(sessionState.headers).length > 0) {
-            try {
-                await page.setExtraHTTPHeaders(sessionState.headers);
-                sdkLogger.info({ headerCount: Object.keys(sessionState.headers).length }, `[${this.type}] Applied session headers`);
-            }
-            catch (error) {
-                sdkLogger.warn({ error }, `[${this.type}] Failed to apply session headers`);
-            }
-        }
-        if (sessionState.localStorage && Object.keys(sessionState.localStorage).length > 0) {
-            sdkLogger.debug({ keyCount: Object.keys(sessionState.localStorage).length }, `[${this.type}] localStorage will be applied after first navigation`);
-        }
-    }
-    /**
-     * Capture browser session state after sync for persistence
-     */
-    async captureSessionState(page) {
-        const sessionState = {};
-        try {
-            const cookies = await page.context().cookies();
-            if (cookies.length > 0) {
-                sessionState.cookies = cookies;
-                sdkLogger.debug({ cookieCount: cookies.length }, `[${this.type}] Captured session cookies`);
-            }
-        }
-        catch (error) {
-            sdkLogger.warn({ error }, `[${this.type}] Failed to capture session cookies`);
-        }
-        try {
-            const localStorage = await page.evaluate(() => {
-                const storage = {};
-                for (let i = 0; i < window.localStorage.length; i++) {
-                    const key = window.localStorage.key(i);
-                    if (key) {
-                        storage[key] = window.localStorage.getItem(key) || '';
-                    }
-                }
-                return storage;
-            });
-            if (Object.keys(localStorage).length > 0) {
-                sessionState.localStorage = localStorage;
-                sdkLogger.debug({ keyCount: Object.keys(localStorage).length }, `[${this.type}] Captured localStorage`);
-            }
-        }
-        catch (error) {
-            sdkLogger.debug({ error }, `[${this.type}] Could not capture localStorage (may be expected)`);
-        }
-        return sessionState;
-    }
-    /**
-     * fetchPage() implementation for browser-based pagination
-     */
-    async fetchPage(cursor, options, _env) {
-        const pageNumber = cursor ? parseInt(cursor, 10) : 1;
-        this._currentPageNumber = pageNumber;
-        const baseUrl = this.getBaseUrl(options);
-        const pageUrl = this.buildPageUrl(baseUrl, pageNumber);
-        const config = this.getBrowserConfig();
-        const paginationConfig = this.getBrowserPaginationConfig();
-        const page = this._page;
-        sdkLogger.info({ pageUrl, pageNumber, maxPages: paginationConfig.pagesPerRun }, `[${this.type}] Fetching page`);
-        await page.goto(pageUrl, {
-            waitUntil: config.waitUntil,
-            timeout: config.navigationTimeout,
-        });
-        if (this._isFirstPage && config.cookieConsent) {
-            await this.handleCookieConsent(page, config.cookieConsent);
-            this._isFirstPage = false;
-        }
-        if (config.captcha?.enabled) {
-            await this.checkForCaptcha(page, config.captcha);
-        }
-        await this.waitForContent(page);
-        const items = await this.extractItems(page);
-        sdkLogger.info({ itemCount: items.length, pageNumber }, `[${this.type}] Extracted items from page`);
-        const hasNext = items.length > 0 &&
-            items.length >= paginationConfig.pageSize &&
-            pageNumber < paginationConfig.pagesPerRun;
-        if (hasNext && paginationConfig.pageDelayMs > 0) {
-            await this.sleep(paginationConfig.pageDelayMs);
-        }
-        return {
-            items,
-            nextToken: hasNext ? String(pageNumber + 1) : null,
-            rawCount: items.length,
-        };
-    }
-    /**
-     * Handle cookie consent banner
-     */
-    async handleCookieConsent(page, config) {
-        try {
-            const timeout = config.timeout ?? 2000;
-            for (const selector of config.bannerSelectors) {
-                try {
-                    const banner = await page.$(selector);
-                    if (banner) {
-                        for (const acceptSelector of config.acceptSelectors) {
-                            try {
-                                const acceptButton = await page.$(acceptSelector);
-                                if (acceptButton) {
-                                    await acceptButton.click();
-                                    await this.sleep(1000);
-                                    sdkLogger.debug(`[${this.type}] Cookie consent accepted`);
-                                    return;
-                                }
-                            }
-                            catch (_e) {
-                                // Try next accept selector
-                            }
-                        }
-                    }
-                }
-                catch (_e) {
-                    // Try next banner selector
-                }
-            }
-            const bannerSelector = config.bannerSelectors.join(', ');
-            try {
-                await page.waitForSelector(bannerSelector, { timeout });
-                const acceptSelector = config.acceptSelectors.join(', ');
-                await page.click(acceptSelector);
-                await this.sleep(1000);
-                sdkLogger.debug(`[${this.type}] Cookie consent accepted (waited)`);
-            }
-            catch (_e) {
-                // No cookie banner or already dismissed
-            }
-        }
-        catch (_e) {
-            sdkLogger.debug(`[${this.type}] No cookie banner found or already handled`);
-        }
-    }
-    /**
-     * Check for CAPTCHA and throw if detected
-     */
-    async checkForCaptcha(page, config) {
-        const hasCaptcha = await page.evaluate((cfg) => {
-            if (cfg.selectors) {
-                for (const selector of cfg.selectors) {
-                    if (document.querySelector(selector))
-                        return true;
-                }
-            }
-            if (cfg.textPatterns) {
-                const bodyText = document.body.textContent || '';
-                for (const pattern of cfg.textPatterns) {
-                    if (bodyText.includes(pattern))
-                        return true;
-                }
-            }
-            return false;
-        }, { selectors: config.selectors, textPatterns: config.textPatterns });
-        if (hasCaptcha) {
-            throw new Error(`CAPTCHA detected - ${this.displayName} blocking access`);
-        }
-    }
-    /**
-     * Get current page number
-     */
-    getCurrentPageNumber() {
-        return this._currentPageNumber;
-    }
-    /**
-     * Get the Playwright page instance
-     */
-    getPage() {
-        return this._page;
-    }
-}
-//# sourceMappingURL=browser-paginated.js.map

package/dist/browser-paginated.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"browser-paginated.js","sourceRoot":"","sources":["../src/browser-paginated.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAC3B;;;;GAIG;AAGH,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAqE/C;;GAEG;AACH,MAAM,OAAgB,oBAGpB,SAAQ,aAAiC;IAChC,OAAO,GAAG,SAAkB,CAAC;IAE9B,QAAQ,GAAmB,IAAI,CAAC;IAChC,KAAK,GAAgB,IAAI,CAAC;IAC1B,cAAc,GAAW,EAAE,CAAC;IAC5B,YAAY,GAAY,IAAI,CAAC;IAC7B,kBAAkB,GAAW,CAAC,CAAC;IA2BvC;;OAEG;IACO,0BAA0B;QAClC,OAAO;YACL,GAAG,IAAI,CAAC,mBAAmB,EAAE;YAC7B,WAAW,EAAE,CAAC;YACd,WAAW,EAAE,IAAI;SAClB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CACR,OAAoB,EACpB,UAA8B,EAC9B,GAAQ,EACR,YAAkC,EAClC,kBAA8D;QAE9D,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACvC,MAAM,mBAAmB,GAAG,YAAsD,CAAC;QAEnF,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACzF,IAAI,CAAC,QAAQ,GAAG,OAAkB,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,EAAE,CAAS,CAAC;QAC/C,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC;QAE5B,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC;gBACnC,YAAY,EAAE,MAAM,CAAC,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,IAAI,mBAAmB,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,kBAAkB,CAAC,CAAC;YAEjF,IAAI,eAAmD,CAAC;YACxD,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAClD,eAAe,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACzD,CAAC;YAED,MAAM,oBAAoB,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAExE,OAAO;gBACL,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,QAAQ,EAAE;oBACR,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;oBACnC,aAAa,EAAE,CAAC;oBAChB,UAAU,EAAE,eAAe;oBAC3B,wBAAwB,EAAE,MAAM,CAAC,qBAAqB;iBACvD;gBACD,WAAW,EAAE,oBAAoB;aAClC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,MAAM,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;YACjF,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACrB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,iBAAiB,CAAC,IAAU,EAAE,YAAiC;QAC7E,IAAI,YAAY,CAAC,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5D,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;gBACtD,SAAS,CAAC,IAAI,CACZ,EAAE,WAAW,EAAE,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,EAC5C,IAAI,IAAI,CAAC,IAAI,2BAA2B,CACzC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,mCAAmC,CAAC,CAAC;YAC9E,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,mBAAmB,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;gBACrD,SAAS,CAAC,IAAI,CACZ,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EACzD,IAAI,IAAI,CAAC,IAAI,2BAA2B,CACzC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,mCAAmC,CAAC,CAAC;YAC9E,CAAC;QACH,CAAC;QAED,IAAI,YAAY,CAAC,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnF,SAAS,CAAC,KAAK,CACb,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,MAAM,EAAE,EAC3D,IAAI,IAAI,CAAC,IAAI,uDAAuD,CACrE,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,IAAU;QAC5C,MAAM,YAAY,GAAwB,EAAE,CAAC;QAE7C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;YAC/C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,YAAY,CAAC,OAAO,GAAG,OAAO,CAAC;gBAC/B,SAAS,CAAC,KAAK,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,4BAA4B,CAAC,CAAC;YAC9F,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,qCAAqC,CAAC,CAAC;QAChF,CAAC;QAED,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBAC5C,MAAM,OAAO,GAA2B,EAAE,CAAC;gBAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACpD,MAAM,GAAG,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBACvC,IAAI,GAAG,EAAE,CAAC;wBACR,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;oBACxD,CAAC;gBACH,CAAC;gBACD,OAAO,OAAO,CAAC;YACjB,CAAC,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzC,YAAY,CAAC,YAAY,GAAG,YAAY,CAAC;gBACzC,SAAS,CAAC,KAAK,CACb,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,EAAE,EAC9C,IAAI,IAAI,CAAC,IAAI,yBAAyB,CACvC,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,IAAI,CAAC,IAAI,oDAAoD,CAAC,CAAC;QAChG,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,SAAS,CACvB,MAAqB,EACrB,OAAoB,EACpB,IAAS;QAET,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC;QAErC,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACvC,MAAM,gBAAgB,GAAG,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAM,CAAC;QAEzB,SAAS,CAAC,IAAI,CACZ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,CAAC,WAAW,EAAE,EAC/D,IAAI,IAAI,CAAC,IAAI,iBAAiB,CAC/B,CAAC;QAEF,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACvB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,MAAM,CAAC,iBAAiB;SAClC,CAAC,CAAC;QAEH,IAAI,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YAC9C,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;YAC3D,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC5B,CAAC;QAED,IAAI,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEhC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAE5C,SAAS,CAAC,IAAI,CACZ,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,EACvC,IAAI,IAAI,CAAC,IAAI,6BAA6B,CAC3C,CAAC;QAEF,MAAM,OAAO,GACX,KAAK,CAAC,MAAM,GAAG,CAAC;YAChB,KAAK,CAAC,MAAM,IAAI,gBAAgB,CAAC,QAAQ;YACzC,UAAU,GAAG,gBAAgB,CAAC,WAAW,CAAC;QAE5C,IAAI,OAAO,IAAI,gBAAgB,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YAChD,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;QACjD,CAAC;QAED,OAAO;YACL,KAAK;YACL,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI;YAClD,QAAQ,EAAE,KAAK,CAAC,MAAM;SACvB,CAAC;IACJ,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,IAAU,EAAE,MAA2B;QACzE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC;YAEvC,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;gBAC9C,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;oBACtC,IAAI,MAAM,EAAE,CAAC;wBACX,KAAK,MAAM,cAAc,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;4BACpD,IAAI,CAAC;gCACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;gCAClD,IAAI,YAAY,EAAE,CAAC;oCACjB,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC;oCAC3B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oCACvB,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,2BAA2B,CAAC,CAAC;oCAC1D,OAAO;gCACT,CAAC;4BACH,CAAC;4BAAC,OAAO,EAAE,EAAE,CAAC;gCACZ,2BAA2B;4BAC7B,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACZ,2BAA2B;gBAC7B,CAAC;YACH,CAAC;YAED,MAAM,cAAc,GAAG,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzD,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;gBACxD,MAAM,cAAc,GAAG,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzD,MAAM,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;gBACjC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvB,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,oCAAoC,CAAC,CAAC;YACrE,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,wCAAwC;YAC1C,CAAC;QACH,CAAC;QAAC,OAAO,EAAE,EAAE,CAAC;YACZ,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,6CAA6C,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,eAAe,CAAC,IAAU,EAAE,MAAqB;QAC/D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CACpC,CAAC,GAAsD,EAAE,EAAE;YACzD,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;gBAClB,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;oBACrC,IAAI,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC;wBAAE,OAAO,IAAI,CAAC;gBACpD,CAAC;YACH,CAAC;YAED,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;gBACjD,KAAK,MAAM,OAAO,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;oBACvC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;wBAAE,OAAO,IAAI,CAAC;gBAC9C,CAAC;YACH,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC,EACD,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE,CACnE,CAAC;QAEF,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,CAAC,WAAW,kBAAkB,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED;;OAEG;IACO,oBAAoB;QAC5B,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IAED;;OAEG;IACO,OAAO;QACf,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;CACF"}

package/dist/http.d.ts

@@ -1,18 +0,0 @@
-import { type KyInstance, type Options } from 'ky';
-/**
- * Create a configured ky instance with custom options
- */
-export declare function createHttpClient(options?: Options): KyInstance;
-/**
- * Default HTTP client for feeds with standard User-Agent
- */
-export declare const httpClient: KyInstance;
-/**
- * Create an HTTP client with authentication headers
- */
-export declare function createAuthenticatedClient(authHeader: string, additionalHeaders?: Record<string, string>): KyInstance;
-/**
- * HTTP client for JSON APIs
- */
-export declare const jsonHttpClient: KyInstance;
-//# sourceMappingURL=http.d.ts.map

package/dist/http.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,OAAW,EAAE,KAAK,UAAU,EAAE,KAAK,OAAO,EAAE,MAAM,IAAI,CAAC;AA8BvD;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,UAAU,CAY9D;AAED;;GAEG;AACH,eAAO,MAAM,UAAU,YAIrB,CAAC;AAEH;;GAEG;AACH,wBAAgB,yBAAyB,CACvC,UAAU,EAAE,MAAM,EAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GACzC,UAAU,CAQZ;AAED;;GAEG;AACH,eAAO,MAAM,cAAc,YAMzB,CAAC"}

package/dist/http.js

@@ -1,74 +0,0 @@
-import ky from 'ky';
-/**
- * Shared HTTP client configuration for all feeds
- */
-/**
- * Default retry configuration
- * - Max 2 retries (3 total attempts)
- * - Only retry transient errors (429, 5xx)
- * - Exponential backoff up to 5 seconds
- * - 30s timeout per request
- */
-const defaultRetryConfig = {
-    retry: {
-        limit: 2, // Max 2 retries (3 total attempts)
-        methods: ['get', 'post'],
-        statusCodes: [
-            408, // Request Timeout
-            429, // Too Many Requests (rate limit)
-            500, // Internal Server Error
-            502, // Bad Gateway
-            503, // Service Unavailable
-            504, // Gateway Timeout
-        ],
-        backoffLimit: 5000, // Max 5 seconds delay between retries
-    },
-    timeout: 30000, // 30 second timeout per request
-};
-/**
- * Create a configured ky instance with custom options
- */
-export function createHttpClient(options) {
-    return ky.create({
-        ...defaultRetryConfig,
-        ...options,
-        // Merge retry config if provided
-        retry: options?.retry
-            ? {
-                ...defaultRetryConfig.retry,
-                ...(typeof options.retry === 'number' ? { limit: options.retry } : options.retry),
-            }
-            : defaultRetryConfig.retry,
-    });
-}
-/**
- * Default HTTP client for feeds with standard User-Agent
- */
-export const httpClient = createHttpClient({
-    headers: {
-        'User-Agent': 'UserResearchBot/1.0',
-    },
-});
-/**
- * Create an HTTP client with authentication headers
- */
-export function createAuthenticatedClient(authHeader, additionalHeaders) {
-    return createHttpClient({
-        headers: {
-            'User-Agent': 'UserResearchBot/1.0',
-            Authorization: authHeader,
-            ...additionalHeaders,
-        },
-    });
-}
-/**
- * HTTP client for JSON APIs
- */
-export const jsonHttpClient = createHttpClient({
-    headers: {
-        'User-Agent': 'UserResearchBot/1.0',
-        Accept: 'application/json',
-        'Content-Type': 'application/json',
-    },
-});
-//# sourceMappingURL=http.js.map

package/dist/http.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqC,MAAM,IAAI,CAAC;AAEvD;;GAEG;AAEH;;;;;;GAMG;AACH,MAAM,kBAAkB,GAAG;IACzB,KAAK,EAAE;QACL,KAAK,EAAE,CAAC,EAAE,mCAAmC;QAC7C,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;QACxB,WAAW,EAAE;YACX,GAAG,EAAE,kBAAkB;YACvB,GAAG,EAAE,iCAAiC;YACtC,GAAG,EAAE,wBAAwB;YAC7B,GAAG,EAAE,cAAc;YACnB,GAAG,EAAE,sBAAsB;YAC3B,GAAG,EAAE,kBAAkB;SACxB;QACD,YAAY,EAAE,IAAI,EAAE,sCAAsC;KAC3D;IACD,OAAO,EAAE,KAAK,EAAE,gCAAgC;CACjD,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAiB;IAChD,OAAO,EAAE,CAAC,MAAM,CAAC;QACf,GAAG,kBAAkB;QACrB,GAAG,OAAO;QACV,iCAAiC;QACjC,KAAK,EAAE,OAAO,EAAE,KAAK;YACnB,CAAC,CAAC;gBACE,GAAG,kBAAkB,CAAC,KAAK;gBAC3B,GAAG,CAAC,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;aAClF;YACH,CAAC,CAAC,kBAAkB,CAAC,KAAK;KAC7B,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;IACzC,OAAO,EAAE;QACP,YAAY,EAAE,qBAAqB;KACpC;CACF,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,UAAU,yBAAyB,CACvC,UAAkB,EAClB,iBAA0C;IAE1C,OAAO,gBAAgB,CAAC;QACtB,OAAO,EAAE;YACP,YAAY,EAAE,qBAAqB;YACnC,aAAa,EAAE,UAAU;YACzB,GAAG,iBAAiB;SACrB;KACF,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,gBAAgB,CAAC;IAC7C,OAAO,EAAE;QACP,YAAY,EAAE,qBAAqB;QACnC,MAAM,EAAE,kBAAkB;QAC1B,cAAc,EAAE,kBAAkB;KACnC;CACF,CAAC,CAAC"}

package/dist/paginated.d.ts

@@ -1,93 +0,0 @@
-/**
- * Paginated Feed Base Class
- *
- * Provides a reusable pagination pattern for feeds that fetch data in pages.
- * Uses the template method pattern - subclasses implement specific methods while
- * the base class handles the pagination loop, rate limiting, and checkpoint management.
- */
-import { BaseFeed } from './base.js';
-import type { Checkpoint, Content, Env, FeedOptions } from './types.js';
-/**
- * Configuration for pagination behavior
- */
-export interface PaginationConfig {
-    /** Maximum pages to fetch (safety limit). Default: 50 */
-    maxPages: number;
-    /** Expected items per page (for partial page detection). Default: 100 */
-    pageSize: number;
-    /** Milliseconds to wait between page requests. Default: 1000 */
-    rateLimitMs: number;
-    /** Whether to update checkpoint after each page (for resume). Default: false */
-    incrementalCheckpoint: boolean;
-}
-/**
- * Result from fetching a single page
- */
-export interface PageFetchResult<TItem> {
-    /** Items from this page */
-    items: TItem[];
-    /** Token/cursor for next page (null = no more pages) */
-    nextToken: string | null;
-    /** Optional: raw item count before any filtering (for logging) */
-    rawCount?: number;
-}
-/**
- * Extended checkpoint interface with pagination token
- */
-export interface PaginatedCheckpoint extends Checkpoint {
-    /** Pagination token for resuming sync */
-    pagination_token?: string | null;
-    /** Whether initial sync completed (used to choose checkpoint boundary) */
-    initial_complete?: boolean;
-}
-/**
- * Result from the paginate() method
- */
-export interface PaginateResult<TCheckpoint extends Checkpoint> {
-    contents: Content[];
-    checkpoint: TCheckpoint;
-    parentMap?: Map<string, string>;
-    nextSyncRecommendedAt?: Date;
-}
-/**
- * Base class for feeds with cursor/page-based pagination
- */
-export declare abstract class PaginatedFeed<TItem, TCheckpoint extends PaginatedCheckpoint = PaginatedCheckpoint> extends BaseFeed {
-    /**
-     * Default pagination configuration
-     */
-    protected getPaginationConfig(): PaginationConfig;
-    /**
-     * ABSTRACT: Fetch a single page of items from the platform
-     */
-    protected abstract fetchPage(cursor: string | null, options: FeedOptions, env: Env): Promise<PageFetchResult<TItem>>;
-    /**
-     * ABSTRACT: Transform a platform item to Content format
-     */
-    protected abstract transformItem(item: TItem, options: FeedOptions): Content;
-    /**
-     * ABSTRACT: Extract published date from platform item
-     */
-    protected abstract getItemDate(item: TItem): Date;
-    /**
-     * OPTIONAL: Filter items before transformation
-     */
-    protected filterItem(_item: TItem, _options: FeedOptions): boolean;
-    /**
-     * OPTIONAL: Extract parent ID for hierarchical content
-     */
-    protected getParentId(_item: TItem): string | null;
-    /**
-     * OPTIONAL: Extract pagination token from checkpoint
-     */
-    protected getPaginationToken(checkpoint: TCheckpoint | null): string | null;
-    /**
-     * OPTIONAL: Create checkpoint with feed-specific fields
-     */
-    protected createCheckpoint(existing: TCheckpoint | null, latestContent: Content | null, nextToken: string | null, itemsProcessed: number): TCheckpoint;
-    /**
-     * Main pagination loop - handles both initial and incremental syncs
-     */
-    protected paginate(options: FeedOptions, checkpoint: TCheckpoint | null, env: Env, updateCheckpointFn?: (checkpoint: Checkpoint) => Promise<void>): Promise<PaginateResult<TCheckpoint>>;
-}
-//# sourceMappingURL=paginated.d.ts.map

package/dist/paginated.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"paginated.d.ts","sourceRoot":"","sources":["../src/paginated.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,QAAQ,EAAkB,MAAM,WAAW,CAAC;AAErD,OAAO,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAExE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,gFAAgF;IAChF,qBAAqB,EAAE,OAAO,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,KAAK;IACpC,2BAA2B;IAC3B,KAAK,EAAE,KAAK,EAAE,CAAC;IACf,wDAAwD;IACxD,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,UAAU;IACrD,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,WAAW,SAAS,UAAU;IAC5D,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,UAAU,EAAE,WAAW,CAAC;IACxB,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,qBAAqB,CAAC,EAAE,IAAI,CAAC;CAC9B;AAED;;GAEG;AACH,8BAAsB,aAAa,CACjC,KAAK,EACL,WAAW,SAAS,mBAAmB,GAAG,mBAAmB,CAC7D,SAAQ,QAAQ;IAChB;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,gBAAgB;IASjD;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,SAAS,CAC1B,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,GAAG,GACP,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAElC;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,WAAW,GAAG,OAAO;IAE5E;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,GAAG,IAAI;IAEjD;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,GAAG,OAAO;IAIlE;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,GAAG,IAAI;IAIlD;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,UAAU,EAAE,WAAW,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI;IAI3E;;OAEG;IACH,SAAS,CAAC,gBAAgB,CACxB,QAAQ,EAAE,WAAW,GAAG,IAAI,EAC5B,aAAa,EAAE,OAAO,GAAG,IAAI,EAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,cAAc,EAAE,MAAM,GACrB,WAAW;IAUd;;OAEG;cACa,QAAQ,CACtB,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,WAAW,GAAG,IAAI,EAC9B,GAAG,EAAE,GAAG,EACR,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAC7D,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;CA6JxC"}

package/dist/paginated.js

@@ -1,167 +0,0 @@
-/**
- * Paginated Feed Base Class
- *
- * Provides a reusable pagination pattern for feeds that fetch data in pages.
- * Uses the template method pattern - subclasses implement specific methods while
- * the base class handles the pagination loop, rate limiting, and checkpoint management.
- */
-import { BaseFeed, RateLimitError } from './base.js';
-import { sdkLogger } from './logger.js';
-/**
- * Base class for feeds with cursor/page-based pagination
- */
-export class PaginatedFeed extends BaseFeed {
-    /**
-     * Default pagination configuration
-     */
-    getPaginationConfig() {
-        return {
-            maxPages: 50,
-            pageSize: 100,
-            rateLimitMs: 1000,
-            incrementalCheckpoint: false,
-        };
-    }
-    /**
-     * OPTIONAL: Filter items before transformation
-     */
-    filterItem(_item, _options) {
-        return true;
-    }
-    /**
-     * OPTIONAL: Extract parent ID for hierarchical content
-     */
-    getParentId(_item) {
-        return null;
-    }
-    /**
-     * OPTIONAL: Extract pagination token from checkpoint
-     */
-    getPaginationToken(checkpoint) {
-        return checkpoint?.pagination_token ?? null;
-    }
-    /**
-     * OPTIONAL: Create checkpoint with feed-specific fields
-     */
-    createCheckpoint(existing, latestContent, nextToken, itemsProcessed) {
-        return {
-            updated_at: new Date(),
-            last_timestamp: latestContent?.occurred_at ?? existing?.last_timestamp,
-            pagination_token: nextToken,
-            total_items_processed: (existing?.total_items_processed || 0) + itemsProcessed,
-            initial_complete: existing?.initial_complete ?? false,
-        };
-    }
-    /**
-     * Main pagination loop - handles both initial and incremental syncs
-     */
-    async paginate(options, checkpoint, env, updateCheckpointFn) {
-        const config = this.getPaginationConfig();
-        const existingToken = this.getPaginationToken(checkpoint);
-        const parentMap = new Map();
-        const initialComplete = checkpoint?.initial_complete ?? (!!checkpoint?.last_timestamp && !existingToken);
-        const isIncremental = initialComplete && !existingToken;
-        const lookbackDate = this.getLookbackDate(options);
-        const checkpointTimestamp = checkpoint?.last_timestamp ?? null;
-        const checkpointDate = checkpointTimestamp
-            ? checkpointTimestamp instanceof Date
-                ? checkpointTimestamp
-                : new Date(checkpointTimestamp)
-            : null;
-        const boundaryDate = initialComplete && checkpointDate ? checkpointDate : lookbackDate;
-        const allContents = [];
-        let cursor = isIncremental ? null : existingToken;
-        let pageCount = 0;
-        let stopReason = null;
-        let rateLimitRetryMs;
-        const modeLabel = isIncremental ? 'incremental' : cursor ? 'resumed' : 'initial';
-        sdkLogger.info(`[${this.type}] Starting ${modeLabel} sync (max ${config.maxPages} pages)`);
-        while (pageCount < config.maxPages) {
-            if (pageCount > 0) {
-                await this.sleep(config.rateLimitMs);
-            }
-            let result;
-            try {
-                result = await this.fetchPage(cursor, options, env);
-            }
-            catch (error) {
-                if (error instanceof RateLimitError) {
-                    sdkLogger.warn({ error: error.message }, `[${this.type}] Rate limit hit after ${pageCount} pages, pausing`);
-                    stopReason = 'rate_limited';
-                    rateLimitRetryMs = error.retryAfterMs;
-                    break;
-                }
-                throw error;
-            }
-            const rawCount = result.rawCount ?? result.items.length;
-            if (result.items.length === 0) {
-                sdkLogger.info(`[${this.type}] Empty page at ${pageCount + 1}, stopping`);
-                stopReason = 'empty_page';
-                break;
-            }
-            const pageContents = [];
-            for (const item of result.items) {
-                if (!this.filterItem(item, options))
-                    continue;
-                const content = this.transformItem(item, options);
-                pageContents.push(content);
-                const parentId = this.getParentId(item);
-                if (parentId) {
-                    parentMap.set(content.origin_id, parentId);
-                    if (!content.origin_parent_id) {
-                        content.origin_parent_id = parentId;
-                    }
-                }
-            }
-            const boundaryContents = pageContents.filter((c) => c.occurred_at >= boundaryDate);
-            allContents.push(...boundaryContents);
-            pageCount++;
-            sdkLogger.info(`[${this.type}] Page ${pageCount}: raw=${rawCount}, ` +
-                `filtered=${pageContents.length}, recent=${boundaryContents.length}, ` +
-                `nextToken=${result.nextToken ? 'yes' : 'no'}`);
-            if (config.incrementalCheckpoint && updateCheckpointFn && allContents.length > 0) {
-                const incrementalCP = this.createCheckpoint(checkpoint, allContents[0], result.nextToken, boundaryContents.length);
-                await updateCheckpointFn(incrementalCP);
-                sdkLogger.debug(`[${this.type}] Saved incremental checkpoint after page ${pageCount}`);
-            }
-            if (boundaryContents.length === 0 && pageContents.length > 0) {
-                sdkLogger.info(`[${this.type}] Reached boundary after ${pageCount} pages`);
-                stopReason = 'boundary';
-                break;
-            }
-            if (result.items.length < config.pageSize) {
-                sdkLogger.info(`[${this.type}] Partial page (${result.items.length}/${config.pageSize}), end reached`);
-                stopReason = 'partial_page';
-                break;
-            }
-            if (!result.nextToken) {
-                sdkLogger.info(`[${this.type}] No next token after page ${pageCount}, end reached`);
-                stopReason = 'no_next_token';
-                break;
-            }
-            cursor = result.nextToken;
-        }
-        if (!stopReason && pageCount >= config.maxPages) {
-            stopReason = 'max_pages';
-        }
-        if (stopReason === 'max_pages') {
-            sdkLogger.warn(`[${this.type}] Hit max page limit (${config.maxPages})`);
-        }
-        sdkLogger.info(`[${this.type}] Pagination complete: ${allContents.length} items from ${pageCount} pages`);
-        const resumeToken = stopReason === 'max_pages' || stopReason === 'rate_limited' ? cursor : null;
-        const shouldMarkComplete = initialComplete ||
-            (!initialComplete && stopReason !== 'max_pages' && stopReason !== 'rate_limited');
-        const nextSyncRecommendedAt = stopReason === 'rate_limited'
-            ? new Date(Date.now() + (rateLimitRetryMs ?? config.rateLimitMs))
-            : undefined;
-        const finalCheckpoint = this.createCheckpoint(checkpoint, allContents[0] || null, resumeToken, allContents.length);
-        finalCheckpoint.initial_complete = shouldMarkComplete;
-        return {
-            contents: allContents,
-            checkpoint: finalCheckpoint,
-            parentMap: parentMap.size > 0 ? parentMap : undefined,
-            nextSyncRecommendedAt,
-        };
-    }
-}
-//# sourceMappingURL=paginated.js.map

package/dist/paginated.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"paginated.js","sourceRoot":"","sources":["../src/paginated.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAiDxC;;GAEG;AACH,MAAM,OAAgB,aAGpB,SAAQ,QAAQ;IAChB;;OAEG;IACO,mBAAmB;QAC3B,OAAO;YACL,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE,GAAG;YACb,WAAW,EAAE,IAAI;YACjB,qBAAqB,EAAE,KAAK;SAC7B,CAAC;IACJ,CAAC;IAqBD;;OAEG;IACO,UAAU,CAAC,KAAY,EAAE,QAAqB;QACtD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,KAAY;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,UAA8B;QACzD,OAAO,UAAU,EAAE,gBAAgB,IAAI,IAAI,CAAC;IAC9C,CAAC;IAED;;OAEG;IACO,gBAAgB,CACxB,QAA4B,EAC5B,aAA6B,EAC7B,SAAwB,EACxB,cAAsB;QAEtB,OAAO;YACL,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,cAAc,EAAE,aAAa,EAAE,WAAW,IAAI,QAAQ,EAAE,cAAc;YACtE,gBAAgB,EAAE,SAAS;YAC3B,qBAAqB,EAAE,CAAC,QAAQ,EAAE,qBAAqB,IAAI,CAAC,CAAC,GAAG,cAAc;YAC9E,gBAAgB,EAAE,QAAQ,EAAE,gBAAgB,IAAI,KAAK;SACvC,CAAC;IACnB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,QAAQ,CACtB,OAAoB,EACpB,UAA8B,EAC9B,GAAQ,EACR,kBAA8D;QAE9D,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,MAAM,eAAe,GACnB,UAAU,EAAE,gBAAgB,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,cAAc,IAAI,CAAC,aAAa,CAAC,CAAC;QACnF,MAAM,aAAa,GAAG,eAAe,IAAI,CAAC,aAAa,CAAC;QACxD,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,mBAAmB,GAAG,UAAU,EAAE,cAAc,IAAI,IAAI,CAAC;QAC/D,MAAM,cAAc,GAAG,mBAAmB;YACxC,CAAC,CAAC,mBAAmB,YAAY,IAAI;gBACnC,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,IAAI,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC,CAAC,IAAI,CAAC;QACT,MAAM,YAAY,GAAG,eAAe,IAAI,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC;QAEvF,MAAM,WAAW,GAAc,EAAE,CAAC;QAClC,IAAI,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC;QAClD,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,UAAU,GAOH,IAAI,CAAC;QAChB,IAAI,gBAAoC,CAAC;QACzC,MAAM,SAAS,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QAEjF,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,cAAc,SAAS,cAAc,MAAM,CAAC,QAAQ,SAAS,CAAC,CAAC;QAE3F,OAAO,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACvC,CAAC;YAED,IAAI,MAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;YACtD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;oBACpC,SAAS,CAAC,IAAI,CACZ,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,EACxB,IAAI,IAAI,CAAC,IAAI,0BAA0B,SAAS,iBAAiB,CAClE,CAAC;oBACF,UAAU,GAAG,cAAc,CAAC;oBAC5B,gBAAgB,GAAG,KAAK,CAAC,YAAY,CAAC;oBACtC,MAAM;gBACR,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;YAExD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,mBAAmB,SAAS,GAAG,CAAC,YAAY,CAAC,CAAC;gBAC1E,UAAU,GAAG,YAAY,CAAC;gBAC1B,MAAM;YACR,CAAC;YAED,MAAM,YAAY,GAAc,EAAE,CAAC;YACnC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;oBAAE,SAAS;gBAE9C,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAClD,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACxC,IAAI,QAAQ,EAAE,CAAC;oBACb,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;oBAC3C,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;wBAC9B,OAAO,CAAC,gBAAgB,GAAG,QAAQ,CAAC;oBACtC,CAAC;gBACH,CAAC;YACH,CAAC;YAED,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,YAAY,CAAC,CAAC;YACnF,WAAW,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAC;YAEtC,SAAS,EAAE,CAAC;YAEZ,SAAS,CAAC,IAAI,CACZ,IAAI,IAAI,CAAC,IAAI,UAAU,SAAS,SAAS,QAAQ,IAAI;gBACnD,YAAY,YAAY,CAAC,MAAM,YAAY,gBAAgB,CAAC,MAAM,IAAI;gBACtE,aAAa,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CACjD,CAAC;YAEF,IAAI,MAAM,CAAC,qBAAqB,IAAI,kBAAkB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjF,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CACzC,UAAU,EACV,WAAW,CAAC,CAAC,CAAC,EACd,MAAM,CAAC,SAAS,EAChB,gBAAgB,CAAC,MAAM,CACxB,CAAC;gBACF,MAAM,kBAAkB,CAAC,aAAa,CAAC,CAAC;gBACxC,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,6CAA6C,SAAS,EAAE,CAAC,CAAC;YACzF,CAAC;YAED,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7D,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,4BAA4B,SAAS,QAAQ,CAAC,CAAC;gBAC3E,UAAU,GAAG,UAAU,CAAC;gBACxB,MAAM;YACR,CAAC;YAED,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAC1C,SAAS,CAAC,IAAI,CACZ,IAAI,IAAI,CAAC,IAAI,mBAAmB,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,gBAAgB,CACvF,CAAC;gBACF,UAAU,GAAG,cAAc,CAAC;gBAC5B,MAAM;YACR,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBACtB,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,8BAA8B,SAAS,eAAe,CAAC,CAAC;gBACpF,UAAU,GAAG,eAAe,CAAC;gBAC7B,MAAM;YACR,CAAC;YAED,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,UAAU,IAAI,SAAS,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAChD,UAAU,GAAG,WAAW,CAAC;QAC3B,CAAC;QAED,IAAI,UAAU,KAAK,WAAW,EAAE,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,yBAAyB,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC3E,CAAC;QAED,SAAS,CAAC,IAAI,CACZ,IAAI,IAAI,CAAC,IAAI,0BAA0B,WAAW,CAAC,MAAM,eAAe,SAAS,QAAQ,CAC1F,CAAC;QAEF,MAAM,WAAW,GAAG,UAAU,KAAK,WAAW,IAAI,UAAU,KAAK,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;QAChG,MAAM,kBAAkB,GACtB,eAAe;YACf,CAAC,CAAC,eAAe,IAAI,UAAU,KAAK,WAAW,IAAI,UAAU,KAAK,cAAc,CAAC,CAAC;QACpF,MAAM,qBAAqB,GACzB,UAAU,KAAK,cAAc;YAC3B,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,gBAAgB,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC;YACjE,CAAC,CAAC,SAAS,CAAC;QAEhB,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAC3C,UAAU,EACV,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,EACtB,WAAW,EACX,WAAW,CAAC,MAAM,CACnB,CAAC;QACF,eAAe,CAAC,gBAAgB,GAAG,kBAAkB,CAAC;QAEtD,OAAO;YACL,QAAQ,EAAE,WAAW;YACrB,UAAU,EAAE,eAAe;YAC3B,SAAS,EAAE,SAAS,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YACrD,qBAAqB;SACtB,CAAC;IACJ,CAAC;CACF"}