@lobu/connector-sdk 6.1.1 → 7.1.0

package/dist/paginated.js

@@ -1,167 +0,0 @@
-/**
- * Paginated Feed Base Class
- *
- * Provides a reusable pagination pattern for feeds that fetch data in pages.
- * Uses the template method pattern - subclasses implement specific methods while
- * the base class handles the pagination loop, rate limiting, and checkpoint management.
- */
-import { BaseFeed, RateLimitError } from './base.js';
-import { sdkLogger } from './logger.js';
-/**
- * Base class for feeds with cursor/page-based pagination
- */
-export class PaginatedFeed extends BaseFeed {
-    /**
-     * Default pagination configuration
-     */
-    getPaginationConfig() {
-        return {
-            maxPages: 50,
-            pageSize: 100,
-            rateLimitMs: 1000,
-            incrementalCheckpoint: false,
-        };
-    }
-    /**
-     * OPTIONAL: Filter items before transformation
-     */
-    filterItem(_item, _options) {
-        return true;
-    }
-    /**
-     * OPTIONAL: Extract parent ID for hierarchical content
-     */
-    getParentId(_item) {
-        return null;
-    }
-    /**
-     * OPTIONAL: Extract pagination token from checkpoint
-     */
-    getPaginationToken(checkpoint) {
-        return checkpoint?.pagination_token ?? null;
-    }
-    /**
-     * OPTIONAL: Create checkpoint with feed-specific fields
-     */
-    createCheckpoint(existing, latestContent, nextToken, itemsProcessed) {
-        return {
-            updated_at: new Date(),
-            last_timestamp: latestContent?.occurred_at ?? existing?.last_timestamp,
-            pagination_token: nextToken,
-            total_items_processed: (existing?.total_items_processed || 0) + itemsProcessed,
-            initial_complete: existing?.initial_complete ?? false,
-        };
-    }
-    /**
-     * Main pagination loop - handles both initial and incremental syncs
-     */
-    async paginate(options, checkpoint, env, updateCheckpointFn) {
-        const config = this.getPaginationConfig();
-        const existingToken = this.getPaginationToken(checkpoint);
-        const parentMap = new Map();
-        const initialComplete = checkpoint?.initial_complete ?? (!!checkpoint?.last_timestamp && !existingToken);
-        const isIncremental = initialComplete && !existingToken;
-        const lookbackDate = this.getLookbackDate(options);
-        const checkpointTimestamp = checkpoint?.last_timestamp ?? null;
-        const checkpointDate = checkpointTimestamp
-            ? checkpointTimestamp instanceof Date
-                ? checkpointTimestamp
-                : new Date(checkpointTimestamp)
-            : null;
-        const boundaryDate = initialComplete && checkpointDate ? checkpointDate : lookbackDate;
-        const allContents = [];
-        let cursor = isIncremental ? null : existingToken;
-        let pageCount = 0;
-        let stopReason = null;
-        let rateLimitRetryMs;
-        const modeLabel = isIncremental ? 'incremental' : cursor ? 'resumed' : 'initial';
-        sdkLogger.info(`[${this.type}] Starting ${modeLabel} sync (max ${config.maxPages} pages)`);
-        while (pageCount < config.maxPages) {
-            if (pageCount > 0) {
-                await this.sleep(config.rateLimitMs);
-            }
-            let result;
-            try {
-                result = await this.fetchPage(cursor, options, env);
-            }
-            catch (error) {
-                if (error instanceof RateLimitError) {
-                    sdkLogger.warn({ error: error.message }, `[${this.type}] Rate limit hit after ${pageCount} pages, pausing`);
-                    stopReason = 'rate_limited';
-                    rateLimitRetryMs = error.retryAfterMs;
-                    break;
-                }
-                throw error;
-            }
-            const rawCount = result.rawCount ?? result.items.length;
-            if (result.items.length === 0) {
-                sdkLogger.info(`[${this.type}] Empty page at ${pageCount + 1}, stopping`);
-                stopReason = 'empty_page';
-                break;
-            }
-            const pageContents = [];
-            for (const item of result.items) {
-                if (!this.filterItem(item, options))
-                    continue;
-                const content = this.transformItem(item, options);
-                pageContents.push(content);
-                const parentId = this.getParentId(item);
-                if (parentId) {
-                    parentMap.set(content.origin_id, parentId);
-                    if (!content.origin_parent_id) {
-                        content.origin_parent_id = parentId;
-                    }
-                }
-            }
-            const boundaryContents = pageContents.filter((c) => c.occurred_at >= boundaryDate);
-            allContents.push(...boundaryContents);
-            pageCount++;
-            sdkLogger.info(`[${this.type}] Page ${pageCount}: raw=${rawCount}, ` +
-                `filtered=${pageContents.length}, recent=${boundaryContents.length}, ` +
-                `nextToken=${result.nextToken ? 'yes' : 'no'}`);
-            if (config.incrementalCheckpoint && updateCheckpointFn && allContents.length > 0) {
-                const incrementalCP = this.createCheckpoint(checkpoint, allContents[0], result.nextToken, boundaryContents.length);
-                await updateCheckpointFn(incrementalCP);
-                sdkLogger.debug(`[${this.type}] Saved incremental checkpoint after page ${pageCount}`);
-            }
-            if (boundaryContents.length === 0 && pageContents.length > 0) {
-                sdkLogger.info(`[${this.type}] Reached boundary after ${pageCount} pages`);
-                stopReason = 'boundary';
-                break;
-            }
-            if (result.items.length < config.pageSize) {
-                sdkLogger.info(`[${this.type}] Partial page (${result.items.length}/${config.pageSize}), end reached`);
-                stopReason = 'partial_page';
-                break;
-            }
-            if (!result.nextToken) {
-                sdkLogger.info(`[${this.type}] No next token after page ${pageCount}, end reached`);
-                stopReason = 'no_next_token';
-                break;
-            }
-            cursor = result.nextToken;
-        }
-        if (!stopReason && pageCount >= config.maxPages) {
-            stopReason = 'max_pages';
-        }
-        if (stopReason === 'max_pages') {
-            sdkLogger.warn(`[${this.type}] Hit max page limit (${config.maxPages})`);
-        }
-        sdkLogger.info(`[${this.type}] Pagination complete: ${allContents.length} items from ${pageCount} pages`);
-        const resumeToken = stopReason === 'max_pages' || stopReason === 'rate_limited' ? cursor : null;
-        const shouldMarkComplete = initialComplete ||
-            (!initialComplete && stopReason !== 'max_pages' && stopReason !== 'rate_limited');
-        const nextSyncRecommendedAt = stopReason === 'rate_limited'
-            ? new Date(Date.now() + (rateLimitRetryMs ?? config.rateLimitMs))
-            : undefined;
-        const finalCheckpoint = this.createCheckpoint(checkpoint, allContents[0] || null, resumeToken, allContents.length);
-        finalCheckpoint.initial_complete = shouldMarkComplete;
-        return {
-            contents: allContents,
-            checkpoint: finalCheckpoint,
-            parentMap: parentMap.size > 0 ? parentMap : undefined,
-            nextSyncRecommendedAt,
-        };
-    }
-}
-//# sourceMappingURL=paginated.js.map

package/dist/paginated.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"paginated.js","sourceRoot":"","sources":["../src/paginated.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAiDxC;;GAEG;AACH,MAAM,OAAgB,aAGpB,SAAQ,QAAQ;IAChB;;OAEG;IACO,mBAAmB;QAC3B,OAAO;YACL,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE,GAAG;YACb,WAAW,EAAE,IAAI;YACjB,qBAAqB,EAAE,KAAK;SAC7B,CAAC;IACJ,CAAC;IAqBD;;OAEG;IACO,UAAU,CAAC,KAAY,EAAE,QAAqB;QACtD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,KAAY;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,UAA8B;QACzD,OAAO,UAAU,EAAE,gBAAgB,IAAI,IAAI,CAAC;IAC9C,CAAC;IAED;;OAEG;IACO,gBAAgB,CACxB,QAA4B,EAC5B,aAA6B,EAC7B,SAAwB,EACxB,cAAsB;QAEtB,OAAO;YACL,UAAU,EAAE,IAAI,IAAI,EAAE;YACtB,cAAc,EAAE,aAAa,EAAE,WAAW,IAAI,QAAQ,EAAE,cAAc;YACtE,gBAAgB,EAAE,SAAS;YAC3B,qBAAqB,EAAE,CAAC,QAAQ,EAAE,qBAAqB,IAAI,CAAC,CAAC,GAAG,cAAc;YAC9E,gBAAgB,EAAE,QAAQ,EAAE,gBAAgB,IAAI,KAAK;SACvC,CAAC;IACnB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,QAAQ,CACtB,OAAoB,EACpB,UAA8B,EAC9B,GAAQ,EACR,kBAA8D;QAE9D,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,MAAM,eAAe,GACnB,UAAU,EAAE,gBAAgB,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,cAAc,IAAI,CAAC,aAAa,CAAC,CAAC;QACnF,MAAM,aAAa,GAAG,eAAe,IAAI,CAAC,aAAa,CAAC;QACxD,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,mBAAmB,GAAG,UAAU,EAAE,cAAc,IAAI,IAAI,CAAC;QAC/D,MAAM,cAAc,GAAG,mBAAmB;YACxC,CAAC,CAAC,mBAAmB,YAAY,IAAI;gBACnC,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,IAAI,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC,CAAC,IAAI,CAAC;QACT,MAAM,YAAY,GAAG,eAAe,IAAI,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC;QAEvF,MAAM,WAAW,GAAc,EAAE,CAAC;QAClC,IAAI,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC;QAClD,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,UAAU,GAOH,IAAI,CAAC;QAChB,IAAI,gBAAoC,CAAC;QACzC,MAAM,SAAS,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QAEjF,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,cAAc,SAAS,cAAc,MAAM,CAAC,QAAQ,SAAS,CAAC,CAAC;QAE3F,OAAO,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;YACnC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACvC,CAAC;YAED,IAAI,MAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;YACtD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;oBACpC,SAAS,CAAC,IAAI,CACZ,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,EACxB,IAAI,IAAI,CAAC,IAAI,0BAA0B,SAAS,iBAAiB,CAClE,CAAC;oBACF,UAAU,GAAG,cAAc,CAAC;oBAC5B,gBAAgB,GAAG,KAAK,CAAC,YAAY,CAAC;oBACtC,MAAM;gBACR,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;YAExD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,mBAAmB,SAAS,GAAG,CAAC,YAAY,CAAC,CAAC;gBAC1E,UAAU,GAAG,YAAY,CAAC;gBAC1B,MAAM;YACR,CAAC;YAED,MAAM,YAAY,GAAc,EAAE,CAAC;YACnC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;oBAAE,SAAS;gBAE9C,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAClD,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAE3B,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACxC,IAAI,QAAQ,EAAE,CAAC;oBACb,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;oBAC3C,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;wBAC9B,OAAO,CAAC,gBAAgB,GAAG,QAAQ,CAAC;oBACtC,CAAC;gBACH,CAAC;YACH,CAAC;YAED,MAAM,gBAAgB,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,YAAY,CAAC,CAAC;YACnF,WAAW,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAC;YAEtC,SAAS,EAAE,CAAC;YAEZ,SAAS,CAAC,IAAI,CACZ,IAAI,IAAI,CAAC,IAAI,UAAU,SAAS,SAAS,QAAQ,IAAI;gBACnD,YAAY,YAAY,CAAC,MAAM,YAAY,gBAAgB,CAAC,MAAM,IAAI;gBACtE,aAAa,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CACjD,CAAC;YAEF,IAAI,MAAM,CAAC,qBAAqB,IAAI,kBAAkB,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjF,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CACzC,UAAU,EACV,WAAW,CAAC,CAAC,CAAC,EACd,MAAM,CAAC,SAAS,EAChB,gBAAgB,CAAC,MAAM,CACxB,CAAC;gBACF,MAAM,kBAAkB,CAAC,aAAa,CAAC,CAAC;gBACxC,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,6CAA6C,SAAS,EAAE,CAAC,CAAC;YACzF,CAAC;YAED,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7D,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,4BAA4B,SAAS,QAAQ,CAAC,CAAC;gBAC3E,UAAU,GAAG,UAAU,CAAC;gBACxB,MAAM;YACR,CAAC;YAED,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAC1C,SAAS,CAAC,IAAI,CACZ,IAAI,IAAI,CAAC,IAAI,mBAAmB,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,gBAAgB,CACvF,CAAC;gBACF,UAAU,GAAG,cAAc,CAAC;gBAC5B,MAAM;YACR,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBACtB,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,8BAA8B,SAAS,eAAe,CAAC,CAAC;gBACpF,UAAU,GAAG,eAAe,CAAC;gBAC7B,MAAM;YACR,CAAC;YAED,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,UAAU,IAAI,SAAS,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAChD,UAAU,GAAG,WAAW,CAAC;QAC3B,CAAC;QAED,IAAI,UAAU,KAAK,WAAW,EAAE,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,yBAAyB,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC3E,CAAC;QAED,SAAS,CAAC,IAAI,CACZ,IAAI,IAAI,CAAC,IAAI,0BAA0B,WAAW,CAAC,MAAM,eAAe,SAAS,QAAQ,CAC1F,CAAC;QAEF,MAAM,WAAW,GAAG,UAAU,KAAK,WAAW,IAAI,UAAU,KAAK,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;QAChG,MAAM,kBAAkB,GACtB,eAAe;YACf,CAAC,CAAC,eAAe,IAAI,UAAU,KAAK,WAAW,IAAI,UAAU,KAAK,cAAc,CAAC,CAAC;QACpF,MAAM,qBAAqB,GACzB,UAAU,KAAK,cAAc;YAC3B,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,gBAAgB,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC;YACjE,CAAC,CAAC,SAAS,CAAC;QAEhB,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAC3C,UAAU,EACV,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,EACtB,WAAW,EACX,WAAW,CAAC,MAAM,CACnB,CAAC;QACF,eAAe,CAAC,gBAAgB,GAAG,kBAAkB,CAAC;QAEtD,OAAO;YACL,QAAQ,EAAE,WAAW;YACrB,UAAU,EAAE,eAAe;YAC3B,SAAS,EAAE,SAAS,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YACrD,qBAAqB;SACtB,CAAC;IACJ,CAAC;CACF"}