@shopimind/integration-kit-js 1.0.0

package/dist/store/repositories.js

@@ -0,0 +1,281 @@
+const nn = (v) => (v === undefined ? null : v);
+/** GCM AAD binding an encrypted secret to its exact location (anti-relocation). */
+const aadFor = (installationId, key) => `${installationId}:${key}`;
+/** Installs (one row per installation). COALESCE upsert: a null field does not overwrite. */
+export class InstallRepo {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    upsert(u) {
+        this.db
+            .prepare(`INSERT INTO installs
+           (installation_id, shop_domain, shop_name, status, installed_at, updated_at)
+         VALUES
+           (@installation_id, @shop_domain, @shop_name, @status, @installed_at, datetime('now'))
+         ON CONFLICT(installation_id) DO UPDATE SET
+           shop_domain  = COALESCE(excluded.shop_domain, shop_domain),
+           shop_name    = COALESCE(excluded.shop_name, shop_name),
+           status       = excluded.status,
+           installed_at = COALESCE(excluded.installed_at, installed_at),
+           updated_at   = datetime('now')`)
+            .run({
+            installation_id: u.installation_id,
+            shop_domain: nn(u.shop_domain),
+            shop_name: nn(u.shop_name),
+            status: u.status,
+            installed_at: nn(u.installed_at),
+        });
+    }
+    setStatus(installationId, status, stamps = {}) {
+        this.db
+            .prepare(`UPDATE installs SET
+           status         = @status,
+           activated_at   = @activated_at,
+           deactivated_at = @deactivated_at,
+           uninstalled_at = @uninstalled_at,
+           updated_at     = datetime('now')
+         WHERE installation_id = @installation_id`)
+            .run({
+            installation_id: installationId,
+            status,
+            activated_at: nn(stamps.activated_at),
+            deactivated_at: nn(stamps.deactivated_at),
+            uninstalled_at: nn(stamps.uninstalled_at),
+        });
+    }
+    /** Sets the INTEGRATOR account associated with the installation (the correlation bridge). */
+    setExternalAccount(installationId, ref, name = null) {
+        this.db
+            .prepare(`UPDATE installs SET
+           external_account_ref  = @ref,
+           external_account_name = @name,
+           updated_at            = datetime('now')
+         WHERE installation_id = @installation_id`)
+            .run({ installation_id: installationId, ref: nn(ref), name: nn(name) });
+    }
+    find(installationId) {
+        return this.db
+            .prepare('SELECT * FROM installs WHERE installation_id = ?')
+            .get(installationId);
+    }
+    listActive() {
+        return this.db.prepare(`SELECT * FROM installs WHERE status = 'active'`).all();
+    }
+}
+/** Webhook log. The `payload_json` MUST already be redacted by the runtime. */
+export class WebhookLogRepo {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    log(entry) {
+        this.db
+            .prepare(`INSERT INTO webhook_log (event, installation_id, signature_ok, payload_json)
+         VALUES (@event, @installation_id, @signature_ok, @payload_json)`)
+            .run({
+            event: nn(entry.event),
+            installation_id: nn(entry.installation_id),
+            signature_ok: entry.signature_ok ? 1 : 0,
+            payload_json: entry.payload_json,
+        });
+    }
+    /** Retention: deletes log rows older than `days` days. Returns the number of rows removed. */
+    purgeOlderThan(days) {
+        return this.db
+            .prepare(`DELETE FROM webhook_log WHERE created_at < datetime('now', @cutoff)`)
+            .run({ cutoff: `-${days} days` }).changes;
+    }
+}
+/** Replay protection for lifecycle webhooks (key derived from the signature). */
+export class WebhookSeenRepo {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    /** Atomically claims processing; `false` if the signature was already seen (replay). */
+    claim(installationId, dedupKey) {
+        const info = this.db
+            .prepare(`INSERT INTO webhook_seen (installation_id, dedup_key) VALUES (?, ?)
+         ON CONFLICT(installation_id, dedup_key) DO NOTHING`)
+            .run(installationId, dedupKey);
+        return info.changes === 1;
+    }
+    /** Releases a claim (failed processing) -> an identical resend can retry. */
+    release(installationId, dedupKey) {
+        this.db
+            .prepare('DELETE FROM webhook_seen WHERE installation_id = ? AND dedup_key = ?')
+            .run(installationId, dedupKey);
+    }
+    /** Retention: deletes dedup rows older than `days` days. Returns the number of rows removed. */
+    purgeOlderThan(days) {
+        return this.db
+            .prepare(`DELETE FROM webhook_seen WHERE created_at < datetime('now', @cutoff)`)
+            .run({ cutoff: `-${days} days` }).changes;
+    }
+}
+/** Sync cursors, scoped by (installation, entity, source_key). */
+export class CursorRepo {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    get(installationId, entity, sourceKey = '') {
+        return this.db
+            .prepare('SELECT * FROM sync_cursor WHERE installation_id = ? AND entity = ? AND source_key = ?')
+            .get(installationId, entity, sourceKey);
+    }
+    set(installationId, entity, sourceKey, w) {
+        this.db
+            .prepare(`INSERT INTO sync_cursor
+           (installation_id, entity, source_key, last_synced_at, last_status, last_error, items, updated_at)
+         VALUES
+           (@installation_id, @entity, @source_key, @last_synced_at, @last_status, @last_error, @items, datetime('now'))
+         ON CONFLICT(installation_id, entity, source_key) DO UPDATE SET
+           last_synced_at = excluded.last_synced_at,
+           last_status    = excluded.last_status,
+           last_error     = excluded.last_error,
+           items          = excluded.items,
+           updated_at     = datetime('now')`)
+            .run({
+            installation_id: installationId,
+            entity,
+            source_key: sourceKey,
+            last_synced_at: w.last_synced_at,
+            last_status: nn(w.last_status),
+            last_error: nn(w.last_error),
+            items: w.items ?? 0,
+        });
+    }
+}
+/** History of sync runs. */
+export class RunRepo {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    start(installationId) {
+        const info = this.db
+            .prepare(`INSERT INTO sync_run (installation_id, status) VALUES (?, 'running')`)
+            .run(installationId);
+        return Number(info.lastInsertRowid);
+    }
+    finish(runId, status, summary) {
+        // Never let a non-serializable summary (circular reference, BigInt, …) throw and
+        // leave the run stuck in 'running': fall back to a safe placeholder instead.
+        let summaryJson;
+        try {
+            summaryJson = JSON.stringify(summary);
+        }
+        catch {
+            summaryJson = '{"error":"unserializable summary"}';
+        }
+        this.db
+            .prepare(`UPDATE sync_run SET status = @status, summary_json = @summary_json, finished_at = datetime('now') WHERE id = @id`)
+            .run({ id: runId, status, summary_json: summaryJson });
+    }
+    recent(installationId, limit = 10) {
+        return this.db
+            .prepare('SELECT * FROM sync_run WHERE installation_id = ? ORDER BY id DESC LIMIT ?')
+            .all(installationId, limit);
+    }
+}
+/**
+ * Private integration state (KV per installation). Sensitive values are encrypted
+ * at rest via the kit's `SecretCipher`.
+ */
+export class IntegrationStateRepo {
+    db;
+    cipher;
+    constructor(db, cipher) {
+        this.db = db;
+        this.cipher = cipher;
+    }
+    write(installationId, key, value, encrypted) {
+        this.db
+            .prepare(`INSERT INTO integration_state (installation_id, key, value, encrypted, updated_at)
+         VALUES (@id, @key, @value, @encrypted, datetime('now'))
+         ON CONFLICT(installation_id, key) DO UPDATE SET
+           value = excluded.value, encrypted = excluded.encrypted, updated_at = datetime('now')`)
+            .run({ id: installationId, key, value, encrypted: encrypted ? 1 : 0 });
+    }
+    set(installationId, key, value) {
+        this.write(installationId, key, value, false);
+    }
+    setSecret(installationId, key, value) {
+        this.write(installationId, key, this.cipher.encrypt(value, aadFor(installationId, key)), true);
+    }
+    get(installationId, key) {
+        const row = this.db
+            .prepare('SELECT value, encrypted FROM integration_state WHERE installation_id = ? AND key = ?')
+            .get(installationId, key);
+        if (!row || row.value === null)
+            return null;
+        return row.encrypted ? this.cipher.decrypt(row.value, aadFor(installationId, key)) : row.value;
+    }
+    delete(installationId, key) {
+        this.db
+            .prepare('DELETE FROM integration_state WHERE installation_id = ? AND key = ?')
+            .run(installationId, key);
+    }
+}
+/** Log of inbound calls (middleware idempotency + audit). */
+export class InboundEventRepo {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    find(installationId, idempotencyKey) {
+        return this.db
+            .prepare('SELECT * FROM inbound_event WHERE installation_id = ? AND idempotency_key = ?')
+            .get(installationId, idempotencyKey);
+    }
+    /**
+     * ATOMICALLY claims the processing of an inbound call (anti-TOCTOU). The INSERT
+     * `ON CONFLICT DO NOTHING` relies on the UNIQUE index (installation_id, idempotency_key):
+     * - `fresh: true` => row created by THIS call -> it is responsible for processing;
+     * - `fresh: false` => a row already existed -> `status` tells where it stands
+     *   ('received' = in progress/already claimed, 'done' = processed, 'failed' = retryable).
+     */
+    claim(installationId, idempotencyKey, action) {
+        const info = this.db
+            .prepare(`INSERT INTO inbound_event (installation_id, idempotency_key, action, status)
+         VALUES (?, ?, ?, 'received')
+         ON CONFLICT(installation_id, idempotency_key) DO NOTHING`)
+            .run(installationId, idempotencyKey, nn(action));
+        if (info.changes === 1)
+            return { rowId: Number(info.lastInsertRowid), fresh: true, status: 'received' };
+        const existing = this.find(installationId, idempotencyKey);
+        if (!existing) {
+            // Extreme race (row deleted between the INSERT and the SELECT): retry once.
+            const retry = this.db
+                .prepare(`INSERT INTO inbound_event (installation_id, idempotency_key, action, status) VALUES (?, ?, ?, 'received') ON CONFLICT(installation_id, idempotency_key) DO NOTHING`)
+                .run(installationId, idempotencyKey, nn(action));
+            const row = this.find(installationId, idempotencyKey);
+            return { rowId: row ? row.id : Number(retry.lastInsertRowid), fresh: retry.changes === 1, status: row ? row.status : 'received' };
+        }
+        return { rowId: existing.id, fresh: false, status: existing.status };
+    }
+    finish(id, status, error = null) {
+        this.db
+            .prepare(`UPDATE inbound_event SET status = @status, error = @error, processed_at = datetime('now') WHERE id = @id`)
+            .run({ id, status, error: nn(error) });
+    }
+    /** Retention: deletes inbound rows older than `days` days. Returns the number of rows removed. */
+    purgeOlderThan(days) {
+        return this.db
+            .prepare(`DELETE FROM inbound_event WHERE received_at < datetime('now', @cutoff)`)
+            .run({ cutoff: `-${days} days` }).changes;
+    }
+}
+export function createRepositories(db, cipher) {
+    return {
+        installs: new InstallRepo(db),
+        webhookLog: new WebhookLogRepo(db),
+        webhookSeen: new WebhookSeenRepo(db),
+        cursors: new CursorRepo(db),
+        runs: new RunRepo(db),
+        state: new IntegrationStateRepo(db, cipher),
+        inboundEvents: new InboundEventRepo(db),
+    };
+}

package/dist/store/types.d.ts

@@ -0,0 +1,62 @@
+export type InstallStatus = 'inactive' | 'active' | 'uninstalled';
+/**
+ * An installation. `installation_id` = OPAQUE token issued by ShopiMind.
+ * `external_account_ref`/`_name` = the reference of the INTEGRATOR's internal
+ * account (the correlation bridge, set by the integration).
+ */
+export interface InstallRow {
+    installation_id: string;
+    shop_domain: string | null;
+    shop_name: string | null;
+    external_account_ref: string | null;
+    external_account_name: string | null;
+    status: string;
+    installed_at: string | null;
+    activated_at: string | null;
+    deactivated_at: string | null;
+    uninstalled_at: string | null;
+    created_at: string;
+    updated_at: string;
+}
+export interface InstallUpsert {
+    installation_id: string;
+    shop_domain?: string | null;
+    shop_name?: string | null;
+    status: InstallStatus;
+    installed_at?: string | null;
+}
+export interface CursorRow {
+    installation_id: string;
+    entity: string;
+    source_key: string;
+    last_synced_at: string | null;
+    last_status: string | null;
+    last_error: string | null;
+    items: number;
+    updated_at: string;
+}
+export interface CursorWrite {
+    last_synced_at: string;
+    last_status?: string;
+    last_error?: string | null;
+    items?: number;
+}
+export interface SyncRunRow {
+    id: number;
+    installation_id: string;
+    status: string;
+    summary_json: string | null;
+    started_at: string;
+    finished_at: string | null;
+}
+/** A row of the inbound call log (inbound routes / middleware). */
+export interface InboundEventRow {
+    id: number;
+    installation_id: string;
+    idempotency_key: string;
+    action: string | null;
+    status: string;
+    error: string | null;
+    received_at: string;
+    processed_at: string | null;
+}

package/dist/store/types.js

@@ -0,0 +1 @@
+export {};

package/dist/sync/concurrency.d.ts

@@ -0,0 +1,12 @@
+/**
+ * BOUNDED-concurrency map (avoids 429s). At most `limit` promises in flight at a
+ * time, with result order preserved - avoids bursts of calls that trigger a 429
+ * on the third-party API side.
+ *
+ * IMPORTANT — this MATERIALIZES the input iterable eagerly (`[...items]`) and
+ * returns a fully-buffered result array. Unlike `paginate`/`streamPages`, it is
+ * NOT streaming: both the input and the output are held in memory at once. Do not
+ * pass a huge lazy stream (e.g. a long backfill) to it — page it with `paginate`
+ * first and map each bounded batch instead.
+ */
+export declare function mapWithConcurrency<I, O>(items: Iterable<I>, limit: number, fn: (item: I, index: number) => Promise<O>): Promise<O[]>;

package/dist/sync/concurrency.js

@@ -0,0 +1,30 @@
+/**
+ * BOUNDED-concurrency map (avoids 429s). At most `limit` promises in flight at a
+ * time, with result order preserved - avoids bursts of calls that trigger a 429
+ * on the third-party API side.
+ *
+ * IMPORTANT — this MATERIALIZES the input iterable eagerly (`[...items]`) and
+ * returns a fully-buffered result array. Unlike `paginate`/`streamPages`, it is
+ * NOT streaming: both the input and the output are held in memory at once. Do not
+ * pass a huge lazy stream (e.g. a long backfill) to it — page it with `paginate`
+ * first and map each bounded batch instead.
+ */
+export async function mapWithConcurrency(items, limit, fn) {
+    if (!Number.isInteger(limit) || limit < 1) {
+        throw new Error('concurrency limit must be an integer >= 1');
+    }
+    const all = [...items];
+    const results = new Array(all.length);
+    let cursor = 0;
+    const worker = async () => {
+        for (;;) {
+            const index = cursor++;
+            if (index >= all.length)
+                return;
+            results[index] = await fn(all[index], index);
+        }
+    };
+    const workerCount = Math.min(limit, all.length);
+    await Promise.all(Array.from({ length: workerCount }, () => worker()));
+    return results;
+}

package/dist/sync/cursor.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Cursor semantics that are SAFE by construction.
+ *
+ * Golden rule (enforced by the ENGINE, never by the integration): the cursor
+ * only advances if the step had NO error. A partial failure replays its window
+ * on the next run - no silent data loss.
+ */
+export type CursorScope = 'global' | 'per-source';
+export interface SyncStepOutcome {
+    errors: string[];
+    /** Upper bound to advance the cursor to IF the run is clean. */
+    advanceCursorTo?: Date;
+}
+export declare function shouldAdvanceCursor(outcome: SyncStepOutcome): boolean;
+/** Returns the new cursor value, or `null` if it should not advance. */
+export declare function nextCursorValue(outcome: SyncStepOutcome): Date | null;

package/dist/sync/cursor.js

@@ -0,0 +1,14 @@
+/**
+ * Cursor semantics that are SAFE by construction.
+ *
+ * Golden rule (enforced by the ENGINE, never by the integration): the cursor
+ * only advances if the step had NO error. A partial failure replays its window
+ * on the next run - no silent data loss.
+ */
+export function shouldAdvanceCursor(outcome) {
+    return outcome.errors.length === 0 && outcome.advanceCursorTo != null;
+}
+/** Returns the new cursor value, or `null` if it should not advance. */
+export function nextCursorValue(outcome) {
+    return shouldAdvanceCursor(outcome) ? outcome.advanceCursorTo : null;
+}

package/dist/sync/engine.d.ts

@@ -0,0 +1,49 @@
+import type { CursorRepo, RunRepo } from '../store/repositories.js';
+import type { Integration, IntegrationContext, SyncWindow } from '../integration/types.js';
+import type { SourceHandle } from '../sdk/source-scope.js';
+import { type SendBulk } from '../sdk/send-bulk.js';
+export interface SyncOptions {
+    fullBackfill?: boolean;
+    backfillDays?: number;
+    /** Injectable for testing; defaults to `() => new Date()`. */
+    now?: () => Date;
+}
+export interface SyncStepSummary {
+    entity: string;
+    sourceKey: string;
+    items: number;
+    /** Items the API rejected during the step (data NOT persisted on the ShopiMind side). */
+    rejected: number;
+    errors: string[];
+    advanced: boolean;
+}
+export interface SyncSummary {
+    runId: number;
+    status: 'ok' | 'partial';
+    steps: SyncStepSummary[];
+    errors: string[];
+}
+export interface SyncDeps {
+    cursors: CursorRepo;
+    runs: RunRepo;
+    /**
+     * Builds `withSource` bound to a given `sendBulk`, so source-scoped pushes
+     * (`withSource(k).send(...)`) feed the step's reject accumulator. Provided by the
+     * runtime, which knows the installation + provisioning state.
+     */
+    makeSource: (sendBulk: SendBulk) => (sourceKey: string) => SourceHandle;
+}
+/**
+ * Runs the enabled sync steps of an integration. The cursor is managed HERE,
+ * never by the integration:
+ *   - scope 'global'      -> one cursor (entity, '')
+ *   - scope 'per-source'  -> one cursor per source (entity, sourceKey)
+ * GOLDEN RULE: the cursor only advances if the step had NO error.
+ */
+export declare function runIntegrationSync<S>(integration: Pick<Integration<S>, 'syncSteps'>, base: IntegrationContext<S>, deps: SyncDeps, opts?: SyncOptions): Promise<SyncSummary>;
+/** Sync window: backfill on the first run / in full mode, otherwise from the cursor. */
+export declare function computeWindow(lastSyncedAt: string | null, opts: {
+    now: () => Date;
+    full: boolean;
+    backfillDays: number;
+}): SyncWindow;

package/dist/sync/engine.js

@@ -0,0 +1,129 @@
+import { makeSendBulk } from '../sdk/send-bulk.js';
+import { paginate } from './paginate.js';
+import { mapWithConcurrency } from './concurrency.js';
+import { shouldAdvanceCursor } from './cursor.js';
+/**
+ * Runs the enabled sync steps of an integration. The cursor is managed HERE,
+ * never by the integration:
+ *   - scope 'global'      -> one cursor (entity, '')
+ *   - scope 'per-source'  -> one cursor per source (entity, sourceKey)
+ * GOLDEN RULE: the cursor only advances if the step had NO error.
+ */
+export async function runIntegrationSync(integration, base, deps, opts = {}) {
+    const now = opts.now ?? (() => new Date());
+    const backfillDays = opts.backfillDays ?? 365;
+    const full = opts.fullBackfill ?? false;
+    const runId = deps.runs.start(base.installationId);
+    const summary = { runId, status: 'ok', steps: [], errors: [] };
+    try {
+        for (const step of integration.syncSteps) {
+            if (!step.enabled(base.settings))
+                continue;
+            const resolved = await resolveSources(step, base);
+            if (resolved.error) {
+                // A misconfigured per-source step (no sources()) is NOT skipped silently:
+                // surface it so the run is marked partial instead of hiding missing data.
+                summary.errors.push(resolved.error);
+                continue;
+            }
+            for (const sourceKey of resolved.sourceKeys) {
+                const stepSummary = await runOneSource(step, base, deps, sourceKey, { now, full, backfillDays });
+                summary.steps.push(stepSummary);
+                summary.errors.push(...stepSummary.errors);
+            }
+        }
+        summary.status = summary.errors.length > 0 ? 'partial' : 'ok';
+        deps.runs.finish(runId, summary.status, summary);
+        return summary;
+    }
+    catch (e) {
+        deps.runs.finish(runId, 'failed', { errors: [errMsg(e)] });
+        throw e;
+    }
+}
+async function resolveSources(step, base) {
+    if (step.cursorScope === 'global')
+        return { sourceKeys: [''] };
+    if (!step.sources) {
+        return { sourceKeys: [], error: `per-source step '${step.entity}' has no sources()` };
+    }
+    return { sourceKeys: await step.sources(base) };
+}
+async function runOneSource(step, base, deps, sourceKey, win) {
+    const cursor = deps.cursors.get(base.installationId, step.entity, sourceKey) ?? null;
+    const window = computeWindow(cursor?.last_synced_at ?? null, win);
+    // Per-step-run reject accumulator: `ctx.sendBulk` and `withSource(k).send` feed it,
+    // so the engine can HOLD the cursor on data loss EVEN IF the step result omits the
+    // count — safe by construction (the dev cannot forget to surface rejections).
+    const rejects = { count: 0 };
+    const stepSendBulk = makeSendBulk(base.spm, base.logger, (n) => {
+        rejects.count += n;
+    });
+    const ctx = {
+        ...base,
+        entity: step.entity,
+        sourceKey,
+        window,
+        cursor,
+        paginate,
+        mapConcurrent: mapWithConcurrency,
+        sendBulk: stepSendBulk,
+        withSource: deps.makeSource(stepSendBulk),
+    };
+    let result;
+    try {
+        result = await step.run(ctx);
+    }
+    catch (e) {
+        result = { items: 0, errors: [`fatal: ${errMsg(e)}`] };
+    }
+    // GOLDEN RULE: do not advance the cursor on (a) a step error OR (b) unhandled
+    // rejections (data the API did NOT persist). `tolerateRejects` only lifts (b) — for
+    // a windowed stream a PERMANENT rejection ("poison pill") would otherwise freeze the
+    // window forever — but rejections stay visible (the warn log + the summary count).
+    const cleanRun = shouldAdvanceCursor(result);
+    const blockedByRejects = rejects.count > 0 && !step.tolerateRejects;
+    const advanced = cleanRun && !blockedByRejects;
+    const errors = [...result.errors];
+    if (blockedByRejects) {
+        errors.push(`${rejects.count} item(s) rejected (cursor held; set tolerateRejects to advance)`);
+    }
+    if (advanced) {
+        const advanceTo = result.advanceCursorTo;
+        // Never advance past the window upper bound: a future cursor would make the
+        // next window empty/inverted and silently skip data.
+        const clamped = advanceTo.getTime() > window.until.getTime() ? window.until : advanceTo;
+        deps.cursors.set(base.installationId, step.entity, sourceKey, {
+            last_synced_at: clamped.toISOString(),
+            last_status: 'ok',
+            items: result.items,
+        });
+    }
+    else if (errors.length > 0) {
+        // Failed/blocked step: record the failure WITHOUT advancing the cursor, so the
+        // same window is replayed next run (no silent data loss). The CursorWrite API
+        // requires a last_synced_at, so we keep the previous value (or null if none yet).
+        deps.cursors.set(base.installationId, step.entity, sourceKey, {
+            // Keep the OLD cursor value (or null on a never-synced source); the column is
+            // nullable at the DB level even though CursorWrite types it as a string.
+            last_synced_at: (cursor?.last_synced_at ?? null),
+            last_status: 'error',
+            last_error: errors.join('; '),
+            items: result.items,
+        });
+    }
+    return { entity: step.entity, sourceKey, items: result.items, rejected: rejects.count, errors, advanced };
+}
+/** Sync window: backfill on the first run / in full mode, otherwise from the cursor. */
+export function computeWindow(lastSyncedAt, opts) {
+    const until = opts.now();
+    if (opts.full || !lastSyncedAt) {
+        const since = new Date(until);
+        since.setDate(since.getDate() - opts.backfillDays);
+        return { since, until };
+    }
+    return { since: new Date(lastSyncedAt), until };
+}
+function errMsg(e) {
+    return e instanceof Error ? e.message : String(e);
+}

package/dist/sync/paginate.d.ts

@@ -0,0 +1,14 @@
+/**
+ * STREAMING pagination (avoids OOM). Iterates page by page without accumulating
+ * the whole dataset in memory - safe even for a long backfill over large volumes.
+ */
+export interface PaginateOptions {
+    /** First page (defaults to 1). */
+    startPage?: number;
+    /** Page size: a shorter page ends the stream. */
+    pageSize?: number;
+}
+/** Iterates items one by one, page after page, without accumulating the dataset. */
+export declare function paginate<T>(fetchPage: (page: number) => Promise<T[]>, opts?: PaginateOptions): AsyncGenerator<T, void, void>;
+/** Variant that yields PAGES (useful for pushing in batches). */
+export declare function streamPages<T>(fetchPage: (page: number) => Promise<T[]>, opts?: PaginateOptions): AsyncGenerator<T[], void, void>;

package/dist/sync/paginate.js

@@ -0,0 +1,42 @@
+/**
+ * STREAMING pagination (avoids OOM). Iterates page by page without accumulating
+ * the whole dataset in memory - safe even for a long backfill over large volumes.
+ */
+/**
+ * Hard upper bound on the number of pages a single pagination may walk through.
+ * A safety net against an infinite loop when `fetchPage` keeps returning full
+ * pages without ever advancing (e.g. a misconfigured backfill that ignores the
+ * page argument). 100k pages is far beyond any legitimate sync.
+ */
+const MAX_PAGES = 100000;
+/** Iterates items one by one, page after page, without accumulating the dataset. */
+export async function* paginate(fetchPage, opts = {}) {
+    const start = opts.startPage ?? 1;
+    for (let page = start;; page++) {
+        if (page - start >= MAX_PAGES) {
+            throw new Error('pagination exceeded MAX_PAGES — fetchPage likely not advancing');
+        }
+        const rows = await fetchPage(page);
+        if (!Array.isArray(rows) || rows.length === 0)
+            return;
+        for (const row of rows)
+            yield row;
+        if (opts.pageSize != null && rows.length < opts.pageSize)
+            return;
+    }
+}
+/** Variant that yields PAGES (useful for pushing in batches). */
+export async function* streamPages(fetchPage, opts = {}) {
+    const start = opts.startPage ?? 1;
+    for (let page = start;; page++) {
+        if (page - start >= MAX_PAGES) {
+            throw new Error('pagination exceeded MAX_PAGES — fetchPage likely not advancing');
+        }
+        const rows = await fetchPage(page);
+        if (!Array.isArray(rows) || rows.length === 0)
+            return;
+        yield rows;
+        if (opts.pageSize != null && rows.length < opts.pageSize)
+            return;
+    }
+}