@shopimind/integration-kit-js 1.3.0 → 1.4.0

package/dist/store/repositories.js

@@ -86,6 +86,13 @@ export class WebhookLogRepo {
             .prepare(`DELETE FROM webhook_log WHERE created_at < datetime('now', @cutoff)`)
             .run({ cutoff: `-${days} days` }).changes;
     }
+    /** Most recent webhook log entries (newest first) — for /admin/overview (E5). */
+    recent(limit = 20) {
+        const capped = Math.max(1, Math.min(limit, 200));
+        return this.db
+            .prepare('SELECT event, installation_id, signature_ok, created_at FROM webhook_log ORDER BY id DESC LIMIT ?')
+            .all(capped);
+    }
 }
 /** Replay protection for lifecycle webhooks (key derived from the signature). */
 export class WebhookSeenRepo {
@@ -128,14 +135,18 @@ export class CursorRepo {
     set(installationId, entity, sourceKey, w) {
         this.db
             .prepare(`INSERT INTO sync_cursor
-           (installation_id, entity, source_key, last_synced_at, last_status, last_error, items, updated_at)
+           (installation_id, entity, source_key, last_synced_at, last_status, last_error, items,
+            consecutive_failures, updated_at)
          VALUES
-           (@installation_id, @entity, @source_key, @last_synced_at, @last_status, @last_error, @items, datetime('now'))
+           (@installation_id, @entity, @source_key, @last_synced_at, @last_status, @last_error, @items,
+            COALESCE(@consecutive_failures, 0), datetime('now'))
          ON CONFLICT(installation_id, entity, source_key) DO UPDATE SET
            last_synced_at = excluded.last_synced_at,
            last_status    = excluded.last_status,
            last_error     = excluded.last_error,
            items          = excluded.items,
+           -- Omitted (@consecutive_failures IS NULL) -> keep the existing counter.
+           consecutive_failures = COALESCE(@consecutive_failures, sync_cursor.consecutive_failures),
            updated_at     = datetime('now')`)
             .run({
             installation_id: installationId,
@@ -145,8 +156,22 @@ export class CursorRepo {
             last_status: nn(w.last_status),
             last_error: nn(w.last_error),
             items: w.items ?? 0,
+            consecutive_failures: nn(w.consecutive_failures),
         });
     }
+    /** All cursors for an installation (for /health, /admin/overview). */
+    listByInstallation(installationId) {
+        return this.db
+            .prepare('SELECT * FROM sync_cursor WHERE installation_id = ? ORDER BY entity, source_key')
+            .all(installationId);
+    }
+    /** Number of cursors currently in the `error` status (health signal). */
+    countInError() {
+        const row = this.db
+            .prepare(`SELECT COUNT(*) AS n FROM sync_cursor WHERE last_status = 'error'`)
+            .get();
+        return row.n;
+    }
 }
 /** History of sync runs. */
 export class RunRepo {
@@ -268,6 +293,44 @@ export class InboundEventRepo {
             .run({ cutoff: `-${days} days` }).changes;
     }
 }
+/**
+ * Dead-letter of per-item REJECTIONS (E4). The engine records here what the
+ * ShopiMind API refused during a bulk push (validation), capped per run so a
+ * poison batch cannot flood the store; an operator inspects/replays via the admin
+ * endpoint. Subject to the same retention purge as the other log tables.
+ */
+export class RejectedItemRepo {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    add(entry) {
+        this.db
+            .prepare(`INSERT INTO rejected_item (installation_id, run_id, entity, source_key, payload_json, reason)
+         VALUES (@installation_id, @run_id, @entity, @source_key, @payload_json, @reason)`)
+            .run({
+            installation_id: entry.installation_id,
+            run_id: nn(entry.run_id),
+            entity: nn(entry.entity),
+            source_key: nn(entry.source_key),
+            payload_json: entry.payload_json,
+            reason: nn(entry.reason),
+        });
+    }
+    /** Most recent rejected items for an installation, newest first (bounded). */
+    listByInstallation(installationId, limit = 100) {
+        const capped = Math.max(1, Math.min(limit, 500));
+        return this.db
+            .prepare('SELECT * FROM rejected_item WHERE installation_id = ? ORDER BY id DESC LIMIT ?')
+            .all(installationId, capped);
+    }
+    /** Retention: deletes rejected-item rows older than `days` days. Returns rows removed. */
+    purgeOlderThan(days) {
+        return this.db
+            .prepare(`DELETE FROM rejected_item WHERE created_at < datetime('now', @cutoff)`)
+            .run({ cutoff: `-${days} days` }).changes;
+    }
+}
 export function createRepositories(db, cipher) {
     return {
         installs: new InstallRepo(db),
@@ -277,5 +340,6 @@ export function createRepositories(db, cipher) {
         runs: new RunRepo(db),
         state: new IntegrationStateRepo(db, cipher),
         inboundEvents: new InboundEventRepo(db),
+        rejectedItems: new RejectedItemRepo(db),
     };
 }

package/dist/store/types.d.ts

@@ -33,13 +33,26 @@ export interface CursorRow {
     last_status: string | null;
     last_error: string | null;
     items: number;
+    /** Consecutive failed/held runs for this cursor (E3). Reset to 0 on a clean advance. */
+    consecutive_failures: number;
     updated_at: string;
 }
 export interface CursorWrite {
-    last_synced_at: string;
+    /**
+     * Upper bound the cursor advanced to (ISO 8601), or `null` when the cursor is
+     * NOT advancing (a failure row that preserves the previous value, possibly
+     * never-synced). Nullable by design — do not cast a null away.
+     */
+    last_synced_at: string | null;
     last_status?: string;
     last_error?: string | null;
     items?: number;
+    /**
+     * Absolute value to persist in `consecutive_failures` (E3). The engine sets 0 on
+     * a clean advance and the incremented count on a failure/hold. Omitted -> left
+     * unchanged (COALESCE), so callers that do not track it keep the old value.
+     */
+    consecutive_failures?: number;
 }
 export interface SyncRunRow {
     id: number;
@@ -60,3 +73,14 @@ export interface InboundEventRow {
     received_at: string;
     processed_at: string | null;
 }
+/** A dead-lettered item the ShopiMind API REJECTED during a bulk push (E4). */
+export interface RejectedItemRow {
+    id: number;
+    installation_id: string;
+    run_id: number | null;
+    entity: string | null;
+    source_key: string | null;
+    payload_json: string | null;
+    reason: string | null;
+    created_at: string;
+}

package/dist/sync/engine.d.ts

@@ -1,4 +1,4 @@
-import type { CursorRepo, RunRepo } from '../store/repositories.js';
+import type { CursorRepo, RunRepo, RejectedItemRepo } from '../store/repositories.js';
 import type { Integration, IntegrationContext, SyncWindow } from '../integration/types.js';
 import type { SourceHandle } from '../sdk/source-scope.js';
 import type { CustomDataHandle } from '../sdk/custom-data-scope.js';
@@ -6,6 +6,13 @@ import { type SendBulk } from '../sdk/send-bulk.js';
 export interface SyncOptions {
     fullBackfill?: boolean;
     backfillDays?: number;
+    /**
+     * Defensive OVERLAP (E9): on an incremental window, shift `since` back by this many
+     * seconds so an event that landed exactly on the previous cursor boundary (or a
+     * source with slightly skewed clocks) is not missed. Harmless: re-fetched items are
+     * idempotent on the ShopiMind side (bulkSave upserts). 0/undefined -> no overlap.
+     */
+    overlapSeconds?: number;
     /** Injectable for testing; defaults to `() => new Date()`. */
     now?: () => Date;
 }
@@ -17,7 +24,30 @@ export interface SyncStepSummary {
     rejected: number;
     errors: string[];
     advanced: boolean;
+    /**
+     * True when the step-source was NOT run because its cursor is in exponential
+     * backoff after repeated failures (E3). `items`/`rejected` are 0, `errors` empty.
+     */
+    skippedBackoff?: boolean;
 }
+/**
+ * Exponential backoff window after `n` consecutive failures: 2^(n-1) minutes,
+ * capped at 24h. n<=0 -> 0 (no wait). While `now < updated_at + window` the engine
+ * SKIPS the step-source; the cursor is untouched (GOLDEN RULE preserved).
+ */
+export declare function backoffWindowMs(consecutiveFailures: number): number;
+/**
+ * Decides whether per-item rejections should be TOLERATED (cursor may advance),
+ * per the step's `tolerateRejects` policy (E8):
+ *   - `undefined`/`false` -> never tolerate (strict hold);
+ *   - `true`              -> always tolerate (poison-pill escape hatch);
+ *   - `{ maxRatio }`      -> tolerate only while rejected/attempted <= maxRatio.
+ * `attempted` = accepted `items` + `rejected` (the reject sink is not counted in
+ * `items`, so we add it back to size the denominator).
+ */
+export declare function rejectsTolerated(policy: boolean | {
+    maxRatio: number;
+} | undefined, rejected: number, items: number): boolean;
 export interface SyncSummary {
     runId: number;
     status: 'ok' | 'partial';
@@ -38,6 +68,12 @@ export interface SyncDeps {
      * inside a step feeds the step's reject accumulator.
      */
     makeCustomData: (sendBulk: SendBulk) => (name: string) => CustomDataHandle;
+    /**
+     * Optional dead-letter sink (E4). When provided, per-item REJECTIONS reported
+     * during a step are recorded here (capped per run) so an operator can inspect and
+     * later replay what the API refused. Best-effort: a store failure never aborts sync.
+     */
+    rejectedItems?: RejectedItemRepo;
 }
 /**
  * Runs the enabled sync steps of an integration. The cursor is managed HERE,
@@ -47,9 +83,15 @@ export interface SyncDeps {
  * GOLDEN RULE: the cursor only advances if the step had NO error.
  */
 export declare function runIntegrationSync<S>(integration: Pick<Integration<S>, 'syncSteps'>, base: IntegrationContext<S>, deps: SyncDeps, opts?: SyncOptions): Promise<SyncSummary>;
-/** Sync window: backfill on the first run / in full mode, otherwise from the cursor. */
+/**
+ * Sync window: backfill on the first run / in full mode, otherwise from the cursor.
+ * On an incremental window, `overlapSeconds` (E9) shifts `since` back defensively so
+ * an item on the previous boundary is not missed (re-fetches are idempotent upserts).
+ * A backfill window is NOT shifted — it already starts far in the past.
+ */
 export declare function computeWindow(lastSyncedAt: string | null, opts: {
     now: () => Date;
     full: boolean;
     backfillDays: number;
+    overlapSeconds?: number;
 }): SyncWindow;

package/dist/sync/engine.js

@@ -2,6 +2,48 @@ import { makeSendBulk } from '../sdk/send-bulk.js';
 import { paginate } from './paginate.js';
 import { mapWithConcurrency } from './concurrency.js';
 import { shouldAdvanceCursor } from './cursor.js';
+/**
+ * Log ERROR once this many consecutive failures pile up on a cursor (E3), so a
+ * persistently broken source escalates from warn-noise to an actionable signal.
+ */
+const ESCALATE_AT_FAILURES = 3;
+/** Backoff is capped at ~24h: 2^(k-1) minutes, never longer than this. */
+const MAX_BACKOFF_MS = 24 * 60 * 60_000;
+/** Base backoff unit (E3): the k-th consecutive failure skips ticks for ~2^(k-1) minutes. */
+const BACKOFF_BASE_MS = 60_000;
+/** Per-run cap on dead-lettered items (E4) — a poison batch must not flood the store. */
+const REJECTED_ITEMS_CAP_PER_RUN = 500;
+/**
+ * Exponential backoff window after `n` consecutive failures: 2^(n-1) minutes,
+ * capped at 24h. n<=0 -> 0 (no wait). While `now < updated_at + window` the engine
+ * SKIPS the step-source; the cursor is untouched (GOLDEN RULE preserved).
+ */
+export function backoffWindowMs(consecutiveFailures) {
+    if (consecutiveFailures <= 0)
+        return 0;
+    const ms = BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1);
+    return Math.min(ms, MAX_BACKOFF_MS);
+}
+/**
+ * Decides whether per-item rejections should be TOLERATED (cursor may advance),
+ * per the step's `tolerateRejects` policy (E8):
+ *   - `undefined`/`false` -> never tolerate (strict hold);
+ *   - `true`              -> always tolerate (poison-pill escape hatch);
+ *   - `{ maxRatio }`      -> tolerate only while rejected/attempted <= maxRatio.
+ * `attempted` = accepted `items` + `rejected` (the reject sink is not counted in
+ * `items`, so we add it back to size the denominator).
+ */
+export function rejectsTolerated(policy, rejected, items) {
+    if (!policy)
+        return false;
+    if (policy === true)
+        return true;
+    const attempted = items + rejected;
+    if (attempted <= 0)
+        return true; // nothing attempted -> nothing to hold on
+    const ratio = rejected / attempted;
+    return ratio <= policy.maxRatio;
+}
 /**
  * Runs the enabled sync steps of an integration. The cursor is managed HERE,
  * never by the integration:
@@ -13,8 +55,11 @@ export async function runIntegrationSync(integration, base, deps, opts = {}) {
     const now = opts.now ?? (() => new Date());
     const backfillDays = opts.backfillDays ?? 365;
     const full = opts.fullBackfill ?? false;
+    const overlapSeconds = opts.overlapSeconds ?? 0;
     const runId = deps.runs.start(base.installationId);
     const summary = { runId, status: 'ok', steps: [], errors: [] };
+    // Per-run budget shared across all step-sources: caps total dead-lettered items (E4).
+    const deadLetterBudget = { remaining: REJECTED_ITEMS_CAP_PER_RUN };
     try {
         for (const step of integration.syncSteps) {
             if (!step.enabled(base.settings))
@@ -27,7 +72,14 @@ export async function runIntegrationSync(integration, base, deps, opts = {}) {
                 continue;
             }
             for (const sourceKey of resolved.sourceKeys) {
-                const stepSummary = await runOneSource(step, base, deps, sourceKey, { now, full, backfillDays });
+                const stepSummary = await runOneSource(step, base, deps, sourceKey, {
+                    now,
+                    full,
+                    backfillDays,
+                    overlapSeconds,
+                    runId,
+                    deadLetterBudget,
+                });
                 summary.steps.push(stepSummary);
                 summary.errors.push(...stepSummary.errors);
             }
@@ -51,13 +103,38 @@ async function resolveSources(step, base) {
 }
 async function runOneSource(step, base, deps, sourceKey, win) {
     const cursor = deps.cursors.get(base.installationId, step.entity, sourceKey) ?? null;
+    // E3 — EXPONENTIAL BACKOFF. A cursor that keeps failing must not hammer a broken
+    // upstream every tick. While inside the backoff window (based on the last failure's
+    // `updated_at`), skip this step-source entirely. The cursor is NOT touched (GOLDEN
+    // RULE preserved) and no error is added — the source will simply retry once the
+    // window elapses. `full` mode (an explicit operator backfill) bypasses backoff.
+    const failures = cursor?.consecutive_failures ?? 0;
+    if (!win.full && failures > 0 && cursor?.updated_at) {
+        const backoffMs = backoffWindowMs(failures);
+        const lastAt = Date.parse(cursor.updated_at + 'Z'); // stored UTC (SQLite datetime('now'))
+        const readyAt = Number.isNaN(lastAt) ? 0 : lastAt + backoffMs;
+        if (win.now().getTime() < readyAt) {
+            base.logger.info(`sync step '${step.entity}' in backoff — skipped this tick`, {
+                entity: step.entity,
+                sourceKey,
+                consecutive_failures: failures,
+                retry_in_ms: readyAt - win.now().getTime(),
+            });
+            return { entity: step.entity, sourceKey, items: 0, rejected: 0, errors: [], advanced: false, skippedBackoff: true };
+        }
+    }
     const window = computeWindow(cursor?.last_synced_at ?? null, win);
     // Per-step-run reject accumulator: `ctx.sendBulk` and `withSource(k).send` feed it,
     // so the engine can HOLD the cursor on data loss EVEN IF the step result omits the
     // count — safe by construction (the dev cannot forget to surface rejections).
     const rejects = { count: 0 };
-    const stepSendBulk = makeSendBulk(base.spm, base.logger, (n) => {
+    const stepSendBulk = makeSendBulk(base.spm, base.logger, (n, items) => {
         rejects.count += n;
+        // E4 — DEAD-LETTER. Persist what the API refused (bounded by the per-run budget)
+        // so it survives the run for inspection/replay. Best-effort: a store hiccup here
+        // must never fail the sync.
+        if (deps.rejectedItems)
+            recordRejects(deps.rejectedItems, base, step, sourceKey, win, items, rejects.count);
     });
     const ctx = {
         ...base,
@@ -78,12 +155,22 @@ async function runOneSource(step, base, deps, sourceKey, win) {
     catch (e) {
         result = { items: 0, errors: [`fatal: ${errMsg(e)}`] };
     }
+    // A step that finished CLEAN (no error) yet returned no `advanceCursorTo` almost
+    // always means the author forgot to advance: the window will be replayed forever
+    // and the cursor is stuck. This is a silent correctness bug (duplicate work, no
+    // progress), so surface it loudly. A step that legitimately never advances (e.g.
+    // a pure fan-out) can suppress this by returning `advanceCursorTo: ctx.window.until`.
+    if (result.errors.length === 0 && result.advanceCursorTo == null) {
+        base.logger.warn(`sync step '${step.entity}' completed clean without advanceCursorTo — cursor not advanced (window will replay)`, { entity: step.entity, sourceKey, items: result.items });
+    }
     // GOLDEN RULE: do not advance the cursor on (a) a step error OR (b) unhandled
     // rejections (data the API did NOT persist). `tolerateRejects` only lifts (b) — for
     // a windowed stream a PERMANENT rejection ("poison pill") would otherwise freeze the
     // window forever — but rejections stay visible (the warn log + the summary count).
+    // E8: `{ maxRatio }` tolerates only while the reject ratio stays within budget.
     const cleanRun = shouldAdvanceCursor(result);
-    const blockedByRejects = rejects.count > 0 && !step.tolerateRejects;
+    const tolerated = rejectsTolerated(step.tolerateRejects, rejects.count, result.items);
+    const blockedByRejects = rejects.count > 0 && !tolerated;
     const advanced = cleanRun && !blockedByRejects;
     const errors = [...result.errors];
     if (blockedByRejects) {
@@ -98,24 +185,77 @@ async function runOneSource(step, base, deps, sourceKey, win) {
             last_synced_at: clamped.toISOString(),
             last_status: 'ok',
             items: result.items,
+            // E3 — a clean advance clears the failure escalation.
+            consecutive_failures: 0,
         });
     }
     else if (errors.length > 0) {
         // Failed/blocked step: record the failure WITHOUT advancing the cursor, so the
-        // same window is replayed next run (no silent data loss). The CursorWrite API
-        // requires a last_synced_at, so we keep the previous value (or null if none yet).
+        // same window is replayed next run (no silent data loss). `last_synced_at` is
+        // nullable by contract (E11) — keep the previous value (or null if never synced).
+        const nextFailures = failures + 1;
+        if (nextFailures >= ESCALATE_AT_FAILURES) {
+            // E3 — escalate to ERROR once failures pile up: a persistently broken source
+            // deserves an actionable signal, not just repeated warns.
+            base.logger.error(`sync step '${step.entity}' failing repeatedly`, {
+                entity: step.entity,
+                sourceKey,
+                consecutive_failures: nextFailures,
+                last_error: errors.join('; '),
+            });
+        }
         deps.cursors.set(base.installationId, step.entity, sourceKey, {
-            // Keep the OLD cursor value (or null on a never-synced source); the column is
-            // nullable at the DB level even though CursorWrite types it as a string.
-            last_synced_at: (cursor?.last_synced_at ?? null),
+            last_synced_at: cursor?.last_synced_at ?? null,
             last_status: 'error',
             last_error: errors.join('; '),
             items: result.items,
+            consecutive_failures: nextFailures,
         });
     }
     return { entity: step.entity, sourceKey, items: result.items, rejected: rejects.count, errors, advanced };
 }
-/** Sync window: backfill on the first run / in full mode, otherwise from the cursor. */
+/**
+ * Dead-letters the rejected items reported by a push (E4), honouring the per-run
+ * budget. Best-effort: any store error is swallowed (a broken dead-letter must never
+ * fail sync) — the rejection is already surfaced via the warn log + cursor hold.
+ */
+function recordRejects(repo, base, step, sourceKey, win, items, reasonCount) {
+    if (win.deadLetterBudget.remaining <= 0)
+        return;
+    try {
+        for (const item of items) {
+            if (win.deadLetterBudget.remaining <= 0)
+                break;
+            win.deadLetterBudget.remaining -= 1;
+            repo.add({
+                installation_id: base.installationId,
+                run_id: win.runId,
+                entity: step.entity,
+                source_key: sourceKey,
+                payload_json: safeJson(item),
+                reason: `rejected during ${step.entity} push`,
+            });
+        }
+    }
+    catch (e) {
+        base.logger.warn('dead-letter write failed (best-effort)', { entity: step.entity, error: errMsg(e), reasonCount });
+    }
+}
+/** JSON-stringifies an item, falling back to a placeholder on a non-serializable value. */
+function safeJson(v) {
+    try {
+        return JSON.stringify(v);
+    }
+    catch {
+        return '"[unserializable rejected item]"';
+    }
+}
+/**
+ * Sync window: backfill on the first run / in full mode, otherwise from the cursor.
+ * On an incremental window, `overlapSeconds` (E9) shifts `since` back defensively so
+ * an item on the previous boundary is not missed (re-fetches are idempotent upserts).
+ * A backfill window is NOT shifted — it already starts far in the past.
+ */
 export function computeWindow(lastSyncedAt, opts) {
     const until = opts.now();
     if (opts.full || !lastSyncedAt) {
@@ -123,7 +263,11 @@ export function computeWindow(lastSyncedAt, opts) {
         since.setDate(since.getDate() - opts.backfillDays);
         return { since, until };
     }
-    return { since: new Date(lastSyncedAt), until };
+    const since = new Date(lastSyncedAt);
+    const overlap = opts.overlapSeconds ?? 0;
+    if (overlap > 0)
+        since.setTime(since.getTime() - overlap * 1000);
+    return { since, until };
 }
 function errMsg(e) {
     return e instanceof Error ? e.message : String(e);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shopimind/integration-kit-js",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Foundation for building ShopiMind integrations: a runtime plus typed, once-tested primitives (HMAC webhook signatures, encryption, log redaction, a safe cursor-based sync engine, pagination/concurrency, persistence, ShopiMind SDK re-export, idempotent provisioning, secured inbound middleware, HTTP server). An integration only writes pure functions and declarations passed to defineIntegration.",
   "license": "SEE LICENSE IN LICENSE",
   "repository": {