@memberjunction/integration-engine 5.38.0 → 5.40.0

package/dist/RateLimiter.d.ts

@@ -0,0 +1,117 @@
+/**
+ * Adaptive, per-key token-bucket rate limiter (plan.md §7 — "peak-aware rate limiting").
+ *
+ * Each key (an IntegrationID) owns an independent token bucket that refills continuously
+ * at an *effective* rate and caps at a burst capacity. `Acquire(key)` waits until at
+ * least one token is available, then consumes it — letting the engine push a source to
+ * its real limits while never exceeding them.
+ *
+ * The "adaptive" part is classic AIMD (Additive-Increase / Multiplicative-Decrease), the
+ * same control law TCP congestion control uses:
+ *   - `ReportThrottle(key, retryAfterMs?)` — the source signalled a 429 / rate-limit header.
+ *     We MULTIPLICATIVELY cut the effective rate (×`ThrottleBackoffFactor`, default 0.5) and,
+ *     if a Retry-After was given, FREEZE refills for that long so in-flight tokens drain.
+ *   - `ReportSuccess(key)` — sustained success. We ADDITIVELY nudge the effective rate back up
+ *     (+`SuccessRampPerCall`) but never past the configured ceiling (`TokensPerSec`). This finds
+ *     the highest safe throughput and re-discovers it after the source recovers.
+ *
+ * Deterministic + testable: the clock (`now`) and the wait primitive (`sleep`) are injected
+ * via constructor options. Production defaults to wall-clock + a real `setTimeout`; tests pass
+ * fakes so NO real timers are involved and behaviour is fully reproducible.
+ *
+ * Pure: no DB, no network, no MJ metadata. The engine owns ONE instance and routes every
+ * outbound source call through `Acquire` / `ReportThrottle` / `ReportSuccess`.
+ */
+/** Returns the current time in milliseconds (injectable for deterministic tests). */
+export type NowFn = () => number;
+/** Resolves after `ms` milliseconds (injectable for deterministic tests). */
+export type SleepFn = (ms: number) => Promise<void>;
+/** Construction options for {@link RateLimiter}. All fields optional; sensible defaults applied. */
+export interface RateLimiterOptions {
+    /**
+     * Ceiling refill rate in tokens per second — the steady-state target a key ramps back
+     * toward and never exceeds. Also the rate a fresh key starts at. Default: 10.
+     */
+    TokensPerSec?: number;
+    /** Bucket capacity (max tokens that can accumulate) — the allowed burst size. Default: `TokensPerSec`. */
+    Burst?: number;
+    /**
+     * Multiplicative-decrease factor applied to the effective rate on each throttle signal
+     * (0 < f < 1). Default: 0.5 (halve the rate, like TCP).
+     */
+    ThrottleBackoffFactor?: number;
+    /**
+     * Additive-increase amount (tokens/sec) added to the effective rate on each success,
+     * clamped to `TokensPerSec`. Default: `TokensPerSec / 10` (≈10 successes to fully recover).
+     */
+    SuccessRampPerCall?: number;
+    /**
+     * Floor the effective rate can be cut to by repeated throttles, so a key never fully stalls
+     * (tokens/sec). Default: `TokensPerSec / 20`. Clamped to be > 0.
+     */
+    MinTokensPerSec?: number;
+    /** Time source. Default: `Date.now`. */
+    now?: NowFn;
+    /** Wait primitive. Default: a real `setTimeout`-based sleep. */
+    sleep?: SleepFn;
+}
+/** Read-only snapshot of a key's live limiter state — for tests, metrics, and debugging. */
+export interface RateLimiterKeyState {
+    /** Current available tokens (fractional). */
+    Tokens: number;
+    /** Current effective refill rate in tokens/sec (between `MinTokensPerSec` and `TokensPerSec`). */
+    EffectiveTokensPerSec: number;
+    /** Bucket capacity (burst). */
+    Burst: number;
+    /** Epoch ms until which refills are frozen by a Retry-After (0 = not frozen). */
+    FrozenUntil: number;
+}
+/**
+ * Adaptive per-key token-bucket rate limiter. See file header for the full design.
+ * One instance per engine; keys are IntegrationIDs (or any stable string).
+ */
+export declare class RateLimiter {
+    private readonly tokensPerSec;
+    private readonly burst;
+    private readonly throttleBackoffFactor;
+    private readonly successRampPerCall;
+    private readonly minTokensPerSec;
+    private readonly nowFn;
+    private readonly sleepFn;
+    private readonly buckets;
+    constructor(options?: RateLimiterOptions);
+    /**
+     * Waits until a token is available for `key`, then consumes exactly one. Returns the
+     * number of milliseconds spent waiting (0 if a token was immediately available) — useful
+     * for the engine to surface "we are being held back" telemetry.
+     */
+    Acquire(key: string): Promise<number>;
+    /**
+     * Multiplicative decrease: the source threw a 429 / rate-limit signal for `key`. Cuts the
+     * effective rate by `ThrottleBackoffFactor` (floored at `MinTokensPerSec`) and, when a
+     * Retry-After is supplied, freezes refills for that long so the bucket drains.
+     */
+    ReportThrottle(key: string, retryAfterMs?: number): void;
+    /**
+     * Additive increase: a call for `key` succeeded. Nudges the effective rate up by
+     * `SuccessRampPerCall`, never above the configured `TokensPerSec` ceiling (AIMD recovery).
+     */
+    ReportSuccess(key: string): void;
+    /** Returns a read-only snapshot of `key`'s state (refilled to "now"). Creates the bucket if absent. */
+    GetState(key: string): RateLimiterKeyState;
+    /** Drops all per-key state. Mainly for tests and engine teardown. */
+    Reset(): void;
+    /** Lazily creates a bucket starting full (one burst of tokens) at the ceiling rate. */
+    private getBucket;
+    /**
+     * Advances `bucket` to the current time: adds `effectiveRate × elapsedSeconds` tokens
+     * (capped at burst). While frozen by a Retry-After, no time accrues toward refill — the
+     * freeze interval is excluded so the bucket genuinely pauses, then resumes from the freeze end.
+     */
+    private refill;
+    /** Earliest time tokens may start accruing again, accounting for any active freeze window. */
+    private accrualStart;
+    /** Milliseconds until the next whole token, given the current (post-refill) deficit and freeze. */
+    private waitMsForToken;
+}
+//# sourceMappingURL=RateLimiter.d.ts.map

package/dist/RateLimiter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RateLimiter.d.ts","sourceRoot":"","sources":["../src/RateLimiter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,qFAAqF;AACrF,MAAM,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC;AAEjC,6EAA6E;AAC7E,MAAM,MAAM,OAAO,GAAG,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEpD,oGAAoG;AACpG,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0GAA0G;IAC1G,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wCAAwC;IACxC,GAAG,CAAC,EAAE,KAAK,CAAC;IACZ,gEAAgE;IAChE,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,4FAA4F;AAC5F,MAAM,WAAW,mBAAmB;IAChC,6CAA6C;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,kGAAkG;IAClG,qBAAqB,EAAE,MAAM,CAAC;IAC9B,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,WAAW,EAAE,MAAM,CAAC;CACvB;AAaD;;;GAGG;AACH,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAQ;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA6B;gBAEzC,OAAO,GAAE,kBAAuB;IAU5C;;;;OAIG;IACU,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAelD;;;;OAIG;IACI,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI;IAU/D;;;OAGG;IACI,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAMvC,uGAAuG;IAChG,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,mBAAmB;IAWjD,qEAAqE;IAC9D,KAAK,IAAI,IAAI;IAIpB,uFAAuF;IACvF,OAAO,CAAC,SAAS;IAcjB;;;;OAIG;IACH,OAAO,CAAC,MAAM;IAad,8FAA8F;IAC9F,OAAO,CAAC,YAAY;IAQpB,mGAAmG;IACnG,OAAO,CAAC,cAAc;CAOzB"}

package/dist/RateLimiter.js

@@ -0,0 +1,159 @@
+/**
+ * Adaptive, per-key token-bucket rate limiter (plan.md §7 — "peak-aware rate limiting").
+ *
+ * Each key (an IntegrationID) owns an independent token bucket that refills continuously
+ * at an *effective* rate and caps at a burst capacity. `Acquire(key)` waits until at
+ * least one token is available, then consumes it — letting the engine push a source to
+ * its real limits while never exceeding them.
+ *
+ * The "adaptive" part is classic AIMD (Additive-Increase / Multiplicative-Decrease), the
+ * same control law TCP congestion control uses:
+ *   - `ReportThrottle(key, retryAfterMs?)` — the source signalled a 429 / rate-limit header.
+ *     We MULTIPLICATIVELY cut the effective rate (×`ThrottleBackoffFactor`, default 0.5) and,
+ *     if a Retry-After was given, FREEZE refills for that long so in-flight tokens drain.
+ *   - `ReportSuccess(key)` — sustained success. We ADDITIVELY nudge the effective rate back up
+ *     (+`SuccessRampPerCall`) but never past the configured ceiling (`TokensPerSec`). This finds
+ *     the highest safe throughput and re-discovers it after the source recovers.
+ *
+ * Deterministic + testable: the clock (`now`) and the wait primitive (`sleep`) are injected
+ * via constructor options. Production defaults to wall-clock + a real `setTimeout`; tests pass
+ * fakes so NO real timers are involved and behaviour is fully reproducible.
+ *
+ * Pure: no DB, no network, no MJ metadata. The engine owns ONE instance and routes every
+ * outbound source call through `Acquire` / `ReportThrottle` / `ReportSuccess`.
+ */
+const DEFAULT_TOKENS_PER_SEC = 10;
+const DEFAULT_BACKOFF_FACTOR = 0.5;
+/**
+ * Adaptive per-key token-bucket rate limiter. See file header for the full design.
+ * One instance per engine; keys are IntegrationIDs (or any stable string).
+ */
+export class RateLimiter {
+    constructor(options = {}) {
+        this.buckets = new Map();
+        this.tokensPerSec = positive(options.TokensPerSec, DEFAULT_TOKENS_PER_SEC);
+        this.burst = positive(options.Burst, this.tokensPerSec);
+        this.throttleBackoffFactor = clamp01Exclusive(options.ThrottleBackoffFactor, DEFAULT_BACKOFF_FACTOR);
+        this.successRampPerCall = positive(options.SuccessRampPerCall, this.tokensPerSec / 10);
+        this.minTokensPerSec = Math.min(positive(options.MinTokensPerSec, this.tokensPerSec / 20), this.tokensPerSec);
+        this.nowFn = options.now ?? (() => Date.now());
+        this.sleepFn = options.sleep ?? realSleep;
+    }
+    /**
+     * Waits until a token is available for `key`, then consumes exactly one. Returns the
+     * number of milliseconds spent waiting (0 if a token was immediately available) — useful
+     * for the engine to surface "we are being held back" telemetry.
+     */
+    async Acquire(key) {
+        const startMs = this.nowFn();
+        // Loop because the computed wait may land just before a refill tick (fractional rounding),
+        // or a concurrent Acquire on the same key may have taken the token we were waiting for.
+        for (;;) {
+            const bucket = this.getBucket(key);
+            this.refill(bucket);
+            if (bucket.tokens >= 1) {
+                bucket.tokens -= 1;
+                return this.nowFn() - startMs;
+            }
+            await this.sleepFn(this.waitMsForToken(bucket));
+        }
+    }
+    /**
+     * Multiplicative decrease: the source threw a 429 / rate-limit signal for `key`. Cuts the
+     * effective rate by `ThrottleBackoffFactor` (floored at `MinTokensPerSec`) and, when a
+     * Retry-After is supplied, freezes refills for that long so the bucket drains.
+     */
+    ReportThrottle(key, retryAfterMs) {
+        const bucket = this.getBucket(key);
+        this.refill(bucket);
+        bucket.effectiveRate = Math.max(this.minTokensPerSec, bucket.effectiveRate * this.throttleBackoffFactor);
+        if (retryAfterMs != null && retryAfterMs > 0) {
+            bucket.frozenUntilMs = this.nowFn() + retryAfterMs;
+            bucket.tokens = 0; // honour the source's "do not call until then"
+        }
+    }
+    /**
+     * Additive increase: a call for `key` succeeded. Nudges the effective rate up by
+     * `SuccessRampPerCall`, never above the configured `TokensPerSec` ceiling (AIMD recovery).
+     */
+    ReportSuccess(key) {
+        const bucket = this.getBucket(key);
+        this.refill(bucket);
+        bucket.effectiveRate = Math.min(this.tokensPerSec, bucket.effectiveRate + this.successRampPerCall);
+    }
+    /** Returns a read-only snapshot of `key`'s state (refilled to "now"). Creates the bucket if absent. */
+    GetState(key) {
+        const bucket = this.getBucket(key);
+        this.refill(bucket);
+        return {
+            Tokens: bucket.tokens,
+            EffectiveTokensPerSec: bucket.effectiveRate,
+            Burst: this.burst,
+            FrozenUntil: bucket.frozenUntilMs,
+        };
+    }
+    /** Drops all per-key state. Mainly for tests and engine teardown. */
+    Reset() {
+        this.buckets.clear();
+    }
+    /** Lazily creates a bucket starting full (one burst of tokens) at the ceiling rate. */
+    getBucket(key) {
+        let bucket = this.buckets.get(key);
+        if (!bucket) {
+            bucket = {
+                tokens: this.burst,
+                effectiveRate: this.tokensPerSec,
+                lastRefillMs: this.nowFn(),
+                frozenUntilMs: 0,
+            };
+            this.buckets.set(key, bucket);
+        }
+        return bucket;
+    }
+    /**
+     * Advances `bucket` to the current time: adds `effectiveRate × elapsedSeconds` tokens
+     * (capped at burst). While frozen by a Retry-After, no time accrues toward refill — the
+     * freeze interval is excluded so the bucket genuinely pauses, then resumes from the freeze end.
+     */
+    refill(bucket) {
+        const nowMs = this.nowFn();
+        if (nowMs <= bucket.lastRefillMs) {
+            return;
+        }
+        const accrualStartMs = this.accrualStart(bucket, nowMs);
+        if (nowMs > accrualStartMs) {
+            const elapsedSec = (nowMs - accrualStartMs) / 1000;
+            bucket.tokens = Math.min(this.burst, bucket.tokens + elapsedSec * bucket.effectiveRate);
+        }
+        bucket.lastRefillMs = nowMs;
+    }
+    /** Earliest time tokens may start accruing again, accounting for any active freeze window. */
+    accrualStart(bucket, nowMs) {
+        if (bucket.frozenUntilMs > bucket.lastRefillMs) {
+            // Some/all of the elapsed window was frozen; accrual only counts from the freeze end.
+            return Math.min(bucket.frozenUntilMs, nowMs);
+        }
+        return bucket.lastRefillMs;
+    }
+    /** Milliseconds until the next whole token, given the current (post-refill) deficit and freeze. */
+    waitMsForToken(bucket) {
+        const nowMs = this.nowFn();
+        const freezeRemainingMs = Math.max(0, bucket.frozenUntilMs - nowMs);
+        const deficit = 1 - bucket.tokens;
+        const refillMs = deficit > 0 ? Math.ceil((deficit / bucket.effectiveRate) * 1000) : 0;
+        return Math.max(1, freezeRemainingMs + refillMs);
+    }
+}
+/** Real wall-clock sleep — the only impure primitive, isolated here and overridden in tests. */
+function realSleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+/** Returns `value` if it is a finite positive number, else `fallback`. */
+function positive(value, fallback) {
+    return value != null && Number.isFinite(value) && value > 0 ? value : fallback;
+}
+/** Returns `value` if it is a finite number strictly within (0, 1), else `fallback`. */
+function clamp01Exclusive(value, fallback) {
+    return value != null && Number.isFinite(value) && value > 0 && value < 1 ? value : fallback;
+}
+//# sourceMappingURL=RateLimiter.js.map

package/dist/RateLimiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RateLimiter.js","sourceRoot":"","sources":["../src/RateLimiter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AA0DH,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAClC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC;;;GAGG;AACH,MAAM,OAAO,WAAW;IAUpB,YAAY,UAA8B,EAAE;QAF3B,YAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;QAGjD,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,sBAAsB,CAAC,CAAC;QAC3E,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACxD,IAAI,CAAC,qBAAqB,GAAG,gBAAgB,CAAC,OAAO,CAAC,qBAAqB,EAAE,sBAAsB,CAAC,CAAC;QACrG,IAAI,CAAC,kBAAkB,GAAG,QAAQ,CAAC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;QACvF,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,EAAE,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9G,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,KAAK,IAAI,SAAS,CAAC;IAC9C,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,OAAO,CAAC,GAAW;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAC7B,2FAA2F;QAC3F,wFAAwF;QACxF,SAAS,CAAC;YACN,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACpB,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACrB,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;gBACnB,OAAO,IAAI,CAAC,KAAK,EAAE,GAAG,OAAO,CAAC;YAClC,CAAC;YACD,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;QACpD,CAAC;IACL,CAAC;IAED;;;;OAIG;IACI,cAAc,CAAC,GAAW,EAAE,YAAqB;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACpB,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACzG,IAAI,YAAY,IAAI,IAAI,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YAC3C,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,YAAY,CAAC;YACnD,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,+CAA+C;QACtE,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,aAAa,CAAC,GAAW;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACpB,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACvG,CAAC;IAED,uGAAuG;IAChG,QAAQ,CAAC,GAAW;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACpB,OAAO;YACH,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,qBAAqB,EAAE,MAAM,CAAC,aAAa;YAC3C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,MAAM,CAAC,aAAa;SACpC,CAAC;IACN,CAAC;IAED,qEAAqE;IAC9D,KAAK;QACR,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IAED,uFAAuF;IAC/E,SAAS,CAAC,GAAW;QACzB,IAAI,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,GAAG;gBACL,MAAM,EAAE,IAAI,CAAC,KAAK;gBAClB,aAAa,EAAE,IAAI,CAAC,YAAY;gBAChC,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE;gBAC1B,aAAa,EAAE,CAAC;aACnB,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,MAAc;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,IAAI,KAAK,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YAC/B,OAAO;QACX,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACxD,IAAI,KAAK,GAAG,cAAc,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,CAAC,KAAK,GAAG,cAAc,CAAC,GAAG,IAAI,CAAC;YACnD,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,GAAG,UAAU,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;QAC5F,CAAC;QACD,MAAM,CAAC,YAAY,GAAG,KAAK,CAAC;IAChC,CAAC;IAED,8FAA8F;IACtF,YAAY,CAAC,MAAc,EAAE,KAAa;QAC9C,IAAI,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7C,sFAAsF;YACtF,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,MAAM,CAAC,YAAY,CAAC;IAC/B,CAAC;IAED,mGAAmG;IAC3F,cAAc,CAAC,MAAc;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAClC,MAAM,QAAQ,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,iBAAiB,GAAG,QAAQ,CAAC,CAAC;IACrD,CAAC;CACJ;AAED,gGAAgG;AAChG,SAAS,SAAS,CAAC,EAAU;IACzB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,0EAA0E;AAC1E,SAAS,QAAQ,CAAC,KAAyB,EAAE,QAAgB;IACzD,OAAO,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AACnF,CAAC;AAED,wFAAwF;AACxF,SAAS,gBAAgB,CAAC,KAAyB,EAAE,QAAgB;IACjE,OAAO,KAAK,IAAI,IAAI,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;AAChG,CAAC"}

package/dist/SyncLogger.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Structured logger for sync runs.
+ *
+ * Every meaningful step in `IntegrationEngine.RunSync()` emits ONE structured
+ * JSON line through this helper so operators tailing the MJAPI log file can
+ * grep + jq the event stream without parsing free-form prose.
+ *
+ * Vocabulary (the `event` field):
+ *
+ *   sync.run.start                — run dispatched; carries CompanyIntegration shape, config snapshot
+ *   sync.config.loaded            — entity-map + field-map counts and names available for the run
+ *   sync.connector.built          — connector instance constructed (class + import path)
+ *   sync.connector.test           — TestConnection result (success + duration)
+ *   sync.entity-map.start         — per-IO sync starting (direction, watermark, externalObjectName)
+ *   sync.fetch.batch.start        — outbound API fetch about to fire (page/offset/cursor + URL hints)
+ *   sync.fetch.batch.complete     — fetch returned (record count, duration, hasMore)
+ *   sync.record.decision          — per-record action (Create/Update/Skip/Delete) + reason + matchKey
+ *   sync.record.saved             — record persisted to MJ (action, mjEntity, mjID, durationMs)
+ *   sync.record.error             — record failed (classified code, MJ-side or external-side, action)
+ *   sync.push.candidates          — outbound push: count of changed records selected since last push
+ *   sync.push.record              — outbound push: about to send to external (externalID, body keys)
+ *   sync.push.response            — outbound push: response (status, externalIDReturned, durationMs)
+ *   sync.entity-map.complete      — per-IO stats rollup + new watermark
+ *   sync.run.complete             — run terminal: rollup + duration + status
+ *   sync.run.fail                 — run terminal: error
+ *   sync.run.cancelled            — run terminal: abort signal fired
+ *
+ * Each line is `{ ts, event, ...data }` — `ts` is ISO 8601.  All fields are
+ * consistently namespaced so a grep / jq pipeline can extract what it needs.
+ *
+ * To filter the log file:
+ *   tail -f /tmp/mjapi.log | grep '"event":"sync\.'                # all sync events
+ *   tail -f /tmp/mjapi.log | grep '"event":"sync\.record\.'         # per-record only
+ *   tail -f /tmp/mjapi.log | grep '"event":"sync\.fetch\.'          # only external fetches
+ *   tail -f /tmp/mjapi.log | grep '"event":"sync\.push\.'           # only outbound writes
+ *   tail -f /tmp/mjapi.log | grep '"event":"sync\..*\.error"'       # only errors
+ */
+export type SyncLogEvent = 'sync.run.start' | 'sync.config.loaded' | 'sync.connector.built' | 'sync.connector.test' | 'sync.entity-map.start' | 'sync.resume.keyset' | 'sync.partition.reconcile' | 'sync.fetch.batch.start' | 'sync.fetch.batch.complete' | 'sync.record.decision' | 'sync.record.saved' | 'sync.record.error' | 'sync.record.archived' | 'sync.record.conflict' | 'sync.push.candidates' | 'sync.push.record' | 'sync.push.response' | 'sync.entity-map.complete' | 'sync.run.complete' | 'sync.run.fail' | 'sync.run.cancelled' | 'sync.warning';
+import type { IntegrationProgressEmitter } from '@memberjunction/integration-progress-artifacts';
+export interface SyncLoggerContext {
+    /** CompanyIntegration ID — the per-run anchor for filtering. */
+    ciId: string;
+    /** Integration name (HubSpot / YourMembership / …). */
+    integration: string | null | undefined;
+    /** Sync run ID for cross-correlation with run details rows. */
+    runId?: string | null;
+}
+export interface SyncLogEntry {
+    ts: string;
+    event: SyncLogEvent;
+    ciId: string;
+    integration?: string | null;
+    runId?: string | null;
+    /** Free-form, event-specific structured data. */
+    [key: string]: unknown;
+}
+/**
+ * Light wrapper that prepends an ISO timestamp + the per-run context to every
+ * line.  Writes to console.log (or console.error for fail events) so the line
+ * lands in whatever stream the wrapper script is teeing to disk.
+ */
+export declare class SyncLogger {
+    private readonly ctx;
+    private emitter?;
+    constructor(ctx: SyncLoggerContext);
+    /** Update the runId once the run record has been created. */
+    attachRunId(runId: string): void;
+    /** Set the integration name once it's resolved from LoadRunConfiguration. */
+    attachIntegrationName(name: string | null | undefined): void;
+    /**
+     * Attach a durable progress-artifact emitter. Once attached, the grep-friendly
+     * console events are ALSO forwarded (selectively, at stage/batch/error
+     * granularity — never per-record, to keep the stream queryable) to the run's
+     * append-only JSONL artifact so the sync is visible over GraphQL and survives an
+     * MJAPI restart. Terminal run.complete/run.fail are owned by the caller (which
+     * awaits the emitter's async terminal write), so they are NOT forwarded here.
+     */
+    attachEmitter(emitter: IntegrationProgressEmitter): void;
+    /**
+     * Persist a resumable CHECKPOINT (the sync position after a committed batch) into the durable
+     * progress artifact (plan.md §8a). On crash/restart the latest checkpoint's resumableState
+     * (watermark / keyset AfterKey / cursor / batchIndex) lets the run pick back up. Best-effort —
+     * no emitter attached → no-op.
+     */
+    checkpoint(stage: string, resumableState: Record<string, unknown>): void;
+    /**
+     * Emit a non-fatal STRUCTURED WARNING. Unlike record/fetch errors (which mark the run
+     * failed), a warning records a notable-but-non-fatal condition — the canonical case being a
+     * second-layer/association object that fetched zero records because its parents weren't
+     * available (the silent-empty). Forwarded to the durable artifact as a SyncWarning so the
+     * condition is visible over GraphQL instead of a swallowed console.warn, WITHOUT affecting
+     * run success. Goes to console.warn so it's also greppable in the tee'd log.
+     */
+    warning(stage: string, code: string, message: string, data?: Record<string, unknown>): void;
+    emit(event: SyncLogEvent, data?: Record<string, unknown>): void;
+    /**
+     * Best-effort mirror of a sync event into the durable artifact stream. Any
+     * failure here is swallowed — structured logging must never break a sync.
+     */
+    private forwardToEmitter;
+    private stageLabel;
+    private num;
+    /** Narrows an unknown to a plain record (or undefined) without a lazy `any` cast. */
+    private asRecord;
+}
+//# sourceMappingURL=SyncLogger.d.ts.map

package/dist/SyncLogger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SyncLogger.d.ts","sourceRoot":"","sources":["../src/SyncLogger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,MAAM,MAAM,YAAY,GAClB,gBAAgB,GAChB,oBAAoB,GACpB,sBAAsB,GACtB,qBAAqB,GACrB,uBAAuB,GACvB,oBAAoB,GACpB,0BAA0B,GAC1B,wBAAwB,GACxB,2BAA2B,GAC3B,sBAAsB,GACtB,mBAAmB,GACnB,mBAAmB,GACnB,sBAAsB,GACtB,sBAAsB,GACtB,sBAAsB,GACtB,kBAAkB,GAClB,oBAAoB,GACpB,0BAA0B,GAC1B,mBAAmB,GACnB,eAAe,GACf,oBAAoB,GACpB,cAAc,CAAC;AAErB,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,gDAAgD,CAAC;AAEjG,MAAM,WAAW,iBAAiB;IAC9B,gEAAgE;IAChE,IAAI,EAAE,MAAM,CAAC;IACb,uDAAuD;IACvD,WAAW,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACvC,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,iDAAiD;IACjD,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED;;;;GAIG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAoB;IACxC,OAAO,CAAC,OAAO,CAAC,CAA6B;gBAEjC,GAAG,EAAE,iBAAiB;IAIlC,6DAA6D;IACtD,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAIvC,6EAA6E;IACtE,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,IAAI;IAInE;;;;;;;OAOG;IACI,aAAa,CAAC,OAAO,EAAE,0BAA0B,GAAG,IAAI;IAI/D;;;;;OAKG;IACI,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI/E;;;;;;;OAOG;IACI,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI;IAI3F,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,IAAI,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GAAG,IAAI;IAoB1E;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAyDxB,OAAO,CAAC,UAAU;IAKlB,OAAO,CAAC,GAAG;IAIX,qFAAqF;IACrF,OAAO,CAAC,QAAQ;CAGnB"}

package/dist/SyncLogger.js

@@ -0,0 +1,176 @@
+/**
+ * Structured logger for sync runs.
+ *
+ * Every meaningful step in `IntegrationEngine.RunSync()` emits ONE structured
+ * JSON line through this helper so operators tailing the MJAPI log file can
+ * grep + jq the event stream without parsing free-form prose.
+ *
+ * Vocabulary (the `event` field):
+ *
+ *   sync.run.start                — run dispatched; carries CompanyIntegration shape, config snapshot
+ *   sync.config.loaded            — entity-map + field-map counts and names available for the run
+ *   sync.connector.built          — connector instance constructed (class + import path)
+ *   sync.connector.test           — TestConnection result (success + duration)
+ *   sync.entity-map.start         — per-IO sync starting (direction, watermark, externalObjectName)
+ *   sync.fetch.batch.start        — outbound API fetch about to fire (page/offset/cursor + URL hints)
+ *   sync.fetch.batch.complete     — fetch returned (record count, duration, hasMore)
+ *   sync.record.decision          — per-record action (Create/Update/Skip/Delete) + reason + matchKey
+ *   sync.record.saved             — record persisted to MJ (action, mjEntity, mjID, durationMs)
+ *   sync.record.error             — record failed (classified code, MJ-side or external-side, action)
+ *   sync.push.candidates          — outbound push: count of changed records selected since last push
+ *   sync.push.record              — outbound push: about to send to external (externalID, body keys)
+ *   sync.push.response            — outbound push: response (status, externalIDReturned, durationMs)
+ *   sync.entity-map.complete      — per-IO stats rollup + new watermark
+ *   sync.run.complete             — run terminal: rollup + duration + status
+ *   sync.run.fail                 — run terminal: error
+ *   sync.run.cancelled            — run terminal: abort signal fired
+ *
+ * Each line is `{ ts, event, ...data }` — `ts` is ISO 8601.  All fields are
+ * consistently namespaced so a grep / jq pipeline can extract what it needs.
+ *
+ * To filter the log file:
+ *   tail -f /tmp/mjapi.log | grep '"event":"sync\.'                # all sync events
+ *   tail -f /tmp/mjapi.log | grep '"event":"sync\.record\.'         # per-record only
+ *   tail -f /tmp/mjapi.log | grep '"event":"sync\.fetch\.'          # only external fetches
+ *   tail -f /tmp/mjapi.log | grep '"event":"sync\.push\.'           # only outbound writes
+ *   tail -f /tmp/mjapi.log | grep '"event":"sync\..*\.error"'       # only errors
+ */
+/**
+ * Light wrapper that prepends an ISO timestamp + the per-run context to every
+ * line.  Writes to console.log (or console.error for fail events) so the line
+ * lands in whatever stream the wrapper script is teeing to disk.
+ */
+export class SyncLogger {
+    constructor(ctx) {
+        this.ctx = ctx;
+    }
+    /** Update the runId once the run record has been created. */
+    attachRunId(runId) {
+        this.ctx.runId = runId;
+    }
+    /** Set the integration name once it's resolved from LoadRunConfiguration. */
+    attachIntegrationName(name) {
+        this.ctx.integration = name;
+    }
+    /**
+     * Attach a durable progress-artifact emitter. Once attached, the grep-friendly
+     * console events are ALSO forwarded (selectively, at stage/batch/error
+     * granularity — never per-record, to keep the stream queryable) to the run's
+     * append-only JSONL artifact so the sync is visible over GraphQL and survives an
+     * MJAPI restart. Terminal run.complete/run.fail are owned by the caller (which
+     * awaits the emitter's async terminal write), so they are NOT forwarded here.
+     */
+    attachEmitter(emitter) {
+        this.emitter = emitter;
+    }
+    /**
+     * Persist a resumable CHECKPOINT (the sync position after a committed batch) into the durable
+     * progress artifact (plan.md §8a). On crash/restart the latest checkpoint's resumableState
+     * (watermark / keyset AfterKey / cursor / batchIndex) lets the run pick back up. Best-effort —
+     * no emitter attached → no-op.
+     */
+    checkpoint(stage, resumableState) {
+        this.emitter?.checkpoint(stage, resumableState);
+    }
+    /**
+     * Emit a non-fatal STRUCTURED WARNING. Unlike record/fetch errors (which mark the run
+     * failed), a warning records a notable-but-non-fatal condition — the canonical case being a
+     * second-layer/association object that fetched zero records because its parents weren't
+     * available (the silent-empty). Forwarded to the durable artifact as a SyncWarning so the
+     * condition is visible over GraphQL instead of a swallowed console.warn, WITHOUT affecting
+     * run success. Goes to console.warn so it's also greppable in the tee'd log.
+     */
+    warning(stage, code, message, data) {
+        this.emit('sync.warning', { stage, code, message, warningData: data });
+    }
+    emit(event, data = {}) {
+        const entry = {
+            ts: new Date().toISOString(),
+            event,
+            ciId: this.ctx.ciId,
+            integration: this.ctx.integration ?? null,
+            runId: this.ctx.runId ?? null,
+            ...data,
+        };
+        const line = JSON.stringify(entry);
+        if (event === 'sync.run.fail' || event === 'sync.record.error') {
+            console.error(line);
+        }
+        else if (event === 'sync.warning') {
+            console.warn(line);
+        }
+        else {
+            console.log(line);
+        }
+        this.forwardToEmitter(event, data);
+    }
+    /**
+     * Best-effort mirror of a sync event into the durable artifact stream. Any
+     * failure here is swallowed — structured logging must never break a sync.
+     */
+    forwardToEmitter(event, data) {
+        const emitter = this.emitter;
+        if (!emitter)
+            return;
+        try {
+            const stage = this.stageLabel(data);
+            switch (event) {
+                case 'sync.entity-map.start':
+                    emitter.stageStart(stage, `Syncing ${stage}`);
+                    break;
+                case 'sync.fetch.batch.complete':
+                    emitter.emit('records.batch.complete', {
+                        stage,
+                        counts: { processed: this.num(data.recordCount) },
+                        data,
+                    });
+                    break;
+                case 'sync.record.error':
+                    emitter.emit('record.error', {
+                        stage,
+                        level: 'error',
+                        message: typeof data.error === 'string' ? data.error : 'record error',
+                        data,
+                    });
+                    break;
+                case 'sync.record.conflict':
+                    emitter.emit('record.error', { stage, level: 'warn', message: 'conflict', data });
+                    break;
+                case 'sync.warning':
+                    emitter.warning(typeof data.stage === 'string' ? data.stage : stage, typeof data.code === 'string' ? data.code : 'WARNING', typeof data.message === 'string' ? data.message : '', this.asRecord(data.warningData));
+                    break;
+                case 'sync.entity-map.complete':
+                    emitter.stageComplete(stage, {
+                        processed: this.num(data.recordsProcessed),
+                        succeeded: this.num(data.recordsCreated) + this.num(data.recordsUpdated),
+                        failed: this.num(data.recordsErrored),
+                        skipped: this.num(data.recordsSkipped),
+                    });
+                    break;
+                case 'sync.push.response':
+                    emitter.emit('external.call.complete', { stage, level: 'debug', data });
+                    break;
+                default:
+                    // run.start / config.loaded / connector.* / fetch.start / record.decision|saved /
+                    // push.candidates|record / terminal events are intentionally not mirrored — the
+                    // caller owns run lifecycle, and per-record events would bloat the stream.
+                    break;
+            }
+        }
+        catch {
+            /* structured-artifact mirroring is best-effort — never break a sync */
+        }
+    }
+    stageLabel(data) {
+        const s = data.externalObjectName ?? data.entityMap ?? data.entity ?? data.objectName;
+        return typeof s === 'string' && s.length > 0 ? s : 'sync';
+    }
+    num(v) {
+        return typeof v === 'number' && Number.isFinite(v) ? v : 0;
+    }
+    /** Narrows an unknown to a plain record (or undefined) without a lazy `any` cast. */
+    asRecord(v) {
+        return v !== null && typeof v === 'object' ? v : undefined;
+    }
+}
+//# sourceMappingURL=SyncLogger.js.map

package/dist/SyncLogger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SyncLogger.js","sourceRoot":"","sources":["../src/SyncLogger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AA+CH;;;;GAIG;AACH,MAAM,OAAO,UAAU;IAInB,YAAY,GAAsB;QAC9B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACnB,CAAC;IAED,6DAA6D;IACtD,WAAW,CAAC,KAAa;QAC5B,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,6EAA6E;IACtE,qBAAqB,CAAC,IAA+B;QACxD,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,IAAI,CAAC;IAChC,CAAC;IAED;;;;;;;OAOG;IACI,aAAa,CAAC,OAAmC;QACpD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAED;;;;;OAKG;IACI,UAAU,CAAC,KAAa,EAAE,cAAuC;QACpE,IAAI,CAAC,OAAO,EAAE,UAAU,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IACpD,CAAC;IAED;;;;;;;OAOG;IACI,OAAO,CAAC,KAAa,EAAE,IAAY,EAAE,OAAe,EAAE,IAA8B;QACvF,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC;IAEM,IAAI,CAAC,KAAmB,EAAE,OAAgC,EAAE;QAC/D,MAAM,KAAK,GAAiB;YACxB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,KAAK;YACL,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI;YACnB,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI;YACzC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI;YAC7B,GAAG,IAAI;SACV,CAAC;QACF,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,KAAK,KAAK,eAAe,IAAI,KAAK,KAAK,mBAAmB,EAAE,CAAC;YAC7D,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,KAAK,KAAK,cAAc,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACK,gBAAgB,CAAC,KAAmB,EAAE,IAA6B;QACvE,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACpC,QAAQ,KAAK,EAAE,CAAC;gBACZ,KAAK,uBAAuB;oBACxB,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,KAAK,EAAE,CAAC,CAAC;oBAC9C,MAAM;gBACV,KAAK,2BAA2B;oBAC5B,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE;wBACnC,KAAK;wBACL,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE;wBACjD,IAAI;qBACP,CAAC,CAAC;oBACH,MAAM;gBACV,KAAK,mBAAmB;oBACpB,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE;wBACzB,KAAK;wBACL,KAAK,EAAE,OAAO;wBACd,OAAO,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc;wBACrE,IAAI;qBACP,CAAC,CAAC;oBACH,MAAM;gBACV,KAAK,sBAAsB;oBACvB,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;oBAClF,MAAM;gBACV,KAAK,cAAc;oBACf,OAAO,CAAC,OAAO,CACX,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EACnD,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EACrD,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EACpD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAClC,CAAC;oBACF,MAAM;gBACV,KAAK,0BAA0B;oBAC3B,OAAO,CAAC,aAAa,CAAC,KAAK,EAAE;wBACzB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC;wBAC1C,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC;wBACxE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC;wBACrC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC;qBACzC,CAAC,CAAC;oBACH,MAAM;gBACV,KAAK,oBAAoB;oBACrB,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;oBACxE,MAAM;gBACV;oBACI,kFAAkF;oBAClF,gFAAgF;oBAChF,2EAA2E;oBAC3E,MAAM;YACd,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACL,uEAAuE;QAC3E,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,IAA6B;QAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,CAAC;QACtF,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC9D,CAAC;IAEO,GAAG,CAAC,CAAU;QAClB,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,qFAAqF;IAC7E,QAAQ,CAAC,CAAU;QACvB,OAAO,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAE,CAA6B,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5F,CAAC;CACJ"}

package/dist/WatermarkService.d.ts

@@ -43,13 +43,48 @@ export declare class WatermarkService {
      * @returns true if the watermark is valid for its type, false otherwise
      */
     ValidateWatermark(watermarkValue: string, watermarkType: WatermarkType): boolean;
+    /**
+     * Persists a keyset/seek resume position for an INTERRUPTED scan (plan.md §8a).
+     *
+     * Stored on the Pull watermark record as `WatermarkType='Cursor'` so {@link Load} surfaces it and
+     * `ProcessPullSync` can resume the seek (`key > AfterKey`) on the next run instead of re-scanning
+     * from the start. Only meaningful for connectors that declare a `StableOrderingKey`; the engine
+     * gates the call on that, so a timestamp connector's record is never written here. Idempotent —
+     * safe to call repeatedly at the checkpoint cadence and once more on graceful early-exit.
+     */
+    SaveKeysetPosition(entityMapID: string, afterKey: string, contextUser: UserInfo): Promise<void>;
+    /**
+     * Clears a persisted keyset resume position after a CLEAN scan (plan.md §8a), so the next run
+     * starts a fresh seek from the beginning. Keeps the record (still `WatermarkType='Cursor'`) for
+     * bookkeeping but nulls the value — a null value is what the restore logic reads as "no scan in
+     * progress". No-op when no watermark record exists yet.
+     */
+    ClearKeysetPosition(entityMapID: string, contextUser: UserInfo): Promise<void>;
+    /**
+     * Persists the partition→rollup map for a partition hash-diff (Merkle) reconcile of a watermark-less
+     * source (plan.md §7). Stored on the Pull watermark record as `WatermarkType='ChangeToken'` (the
+     * map IS an opaque change-detection token), JSON-encoded. The object is no-watermark, so this field
+     * is free; a partition-reconcile object never also carries a timestamp/cursor. Idempotent upsert.
+     */
+    SavePartitionRollups(entityMapID: string, rollups: Map<string, string>, contextUser: UserInfo): Promise<void>;
+    /**
+     * Loads the previously-stored partition→rollup map (empty map when none / first sync / wrong type),
+     * so the next reconcile can diff against it and deep-sync only the partitions whose rollup moved.
+     */
+    LoadPartitionRollups(entityMapID: string, contextUser: UserInfo): Promise<Map<string, string>>;
     /**
      * Updates an existing watermark record with a new value and timestamp.
      */
     private UpdateExistingWatermark;
     /**
-     * Creates a new watermark record for the given entity map.
+     * Creates a new watermark record for the given entity map (Timestamp type — the default for
+     * incremental connectors).
      */
     private CreateNewWatermark;
+    /**
+     * Shared creator for a new watermark row. Factored out so timestamp watermarks and keyset resume
+     * positions ({@link SaveKeysetPosition}) share one persistence path with a per-call WatermarkType.
+     */
+    private createWatermark;
 }
 //# sourceMappingURL=WatermarkService.d.ts.map

package/dist/WatermarkService.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"WatermarkService.d.ts","sourceRoot":"","sources":["../src/WatermarkService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAExE,OAAO,KAAK,EAAE,gCAAgC,EAAE,MAAM,mBAAmB,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD;;;GAGG;AACH,qBAAa,gBAAgB;IACzB;;;;;;OAMG;IACU,IAAI,CACb,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,QAAQ,EACrB,SAAS,GAAE,MAAM,GAAG,MAAe,GACpC,OAAO,CAAC,gCAAgC,GAAG,IAAI,CAAC;IAcnD;;;;;;;;OAQG;IACU,MAAM,CACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,QAAQ,EACrB,SAAS,GAAE,MAAM,GAAG,MAAe,GACpC,OAAO,CAAC,IAAI,CAAC;IAShB;;;;;;;;;OASG;IACU,cAAc,CACvB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,IAAI,CAAC;IAShB;;;;;;OAMG;IACI,iBAAiB,CAAC,cAAc,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO;IAkBvF;;OAEG;YACW,uBAAuB;IAYrC;;OAEG;YACW,kBAAkB;CAyBnC"}
+{"version":3,"file":"WatermarkService.d.ts","sourceRoot":"","sources":["../src/WatermarkService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAqB,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAExE,OAAO,KAAK,EAAE,gCAAgC,EAAE,MAAM,mBAAmB,CAAC;AAC1E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD;;;GAGG;AACH,qBAAa,gBAAgB;IACzB;;;;;;OAMG;IACU,IAAI,CACb,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,QAAQ,EACrB,SAAS,GAAE,MAAM,GAAG,MAAe,GACpC,OAAO,CAAC,gCAAgC,GAAG,IAAI,CAAC;IAsBnD;;;;;;;;OAQG;IACU,MAAM,CACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,QAAQ,EACrB,SAAS,GAAE,MAAM,GAAG,MAAe,GACpC,OAAO,CAAC,IAAI,CAAC;IAShB;;;;;;;;;OASG;IACU,cAAc,CACvB,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,IAAI,CAAC;IAShB;;;;;;OAMG;IACI,iBAAiB,CAAC,cAAc,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO;IAkBvF;;;;;;;;OAQG;IACU,kBAAkB,CAC3B,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,IAAI,CAAC;IAehB;;;;;OAKG;IACU,mBAAmB,CAAC,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAY3F;;;;;OAKG;IACU,oBAAoB,CAC7B,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC5B,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,IAAI,CAAC;IAchB;;;OAGG;IACU,oBAAoB,CAC7B,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,QAAQ,GACtB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAa/B;;OAEG;YACW,uBAAuB;IAYrC;;;OAGG;YACW,kBAAkB;IAShC;;;OAGG;YACW,eAAe;CAsChC"}