catalist-support-agent 1.0.0

package/dist/extraction/rate-limiter.js

@@ -0,0 +1,242 @@
+/**
+ * Rate Limiter Module
+ *
+ * Implements rate limiting for API calls (Gmail API, OpenAI API).
+ * Uses token bucket algorithm with sliding window for smooth rate limiting.
+ */
+// =============================================================================
+// Token Bucket Rate Limiter
+// =============================================================================
+export class RateLimiter {
+    config;
+    tokens;
+    lastRefill;
+    currentConcurrent = 0;
+    requestTimes = [];
+    constructor(config) {
+        this.config = {
+            requestsPerMinute: config.requestsPerMinute,
+            requestsPerSecond: config.requestsPerSecond ?? Math.ceil(config.requestsPerMinute / 60),
+            maxConcurrent: config.maxConcurrent ?? 10,
+            enableRetry: config.enableRetry ?? true,
+            maxRetries: config.maxRetries ?? 3,
+            baseDelayMs: config.baseDelayMs ?? 1000,
+        };
+        this.tokens = this.config.requestsPerMinute;
+        this.lastRefill = Date.now();
+    }
+    /**
+     * Acquire a token (blocks until available)
+     */
+    async acquire() {
+        while (true) {
+            const status = this.getStatus();
+            if (status.available) {
+                this.consumeToken();
+                return;
+            }
+            // Wait until we can proceed
+            const waitTime = Math.max(50, status.resetAtMs - Date.now());
+            await this.delay(Math.min(waitTime, 1000));
+        }
+    }
+    /**
+     * Try to acquire a token without blocking
+     */
+    tryAcquire() {
+        const status = this.getStatus();
+        if (status.available) {
+            this.consumeToken();
+            return true;
+        }
+        return false;
+    }
+    /**
+     * Release a concurrent slot
+     */
+    release() {
+        if (this.currentConcurrent > 0) {
+            this.currentConcurrent--;
+        }
+    }
+    /**
+     * Execute a function with rate limiting
+     */
+    async execute(fn) {
+        await this.acquire();
+        this.currentConcurrent++;
+        try {
+            return await fn();
+        }
+        finally {
+            this.release();
+        }
+    }
+    /**
+     * Execute with retry on rate limit errors
+     */
+    async executeWithRetry(fn, isRateLimitError = this.defaultIsRateLimitError) {
+        let lastError = null;
+        for (let attempt = 0; attempt <= this.config.maxRetries; attempt++) {
+            try {
+                return await this.execute(fn);
+            }
+            catch (error) {
+                lastError = error instanceof Error ? error : new Error(String(error));
+                if (!this.config.enableRetry || !isRateLimitError(error)) {
+                    throw error;
+                }
+                if (attempt < this.config.maxRetries) {
+                    // Exponential backoff with jitter
+                    const delay = this.config.baseDelayMs * Math.pow(2, attempt) * (0.5 + Math.random() * 0.5);
+                    console.warn(`Rate limit hit, retrying in ${Math.round(delay)}ms (attempt ${attempt + 1}/${this.config.maxRetries})`);
+                    await this.delay(delay);
+                }
+            }
+        }
+        throw lastError || new Error('Rate limit exceeded after retries');
+    }
+    /**
+     * Get current rate limit status
+     */
+    getStatus() {
+        this.refillTokens();
+        const now = Date.now();
+        const oneMinuteAgo = now - 60000;
+        const oneSecondAgo = now - 1000;
+        // Clean up old request times
+        this.requestTimes = this.requestTimes.filter((t) => t > oneMinuteAgo);
+        // Count requests in last second
+        const requestsLastSecond = this.requestTimes.filter((t) => t > oneSecondAgo).length;
+        // Check all conditions
+        const hasTokens = this.tokens > 0;
+        const underSecondLimit = requestsLastSecond < this.config.requestsPerSecond;
+        const underConcurrentLimit = this.currentConcurrent < this.config.maxConcurrent;
+        const available = hasTokens && underSecondLimit && underConcurrentLimit;
+        // Calculate reset time
+        let resetAtMs = now;
+        if (!hasTokens) {
+            const nextRefillTime = this.lastRefill + 60000;
+            resetAtMs = Math.max(resetAtMs, nextRefillTime);
+        }
+        if (!underSecondLimit) {
+            const oldestSecondRequest = this.requestTimes.find((t) => t > oneSecondAgo);
+            if (oldestSecondRequest) {
+                resetAtMs = Math.max(resetAtMs, oldestSecondRequest + 1000);
+            }
+        }
+        return {
+            available,
+            remainingRequests: Math.floor(this.tokens),
+            resetAtMs,
+            currentConcurrent: this.currentConcurrent,
+        };
+    }
+    /**
+     * Consume a token
+     */
+    consumeToken() {
+        this.tokens--;
+        this.requestTimes.push(Date.now());
+    }
+    /**
+     * Refill tokens based on elapsed time
+     */
+    refillTokens() {
+        const now = Date.now();
+        const elapsed = now - this.lastRefill;
+        // Refill rate: tokens per millisecond
+        const refillRate = this.config.requestsPerMinute / 60000;
+        const tokensToAdd = elapsed * refillRate;
+        this.tokens = Math.min(this.config.requestsPerMinute, this.tokens + tokensToAdd);
+        this.lastRefill = now;
+    }
+    /**
+     * Default rate limit error detector
+     */
+    defaultIsRateLimitError(error) {
+        if (error instanceof Error) {
+            const message = error.message.toLowerCase();
+            return (message.includes('rate limit') ||
+                message.includes('too many requests') ||
+                message.includes('429'));
+        }
+        return false;
+    }
+    /**
+     * Delay helper
+     */
+    delay(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    /**
+     * Reset the rate limiter
+     */
+    reset() {
+        this.tokens = this.config.requestsPerMinute;
+        this.lastRefill = Date.now();
+        this.currentConcurrent = 0;
+        this.requestTimes = [];
+    }
+    /**
+     * Get configuration
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+}
+// =============================================================================
+// Pre-configured Rate Limiters
+// =============================================================================
+// Gmail API: 250 queries per second (but we'll be more conservative)
+export const gmailRateLimiter = new RateLimiter({
+    requestsPerMinute: 60, // Conservative limit
+    requestsPerSecond: 5,
+    maxConcurrent: 5,
+    enableRetry: true,
+    maxRetries: 3,
+    baseDelayMs: 2000,
+});
+// OpenAI API: 3000 RPM for embeddings
+export const openaiRateLimiter = new RateLimiter({
+    requestsPerMinute: 3000,
+    requestsPerSecond: 50,
+    maxConcurrent: 20,
+    enableRetry: true,
+    maxRetries: 3,
+    baseDelayMs: 1000,
+});
+// =============================================================================
+// Utility Functions
+// =============================================================================
+/**
+ * Create a rate-limited function
+ */
+export function rateLimited(fn, limiter) {
+    return (async (...args) => {
+        return limiter.execute(() => fn(...args));
+    });
+}
+/**
+ * Batch execute with rate limiting
+ */
+export async function batchExecute(items, fn, limiter, options = {}) {
+    const { concurrency = 5, onProgress } = options;
+    const results = [];
+    let completed = 0;
+    // Process in chunks
+    for (let i = 0; i < items.length; i += concurrency) {
+        const chunk = items.slice(i, i + concurrency);
+        const chunkResults = await Promise.all(chunk.map(async (item) => {
+            const result = await limiter.execute(() => fn(item));
+            completed++;
+            if (onProgress) {
+                onProgress(completed, items.length);
+            }
+            return result;
+        }));
+        results.push(...chunkResults);
+    }
+    return results;
+}
+//# sourceMappingURL=rate-limiter.js.map

package/dist/extraction/rate-limiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rate-limiter.js","sourceRoot":"","sources":["../../src/extraction/rate-limiter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAiCH,gFAAgF;AAChF,4BAA4B;AAC5B,gFAAgF;AAEhF,MAAM,OAAO,WAAW;IACd,MAAM,CAA8B;IACpC,MAAM,CAAS;IACf,UAAU,CAAS;IACnB,iBAAiB,GAAW,CAAC,CAAC;IAC9B,YAAY,GAAa,EAAE,CAAC;IAEpC,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG;YACZ,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,GAAG,EAAE,CAAC;YACvF,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;YACzC,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI;YACvC,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC;YAClC,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI;SACxC,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC5C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YAEhC,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,IAAI,CAAC,YAAY,EAAE,CAAC;gBACpB,OAAO;YACT,CAAC;YAED,4BAA4B;YAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7D,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU;QACR,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;QAEhC,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,IAAI,CAAC,iBAAiB,GAAG,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAI,EAAoB;QACnC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACrB,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAEzB,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CACpB,EAAoB,EACpB,mBAAgD,IAAI,CAAC,uBAAuB;QAE5E,IAAI,SAAS,GAAiB,IAAI,CAAC;QAEnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACnE,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAChC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAEtE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;oBACzD,MAAM,KAAK,CAAC;gBACd,CAAC;gBAED,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;oBACrC,kCAAkC;oBAClC,MAAM,KAAK,GACT,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;oBAC/E,OAAO,CAAC,IAAI,CACV,+BAA+B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,eAAe,OAAO,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,CACxG,CAAC;oBACF,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,SAAS;QACP,IAAI,CAAC,YAAY,EAAE,CAAC;QAEpB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,YAAY,GAAG,GAAG,GAAG,KAAK,CAAC;QACjC,MAAM,YAAY,GAAG,GAAG,GAAG,IAAI,CAAC;QAEhC,6BAA6B;QAC7B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC;QAEtE,gCAAgC;QAChC,MAAM,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,MAAM,CAAC;QAEpF,uBAAuB;QACvB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,MAAM,gBAAgB,GAAG,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC5E,MAAM,oBAAoB,GAAG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;QAEhF,MAAM,SAAS,GAAG,SAAS,IAAI,gBAAgB,IAAI,oBAAoB,CAAC;QAExE,uBAAuB;QACvB,IAAI,SAAS,GAAG,GAAG,CAAC;QACpB,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;YAC/C,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,MAAM,mBAAmB,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC;YAC5E,IAAI,mBAAmB,EAAE,CAAC;gBACxB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,mBAAmB,GAAG,IAAI,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,OAAO;YACL,SAAS;YACT,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;YAC1C,SAAS;YACT,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;SAC1C,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC;QAEtC,sCAAsC;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,GAAG,KAAK,CAAC;QACzD,MAAM,WAAW,GAAG,OAAO,GAAG,UAAU,CAAC;QAEzC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;QACjF,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;IACxB,CAAC;IAED;;OAEG;IACK,uBAAuB,CAAC,KAAc;QAC5C,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC5C,OAAO,CACL,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAC9B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;gBACrC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CACxB,CAAC;QACJ,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAC5C,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;CACF;AAED,gFAAgF;AAChF,+BAA+B;AAC/B,gFAAgF;AAEhF,qEAAqE;AACrE,MAAM,CAAC,MAAM,gBAAgB,GAAG,IAAI,WAAW,CAAC;IAC9C,iBAAiB,EAAE,EAAE,EAAE,qBAAqB;IAC5C,iBAAiB,EAAE,CAAC;IACpB,aAAa,EAAE,CAAC;IAChB,WAAW,EAAE,IAAI;IACjB,UAAU,EAAE,CAAC;IACb,WAAW,EAAE,IAAI;CAClB,CAAC,CAAC;AAEH,sCAAsC;AACtC,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,WAAW,CAAC;IAC/C,iBAAiB,EAAE,IAAI;IACvB,iBAAiB,EAAE,EAAE;IACrB,aAAa,EAAE,EAAE;IACjB,WAAW,EAAE,IAAI;IACjB,UAAU,EAAE,CAAC;IACb,WAAW,EAAE,IAAI;CAClB,CAAC,CAAC;AAEH,gFAAgF;AAChF,oBAAoB;AACpB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,WAAW,CACzB,EAAK,EACL,OAAoB;IAEpB,OAAO,CAAC,KAAK,EAAE,GAAG,IAAmB,EAAE,EAAE;QACvC,OAAO,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAM,CAAC;AACV,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAU,EACV,EAA2B,EAC3B,OAAoB,EACpB,UAA6F,EAAE;IAE/F,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,OAAO,GAAQ,EAAE,CAAC;IACxB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,oBAAoB;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QAE9C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;YACrD,SAAS,EAAE,CAAC;YACZ,IAAI,UAAU,EAAE,CAAC;gBACf,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YACtC,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CACH,CAAC;QAEF,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/extraction/state-manager.d.ts

@@ -0,0 +1,126 @@
+/**
+ * State Manager Module
+ *
+ * Tracks extraction state for incremental Gmail sync.
+ * Uses Gmail History API to only process new emails since last extraction.
+ */
+export interface ExtractionState {
+    id: number;
+    emailAddress: string;
+    lastHistoryId: string | null;
+    lastExtractedAt: Date | null;
+    totalEntriesExtracted: number;
+    lastBatchId: string | null;
+    consecutiveErrors: number;
+    lastError: string | null;
+    lastErrorAt: Date | null;
+    createdAt: Date;
+    updatedAt: Date;
+}
+export interface BatchInfo {
+    id: string;
+    emailAccount: string;
+    dateRangeStart: Date;
+    dateRangeEnd: Date;
+    status: 'pending' | 'in_progress' | 'completed' | 'failed' | 'partial';
+    totalThreadsFound: number;
+    threadsProcessed: number;
+    entriesCreated: number;
+    entriesSkipped: number;
+    errorsEncountered: number;
+    startedAt: Date | null;
+    completedAt: Date | null;
+    errorLog: Array<{
+        timestamp: string;
+        error: string;
+    }>;
+    configUsed: Record<string, unknown>;
+    createdAt: Date;
+    updatedAt: Date;
+}
+export interface StateManagerConfig {
+    supabaseUrl: string;
+    supabaseServiceRoleKey: string;
+}
+export declare class StateManager {
+    private supabase;
+    constructor(config: StateManagerConfig);
+    /**
+     * Get extraction state for an email account
+     */
+    getState(emailAddress: string): Promise<ExtractionState | null>;
+    /**
+     * Create or get extraction state
+     */
+    getOrCreateState(emailAddress: string): Promise<ExtractionState>;
+    /**
+     * Update extraction state after successful extraction
+     */
+    updateStateSuccess(emailAddress: string, updates: {
+        historyId?: string;
+        entriesExtracted?: number;
+        batchId?: string;
+    }): Promise<void>;
+    /**
+     * Record extraction error
+     */
+    recordError(emailAddress: string, errorMessage: string): Promise<void>;
+    /**
+     * Check if we should back off due to errors
+     */
+    shouldBackOff(emailAddress: string, maxConsecutiveErrors?: number): Promise<boolean>;
+    /**
+     * Create a new import batch
+     */
+    createBatch(emailAccount: string, dateRangeStart: Date, dateRangeEnd: Date, config?: Record<string, unknown>): Promise<string>;
+    /**
+     * Start a batch (mark as in_progress)
+     */
+    startBatch(batchId: string, totalThreads: number): Promise<void>;
+    /**
+     * Update batch progress
+     */
+    updateBatchProgress(batchId: string, progress: {
+        threadsProcessed?: number;
+        entriesCreated?: number;
+        entriesSkipped?: number;
+        errorsEncountered?: number;
+    }): Promise<void>;
+    /**
+     * Add error to batch error log
+     */
+    addBatchError(batchId: string, errorMessage: string): Promise<void>;
+    /**
+     * Complete a batch
+     */
+    completeBatch(batchId: string, status?: 'completed' | 'failed' | 'partial'): Promise<void>;
+    /**
+     * Get batch info
+     */
+    getBatch(batchId: string): Promise<BatchInfo | null>;
+    /**
+     * Get recent batches for an email account
+     */
+    getRecentBatches(emailAccount: string, limit?: number): Promise<BatchInfo[]>;
+    /**
+     * Get overall extraction statistics
+     */
+    getStats(): Promise<{
+        totalEntries: number;
+        pendingEntries: number;
+        approvedEntries: number;
+        entriesWithEmbeddings: number;
+        recentBatches: BatchInfo[];
+    }>;
+    private mapStateRow;
+    private mapBatchRow;
+}
+/**
+ * Get singleton state manager instance
+ */
+export declare function getStateManager(): StateManager;
+/**
+ * Reset the singleton (for testing)
+ */
+export declare function resetStateManager(): void;
+//# sourceMappingURL=state-manager.d.ts.map

package/dist/extraction/state-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"state-manager.d.ts","sourceRoot":"","sources":["../../src/extraction/state-manager.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,eAAe,EAAE,IAAI,GAAG,IAAI,CAAC;IAC7B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,IAAI,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,IAAI,CAAC;IACrB,YAAY,EAAE,IAAI,CAAC;IACnB,MAAM,EAAE,SAAS,GAAG,aAAa,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;IACvE,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;IACvB,WAAW,EAAE,IAAI,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACtD,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAMD,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAiB;gBAErB,MAAM,EAAE,kBAAkB;IAQtC;;OAEG;IACG,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAcrE;;OAEG;IACG,gBAAgB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IA0BtE;;OAEG;IACG,kBAAkB,CACtB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE;QACP,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,GACA,OAAO,CAAC,IAAI,CAAC;IAsChB;;OAEG;IACG,WAAW,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB5E;;OAEG;IACG,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,oBAAoB,GAAE,MAAU,GAAG,OAAO,CAAC,OAAO,CAAC;IAa7F;;OAEG;IACG,WAAW,CACf,YAAY,EAAE,MAAM,EACpB,cAAc,EAAE,IAAI,EACpB,YAAY,EAAE,IAAI,EAClB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC,MAAM,CAAC;IAoBlB;;OAEG;IACG,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAetE;;OAEG;IACG,mBAAmB,CACvB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE;QACR,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B,GACA,OAAO,CAAC,IAAI,CAAC;IA0BhB;;OAEG;IACG,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BzE;;OAEG;IACG,aAAa,CACjB,OAAO,EAAE,MAAM,EACf,MAAM,GAAE,WAAW,GAAG,QAAQ,GAAG,SAAuB,GACvD,OAAO,CAAC,IAAI,CAAC;IAchB;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;IAc1D;;OAEG;IACG,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAmBtF;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC;QACxB,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,qBAAqB,EAAE,MAAM,CAAC;QAC9B,aAAa,EAAE,SAAS,EAAE,CAAC;KAC5B,CAAC;IA0BF,OAAO,CAAC,WAAW;IAgBnB,OAAO,CAAC,WAAW;CAoBpB;AAQD;;GAEG;AACH,wBAAgB,eAAe,IAAI,YAAY,CAgB9C;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}