@rulecatch/ai-pooler 0.4.0

package/dist/flush.js

@@ -0,0 +1,1114 @@
+#!/usr/bin/env node
+import { existsSync, readFileSync, writeFileSync, appendFileSync, unlinkSync, readdirSync } from 'fs';
+import { homedir } from 'os';
+import { join } from 'path';
+import { randomBytes, createCipheriv, createHash, pbkdf2Sync } from 'crypto';
+
+/**
+ * Client-side encryption for PII (Zero-Knowledge Architecture)
+ *
+ * All PII (emails, usernames, file paths) is encrypted on the client
+ * before being sent to Rulecatch API. We never see plaintext PII.
+ *
+ * The user's privacy key is derived from their password or a separate
+ * key they set during setup. Without this key, the data is unreadable.
+ */
+const ALGORITHM = 'aes-256-gcm';
+const IV_LENGTH = 16;
+/**
+ * Encrypt a PII field (email, username, file path, etc.)
+ * Returns ciphertext + IV + auth tag for storage
+ */
+function encryptPII(plaintext, key) {
+    if (!plaintext) {
+        return { ciphertext: '', iv: '', tag: '' };
+    }
+    const iv = randomBytes(IV_LENGTH);
+    const cipher = createCipheriv(ALGORITHM, key, iv);
+    const encrypted = Buffer.concat([
+        cipher.update(plaintext, 'utf8'),
+        cipher.final(),
+    ]);
+    const tag = cipher.getAuthTag();
+    return {
+        ciphertext: encrypted.toString('base64'),
+        iv: iv.toString('base64'),
+        tag: tag.toString('base64'),
+    };
+}
+/**
+ * Create a one-way hash for indexing/grouping
+ * Cannot be reversed, but same input = same hash (for deduplication)
+ *
+ * We use a truncated hash (16 chars) to prevent rainbow table attacks
+ * while still allowing grouping by the same identifier.
+ */
+function hashForIndex(plaintext, salt) {
+    if (!plaintext) {
+        return '';
+    }
+    return createHash('sha256')
+        .update(plaintext + salt)
+        .digest('hex')
+        .slice(0, 16);
+}
+/**
+ * Encrypt all PII fields in an event payload
+ * Non-PII fields are passed through unchanged
+ */
+function encryptEventPII(event, key, salt) {
+    const piiFields = [
+        'accountEmail',
+        'gitEmail',
+        'gitUsername',
+        'filePath',
+        'cwd',
+        'projectId',
+    ];
+    const arrayPiiFields = ['filesModified'];
+    const result = { ...event };
+    // Encrypt single PII fields
+    for (const field of piiFields) {
+        const value = event[field];
+        if (typeof value === 'string' && value) {
+            const encrypted = encryptPII(value, key);
+            result[`${field}_encrypted`] = encrypted.ciphertext;
+            result[`${field}_iv`] = encrypted.iv;
+            result[`${field}_tag`] = encrypted.tag;
+            result[`${field}_hash`] = hashForIndex(value, salt);
+            delete result[field]; // Remove plaintext
+        }
+    }
+    // Encrypt array PII fields
+    for (const field of arrayPiiFields) {
+        const value = event[field];
+        if (Array.isArray(value)) {
+            const encryptedArray = value.map((item) => {
+                if (typeof item === 'string') {
+                    const encrypted = encryptPII(item, key);
+                    return {
+                        encrypted: encrypted.ciphertext,
+                        iv: encrypted.iv,
+                        tag: encrypted.tag,
+                        hash: hashForIndex(item, salt),
+                    };
+                }
+                return item;
+            });
+            result[`${field}_encrypted`] = encryptedArray;
+            result[`${field}_hashes`] = value.map((item) => typeof item === 'string' ? hashForIndex(item, salt) : '');
+            delete result[field]; // Remove plaintext
+        }
+    }
+    return result;
+}
+
+/**
+ * Backpressure & Flow Control for Rulecatch AI Pooler
+ *
+ * Implements smart throttling that:
+ * 1. Asks server "how much can I send?" before flushing
+ * 2. Respects rate limits (429) with exponential backoff
+ * 3. Gradually drains buffer when server recovers
+ * 4. Prevents thundering herd after outages
+ */
+// Paths
+const RULECATCH_DIR$1 = join(homedir(), '.claude', 'rulecatch');
+const BACKPRESSURE_STATE_FILE = join(RULECATCH_DIR$1, '.backpressure-state');
+const LOG_FILE$1 = join(RULECATCH_DIR$1, 'flush.log');
+// API version
+const API_VERSION$1 = '/api/v1';
+/**
+ * Default capacity when server is unreachable
+ */
+const DEFAULT_CAPACITY = {
+    ready: false,
+    maxBatchSize: 10,
+    delayBetweenBatches: 5000,
+    retryAfter: 30,
+};
+/**
+ * Backoff configuration
+ */
+const BACKOFF_CONFIG = {
+    /** Base delay in ms */
+    baseDelay: 1000,
+    /** Maximum delay in ms (5 minutes) */
+    maxDelay: 300000,
+    /** Multiplier for each failure */
+    multiplier: 2,
+    /** Max consecutive failures before circuit breaker */
+    maxFailures: 10};
+function log$1(message) {
+    const timestamp = new Date().toISOString();
+    try {
+        appendFileSync(LOG_FILE$1, `[${timestamp}] [backpressure] ${message}\n`);
+    }
+    catch {
+        // Ignore
+    }
+}
+function getBaseUrl$1(region, endpointOverride) {
+    // Allow override for local testing (e.g., RULECATCH_API_URL=http://localhost:3001)
+    if (process.env.RULECATCH_API_URL) {
+        return process.env.RULECATCH_API_URL;
+    }
+    if (endpointOverride) {
+        return endpointOverride;
+    }
+    return region === 'eu'
+        ? 'https://api-eu.rulecatch.ai'
+        : 'https://api.rulecatch.ai';
+}
+/**
+ * Load persisted backpressure state
+ */
+function loadState() {
+    try {
+        if (existsSync(BACKPRESSURE_STATE_FILE)) {
+            const content = readFileSync(BACKPRESSURE_STATE_FILE, 'utf-8');
+            return JSON.parse(content);
+        }
+    }
+    catch {
+        // Corrupt state, start fresh
+    }
+    return {
+        backoffLevel: 0,
+        nextAttemptAfter: 0,
+        lastCapacity: null,
+        consecutiveFailures: 0,
+        lastSuccessTime: 0,
+        pendingEventCount: 0,
+    };
+}
+/**
+ * Save backpressure state
+ */
+function saveState(state) {
+    try {
+        writeFileSync(BACKPRESSURE_STATE_FILE, JSON.stringify(state, null, 2));
+    }
+    catch {
+        log$1('Failed to save backpressure state');
+    }
+}
+/**
+ * Calculate backoff delay based on failure count
+ */
+function calculateBackoffDelay(backoffLevel) {
+    const delay = BACKOFF_CONFIG.baseDelay * Math.pow(BACKOFF_CONFIG.multiplier, backoffLevel);
+    return Math.min(delay, BACKOFF_CONFIG.maxDelay);
+}
+/**
+ * Check if we're allowed to attempt a flush
+ */
+function canAttemptFlush(state) {
+    const now = Date.now();
+    // Check circuit breaker
+    if (state.consecutiveFailures >= BACKOFF_CONFIG.maxFailures) {
+        const waitMs = state.nextAttemptAfter - now;
+        if (waitMs > 0) {
+            return {
+                allowed: false,
+                waitMs,
+                reason: `Circuit breaker open: ${state.consecutiveFailures} consecutive failures. Retry in ${Math.ceil(waitMs / 1000)}s`,
+            };
+        }
+    }
+    // Check backoff timer
+    if (state.nextAttemptAfter > now) {
+        const waitMs = state.nextAttemptAfter - now;
+        return {
+            allowed: false,
+            waitMs,
+            reason: `Backing off: retry in ${Math.ceil(waitMs / 1000)}s (level ${state.backoffLevel})`,
+        };
+    }
+    return { allowed: true, waitMs: 0, reason: 'OK' };
+}
+/**
+ * Ask server how much we can send
+ */
+async function getServerCapacity(apiKey, region, sessionToken, pendingCount, endpointOverride) {
+    const baseUrl = getBaseUrl$1(region, endpointOverride);
+    const endpoint = `${baseUrl}${API_VERSION$1}/ai/pooler/capacity`;
+    try {
+        const headers = {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${apiKey}`,
+        };
+        if (sessionToken) {
+            headers['X-Pooler-Token'] = sessionToken;
+        }
+        const response = await fetch(endpoint, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({
+                pendingEventCount: pendingCount,
+                clientVersion: '0.4.0',
+            }),
+            signal: AbortSignal.timeout(10000), // 10s timeout
+        });
+        if (response.status === 429) {
+            // Rate limited - parse Retry-After header
+            const retryAfter = parseInt(response.headers.get('Retry-After') || '60', 10);
+            log$1(`Rate limited by server, retry after ${retryAfter}s`);
+            return {
+                ready: false,
+                maxBatchSize: 0,
+                delayBetweenBatches: 0,
+                retryAfter,
+                message: 'Rate limited',
+            };
+        }
+        if (response.status === 503) {
+            // Server overloaded
+            const retryAfter = parseInt(response.headers.get('Retry-After') || '120', 10);
+            log$1(`Server overloaded, retry after ${retryAfter}s`);
+            return {
+                ready: false,
+                maxBatchSize: 0,
+                delayBetweenBatches: 0,
+                retryAfter,
+                message: 'Server overloaded',
+            };
+        }
+        if (!response.ok) {
+            log$1(`Capacity check failed: ${response.status}`);
+            return DEFAULT_CAPACITY;
+        }
+        const data = await response.json();
+        log$1(`Server capacity: ready=${data.ready}, maxBatch=${data.maxBatchSize}, delay=${data.delayBetweenBatches}ms, load=${data.loadPercent}%`);
+        return data;
+    }
+    catch (err) {
+        log$1(`Failed to get server capacity: ${err}`);
+        return DEFAULT_CAPACITY;
+    }
+}
+/**
+ * Record a successful flush
+ */
+function recordSuccess(state, eventsSent) {
+    const now = Date.now();
+    // Reset backoff on success
+    const newState = {
+        ...state,
+        backoffLevel: Math.max(0, state.backoffLevel - 1), // Gradually reduce backoff
+        consecutiveFailures: 0,
+        lastSuccessTime: now,
+        nextAttemptAfter: 0, // Can attempt immediately
+        pendingEventCount: Math.max(0, state.pendingEventCount - eventsSent),
+    };
+    log$1(`Success: sent ${eventsSent} events, backoff level now ${newState.backoffLevel}`);
+    return newState;
+}
+/**
+ * Record a failed flush attempt
+ */
+function recordFailure(state, statusCode, retryAfterHeader) {
+    const now = Date.now();
+    const newFailureCount = state.consecutiveFailures + 1;
+    const newBackoffLevel = Math.min(state.backoffLevel + 1, 10); // Cap at level 10
+    let nextAttemptDelay;
+    // If server provided Retry-After, use that
+    if (retryAfterHeader) {
+        nextAttemptDelay = parseInt(retryAfterHeader, 10) * 1000;
+    }
+    else if (statusCode === 429 || statusCode === 503) {
+        // Rate limited or overloaded - use longer delay
+        nextAttemptDelay = calculateBackoffDelay(newBackoffLevel) * 2;
+    }
+    else {
+        // Other error - standard backoff
+        nextAttemptDelay = calculateBackoffDelay(newBackoffLevel);
+    }
+    const newState = {
+        ...state,
+        backoffLevel: newBackoffLevel,
+        consecutiveFailures: newFailureCount,
+        nextAttemptAfter: now + nextAttemptDelay,
+    };
+    log$1(`Failure (${statusCode}): count=${newFailureCount}, backoff level=${newBackoffLevel}, retry in ${Math.ceil(nextAttemptDelay / 1000)}s`);
+    return newState;
+}
+/**
+ * Update pending event count
+ */
+function updatePendingCount(state, count) {
+    return { ...state, pendingEventCount: count };
+}
+/**
+ * Get human-readable status for CLI
+ */
+function getStatusSummary(state) {
+    const lines = [];
+    if (state.consecutiveFailures > 0) {
+        lines.push(`Consecutive failures: ${state.consecutiveFailures}`);
+    }
+    if (state.backoffLevel > 0) {
+        lines.push(`Backoff level: ${state.backoffLevel}/10`);
+    }
+    if (state.nextAttemptAfter > Date.now()) {
+        const waitSec = Math.ceil((state.nextAttemptAfter - Date.now()) / 1000);
+        lines.push(`Next attempt in: ${waitSec}s`);
+    }
+    if (state.pendingEventCount > 0) {
+        lines.push(`Pending events: ${state.pendingEventCount}`);
+    }
+    if (state.lastSuccessTime > 0) {
+        const ago = Math.floor((Date.now() - state.lastSuccessTime) / 1000);
+        lines.push(`Last success: ${ago}s ago`);
+    }
+    if (state.lastCapacity) {
+        lines.push(`Server load: ${state.lastCapacity.loadPercent || 'unknown'}%`);
+        lines.push(`Max batch: ${state.lastCapacity.maxBatchSize}`);
+    }
+    return lines.length > 0 ? lines.join('\n') : 'Healthy (no backpressure)';
+}
+
+/**
+ * Rulecatch Flush Script with Backpressure Control
+ *
+ * Reads buffered event files from ~/.claude/rulecatch/buffer/,
+ * encrypts PII, acquires/reuses session token, POSTs to API
+ * with intelligent flow control to prevent overwhelming the server.
+ *
+ * Features:
+ * - Asks server "how much can I send?" before flushing
+ * - Respects rate limits (429) with exponential backoff
+ * - Gradually drains buffer when server recovers
+ * - Prevents thundering herd after outages
+ *
+ * Usage:
+ *   node flush.js          # Flush if batch threshold met
+ *   node flush.js --force  # Flush all regardless of threshold
+ *   node flush.js --status # Show backpressure status
+ */
+// Paths
+const RULECATCH_DIR = join(homedir(), '.claude', 'rulecatch');
+const CONFIG_PATH = join(RULECATCH_DIR, 'config.json');
+const BUFFER_DIR = join(RULECATCH_DIR, 'buffer');
+const SESSION_FILE = join(RULECATCH_DIR, '.session');
+const LOCK_FILE = join(RULECATCH_DIR, '.flush-lock');
+const LOG_FILE = join(RULECATCH_DIR, 'flush.log');
+const PAUSED_FILE = join(RULECATCH_DIR, '.paused');
+// API version
+const API_VERSION = '/api/v1';
+function getDebugLevel(config) {
+    return config?.debugLevel || 'verbose';
+}
+const DEFAULT_DEBUG_LOG_FILE = '/var/log/ai-pooler.log';
+/**
+ * Sanitize text to remove any potential PII.
+ * All sensitive data is masked with exactly 6 Xs: XXXXXX
+ *
+ * Catches:
+ * - Home directory paths (e.g., /home/username/, /Users/username/)
+ * - Windows user paths (C:\Users\username\)
+ * - Email addresses
+ * - API keys, tokens, passwords
+ * - Common PII field patterns
+ */
+function sanitizeForErrorReport(text) {
+    return text
+        // Unix home paths: /home/username/ or /Users/username/
+        .replace(/\/(?:home|Users)\/[^\/\s]+/g, '/home/XXXXXX')
+        // Windows paths: C:\Users\username\
+        .replace(/[A-Z]:\\Users\\[^\\s]+/gi, 'C:\\Users\\XXXXXX')
+        // Email addresses
+        .replace(/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g, 'XXXXXX@XXXXXX')
+        // API keys (dc_...)
+        .replace(/dc_[a-zA-Z0-9_-]+/g, 'dc_XXXXXX')
+        // Pooler tokens
+        .replace(/pooler_[a-zA-Z0-9_]+/g, 'pooler_XXXXXX')
+        // Bearer tokens
+        .replace(/Bearer\s+[a-zA-Z0-9+/=_-]{20,}/g, 'Bearer XXXXXX')
+        // Password patterns (password=..., password:..., "password":"...")
+        .replace(/password["']?\s*[:=]\s*["']?[^"'\s,}]+/gi, 'password=XXXXXX')
+        // Secret patterns
+        .replace(/secret["']?\s*[:=]\s*["']?[^"'\s,}]+/gi, 'secret=XXXXXX')
+        // Key patterns (apiKey, api_key, etc.)
+        .replace(/(?:api[_-]?key|apikey)["']?\s*[:=]\s*["']?[^"'\s,}]+/gi, 'apiKey=XXXXXX')
+        // MongoDB connection strings
+        .replace(/mongodb(\+srv)?:\/\/[^@]+@/gi, 'mongodb$1://XXXXXX:XXXXXX@')
+        // Generic URLs with credentials
+        .replace(/:\/\/[^:]+:[^@]+@/g, '://XXXXXX:XXXXXX@');
+}
+/**
+ * Report an error to the Rulecatch error tracking endpoint.
+ * Fires and forgets - we don't want error reporting to block exit.
+ *
+ * IMPORTANT: All PII is stripped before sending:
+ * - File paths with usernames are redacted
+ * - Email addresses are redacted
+ * - API keys and tokens are redacted
+ */
+async function reportError(error, phase, config) {
+    try {
+        const rawMessage = error instanceof Error ? error.message : String(error);
+        const rawStack = error instanceof Error ? error.stack : undefined;
+        // Sanitize to remove any PII
+        const errorMessage = sanitizeForErrorReport(rawMessage);
+        const stack = rawStack ? sanitizeForErrorReport(rawStack) : undefined;
+        const region = config?.region || 'us';
+        const baseUrl = region === 'eu'
+            ? 'https://api-eu.rulecatch.ai'
+            : 'https://api.rulecatch.ai';
+        const report = {
+            error: errorMessage,
+            stack,
+            phase,
+            region,
+            // Note: projectId could be PII if user named it after themselves
+            // We hash it for grouping but don't send raw
+            projectIdHash: config?.projectId
+                ? require('crypto').createHash('sha256').update(config.projectId).digest('hex').slice(0, 16)
+                : undefined,
+            nodeVersion: process.version,
+            platform: process.platform,
+            arch: process.arch,
+            timestamp: new Date().toISOString(),
+            // Config info (only boolean flags, no values)
+            hasApiKey: !!config?.apiKey,
+            hasEncryptionKey: !!config?.encryptionKey,
+            hasSalt: !!config?.salt,
+            hasBatchSize: !!config?.batchSize,
+            hasDebugEnabled: !!config?.debug,
+        };
+        // Fire and forget - don't await, use short timeout
+        fetch(`${baseUrl}/api/v1/pooler/crash-reports`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(report),
+            signal: AbortSignal.timeout(5000), // 5s max
+        }).catch(() => {
+            // Silently ignore - we tried our best
+        });
+        // Give it a moment to send before process exits
+        await new Promise(resolve => setTimeout(resolve, 100));
+    }
+    catch {
+        // Error reporting itself failed - nothing we can do
+    }
+}
+/**
+ * Expand ~ to home directory in paths
+ */
+function expandPath(path) {
+    if (path.startsWith('~/')) {
+        return join(homedir(), path.slice(2));
+    }
+    return path;
+}
+/**
+ * Get the debug log file path from config or default
+ */
+function getDebugLogPath(config) {
+    const configPath = config?.debugLogFile;
+    if (configPath) {
+        return expandPath(configPath);
+    }
+    return DEFAULT_DEBUG_LOG_FILE;
+}
+// ANSI color codes for terminal output
+const colors = {
+    reset: '\x1b[0m',
+    bold: '\x1b[1m',
+    cyan: '\x1b[36m',
+    green: '\x1b[32m',
+    yellow: '\x1b[33m',
+    magenta: '\x1b[35m',
+    purple: '\x1b[95m', // Bright magenta/purple
+    dim: '\x1b[2m',
+};
+/**
+ * Debug log - only outputs when debug mode is enabled
+ * Writes to debugLogFile (default: /var/log/ai-pooler.log)
+ *
+ * @param config - Config object
+ * @param message - Log message
+ * @param details - Optional key-value details
+ * @param level - Minimum level required to log this message (default: 'verbose')
+ */
+function debugLog(config, message, details, level = 'verbose') {
+    if (!config?.debug)
+        return;
+    const configLevel = getDebugLevel(config);
+    const levelPriority = { minimal: 1, verbose: 2, full: 3 };
+    // Only log if config level is >= required level
+    if (levelPriority[configLevel] < levelPriority[level])
+        return;
+    const logPath = getDebugLogPath(config);
+    const timestamp = new Date().toISOString();
+    // Strip ANSI color codes for file output
+    const cleanMessage = message.replace(/\x1b\[[0-9;]*m/g, '');
+    let logLine = `[${timestamp}] ${cleanMessage}`;
+    if (details) {
+        const detailsStr = Object.entries(details)
+            .map(([key, value]) => {
+            // Strip ANSI codes from values too
+            const cleanValue = String(value).replace(/\x1b\[[0-9;]*m/g, '');
+            return `${key}=${cleanValue}`;
+        })
+            .join(' | ');
+        logLine += ` | ${detailsStr}`;
+    }
+    try {
+        appendFileSync(logPath, logLine + '\n');
+    }
+    catch {
+        // If we can't write to the log file, silently ignore
+        // (e.g., /var/log not writable without sudo)
+    }
+}
+/**
+ * Log full JSON payload (only at 'full' debug level)
+ */
+function debugLogJson(config, label, data) {
+    if (!config?.debug)
+        return;
+    if (getDebugLevel(config) !== 'full')
+        return;
+    const logPath = getDebugLogPath(config);
+    const timestamp = new Date().toISOString();
+    try {
+        const jsonStr = JSON.stringify(data, null, 2);
+        appendFileSync(logPath, `[${timestamp}] === ${label} ===\n${jsonStr}\n\n`);
+    }
+    catch {
+        // Silently ignore
+    }
+}
+/**
+ * Check if data collection is paused due to subscription issues.
+ */
+function isPaused() {
+    try {
+        if (existsSync(PAUSED_FILE)) {
+            return JSON.parse(readFileSync(PAUSED_FILE, 'utf-8'));
+        }
+    }
+    catch {
+        // Corrupt file - ignore
+    }
+    return null;
+}
+/**
+ * Write paused marker file when subscription is invalid.
+ */
+function writePausedFile(info) {
+    const pausedInfo = {
+        ...info,
+        pausedAt: new Date().toISOString(),
+    };
+    writeFileSync(PAUSED_FILE, JSON.stringify(pausedInfo, null, 2), { mode: 0o600 });
+}
+function log(message) {
+    const timestamp = new Date().toISOString();
+    try {
+        appendFileSync(LOG_FILE, `[${timestamp}] ${message}\n`);
+    }
+    catch {
+        // Ignore
+    }
+}
+function getBaseUrl(region, endpointOverride) {
+    // Allow override for local testing (e.g., RULECATCH_API_URL=http://localhost:3001)
+    if (process.env.RULECATCH_API_URL) {
+        return process.env.RULECATCH_API_URL;
+    }
+    // Allow override from config.endpoint
+    if (endpointOverride) {
+        return endpointOverride;
+    }
+    return region === 'eu'
+        ? 'https://api-eu.rulecatch.ai'
+        : 'https://api.rulecatch.ai';
+}
+function deriveEncryptionKey(password, salt) {
+    return pbkdf2Sync(password, salt, 100000, 32, 'sha256');
+}
+/**
+ * Acquire a file lock to prevent concurrent flushes.
+ */
+function acquireLock() {
+    try {
+        if (existsSync(LOCK_FILE)) {
+            const lockContent = readFileSync(LOCK_FILE, 'utf-8').trim();
+            const lockTime = parseInt(lockContent, 10);
+            // Stale lock (older than 60 seconds)
+            if (Date.now() - lockTime > 60000) {
+                unlinkSync(LOCK_FILE);
+            }
+            else {
+                return false;
+            }
+        }
+        writeFileSync(LOCK_FILE, String(Date.now()));
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function releaseLock() {
+    try {
+        if (existsSync(LOCK_FILE)) {
+            unlinkSync(LOCK_FILE);
+        }
+    }
+    catch {
+        // Ignore
+    }
+}
+/**
+ * Load or acquire session token.
+ */
+async function getSessionToken(config) {
+    // Check cached token
+    if (existsSync(SESSION_FILE)) {
+        try {
+            const cached = JSON.parse(readFileSync(SESSION_FILE, 'utf-8'));
+            if (cached.token && cached.expiry > Date.now()) {
+                return cached.token;
+            }
+        }
+        catch {
+            // Expired or corrupt, re-acquire
+        }
+    }
+    // Acquire new token
+    const baseUrl = getBaseUrl(config.region, config.endpoint);
+    const tokenEndpoint = `${baseUrl}${API_VERSION}/ai/pooler/session`;
+    const regionLabel = config.region === 'eu' ? 'api-eu' : 'api';
+    const startTime = Date.now();
+    try {
+        const response = await fetch(tokenEndpoint, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${config.apiKey}`,
+            },
+            body: JSON.stringify({
+                projectId: config.projectId,
+                region: config.region,
+                encrypted: true,
+            }),
+        });
+        const durationMs = Date.now() - startTime;
+        if (!response.ok) {
+            // Check if this is a subscription pause response (403 with status: 'paused')
+            if (response.status === 403) {
+                try {
+                    const errorBody = await response.json();
+                    if (errorBody.status === 'paused') {
+                        // Write paused marker file
+                        writePausedFile({
+                            reason: errorBody.reason || 'subscription_expired',
+                            message: errorBody.message || 'Subscription expired.',
+                            region: errorBody.region || config.region,
+                            dashboardUrl: errorBody.dashboardUrl || `https://dashboard${config.region === 'eu' ? '-eu' : ''}.rulecatch.ai`,
+                            billingUrl: errorBody.billingUrl || `https://dashboard${config.region === 'eu' ? '-eu' : ''}.rulecatch.ai/billing`,
+                        });
+                        log(`Subscription paused: ${errorBody.reason}`);
+                        debugLog(config, `${colors.yellow}⚠ SUBSCRIPTION PAUSED${colors.reset}`, {
+                            reason: errorBody.reason,
+                            billingUrl: errorBody.billingUrl,
+                        });
+                        // Return special marker
+                        return 'PAUSED';
+                    }
+                }
+                catch {
+                    // Couldn't parse as pause response - fall through to generic error
+                }
+            }
+            log(`Failed to acquire session token: ${response.status}`);
+            debugLog(config, `${colors.yellow}✗ Session token failed${colors.reset}`, {
+                region: `${colors.cyan}${config.region.toUpperCase()}${colors.reset}`,
+                endpoint: regionLabel,
+                status: response.status,
+                duration: `${durationMs}ms`,
+            });
+            return null;
+        }
+        const result = await response.json();
+        const cache = {
+            token: result.token,
+            // Refresh 1 hour before expiry
+            expiry: Date.now() + (result.expiresIn - 3600) * 1000,
+        };
+        writeFileSync(SESSION_FILE, JSON.stringify(cache), { mode: 0o600 });
+        log(`Session token acquired (expires in ${result.expiresIn}s)`);
+        debugLog(config, `${colors.green}✓ Session token acquired${colors.reset}`, {
+            region: `${colors.cyan}${config.region.toUpperCase()}${colors.reset}`,
+            endpoint: `${colors.cyan}${regionLabel}.rulecatch.ai${colors.reset}`,
+            duration: `${colors.green}${durationMs}ms${colors.reset}`,
+            expiresIn: `${Math.round(result.expiresIn / 3600)}h`,
+        });
+        return result.token;
+    }
+    catch (err) {
+        const durationMs = Date.now() - startTime;
+        log(`Error acquiring session token: ${err}`);
+        debugLog(config, `${colors.yellow}✗ Session token error${colors.reset}`, {
+            region: `${colors.cyan}${config.region.toUpperCase()}${colors.reset}`,
+            endpoint: regionLabel,
+            error: String(err),
+            duration: `${durationMs}ms`,
+        });
+        return null;
+    }
+}
+/**
+ * Read all buffer files and return events with file paths
+ */
+function readBufferFiles() {
+    if (!existsSync(BUFFER_DIR)) {
+        return { events: [], filePaths: [] };
+    }
+    let files;
+    try {
+        files = readdirSync(BUFFER_DIR)
+            .filter(f => f.endsWith('.json'))
+            .sort(); // Process in chronological order
+    }
+    catch {
+        return { events: [], filePaths: [] };
+    }
+    const events = [];
+    const filePaths = [];
+    for (const file of files) {
+        const filePath = join(BUFFER_DIR, file);
+        try {
+            const content = readFileSync(filePath, 'utf-8');
+            const event = JSON.parse(content);
+            events.push(event);
+            filePaths.push(filePath);
+        }
+        catch (err) {
+            log(`Failed to read buffer file ${file}: ${err}`);
+            // Delete corrupt files
+            try {
+                unlinkSync(filePath);
+            }
+            catch { /* ignore */ }
+        }
+    }
+    return { events, filePaths };
+}
+/**
+ * Delete successfully sent files
+ */
+function deleteFiles(filePaths) {
+    for (const filePath of filePaths) {
+        try {
+            unlinkSync(filePath);
+        }
+        catch {
+            // Ignore
+        }
+    }
+}
+/**
+ * Send a batch of events to the API
+ */
+async function sendBatch(events, config, sessionToken) {
+    const baseUrl = getBaseUrl(config.region, config.endpoint);
+    const endpoint = `${baseUrl}${API_VERSION}/ai/ingest`;
+    const regionLabel = config.region === 'eu' ? 'api-eu' : 'api';
+    // FULL: Log raw events received from buffer
+    debugLogJson(config, 'RAW EVENTS FROM BUFFER (before encryption)', events);
+    // Actual headers for the request
+    const headers = {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${config.apiKey}`,
+    };
+    if (sessionToken) {
+        headers['X-Pooler-Token'] = sessionToken;
+    }
+    // FULL: Log endpoint and headers (masked for security)
+    debugLog(config, 'REQUEST DETAILS', {
+        endpoint,
+        method: 'POST',
+        headers: JSON.stringify({
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${config.apiKey.slice(0, 10)}...`,
+            'X-Pooler-Token': sessionToken ? `${sessionToken.slice(0, 20)}...` : 'none',
+        }),
+    }, 'full');
+    // Encrypt PII fields
+    const key = deriveEncryptionKey(config.encryptionKey, config.salt);
+    const processedEvents = events.map(event => encryptEventPII(event, key, config.salt));
+    // FULL: Log processed events after encryption
+    debugLogJson(config, 'PROCESSED EVENTS (after encryption)', processedEvents);
+    const requestBody = {
+        projectId: config.projectId, // REQUIRED: must be at top level
+        events: processedEvents,
+        encryptionSalt: config.salt,
+        region: config.region,
+        endpoint: endpoint,
+        sentAt: new Date().toISOString(),
+        ...(config.autoGeneratedKey && { autoGeneratedKey: true }),
+    };
+    // FULL: Log full request body
+    debugLogJson(config, 'FULL REQUEST BODY', requestBody);
+    const body = JSON.stringify(requestBody);
+    const startTime = Date.now();
+    try {
+        const response = await fetch(endpoint, {
+            method: 'POST',
+            headers,
+            body,
+            signal: AbortSignal.timeout(30000), // 30s timeout
+        });
+        const durationMs = Date.now() - startTime;
+        const retryAfter = response.headers.get('Retry-After') || undefined;
+        if (!response.ok) {
+            const text = await response.text().catch(() => '');
+            log(`API error ${response.status}: ${text}`);
+            // Check if this is a subscription pause response (403 with status: 'paused')
+            if (response.status === 403) {
+                try {
+                    const errorBody = JSON.parse(text);
+                    if (errorBody.status === 'paused') {
+                        // Write paused marker file
+                        writePausedFile({
+                            reason: errorBody.reason || 'subscription_expired',
+                            message: errorBody.message || 'Subscription expired.',
+                            region: errorBody.region || config.region,
+                            dashboardUrl: errorBody.dashboardUrl || `https://dashboard${config.region === 'eu' ? '-eu' : ''}.rulecatch.ai`,
+                            billingUrl: errorBody.billingUrl || `https://dashboard${config.region === 'eu' ? '-eu' : ''}.rulecatch.ai/billing`,
+                        });
+                        log(`Subscription paused by ingest endpoint: ${errorBody.reason}`);
+                        debugLog(config, `${colors.yellow}⚠ SUBSCRIPTION PAUSED (ingest rejected)${colors.reset}`, {
+                            reason: errorBody.reason,
+                            billingUrl: errorBody.billingUrl,
+                        });
+                        // Return special marker to indicate paused state
+                        return { ok: false, status: response.status, retryAfter: undefined, durationMs, paused: true };
+                    }
+                }
+                catch {
+                    // Couldn't parse as pause response - continue with generic error handling
+                }
+            }
+            // Debug log for failed request
+            debugLog(config, `${colors.yellow}✗ POST failed${colors.reset}`, {
+                region: `${colors.cyan}${config.region.toUpperCase()}${colors.reset}`,
+                endpoint: regionLabel,
+                status: response.status,
+                duration: `${durationMs}ms`,
+            });
+        }
+        else {
+            // Build summary of what was sent
+            const toolCounts = {};
+            for (const event of events) {
+                const toolName = event.toolName;
+                const eventType = event.type;
+                const label = toolName || eventType || 'unknown';
+                toolCounts[label] = (toolCounts[label] || 0) + 1;
+            }
+            const toolsSummary = Object.entries(toolCounts)
+                .map(([name, count]) => count > 1 ? `${name}(${count})` : name)
+                .join(', ');
+            // Debug log for successful request
+            debugLog(config, `${colors.green}✓ ${events.length} events sent${colors.reset}`, {
+                region: `${colors.cyan}${config.region.toUpperCase()}${colors.reset}`,
+                endpoint: `${colors.cyan}${regionLabel}.rulecatch.ai${colors.reset}`,
+                events: `${colors.cyan}${toolsSummary}${colors.reset}`,
+                duration: `${colors.green}${durationMs}ms${colors.reset}`,
+            });
+        }
+        return { ok: response.ok, status: response.status, retryAfter, durationMs };
+    }
+    catch (err) {
+        const durationMs = Date.now() - startTime;
+        log(`Network error: ${err}`);
+        // Debug log for network error
+        debugLog(config, `${colors.yellow}✗ Network error${colors.reset}`, {
+            region: `${colors.cyan}${config.region.toUpperCase()}${colors.reset}`,
+            endpoint: regionLabel,
+            error: String(err),
+            duration: `${durationMs}ms`,
+        });
+        return { ok: false, status: 0, retryAfter: '30', durationMs };
+    }
+}
+async function main() {
+    const args = process.argv.slice(2);
+    const force = args.includes('--force');
+    const showStatus = args.includes('--status');
+    // Show status and exit
+    if (showStatus) {
+        const state = loadState();
+        console.log('\nBackpressure Status:');
+        console.log('-------------------');
+        console.log(getStatusSummary(state));
+        console.log('');
+        process.exit(0);
+    }
+    // No config = not set up
+    if (!existsSync(CONFIG_PATH)) {
+        process.exit(0);
+    }
+    // Check if paused due to subscription issues
+    const pausedInfo = isPaused();
+    if (pausedInfo) {
+        // Silently exit - data collection is paused
+        // User must run `npx @rulecatch/ai-pooler reactivate` to resume
+        log('Data collection paused - subscription issue. Run: npx @rulecatch/ai-pooler reactivate');
+        process.exit(0);
+    }
+    let config;
+    try {
+        config = JSON.parse(readFileSync(CONFIG_PATH, 'utf-8'));
+        globalConfig = config; // Store for error reporting
+    }
+    catch (err) {
+        log('Failed to parse config.json');
+        await reportError(err, 'config', null);
+        process.exit(1);
+    }
+    if (!config.apiKey) {
+        process.exit(0);
+    }
+    // Log startup if debug enabled
+    debugLog(config, 'Flush script started', {
+        debug: config.debug,
+        debugLevel: config.debugLevel || 'verbose',
+        debugLogFile: config.debugLogFile || DEFAULT_DEBUG_LOG_FILE,
+        region: config.region,
+    }, 'minimal');
+    // Read buffer files
+    const { events, filePaths } = readBufferFiles();
+    // FULL: Log buffer file paths
+    if (filePaths.length > 0) {
+        debugLog(config, `Read ${filePaths.length} buffer files`, {
+            files: filePaths.map(f => f.split('/').pop()).join(', '),
+        }, 'full');
+    }
+    if (events.length === 0) {
+        process.exit(0);
+    }
+    const defaultBatchSize = config.batchSize || 20;
+    // If not forcing and below threshold, exit
+    if (!force && events.length < defaultBatchSize) {
+        process.exit(0);
+    }
+    // Load backpressure state
+    let state = loadState();
+    state = updatePendingCount(state, events.length);
+    // Check if we're allowed to flush
+    const canFlush = canAttemptFlush(state);
+    if (!canFlush.allowed) {
+        log(canFlush.reason);
+        console.log(canFlush.reason);
+        saveState(state);
+        process.exit(0);
+    }
+    // Acquire lock
+    if (!acquireLock()) {
+        log('Another flush is running, exiting');
+        process.exit(0);
+    }
+    try {
+        // Get session token
+        const sessionToken = await getSessionToken(config);
+        // Check if subscription is paused
+        if (sessionToken === 'PAUSED') {
+            // Paused file was written by getSessionToken
+            // Exit gracefully - user needs to reactivate
+            log('Subscription paused. Run: npx @rulecatch/ai-pooler reactivate');
+            releaseLock();
+            process.exit(0);
+        }
+        // Ask server how much we can send
+        const capacity = await getServerCapacity(config.apiKey, config.region, sessionToken, events.length, config.endpoint);
+        state.lastCapacity = capacity;
+        if (!capacity.ready) {
+            // Server not ready - back off
+            state.nextAttemptAfter = Date.now() + (capacity.retryAfter * 1000);
+            log(`Server not ready: ${capacity.message || 'backing off for ' + capacity.retryAfter + 's'}`);
+            console.log(`Server not ready, will retry in ${capacity.retryAfter}s`);
+            saveState(state);
+            releaseLock();
+            process.exit(0);
+        }
+        // Send events in batches with controlled pace
+        let totalSent = 0;
+        let currentIndex = 0;
+        while (currentIndex < events.length) {
+            // Take a batch (server-controlled size)
+            const batchSize = Math.min(capacity.maxBatchSize, events.length - currentIndex);
+            const batchEvents = events.slice(currentIndex, currentIndex + batchSize);
+            const batchFilePaths = filePaths.slice(currentIndex, currentIndex + batchSize);
+            log(`Sending batch ${Math.floor(currentIndex / capacity.maxBatchSize) + 1}: ${batchSize} events`);
+            const result = await sendBatch(batchEvents, config, sessionToken);
+            if (result.ok) {
+                // Success - delete sent files
+                deleteFiles(batchFilePaths);
+                totalSent += batchSize;
+                currentIndex += batchSize;
+                state = recordSuccess(state, batchSize);
+                // If more events, wait before next batch (server-controlled delay)
+                if (currentIndex < events.length && capacity.delayBetweenBatches > 0) {
+                    log(`Waiting ${capacity.delayBetweenBatches}ms before next batch`);
+                    await new Promise(resolve => setTimeout(resolve, capacity.delayBetweenBatches));
+                    // Re-check capacity every 100 events during large drains
+                    if (totalSent > 0 && totalSent % 100 === 0) {
+                        const newCapacity = await getServerCapacity(config.apiKey, config.region, sessionToken, events.length - currentIndex, config.endpoint);
+                        state.lastCapacity = newCapacity;
+                        if (!newCapacity.ready) {
+                            log(`Server requested pause during drain, ${events.length - currentIndex} events remaining`);
+                            break;
+                        }
+                    }
+                }
+            }
+            else {
+                // Check if subscription was paused by the server
+                if (result.paused) {
+                    // Paused file was written by sendBatch
+                    log('Subscription paused by server. Run: npx @rulecatch/ai-pooler reactivate');
+                    // Don't record as failure - it's a subscription issue, not a transient error
+                    break;
+                }
+                // Failure - record and stop
+                state = recordFailure(state, result.status, result.retryAfter);
+                log(`Batch failed, stopping flush. ${events.length - currentIndex} events remaining.`);
+                break;
+            }
+        }
+        const remaining = events.length - totalSent;
+        state = updatePendingCount(state, remaining);
+        saveState(state);
+        log(`Flush complete: sent ${totalSent}/${events.length} events, ${remaining} remaining`);
+        if (totalSent > 0) {
+            console.log(`Sent ${totalSent} events${remaining > 0 ? ` (${remaining} remaining)` : ''}`);
+        }
+    }
+    catch (err) {
+        log(`Flush error: ${err}`);
+        state = recordFailure(state, 0);
+        saveState(state);
+        // Report error to API
+        await reportError(err, 'send', config);
+    }
+    finally {
+        releaseLock();
+    }
+}
+// Track config globally for error reporting in fatal handler
+let globalConfig = null;
+main().catch(async (err) => {
+    log(`Fatal: ${err}`);
+    // Report fatal error before exiting
+    await reportError(err, 'unknown', globalConfig);
+    releaseLock();
+    process.exit(1);
+});
+//# sourceMappingURL=flush.js.map