@teamvibe/poller 0.2.3 → 0.2.5

package/dist/auth-provider.d.ts

@@ -14,5 +14,10 @@ export declare function getCredentials(): AuthCredentials;
 export declare function getAuthConfig(): AuthConfig;
 export declare function getPollerId(): string | null;
 export declare function isTokenMode(): boolean;
+/**
+ * Force an immediate credential refresh. Used when SQS returns ExpiredTokenException.
+ * Throws if refresh fails (caller should handle).
+ */
+export declare function forceRefresh(): Promise<void>;
 export declare function stopRefresh(): void;
 export {};

package/dist/auth-provider.js

@@ -6,6 +6,9 @@ let currentConfig = null;
 let currentPollerId = null;
 let refreshTimer = null;
 let heartbeatTimer = null;
+// Stored for on-demand refresh
+let storedApiUrl = null;
+let storedToken = null;
 const REFRESH_INTERVAL_MS = 50 * 60 * 1000; // 50 minutes (before 1hr expiry)
 const HEARTBEAT_INTERVAL_MS = 60 * 1000; // 1 minute
 async function sendHeartbeat(apiUrl, token, version) {
@@ -44,6 +47,8 @@ async function fetchCredentials(apiUrl, token) {
 }
 export async function initAuth(apiUrl, token) {
     logger.info('Authenticating with TeamVibe API...');
+    storedApiUrl = apiUrl;
+    storedToken = token;
     const auth = await fetchCredentials(apiUrl, token);
     currentCredentials = auth.credentials;
     currentConfig = auth.config;
@@ -87,6 +92,20 @@ export function getPollerId() {
 export function isTokenMode() {
     return currentCredentials !== null;
 }
+/**
+ * Force an immediate credential refresh. Used when SQS returns ExpiredTokenException.
+ * Throws if refresh fails (caller should handle).
+ */
+export async function forceRefresh() {
+    if (!storedApiUrl || !storedToken) {
+        throw new Error('Cannot force refresh: auth not initialized (no stored API URL/token)');
+    }
+    logger.info('Force-refreshing credentials (token expired)...');
+    const auth = await fetchCredentials(storedApiUrl, storedToken);
+    currentCredentials = auth.credentials;
+    currentConfig = auth.config;
+    logger.info('Credentials force-refreshed successfully');
+}
 export function stopRefresh() {
     if (refreshTimer) {
         clearInterval(refreshTimer);

package/dist/config.d.ts

@@ -16,6 +16,7 @@ declare const configSchema: z.ZodObject<{
     IDLE_WATCHDOG_WARN_MS: z.ZodDefault<z.ZodNumber>;
     IDLE_WATCHDOG_KILL_MS: z.ZodDefault<z.ZodNumber>;
     NETWORK_ERROR_KILL_MS: z.ZodDefault<z.ZodNumber>;
+    MAX_CONSECUTIVE_NET_FAILURES: z.ZodDefault<z.ZodNumber>;
     TEAMVIBE_DATA_DIR: z.ZodDefault<z.ZodString>;
     BRAINS_PATH: z.ZodDefault<z.ZodString>;
     DEFAULT_BRAIN_PATH: z.ZodDefault<z.ZodString>;
@@ -45,6 +46,7 @@ declare const configSchema: z.ZodObject<{
     IDLE_WATCHDOG_WARN_MS: number;
     IDLE_WATCHDOG_KILL_MS: number;
     NETWORK_ERROR_KILL_MS: number;
+    MAX_CONSECUTIVE_NET_FAILURES: number;
     TEAMVIBE_DATA_DIR: string;
     BRAINS_PATH: string;
     DEFAULT_BRAIN_PATH: string;
@@ -80,6 +82,7 @@ declare const configSchema: z.ZodObject<{
     IDLE_WATCHDOG_WARN_MS?: number | undefined;
     IDLE_WATCHDOG_KILL_MS?: number | undefined;
     NETWORK_ERROR_KILL_MS?: number | undefined;
+    MAX_CONSECUTIVE_NET_FAILURES?: number | undefined;
     TEAMVIBE_DATA_DIR?: string | undefined;
     BRAINS_PATH?: string | undefined;
     DEFAULT_BRAIN_PATH?: string | undefined;
@@ -112,6 +115,7 @@ export declare const config: {
     IDLE_WATCHDOG_WARN_MS: number;
     IDLE_WATCHDOG_KILL_MS: number;
     NETWORK_ERROR_KILL_MS: number;
+    MAX_CONSECUTIVE_NET_FAILURES: number;
     TEAMVIBE_DATA_DIR: string;
     BRAINS_PATH: string;
     DEFAULT_BRAIN_PATH: string;

package/dist/config.js

@@ -24,7 +24,8 @@ const configSchema = z.object({
     LOCK_RETRY_INTERVAL_MS: z.coerce.number().default(30000), // 30 seconds
     IDLE_WATCHDOG_WARN_MS: z.coerce.number().default(300000), // 5 minutes
     IDLE_WATCHDOG_KILL_MS: z.coerce.number().default(600000), // 10 minutes
-    NETWORK_ERROR_KILL_MS: z.coerce.number().default(120000), // 2 minutes of continuous network errors → kill
+    NETWORK_ERROR_KILL_MS: z.coerce.number().default(120000), // Per-session: kill stuck session after 2 min of network errors
+    MAX_CONSECUTIVE_NET_FAILURES: z.coerce.number().default(5), // Poll loop: DNS flush at N, exit(1) at 2×N consecutive failures
     // Paths
     TEAMVIBE_DATA_DIR: z.string().default(DEFAULT_DATA_DIR),
     BRAINS_PATH: z.string().default(''),
@@ -43,7 +44,7 @@ const configSchema = z.object({
         .transform((v) => v.toLowerCase() === 'true'),
     BRAIN_UPDATE_COOLDOWN_MS: z.coerce.number().default(300000), // 5 minutes
     // Brain heartbeat & maintenance (cron-based)
-    HEARTBEAT_CRON: z.string().default('*/5 * * * *'), // Every 5 minutes
+    HEARTBEAT_CRON: z.string().default('0 */6 * * *'), // Every 6 hours
     MAINTENANCE_CRON: z.string().default('30 3 * * *'), // Daily at 03:30 UTC
     MAINTENANCE_ENABLED: z.string().default('true').transform((v) => v.toLowerCase() === 'true'),
     // Auto-update settings

package/dist/poller.js

@@ -13,6 +13,10 @@ import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocesse
 import { Cron } from 'croner';
 import { isControlMessage, handleControlMessage } from './control-handler.js';
 import { checkForUpdate } from './auto-update.js';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import { platform } from 'os';
+const execAsync = promisify(exec);
 // Track active message processing
 const processingMessages = new Set();
 // Poller auto-update cron (separate from brain heartbeats)
@@ -49,6 +53,38 @@ function stopAutoUpdateCron() {
         autoUpdateJob = null;
     }
 }
+// --- Circuit breaker ---
+let consecutiveNetFailures = 0;
+const NETWORK_ERROR_CODES = ['ENOTFOUND', 'ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED'];
+function isNetworkError(error) {
+    if (error instanceof Error) {
+        const code = error.code || '';
+        const message = error.message || '';
+        return (NETWORK_ERROR_CODES.includes(code) ||
+            message.includes('getaddrinfo') ||
+            message.includes('socket hang up'));
+    }
+    return false;
+}
+async function attemptDnsFlush() {
+    const cmd = platform() === 'darwin'
+        ? 'dscacheutil -flushcache'
+        : platform() === 'linux'
+            ? 'resolvectl flush-caches'
+            : null;
+    if (!cmd) {
+        logger.warn(`Circuit breaker: DNS flush not supported on platform "${platform()}"`);
+        return;
+    }
+    logger.warn('Circuit breaker: Attempting DNS cache flush...');
+    try {
+        await execAsync(cmd, { timeout: 5000 });
+        logger.info('Circuit breaker: DNS cache flush completed');
+    }
+    catch (err) {
+        logger.warn(`Circuit breaker: DNS cache flush failed: ${err}`);
+    }
+}
 // Track which threads have an active processing loop
 const activeLoops = new Set();
 async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
@@ -302,10 +338,34 @@ function deduplicateMessages(messages) {
     }
     return { unique, duplicates };
 }
+// Circuit breaker for consecutive auth/infrastructure failures
+const AUTH_CIRCUIT_BREAKER_THRESHOLD = 3;
+const AUTH_CIRCUIT_BREAKER_MAX_BACKOFF_MS = 60_000;
+let consecutiveAuthFailures = 0;
+function isAuthError(error) {
+    if (!(error instanceof Error))
+        return false;
+    const name = error.name ?? '';
+    const code = error.Code ?? '';
+    const message = error.message ?? '';
+    return (name === 'ExpiredTokenException' ||
+        code === 'ExpiredToken' ||
+        code === 'InvalidClientTokenId' ||
+        code === 'UnrecognizedClientException' ||
+        message.includes('security token') ||
+        message.includes('Auth failed'));
+}
 async function pollLoop() {
     logger.info('Poll loop started');
     while (!shuttingDown) {
         try {
+            // Circuit breaker: if too many consecutive auth failures, back off
+            if (consecutiveAuthFailures >= AUTH_CIRCUIT_BREAKER_THRESHOLD) {
+                const backoffMs = Math.min(5_000 * Math.pow(2, consecutiveAuthFailures - AUTH_CIRCUIT_BREAKER_THRESHOLD), AUTH_CIRCUIT_BREAKER_MAX_BACKOFF_MS);
+                logger.warn(`[CircuitBreaker] Auth degraded (${consecutiveAuthFailures} consecutive failures), ` +
+                    `backing off ${backoffMs}ms before retry...`);
+                await sleep(backoffMs);
+            }
             if (isAtCapacity()) {
                 logger.debug(`At capacity (${getActiveProcessCount()}/${config.MAX_CONCURRENT_SESSIONS}), waiting...`);
                 await sleep(1000);
@@ -314,6 +374,12 @@ async function pollLoop() {
             const availableSlots = config.MAX_CONCURRENT_SESSIONS - getActiveProcessCount();
             logger.debug(`Polling for up to ${availableSlots} messages...`);
             const messages = await pollMessages(availableSlots);
+            consecutiveNetFailures = 0; // Successful poll — reset circuit breaker
+            // Successful poll — reset circuit breaker
+            if (consecutiveAuthFailures > 0) {
+                logger.info(`[CircuitBreaker] Auth recovered after ${consecutiveAuthFailures} failure(s)`);
+                consecutiveAuthFailures = 0;
+            }
             if (messages.length === 0) {
                 continue;
             }
@@ -347,8 +413,41 @@ async function pollLoop() {
             Promise.all(processPromises);
         }
         catch (error) {
-            logger.error('Error in poll loop:', error);
-            await sleep(5000);
+            if (isAuthError(error)) {
+                consecutiveAuthFailures++;
+                logger.error(`[CircuitBreaker] Auth failure #${consecutiveAuthFailures}: ${error instanceof Error ? error.message : error}`);
+                emitEvent('poller.auth_failure', '', { consecutiveFailures: consecutiveAuthFailures });
+                await sleep(5000);
+            }
+            else if (isNetworkError(error)) {
+                consecutiveNetFailures++;
+                logger.error(`Network error in poll loop (${consecutiveNetFailures}/${config.MAX_CONSECUTIVE_NET_FAILURES}):`, error);
+                if (consecutiveNetFailures === config.MAX_CONSECUTIVE_NET_FAILURES) {
+                    await attemptDnsFlush();
+                }
+                if (consecutiveNetFailures >= config.MAX_CONSECUTIVE_NET_FAILURES * 2) {
+                    logger.error(`Circuit breaker triggered: ${consecutiveNetFailures} consecutive network failures. Exiting for restart.`);
+                    emitEvent('poller.circuit_breaker', '', {
+                        reason: 'sustained_network_failure',
+                        count: consecutiveNetFailures,
+                    });
+                    // Hard timeout on shutdown — network may be down, don't hang
+                    await Promise.race([
+                        shutdown('CIRCUIT_BREAKER', 1),
+                        sleep(15000),
+                    ]);
+                    process.exit(1); // Fallback if shutdown didn't call process.exit (e.g. timeout)
+                }
+                // Exponential backoff: 5s, 10s, 20s, 40s... capped at 60s
+                const backoffMs = Math.min(5000 * Math.pow(2, consecutiveNetFailures - 1), 60000);
+                await sleep(backoffMs);
+            }
+            else {
+                // Non-network, non-auth errors — reset auth circuit breaker (infra is reachable)
+                consecutiveAuthFailures = 0;
+                logger.error('Error in poll loop:', error);
+                await sleep(5000);
+            }
         }
     }
 }
@@ -357,7 +456,7 @@ function sleep(ms) {
 }
 // Graceful shutdown
 let shuttingDown = false;
-async function shutdown(signal) {
+async function shutdown(signal, exitCode = 0) {
     if (shuttingDown)
         return;
     shuttingDown = true;
@@ -382,7 +481,7 @@ async function shutdown(signal) {
     // Final flush after all processes stopped — captures errors emitted during shutdown
     await flushEventQueue();
     logger.info('Shutdown complete');
-    process.exit(0);
+    process.exit(exitCode);
 }
 export async function startPoller() {
     logger.info('TeamVibe Poller starting...');

package/dist/sqs-poller.js

@@ -1,8 +1,15 @@
 import { SQSClient, ReceiveMessageCommand, DeleteMessageCommand, ChangeMessageVisibilityCommand, SendMessageCommand, } from '@aws-sdk/client-sqs';
 import { config } from './config.js';
 import { logger } from './logger.js';
-import { isTokenMode, getCredentials, getAuthConfig } from './auth-provider.js';
+import { isTokenMode, getCredentials, getAuthConfig, forceRefresh } from './auth-provider.js';
 import { emitError } from './event-queue.js';
+function isExpiredTokenError(error) {
+    if (!(error instanceof Error))
+        return false;
+    const name = error.name ?? '';
+    const code = error.Code ?? '';
+    return name === 'ExpiredTokenException' || code === 'ExpiredToken';
+}
 function createSqsClient() {
     if (isTokenMode()) {
         const creds = getCredentials();
@@ -42,6 +49,24 @@ export async function pollMessages(maxMessages = 10) {
         return response.Messages.map((msg) => parseMessage(msg)).filter((m) => m !== null);
     }
     catch (error) {
+        // Expired AWS STS token — force-refresh credentials and retry once
+        if (isTokenMode() && isExpiredTokenError(error)) {
+            logger.warn('SQS poll failed with ExpiredToken, attempting credential refresh...');
+            await forceRefresh(); // throws if refresh itself fails
+            // Retry poll with fresh credentials
+            const retryResponse = await createSqsClient().send(new ReceiveMessageCommand({
+                QueueUrl: getQueueUrl(),
+                MaxNumberOfMessages: Math.min(maxMessages, 10),
+                WaitTimeSeconds: config.POLL_WAIT_TIME_SECONDS,
+                VisibilityTimeout: config.VISIBILITY_TIMEOUT_SECONDS,
+                MessageAttributeNames: ['All'],
+            }));
+            if (!retryResponse.Messages || retryResponse.Messages.length === 0) {
+                return [];
+            }
+            logger.info(`Received ${retryResponse.Messages.length} message(s) after credential refresh`);
+            return retryResponse.Messages.map((msg) => parseMessage(msg)).filter((m) => m !== null);
+        }
         logger.error('Error polling SQS:', error);
         emitError('sqs', `Error polling SQS: ${error instanceof Error ? error.message : String(error)}`);
         throw error;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@teamvibe/poller",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "type": "module",
   "main": "./dist/index.js",
   "bin": {