@teamvibe/poller 0.1.57 → 0.1.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-provider.js +4 -0
- package/dist/brain-manager.js +5 -0
- package/dist/claude-spawner.d.ts +3 -1
- package/dist/claude-spawner.js +65 -1
- package/dist/config.d.ts +8 -0
- package/dist/config.js +2 -0
- package/dist/event-queue.d.ts +14 -0
- package/dist/event-queue.js +113 -0
- package/dist/heartbeat-manager.js +5 -0
- package/dist/mailbox-manager.d.ts +22 -0
- package/dist/mailbox-manager.js +86 -0
- package/dist/maintenance-reporter.js +6 -0
- package/dist/poller.js +99 -184
- package/dist/session-store.js +5 -0
- package/dist/slack-client.js +4 -0
- package/dist/sqs-poller.js +4 -0
- package/dist/types.d.ts +11 -0
- package/package.json +1 -1
package/dist/auth-provider.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { POLLER_VERSION } from './config.js';
|
|
2
2
|
import { logger } from './logger.js';
|
|
3
|
+
import { emitError } from './event-queue.js';
|
|
3
4
|
let currentCredentials = null;
|
|
4
5
|
let currentConfig = null;
|
|
5
6
|
let currentPollerId = null;
|
|
@@ -19,10 +20,12 @@ async function sendHeartbeat(apiUrl, token, version) {
|
|
|
19
20
|
});
|
|
20
21
|
if (!response.ok) {
|
|
21
22
|
logger.warn(`Heartbeat failed (${response.status})`);
|
|
23
|
+
emitError('auth', `Heartbeat failed (${response.status})`, { severity: 'warn' });
|
|
22
24
|
}
|
|
23
25
|
}
|
|
24
26
|
catch (error) {
|
|
25
27
|
logger.warn(`Heartbeat error: ${error instanceof Error ? error.message : error}`);
|
|
28
|
+
emitError('auth', `Heartbeat error: ${error instanceof Error ? error.message : String(error)}`, { severity: 'warn' });
|
|
26
29
|
}
|
|
27
30
|
}
|
|
28
31
|
async function fetchCredentials(apiUrl, token) {
|
|
@@ -62,6 +65,7 @@ export async function initAuth(apiUrl, token) {
|
|
|
62
65
|
}
|
|
63
66
|
catch (error) {
|
|
64
67
|
logger.error(`Credential refresh failed: ${error instanceof Error ? error.message : error}`);
|
|
68
|
+
emitError('auth', `Credential refresh failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
65
69
|
}
|
|
66
70
|
}, REFRESH_INTERVAL_MS);
|
|
67
71
|
}
|
package/dist/brain-manager.js
CHANGED
|
@@ -4,6 +4,7 @@ import { existsSync, mkdirSync } from 'fs';
|
|
|
4
4
|
import { join } from 'path';
|
|
5
5
|
import { config } from './config.js';
|
|
6
6
|
import { logger } from './logger.js';
|
|
7
|
+
import { emitError } from './event-queue.js';
|
|
7
8
|
const execAsync = promisify(exec);
|
|
8
9
|
// Clean env for git commands:
|
|
9
10
|
// - npm/npx sets GIT_ASKPASS which prevents credential helpers from working
|
|
@@ -42,10 +43,12 @@ async function reportBrainSync(brainId, workspaceId, event) {
|
|
|
42
43
|
});
|
|
43
44
|
if (!response.ok) {
|
|
44
45
|
logger.warn(`Brain sync report failed (${response.status})`);
|
|
46
|
+
emitError('brain', `Brain sync report failed (${response.status})`, { severity: 'warn', context: { brainId, event } });
|
|
45
47
|
}
|
|
46
48
|
}
|
|
47
49
|
catch (error) {
|
|
48
50
|
logger.warn(`Brain sync report error: ${error instanceof Error ? error.message : error}`);
|
|
51
|
+
emitError('brain', `Brain sync report error: ${error instanceof Error ? error.message : String(error)}`, { severity: 'warn', context: { brainId } });
|
|
49
52
|
}
|
|
50
53
|
}
|
|
51
54
|
export async function getBrainPath(brain, channelId, workspaceId) {
|
|
@@ -107,6 +110,7 @@ async function updateBrain(brainDir, brainId, branch) {
|
|
|
107
110
|
catch (error) {
|
|
108
111
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
109
112
|
logger.error(`Failed to update brain ${brainId}: ${errorMessage}`);
|
|
113
|
+
emitError('brain', `Failed to update brain ${brainId}: ${errorMessage}`, { context: { brainId, brainDir } });
|
|
110
114
|
}
|
|
111
115
|
}
|
|
112
116
|
/**
|
|
@@ -170,6 +174,7 @@ export async function pushBrainChanges(brainDir, brainId, workspaceId) {
|
|
|
170
174
|
catch (error) {
|
|
171
175
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
172
176
|
logger.warn(`Failed to push brain ${brainId} changes: ${errorMessage}`);
|
|
177
|
+
emitError('brain', `Failed to push brain ${brainId} changes: ${errorMessage}`, { severity: 'warn', context: { brainId } });
|
|
173
178
|
}
|
|
174
179
|
}
|
|
175
180
|
/**
|
package/dist/claude-spawner.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { TeamVibeQueueMessage } from './types.js';
|
|
1
|
+
import type { TeamVibeQueueMessage, SessionMetrics } from './types.js';
|
|
2
2
|
import { type SessionLogger } from './logger.js';
|
|
3
3
|
import type { TypingIndicator } from './slack-client.js';
|
|
4
4
|
export interface SpawnResult {
|
|
@@ -6,7 +6,9 @@ export interface SpawnResult {
|
|
|
6
6
|
output: string;
|
|
7
7
|
exitCode: number | null;
|
|
8
8
|
error?: string;
|
|
9
|
+
metrics?: SessionMetrics;
|
|
9
10
|
}
|
|
11
|
+
export declare function buildPrompt(msg: TeamVibeQueueMessage, resumeHint?: string): string;
|
|
10
12
|
export declare function spawnClaudeCode(msg: TeamVibeQueueMessage, sessionLog: SessionLogger, cwd: string, sessionId?: string, isFirstMessage?: boolean, lastMessageTs?: string, onMessageSent?: () => void, typingIndicator?: TypingIndicator): Promise<SpawnResult & {
|
|
11
13
|
newSessionId?: string;
|
|
12
14
|
}>;
|
package/dist/claude-spawner.js
CHANGED
|
@@ -6,6 +6,7 @@ import { config } from './config.js';
|
|
|
6
6
|
import { logger } from './logger.js';
|
|
7
7
|
import { getBaseBrainPath } from './brain-manager.js';
|
|
8
8
|
import { getPollerId } from './auth-provider.js';
|
|
9
|
+
import { emitError } from './event-queue.js';
|
|
9
10
|
/**
|
|
10
11
|
* Maps tool names (or prefixes) to human-readable status messages.
|
|
11
12
|
* More specific patterns are checked first.
|
|
@@ -80,6 +81,7 @@ async function countNewThreadMessages(botToken, channel, threadTs, sinceTs) {
|
|
|
80
81
|
}
|
|
81
82
|
catch (error) {
|
|
82
83
|
logger.warn(`Failed to count new thread messages: ${error}`);
|
|
84
|
+
emitError('slack', `Failed to count new thread messages: ${error instanceof Error ? error.message : String(error)}`, { severity: 'warn' });
|
|
83
85
|
return 0;
|
|
84
86
|
}
|
|
85
87
|
}
|
|
@@ -97,10 +99,11 @@ function readBrainSettings(cwd) {
|
|
|
97
99
|
}
|
|
98
100
|
catch (error) {
|
|
99
101
|
logger.warn(`Failed to read brain settings at ${settingsPath}: ${error}`);
|
|
102
|
+
emitError('brain', `Failed to read brain settings at ${settingsPath}: ${error instanceof Error ? error.message : String(error)}`, { severity: 'warn' });
|
|
100
103
|
return {};
|
|
101
104
|
}
|
|
102
105
|
}
|
|
103
|
-
function buildPrompt(msg, resumeHint) {
|
|
106
|
+
export function buildPrompt(msg, resumeHint) {
|
|
104
107
|
// Heartbeat messages get a minimal prompt — CLAUDE.md/MAINTENANCE.md handle the rest
|
|
105
108
|
if (msg.source === 'heartbeat') {
|
|
106
109
|
return 'heartbeat';
|
|
@@ -268,6 +271,8 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
268
271
|
args.push('--model', model);
|
|
269
272
|
sessionLog.info(`Using model: ${model}`);
|
|
270
273
|
}
|
|
274
|
+
// Mailbox protocol: instruct Claude to check for mid-task messages
|
|
275
|
+
args.push('--append-system-prompt', 'Between major steps, check for new user messages: run `ls .mailbox/new/ 2>/dev/null`. If files exist, read each (oldest first), delete it, and act on the message immediately. If empty, continue your current work. Check before starting, every 2-3 tool calls, and after finishing.');
|
|
271
276
|
// Handle session continuity
|
|
272
277
|
if (sessionId) {
|
|
273
278
|
if (isFirstMessage) {
|
|
@@ -319,6 +324,8 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
319
324
|
let stdout = '';
|
|
320
325
|
let stderr = '';
|
|
321
326
|
let stdoutBuffer = '';
|
|
327
|
+
let metrics;
|
|
328
|
+
let lastActivityTime = Date.now();
|
|
322
329
|
proc.stdout.on('data', (data) => {
|
|
323
330
|
const chunk = data.toString();
|
|
324
331
|
stdout += chunk;
|
|
@@ -332,6 +339,26 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
332
339
|
for (const line of lines) {
|
|
333
340
|
try {
|
|
334
341
|
const event = JSON.parse(line);
|
|
342
|
+
// Track activity for idle watchdog
|
|
343
|
+
lastActivityTime = Date.now();
|
|
344
|
+
// Extract metrics from result event
|
|
345
|
+
if (event.type === 'result') {
|
|
346
|
+
metrics = {
|
|
347
|
+
duration_ms: event.duration_ms || 0,
|
|
348
|
+
duration_api_ms: event.duration_api_ms || 0,
|
|
349
|
+
num_turns: event.num_turns || 0,
|
|
350
|
+
total_cost_usd: event.total_cost_usd || 0,
|
|
351
|
+
input_tokens: event.usage?.input_tokens || 0,
|
|
352
|
+
output_tokens: event.usage?.output_tokens || 0,
|
|
353
|
+
cache_read_input_tokens: event.usage?.cache_read_input_tokens || 0,
|
|
354
|
+
cache_creation_input_tokens: event.usage?.cache_creation_input_tokens || 0,
|
|
355
|
+
model: Object.keys(event.modelUsage || {})[0] || 'unknown',
|
|
356
|
+
};
|
|
357
|
+
sessionLog.info(`[Metrics] Cost: $${metrics.total_cost_usd.toFixed(4)}, ` +
|
|
358
|
+
`Duration: ${metrics.duration_ms}ms, ` +
|
|
359
|
+
`Turns: ${metrics.num_turns}, ` +
|
|
360
|
+
`Tokens: ${metrics.input_tokens}in/${metrics.output_tokens}out`);
|
|
361
|
+
}
|
|
335
362
|
const formatted = formatStreamEvent(event);
|
|
336
363
|
if (formatted) {
|
|
337
364
|
sessionLog.claude('stdout', formatted);
|
|
@@ -373,16 +400,47 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
373
400
|
const timeout = setTimeout(() => {
|
|
374
401
|
timedOut = true;
|
|
375
402
|
logger.error(`Claude Code process timed out after ${timeoutMinutes} minutes`);
|
|
403
|
+
emitError('session', `Claude Code process timed out after ${timeoutMinutes} minutes`, {
|
|
404
|
+
context: { channelId: msg.teamvibe.channelId, threadId: msg.thread_id },
|
|
405
|
+
channelId: msg.teamvibe.channelId,
|
|
406
|
+
});
|
|
376
407
|
proc.kill('SIGTERM');
|
|
377
408
|
setTimeout(() => proc.kill('SIGKILL'), 5000);
|
|
378
409
|
}, config.CLAUDE_TIMEOUT_MS);
|
|
410
|
+
// Idle watchdog: kill if no stdout activity for too long
|
|
411
|
+
let idleWarned = false;
|
|
412
|
+
const idleWatchdog = setInterval(() => {
|
|
413
|
+
const idleMs = Date.now() - lastActivityTime;
|
|
414
|
+
if (idleMs > config.IDLE_WATCHDOG_KILL_MS) {
|
|
415
|
+
const idleMin = Math.round(idleMs / 60000);
|
|
416
|
+
logger.error(`Claude Code idle for ${idleMin} minutes, killing process`);
|
|
417
|
+
emitError('session', `Claude Code idle for ${idleMin} minutes — killing hung process`, {
|
|
418
|
+
context: { channelId: msg.teamvibe.channelId, threadId: msg.thread_id },
|
|
419
|
+
channelId: msg.teamvibe.channelId,
|
|
420
|
+
});
|
|
421
|
+
proc.kill('SIGTERM');
|
|
422
|
+
setTimeout(() => proc.kill('SIGKILL'), 5000);
|
|
423
|
+
}
|
|
424
|
+
else if (!idleWarned && idleMs > config.IDLE_WATCHDOG_WARN_MS) {
|
|
425
|
+
const idleMin = Math.round(idleMs / 60000);
|
|
426
|
+
logger.warn(`Claude Code idle for ${idleMin} minutes (will kill at ${Math.round(config.IDLE_WATCHDOG_KILL_MS / 60000)} min)`);
|
|
427
|
+
emitError('session', `Claude Code idle for ${idleMin} minutes`, {
|
|
428
|
+
severity: 'warn',
|
|
429
|
+
context: { channelId: msg.teamvibe.channelId, threadId: msg.thread_id },
|
|
430
|
+
channelId: msg.teamvibe.channelId,
|
|
431
|
+
});
|
|
432
|
+
idleWarned = true;
|
|
433
|
+
}
|
|
434
|
+
}, 30000); // Check every 30 seconds
|
|
379
435
|
proc.on('close', (code) => {
|
|
380
436
|
clearTimeout(timeout);
|
|
437
|
+
clearInterval(idleWatchdog);
|
|
381
438
|
activeProcesses.delete(msg.id);
|
|
382
439
|
const result = {
|
|
383
440
|
success: code === 0,
|
|
384
441
|
output: stdout,
|
|
385
442
|
exitCode: code,
|
|
443
|
+
metrics,
|
|
386
444
|
};
|
|
387
445
|
if (timedOut) {
|
|
388
446
|
result.error = `Session timed out after ${timeoutMinutes} minutes. The task was still running when the time limit was reached. Consider breaking complex tasks into smaller steps.`;
|
|
@@ -394,7 +452,12 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
394
452
|
});
|
|
395
453
|
proc.on('error', (error) => {
|
|
396
454
|
clearTimeout(timeout);
|
|
455
|
+
clearInterval(idleWatchdog);
|
|
397
456
|
activeProcesses.delete(msg.id);
|
|
457
|
+
emitError('session', `Claude Code process spawn error: ${error.message}`, {
|
|
458
|
+
channelId: msg.teamvibe.channelId,
|
|
459
|
+
context: { threadId: msg.thread_id },
|
|
460
|
+
});
|
|
398
461
|
resolve({
|
|
399
462
|
success: false,
|
|
400
463
|
output: stdout,
|
|
@@ -440,6 +503,7 @@ export function killAllActiveProcesses() {
|
|
|
440
503
|
}
|
|
441
504
|
const forceKillTimer = setTimeout(() => {
|
|
442
505
|
logger.warn(`Process ${id} did not exit after SIGTERM, sending SIGKILL`);
|
|
506
|
+
emitError('session', `Process ${id} did not exit after SIGTERM, sending SIGKILL`, { severity: 'warn' });
|
|
443
507
|
proc.kill('SIGKILL');
|
|
444
508
|
}, 5000);
|
|
445
509
|
proc.once('close', () => {
|
package/dist/config.d.ts
CHANGED
|
@@ -13,6 +13,8 @@ declare const configSchema: z.ZodObject<{
|
|
|
13
13
|
CLAUDE_TIMEOUT_MS: z.ZodDefault<z.ZodNumber>;
|
|
14
14
|
STALE_LOCK_TIMEOUT_MS: z.ZodDefault<z.ZodNumber>;
|
|
15
15
|
LOCK_RETRY_INTERVAL_MS: z.ZodDefault<z.ZodNumber>;
|
|
16
|
+
IDLE_WATCHDOG_WARN_MS: z.ZodDefault<z.ZodNumber>;
|
|
17
|
+
IDLE_WATCHDOG_KILL_MS: z.ZodDefault<z.ZodNumber>;
|
|
16
18
|
TEAMVIBE_DATA_DIR: z.ZodDefault<z.ZodString>;
|
|
17
19
|
BRAINS_PATH: z.ZodDefault<z.ZodString>;
|
|
18
20
|
DEFAULT_BRAIN_PATH: z.ZodDefault<z.ZodString>;
|
|
@@ -37,6 +39,8 @@ declare const configSchema: z.ZodObject<{
|
|
|
37
39
|
CLAUDE_TIMEOUT_MS: number;
|
|
38
40
|
STALE_LOCK_TIMEOUT_MS: number;
|
|
39
41
|
LOCK_RETRY_INTERVAL_MS: number;
|
|
42
|
+
IDLE_WATCHDOG_WARN_MS: number;
|
|
43
|
+
IDLE_WATCHDOG_KILL_MS: number;
|
|
40
44
|
TEAMVIBE_DATA_DIR: string;
|
|
41
45
|
BRAINS_PATH: string;
|
|
42
46
|
DEFAULT_BRAIN_PATH: string;
|
|
@@ -67,6 +71,8 @@ declare const configSchema: z.ZodObject<{
|
|
|
67
71
|
CLAUDE_TIMEOUT_MS?: number | undefined;
|
|
68
72
|
STALE_LOCK_TIMEOUT_MS?: number | undefined;
|
|
69
73
|
LOCK_RETRY_INTERVAL_MS?: number | undefined;
|
|
74
|
+
IDLE_WATCHDOG_WARN_MS?: number | undefined;
|
|
75
|
+
IDLE_WATCHDOG_KILL_MS?: number | undefined;
|
|
70
76
|
TEAMVIBE_DATA_DIR?: string | undefined;
|
|
71
77
|
BRAINS_PATH?: string | undefined;
|
|
72
78
|
DEFAULT_BRAIN_PATH?: string | undefined;
|
|
@@ -94,6 +100,8 @@ export declare const config: {
|
|
|
94
100
|
CLAUDE_TIMEOUT_MS: number;
|
|
95
101
|
STALE_LOCK_TIMEOUT_MS: number;
|
|
96
102
|
LOCK_RETRY_INTERVAL_MS: number;
|
|
103
|
+
IDLE_WATCHDOG_WARN_MS: number;
|
|
104
|
+
IDLE_WATCHDOG_KILL_MS: number;
|
|
97
105
|
TEAMVIBE_DATA_DIR: string;
|
|
98
106
|
BRAINS_PATH: string;
|
|
99
107
|
DEFAULT_BRAIN_PATH: string;
|
package/dist/config.js
CHANGED
|
@@ -22,6 +22,8 @@ const configSchema = z.object({
|
|
|
22
22
|
CLAUDE_TIMEOUT_MS: z.coerce.number().default(3600000), // 60 minutes
|
|
23
23
|
STALE_LOCK_TIMEOUT_MS: z.coerce.number().default(3900000), // 65 minutes
|
|
24
24
|
LOCK_RETRY_INTERVAL_MS: z.coerce.number().default(30000), // 30 seconds
|
|
25
|
+
IDLE_WATCHDOG_WARN_MS: z.coerce.number().default(300000), // 5 minutes
|
|
26
|
+
IDLE_WATCHDOG_KILL_MS: z.coerce.number().default(600000), // 10 minutes
|
|
25
27
|
// Paths
|
|
26
28
|
TEAMVIBE_DATA_DIR: z.string().default(DEFAULT_DATA_DIR),
|
|
27
29
|
BRAINS_PATH: z.string().default(''),
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type ErrorCategory = 'sqs' | 'brain' | 'auth' | 'session' | 'slack' | 'maintenance';
|
|
2
|
+
export type ErrorSeverity = 'warn' | 'error';
|
|
3
|
+
export declare function flush(): Promise<void>;
|
|
4
|
+
export declare function startEventQueue(): void;
|
|
5
|
+
export declare function stopEventQueue(): void;
|
|
6
|
+
export declare function emitError(category: ErrorCategory, message: string, options?: {
|
|
7
|
+
severity?: ErrorSeverity;
|
|
8
|
+
channelId?: string;
|
|
9
|
+
context?: Record<string, unknown>;
|
|
10
|
+
}): void;
|
|
11
|
+
/** For testing: get current queue length */
|
|
12
|
+
export declare function getQueueLength(): number;
|
|
13
|
+
/** For testing: clear the queue */
|
|
14
|
+
export declare function clearQueue(): void;
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
// packages/poller/src/event-queue.ts
|
|
2
|
+
import { config } from './config.js';
|
|
3
|
+
import { logger } from './logger.js';
|
|
4
|
+
const MAX_QUEUE_SIZE = 200;
|
|
5
|
+
const FLUSH_INTERVAL_MS = 30_000;
|
|
6
|
+
const DEDUP_WINDOW_MS = 60_000;
|
|
7
|
+
let queue = [];
|
|
8
|
+
let flushTimer = null;
|
|
9
|
+
let isFlushing = false;
|
|
10
|
+
function dedupKey(channelId, category, message) {
|
|
11
|
+
return `${channelId}:${category}:${message}`;
|
|
12
|
+
}
|
|
13
|
+
function enqueue(event) {
|
|
14
|
+
// Dedup: collapse identical category+message within window
|
|
15
|
+
const key = dedupKey(event.channelId, event.metadata.category, event.metadata.message);
|
|
16
|
+
const now = Date.now();
|
|
17
|
+
const windowStart = new Date(now - DEDUP_WINDOW_MS).toISOString();
|
|
18
|
+
const existing = queue.find((e) => dedupKey(e.channelId, e.metadata.category, e.metadata.message) === key &&
|
|
19
|
+
e.timestamp >= windowStart);
|
|
20
|
+
if (existing) {
|
|
21
|
+
existing.metadata.count += 1;
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
if (queue.length >= MAX_QUEUE_SIZE) {
|
|
25
|
+
queue.shift(); // Drop oldest
|
|
26
|
+
}
|
|
27
|
+
queue.push(event);
|
|
28
|
+
}
|
|
29
|
+
async function flushOne(event) {
|
|
30
|
+
try {
|
|
31
|
+
const response = await fetch(`${config.TEAMVIBE_API_URL}/events`, {
|
|
32
|
+
method: 'POST',
|
|
33
|
+
headers: {
|
|
34
|
+
'Content-Type': 'application/json',
|
|
35
|
+
Authorization: `Bearer ${config.TEAMVIBE_POLLER_TOKEN}`,
|
|
36
|
+
},
|
|
37
|
+
body: JSON.stringify({
|
|
38
|
+
eventType: event.eventType,
|
|
39
|
+
channelId: event.channelId,
|
|
40
|
+
metadata: event.metadata,
|
|
41
|
+
error: event.error,
|
|
42
|
+
}),
|
|
43
|
+
});
|
|
44
|
+
return response.ok;
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
export async function flush() {
|
|
51
|
+
if (queue.length === 0 || isFlushing)
|
|
52
|
+
return;
|
|
53
|
+
isFlushing = true;
|
|
54
|
+
try {
|
|
55
|
+
const batch = [...queue];
|
|
56
|
+
const failed = [];
|
|
57
|
+
for (const event of batch) {
|
|
58
|
+
const ok = await flushOne(event);
|
|
59
|
+
if (!ok) {
|
|
60
|
+
failed.push(event);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
queue = failed;
|
|
64
|
+
if (failed.length > 0) {
|
|
65
|
+
logger.debug(`[EventQueue] ${failed.length} event(s) failed to flush, will retry`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
finally {
|
|
69
|
+
isFlushing = false;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
export function startEventQueue() {
|
|
73
|
+
if (!config.TEAMVIBE_API_URL || !config.TEAMVIBE_POLLER_TOKEN) {
|
|
74
|
+
return; // No-op in direct mode
|
|
75
|
+
}
|
|
76
|
+
flushTimer = setInterval(() => {
|
|
77
|
+
flush().catch(() => { });
|
|
78
|
+
}, FLUSH_INTERVAL_MS);
|
|
79
|
+
}
|
|
80
|
+
export function stopEventQueue() {
|
|
81
|
+
if (flushTimer) {
|
|
82
|
+
clearInterval(flushTimer);
|
|
83
|
+
flushTimer = null;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
export function emitError(category, message, options) {
|
|
87
|
+
if (!config.TEAMVIBE_API_URL || !config.TEAMVIBE_POLLER_TOKEN) {
|
|
88
|
+
return; // No-op in direct mode
|
|
89
|
+
}
|
|
90
|
+
const severity = options?.severity ?? 'error';
|
|
91
|
+
const channelId = options?.channelId ?? '_system';
|
|
92
|
+
enqueue({
|
|
93
|
+
eventType: 'poller.error',
|
|
94
|
+
channelId,
|
|
95
|
+
metadata: {
|
|
96
|
+
category,
|
|
97
|
+
severity,
|
|
98
|
+
message,
|
|
99
|
+
count: 1,
|
|
100
|
+
context: options?.context,
|
|
101
|
+
},
|
|
102
|
+
error: { code: category, message },
|
|
103
|
+
timestamp: new Date().toISOString(),
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
/** For testing: get current queue length */
|
|
107
|
+
export function getQueueLength() {
|
|
108
|
+
return queue.length;
|
|
109
|
+
}
|
|
110
|
+
/** For testing: clear the queue */
|
|
111
|
+
export function clearQueue() {
|
|
112
|
+
queue = [];
|
|
113
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { config } from './config.js';
|
|
2
2
|
import { logger } from './logger.js';
|
|
3
3
|
import { isAtCapacity } from './claude-spawner.js';
|
|
4
|
+
import { emitError } from './event-queue.js';
|
|
4
5
|
const trackedBrains = new Map();
|
|
5
6
|
let heartbeatInterval = null;
|
|
6
7
|
/**
|
|
@@ -85,6 +86,10 @@ export function startHeartbeatLoop(processMessageFn) {
|
|
|
85
86
|
}
|
|
86
87
|
catch (error) {
|
|
87
88
|
logger.error(`[Heartbeat] Failed maintenance for ${brainId}: ${error instanceof Error ? error.message : error}`);
|
|
89
|
+
emitError('session', `Heartbeat maintenance failed for ${brainId}: ${error instanceof Error ? error.message : String(error)}`, {
|
|
90
|
+
channelId: info.channelId,
|
|
91
|
+
context: { brainId },
|
|
92
|
+
});
|
|
88
93
|
}
|
|
89
94
|
}
|
|
90
95
|
}, config.MAINTENANCE_CHECK_INTERVAL_MS);
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Write a new message to the mailbox.
|
|
3
|
+
* Returns the filename of the created message file.
|
|
4
|
+
*/
|
|
5
|
+
export declare function writeMessage(brainPath: string, content: string): string;
|
|
6
|
+
/**
|
|
7
|
+
* Pick the oldest message from processing/ (crash recovery) or new/.
|
|
8
|
+
* If found in new/, moves it to processing/.
|
|
9
|
+
* Returns { filename, content } or null if no messages.
|
|
10
|
+
*/
|
|
11
|
+
export declare function pickNextMessage(brainPath: string): {
|
|
12
|
+
filename: string;
|
|
13
|
+
content: string;
|
|
14
|
+
} | null;
|
|
15
|
+
/**
|
|
16
|
+
* Delete a message file from processing/ (marks it as done).
|
|
17
|
+
*/
|
|
18
|
+
export declare function markProcessed(brainPath: string, filename: string): void;
|
|
19
|
+
/**
|
|
20
|
+
* Check if there are unprocessed messages in new/ or processing/.
|
|
21
|
+
*/
|
|
22
|
+
export declare function hasUnprocessed(brainPath: string): boolean;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { mkdirSync, writeFileSync, readFileSync, renameSync, unlinkSync, readdirSync, existsSync } from 'fs';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { logger } from './logger.js';
|
|
4
|
+
const MAILBOX_DIR = '.mailbox';
|
|
5
|
+
const NEW_DIR = 'new';
|
|
6
|
+
const PROCESSING_DIR = 'processing';
|
|
7
|
+
function ensureMailboxDirs(brainPath) {
|
|
8
|
+
const mailbox = join(brainPath, MAILBOX_DIR);
|
|
9
|
+
const newDir = join(mailbox, NEW_DIR);
|
|
10
|
+
const processingDir = join(mailbox, PROCESSING_DIR);
|
|
11
|
+
mkdirSync(newDir, { recursive: true });
|
|
12
|
+
mkdirSync(processingDir, { recursive: true });
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Write a new message to the mailbox.
|
|
16
|
+
* Returns the filename of the created message file.
|
|
17
|
+
*/
|
|
18
|
+
export function writeMessage(brainPath, content) {
|
|
19
|
+
ensureMailboxDirs(brainPath);
|
|
20
|
+
const timestamp = process.hrtime.bigint().toString();
|
|
21
|
+
const filename = `${timestamp}.txt`;
|
|
22
|
+
const filepath = join(brainPath, MAILBOX_DIR, NEW_DIR, filename);
|
|
23
|
+
writeFileSync(filepath, content, 'utf-8');
|
|
24
|
+
logger.info(`[Mailbox] Wrote message: ${filename} (${content.length} chars)`);
|
|
25
|
+
return filename;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Pick the oldest message from processing/ (crash recovery) or new/.
|
|
29
|
+
* If found in new/, moves it to processing/.
|
|
30
|
+
* Returns { filename, content } or null if no messages.
|
|
31
|
+
*/
|
|
32
|
+
export function pickNextMessage(brainPath) {
|
|
33
|
+
ensureMailboxDirs(brainPath);
|
|
34
|
+
const processingDir = join(brainPath, MAILBOX_DIR, PROCESSING_DIR);
|
|
35
|
+
const newDir = join(brainPath, MAILBOX_DIR, NEW_DIR);
|
|
36
|
+
// First check processing/ (crash recovery)
|
|
37
|
+
const processingFiles = readdirSync(processingDir).filter(f => f.endsWith('.txt')).sort();
|
|
38
|
+
if (processingFiles.length > 0) {
|
|
39
|
+
const filename = processingFiles[0];
|
|
40
|
+
const content = readFileSync(join(processingDir, filename), 'utf-8');
|
|
41
|
+
logger.info(`[Mailbox] Resuming processing message: ${filename}`);
|
|
42
|
+
return { filename, content };
|
|
43
|
+
}
|
|
44
|
+
// Then check new/
|
|
45
|
+
const newFiles = readdirSync(newDir).filter(f => f.endsWith('.txt')).sort();
|
|
46
|
+
if (newFiles.length > 0) {
|
|
47
|
+
const filename = newFiles[0];
|
|
48
|
+
const srcPath = join(newDir, filename);
|
|
49
|
+
const dstPath = join(processingDir, filename);
|
|
50
|
+
const content = readFileSync(srcPath, 'utf-8');
|
|
51
|
+
renameSync(srcPath, dstPath);
|
|
52
|
+
logger.info(`[Mailbox] Picked message: ${filename} (moved to processing)`);
|
|
53
|
+
return { filename, content };
|
|
54
|
+
}
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Delete a message file from processing/ (marks it as done).
|
|
59
|
+
*/
|
|
60
|
+
export function markProcessed(brainPath, filename) {
|
|
61
|
+
const filepath = join(brainPath, MAILBOX_DIR, PROCESSING_DIR, filename);
|
|
62
|
+
try {
|
|
63
|
+
if (existsSync(filepath)) {
|
|
64
|
+
unlinkSync(filepath);
|
|
65
|
+
logger.info(`[Mailbox] Deleted processed message: ${filename}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
logger.warn(`[Mailbox] Failed to delete ${filename}: ${error}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Check if there are unprocessed messages in new/ or processing/.
|
|
74
|
+
*/
|
|
75
|
+
export function hasUnprocessed(brainPath) {
|
|
76
|
+
const newDir = join(brainPath, MAILBOX_DIR, NEW_DIR);
|
|
77
|
+
const processingDir = join(brainPath, MAILBOX_DIR, PROCESSING_DIR);
|
|
78
|
+
try {
|
|
79
|
+
const newCount = existsSync(newDir) ? readdirSync(newDir).filter(f => f.endsWith('.txt')).length : 0;
|
|
80
|
+
const procCount = existsSync(processingDir) ? readdirSync(processingDir).filter(f => f.endsWith('.txt')).length : 0;
|
|
81
|
+
return (newCount + procCount) > 0;
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -4,6 +4,7 @@ import { execFile } from 'child_process';
|
|
|
4
4
|
import { promisify } from 'util';
|
|
5
5
|
import { config } from './config.js';
|
|
6
6
|
import { logger } from './logger.js';
|
|
7
|
+
import { emitError } from './event-queue.js';
|
|
7
8
|
const execFileAsync = promisify(execFile);
|
|
8
9
|
const OPERATION_TO_EVENT_TYPE = {
|
|
9
10
|
consolidation: 'maintenance.consolidation',
|
|
@@ -32,11 +33,13 @@ export async function postMaintenanceReports(brainPath, channelId) {
|
|
|
32
33
|
const report = JSON.parse(content);
|
|
33
34
|
if (!report.operationType) {
|
|
34
35
|
logger.warn(`[MaintenanceReporter] Skipping ${file}: missing operationType`);
|
|
36
|
+
emitError('maintenance', `Skipping report ${file}: missing operationType`, { severity: 'warn' });
|
|
35
37
|
continue;
|
|
36
38
|
}
|
|
37
39
|
const eventType = OPERATION_TO_EVENT_TYPE[report.operationType];
|
|
38
40
|
if (!eventType) {
|
|
39
41
|
logger.warn(`[MaintenanceReporter] Skipping ${file}: unknown operationType "${report.operationType}"`);
|
|
42
|
+
emitError('maintenance', `Skipping report ${file}: unknown operationType "${report.operationType}"`, { severity: 'warn' });
|
|
40
43
|
continue;
|
|
41
44
|
}
|
|
42
45
|
const payload = {
|
|
@@ -60,10 +63,12 @@ export async function postMaintenanceReports(brainPath, channelId) {
|
|
|
60
63
|
else {
|
|
61
64
|
const errorBody = await response.text().catch(() => 'unknown');
|
|
62
65
|
logger.warn(`[MaintenanceReporter] Failed to POST ${file}: ${response.status} ${errorBody}`);
|
|
66
|
+
emitError('maintenance', `Failed to POST report ${file}: ${response.status}`, { severity: 'warn', channelId });
|
|
63
67
|
}
|
|
64
68
|
}
|
|
65
69
|
catch (err) {
|
|
66
70
|
logger.warn(`[MaintenanceReporter] Error processing ${file}: ${err instanceof Error ? err.message : err}`);
|
|
71
|
+
emitError('maintenance', `Error processing report ${file}: ${err instanceof Error ? err.message : String(err)}`, { severity: 'warn', channelId });
|
|
67
72
|
}
|
|
68
73
|
}
|
|
69
74
|
// Remove posted JSON reports from git so they don't resurrect on next pull
|
|
@@ -80,5 +85,6 @@ export async function postMaintenanceReports(brainPath, channelId) {
|
|
|
80
85
|
}
|
|
81
86
|
catch (err) {
|
|
82
87
|
logger.warn(`[MaintenanceReporter] Error scanning reports: ${err instanceof Error ? err.message : err}`);
|
|
88
|
+
emitError('maintenance', `Error scanning reports: ${err instanceof Error ? err.message : String(err)}`, { severity: 'warn' });
|
|
83
89
|
}
|
|
84
90
|
}
|
package/dist/poller.js
CHANGED
|
@@ -1,86 +1,101 @@
|
|
|
1
1
|
import { config } from './config.js';
|
|
2
2
|
import { logger } from './logger.js';
|
|
3
|
-
import { pollMessages, deleteMessage,
|
|
4
|
-
import { spawnClaudeCode, isAtCapacity, getActiveProcessCount, killAllActiveProcesses } from './claude-spawner.js';
|
|
3
|
+
import { pollMessages, deleteMessage, } from './sqs-poller.js';
|
|
4
|
+
import { spawnClaudeCode, isAtCapacity, getActiveProcessCount, killAllActiveProcesses, buildPrompt } from './claude-spawner.js';
|
|
5
5
|
import { sendSlackError, addReaction, getUserInfo, startTypingIndicator } from './slack-client.js';
|
|
6
6
|
import { acquireSessionLock, releaseSessionLock, updateSessionId } from './session-store.js';
|
|
7
7
|
import { getBrainPath, ensureDirectories, ensureBaseBrain, pushBrainChanges } from './brain-manager.js';
|
|
8
8
|
import { postMaintenanceReports } from './maintenance-reporter.js';
|
|
9
9
|
import { initAuth, stopRefresh } from './auth-provider.js';
|
|
10
10
|
import { registerBrain, startHeartbeatLoop, stopHeartbeatLoop } from './heartbeat-manager.js';
|
|
11
|
+
import { startEventQueue, stopEventQueue, flush as flushEventQueue } from './event-queue.js';
|
|
12
|
+
import { writeMessage, pickNextMessage, markProcessed, hasUnprocessed } from './mailbox-manager.js';
|
|
11
13
|
// Track active message processing
|
|
12
14
|
const processingMessages = new Set();
|
|
13
|
-
//
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
function logQueueState(context) {
|
|
28
|
-
const stats = getQueueStats();
|
|
29
|
-
logger.info(`[Queue] ${context} | Processing: ${stats.processing}, Threads with waiting: ${stats.threadsWithWaiting}, Total waiting: ${stats.totalWaiting}`);
|
|
30
|
-
}
|
|
31
|
-
function waitForThreadCompletion(threadId) {
|
|
32
|
-
let resolveFn;
|
|
33
|
-
const promise = new Promise((resolve) => {
|
|
34
|
-
resolveFn = resolve;
|
|
35
|
-
const signals = threadCompletionSignals.get(threadId) || [];
|
|
36
|
-
signals.push(resolve);
|
|
37
|
-
threadCompletionSignals.set(threadId, signals);
|
|
38
|
-
const currentCount = waitingCountByThread.get(threadId) || 0;
|
|
39
|
-
waitingCountByThread.set(threadId, currentCount + 1);
|
|
40
|
-
logger.info(`[Queue] Thread ${threadId}: ${currentCount + 1} message(s) now waiting`);
|
|
41
|
-
logQueueState('After enqueue');
|
|
42
|
-
});
|
|
43
|
-
const cancel = () => {
|
|
44
|
-
// Remove this specific resolve from the signals array
|
|
45
|
-
const signals = threadCompletionSignals.get(threadId);
|
|
46
|
-
if (signals) {
|
|
47
|
-
const idx = signals.indexOf(resolveFn);
|
|
48
|
-
if (idx !== -1)
|
|
49
|
-
signals.splice(idx, 1);
|
|
50
|
-
if (signals.length === 0)
|
|
51
|
-
threadCompletionSignals.delete(threadId);
|
|
15
|
+
// Track which threads have an active processing loop
|
|
16
|
+
const activeLoops = new Set();
|
|
17
|
+
async function runMailboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
18
|
+
if (activeLoops.has(threadId)) {
|
|
19
|
+
sessionLog.info(`Loop already running for thread ${threadId}, skipping`);
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
activeLoops.add(threadId);
|
|
23
|
+
let iteration = 0;
|
|
24
|
+
try {
|
|
25
|
+
let lockResult = await acquireSessionLock(threadId, kbPath);
|
|
26
|
+
if (!lockResult.success) {
|
|
27
|
+
sessionLog.info(`Session ${threadId} is locked, mailbox message will be picked up by active session`);
|
|
28
|
+
return;
|
|
52
29
|
}
|
|
53
|
-
const
|
|
54
|
-
|
|
55
|
-
|
|
30
|
+
const { session, lockToken } = lockResult;
|
|
31
|
+
while (true) {
|
|
32
|
+
const message = pickNextMessage(kbPath);
|
|
33
|
+
if (!message) {
|
|
34
|
+
sessionLog.info(`[Loop] No more messages for thread ${threadId}`);
|
|
35
|
+
break;
|
|
36
|
+
}
|
|
37
|
+
iteration++;
|
|
38
|
+
const isFirstMessage = session.message_count === 1 && iteration === 1;
|
|
39
|
+
sessionLog.info(`[Loop] Iteration #${iteration} — processing: ${message.filename}`);
|
|
40
|
+
await ensureBaseBrain();
|
|
41
|
+
const hasSlackContext = Boolean(queueMessage.response_context.slack?.channel &&
|
|
42
|
+
queueMessage.response_context.slack?.message_ts);
|
|
43
|
+
const typing = hasSlackContext
|
|
44
|
+
? startTypingIndicator(queueMessage)
|
|
45
|
+
: undefined;
|
|
46
|
+
try {
|
|
47
|
+
// Build a temporary message with the mailbox content as text
|
|
48
|
+
const mailboxMsg = {
|
|
49
|
+
...queueMessage,
|
|
50
|
+
text: message.content,
|
|
51
|
+
};
|
|
52
|
+
const result = await spawnClaudeCode(mailboxMsg, sessionLog, kbPath, session.session_id || undefined, isFirstMessage, session.last_message_ts, () => typing?.stop(), typing);
|
|
53
|
+
typing?.stop();
|
|
54
|
+
if (result.newSessionId && lockToken) {
|
|
55
|
+
await updateSessionId(threadId, lockToken, result.newSessionId);
|
|
56
|
+
}
|
|
57
|
+
if (result.metrics) {
|
|
58
|
+
sessionLog.info(`[Loop] Iteration #${iteration} complete — ` +
|
|
59
|
+
`cost: $${result.metrics.total_cost_usd.toFixed(4)}, ` +
|
|
60
|
+
`duration: ${result.metrics.duration_ms}ms`);
|
|
61
|
+
}
|
|
62
|
+
if (result.success) {
|
|
63
|
+
sessionLog.info(`[Loop] Claude completed iteration #${iteration}`);
|
|
64
|
+
if (queueMessage.teamvibe.brain?.brainId) {
|
|
65
|
+
await pushBrainChanges(kbPath, queueMessage.teamvibe.brain.brainId, queueMessage.teamvibe.workspaceId);
|
|
66
|
+
}
|
|
67
|
+
await postMaintenanceReports(kbPath, queueMessage.teamvibe.channelId);
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
sessionLog.error(`[Loop] Claude failed iteration #${iteration}: ${result.error}`);
|
|
71
|
+
if (hasSlackContext) {
|
|
72
|
+
const errorMessage = result.error || `Process exited with code ${result.exitCode}`;
|
|
73
|
+
const isTimeout = errorMessage.includes('timed out');
|
|
74
|
+
await sendSlackError(queueMessage, errorMessage);
|
|
75
|
+
await addReaction(queueMessage, isTimeout ? 'hourglass' : 'x');
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
typing?.stop();
|
|
81
|
+
sessionLog.error(`[Loop] Error in iteration #${iteration}: ${error instanceof Error ? error.message : error}`);
|
|
82
|
+
}
|
|
83
|
+
// Safety net: ensure the processing file is cleaned up
|
|
84
|
+
markProcessed(kbPath, message.filename);
|
|
85
|
+
if (!hasUnprocessed(kbPath)) {
|
|
86
|
+
break;
|
|
87
|
+
}
|
|
88
|
+
sessionLog.info(`[Loop] More messages found, continuing...`);
|
|
56
89
|
}
|
|
57
|
-
|
|
58
|
-
|
|
90
|
+
if (lockToken) {
|
|
91
|
+
const lastMessageTs = queueMessage.response_context.slack?.message_ts;
|
|
92
|
+
await releaseSessionLock(threadId, lockToken, 'idle', lastMessageTs);
|
|
59
93
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
const signals = threadCompletionSignals.get(threadId);
|
|
65
|
-
const waitingCount = signals?.length || 0;
|
|
66
|
-
if (signals && signals.length > 0) {
|
|
67
|
-
logger.info(`[Queue] Thread ${threadId} completed, waking ${waitingCount} waiting message(s)`);
|
|
68
|
-
signals.forEach((resolve) => resolve());
|
|
69
|
-
threadCompletionSignals.delete(threadId);
|
|
70
|
-
waitingCountByThread.delete(threadId);
|
|
71
|
-
logQueueState('After signal');
|
|
94
|
+
sessionLog.info(`[Loop] Complete. Iterations: ${iteration}`);
|
|
95
|
+
}
|
|
96
|
+
finally {
|
|
97
|
+
activeLoops.delete(threadId);
|
|
72
98
|
}
|
|
73
|
-
}
|
|
74
|
-
function startHeartbeat(receiptHandle, sessionLog) {
|
|
75
|
-
return setInterval(async () => {
|
|
76
|
-
try {
|
|
77
|
-
await extendVisibility(receiptHandle, config.VISIBILITY_TIMEOUT_SECONDS);
|
|
78
|
-
sessionLog.info(`Heartbeat: extended visibility by ${config.VISIBILITY_TIMEOUT_SECONDS}s`);
|
|
79
|
-
}
|
|
80
|
-
catch (error) {
|
|
81
|
-
sessionLog.error(`Heartbeat failed: ${error instanceof Error ? error.message : error}`);
|
|
82
|
-
}
|
|
83
|
-
}, config.HEARTBEAT_INTERVAL_MS);
|
|
84
99
|
}
|
|
85
100
|
async function processMessage(received) {
|
|
86
101
|
const { queueMessage, receiptHandle, messageId } = received;
|
|
@@ -126,99 +141,21 @@ async function processMessage(received) {
|
|
|
126
141
|
}
|
|
127
142
|
// Register brain for heartbeat tracking
|
|
128
143
|
registerBrain(queueMessage);
|
|
129
|
-
//
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
const waitStart = Date.now();
|
|
138
|
-
while (Date.now() - waitStart < maxWaitMs) {
|
|
139
|
-
const waiter = waitForThreadCompletion(threadId);
|
|
140
|
-
// Wait for either: completion signal OR retry interval timeout
|
|
141
|
-
await Promise.race([
|
|
142
|
-
waiter.promise,
|
|
143
|
-
sleep(retryIntervalMs),
|
|
144
|
-
]);
|
|
145
|
-
// Clean up the waiter (handles the timeout case where signal didn't fire)
|
|
146
|
-
waiter.cancel();
|
|
147
|
-
lockResult = await acquireSessionLock(threadId, kbPath);
|
|
148
|
-
if (lockResult.success)
|
|
149
|
-
break;
|
|
150
|
-
sessionLog.info(`Lock still held for ${threadId}, retrying in ${retryIntervalMs / 1000}s...`);
|
|
151
|
-
}
|
|
152
|
-
if (!lockResult.success) {
|
|
153
|
-
sessionLog.error(`Gave up waiting for lock on ${threadId} after ${maxWaitMs / 1000}s`);
|
|
154
|
-
await deleteMessage(receiptHandle);
|
|
155
|
-
return;
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
finally {
|
|
159
|
-
clearInterval(waitHeartbeat);
|
|
160
|
-
}
|
|
144
|
+
// Build prompt and write to mailbox
|
|
145
|
+
const prompt = buildPrompt(queueMessage);
|
|
146
|
+
writeMessage(kbPath, prompt);
|
|
147
|
+
sessionLog.info(`Wrote message to mailbox (${prompt.length} chars)`);
|
|
148
|
+
// Delete SQS message immediately — it's now persisted in the mailbox
|
|
149
|
+
if (receiptHandle) {
|
|
150
|
+
await deleteMessage(receiptHandle);
|
|
151
|
+
sessionLog.info('SQS message deleted (persisted in mailbox)');
|
|
161
152
|
}
|
|
162
|
-
const { session, lockToken } = lockResult;
|
|
163
|
-
const isFirstMessage = session.message_count === 1;
|
|
164
|
-
sessionLog.info(`Acquired lock on session ${threadId}, Claude session: ${session.session_id}, mode: ${isFirstMessage ? 'new' : 'resume'}, message #${session.message_count}`);
|
|
165
|
-
logQueueState(`Lock acquired for ${threadId}`);
|
|
166
153
|
processingMessages.add(messageId);
|
|
167
|
-
const heartbeat = startHeartbeat(receiptHandle, sessionLog);
|
|
168
|
-
// Start typing indicator
|
|
169
|
-
const typing = hasSlackContext
|
|
170
|
-
? startTypingIndicator(queueMessage)
|
|
171
|
-
: undefined;
|
|
172
154
|
try {
|
|
173
|
-
|
|
174
|
-
await ensureBaseBrain();
|
|
175
|
-
const result = await spawnClaudeCode(queueMessage, sessionLog, kbPath, session.session_id || undefined, isFirstMessage, session.last_message_ts, () => typing?.stop(), typing);
|
|
176
|
-
typing?.stop();
|
|
177
|
-
// If Claude generated a new session ID (stale lock recovery), persist it
|
|
178
|
-
if (result.newSessionId && lockToken) {
|
|
179
|
-
await updateSessionId(threadId, lockToken, result.newSessionId);
|
|
180
|
-
}
|
|
181
|
-
if (result.success) {
|
|
182
|
-
sessionLog.info('Claude Code completed successfully');
|
|
183
|
-
// Push any changes in the channel brain repo
|
|
184
|
-
if (queueMessage.teamvibe.brain?.brainId) {
|
|
185
|
-
await pushBrainChanges(kbPath, queueMessage.teamvibe.brain.brainId, queueMessage.teamvibe.workspaceId);
|
|
186
|
-
}
|
|
187
|
-
// Post any pending maintenance reports (fire-and-forget)
|
|
188
|
-
await postMaintenanceReports(kbPath, queueMessage.teamvibe.channelId);
|
|
189
|
-
if (lockToken) {
|
|
190
|
-
const lastMessageTs = queueMessage.response_context.slack?.message_ts;
|
|
191
|
-
await releaseSessionLock(threadId, lockToken, 'idle', lastMessageTs);
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
else {
|
|
195
|
-
sessionLog.error(`Claude Code failed: ${result.error}`);
|
|
196
|
-
if (hasSlackContext) {
|
|
197
|
-
const errorMessage = result.error || `Process exited with code ${result.exitCode}`;
|
|
198
|
-
const isTimeout = errorMessage.includes('timed out');
|
|
199
|
-
await sendSlackError(queueMessage, errorMessage);
|
|
200
|
-
await addReaction(queueMessage, isTimeout ? 'hourglass' : 'x');
|
|
201
|
-
}
|
|
202
|
-
if (lockToken) {
|
|
203
|
-
await releaseSessionLock(threadId, lockToken, 'idle');
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
if (receiptHandle) {
|
|
207
|
-
await deleteMessage(receiptHandle);
|
|
208
|
-
}
|
|
209
|
-
sessionLog.info('Message processed and deleted');
|
|
155
|
+
await runMailboxLoop(threadId, kbPath, queueMessage, sessionLog);
|
|
210
156
|
}
|
|
211
157
|
catch (error) {
|
|
212
|
-
|
|
213
|
-
sessionLog.error(`Error processing message: ${error instanceof Error ? error.message : error}`);
|
|
214
|
-
if (lockToken) {
|
|
215
|
-
try {
|
|
216
|
-
await releaseSessionLock(threadId, lockToken, 'idle');
|
|
217
|
-
}
|
|
218
|
-
catch (releaseError) {
|
|
219
|
-
sessionLog.error(`Failed to release lock: ${releaseError}`);
|
|
220
|
-
}
|
|
221
|
-
}
|
|
158
|
+
sessionLog.error(`Error in mailbox loop: ${error instanceof Error ? error.message : error}`);
|
|
222
159
|
if (hasSlackContext) {
|
|
223
160
|
try {
|
|
224
161
|
const errMsg = error instanceof Error ? error.message : 'Unknown error';
|
|
@@ -238,9 +175,7 @@ async function processMessage(received) {
|
|
|
238
175
|
throw error;
|
|
239
176
|
}
|
|
240
177
|
finally {
|
|
241
|
-
clearInterval(heartbeat);
|
|
242
178
|
processingMessages.delete(messageId);
|
|
243
|
-
signalThreadCompletion(threadId);
|
|
244
179
|
}
|
|
245
180
|
}
|
|
246
181
|
function deduplicateMessages(messages) {
|
|
@@ -278,7 +213,6 @@ async function pollLoop() {
|
|
|
278
213
|
continue;
|
|
279
214
|
}
|
|
280
215
|
logger.info(`Received ${messages.length} message(s) from SQS`);
|
|
281
|
-
logQueueState('After SQS poll');
|
|
282
216
|
// Deduplicate: if multiple messages for the same thread, keep only the latest
|
|
283
217
|
const { unique, duplicates } = deduplicateMessages(messages);
|
|
284
218
|
for (const dup of duplicates) {
|
|
@@ -312,13 +246,7 @@ async function shutdown(signal) {
|
|
|
312
246
|
logger.info(`${signal} received, shutting down gracefully...`);
|
|
313
247
|
stopRefresh();
|
|
314
248
|
stopHeartbeatLoop();
|
|
315
|
-
|
|
316
|
-
for (const [threadId, signals] of threadCompletionSignals.entries()) {
|
|
317
|
-
logger.info(`Unblocking ${signals.length} waiting message(s) on thread ${threadId}`);
|
|
318
|
-
signals.forEach((resolve) => resolve());
|
|
319
|
-
}
|
|
320
|
-
threadCompletionSignals.clear();
|
|
321
|
-
waitingCountByThread.clear();
|
|
249
|
+
stopEventQueue();
|
|
322
250
|
// Kill all active Claude child processes
|
|
323
251
|
await killAllActiveProcesses();
|
|
324
252
|
// Wait for processingMessages to drain (with timeout)
|
|
@@ -332,6 +260,8 @@ async function shutdown(signal) {
|
|
|
332
260
|
logger.info(`Waiting for ${processingMessages.size} message(s) to complete...`);
|
|
333
261
|
await sleep(1000);
|
|
334
262
|
}
|
|
263
|
+
// Final flush after all processes stopped — captures errors emitted during shutdown
|
|
264
|
+
await flushEventQueue();
|
|
335
265
|
logger.info('Shutdown complete');
|
|
336
266
|
process.exit(0);
|
|
337
267
|
}
|
|
@@ -349,24 +279,9 @@ export async function startPoller() {
|
|
|
349
279
|
logger.info(` Queue: ${config.SQS_QUEUE_URL}`);
|
|
350
280
|
logger.info(` Sessions table: ${config.SESSIONS_TABLE}`);
|
|
351
281
|
}
|
|
282
|
+
startEventQueue();
|
|
352
283
|
await ensureDirectories();
|
|
353
284
|
await ensureBaseBrain();
|
|
354
285
|
startHeartbeatLoop(processMessage);
|
|
355
|
-
|
|
356
|
-
const healthCheckInterval = setInterval(() => {
|
|
357
|
-
const stats = getQueueStats();
|
|
358
|
-
if (stats.totalWaiting > 0) {
|
|
359
|
-
logger.warn(`[Queue Health] ${stats.totalWaiting} message(s) waiting across ${stats.threadsWithWaiting} thread(s), ` +
|
|
360
|
-
`${stats.processing} actively processing`);
|
|
361
|
-
for (const [threadId, count] of waitingCountByThread.entries()) {
|
|
362
|
-
logger.warn(`[Queue Health] Thread ${threadId}: ${count} waiting`);
|
|
363
|
-
}
|
|
364
|
-
}
|
|
365
|
-
}, 60000);
|
|
366
|
-
try {
|
|
367
|
-
await pollLoop();
|
|
368
|
-
}
|
|
369
|
-
finally {
|
|
370
|
-
clearInterval(healthCheckInterval);
|
|
371
|
-
}
|
|
286
|
+
await pollLoop();
|
|
372
287
|
}
|
package/dist/session-store.js
CHANGED
|
@@ -4,6 +4,7 @@ import { randomUUID } from 'crypto';
|
|
|
4
4
|
import { config } from './config.js';
|
|
5
5
|
import { logger } from './logger.js';
|
|
6
6
|
import { isTokenMode, getCredentials, getAuthConfig } from './auth-provider.js';
|
|
7
|
+
import { emitError } from './event-queue.js';
|
|
7
8
|
function createDocClient() {
|
|
8
9
|
if (isTokenMode()) {
|
|
9
10
|
const creds = getCredentials();
|
|
@@ -94,6 +95,7 @@ export async function acquireSessionLock(threadId, workspacePath) {
|
|
|
94
95
|
const staleLockTimeout = config.STALE_LOCK_TIMEOUT_MS;
|
|
95
96
|
if (lockAge > staleLockTimeout) {
|
|
96
97
|
logger.warn(`Session ${threadId} has stale lock (age: ${Math.round(lockAge / 1000)}s), attempting recovery...`);
|
|
98
|
+
emitError('session', `Stale lock detected on session ${threadId} (age: ${Math.round(lockAge / 1000)}s)`, { severity: 'warn' });
|
|
97
99
|
const released = await forceReleaseStalelock(threadId, staleLockTimeout);
|
|
98
100
|
if (released) {
|
|
99
101
|
logger.info(`Stale lock released for ${threadId}, retrying acquisition`);
|
|
@@ -164,6 +166,7 @@ export async function updateSessionId(threadId, lockToken, newSessionId) {
|
|
|
164
166
|
catch (error) {
|
|
165
167
|
if (error.name === 'ConditionalCheckFailedException') {
|
|
166
168
|
logger.warn(`Lock token mismatch when updating session_id for ${threadId}`);
|
|
169
|
+
emitError('session', `Lock token mismatch when updating session_id for ${threadId}`, { severity: 'warn' });
|
|
167
170
|
return;
|
|
168
171
|
}
|
|
169
172
|
throw error;
|
|
@@ -201,6 +204,7 @@ export async function releaseSessionLock(threadId, lockToken, newStatus = 'idle'
|
|
|
201
204
|
catch (error) {
|
|
202
205
|
if (error.name === 'ConditionalCheckFailedException') {
|
|
203
206
|
logger.warn(`Lock token mismatch when releasing session ${threadId}`);
|
|
207
|
+
emitError('session', `Lock token mismatch when releasing session ${threadId}`, { severity: 'warn' });
|
|
204
208
|
return;
|
|
205
209
|
}
|
|
206
210
|
throw error;
|
|
@@ -232,6 +236,7 @@ export async function forceReleaseStalelock(threadId, maxAgeMs) {
|
|
|
232
236
|
},
|
|
233
237
|
}));
|
|
234
238
|
logger.warn(`Force released stale lock on session ${threadId} (age: ${lockAge}ms)`);
|
|
239
|
+
emitError('session', `Force released stale lock on session ${threadId}`, { severity: 'warn' });
|
|
235
240
|
return true;
|
|
236
241
|
}
|
|
237
242
|
catch (error) {
|
package/dist/slack-client.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { WebClient } from '@slack/web-api';
|
|
2
2
|
import { logger } from './logger.js';
|
|
3
|
+
import { emitError } from './event-queue.js';
|
|
3
4
|
// Cache user info to avoid repeated API calls
|
|
4
5
|
const userCache = new Map();
|
|
5
6
|
// Per-token Slack client cache
|
|
@@ -34,6 +35,7 @@ export async function getUserInfo(botToken, userId) {
|
|
|
34
35
|
}
|
|
35
36
|
else {
|
|
36
37
|
logger.warn(`Failed to fetch user info for ${userId}:`, error?.data?.error || error);
|
|
38
|
+
emitError('slack', `Failed to fetch user info for ${userId}`, { severity: 'warn' });
|
|
37
39
|
}
|
|
38
40
|
}
|
|
39
41
|
const fallback = { name: userId, realName: userId };
|
|
@@ -84,6 +86,7 @@ export async function addReaction(msg, emoji) {
|
|
|
84
86
|
}
|
|
85
87
|
catch (error) {
|
|
86
88
|
logger.warn(`Failed to add reaction ${emoji}:`, error);
|
|
89
|
+
emitError('slack', `Failed to add reaction ${emoji}`, { severity: 'warn' });
|
|
87
90
|
}
|
|
88
91
|
}
|
|
89
92
|
export async function setThreadStatus(msg, status) {
|
|
@@ -100,6 +103,7 @@ export async function setThreadStatus(msg, status) {
|
|
|
100
103
|
}
|
|
101
104
|
catch (error) {
|
|
102
105
|
logger.warn(`Failed to set thread status: ${error}`);
|
|
106
|
+
emitError('slack', `Failed to set thread status`, { severity: 'warn' });
|
|
103
107
|
}
|
|
104
108
|
}
|
|
105
109
|
export function startTypingIndicator(msg, statusText = 'is thinking...') {
|
package/dist/sqs-poller.js
CHANGED
|
@@ -2,6 +2,7 @@ import { SQSClient, ReceiveMessageCommand, DeleteMessageCommand, ChangeMessageVi
|
|
|
2
2
|
import { config } from './config.js';
|
|
3
3
|
import { logger } from './logger.js';
|
|
4
4
|
import { isTokenMode, getCredentials, getAuthConfig } from './auth-provider.js';
|
|
5
|
+
import { emitError } from './event-queue.js';
|
|
5
6
|
function createSqsClient() {
|
|
6
7
|
if (isTokenMode()) {
|
|
7
8
|
const creds = getCredentials();
|
|
@@ -42,12 +43,14 @@ export async function pollMessages(maxMessages = 10) {
|
|
|
42
43
|
}
|
|
43
44
|
catch (error) {
|
|
44
45
|
logger.error('Error polling SQS:', error);
|
|
46
|
+
emitError('sqs', `Error polling SQS: ${error instanceof Error ? error.message : String(error)}`);
|
|
45
47
|
throw error;
|
|
46
48
|
}
|
|
47
49
|
}
|
|
48
50
|
function parseMessage(msg) {
|
|
49
51
|
if (!msg.Body || !msg.ReceiptHandle || !msg.MessageId) {
|
|
50
52
|
logger.error('Invalid SQS message - missing required fields');
|
|
53
|
+
emitError('sqs', 'Invalid SQS message - missing required fields');
|
|
51
54
|
return null;
|
|
52
55
|
}
|
|
53
56
|
try {
|
|
@@ -60,6 +63,7 @@ function parseMessage(msg) {
|
|
|
60
63
|
}
|
|
61
64
|
catch (error) {
|
|
62
65
|
logger.error('Failed to parse message body:', error);
|
|
66
|
+
emitError('sqs', `Failed to parse message body: ${error instanceof Error ? error.message : String(error)}`);
|
|
63
67
|
return null;
|
|
64
68
|
}
|
|
65
69
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -67,3 +67,14 @@ export interface SessionRecord {
|
|
|
67
67
|
ttl: number;
|
|
68
68
|
}
|
|
69
69
|
export type SessionStatus = 'processing' | 'idle' | 'waiting_for_input';
|
|
70
|
+
export interface SessionMetrics {
|
|
71
|
+
duration_ms: number;
|
|
72
|
+
duration_api_ms: number;
|
|
73
|
+
num_turns: number;
|
|
74
|
+
total_cost_usd: number;
|
|
75
|
+
input_tokens: number;
|
|
76
|
+
output_tokens: number;
|
|
77
|
+
cache_read_input_tokens: number;
|
|
78
|
+
cache_creation_input_tokens: number;
|
|
79
|
+
model: string;
|
|
80
|
+
}
|