@teamvibe/poller 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-provider.d.ts +5 -0
- package/dist/auth-provider.js +19 -0
- package/dist/config.js +1 -1
- package/dist/heartbeat-manager.js +22 -0
- package/dist/metrics/collector.d.ts +28 -0
- package/dist/metrics/collector.js +214 -0
- package/dist/metrics/scheduler.d.ts +52 -0
- package/dist/metrics/scheduler.js +127 -0
- package/dist/metrics/sender.d.ts +10 -0
- package/dist/metrics/sender.js +58 -0
- package/dist/poller.js +41 -2
- package/dist/sqs-poller.js +26 -1
- package/package.json +1 -1
package/dist/auth-provider.d.ts
CHANGED
|
@@ -14,5 +14,10 @@ export declare function getCredentials(): AuthCredentials;
|
|
|
14
14
|
export declare function getAuthConfig(): AuthConfig;
|
|
15
15
|
export declare function getPollerId(): string | null;
|
|
16
16
|
export declare function isTokenMode(): boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Force an immediate credential refresh. Used when SQS returns ExpiredTokenException.
|
|
19
|
+
* Throws if refresh fails (caller should handle).
|
|
20
|
+
*/
|
|
21
|
+
export declare function forceRefresh(): Promise<void>;
|
|
17
22
|
export declare function stopRefresh(): void;
|
|
18
23
|
export {};
|
package/dist/auth-provider.js
CHANGED
|
@@ -6,6 +6,9 @@ let currentConfig = null;
|
|
|
6
6
|
let currentPollerId = null;
|
|
7
7
|
let refreshTimer = null;
|
|
8
8
|
let heartbeatTimer = null;
|
|
9
|
+
// Stored for on-demand refresh
|
|
10
|
+
let storedApiUrl = null;
|
|
11
|
+
let storedToken = null;
|
|
9
12
|
const REFRESH_INTERVAL_MS = 50 * 60 * 1000; // 50 minutes (before 1hr expiry)
|
|
10
13
|
const HEARTBEAT_INTERVAL_MS = 60 * 1000; // 1 minute
|
|
11
14
|
async function sendHeartbeat(apiUrl, token, version) {
|
|
@@ -44,6 +47,8 @@ async function fetchCredentials(apiUrl, token) {
|
|
|
44
47
|
}
|
|
45
48
|
export async function initAuth(apiUrl, token) {
|
|
46
49
|
logger.info('Authenticating with TeamVibe API...');
|
|
50
|
+
storedApiUrl = apiUrl;
|
|
51
|
+
storedToken = token;
|
|
47
52
|
const auth = await fetchCredentials(apiUrl, token);
|
|
48
53
|
currentCredentials = auth.credentials;
|
|
49
54
|
currentConfig = auth.config;
|
|
@@ -87,6 +92,20 @@ export function getPollerId() {
|
|
|
87
92
|
export function isTokenMode() {
|
|
88
93
|
return currentCredentials !== null;
|
|
89
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* Force an immediate credential refresh. Used when SQS returns ExpiredTokenException.
|
|
97
|
+
* Throws if refresh fails (caller should handle).
|
|
98
|
+
*/
|
|
99
|
+
export async function forceRefresh() {
|
|
100
|
+
if (!storedApiUrl || !storedToken) {
|
|
101
|
+
throw new Error('Cannot force refresh: auth not initialized (no stored API URL/token)');
|
|
102
|
+
}
|
|
103
|
+
logger.info('Force-refreshing credentials (token expired)...');
|
|
104
|
+
const auth = await fetchCredentials(storedApiUrl, storedToken);
|
|
105
|
+
currentCredentials = auth.credentials;
|
|
106
|
+
currentConfig = auth.config;
|
|
107
|
+
logger.info('Credentials force-refreshed successfully');
|
|
108
|
+
}
|
|
90
109
|
export function stopRefresh() {
|
|
91
110
|
if (refreshTimer) {
|
|
92
111
|
clearInterval(refreshTimer);
|
package/dist/config.js
CHANGED
|
@@ -44,7 +44,7 @@ const configSchema = z.object({
|
|
|
44
44
|
.transform((v) => v.toLowerCase() === 'true'),
|
|
45
45
|
BRAIN_UPDATE_COOLDOWN_MS: z.coerce.number().default(300000), // 5 minutes
|
|
46
46
|
// Brain heartbeat & maintenance (cron-based)
|
|
47
|
-
HEARTBEAT_CRON: z.string().default('*/
|
|
47
|
+
HEARTBEAT_CRON: z.string().default('0 */6 * * *'), // Every 6 hours
|
|
48
48
|
MAINTENANCE_CRON: z.string().default('30 3 * * *'), // Daily at 03:30 UTC
|
|
49
49
|
MAINTENANCE_ENABLED: z.string().default('true').transform((v) => v.toLowerCase() === 'true'),
|
|
50
50
|
// Auto-update settings
|
|
@@ -3,6 +3,10 @@ import { config } from './config.js';
|
|
|
3
3
|
import { logger } from './logger.js';
|
|
4
4
|
import { isAtCapacity } from './claude-spawner.js';
|
|
5
5
|
import { emitError } from './event-queue.js';
|
|
6
|
+
import { registerBrainForMetrics } from './metrics/scheduler.js';
|
|
7
|
+
import { collectMetrics } from './metrics/collector.js';
|
|
8
|
+
import { sendMetrics } from './metrics/sender.js';
|
|
9
|
+
import { join } from 'path';
|
|
6
10
|
const trackedBrains = new Map();
|
|
7
11
|
let heartbeatJob = null;
|
|
8
12
|
let maintenanceJob = null;
|
|
@@ -17,6 +21,7 @@ export function registerBrain(msg) {
|
|
|
17
21
|
// Don't register from heartbeat/maintenance messages (avoid self-referential loop)
|
|
18
22
|
if (msg.source === 'heartbeat' || msg.source === 'maintenance')
|
|
19
23
|
return;
|
|
24
|
+
const isNew = !trackedBrains.has(brain.brainId);
|
|
20
25
|
trackedBrains.set(brain.brainId, {
|
|
21
26
|
brainId: brain.brainId,
|
|
22
27
|
gitRepoUrl: brain.gitRepoUrl,
|
|
@@ -27,6 +32,23 @@ export function registerBrain(msg) {
|
|
|
27
32
|
channelId: msg.teamvibe.channelId,
|
|
28
33
|
});
|
|
29
34
|
logger.debug(`[Heartbeat] Registered brain: ${brain.brainId}`);
|
|
35
|
+
// Metrics pipeline (teamvibe.ai#110): register for hourly collection. On the
|
|
36
|
+
// first sighting of this brain, fire one immediate collect+send so we don't
|
|
37
|
+
// wait up to an hour for the first sample.
|
|
38
|
+
registerBrainForMetrics(brain.brainId);
|
|
39
|
+
if (isNew) {
|
|
40
|
+
const brainId = brain.brainId;
|
|
41
|
+
void (async () => {
|
|
42
|
+
try {
|
|
43
|
+
const brainPath = join(config.BRAINS_PATH, brainId);
|
|
44
|
+
const payload = await collectMetrics(brainPath, brainId);
|
|
45
|
+
await sendMetrics(payload);
|
|
46
|
+
}
|
|
47
|
+
catch (err) {
|
|
48
|
+
logger.warn(`[Metrics] Initial send failed for ${brainId}: ${err instanceof Error ? err.message : err}`);
|
|
49
|
+
}
|
|
50
|
+
})();
|
|
51
|
+
}
|
|
30
52
|
}
|
|
31
53
|
/**
|
|
32
54
|
* Send a synthetic message to all tracked brains.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v1 metrics payload — see teamvibeai/teamvibe.ai#110 (revised post-review 2026-05-21).
|
|
3
|
+
*
|
|
4
|
+
* All measures are OPTIONAL: a missing field means "this measure could not be
|
|
5
|
+
* collected" (file absent, glob failed, command unavailable, etc.). Timestream
|
|
6
|
+
* handles sparse measures natively — never send a zero in place of a real omission.
|
|
7
|
+
*/
|
|
8
|
+
export interface MetricsPayload {
|
|
9
|
+
brainId: string;
|
|
10
|
+
todayMdBytes?: number;
|
|
11
|
+
todayMdLines?: number;
|
|
12
|
+
summaryMdBytes?: number;
|
|
13
|
+
summaryMdLines?: number;
|
|
14
|
+
brainDirBytes?: number;
|
|
15
|
+
vmDiskFreeBytes?: number;
|
|
16
|
+
vmDiskUsedBytes?: number;
|
|
17
|
+
inboxUnreadFiles?: number;
|
|
18
|
+
inboxTotalFiles?: number;
|
|
19
|
+
oldestInboxMessageAgeSeconds?: number;
|
|
20
|
+
uncommittedChangeCount?: number;
|
|
21
|
+
daysSinceMaintenance?: number;
|
|
22
|
+
timestamp?: string;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Collect all v1 metrics for a brain. Never throws — individual collector
|
|
26
|
+
* failures cause the corresponding measure to be omitted from the result.
|
|
27
|
+
*/
|
|
28
|
+
export declare function collectMetrics(brainPath: string, brainId: string): Promise<MetricsPayload>;
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { stat, readdir, readFile } from 'fs/promises';
|
|
2
|
+
import { statfs } from 'fs';
|
|
3
|
+
import { promisify } from 'util';
|
|
4
|
+
import { execFile } from 'child_process';
|
|
5
|
+
import { join } from 'path';
|
|
6
|
+
const statfsAsync = promisify(statfs);
|
|
7
|
+
const execFileAsync = promisify(execFile);
|
|
8
|
+
const MS_PER_DAY = 86_400_000;
|
|
9
|
+
/**
|
|
10
|
+
* Recursively sum file sizes under a directory.
|
|
11
|
+
* Skips entries we cannot stat (permission errors, broken symlinks).
|
|
12
|
+
*/
|
|
13
|
+
async function dirSizeBytes(path) {
|
|
14
|
+
let total = 0;
|
|
15
|
+
let entries;
|
|
16
|
+
try {
|
|
17
|
+
entries = await readdir(path, { withFileTypes: true });
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return 0;
|
|
21
|
+
}
|
|
22
|
+
for (const entry of entries) {
|
|
23
|
+
const full = join(path, entry.name);
|
|
24
|
+
try {
|
|
25
|
+
if (entry.isDirectory()) {
|
|
26
|
+
total += await dirSizeBytes(full);
|
|
27
|
+
}
|
|
28
|
+
else if (entry.isFile()) {
|
|
29
|
+
const s = await stat(full);
|
|
30
|
+
total += s.size;
|
|
31
|
+
}
|
|
32
|
+
// Skip symlinks/sockets/etc.
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
// Single broken entry shouldn't tank the whole walk.
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return total;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Count newline characters in a file. Returns line count using newline-count
|
|
42
|
+
* semantics (a trailing newline = N lines, no trailing newline = N-1 lines —
|
|
43
|
+
* matches what `wc -l` would have reported).
|
|
44
|
+
*/
|
|
45
|
+
async function countLines(path) {
|
|
46
|
+
const content = await readFile(path, 'utf-8');
|
|
47
|
+
let count = 0;
|
|
48
|
+
for (let i = 0; i < content.length; i++) {
|
|
49
|
+
if (content.charCodeAt(i) === 10 /* \n */)
|
|
50
|
+
count++;
|
|
51
|
+
}
|
|
52
|
+
return count;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Find files matching .inbox/<thread>/new/* under brainPath.
|
|
56
|
+
* Returns absolute paths. Throws if the .inbox directory itself is unreadable —
|
|
57
|
+
* caller wraps in try/catch and omits the measure on any failure.
|
|
58
|
+
*/
|
|
59
|
+
async function listInboxUnreadFiles(brainPath) {
|
|
60
|
+
const inboxRoot = join(brainPath, '.inbox');
|
|
61
|
+
const threads = (await readdir(inboxRoot, { withFileTypes: true }));
|
|
62
|
+
const files = [];
|
|
63
|
+
for (const t of threads) {
|
|
64
|
+
if (!t.isDirectory())
|
|
65
|
+
continue;
|
|
66
|
+
const newDir = join(inboxRoot, t.name, 'new');
|
|
67
|
+
let entries;
|
|
68
|
+
try {
|
|
69
|
+
entries = (await readdir(newDir, { withFileTypes: true }));
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
for (const e of entries) {
|
|
75
|
+
if (e.isFile())
|
|
76
|
+
files.push(join(newDir, e.name));
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return files;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Count regular files anywhere under brainPath/.inbox.
|
|
83
|
+
*/
|
|
84
|
+
async function countInboxTotalFiles(brainPath) {
|
|
85
|
+
const inboxRoot = join(brainPath, '.inbox');
|
|
86
|
+
let count = 0;
|
|
87
|
+
async function walk(dir) {
|
|
88
|
+
let entries;
|
|
89
|
+
try {
|
|
90
|
+
entries = (await readdir(dir, { withFileTypes: true }));
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
for (const e of entries) {
|
|
96
|
+
const full = join(dir, e.name);
|
|
97
|
+
if (e.isDirectory()) {
|
|
98
|
+
await walk(full);
|
|
99
|
+
}
|
|
100
|
+
else if (e.isFile()) {
|
|
101
|
+
count++;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
await walk(inboxRoot);
|
|
106
|
+
return count;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Collect all v1 metrics for a brain. Never throws — individual collector
|
|
110
|
+
* failures cause the corresponding measure to be omitted from the result.
|
|
111
|
+
*/
|
|
112
|
+
export async function collectMetrics(brainPath, brainId) {
|
|
113
|
+
const payload = {
|
|
114
|
+
brainId,
|
|
115
|
+
timestamp: new Date().toISOString(),
|
|
116
|
+
};
|
|
117
|
+
// memory/TODAY.md size + line count
|
|
118
|
+
try {
|
|
119
|
+
const todayPath = join(brainPath, 'memory/TODAY.md');
|
|
120
|
+
const s = await stat(todayPath);
|
|
121
|
+
payload.todayMdBytes = s.size;
|
|
122
|
+
payload.todayMdLines = await countLines(todayPath);
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
// File missing or unreadable — omit both measures.
|
|
126
|
+
}
|
|
127
|
+
// memory/SUMMARY.md size + line count
|
|
128
|
+
try {
|
|
129
|
+
const summaryPath = join(brainPath, 'memory/SUMMARY.md');
|
|
130
|
+
const s = await stat(summaryPath);
|
|
131
|
+
payload.summaryMdBytes = s.size;
|
|
132
|
+
payload.summaryMdLines = await countLines(summaryPath);
|
|
133
|
+
}
|
|
134
|
+
catch {
|
|
135
|
+
// omit
|
|
136
|
+
}
|
|
137
|
+
// brain dir total size
|
|
138
|
+
try {
|
|
139
|
+
payload.brainDirBytes = await dirSizeBytes(brainPath);
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
// omit
|
|
143
|
+
}
|
|
144
|
+
// VM disk usage (root filesystem)
|
|
145
|
+
try {
|
|
146
|
+
const fs = await statfsAsync('/');
|
|
147
|
+
// fs.bsize × fs.bfree = free bytes
|
|
148
|
+
// fs.bsize × (fs.blocks - fs.bfree) = used bytes
|
|
149
|
+
payload.vmDiskFreeBytes = fs.bsize * fs.bfree;
|
|
150
|
+
payload.vmDiskUsedBytes = fs.bsize * (fs.blocks - fs.bfree);
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
// omit
|
|
154
|
+
}
|
|
155
|
+
// days since maintenance — fractional, from .last_consolidation mtime
|
|
156
|
+
try {
|
|
157
|
+
const guardPath = join(brainPath, 'memory/.last_consolidation');
|
|
158
|
+
const s = await stat(guardPath);
|
|
159
|
+
payload.daysSinceMaintenance = (Date.now() - s.mtimeMs) / MS_PER_DAY;
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
// Guard file may legitimately not exist (new brain, never consolidated).
|
|
163
|
+
// Omit rather than send a misleading 0.
|
|
164
|
+
}
|
|
165
|
+
// Inbox: unread files (.inbox/<thread>/new/*)
|
|
166
|
+
try {
|
|
167
|
+
const unread = await listInboxUnreadFiles(brainPath);
|
|
168
|
+
payload.inboxUnreadFiles = unread.length;
|
|
169
|
+
// Oldest unread message age — same source, single walk would be cheaper
|
|
170
|
+
// but we already have the paths and the volume is small.
|
|
171
|
+
if (unread.length === 0) {
|
|
172
|
+
payload.oldestInboxMessageAgeSeconds = 0;
|
|
173
|
+
}
|
|
174
|
+
else {
|
|
175
|
+
let minMtime = Infinity;
|
|
176
|
+
for (const f of unread) {
|
|
177
|
+
try {
|
|
178
|
+
const s = await stat(f);
|
|
179
|
+
if (s.mtimeMs < minMtime)
|
|
180
|
+
minMtime = s.mtimeMs;
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
// skip
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
if (minMtime !== Infinity) {
|
|
187
|
+
payload.oldestInboxMessageAgeSeconds = Math.floor((Date.now() - minMtime) / 1000);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
// Per Jarvis's review: different poller versions may lay out .inbox/ differently.
|
|
193
|
+
// Omit measures rather than fail the batch.
|
|
194
|
+
}
|
|
195
|
+
// Inbox: total files (anywhere under .inbox)
|
|
196
|
+
try {
|
|
197
|
+
payload.inboxTotalFiles = await countInboxTotalFiles(brainPath);
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
// omit
|
|
201
|
+
}
|
|
202
|
+
// Uncommitted change count — git status --porcelain
|
|
203
|
+
try {
|
|
204
|
+
const { stdout } = await execFileAsync('git', ['-C', brainPath, 'status', '--porcelain'], {
|
|
205
|
+
timeout: 5000,
|
|
206
|
+
});
|
|
207
|
+
// Each non-empty line = one tracked change.
|
|
208
|
+
payload.uncommittedChangeCount = stdout.split('\n').filter((l) => l.length > 0).length;
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
// Non-git brain, missing git binary, timeout — omit.
|
|
212
|
+
}
|
|
213
|
+
return payload;
|
|
214
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
interface RegisteredBrain {
|
|
2
|
+
brainId: string;
|
|
3
|
+
}
|
|
4
|
+
/**
|
|
5
|
+
* Register (or update) a brain for hourly metrics collection.
|
|
6
|
+
* Safe to call repeatedly with the same brainId.
|
|
7
|
+
*/
|
|
8
|
+
export declare function registerBrainForMetrics(brainId: string): void;
|
|
9
|
+
/**
|
|
10
|
+
* Deterministic 32-bit FNV-1a hash of a string. Used to derive a stable
|
|
11
|
+
* per-brain jitter offset for hourly cron alignment (see #110 review):
|
|
12
|
+
* at fleet scale (100+ brains) we don't want every brain to fire at exactly
|
|
13
|
+
* UTC :00:00 — Lambda cold-start storm + Timestream write throttle risk.
|
|
14
|
+
*
|
|
15
|
+
* No crypto needed; we only need a stable, well-distributed integer.
|
|
16
|
+
*/
|
|
17
|
+
export declare function hashBrainId(s: string): number;
|
|
18
|
+
/**
|
|
19
|
+
* Per-brain deterministic jitter offset in [0, JITTER_WINDOW_MS).
|
|
20
|
+
* Two brains with different ids should normally land on different offsets;
|
|
21
|
+
* the same brain id always lands on the same offset.
|
|
22
|
+
*/
|
|
23
|
+
export declare function jitterMsFor(brainId: string): number;
|
|
24
|
+
/**
|
|
25
|
+
* Cron handler: fan out to every registered brain on its deterministic
|
|
26
|
+
* per-brain jitter offset (0–10 min after UTC :00). This avoids a thundering
|
|
27
|
+
* herd of metric writes at the exact hour boundary while keeping the
|
|
28
|
+
* per-brain offset stable (= no clock drift across hours).
|
|
29
|
+
*
|
|
30
|
+
* Exported for tests so they can fire the handler manually.
|
|
31
|
+
*/
|
|
32
|
+
export declare function collectAndSendAll(): void;
|
|
33
|
+
/**
|
|
34
|
+
* Start the hourly metrics cron. Fires once per UTC hour boundary; each tick
|
|
35
|
+
* fans out per-brain on a deterministic jitter offset (0–10min).
|
|
36
|
+
*
|
|
37
|
+
* Idempotent: re-invocation while already running is a no-op.
|
|
38
|
+
*/
|
|
39
|
+
export declare function startMetricsCron(): void;
|
|
40
|
+
/**
|
|
41
|
+
* Stop the metrics cron. Safe to call multiple times.
|
|
42
|
+
*/
|
|
43
|
+
export declare function stopMetricsCron(): void;
|
|
44
|
+
/**
|
|
45
|
+
* Test-only: clear registered brain list.
|
|
46
|
+
*/
|
|
47
|
+
export declare function _resetForTests(): void;
|
|
48
|
+
/**
|
|
49
|
+
* Test-only: read registered brains (returns a snapshot copy).
|
|
50
|
+
*/
|
|
51
|
+
export declare function _getRegisteredForTests(): RegisteredBrain[];
|
|
52
|
+
export {};
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { join } from 'path';
|
|
2
|
+
import { Cron } from 'croner';
|
|
3
|
+
import { config } from '../config.js';
|
|
4
|
+
import { logger } from '../logger.js';
|
|
5
|
+
import { collectMetrics } from './collector.js';
|
|
6
|
+
import { sendMetrics } from './sender.js';
|
|
7
|
+
const JITTER_WINDOW_MS = 10 * 60 * 1000; // 10 minutes spread per #110 review
|
|
8
|
+
const registered = new Map();
|
|
9
|
+
const pendingJitterTimers = new Set();
|
|
10
|
+
let metricsJob = null;
|
|
11
|
+
/**
|
|
12
|
+
* Register (or update) a brain for hourly metrics collection.
|
|
13
|
+
* Safe to call repeatedly with the same brainId.
|
|
14
|
+
*/
|
|
15
|
+
export function registerBrainForMetrics(brainId) {
|
|
16
|
+
if (!brainId)
|
|
17
|
+
return;
|
|
18
|
+
registered.set(brainId, { brainId });
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Collect + send metrics for a single brain. Never throws.
|
|
22
|
+
*/
|
|
23
|
+
async function collectAndSendOne(brain) {
|
|
24
|
+
try {
|
|
25
|
+
const brainPath = join(config.BRAINS_PATH, brain.brainId);
|
|
26
|
+
const payload = await collectMetrics(brainPath, brain.brainId);
|
|
27
|
+
await sendMetrics(payload);
|
|
28
|
+
}
|
|
29
|
+
catch (err) {
|
|
30
|
+
// Defensive: collector/sender already swallow their own errors, but if
|
|
31
|
+
// anything escapes (e.g. path resolution, unexpected runtime fault) we
|
|
32
|
+
// log and move on rather than crashing the cron tick.
|
|
33
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
34
|
+
logger.warn(`[Metrics] Unexpected failure for brain ${brain.brainId}: ${msg}`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Deterministic 32-bit FNV-1a hash of a string. Used to derive a stable
|
|
39
|
+
* per-brain jitter offset for hourly cron alignment (see #110 review):
|
|
40
|
+
* at fleet scale (100+ brains) we don't want every brain to fire at exactly
|
|
41
|
+
* UTC :00:00 — Lambda cold-start storm + Timestream write throttle risk.
|
|
42
|
+
*
|
|
43
|
+
* No crypto needed; we only need a stable, well-distributed integer.
|
|
44
|
+
*/
|
|
45
|
+
export function hashBrainId(s) {
|
|
46
|
+
let h = 0x811c9dc5; // FNV offset basis
|
|
47
|
+
for (let i = 0; i < s.length; i++) {
|
|
48
|
+
h ^= s.charCodeAt(i);
|
|
49
|
+
// FNV prime is 0x01000193; use Math.imul for 32-bit overflow semantics.
|
|
50
|
+
h = Math.imul(h, 0x01000193);
|
|
51
|
+
}
|
|
52
|
+
// Coerce to unsigned 32-bit.
|
|
53
|
+
return h >>> 0;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Per-brain deterministic jitter offset in [0, JITTER_WINDOW_MS).
|
|
57
|
+
* Two brains with different ids should normally land on different offsets;
|
|
58
|
+
* the same brain id always lands on the same offset.
|
|
59
|
+
*/
|
|
60
|
+
export function jitterMsFor(brainId) {
|
|
61
|
+
return hashBrainId(brainId) % JITTER_WINDOW_MS;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Cron handler: fan out to every registered brain on its deterministic
|
|
65
|
+
* per-brain jitter offset (0–10 min after UTC :00). This avoids a thundering
|
|
66
|
+
* herd of metric writes at the exact hour boundary while keeping the
|
|
67
|
+
* per-brain offset stable (= no clock drift across hours).
|
|
68
|
+
*
|
|
69
|
+
* Exported for tests so they can fire the handler manually.
|
|
70
|
+
*/
|
|
71
|
+
export function collectAndSendAll() {
|
|
72
|
+
if (registered.size === 0)
|
|
73
|
+
return;
|
|
74
|
+
const brains = Array.from(registered.values());
|
|
75
|
+
for (const brain of brains) {
|
|
76
|
+
const delay = jitterMsFor(brain.brainId);
|
|
77
|
+
const timer = setTimeout(() => {
|
|
78
|
+
pendingJitterTimers.delete(timer);
|
|
79
|
+
void collectAndSendOne(brain);
|
|
80
|
+
}, delay);
|
|
81
|
+
pendingJitterTimers.add(timer);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Start the hourly metrics cron. Fires once per UTC hour boundary; each tick
|
|
86
|
+
* fans out per-brain on a deterministic jitter offset (0–10min).
|
|
87
|
+
*
|
|
88
|
+
* Idempotent: re-invocation while already running is a no-op.
|
|
89
|
+
*/
|
|
90
|
+
export function startMetricsCron() {
|
|
91
|
+
if (metricsJob)
|
|
92
|
+
return;
|
|
93
|
+
if (!config.TEAMVIBE_API_URL || !config.TEAMVIBE_POLLER_TOKEN) {
|
|
94
|
+
// No-op in direct (non-token) mode.
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
logger.info('[Metrics] Starting hourly cron: "0 * * * *" UTC (per-brain jitter 0–10min)');
|
|
98
|
+
metricsJob = new Cron('0 * * * *', { timezone: 'UTC', protect: true }, () => {
|
|
99
|
+
collectAndSendAll();
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Stop the metrics cron. Safe to call multiple times.
|
|
104
|
+
*/
|
|
105
|
+
export function stopMetricsCron() {
|
|
106
|
+
if (metricsJob) {
|
|
107
|
+
metricsJob.stop();
|
|
108
|
+
metricsJob = null;
|
|
109
|
+
}
|
|
110
|
+
for (const timer of pendingJitterTimers) {
|
|
111
|
+
clearTimeout(timer);
|
|
112
|
+
}
|
|
113
|
+
pendingJitterTimers.clear();
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Test-only: clear registered brain list.
|
|
117
|
+
*/
|
|
118
|
+
export function _resetForTests() {
|
|
119
|
+
registered.clear();
|
|
120
|
+
stopMetricsCron();
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Test-only: read registered brains (returns a snapshot copy).
|
|
124
|
+
*/
|
|
125
|
+
export function _getRegisteredForTests() {
|
|
126
|
+
return Array.from(registered.values());
|
|
127
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { MetricsPayload } from './collector.js';
|
|
2
|
+
/**
|
|
3
|
+
* Send a metrics payload to the platform. Fail-safe:
|
|
4
|
+
* - 5s timeout per attempt
|
|
5
|
+
* - 1 retry after 30s on non-2xx or network failure
|
|
6
|
+
* - Never throws
|
|
7
|
+
*
|
|
8
|
+
* Returns true on success, false if both attempts failed (caller can log/ignore).
|
|
9
|
+
*/
|
|
10
|
+
export declare function sendMetrics(payload: MetricsPayload): Promise<boolean>;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { config } from '../config.js';
|
|
2
|
+
import { logger } from '../logger.js';
|
|
3
|
+
const SEND_TIMEOUT_MS = 5_000;
|
|
4
|
+
const RETRY_DELAY_MS = 30_000;
|
|
5
|
+
function sleep(ms) {
|
|
6
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
7
|
+
}
|
|
8
|
+
async function postOnce(payload) {
|
|
9
|
+
const controller = new AbortController();
|
|
10
|
+
const timer = setTimeout(() => controller.abort(), SEND_TIMEOUT_MS);
|
|
11
|
+
try {
|
|
12
|
+
const response = await fetch(`${config.TEAMVIBE_API_URL}/metrics`, {
|
|
13
|
+
method: 'POST',
|
|
14
|
+
headers: {
|
|
15
|
+
'Content-Type': 'application/json',
|
|
16
|
+
Authorization: `Bearer ${config.TEAMVIBE_POLLER_TOKEN}`,
|
|
17
|
+
},
|
|
18
|
+
body: JSON.stringify(payload),
|
|
19
|
+
signal: controller.signal,
|
|
20
|
+
});
|
|
21
|
+
if (!response.ok) {
|
|
22
|
+
logger.warn(`[Metrics] Send failed: ${response.status} for brain ${payload.brainId}`);
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
return true;
|
|
26
|
+
}
|
|
27
|
+
catch (err) {
|
|
28
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
29
|
+
logger.warn(`[Metrics] Send error for brain ${payload.brainId}: ${msg}`);
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
finally {
|
|
33
|
+
clearTimeout(timer);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Send a metrics payload to the platform. Fail-safe:
|
|
38
|
+
* - 5s timeout per attempt
|
|
39
|
+
* - 1 retry after 30s on non-2xx or network failure
|
|
40
|
+
* - Never throws
|
|
41
|
+
*
|
|
42
|
+
* Returns true on success, false if both attempts failed (caller can log/ignore).
|
|
43
|
+
*/
|
|
44
|
+
export async function sendMetrics(payload) {
|
|
45
|
+
if (!config.TEAMVIBE_API_URL || !config.TEAMVIBE_POLLER_TOKEN) {
|
|
46
|
+
// Direct (non-token) mode — metrics endpoint isn't reachable. No-op.
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
const ok = await postOnce(payload);
|
|
50
|
+
if (ok)
|
|
51
|
+
return true;
|
|
52
|
+
await sleep(RETRY_DELAY_MS);
|
|
53
|
+
const retried = await postOnce(payload);
|
|
54
|
+
if (!retried) {
|
|
55
|
+
logger.warn(`[Metrics] Dropping payload for brain ${payload.brainId} after 1 retry`);
|
|
56
|
+
}
|
|
57
|
+
return retried;
|
|
58
|
+
}
|
package/dist/poller.js
CHANGED
|
@@ -9,6 +9,7 @@ import { postMaintenanceReports } from './maintenance-reporter.js';
|
|
|
9
9
|
import { initAuth, stopRefresh } from './auth-provider.js';
|
|
10
10
|
import { registerBrain, startHeartbeatLoop, stopHeartbeatLoop } from './heartbeat-manager.js';
|
|
11
11
|
import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent } from './event-queue.js';
|
|
12
|
+
import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
|
|
12
13
|
import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed } from './inbox-manager.js';
|
|
13
14
|
import { Cron } from 'croner';
|
|
14
15
|
import { isControlMessage, handleControlMessage } from './control-handler.js';
|
|
@@ -338,10 +339,34 @@ function deduplicateMessages(messages) {
|
|
|
338
339
|
}
|
|
339
340
|
return { unique, duplicates };
|
|
340
341
|
}
|
|
342
|
+
// Circuit breaker for consecutive auth/infrastructure failures
|
|
343
|
+
const AUTH_CIRCUIT_BREAKER_THRESHOLD = 3;
|
|
344
|
+
const AUTH_CIRCUIT_BREAKER_MAX_BACKOFF_MS = 60_000;
|
|
345
|
+
let consecutiveAuthFailures = 0;
|
|
346
|
+
function isAuthError(error) {
|
|
347
|
+
if (!(error instanceof Error))
|
|
348
|
+
return false;
|
|
349
|
+
const name = error.name ?? '';
|
|
350
|
+
const code = error.Code ?? '';
|
|
351
|
+
const message = error.message ?? '';
|
|
352
|
+
return (name === 'ExpiredTokenException' ||
|
|
353
|
+
code === 'ExpiredToken' ||
|
|
354
|
+
code === 'InvalidClientTokenId' ||
|
|
355
|
+
code === 'UnrecognizedClientException' ||
|
|
356
|
+
message.includes('security token') ||
|
|
357
|
+
message.includes('Auth failed'));
|
|
358
|
+
}
|
|
341
359
|
async function pollLoop() {
|
|
342
360
|
logger.info('Poll loop started');
|
|
343
361
|
while (!shuttingDown) {
|
|
344
362
|
try {
|
|
363
|
+
// Circuit breaker: if too many consecutive auth failures, back off
|
|
364
|
+
if (consecutiveAuthFailures >= AUTH_CIRCUIT_BREAKER_THRESHOLD) {
|
|
365
|
+
const backoffMs = Math.min(5_000 * Math.pow(2, consecutiveAuthFailures - AUTH_CIRCUIT_BREAKER_THRESHOLD), AUTH_CIRCUIT_BREAKER_MAX_BACKOFF_MS);
|
|
366
|
+
logger.warn(`[CircuitBreaker] Auth degraded (${consecutiveAuthFailures} consecutive failures), ` +
|
|
367
|
+
`backing off ${backoffMs}ms before retry...`);
|
|
368
|
+
await sleep(backoffMs);
|
|
369
|
+
}
|
|
345
370
|
if (isAtCapacity()) {
|
|
346
371
|
logger.debug(`At capacity (${getActiveProcessCount()}/${config.MAX_CONCURRENT_SESSIONS}), waiting...`);
|
|
347
372
|
await sleep(1000);
|
|
@@ -351,6 +376,11 @@ async function pollLoop() {
|
|
|
351
376
|
logger.debug(`Polling for up to ${availableSlots} messages...`);
|
|
352
377
|
const messages = await pollMessages(availableSlots);
|
|
353
378
|
consecutiveNetFailures = 0; // Successful poll — reset circuit breaker
|
|
379
|
+
// Successful poll — reset circuit breaker
|
|
380
|
+
if (consecutiveAuthFailures > 0) {
|
|
381
|
+
logger.info(`[CircuitBreaker] Auth recovered after ${consecutiveAuthFailures} failure(s)`);
|
|
382
|
+
consecutiveAuthFailures = 0;
|
|
383
|
+
}
|
|
354
384
|
if (messages.length === 0) {
|
|
355
385
|
continue;
|
|
356
386
|
}
|
|
@@ -384,7 +414,13 @@ async function pollLoop() {
|
|
|
384
414
|
Promise.all(processPromises);
|
|
385
415
|
}
|
|
386
416
|
catch (error) {
|
|
387
|
-
if (
|
|
417
|
+
if (isAuthError(error)) {
|
|
418
|
+
consecutiveAuthFailures++;
|
|
419
|
+
logger.error(`[CircuitBreaker] Auth failure #${consecutiveAuthFailures}: ${error instanceof Error ? error.message : error}`);
|
|
420
|
+
emitEvent('poller.auth_failure', '', { consecutiveFailures: consecutiveAuthFailures });
|
|
421
|
+
await sleep(5000);
|
|
422
|
+
}
|
|
423
|
+
else if (isNetworkError(error)) {
|
|
388
424
|
consecutiveNetFailures++;
|
|
389
425
|
logger.error(`Network error in poll loop (${consecutiveNetFailures}/${config.MAX_CONSECUTIVE_NET_FAILURES}):`, error);
|
|
390
426
|
if (consecutiveNetFailures === config.MAX_CONSECUTIVE_NET_FAILURES) {
|
|
@@ -408,7 +444,8 @@ async function pollLoop() {
|
|
|
408
444
|
await sleep(backoffMs);
|
|
409
445
|
}
|
|
410
446
|
else {
|
|
411
|
-
// Non-network errors
|
|
447
|
+
// Non-network, non-auth errors — reset auth circuit breaker (infra is reachable)
|
|
448
|
+
consecutiveAuthFailures = 0;
|
|
412
449
|
logger.error('Error in poll loop:', error);
|
|
413
450
|
await sleep(5000);
|
|
414
451
|
}
|
|
@@ -428,6 +465,7 @@ async function shutdown(signal, exitCode = 0) {
|
|
|
428
465
|
stopRefresh();
|
|
429
466
|
stopHeartbeatLoop();
|
|
430
467
|
stopAutoUpdateCron();
|
|
468
|
+
stopMetricsCron();
|
|
431
469
|
stopEventQueue();
|
|
432
470
|
// Kill all active Claude child processes
|
|
433
471
|
await killAllActiveProcesses();
|
|
@@ -466,5 +504,6 @@ export async function startPoller() {
|
|
|
466
504
|
await ensureBaseBrain();
|
|
467
505
|
startHeartbeatLoop(processMessage);
|
|
468
506
|
startAutoUpdateCron();
|
|
507
|
+
startMetricsCron();
|
|
469
508
|
await pollLoop();
|
|
470
509
|
}
|
package/dist/sqs-poller.js
CHANGED
|
@@ -1,8 +1,15 @@
|
|
|
1
1
|
import { SQSClient, ReceiveMessageCommand, DeleteMessageCommand, ChangeMessageVisibilityCommand, SendMessageCommand, } from '@aws-sdk/client-sqs';
|
|
2
2
|
import { config } from './config.js';
|
|
3
3
|
import { logger } from './logger.js';
|
|
4
|
-
import { isTokenMode, getCredentials, getAuthConfig } from './auth-provider.js';
|
|
4
|
+
import { isTokenMode, getCredentials, getAuthConfig, forceRefresh } from './auth-provider.js';
|
|
5
5
|
import { emitError } from './event-queue.js';
|
|
6
|
+
function isExpiredTokenError(error) {
|
|
7
|
+
if (!(error instanceof Error))
|
|
8
|
+
return false;
|
|
9
|
+
const name = error.name ?? '';
|
|
10
|
+
const code = error.Code ?? '';
|
|
11
|
+
return name === 'ExpiredTokenException' || code === 'ExpiredToken';
|
|
12
|
+
}
|
|
6
13
|
function createSqsClient() {
|
|
7
14
|
if (isTokenMode()) {
|
|
8
15
|
const creds = getCredentials();
|
|
@@ -42,6 +49,24 @@ export async function pollMessages(maxMessages = 10) {
|
|
|
42
49
|
return response.Messages.map((msg) => parseMessage(msg)).filter((m) => m !== null);
|
|
43
50
|
}
|
|
44
51
|
catch (error) {
|
|
52
|
+
// Expired AWS STS token — force-refresh credentials and retry once
|
|
53
|
+
if (isTokenMode() && isExpiredTokenError(error)) {
|
|
54
|
+
logger.warn('SQS poll failed with ExpiredToken, attempting credential refresh...');
|
|
55
|
+
await forceRefresh(); // throws if refresh itself fails
|
|
56
|
+
// Retry poll with fresh credentials
|
|
57
|
+
const retryResponse = await createSqsClient().send(new ReceiveMessageCommand({
|
|
58
|
+
QueueUrl: getQueueUrl(),
|
|
59
|
+
MaxNumberOfMessages: Math.min(maxMessages, 10),
|
|
60
|
+
WaitTimeSeconds: config.POLL_WAIT_TIME_SECONDS,
|
|
61
|
+
VisibilityTimeout: config.VISIBILITY_TIMEOUT_SECONDS,
|
|
62
|
+
MessageAttributeNames: ['All'],
|
|
63
|
+
}));
|
|
64
|
+
if (!retryResponse.Messages || retryResponse.Messages.length === 0) {
|
|
65
|
+
return [];
|
|
66
|
+
}
|
|
67
|
+
logger.info(`Received ${retryResponse.Messages.length} message(s) after credential refresh`);
|
|
68
|
+
return retryResponse.Messages.map((msg) => parseMessage(msg)).filter((m) => m !== null);
|
|
69
|
+
}
|
|
45
70
|
logger.error('Error polling SQS:', error);
|
|
46
71
|
emitError('sqs', `Error polling SQS: ${error instanceof Error ? error.message : String(error)}`);
|
|
47
72
|
throw error;
|