@teamvibe/poller 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brain-manager.d.ts +43 -1
- package/dist/brain-manager.js +259 -6
- package/dist/classify-failed-iteration.d.ts +1 -1
- package/dist/classify-failed-iteration.js +33 -1
- package/dist/claude-spawner.d.ts +178 -1
- package/dist/claude-spawner.js +690 -92
- package/dist/config.d.ts +32 -0
- package/dist/config.js +23 -2
- package/dist/cron-telemetry.d.ts +3 -0
- package/dist/cron-telemetry.js +16 -0
- package/dist/event-queue.d.ts +30 -0
- package/dist/event-queue.js +4 -0
- package/dist/inbox-manager.d.ts +25 -0
- package/dist/inbox-manager.js +47 -0
- package/dist/inbox-watcher.d.ts +67 -0
- package/dist/inbox-watcher.js +262 -0
- package/dist/maintenance-activity.d.ts +18 -0
- package/dist/maintenance-activity.js +28 -0
- package/dist/maintenance-cron.d.ts +22 -0
- package/dist/maintenance-cron.js +24 -1
- package/dist/maintenance-reporter.js +193 -88
- package/dist/metrics/collector.d.ts +2 -0
- package/dist/metrics/collector.js +7 -0
- package/dist/poller.d.ts +19 -0
- package/dist/poller.js +171 -5
- package/dist/types.d.ts +6 -0
- package/package.json +1 -1
|
@@ -1,16 +1,72 @@
|
|
|
1
|
-
import { readdir, readFile, unlink, writeFile } from 'fs/promises';
|
|
1
|
+
import { readdir, readFile, unlink, writeFile, mkdir, rename, } from 'fs/promises';
|
|
2
2
|
import { join } from 'path';
|
|
3
3
|
import { execFile } from 'child_process';
|
|
4
4
|
import { promisify } from 'util';
|
|
5
5
|
import { config, POLLER_VERSION } from './config.js';
|
|
6
6
|
import { logger } from './logger.js';
|
|
7
7
|
import { emitError } from './event-queue.js';
|
|
8
|
-
import { getBaseBrainCommitSha, getBaseBrainAgeHours } from './base-brain-metadata.js';
|
|
8
|
+
import { getBaseBrainCommitSha, getBaseBrainAgeHours, } from './base-brain-metadata.js';
|
|
9
9
|
const execFileAsync = promisify(execFile);
|
|
10
10
|
const OPERATION_TO_EVENT_TYPE = {
|
|
11
11
|
consolidation: 'maintenance.consolidation',
|
|
12
12
|
reflection: 'maintenance.reflection',
|
|
13
13
|
};
|
|
14
|
+
// teamvibe.ai#222 — malformed reports used to error forever because nothing
|
|
15
|
+
// ever removed them from the brain repo. They're quarantined instead of
|
|
16
|
+
// deleted outright so the raw payload survives for forensic inspection
|
|
17
|
+
// (why did the generator emit invalid JSON?), then purged after a retention
|
|
18
|
+
// window so the quarantine folder doesn't grow unbounded.
|
|
19
|
+
const QUARANTINE_DIR_NAME = '.quarantine';
|
|
20
|
+
const QUARANTINE_RETENTION_DAYS = 30;
|
|
21
|
+
const QUARANTINE_NAME_RE = /^(\d{4}-\d{2}-\d{2})__(.+)$/;
|
|
22
|
+
function todayIso() {
|
|
23
|
+
return new Date().toISOString().slice(0, 10);
|
|
24
|
+
}
|
|
25
|
+
// Date is parsed from the quarantine filename prefix, never from fs mtime —
|
|
26
|
+
// brains are re-checked-out from git per session, so mtime reflects the last
|
|
27
|
+
// checkout, not when the file was actually quarantined.
|
|
28
|
+
function daysSince(isoDate) {
|
|
29
|
+
const from = new Date(`${isoDate}T00:00:00Z`).getTime();
|
|
30
|
+
const now = new Date(`${todayIso()}T00:00:00Z`).getTime();
|
|
31
|
+
return Math.floor((now - from) / (24 * 60 * 60 * 1000));
|
|
32
|
+
}
|
|
33
|
+
async function quarantineFile(brainPath, file) {
|
|
34
|
+
const quarantineDir = join(brainPath, 'reports', QUARANTINE_DIR_NAME);
|
|
35
|
+
await mkdir(quarantineDir, { recursive: true });
|
|
36
|
+
const quarantineName = `${todayIso()}__${file}`;
|
|
37
|
+
await rename(join(brainPath, 'reports', file), join(quarantineDir, quarantineName));
|
|
38
|
+
return quarantineName;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Sweep reports/.quarantine/ for entries past the retention window and
|
|
42
|
+
* remove them from disk. Runs unconditionally (independent of whether
|
|
43
|
+
* there are any new reports/*.json to process this session) so cleanup
|
|
44
|
+
* doesn't stall on brains with a low reporting cadence.
|
|
45
|
+
*/
|
|
46
|
+
async function sweepExpiredQuarantine(brainPath) {
|
|
47
|
+
const quarantineDir = join(brainPath, 'reports', QUARANTINE_DIR_NAME);
|
|
48
|
+
const entries = await readdir(quarantineDir).catch(() => []);
|
|
49
|
+
const expired = [];
|
|
50
|
+
for (const entry of entries) {
|
|
51
|
+
const match = entry.match(QUARANTINE_NAME_RE);
|
|
52
|
+
const quarantinedOn = match?.[1];
|
|
53
|
+
if (!quarantinedOn)
|
|
54
|
+
continue; // unexpected name — leave alone rather than guess an age
|
|
55
|
+
if (daysSince(quarantinedOn) >= QUARANTINE_RETENTION_DAYS) {
|
|
56
|
+
try {
|
|
57
|
+
await unlink(join(quarantineDir, entry));
|
|
58
|
+
expired.push(`reports/${QUARANTINE_DIR_NAME}/${entry}`);
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
// Don't report this entry as expired if the unlink didn't actually
|
|
62
|
+
// happen — otherwise git rm --cached untracks it while it's still
|
|
63
|
+
// sitting on disk, and future sweeps can't see it's already "gone".
|
|
64
|
+
logger.warn(`[MaintenanceReporter] Failed to purge expired quarantine entry ${entry}: ${err instanceof Error ? err.message : err}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return expired;
|
|
69
|
+
}
|
|
14
70
|
/**
|
|
15
71
|
* Scan for JSON maintenance reports in the brain's reports/ directory
|
|
16
72
|
* and POST each to the /events endpoint. Fire-and-forget — never throws.
|
|
@@ -21,109 +77,158 @@ export async function postMaintenanceReports(brainPath, channelId) {
|
|
|
21
77
|
}
|
|
22
78
|
const reportsDir = join(brainPath, 'reports');
|
|
23
79
|
try {
|
|
80
|
+
const expiredQuarantineFiles = await sweepExpiredQuarantine(brainPath);
|
|
24
81
|
const files = await readdir(reportsDir).catch(() => []);
|
|
25
82
|
const jsonFiles = files.filter((f) => f.endsWith('.json'));
|
|
26
|
-
if (jsonFiles.length === 0)
|
|
27
|
-
return;
|
|
28
|
-
logger.info(`[MaintenanceReporter] Found ${jsonFiles.length} JSON report(s) in ${reportsDir}`);
|
|
29
|
-
// Fleet visibility fields — sampled once per run. See teamvibe.ai#129.
|
|
30
|
-
const [baseBrainCommitSha, baseBrainAgeHours] = await Promise.all([
|
|
31
|
-
getBaseBrainCommitSha(config.BASE_BRAIN_PATH),
|
|
32
|
-
getBaseBrainAgeHours(config.BASE_BRAIN_PATH),
|
|
33
|
-
]);
|
|
34
83
|
const postedFiles = [];
|
|
35
84
|
const removedFiles = [];
|
|
85
|
+
const quarantinedFrom = [];
|
|
86
|
+
const quarantinedTo = [];
|
|
36
87
|
let consolidationGuardTouched = false;
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
88
|
+
if (jsonFiles.length > 0) {
|
|
89
|
+
logger.info(`[MaintenanceReporter] Found ${jsonFiles.length} JSON report(s) in ${reportsDir}`);
|
|
90
|
+
// Fleet visibility fields — sampled once per run. See teamvibe.ai#129.
|
|
91
|
+
const [baseBrainCommitSha, baseBrainAgeHours] = await Promise.all([
|
|
92
|
+
getBaseBrainCommitSha(config.BASE_BRAIN_PATH),
|
|
93
|
+
getBaseBrainAgeHours(config.BASE_BRAIN_PATH),
|
|
94
|
+
]);
|
|
95
|
+
for (const file of jsonFiles) {
|
|
96
|
+
const filePath = join(reportsDir, file);
|
|
97
|
+
try {
|
|
98
|
+
const content = await readFile(filePath, 'utf-8');
|
|
99
|
+
let report;
|
|
100
|
+
try {
|
|
101
|
+
report = JSON.parse(content);
|
|
102
|
+
}
|
|
103
|
+
catch (parseErr) {
|
|
104
|
+
// Unlike network/IO errors below, a JSON.parse failure is never
|
|
105
|
+
// going to recover on retry — the payload itself is broken.
|
|
106
|
+
// Quarantine it (not delete) so the content survives for later
|
|
107
|
+
// inspection into why the generator emitted invalid JSON.
|
|
108
|
+
const message = parseErr instanceof Error ? parseErr.message : String(parseErr);
|
|
109
|
+
logger.warn(`[MaintenanceReporter] Quarantining malformed report ${file}: ${message}`);
|
|
110
|
+
emitError('maintenance', `Quarantining malformed report ${file}: ${message}`, { severity: 'warn', channelId });
|
|
111
|
+
const quarantineName = await quarantineFile(brainPath, file);
|
|
112
|
+
quarantinedFrom.push(`reports/${file}`);
|
|
113
|
+
quarantinedTo.push(`reports/${QUARANTINE_DIR_NAME}/${quarantineName}`);
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
if (!report.operationType) {
|
|
117
|
+
logger.warn(`[MaintenanceReporter] Removing invalid report ${file}: missing operationType`);
|
|
118
|
+
emitError('maintenance', `Removing invalid report ${file}: missing operationType`, { severity: 'warn' });
|
|
119
|
+
await unlink(filePath).catch(() => { });
|
|
120
|
+
removedFiles.push(`reports/${file}`);
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
const eventType = OPERATION_TO_EVENT_TYPE[report.operationType];
|
|
124
|
+
if (!eventType) {
|
|
125
|
+
logger.warn(`[MaintenanceReporter] Removing report ${file}: unknown operationType "${report.operationType}"`);
|
|
126
|
+
emitError('maintenance', `Removing report ${file}: unknown operationType "${report.operationType}"`, { severity: 'warn' });
|
|
127
|
+
await unlink(filePath).catch(() => { });
|
|
128
|
+
removedFiles.push(`reports/${file}`);
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
const payload = {
|
|
132
|
+
eventType,
|
|
133
|
+
channelId,
|
|
134
|
+
pollerVersion: POLLER_VERSION,
|
|
135
|
+
baseBrainCommitSha,
|
|
136
|
+
baseBrainAgeHours,
|
|
137
|
+
metadata: report,
|
|
138
|
+
};
|
|
139
|
+
const response = await fetch(`${config.TEAMVIBE_API_URL}/events`, {
|
|
140
|
+
method: 'POST',
|
|
141
|
+
headers: {
|
|
142
|
+
'Content-Type': 'application/json',
|
|
143
|
+
Authorization: `Bearer ${config.TEAMVIBE_POLLER_TOKEN}`,
|
|
144
|
+
},
|
|
145
|
+
body: JSON.stringify(payload),
|
|
146
|
+
});
|
|
147
|
+
if (response.ok) {
|
|
148
|
+
logger.info(`[MaintenanceReporter] Posted ${file} as ${eventType}`);
|
|
149
|
+
await unlink(filePath).catch(() => { });
|
|
150
|
+
postedFiles.push(`reports/${file}`);
|
|
151
|
+
// Touch the consolidation guard file so the poller-local
|
|
152
|
+
// `daysSinceMaintenance` metric reflects this successful run.
|
|
153
|
+
// The metric reads `stat(memory/.last_consolidation).mtimeMs`, so any
|
|
154
|
+
// skip of the consolidation skill's "update timestamp" step causes
|
|
155
|
+
// silent drift. See poller-brain#109, poller-brain-eval#150.
|
|
156
|
+
if (report.operationType === 'consolidation') {
|
|
157
|
+
const guardPath = join(brainPath, 'memory', '.last_consolidation');
|
|
158
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
159
|
+
try {
|
|
160
|
+
await writeFile(guardPath, `${today}\n`);
|
|
161
|
+
consolidationGuardTouched = true;
|
|
162
|
+
logger.info(`[MaintenanceReporter] Touched ${guardPath}`);
|
|
163
|
+
}
|
|
164
|
+
catch (err) {
|
|
165
|
+
logger.warn(`[MaintenanceReporter] Failed to touch ${guardPath}: ${err instanceof Error ? err.message : err}`);
|
|
166
|
+
}
|
|
92
167
|
}
|
|
93
168
|
}
|
|
169
|
+
else {
|
|
170
|
+
const errorBody = await response.text().catch(() => 'unknown');
|
|
171
|
+
logger.warn(`[MaintenanceReporter] Failed to POST ${file}: ${response.status} ${errorBody}`);
|
|
172
|
+
emitError('maintenance', `Failed to POST report ${file}: ${response.status}`, { severity: 'warn', channelId });
|
|
173
|
+
}
|
|
94
174
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
emitError('maintenance', `Failed to POST report ${file}: ${response.status}`, { severity: 'warn', channelId });
|
|
175
|
+
catch (err) {
|
|
176
|
+
logger.warn(`[MaintenanceReporter] Error processing ${file}: ${err instanceof Error ? err.message : err}`);
|
|
177
|
+
emitError('maintenance', `Error processing report ${file}: ${err instanceof Error ? err.message : String(err)}`, { severity: 'warn', channelId });
|
|
99
178
|
}
|
|
100
179
|
}
|
|
101
|
-
catch (err) {
|
|
102
|
-
logger.warn(`[MaintenanceReporter] Error processing ${file}: ${err instanceof Error ? err.message : err}`);
|
|
103
|
-
emitError('maintenance', `Error processing report ${file}: ${err instanceof Error ? err.message : String(err)}`, { severity: 'warn', channelId });
|
|
104
|
-
}
|
|
105
180
|
}
|
|
106
|
-
// Remove posted/invalid JSON reports from git so they
|
|
107
|
-
//
|
|
108
|
-
|
|
109
|
-
|
|
181
|
+
// Remove posted/invalid/expired-quarantine JSON reports from git so they
|
|
182
|
+
// don't resurrect on next pull, stage newly quarantined reports at their
|
|
183
|
+
// new path, and stage the consolidation guard file if it was touched.
|
|
184
|
+
// Runs regardless of jsonFiles.length so the quarantine-expiry sweep
|
|
185
|
+
// still gets committed on sessions with no new reports to process.
|
|
186
|
+
const gitRemovals = [
|
|
187
|
+
...postedFiles,
|
|
188
|
+
...removedFiles,
|
|
189
|
+
...quarantinedFrom,
|
|
190
|
+
...expiredQuarantineFiles,
|
|
191
|
+
];
|
|
192
|
+
const gitAdditions = [...quarantinedTo];
|
|
193
|
+
if (gitRemovals.length > 0 ||
|
|
194
|
+
gitAdditions.length > 0 ||
|
|
195
|
+
consolidationGuardTouched) {
|
|
110
196
|
try {
|
|
111
|
-
if (
|
|
112
|
-
await execFileAsync('git', ['rm', '--cached', '--ignore-unmatch', ...
|
|
197
|
+
if (gitRemovals.length > 0) {
|
|
198
|
+
await execFileAsync('git', ['rm', '--cached', '--ignore-unmatch', ...gitRemovals], { cwd: brainPath });
|
|
199
|
+
}
|
|
200
|
+
if (gitAdditions.length > 0) {
|
|
201
|
+
// -f: quarantine payloads must survive regardless of any brain
|
|
202
|
+
// .gitignore rule matching dotdirs (e.g. a blanket `.*`) — a
|
|
203
|
+
// plain `git add` silently skips ignored paths, which would
|
|
204
|
+
// defeat the whole point of quarantining (forensic survival)
|
|
205
|
+
// the moment the next checkout drops the untracked file.
|
|
206
|
+
await execFileAsync('git', ['add', '-f', ...gitAdditions], {
|
|
207
|
+
cwd: brainPath,
|
|
208
|
+
});
|
|
113
209
|
}
|
|
114
210
|
if (consolidationGuardTouched) {
|
|
115
|
-
await execFileAsync('git', ['add', 'memory/.last_consolidation'], {
|
|
211
|
+
await execFileAsync('git', ['add', 'memory/.last_consolidation'], {
|
|
212
|
+
cwd: brainPath,
|
|
213
|
+
});
|
|
116
214
|
}
|
|
117
|
-
const
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
215
|
+
const messageParts = [];
|
|
216
|
+
if (postedFiles.length > 0 || removedFiles.length > 0)
|
|
217
|
+
messageParts.push('process maintenance reports');
|
|
218
|
+
if (quarantinedFrom.length > 0)
|
|
219
|
+
messageParts.push('quarantine malformed reports');
|
|
220
|
+
if (expiredQuarantineFiles.length > 0)
|
|
221
|
+
messageParts.push('purge expired quarantine');
|
|
222
|
+
if (consolidationGuardTouched)
|
|
223
|
+
messageParts.push('refresh consolidation guard');
|
|
224
|
+
const message = `chore: ${messageParts.join(' + ')}`;
|
|
225
|
+
await execFileAsync('git', ['commit', '-m', message], {
|
|
226
|
+
cwd: brainPath,
|
|
227
|
+
});
|
|
228
|
+
logger.info(`[MaintenanceReporter] Committed ${gitRemovals.length} removal(s), ${gitAdditions.length} quarantine addition(s)${consolidationGuardTouched ? ' + guard refresh' : ''}`);
|
|
124
229
|
}
|
|
125
230
|
catch (err) {
|
|
126
|
-
logger.debug(`[MaintenanceReporter] Could not git-rm reports (may not be a git repo): ${err instanceof Error ? err.message : err}`);
|
|
231
|
+
logger.debug(`[MaintenanceReporter] Could not git-rm/add reports (may not be a git repo): ${err instanceof Error ? err.message : err}`);
|
|
127
232
|
}
|
|
128
233
|
}
|
|
129
234
|
}
|
|
@@ -21,6 +21,8 @@ export interface MetricsPayload {
|
|
|
21
21
|
daysSinceMaintenance?: number;
|
|
22
22
|
lastMaintenanceJobMs?: number;
|
|
23
23
|
lastMaintenanceJobSuccessMs?: number;
|
|
24
|
+
maintenanceSkippedCountLastRun?: number;
|
|
25
|
+
lastMaintenanceJobSkippedMs?: number;
|
|
24
26
|
lastAutoUpdateJobMs?: number;
|
|
25
27
|
lastAutoUpdateJobSuccessMs?: number;
|
|
26
28
|
timestamp?: string;
|
|
@@ -228,6 +228,13 @@ export async function collectMetrics(brainPath, brainId) {
|
|
|
228
228
|
if (cron.lastMaintenanceJobSuccessAt !== null) {
|
|
229
229
|
payload.lastMaintenanceJobSuccessMs = now - cron.lastMaintenanceJobSuccessAt;
|
|
230
230
|
}
|
|
231
|
+
// #131 — activity-based skip counter. Always sent (0 is a real "no skips
|
|
232
|
+
// this run" value, not an omission) so a stuck-high count is visible
|
|
233
|
+
// fleet-wide without per-brain digging.
|
|
234
|
+
payload.maintenanceSkippedCountLastRun = cron.maintenanceSkippedCountLastRun;
|
|
235
|
+
if (cron.lastMaintenanceJobSkippedAt !== null) {
|
|
236
|
+
payload.lastMaintenanceJobSkippedMs = now - cron.lastMaintenanceJobSkippedAt;
|
|
237
|
+
}
|
|
231
238
|
if (cron.lastAutoUpdateJobFiredAt !== null) {
|
|
232
239
|
payload.lastAutoUpdateJobMs = now - cron.lastAutoUpdateJobFiredAt;
|
|
233
240
|
}
|
package/dist/poller.d.ts
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import { logger } from './logger.js';
|
|
2
|
+
import { type SpawnResult } from './claude-spawner.js';
|
|
3
|
+
import { type KilledByPoller } from './classify-failed-iteration.js';
|
|
4
|
+
import type { TeamVibeQueueMessage } from './types.js';
|
|
1
5
|
interface ReleaseLogger {
|
|
2
6
|
info: (msg: string) => void;
|
|
3
7
|
warn: (msg: string) => void;
|
|
@@ -13,5 +17,20 @@ export declare function releaseAllActiveLocks(locks: Map<string, string>, releas
|
|
|
13
17
|
* in morning-check telemetry. Same DI shape as runMaintenanceJob.
|
|
14
18
|
*/
|
|
15
19
|
export declare function runAutoUpdateJob(checkForUpdateFn?: () => Promise<boolean>): Promise<void>;
|
|
20
|
+
/**
|
|
21
|
+
* Whether a runInboxLoop is currently active for this thread. Used by the
|
|
22
|
+
* inbox watcher (teamvibe.ai#242) to avoid starting a second loop for a
|
|
23
|
+
* thread that's already being served — its existing pickNextMessage() call
|
|
24
|
+
* will pick up any new file on its own.
|
|
25
|
+
*/
|
|
26
|
+
export declare function isInboxLoopActive(threadId: string): boolean;
|
|
27
|
+
export declare function isIntentionalKillExitCode(exitCode: number | null): boolean;
|
|
28
|
+
export declare function isPollerInitiatedFailure(result: {
|
|
29
|
+
exitCode: number | null;
|
|
30
|
+
killedByPoller?: KilledByPoller;
|
|
31
|
+
}): boolean;
|
|
32
|
+
export declare function endOfTurnFallbackText(hasSlackContext: boolean, result: Pick<SpawnResult, 'success' | 'sawSendMessageCall' | 'lastPlainTextContent'>): string | undefined;
|
|
33
|
+
export declare function runInboxLoop(threadId: string, kbPath: string, queueMessage: TeamVibeQueueMessage, sessionLog: ReturnType<typeof logger.createSession>): Promise<void>;
|
|
34
|
+
export declare function __setShuttingDownForTest(value: boolean): void;
|
|
16
35
|
export declare function startPoller(): Promise<void>;
|
|
17
36
|
export {};
|
package/dist/poller.js
CHANGED
|
@@ -12,7 +12,8 @@ import { initAuth, stopRefresh, getPollerId } from './auth-provider.js';
|
|
|
12
12
|
import { registerBrain, startMaintenanceLoop, stopMaintenanceLoop } from './maintenance-cron.js';
|
|
13
13
|
import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent, emitError } from './event-queue.js';
|
|
14
14
|
import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
|
|
15
|
-
import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed } from './inbox-manager.js';
|
|
15
|
+
import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage } from './inbox-manager.js';
|
|
16
|
+
import { startInboxWatcher, stopInboxWatcher } from './inbox-watcher.js';
|
|
16
17
|
import { Cron } from 'croner';
|
|
17
18
|
import { isControlMessage, handleControlMessage } from './control-handler.js';
|
|
18
19
|
import { checkForUpdate } from './auto-update.js';
|
|
@@ -122,7 +123,67 @@ async function attemptDnsFlush() {
|
|
|
122
123
|
}
|
|
123
124
|
// Track which threads have an active processing loop
|
|
124
125
|
const activeLoops = new Set();
|
|
125
|
-
|
|
126
|
+
/**
|
|
127
|
+
* Whether a runInboxLoop is currently active for this thread. Used by the
|
|
128
|
+
* inbox watcher (teamvibe.ai#242) to avoid starting a second loop for a
|
|
129
|
+
* thread that's already being served — its existing pickNextMessage() call
|
|
130
|
+
* will pick up any new file on its own.
|
|
131
|
+
*/
|
|
132
|
+
export function isInboxLoopActive(threadId) {
|
|
133
|
+
return activeLoops.has(threadId);
|
|
134
|
+
}
|
|
135
|
+
// Exported for testing (teamvibe.ai#32). SIGTERM (143) / SIGKILL (137) exit
|
|
136
|
+
// codes are the classic 128+signal convention for "the poller itself killed
|
|
137
|
+
// this process on purpose" (timeout, idle, network, zombie, or — as of #32 —
|
|
138
|
+
// a compaction-ceiling watchdog kill). Historically all of these set
|
|
139
|
+
// `killedByPoller` via the same `proc.kill('SIGTERM')` path in
|
|
140
|
+
// claude-spawner.ts and were observed to surface as 143/137 here.
|
|
141
|
+
//
|
|
142
|
+
// PR#308 round-2 review (teamvibe.ai#73): this exit-code convention is
|
|
143
|
+
// fragile on its own — the Claude CLI can legitimately exit 143 for
|
|
144
|
+
// unrelated reasons, and #73's move to group-kill (`process.kill(-pid, ...)`
|
|
145
|
+
// instead of a direct `proc.kill()`) means a signal-terminated close now
|
|
146
|
+
// reports `code: null` with a separate `signal` field (verified empirically
|
|
147
|
+
// against a synthetic detached child in this repo's environment — Node's
|
|
148
|
+
// `close` event reports `code: null` for any signal-terminated process,
|
|
149
|
+
// group-killed or not). Kept as a defensive fallback only — see
|
|
150
|
+
// isPollerInitiatedFailure() below, which is what callers should actually use.
|
|
151
|
+
export function isIntentionalKillExitCode(exitCode) {
|
|
152
|
+
return exitCode === 143 || exitCode === 137;
|
|
153
|
+
}
|
|
154
|
+
// The actual retry-classification signal (PR#308 round-2 review item 1):
|
|
155
|
+
// `killedByPoller` is the purpose-built marker for "the poller killed this
|
|
156
|
+
// process on purpose, not an application failure" (see SpawnResult /
|
|
157
|
+
// classify-failed-iteration.ts) and is set on every poller-initiated kill
|
|
158
|
+
// path this branch touches — timeout, idle watchdog, network watchdog,
|
|
159
|
+
// zombie-kill, and (added by this fix) poller shutdown. Prefer it, and fall
|
|
160
|
+
// back to the exit-code heuristic only when it's absent (e.g. hand-built
|
|
161
|
+
// SpawnResult literals in older/unrelated call sites) so behavior doesn't
|
|
162
|
+
// regress for anything not yet covered by the marker.
|
|
163
|
+
export function isPollerInitiatedFailure(result) {
|
|
164
|
+
return Boolean(result.killedByPoller) || isIntentionalKillExitCode(result.exitCode);
|
|
165
|
+
}
|
|
166
|
+
// teamvibe.ai#284 (poller-brain#235): end-of-turn guard. A successful Claude
|
|
167
|
+
// run can end its turn having only written plain assistant text — no
|
|
168
|
+
// `mcp__slack__send_message` tool call — which the platform otherwise drops
|
|
169
|
+
// silently (no error, no signal; see poller-brain#235's 4x-in-one-session
|
|
170
|
+
// recurrence). Decides whether this run needs the deterministic fallback
|
|
171
|
+
// delivery, and if so, what text to send.
|
|
172
|
+
//
|
|
173
|
+
// Pure/exported (mirrors classifyFailedIteration's pattern) so the gating
|
|
174
|
+
// logic — "Slack context AND no send_message call AND non-empty trailing
|
|
175
|
+
// text" — is unit-testable without mocking spawnClaudeCode or sendSlackMessage.
|
|
176
|
+
// Deliberately does NOT fire on a failed run (result.success === false):
|
|
177
|
+
// that path already has its own delivery via classifyFailedIteration +
|
|
178
|
+
// sendSlackMessage/sendSlackError, and firing here too would double-deliver.
|
|
179
|
+
export function endOfTurnFallbackText(hasSlackContext, result) {
|
|
180
|
+
if (!hasSlackContext || !result.success || result.sawSendMessageCall) {
|
|
181
|
+
return undefined;
|
|
182
|
+
}
|
|
183
|
+
const text = result.lastPlainTextContent?.trim();
|
|
184
|
+
return text ? text : undefined;
|
|
185
|
+
}
|
|
186
|
+
export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
126
187
|
if (activeLoops.has(threadId)) {
|
|
127
188
|
sessionLog.info(`Loop already running for thread ${threadId}, skipping`);
|
|
128
189
|
return;
|
|
@@ -150,7 +211,25 @@ async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
150
211
|
activeLocks.set(threadId, lockToken);
|
|
151
212
|
}
|
|
152
213
|
let currentSessionId = session.session_id || undefined;
|
|
153
|
-
|
|
214
|
+
// PR#308 round-3 review (teamvibe.ai#73): `shuttingDown` (declared below,
|
|
215
|
+
// referenced here via the same forward-reference pattern already used by
|
|
216
|
+
// pollLoop()'s `while (!shuttingDown)`) gates every new iteration of this
|
|
217
|
+
// loop, not just the retry-classification decision inside it. Without
|
|
218
|
+
// this, a message killed with `killedByPoller: 'shutdown'` (added by this
|
|
219
|
+
// branch) falls into the same "poller-initiated, retry immediately, no
|
|
220
|
+
// backoff" bucket as timeout/idle/network/zombie (see
|
|
221
|
+
// isPollerInitiatedFailure() below) — correctly skipping markProcessed/
|
|
222
|
+
// markFailed so the message stays unprocessed for a future boot, but with
|
|
223
|
+
// nothing to stop the loop from immediately calling pickNextMessage()
|
|
224
|
+
// again and spawning a NEW `claude` process for that same message, even
|
|
225
|
+
// though killAllActiveProcesses() (shutdown()'s first step) already ran.
|
|
226
|
+
// Gating the loop *condition* itself (rather than special-casing
|
|
227
|
+
// 'shutdown' inside the retry decision) closes this for every
|
|
228
|
+
// killedByPoller reason uniformly, and — unlike a `break` — falls through
|
|
229
|
+
// to the exact same post-loop cleanup (releaseSessionLock etc., below)
|
|
230
|
+
// that the existing `if (!message) break` / `if (!hasUnprocessed(...))
|
|
231
|
+
// break` exits already use, so lock release is unaffected.
|
|
232
|
+
while (!shuttingDown) {
|
|
154
233
|
const message = pickNextMessage(kbPath, threadId);
|
|
155
234
|
if (!message) {
|
|
156
235
|
sessionLog.info(`[Loop] No more messages for thread ${threadId}`);
|
|
@@ -167,6 +246,7 @@ async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
167
246
|
: undefined;
|
|
168
247
|
let succeeded = false;
|
|
169
248
|
let lastExitCode = null;
|
|
249
|
+
let lastKilledByPoller;
|
|
170
250
|
try {
|
|
171
251
|
// Reconstruct queue message from stored inbox data
|
|
172
252
|
const storedData = JSON.parse(message.content);
|
|
@@ -217,6 +297,28 @@ async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
217
297
|
modelSource: result.modelSource,
|
|
218
298
|
});
|
|
219
299
|
}
|
|
300
|
+
// teamvibe.ai#284 (poller-brain#235): end-of-turn guard. A
|
|
301
|
+
// successful run that never called send_message but left trailing
|
|
302
|
+
// plain text would otherwise vanish silently — deliver it via the
|
|
303
|
+
// same sendSlackMessage() helper used for the failure-path neutral
|
|
304
|
+
// notices, and emit a warning-level event so the near-miss is
|
|
305
|
+
// visible (today session.completed looks identical whether or not
|
|
306
|
+
// this fired).
|
|
307
|
+
const fallbackText = endOfTurnFallbackText(hasSlackContext, result);
|
|
308
|
+
if (fallbackText) {
|
|
309
|
+
sessionLog.warn(`[Loop] Iteration #${iteration}: no send_message call detected, delivering trailing assistant text via fallback`);
|
|
310
|
+
try {
|
|
311
|
+
await sendSlackMessage(queueMessage, fallbackText);
|
|
312
|
+
}
|
|
313
|
+
catch (err) {
|
|
314
|
+
sessionLog.error(`[Loop] End-of-turn fallback delivery failed: ${err instanceof Error ? err.message : err}`);
|
|
315
|
+
}
|
|
316
|
+
emitError('session', 'End-of-turn guard: delivered assistant text via fallback (no send_message tool call detected)', {
|
|
317
|
+
severity: 'warn',
|
|
318
|
+
channelId: queueMessage.teamvibe.channelId,
|
|
319
|
+
context: { threadId: queueMessage.thread_id },
|
|
320
|
+
});
|
|
321
|
+
}
|
|
220
322
|
if (queueMessage.teamvibe.brain?.brainId) {
|
|
221
323
|
await pushBrainChanges(kbPath, queueMessage.teamvibe.brain.brainId, queueMessage.teamvibe.workspaceId);
|
|
222
324
|
}
|
|
@@ -224,6 +326,7 @@ async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
224
326
|
}
|
|
225
327
|
else {
|
|
226
328
|
lastExitCode = result.exitCode;
|
|
329
|
+
lastKilledByPoller = result.killedByPoller;
|
|
227
330
|
sessionLog.error(`[Loop] Claude failed iteration #${iteration}: ${result.error}`);
|
|
228
331
|
if (hasSlackContext) {
|
|
229
332
|
const decision = classifyFailedIteration(result);
|
|
@@ -245,8 +348,19 @@ async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
245
348
|
markProcessed(kbPath, threadId, message.filename);
|
|
246
349
|
}
|
|
247
350
|
else {
|
|
248
|
-
//
|
|
249
|
-
|
|
351
|
+
// Poller-initiated kill (timeout/idle/network/zombie/shutdown, via
|
|
352
|
+
// `killedByPoller`, or the legacy 143/137 exit-code fallback) → never
|
|
353
|
+
// counts as an application failure, so markFailed/backoff is skipped
|
|
354
|
+
// below regardless of reason. For timeout/idle/network/zombie this
|
|
355
|
+
// really does mean "retry immediately" — the `while (!shuttingDown)`
|
|
356
|
+
// condition above lets the loop come straight back around. For
|
|
357
|
+
// 'shutdown' specifically, `shuttingDown` is already true by the time
|
|
358
|
+
// this reads, so the loop condition above stops it from actually
|
|
359
|
+
// retrying inline — instead the branch below requeues the message
|
|
360
|
+
// from processing/ back to new/ (PR#308 round-5) so a future boot's
|
|
361
|
+
// inbox watcher finds it. See isPollerInitiatedFailure() and the
|
|
362
|
+
// loop-condition comment above.
|
|
363
|
+
const isKilledSignal = isPollerInitiatedFailure({ exitCode: lastExitCode, killedByPoller: lastKilledByPoller });
|
|
250
364
|
if (!isKilledSignal) {
|
|
251
365
|
const failures = (failureCounts.get(message.filename) || 0) + 1;
|
|
252
366
|
failureCounts.set(message.filename, failures);
|
|
@@ -261,6 +375,43 @@ async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
261
375
|
await sleep(backoffMs);
|
|
262
376
|
}
|
|
263
377
|
}
|
|
378
|
+
else if (lastKilledByPoller === 'shutdown') {
|
|
379
|
+
// PR#308 round-5 review (teamvibe.ai#73): pickNextMessage() moves a
|
|
380
|
+
// message's file to processing/ as soon as it's picked up, not once
|
|
381
|
+
// it completes — so a message killed here by shutdown is left
|
|
382
|
+
// sitting in processing/, not new/. The watcher's wake condition
|
|
383
|
+
// (hasNewMessage(), inbox-watcher.ts) only scans new/, so nothing
|
|
384
|
+
// would start a fresh runInboxLoop() for this thread on next boot.
|
|
385
|
+
// Requeue it back to new/ so a future boot's watcher scan actually
|
|
386
|
+
// finds it — this is what makes "your message is saved" (the
|
|
387
|
+
// shutdown notice text below, classify-failed-iteration.ts)
|
|
388
|
+
// literally true. It does NOT make resumption automatic: the
|
|
389
|
+
// watcher's checkInboxForNewSessions() (inbox-watcher.ts) only
|
|
390
|
+
// scans brains already in the in-memory trackedBrains map, which
|
|
391
|
+
// is empty after a fresh poller start until some SQS activity for
|
|
392
|
+
// this brain re-registers it (see teamvibe.ai#310) — hence the
|
|
393
|
+
// notice text also naming "next message or scheduled run" rather
|
|
394
|
+
// than claiming automatic resumption.
|
|
395
|
+
//
|
|
396
|
+
// Scoped to 'shutdown' only, deliberately not the other four
|
|
397
|
+
// poller-initiated reasons (timeout/idle/network/zombie) — those
|
|
398
|
+
// retry immediately within this same still-running loop iteration
|
|
399
|
+
// (the `while (!shuttingDown)` condition above lets it come
|
|
400
|
+
// straight back around), so the existing processing/
|
|
401
|
+
// crash-recovery branch inside pickNextMessage() already covers
|
|
402
|
+
// them; requeueing those too would be redundant and could
|
|
403
|
+
// double-process.
|
|
404
|
+
//
|
|
405
|
+
// Safe against waking a brand-new loop for this thread before this
|
|
406
|
+
// poller process actually exits: stopInboxWatcher() runs earlier in
|
|
407
|
+
// shutdown() (poller.ts), BEFORE killAllActiveProcesses() (whose
|
|
408
|
+
// completion is what unblocks this iteration and reaches this
|
|
409
|
+
// branch) — and stopInboxWatcher() closes the real fs.watch
|
|
410
|
+
// handlers themselves (inbox-watcher.ts stopInboxWatcher(), not
|
|
411
|
+
// just an interval/poll timer), so by the time this requeue runs,
|
|
412
|
+
// nothing is watching new/ to react to the file reappearing there.
|
|
413
|
+
requeueMessage(kbPath, threadId, message.filename);
|
|
414
|
+
}
|
|
264
415
|
}
|
|
265
416
|
if (!hasUnprocessed(kbPath, threadId)) {
|
|
266
417
|
break;
|
|
@@ -505,6 +656,19 @@ function sleep(ms) {
|
|
|
505
656
|
}
|
|
506
657
|
// Graceful shutdown
|
|
507
658
|
let shuttingDown = false;
|
|
659
|
+
// Exported for testing only (PR#308 round-3 review item 1). `shuttingDown`
|
|
660
|
+
// has no other external mutator — production code only ever flips it via
|
|
661
|
+
// the real shutdown() below, which also runs killAllActiveProcesses(),
|
|
662
|
+
// releaseAllActiveLocks(), the drain loop, and process.exit(). This setter
|
|
663
|
+
// lets a test put runInboxLoop's `while (!shuttingDown)` gate into the
|
|
664
|
+
// "shutdown in progress" state directly (mirroring the exact race
|
|
665
|
+
// shutdown() itself creates — killAllActiveProcesses() kills the in-flight
|
|
666
|
+
// spawn while its promise is still pending) without invoking any of those
|
|
667
|
+
// real side effects. Same pattern as __resetCronTelemetryForTest /
|
|
668
|
+
// __resetTrackedBrainsForTest elsewhere in this package.
|
|
669
|
+
export function __setShuttingDownForTest(value) {
|
|
670
|
+
shuttingDown = value;
|
|
671
|
+
}
|
|
508
672
|
async function shutdown(signal, exitCode = 0) {
|
|
509
673
|
if (shuttingDown)
|
|
510
674
|
return;
|
|
@@ -514,6 +678,7 @@ async function shutdown(signal, exitCode = 0) {
|
|
|
514
678
|
stopMaintenanceLoop();
|
|
515
679
|
stopAutoUpdateCron();
|
|
516
680
|
stopMetricsCron();
|
|
681
|
+
stopInboxWatcher();
|
|
517
682
|
stopEventQueue();
|
|
518
683
|
// Kill all active Claude child processes
|
|
519
684
|
await killAllActiveProcesses();
|
|
@@ -592,5 +757,6 @@ export async function startPoller() {
|
|
|
592
757
|
startMaintenanceLoop(processMessage);
|
|
593
758
|
startAutoUpdateCron();
|
|
594
759
|
startMetricsCron();
|
|
760
|
+
startInboxWatcher();
|
|
595
761
|
await pollLoop();
|
|
596
762
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -19,6 +19,12 @@ export interface TeamVibeQueueMessage {
|
|
|
19
19
|
filename: string;
|
|
20
20
|
mimetype: string;
|
|
21
21
|
}>;
|
|
22
|
+
inlineImages?: Array<{
|
|
23
|
+
mediaType: string;
|
|
24
|
+
data?: string;
|
|
25
|
+
url?: string;
|
|
26
|
+
expiresAt?: string;
|
|
27
|
+
}>;
|
|
22
28
|
response_context: {
|
|
23
29
|
slack?: {
|
|
24
30
|
channel: string;
|