@teamvibe/poller 0.4.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/base-brain-metadata.d.ts +18 -0
- package/dist/base-brain-metadata.js +52 -0
- package/dist/claude-spawner.js +54 -20
- package/dist/config.d.ts +4 -0
- package/dist/config.js +6 -1
- package/dist/heartbeat-manager.js +4 -4
- package/dist/logger.d.ts +1 -0
- package/dist/logger.js +6 -0
- package/dist/maintenance-reporter.js +45 -7
- package/dist/poller.d.ts +6 -0
- package/dist/poller.js +32 -0
- package/dist/worktree-manager.d.ts +36 -0
- package/dist/worktree-manager.js +147 -0
- package/package.json +1 -1
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read the current HEAD commit SHA of a git repo. Returns null on any error
|
|
3
|
+
* (not a git repo, git missing, dir missing, etc.) so callers can attach a
|
|
4
|
+
* best-effort metric without breaking the report flow.
|
|
5
|
+
*/
|
|
6
|
+
export declare function getBaseBrainCommitSha(repoDir: string): Promise<string | null>;
|
|
7
|
+
/**
|
|
8
|
+
* Age of the base-brain checkout in hours.
|
|
9
|
+
*
|
|
10
|
+
* Defined as: now - max(commitDate of HEAD, mtime of .git/FETCH_HEAD).
|
|
11
|
+
* FETCH_HEAD mtime catches the case where `git pull` runs but the upstream
|
|
12
|
+
* has nothing new (or the fast-forward silently fails) — commitDate only
|
|
13
|
+
* advances on a successful integration, FETCH_HEAD on every pull attempt.
|
|
14
|
+
* Taking the max gives the most defensible "freshness" signal.
|
|
15
|
+
*
|
|
16
|
+
* Returns null on any error.
|
|
17
|
+
*/
|
|
18
|
+
export declare function getBaseBrainAgeHours(repoDir: string): Promise<number | null>;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { stat } from 'fs/promises';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { execFile } from 'child_process';
|
|
4
|
+
import { promisify } from 'util';
|
|
5
|
+
const execFileAsync = promisify(execFile);
|
|
6
|
+
/**
|
|
7
|
+
* Read the current HEAD commit SHA of a git repo. Returns null on any error
|
|
8
|
+
* (not a git repo, git missing, dir missing, etc.) so callers can attach a
|
|
9
|
+
* best-effort metric without breaking the report flow.
|
|
10
|
+
*/
|
|
11
|
+
export async function getBaseBrainCommitSha(repoDir) {
|
|
12
|
+
try {
|
|
13
|
+
const { stdout } = await execFileAsync('git', ['rev-parse', 'HEAD'], { cwd: repoDir });
|
|
14
|
+
const sha = stdout.trim();
|
|
15
|
+
return sha || null;
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return null;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Age of the base-brain checkout in hours.
|
|
23
|
+
*
|
|
24
|
+
* Defined as: now - max(commitDate of HEAD, mtime of .git/FETCH_HEAD).
|
|
25
|
+
* FETCH_HEAD mtime catches the case where `git pull` runs but the upstream
|
|
26
|
+
* has nothing new (or the fast-forward silently fails) — commitDate only
|
|
27
|
+
* advances on a successful integration, FETCH_HEAD on every pull attempt.
|
|
28
|
+
* Taking the max gives the most defensible "freshness" signal.
|
|
29
|
+
*
|
|
30
|
+
* Returns null on any error.
|
|
31
|
+
*/
|
|
32
|
+
export async function getBaseBrainAgeHours(repoDir) {
|
|
33
|
+
try {
|
|
34
|
+
const { stdout } = await execFileAsync('git', ['log', '-1', '--format=%cI', 'HEAD'], { cwd: repoDir });
|
|
35
|
+
const commitDateMs = new Date(stdout.trim()).getTime();
|
|
36
|
+
if (Number.isNaN(commitDateMs))
|
|
37
|
+
return null;
|
|
38
|
+
let fetchHeadMs = 0;
|
|
39
|
+
try {
|
|
40
|
+
const s = await stat(join(repoDir, '.git', 'FETCH_HEAD'));
|
|
41
|
+
fetchHeadMs = s.mtimeMs;
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
// .git/FETCH_HEAD absent on a never-pulled clone — fall back to commitDate
|
|
45
|
+
}
|
|
46
|
+
const freshestMs = Math.max(commitDateMs, fetchHeadMs);
|
|
47
|
+
return (Date.now() - freshestMs) / 3_600_000;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
package/dist/claude-spawner.js
CHANGED
|
@@ -7,6 +7,7 @@ import { logger } from './logger.js';
|
|
|
7
7
|
import { getBaseBrainPath } from './brain-manager.js';
|
|
8
8
|
import { getPollerId } from './auth-provider.js';
|
|
9
9
|
import { emitError } from './event-queue.js';
|
|
10
|
+
import { createWorktree, removeWorktree, isWorktreeIsolationEnabled } from './worktree-manager.js';
|
|
10
11
|
/**
|
|
11
12
|
* Maps tool names (or prefixes) to human-readable status messages.
|
|
12
13
|
* More specific patterns are checked first.
|
|
@@ -527,30 +528,63 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
527
528
|
});
|
|
528
529
|
});
|
|
529
530
|
}
|
|
531
|
+
/**
|
|
532
|
+
* Sources whose sessions should run in an isolated git worktree (teamvibe.ai#120).
|
|
533
|
+
* Foreground Slack messages always run in the main working dir.
|
|
534
|
+
*/
|
|
535
|
+
const ISOLATED_SOURCES = new Set(['cron', 'heartbeat', 'maintenance']);
|
|
536
|
+
function shouldIsolateInWorktree(msg) {
|
|
537
|
+
return isWorktreeIsolationEnabled() && ISOLATED_SOURCES.has(msg.source);
|
|
538
|
+
}
|
|
530
539
|
export async function spawnClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = true, lastMessageTs, onMessageSent, typingIndicator) {
|
|
531
|
-
|
|
532
|
-
//
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
540
|
+
// Worktree isolation: spawned (non-foreground) sessions run in
|
|
541
|
+
// <brain>/.worktrees/<thread_id>/ so the sync-watchdog's reset --hard on
|
|
542
|
+
// the main working dir can't trample in-flight writes or pre-push commits.
|
|
543
|
+
// See teamvibe.ai#120. cwd remains the main brain path on fallback / failure.
|
|
544
|
+
let effectiveCwd = cwd;
|
|
545
|
+
let worktreeOwner;
|
|
546
|
+
if (shouldIsolateInWorktree(msg)) {
|
|
547
|
+
try {
|
|
548
|
+
effectiveCwd = await createWorktree(cwd, msg.thread_id);
|
|
549
|
+
worktreeOwner = { brainPath: cwd, id: msg.thread_id };
|
|
550
|
+
sessionLog.info(`[Worktree] Spawned session running in ${effectiveCwd}`);
|
|
551
|
+
}
|
|
552
|
+
catch (err) {
|
|
553
|
+
sessionLog.warn(`[Worktree] Failed to create worktree, falling back to main brain dir: ${err instanceof Error ? err.message : err}`);
|
|
554
|
+
}
|
|
537
555
|
}
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
//
|
|
541
|
-
|
|
542
|
-
if (
|
|
543
|
-
sessionLog.info('
|
|
544
|
-
|
|
545
|
-
|
|
556
|
+
try {
|
|
557
|
+
const result = await runClaudeCode(msg, sessionLog, effectiveCwd, sessionId, isFirstMessage, lastMessageTs, onMessageSent, typingIndicator);
|
|
558
|
+
// If --resume failed, retry as a fresh session (session files may have been lost on container restart)
|
|
559
|
+
let retryResult = result;
|
|
560
|
+
if (!result.success && sessionId && !isFirstMessage) {
|
|
561
|
+
sessionLog.info('Resume failed, retrying as fresh session (session files may have been lost)');
|
|
562
|
+
retryResult = await runClaudeCode(msg, sessionLog, effectiveCwd, sessionId, true, lastMessageTs, onMessageSent, typingIndicator);
|
|
563
|
+
}
|
|
564
|
+
// If session ID is "already in use" (stale lock file), kill any zombie and retry with fresh ID
|
|
565
|
+
if (!retryResult.success && sessionId && retryResult.error?.includes('already in use')) {
|
|
566
|
+
// Kill zombie process for this thread if still alive
|
|
567
|
+
const killed = killZombieForThread(msg.thread_id);
|
|
568
|
+
if (killed) {
|
|
569
|
+
sessionLog.info('Killed zombie process before fresh session retry');
|
|
570
|
+
// Give the process a moment to fully exit and release lock files
|
|
571
|
+
await new Promise((resolve) => setTimeout(resolve, 2000));
|
|
572
|
+
}
|
|
573
|
+
sessionLog.info('Session ID already in use (stale lock), retrying with fresh session ID');
|
|
574
|
+
const { randomUUID } = await import('crypto');
|
|
575
|
+
const freshId = randomUUID();
|
|
576
|
+
const freshResult = await runClaudeCode(msg, sessionLog, effectiveCwd, freshId, true, lastMessageTs, onMessageSent, typingIndicator);
|
|
577
|
+
return { ...freshResult, newSessionId: freshId };
|
|
578
|
+
}
|
|
579
|
+
return retryResult;
|
|
580
|
+
}
|
|
581
|
+
finally {
|
|
582
|
+
if (worktreeOwner) {
|
|
583
|
+
removeWorktree(worktreeOwner.brainPath, worktreeOwner.id).catch((err) => {
|
|
584
|
+
sessionLog.warn(`[Worktree] Cleanup failed (will be retried by orphan janitor): ${err instanceof Error ? err.message : err}`);
|
|
585
|
+
});
|
|
546
586
|
}
|
|
547
|
-
sessionLog.info('Session ID already in use (stale lock), retrying with fresh session ID');
|
|
548
|
-
const { randomUUID } = await import('crypto');
|
|
549
|
-
const freshId = randomUUID();
|
|
550
|
-
const freshResult = await runClaudeCode(msg, sessionLog, cwd, freshId, true, lastMessageTs, onMessageSent, typingIndicator);
|
|
551
|
-
return { ...freshResult, newSessionId: freshId };
|
|
552
587
|
}
|
|
553
|
-
return retryResult;
|
|
554
588
|
}
|
|
555
589
|
const activeProcesses = new Map();
|
|
556
590
|
const activeProcessesByThread = new Map();
|
package/dist/config.d.ts
CHANGED
|
@@ -28,6 +28,7 @@ declare const configSchema: z.ZodObject<{
|
|
|
28
28
|
BASE_BRAIN_PATH: z.ZodDefault<z.ZodString>;
|
|
29
29
|
BRAIN_AUTO_UPDATE: z.ZodEffects<z.ZodDefault<z.ZodString>, boolean, string | undefined>;
|
|
30
30
|
BRAIN_UPDATE_COOLDOWN_MS: z.ZodDefault<z.ZodNumber>;
|
|
31
|
+
POLLER_TZ: z.ZodDefault<z.ZodString>;
|
|
31
32
|
HEARTBEAT_CRON: z.ZodDefault<z.ZodString>;
|
|
32
33
|
MAINTENANCE_CRON: z.ZodDefault<z.ZodString>;
|
|
33
34
|
MAINTENANCE_ENABLED: z.ZodEffects<z.ZodDefault<z.ZodString>, boolean, string | undefined>;
|
|
@@ -58,6 +59,7 @@ declare const configSchema: z.ZodObject<{
|
|
|
58
59
|
BASE_BRAIN_PATH: string;
|
|
59
60
|
BRAIN_AUTO_UPDATE: boolean;
|
|
60
61
|
BRAIN_UPDATE_COOLDOWN_MS: number;
|
|
62
|
+
POLLER_TZ: string;
|
|
61
63
|
HEARTBEAT_CRON: string;
|
|
62
64
|
MAINTENANCE_CRON: string;
|
|
63
65
|
MAINTENANCE_ENABLED: boolean;
|
|
@@ -94,6 +96,7 @@ declare const configSchema: z.ZodObject<{
|
|
|
94
96
|
BASE_BRAIN_PATH?: string | undefined;
|
|
95
97
|
BRAIN_AUTO_UPDATE?: string | undefined;
|
|
96
98
|
BRAIN_UPDATE_COOLDOWN_MS?: number | undefined;
|
|
99
|
+
POLLER_TZ?: string | undefined;
|
|
97
100
|
HEARTBEAT_CRON?: string | undefined;
|
|
98
101
|
MAINTENANCE_CRON?: string | undefined;
|
|
99
102
|
MAINTENANCE_ENABLED?: string | undefined;
|
|
@@ -127,6 +130,7 @@ export declare const config: {
|
|
|
127
130
|
BASE_BRAIN_PATH: string;
|
|
128
131
|
BRAIN_AUTO_UPDATE: boolean;
|
|
129
132
|
BRAIN_UPDATE_COOLDOWN_MS: number;
|
|
133
|
+
POLLER_TZ: string;
|
|
130
134
|
HEARTBEAT_CRON: string;
|
|
131
135
|
MAINTENANCE_CRON: string;
|
|
132
136
|
MAINTENANCE_ENABLED: boolean;
|
package/dist/config.js
CHANGED
|
@@ -44,8 +44,13 @@ const configSchema = z.object({
|
|
|
44
44
|
.transform((v) => v.toLowerCase() === 'true'),
|
|
45
45
|
BRAIN_UPDATE_COOLDOWN_MS: z.coerce.number().default(300000), // 5 minutes
|
|
46
46
|
// Brain heartbeat & maintenance (cron-based)
|
|
47
|
+
// POLLER_TZ: IANA timezone used to interpret HEARTBEAT_CRON & MAINTENANCE_CRON.
|
|
48
|
+
// Default Europe/Prague — current fleet is CZ-based; ensures `30 3 * * *`
|
|
49
|
+
// means "03:30 local night," not "03:30 UTC = 05:30 CEST." DST is handled
|
|
50
|
+
// by croner automatically. Override with `POLLER_TZ=UTC` (or any IANA name).
|
|
51
|
+
POLLER_TZ: z.string().default('Europe/Prague'),
|
|
47
52
|
HEARTBEAT_CRON: z.string().default('0 */6 * * *'), // Every 6 hours
|
|
48
|
-
MAINTENANCE_CRON: z.string().default('30 3 * * *'), // Daily at 03:30
|
|
53
|
+
MAINTENANCE_CRON: z.string().default('30 3 * * *'), // Daily at 03:30 in POLLER_TZ
|
|
49
54
|
MAINTENANCE_ENABLED: z.string().default('true').transform((v) => v.toLowerCase() === 'true'),
|
|
50
55
|
// Auto-update settings
|
|
51
56
|
AUTO_UPDATE_POLICY: z.enum(['patch', 'minor', 'off']).default('patch'),
|
|
@@ -114,13 +114,13 @@ export function startHeartbeatLoop(processMessageFn) {
|
|
|
114
114
|
return;
|
|
115
115
|
}
|
|
116
116
|
// Heartbeat cron — frequent, lightweight
|
|
117
|
-
logger.info(`[Heartbeat] Starting heartbeat cron: "${config.HEARTBEAT_CRON}"`);
|
|
118
|
-
heartbeatJob = new Cron(config.HEARTBEAT_CRON, { timezone:
|
|
117
|
+
logger.info(`[Heartbeat] Starting heartbeat cron: "${config.HEARTBEAT_CRON}" (tz: ${config.POLLER_TZ})`);
|
|
118
|
+
heartbeatJob = new Cron(config.HEARTBEAT_CRON, { timezone: config.POLLER_TZ, protect: true }, async () => {
|
|
119
119
|
await runHeartbeatJob(processMessageFn);
|
|
120
120
|
});
|
|
121
121
|
// Maintenance cron — daily, heavy (consolidation, memory cleanup)
|
|
122
|
-
logger.info(`[Maintenance] Starting maintenance cron: "${config.MAINTENANCE_CRON}"`);
|
|
123
|
-
maintenanceJob = new Cron(config.MAINTENANCE_CRON, { timezone:
|
|
122
|
+
logger.info(`[Maintenance] Starting maintenance cron: "${config.MAINTENANCE_CRON}" (tz: ${config.POLLER_TZ})`);
|
|
123
|
+
maintenanceJob = new Cron(config.MAINTENANCE_CRON, { timezone: config.POLLER_TZ, protect: true }, async () => {
|
|
124
124
|
await runMaintenanceJob(processMessageFn);
|
|
125
125
|
});
|
|
126
126
|
}
|
package/dist/logger.d.ts
CHANGED
|
@@ -5,6 +5,7 @@ export declare class SessionLogger {
|
|
|
5
5
|
private write;
|
|
6
6
|
info(message: string): void;
|
|
7
7
|
error(message: string): void;
|
|
8
|
+
warn(message: string): void;
|
|
8
9
|
claude(stream: 'stdout' | 'stderr', message: string): void;
|
|
9
10
|
logPrompt(prompt: string): void;
|
|
10
11
|
getLogFile(): string;
|
package/dist/logger.js
CHANGED
|
@@ -56,6 +56,12 @@ export class SessionLogger {
|
|
|
56
56
|
this.write('ERROR', message, ts);
|
|
57
57
|
writeToFile('ERROR', `[${this.sessionId}] ${message}`, ts);
|
|
58
58
|
}
|
|
59
|
+
warn(message) {
|
|
60
|
+
const ts = now();
|
|
61
|
+
console.warn(`[${ts}] [${this.sessionId}] ${message}`);
|
|
62
|
+
this.write('WARN', message, ts);
|
|
63
|
+
writeToFile('WARN', `[${this.sessionId}] ${message}`, ts);
|
|
64
|
+
}
|
|
59
65
|
claude(stream, message) {
|
|
60
66
|
const ts = now();
|
|
61
67
|
const prefix = `[Claude ${stream}]`;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { readdir, readFile, unlink } from 'fs/promises';
|
|
1
|
+
import { readdir, readFile, unlink, writeFile } from 'fs/promises';
|
|
2
2
|
import { join } from 'path';
|
|
3
3
|
import { execFile } from 'child_process';
|
|
4
4
|
import { promisify } from 'util';
|
|
5
|
-
import { config } from './config.js';
|
|
5
|
+
import { config, POLLER_VERSION } from './config.js';
|
|
6
6
|
import { logger } from './logger.js';
|
|
7
7
|
import { emitError } from './event-queue.js';
|
|
8
|
+
import { getBaseBrainCommitSha, getBaseBrainAgeHours } from './base-brain-metadata.js';
|
|
8
9
|
const execFileAsync = promisify(execFile);
|
|
9
10
|
const OPERATION_TO_EVENT_TYPE = {
|
|
10
11
|
consolidation: 'maintenance.consolidation',
|
|
@@ -25,8 +26,14 @@ export async function postMaintenanceReports(brainPath, channelId) {
|
|
|
25
26
|
if (jsonFiles.length === 0)
|
|
26
27
|
return;
|
|
27
28
|
logger.info(`[MaintenanceReporter] Found ${jsonFiles.length} JSON report(s) in ${reportsDir}`);
|
|
29
|
+
// Fleet visibility fields — sampled once per run. See teamvibe.ai#129.
|
|
30
|
+
const [baseBrainCommitSha, baseBrainAgeHours] = await Promise.all([
|
|
31
|
+
getBaseBrainCommitSha(config.BASE_BRAIN_PATH),
|
|
32
|
+
getBaseBrainAgeHours(config.BASE_BRAIN_PATH),
|
|
33
|
+
]);
|
|
28
34
|
const postedFiles = [];
|
|
29
35
|
const removedFiles = [];
|
|
36
|
+
let consolidationGuardTouched = false;
|
|
30
37
|
for (const file of jsonFiles) {
|
|
31
38
|
const filePath = join(reportsDir, file);
|
|
32
39
|
try {
|
|
@@ -50,6 +57,9 @@ export async function postMaintenanceReports(brainPath, channelId) {
|
|
|
50
57
|
const payload = {
|
|
51
58
|
eventType,
|
|
52
59
|
channelId,
|
|
60
|
+
pollerVersion: POLLER_VERSION,
|
|
61
|
+
baseBrainCommitSha,
|
|
62
|
+
baseBrainAgeHours,
|
|
53
63
|
metadata: report,
|
|
54
64
|
};
|
|
55
65
|
const response = await fetch(`${config.TEAMVIBE_API_URL}/events`, {
|
|
@@ -64,6 +74,23 @@ export async function postMaintenanceReports(brainPath, channelId) {
|
|
|
64
74
|
logger.info(`[MaintenanceReporter] Posted ${file} as ${eventType}`);
|
|
65
75
|
await unlink(filePath).catch(() => { });
|
|
66
76
|
postedFiles.push(`reports/${file}`);
|
|
77
|
+
// Touch the consolidation guard file so the poller-local
|
|
78
|
+
// `daysSinceMaintenance` metric reflects this successful run.
|
|
79
|
+
// The metric reads `stat(memory/.last_consolidation).mtimeMs`, so any
|
|
80
|
+
// skip of the consolidation skill's "update timestamp" step causes
|
|
81
|
+
// silent drift. See poller-brain#109, poller-brain-eval#150.
|
|
82
|
+
if (report.operationType === 'consolidation') {
|
|
83
|
+
const guardPath = join(brainPath, 'memory', '.last_consolidation');
|
|
84
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
85
|
+
try {
|
|
86
|
+
await writeFile(guardPath, `${today}\n`);
|
|
87
|
+
consolidationGuardTouched = true;
|
|
88
|
+
logger.info(`[MaintenanceReporter] Touched ${guardPath}`);
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
logger.warn(`[MaintenanceReporter] Failed to touch ${guardPath}: ${err instanceof Error ? err.message : err}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
67
94
|
}
|
|
68
95
|
else {
|
|
69
96
|
const errorBody = await response.text().catch(() => 'unknown');
|
|
@@ -76,13 +103,24 @@ export async function postMaintenanceReports(brainPath, channelId) {
|
|
|
76
103
|
emitError('maintenance', `Error processing report ${file}: ${err instanceof Error ? err.message : String(err)}`, { severity: 'warn', channelId });
|
|
77
104
|
}
|
|
78
105
|
}
|
|
79
|
-
// Remove posted/invalid JSON reports from git so they don't resurrect on next pull
|
|
106
|
+
// Remove posted/invalid JSON reports from git so they don't resurrect on next pull,
|
|
107
|
+
// and stage the consolidation guard file if it was touched this run.
|
|
80
108
|
const allRemovedFromGit = [...postedFiles, ...removedFiles];
|
|
81
|
-
if (allRemovedFromGit.length > 0) {
|
|
109
|
+
if (allRemovedFromGit.length > 0 || consolidationGuardTouched) {
|
|
82
110
|
try {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
111
|
+
if (allRemovedFromGit.length > 0) {
|
|
112
|
+
await execFileAsync('git', ['rm', '--cached', '--ignore-unmatch', ...allRemovedFromGit], { cwd: brainPath });
|
|
113
|
+
}
|
|
114
|
+
if (consolidationGuardTouched) {
|
|
115
|
+
await execFileAsync('git', ['add', 'memory/.last_consolidation'], { cwd: brainPath });
|
|
116
|
+
}
|
|
117
|
+
const message = consolidationGuardTouched && allRemovedFromGit.length > 0
|
|
118
|
+
? 'chore: process maintenance reports and refresh consolidation guard'
|
|
119
|
+
: consolidationGuardTouched
|
|
120
|
+
? 'chore: refresh consolidation guard'
|
|
121
|
+
: 'chore: remove processed maintenance reports';
|
|
122
|
+
await execFileAsync('git', ['commit', '-m', message], { cwd: brainPath });
|
|
123
|
+
logger.info(`[MaintenanceReporter] Committed ${allRemovedFromGit.length} report removal(s)${consolidationGuardTouched ? ' + guard refresh' : ''}`);
|
|
86
124
|
}
|
|
87
125
|
catch (err) {
|
|
88
126
|
logger.debug(`[MaintenanceReporter] Could not git-rm reports (may not be a git repo): ${err instanceof Error ? err.message : err}`);
|
package/dist/poller.d.ts
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
interface ReleaseLogger {
|
|
2
|
+
info: (msg: string) => void;
|
|
3
|
+
warn: (msg: string) => void;
|
|
4
|
+
}
|
|
5
|
+
export declare function releaseAllActiveLocks(locks: Map<string, string>, release: (threadId: string, lockToken: string) => Promise<void>, log?: ReleaseLogger): Promise<void>;
|
|
1
6
|
/**
|
|
2
7
|
* Run the auto-update cron handler with telemetry instrumentation (#119).
|
|
3
8
|
* Exported for tests that fire the handler manually.
|
|
@@ -9,3 +14,4 @@
|
|
|
9
14
|
*/
|
|
10
15
|
export declare function runAutoUpdateJob(checkForUpdateFn?: () => Promise<boolean>): Promise<void>;
|
|
11
16
|
export declare function startPoller(): Promise<void>;
|
|
17
|
+
export {};
|
package/dist/poller.js
CHANGED
|
@@ -5,6 +5,7 @@ import { spawnClaudeCode, isAtCapacity, getActiveProcessCount, killAllActiveProc
|
|
|
5
5
|
import { sendSlackError, addReaction, getUserInfo, startTypingIndicator } from './slack-client.js';
|
|
6
6
|
import { acquireSessionLock, releaseSessionLock, updateSessionId } from './session-store.js';
|
|
7
7
|
import { getBrainPath, ensureDirectories, ensureBaseBrain, pushBrainChanges } from './brain-manager.js';
|
|
8
|
+
import { cleanupOrphanWorktreesForAllBrains, isWorktreeIsolationEnabled } from './worktree-manager.js';
|
|
8
9
|
import { postMaintenanceReports } from './maintenance-reporter.js';
|
|
9
10
|
import { initAuth, stopRefresh } from './auth-provider.js';
|
|
10
11
|
import { registerBrain, startHeartbeatLoop, stopHeartbeatLoop } from './heartbeat-manager.js';
|
|
@@ -21,6 +22,23 @@ import { platform } from 'os';
|
|
|
21
22
|
const execAsync = promisify(exec);
|
|
22
23
|
// Track active message processing
|
|
23
24
|
const processingMessages = new Set();
|
|
25
|
+
// Track DynamoDB session locks held by in-flight runInboxLoop iterations.
|
|
26
|
+
// On SIGTERM/SIGINT, shutdown() releases these so the next message arriving for
|
|
27
|
+
// the same thread doesn't block on the 15-min STALE_LOCK_TIMEOUT_MS.
|
|
28
|
+
// teamvibe.ai#106 (Tier 1)
|
|
29
|
+
const activeLocks = new Map();
|
|
30
|
+
// Exported for testing. Releases every entry in `locks` via `release`,
|
|
31
|
+
// using Promise.allSettled so one bad release doesn't abort the others.
|
|
32
|
+
// `locks` is cleared on completion regardless of per-entry outcome.
|
|
33
|
+
export async function releaseAllActiveLocks(locks, release, log = logger) {
|
|
34
|
+
if (locks.size === 0)
|
|
35
|
+
return;
|
|
36
|
+
log.info(`Releasing ${locks.size} active session lock(s)...`);
|
|
37
|
+
await Promise.allSettled(Array.from(locks.entries()).map(([threadId, lockToken]) => release(threadId, lockToken).catch((err) => {
|
|
38
|
+
log.warn(`Failed to release lock for ${threadId} during shutdown: ${err instanceof Error ? err.message : err}`);
|
|
39
|
+
})));
|
|
40
|
+
locks.clear();
|
|
41
|
+
}
|
|
24
42
|
// Poller auto-update cron (separate from brain heartbeats)
|
|
25
43
|
let autoUpdateJob = null;
|
|
26
44
|
/**
|
|
@@ -127,6 +145,9 @@ async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
127
145
|
}
|
|
128
146
|
}
|
|
129
147
|
const { session, lockToken } = lockResult;
|
|
148
|
+
if (lockToken) {
|
|
149
|
+
activeLocks.set(threadId, lockToken);
|
|
150
|
+
}
|
|
130
151
|
let currentSessionId = session.session_id || undefined;
|
|
131
152
|
while (true) {
|
|
132
153
|
const message = pickNextMessage(kbPath, threadId);
|
|
@@ -239,11 +260,13 @@ async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
239
260
|
if (lockToken) {
|
|
240
261
|
const lastMessageTs = queueMessage.response_context.slack?.message_ts;
|
|
241
262
|
await releaseSessionLock(threadId, lockToken, 'idle', lastMessageTs);
|
|
263
|
+
activeLocks.delete(threadId);
|
|
242
264
|
}
|
|
243
265
|
sessionLog.info(`[Loop] Complete. Iterations: ${iteration}`);
|
|
244
266
|
}
|
|
245
267
|
finally {
|
|
246
268
|
activeLoops.delete(threadId);
|
|
269
|
+
activeLocks.delete(threadId);
|
|
247
270
|
}
|
|
248
271
|
}
|
|
249
272
|
async function processMessage(received) {
|
|
@@ -484,6 +507,10 @@ async function shutdown(signal, exitCode = 0) {
|
|
|
484
507
|
stopEventQueue();
|
|
485
508
|
// Kill all active Claude child processes
|
|
486
509
|
await killAllActiveProcesses();
|
|
510
|
+
// Release any DynamoDB session locks held by interrupted runInboxLoop
|
|
511
|
+
// iterations. Without this, the next message arriving for the same thread
|
|
512
|
+
// would block until STALE_LOCK_TIMEOUT_MS (15 min) elapses. teamvibe.ai#106
|
|
513
|
+
await releaseAllActiveLocks(activeLocks, (threadId, lockToken) => releaseSessionLock(threadId, lockToken, 'idle'));
|
|
487
514
|
// Wait for processingMessages to drain (with timeout)
|
|
488
515
|
const shutdownTimeout = 30000;
|
|
489
516
|
const startTime = Date.now();
|
|
@@ -517,6 +544,11 @@ export async function startPoller() {
|
|
|
517
544
|
startEventQueue();
|
|
518
545
|
await ensureDirectories();
|
|
519
546
|
await ensureBaseBrain();
|
|
547
|
+
if (isWorktreeIsolationEnabled()) {
|
|
548
|
+
logger.info(' Worktree isolation: ENABLED (POLLER_USE_WORKTREES=true)');
|
|
549
|
+
const sweep = await cleanupOrphanWorktreesForAllBrains(config.BRAINS_PATH);
|
|
550
|
+
logger.info(` Orphan worktree sweep: scanned ${sweep.scanned} brains, removed ${sweep.removed} stale worktrees`);
|
|
551
|
+
}
|
|
520
552
|
startHeartbeatLoop(processMessage);
|
|
521
553
|
startAutoUpdateCron();
|
|
522
554
|
startMetricsCron();
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Create a fresh worktree for a spawned session. Returns the absolute path of
|
|
3
|
+
* the worktree, which the caller should use as the spawned process's `cwd`.
|
|
4
|
+
*
|
|
5
|
+
* Idempotent against existing worktrees at the same path: if one is found,
|
|
6
|
+
* it's removed and recreated to guarantee a clean start.
|
|
7
|
+
*/
|
|
8
|
+
export declare function createWorktree(brainPath: string, sessionId: string, branch?: string): Promise<string>;
|
|
9
|
+
/**
|
|
10
|
+
* Remove a worktree by session ID. Force-removes so dirty state from a crashed
|
|
11
|
+
* session doesn't block cleanup — the contract is that any session work the
|
|
12
|
+
* caller wanted to preserve has already been pushed to `origin/<branch>`.
|
|
13
|
+
*/
|
|
14
|
+
export declare function removeWorktree(brainPath: string, sessionId: string): Promise<void>;
|
|
15
|
+
/**
|
|
16
|
+
* Sweep orphan worktrees left behind by crashed sessions. Removes every
|
|
17
|
+
* worktree under `<brainPath>/.worktrees/` whose directory mtime is older
|
|
18
|
+
* than `maxAgeMs`. Intended to run once at poller startup.
|
|
19
|
+
*/
|
|
20
|
+
export declare function cleanupOrphanWorktrees(brainPath: string, maxAgeMs?: number): Promise<{
|
|
21
|
+
removed: string[];
|
|
22
|
+
}>;
|
|
23
|
+
/**
|
|
24
|
+
* Sweep orphan worktrees across every brain directory under `brainsRoot`.
|
|
25
|
+
* Intended to run once at poller startup. Tolerates the entries that aren't
|
|
26
|
+
* brain repos (no `.git` subdir) by skipping them silently.
|
|
27
|
+
*/
|
|
28
|
+
export declare function cleanupOrphanWorktreesForAllBrains(brainsRoot: string, maxAgeMs?: number): Promise<{
|
|
29
|
+
scanned: number;
|
|
30
|
+
removed: number;
|
|
31
|
+
}>;
|
|
32
|
+
/**
|
|
33
|
+
* Whether the spawner should isolate spawned sessions in worktrees. Gated by
|
|
34
|
+
* env var `POLLER_USE_WORKTREES`; defaults off until the rollout is verified.
|
|
35
|
+
*/
|
|
36
|
+
export declare function isWorktreeIsolationEnabled(): boolean;
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { mkdir, readdir, stat, appendFile, readFile } from 'fs/promises';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { execFile } from 'child_process';
|
|
4
|
+
import { promisify } from 'util';
|
|
5
|
+
const execFileAsync = promisify(execFile);
|
|
6
|
+
const WORKTREES_SUBDIR = '.worktrees';
|
|
7
|
+
const EXCLUDE_ENTRY = `${WORKTREES_SUBDIR}/\n`;
|
|
8
|
+
/**
|
|
9
|
+
* Spawned (non-foreground) Claude sessions run in isolated git worktrees off
|
|
10
|
+
* the brain repo to avoid the sync-watchdog race in teamvibe.ai#120.
|
|
11
|
+
*
|
|
12
|
+
* Layout:
|
|
13
|
+
*
|
|
14
|
+
* <brainPath>/ ← main working dir (foreground)
|
|
15
|
+
* <brainPath>/.worktrees/<sessionId>/ ← detached-HEAD worktree (spawned)
|
|
16
|
+
*
|
|
17
|
+
* The worktree is created from the current `origin/<branch>` tip in detached
|
|
18
|
+
* HEAD so the spawned session can commit + `git push origin HEAD:<branch>`
|
|
19
|
+
* without conflicting with the main worktree's branch checkout. The sync
|
|
20
|
+
* watchdog runs only against the main worktree and never traverses
|
|
21
|
+
* `.worktrees/*`, so commits + in-flight tool writes in a worktree are safe.
|
|
22
|
+
*/
|
|
23
|
+
async function ensureExclude(brainPath) {
|
|
24
|
+
const excludePath = join(brainPath, '.git', 'info', 'exclude');
|
|
25
|
+
let current = '';
|
|
26
|
+
try {
|
|
27
|
+
current = await readFile(excludePath, 'utf-8');
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
// .git/info/exclude may not exist (rare); fall back to creating it
|
|
31
|
+
}
|
|
32
|
+
if (!current.split('\n').includes(WORKTREES_SUBDIR + '/')) {
|
|
33
|
+
await appendFile(excludePath, EXCLUDE_ENTRY);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Create a fresh worktree for a spawned session. Returns the absolute path of
|
|
38
|
+
* the worktree, which the caller should use as the spawned process's `cwd`.
|
|
39
|
+
*
|
|
40
|
+
* Idempotent against existing worktrees at the same path: if one is found,
|
|
41
|
+
* it's removed and recreated to guarantee a clean start.
|
|
42
|
+
*/
|
|
43
|
+
export async function createWorktree(brainPath, sessionId, branch = 'main') {
|
|
44
|
+
await mkdir(join(brainPath, WORKTREES_SUBDIR), { recursive: true });
|
|
45
|
+
await ensureExclude(brainPath);
|
|
46
|
+
const worktreePath = join(brainPath, WORKTREES_SUBDIR, sessionId);
|
|
47
|
+
// Defensive cleanup in case a previous attempt left an orphan
|
|
48
|
+
try {
|
|
49
|
+
await execFileAsync('git', ['worktree', 'remove', '--force', worktreePath], { cwd: brainPath });
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
// Not registered as a worktree — fine
|
|
53
|
+
}
|
|
54
|
+
await execFileAsync('git', ['fetch', 'origin', branch, '--quiet'], { cwd: brainPath });
|
|
55
|
+
await execFileAsync('git', ['worktree', 'add', '--detach', worktreePath, `origin/${branch}`], { cwd: brainPath });
|
|
56
|
+
return worktreePath;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Remove a worktree by session ID. Force-removes so dirty state from a crashed
|
|
60
|
+
* session doesn't block cleanup — the contract is that any session work the
|
|
61
|
+
* caller wanted to preserve has already been pushed to `origin/<branch>`.
|
|
62
|
+
*/
|
|
63
|
+
export async function removeWorktree(brainPath, sessionId) {
|
|
64
|
+
const worktreePath = join(brainPath, WORKTREES_SUBDIR, sessionId);
|
|
65
|
+
try {
|
|
66
|
+
await execFileAsync('git', ['worktree', 'remove', '--force', worktreePath], { cwd: brainPath });
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
// Already gone, or never registered — fine
|
|
70
|
+
}
|
|
71
|
+
// Prune any stale administrative bookkeeping in .git/worktrees/
|
|
72
|
+
try {
|
|
73
|
+
await execFileAsync('git', ['worktree', 'prune'], { cwd: brainPath });
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// best-effort
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Sweep orphan worktrees left behind by crashed sessions. Removes every
|
|
81
|
+
* worktree under `<brainPath>/.worktrees/` whose directory mtime is older
|
|
82
|
+
* than `maxAgeMs`. Intended to run once at poller startup.
|
|
83
|
+
*/
|
|
84
|
+
export async function cleanupOrphanWorktrees(brainPath, maxAgeMs = 24 * 60 * 60 * 1000) {
|
|
85
|
+
const worktreesDir = join(brainPath, WORKTREES_SUBDIR);
|
|
86
|
+
const removed = [];
|
|
87
|
+
let entries = [];
|
|
88
|
+
try {
|
|
89
|
+
entries = await readdir(worktreesDir);
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
return { removed }; // dir doesn't exist yet — nothing to do
|
|
93
|
+
}
|
|
94
|
+
const cutoffMs = Date.now() - maxAgeMs;
|
|
95
|
+
for (const entry of entries) {
|
|
96
|
+
const path = join(worktreesDir, entry);
|
|
97
|
+
try {
|
|
98
|
+
const s = await stat(path);
|
|
99
|
+
if (s.mtimeMs < cutoffMs) {
|
|
100
|
+
await removeWorktree(brainPath, entry);
|
|
101
|
+
removed.push(entry);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
// skip entries we can't stat
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return { removed };
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Sweep orphan worktrees across every brain directory under `brainsRoot`.
|
|
112
|
+
* Intended to run once at poller startup. Tolerates the entries that aren't
|
|
113
|
+
* brain repos (no `.git` subdir) by skipping them silently.
|
|
114
|
+
*/
|
|
115
|
+
export async function cleanupOrphanWorktreesForAllBrains(brainsRoot, maxAgeMs = 24 * 60 * 60 * 1000) {
|
|
116
|
+
let entries = [];
|
|
117
|
+
try {
|
|
118
|
+
entries = await readdir(brainsRoot);
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
return { scanned: 0, removed: 0 };
|
|
122
|
+
}
|
|
123
|
+
let scanned = 0;
|
|
124
|
+
let removed = 0;
|
|
125
|
+
for (const entry of entries) {
|
|
126
|
+
const brainPath = join(brainsRoot, entry);
|
|
127
|
+
try {
|
|
128
|
+
const gitStat = await stat(join(brainPath, '.git'));
|
|
129
|
+
if (!gitStat.isDirectory())
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
continue; // not a brain repo
|
|
134
|
+
}
|
|
135
|
+
scanned++;
|
|
136
|
+
const { removed: gone } = await cleanupOrphanWorktrees(brainPath, maxAgeMs);
|
|
137
|
+
removed += gone.length;
|
|
138
|
+
}
|
|
139
|
+
return { scanned, removed };
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Whether the spawner should isolate spawned sessions in worktrees. Gated by
|
|
143
|
+
* env var `POLLER_USE_WORKTREES`; defaults off until the rollout is verified.
|
|
144
|
+
*/
|
|
145
|
+
export function isWorktreeIsolationEnabled() {
|
|
146
|
+
return (process.env['POLLER_USE_WORKTREES'] || '').toLowerCase() === 'true';
|
|
147
|
+
}
|