crewly 1.20.295 → 1.20.297
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/skills/_common/lib.sh +6 -0
- package/config/skills/agent/core/app-data/SKILL.md +13 -1
- package/config/skills/agent/core/app-data/execute.sh +60 -6
- package/config/skills/agent/core/app-data/execute.test.sh +42 -0
- package/config/skills/agent/core/complete-task/SKILL.md +4 -2
- package/config/skills/agent/core/complete-task/execute.sh +7 -0
- package/config/skills/agent/core/complete-task/explicit-id.test.sh +56 -0
- package/config/skills/agent/core/docs-comment/SKILL.md +49 -4
- package/config/skills/agent/core/docs-comment/execute.sh +38 -2
- package/config/skills/agent/core/docs-comment/execute.test.sh +28 -0
- package/config/skills/agent/core/poll-tasks/execute.sh +13 -1
- package/config/skills/agent/core/poll-tasks/execute.test.sh +37 -0
- package/config/skills/team-leader/delegate-task/execute.sh +13 -2
- package/config/skills/team-leader/verify-output/SKILL.md +9 -0
- package/dist/backend/backend/src/constants.d.ts +39 -0
- package/dist/backend/backend/src/constants.d.ts.map +1 -1
- package/dist/backend/backend/src/constants.js +39 -0
- package/dist/backend/backend/src/constants.js.map +1 -1
- package/dist/backend/backend/src/controllers/apps/apps.controller.d.ts +2 -0
- package/dist/backend/backend/src/controllers/apps/apps.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/apps/apps.controller.js +2 -0
- package/dist/backend/backend/src/controllers/apps/apps.controller.js.map +1 -1
- package/dist/backend/backend/src/controllers/apps/apps.routes.d.ts +1 -0
- package/dist/backend/backend/src/controllers/apps/apps.routes.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/apps/apps.routes.js +3 -1
- package/dist/backend/backend/src/controllers/apps/apps.routes.js.map +1 -1
- package/dist/backend/backend/src/controllers/browser/browser.controller.d.ts +20 -0
- package/dist/backend/backend/src/controllers/browser/browser.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/browser/browser.controller.js +52 -9
- package/dist/backend/backend/src/controllers/browser/browser.controller.js.map +1 -1
- package/dist/backend/backend/src/controllers/google/google.controller.d.ts +31 -0
- package/dist/backend/backend/src/controllers/google/google.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/google/google.controller.js +198 -5
- package/dist/backend/backend/src/controllers/google/google.controller.js.map +1 -1
- package/dist/backend/backend/src/controllers/google/google.routes.d.ts +4 -0
- package/dist/backend/backend/src/controllers/google/google.routes.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/google/google.routes.js +9 -1
- package/dist/backend/backend/src/controllers/google/google.routes.js.map +1 -1
- package/dist/backend/backend/src/controllers/monitoring/terminal.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/monitoring/terminal.controller.js +28 -8
- package/dist/backend/backend/src/controllers/monitoring/terminal.controller.js.map +1 -1
- package/dist/backend/backend/src/controllers/slack/slack.controller.d.ts.map +1 -1
- package/dist/backend/backend/src/controllers/slack/slack.controller.js +28 -1
- package/dist/backend/backend/src/controllers/slack/slack.controller.js.map +1 -1
- package/dist/backend/backend/src/index.d.ts.map +1 -1
- package/dist/backend/backend/src/index.js +35 -5
- package/dist/backend/backend/src/index.js.map +1 -1
- package/dist/backend/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/agent/agent-registration.service.js +4 -2
- package/dist/backend/backend/src/services/agent/agent-registration.service.js.map +1 -1
- package/dist/backend/backend/src/services/ai/prompt-modules/skills-reference.module.js +1 -1
- package/dist/backend/backend/src/services/ai/prompt-modules/skills-reference.module.js.map +1 -1
- package/dist/backend/backend/src/services/apps/app-wake-message.d.ts +33 -0
- package/dist/backend/backend/src/services/apps/app-wake-message.d.ts.map +1 -1
- package/dist/backend/backend/src/services/apps/app-wake-message.js +48 -0
- package/dist/backend/backend/src/services/apps/app-wake-message.js.map +1 -1
- package/dist/backend/backend/src/services/apps/apps.service.d.ts +23 -0
- package/dist/backend/backend/src/services/apps/apps.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/apps/apps.service.js +57 -6
- package/dist/backend/backend/src/services/apps/apps.service.js.map +1 -1
- package/dist/backend/backend/src/services/google/docs-comment-gate.d.ts +166 -0
- package/dist/backend/backend/src/services/google/docs-comment-gate.d.ts.map +1 -0
- package/dist/backend/backend/src/services/google/docs-comment-gate.js +176 -0
- package/dist/backend/backend/src/services/google/docs-comment-gate.js.map +1 -0
- package/dist/backend/backend/src/services/google/docs-comments.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/google/docs-comments.service.js +2 -2
- package/dist/backend/backend/src/services/google/docs-comments.service.js.map +1 -1
- package/dist/backend/backend/src/services/google/google-api.client.d.ts +15 -0
- package/dist/backend/backend/src/services/google/google-api.client.d.ts.map +1 -1
- package/dist/backend/backend/src/services/google/google-api.client.js +16 -2
- package/dist/backend/backend/src/services/google/google-api.client.js.map +1 -1
- package/dist/backend/backend/src/services/google/google-reauth.wiring.js +1 -1
- package/dist/backend/backend/src/services/google/google-reauth.wiring.js.map +1 -1
- package/dist/backend/backend/src/services/google/google-workspace-token.service.d.ts +12 -0
- package/dist/backend/backend/src/services/google/google-workspace-token.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/google/google-workspace-token.service.js +18 -0
- package/dist/backend/backend/src/services/google/google-workspace-token.service.js.map +1 -1
- package/dist/backend/backend/src/services/messaging/sub-agent-message-queue.service.d.ts +18 -4
- package/dist/backend/backend/src/services/messaging/sub-agent-message-queue.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/messaging/sub-agent-message-queue.service.js +36 -8
- package/dist/backend/backend/src/services/messaging/sub-agent-message-queue.service.js.map +1 -1
- package/dist/backend/backend/src/services/orc/reply-destination-resolver.d.ts +6 -0
- package/dist/backend/backend/src/services/orc/reply-destination-resolver.d.ts.map +1 -1
- package/dist/backend/backend/src/services/orc/reply-destination-resolver.js +18 -0
- package/dist/backend/backend/src/services/orc/reply-destination-resolver.js.map +1 -1
- package/dist/backend/backend/src/services/orc/reply-destination.wiring.d.ts +2 -0
- package/dist/backend/backend/src/services/orc/reply-destination.wiring.d.ts.map +1 -1
- package/dist/backend/backend/src/services/orc/reply-destination.wiring.js +81 -8
- package/dist/backend/backend/src/services/orc/reply-destination.wiring.js.map +1 -1
- package/dist/backend/backend/src/services/project-tickets/project-ticket-workflow.service.d.ts +8 -0
- package/dist/backend/backend/src/services/project-tickets/project-ticket-workflow.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/project-tickets/project-ticket-workflow.service.js +22 -0
- package/dist/backend/backend/src/services/project-tickets/project-ticket-workflow.service.js.map +1 -1
- package/dist/backend/backend/src/services/session/session-command-helper.d.ts +17 -0
- package/dist/backend/backend/src/services/session/session-command-helper.d.ts.map +1 -1
- package/dist/backend/backend/src/services/session/session-command-helper.js +32 -0
- package/dist/backend/backend/src/services/session/session-command-helper.js.map +1 -1
- package/dist/backend/backend/src/services/slack/orc-owner-dm-target.d.ts +27 -0
- package/dist/backend/backend/src/services/slack/orc-owner-dm-target.d.ts.map +1 -0
- package/dist/backend/backend/src/services/slack/orc-owner-dm-target.js +48 -0
- package/dist/backend/backend/src/services/slack/orc-owner-dm-target.js.map +1 -0
- package/dist/backend/backend/src/services/task-pool/pool-archive-migration.d.ts +18 -0
- package/dist/backend/backend/src/services/task-pool/pool-archive-migration.d.ts.map +1 -1
- package/dist/backend/backend/src/services/task-pool/pool-archive-migration.js +9 -3
- package/dist/backend/backend/src/services/task-pool/pool-archive-migration.js.map +1 -1
- package/dist/backend/backend/src/services/task-pool/pool-prune.d.ts +76 -0
- package/dist/backend/backend/src/services/task-pool/pool-prune.d.ts.map +1 -0
- package/dist/backend/backend/src/services/task-pool/pool-prune.js +129 -0
- package/dist/backend/backend/src/services/task-pool/pool-prune.js.map +1 -0
- package/dist/backend/backend/src/services/task-pool/pool-storage.d.ts +9 -1
- package/dist/backend/backend/src/services/task-pool/pool-storage.d.ts.map +1 -1
- package/dist/backend/backend/src/services/task-pool/pool-storage.js +34 -3
- package/dist/backend/backend/src/services/task-pool/pool-storage.js.map +1 -1
- package/dist/backend/backend/src/services/worktree/runaway-file-watcher.d.ts +131 -0
- package/dist/backend/backend/src/services/worktree/runaway-file-watcher.d.ts.map +1 -0
- package/dist/backend/backend/src/services/worktree/runaway-file-watcher.js +251 -0
- package/dist/backend/backend/src/services/worktree/runaway-file-watcher.js.map +1 -0
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.d.ts +30 -0
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.d.ts.map +1 -1
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.js +82 -0
- package/dist/backend/backend/src/services/worktree/worktree-janitor.service.js.map +1 -1
- package/dist/backend/backend/src/utils/screen-excerpt.d.ts +34 -0
- package/dist/backend/backend/src/utils/screen-excerpt.d.ts.map +1 -0
- package/dist/backend/backend/src/utils/screen-excerpt.js +56 -0
- package/dist/backend/backend/src/utils/screen-excerpt.js.map +1 -0
- package/dist/backend/build-info.json +2 -2
- package/dist/cli/backend/src/constants.d.ts +39 -0
- package/dist/cli/backend/src/constants.d.ts.map +1 -1
- package/dist/cli/backend/src/constants.js +39 -0
- package/dist/cli/backend/src/constants.js.map +1 -1
- package/dist/cli/backend/src/services/agent/agent-registration.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/agent/agent-registration.service.js +4 -2
- package/dist/cli/backend/src/services/agent/agent-registration.service.js.map +1 -1
- package/dist/cli/backend/src/services/ai/prompt-modules/skills-reference.module.js +1 -1
- package/dist/cli/backend/src/services/ai/prompt-modules/skills-reference.module.js.map +1 -1
- package/dist/cli/backend/src/services/messaging/sub-agent-message-queue.service.d.ts +18 -4
- package/dist/cli/backend/src/services/messaging/sub-agent-message-queue.service.d.ts.map +1 -1
- package/dist/cli/backend/src/services/messaging/sub-agent-message-queue.service.js +36 -8
- package/dist/cli/backend/src/services/messaging/sub-agent-message-queue.service.js.map +1 -1
- package/dist/cli/backend/src/services/session/session-command-helper.d.ts +17 -0
- package/dist/cli/backend/src/services/session/session-command-helper.d.ts.map +1 -1
- package/dist/cli/backend/src/services/session/session-command-helper.js +32 -0
- package/dist/cli/backend/src/services/session/session-command-helper.js.map +1 -1
- package/dist/cli/backend/src/services/task-pool/pool-storage.d.ts +9 -1
- package/dist/cli/backend/src/services/task-pool/pool-storage.d.ts.map +1 -1
- package/dist/cli/backend/src/services/task-pool/pool-storage.js +34 -3
- package/dist/cli/backend/src/services/task-pool/pool-storage.js.map +1 -1
- package/dist/cli/backend/src/utils/screen-excerpt.d.ts +34 -0
- package/dist/cli/backend/src/utils/screen-excerpt.d.ts.map +1 -0
- package/dist/cli/backend/src/utils/screen-excerpt.js +56 -0
- package/dist/cli/backend/src/utils/screen-excerpt.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runaway-file watcher — catches a single file that grows without bound in an
|
|
3
|
+
* agent's scratch dir or worktree (crewly#1078: a 40 GB `all.md` filled a Mac
|
|
4
|
+
* before the low-disk alert, which only reports free space, could help).
|
|
5
|
+
*
|
|
6
|
+
* ## What it does, once a minute
|
|
7
|
+
* 1. Walks the watched dirs (Claude scratch session dirs, `crewly-worktrees`,
|
|
8
|
+
* each registered project's `.claude/worktrees` and `.crewly/worktrees`),
|
|
9
|
+
* skipping `node_modules` and `.git`, and `lstat`s regular files only
|
|
10
|
+
* (symlinks are never followed).
|
|
11
|
+
* 2. Compares each file's size with its size at the previous scan. A file is
|
|
12
|
+
* a **runaway** when
|
|
13
|
+
* - it is at least RUNAWAY_FILE_BYTES (5 GB) AND it grew since the last
|
|
14
|
+
* scan, or
|
|
15
|
+
* - it grew by at least RUNAWAY_GROWTH_BYTES_PER_MIN (1 GB/min).
|
|
16
|
+
* 3. For a runaway: stops the process(es) that have the file open (`lsof`,
|
|
17
|
+
* SIGTERM then SIGKILL; never this process or its parent), cuts the file
|
|
18
|
+
* down to its first 1 MB (so the disk is freed and the agent can still
|
|
19
|
+
* see what it was writing), tells the agent working there, and alerts the
|
|
20
|
+
* owner through the existing low-disk alert path. Each path alerts the
|
|
21
|
+
* owner once.
|
|
22
|
+
* 4. A file that is at least 5 GB but NOT growing is only reported to the
|
|
23
|
+
* owner (once): it may be legitimate, so nothing is killed or cut.
|
|
24
|
+
*
|
|
25
|
+
* The truncate step only runs when the file is not open by a process we could
|
|
26
|
+
* not stop. Files outside the watched dirs are never touched.
|
|
27
|
+
*
|
|
28
|
+
* @module services/worktree/runaway-file-watcher
|
|
29
|
+
*/
|
|
30
|
+
/** Why a file was flagged. */
|
|
31
|
+
export type RunawayKind = 'growing' | 'oversize';
|
|
32
|
+
/** One flagged file. */
|
|
33
|
+
export interface RunawayFinding {
|
|
34
|
+
/** Absolute path */
|
|
35
|
+
path: string;
|
|
36
|
+
/** Current size in bytes */
|
|
37
|
+
bytes: number;
|
|
38
|
+
/** Growth since the previous scan, bytes per minute (0 when unknown) */
|
|
39
|
+
growthPerMin: number;
|
|
40
|
+
/** `growing` (acted on) or `oversize` (reported only) */
|
|
41
|
+
kind: RunawayKind;
|
|
42
|
+
}
|
|
43
|
+
/** What was done about a flagged file. */
|
|
44
|
+
export interface RunawayOutcome extends RunawayFinding {
|
|
45
|
+
/** Pids that were sent SIGTERM/SIGKILL */
|
|
46
|
+
stoppedPids: number[];
|
|
47
|
+
/** True when the file was cut down to its head */
|
|
48
|
+
truncated: boolean;
|
|
49
|
+
/** Bytes released by the truncate */
|
|
50
|
+
freedBytes: number;
|
|
51
|
+
/** Agent sessions that were told */
|
|
52
|
+
notifiedAgents: string[];
|
|
53
|
+
/** True when the owner notice was delivered */
|
|
54
|
+
ownerNotified: boolean;
|
|
55
|
+
}
|
|
56
|
+
/** Injectable I/O (defaults talk to the OS). */
|
|
57
|
+
export interface RunawayWatcherDeps {
|
|
58
|
+
/** Directories to scan this tick */
|
|
59
|
+
listRoots: () => Promise<string[]>;
|
|
60
|
+
/** Pids that have `file` open (default `lsof -t`) */
|
|
61
|
+
findWriters?: (file: string) => Promise<number[]>;
|
|
62
|
+
/** Signal a pid (default process.kill) */
|
|
63
|
+
kill?: (pid: number, signal: NodeJS.Signals) => void;
|
|
64
|
+
/** Whether a pid is still alive (default signal 0) */
|
|
65
|
+
isAlive?: (pid: number) => boolean;
|
|
66
|
+
/** Sleep (default setTimeout) */
|
|
67
|
+
sleep?: (ms: number) => Promise<void>;
|
|
68
|
+
/** Tell the agent(s) working near `file`; returns the sessions told */
|
|
69
|
+
notifyAgents?: (file: string, message: string) => Promise<string[]>;
|
|
70
|
+
/** Tell the owner; true when delivered */
|
|
71
|
+
notifyOwner?: (notice: {
|
|
72
|
+
title: string;
|
|
73
|
+
message: string;
|
|
74
|
+
urgent: boolean;
|
|
75
|
+
}) => Promise<boolean>;
|
|
76
|
+
/** Clock (default Date.now) */
|
|
77
|
+
now?: () => number;
|
|
78
|
+
/** Logger */
|
|
79
|
+
logger?: {
|
|
80
|
+
info(m: string, c?: Record<string, unknown>): void;
|
|
81
|
+
warn(m: string, c?: Record<string, unknown>): void;
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Stateful watcher; call {@link RunawayFileWatcher.scan} on a timer.
|
|
86
|
+
*
|
|
87
|
+
* @example
|
|
88
|
+
* ```typescript
|
|
89
|
+
* const w = new RunawayFileWatcher({ listRoots: async () => ['/private/tmp/claude-501'] });
|
|
90
|
+
* setInterval(() => void w.scan(), 60_000);
|
|
91
|
+
* ```
|
|
92
|
+
*/
|
|
93
|
+
export declare class RunawayFileWatcher {
|
|
94
|
+
private readonly deps;
|
|
95
|
+
private seen;
|
|
96
|
+
private alerted;
|
|
97
|
+
/**
|
|
98
|
+
* @param deps - Roots to scan and the side-effect hooks
|
|
99
|
+
*/
|
|
100
|
+
constructor(deps: RunawayWatcherDeps);
|
|
101
|
+
/**
|
|
102
|
+
* One scan: find runaway files, act on them. Never throws.
|
|
103
|
+
*
|
|
104
|
+
* @returns Outcome per flagged file (empty when nothing is wrong)
|
|
105
|
+
*/
|
|
106
|
+
scan(): Promise<RunawayOutcome[]>;
|
|
107
|
+
private walk;
|
|
108
|
+
private handle;
|
|
109
|
+
private readonly notSelf;
|
|
110
|
+
/** SIGTERM the file's writers, SIGKILL those still alive after the grace period. */
|
|
111
|
+
private stopWriters;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Pids of this user's processes that have `file` open.
|
|
115
|
+
*
|
|
116
|
+
* @param file - Absolute path
|
|
117
|
+
* @returns Pids (empty when none, or when `lsof` is unavailable)
|
|
118
|
+
*/
|
|
119
|
+
export declare function defaultFindWriters(file: string): Promise<number[]>;
|
|
120
|
+
/**
|
|
121
|
+
* Directories to watch: Claude scratch roots, `<tmp>/crewly-worktrees`, and
|
|
122
|
+
* each repo's `.claude/worktrees` and `.crewly/worktrees`. Missing dirs are
|
|
123
|
+
* dropped.
|
|
124
|
+
*
|
|
125
|
+
* @param scratchRoots - Claude Code scratch roots
|
|
126
|
+
* @param tmpRoots - Temp roots whose `crewly-worktrees/` holds agent worktrees
|
|
127
|
+
* @param repoPaths - Registered project / repo paths
|
|
128
|
+
* @returns Existing directories to scan
|
|
129
|
+
*/
|
|
130
|
+
export declare function runawayWatchRoots(scratchRoots: string[], tmpRoots: string[], repoPaths: string[]): string[];
|
|
131
|
+
//# sourceMappingURL=runaway-file-watcher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runaway-file-watcher.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/worktree/runaway-file-watcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAOH,8BAA8B;AAC9B,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,UAAU,CAAC;AAEjD,wBAAwB;AACxB,MAAM,WAAW,cAAc;IAC9B,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,wEAAwE;IACxE,YAAY,EAAE,MAAM,CAAC;IACrB,yDAAyD;IACzD,IAAI,EAAE,WAAW,CAAC;CAClB;AAED,0CAA0C;AAC1C,MAAM,WAAW,cAAe,SAAQ,cAAc;IACrD,0CAA0C;IAC1C,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,kDAAkD;IAClD,SAAS,EAAE,OAAO,CAAC;IACnB,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,oCAAoC;IACpC,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,+CAA+C;IAC/C,aAAa,EAAE,OAAO,CAAC;CACvB;AAED,gDAAgD;AAChD,MAAM,WAAW,kBAAkB;IAClC,oCAAoC;IACpC,SAAS,EAAE,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACnC,qDAAqD;IACrD,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,0CAA0C;IAC1C,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC;IACrD,sDAAsD;IACtD,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACnC,iCAAiC;IACjC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,uEAAuE;IACvE,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACpE,0CAA0C;IAC1C,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChG,+BAA+B;IAC/B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,aAAa;IACb,MAAM,CAAC,EAAE;QAAE,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;QAAC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;KAAE,CAAC;CACpH;AAOD;;;;;;;;GAQG;AACH,qBAAa,kBAAkB;IAOlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IANjC,OAAO,CAAC,IAAI,CAA2B;IACvC,OAAO,CAAC,OAAO,CAAqB;IAEpC;;OAEG;gBAC0B,IAAI,EAAE,kBAAkB;IAErD;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAgCzB,IAAI;YAiCJ,MAAM;IA0CpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyE;IAEjG,oFAAoF;YACtE,WAAW;CAyBzB;AAWD;;;;;GAKG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAQxE;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,YAAY,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAe3G"}
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runaway-file watcher — catches a single file that grows without bound in an
|
|
3
|
+
* agent's scratch dir or worktree (crewly#1078: a 40 GB `all.md` filled a Mac
|
|
4
|
+
* before the low-disk alert, which only reports free space, could help).
|
|
5
|
+
*
|
|
6
|
+
* ## What it does, once a minute
|
|
7
|
+
* 1. Walks the watched dirs (Claude scratch session dirs, `crewly-worktrees`,
|
|
8
|
+
* each registered project's `.claude/worktrees` and `.crewly/worktrees`),
|
|
9
|
+
* skipping `node_modules` and `.git`, and `lstat`s regular files only
|
|
10
|
+
* (symlinks are never followed).
|
|
11
|
+
* 2. Compares each file's size with its size at the previous scan. A file is
|
|
12
|
+
* a **runaway** when
|
|
13
|
+
* - it is at least RUNAWAY_FILE_BYTES (5 GB) AND it grew since the last
|
|
14
|
+
* scan, or
|
|
15
|
+
* - it grew by at least RUNAWAY_GROWTH_BYTES_PER_MIN (1 GB/min).
|
|
16
|
+
* 3. For a runaway: stops the process(es) that have the file open (`lsof`,
|
|
17
|
+
* SIGTERM then SIGKILL; never this process or its parent), cuts the file
|
|
18
|
+
* down to its first 1 MB (so the disk is freed and the agent can still
|
|
19
|
+
* see what it was writing), tells the agent working there, and alerts the
|
|
20
|
+
* owner through the existing low-disk alert path. Each path alerts the
|
|
21
|
+
* owner once.
|
|
22
|
+
* 4. A file that is at least 5 GB but NOT growing is only reported to the
|
|
23
|
+
* owner (once): it may be legitimate, so nothing is killed or cut.
|
|
24
|
+
*
|
|
25
|
+
* The truncate step only runs when the file is not open by a process we could
|
|
26
|
+
* not stop. Files outside the watched dirs are never touched.
|
|
27
|
+
*
|
|
28
|
+
* @module services/worktree/runaway-file-watcher
|
|
29
|
+
*/
|
|
30
|
+
import * as fs from 'fs';
|
|
31
|
+
import * as path from 'path';
|
|
32
|
+
import { WORKTREE_JANITOR_CONSTANTS as K } from '../../constants.js';
|
|
33
|
+
import { EventLoopYielder, formatBytes, isPathInside, runCommand } from './worktree-janitor.git.js';
|
|
34
|
+
/**
|
|
35
|
+
* Stateful watcher; call {@link RunawayFileWatcher.scan} on a timer.
|
|
36
|
+
*
|
|
37
|
+
* @example
|
|
38
|
+
* ```typescript
|
|
39
|
+
* const w = new RunawayFileWatcher({ listRoots: async () => ['/private/tmp/claude-501'] });
|
|
40
|
+
* setInterval(() => void w.scan(), 60_000);
|
|
41
|
+
* ```
|
|
42
|
+
*/
|
|
43
|
+
export class RunawayFileWatcher {
|
|
44
|
+
deps;
|
|
45
|
+
seen = new Map();
|
|
46
|
+
alerted = new Set();
|
|
47
|
+
/**
|
|
48
|
+
* @param deps - Roots to scan and the side-effect hooks
|
|
49
|
+
*/
|
|
50
|
+
constructor(deps) {
|
|
51
|
+
this.deps = deps;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* One scan: find runaway files, act on them. Never throws.
|
|
55
|
+
*
|
|
56
|
+
* @returns Outcome per flagged file (empty when nothing is wrong)
|
|
57
|
+
*/
|
|
58
|
+
async scan() {
|
|
59
|
+
const now = (this.deps.now ?? Date.now)();
|
|
60
|
+
const outcomes = [];
|
|
61
|
+
try {
|
|
62
|
+
const roots = await this.deps.listRoots();
|
|
63
|
+
const next = new Map();
|
|
64
|
+
const findings = [];
|
|
65
|
+
const budget = { left: K.RUNAWAY_SCAN_MAX_ENTRIES };
|
|
66
|
+
const yielder = new EventLoopYielder();
|
|
67
|
+
for (const root of roots) {
|
|
68
|
+
await this.walk(root, 0, budget, yielder, async (file, bytes) => {
|
|
69
|
+
if (bytes < K.RUNAWAY_TRACK_MIN_BYTES && !this.seen.has(file))
|
|
70
|
+
return;
|
|
71
|
+
const prev = this.seen.get(file);
|
|
72
|
+
if (bytes >= K.RUNAWAY_TRACK_MIN_BYTES)
|
|
73
|
+
next.set(file, { bytes, at: now });
|
|
74
|
+
const grew = prev ? bytes > prev.bytes : false;
|
|
75
|
+
const minutes = prev ? Math.max((now - prev.at) / 60_000, 1 / 60) : 0;
|
|
76
|
+
const growthPerMin = prev && grew ? Math.round((bytes - prev.bytes) / minutes) : 0;
|
|
77
|
+
if (grew && (bytes >= K.RUNAWAY_FILE_BYTES || growthPerMin >= K.RUNAWAY_GROWTH_BYTES_PER_MIN)) {
|
|
78
|
+
findings.push({ path: file, bytes, growthPerMin, kind: 'growing' });
|
|
79
|
+
}
|
|
80
|
+
else if (bytes >= K.RUNAWAY_FILE_BYTES && !this.alerted.has(file)) {
|
|
81
|
+
findings.push({ path: file, bytes, growthPerMin: 0, kind: 'oversize' });
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
this.seen = next;
|
|
86
|
+
for (const f of findings)
|
|
87
|
+
outcomes.push(await this.handle(f));
|
|
88
|
+
}
|
|
89
|
+
catch (err) {
|
|
90
|
+
this.deps.logger?.warn('Runaway-file scan failed', { error: err instanceof Error ? err.message : String(err) });
|
|
91
|
+
}
|
|
92
|
+
return outcomes;
|
|
93
|
+
}
|
|
94
|
+
async walk(dir, depth, budget, yielder, onFile) {
|
|
95
|
+
if (depth > K.RUNAWAY_SCAN_DEPTH || budget.left <= 0)
|
|
96
|
+
return;
|
|
97
|
+
let entries;
|
|
98
|
+
try {
|
|
99
|
+
entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
for (const e of entries) {
|
|
105
|
+
if (budget.left-- <= 0)
|
|
106
|
+
return;
|
|
107
|
+
await yielder.tick();
|
|
108
|
+
const full = path.join(dir, e.name);
|
|
109
|
+
if (e.isSymbolicLink())
|
|
110
|
+
continue;
|
|
111
|
+
if (e.isDirectory()) {
|
|
112
|
+
if (K.SEARCH_SKIP_DIRS.includes(e.name))
|
|
113
|
+
continue;
|
|
114
|
+
await this.walk(full, depth + 1, budget, yielder, onFile);
|
|
115
|
+
}
|
|
116
|
+
else if (e.isFile()) {
|
|
117
|
+
try {
|
|
118
|
+
const st = await fs.promises.lstat(full);
|
|
119
|
+
if (st.isFile())
|
|
120
|
+
await onFile(full, st.size);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
// vanished between readdir and lstat
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
async handle(f) {
|
|
129
|
+
const out = { ...f, stoppedPids: [], truncated: false, freedBytes: 0, notifiedAgents: [], ownerNotified: false };
|
|
130
|
+
const log = this.deps.logger;
|
|
131
|
+
if (f.kind === 'growing') {
|
|
132
|
+
out.stoppedPids = await this.stopWriters(f.path);
|
|
133
|
+
const stillOpen = (await (this.deps.findWriters ?? defaultFindWriters)(f.path).catch(() => [])).filter(this.notSelf);
|
|
134
|
+
if (stillOpen.length === 0) {
|
|
135
|
+
try {
|
|
136
|
+
const before = (await fs.promises.lstat(f.path)).size;
|
|
137
|
+
await fs.promises.truncate(f.path, Math.min(before, K.RUNAWAY_KEEP_HEAD_BYTES));
|
|
138
|
+
out.truncated = true;
|
|
139
|
+
out.freedBytes = Math.max(0, before - K.RUNAWAY_KEEP_HEAD_BYTES);
|
|
140
|
+
}
|
|
141
|
+
catch (err) {
|
|
142
|
+
log?.warn('Could not cut a runaway file', { path: f.path, error: err instanceof Error ? err.message : String(err) });
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
this.seen.delete(f.path);
|
|
146
|
+
out.notifiedAgents = await (this.deps.notifyAgents ?? (async () => []))(f.path, `[DISK GUARD] ${f.path} was growing without bound (${formatBytes(f.bytes)}, +${formatBytes(f.growthPerMin)}/min). ` +
|
|
147
|
+
`Crewly stopped the process writing it${out.truncated ? ' and cut the file to its first 1 MB' : ''}. ` +
|
|
148
|
+
'Find what was producing the output, fix it, and cap anything you redirect (| head -c 50M).').catch(() => []);
|
|
149
|
+
}
|
|
150
|
+
if (!this.alerted.has(f.path)) {
|
|
151
|
+
this.alerted.add(f.path);
|
|
152
|
+
const what = f.kind === 'growing'
|
|
153
|
+
? `A file was growing out of control in an agent's working dir: ${f.path} (${formatBytes(f.bytes)}${f.growthPerMin ? `, +${formatBytes(f.growthPerMin)}/min` : ''}). ` +
|
|
154
|
+
`Crewly stopped ${out.stoppedPids.length} process(es) writing it${out.truncated ? ` and cut the file, freeing ${formatBytes(out.freedBytes)}` : ' but could not cut it (still open)'}.` +
|
|
155
|
+
(out.notifiedAgents.length ? ` Told: ${out.notifiedAgents.join(', ')}.` : '')
|
|
156
|
+
: `A very large file sits in an agent's working dir: ${f.path} (${formatBytes(f.bytes)}). It is not growing, so nothing was touched; delete it if it is not needed.`;
|
|
157
|
+
out.ownerNotified = await (this.deps.notifyOwner ?? (async () => false))({
|
|
158
|
+
title: f.kind === 'growing' ? 'Runaway file stopped' : 'Very large file in an agent dir',
|
|
159
|
+
message: what,
|
|
160
|
+
urgent: f.kind === 'growing',
|
|
161
|
+
}).catch(() => false);
|
|
162
|
+
}
|
|
163
|
+
log?.warn(`Runaway file (${f.kind}): ${f.path} ${formatBytes(f.bytes)}`, { stoppedPids: out.stoppedPids, truncated: out.truncated });
|
|
164
|
+
return out;
|
|
165
|
+
}
|
|
166
|
+
notSelf = (pid) => pid !== process.pid && pid !== process.ppid;
|
|
167
|
+
/** SIGTERM the file's writers, SIGKILL those still alive after the grace period. */
|
|
168
|
+
async stopWriters(file) {
|
|
169
|
+
const kill = this.deps.kill ?? ((pid, sig) => process.kill(pid, sig));
|
|
170
|
+
const alive = this.deps.isAlive ?? defaultIsAlive;
|
|
171
|
+
const sleep = this.deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
172
|
+
const pids = (await (this.deps.findWriters ?? defaultFindWriters)(file).catch(() => [])).filter(this.notSelf);
|
|
173
|
+
for (const pid of pids) {
|
|
174
|
+
try {
|
|
175
|
+
kill(pid, 'SIGTERM');
|
|
176
|
+
}
|
|
177
|
+
catch {
|
|
178
|
+
// already gone, or not ours
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
if (pids.length > 0) {
|
|
182
|
+
await sleep(K.RUNAWAY_KILL_GRACE_MS);
|
|
183
|
+
for (const pid of pids) {
|
|
184
|
+
if (!alive(pid))
|
|
185
|
+
continue;
|
|
186
|
+
try {
|
|
187
|
+
kill(pid, 'SIGKILL');
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
// ignore
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
return pids;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
function defaultIsAlive(pid) {
|
|
198
|
+
try {
|
|
199
|
+
process.kill(pid, 0);
|
|
200
|
+
return true;
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
return false;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Pids of this user's processes that have `file` open.
|
|
208
|
+
*
|
|
209
|
+
* @param file - Absolute path
|
|
210
|
+
* @returns Pids (empty when none, or when `lsof` is unavailable)
|
|
211
|
+
*/
|
|
212
|
+
export async function defaultFindWriters(file) {
|
|
213
|
+
if (process.platform === 'win32')
|
|
214
|
+
return [];
|
|
215
|
+
const r = await runCommand('lsof', ['-t', '-w', '--', file], { timeoutMs: K.RUNAWAY_LSOF_TIMEOUT_MS });
|
|
216
|
+
if (!r.stdout)
|
|
217
|
+
return [];
|
|
218
|
+
return r.stdout
|
|
219
|
+
.split('\n')
|
|
220
|
+
.map((l) => Number(l.trim()))
|
|
221
|
+
.filter((n) => Number.isInteger(n) && n > 1);
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Directories to watch: Claude scratch roots, `<tmp>/crewly-worktrees`, and
|
|
225
|
+
* each repo's `.claude/worktrees` and `.crewly/worktrees`. Missing dirs are
|
|
226
|
+
* dropped.
|
|
227
|
+
*
|
|
228
|
+
* @param scratchRoots - Claude Code scratch roots
|
|
229
|
+
* @param tmpRoots - Temp roots whose `crewly-worktrees/` holds agent worktrees
|
|
230
|
+
* @param repoPaths - Registered project / repo paths
|
|
231
|
+
* @returns Existing directories to scan
|
|
232
|
+
*/
|
|
233
|
+
export function runawayWatchRoots(scratchRoots, tmpRoots, repoPaths) {
|
|
234
|
+
const candidates = [
|
|
235
|
+
...scratchRoots,
|
|
236
|
+
...tmpRoots.map((r) => path.join(r, K.TMP_WORKTREE_DIR)),
|
|
237
|
+
...repoPaths.flatMap((r) => [path.join(r, K.AGENT_WORKTREE_DIR), path.join(r, K.MANAGED_WORKTREE_DIR)]),
|
|
238
|
+
];
|
|
239
|
+
const out = [];
|
|
240
|
+
for (const c of candidates) {
|
|
241
|
+
try {
|
|
242
|
+
if (fs.statSync(c).isDirectory() && !out.some((o) => isPathInside(c, o)))
|
|
243
|
+
out.push(c);
|
|
244
|
+
}
|
|
245
|
+
catch {
|
|
246
|
+
// missing
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return out;
|
|
250
|
+
}
|
|
251
|
+
//# sourceMappingURL=runaway-file-watcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runaway-file-watcher.js","sourceRoot":"","sources":["../../../../../../backend/src/services/worktree/runaway-file-watcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,0BAA0B,IAAI,CAAC,EAAE,MAAM,oBAAoB,CAAC;AACrE,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AA0DpG;;;;;;;;GAQG;AACH,MAAM,OAAO,kBAAkB;IAOD;IANrB,IAAI,GAAG,IAAI,GAAG,EAAgB,CAAC;IAC/B,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpC;;OAEG;IACH,YAA6B,IAAwB;QAAxB,SAAI,GAAJ,IAAI,CAAoB;IAAG,CAAC;IAEzD;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACT,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAqB,EAAE,CAAC;QACtC,IAAI,CAAC;YACJ,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAgB,CAAC;YACrC,MAAM,QAAQ,GAAqB,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,wBAAwB,EAAE,CAAC;YACpD,MAAM,OAAO,GAAG,IAAI,gBAAgB,EAAE,CAAC;YACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;oBAC/D,IAAI,KAAK,GAAG,CAAC,CAAC,uBAAuB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;wBAAE,OAAO;oBACtE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACjC,IAAI,KAAK,IAAI,CAAC,CAAC,uBAAuB;wBAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;oBAC3E,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;oBAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACtE,MAAM,YAAY,GAAG,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACnF,IAAI,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,kBAAkB,IAAI,YAAY,IAAI,CAAC,CAAC,4BAA4B,CAAC,EAAE,CAAC;wBAC/F,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;oBACrE,CAAC;yBAAM,IAAI,KAAK,IAAI,CAAC,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;wBACrE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;oBACzE,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,KAAK,MAAM,CAAC,IAAI,QAAQ;gBAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjH,CAAC;QACD,OAAO,QAAQ,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,IAAI,CACjB,GAAW,EACX,KAAa,EACb,MAAwB,EACxB,OAAyB,EACzB,MAAsD;QAEtD,IAAI,KAAK,GAAG,CAAC,CAAC,kBAAkB,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC;YAAE,OAAO;QAC7D,IAAI,OAAoB,CAAC;QACzB,IAAI,CAAC;YACJ,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,CAAC;QAAC,MAAM,CAAC;YACR,OAAO;QACR,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC;gBAAE,OAAO;YAC/B,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,CAAC,CAAC,cAAc,EAAE;gBAAE,SAAS;YACjC,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;gBACrB,IAAI,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAClD,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3D,CAAC;iBAAM,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBACvB,IAAI,CAAC;oBACJ,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACzC,IAAI,EAAE,CAAC,MAAM,EAAE;wBAAE,MAAM,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;gBAC9C,CAAC;gBAAC,MAAM,CAAC;oBACR,qCAAqC;gBACtC,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,CAAiB;QACrC,MAAM,GAAG,GAAmB,EAAE,GAAG,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QACjI,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC1B,GAAG,CAAC,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACjD,MAAM,SAAS,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAc,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACjI,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACJ,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;oBACtD,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC;oBAChF,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC;oBACrB,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,uBAAuB,CAAC,CAAC;gBAClE,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,GAAG,EAAE,IAAI,CAAC,8BAA8B,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtH,CAAC;YACF,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACzB,GAAG,CAAC,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CACtE,CAAC,CAAC,IAAI,EACN,gBAAgB,CAAC,CAAC,IAAI,+BAA+B,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS;gBAClH,wCAAwC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,qCAAqC,CAAC,CAAC,CAAC,EAAE,IAAI;gBACtG,4FAA4F,CAC7F,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAc,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACzB,MAAM,IAAI,GACT,CAAC,CAAC,IAAI,KAAK,SAAS;gBACnB,CAAC,CAAC,gEAAgE,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK;oBACrK,kBAAkB,GAAG,CAAC,WAAW,CAAC,MAAM,0BAA0B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,8BAA8B,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,oCAAoC,GAAG;oBACvL,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9E,CAAC,CAAC,qDAAqD,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,8EAA8E,CAAC;YACvK,GAAG,CAAC,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;gBACxE,KAAK,EAAE,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,iCAAiC;gBACxF,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,CAAC,CAAC,IAAI,KAAK,SAAS;aAC5B,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,GAAG,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,IAAI,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QACrI,OAAO,GAAG,CAAC;IACZ,CAAC;IAEgB,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE,CAAC,GAAG,KAAK,OAAO,CAAC,GAAG,IAAI,GAAG,KAAK,OAAO,CAAC,IAAI,CAAC;IAEjG,oFAAoF;IAC5E,KAAK,CAAC,WAAW,CAAC,IAAY;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,cAAc,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QACvF,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,kBAAkB,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAc,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1H,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,CAAC;gBACJ,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;YACtB,CAAC;YAAC,MAAM,CAAC;gBACR,4BAA4B;YAC7B,CAAC;QACF,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,KAAK,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC;YACrC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAC1B,IAAI,CAAC;oBACJ,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;gBACtB,CAAC;gBAAC,MAAM,CAAC;oBACR,SAAS;gBACV,CAAC;YACF,CAAC;QACF,CAAC;QACD,OAAO,IAAI,CAAC;IACb,CAAC;CACD;AAED,SAAS,cAAc,CAAC,GAAW;IAClC,IAAI,CAAC;QACJ,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,KAAK,CAAC;IACd,CAAC;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY;IACpD,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,EAAE,CAAC;IAC5C,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,uBAAuB,EAAE,CAAC,CAAC;IACvG,IAAI,CAAC,CAAC,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IACzB,OAAO,CAAC,CAAC,MAAM;SACb,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SAC5B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAAC,YAAsB,EAAE,QAAkB,EAAE,SAAmB;IAChG,MAAM,UAAU,GAAG;QAClB,GAAG,YAAY;QACf,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;QACxD,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,kBAAkB,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC;KACvG,CAAC;IACF,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,IAAI,CAAC;YACJ,IAAI,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvF,CAAC;QAAC,MAAM,CAAC;YACR,UAAU;QACX,CAAC;IACF,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC"}
|
|
@@ -64,6 +64,15 @@
|
|
|
64
64
|
* the janitor left and why; below 5 GB the notice is urgent and repeats at
|
|
65
65
|
* most every 6 h.
|
|
66
66
|
*
|
|
67
|
+
* ## Runaway files
|
|
68
|
+
* Once a minute a watcher scans agent scratch dirs and worktrees for a single
|
|
69
|
+
* file that grows without bound (a 40 GB `all.md` filled a disk, crewly#1078):
|
|
70
|
+
* a file over 5 GB that is still growing, or any file growing over 1 GB/min,
|
|
71
|
+
* has its writer stopped, is cut to its first 1 MB, the agent working there is
|
|
72
|
+
* told and the owner is alerted once through the low-disk notifier. A big file
|
|
73
|
+
* that is not growing is only reported. See
|
|
74
|
+
* {@link module:services/worktree/runaway-file-watcher}.
|
|
75
|
+
*
|
|
67
76
|
* Runs every 30 minutes (first run 10 minutes after boot). Kill switch:
|
|
68
77
|
* `CREWLY_WORKTREE_JANITOR=0` (also stops the low-disk guard). Never throws;
|
|
69
78
|
* logs one summary line per sweep.
|
|
@@ -73,6 +82,7 @@
|
|
|
73
82
|
import { type StatFsFn } from './low-disk-guard.js';
|
|
74
83
|
import { type ScratchSweepSummary } from './scratch-janitor.js';
|
|
75
84
|
import { type DockerReclaimResult } from './docker-reclaim.js';
|
|
85
|
+
import { type RunawayOutcome } from './runaway-file-watcher.js';
|
|
76
86
|
import { type TraceDiskResult } from '../trace/trace-store.js';
|
|
77
87
|
/**
|
|
78
88
|
* Why a worktree is kept, or why it may go. (`not-agent-worktree` is retired:
|
|
@@ -234,6 +244,10 @@ export declare class WorktreeJanitorService {
|
|
|
234
244
|
private lastPassAt;
|
|
235
245
|
private memoryNoticeState;
|
|
236
246
|
private notifier;
|
|
247
|
+
private agentNotifier;
|
|
248
|
+
private runawayTimer;
|
|
249
|
+
private runawayWatcher;
|
|
250
|
+
private runawayInFlight;
|
|
237
251
|
/**
|
|
238
252
|
* @param options - Injectable dependencies; defaults use the live backend
|
|
239
253
|
*/
|
|
@@ -245,6 +259,22 @@ export declare class WorktreeJanitorService {
|
|
|
245
259
|
* @param notifier - Sender, or null to disable notices
|
|
246
260
|
*/
|
|
247
261
|
setLowDiskNotifier(notifier: LowDiskNotifier | null): void;
|
|
262
|
+
/**
|
|
263
|
+
* Set how an agent is told its working dir had a runaway file (the server
|
|
264
|
+
* wires the normal agent message delivery here).
|
|
265
|
+
*
|
|
266
|
+
* @param notifier - Sender (true when delivered), or null to skip agent notices
|
|
267
|
+
*/
|
|
268
|
+
setAgentNotifier(notifier: ((sessionName: string, message: string) => Promise<boolean>) | null): void;
|
|
269
|
+
/**
|
|
270
|
+
* One runaway-file scan now (also what the minute timer runs). Never
|
|
271
|
+
* throws; overlapping scans are skipped. Honours the kill switch.
|
|
272
|
+
*
|
|
273
|
+
* @returns What was flagged and done (empty when nothing is wrong)
|
|
274
|
+
*/
|
|
275
|
+
checkRunawayFiles(): Promise<RunawayOutcome[]>;
|
|
276
|
+
/** Sessions whose cwd holds `file`, or sits in the same scratch session dir; told via the agent notifier. */
|
|
277
|
+
private notifyAgentsNear;
|
|
248
278
|
/**
|
|
249
279
|
* Process-wide instance used by the server and the HTTP routes.
|
|
250
280
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worktree-janitor.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/worktree/worktree-janitor.service.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"worktree-janitor.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/worktree/worktree-janitor.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgFG;AASH,OAAO,EASN,KAAK,QAAQ,EACb,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAqC,KAAK,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AACnG,OAAO,EAAiB,KAAK,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAyC,KAAK,cAAc,EAAE,MAAM,2BAA2B,CAAC;AACvG,OAAO,EAAiB,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAoB9E;;;GAGG;AACH,MAAM,MAAM,cAAc,GACvB,eAAe,GACf,MAAM,GACN,UAAU,GACV,QAAQ,GACR,+BAA+B,GAC/B,sBAAsB,GACtB,gBAAgB,GAChB,kBAAkB,GAClB,QAAQ,GACR,OAAO,GACP,eAAe,GACf,YAAY,GACZ,QAAQ,GACR,WAAW,CAAC;AAEf,qDAAqD;AACrD,MAAM,WAAW,eAAe;IAC/B,yDAAyD;IACzD,IAAI,EAAE,MAAM,CAAC;IACb,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,kBAAkB;IAClB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,gCAAgC;IAChC,QAAQ,EAAE,QAAQ,GAAG,MAAM,CAAC;IAC5B,wBAAwB;IACxB,MAAM,EAAE,cAAc,CAAC;IACvB,0EAA0E;IAC1E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wCAAwC;AACxC,MAAM,WAAW,eAAgB,SAAQ,eAAe;IACvD,mDAAmD;IACnD,OAAO,EAAE,OAAO,CAAC;IACjB,0EAA0E;IAC1E,MAAM,EAAE,OAAO,CAAC;IAChB,6CAA6C;IAC7C,aAAa,EAAE,OAAO,CAAC;IACvB,4EAA4E;IAC5E,KAAK,CAAC,EAAE,MAAM,CAAC;CACf;AAED,kCAAkC;AAClC,MAAM,WAAW,iBAAiB;IACjC,qBAAqB;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,qBAAqB;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,QAAQ,EAAE,OAAO,CAAC;IAClB,qDAAqD;IACrD,MAAM,EAAE,OAAO,CAAC;IAChB,sCAAsC;IACtC,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,wBAAwB;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,wEAAwE;IACxE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,2BAA2B;IAC3B,SAAS,EAAE,eAAe,EAAE,CAAC;IAC7B,qFAAqF;IACrF,OAAO,EAAE,OAAO,CAAC;IACjB,8EAA8E;IAC9E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,iCAAiC;IACjC,OAAO,EAAE,mBAAmB,GAAG,IAAI,CAAC;IACpC,4EAA4E;IAC5E,MAAM,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;CAChC;AAED,mCAAmC;AACnC,MAAM,WAAW,kBAAkB;IAClC,kDAAkD;IAClD,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,0EAA0E;IAC1E,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,mCAAmC;IACnC,OAAO,EAAE,OAAO,CAAC;IACjB,gCAAgC;IAChC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;CACrC;AAED,uEAAuE;AACvE,MAAM,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAEhH,qDAAqD;AACrD,MAAM,WAAW,OAAO;IACvB,yBAAyB;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,uEAAuE;IACvE,MAAM,EAAE,eAAe,GAAG,SAAS,CAAC;CACpC;AAED,uDAAuD;AACvD,MAAM,WAAW,aAAa;IAC7B,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC/D,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC/D;AAED,mEAAmE;AACnE,MAAM,WAAW,sBAAsB;IACtC,4EAA4E;IAC5E,aAAa,CAAC,EAAE,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACxC,oGAAoG;IACpG,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IAC/C,qCAAqC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,+BAA+B;IAC/B,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,0EAA0E;IAC1E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kEAAkE;IAClE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iIAAiI;IACjI,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,mFAAmF;IACnF,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,oDAAoD;IACpD,MAAM,CAAC,EAAE,QAAQ,CAAC;IAClB,yDAAyD;IACzD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6FAA6F;IAC7F,WAAW,CAAC,EAAE,eAAe,GAAG,IAAI,CAAC;IACrC,6DAA6D;IAC7D,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC/C,qGAAqG;IACrG,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,0FAA0F;IAC1F,aAAa,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,mBAAmB,CAAC,CAAC,GAAG,IAAI,CAAC;IAC5D,4DAA4D;IAC5D,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,2DAA2D;IAC3D,MAAM,CAAC,EAAE,aAAa,CAAC;CACvB;AAUD;;;;;;;;;;;GAWG;AACH,qBAAa,sBAAsB;IAClC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAuC;IAE9D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAyB;IAC9C,OAAO,CAAC,aAAa,CAA+B;IACpD,OAAO,CAAC,aAAa,CAA+B;IACpD,OAAO,CAAC,QAAQ,CAA2C;IAC3D,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,WAAW,CAAkC;IACrD,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,iBAAiB,CAA4C;IACrE,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,iBAAiB,CAA0B;IACnD,OAAO,CAAC,QAAQ,CAAyB;IACzC,OAAO,CAAC,aAAa,CAA6E;IAClG,OAAO,CAAC,YAAY,CAA+B;IACnD,OAAO,CAAC,cAAc,CAAmC;IACzD,OAAO,CAAC,eAAe,CAAS;IAEhC;;OAEG;gBACS,OAAO,GAAE,sBAA2B;IAKhD;;;;;OAKG;IACH,kBAAkB,CAAC,QAAQ,EAAE,eAAe,GAAG,IAAI,GAAG,IAAI;IAI1D;;;;;OAKG;IACH,gBAAgB,CAAC,QAAQ,EAAE,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI;IAIrG;;;;;OAKG;IACG,iBAAiB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAsBpD,6GAA6G;YAC/F,gBAAgB;IAqB9B;;;;OAIG;IACH,MAAM,CAAC,WAAW,IAAI,sBAAsB;IAK5C,mCAAmC;IACnC,MAAM,CAAC,aAAa,IAAI,IAAI;IAK5B;;;;OAIG;IACH,UAAU,IAAI,OAAO;IAIrB;;;;;;OAMG;IACH,KAAK,IAAI,OAAO;IAgBhB,+BAA+B;IAC/B,IAAI,IAAI,IAAI;IAWZ;;;;OAIG;IACG,UAAU,IAAI,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,KAAK,EAAE,IAAI,GAAG,KAAK,GAAG,UAAU,GAAG,SAAS,CAAA;KAAE,CAAC;IAMrH;;;;;;;OAOG;IACG,SAAS,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAQ9C;;;;OAIG;IACH,cAAc,IAAI,iBAAiB,GAAG,IAAI;IAI1C;;;;;OAKG;IACG,IAAI,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAIxC;;;;;OAKG;IACG,GAAG,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAevC,OAAO,KAAK,MAAM,GAIjB;IAED,OAAO,CAAC,GAAG;IAIX,OAAO,CAAC,QAAQ;IAIhB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,QAAQ;IAIhB,OAAO,CAAC,SAAS;YAKH,eAAe;YAWf,gBAAgB;YAYhB,WAAW;IAsCzB;;;OAGG;YACW,kBAAkB;IAgBhC,2EAA2E;YAC7D,SAAS;IAoBvB,OAAO,CAAC,GAAG;YAIG,IAAI;YAmGJ,YAAY;YAwBZ,YAAY;IAQ1B,+EAA+E;IAC/E,OAAO,CAAC,eAAe;IASvB;;OAEG;YACW,QAAQ;IA0DtB,gGAAgG;YAClF,UAAU;IAyCxB,+EAA+E;YACjE,WAAW;IAwBzB,oFAAoF;YACtE,MAAM;CA6BpB;AAED;;;;;GAKG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAiB9D;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAgBrE"}
|
|
@@ -64,6 +64,15 @@
|
|
|
64
64
|
* the janitor left and why; below 5 GB the notice is urgent and repeats at
|
|
65
65
|
* most every 6 h.
|
|
66
66
|
*
|
|
67
|
+
* ## Runaway files
|
|
68
|
+
* Once a minute a watcher scans agent scratch dirs and worktrees for a single
|
|
69
|
+
* file that grows without bound (a 40 GB `all.md` filled a disk, crewly#1078):
|
|
70
|
+
* a file over 5 GB that is still growing, or any file growing over 1 GB/min,
|
|
71
|
+
* has its writer stopped, is cut to its first 1 MB, the agent working there is
|
|
72
|
+
* told and the owner is alerted once through the low-disk notifier. A big file
|
|
73
|
+
* that is not growing is only reported. See
|
|
74
|
+
* {@link module:services/worktree/runaway-file-watcher}.
|
|
75
|
+
*
|
|
67
76
|
* Runs every 30 minutes (first run 10 minutes after boot). Kill switch:
|
|
68
77
|
* `CREWLY_WORKTREE_JANITOR=0` (also stops the low-disk guard). Never throws;
|
|
69
78
|
* logs one summary line per sweep.
|
|
@@ -80,6 +89,7 @@ import { safeReadJson, modifyJsonFile } from '../../utils/file-io.utils.js';
|
|
|
80
89
|
import { buildLowDiskNotice, diskLevel, idleThreshold, noticeToSend, readFreeBytes, recordNotice, } from './low-disk-guard.js';
|
|
81
90
|
import { defaultScratchRoots, sweepScratch } from './scratch-janitor.js';
|
|
82
91
|
import { reclaimDocker } from './docker-reclaim.js';
|
|
92
|
+
import { RunawayFileWatcher, runawayWatchRoots } from './runaway-file-watcher.js';
|
|
83
93
|
import { getTraceStore } from '../trace/trace-store.js';
|
|
84
94
|
import { canonicalPath, diskUsageBytes, discoverLinkedWorktreeRepos, formatBytes, isJanitorDisabled, isPathInside, latestWorktreeMtimeMs, listProcessCwds, parsePorcelainWorktrees, runCommand, } from './worktree-janitor.git.js';
|
|
85
95
|
/** Milliseconds per minute (for the `recent` detail text). */
|
|
@@ -110,6 +120,10 @@ export class WorktreeJanitorService {
|
|
|
110
120
|
lastPassAt = null;
|
|
111
121
|
memoryNoticeState = {};
|
|
112
122
|
notifier;
|
|
123
|
+
agentNotifier = null;
|
|
124
|
+
runawayTimer = null;
|
|
125
|
+
runawayWatcher = null;
|
|
126
|
+
runawayInFlight = false;
|
|
113
127
|
/**
|
|
114
128
|
* @param options - Injectable dependencies; defaults use the live backend
|
|
115
129
|
*/
|
|
@@ -126,6 +140,69 @@ export class WorktreeJanitorService {
|
|
|
126
140
|
setLowDiskNotifier(notifier) {
|
|
127
141
|
this.notifier = notifier;
|
|
128
142
|
}
|
|
143
|
+
/**
|
|
144
|
+
* Set how an agent is told its working dir had a runaway file (the server
|
|
145
|
+
* wires the normal agent message delivery here).
|
|
146
|
+
*
|
|
147
|
+
* @param notifier - Sender (true when delivered), or null to skip agent notices
|
|
148
|
+
*/
|
|
149
|
+
setAgentNotifier(notifier) {
|
|
150
|
+
this.agentNotifier = notifier;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* One runaway-file scan now (also what the minute timer runs). Never
|
|
154
|
+
* throws; overlapping scans are skipped. Honours the kill switch.
|
|
155
|
+
*
|
|
156
|
+
* @returns What was flagged and done (empty when nothing is wrong)
|
|
157
|
+
*/
|
|
158
|
+
async checkRunawayFiles() {
|
|
159
|
+
if (this.isDisabled() || this.runawayInFlight)
|
|
160
|
+
return [];
|
|
161
|
+
this.runawayInFlight = true;
|
|
162
|
+
try {
|
|
163
|
+
if (!this.runawayWatcher) {
|
|
164
|
+
this.runawayWatcher = new RunawayFileWatcher({
|
|
165
|
+
listRoots: async () => {
|
|
166
|
+
const repos = await (this.opts.listRepoPaths ?? defaultListRepoPaths)().catch(() => []);
|
|
167
|
+
return runawayWatchRoots(this.scratchRoots(), this.tmpRoots(), repos);
|
|
168
|
+
},
|
|
169
|
+
notifyOwner: async (notice) => (this.notifier ? this.notifier(notice) : false),
|
|
170
|
+
notifyAgents: (file, message) => this.notifyAgentsNear(file, message),
|
|
171
|
+
now: () => this.now(),
|
|
172
|
+
logger: this.logger,
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
return await this.runawayWatcher.scan();
|
|
176
|
+
}
|
|
177
|
+
finally {
|
|
178
|
+
this.runawayInFlight = false;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/** Sessions whose cwd holds `file`, or sits in the same scratch session dir; told via the agent notifier. */
|
|
182
|
+
async notifyAgentsNear(file, message) {
|
|
183
|
+
const notifier = this.agentNotifier;
|
|
184
|
+
if (!notifier)
|
|
185
|
+
return [];
|
|
186
|
+
const told = [];
|
|
187
|
+
try {
|
|
188
|
+
const { getSessionBackendSync } = await import('../session/session-backend.factory.js');
|
|
189
|
+
const backend = getSessionBackendSync();
|
|
190
|
+
const unit = /^(.*[\\/]claude-\d+[\\/][^\\/]+[\\/][0-9a-f-]{36})(?:[\\/]|$)/i.exec(file)?.[1];
|
|
191
|
+
for (const name of backend?.listSessions() ?? []) {
|
|
192
|
+
const cwd = backend?.getSession(name)?.cwd;
|
|
193
|
+
if (!cwd)
|
|
194
|
+
continue;
|
|
195
|
+
if (isPathInside(file, cwd) || (unit && isPathInside(cwd, unit))) {
|
|
196
|
+
if (await notifier(name, message).catch(() => false))
|
|
197
|
+
told.push(name);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
catch {
|
|
202
|
+
// no session backend — the owner alert still goes out
|
|
203
|
+
}
|
|
204
|
+
return told;
|
|
205
|
+
}
|
|
129
206
|
/**
|
|
130
207
|
* Process-wide instance used by the server and the HTTP routes.
|
|
131
208
|
*
|
|
@@ -168,6 +245,8 @@ export class WorktreeJanitorService {
|
|
|
168
245
|
this.firstRunTimer.unref?.();
|
|
169
246
|
this.diskTimer = setInterval(() => void this.checkDisk(), WORKTREE_JANITOR_CONSTANTS.LOW_DISK_CHECK_INTERVAL_MS);
|
|
170
247
|
this.diskTimer.unref?.();
|
|
248
|
+
this.runawayTimer = setInterval(() => void this.checkRunawayFiles(), WORKTREE_JANITOR_CONSTANTS.RUNAWAY_CHECK_INTERVAL_MS);
|
|
249
|
+
this.runawayTimer.unref?.();
|
|
171
250
|
return true;
|
|
172
251
|
}
|
|
173
252
|
/** Cancel scheduled passes. */
|
|
@@ -178,6 +257,9 @@ export class WorktreeJanitorService {
|
|
|
178
257
|
clearInterval(this.intervalTimer);
|
|
179
258
|
if (this.diskTimer)
|
|
180
259
|
clearInterval(this.diskTimer);
|
|
260
|
+
if (this.runawayTimer)
|
|
261
|
+
clearInterval(this.runawayTimer);
|
|
262
|
+
this.runawayTimer = null;
|
|
181
263
|
this.firstRunTimer = null;
|
|
182
264
|
this.intervalTimer = null;
|
|
183
265
|
this.diskTimer = null;
|