@triflux/core 10.41.1 → 10.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/hooks/keyword-rules.json +39 -5
- package/hub/lib/cto-env.mjs +48 -0
- package/hub/lib/memory-store.mjs +9 -0
- package/hub/log-retention.mjs +726 -0
- package/hub/router.mjs +24 -1
- package/package.json +1 -1
package/hooks/keyword-rules.json
CHANGED
|
@@ -62,6 +62,40 @@
|
|
|
62
62
|
"state": null,
|
|
63
63
|
"mcp_route": null
|
|
64
64
|
},
|
|
65
|
+
{
|
|
66
|
+
"id": "tfx-harness",
|
|
67
|
+
"patterns": [
|
|
68
|
+
{
|
|
69
|
+
"source": "\\btfx[\\s-]?harness\\b",
|
|
70
|
+
"flags": "i"
|
|
71
|
+
}
|
|
72
|
+
],
|
|
73
|
+
"skill": "tfx-harness",
|
|
74
|
+
"priority": 1,
|
|
75
|
+
"supersedes": ["tfx-unified"],
|
|
76
|
+
"exclusive": false,
|
|
77
|
+
"state": null,
|
|
78
|
+
"mcp_route": null
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"id": "tfx-harness-meta",
|
|
82
|
+
"patterns": [
|
|
83
|
+
{
|
|
84
|
+
"source": "(?:어떤\\s*스킬|스킬\\s*추천|스킬\\s*라우팅|작업\\s*라우팅|메타\\s*라우팅)",
|
|
85
|
+
"flags": "i"
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
"source": "\\bwhich\\s+skill\\b",
|
|
89
|
+
"flags": "i"
|
|
90
|
+
}
|
|
91
|
+
],
|
|
92
|
+
"skill": "tfx-harness",
|
|
93
|
+
"priority": 2,
|
|
94
|
+
"supersedes": ["tfx-unified"],
|
|
95
|
+
"exclusive": false,
|
|
96
|
+
"state": null,
|
|
97
|
+
"mcp_route": null
|
|
98
|
+
},
|
|
65
99
|
{
|
|
66
100
|
"id": "tfx-unified",
|
|
67
101
|
"patterns": [
|
|
@@ -160,7 +194,7 @@
|
|
|
160
194
|
}
|
|
161
195
|
],
|
|
162
196
|
"skill": "tfx-hub",
|
|
163
|
-
"priority":
|
|
197
|
+
"priority": 1,
|
|
164
198
|
"supersedes": [],
|
|
165
199
|
"exclusive": false,
|
|
166
200
|
"state": null,
|
|
@@ -175,7 +209,7 @@
|
|
|
175
209
|
}
|
|
176
210
|
],
|
|
177
211
|
"skill": "tfx-doctor",
|
|
178
|
-
"priority":
|
|
212
|
+
"priority": 1,
|
|
179
213
|
"supersedes": [],
|
|
180
214
|
"exclusive": false,
|
|
181
215
|
"state": null,
|
|
@@ -190,7 +224,7 @@
|
|
|
190
224
|
}
|
|
191
225
|
],
|
|
192
226
|
"skill": "tfx-setup",
|
|
193
|
-
"priority":
|
|
227
|
+
"priority": 1,
|
|
194
228
|
"supersedes": [],
|
|
195
229
|
"exclusive": false,
|
|
196
230
|
"state": null,
|
|
@@ -205,7 +239,7 @@
|
|
|
205
239
|
}
|
|
206
240
|
],
|
|
207
241
|
"skill": "tfx-research",
|
|
208
|
-
"priority":
|
|
242
|
+
"priority": 1,
|
|
209
243
|
"supersedes": [],
|
|
210
244
|
"exclusive": false,
|
|
211
245
|
"state": null,
|
|
@@ -220,7 +254,7 @@
|
|
|
220
254
|
}
|
|
221
255
|
],
|
|
222
256
|
"skill": "tfx-interview",
|
|
223
|
-
"priority":
|
|
257
|
+
"priority": 1,
|
|
224
258
|
"supersedes": [],
|
|
225
259
|
"exclusive": false,
|
|
226
260
|
"state": null,
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
const OFF_VALUES = new Set(["0", "false", "off", "no"]);
|
|
2
|
+
const ON_VALUES = new Set(["1", "true", "on", "yes"]);
|
|
3
|
+
|
|
4
|
+
function normalizeEnvValue(name) {
|
|
5
|
+
return String(process.env[name] ?? "")
|
|
6
|
+
.trim()
|
|
7
|
+
.toLowerCase();
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function isExplicitlyOff(name) {
|
|
11
|
+
return OFF_VALUES.has(normalizeEnvValue(name));
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function isExplicitlyOn(name) {
|
|
15
|
+
return ON_VALUES.has(normalizeEnvValue(name));
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function isCtoDisabled() {
|
|
19
|
+
return isExplicitlyOff("TFX_CTO");
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function isCtoManagerEnabled() {
|
|
23
|
+
if (isCtoDisabled()) return false;
|
|
24
|
+
return isExplicitlyOn("TFX_CTO_MANAGER");
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function getCtoHygieneApplyMode() {
|
|
28
|
+
if (isExplicitlyOff("TFX_CTO")) return "off";
|
|
29
|
+
return normalizeEnvValue("TFX_CTO_HYGIENE_APPLY") === "archive"
|
|
30
|
+
? "archive"
|
|
31
|
+
: "off";
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function isCtoRetentionEnabled() {
|
|
35
|
+
if (isExplicitlyOff("TFX_CTO")) return false;
|
|
36
|
+
return isExplicitlyOn("TFX_CTO_RETENTION");
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function getCtoMode() {
|
|
40
|
+
return normalizeEnvValue("TFX_CTO_MODE") === "bridge" ? "bridge" : "bounded";
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function getCtoMaxTokens() {
|
|
44
|
+
const raw = normalizeEnvValue("TFX_CTO_MAX_TOKENS");
|
|
45
|
+
if (!raw) return 0;
|
|
46
|
+
const value = Number(raw);
|
|
47
|
+
return Number.isSafeInteger(value) && value > 0 ? value : 0;
|
|
48
|
+
}
|
package/hub/lib/memory-store.mjs
CHANGED
|
@@ -177,6 +177,15 @@ export function createMemoryStore() {
|
|
|
177
177
|
lease_id: uuidv7(),
|
|
178
178
|
lease_expires_ms: next.lease_expires_ms,
|
|
179
179
|
server_time_ms: now,
|
|
180
|
+
effective: {
|
|
181
|
+
updated: true,
|
|
182
|
+
online: next.status === "online",
|
|
183
|
+
status: next.status,
|
|
184
|
+
last_seen_ms: next.last_seen_ms,
|
|
185
|
+
lease_expires_ms: next.lease_expires_ms,
|
|
186
|
+
store_type: "memory",
|
|
187
|
+
db_path: null,
|
|
188
|
+
},
|
|
180
189
|
};
|
|
181
190
|
},
|
|
182
191
|
|
|
@@ -0,0 +1,726 @@
|
|
|
1
|
+
import { once } from "node:events";
|
|
2
|
+
import {
|
|
3
|
+
closeSync,
|
|
4
|
+
createReadStream,
|
|
5
|
+
createWriteStream,
|
|
6
|
+
existsSync,
|
|
7
|
+
mkdirSync,
|
|
8
|
+
openSync,
|
|
9
|
+
readdirSync,
|
|
10
|
+
readSync,
|
|
11
|
+
renameSync,
|
|
12
|
+
rmSync,
|
|
13
|
+
statSync,
|
|
14
|
+
unlinkSync,
|
|
15
|
+
writeFileSync,
|
|
16
|
+
} from "node:fs";
|
|
17
|
+
import { homedir } from "node:os";
|
|
18
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
19
|
+
import { createGzip } from "node:zlib";
|
|
20
|
+
|
|
21
|
+
import { isCtoRetentionEnabled } from "./lib/cto-env.mjs";
|
|
22
|
+
|
|
23
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
24
|
+
const MB = 1024 * 1024;
|
|
25
|
+
const DEFAULT_TAIL_READ_BYTES = 2 * MB;
|
|
26
|
+
|
|
27
|
+
export const LOG_RETENTION_DEFAULTS = Object.freeze({
|
|
28
|
+
ledger: Object.freeze({
|
|
29
|
+
maxBytes: 50 * MB,
|
|
30
|
+
maxAgeMs: 30 * DAY_MS,
|
|
31
|
+
tailBytes: 10 * MB,
|
|
32
|
+
tailAgeMs: 14 * DAY_MS,
|
|
33
|
+
}),
|
|
34
|
+
swarm: Object.freeze({
|
|
35
|
+
maxBytes: 25 * MB,
|
|
36
|
+
maxAgeMs: 14 * DAY_MS,
|
|
37
|
+
activeMtimeGraceMs: 15 * 60 * 1000,
|
|
38
|
+
}),
|
|
39
|
+
batch: Object.freeze({
|
|
40
|
+
rotateAfterLines: 200,
|
|
41
|
+
keepLines: 100,
|
|
42
|
+
}),
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
function timestampForPath(now = new Date()) {
|
|
46
|
+
return now
|
|
47
|
+
.toISOString()
|
|
48
|
+
.replace(/[-:]/g, "")
|
|
49
|
+
.replace(/\.\d{3}Z$/u, "");
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function archiveDay(now = new Date()) {
|
|
53
|
+
return timestampForPath(now).slice(0, 8);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function normalizeTimeMs(value) {
|
|
57
|
+
if (value instanceof Date) return value.getTime();
|
|
58
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
59
|
+
if (typeof value === "string" && value.trim()) {
|
|
60
|
+
const parsed = Date.parse(value);
|
|
61
|
+
return Number.isFinite(parsed) ? parsed : 0;
|
|
62
|
+
}
|
|
63
|
+
return 0;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function entryTimeMs(entry) {
|
|
67
|
+
return normalizeTimeMs(
|
|
68
|
+
entry?.ts ??
|
|
69
|
+
entry?.time ??
|
|
70
|
+
entry?.timestamp ??
|
|
71
|
+
entry?.created_at ??
|
|
72
|
+
entry?.createdAt,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function parseJsonLine(line) {
|
|
77
|
+
const trimmed = String(line ?? "").trim();
|
|
78
|
+
if (!trimmed) return null;
|
|
79
|
+
try {
|
|
80
|
+
return JSON.parse(trimmed);
|
|
81
|
+
} catch {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function statOrNull(path) {
|
|
87
|
+
try {
|
|
88
|
+
return statSync(path);
|
|
89
|
+
} catch {
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function mkdirForFile(path) {
|
|
95
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function writeStream(stream, chunk) {
|
|
99
|
+
if (stream.write(chunk)) return Promise.resolve();
|
|
100
|
+
return once(stream, "drain").then(() => {});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async function closeWritable(stream) {
|
|
104
|
+
return new Promise((resolve, reject) => {
|
|
105
|
+
stream.once("error", reject);
|
|
106
|
+
stream.end(() => {
|
|
107
|
+
stream.removeListener("error", reject);
|
|
108
|
+
resolve();
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function acquireLock(lockPath, opts = {}) {
|
|
114
|
+
mkdirForFile(lockPath);
|
|
115
|
+
const staleMs = Number.isFinite(opts.staleMs) ? opts.staleMs : 10 * 60_000;
|
|
116
|
+
try {
|
|
117
|
+
const current = statSync(lockPath);
|
|
118
|
+
if (Date.now() - current.mtimeMs > staleMs) unlinkSync(lockPath);
|
|
119
|
+
} catch (error) {
|
|
120
|
+
if (error?.code !== "ENOENT") throw error;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const fd = openSync(lockPath, "wx", 0o600);
|
|
124
|
+
writeFileSync(
|
|
125
|
+
fd,
|
|
126
|
+
JSON.stringify({ pid: process.pid, created_at: new Date().toISOString() }),
|
|
127
|
+
);
|
|
128
|
+
let released = false;
|
|
129
|
+
return {
|
|
130
|
+
path: lockPath,
|
|
131
|
+
release() {
|
|
132
|
+
if (released) return;
|
|
133
|
+
released = true;
|
|
134
|
+
closeSync(fd);
|
|
135
|
+
try {
|
|
136
|
+
unlinkSync(lockPath);
|
|
137
|
+
} catch {}
|
|
138
|
+
},
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async function withLock(lockPath, opts, fn) {
|
|
143
|
+
let lock = null;
|
|
144
|
+
try {
|
|
145
|
+
lock = acquireLock(lockPath, opts);
|
|
146
|
+
} catch (error) {
|
|
147
|
+
if (error?.code === "EEXIST") {
|
|
148
|
+
return { rotated: false, skipped: true, reason: "lock_busy", lockPath };
|
|
149
|
+
}
|
|
150
|
+
throw error;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
try {
|
|
154
|
+
return await fn(lock);
|
|
155
|
+
} finally {
|
|
156
|
+
lock.release();
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function collectActive(value) {
|
|
161
|
+
const sessions = Array.isArray(value)
|
|
162
|
+
? value
|
|
163
|
+
: Array.isArray(value?.sessions)
|
|
164
|
+
? value.sessions
|
|
165
|
+
: Array.isArray(value?.live_sessions)
|
|
166
|
+
? value.live_sessions
|
|
167
|
+
: [];
|
|
168
|
+
const ids = new Set();
|
|
169
|
+
const paths = new Set();
|
|
170
|
+
|
|
171
|
+
function addId(value) {
|
|
172
|
+
if (typeof value === "string" && value.trim()) ids.add(value.trim());
|
|
173
|
+
else if (typeof value === "number" && Number.isFinite(value)) {
|
|
174
|
+
ids.add(String(value));
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function addPath(value) {
|
|
179
|
+
if (typeof value === "string" && value.trim()) paths.add(resolve(value));
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
for (const session of sessions) {
|
|
183
|
+
addId(session?.sessionId);
|
|
184
|
+
addId(session?.session_id);
|
|
185
|
+
addId(session?.runId);
|
|
186
|
+
addId(session?.run_id);
|
|
187
|
+
addId(session?.id);
|
|
188
|
+
addId(session?.agent_id);
|
|
189
|
+
addId(session?.agentId);
|
|
190
|
+
|
|
191
|
+
for (const key of [
|
|
192
|
+
"logPath",
|
|
193
|
+
"log_path",
|
|
194
|
+
"eventLogPath",
|
|
195
|
+
"event_log_path",
|
|
196
|
+
"filePath",
|
|
197
|
+
"path",
|
|
198
|
+
"runDir",
|
|
199
|
+
"run_dir",
|
|
200
|
+
"logsDir",
|
|
201
|
+
"logs_dir",
|
|
202
|
+
]) {
|
|
203
|
+
addPath(session?.[key]);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
for (const key of ["files", "dirtyFiles", "claimedPaths", "logPaths"]) {
|
|
207
|
+
const values = Array.isArray(session?.[key]) ? session[key] : [];
|
|
208
|
+
for (const value of values) addPath(value);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
return { ids, paths };
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
async function readActive(opts = {}) {
|
|
216
|
+
if (opts.active) return collectActive(opts.active);
|
|
217
|
+
if (typeof opts.getActive !== "function") return collectActive([]);
|
|
218
|
+
return collectActive(await opts.getActive());
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function isOwnedByActivePath(path, active) {
|
|
222
|
+
const resolved = resolve(path);
|
|
223
|
+
for (const activePath of active.paths) {
|
|
224
|
+
if (resolved === activePath || resolved.startsWith(`${activePath}/`)) {
|
|
225
|
+
return true;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return false;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function runIdForDir(runDir) {
|
|
232
|
+
const name = basename(runDir);
|
|
233
|
+
return name.startsWith("run-") ? name.slice(4) : name;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function isTerminalSwarmEvent(entry) {
|
|
237
|
+
if (entry?.event === "integration_complete") return true;
|
|
238
|
+
if (entry?.event !== "swarm_state") return false;
|
|
239
|
+
return entry?.to === "completed" || entry?.to === "failed";
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function isActiveRun(runDir, eventPath, active, stat, policy, nowMs) {
|
|
243
|
+
if (
|
|
244
|
+
isOwnedByActivePath(runDir, active) ||
|
|
245
|
+
isOwnedByActivePath(eventPath, active)
|
|
246
|
+
) {
|
|
247
|
+
return true;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
const runId = runIdForDir(runDir);
|
|
251
|
+
const candidateIds = new Set([runId, `run-${runId}`, `swarm-${runId}`]);
|
|
252
|
+
for (const id of candidateIds) {
|
|
253
|
+
if (active.ids.has(id)) return true;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
const tail = readJsonLines(eventPath, { limit: 20, maxBytes: 256 * 1024 });
|
|
257
|
+
if (tail.some((entry) => active.ids.has(String(entry?.sessionId || "")))) {
|
|
258
|
+
return true;
|
|
259
|
+
}
|
|
260
|
+
if (tail.some(isTerminalSwarmEvent)) return false;
|
|
261
|
+
|
|
262
|
+
const graceMs = Number.isFinite(policy.activeMtimeGraceMs)
|
|
263
|
+
? policy.activeMtimeGraceMs
|
|
264
|
+
: 0;
|
|
265
|
+
return graceMs > 0 && stat && nowMs - stat.mtimeMs <= graceMs;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function firstJsonLineTimeMs(path) {
|
|
269
|
+
const fd = openSync(path, "r");
|
|
270
|
+
try {
|
|
271
|
+
const stat = statSync(path);
|
|
272
|
+
const size = Math.min(stat.size, 256 * 1024);
|
|
273
|
+
if (size <= 0) return 0;
|
|
274
|
+
const buffer = Buffer.alloc(size);
|
|
275
|
+
const bytesRead = readSync(fd, buffer, 0, size, 0);
|
|
276
|
+
const text = buffer.subarray(0, bytesRead).toString("utf8");
|
|
277
|
+
for (const line of text.split(/\r?\n/u)) {
|
|
278
|
+
const parsed = parseJsonLine(line);
|
|
279
|
+
const timeMs = entryTimeMs(parsed);
|
|
280
|
+
if (timeMs > 0) return timeMs;
|
|
281
|
+
}
|
|
282
|
+
return 0;
|
|
283
|
+
} finally {
|
|
284
|
+
closeSync(fd);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function shouldRotateBySizeOrAge(path, policy, nowMs) {
|
|
289
|
+
const stat = statOrNull(path);
|
|
290
|
+
if (!stat?.isFile()) {
|
|
291
|
+
return { rotate: false, reason: "missing", stat };
|
|
292
|
+
}
|
|
293
|
+
if (Number.isFinite(policy.maxBytes) && stat.size > policy.maxBytes) {
|
|
294
|
+
return { rotate: true, reason: "size", stat };
|
|
295
|
+
}
|
|
296
|
+
const maxAgeMs = Number.isFinite(policy.maxAgeMs) ? policy.maxAgeMs : 0;
|
|
297
|
+
if (maxAgeMs > 0) {
|
|
298
|
+
const firstMs =
|
|
299
|
+
firstJsonLineTimeMs(path) || stat.birthtimeMs || stat.mtimeMs;
|
|
300
|
+
if (firstMs > 0 && nowMs - firstMs > maxAgeMs) {
|
|
301
|
+
return { rotate: true, reason: "age", stat };
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
return { rotate: false, reason: "below_threshold", stat };
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function lineCountUpTo(path, maxLines) {
|
|
308
|
+
if (!existsSync(path)) return 0;
|
|
309
|
+
const fd = openSync(path, "r");
|
|
310
|
+
try {
|
|
311
|
+
const buffer = Buffer.alloc(64 * 1024);
|
|
312
|
+
let count = 0;
|
|
313
|
+
let position = 0;
|
|
314
|
+
while (count <= maxLines) {
|
|
315
|
+
const bytesRead = readSync(fd, buffer, 0, buffer.length, position);
|
|
316
|
+
if (bytesRead <= 0) break;
|
|
317
|
+
position += bytesRead;
|
|
318
|
+
for (let i = 0; i < bytesRead; i += 1) {
|
|
319
|
+
if (buffer[i] === 10) count += 1;
|
|
320
|
+
if (count > maxLines) break;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
return count;
|
|
324
|
+
} finally {
|
|
325
|
+
closeSync(fd);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function readFromTail(path, maxBytes) {
|
|
330
|
+
const stat = statSync(path);
|
|
331
|
+
const bytes = Math.min(stat.size, maxBytes);
|
|
332
|
+
const fd = openSync(path, "r");
|
|
333
|
+
try {
|
|
334
|
+
const buffer = Buffer.alloc(bytes);
|
|
335
|
+
const offset = stat.size - bytes;
|
|
336
|
+
const read = readSync(fd, buffer, 0, bytes, offset);
|
|
337
|
+
return {
|
|
338
|
+
size: stat.size,
|
|
339
|
+
offset,
|
|
340
|
+
buffer: buffer.subarray(0, read),
|
|
341
|
+
};
|
|
342
|
+
} finally {
|
|
343
|
+
closeSync(fd);
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
export function readJsonLines(path, opts = {}) {
|
|
348
|
+
if (!existsSync(path)) return [];
|
|
349
|
+
const limit = Number.isFinite(opts.limit) ? Math.max(0, opts.limit) : 100;
|
|
350
|
+
const maxBytes = Number.isFinite(opts.maxBytes)
|
|
351
|
+
? Math.max(1, opts.maxBytes)
|
|
352
|
+
: DEFAULT_TAIL_READ_BYTES;
|
|
353
|
+
const { size, offset, buffer } = readFromTail(path, maxBytes);
|
|
354
|
+
let text = buffer.toString("utf8");
|
|
355
|
+
if (offset > 0) {
|
|
356
|
+
const newline = text.indexOf("\n");
|
|
357
|
+
text = newline >= 0 ? text.slice(newline + 1) : "";
|
|
358
|
+
}
|
|
359
|
+
const parsed = text
|
|
360
|
+
.split(/\r?\n/u)
|
|
361
|
+
.map((line) => line.trim())
|
|
362
|
+
.filter(Boolean)
|
|
363
|
+
.flatMap((line) => {
|
|
364
|
+
const entry = parseJsonLine(line);
|
|
365
|
+
return entry ? [entry] : [];
|
|
366
|
+
});
|
|
367
|
+
if (limit === 0) return parsed;
|
|
368
|
+
if (size <= maxBytes) return parsed.slice(-limit);
|
|
369
|
+
return parsed.slice(-limit);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
async function splitJsonlToArchiveAndTail({
|
|
373
|
+
sourcePath,
|
|
374
|
+
tailPath,
|
|
375
|
+
archivePath,
|
|
376
|
+
keepLine,
|
|
377
|
+
}) {
|
|
378
|
+
mkdirForFile(tailPath);
|
|
379
|
+
mkdirForFile(archivePath);
|
|
380
|
+
|
|
381
|
+
const input = createReadStream(sourcePath);
|
|
382
|
+
const tail = createWriteStream(tailPath, { flags: "wx" });
|
|
383
|
+
const archiveFile = createWriteStream(archivePath, { flags: "wx" });
|
|
384
|
+
const archiveClosed = once(archiveFile, "close");
|
|
385
|
+
const gzip = createGzip();
|
|
386
|
+
gzip.pipe(archiveFile);
|
|
387
|
+
|
|
388
|
+
let carry = Buffer.alloc(0);
|
|
389
|
+
let lineIndex = 0;
|
|
390
|
+
let offset = 0;
|
|
391
|
+
let archivedLines = 0;
|
|
392
|
+
let retainedLines = 0;
|
|
393
|
+
let archivedBytes = 0;
|
|
394
|
+
let retainedBytes = 0;
|
|
395
|
+
|
|
396
|
+
async function handleRecord(record) {
|
|
397
|
+
const body = record.toString("utf8").replace(/\r?\n$/u, "");
|
|
398
|
+
const startOffset = offset;
|
|
399
|
+
const endOffset = offset + record.length;
|
|
400
|
+
const parsed = parseJsonLine(body);
|
|
401
|
+
const keep = keepLine({
|
|
402
|
+
line: body,
|
|
403
|
+
entry: parsed,
|
|
404
|
+
lineIndex,
|
|
405
|
+
startOffset,
|
|
406
|
+
endOffset,
|
|
407
|
+
bytes: record.length,
|
|
408
|
+
});
|
|
409
|
+
lineIndex += 1;
|
|
410
|
+
offset = endOffset;
|
|
411
|
+
if (keep) {
|
|
412
|
+
retainedLines += 1;
|
|
413
|
+
retainedBytes += record.length;
|
|
414
|
+
await writeStream(tail, record);
|
|
415
|
+
} else {
|
|
416
|
+
archivedLines += 1;
|
|
417
|
+
archivedBytes += record.length;
|
|
418
|
+
await writeStream(gzip, record);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
try {
|
|
423
|
+
for await (const chunk of input) {
|
|
424
|
+
const buffer = carry.length ? Buffer.concat([carry, chunk]) : chunk;
|
|
425
|
+
let start = 0;
|
|
426
|
+
while (true) {
|
|
427
|
+
const newline = buffer.indexOf(10, start);
|
|
428
|
+
if (newline < 0) break;
|
|
429
|
+
await handleRecord(buffer.subarray(start, newline + 1));
|
|
430
|
+
start = newline + 1;
|
|
431
|
+
}
|
|
432
|
+
carry = buffer.subarray(start);
|
|
433
|
+
}
|
|
434
|
+
if (carry.length > 0) await handleRecord(carry);
|
|
435
|
+
await closeWritable(tail);
|
|
436
|
+
await closeWritable(gzip);
|
|
437
|
+
await archiveClosed;
|
|
438
|
+
return {
|
|
439
|
+
archivedLines,
|
|
440
|
+
retainedLines,
|
|
441
|
+
archivedBytes,
|
|
442
|
+
retainedBytes,
|
|
443
|
+
};
|
|
444
|
+
} catch (error) {
|
|
445
|
+
tail.destroy();
|
|
446
|
+
gzip.destroy();
|
|
447
|
+
archiveFile.destroy();
|
|
448
|
+
throw error;
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
async function gzipWholeFile(sourcePath, archivePath) {
|
|
453
|
+
mkdirForFile(archivePath);
|
|
454
|
+
const input = createReadStream(sourcePath);
|
|
455
|
+
const gzip = createGzip();
|
|
456
|
+
const output = createWriteStream(archivePath, { flags: "wx" });
|
|
457
|
+
input.pipe(gzip).pipe(output);
|
|
458
|
+
await once(output, "close");
|
|
459
|
+
const stat = statSync(sourcePath);
|
|
460
|
+
return { archivedBytes: stat.size };
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
async function rotateLedger({ ledgerPath, lakeRoot, policy, active, now }) {
|
|
464
|
+
const nowMs = now.getTime();
|
|
465
|
+
if (isOwnedByActivePath(ledgerPath, active)) {
|
|
466
|
+
return { log: "ledger", rotated: false, skipped: true, reason: "active" };
|
|
467
|
+
}
|
|
468
|
+
const decision = shouldRotateBySizeOrAge(ledgerPath, policy, nowMs);
|
|
469
|
+
if (!decision.rotate) {
|
|
470
|
+
return { log: "ledger", rotated: false, reason: decision.reason };
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
const lockPath = join(lakeRoot, "retention.lock");
|
|
474
|
+
return await withLock(lockPath, {}, async () => {
|
|
475
|
+
const ledgerLockPath = `${ledgerPath}.lock`;
|
|
476
|
+
return await withLock(ledgerLockPath, {}, async () => {
|
|
477
|
+
const ts = timestampForPath(now);
|
|
478
|
+
const archiveDir = join(lakeRoot, "archive", archiveDay(now));
|
|
479
|
+
const movedPath = join(archiveDir, `.ledger-${ts}.jsonl.rotating`);
|
|
480
|
+
const archivePath = join(archiveDir, `ledger-${ts}.jsonl.gz`);
|
|
481
|
+
const tailTmpPath = `${ledgerPath}.${process.pid}.${ts}.tmp`;
|
|
482
|
+
mkdirSync(archiveDir, { recursive: true });
|
|
483
|
+
renameSync(ledgerPath, movedPath);
|
|
484
|
+
|
|
485
|
+
try {
|
|
486
|
+
const sourceSize = decision.stat.size;
|
|
487
|
+
const tailStart = Math.max(0, sourceSize - policy.tailBytes);
|
|
488
|
+
const cutoffMs = nowMs - policy.tailAgeMs;
|
|
489
|
+
const split = await splitJsonlToArchiveAndTail({
|
|
490
|
+
sourcePath: movedPath,
|
|
491
|
+
tailPath: tailTmpPath,
|
|
492
|
+
archivePath,
|
|
493
|
+
keepLine({ entry, startOffset }) {
|
|
494
|
+
const timeMs = entryTimeMs(entry);
|
|
495
|
+
return (
|
|
496
|
+
startOffset >= tailStart || (timeMs > 0 && timeMs >= cutoffMs)
|
|
497
|
+
);
|
|
498
|
+
},
|
|
499
|
+
});
|
|
500
|
+
renameSync(tailTmpPath, ledgerPath);
|
|
501
|
+
rmSync(movedPath, { force: true });
|
|
502
|
+
return {
|
|
503
|
+
log: "ledger",
|
|
504
|
+
rotated: true,
|
|
505
|
+
reason: decision.reason,
|
|
506
|
+
archivePath,
|
|
507
|
+
retainedPath: ledgerPath,
|
|
508
|
+
...split,
|
|
509
|
+
};
|
|
510
|
+
} catch (error) {
|
|
511
|
+
try {
|
|
512
|
+
if (!existsSync(ledgerPath) && existsSync(movedPath)) {
|
|
513
|
+
renameSync(movedPath, ledgerPath);
|
|
514
|
+
}
|
|
515
|
+
} catch {}
|
|
516
|
+
rmSync(tailTmpPath, { force: true });
|
|
517
|
+
throw error;
|
|
518
|
+
}
|
|
519
|
+
});
|
|
520
|
+
});
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
async function rotateSwarmRun({
|
|
524
|
+
runDir,
|
|
525
|
+
eventPath,
|
|
526
|
+
policy,
|
|
527
|
+
active,
|
|
528
|
+
now,
|
|
529
|
+
archiveRoot,
|
|
530
|
+
}) {
|
|
531
|
+
const stat = statOrNull(eventPath);
|
|
532
|
+
if (!stat?.isFile()) {
|
|
533
|
+
return { log: "swarm-events", runDir, rotated: false, reason: "missing" };
|
|
534
|
+
}
|
|
535
|
+
const nowMs = now.getTime();
|
|
536
|
+
if (isActiveRun(runDir, eventPath, active, stat, policy, nowMs)) {
|
|
537
|
+
return {
|
|
538
|
+
log: "swarm-events",
|
|
539
|
+
runDir,
|
|
540
|
+
rotated: false,
|
|
541
|
+
skipped: true,
|
|
542
|
+
reason: "active",
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
const decision = shouldRotateBySizeOrAge(eventPath, policy, nowMs);
|
|
546
|
+
if (!decision.rotate) {
|
|
547
|
+
return {
|
|
548
|
+
log: "swarm-events",
|
|
549
|
+
runDir,
|
|
550
|
+
rotated: false,
|
|
551
|
+
reason: decision.reason,
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
return await withLock(join(runDir, ".retention.lock"), {}, async () => {
|
|
556
|
+
const ts = timestampForPath(now);
|
|
557
|
+
const runName = basename(runDir);
|
|
558
|
+
const archiveDir = join(archiveRoot, archiveDay(now), runName);
|
|
559
|
+
const movedPath = join(archiveDir, `swarm-events-${ts}.jsonl`);
|
|
560
|
+
const archivePath = `${movedPath}.gz`;
|
|
561
|
+
mkdirSync(archiveDir, { recursive: true });
|
|
562
|
+
renameSync(eventPath, movedPath);
|
|
563
|
+
try {
|
|
564
|
+
const result = await gzipWholeFile(movedPath, archivePath);
|
|
565
|
+
rmSync(movedPath, { force: true });
|
|
566
|
+
return {
|
|
567
|
+
log: "swarm-events",
|
|
568
|
+
runDir,
|
|
569
|
+
rotated: true,
|
|
570
|
+
reason: decision.reason,
|
|
571
|
+
archivePath,
|
|
572
|
+
...result,
|
|
573
|
+
};
|
|
574
|
+
} catch (error) {
|
|
575
|
+
try {
|
|
576
|
+
if (!existsSync(eventPath) && existsSync(movedPath)) {
|
|
577
|
+
renameSync(movedPath, eventPath);
|
|
578
|
+
}
|
|
579
|
+
} catch {}
|
|
580
|
+
throw error;
|
|
581
|
+
}
|
|
582
|
+
});
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
async function rotateBatchEvents({ batchPath, policy, active, now }) {
|
|
586
|
+
if (!existsSync(batchPath)) {
|
|
587
|
+
return { log: "batch-events", rotated: false, reason: "missing" };
|
|
588
|
+
}
|
|
589
|
+
if (isOwnedByActivePath(batchPath, active)) {
|
|
590
|
+
return {
|
|
591
|
+
log: "batch-events",
|
|
592
|
+
rotated: false,
|
|
593
|
+
skipped: true,
|
|
594
|
+
reason: "active",
|
|
595
|
+
};
|
|
596
|
+
}
|
|
597
|
+
const count = lineCountUpTo(batchPath, policy.rotateAfterLines + 1);
|
|
598
|
+
if (count <= policy.rotateAfterLines) {
|
|
599
|
+
return { log: "batch-events", rotated: false, reason: "below_threshold" };
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
const lockPath = join(dirname(batchPath), "retention.lock");
|
|
603
|
+
return await withLock(lockPath, {}, async () => {
|
|
604
|
+
const ts = timestampForPath(now);
|
|
605
|
+
const archiveDir = join(dirname(batchPath), "archive", archiveDay(now));
|
|
606
|
+
const movedPath = join(archiveDir, `.batch-events-${ts}.jsonl.rotating`);
|
|
607
|
+
const archivePath = join(archiveDir, `batch-events-${ts}.jsonl.gz`);
|
|
608
|
+
const tailTmpPath = `${batchPath}.${process.pid}.${ts}.tmp`;
|
|
609
|
+
mkdirSync(archiveDir, { recursive: true });
|
|
610
|
+
renameSync(batchPath, movedPath);
|
|
611
|
+
try {
|
|
612
|
+
const totalLines = lineCountUpTo(movedPath, Number.MAX_SAFE_INTEGER);
|
|
613
|
+
const keepFrom = Math.max(0, totalLines - policy.keepLines);
|
|
614
|
+
const split = await splitJsonlToArchiveAndTail({
|
|
615
|
+
sourcePath: movedPath,
|
|
616
|
+
tailPath: tailTmpPath,
|
|
617
|
+
archivePath,
|
|
618
|
+
keepLine({ lineIndex }) {
|
|
619
|
+
return lineIndex >= keepFrom;
|
|
620
|
+
},
|
|
621
|
+
});
|
|
622
|
+
renameSync(tailTmpPath, batchPath);
|
|
623
|
+
rmSync(movedPath, { force: true });
|
|
624
|
+
return {
|
|
625
|
+
log: "batch-events",
|
|
626
|
+
rotated: true,
|
|
627
|
+
reason: "line_count",
|
|
628
|
+
archivePath,
|
|
629
|
+
retainedPath: batchPath,
|
|
630
|
+
...split,
|
|
631
|
+
};
|
|
632
|
+
} catch (error) {
|
|
633
|
+
try {
|
|
634
|
+
if (!existsSync(batchPath) && existsSync(movedPath)) {
|
|
635
|
+
renameSync(movedPath, batchPath);
|
|
636
|
+
}
|
|
637
|
+
} catch {}
|
|
638
|
+
rmSync(tailTmpPath, { force: true });
|
|
639
|
+
throw error;
|
|
640
|
+
}
|
|
641
|
+
});
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
function listSwarmEventPaths(swarmLogsRoot) {
|
|
645
|
+
if (!existsSync(swarmLogsRoot)) return [];
|
|
646
|
+
return readdirSync(swarmLogsRoot, { withFileTypes: true })
|
|
647
|
+
.filter((entry) => entry.isDirectory())
|
|
648
|
+
.map((entry) => {
|
|
649
|
+
const runDir = join(swarmLogsRoot, entry.name);
|
|
650
|
+
return { runDir, eventPath: join(runDir, "swarm-events.jsonl") };
|
|
651
|
+
})
|
|
652
|
+
.filter(({ eventPath }) => existsSync(eventPath));
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
function mergePolicies(overrides = {}) {
|
|
656
|
+
return {
|
|
657
|
+
ledger: { ...LOG_RETENTION_DEFAULTS.ledger, ...(overrides.ledger || {}) },
|
|
658
|
+
swarm: { ...LOG_RETENTION_DEFAULTS.swarm, ...(overrides.swarm || {}) },
|
|
659
|
+
batch: { ...LOG_RETENTION_DEFAULTS.batch, ...(overrides.batch || {}) },
|
|
660
|
+
};
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
export async function runLogRetention(opts = {}) {
|
|
664
|
+
const enabled =
|
|
665
|
+
typeof opts.enabled === "boolean" ? opts.enabled : isCtoRetentionEnabled();
|
|
666
|
+
if (!enabled) {
|
|
667
|
+
return { enabled: false, applied: false, results: [] };
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
const rootDir = resolve(opts.rootDir || process.cwd());
|
|
671
|
+
const lakeRoot = resolve(opts.lakeRoot || join(rootDir, ".triflux", "lake"));
|
|
672
|
+
const swarmLogsRoot = resolve(
|
|
673
|
+
opts.swarmLogsRoot || join(rootDir, ".triflux", "swarm-logs"),
|
|
674
|
+
);
|
|
675
|
+
const batchPath = resolve(
|
|
676
|
+
opts.batchEventsPath ||
|
|
677
|
+
join(homedir(), ".claude", "cache", "batch-events.jsonl"),
|
|
678
|
+
);
|
|
679
|
+
const now =
|
|
680
|
+
opts.now instanceof Date ? opts.now : new Date(opts.now || Date.now());
|
|
681
|
+
const policies = mergePolicies(opts.policies);
|
|
682
|
+
const active = await readActive(opts);
|
|
683
|
+
const results = [];
|
|
684
|
+
|
|
685
|
+
mkdirSync(lakeRoot, { recursive: true });
|
|
686
|
+
results.push(
|
|
687
|
+
await rotateLedger({
|
|
688
|
+
ledgerPath: join(lakeRoot, "ledger.jsonl"),
|
|
689
|
+
lakeRoot,
|
|
690
|
+
policy: policies.ledger,
|
|
691
|
+
active,
|
|
692
|
+
now,
|
|
693
|
+
}),
|
|
694
|
+
);
|
|
695
|
+
|
|
696
|
+
const swarmArchiveRoot =
|
|
697
|
+
opts.swarmArchiveRoot || join(swarmLogsRoot, "archive");
|
|
698
|
+
for (const entry of listSwarmEventPaths(swarmLogsRoot)) {
|
|
699
|
+
results.push(
|
|
700
|
+
await rotateSwarmRun({
|
|
701
|
+
...entry,
|
|
702
|
+
policy: policies.swarm,
|
|
703
|
+
active,
|
|
704
|
+
now,
|
|
705
|
+
archiveRoot: swarmArchiveRoot,
|
|
706
|
+
}),
|
|
707
|
+
);
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
results.push(
|
|
711
|
+
await rotateBatchEvents({
|
|
712
|
+
batchPath,
|
|
713
|
+
policy: policies.batch,
|
|
714
|
+
active,
|
|
715
|
+
now,
|
|
716
|
+
}),
|
|
717
|
+
);
|
|
718
|
+
|
|
719
|
+
return {
|
|
720
|
+
enabled: true,
|
|
721
|
+
applied: results.some((result) => result.rotated),
|
|
722
|
+
results,
|
|
723
|
+
};
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
export const applyLogRetention = runLogRetention;
|
package/hub/router.mjs
CHANGED
|
@@ -260,6 +260,26 @@ export function createRouter(store) {
|
|
|
260
260
|
);
|
|
261
261
|
}
|
|
262
262
|
|
|
263
|
+
function registerPresenceIsEffective(result) {
|
|
264
|
+
const effective = result?.effective;
|
|
265
|
+
return (
|
|
266
|
+
effective?.online === true &&
|
|
267
|
+
effective.status === "online" &&
|
|
268
|
+
Number(effective.lease_expires_ms) === Number(result?.lease_expires_ms)
|
|
269
|
+
);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function registerPresenceFailure(agentId, result) {
|
|
273
|
+
return {
|
|
274
|
+
ok: false,
|
|
275
|
+
error: {
|
|
276
|
+
code: "REGISTER_PRESENCE_NOT_UPDATED",
|
|
277
|
+
message: `${agentId} register did not update effective presence`,
|
|
278
|
+
},
|
|
279
|
+
data: { ...result, hub_pid: process.pid },
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
|
|
263
283
|
function sortRoleCandidates(candidates = []) {
|
|
264
284
|
return [...candidates].sort((left, right) => {
|
|
265
285
|
const sourceDelta =
|
|
@@ -802,6 +822,9 @@ export function createRouter(store) {
|
|
|
802
822
|
|
|
803
823
|
registerAgent(args) {
|
|
804
824
|
const result = store.registerAgent(args);
|
|
825
|
+
if (!registerPresenceIsEffective(result)) {
|
|
826
|
+
return registerPresenceFailure(args.agent_id, result);
|
|
827
|
+
}
|
|
805
828
|
upsertRuntimeTopics(args.agent_id, args.topics || [], { replace: true });
|
|
806
829
|
refreshRoleCandidateForAgent(args.agent_id);
|
|
807
830
|
for (const roleName of ROLE_TOPICS) {
|
|
@@ -810,7 +833,7 @@ export function createRouter(store) {
|
|
|
810
833
|
transferBacklog: true,
|
|
811
834
|
});
|
|
812
835
|
}
|
|
813
|
-
return result;
|
|
836
|
+
return { ok: true, data: { ...result, hub_pid: process.pid } };
|
|
814
837
|
},
|
|
815
838
|
|
|
816
839
|
refreshAgentLease(agentId, ttlMs = 30000) {
|