@openagents-org/agent-launcher 0.2.192 → 0.2.193
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/adapters/base.js +55 -26
- package/src/adapters/failure-streak.js +89 -0
- package/src/config.js +12 -3
- package/src/daemon.js +30 -35
- package/src/log-writer.js +351 -0
- package/src/workspace-client.js +44 -14
package/package.json
CHANGED
package/src/adapters/base.js
CHANGED
|
@@ -35,6 +35,7 @@ const {
|
|
|
35
35
|
classifyHeartbeatError,
|
|
36
36
|
httpStatusOf,
|
|
37
37
|
} = require('./health-status');
|
|
38
|
+
const { FailureStreak, isTransientNetworkError, formatDuration } = require('./failure-streak');
|
|
38
39
|
|
|
39
40
|
const DEFAULT_ENDPOINT = 'https://workspace-endpoint.openagents.org';
|
|
40
41
|
|
|
@@ -104,7 +105,7 @@ class BaseAdapter {
|
|
|
104
105
|
* @param {string} opts.agentName
|
|
105
106
|
* @param {string} [opts.endpoint]
|
|
106
107
|
*/
|
|
107
|
-
constructor({ workspaceId, channelName, token, agentName, endpoint, agentEnv, agentType, workingDir, onStatus }) {
|
|
108
|
+
constructor({ workspaceId, channelName, token, agentName, endpoint, agentEnv, agentType, workingDir, onStatus, logSink }) {
|
|
108
109
|
this.workspaceId = workspaceId;
|
|
109
110
|
this.channelName = channelName;
|
|
110
111
|
this.token = token;
|
|
@@ -198,10 +199,17 @@ class BaseAdapter {
|
|
|
198
199
|
// (e.g. after a `restart` IPC bounce) so uptime tracks "time since last
|
|
199
200
|
// restart" rather than the long-running daemon's process uptime.
|
|
200
201
|
this._startedAt = Date.now();
|
|
201
|
-
|
|
202
|
+
// Inside the daemon, `logSink` routes lines through its daemon.log writer
|
|
203
|
+
// (batching, repeat suppression, rotation). Standalone, they go to stdout.
|
|
204
|
+
const sink = typeof logSink === 'function' ? logSink : null;
|
|
205
|
+
this._log = (msg, level = 'info') => {
|
|
206
|
+
if (sink) return sink(level, msg);
|
|
202
207
|
const ts = new Date().toISOString();
|
|
203
|
-
console.log(`${ts}
|
|
208
|
+
console.log(`${ts} ${String(level).toUpperCase()} adapter [${this.agentName}]: ${msg}`);
|
|
204
209
|
};
|
|
210
|
+
// Runs of identical background failures (see failure-streak.js).
|
|
211
|
+
this._pollFailures = new FailureStreak();
|
|
212
|
+
this._heartbeatFailures = new FailureStreak();
|
|
205
213
|
}
|
|
206
214
|
|
|
207
215
|
// ------------------------------------------------------------------
|
|
@@ -337,7 +345,7 @@ class BaseAdapter {
|
|
|
337
345
|
this._log(`Synced model from workspace: ${this.workspaceModel}`);
|
|
338
346
|
}
|
|
339
347
|
} catch (e) {
|
|
340
|
-
this._log(`Warning: skill sync failed (non-fatal): ${e.message}
|
|
348
|
+
this._log(`Warning: skill sync failed (non-fatal): ${e.message}`, 'warn');
|
|
341
349
|
}
|
|
342
350
|
|
|
343
351
|
// Fast-path operations (control-event cursor + heartbeat + control poll)
|
|
@@ -352,7 +360,7 @@ class BaseAdapter {
|
|
|
352
360
|
try {
|
|
353
361
|
// Send initial heartbeat
|
|
354
362
|
try { await this._heartbeat(); } catch (e) {
|
|
355
|
-
this._log(`Heartbeat failed (non-fatal): ${e.message}
|
|
363
|
+
this._log(`Heartbeat failed (non-fatal): ${e.message}`, 'warn');
|
|
356
364
|
}
|
|
357
365
|
// Slow path: only the message-poll loop waits for this.
|
|
358
366
|
await this._skipExistingEvents();
|
|
@@ -412,7 +420,7 @@ class BaseAdapter {
|
|
|
412
420
|
} catch (e) {
|
|
413
421
|
if (!this._warnedCursorPersist) {
|
|
414
422
|
this._warnedCursorPersist = true;
|
|
415
|
-
this._log(`Could not persist poll cursor (${e.message}) — a restart will skip messages that arrive while offline
|
|
423
|
+
this._log(`Could not persist poll cursor (${e.message}) — a restart will skip messages that arrive while offline`, 'warn');
|
|
416
424
|
}
|
|
417
425
|
}
|
|
418
426
|
}
|
|
@@ -457,10 +465,14 @@ class BaseAdapter {
|
|
|
457
465
|
try {
|
|
458
466
|
await this.client.heartbeat(this.workspaceId, this.agentName, this.token, this._sessionId);
|
|
459
467
|
this._heartbeatFailStreak = 0;
|
|
468
|
+
const recovered = this._heartbeatFailures.succeed();
|
|
469
|
+
if (recovered) {
|
|
470
|
+
this._log(`Workspace heartbeat recovered after ${recovered.count} unsuccessful attempt(s) over ${formatDuration(recovered.ms)}`);
|
|
471
|
+
}
|
|
460
472
|
this._reportStatus(null); // alive → clear any prior connectivity error
|
|
461
473
|
} catch (e) {
|
|
462
474
|
if (e instanceof SessionRevokedError) {
|
|
463
|
-
this._log(`SESSION REVOKED: another client joined as '${this.agentName}'. Stopping adapter
|
|
475
|
+
this._log(`SESSION REVOKED: another client joined as '${this.agentName}'. Stopping adapter.`, 'warn');
|
|
464
476
|
// Terminal (not a user stop): record so the daemon can show why it ended.
|
|
465
477
|
this._setExitInfo(REASON.SESSION_REVOKED, 'Workspace session revoked — another client joined with the same agent name');
|
|
466
478
|
this._reportStatus(REASON.SESSION_REVOKED, 'Workspace session revoked');
|
|
@@ -471,7 +483,9 @@ class BaseAdapter {
|
|
|
471
483
|
// single transient blip (or an expected brief reconnect) isn't mislabeled.
|
|
472
484
|
this._heartbeatFailStreak++;
|
|
473
485
|
const { reason, message } = classifyHeartbeatError(e);
|
|
474
|
-
|
|
486
|
+
if (this._heartbeatFailures.fail()) {
|
|
487
|
+
this._log(`${message} (consecutive failures: ${this._heartbeatFailStreak})`, 'warn');
|
|
488
|
+
}
|
|
475
489
|
if (this._heartbeatFailStreak >= HEARTBEAT_ERROR_THRESHOLD) {
|
|
476
490
|
this._reportStatus(reason, message);
|
|
477
491
|
}
|
|
@@ -593,7 +607,7 @@ class BaseAdapter {
|
|
|
593
607
|
}
|
|
594
608
|
if (!channel) {
|
|
595
609
|
try { await this._stopSharedWork(); } catch (e) {
|
|
596
|
-
this._log(`Stop: shared work did not stop cleanly: ${e && e.message ? e.message : e}
|
|
610
|
+
this._log(`Stop: shared work did not stop cleanly: ${e && e.message ? e.message : e}`, 'warn');
|
|
597
611
|
}
|
|
598
612
|
}
|
|
599
613
|
for (const ch of channels) {
|
|
@@ -601,7 +615,7 @@ class BaseAdapter {
|
|
|
601
615
|
try {
|
|
602
616
|
outcome = await this._stopChannelWork(ch);
|
|
603
617
|
} catch (e) {
|
|
604
|
-
this._log(`Stop: ${ch} did not stop cleanly: ${e && e.message ? e.message : e}
|
|
618
|
+
this._log(`Stop: ${ch} did not stop cleanly: ${e && e.message ? e.message : e}`, 'warn');
|
|
605
619
|
outcome = 'failed';
|
|
606
620
|
}
|
|
607
621
|
// The pressed channel is always answered, so its UI settles even when
|
|
@@ -788,7 +802,7 @@ class BaseAdapter {
|
|
|
788
802
|
skillId, state: 'installing',
|
|
789
803
|
});
|
|
790
804
|
} catch (e) {
|
|
791
|
-
this._log(`skill.install: could not report 'installing' (non-fatal): ${e && e.message ? e.message : e}
|
|
805
|
+
this._log(`skill.install: could not report 'installing' (non-fatal): ${e && e.message ? e.message : e}`, 'warn');
|
|
792
806
|
}
|
|
793
807
|
|
|
794
808
|
try {
|
|
@@ -822,19 +836,19 @@ class BaseAdapter {
|
|
|
822
836
|
skillId, state: 'installed', path: result.path, partial: result.partial === true,
|
|
823
837
|
});
|
|
824
838
|
} catch (e) {
|
|
825
|
-
this._log(`skill.install: installed on disk but failed to report 'installed': ${e && e.message ? e.message : e}
|
|
839
|
+
this._log(`skill.install: installed on disk but failed to report 'installed': ${e && e.message ? e.message : e}`, 'warn');
|
|
826
840
|
}
|
|
827
841
|
this._log(`skill.install: SUCCESS "${skillId}" → ${result.path}${result.partial ? ' (partial)' : ''}`);
|
|
828
842
|
await this._onSkillsChanged();
|
|
829
843
|
} catch (e) {
|
|
830
844
|
const msg = e && e.message ? e.message : String(e);
|
|
831
|
-
this._log(`skill.install: FAILED "${skillId}": ${msg}
|
|
845
|
+
this._log(`skill.install: FAILED "${skillId}": ${msg}`, 'error');
|
|
832
846
|
try {
|
|
833
847
|
await this.client.reportSkillStatus(this.workspaceId, this.agentName, this.token, {
|
|
834
848
|
skillId, state: 'failed', error: msg,
|
|
835
849
|
});
|
|
836
850
|
} catch (e2) {
|
|
837
|
-
this._log(`skill.install: also failed to report 'failed': ${e2 && e2.message ? e2.message : e2}
|
|
851
|
+
this._log(`skill.install: also failed to report 'failed': ${e2 && e2.message ? e2.message : e2}`, 'warn');
|
|
838
852
|
}
|
|
839
853
|
}
|
|
840
854
|
}
|
|
@@ -863,12 +877,12 @@ class BaseAdapter {
|
|
|
863
877
|
skillId, state: 'uninstalled',
|
|
864
878
|
});
|
|
865
879
|
} catch (e) {
|
|
866
|
-
this._log(`skill.uninstall: failed to report status: ${e && e.message ? e.message : e}
|
|
880
|
+
this._log(`skill.uninstall: failed to report status: ${e && e.message ? e.message : e}`, 'warn');
|
|
867
881
|
}
|
|
868
882
|
await this._onSkillsChanged();
|
|
869
883
|
} catch (e) {
|
|
870
884
|
const msg = e && e.message ? e.message : String(e);
|
|
871
|
-
this._log(`skill.uninstall: FAILED "${skillId}": ${msg}
|
|
885
|
+
this._log(`skill.uninstall: FAILED "${skillId}": ${msg}`, 'error');
|
|
872
886
|
}
|
|
873
887
|
}
|
|
874
888
|
|
|
@@ -923,7 +937,7 @@ class BaseAdapter {
|
|
|
923
937
|
sessionId: this._sessionId,
|
|
924
938
|
});
|
|
925
939
|
} catch (e) {
|
|
926
|
-
this._log(`Status: failed to post: ${e && e.message ? e.message : e}
|
|
940
|
+
this._log(`Status: failed to post: ${e && e.message ? e.message : e}`, 'warn');
|
|
927
941
|
}
|
|
928
942
|
}
|
|
929
943
|
|
|
@@ -950,7 +964,7 @@ class BaseAdapter {
|
|
|
950
964
|
(r) => r.created_by === prefixed || r.created_by === this.agentName,
|
|
951
965
|
);
|
|
952
966
|
} catch (e) {
|
|
953
|
-
this._log(`Routines: failed to list: ${e && e.message ? e.message : e}
|
|
967
|
+
this._log(`Routines: failed to list: ${e && e.message ? e.message : e}`, 'warn');
|
|
954
968
|
try {
|
|
955
969
|
await this.client.sendMessage(
|
|
956
970
|
this.workspaceId, channel, this.token,
|
|
@@ -991,7 +1005,7 @@ class BaseAdapter {
|
|
|
991
1005
|
sessionId: this._sessionId,
|
|
992
1006
|
});
|
|
993
1007
|
} catch (e) {
|
|
994
|
-
this._log(`Routines: failed to post: ${e && e.message ? e.message : e}
|
|
1008
|
+
this._log(`Routines: failed to post: ${e && e.message ? e.message : e}`, 'warn');
|
|
995
1009
|
}
|
|
996
1010
|
}
|
|
997
1011
|
|
|
@@ -1049,11 +1063,26 @@ class BaseAdapter {
|
|
|
1049
1063
|
rawCursor = result.cursor;
|
|
1050
1064
|
composingActive = !!result.composing;
|
|
1051
1065
|
this._pollAuthFailStreak = 0;
|
|
1066
|
+
const recovered = this._pollFailures.succeed();
|
|
1067
|
+
if (recovered) {
|
|
1068
|
+
this._log(`Poll recovered after ${recovered.count} unsuccessful attempt(s) over ${formatDuration(recovered.ms)}`);
|
|
1069
|
+
}
|
|
1052
1070
|
if (pollCount <= 3 || pollCount % 20 === 0) {
|
|
1053
1071
|
this._log(`Poll #${pollCount}: ${messages.length} messages, cursor=${rawCursor || 'none'}${composingActive ? ' composing' : ''}`);
|
|
1054
1072
|
}
|
|
1055
1073
|
} catch (e) {
|
|
1056
|
-
|
|
1074
|
+
// A workspace that is down or slow fails every poll until it is back;
|
|
1075
|
+
// log the streak, not every attempt (see failure-streak.js).
|
|
1076
|
+
if (this._pollFailures.fail()) {
|
|
1077
|
+
const count = this._pollFailures.count;
|
|
1078
|
+
const streak = count > 1 ? ` (consecutive failures: ${count})` : '';
|
|
1079
|
+
if (isTransientNetworkError(e)) {
|
|
1080
|
+
this._log(`Poll #${pollCount} failed: ${e.message}${streak} — retrying every 5s`, 'warn');
|
|
1081
|
+
} else {
|
|
1082
|
+
const stack = count === 1 && e && e.stack ? `\nStack: ${e.stack}` : '';
|
|
1083
|
+
this._log(`Poll #${pollCount} failed: ${e && e.message}${streak}${stack}`, 'error');
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1057
1086
|
// A rejected credential never heals by retrying with the same token —
|
|
1058
1087
|
// it means the device was re-paired/unpaired and this adapter is
|
|
1059
1088
|
// running on a revoked one. After a few consecutive rejections (not
|
|
@@ -1249,7 +1278,7 @@ class BaseAdapter {
|
|
|
1249
1278
|
await this._handleMessage(current);
|
|
1250
1279
|
}
|
|
1251
1280
|
} catch (e) {
|
|
1252
|
-
this._log(`Error ${first ? 'in channel worker' : 'processing queued message'} for ${channel}: ${e.message}
|
|
1281
|
+
this._log(`Error ${first ? 'in channel worker' : 'processing queued message'} for ${channel}: ${e.message}`, 'error');
|
|
1253
1282
|
try { await this.sendError(channel, `Agent error: ${e.message}`); } catch {}
|
|
1254
1283
|
}
|
|
1255
1284
|
delete this._channelRunGeneration[channel];
|
|
@@ -1333,7 +1362,7 @@ class BaseAdapter {
|
|
|
1333
1362
|
delete cache[channel];
|
|
1334
1363
|
// Entry is gone — fall through to list+match below.
|
|
1335
1364
|
} else {
|
|
1336
|
-
this._log(`${label} read failed (${e.message}) — reusing last known state
|
|
1365
|
+
this._log(`${label} read failed (${e.message}) — reusing last known state`, 'warn');
|
|
1337
1366
|
return { available: true, state: 'found', entryId: cachedId, title: titles[0], content: null, error: true };
|
|
1338
1367
|
}
|
|
1339
1368
|
}
|
|
@@ -1369,7 +1398,7 @@ class BaseAdapter {
|
|
|
1369
1398
|
}
|
|
1370
1399
|
return { available: true, state: 'found', entryId: entry.id, title: matchedTitle, content: (full && full.content) || '', error: false };
|
|
1371
1400
|
} catch (e) {
|
|
1372
|
-
this._log(`${label} fetch failed (${e.message}) — reusing last known state
|
|
1401
|
+
this._log(`${label} fetch failed (${e.message}) — reusing last known state`, 'warn');
|
|
1373
1402
|
const knownId = cache[channel] || null;
|
|
1374
1403
|
return { available: true, state: knownId ? 'found' : 'unknown', entryId: knownId, title: knownId ? titles[0] : null, content: null, error: true };
|
|
1375
1404
|
}
|
|
@@ -1427,7 +1456,7 @@ class BaseAdapter {
|
|
|
1427
1456
|
}
|
|
1428
1457
|
this._pinnedContext[channel] = fresh;
|
|
1429
1458
|
} catch (e) {
|
|
1430
|
-
this._log(`Pinned-context fetch failed (${e && e.message ? e.message : e}) — reusing the last pinned context
|
|
1459
|
+
this._log(`Pinned-context fetch failed (${e && e.message ? e.message : e}) — reusing the last pinned context`, 'warn');
|
|
1431
1460
|
}
|
|
1432
1461
|
}
|
|
1433
1462
|
|
|
@@ -1475,7 +1504,7 @@ class BaseAdapter {
|
|
|
1475
1504
|
this._log(`Auto-titled channel: ${title}`);
|
|
1476
1505
|
}
|
|
1477
1506
|
} catch (e) {
|
|
1478
|
-
this._log(`Failed to auto-title channel: ${e.message}
|
|
1507
|
+
this._log(`Failed to auto-title channel: ${e.message}`, 'warn');
|
|
1479
1508
|
}
|
|
1480
1509
|
}
|
|
1481
1510
|
|
|
@@ -1625,7 +1654,7 @@ class BaseAdapter {
|
|
|
1625
1654
|
}
|
|
1626
1655
|
|
|
1627
1656
|
_onSessionRevoked() {
|
|
1628
|
-
this._log(`SESSION REVOKED: another client joined as '${this.agentName}'. Stopping adapter
|
|
1657
|
+
this._log(`SESSION REVOKED: another client joined as '${this.agentName}'. Stopping adapter.`, 'warn');
|
|
1629
1658
|
this._running = false;
|
|
1630
1659
|
}
|
|
1631
1660
|
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Logging policy for background calls that fail repeatedly — the message poll
|
|
5
|
+
* and the workspace heartbeat.
|
|
6
|
+
*
|
|
7
|
+
* Both retry forever, so an unreachable or slow workspace used to produce one
|
|
8
|
+
* entry per attempt: the poll wrote a full stack every ~20s (15s deadline + 5s
|
|
9
|
+
* back-off) at INFO, and the Logs page — inferring severity from "failed" plus
|
|
10
|
+
* the stack — showed a wall of red ERRORs for what was a network outage.
|
|
11
|
+
*
|
|
12
|
+
* A streak logs its 1st, 3rd, 10th, 30th and then every 100th failure, and one
|
|
13
|
+
* line when it ends. Network-level failures are expected in the field and are
|
|
14
|
+
* WARN without a stack; anything else is an ERROR, with its stack once.
|
|
15
|
+
*
|
|
16
|
+
* Pure and dependency-light, so it is unit-testable without an adapter.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const { httpStatusOf } = require('./health-status');
|
|
20
|
+
|
|
21
|
+
const TRANSIENT_CODES = new Set([
|
|
22
|
+
'ETIMEDOUT',
|
|
23
|
+
'ECONNRESET',
|
|
24
|
+
'ECONNREFUSED',
|
|
25
|
+
'ECONNABORTED',
|
|
26
|
+
'ENOTFOUND',
|
|
27
|
+
'EAI_AGAIN',
|
|
28
|
+
'ENETUNREACH',
|
|
29
|
+
'ENETDOWN',
|
|
30
|
+
'EHOSTUNREACH',
|
|
31
|
+
'EPIPE',
|
|
32
|
+
'UND_ERR_SOCKET',
|
|
33
|
+
'UND_ERR_CONNECT_TIMEOUT',
|
|
34
|
+
]);
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* True for failures of the network or of an overloaded server — the kind a
|
|
38
|
+
* retry is expected to fix, not a bug or a rejected credential.
|
|
39
|
+
*/
|
|
40
|
+
function isTransientNetworkError(err) {
|
|
41
|
+
if (!err) return false;
|
|
42
|
+
if (err.code && TRANSIENT_CODES.has(err.code)) return true;
|
|
43
|
+
if (err.name === 'AbortError' || err.name === 'TimeoutError') return true;
|
|
44
|
+
const status = httpStatusOf(err);
|
|
45
|
+
if (typeof status === 'number') return status === 429 || status >= 500;
|
|
46
|
+
return /\b(timed out|socket hang up|network)\b/i.test(err.message || '');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Which failure counts in a streak get a log line. */
|
|
50
|
+
function isLogPoint(count) {
|
|
51
|
+
return count === 1 || count === 3 || count === 10 || count === 30 || count % 100 === 0;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
class FailureStreak {
|
|
55
|
+
constructor() {
|
|
56
|
+
this.count = 0;
|
|
57
|
+
this.since = 0;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Record one failure. True when this one should be logged. */
|
|
61
|
+
fail(now = Date.now()) {
|
|
62
|
+
this.count++;
|
|
63
|
+
if (this.count === 1) this.since = now;
|
|
64
|
+
return isLogPoint(this.count);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Record a success. Returns the streak it ended — `{ count, ms }` — or null
|
|
69
|
+
* when there was none.
|
|
70
|
+
*/
|
|
71
|
+
succeed(now = Date.now()) {
|
|
72
|
+
if (this.count === 0) return null;
|
|
73
|
+
const ended = { count: this.count, ms: now - this.since };
|
|
74
|
+
this.count = 0;
|
|
75
|
+
this.since = 0;
|
|
76
|
+
return ended;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** "45s" / "12m" / "3h" — for the line that closes a streak. */
|
|
81
|
+
function formatDuration(ms) {
|
|
82
|
+
const s = Math.max(0, Math.round(ms / 1000));
|
|
83
|
+
if (s < 120) return `${s}s`;
|
|
84
|
+
const m = Math.round(s / 60);
|
|
85
|
+
if (m < 120) return `${m}m`;
|
|
86
|
+
return `${Math.round(m / 60)}h`;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
module.exports = { FailureStreak, isTransientNetworkError, isLogPoint, formatDuration };
|
package/src/config.js
CHANGED
|
@@ -269,9 +269,18 @@ class Config {
|
|
|
269
269
|
const { keptLines, removed } = filterLogsByTimeRange(allLines, start, end);
|
|
270
270
|
|
|
271
271
|
const nextContent = keptLines.join('\n') + (hasTrailingNewline && keptLines.length > 0 ? '\n' : '');
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
272
|
+
// Rewrite in place, not write-temp + rename: the running daemon's stdout
|
|
273
|
+
// and stderr are append-mode handles to daemon.log. A rename leaves them
|
|
274
|
+
// writing into the replaced file (and fails outright on Windows while they
|
|
275
|
+
// are open); truncating in place keeps them appending to daemon.log.
|
|
276
|
+
const nextBytes = Buffer.from(nextContent, 'utf-8');
|
|
277
|
+
const fd = fs.openSync(this.logFile, 'r+');
|
|
278
|
+
try {
|
|
279
|
+
if (nextBytes.length > 0) fs.writeSync(fd, nextBytes, 0, nextBytes.length, 0);
|
|
280
|
+
fs.ftruncateSync(fd, nextBytes.length);
|
|
281
|
+
} finally {
|
|
282
|
+
fs.closeSync(fd);
|
|
283
|
+
}
|
|
275
284
|
|
|
276
285
|
return { removed, remaining: keptLines.length };
|
|
277
286
|
}
|
package/src/daemon.js
CHANGED
|
@@ -12,6 +12,7 @@ const { WorkspaceClient } = require('./workspace-client');
|
|
|
12
12
|
const { listEndpointModels } = require('./model-list');
|
|
13
13
|
const { AUTH_MODE_KEY, isCliLogin, stripForCliLogin, typeEnvFor } = require('./env');
|
|
14
14
|
const { getEnhancedEnv, whichBinary, IS_WINDOWS, defaultAgentWorkdir } = require('./paths');
|
|
15
|
+
const { LogWriter } = require('./log-writer');
|
|
15
16
|
|
|
16
17
|
/**
|
|
17
18
|
* Mask an API key for display (same shape the workspace backend uses for
|
|
@@ -247,7 +248,7 @@ class Daemon {
|
|
|
247
248
|
this._log(`${name}: workspace credential for '${info.networkRef}' rotated (device re-paired) — restarting to pick up the new token`);
|
|
248
249
|
this.restartAgent(name).catch((e) => {
|
|
249
250
|
info._credRestartPending = false;
|
|
250
|
-
this._log(`${name}: credential-rotation restart failed: ${e.message}
|
|
251
|
+
this._log(`${name}: credential-rotation restart failed: ${e.message}`, 'warn');
|
|
251
252
|
});
|
|
252
253
|
}
|
|
253
254
|
}
|
|
@@ -583,7 +584,7 @@ class Daemon {
|
|
|
583
584
|
}
|
|
584
585
|
for (const [key, value] of Object.entries(config)) {
|
|
585
586
|
if (!allowed.has(key)) {
|
|
586
|
-
this._log(`config for '${type}': refused non-env_config key '${key}'
|
|
587
|
+
this._log(`config for '${type}': refused non-env_config key '${key}'`, 'warn');
|
|
587
588
|
continue;
|
|
588
589
|
}
|
|
589
590
|
if (value === null || value === undefined) continue;
|
|
@@ -805,11 +806,11 @@ async _runNodeCommand(n, cmd) {
|
|
|
805
806
|
this._crashGuardsInstalled = true;
|
|
806
807
|
process.on('unhandledRejection', (reason) => {
|
|
807
808
|
const msg = reason && reason.stack ? reason.stack : String(reason);
|
|
808
|
-
this._log(`UNHANDLED REJECTION (daemon kept alive): ${msg}
|
|
809
|
+
this._log(`UNHANDLED REJECTION (daemon kept alive): ${msg}`, 'error');
|
|
809
810
|
});
|
|
810
811
|
process.on('uncaughtException', (err) => {
|
|
811
812
|
const msg = err && err.stack ? err.stack : String(err);
|
|
812
|
-
this._log(`UNCAUGHT EXCEPTION (daemon kept alive): ${msg}
|
|
813
|
+
this._log(`UNCAUGHT EXCEPTION (daemon kept alive): ${msg}`, 'error');
|
|
813
814
|
});
|
|
814
815
|
}
|
|
815
816
|
|
|
@@ -919,6 +920,7 @@ async _runNodeCommand(n, cmd) {
|
|
|
919
920
|
this._writeStatus();
|
|
920
921
|
this._cleanupPid();
|
|
921
922
|
this._log('Daemon stopped');
|
|
923
|
+
if (this._logWriter) await this._logWriter.flush();
|
|
922
924
|
|
|
923
925
|
if (this._shutdownResolve) this._shutdownResolve();
|
|
924
926
|
}
|
|
@@ -947,7 +949,7 @@ async _runNodeCommand(n, cmd) {
|
|
|
947
949
|
// a hung adapter.run() promise prevents the slot from ever being
|
|
948
950
|
// released, and subsequent start/restart commands see "already running".
|
|
949
951
|
if (this._adapters && this._adapters[agentName]) {
|
|
950
|
-
this._log(`WARNING: ${agentName} adapter did not exit after stop — force-releasing slot
|
|
952
|
+
this._log(`WARNING: ${agentName} adapter did not exit after stop — force-releasing slot`, 'warn');
|
|
951
953
|
try { this._adapters[agentName].stop(); } catch {}
|
|
952
954
|
delete this._adapters[agentName];
|
|
953
955
|
}
|
|
@@ -1282,7 +1284,7 @@ async _runNodeCommand(n, cmd) {
|
|
|
1282
1284
|
const exitCode = await new Promise((resolve) => {
|
|
1283
1285
|
proc.on('exit', (code) => resolve(code));
|
|
1284
1286
|
proc.on('error', (err) => {
|
|
1285
|
-
this._log(`${name} spawn error: ${err.message}
|
|
1287
|
+
this._log(`${name} spawn error: ${err.message}`, 'error');
|
|
1286
1288
|
resolve(1);
|
|
1287
1289
|
});
|
|
1288
1290
|
});
|
|
@@ -1309,13 +1311,13 @@ async _runNodeCommand(n, cmd) {
|
|
|
1309
1311
|
this._writeStatus();
|
|
1310
1312
|
|
|
1311
1313
|
if (info.restarts >= 10) {
|
|
1312
|
-
this._log(`${name} crashed ${info.restarts} times, giving up. Fix the issue and restart manually
|
|
1314
|
+
this._log(`${name} crashed ${info.restarts} times, giving up. Fix the issue and restart manually.`, 'error');
|
|
1313
1315
|
info.state = 'stopped';
|
|
1314
1316
|
this._writeStatus();
|
|
1315
1317
|
break;
|
|
1316
1318
|
}
|
|
1317
1319
|
|
|
1318
|
-
this._log(`${name} crashed: ${info.lastError}, restarting in ${info._backoff}s (attempt ${info.restarts})
|
|
1320
|
+
this._log(`${name} crashed: ${info.lastError}, restarting in ${info._backoff}s (attempt ${info.restarts})`, 'warn');
|
|
1319
1321
|
await this._sleep(info._backoff * 1000);
|
|
1320
1322
|
info._backoff = Math.min(info._backoff * 2, 60);
|
|
1321
1323
|
}
|
|
@@ -1384,6 +1386,7 @@ async _runNodeCommand(n, cmd) {
|
|
|
1384
1386
|
toolMode: agentCfg.tool_mode || 'skills',
|
|
1385
1387
|
// Live runtime/connectivity status → daemon.status.json (Agents list/TUI).
|
|
1386
1388
|
onStatus: (update) => this._applyAdapterStatus(name, info, update),
|
|
1389
|
+
logSink: (level, msg) => this._logAs(`adapter [${name}]`, level, msg),
|
|
1387
1390
|
});
|
|
1388
1391
|
} catch (e) {
|
|
1389
1392
|
// A construction failure (e.g. unknown agent type in this core) is a hard
|
|
@@ -1391,7 +1394,7 @@ async _runNodeCommand(n, cmd) {
|
|
|
1391
1394
|
info.state = 'error';
|
|
1392
1395
|
info.errorReason = 'adapter_crashed';
|
|
1393
1396
|
info.lastError = redactDiagnostic(e.message || String(e));
|
|
1394
|
-
this._log(`${name} failed to create ${agentType} adapter: ${info.lastError}
|
|
1397
|
+
this._log(`${name} failed to create ${agentType} adapter: ${info.lastError}`, 'error');
|
|
1395
1398
|
this._writeStatus();
|
|
1396
1399
|
return;
|
|
1397
1400
|
}
|
|
@@ -1408,11 +1411,11 @@ async _runNodeCommand(n, cmd) {
|
|
|
1408
1411
|
info.errorReason = pf.reason || 'runtime_missing';
|
|
1409
1412
|
info.lastError = redactDiagnostic(pf.message || 'runtime not available');
|
|
1410
1413
|
this._writeStatus();
|
|
1411
|
-
this._log(`${name} preflight failed (${info.errorReason}): ${info.lastError}
|
|
1414
|
+
this._log(`${name} preflight failed (${info.errorReason}): ${info.lastError}`, 'warn');
|
|
1412
1415
|
return;
|
|
1413
1416
|
}
|
|
1414
1417
|
} catch (e) {
|
|
1415
|
-
this._log(`${name} preflight error (ignored, continuing): ${e.message}
|
|
1418
|
+
this._log(`${name} preflight error (ignored, continuing): ${e.message}`, 'warn');
|
|
1416
1419
|
}
|
|
1417
1420
|
|
|
1418
1421
|
// Store adapter reference for stop and duplicate detection
|
|
@@ -1440,7 +1443,7 @@ async _runNodeCommand(n, cmd) {
|
|
|
1440
1443
|
} catch (e) {
|
|
1441
1444
|
info.errorReason = info.errorReason || 'adapter_crashed';
|
|
1442
1445
|
info.lastError = redactDiagnostic((e.message || String(e)).slice(0, 200));
|
|
1443
|
-
this._log(`${name} adapter error: ${info.lastError}
|
|
1446
|
+
this._log(`${name} adapter error: ${info.lastError}`, 'error');
|
|
1444
1447
|
}
|
|
1445
1448
|
|
|
1446
1449
|
delete this._adapters[name];
|
|
@@ -1756,7 +1759,7 @@ async _runNodeCommand(n, cmd) {
|
|
|
1756
1759
|
// The old adapter will exit on its next poll iteration since its
|
|
1757
1760
|
// entry in _stoppedAgents triggers adapter.stop() via checkStop.
|
|
1758
1761
|
if (this._adapters && this._adapters[name]) {
|
|
1759
|
-
this._log(`WARNING: adapter '${name}' did not clear after 10s — force-releasing slot to avoid duplicate
|
|
1762
|
+
this._log(`WARNING: adapter '${name}' did not clear after 10s — force-releasing slot to avoid duplicate`, 'warn');
|
|
1760
1763
|
try { this._adapters[name].stop(); } catch {}
|
|
1761
1764
|
delete this._adapters[name];
|
|
1762
1765
|
}
|
|
@@ -1773,33 +1776,25 @@ async _runNodeCommand(n, cmd) {
|
|
|
1773
1776
|
try { fs.unlinkSync(this.config.statusFile); } catch {}
|
|
1774
1777
|
}
|
|
1775
1778
|
|
|
1776
|
-
_log(msg) {
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1779
|
+
_log(msg, level = 'info') {
|
|
1780
|
+
this._logAs('daemon', level, msg);
|
|
1781
|
+
}
|
|
1782
|
+
|
|
1783
|
+
/**
|
|
1784
|
+
* One entry in daemon.log. Adapters log through here too (see the `logSink`
|
|
1785
|
+
* passed in _adapterLoop), so batching, repeat suppression and rotation cover
|
|
1786
|
+
* their lines as well — they used to bypass all of it via console.log.
|
|
1787
|
+
*/
|
|
1788
|
+
_logAs(scope, level, msg) {
|
|
1789
|
+
if (!this._logWriter) this._logWriter = new LogWriter(this.config.logFile);
|
|
1790
|
+
const written = this._logWriter.write(level, scope, msg);
|
|
1783
1791
|
// Only log to console if stdout is a TTY (not redirected to log file)
|
|
1784
1792
|
// to avoid duplicate lines when daemonized
|
|
1785
|
-
if (!this._shuttingDown && process.stdout.isTTY) {
|
|
1786
|
-
console.log(
|
|
1793
|
+
if (written && !this._shuttingDown && process.stdout.isTTY) {
|
|
1794
|
+
console.log(`${new Date().toISOString()} ${String(level).toUpperCase()} ${scope}: ${msg}`);
|
|
1787
1795
|
}
|
|
1788
1796
|
}
|
|
1789
1797
|
|
|
1790
|
-
_maybeRotateLog() {
|
|
1791
|
-
// Rotate at 10MB, keep 1 backup
|
|
1792
|
-
const MAX_SIZE = 10 * 1024 * 1024;
|
|
1793
|
-
try {
|
|
1794
|
-
const stat = fs.statSync(this.config.logFile);
|
|
1795
|
-
if (stat.size > MAX_SIZE) {
|
|
1796
|
-
const backup = this.config.logFile + '.1';
|
|
1797
|
-
try { fs.unlinkSync(backup); } catch {}
|
|
1798
|
-
fs.renameSync(this.config.logFile, backup);
|
|
1799
|
-
}
|
|
1800
|
-
} catch {}
|
|
1801
|
-
}
|
|
1802
|
-
|
|
1803
1798
|
_sleep(ms) {
|
|
1804
1799
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1805
1800
|
}
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The daemon's writer for daemon.log — one sink shared by the daemon and every
|
|
5
|
+
* adapter running inside it.
|
|
6
|
+
*
|
|
7
|
+
* The line format is unchanged (`<ISO time> <LEVEL> <scope>: <message>`, extra
|
|
8
|
+
* message lines following as continuation lines): Launchers already in the
|
|
9
|
+
* field parse this file, and they may run a newer core than they shipped with.
|
|
10
|
+
* What changes is how lines get there:
|
|
11
|
+
*
|
|
12
|
+
* - Batched, asynchronous writes. The old writer did an appendFileSync — an
|
|
13
|
+
* open/write/close on the event loop — plus a statSync for every line. Lines
|
|
14
|
+
* now collect in memory and go out in one append per `flushMs`, or sooner
|
|
15
|
+
* once `flushBytes` are waiting. The size check runs once per batch.
|
|
16
|
+
*
|
|
17
|
+
* - Repeat suppression. A failure that repeats every few seconds used to fill
|
|
18
|
+
* the file with identical entries and rotate out everything useful. Within
|
|
19
|
+
* each `rateWindowMs`, lines with the same level, scope and message shape
|
|
20
|
+
* (digits and ids masked) pass `rateBurst` times; the rest are counted and
|
|
21
|
+
* reported in one line when the window ends. ERROR lines always pass.
|
|
22
|
+
*
|
|
23
|
+
* - Copy-truncate rotation with gzip generations. The old rename-based
|
|
24
|
+
* rotation moved the file out from under every process that inherited a
|
|
25
|
+
* handle to it — the daemon's own stdout/stderr, and with them every
|
|
26
|
+
* adapter line — so those kept writing into the renamed backup, which the
|
|
27
|
+
* next rotation deleted while still open. Copying the file aside and then
|
|
28
|
+
* truncating it keeps every holder writing to daemon.log (they all opened
|
|
29
|
+
* it in append mode). The copy is gzipped into daemon.log.1.gz, and the
|
|
30
|
+
* older generations shift up to `keep`.
|
|
31
|
+
*
|
|
32
|
+
* Writes are best-effort, as they always were: a log that cannot be written
|
|
33
|
+
* must never take the daemon down.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
const fs = require('fs');
|
|
37
|
+
const zlib = require('zlib');
|
|
38
|
+
const { pipeline } = require('stream');
|
|
39
|
+
|
|
40
|
+
const LEVEL_TOKENS = { debug: 'DEBUG', info: 'INFO', warn: 'WARN', error: 'ERROR' };
|
|
41
|
+
|
|
42
|
+
const DEFAULTS = {
|
|
43
|
+
flushMs: 250,
|
|
44
|
+
flushBytes: 64 * 1024,
|
|
45
|
+
// Backlog bound for a log that cannot keep up (a stalled disk). Past this the
|
|
46
|
+
// oldest waiting lines are dropped and the drop is reported in the log.
|
|
47
|
+
maxPendingBytes: 4 * 1024 * 1024,
|
|
48
|
+
maxBytes: 10 * 1024 * 1024,
|
|
49
|
+
keep: 5,
|
|
50
|
+
rateWindowMs: 60 * 1000,
|
|
51
|
+
rateBurst: 20,
|
|
52
|
+
// Distinct message shapes tracked per window. A shape seen after the table
|
|
53
|
+
// is full is let through rather than tracked.
|
|
54
|
+
maxRateKeys: 1024,
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
/** Every writer with lines that may still be in memory; flushed on exit. */
|
|
58
|
+
const live = new Set();
|
|
59
|
+
let exitHookInstalled = false;
|
|
60
|
+
|
|
61
|
+
function installExitHook() {
|
|
62
|
+
if (exitHookInstalled) return;
|
|
63
|
+
exitHookInstalled = true;
|
|
64
|
+
process.once('exit', () => {
|
|
65
|
+
for (const w of live) w.flushSync();
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function normalizeLevel(level) {
|
|
70
|
+
return Object.prototype.hasOwnProperty.call(LEVEL_TOKENS, level) ? level : 'info';
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* The shape of a message: its first line with ids and numbers masked, so
|
|
75
|
+
* `Poll #257 failed` and `Poll #258 failed` count as the same line.
|
|
76
|
+
*/
|
|
77
|
+
function messageShape(message) {
|
|
78
|
+
const first = String(message).split('\n', 1)[0];
|
|
79
|
+
return first
|
|
80
|
+
.replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '<id>')
|
|
81
|
+
.replace(/\d+/g, '#')
|
|
82
|
+
.slice(0, 200);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
class LogWriter {
|
|
86
|
+
/**
|
|
87
|
+
* @param {string} file - daemon.log
|
|
88
|
+
* @param {object} [opts] - overrides for DEFAULTS
|
|
89
|
+
*/
|
|
90
|
+
constructor(file, opts = {}) {
|
|
91
|
+
this.file = file;
|
|
92
|
+
this.opts = { ...DEFAULTS, ...opts };
|
|
93
|
+
|
|
94
|
+
this._pending = [];
|
|
95
|
+
this._pendingBytes = 0;
|
|
96
|
+
this._timer = null;
|
|
97
|
+
this._writing = false;
|
|
98
|
+
this._rotating = false;
|
|
99
|
+
this._waiters = [];
|
|
100
|
+
|
|
101
|
+
this._rate = new Map();
|
|
102
|
+
this._windowStart = Date.now();
|
|
103
|
+
this._windowTimer = null;
|
|
104
|
+
|
|
105
|
+
this.stats = { written: 0, suppressed: 0, dropped: 0, writeErrors: 0, rotations: 0 };
|
|
106
|
+
this._droppedSinceNote = 0;
|
|
107
|
+
|
|
108
|
+
live.add(this);
|
|
109
|
+
installExitHook();
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Queue one entry. Returns false when it was suppressed as a repeat. */
|
|
113
|
+
write(level, scope, message) {
|
|
114
|
+
const lvl = normalizeLevel(level);
|
|
115
|
+
const now = Date.now();
|
|
116
|
+
if (!this._admit(lvl, scope, message, now)) return false;
|
|
117
|
+
this._enqueue(lvl, scope, message, now);
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Resolves once everything queued so far is on disk (or failed to be). */
|
|
122
|
+
flush() {
|
|
123
|
+
return new Promise((resolve) => {
|
|
124
|
+
this._rollWindow(Date.now());
|
|
125
|
+
if (!this._writing && this._pending.length === 0) return resolve();
|
|
126
|
+
this._waiters.push(resolve);
|
|
127
|
+
if (this._timer) { clearTimeout(this._timer); this._timer = null; }
|
|
128
|
+
this._flush();
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Write whatever is still queued, synchronously. For process exit, where an
|
|
134
|
+
* asynchronous write would never complete.
|
|
135
|
+
*/
|
|
136
|
+
flushSync() {
|
|
137
|
+
this._rollWindow(Date.now());
|
|
138
|
+
if (this._timer) { clearTimeout(this._timer); this._timer = null; }
|
|
139
|
+
const data = this._takePending();
|
|
140
|
+
if (!data) return;
|
|
141
|
+
try {
|
|
142
|
+
fs.appendFileSync(this.file, data, 'utf-8');
|
|
143
|
+
this.stats.written += data.length;
|
|
144
|
+
} catch {
|
|
145
|
+
this.stats.writeErrors++;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Flush synchronously and stop tracking this writer. */
|
|
150
|
+
close() {
|
|
151
|
+
this.flushSync();
|
|
152
|
+
if (this._windowTimer) { clearTimeout(this._windowTimer); this._windowTimer = null; }
|
|
153
|
+
live.delete(this);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// ------------------------------------------------------------------
|
|
157
|
+
// Repeat suppression
|
|
158
|
+
// ------------------------------------------------------------------
|
|
159
|
+
|
|
160
|
+
_admit(level, scope, message, now) {
|
|
161
|
+
if (level === 'error') return true;
|
|
162
|
+
if (now - this._windowStart >= this.opts.rateWindowMs) this._rollWindow(now);
|
|
163
|
+
|
|
164
|
+
const key = `${level}\u0000${scope}\u0000${messageShape(message)}`;
|
|
165
|
+
let slot = this._rate.get(key);
|
|
166
|
+
if (!slot) {
|
|
167
|
+
if (this._rate.size >= this.opts.maxRateKeys) return true;
|
|
168
|
+
slot = { level, scope, sample: String(message).split('\n', 1)[0], seen: 0, suppressed: 0 };
|
|
169
|
+
this._rate.set(key, slot);
|
|
170
|
+
}
|
|
171
|
+
slot.seen++;
|
|
172
|
+
if (slot.seen <= this.opts.rateBurst) return true;
|
|
173
|
+
|
|
174
|
+
slot.suppressed++;
|
|
175
|
+
this.stats.suppressed++;
|
|
176
|
+
// Report at the end of this window even if nothing else is ever logged.
|
|
177
|
+
if (!this._windowTimer) {
|
|
178
|
+
const wait = Math.max(0, this._windowStart + this.opts.rateWindowMs - now);
|
|
179
|
+
this._windowTimer = setTimeout(() => {
|
|
180
|
+
this._windowTimer = null;
|
|
181
|
+
this._rollWindow(Date.now());
|
|
182
|
+
}, wait);
|
|
183
|
+
if (this._windowTimer.unref) this._windowTimer.unref();
|
|
184
|
+
}
|
|
185
|
+
return false;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** Close the current window: one summary line per suppressed shape. */
|
|
189
|
+
_rollWindow(now) {
|
|
190
|
+
const secs = Math.max(1, Math.round((now - this._windowStart) / 1000));
|
|
191
|
+
for (const slot of this._rate.values()) {
|
|
192
|
+
if (slot.suppressed > 0) {
|
|
193
|
+
this._enqueue(
|
|
194
|
+
slot.level,
|
|
195
|
+
slot.scope,
|
|
196
|
+
`Suppressed ${slot.suppressed} more like this in the last ${secs}s: ${slot.sample}`,
|
|
197
|
+
now,
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
this._rate.clear();
|
|
202
|
+
this._windowStart = now;
|
|
203
|
+
if (this._windowTimer) { clearTimeout(this._windowTimer); this._windowTimer = null; }
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// ------------------------------------------------------------------
|
|
207
|
+
// Batching
|
|
208
|
+
// ------------------------------------------------------------------
|
|
209
|
+
|
|
210
|
+
_format(level, scope, message, now) {
|
|
211
|
+
return `${new Date(now).toISOString()} ${LEVEL_TOKENS[level]} ${scope}: ${message}\n`;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
_enqueue(level, scope, message, now) {
|
|
215
|
+
const line = this._format(level, scope, message, now);
|
|
216
|
+
this._pending.push(line);
|
|
217
|
+
this._pendingBytes += line.length;
|
|
218
|
+
|
|
219
|
+
while (this._pendingBytes > this.opts.maxPendingBytes && this._pending.length > 1) {
|
|
220
|
+
this._pendingBytes -= this._pending.shift().length;
|
|
221
|
+
this.stats.dropped++;
|
|
222
|
+
this._droppedSinceNote++;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
this._schedule(this._pendingBytes >= this.opts.flushBytes ? 0 : this.opts.flushMs);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
_schedule(delay) {
|
|
229
|
+
// A write in progress picks up the backlog when it completes.
|
|
230
|
+
if (this._writing) return;
|
|
231
|
+
if (this._timer) {
|
|
232
|
+
if (delay > 0) return;
|
|
233
|
+
clearTimeout(this._timer);
|
|
234
|
+
}
|
|
235
|
+
this._timer = setTimeout(() => {
|
|
236
|
+
this._timer = null;
|
|
237
|
+
this._flush();
|
|
238
|
+
}, delay);
|
|
239
|
+
if (this._timer.unref) this._timer.unref();
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
_takePending() {
|
|
243
|
+
if (this._droppedSinceNote > 0) {
|
|
244
|
+
this._pending.unshift(this._format(
|
|
245
|
+
'warn', 'daemon',
|
|
246
|
+
`Log writer fell behind — dropped ${this._droppedSinceNote} line(s)`,
|
|
247
|
+
Date.now(),
|
|
248
|
+
));
|
|
249
|
+
this._droppedSinceNote = 0;
|
|
250
|
+
}
|
|
251
|
+
if (this._pending.length === 0) return '';
|
|
252
|
+
const data = this._pending.join('');
|
|
253
|
+
this._pending = [];
|
|
254
|
+
this._pendingBytes = 0;
|
|
255
|
+
return data;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
_flush() {
|
|
259
|
+
if (this._writing) return;
|
|
260
|
+
const data = this._takePending();
|
|
261
|
+
if (!data) return this._settle();
|
|
262
|
+
|
|
263
|
+
this._writing = true;
|
|
264
|
+
fs.appendFile(this.file, data, 'utf-8', (err) => {
|
|
265
|
+
if (err) this.stats.writeErrors++;
|
|
266
|
+
else this.stats.written += data.length;
|
|
267
|
+
this._maybeRotate(() => {
|
|
268
|
+
this._writing = false;
|
|
269
|
+
if (this._pending.length > 0) this._flush();
|
|
270
|
+
else this._settle();
|
|
271
|
+
});
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
_settle() {
|
|
276
|
+
if (this._writing || this._pending.length > 0) return;
|
|
277
|
+
const waiters = this._waiters;
|
|
278
|
+
this._waiters = [];
|
|
279
|
+
for (const resolve of waiters) resolve();
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// ------------------------------------------------------------------
|
|
283
|
+
// Rotation
|
|
284
|
+
// ------------------------------------------------------------------
|
|
285
|
+
|
|
286
|
+
_gen(n) {
|
|
287
|
+
return `${this.file}.${n}.gz`;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
_maybeRotate(done) {
|
|
291
|
+
if (this._rotating) return done();
|
|
292
|
+
fs.stat(this.file, (err, st) => {
|
|
293
|
+
if (err || st.size < this.opts.maxBytes) return done();
|
|
294
|
+
this._rotate(done);
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* Copy the full log aside, truncate it in place, then compress the copy in
|
|
300
|
+
* the background. `done` runs as soon as the truncate lands, so writing
|
|
301
|
+
* continues into the emptied file while the copy is compressed.
|
|
302
|
+
*
|
|
303
|
+
* A line another process appends between the copy and the truncate is lost.
|
|
304
|
+
* That window is the duration of one file copy, once per `maxBytes` of log.
|
|
305
|
+
*/
|
|
306
|
+
_rotate(done) {
|
|
307
|
+
this._rotating = true;
|
|
308
|
+
const copy = `${this.file}.rotating`;
|
|
309
|
+
try {
|
|
310
|
+
fs.copyFileSync(this.file, copy);
|
|
311
|
+
fs.truncateSync(this.file, 0);
|
|
312
|
+
} catch {
|
|
313
|
+
try { fs.unlinkSync(copy); } catch { /* not created */ }
|
|
314
|
+
this._rotating = false;
|
|
315
|
+
return done();
|
|
316
|
+
}
|
|
317
|
+
this.stats.rotations++;
|
|
318
|
+
done();
|
|
319
|
+
|
|
320
|
+
this._shiftGenerations();
|
|
321
|
+
const target = this._gen(1);
|
|
322
|
+
const partial = `${target}.tmp`;
|
|
323
|
+
pipeline(
|
|
324
|
+
fs.createReadStream(copy),
|
|
325
|
+
zlib.createGzip(),
|
|
326
|
+
fs.createWriteStream(partial),
|
|
327
|
+
(err) => {
|
|
328
|
+
try { fs.unlinkSync(copy); } catch { /* already gone */ }
|
|
329
|
+
if (err) {
|
|
330
|
+
try { fs.unlinkSync(partial); } catch { /* not created */ }
|
|
331
|
+
} else {
|
|
332
|
+
try { fs.renameSync(partial, target); } catch { /* best effort */ }
|
|
333
|
+
}
|
|
334
|
+
this._rotating = false;
|
|
335
|
+
},
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
_shiftGenerations() {
|
|
340
|
+
const { keep } = this.opts;
|
|
341
|
+
try { fs.unlinkSync(this._gen(keep)); } catch { /* none yet */ }
|
|
342
|
+
for (let n = keep - 1; n >= 1; n--) {
|
|
343
|
+
try { fs.renameSync(this._gen(n), this._gen(n + 1)); } catch { /* none yet */ }
|
|
344
|
+
}
|
|
345
|
+
// The uncompressed backup the previous writer kept. It was always deleted
|
|
346
|
+
// at the next rotation; this is that rotation.
|
|
347
|
+
try { fs.unlinkSync(`${this.file}.1`); } catch { /* none */ }
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
module.exports = { LogWriter, messageShape, DEFAULTS };
|
package/src/workspace-client.js
CHANGED
|
@@ -5,6 +5,29 @@ const http = require('http');
|
|
|
5
5
|
|
|
6
6
|
const DEFAULT_ENDPOINT = 'https://workspace-endpoint.openagents.org';
|
|
7
7
|
|
|
8
|
+
/**
|
|
9
|
+
* The error a request deadline produces. Every request carries a hard
|
|
10
|
+
* AbortSignal.timeout deadline (see _get), and when it fires Node rejects with
|
|
11
|
+
* a bare "AbortError: The operation was aborted" — which reads like somebody
|
|
12
|
+
* cancelled the call and names no cause. The socket-inactivity timeout said
|
|
13
|
+
* "Request timed out". Both now say the same thing, with the deadline, and
|
|
14
|
+
* carry ETIMEDOUT so callers can tell a slow network from a bug.
|
|
15
|
+
*/
|
|
16
|
+
function requestTimeoutError(timeout) {
|
|
17
|
+
const after = timeout >= 1000 ? `${Math.round(timeout / 1000)}s` : `${timeout}ms`;
|
|
18
|
+
const err = new Error(`Request timed out after ${after}`);
|
|
19
|
+
err.code = 'ETIMEDOUT';
|
|
20
|
+
return err;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** A request 'error' handler that reports a fired deadline as a timeout. */
|
|
24
|
+
function rejectWithDeadline(reject, timeout) {
|
|
25
|
+
return (err) => {
|
|
26
|
+
const aborted = err && (err.name === 'AbortError' || err.code === 'ABORT_ERR');
|
|
27
|
+
reject(aborted ? requestTimeoutError(timeout) : err);
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
8
31
|
/**
|
|
9
32
|
* Thrown when the workspace rejects a request because our session_id has
|
|
10
33
|
* been revoked by a newer /v1/join as the same agent. Callers should
|
|
@@ -967,18 +990,25 @@ class WorkspaceClient {
|
|
|
967
990
|
try {
|
|
968
991
|
const parsed = JSON.parse(data);
|
|
969
992
|
if (res.statusCode >= 400) {
|
|
970
|
-
|
|
993
|
+
const err = new Error(parsed.message || `HTTP ${res.statusCode}`);
|
|
994
|
+
// Lets the poll loop tell a rejected credential (401/403) and an
|
|
995
|
+
// overloaded server (5xx) apart from a bug.
|
|
996
|
+
err.status = res.statusCode;
|
|
997
|
+
reject(err);
|
|
971
998
|
} else {
|
|
972
999
|
resolve(parsed);
|
|
973
1000
|
}
|
|
974
1001
|
} catch {
|
|
975
|
-
|
|
1002
|
+
// A proxy's HTML error page (502/504) lands here.
|
|
1003
|
+
const err = new Error(`Invalid response: ${data.slice(0, 200)}`);
|
|
1004
|
+
if (res.statusCode >= 400) err.status = res.statusCode;
|
|
1005
|
+
reject(err);
|
|
976
1006
|
}
|
|
977
1007
|
});
|
|
978
1008
|
});
|
|
979
1009
|
|
|
980
|
-
req.on('error', reject);
|
|
981
|
-
req.on('timeout', () => { req.destroy(); reject(
|
|
1010
|
+
req.on('error', rejectWithDeadline(reject, timeout));
|
|
1011
|
+
req.on('timeout', () => { req.destroy(); reject(requestTimeoutError(timeout)); });
|
|
982
1012
|
req.end();
|
|
983
1013
|
});
|
|
984
1014
|
}
|
|
@@ -1008,8 +1038,8 @@ class WorkspaceClient {
|
|
|
1008
1038
|
});
|
|
1009
1039
|
});
|
|
1010
1040
|
|
|
1011
|
-
req.on('error', reject);
|
|
1012
|
-
req.on('timeout', () => { req.destroy(); reject(
|
|
1041
|
+
req.on('error', rejectWithDeadline(reject, timeout));
|
|
1042
|
+
req.on('timeout', () => { req.destroy(); reject(requestTimeoutError(timeout)); });
|
|
1013
1043
|
req.end();
|
|
1014
1044
|
});
|
|
1015
1045
|
}
|
|
@@ -1058,8 +1088,8 @@ class WorkspaceClient {
|
|
|
1058
1088
|
});
|
|
1059
1089
|
});
|
|
1060
1090
|
|
|
1061
|
-
req.on('error', reject);
|
|
1062
|
-
req.on('timeout', () => { req.destroy(); reject(
|
|
1091
|
+
req.on('error', rejectWithDeadline(reject, timeout));
|
|
1092
|
+
req.on('timeout', () => { req.destroy(); reject(requestTimeoutError(timeout)); });
|
|
1063
1093
|
req.write(jsonBody);
|
|
1064
1094
|
req.end();
|
|
1065
1095
|
});
|
|
@@ -1101,8 +1131,8 @@ class WorkspaceClient {
|
|
|
1101
1131
|
});
|
|
1102
1132
|
});
|
|
1103
1133
|
|
|
1104
|
-
req.on('error', reject);
|
|
1105
|
-
req.on('timeout', () => { req.destroy(); reject(
|
|
1134
|
+
req.on('error', rejectWithDeadline(reject, timeout));
|
|
1135
|
+
req.on('timeout', () => { req.destroy(); reject(requestTimeoutError(timeout)); });
|
|
1106
1136
|
req.write(jsonBody);
|
|
1107
1137
|
req.end();
|
|
1108
1138
|
});
|
|
@@ -1139,8 +1169,8 @@ class WorkspaceClient {
|
|
|
1139
1169
|
});
|
|
1140
1170
|
});
|
|
1141
1171
|
|
|
1142
|
-
req.on('error', reject);
|
|
1143
|
-
req.on('timeout', () => { req.destroy(); reject(
|
|
1172
|
+
req.on('error', rejectWithDeadline(reject, timeout));
|
|
1173
|
+
req.on('timeout', () => { req.destroy(); reject(requestTimeoutError(timeout)); });
|
|
1144
1174
|
req.write(jsonBody);
|
|
1145
1175
|
req.end();
|
|
1146
1176
|
});
|
|
@@ -1175,8 +1205,8 @@ class WorkspaceClient {
|
|
|
1175
1205
|
});
|
|
1176
1206
|
});
|
|
1177
1207
|
|
|
1178
|
-
req.on('error', reject);
|
|
1179
|
-
req.on('timeout', () => { req.destroy(); reject(
|
|
1208
|
+
req.on('error', rejectWithDeadline(reject, 15000));
|
|
1209
|
+
req.on('timeout', () => { req.destroy(); reject(requestTimeoutError(15000)); });
|
|
1180
1210
|
req.end();
|
|
1181
1211
|
});
|
|
1182
1212
|
}
|