clementine-agent 1.0.9 → 1.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/assistant.js +48 -7
- package/dist/agent/metacognition.js +16 -1
- package/dist/agent/stall-guard.d.ts +4 -0
- package/dist/agent/stall-guard.js +4 -0
- package/dist/channels/discord-utils.js +25 -8
- package/dist/gateway/cron-scheduler.js +2 -1
- package/dist/gateway/heartbeat-scheduler.js +10 -1
- package/dist/gateway/router.d.ts +5 -2
- package/dist/gateway/router.js +59 -10
- package/package.json +1 -1
package/dist/agent/assistant.js
CHANGED
|
@@ -1357,6 +1357,13 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
|
|
|
1357
1357
|
if (stallGuard) {
|
|
1358
1358
|
const stallCheck = stallGuard.shouldBlockTool(toolName);
|
|
1359
1359
|
if (stallCheck.block) {
|
|
1360
|
+
// When the breaker engages we also abort the whole query —
|
|
1361
|
+
// denying a single tool isn't enough for a runaway loop,
|
|
1362
|
+
// the agent will just try the next read-only tool.
|
|
1363
|
+
if (abortController && !abortController.signal.aborted) {
|
|
1364
|
+
logger.warn({ sessionKey, toolName }, 'StallGuard breaker engaged — aborting query');
|
|
1365
|
+
abortController.abort();
|
|
1366
|
+
}
|
|
1360
1367
|
return { behavior: 'deny', message: stallCheck.message ?? 'Stall breaker.' };
|
|
1361
1368
|
}
|
|
1362
1369
|
}
|
|
@@ -2034,9 +2041,29 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
|
|
|
2034
2041
|
catch (e) {
|
|
2035
2042
|
const errStr = String(e).toLowerCase();
|
|
2036
2043
|
if (errStr.includes('abort') || errStr.includes('cancel')) {
|
|
2037
|
-
// Query was aborted
|
|
2038
|
-
|
|
2039
|
-
|
|
2044
|
+
// Query was aborted. Four sources: timeout, user cancel, StallGuard
|
|
2045
|
+
// tripped (runaway loop), or interrupted by a new user message.
|
|
2046
|
+
const stallAbort = !!stallGuard?.isBreakerActive();
|
|
2047
|
+
const abortReason = abortController?.signal.reason;
|
|
2048
|
+
const interruptAbort = abortReason === 'interrupted-by-new-message';
|
|
2049
|
+
logger.warn({ sessionKey, stallAbort, interruptAbort }, 'Chat query aborted');
|
|
2050
|
+
if (interruptAbort) {
|
|
2051
|
+
// New message came in — let the next query answer. Just mark
|
|
2052
|
+
// the partial response so the user knows this one was cut off.
|
|
2053
|
+
// (The next handleMessage call will fold this partial into its prompt.)
|
|
2054
|
+
responseText = responseText
|
|
2055
|
+
? responseText + '\n\n*(interrupted — answering your new message…)*'
|
|
2056
|
+
: '*(interrupted — switching to your new message…)*';
|
|
2057
|
+
}
|
|
2058
|
+
else if (stallAbort) {
|
|
2059
|
+
const reason = stallGuard?.getBreakerReason() ?? 'runaway loop';
|
|
2060
|
+
const stallMsg = `I got stuck in a loop — ${reason} ` +
|
|
2061
|
+
`I stopped to save budget. Options:\n` +
|
|
2062
|
+
`• Rephrase your request more specifically\n` +
|
|
2063
|
+
`• Reply "deep mode" to queue this as a background task with a bigger budget`;
|
|
2064
|
+
responseText = responseText ? responseText + '\n\n' + stallMsg : stallMsg;
|
|
2065
|
+
}
|
|
2066
|
+
else if (!responseText) {
|
|
2040
2067
|
responseText = 'I ran out of time on this one. Let me know if you want me to pick it back up.';
|
|
2041
2068
|
}
|
|
2042
2069
|
else {
|
|
@@ -2073,7 +2100,8 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
|
|
|
2073
2100
|
responseText = responseText || 'The conversation context filled up from large tool outputs. I\'ve reset the session — please try again, and I\'ll keep query results smaller this time.';
|
|
2074
2101
|
}
|
|
2075
2102
|
else if (errStr.includes('prompt is too long') || errStr.includes('prompt too long') || errStr.includes('context_length')) {
|
|
2076
|
-
responseText = responseText || '
|
|
2103
|
+
responseText = responseText || ('The conversation got too large to process (tool responses filled the context window). ' +
|
|
2104
|
+
"I've reset the session. Try again — I'll keep result sets smaller this time.");
|
|
2077
2105
|
}
|
|
2078
2106
|
else if (errStr.includes('no conversation found') || errStr.includes('conversation not found') || errStr.includes('session not found')) {
|
|
2079
2107
|
// Stale session — clear and retry
|
|
@@ -2094,9 +2122,20 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
|
|
|
2094
2122
|
else {
|
|
2095
2123
|
logger.error({ err: e, sessionKey }, 'SDK query failed');
|
|
2096
2124
|
if (!responseText) {
|
|
2097
|
-
//
|
|
2125
|
+
// Classify so the user gets a useful suggestion instead of raw error text.
|
|
2098
2126
|
const shortErr = String(e).replace(/\n.*$/s, '').slice(0, 200);
|
|
2099
|
-
|
|
2127
|
+
const lowerErr = String(e).toLowerCase();
|
|
2128
|
+
let hint = '';
|
|
2129
|
+
if (lowerErr.includes('econnrefused') || lowerErr.includes('socket') || lowerErr.includes('network')) {
|
|
2130
|
+
hint = 'Looks like a network issue — check your internet and try again.';
|
|
2131
|
+
}
|
|
2132
|
+
else if (lowerErr.includes('spawn') || lowerErr.includes('enoent')) {
|
|
2133
|
+
hint = 'A required binary seems to be missing. Try `clementine doctor` to diagnose.';
|
|
2134
|
+
}
|
|
2135
|
+
else {
|
|
2136
|
+
hint = 'Try again, or `!clear` to reset the session. If it keeps happening, check `~/.clementine/logs/clementine.log`.';
|
|
2137
|
+
}
|
|
2138
|
+
responseText = `I hit an error: ${shortErr}\n\n${hint}`;
|
|
2100
2139
|
}
|
|
2101
2140
|
}
|
|
2102
2141
|
}
|
|
@@ -3644,7 +3683,9 @@ You have a cost budget per message — not a hard turn limit. Work until the tas
|
|
|
3644
3683
|
appendProgress({ event: 'aborted', phase, reason: `${MAX_CONSECUTIVE_ERRORS} consecutive phase errors` });
|
|
3645
3684
|
writeStatus({ jobName, status: 'error', phase, startedAt, finishedAt: new Date().toISOString() });
|
|
3646
3685
|
logger.error(`Unleashed task ${jobName} aborted after ${MAX_CONSECUTIVE_ERRORS} consecutive errors`);
|
|
3647
|
-
const errorResult = lastOutput || `Task "${jobName}" aborted after ${MAX_CONSECUTIVE_ERRORS} consecutive phase errors
|
|
3686
|
+
const errorResult = lastOutput || (`Task "${jobName}" aborted after ${MAX_CONSECUTIVE_ERRORS} consecutive phase errors. ` +
|
|
3687
|
+
`Check \`clementine cron runs ${jobName}\` for the failing phase, or retry with ` +
|
|
3688
|
+
`\`clementine cron run ${jobName}\`.`);
|
|
3648
3689
|
if (this.onUnleashedComplete) {
|
|
3649
3690
|
try {
|
|
3650
3691
|
this.onUnleashedComplete(jobName, errorResult);
|
|
@@ -94,7 +94,22 @@ export class MetacognitiveMonitor {
|
|
|
94
94
|
this.interventionCount++;
|
|
95
95
|
return signal;
|
|
96
96
|
}
|
|
97
|
-
// Signal: excessive tool calls
|
|
97
|
+
// Signal: excessive tool calls with near-zero output.
|
|
98
|
+
// Warn at 20, intervene (hard stop) at 60 — beyond 60 the agent is
|
|
99
|
+
// almost certainly in a runaway loop that will burn through the
|
|
100
|
+
// budget cap with nothing to show for it.
|
|
101
|
+
if (this.toolCalls.length >= 60 && this.outputCharCount < 200) {
|
|
102
|
+
this.confidence = 'low';
|
|
103
|
+
if (!this.signals.includes('high_effort_low_output')) {
|
|
104
|
+
this.signals.push('high_effort_low_output');
|
|
105
|
+
}
|
|
106
|
+
this.interventionCount++;
|
|
107
|
+
return {
|
|
108
|
+
type: 'intervene',
|
|
109
|
+
reason: 'high_effort_low_output',
|
|
110
|
+
guidance: `You've made ${this.toolCalls.length} tool calls across ${this.uniqueTools.size} tools with only ${this.outputCharCount} chars of output. This is a runaway loop. Stopping now to prevent budget waste.`,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
98
113
|
if (this.toolCalls.length > 20 && this.outputCharCount < 200) {
|
|
99
114
|
this.confidence = 'low';
|
|
100
115
|
if (!this.signals.includes('high_effort_low_output')) {
|
|
@@ -33,6 +33,10 @@ export declare class StallGuard {
|
|
|
33
33
|
block: boolean;
|
|
34
34
|
message?: string;
|
|
35
35
|
};
|
|
36
|
+
/** True when the stall breaker has been engaged during this query. */
|
|
37
|
+
isBreakerActive(): boolean;
|
|
38
|
+
/** Reason string set when the breaker engaged (empty if not active). */
|
|
39
|
+
getBreakerReason(): string;
|
|
36
40
|
/**
|
|
37
41
|
* Record a tool call. Runs loop detection and metacognition.
|
|
38
42
|
* Activates the breaker if either detector fires.
|
|
@@ -41,6 +41,10 @@ export class StallGuard {
|
|
|
41
41
|
}
|
|
42
42
|
return { block: false };
|
|
43
43
|
}
|
|
44
|
+
/** True when the stall breaker has been engaged during this query. */
|
|
45
|
+
isBreakerActive() { return this.breakerActive; }
|
|
46
|
+
/** Reason string set when the breaker engaged (empty if not active). */
|
|
47
|
+
getBreakerReason() { return this.breakerReason; }
|
|
44
48
|
/**
|
|
45
49
|
* Record a tool call. Runs loop detection and metacognition.
|
|
46
50
|
* Activates the breaker if either detector fires.
|
|
@@ -161,20 +161,37 @@ export class DiscordStreamingMessage {
|
|
|
161
161
|
this.progressTimer = null;
|
|
162
162
|
}
|
|
163
163
|
if (!text)
|
|
164
|
-
text =
|
|
164
|
+
text = "*(I didn't have anything to respond with — try rephrasing or giving me more context.)*";
|
|
165
165
|
text = sanitizeResponse(text);
|
|
166
|
-
|
|
167
|
-
if (
|
|
168
|
-
|
|
169
|
-
|
|
166
|
+
try {
|
|
167
|
+
if (this.message) {
|
|
168
|
+
if (text.length <= 1900) {
|
|
169
|
+
await this.message.edit(text);
|
|
170
|
+
this.messageId = this.message.id;
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
await this.message.delete().catch(() => { });
|
|
174
|
+
await sendChunked(this.channel, text);
|
|
175
|
+
}
|
|
170
176
|
}
|
|
171
177
|
else {
|
|
172
|
-
await this.message.delete().catch(() => { });
|
|
173
178
|
await sendChunked(this.channel, text);
|
|
174
179
|
}
|
|
175
180
|
}
|
|
176
|
-
|
|
177
|
-
|
|
181
|
+
catch (err) {
|
|
182
|
+
// Delivery failed after the agent already generated a response.
|
|
183
|
+
// Log loudly + persist the response text to the daily note so it isn't
|
|
184
|
+
// lost silently. Don't re-throw — the callers don't have try/catch
|
|
185
|
+
// around finalize() and we don't want to introduce crashes.
|
|
186
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
187
|
+
try {
|
|
188
|
+
const pino = (await import('pino')).default;
|
|
189
|
+
pino({ name: 'clementine.discord' }).warn({ err: errMsg, channelId: this.channel.id }, 'Discord delivery failed — response text saved to daily note');
|
|
190
|
+
const { logToDailyNote } = await import('../gateway/cron-scheduler.js');
|
|
191
|
+
const preview = text.slice(0, 1500);
|
|
192
|
+
logToDailyNote(`**[Discord delivery failed]** Channel \`${this.channel.id ?? 'unknown'}\` — response was:\n\n${preview}`);
|
|
193
|
+
}
|
|
194
|
+
catch { /* best-effort */ }
|
|
178
195
|
}
|
|
179
196
|
}
|
|
180
197
|
/** Format elapsed milliseconds as human-readable duration. */
|
|
@@ -1184,7 +1184,8 @@ export class CronScheduler {
|
|
|
1184
1184
|
// Truncate
|
|
1185
1185
|
if (msg.length > 300)
|
|
1186
1186
|
msg = msg.slice(0, 297) + '...';
|
|
1187
|
-
return
|
|
1187
|
+
return (`Cron \`${jobName}\` failed: ${msg.trim()}\n` +
|
|
1188
|
+
`Check \`clementine cron runs ${jobName}\` for details, or retry with \`clementine cron run ${jobName}\`.`);
|
|
1188
1189
|
}
|
|
1189
1190
|
listJobs() {
|
|
1190
1191
|
if (this.jobs.length === 0) {
|
|
@@ -135,7 +135,16 @@ export class HeartbeatScheduler {
|
|
|
135
135
|
this.dispatcher.send(`**Self-Improvement Failed (nightly)**\n` +
|
|
136
136
|
`The self-improvement loop crashed: ${String(err).slice(0, 200)}\n\n` +
|
|
137
137
|
`This will keep failing every night until the root cause is fixed. ` +
|
|
138
|
-
`Ask me to check the self-improvement status for details.`, {}).catch(() => {
|
|
138
|
+
`Ask me to check the self-improvement status for details.`, {}).catch(async (sendErr) => {
|
|
139
|
+
// If the notification about the failure also failed, surface it to the daily note
|
|
140
|
+
// so the user sees it on their next check-in instead of it vanishing into logs.
|
|
141
|
+
logger.warn({ err: sendErr }, 'Failed to notify about self-improvement failure — writing to daily note');
|
|
142
|
+
try {
|
|
143
|
+
const { logToDailyNote } = await import('./cron-scheduler.js');
|
|
144
|
+
logToDailyNote(`**[Self-improvement crashed]** ${String(err).slice(0, 400)}`);
|
|
145
|
+
}
|
|
146
|
+
catch { /* best-effort */ }
|
|
147
|
+
});
|
|
139
148
|
});
|
|
140
149
|
}
|
|
141
150
|
// Weekly per-agent improvement: one agent per day at 2 AM, cycling through
|
package/dist/gateway/router.d.ts
CHANGED
|
@@ -121,8 +121,11 @@ export declare class Gateway {
|
|
|
121
121
|
*/
|
|
122
122
|
stopSession(sessionKey: string): boolean;
|
|
123
123
|
/**
|
|
124
|
-
* Serialize access to a session.
|
|
125
|
-
*
|
|
124
|
+
* Serialize access to a session. If a query is already in-flight when a new
|
|
125
|
+
* message arrives, we interrupt it — abort the running query, capture its
|
|
126
|
+
* partial output so the next handler can fold it into the new prompt, then
|
|
127
|
+
* wait for the aborted handler to release the lock. This lets users redirect
|
|
128
|
+
* or correct the agent mid-response instead of queuing behind a long query.
|
|
126
129
|
*/
|
|
127
130
|
private acquireSessionLock;
|
|
128
131
|
handleMessage(sessionKey: string, text: string, onText?: OnTextCallback, model?: string, maxTurns?: number, onToolActivity?: OnToolActivityCallback): Promise<string>;
|
package/dist/gateway/router.js
CHANGED
|
@@ -193,7 +193,15 @@ export class Gateway {
|
|
|
193
193
|
logger.warn({ err, sessionKey }, 'Deep mode agent follow-up failed — using raw fallback');
|
|
194
194
|
if (rawFallback.trim()) {
|
|
195
195
|
await this._dispatcher?.send(rawFallback.slice(0, 1500))
|
|
196
|
-
.catch(e =>
|
|
196
|
+
.catch(async (e) => {
|
|
197
|
+
// Both paths failed — surface it instead of swallowing at debug level.
|
|
198
|
+
logger.warn({ err: e, sessionKey }, 'Deep mode fallback delivery failed — persisting to daily note');
|
|
199
|
+
try {
|
|
200
|
+
const { logToDailyNote } = await import('./cron-scheduler.js');
|
|
201
|
+
logToDailyNote(`**[Deep mode delivery failed]** Session ${sessionKey} — result was:\n\n${rawFallback.slice(0, 1500)}`);
|
|
202
|
+
}
|
|
203
|
+
catch { /* best-effort */ }
|
|
204
|
+
});
|
|
197
205
|
}
|
|
198
206
|
}
|
|
199
207
|
}
|
|
@@ -524,16 +532,30 @@ export class Gateway {
|
|
|
524
532
|
return false;
|
|
525
533
|
}
|
|
526
534
|
/**
|
|
527
|
-
* Serialize access to a session.
|
|
528
|
-
*
|
|
535
|
+
* Serialize access to a session. If a query is already in-flight when a new
|
|
536
|
+
* message arrives, we interrupt it — abort the running query, capture its
|
|
537
|
+
* partial output so the next handler can fold it into the new prompt, then
|
|
538
|
+
* wait for the aborted handler to release the lock. This lets users redirect
|
|
539
|
+
* or correct the agent mid-response instead of queuing behind a long query.
|
|
529
540
|
*/
|
|
530
541
|
async acquireSessionLock(sessionKey) {
|
|
531
|
-
// Wait for any existing lock to resolve
|
|
532
542
|
let s = this.getSession(sessionKey);
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
543
|
+
// If a query is in-flight, interrupt it rather than wait indefinitely.
|
|
544
|
+
if (s.lock) {
|
|
545
|
+
if (s.abortController && !s.abortController.signal.aborted) {
|
|
546
|
+
const partial = s.lastStreamedText ?? '';
|
|
547
|
+
s.pendingInterrupt = { partial, interruptedAt: Date.now() };
|
|
548
|
+
logger.info({ sessionKey, partialLen: partial.length }, 'New message arrived — interrupting in-flight query');
|
|
549
|
+
// Pass a reason string so assistant.ts can distinguish this from a
|
|
550
|
+
// timeout abort and show the right final message.
|
|
551
|
+
s.abortController.abort('interrupted-by-new-message');
|
|
552
|
+
}
|
|
553
|
+
// Drain any remaining lock promises (the aborted handler still needs to
|
|
554
|
+
// finish its finally block before we can proceed).
|
|
555
|
+
while (s.lock) {
|
|
556
|
+
await s.lock;
|
|
557
|
+
s = this.getSession(sessionKey);
|
|
558
|
+
}
|
|
537
559
|
}
|
|
538
560
|
// Create a new lock (a promise + its resolver)
|
|
539
561
|
let releaseFn;
|
|
@@ -725,8 +747,17 @@ export class Gateway {
|
|
|
725
747
|
let toolActivityCount = 0;
|
|
726
748
|
let lastStreamedText = '';
|
|
727
749
|
let lastProgressEmitAt = Date.now();
|
|
750
|
+
const sessState = this.getSession(sessionKey);
|
|
728
751
|
const wrappedOnText = onText
|
|
729
|
-
? async (token) => {
|
|
752
|
+
? async (token) => {
|
|
753
|
+
resetIdleTimer();
|
|
754
|
+
lastStreamedText = token;
|
|
755
|
+
// Mirror to session state so a concurrent acquireSessionLock()
|
|
756
|
+
// can capture the partial output on interrupt.
|
|
757
|
+
sessState.lastStreamedText = token;
|
|
758
|
+
lastProgressEmitAt = Date.now();
|
|
759
|
+
return onText(token);
|
|
760
|
+
}
|
|
730
761
|
: undefined;
|
|
731
762
|
// Progress streaming: emit brief status indicators during long tool chains
|
|
732
763
|
// so the user doesn't see silence while the agent works
|
|
@@ -762,6 +793,24 @@ export class Gateway {
|
|
|
762
793
|
]);
|
|
763
794
|
}, CHAT_MAX_WALL_MS);
|
|
764
795
|
});
|
|
796
|
+
// If the previous query on this session was interrupted by this
|
|
797
|
+
// incoming message, fold the partial output in so the agent can pivot
|
|
798
|
+
// smoothly instead of re-planning from scratch.
|
|
799
|
+
let chatPrompt = text;
|
|
800
|
+
const interrupt = sessState.pendingInterrupt;
|
|
801
|
+
if (interrupt && interrupt.partial.trim()) {
|
|
802
|
+
delete sessState.pendingInterrupt;
|
|
803
|
+
const partialPreview = interrupt.partial.slice(0, 1500);
|
|
804
|
+
chatPrompt =
|
|
805
|
+
`[You were mid-response when the user sent a new message — they chose not to wait. ` +
|
|
806
|
+
`Here's what you had said so far (may be mid-sentence):\n---\n${partialPreview}\n---\n` +
|
|
807
|
+
`New message from user:]\n\n${text}`;
|
|
808
|
+
logger.info({ sessionKey, partialLen: interrupt.partial.length }, 'Folding interrupted partial into new prompt');
|
|
809
|
+
}
|
|
810
|
+
else if (interrupt) {
|
|
811
|
+
// Interrupt flag was set but no useful partial text — just clear it.
|
|
812
|
+
delete sessState.pendingInterrupt;
|
|
813
|
+
}
|
|
765
814
|
try {
|
|
766
815
|
// No artificial turn cap — let the agent work until done.
|
|
767
816
|
// Primary guardrail is cost budget (maxBudgetUsd in buildOptions).
|
|
@@ -769,7 +818,7 @@ export class Gateway {
|
|
|
769
818
|
events.emit('query:start', { sessionKey, model: effectiveModel, maxTurns: maxTurns, timestamp: Date.now() });
|
|
770
819
|
const queryStartMs = Date.now();
|
|
771
820
|
const [response] = await Promise.race([
|
|
772
|
-
this.assistant.chat(
|
|
821
|
+
this.assistant.chat(chatPrompt, effectiveSessionKey, { onText: wrappedOnText, onToolActivity: wrappedOnToolActivity, model: effectiveModel, maxTurns: maxTurns, securityAnnotation, projectOverride, profile: resolvedProfile, verboseLevel, abortController: chatAc }),
|
|
773
822
|
hardWallPromise,
|
|
774
823
|
]);
|
|
775
824
|
clearTimeout(chatTimer);
|