clementine-agent 1.18.20 → 1.18.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/dist/agent/action-enforcer.d.ts +29 -0
- package/dist/agent/action-enforcer.js +120 -0
- package/dist/agent/assistant.d.ts +12 -0
- package/dist/agent/assistant.js +165 -31
- package/dist/agent/auto-update.js +46 -2
- package/dist/agent/local-turn.d.ts +16 -0
- package/dist/agent/local-turn.js +54 -1
- package/dist/agent/route-classifier.d.ts +1 -0
- package/dist/agent/route-classifier.js +30 -3
- package/dist/agent/toolsets.d.ts +14 -0
- package/dist/agent/toolsets.js +68 -0
- package/dist/brain/ingestion-pipeline.d.ts +7 -0
- package/dist/brain/ingestion-pipeline.js +107 -21
- package/dist/channels/discord.js +38 -7
- package/dist/channels/telegram.js +5 -6
- package/dist/cli/dashboard.js +56 -6
- package/dist/cli/index.js +174 -0
- package/dist/cli/ingest.js +8 -2
- package/dist/gateway/context-hygiene.d.ts +17 -0
- package/dist/gateway/context-hygiene.js +31 -0
- package/dist/gateway/heartbeat-scheduler.d.ts +20 -0
- package/dist/gateway/heartbeat-scheduler.js +27 -10
- package/dist/gateway/router.d.ts +7 -0
- package/dist/gateway/router.js +303 -9
- package/dist/gateway/turn-ledger.d.ts +32 -0
- package/dist/gateway/turn-ledger.js +55 -0
- package/dist/memory/embeddings.d.ts +2 -0
- package/dist/memory/embeddings.js +8 -1
- package/dist/memory/store.d.ts +88 -1
- package/dist/memory/store.js +349 -18
- package/dist/memory/write-queue.d.ts +16 -0
- package/dist/memory/write-queue.js +5 -0
- package/dist/tools/shared.d.ts +89 -0
- package/dist/types.d.ts +11 -0
- package/package.json +1 -1
- package/scripts/postinstall.js +56 -6
package/dist/channels/discord.js
CHANGED
|
@@ -13,6 +13,8 @@ import path from 'node:path';
|
|
|
13
13
|
import { chunkText, sendChunked, DiscordStreamingMessage, friendlyToolName, formatCronEmbed, rehydrateStatusEmbed, setSavedStatusEmbed, } from './discord-utils.js';
|
|
14
14
|
import { DISCORD_TOKEN, DISCORD_OWNER_ID, DISCORD_WATCHED_CHANNELS, MODELS, ASSISTANT_NAME, OWNER_NAME, PKG_DIR, VAULT_DIR, BASE_DIR, DEFAULT_MODEL_TIER, } from '../config.js';
|
|
15
15
|
import { findProjectByName, getLinkedProjects } from '../agent/assistant.js';
|
|
16
|
+
import { detectApprovalReply } from '../agent/local-turn.js';
|
|
17
|
+
import { normalizeToolsetName } from '../agent/toolsets.js';
|
|
16
18
|
import * as cronParser from 'cron-parser';
|
|
17
19
|
const logger = pino({ name: 'clementine.discord' });
|
|
18
20
|
const BOT_MESSAGE_TRACKING_LIMIT = 100;
|
|
@@ -35,6 +37,12 @@ const slashCommands = [
|
|
|
35
37
|
.addStringOption(o => o.setName('job').setDescription('Job name (for run/enable/disable)').setAutocomplete(true)),
|
|
36
38
|
new SlashCommandBuilder().setName('heartbeat').setDescription('Run heartbeat check manually'),
|
|
37
39
|
new SlashCommandBuilder().setName('tools').setDescription('List available MCP tools'),
|
|
40
|
+
new SlashCommandBuilder().setName('toolset').setDescription('Set this chat tool mode')
|
|
41
|
+
.addStringOption(o => o.setName('mode').setDescription('Tool mode').setRequired(true)
|
|
42
|
+
.addChoices({ name: 'Auto', value: 'auto' }, { name: 'Safe', value: 'safe' }, { name: 'Diagnostic', value: 'diagnostic' }, { name: 'Communications', value: 'communications' }, { name: 'Memory', value: 'memory' }, { name: 'Full', value: 'full' })),
|
|
43
|
+
new SlashCommandBuilder().setName('compress').setDescription('Compact this conversation context into memory'),
|
|
44
|
+
new SlashCommandBuilder().setName('usage').setDescription('Show recent turn/tool usage for this chat'),
|
|
45
|
+
new SlashCommandBuilder().setName('debug').setDescription('Show session diagnostics for this chat'),
|
|
38
46
|
new SlashCommandBuilder().setName('project').setDescription('Set active project context')
|
|
39
47
|
.addStringOption(o => o.setName('action').setDescription('Action').setRequired(true)
|
|
40
48
|
.addChoices({ name: 'List projects', value: 'list' }, { name: 'Set active project', value: 'set' }, { name: 'Clear active project', value: 'clear' }, { name: 'Show current', value: 'status' }))
|
|
@@ -269,6 +277,8 @@ function handleHelp() {
|
|
|
269
277
|
'`!self-improve run|status|history|pending|apply|deny` \u2014 Self-improvement',
|
|
270
278
|
'`!team setup|list|status|messages|topology` \u2014 Manage agent team',
|
|
271
279
|
'`!status [job]` \u2014 Check unleashed task progress',
|
|
280
|
+
'`/toolset` \u2014 Set tool mode \u00b7 `/compress` \u2014 Compact context \u00b7 `/usage` \u2014 Usage snapshot',
|
|
281
|
+
'`/debug` \u2014 Session diagnostics',
|
|
272
282
|
'`!dashboard` \u2014 Send a fresh system status embed',
|
|
273
283
|
'`!heartbeat` \u2014 Run heartbeat \u00b7 `!tools` \u2014 List tools \u00b7 `!clear` \u2014 Reset',
|
|
274
284
|
'`!stop` \u2014 Interrupt current response',
|
|
@@ -1074,15 +1084,12 @@ export async function startDiscord(gateway, heartbeat, cronScheduler, dispatcher
|
|
|
1074
1084
|
}
|
|
1075
1085
|
// ── Approval responses (DM only) ────────────────────────────────
|
|
1076
1086
|
if (isDm) {
|
|
1077
|
-
const
|
|
1078
|
-
if (
|
|
1087
|
+
const approvalReply = detectApprovalReply(text);
|
|
1088
|
+
if (approvalReply !== null) {
|
|
1079
1089
|
const approvals = gateway.getPendingApprovals();
|
|
1080
1090
|
if (approvals.length > 0) {
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
(lower === 'yes' || lower === 'approve' || lower === 'go');
|
|
1084
|
-
gateway.resolveApproval(approvals[approvals.length - 1], result);
|
|
1085
|
-
await message.react(lower === 'no' || lower === 'deny' || lower === 'skip' ? '\u274c' : '\u2705');
|
|
1091
|
+
gateway.resolveApproval(approvals[approvals.length - 1], approvalReply);
|
|
1092
|
+
await message.react(approvalReply === false ? '\u274c' : '\u2705');
|
|
1086
1093
|
return;
|
|
1087
1094
|
}
|
|
1088
1095
|
}
|
|
@@ -1227,6 +1234,30 @@ export async function startDiscord(gateway, heartbeat, cronScheduler, dispatcher
|
|
|
1227
1234
|
await cmd.reply(formatToolsList());
|
|
1228
1235
|
return;
|
|
1229
1236
|
}
|
|
1237
|
+
if (name === 'toolset') {
|
|
1238
|
+
const mode = normalizeToolsetName(cmd.options.getString('mode', true));
|
|
1239
|
+
if (!mode) {
|
|
1240
|
+
await cmd.reply({ content: 'Unknown toolset.', ephemeral: true });
|
|
1241
|
+
return;
|
|
1242
|
+
}
|
|
1243
|
+
gateway.setSessionToolset(sessionKey, mode);
|
|
1244
|
+
await cmd.reply({ content: `Toolset set to **${mode}**.`, ephemeral: true });
|
|
1245
|
+
updatePresence(sessionKey);
|
|
1246
|
+
return;
|
|
1247
|
+
}
|
|
1248
|
+
if (name === 'compress') {
|
|
1249
|
+
await cmd.reply({ content: gateway.compactSessionForUser(sessionKey), ephemeral: true });
|
|
1250
|
+
updatePresence(sessionKey);
|
|
1251
|
+
return;
|
|
1252
|
+
}
|
|
1253
|
+
if (name === 'usage') {
|
|
1254
|
+
await cmd.reply({ content: gateway.describeSessionUsage(sessionKey), ephemeral: true });
|
|
1255
|
+
return;
|
|
1256
|
+
}
|
|
1257
|
+
if (name === 'debug') {
|
|
1258
|
+
await cmd.reply({ content: gateway.describeSessionDebug(sessionKey).slice(0, 1900), ephemeral: true });
|
|
1259
|
+
return;
|
|
1260
|
+
}
|
|
1230
1261
|
if (name === 'status') {
|
|
1231
1262
|
const jobArg = cmd.options.getString('job') ?? undefined;
|
|
1232
1263
|
await cmd.reply(handleUnleashedStatus(jobArg));
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
import { Bot } from 'grammy';
|
|
8
8
|
import pino from 'pino';
|
|
9
9
|
import { TELEGRAM_BOT_TOKEN, TELEGRAM_OWNER_ID, } from '../config.js';
|
|
10
|
+
import { detectApprovalReply } from '../agent/local-turn.js';
|
|
10
11
|
const logger = pino({ name: 'clementine.telegram' });
|
|
11
12
|
const STREAM_UPDATE_INTERVAL = 1500; // ms
|
|
12
13
|
const TELEGRAM_MSG_LIMIT = 4096;
|
|
@@ -139,14 +140,12 @@ export async function startTelegram(gateway, dispatcher) {
|
|
|
139
140
|
const chatId = ctx.chat.id;
|
|
140
141
|
const sessionKey = `telegram:user:${userId}`;
|
|
141
142
|
// ── Approval responses ──────────────────────────────────────────
|
|
142
|
-
const
|
|
143
|
-
if (
|
|
143
|
+
const approvalReply = detectApprovalReply(text);
|
|
144
|
+
if (approvalReply !== null) {
|
|
144
145
|
const approvals = gateway.getPendingApprovals();
|
|
145
146
|
if (approvals.length > 0) {
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
gateway.resolveApproval(approvals[approvals.length - 1], result);
|
|
149
|
-
const approved = result !== false;
|
|
147
|
+
gateway.resolveApproval(approvals[approvals.length - 1], approvalReply);
|
|
148
|
+
const approved = approvalReply !== false;
|
|
150
149
|
await ctx.reply(approved ? '✅ Approved.' : '❌ Denied.');
|
|
151
150
|
return;
|
|
152
151
|
}
|
package/dist/cli/dashboard.js
CHANGED
|
@@ -6572,6 +6572,22 @@ If the tool returns nothing or errors, return an empty array \`[]\`.`,
|
|
|
6572
6572
|
res.json({ ok: true, action, report });
|
|
6573
6573
|
return;
|
|
6574
6574
|
}
|
|
6575
|
+
if (action === 'install-dense-model') {
|
|
6576
|
+
const embeddings = await import('../memory/embeddings.js');
|
|
6577
|
+
const ready = await embeddings.probeDenseReady();
|
|
6578
|
+
if (!ready) {
|
|
6579
|
+
res.status(503).json({ error: 'Dense embedding model failed to load' });
|
|
6580
|
+
return;
|
|
6581
|
+
}
|
|
6582
|
+
res.json({
|
|
6583
|
+
ok: true,
|
|
6584
|
+
action,
|
|
6585
|
+
model: embeddings.currentDenseModel(),
|
|
6586
|
+
dimension: embeddings.denseDimension(),
|
|
6587
|
+
cacheDir: embeddings.denseModelCacheDir(),
|
|
6588
|
+
});
|
|
6589
|
+
return;
|
|
6590
|
+
}
|
|
6575
6591
|
if (action === 'reembed-dense') {
|
|
6576
6592
|
// Run backfill in the background — first call also pays the model
|
|
6577
6593
|
// load cost (~440MB download on first ever run). We respond immediately
|
|
@@ -22854,7 +22870,7 @@ async function refreshCoverageStrip() {
|
|
|
22854
22870
|
if (!d.ok || !d.health) { el.innerHTML = ''; return; }
|
|
22855
22871
|
var h = d.health;
|
|
22856
22872
|
var total = (h.chunks && h.chunks.total) || 0;
|
|
22857
|
-
var de = h.denseEmbeddings || { withDense: 0, total: total, currentModel: '', ready: false };
|
|
22873
|
+
var de = h.denseEmbeddings || { withDense: 0, total: total, currentModel: '', ready: false, installed: false, cacheSize: '0 B' };
|
|
22858
22874
|
var sparseCovered = (h.chunks && h.chunks.withSparseEmbedding != null) ? h.chunks.withSparseEmbedding : null;
|
|
22859
22875
|
var densePct = de.total > 0 ? Math.round((de.withDense / de.total) * 100) : 0;
|
|
22860
22876
|
var sparsePct = (sparseCovered != null && total > 0) ? Math.round((sparseCovered / total) * 100) : null;
|
|
@@ -22866,7 +22882,26 @@ async function refreshCoverageStrip() {
|
|
|
22866
22882
|
if (sparsePct != null) html += '<span><span style="color:#10b981">●</span> Sparse ' + sparsePct + '%</span>';
|
|
22867
22883
|
html += '<span><span style="color:' + denseColor + '">●</span> Dense ' + densePct + '%'
|
|
22868
22884
|
+ (modelLabel ? ' <span style="color:var(--text-muted)">(' + esc(modelLabel) + ')</span>' : '') + '</span>';
|
|
22869
|
-
if (de.
|
|
22885
|
+
if (!de.installed) {
|
|
22886
|
+
html += '<span style="margin-left:auto;display:flex;align-items:center;gap:8px">'
|
|
22887
|
+
+ '<span style="color:#f59e0b">Model not installed</span>'
|
|
22888
|
+
+ '<button class="btn-sm" onclick="memoryHealthAction(\\'install-dense-model\\')">Install model</button>'
|
|
22889
|
+
+ '</span>';
|
|
22890
|
+
} else if (!de.ready) {
|
|
22891
|
+
html += '<span style="margin-left:auto;display:flex;align-items:center;gap:8px">'
|
|
22892
|
+
+ '<span style="color:#f59e0b">Model not verified</span>'
|
|
22893
|
+
+ '<button class="btn-sm" onclick="memoryHealthAction(\\'install-dense-model\\')">Verify model</button>'
|
|
22894
|
+
+ '</span>';
|
|
22895
|
+
} else if (!de.ready) {
|
|
22896
|
+
html += '<div class="card" style="margin-bottom:16px;border-left:3px solid #f59e0b">';
|
|
22897
|
+
html += '<div class="card-body" style="padding:14px;display:flex;align-items:center;gap:14px;flex-wrap:wrap">';
|
|
22898
|
+
html += '<div style="flex:1;min-width:240px">';
|
|
22899
|
+
html += '<div style="font-weight:600;margin-bottom:4px">Embedding model is cached but has not been verified in this daemon</div>';
|
|
22900
|
+
html += '<div style="font-size:12px;color:var(--text-muted)">Run a quick load check to confirm the local model is usable before relying on dense recall.</div>';
|
|
22901
|
+
html += '</div>';
|
|
22902
|
+
html += '<button class="btn-sm" onclick="memoryHealthAction(\\'install-dense-model\\')" title="Load and verify the cached model">Verify model</button>';
|
|
22903
|
+
html += '</div></div>';
|
|
22904
|
+
} else if (de.total > 0 && de.withDense < de.total) {
|
|
22870
22905
|
var missing = de.total - de.withDense;
|
|
22871
22906
|
html += '<span style="margin-left:auto;display:flex;gap:6px">'
|
|
22872
22907
|
+ '<span style="color:var(--text-muted)">' + missing.toLocaleString() + ' missing</span>'
|
|
@@ -22938,12 +22973,17 @@ async function refreshRecentWrites() {
|
|
|
22938
22973
|
}
|
|
22939
22974
|
|
|
22940
22975
|
async function memoryHealthAction(action, extra) {
|
|
22941
|
-
var labels = { 'janitor': 'cleanup', 'rebuild-fts': 'FTS rebuild', 'fix-orphans': 'orphan fix', 'reembed-dense': 'dense embedding backfill' };
|
|
22976
|
+
var labels = { 'janitor': 'cleanup', 'rebuild-fts': 'FTS rebuild', 'fix-orphans': 'orphan fix', 'install-dense-model': 'local embedding model install/verify', 'reembed-dense': 'dense embedding backfill' };
|
|
22942
22977
|
if (!confirm('Run ' + (labels[action] || action) + ' now?')) return;
|
|
22943
22978
|
try {
|
|
22944
22979
|
var body = Object.assign({ action: action }, extra || {});
|
|
22945
22980
|
var r = await apiJson('POST', '/api/memory/health/action', body);
|
|
22946
22981
|
if (r.error) { toast('Action failed: ' + r.error, 'error'); return; }
|
|
22982
|
+
if (action === 'install-dense-model') {
|
|
22983
|
+
toast('Embedding model verified: ' + (r.model || 'local model'), 'success');
|
|
22984
|
+
refreshMemoryHealth();
|
|
22985
|
+
return;
|
|
22986
|
+
}
|
|
22947
22987
|
if (action === 'reembed-dense' && r.started) {
|
|
22948
22988
|
toast('Backfill started in background (' + (r.limit || '?') + ' chunks). Refreshing every 10s…', 'info');
|
|
22949
22989
|
// Poll coverage updates so the user sees progress without manually refreshing.
|
|
@@ -23185,7 +23225,7 @@ async function refreshMemoryHealth() {
|
|
|
23185
23225
|
|
|
23186
23226
|
// Dense embedding coverage — the leading indicator for retrieval quality.
|
|
23187
23227
|
// <50% means the agent is mostly searching on TF-IDF and missing semantic matches.
|
|
23188
|
-
var de = h.denseEmbeddings || { withDense: 0, total: 0, models: [], currentModel: '', ready: false };
|
|
23228
|
+
var de = h.denseEmbeddings || { withDense: 0, total: 0, models: [], currentModel: '', ready: false, installed: false, cacheSize: '0 B' };
|
|
23189
23229
|
var densePct = de.total > 0 ? ((de.withDense / de.total) * 100).toFixed(1) : '0.0';
|
|
23190
23230
|
var denseColor = de.total === 0 ? 'var(--text-muted)'
|
|
23191
23231
|
: (de.withDense / Math.max(1, de.total)) >= 0.95 ? 'var(--success, #10b981)'
|
|
@@ -23196,12 +23236,22 @@ async function refreshMemoryHealth() {
|
|
|
23196
23236
|
+ '<div class="metric-hero-value" style="color:' + denseColor + '">' + densePct + '%</div>'
|
|
23197
23237
|
+ '<div class="metric-hero-label">Semantic Coverage</div>'
|
|
23198
23238
|
+ '<div class="metric-hero-sub">' + (de.withDense || 0) + ' of ' + (de.total || 0)
|
|
23199
|
-
+ ' chunks · ' + esc(modelLabel) + '</div></div>';
|
|
23239
|
+
+ ' chunks · ' + esc(modelLabel) + ' · model ' + (de.installed ? esc(de.cacheSize || 'cached') : 'not installed') + '</div></div>';
|
|
23200
23240
|
|
|
23201
23241
|
html += '</div>';
|
|
23202
23242
|
|
|
23203
23243
|
// Coverage call-to-action — only render when there's work to do.
|
|
23204
|
-
if (de.
|
|
23244
|
+
if (!de.installed) {
|
|
23245
|
+
html += '<div class="card" style="margin-bottom:16px;border-left:3px solid #f59e0b">';
|
|
23246
|
+
html += '<div class="card-body" style="padding:14px;display:flex;align-items:center;gap:14px;flex-wrap:wrap">';
|
|
23247
|
+
html += '<div style="flex:1;min-width:240px">';
|
|
23248
|
+
html += '<div style="font-weight:600;margin-bottom:4px">Local embedding model is not installed yet</div>';
|
|
23249
|
+
html += '<div style="font-size:12px;color:var(--text-muted)">Install once to enable dense semantic recall without waiting for the first chat or backfill to download it.</div>';
|
|
23250
|
+
if (de.cacheDir) html += '<div style="font-size:11px;color:var(--text-muted);margin-top:4px;font-family:\\x27JetBrains Mono\\x27,monospace">' + esc(de.cacheDir) + '</div>';
|
|
23251
|
+
html += '</div>';
|
|
23252
|
+
html += '<button class="btn-sm" onclick="memoryHealthAction(\\'install-dense-model\\')" title="Download and verify the local dense embedding model">Install model</button>';
|
|
23253
|
+
html += '</div></div>';
|
|
23254
|
+
} else if (de.total > 0 && de.withDense < de.total) {
|
|
23205
23255
|
var missing = de.total - de.withDense;
|
|
23206
23256
|
html += '<div class="card" style="margin-bottom:16px;border-left:3px solid ' + denseColor + '">';
|
|
23207
23257
|
html += '<div class="card-body" style="padding:14px;display:flex;align-items:center;gap:14px;flex-wrap:wrap">';
|
package/dist/cli/index.js
CHANGED
|
@@ -64,6 +64,45 @@ function getLaunchdPlistPath() {
|
|
|
64
64
|
function getSystemdServiceName() {
|
|
65
65
|
return `${getAssistantName().toLowerCase()}.service`;
|
|
66
66
|
}
|
|
67
|
+
function formatBytes(n) {
|
|
68
|
+
if (!Number.isFinite(n) || n < 0)
|
|
69
|
+
return '0 B';
|
|
70
|
+
if (n < 1024)
|
|
71
|
+
return `${n} B`;
|
|
72
|
+
if (n < 1024 * 1024)
|
|
73
|
+
return `${(n / 1024).toFixed(1)} KB`;
|
|
74
|
+
if (n < 1024 * 1024 * 1024)
|
|
75
|
+
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
|
|
76
|
+
return `${(n / (1024 * 1024 * 1024)).toFixed(2)} GB`;
|
|
77
|
+
}
|
|
78
|
+
function dirSizeBytes(dir) {
|
|
79
|
+
if (!existsSync(dir))
|
|
80
|
+
return 0;
|
|
81
|
+
let total = 0;
|
|
82
|
+
try {
|
|
83
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
84
|
+
const full = path.join(dir, entry.name);
|
|
85
|
+
if (entry.isDirectory())
|
|
86
|
+
total += dirSizeBytes(full);
|
|
87
|
+
else if (entry.isFile())
|
|
88
|
+
total += statSync(full).size;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
return total;
|
|
93
|
+
}
|
|
94
|
+
return total;
|
|
95
|
+
}
|
|
96
|
+
async function suppressStdout(fn) {
|
|
97
|
+
const originalWrite = process.stdout.write.bind(process.stdout);
|
|
98
|
+
process.stdout.write = () => true;
|
|
99
|
+
try {
|
|
100
|
+
return await fn();
|
|
101
|
+
}
|
|
102
|
+
finally {
|
|
103
|
+
process.stdout.write = originalWrite;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
67
106
|
function getSystemdServicePath() {
|
|
68
107
|
const home = process.env.HOME ?? '';
|
|
69
108
|
return path.join(home, '.config', 'systemd', 'user', getSystemdServiceName());
|
|
@@ -797,6 +836,24 @@ function cmdDoctor(opts = {}) {
|
|
|
797
836
|
else {
|
|
798
837
|
console.log(` ${DIM} ○ memory database (created on first launch)${RESET}`);
|
|
799
838
|
}
|
|
839
|
+
// Local dense embedding model cache. Doctor does a cheap filesystem check;
|
|
840
|
+
// `--fix` runs the real model probe/installer so users can verify the model
|
|
841
|
+
// without needing to know the hidden Transformers.js cache mechanics.
|
|
842
|
+
const modelCacheDir = path.join(BASE_DIR, 'models');
|
|
843
|
+
const modelCacheBytes = dirSizeBytes(modelCacheDir);
|
|
844
|
+
if (modelCacheBytes >= 1024 * 1024) {
|
|
845
|
+
console.log(` ${GREEN}OK${RESET} local embedding model cache (${formatBytes(modelCacheBytes)})`);
|
|
846
|
+
console.log(` ${DIM}Verify load: clementine memory model status --probe${RESET}`);
|
|
847
|
+
}
|
|
848
|
+
else {
|
|
849
|
+
console.log(` ${YELLOW}WARN${RESET} local embedding model not installed/verified`);
|
|
850
|
+
const installCmd = `"${process.execPath}" "${path.join(PACKAGE_ROOT, 'dist', 'cli', 'index.js')}" memory model install`;
|
|
851
|
+
if (!tryFix('local embedding model', installCmd, { cwd: PACKAGE_ROOT, timeout: 10 * 60_000 })) {
|
|
852
|
+
console.log(` Install: ${CYAN}clementine memory model install${RESET}`);
|
|
853
|
+
console.log(` Auto-prefetch on updates: ${CYAN}clementine config set CLEMENTINE_PREFETCH_EMBEDDINGS 1${RESET}`);
|
|
854
|
+
issues++;
|
|
855
|
+
}
|
|
856
|
+
}
|
|
800
857
|
// Channel tokens (informational)
|
|
801
858
|
if (existsSync(ENV_PATH)) {
|
|
802
859
|
const env = readFileSync(ENV_PATH, 'utf-8');
|
|
@@ -3100,6 +3157,123 @@ memoryCmd
|
|
|
3100
3157
|
process.exit(1);
|
|
3101
3158
|
}
|
|
3102
3159
|
});
|
|
3160
|
+
const memoryModelCmd = memoryCmd
|
|
3161
|
+
.command('model')
|
|
3162
|
+
.description('Inspect or install the local dense embedding model used for semantic recall');
|
|
3163
|
+
memoryModelCmd
|
|
3164
|
+
.command('status')
|
|
3165
|
+
.description('Show whether the local dense embedding model is cached and optionally verify it loads')
|
|
3166
|
+
.option('--probe', 'Load the model to verify the cache is usable; first run may download weights')
|
|
3167
|
+
.option('--json', 'Emit machine-readable JSON')
|
|
3168
|
+
.action(async (opts) => {
|
|
3169
|
+
const BOLD = '\x1b[1m';
|
|
3170
|
+
const DIM = '\x1b[0;90m';
|
|
3171
|
+
const GREEN = '\x1b[0;32m';
|
|
3172
|
+
const YELLOW = '\x1b[0;33m';
|
|
3173
|
+
const RED = '\x1b[0;31m';
|
|
3174
|
+
const RESET = '\x1b[0m';
|
|
3175
|
+
try {
|
|
3176
|
+
if (opts.json)
|
|
3177
|
+
process.env.CLEMENTINE_EMBEDDINGS_LOG_LEVEL = process.env.CLEMENTINE_EMBEDDINGS_LOG_LEVEL || 'silent';
|
|
3178
|
+
const embeddings = await import('../memory/embeddings.js');
|
|
3179
|
+
const cacheDir = embeddings.denseModelCacheDir();
|
|
3180
|
+
const cacheBytes = dirSizeBytes(cacheDir);
|
|
3181
|
+
const probeRan = !!opts.probe;
|
|
3182
|
+
let ready = embeddings.isDenseReady();
|
|
3183
|
+
if (opts.probe) {
|
|
3184
|
+
ready = opts.json
|
|
3185
|
+
? await suppressStdout(() => embeddings.probeDenseReady())
|
|
3186
|
+
: await embeddings.probeDenseReady();
|
|
3187
|
+
}
|
|
3188
|
+
const status = {
|
|
3189
|
+
model: embeddings.currentDenseModel(),
|
|
3190
|
+
dimension: embeddings.denseDimension(),
|
|
3191
|
+
cacheDir,
|
|
3192
|
+
cacheExists: existsSync(cacheDir),
|
|
3193
|
+
cacheBytes,
|
|
3194
|
+
cacheSize: formatBytes(cacheBytes),
|
|
3195
|
+
readyInThisProcess: embeddings.isDenseReady(),
|
|
3196
|
+
verified: probeRan ? ready : false,
|
|
3197
|
+
probeRan,
|
|
3198
|
+
};
|
|
3199
|
+
if (opts.json) {
|
|
3200
|
+
console.log(JSON.stringify(status, null, 2));
|
|
3201
|
+
return;
|
|
3202
|
+
}
|
|
3203
|
+
console.log();
|
|
3204
|
+
console.log(` ${BOLD}Local embedding model${RESET}`);
|
|
3205
|
+
console.log(` Model: ${status.model}`);
|
|
3206
|
+
console.log(` Dimension: ${status.dimension}`);
|
|
3207
|
+
console.log(` Cache: ${status.cacheExists ? `${GREEN}${status.cacheSize}${RESET}` : `${YELLOW}missing${RESET}`} ${DIM}${cacheDir}${RESET}`);
|
|
3208
|
+
if (probeRan) {
|
|
3209
|
+
console.log(` Load check: ${ready ? `${GREEN}verified${RESET}` : `${RED}failed${RESET}`}`);
|
|
3210
|
+
}
|
|
3211
|
+
else {
|
|
3212
|
+
console.log(` Load check: ${DIM}not run (use --probe to verify)${RESET}`);
|
|
3213
|
+
}
|
|
3214
|
+
if (!status.cacheExists || status.cacheBytes < 1024 * 1024) {
|
|
3215
|
+
console.log();
|
|
3216
|
+
console.log(` ${DIM}Install with: clementine memory model install${RESET}`);
|
|
3217
|
+
}
|
|
3218
|
+
console.log();
|
|
3219
|
+
}
|
|
3220
|
+
catch (err) {
|
|
3221
|
+
console.error(` ${RED}Error reading model status${RESET}: ${err}`);
|
|
3222
|
+
process.exit(1);
|
|
3223
|
+
}
|
|
3224
|
+
});
|
|
3225
|
+
memoryModelCmd
|
|
3226
|
+
.command('install')
|
|
3227
|
+
.description('Download/cache and verify the local dense embedding model; optionally backfill memory chunks')
|
|
3228
|
+
.option('--model <id>', 'Override embedding model id (default: Snowflake/snowflake-arctic-embed-m-v1.5)')
|
|
3229
|
+
.option('--backfill', 'After installing, backfill dense embeddings for existing chunks')
|
|
3230
|
+
.option('--limit <n>', 'Backfill at most N chunks when --backfill is used')
|
|
3231
|
+
.action(async (opts) => {
|
|
3232
|
+
const BOLD = '\x1b[1m';
|
|
3233
|
+
const DIM = '\x1b[0;90m';
|
|
3234
|
+
const GREEN = '\x1b[0;32m';
|
|
3235
|
+
const YELLOW = '\x1b[0;33m';
|
|
3236
|
+
const RED = '\x1b[0;31m';
|
|
3237
|
+
const RESET = '\x1b[0m';
|
|
3238
|
+
try {
|
|
3239
|
+
if (opts.model)
|
|
3240
|
+
process.env.EMBEDDING_DENSE_MODEL = opts.model;
|
|
3241
|
+
const embeddings = await import('../memory/embeddings.js');
|
|
3242
|
+
console.log();
|
|
3243
|
+
console.log(` ${BOLD}Installing local embedding model${RESET}`);
|
|
3244
|
+
console.log(` Model: ${embeddings.currentDenseModel()}`);
|
|
3245
|
+
console.log(` Cache: ${embeddings.denseModelCacheDir()}`);
|
|
3246
|
+
console.log(` ${DIM}First run may download model weights; later runs use the local cache.${RESET}`);
|
|
3247
|
+
const ready = await embeddings.probeDenseReady();
|
|
3248
|
+
if (!ready) {
|
|
3249
|
+
console.error(` ${RED}Failed to load dense embedding model.${RESET}`);
|
|
3250
|
+
console.error(` ${DIM}Check network access for the first download, then re-run this command.${RESET}`);
|
|
3251
|
+
process.exit(1);
|
|
3252
|
+
}
|
|
3253
|
+
const cacheBytes = dirSizeBytes(embeddings.denseModelCacheDir());
|
|
3254
|
+
console.log(` ${GREEN}✓${RESET} Model ready (${embeddings.denseDimension()}-dim, ${formatBytes(cacheBytes)} cached).`);
|
|
3255
|
+
if (opts.backfill) {
|
|
3256
|
+
const limit = opts.limit ? parseInt(opts.limit, 10) : undefined;
|
|
3257
|
+
const { MemoryStore } = await import('../memory/store.js');
|
|
3258
|
+
const VAULT_DIR = path.join(BASE_DIR, 'vault');
|
|
3259
|
+
const DB_PATH = path.join(VAULT_DIR, '.memory.db');
|
|
3260
|
+
const store = new MemoryStore(DB_PATH, VAULT_DIR);
|
|
3261
|
+
store.initialize();
|
|
3262
|
+
console.log();
|
|
3263
|
+
console.log(` ${BOLD}Backfilling memory chunks${RESET}${limit ? ` ${DIM}(limit ${limit})${RESET}` : ''}`);
|
|
3264
|
+
const result = await store.backfillDenseEmbeddings({ limit });
|
|
3265
|
+
console.log(` ${GREEN}✓${RESET} Embedded ${result.embedded.toLocaleString()} chunk${result.embedded === 1 ? '' : 's'}.`);
|
|
3266
|
+
if (result.failed > 0) {
|
|
3267
|
+
console.log(` ${YELLOW}!${RESET} Failed ${result.failed.toLocaleString()} chunk${result.failed === 1 ? '' : 's'}.`);
|
|
3268
|
+
}
|
|
3269
|
+
}
|
|
3270
|
+
console.log();
|
|
3271
|
+
}
|
|
3272
|
+
catch (err) {
|
|
3273
|
+
console.error(` ${RED}Error installing model${RESET}: ${err}`);
|
|
3274
|
+
process.exit(1);
|
|
3275
|
+
}
|
|
3276
|
+
});
|
|
3103
3277
|
memoryCmd
|
|
3104
3278
|
.command('reembed')
|
|
3105
3279
|
.description('Backfill dense neural embeddings for all chunks (or all stale chunks if model changed). Default model: Snowflake/snowflake-arctic-embed-m-v1.5 — first run downloads ~440MB to ~/.clementine/models/.')
|
package/dist/cli/ingest.js
CHANGED
|
@@ -62,7 +62,11 @@ export async function cmdIngestSeed(inputPath, opts) {
|
|
|
62
62
|
console.log(` Records in: ${result.recordsIn}`);
|
|
63
63
|
console.log(` Records written: ${result.recordsWritten}`);
|
|
64
64
|
console.log(` Records skipped: ${result.recordsSkipped}`);
|
|
65
|
+
console.log(` Records unchanged: ${result.recordsUnchanged}`);
|
|
65
66
|
console.log(` Records failed: ${result.recordsFailed}`);
|
|
67
|
+
if (result.recallCheckStatus) {
|
|
68
|
+
console.log(` Recall check: ${result.recallCheckStatus}${result.recallCheck ? ` (${result.recallCheck.hits}/${result.recallCheck.checked})` : ''}`);
|
|
69
|
+
}
|
|
66
70
|
if (result.overviewNotePath) {
|
|
67
71
|
console.log(` Overview note: ${result.overviewNotePath}`);
|
|
68
72
|
}
|
|
@@ -94,7 +98,9 @@ export async function cmdIngestRun(slug) {
|
|
|
94
98
|
},
|
|
95
99
|
});
|
|
96
100
|
process.stdout.write('\n');
|
|
97
|
-
console.log(` written=${result.recordsWritten} skipped=${result.recordsSkipped} failed=${result.recordsFailed}`);
|
|
101
|
+
console.log(` written=${result.recordsWritten} unchanged=${result.recordsUnchanged} skipped=${result.recordsSkipped} failed=${result.recordsFailed}`);
|
|
102
|
+
if (result.recallCheckStatus)
|
|
103
|
+
console.log(` recall=${result.recallCheckStatus}`);
|
|
98
104
|
if (result.overviewNotePath) {
|
|
99
105
|
console.log(` overview: ${result.overviewNotePath}`);
|
|
100
106
|
}
|
|
@@ -130,7 +136,7 @@ export async function cmdIngestStatus(slug) {
|
|
|
130
136
|
}
|
|
131
137
|
console.log(`\nRecent runs (${runs.length}):\n`);
|
|
132
138
|
for (const r of runs) {
|
|
133
|
-
console.log(` #${r.id} ${r.startedAt} ${r.status.padEnd(8)} in=${r.recordsIn} written=${r.recordsWritten} skipped=${r.recordsSkipped} failed=${r.recordsFailed}`);
|
|
139
|
+
console.log(` #${r.id} ${r.startedAt} ${r.status.padEnd(8)} in=${r.recordsIn} written=${r.recordsWritten} unchanged=${r.recordsUnchanged} skipped=${r.recordsSkipped} failed=${r.recordsFailed} recall=${r.recallCheckStatus ?? '—'}`);
|
|
134
140
|
if (r.overviewNotePath)
|
|
135
141
|
console.log(` overview: ${r.overviewNotePath}`);
|
|
136
142
|
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface GatewayContextSnapshot {
|
|
2
|
+
sessionKey: string;
|
|
3
|
+
textChars: number;
|
|
4
|
+
exchangeCount: number;
|
|
5
|
+
pendingContextChars?: number;
|
|
6
|
+
recentTranscriptChars?: number;
|
|
7
|
+
}
|
|
8
|
+
export interface GatewayContextHygieneDecision {
|
|
9
|
+
shouldCompact: boolean;
|
|
10
|
+
reason: string;
|
|
11
|
+
estimatedTokens: number;
|
|
12
|
+
}
|
|
13
|
+
export declare const GATEWAY_CONTEXT_COMPACT_EXCHANGES = 30;
|
|
14
|
+
export declare const GATEWAY_CONTEXT_COMPACT_TOKENS = 90000;
|
|
15
|
+
export declare function assessGatewayContextHygiene(snapshot: GatewayContextSnapshot): GatewayContextHygieneDecision;
|
|
16
|
+
export declare function formatGatewayHygieneAnnotation(decision: GatewayContextHygieneDecision): string;
|
|
17
|
+
//# sourceMappingURL=context-hygiene.d.ts.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { estimateTokensApprox } from './turn-ledger.js';
|
|
2
|
+
export const GATEWAY_CONTEXT_COMPACT_EXCHANGES = 30;
|
|
3
|
+
export const GATEWAY_CONTEXT_COMPACT_TOKENS = 90_000;
|
|
4
|
+
export function assessGatewayContextHygiene(snapshot) {
|
|
5
|
+
const totalChars = snapshot.textChars + (snapshot.pendingContextChars ?? 0) + (snapshot.recentTranscriptChars ?? 0);
|
|
6
|
+
const estimatedTokens = estimateTokensApprox('x'.repeat(Math.min(totalChars, 400_000)))
|
|
7
|
+
+ Math.max(0, Math.ceil((totalChars - 400_000) / 4));
|
|
8
|
+
if (snapshot.exchangeCount >= GATEWAY_CONTEXT_COMPACT_EXCHANGES) {
|
|
9
|
+
return {
|
|
10
|
+
shouldCompact: true,
|
|
11
|
+
reason: `exchange_count_${snapshot.exchangeCount}`,
|
|
12
|
+
estimatedTokens,
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
if (estimatedTokens >= GATEWAY_CONTEXT_COMPACT_TOKENS) {
|
|
16
|
+
return {
|
|
17
|
+
shouldCompact: true,
|
|
18
|
+
reason: `estimated_tokens_${estimatedTokens}`,
|
|
19
|
+
estimatedTokens,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
return {
|
|
23
|
+
shouldCompact: false,
|
|
24
|
+
reason: 'within_budget',
|
|
25
|
+
estimatedTokens,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
export function formatGatewayHygieneAnnotation(decision) {
|
|
29
|
+
return `[Context hygiene: compacted older session context before this turn (${decision.reason}, approx ${decision.estimatedTokens} tokens in visible gateway inputs). Continuity was saved to session summaries and lineage; use transcript_search/memory for exact details.]`;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=context-hygiene.js.map
|
|
@@ -8,6 +8,26 @@ import type { HeartbeatWorkItem } from '../types.js';
|
|
|
8
8
|
import type { CronScheduler } from './cron-scheduler.js';
|
|
9
9
|
import type { NotificationDispatcher } from './notifications.js';
|
|
10
10
|
import type { Gateway } from './router.js';
|
|
11
|
+
export declare function buildInsightCheckCronCall(prompt: string): {
|
|
12
|
+
jobName: 'insight-check';
|
|
13
|
+
jobPrompt: string;
|
|
14
|
+
tier: 1;
|
|
15
|
+
maxTurns: 1;
|
|
16
|
+
model: 'haiku';
|
|
17
|
+
opts: {
|
|
18
|
+
disableAllTools: true;
|
|
19
|
+
};
|
|
20
|
+
};
|
|
21
|
+
export declare function buildConsolidationCronCall(prompt: string): {
|
|
22
|
+
jobName: 'consolidation-llm';
|
|
23
|
+
jobPrompt: string;
|
|
24
|
+
tier: 1;
|
|
25
|
+
maxTurns: 1;
|
|
26
|
+
model: 'haiku';
|
|
27
|
+
opts: {
|
|
28
|
+
disableAllTools: true;
|
|
29
|
+
};
|
|
30
|
+
};
|
|
11
31
|
export declare class HeartbeatScheduler {
|
|
12
32
|
private readonly stateFile;
|
|
13
33
|
private gateway;
|
|
@@ -17,6 +17,26 @@ import { recentDecisions, recordDecision, recordDecisionOutcome, wasRecentlyDeci
|
|
|
17
17
|
import { CronRunLog, logToDailyNote, todayISO } from './cron-scheduler.js';
|
|
18
18
|
const logger = pino({ name: 'clementine.heartbeat' });
|
|
19
19
|
const PROACTIVE_DECISION_DEDUPE_MS = 24 * 60 * 60 * 1000;
|
|
20
|
+
export function buildInsightCheckCronCall(prompt) {
|
|
21
|
+
return {
|
|
22
|
+
jobName: 'insight-check',
|
|
23
|
+
jobPrompt: prompt,
|
|
24
|
+
tier: 1,
|
|
25
|
+
maxTurns: 1,
|
|
26
|
+
model: 'haiku',
|
|
27
|
+
opts: { disableAllTools: true },
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export function buildConsolidationCronCall(prompt) {
|
|
31
|
+
return {
|
|
32
|
+
jobName: 'consolidation-llm',
|
|
33
|
+
jobPrompt: prompt,
|
|
34
|
+
tier: 1,
|
|
35
|
+
maxTurns: 1,
|
|
36
|
+
model: 'haiku',
|
|
37
|
+
opts: { disableAllTools: true },
|
|
38
|
+
};
|
|
39
|
+
}
|
|
20
40
|
// ── HeartbeatScheduler ────────────────────────────────────────────────
|
|
21
41
|
export class HeartbeatScheduler {
|
|
22
42
|
stateFile;
|
|
@@ -265,7 +285,8 @@ export class HeartbeatScheduler {
|
|
|
265
285
|
}
|
|
266
286
|
// LLM callback for summarization/principle extraction
|
|
267
287
|
const llmCall = async (prompt) => {
|
|
268
|
-
const
|
|
288
|
+
const cronCall = buildConsolidationCronCall(prompt);
|
|
289
|
+
const result = await this.gateway.handleCronJob(cronCall.jobName, cronCall.jobPrompt, cronCall.tier, cronCall.maxTurns, cronCall.model, undefined, 'standard', undefined, undefined, undefined, undefined, cronCall.opts);
|
|
269
290
|
return result || '';
|
|
270
291
|
};
|
|
271
292
|
const result = await runConsolidation(store, llmCall);
|
|
@@ -905,18 +926,14 @@ export class HeartbeatScheduler {
|
|
|
905
926
|
const prompt = buildInsightPrompt(signals);
|
|
906
927
|
if (!prompt)
|
|
907
928
|
return;
|
|
908
|
-
// Run
|
|
909
|
-
//
|
|
910
|
-
//
|
|
911
|
-
// tool calls (activity_history, outlook_inbox, goal_list, task_list)
|
|
912
|
-
// before composing its rating — at 1 turn it always crashes with
|
|
913
|
-
// "Reached maximum number of turns".
|
|
929
|
+
// Run a no-tool classifier call via gateway. gatherInsightSignals()
|
|
930
|
+
// already assembled the local signal list; attaching MCP schemas here can
|
|
931
|
+
// make the prompt too large before the model ever evaluates urgency.
|
|
914
932
|
const icStartedAt = new Date();
|
|
915
933
|
let response = null;
|
|
916
934
|
try {
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
'haiku');
|
|
935
|
+
const cronCall = buildInsightCheckCronCall(prompt);
|
|
936
|
+
response = await this.gateway.handleCronJob(cronCall.jobName, cronCall.jobPrompt, cronCall.tier, cronCall.maxTurns, cronCall.model, undefined, 'standard', undefined, undefined, undefined, undefined, cronCall.opts);
|
|
920
937
|
this.runLog.append({
|
|
921
938
|
jobName: 'insight-check',
|
|
922
939
|
startedAt: icStartedAt.toISOString(),
|
package/dist/gateway/router.d.ts
CHANGED
|
@@ -11,10 +11,12 @@ import { TeamRouter } from '../agent/team-router.js';
|
|
|
11
11
|
import { TeamBus } from '../agent/team-bus.js';
|
|
12
12
|
import type { NotificationDispatcher } from './notifications.js';
|
|
13
13
|
import { type ProactiveNotificationInput } from './notification-context.js';
|
|
14
|
+
import { type ToolsetName } from '../agent/toolsets.js';
|
|
14
15
|
export type ChatErrorKind = 'rate_limit' | 'one_million_context' | 'context_overflow' | 'auth' | 'billing' | 'transient' | 'unknown';
|
|
15
16
|
export declare function classifyChatError(err: unknown): ChatErrorKind;
|
|
16
17
|
/** Detect auth-like errors in response text that the SDK returned as "successful" results. */
|
|
17
18
|
export declare function looksLikeAuthError(text: string): boolean;
|
|
19
|
+
export declare function isLiveUnleashedStatus(status: Record<string, unknown>, nowMs?: number): boolean;
|
|
18
20
|
export declare class Gateway {
|
|
19
21
|
readonly assistant: PersonalAssistant;
|
|
20
22
|
/** Resolvers for pending approvals. `true` = approved, `false` = denied, `string` = revision feedback. */
|
|
@@ -143,6 +145,8 @@ export declare class Gateway {
|
|
|
143
145
|
}): Promise<string>;
|
|
144
146
|
setSessionVerboseLevel(sessionKey: string, level: VerboseLevel): void;
|
|
145
147
|
getSessionVerboseLevel(sessionKey: string): VerboseLevel | undefined;
|
|
148
|
+
setSessionToolset(sessionKey: string, toolset: ToolsetName): void;
|
|
149
|
+
getSessionToolset(sessionKey: string): ToolsetName;
|
|
146
150
|
setSessionModel(sessionKey: string, modelId: string): void;
|
|
147
151
|
getSessionModel(sessionKey: string): string | undefined;
|
|
148
152
|
setSessionProject(sessionKey: string, project: ProjectMeta): void;
|
|
@@ -238,6 +242,9 @@ export declare class Gateway {
|
|
|
238
242
|
maxExchanges: number;
|
|
239
243
|
memoryCount: number;
|
|
240
244
|
};
|
|
245
|
+
compactSessionForUser(sessionKey: string): string;
|
|
246
|
+
describeSessionUsage(sessionKey: string): string;
|
|
247
|
+
describeSessionDebug(sessionKey: string): string;
|
|
241
248
|
clearSession(sessionKey: string): void;
|
|
242
249
|
/** Get the last auto-matched project for a session. */
|
|
243
250
|
getLastMatchedProject(sessionKey: string): {
|