clementine-agent 1.18.19 → 1.18.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/dist/agent/action-enforcer.d.ts +29 -0
- package/dist/agent/action-enforcer.js +120 -0
- package/dist/agent/assistant.d.ts +14 -0
- package/dist/agent/assistant.js +190 -35
- package/dist/agent/auto-update.js +46 -2
- package/dist/agent/local-turn.d.ts +16 -0
- package/dist/agent/local-turn.js +54 -1
- package/dist/agent/route-classifier.d.ts +1 -0
- package/dist/agent/route-classifier.js +30 -3
- package/dist/agent/toolsets.d.ts +14 -0
- package/dist/agent/toolsets.js +68 -0
- package/dist/brain/ingestion-pipeline.d.ts +7 -0
- package/dist/brain/ingestion-pipeline.js +107 -21
- package/dist/channels/discord.js +38 -7
- package/dist/channels/telegram.js +5 -6
- package/dist/cli/dashboard.js +112 -6
- package/dist/cli/index.js +174 -0
- package/dist/cli/ingest.js +8 -2
- package/dist/gateway/context-hygiene.d.ts +17 -0
- package/dist/gateway/context-hygiene.js +31 -0
- package/dist/gateway/heartbeat-scheduler.d.ts +20 -0
- package/dist/gateway/heartbeat-scheduler.js +27 -10
- package/dist/gateway/router.d.ts +8 -1
- package/dist/gateway/router.js +326 -12
- package/dist/gateway/turn-ledger.d.ts +32 -0
- package/dist/gateway/turn-ledger.js +55 -0
- package/dist/memory/embeddings.d.ts +2 -0
- package/dist/memory/embeddings.js +8 -1
- package/dist/memory/store.d.ts +88 -1
- package/dist/memory/store.js +349 -18
- package/dist/memory/write-queue.d.ts +16 -0
- package/dist/memory/write-queue.js +5 -0
- package/dist/tools/shared.d.ts +89 -0
- package/dist/types.d.ts +11 -0
- package/package.json +1 -1
- package/scripts/postinstall.js +56 -6
package/dist/cli/dashboard.js
CHANGED
|
@@ -39,6 +39,7 @@ const HEARTBEAT_WORK_QUEUE_FILE = path.join(BASE_DIR, 'heartbeat', 'work-queue.j
|
|
|
39
39
|
const MEMORY_DB_PATH = path.join(VAULT_DIR, '.memory.db');
|
|
40
40
|
const PROJECTS_META_FILE = path.join(BASE_DIR, 'projects.json');
|
|
41
41
|
const DASHBOARD_PID_FILE = path.join(BASE_DIR, '.dashboard.pid');
|
|
42
|
+
const INTERACTIVE_FAILURE_LOG = path.join(BASE_DIR, 'self-improve', 'interactive-failures.jsonl');
|
|
42
43
|
/**
|
|
43
44
|
* Kill all existing dashboard processes before starting a new one.
|
|
44
45
|
* Uses both the PID file and a process sweep to catch orphans.
|
|
@@ -5679,6 +5680,43 @@ If the tool returns nothing or errors, return an empty array \`[]\`.`,
|
|
|
5679
5680
|
process.env[key] = normalized;
|
|
5680
5681
|
return { ok: true, value: normalized };
|
|
5681
5682
|
}
|
|
5683
|
+
function readRecentDashboardChatFailures(limit = 5) {
|
|
5684
|
+
try {
|
|
5685
|
+
if (!existsSync(INTERACTIVE_FAILURE_LOG))
|
|
5686
|
+
return [];
|
|
5687
|
+
const lines = readFileSync(INTERACTIVE_FAILURE_LOG, 'utf-8')
|
|
5688
|
+
.trim()
|
|
5689
|
+
.split('\n')
|
|
5690
|
+
.filter(Boolean)
|
|
5691
|
+
.slice(-80)
|
|
5692
|
+
.reverse();
|
|
5693
|
+
const out = [];
|
|
5694
|
+
for (const line of lines) {
|
|
5695
|
+
try {
|
|
5696
|
+
const item = JSON.parse(line);
|
|
5697
|
+
const error = String(item.error ?? '');
|
|
5698
|
+
const stage = String(item.stage ?? '');
|
|
5699
|
+
const haystack = `${stage} ${error}`;
|
|
5700
|
+
if (!/1m|context|budget|credit|api error|rate.?limit/i.test(haystack))
|
|
5701
|
+
continue;
|
|
5702
|
+
out.push({
|
|
5703
|
+
createdAt: String(item.createdAt ?? ''),
|
|
5704
|
+
stage,
|
|
5705
|
+
sessionKey: String(item.sessionKey ?? ''),
|
|
5706
|
+
textPreview: String(item.textPreview ?? '').slice(0, 220),
|
|
5707
|
+
error: error.slice(0, 500),
|
|
5708
|
+
});
|
|
5709
|
+
if (out.length >= limit)
|
|
5710
|
+
break;
|
|
5711
|
+
}
|
|
5712
|
+
catch { /* skip malformed lines */ }
|
|
5713
|
+
}
|
|
5714
|
+
return out;
|
|
5715
|
+
}
|
|
5716
|
+
catch {
|
|
5717
|
+
return [];
|
|
5718
|
+
}
|
|
5719
|
+
}
|
|
5682
5720
|
const ASSISTANT_PREF_OPTIONS = {
|
|
5683
5721
|
proactivity: ['quiet', 'balanced', 'proactive', 'operator'],
|
|
5684
5722
|
responseStyle: ['concise', 'balanced', 'detailed'],
|
|
@@ -5797,6 +5835,7 @@ If the tool returns nothing or errors, return an empty array \`[]\`.`,
|
|
|
5797
5835
|
legacyMode,
|
|
5798
5836
|
},
|
|
5799
5837
|
findings,
|
|
5838
|
+
recentFailures: readRecentDashboardChatFailures(),
|
|
5800
5839
|
counts: doctor.counts,
|
|
5801
5840
|
});
|
|
5802
5841
|
}
|
|
@@ -6533,6 +6572,22 @@ If the tool returns nothing or errors, return an empty array \`[]\`.`,
|
|
|
6533
6572
|
res.json({ ok: true, action, report });
|
|
6534
6573
|
return;
|
|
6535
6574
|
}
|
|
6575
|
+
if (action === 'install-dense-model') {
|
|
6576
|
+
const embeddings = await import('../memory/embeddings.js');
|
|
6577
|
+
const ready = await embeddings.probeDenseReady();
|
|
6578
|
+
if (!ready) {
|
|
6579
|
+
res.status(503).json({ error: 'Dense embedding model failed to load' });
|
|
6580
|
+
return;
|
|
6581
|
+
}
|
|
6582
|
+
res.json({
|
|
6583
|
+
ok: true,
|
|
6584
|
+
action,
|
|
6585
|
+
model: embeddings.currentDenseModel(),
|
|
6586
|
+
dimension: embeddings.denseDimension(),
|
|
6587
|
+
cacheDir: embeddings.denseModelCacheDir(),
|
|
6588
|
+
});
|
|
6589
|
+
return;
|
|
6590
|
+
}
|
|
6536
6591
|
if (action === 'reembed-dense') {
|
|
6537
6592
|
// Run backfill in the background — first call also pays the model
|
|
6538
6593
|
// load cost (~440MB download on first ever run). We respond immediately
|
|
@@ -20083,6 +20138,7 @@ async function refreshBudgetHealth() {
|
|
|
20083
20138
|
var modeClass = mode === 'off' ? 'badge-green' : mode === 'on' ? 'badge-yellow' : 'badge-blue';
|
|
20084
20139
|
var rows = d.budgets || [];
|
|
20085
20140
|
var findings = d.findings || [];
|
|
20141
|
+
var recentFailures = d.recentFailures || [];
|
|
20086
20142
|
var html = '<div class="card">'
|
|
20087
20143
|
+ '<div class="card-header" style="display:flex;align-items:center;justify-content:space-between;gap:12px">'
|
|
20088
20144
|
+ '<div style="display:flex;align-items:center;gap:8px"><span>Spend Guards & Context Health</span><span class="badge ' + modeClass + '" style="font-size:10px">1M ' + esc(mode) + '</span></div>'
|
|
@@ -20131,6 +20187,22 @@ async function refreshBudgetHealth() {
|
|
|
20131
20187
|
+ '<div style="font-size:12px;color:var(--text-secondary);margin-top:4px">Safe Recovery lowers autonomous spend and disables 1M context for accounts seeing credit or entitlement errors.</div>'
|
|
20132
20188
|
+ '<div style="font-size:11px;color:var(--text-muted);margin-top:6px">Restart the daemon after changing budgets or context mode.</div>'
|
|
20133
20189
|
+ '</div></div>';
|
|
20190
|
+
if (recentFailures.length) {
|
|
20191
|
+
html += '<div style="border-top:1px solid var(--border);padding-top:10px;margin-bottom:10px">'
|
|
20192
|
+
+ '<div style="font-weight:600;font-size:13px;margin-bottom:6px">Recent chat failures</div>';
|
|
20193
|
+
for (var rf = 0; rf < recentFailures.length; rf++) {
|
|
20194
|
+
var fail = recentFailures[rf] || {};
|
|
20195
|
+
html += '<div style="padding:8px 0;border-bottom:1px solid rgba(127,127,127,0.12)">'
|
|
20196
|
+
+ '<div style="display:flex;gap:8px;align-items:center;flex-wrap:wrap">'
|
|
20197
|
+
+ '<span class="badge badge-yellow" style="font-size:10px">' + esc(fail.stage || 'failure') + '</span>'
|
|
20198
|
+
+ '<span style="font-size:11px;color:var(--text-muted)">' + esc(fail.createdAt || '') + '</span>'
|
|
20199
|
+
+ '</div>'
|
|
20200
|
+
+ '<div style="font-size:12px;color:var(--text-secondary);margin-top:4px">' + esc(fail.error || '') + '</div>'
|
|
20201
|
+
+ (fail.textPreview ? '<div style="font-size:11px;color:var(--text-muted);margin-top:3px">Prompt: ' + esc(fail.textPreview) + '</div>' : '')
|
|
20202
|
+
+ '</div>';
|
|
20203
|
+
}
|
|
20204
|
+
html += '</div>';
|
|
20205
|
+
}
|
|
20134
20206
|
if (findings.length) {
|
|
20135
20207
|
html += '<div style="border-top:1px solid var(--border);padding-top:10px">'
|
|
20136
20208
|
+ '<div style="font-weight:600;font-size:13px;margin-bottom:6px">Potential causes</div>';
|
|
@@ -22798,7 +22870,7 @@ async function refreshCoverageStrip() {
|
|
|
22798
22870
|
if (!d.ok || !d.health) { el.innerHTML = ''; return; }
|
|
22799
22871
|
var h = d.health;
|
|
22800
22872
|
var total = (h.chunks && h.chunks.total) || 0;
|
|
22801
|
-
var de = h.denseEmbeddings || { withDense: 0, total: total, currentModel: '', ready: false };
|
|
22873
|
+
var de = h.denseEmbeddings || { withDense: 0, total: total, currentModel: '', ready: false, installed: false, cacheSize: '0 B' };
|
|
22802
22874
|
var sparseCovered = (h.chunks && h.chunks.withSparseEmbedding != null) ? h.chunks.withSparseEmbedding : null;
|
|
22803
22875
|
var densePct = de.total > 0 ? Math.round((de.withDense / de.total) * 100) : 0;
|
|
22804
22876
|
var sparsePct = (sparseCovered != null && total > 0) ? Math.round((sparseCovered / total) * 100) : null;
|
|
@@ -22810,7 +22882,26 @@ async function refreshCoverageStrip() {
|
|
|
22810
22882
|
if (sparsePct != null) html += '<span><span style="color:#10b981">●</span> Sparse ' + sparsePct + '%</span>';
|
|
22811
22883
|
html += '<span><span style="color:' + denseColor + '">●</span> Dense ' + densePct + '%'
|
|
22812
22884
|
+ (modelLabel ? ' <span style="color:var(--text-muted)">(' + esc(modelLabel) + ')</span>' : '') + '</span>';
|
|
22813
|
-
if (de.
|
|
22885
|
+
if (!de.installed) {
|
|
22886
|
+
html += '<span style="margin-left:auto;display:flex;align-items:center;gap:8px">'
|
|
22887
|
+
+ '<span style="color:#f59e0b">Model not installed</span>'
|
|
22888
|
+
+ '<button class="btn-sm" onclick="memoryHealthAction(\\'install-dense-model\\')">Install model</button>'
|
|
22889
|
+
+ '</span>';
|
|
22890
|
+
} else if (!de.ready) {
|
|
22891
|
+
html += '<span style="margin-left:auto;display:flex;align-items:center;gap:8px">'
|
|
22892
|
+
+ '<span style="color:#f59e0b">Model not verified</span>'
|
|
22893
|
+
+ '<button class="btn-sm" onclick="memoryHealthAction(\\'install-dense-model\\')">Verify model</button>'
|
|
22894
|
+
+ '</span>';
|
|
22895
|
+
} else if (!de.ready) {
|
|
22896
|
+
html += '<div class="card" style="margin-bottom:16px;border-left:3px solid #f59e0b">';
|
|
22897
|
+
html += '<div class="card-body" style="padding:14px;display:flex;align-items:center;gap:14px;flex-wrap:wrap">';
|
|
22898
|
+
html += '<div style="flex:1;min-width:240px">';
|
|
22899
|
+
html += '<div style="font-weight:600;margin-bottom:4px">Embedding model is cached but has not been verified in this daemon</div>';
|
|
22900
|
+
html += '<div style="font-size:12px;color:var(--text-muted)">Run a quick load check to confirm the local model is usable before relying on dense recall.</div>';
|
|
22901
|
+
html += '</div>';
|
|
22902
|
+
html += '<button class="btn-sm" onclick="memoryHealthAction(\\'install-dense-model\\')" title="Load and verify the cached model">Verify model</button>';
|
|
22903
|
+
html += '</div></div>';
|
|
22904
|
+
} else if (de.total > 0 && de.withDense < de.total) {
|
|
22814
22905
|
var missing = de.total - de.withDense;
|
|
22815
22906
|
html += '<span style="margin-left:auto;display:flex;gap:6px">'
|
|
22816
22907
|
+ '<span style="color:var(--text-muted)">' + missing.toLocaleString() + ' missing</span>'
|
|
@@ -22882,12 +22973,17 @@ async function refreshRecentWrites() {
|
|
|
22882
22973
|
}
|
|
22883
22974
|
|
|
22884
22975
|
async function memoryHealthAction(action, extra) {
|
|
22885
|
-
var labels = { 'janitor': 'cleanup', 'rebuild-fts': 'FTS rebuild', 'fix-orphans': 'orphan fix', 'reembed-dense': 'dense embedding backfill' };
|
|
22976
|
+
var labels = { 'janitor': 'cleanup', 'rebuild-fts': 'FTS rebuild', 'fix-orphans': 'orphan fix', 'install-dense-model': 'local embedding model install/verify', 'reembed-dense': 'dense embedding backfill' };
|
|
22886
22977
|
if (!confirm('Run ' + (labels[action] || action) + ' now?')) return;
|
|
22887
22978
|
try {
|
|
22888
22979
|
var body = Object.assign({ action: action }, extra || {});
|
|
22889
22980
|
var r = await apiJson('POST', '/api/memory/health/action', body);
|
|
22890
22981
|
if (r.error) { toast('Action failed: ' + r.error, 'error'); return; }
|
|
22982
|
+
if (action === 'install-dense-model') {
|
|
22983
|
+
toast('Embedding model verified: ' + (r.model || 'local model'), 'success');
|
|
22984
|
+
refreshMemoryHealth();
|
|
22985
|
+
return;
|
|
22986
|
+
}
|
|
22891
22987
|
if (action === 'reembed-dense' && r.started) {
|
|
22892
22988
|
toast('Backfill started in background (' + (r.limit || '?') + ' chunks). Refreshing every 10s…', 'info');
|
|
22893
22989
|
// Poll coverage updates so the user sees progress without manually refreshing.
|
|
@@ -23129,7 +23225,7 @@ async function refreshMemoryHealth() {
|
|
|
23129
23225
|
|
|
23130
23226
|
// Dense embedding coverage — the leading indicator for retrieval quality.
|
|
23131
23227
|
// <50% means the agent is mostly searching on TF-IDF and missing semantic matches.
|
|
23132
|
-
var de = h.denseEmbeddings || { withDense: 0, total: 0, models: [], currentModel: '', ready: false };
|
|
23228
|
+
var de = h.denseEmbeddings || { withDense: 0, total: 0, models: [], currentModel: '', ready: false, installed: false, cacheSize: '0 B' };
|
|
23133
23229
|
var densePct = de.total > 0 ? ((de.withDense / de.total) * 100).toFixed(1) : '0.0';
|
|
23134
23230
|
var denseColor = de.total === 0 ? 'var(--text-muted)'
|
|
23135
23231
|
: (de.withDense / Math.max(1, de.total)) >= 0.95 ? 'var(--success, #10b981)'
|
|
@@ -23140,12 +23236,22 @@ async function refreshMemoryHealth() {
|
|
|
23140
23236
|
+ '<div class="metric-hero-value" style="color:' + denseColor + '">' + densePct + '%</div>'
|
|
23141
23237
|
+ '<div class="metric-hero-label">Semantic Coverage</div>'
|
|
23142
23238
|
+ '<div class="metric-hero-sub">' + (de.withDense || 0) + ' of ' + (de.total || 0)
|
|
23143
|
-
+ ' chunks · ' + esc(modelLabel) + '</div></div>';
|
|
23239
|
+
+ ' chunks · ' + esc(modelLabel) + ' · model ' + (de.installed ? esc(de.cacheSize || 'cached') : 'not installed') + '</div></div>';
|
|
23144
23240
|
|
|
23145
23241
|
html += '</div>';
|
|
23146
23242
|
|
|
23147
23243
|
// Coverage call-to-action — only render when there's work to do.
|
|
23148
|
-
if (de.
|
|
23244
|
+
if (!de.installed) {
|
|
23245
|
+
html += '<div class="card" style="margin-bottom:16px;border-left:3px solid #f59e0b">';
|
|
23246
|
+
html += '<div class="card-body" style="padding:14px;display:flex;align-items:center;gap:14px;flex-wrap:wrap">';
|
|
23247
|
+
html += '<div style="flex:1;min-width:240px">';
|
|
23248
|
+
html += '<div style="font-weight:600;margin-bottom:4px">Local embedding model is not installed yet</div>';
|
|
23249
|
+
html += '<div style="font-size:12px;color:var(--text-muted)">Install once to enable dense semantic recall without waiting for the first chat or backfill to download it.</div>';
|
|
23250
|
+
if (de.cacheDir) html += '<div style="font-size:11px;color:var(--text-muted);margin-top:4px;font-family:\\x27JetBrains Mono\\x27,monospace">' + esc(de.cacheDir) + '</div>';
|
|
23251
|
+
html += '</div>';
|
|
23252
|
+
html += '<button class="btn-sm" onclick="memoryHealthAction(\\'install-dense-model\\')" title="Download and verify the local dense embedding model">Install model</button>';
|
|
23253
|
+
html += '</div></div>';
|
|
23254
|
+
} else if (de.total > 0 && de.withDense < de.total) {
|
|
23149
23255
|
var missing = de.total - de.withDense;
|
|
23150
23256
|
html += '<div class="card" style="margin-bottom:16px;border-left:3px solid ' + denseColor + '">';
|
|
23151
23257
|
html += '<div class="card-body" style="padding:14px;display:flex;align-items:center;gap:14px;flex-wrap:wrap">';
|
package/dist/cli/index.js
CHANGED
|
@@ -64,6 +64,45 @@ function getLaunchdPlistPath() {
|
|
|
64
64
|
function getSystemdServiceName() {
|
|
65
65
|
return `${getAssistantName().toLowerCase()}.service`;
|
|
66
66
|
}
|
|
67
|
+
function formatBytes(n) {
|
|
68
|
+
if (!Number.isFinite(n) || n < 0)
|
|
69
|
+
return '0 B';
|
|
70
|
+
if (n < 1024)
|
|
71
|
+
return `${n} B`;
|
|
72
|
+
if (n < 1024 * 1024)
|
|
73
|
+
return `${(n / 1024).toFixed(1)} KB`;
|
|
74
|
+
if (n < 1024 * 1024 * 1024)
|
|
75
|
+
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
|
|
76
|
+
return `${(n / (1024 * 1024 * 1024)).toFixed(2)} GB`;
|
|
77
|
+
}
|
|
78
|
+
function dirSizeBytes(dir) {
|
|
79
|
+
if (!existsSync(dir))
|
|
80
|
+
return 0;
|
|
81
|
+
let total = 0;
|
|
82
|
+
try {
|
|
83
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
84
|
+
const full = path.join(dir, entry.name);
|
|
85
|
+
if (entry.isDirectory())
|
|
86
|
+
total += dirSizeBytes(full);
|
|
87
|
+
else if (entry.isFile())
|
|
88
|
+
total += statSync(full).size;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
return total;
|
|
93
|
+
}
|
|
94
|
+
return total;
|
|
95
|
+
}
|
|
96
|
+
async function suppressStdout(fn) {
|
|
97
|
+
const originalWrite = process.stdout.write.bind(process.stdout);
|
|
98
|
+
process.stdout.write = () => true;
|
|
99
|
+
try {
|
|
100
|
+
return await fn();
|
|
101
|
+
}
|
|
102
|
+
finally {
|
|
103
|
+
process.stdout.write = originalWrite;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
67
106
|
function getSystemdServicePath() {
|
|
68
107
|
const home = process.env.HOME ?? '';
|
|
69
108
|
return path.join(home, '.config', 'systemd', 'user', getSystemdServiceName());
|
|
@@ -797,6 +836,24 @@ function cmdDoctor(opts = {}) {
|
|
|
797
836
|
else {
|
|
798
837
|
console.log(` ${DIM} ○ memory database (created on first launch)${RESET}`);
|
|
799
838
|
}
|
|
839
|
+
// Local dense embedding model cache. Doctor does a cheap filesystem check;
|
|
840
|
+
// `--fix` runs the real model probe/installer so users can verify the model
|
|
841
|
+
// without needing to know the hidden Transformers.js cache mechanics.
|
|
842
|
+
const modelCacheDir = path.join(BASE_DIR, 'models');
|
|
843
|
+
const modelCacheBytes = dirSizeBytes(modelCacheDir);
|
|
844
|
+
if (modelCacheBytes >= 1024 * 1024) {
|
|
845
|
+
console.log(` ${GREEN}OK${RESET} local embedding model cache (${formatBytes(modelCacheBytes)})`);
|
|
846
|
+
console.log(` ${DIM}Verify load: clementine memory model status --probe${RESET}`);
|
|
847
|
+
}
|
|
848
|
+
else {
|
|
849
|
+
console.log(` ${YELLOW}WARN${RESET} local embedding model not installed/verified`);
|
|
850
|
+
const installCmd = `"${process.execPath}" "${path.join(PACKAGE_ROOT, 'dist', 'cli', 'index.js')}" memory model install`;
|
|
851
|
+
if (!tryFix('local embedding model', installCmd, { cwd: PACKAGE_ROOT, timeout: 10 * 60_000 })) {
|
|
852
|
+
console.log(` Install: ${CYAN}clementine memory model install${RESET}`);
|
|
853
|
+
console.log(` Auto-prefetch on updates: ${CYAN}clementine config set CLEMENTINE_PREFETCH_EMBEDDINGS 1${RESET}`);
|
|
854
|
+
issues++;
|
|
855
|
+
}
|
|
856
|
+
}
|
|
800
857
|
// Channel tokens (informational)
|
|
801
858
|
if (existsSync(ENV_PATH)) {
|
|
802
859
|
const env = readFileSync(ENV_PATH, 'utf-8');
|
|
@@ -3100,6 +3157,123 @@ memoryCmd
|
|
|
3100
3157
|
process.exit(1);
|
|
3101
3158
|
}
|
|
3102
3159
|
});
|
|
3160
|
+
const memoryModelCmd = memoryCmd
|
|
3161
|
+
.command('model')
|
|
3162
|
+
.description('Inspect or install the local dense embedding model used for semantic recall');
|
|
3163
|
+
memoryModelCmd
|
|
3164
|
+
.command('status')
|
|
3165
|
+
.description('Show whether the local dense embedding model is cached and optionally verify it loads')
|
|
3166
|
+
.option('--probe', 'Load the model to verify the cache is usable; first run may download weights')
|
|
3167
|
+
.option('--json', 'Emit machine-readable JSON')
|
|
3168
|
+
.action(async (opts) => {
|
|
3169
|
+
const BOLD = '\x1b[1m';
|
|
3170
|
+
const DIM = '\x1b[0;90m';
|
|
3171
|
+
const GREEN = '\x1b[0;32m';
|
|
3172
|
+
const YELLOW = '\x1b[0;33m';
|
|
3173
|
+
const RED = '\x1b[0;31m';
|
|
3174
|
+
const RESET = '\x1b[0m';
|
|
3175
|
+
try {
|
|
3176
|
+
if (opts.json)
|
|
3177
|
+
process.env.CLEMENTINE_EMBEDDINGS_LOG_LEVEL = process.env.CLEMENTINE_EMBEDDINGS_LOG_LEVEL || 'silent';
|
|
3178
|
+
const embeddings = await import('../memory/embeddings.js');
|
|
3179
|
+
const cacheDir = embeddings.denseModelCacheDir();
|
|
3180
|
+
const cacheBytes = dirSizeBytes(cacheDir);
|
|
3181
|
+
const probeRan = !!opts.probe;
|
|
3182
|
+
let ready = embeddings.isDenseReady();
|
|
3183
|
+
if (opts.probe) {
|
|
3184
|
+
ready = opts.json
|
|
3185
|
+
? await suppressStdout(() => embeddings.probeDenseReady())
|
|
3186
|
+
: await embeddings.probeDenseReady();
|
|
3187
|
+
}
|
|
3188
|
+
const status = {
|
|
3189
|
+
model: embeddings.currentDenseModel(),
|
|
3190
|
+
dimension: embeddings.denseDimension(),
|
|
3191
|
+
cacheDir,
|
|
3192
|
+
cacheExists: existsSync(cacheDir),
|
|
3193
|
+
cacheBytes,
|
|
3194
|
+
cacheSize: formatBytes(cacheBytes),
|
|
3195
|
+
readyInThisProcess: embeddings.isDenseReady(),
|
|
3196
|
+
verified: probeRan ? ready : false,
|
|
3197
|
+
probeRan,
|
|
3198
|
+
};
|
|
3199
|
+
if (opts.json) {
|
|
3200
|
+
console.log(JSON.stringify(status, null, 2));
|
|
3201
|
+
return;
|
|
3202
|
+
}
|
|
3203
|
+
console.log();
|
|
3204
|
+
console.log(` ${BOLD}Local embedding model${RESET}`);
|
|
3205
|
+
console.log(` Model: ${status.model}`);
|
|
3206
|
+
console.log(` Dimension: ${status.dimension}`);
|
|
3207
|
+
console.log(` Cache: ${status.cacheExists ? `${GREEN}${status.cacheSize}${RESET}` : `${YELLOW}missing${RESET}`} ${DIM}${cacheDir}${RESET}`);
|
|
3208
|
+
if (probeRan) {
|
|
3209
|
+
console.log(` Load check: ${ready ? `${GREEN}verified${RESET}` : `${RED}failed${RESET}`}`);
|
|
3210
|
+
}
|
|
3211
|
+
else {
|
|
3212
|
+
console.log(` Load check: ${DIM}not run (use --probe to verify)${RESET}`);
|
|
3213
|
+
}
|
|
3214
|
+
if (!status.cacheExists || status.cacheBytes < 1024 * 1024) {
|
|
3215
|
+
console.log();
|
|
3216
|
+
console.log(` ${DIM}Install with: clementine memory model install${RESET}`);
|
|
3217
|
+
}
|
|
3218
|
+
console.log();
|
|
3219
|
+
}
|
|
3220
|
+
catch (err) {
|
|
3221
|
+
console.error(` ${RED}Error reading model status${RESET}: ${err}`);
|
|
3222
|
+
process.exit(1);
|
|
3223
|
+
}
|
|
3224
|
+
});
|
|
3225
|
+
memoryModelCmd
|
|
3226
|
+
.command('install')
|
|
3227
|
+
.description('Download/cache and verify the local dense embedding model; optionally backfill memory chunks')
|
|
3228
|
+
.option('--model <id>', 'Override embedding model id (default: Snowflake/snowflake-arctic-embed-m-v1.5)')
|
|
3229
|
+
.option('--backfill', 'After installing, backfill dense embeddings for existing chunks')
|
|
3230
|
+
.option('--limit <n>', 'Backfill at most N chunks when --backfill is used')
|
|
3231
|
+
.action(async (opts) => {
|
|
3232
|
+
const BOLD = '\x1b[1m';
|
|
3233
|
+
const DIM = '\x1b[0;90m';
|
|
3234
|
+
const GREEN = '\x1b[0;32m';
|
|
3235
|
+
const YELLOW = '\x1b[0;33m';
|
|
3236
|
+
const RED = '\x1b[0;31m';
|
|
3237
|
+
const RESET = '\x1b[0m';
|
|
3238
|
+
try {
|
|
3239
|
+
if (opts.model)
|
|
3240
|
+
process.env.EMBEDDING_DENSE_MODEL = opts.model;
|
|
3241
|
+
const embeddings = await import('../memory/embeddings.js');
|
|
3242
|
+
console.log();
|
|
3243
|
+
console.log(` ${BOLD}Installing local embedding model${RESET}`);
|
|
3244
|
+
console.log(` Model: ${embeddings.currentDenseModel()}`);
|
|
3245
|
+
console.log(` Cache: ${embeddings.denseModelCacheDir()}`);
|
|
3246
|
+
console.log(` ${DIM}First run may download model weights; later runs use the local cache.${RESET}`);
|
|
3247
|
+
const ready = await embeddings.probeDenseReady();
|
|
3248
|
+
if (!ready) {
|
|
3249
|
+
console.error(` ${RED}Failed to load dense embedding model.${RESET}`);
|
|
3250
|
+
console.error(` ${DIM}Check network access for the first download, then re-run this command.${RESET}`);
|
|
3251
|
+
process.exit(1);
|
|
3252
|
+
}
|
|
3253
|
+
const cacheBytes = dirSizeBytes(embeddings.denseModelCacheDir());
|
|
3254
|
+
console.log(` ${GREEN}✓${RESET} Model ready (${embeddings.denseDimension()}-dim, ${formatBytes(cacheBytes)} cached).`);
|
|
3255
|
+
if (opts.backfill) {
|
|
3256
|
+
const limit = opts.limit ? parseInt(opts.limit, 10) : undefined;
|
|
3257
|
+
const { MemoryStore } = await import('../memory/store.js');
|
|
3258
|
+
const VAULT_DIR = path.join(BASE_DIR, 'vault');
|
|
3259
|
+
const DB_PATH = path.join(VAULT_DIR, '.memory.db');
|
|
3260
|
+
const store = new MemoryStore(DB_PATH, VAULT_DIR);
|
|
3261
|
+
store.initialize();
|
|
3262
|
+
console.log();
|
|
3263
|
+
console.log(` ${BOLD}Backfilling memory chunks${RESET}${limit ? ` ${DIM}(limit ${limit})${RESET}` : ''}`);
|
|
3264
|
+
const result = await store.backfillDenseEmbeddings({ limit });
|
|
3265
|
+
console.log(` ${GREEN}✓${RESET} Embedded ${result.embedded.toLocaleString()} chunk${result.embedded === 1 ? '' : 's'}.`);
|
|
3266
|
+
if (result.failed > 0) {
|
|
3267
|
+
console.log(` ${YELLOW}!${RESET} Failed ${result.failed.toLocaleString()} chunk${result.failed === 1 ? '' : 's'}.`);
|
|
3268
|
+
}
|
|
3269
|
+
}
|
|
3270
|
+
console.log();
|
|
3271
|
+
}
|
|
3272
|
+
catch (err) {
|
|
3273
|
+
console.error(` ${RED}Error installing model${RESET}: ${err}`);
|
|
3274
|
+
process.exit(1);
|
|
3275
|
+
}
|
|
3276
|
+
});
|
|
3103
3277
|
memoryCmd
|
|
3104
3278
|
.command('reembed')
|
|
3105
3279
|
.description('Backfill dense neural embeddings for all chunks (or all stale chunks if model changed). Default model: Snowflake/snowflake-arctic-embed-m-v1.5 — first run downloads ~440MB to ~/.clementine/models/.')
|
package/dist/cli/ingest.js
CHANGED
|
@@ -62,7 +62,11 @@ export async function cmdIngestSeed(inputPath, opts) {
|
|
|
62
62
|
console.log(` Records in: ${result.recordsIn}`);
|
|
63
63
|
console.log(` Records written: ${result.recordsWritten}`);
|
|
64
64
|
console.log(` Records skipped: ${result.recordsSkipped}`);
|
|
65
|
+
console.log(` Records unchanged: ${result.recordsUnchanged}`);
|
|
65
66
|
console.log(` Records failed: ${result.recordsFailed}`);
|
|
67
|
+
if (result.recallCheckStatus) {
|
|
68
|
+
console.log(` Recall check: ${result.recallCheckStatus}${result.recallCheck ? ` (${result.recallCheck.hits}/${result.recallCheck.checked})` : ''}`);
|
|
69
|
+
}
|
|
66
70
|
if (result.overviewNotePath) {
|
|
67
71
|
console.log(` Overview note: ${result.overviewNotePath}`);
|
|
68
72
|
}
|
|
@@ -94,7 +98,9 @@ export async function cmdIngestRun(slug) {
|
|
|
94
98
|
},
|
|
95
99
|
});
|
|
96
100
|
process.stdout.write('\n');
|
|
97
|
-
console.log(` written=${result.recordsWritten} skipped=${result.recordsSkipped} failed=${result.recordsFailed}`);
|
|
101
|
+
console.log(` written=${result.recordsWritten} unchanged=${result.recordsUnchanged} skipped=${result.recordsSkipped} failed=${result.recordsFailed}`);
|
|
102
|
+
if (result.recallCheckStatus)
|
|
103
|
+
console.log(` recall=${result.recallCheckStatus}`);
|
|
98
104
|
if (result.overviewNotePath) {
|
|
99
105
|
console.log(` overview: ${result.overviewNotePath}`);
|
|
100
106
|
}
|
|
@@ -130,7 +136,7 @@ export async function cmdIngestStatus(slug) {
|
|
|
130
136
|
}
|
|
131
137
|
console.log(`\nRecent runs (${runs.length}):\n`);
|
|
132
138
|
for (const r of runs) {
|
|
133
|
-
console.log(` #${r.id} ${r.startedAt} ${r.status.padEnd(8)} in=${r.recordsIn} written=${r.recordsWritten} skipped=${r.recordsSkipped} failed=${r.recordsFailed}`);
|
|
139
|
+
console.log(` #${r.id} ${r.startedAt} ${r.status.padEnd(8)} in=${r.recordsIn} written=${r.recordsWritten} unchanged=${r.recordsUnchanged} skipped=${r.recordsSkipped} failed=${r.recordsFailed} recall=${r.recallCheckStatus ?? '—'}`);
|
|
134
140
|
if (r.overviewNotePath)
|
|
135
141
|
console.log(` overview: ${r.overviewNotePath}`);
|
|
136
142
|
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface GatewayContextSnapshot {
|
|
2
|
+
sessionKey: string;
|
|
3
|
+
textChars: number;
|
|
4
|
+
exchangeCount: number;
|
|
5
|
+
pendingContextChars?: number;
|
|
6
|
+
recentTranscriptChars?: number;
|
|
7
|
+
}
|
|
8
|
+
export interface GatewayContextHygieneDecision {
|
|
9
|
+
shouldCompact: boolean;
|
|
10
|
+
reason: string;
|
|
11
|
+
estimatedTokens: number;
|
|
12
|
+
}
|
|
13
|
+
export declare const GATEWAY_CONTEXT_COMPACT_EXCHANGES = 30;
|
|
14
|
+
export declare const GATEWAY_CONTEXT_COMPACT_TOKENS = 90000;
|
|
15
|
+
export declare function assessGatewayContextHygiene(snapshot: GatewayContextSnapshot): GatewayContextHygieneDecision;
|
|
16
|
+
export declare function formatGatewayHygieneAnnotation(decision: GatewayContextHygieneDecision): string;
|
|
17
|
+
//# sourceMappingURL=context-hygiene.d.ts.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { estimateTokensApprox } from './turn-ledger.js';
|
|
2
|
+
export const GATEWAY_CONTEXT_COMPACT_EXCHANGES = 30;
|
|
3
|
+
export const GATEWAY_CONTEXT_COMPACT_TOKENS = 90_000;
|
|
4
|
+
export function assessGatewayContextHygiene(snapshot) {
|
|
5
|
+
const totalChars = snapshot.textChars + (snapshot.pendingContextChars ?? 0) + (snapshot.recentTranscriptChars ?? 0);
|
|
6
|
+
const estimatedTokens = estimateTokensApprox('x'.repeat(Math.min(totalChars, 400_000)))
|
|
7
|
+
+ Math.max(0, Math.ceil((totalChars - 400_000) / 4));
|
|
8
|
+
if (snapshot.exchangeCount >= GATEWAY_CONTEXT_COMPACT_EXCHANGES) {
|
|
9
|
+
return {
|
|
10
|
+
shouldCompact: true,
|
|
11
|
+
reason: `exchange_count_${snapshot.exchangeCount}`,
|
|
12
|
+
estimatedTokens,
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
if (estimatedTokens >= GATEWAY_CONTEXT_COMPACT_TOKENS) {
|
|
16
|
+
return {
|
|
17
|
+
shouldCompact: true,
|
|
18
|
+
reason: `estimated_tokens_${estimatedTokens}`,
|
|
19
|
+
estimatedTokens,
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
return {
|
|
23
|
+
shouldCompact: false,
|
|
24
|
+
reason: 'within_budget',
|
|
25
|
+
estimatedTokens,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
export function formatGatewayHygieneAnnotation(decision) {
|
|
29
|
+
return `[Context hygiene: compacted older session context before this turn (${decision.reason}, approx ${decision.estimatedTokens} tokens in visible gateway inputs). Continuity was saved to session summaries and lineage; use transcript_search/memory for exact details.]`;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=context-hygiene.js.map
|
|
@@ -8,6 +8,26 @@ import type { HeartbeatWorkItem } from '../types.js';
|
|
|
8
8
|
import type { CronScheduler } from './cron-scheduler.js';
|
|
9
9
|
import type { NotificationDispatcher } from './notifications.js';
|
|
10
10
|
import type { Gateway } from './router.js';
|
|
11
|
+
export declare function buildInsightCheckCronCall(prompt: string): {
|
|
12
|
+
jobName: 'insight-check';
|
|
13
|
+
jobPrompt: string;
|
|
14
|
+
tier: 1;
|
|
15
|
+
maxTurns: 1;
|
|
16
|
+
model: 'haiku';
|
|
17
|
+
opts: {
|
|
18
|
+
disableAllTools: true;
|
|
19
|
+
};
|
|
20
|
+
};
|
|
21
|
+
export declare function buildConsolidationCronCall(prompt: string): {
|
|
22
|
+
jobName: 'consolidation-llm';
|
|
23
|
+
jobPrompt: string;
|
|
24
|
+
tier: 1;
|
|
25
|
+
maxTurns: 1;
|
|
26
|
+
model: 'haiku';
|
|
27
|
+
opts: {
|
|
28
|
+
disableAllTools: true;
|
|
29
|
+
};
|
|
30
|
+
};
|
|
11
31
|
export declare class HeartbeatScheduler {
|
|
12
32
|
private readonly stateFile;
|
|
13
33
|
private gateway;
|
|
@@ -17,6 +17,26 @@ import { recentDecisions, recordDecision, recordDecisionOutcome, wasRecentlyDeci
|
|
|
17
17
|
import { CronRunLog, logToDailyNote, todayISO } from './cron-scheduler.js';
|
|
18
18
|
const logger = pino({ name: 'clementine.heartbeat' });
|
|
19
19
|
const PROACTIVE_DECISION_DEDUPE_MS = 24 * 60 * 60 * 1000;
|
|
20
|
+
export function buildInsightCheckCronCall(prompt) {
|
|
21
|
+
return {
|
|
22
|
+
jobName: 'insight-check',
|
|
23
|
+
jobPrompt: prompt,
|
|
24
|
+
tier: 1,
|
|
25
|
+
maxTurns: 1,
|
|
26
|
+
model: 'haiku',
|
|
27
|
+
opts: { disableAllTools: true },
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export function buildConsolidationCronCall(prompt) {
|
|
31
|
+
return {
|
|
32
|
+
jobName: 'consolidation-llm',
|
|
33
|
+
jobPrompt: prompt,
|
|
34
|
+
tier: 1,
|
|
35
|
+
maxTurns: 1,
|
|
36
|
+
model: 'haiku',
|
|
37
|
+
opts: { disableAllTools: true },
|
|
38
|
+
};
|
|
39
|
+
}
|
|
20
40
|
// ── HeartbeatScheduler ────────────────────────────────────────────────
|
|
21
41
|
export class HeartbeatScheduler {
|
|
22
42
|
stateFile;
|
|
@@ -265,7 +285,8 @@ export class HeartbeatScheduler {
|
|
|
265
285
|
}
|
|
266
286
|
// LLM callback for summarization/principle extraction
|
|
267
287
|
const llmCall = async (prompt) => {
|
|
268
|
-
const
|
|
288
|
+
const cronCall = buildConsolidationCronCall(prompt);
|
|
289
|
+
const result = await this.gateway.handleCronJob(cronCall.jobName, cronCall.jobPrompt, cronCall.tier, cronCall.maxTurns, cronCall.model, undefined, 'standard', undefined, undefined, undefined, undefined, cronCall.opts);
|
|
269
290
|
return result || '';
|
|
270
291
|
};
|
|
271
292
|
const result = await runConsolidation(store, llmCall);
|
|
@@ -905,18 +926,14 @@ export class HeartbeatScheduler {
|
|
|
905
926
|
const prompt = buildInsightPrompt(signals);
|
|
906
927
|
if (!prompt)
|
|
907
928
|
return;
|
|
908
|
-
// Run
|
|
909
|
-
//
|
|
910
|
-
//
|
|
911
|
-
// tool calls (activity_history, outlook_inbox, goal_list, task_list)
|
|
912
|
-
// before composing its rating — at 1 turn it always crashes with
|
|
913
|
-
// "Reached maximum number of turns".
|
|
929
|
+
// Run a no-tool classifier call via gateway. gatherInsightSignals()
|
|
930
|
+
// already assembled the local signal list; attaching MCP schemas here can
|
|
931
|
+
// make the prompt too large before the model ever evaluates urgency.
|
|
914
932
|
const icStartedAt = new Date();
|
|
915
933
|
let response = null;
|
|
916
934
|
try {
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
'haiku');
|
|
935
|
+
const cronCall = buildInsightCheckCronCall(prompt);
|
|
936
|
+
response = await this.gateway.handleCronJob(cronCall.jobName, cronCall.jobPrompt, cronCall.tier, cronCall.maxTurns, cronCall.model, undefined, 'standard', undefined, undefined, undefined, undefined, cronCall.opts);
|
|
920
937
|
this.runLog.append({
|
|
921
938
|
jobName: 'insight-check',
|
|
922
939
|
startedAt: icStartedAt.toISOString(),
|
package/dist/gateway/router.d.ts
CHANGED
|
@@ -11,10 +11,12 @@ import { TeamRouter } from '../agent/team-router.js';
|
|
|
11
11
|
import { TeamBus } from '../agent/team-bus.js';
|
|
12
12
|
import type { NotificationDispatcher } from './notifications.js';
|
|
13
13
|
import { type ProactiveNotificationInput } from './notification-context.js';
|
|
14
|
-
|
|
14
|
+
import { type ToolsetName } from '../agent/toolsets.js';
|
|
15
|
+
export type ChatErrorKind = 'rate_limit' | 'one_million_context' | 'context_overflow' | 'auth' | 'billing' | 'transient' | 'unknown';
|
|
15
16
|
export declare function classifyChatError(err: unknown): ChatErrorKind;
|
|
16
17
|
/** Detect auth-like errors in response text that the SDK returned as "successful" results. */
|
|
17
18
|
export declare function looksLikeAuthError(text: string): boolean;
|
|
19
|
+
export declare function isLiveUnleashedStatus(status: Record<string, unknown>, nowMs?: number): boolean;
|
|
18
20
|
export declare class Gateway {
|
|
19
21
|
readonly assistant: PersonalAssistant;
|
|
20
22
|
/** Resolvers for pending approvals. `true` = approved, `false` = denied, `string` = revision feedback. */
|
|
@@ -143,6 +145,8 @@ export declare class Gateway {
|
|
|
143
145
|
}): Promise<string>;
|
|
144
146
|
setSessionVerboseLevel(sessionKey: string, level: VerboseLevel): void;
|
|
145
147
|
getSessionVerboseLevel(sessionKey: string): VerboseLevel | undefined;
|
|
148
|
+
setSessionToolset(sessionKey: string, toolset: ToolsetName): void;
|
|
149
|
+
getSessionToolset(sessionKey: string): ToolsetName;
|
|
146
150
|
setSessionModel(sessionKey: string, modelId: string): void;
|
|
147
151
|
getSessionModel(sessionKey: string): string | undefined;
|
|
148
152
|
setSessionProject(sessionKey: string, project: ProjectMeta): void;
|
|
@@ -238,6 +242,9 @@ export declare class Gateway {
|
|
|
238
242
|
maxExchanges: number;
|
|
239
243
|
memoryCount: number;
|
|
240
244
|
};
|
|
245
|
+
compactSessionForUser(sessionKey: string): string;
|
|
246
|
+
describeSessionUsage(sessionKey: string): string;
|
|
247
|
+
describeSessionDebug(sessionKey: string): string;
|
|
241
248
|
clearSession(sessionKey: string): void;
|
|
242
249
|
/** Get the last auto-matched project for a session. */
|
|
243
250
|
getLastMatchedProject(sessionKey: string): {
|