clementine-agent 1.1.19 → 1.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/self-improve.js +22 -6
- package/dist/cli/index.js +384 -1
- package/dist/memory/store.d.ts +88 -0
- package/dist/memory/store.js +224 -1
- package/dist/tools/memory-tools.js +92 -2
- package/dist/tools/tool-meta.js +13 -3
- package/dist/types.d.ts +9 -1
- package/package.json +1 -1
|
@@ -23,6 +23,7 @@ const DEFAULT_CONFIG = {
|
|
|
23
23
|
iterationBudgetMs: 300_000, // 5 min
|
|
24
24
|
maxDurationMs: 3_600_000, // 1 hour
|
|
25
25
|
acceptThreshold: 0.7,
|
|
26
|
+
surfaceThreshold: 0.85,
|
|
26
27
|
plateauLimit: 3,
|
|
27
28
|
// 'source' deprecated — self-improvement produces data, not engine TS edits.
|
|
28
29
|
// 'advisor-rule' writes YAML to ~/.clementine/advisor-rules/user/.
|
|
@@ -300,12 +301,25 @@ export class SelfImproveLoop {
|
|
|
300
301
|
const score = evaluation?.score ?? 0;
|
|
301
302
|
const normalizedScore = score / 10; // Convert 0-10 to 0-1
|
|
302
303
|
const accepted = normalizedScore >= this.config.acceptThreshold;
|
|
304
|
+
// Surface gate: even when accepted, only score >= surfaceThreshold
|
|
305
|
+
// reaches the user's pending-changes inbox. Below that floor we
|
|
306
|
+
// keep the experiment in the trend log but don't ping the user.
|
|
307
|
+
const surfaceFloor = this.config.surfaceThreshold ?? this.config.acceptThreshold;
|
|
308
|
+
const surfaced = normalizedScore >= surfaceFloor;
|
|
303
309
|
const priorScores = history
|
|
304
310
|
.filter(e => e.area === proposal.area && e.target === proposal.target && e.score > 0)
|
|
305
311
|
.map(e => e.score);
|
|
306
312
|
const baselineScore = priorScores.length > 0
|
|
307
313
|
? priorScores.reduce((a, b) => a + b, 0) / priorScores.length
|
|
308
314
|
: 0.5;
|
|
315
|
+
const initialStatus = accepted
|
|
316
|
+
? (surfaced ? 'pending' : 'unsurfaced')
|
|
317
|
+
: 'denied';
|
|
318
|
+
const reason = accepted
|
|
319
|
+
? (surfaced
|
|
320
|
+
? `Score ${score}/10 exceeds surface threshold — pending approval`
|
|
321
|
+
: `Score ${score}/10 accepted but below surface floor (${surfaceFloor * 10}/10) — kept in trend log only`)
|
|
322
|
+
: `Score ${score}/10 below accept threshold (${this.config.acceptThreshold * 10}/10)`;
|
|
309
323
|
const experiment = {
|
|
310
324
|
id,
|
|
311
325
|
iteration: i,
|
|
@@ -319,17 +333,19 @@ export class SelfImproveLoop {
|
|
|
319
333
|
baselineScore,
|
|
320
334
|
score: normalizedScore,
|
|
321
335
|
accepted,
|
|
322
|
-
approvalStatus:
|
|
323
|
-
reason
|
|
324
|
-
? `Score ${score}/10 exceeds threshold — pending approval`
|
|
325
|
-
: `Score ${score}/10 below threshold (${this.config.acceptThreshold * 10}/10)`,
|
|
336
|
+
approvalStatus: initialStatus,
|
|
337
|
+
reason,
|
|
326
338
|
};
|
|
327
339
|
// Step 7: Log
|
|
328
340
|
this.appendExperimentLog(experiment);
|
|
329
341
|
history.push(experiment);
|
|
330
342
|
state.totalExperiments++;
|
|
331
|
-
|
|
332
|
-
|
|
343
|
+
if (accepted && !surfaced) {
|
|
344
|
+
logger.info({ id, area: proposal.area, target: proposal.target, score: score, surfaceFloor: surfaceFloor * 10 }, 'Proposal accepted but unsurfaced — below noise floor, not added to review queue');
|
|
345
|
+
}
|
|
346
|
+
// Step 6: Gate — save pending change + notify (tiered by risk).
|
|
347
|
+
// Only proposals that ALSO clear the surface floor reach the inbox.
|
|
348
|
+
if (accepted && surfaced) {
|
|
333
349
|
const risk = classifyRisk(proposal.area);
|
|
334
350
|
if (this.config.autoApply && risk === 'low') {
|
|
335
351
|
// Low-risk + auto-apply enabled: apply immediately without approval
|
package/dist/cli/index.js
CHANGED
|
@@ -2090,10 +2090,390 @@ configCmd
|
|
|
2090
2090
|
console.error(` Failed to open editor: ${editor}`);
|
|
2091
2091
|
}
|
|
2092
2092
|
});
|
|
2093
|
+
// ── Agent commands ──────────────────────────────────────────────────
|
|
2094
|
+
const agentCmd = program
|
|
2095
|
+
.command('agent')
|
|
2096
|
+
.description('Hire, list, and manage specialist agents');
|
|
2097
|
+
agentCmd
|
|
2098
|
+
.command('list')
|
|
2099
|
+
.description('List all agents with status and tier')
|
|
2100
|
+
.option('--json', 'Emit machine-readable JSON')
|
|
2101
|
+
.action(async (opts) => {
|
|
2102
|
+
const BOLD = '\x1b[1m';
|
|
2103
|
+
const DIM = '\x1b[0;90m';
|
|
2104
|
+
const GREEN = '\x1b[0;32m';
|
|
2105
|
+
const YELLOW = '\x1b[0;33m';
|
|
2106
|
+
const RED = '\x1b[0;31m';
|
|
2107
|
+
const RESET = '\x1b[0m';
|
|
2108
|
+
try {
|
|
2109
|
+
const { AgentManager } = await import('../agent/agent-manager.js');
|
|
2110
|
+
const AGENTS_DIR = path.join(BASE_DIR, 'agents');
|
|
2111
|
+
const mgr = new AgentManager(AGENTS_DIR);
|
|
2112
|
+
const agents = mgr.listAll();
|
|
2113
|
+
if (opts.json) {
|
|
2114
|
+
console.log(JSON.stringify(agents.map(a => ({
|
|
2115
|
+
slug: a.slug, name: a.name, status: a.status, tier: a.tier,
|
|
2116
|
+
description: a.description, hasChannel: !!a.team?.channelName,
|
|
2117
|
+
})), null, 2));
|
|
2118
|
+
return;
|
|
2119
|
+
}
|
|
2120
|
+
if (agents.length === 0) {
|
|
2121
|
+
console.log(`\n No agents found in ${AGENTS_DIR}.`);
|
|
2122
|
+
console.log(` Hire one: ${BOLD}clementine agent new <slug>${RESET}\n`);
|
|
2123
|
+
return;
|
|
2124
|
+
}
|
|
2125
|
+
console.log();
|
|
2126
|
+
console.log(` ${BOLD}${'SLUG'.padEnd(28)}${'NAME'.padEnd(24)}${'STATUS'.padEnd(12)}${'TIER'.padEnd(6)}${RESET}`);
|
|
2127
|
+
console.log(` ${DIM}${'─'.repeat(70)}${RESET}`);
|
|
2128
|
+
for (const a of agents) {
|
|
2129
|
+
const statusColor = a.status === 'active' ? GREEN : a.status === 'paused' ? YELLOW : RED;
|
|
2130
|
+
const statusStr = `${statusColor}${(a.status ?? 'active').padEnd(10)}${RESET}`;
|
|
2131
|
+
console.log(` ${a.slug.padEnd(28)}${(a.name ?? '').slice(0, 22).padEnd(24)}${statusStr} ${String(a.tier).padEnd(6)}`);
|
|
2132
|
+
}
|
|
2133
|
+
console.log();
|
|
2134
|
+
}
|
|
2135
|
+
catch (err) {
|
|
2136
|
+
console.error(` Error listing agents: ${err}`);
|
|
2137
|
+
process.exit(1);
|
|
2138
|
+
}
|
|
2139
|
+
});
|
|
2140
|
+
agentCmd
|
|
2141
|
+
.command('new <slug>')
|
|
2142
|
+
.description('Scaffold a new agent at agents/<slug>/agent.md (does not overwrite)')
|
|
2143
|
+
.option('-n, --name <name>', 'Display name (default: title-case of slug)')
|
|
2144
|
+
.option('-d, --description <text>', 'One-line description of what the agent does')
|
|
2145
|
+
.option('-r, --role <role>', 'Role template — auto-scaffolds CRON.md / PLAYBOOK.md (sdr, researcher)')
|
|
2146
|
+
.option('-t, --tier <tier>', 'Security tier — 1 = vault-only, 2 = bash/git allowed', '1')
|
|
2147
|
+
.option('-m, --model <model>', 'Default model (sonnet, haiku, opus). Inherits global default if omitted.')
|
|
2148
|
+
.option('--channel <name>', 'Discord/Slack channel name the agent listens in (enables team mode)')
|
|
2149
|
+
.action(async (slug, opts) => {
|
|
2150
|
+
const BOLD = '\x1b[1m';
|
|
2151
|
+
const DIM = '\x1b[0;90m';
|
|
2152
|
+
const GREEN = '\x1b[0;32m';
|
|
2153
|
+
const RED = '\x1b[0;31m';
|
|
2154
|
+
const RESET = '\x1b[0m';
|
|
2155
|
+
// Validate slug — lowercase, dashes, alphanumeric, no leading/trailing dash.
|
|
2156
|
+
if (!/^[a-z0-9][a-z0-9-]*[a-z0-9]$/.test(slug) || slug.length < 3) {
|
|
2157
|
+
console.error(` ${RED}Invalid slug${RESET}: must be 3+ chars, lowercase letters/digits/dashes, no leading/trailing dash. Got: "${slug}"`);
|
|
2158
|
+
process.exit(1);
|
|
2159
|
+
}
|
|
2160
|
+
if (slug === 'clementine') {
|
|
2161
|
+
console.error(` ${RED}Reserved slug${RESET}: "clementine" is the master assistant. Pick a different name.`);
|
|
2162
|
+
process.exit(1);
|
|
2163
|
+
}
|
|
2164
|
+
const tier = Math.max(1, Math.min(2, parseInt(opts.tier ?? '1', 10) || 1));
|
|
2165
|
+
const name = opts.name ?? slug.split('-').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ');
|
|
2166
|
+
const description = opts.description ?? `${name} — specialist agent.`;
|
|
2167
|
+
try {
|
|
2168
|
+
const { AgentManager } = await import('../agent/agent-manager.js');
|
|
2169
|
+
const AGENTS_DIR = path.join(BASE_DIR, 'agents');
|
|
2170
|
+
const mgr = new AgentManager(AGENTS_DIR);
|
|
2171
|
+
// Refuse if already taken (createAgent throws but we want a friendly error)
|
|
2172
|
+
if (mgr.get(slug)) {
|
|
2173
|
+
console.error(` ${RED}Agent "${slug}" already exists${RESET}. Edit ${path.join(AGENTS_DIR, slug, 'agent.md')} or pick a different slug.`);
|
|
2174
|
+
process.exit(1);
|
|
2175
|
+
}
|
|
2176
|
+
const config = {
|
|
2177
|
+
name,
|
|
2178
|
+
description,
|
|
2179
|
+
tier,
|
|
2180
|
+
};
|
|
2181
|
+
if (opts.model)
|
|
2182
|
+
config.model = opts.model;
|
|
2183
|
+
if (opts.channel)
|
|
2184
|
+
config.channelName = opts.channel;
|
|
2185
|
+
if (opts.role)
|
|
2186
|
+
config.role = opts.role;
|
|
2187
|
+
const profile = mgr.createAgent(config);
|
|
2188
|
+
const agentMdPath = path.join(profile.agentDir ?? path.join(AGENTS_DIR, slug), 'agent.md');
|
|
2189
|
+
console.log();
|
|
2190
|
+
console.log(` ${GREEN}✓${RESET} Created agent ${BOLD}${profile.name}${RESET} (${profile.slug})`);
|
|
2191
|
+
console.log(` ${DIM}File: ${agentMdPath}${RESET}`);
|
|
2192
|
+
if (opts.role) {
|
|
2193
|
+
console.log(` ${DIM}Role scaffold "${opts.role}" wrote CRON.md / PLAYBOOK.md / SEQUENCES.md if applicable.${RESET}`);
|
|
2194
|
+
}
|
|
2195
|
+
console.log();
|
|
2196
|
+
console.log(` Next steps:`);
|
|
2197
|
+
console.log(` 1. Edit ${BOLD}agent.md${RESET} to refine personality / standing instructions`);
|
|
2198
|
+
if (!opts.channel) {
|
|
2199
|
+
console.log(` 2. Add a ${BOLD}channelName${RESET} (and DISCORD/SLACK token via 'clementine config set') to give them their own bot`);
|
|
2200
|
+
}
|
|
2201
|
+
console.log(` 3. Edit ${BOLD}~/.clementine/agents/${slug}/CRON.md${RESET} to schedule autonomous work`);
|
|
2202
|
+
console.log(` 4. Restart the daemon: ${BOLD}clementine restart${RESET}`);
|
|
2203
|
+
console.log();
|
|
2204
|
+
}
|
|
2205
|
+
catch (err) {
|
|
2206
|
+
console.error(` ${RED}Failed to create agent${RESET}: ${err instanceof Error ? err.message : String(err)}`);
|
|
2207
|
+
process.exit(1);
|
|
2208
|
+
}
|
|
2209
|
+
});
|
|
2210
|
+
agentCmd
|
|
2211
|
+
.command('show <slug>')
|
|
2212
|
+
.description('Show an agent\'s file path and parsed profile summary')
|
|
2213
|
+
.action(async (slug) => {
|
|
2214
|
+
const BOLD = '\x1b[1m';
|
|
2215
|
+
const DIM = '\x1b[0;90m';
|
|
2216
|
+
const RED = '\x1b[0;31m';
|
|
2217
|
+
const RESET = '\x1b[0m';
|
|
2218
|
+
try {
|
|
2219
|
+
const { AgentManager } = await import('../agent/agent-manager.js');
|
|
2220
|
+
const AGENTS_DIR = path.join(BASE_DIR, 'agents');
|
|
2221
|
+
const mgr = new AgentManager(AGENTS_DIR);
|
|
2222
|
+
const profile = mgr.get(slug);
|
|
2223
|
+
if (!profile) {
|
|
2224
|
+
console.error(` ${RED}Agent "${slug}" not found${RESET}.`);
|
|
2225
|
+
console.error(` List agents: ${BOLD}clementine agent list${RESET}`);
|
|
2226
|
+
process.exit(1);
|
|
2227
|
+
}
|
|
2228
|
+
const agentMdPath = path.join(profile.agentDir ?? path.join(AGENTS_DIR, slug), 'agent.md');
|
|
2229
|
+
console.log();
|
|
2230
|
+
console.log(` ${BOLD}${profile.name}${RESET} ${DIM}(${profile.slug})${RESET}`);
|
|
2231
|
+
console.log(` ${DIM}${agentMdPath}${RESET}`);
|
|
2232
|
+
console.log();
|
|
2233
|
+
console.log(` Status: ${profile.status ?? 'active'}`);
|
|
2234
|
+
console.log(` Tier: ${profile.tier}`);
|
|
2235
|
+
if (profile.model)
|
|
2236
|
+
console.log(` Model: ${profile.model}`);
|
|
2237
|
+
console.log(` Description: ${profile.description || DIM + '(none)' + RESET}`);
|
|
2238
|
+
if (profile.team?.channelName) {
|
|
2239
|
+
const ch = Array.isArray(profile.team.channelName) ? profile.team.channelName.join(', ') : profile.team.channelName;
|
|
2240
|
+
console.log(` Channel: ${ch}`);
|
|
2241
|
+
}
|
|
2242
|
+
if (profile.activeHours) {
|
|
2243
|
+
console.log(` Active hours: ${profile.activeHours.start.toFixed(2)}–${profile.activeHours.end.toFixed(2)} (decimal hours)`);
|
|
2244
|
+
}
|
|
2245
|
+
if (profile.budgetMonthlyCents != null && profile.budgetMonthlyCents > 0) {
|
|
2246
|
+
console.log(` Budget/mo: $${(profile.budgetMonthlyCents / 100).toFixed(2)}`);
|
|
2247
|
+
}
|
|
2248
|
+
console.log(` Strict mem: ${profile.strictMemoryIsolation === false ? 'no (soft boost)' : 'yes (default)'}`);
|
|
2249
|
+
console.log();
|
|
2250
|
+
}
|
|
2251
|
+
catch (err) {
|
|
2252
|
+
console.error(` Error reading agent: ${err}`);
|
|
2253
|
+
process.exit(1);
|
|
2254
|
+
}
|
|
2255
|
+
});
|
|
2093
2256
|
// ── Memory commands ─────────────────────────────────────────────────
|
|
2094
2257
|
const memoryCmd = program
|
|
2095
2258
|
.command('memory')
|
|
2096
2259
|
.description('Search and manage memory');
|
|
2260
|
+
memoryCmd
|
|
2261
|
+
.command('status')
|
|
2262
|
+
.description('Show memory store stats — chunk count, embeddings coverage, agent/category breakdown, salience')
|
|
2263
|
+
.option('--json', 'Emit machine-readable JSON')
|
|
2264
|
+
.action(async (opts) => {
|
|
2265
|
+
const BOLD = '\x1b[1m';
|
|
2266
|
+
const DIM = '\x1b[0;90m';
|
|
2267
|
+
const CYAN = '\x1b[0;36m';
|
|
2268
|
+
const RESET = '\x1b[0m';
|
|
2269
|
+
try {
|
|
2270
|
+
const { MemoryStore } = await import('../memory/store.js');
|
|
2271
|
+
const VAULT_DIR = path.join(BASE_DIR, 'vault');
|
|
2272
|
+
const DB_PATH = path.join(VAULT_DIR, '.memory.db');
|
|
2273
|
+
const store = new MemoryStore(DB_PATH, VAULT_DIR);
|
|
2274
|
+
const stats = store.getMemoryStats();
|
|
2275
|
+
if (opts.json) {
|
|
2276
|
+
console.log(JSON.stringify(stats, null, 2));
|
|
2277
|
+
return;
|
|
2278
|
+
}
|
|
2279
|
+
const pct = stats.totalChunks > 0
|
|
2280
|
+
? ((stats.chunksWithEmbeddings / stats.totalChunks) * 100).toFixed(1)
|
|
2281
|
+
: '0.0';
|
|
2282
|
+
console.log();
|
|
2283
|
+
console.log(` ${BOLD}Memory store${RESET} ${DIM}${DB_PATH}${RESET}`);
|
|
2284
|
+
console.log();
|
|
2285
|
+
console.log(` Total chunks: ${BOLD}${stats.totalChunks.toLocaleString()}${RESET}`);
|
|
2286
|
+
console.log(` With embeddings: ${stats.chunksWithEmbeddings.toLocaleString()} ${DIM}(${pct}%, TF-IDF 512-dim)${RESET}`);
|
|
2287
|
+
console.log(` Pinned (manual): ${stats.pinnedChunks}`);
|
|
2288
|
+
console.log(` Avg salience: ${stats.avgSalience.toFixed(3)} ${DIM}(0 = no access boost; >1 = strong reinforcement)${RESET}`);
|
|
2289
|
+
if (stats.oldestUpdated) {
|
|
2290
|
+
console.log(` Date range: ${stats.oldestUpdated.slice(0, 10)} → ${stats.newestUpdated?.slice(0, 10)}`);
|
|
2291
|
+
}
|
|
2292
|
+
console.log();
|
|
2293
|
+
console.log(` ${BOLD}Per agent${RESET}`);
|
|
2294
|
+
for (const a of stats.perAgent.slice(0, 10)) {
|
|
2295
|
+
console.log(` ${CYAN}${a.agentSlug.padEnd(28)}${RESET}${a.count.toLocaleString().padStart(8)}`);
|
|
2296
|
+
}
|
|
2297
|
+
if (stats.perAgent.length > 10)
|
|
2298
|
+
console.log(` ${DIM}…and ${stats.perAgent.length - 10} more${RESET}`);
|
|
2299
|
+
console.log();
|
|
2300
|
+
console.log(` ${BOLD}Per category${RESET}`);
|
|
2301
|
+
for (const c of stats.perCategory.slice(0, 10)) {
|
|
2302
|
+
console.log(` ${CYAN}${c.category.padEnd(28)}${RESET}${c.count.toLocaleString().padStart(8)}`);
|
|
2303
|
+
}
|
|
2304
|
+
console.log();
|
|
2305
|
+
}
|
|
2306
|
+
catch (err) {
|
|
2307
|
+
console.error(` Error reading memory stats: ${err}`);
|
|
2308
|
+
process.exit(1);
|
|
2309
|
+
}
|
|
2310
|
+
});
|
|
2311
|
+
memoryCmd
|
|
2312
|
+
.command('pin <chunkId>')
|
|
2313
|
+
.description('Pin a chunk — gives its score a 2x boost in recall (use chunk IDs from `memory search`)')
|
|
2314
|
+
.action(async (chunkIdStr) => {
|
|
2315
|
+
const GREEN = '\x1b[0;32m';
|
|
2316
|
+
const RED = '\x1b[0;31m';
|
|
2317
|
+
const RESET = '\x1b[0m';
|
|
2318
|
+
const chunkId = parseInt(chunkIdStr, 10);
|
|
2319
|
+
if (!Number.isFinite(chunkId) || chunkId <= 0) {
|
|
2320
|
+
console.error(` ${RED}Invalid chunk id${RESET}: "${chunkIdStr}". Use IDs from \`clementine memory search\`.`);
|
|
2321
|
+
process.exit(1);
|
|
2322
|
+
}
|
|
2323
|
+
try {
|
|
2324
|
+
const { MemoryStore } = await import('../memory/store.js');
|
|
2325
|
+
const VAULT_DIR = path.join(BASE_DIR, 'vault');
|
|
2326
|
+
const DB_PATH = path.join(VAULT_DIR, '.memory.db');
|
|
2327
|
+
const store = new MemoryStore(DB_PATH, VAULT_DIR);
|
|
2328
|
+
const ok = store.setPinned(chunkId, true);
|
|
2329
|
+
if (!ok) {
|
|
2330
|
+
console.error(` ${RED}Chunk ${chunkId} not found.${RESET}`);
|
|
2331
|
+
process.exit(1);
|
|
2332
|
+
}
|
|
2333
|
+
console.log(` ${GREEN}✓${RESET} Pinned chunk ${chunkId}. It now gets a 2× boost in memory_recall.`);
|
|
2334
|
+
}
|
|
2335
|
+
catch (err) {
|
|
2336
|
+
console.error(` Error pinning chunk: ${err}`);
|
|
2337
|
+
process.exit(1);
|
|
2338
|
+
}
|
|
2339
|
+
});
|
|
2340
|
+
memoryCmd
|
|
2341
|
+
.command('unpin <chunkId>')
|
|
2342
|
+
.description('Unpin a chunk — removes the manual 2x boost, leaves automatic salience untouched')
|
|
2343
|
+
.action(async (chunkIdStr) => {
|
|
2344
|
+
const GREEN = '\x1b[0;32m';
|
|
2345
|
+
const RED = '\x1b[0;31m';
|
|
2346
|
+
const RESET = '\x1b[0m';
|
|
2347
|
+
const chunkId = parseInt(chunkIdStr, 10);
|
|
2348
|
+
if (!Number.isFinite(chunkId) || chunkId <= 0) {
|
|
2349
|
+
console.error(` ${RED}Invalid chunk id${RESET}: "${chunkIdStr}".`);
|
|
2350
|
+
process.exit(1);
|
|
2351
|
+
}
|
|
2352
|
+
try {
|
|
2353
|
+
const { MemoryStore } = await import('../memory/store.js');
|
|
2354
|
+
const VAULT_DIR = path.join(BASE_DIR, 'vault');
|
|
2355
|
+
const DB_PATH = path.join(VAULT_DIR, '.memory.db');
|
|
2356
|
+
const store = new MemoryStore(DB_PATH, VAULT_DIR);
|
|
2357
|
+
const ok = store.setPinned(chunkId, false);
|
|
2358
|
+
if (!ok) {
|
|
2359
|
+
console.error(` ${RED}Chunk ${chunkId} not found.${RESET}`);
|
|
2360
|
+
process.exit(1);
|
|
2361
|
+
}
|
|
2362
|
+
console.log(` ${GREEN}✓${RESET} Unpinned chunk ${chunkId}.`);
|
|
2363
|
+
}
|
|
2364
|
+
catch (err) {
|
|
2365
|
+
console.error(` Error unpinning chunk: ${err}`);
|
|
2366
|
+
process.exit(1);
|
|
2367
|
+
}
|
|
2368
|
+
});
|
|
2369
|
+
memoryCmd
|
|
2370
|
+
.command('dedup')
|
|
2371
|
+
.description('Find near-duplicate chunks via embedding cosine similarity. Dry-run by default.')
|
|
2372
|
+
.option('--threshold <n>', 'Cosine similarity threshold (0-1)', '0.95')
|
|
2373
|
+
.option('--apply', 'Actually delete duplicates (default: dry-run preview only)')
|
|
2374
|
+
.option('--limit <n>', 'Max clusters to report', '50')
|
|
2375
|
+
.action(async (opts) => {
|
|
2376
|
+
const BOLD = '\x1b[1m';
|
|
2377
|
+
const DIM = '\x1b[0;90m';
|
|
2378
|
+
const GREEN = '\x1b[0;32m';
|
|
2379
|
+
const YELLOW = '\x1b[0;33m';
|
|
2380
|
+
const RESET = '\x1b[0m';
|
|
2381
|
+
const threshold = parseFloat(opts.threshold);
|
|
2382
|
+
const limit = parseInt(opts.limit, 10);
|
|
2383
|
+
try {
|
|
2384
|
+
const { MemoryStore } = await import('../memory/store.js');
|
|
2385
|
+
const VAULT_DIR = path.join(BASE_DIR, 'vault');
|
|
2386
|
+
const DB_PATH = path.join(VAULT_DIR, '.memory.db');
|
|
2387
|
+
const store = new MemoryStore(DB_PATH, VAULT_DIR);
|
|
2388
|
+
const clusters = store.findNearDuplicates({ threshold, limit });
|
|
2389
|
+
if (clusters.length === 0) {
|
|
2390
|
+
console.log(` ${GREEN}No near-duplicates found above threshold ${threshold}.${RESET}`);
|
|
2391
|
+
return;
|
|
2392
|
+
}
|
|
2393
|
+
const totalDupes = clusters.reduce((sum, c) => sum + c.duplicates.length, 0);
|
|
2394
|
+
console.log();
|
|
2395
|
+
console.log(` ${BOLD}Found ${clusters.length} cluster${clusters.length === 1 ? '' : 's'} (${totalDupes} duplicate chunk${totalDupes === 1 ? '' : 's'})${RESET}`);
|
|
2396
|
+
console.log(` ${DIM}Keeping the most-recent chunk per cluster; older copies will be removed if --apply is passed.${RESET}`);
|
|
2397
|
+
console.log();
|
|
2398
|
+
for (const cluster of clusters.slice(0, 20)) {
|
|
2399
|
+
const keepLabel = `${cluster.keep.sourceFile} > ${cluster.keep.section}`;
|
|
2400
|
+
const agent = cluster.keep.agentSlug ?? 'global';
|
|
2401
|
+
console.log(` ${BOLD}KEEP${RESET} #${cluster.keep.chunkId} ${DIM}[${agent}]${RESET} ${keepLabel}`);
|
|
2402
|
+
for (const dup of cluster.duplicates) {
|
|
2403
|
+
const dupLabel = `${dup.sourceFile} > ${dup.section}`;
|
|
2404
|
+
console.log(` ${YELLOW}drop${RESET} #${dup.chunkId} sim=${dup.similarity.toFixed(3)} ${DIM}${dupLabel}${RESET}`);
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
if (clusters.length > 20) {
|
|
2408
|
+
console.log(` ${DIM}…and ${clusters.length - 20} more clusters (raise --limit to see them).${RESET}`);
|
|
2409
|
+
}
|
|
2410
|
+
console.log();
|
|
2411
|
+
if (opts.apply) {
|
|
2412
|
+
const allDupeIds = clusters.flatMap(c => c.duplicates.map(d => d.chunkId));
|
|
2413
|
+
const removed = store.deleteChunks(allDupeIds);
|
|
2414
|
+
console.log(` ${GREEN}✓${RESET} Deleted ${removed} duplicate chunk${removed === 1 ? '' : 's'}.`);
|
|
2415
|
+
}
|
|
2416
|
+
else {
|
|
2417
|
+
console.log(` ${DIM}This was a preview. Re-run with ${BOLD}--apply${RESET}${DIM} to delete the duplicates.${RESET}`);
|
|
2418
|
+
}
|
|
2419
|
+
console.log();
|
|
2420
|
+
}
|
|
2421
|
+
catch (err) {
|
|
2422
|
+
console.error(` Error during dedup: ${err}`);
|
|
2423
|
+
process.exit(1);
|
|
2424
|
+
}
|
|
2425
|
+
});
|
|
2426
|
+
memoryCmd
|
|
2427
|
+
.command('cross-agent')
|
|
2428
|
+
.description('Surface chunks that recur across 3+ agents — candidates for promotion to global memory')
|
|
2429
|
+
.option('--threshold <n>', 'Cosine similarity threshold for "same idea" (0-1)', '0.88')
|
|
2430
|
+
.option('--min-agents <n>', 'Minimum distinct agents touched by a cluster', '3')
|
|
2431
|
+
.option('--limit <n>', 'Max clusters to report', '30')
|
|
2432
|
+
.action(async (opts) => {
|
|
2433
|
+
const BOLD = '\x1b[1m';
|
|
2434
|
+
const DIM = '\x1b[0;90m';
|
|
2435
|
+
const CYAN = '\x1b[0;36m';
|
|
2436
|
+
const GREEN = '\x1b[0;32m';
|
|
2437
|
+
const RESET = '\x1b[0m';
|
|
2438
|
+
try {
|
|
2439
|
+
const { MemoryStore } = await import('../memory/store.js');
|
|
2440
|
+
const VAULT_DIR = path.join(BASE_DIR, 'vault');
|
|
2441
|
+
const DB_PATH = path.join(VAULT_DIR, '.memory.db');
|
|
2442
|
+
const store = new MemoryStore(DB_PATH, VAULT_DIR);
|
|
2443
|
+
const clusters = store.findCrossAgentRecurrence({
|
|
2444
|
+
threshold: parseFloat(opts.threshold),
|
|
2445
|
+
minAgents: parseInt(opts.minAgents, 10),
|
|
2446
|
+
limit: parseInt(opts.limit, 10),
|
|
2447
|
+
});
|
|
2448
|
+
if (clusters.length === 0) {
|
|
2449
|
+
console.log(` ${GREEN}No cross-agent recurrence found above threshold ${opts.threshold} touching ${opts.minAgents}+ agents.${RESET}`);
|
|
2450
|
+
return;
|
|
2451
|
+
}
|
|
2452
|
+
console.log();
|
|
2453
|
+
console.log(` ${BOLD}Found ${clusters.length} cluster${clusters.length === 1 ? '' : 's'} recurring across ${opts.minAgents}+ agents${RESET}`);
|
|
2454
|
+
console.log(` ${DIM}These are candidates for promotion to global memory — facts the team has independently arrived at.${RESET}`);
|
|
2455
|
+
console.log();
|
|
2456
|
+
for (const c of clusters) {
|
|
2457
|
+
const preview = c.representative.content.replace(/\n/g, ' ').slice(0, 140);
|
|
2458
|
+
console.log(` ${BOLD}Cluster (${c.agents.length} agents)${RESET} ${CYAN}${c.agents.join(', ')}${RESET}`);
|
|
2459
|
+
console.log(` representative #${c.representative.chunkId} ${DIM}${c.representative.sourceFile} > ${c.representative.section}${RESET}`);
|
|
2460
|
+
console.log(` ${DIM}${preview}${preview.length >= 140 ? '…' : ''}${RESET}`);
|
|
2461
|
+
for (const m of c.members.slice(1, 4)) {
|
|
2462
|
+
console.log(` ${DIM}└─ #${m.chunkId} [${m.agentSlug}] sim=${m.similarity.toFixed(3)}${RESET}`);
|
|
2463
|
+
}
|
|
2464
|
+
if (c.members.length > 4) {
|
|
2465
|
+
console.log(` ${DIM}└─ +${c.members.length - 4} more${RESET}`);
|
|
2466
|
+
}
|
|
2467
|
+
console.log();
|
|
2468
|
+
}
|
|
2469
|
+
console.log(` ${DIM}To promote a chunk to global, use the agent-side ${BOLD}memory_promote${RESET}${DIM} tool with the chunk id, or pin it with ${BOLD}clementine memory pin <id>${RESET}${DIM} for now.${RESET}`);
|
|
2470
|
+
console.log();
|
|
2471
|
+
}
|
|
2472
|
+
catch (err) {
|
|
2473
|
+
console.error(` Error finding cross-agent recurrence: ${err}`);
|
|
2474
|
+
process.exit(1);
|
|
2475
|
+
}
|
|
2476
|
+
});
|
|
2097
2477
|
memoryCmd
|
|
2098
2478
|
.command('search <query>')
|
|
2099
2479
|
.description('Search memory (full-text)')
|
|
@@ -2118,10 +2498,13 @@ memoryCmd
|
|
|
2118
2498
|
const source = r.sourceFile ? path.basename(r.sourceFile) : 'unknown';
|
|
2119
2499
|
const section = r.section || '';
|
|
2120
2500
|
const snippet = r.content.replace(/\n/g, ' ').slice(0, 120);
|
|
2121
|
-
|
|
2501
|
+
const pinned = r.pinned ? ' 📌' : '';
|
|
2502
|
+
console.log(` ${DIM}#${r.chunkId}${RESET} ${BOLD}${source}${RESET}${section ? ` › ${CYAN}${section}${RESET}` : ''}${pinned}`);
|
|
2122
2503
|
console.log(` ${DIM}${snippet}${snippet.length >= 120 ? '…' : ''}${RESET}`);
|
|
2123
2504
|
console.log();
|
|
2124
2505
|
}
|
|
2506
|
+
console.log(` ${DIM}Tip: pin a chunk to boost its score in recall — ${BOLD}clementine memory pin <id>${RESET}`);
|
|
2507
|
+
console.log();
|
|
2125
2508
|
}
|
|
2126
2509
|
catch (err) {
|
|
2127
2510
|
console.error(` Error searching memory: ${err}`);
|
package/dist/memory/store.d.ts
CHANGED
|
@@ -50,6 +50,94 @@ export declare class MemoryStore {
|
|
|
50
50
|
private get conn();
|
|
51
51
|
/** Return the total number of indexed chunks. */
|
|
52
52
|
getChunkCount(): number;
|
|
53
|
+
/** Toggle the manual pin flag on a chunk. Pinned chunks get a 2x score boost in recall. */
|
|
54
|
+
setPinned(chunkId: number, pinned: boolean): boolean;
|
|
55
|
+
/**
|
|
56
|
+
* Aggregate stats for the memory store — used by `clementine memory status`.
|
|
57
|
+
* Single-pass scans so it stays fast even on large chunk tables.
|
|
58
|
+
*/
|
|
59
|
+
getMemoryStats(): {
|
|
60
|
+
totalChunks: number;
|
|
61
|
+
chunksWithEmbeddings: number;
|
|
62
|
+
pinnedChunks: number;
|
|
63
|
+
perAgent: Array<{
|
|
64
|
+
agentSlug: string;
|
|
65
|
+
count: number;
|
|
66
|
+
}>;
|
|
67
|
+
perCategory: Array<{
|
|
68
|
+
category: string;
|
|
69
|
+
count: number;
|
|
70
|
+
}>;
|
|
71
|
+
avgSalience: number;
|
|
72
|
+
oldestUpdated: string | null;
|
|
73
|
+
newestUpdated: string | null;
|
|
74
|
+
};
|
|
75
|
+
/**
|
|
76
|
+
* Find clusters of near-duplicate chunks using embedding cosine similarity.
|
|
77
|
+
* Returns clusters where at least 2 chunks score above the threshold.
|
|
78
|
+
*
|
|
79
|
+
* Caller decides what to do — typical use is `clementine memory dedup` to
|
|
80
|
+
* preview / merge / mark-superseded. Per-pair O(n²) within agent scope to
|
|
81
|
+
* keep the search space tractable; cross-agent dupes are surfaced separately
|
|
82
|
+
* by the auto-promote flow.
|
|
83
|
+
*/
|
|
84
|
+
findNearDuplicates(opts?: {
|
|
85
|
+
threshold?: number;
|
|
86
|
+
minLen?: number;
|
|
87
|
+
limit?: number;
|
|
88
|
+
}): Array<{
|
|
89
|
+
keep: {
|
|
90
|
+
chunkId: number;
|
|
91
|
+
sourceFile: string;
|
|
92
|
+
section: string;
|
|
93
|
+
content: string;
|
|
94
|
+
agentSlug: string | null;
|
|
95
|
+
updatedAt: string | null;
|
|
96
|
+
};
|
|
97
|
+
duplicates: Array<{
|
|
98
|
+
chunkId: number;
|
|
99
|
+
sourceFile: string;
|
|
100
|
+
section: string;
|
|
101
|
+
content: string;
|
|
102
|
+
agentSlug: string | null;
|
|
103
|
+
updatedAt: string | null;
|
|
104
|
+
similarity: number;
|
|
105
|
+
}>;
|
|
106
|
+
}>;
|
|
107
|
+
/** Delete chunks by id. Used by dedup --apply. */
|
|
108
|
+
deleteChunks(chunkIds: number[]): number;
|
|
109
|
+
/**
|
|
110
|
+
* Find chunks whose semantic content recurs across 3+ different agents —
|
|
111
|
+
* candidates for promotion to global memory. Detection-only; surfacing.
|
|
112
|
+
* The user (or a future cron) decides whether to actually promote.
|
|
113
|
+
*
|
|
114
|
+
* Approach: scan agent-scoped chunks with embeddings, cluster cross-agent
|
|
115
|
+
* pairs above the similarity threshold, return clusters touching >= minAgents
|
|
116
|
+
* distinct agents. Limits keep the O(n²) scan tractable on large stores.
|
|
117
|
+
*/
|
|
118
|
+
findCrossAgentRecurrence(opts?: {
|
|
119
|
+
threshold?: number;
|
|
120
|
+
minAgents?: number;
|
|
121
|
+
minLen?: number;
|
|
122
|
+
limit?: number;
|
|
123
|
+
}): Array<{
|
|
124
|
+
representative: {
|
|
125
|
+
chunkId: number;
|
|
126
|
+
sourceFile: string;
|
|
127
|
+
section: string;
|
|
128
|
+
content: string;
|
|
129
|
+
agentSlug: string;
|
|
130
|
+
};
|
|
131
|
+
members: Array<{
|
|
132
|
+
chunkId: number;
|
|
133
|
+
sourceFile: string;
|
|
134
|
+
section: string;
|
|
135
|
+
agentSlug: string;
|
|
136
|
+
similarity: number;
|
|
137
|
+
updatedAt: string | null;
|
|
138
|
+
}>;
|
|
139
|
+
agents: string[];
|
|
140
|
+
}>;
|
|
53
141
|
/**
|
|
54
142
|
* Scan the entire vault, hash-compare, and re-index changed files.
|
|
55
143
|
*/
|
package/dist/memory/store.js
CHANGED
|
@@ -173,6 +173,15 @@ export class MemoryStore {
|
|
|
173
173
|
catch {
|
|
174
174
|
// Column already exists
|
|
175
175
|
}
|
|
176
|
+
// Add pinned flag — manual salience reinforcement. When true, recall
|
|
177
|
+
// applies an extra score boost on top of the access-pattern salience.
|
|
178
|
+
// Toggled by `clementine memory pin/unpin <chunkId>` (or the dashboard).
|
|
179
|
+
try {
|
|
180
|
+
this.conn.exec('ALTER TABLE chunks ADD COLUMN pinned INTEGER DEFAULT 0');
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
// Column already exists
|
|
184
|
+
}
|
|
176
185
|
// Indexes for category/topic filtering
|
|
177
186
|
try {
|
|
178
187
|
this.conn.exec('CREATE INDEX idx_chunks_category ON chunks(category)');
|
|
@@ -737,6 +746,213 @@ export class MemoryStore {
|
|
|
737
746
|
return 0;
|
|
738
747
|
}
|
|
739
748
|
}
|
|
749
|
+
/** Toggle the manual pin flag on a chunk. Pinned chunks get a 2x score boost in recall. */
|
|
750
|
+
setPinned(chunkId, pinned) {
|
|
751
|
+
try {
|
|
752
|
+
const result = this.conn.prepare('UPDATE chunks SET pinned = ? WHERE id = ?')
|
|
753
|
+
.run(pinned ? 1 : 0, chunkId);
|
|
754
|
+
return result.changes > 0;
|
|
755
|
+
}
|
|
756
|
+
catch {
|
|
757
|
+
return false;
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
/**
|
|
761
|
+
* Aggregate stats for the memory store — used by `clementine memory status`.
|
|
762
|
+
* Single-pass scans so it stays fast even on large chunk tables.
|
|
763
|
+
*/
|
|
764
|
+
getMemoryStats() {
|
|
765
|
+
const totalChunks = this.getChunkCount();
|
|
766
|
+
const chunksWithEmbeddings = this.conn
|
|
767
|
+
.prepare('SELECT COUNT(*) as cnt FROM chunks WHERE embedding IS NOT NULL')
|
|
768
|
+
.get()?.cnt ?? 0;
|
|
769
|
+
const pinnedChunks = this.conn
|
|
770
|
+
.prepare('SELECT COUNT(*) as cnt FROM chunks WHERE pinned = 1')
|
|
771
|
+
.get()?.cnt ?? 0;
|
|
772
|
+
const perAgent = this.conn
|
|
773
|
+
.prepare(`SELECT COALESCE(agent_slug, 'global') as agentSlug, COUNT(*) as count
|
|
774
|
+
FROM chunks GROUP BY agent_slug ORDER BY count DESC`)
|
|
775
|
+
.all();
|
|
776
|
+
const perCategory = this.conn
|
|
777
|
+
.prepare(`SELECT COALESCE(category, '(none)') as category, COUNT(*) as count
|
|
778
|
+
FROM chunks GROUP BY category ORDER BY count DESC`)
|
|
779
|
+
.all();
|
|
780
|
+
const avgRow = this.conn
|
|
781
|
+
.prepare('SELECT AVG(salience) as avg FROM chunks WHERE salience > 0')
|
|
782
|
+
.get();
|
|
783
|
+
const dateRow = this.conn
|
|
784
|
+
.prepare('SELECT MIN(updated_at) as oldest, MAX(updated_at) as newest FROM chunks WHERE updated_at IS NOT NULL')
|
|
785
|
+
.get();
|
|
786
|
+
return {
|
|
787
|
+
totalChunks,
|
|
788
|
+
chunksWithEmbeddings,
|
|
789
|
+
pinnedChunks,
|
|
790
|
+
perAgent,
|
|
791
|
+
perCategory,
|
|
792
|
+
avgSalience: avgRow?.avg ?? 0,
|
|
793
|
+
oldestUpdated: dateRow?.oldest ?? null,
|
|
794
|
+
newestUpdated: dateRow?.newest ?? null,
|
|
795
|
+
};
|
|
796
|
+
}
|
|
797
|
+
/**
|
|
798
|
+
* Find clusters of near-duplicate chunks using embedding cosine similarity.
|
|
799
|
+
* Returns clusters where at least 2 chunks score above the threshold.
|
|
800
|
+
*
|
|
801
|
+
* Caller decides what to do — typical use is `clementine memory dedup` to
|
|
802
|
+
* preview / merge / mark-superseded. Per-pair O(n²) within agent scope to
|
|
803
|
+
* keep the search space tractable; cross-agent dupes are surfaced separately
|
|
804
|
+
* by the auto-promote flow.
|
|
805
|
+
*/
|
|
806
|
+
findNearDuplicates(opts = {}) {
|
|
807
|
+
const threshold = opts.threshold ?? 0.95;
|
|
808
|
+
const minLen = opts.minLen ?? 80; // skip very short chunks — too easily collide
|
|
809
|
+
const limitClusters = opts.limit ?? 50; // cap results so the CLI stays readable
|
|
810
|
+
if (!embeddingsModule.isReady())
|
|
811
|
+
return [];
|
|
812
|
+
const rows = this.conn.prepare(`SELECT id, source_file, section, content, embedding, agent_slug, updated_at
|
|
813
|
+
FROM chunks
|
|
814
|
+
WHERE embedding IS NOT NULL AND length(content) >= ?
|
|
815
|
+
ORDER BY agent_slug, updated_at DESC`).all(minLen);
|
|
816
|
+
// Group by agent first — only compare within the same scope to bound the
|
|
817
|
+
// O(n²) blow-up. Cross-agent dedup is the auto-promote flow's job.
|
|
818
|
+
const buckets = new Map();
|
|
819
|
+
for (const r of rows) {
|
|
820
|
+
const key = r.agent_slug ?? '__global__';
|
|
821
|
+
if (!buckets.has(key))
|
|
822
|
+
buckets.set(key, []);
|
|
823
|
+
buckets.get(key).push(r);
|
|
824
|
+
}
|
|
825
|
+
const clusters = [];
|
|
826
|
+
const consumed = new Set();
|
|
827
|
+
for (const bucket of buckets.values()) {
|
|
828
|
+
// Decode embeddings once per row.
|
|
829
|
+
const decoded = bucket.map(r => ({
|
|
830
|
+
...r,
|
|
831
|
+
vec: embeddingsModule.deserializeEmbedding(r.embedding),
|
|
832
|
+
}));
|
|
833
|
+
for (let i = 0; i < decoded.length; i++) {
|
|
834
|
+
if (consumed.has(decoded[i].id))
|
|
835
|
+
continue;
|
|
836
|
+
const head = decoded[i];
|
|
837
|
+
const dupes = [];
|
|
838
|
+
for (let j = i + 1; j < decoded.length; j++) {
|
|
839
|
+
if (consumed.has(decoded[j].id))
|
|
840
|
+
continue;
|
|
841
|
+
const sim = embeddingsModule.cosineSimilarity(head.vec, decoded[j].vec);
|
|
842
|
+
if (sim >= threshold) {
|
|
843
|
+
dupes.push({
|
|
844
|
+
chunkId: decoded[j].id,
|
|
845
|
+
sourceFile: decoded[j].source_file,
|
|
846
|
+
section: decoded[j].section,
|
|
847
|
+
content: decoded[j].content,
|
|
848
|
+
agentSlug: decoded[j].agent_slug,
|
|
849
|
+
updatedAt: decoded[j].updated_at,
|
|
850
|
+
similarity: sim,
|
|
851
|
+
});
|
|
852
|
+
consumed.add(decoded[j].id);
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
if (dupes.length > 0) {
|
|
856
|
+
consumed.add(head.id);
|
|
857
|
+
clusters.push({
|
|
858
|
+
keep: {
|
|
859
|
+
chunkId: head.id,
|
|
860
|
+
sourceFile: head.source_file,
|
|
861
|
+
section: head.section,
|
|
862
|
+
content: head.content,
|
|
863
|
+
agentSlug: head.agent_slug,
|
|
864
|
+
updatedAt: head.updated_at,
|
|
865
|
+
},
|
|
866
|
+
duplicates: dupes,
|
|
867
|
+
});
|
|
868
|
+
if (clusters.length >= limitClusters)
|
|
869
|
+
return clusters;
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
return clusters;
|
|
874
|
+
}
|
|
875
|
+
/** Delete chunks by id. Used by dedup --apply. */
|
|
876
|
+
deleteChunks(chunkIds) {
|
|
877
|
+
if (!chunkIds.length)
|
|
878
|
+
return 0;
|
|
879
|
+
const placeholders = chunkIds.map(() => '?').join(',');
|
|
880
|
+
const result = this.conn.prepare(`DELETE FROM chunks WHERE id IN (${placeholders})`).run(...chunkIds);
|
|
881
|
+
return result.changes;
|
|
882
|
+
}
|
|
883
|
+
/**
|
|
884
|
+
* Find chunks whose semantic content recurs across 3+ different agents —
|
|
885
|
+
* candidates for promotion to global memory. Detection-only; surfacing.
|
|
886
|
+
* The user (or a future cron) decides whether to actually promote.
|
|
887
|
+
*
|
|
888
|
+
* Approach: scan agent-scoped chunks with embeddings, cluster cross-agent
|
|
889
|
+
* pairs above the similarity threshold, return clusters touching >= minAgents
|
|
890
|
+
* distinct agents. Limits keep the O(n²) scan tractable on large stores.
|
|
891
|
+
*/
|
|
892
|
+
findCrossAgentRecurrence(opts = {}) {
|
|
893
|
+
const threshold = opts.threshold ?? 0.88; // looser than dedup — paraphrases count
|
|
894
|
+
const minAgents = opts.minAgents ?? 3;
|
|
895
|
+
const minLen = opts.minLen ?? 100;
|
|
896
|
+
const limitClusters = opts.limit ?? 30;
|
|
897
|
+
if (!embeddingsModule.isReady())
|
|
898
|
+
return [];
|
|
899
|
+
// Only consider chunks that ARE agent-scoped (NULL = already global).
|
|
900
|
+
const rows = this.conn.prepare(`SELECT id, source_file, section, content, embedding, agent_slug, updated_at
|
|
901
|
+
FROM chunks
|
|
902
|
+
WHERE embedding IS NOT NULL
|
|
903
|
+
AND agent_slug IS NOT NULL
|
|
904
|
+
AND length(content) >= ?
|
|
905
|
+
ORDER BY updated_at DESC`).all(minLen);
|
|
906
|
+
if (rows.length < minAgents)
|
|
907
|
+
return [];
|
|
908
|
+
const decoded = rows.map(r => ({ ...r, vec: embeddingsModule.deserializeEmbedding(r.embedding) }));
|
|
909
|
+
const clusters = [];
|
|
910
|
+
const consumed = new Set();
|
|
911
|
+
for (let i = 0; i < decoded.length; i++) {
|
|
912
|
+
if (consumed.has(decoded[i].id))
|
|
913
|
+
continue;
|
|
914
|
+
const head = decoded[i];
|
|
915
|
+
const members = [
|
|
916
|
+
{ chunkId: head.id, sourceFile: head.source_file, section: head.section, agentSlug: head.agent_slug, similarity: 1.0, updatedAt: head.updated_at },
|
|
917
|
+
];
|
|
918
|
+
const agentsTouched = new Set([head.agent_slug]);
|
|
919
|
+
for (let j = i + 1; j < decoded.length; j++) {
|
|
920
|
+
if (consumed.has(decoded[j].id))
|
|
921
|
+
continue;
|
|
922
|
+
const sim = embeddingsModule.cosineSimilarity(head.vec, decoded[j].vec);
|
|
923
|
+
if (sim >= threshold) {
|
|
924
|
+
members.push({
|
|
925
|
+
chunkId: decoded[j].id,
|
|
926
|
+
sourceFile: decoded[j].source_file,
|
|
927
|
+
section: decoded[j].section,
|
|
928
|
+
agentSlug: decoded[j].agent_slug,
|
|
929
|
+
similarity: sim,
|
|
930
|
+
updatedAt: decoded[j].updated_at,
|
|
931
|
+
});
|
|
932
|
+
agentsTouched.add(decoded[j].agent_slug);
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
if (agentsTouched.size >= minAgents) {
|
|
936
|
+
// Mark all in this cluster consumed so we don't re-cluster around them.
|
|
937
|
+
for (const m of members)
|
|
938
|
+
consumed.add(m.chunkId);
|
|
939
|
+
clusters.push({
|
|
940
|
+
representative: {
|
|
941
|
+
chunkId: head.id,
|
|
942
|
+
sourceFile: head.source_file,
|
|
943
|
+
section: head.section,
|
|
944
|
+
content: head.content,
|
|
945
|
+
agentSlug: head.agent_slug,
|
|
946
|
+
},
|
|
947
|
+
members,
|
|
948
|
+
agents: Array.from(agentsTouched).sort(),
|
|
949
|
+
});
|
|
950
|
+
if (clusters.length >= limitClusters)
|
|
951
|
+
break;
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
return clusters;
|
|
955
|
+
}
|
|
740
956
|
// ── Full Sync ──────────────────────────────────────────────────────
|
|
741
957
|
/**
|
|
742
958
|
* Scan the entire vault, hash-compare, and re-index changed files.
|
|
@@ -878,7 +1094,7 @@ export class MemoryStore {
|
|
|
878
1094
|
try {
|
|
879
1095
|
let sql = `SELECT c.id, c.source_file, c.section, c.content, c.chunk_type,
|
|
880
1096
|
c.updated_at, c.salience, c.last_outcome_score, c.agent_slug, c.category, c.topic,
|
|
881
|
-
bm25(chunks_fts) as score
|
|
1097
|
+
c.pinned, bm25(chunks_fts) as score
|
|
882
1098
|
FROM chunks_fts f
|
|
883
1099
|
JOIN chunks c ON c.id = f.rowid
|
|
884
1100
|
WHERE chunks_fts MATCH ?`;
|
|
@@ -912,6 +1128,7 @@ export class MemoryStore {
|
|
|
912
1128
|
agentSlug: row.agent_slug ?? null,
|
|
913
1129
|
category: row.category,
|
|
914
1130
|
topic: row.topic,
|
|
1131
|
+
pinned: row.pinned === 1,
|
|
915
1132
|
}));
|
|
916
1133
|
}
|
|
917
1134
|
catch {
|
|
@@ -1030,6 +1247,12 @@ export class MemoryStore {
|
|
|
1030
1247
|
if (r.salience > 0) {
|
|
1031
1248
|
r.score *= 1.0 + r.salience;
|
|
1032
1249
|
}
|
|
1250
|
+
// Manual pin: stronger boost than access-pattern salience. Toggled via
|
|
1251
|
+
// `clementine memory pin <chunkId>`. Doubles the relevance score so
|
|
1252
|
+
// pinned chunks consistently rank near the top within their relevance band.
|
|
1253
|
+
if (r.pinned) {
|
|
1254
|
+
r.score *= 2.0;
|
|
1255
|
+
}
|
|
1033
1256
|
// Outcome-driven adjustment: chunks that recently got cited in
|
|
1034
1257
|
// responses get a small boost; chunks that were pulled in and
|
|
1035
1258
|
// ignored get a small penalty. Bounded to ±30% so outcome noise
|
|
@@ -108,6 +108,51 @@ export function registerMemoryTools(server) {
|
|
|
108
108
|
}
|
|
109
109
|
}
|
|
110
110
|
});
|
|
111
|
+
// ── 0b. team_scratchpad ────────────────────────────────────────────────
|
|
112
|
+
//
|
|
113
|
+
// Cross-agent shared scratchpad. Unlike working_memory (per-agent), this
|
|
114
|
+
// is a single shared markdown file every agent can read and append to.
|
|
115
|
+
// Use cases: live coordination ("Sasha is drafting the brief, Ross hold
|
|
116
|
+
// outbound for 30m"), cross-agent context drops, async hand-offs that
|
|
117
|
+
// don't warrant a full goal_create or task_add. Append tags every entry
|
|
118
|
+
// with the author's agent slug + ISO timestamp so the trail stays clear.
|
|
119
|
+
const TEAM_SCRATCHPAD_FILE = path.join(BASE_DIR, 'team-scratchpad.md');
|
|
120
|
+
server.tool('team_scratchpad', getToolDescription('team_scratchpad') ?? 'Cross-agent shared scratchpad for live team coordination. All agents read/write the same file. Use for hand-offs, "I am working on X", short-term context drops. For durable facts, use memory_write/MEMORY.md instead.', {
|
|
121
|
+
action: z.enum(['read', 'append', 'replace', 'clear']).describe('What to do with the team scratchpad'),
|
|
122
|
+
content: z.string().optional().describe('Text to append or replace with (required for append/replace)'),
|
|
123
|
+
}, async ({ action, content }) => {
|
|
124
|
+
const author = ACTIVE_AGENT_SLUG ?? 'clementine';
|
|
125
|
+
switch (action) {
|
|
126
|
+
case 'read': {
|
|
127
|
+
if (!existsSync(TEAM_SCRATCHPAD_FILE)) {
|
|
128
|
+
return textResult('Team scratchpad is empty.');
|
|
129
|
+
}
|
|
130
|
+
return textResult(readFileSync(TEAM_SCRATCHPAD_FILE, 'utf-8'));
|
|
131
|
+
}
|
|
132
|
+
case 'append': {
|
|
133
|
+
if (!content)
|
|
134
|
+
return textResult('Error: content is required for append.');
|
|
135
|
+
const stamp = new Date().toISOString();
|
|
136
|
+
const entry = `\n- **[${author}@${stamp}]** ${content}\n`;
|
|
137
|
+
const existing = existsSync(TEAM_SCRATCHPAD_FILE) ? readFileSync(TEAM_SCRATCHPAD_FILE, 'utf-8') : '# Team Scratchpad\n\nShared across all agents. Append tags entries with author + timestamp.\n';
|
|
138
|
+
writeFileSync(TEAM_SCRATCHPAD_FILE, existing + entry);
|
|
139
|
+
return textResult(`Appended to team scratchpad as ${author}.`);
|
|
140
|
+
}
|
|
141
|
+
case 'replace': {
|
|
142
|
+
if (!content)
|
|
143
|
+
return textResult('Error: content is required for replace.');
|
|
144
|
+
const stamp = new Date().toISOString();
|
|
145
|
+
const header = `# Team Scratchpad\n\n_Replaced by ${author} at ${stamp}._\n\n`;
|
|
146
|
+
writeFileSync(TEAM_SCRATCHPAD_FILE, header + content + '\n');
|
|
147
|
+
return textResult(`Team scratchpad replaced by ${author}.`);
|
|
148
|
+
}
|
|
149
|
+
case 'clear': {
|
|
150
|
+
if (existsSync(TEAM_SCRATCHPAD_FILE))
|
|
151
|
+
unlinkSync(TEAM_SCRATCHPAD_FILE);
|
|
152
|
+
return textResult('Team scratchpad cleared.');
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
});
|
|
111
156
|
// ── 1. memory_read ─────────────────────────────────────────────────────
|
|
112
157
|
server.tool('memory_read', getToolDescription('memory_read') ?? "Read a note from the Obsidian vault. Shortcuts: 'today', 'yesterday', 'memory', 'tasks', 'heartbeat', 'cron', 'soul'. Or pass a relative path or note name.", {
|
|
113
158
|
name: z.string().describe('Note name, path, or shortcut'),
|
|
@@ -305,7 +350,7 @@ export function registerMemoryTools(server) {
|
|
|
305
350
|
return textResult(results.join('\n'));
|
|
306
351
|
});
|
|
307
352
|
// ── 4. memory_recall ───────────────────────────────────────────────────
|
|
308
|
-
server.tool('memory_recall', getToolDescription('memory_recall') ?? 'Context retrieval combining FTS5 relevance + recency search
|
|
353
|
+
server.tool('memory_recall', getToolDescription('memory_recall') ?? 'Context retrieval combining FTS5 relevance + recency search, scoped to your memory + global. For cross-agent synthesis use brain_recall.', {
|
|
309
354
|
query: z.string().describe('Natural language search query'),
|
|
310
355
|
category: z.enum(['facts', 'events', 'discoveries', 'preferences', 'advice']).optional().describe('Filter by category'),
|
|
311
356
|
topic: z.string().optional().describe('Filter by topic'),
|
|
@@ -321,11 +366,56 @@ export function registerMemoryTools(server) {
|
|
|
321
366
|
store.recordAccess(chunkIds);
|
|
322
367
|
const lines = results.map(r => {
|
|
323
368
|
const label = `[${r.matchType}]`;
|
|
369
|
+
const agentTag = r.agentSlug ? ` [agent: ${r.agentSlug}]` : '';
|
|
324
370
|
const preview = r.content.slice(0, 300).replace(/\n/g, ' ');
|
|
325
|
-
return `**${r.sourceFile} > ${r.section}** ${label} (score: ${r.score.toFixed(3)})\n${preview}\n`;
|
|
371
|
+
return `**${r.sourceFile} > ${r.section}** ${label}${agentTag} (score: ${r.score.toFixed(3)})\n${preview}\n`;
|
|
326
372
|
});
|
|
327
373
|
return textResult(lines.join('\n'));
|
|
328
374
|
});
|
|
375
|
+
// ── 4b. brain_recall ──────────────────────────────────────────────────
|
|
376
|
+
//
|
|
377
|
+
// Cross-agent unified recall. Differs from memory_recall in two ways:
|
|
378
|
+
// 1. No agentSlug scope — pulls from every agent's memory + global.
|
|
379
|
+
// 2. Always tags each result with [agent: <slug>] so the caller can
|
|
380
|
+
// see provenance (which agent's memory the chunk came from).
|
|
381
|
+
//
|
|
382
|
+
// Intended caller: Clementine herself. Specialist agents normally stay in
|
|
383
|
+
// memory_recall (which respects strict isolation). brain_recall is the
|
|
384
|
+
// "single brain" view that lets the master assistant synthesize across
|
|
385
|
+
// the whole team.
|
|
386
|
+
server.tool('brain_recall', getToolDescription('brain_recall') ?? 'Cross-agent unified recall — searches across all agents with source-agent attribution. Use for synthesis questions or when you need the full picture, not just your own scope.', {
|
|
387
|
+
query: z.string().describe('Natural language query — what to find across all agents'),
|
|
388
|
+
category: z.enum(['facts', 'events', 'discoveries', 'preferences', 'advice']).optional().describe('Filter by category'),
|
|
389
|
+
topic: z.string().optional().describe('Filter by topic'),
|
|
390
|
+
limit: z.number().optional().describe('Max results across all agents (default 12)'),
|
|
391
|
+
}, async ({ query, category, topic, limit }) => {
|
|
392
|
+
const store = await getStore();
|
|
393
|
+
// Intentionally omit agentSlug — we want the unscoped, cross-agent view.
|
|
394
|
+
const results = store.searchContext(query, { category, topic, limit: limit ?? 12 });
|
|
395
|
+
if (!results.length) {
|
|
396
|
+
return textResult(`No results for: ${query}`);
|
|
397
|
+
}
|
|
398
|
+
const chunkIds = results.map(r => r.chunkId).filter(Boolean);
|
|
399
|
+
if (chunkIds.length)
|
|
400
|
+
store.recordAccess(chunkIds);
|
|
401
|
+
// Group attribution counts so the agent gets a quick summary of the spread.
|
|
402
|
+
const perAgent = new Map();
|
|
403
|
+
for (const r of results) {
|
|
404
|
+
const key = r.agentSlug ?? 'global';
|
|
405
|
+
perAgent.set(key, (perAgent.get(key) ?? 0) + 1);
|
|
406
|
+
}
|
|
407
|
+
const spread = Array.from(perAgent.entries())
|
|
408
|
+
.sort((a, b) => b[1] - a[1])
|
|
409
|
+
.map(([slug, n]) => `${slug}:${n}`)
|
|
410
|
+
.join(', ');
|
|
411
|
+
const lines = results.map(r => {
|
|
412
|
+
const agent = r.agentSlug ?? 'global';
|
|
413
|
+
const label = `[${r.matchType}]`;
|
|
414
|
+
const preview = r.content.slice(0, 300).replace(/\n/g, ' ');
|
|
415
|
+
return `**${r.sourceFile} > ${r.section}** ${label} [agent: ${agent}] (score: ${r.score.toFixed(3)})\n${preview}\n`;
|
|
416
|
+
});
|
|
417
|
+
return textResult(`Cross-agent spread: ${spread}\n\n${lines.join('\n')}`);
|
|
418
|
+
});
|
|
329
419
|
// ── 10. memory_connections ─────────────────────────────────────────────
|
|
330
420
|
server.tool('memory_connections', 'Query the wikilink graph — find all notes connected to/from a given note.', {
|
|
331
421
|
note_name: z.string().describe('Note name (without .md) to find connections for'),
|
package/dist/tools/tool-meta.js
CHANGED
|
@@ -11,10 +11,15 @@
|
|
|
11
11
|
const TOOL_META = {
|
|
12
12
|
// ── Memory & Vault ────────────────────────────────────────────────
|
|
13
13
|
working_memory: {
|
|
14
|
-
description: '
|
|
14
|
+
description: 'Per-agent persistent scratchpad — only YOU see it. Survives across conversations. Use for current project context, TODOs, reminders, or anything you need to remember for next time. Actions: read, append, replace, clear. ALWAYS read before replacing. For cross-agent coordination, use team_scratchpad instead.',
|
|
15
15
|
exampleUsage: 'Before starting complex work, read working_memory to check for context from prior sessions.',
|
|
16
16
|
returnHint: 'Full working memory contents (markdown text).',
|
|
17
17
|
},
|
|
18
|
+
team_scratchpad: {
|
|
19
|
+
description: 'Cross-agent shared scratchpad — every agent on the team reads and writes the same file. Use for live coordination, hand-offs, "I am working on X — back off until Y", short-lived context drops. For durable facts that should outlive coordination noise, use memory_write to MEMORY.md instead. Append tags entries with author slug + timestamp.',
|
|
20
|
+
exampleUsage: 'Before starting outbound work, read team_scratchpad to see if another agent has already claimed a prospect or paused outreach.',
|
|
21
|
+
returnHint: 'Full scratchpad contents with per-entry author + ISO timestamp.',
|
|
22
|
+
},
|
|
18
23
|
memory_search: {
|
|
19
24
|
description: 'Full-text search across all vault notes. Best for finding specific keywords or phrases. For broader semantic matching, use memory_recall instead. Results include file path, section heading, and relevance score.',
|
|
20
25
|
exampleUsage: 'Use when the user asks "what did we discuss about X" or you need to find a specific note.',
|
|
@@ -22,10 +27,15 @@ const TOOL_META = {
|
|
|
22
27
|
paginationNote: 'Default limit is 20 results. For broad queries, start with limit=5 and increase only if needed.',
|
|
23
28
|
},
|
|
24
29
|
memory_recall: {
|
|
25
|
-
description: 'Context retrieval combining text relevance + recency
|
|
26
|
-
exampleUsage: 'Use before responding to questions about people, projects, or topics the user has discussed before.',
|
|
30
|
+
description: 'Context retrieval combining text relevance + recency, scoped to YOUR memory + global memory. Best as your default "what do I know about X" tool when the question lives in your domain. For cross-agent synthesis (what your team as a whole knows), use brain_recall instead.',
|
|
31
|
+
exampleUsage: 'Use before responding to questions about people, projects, or topics the user has discussed with you before.',
|
|
27
32
|
returnHint: 'Ranked chunks with source file, category, and content preview.',
|
|
28
33
|
},
|
|
34
|
+
brain_recall: {
|
|
35
|
+
description: 'CROSS-AGENT unified recall — searches the entire memory store across YOU and every team agent, with source-agent attribution per result. Use for synthesis questions ("what does my team know about X?", "have any of my agents discussed Y?"), or whenever you need the full picture instead of just your own scope. Specialist agents normally stay in memory_recall (which respects their isolation); brain_recall is the single-brain view, primarily for Clementine.',
|
|
36
|
+
exampleUsage: 'When the user asks "what have we collectively learned about Acme this quarter", brain_recall returns chunks from every agent that touched the topic.',
|
|
37
|
+
returnHint: 'Ranked chunks tagged [agent: <slug>] or [agent: global], plus source file and content preview.',
|
|
38
|
+
},
|
|
29
39
|
memory_read: {
|
|
30
40
|
description: "Read a note from the Obsidian vault. Shortcuts: 'today' (daily note), 'yesterday', 'memory' (MEMORY.md), 'tasks' (TASKS.md), 'heartbeat', 'cron', 'soul'. Or pass a relative path like '03-Projects/my-project.md'.",
|
|
31
41
|
exampleUsage: "memory_read('today') to check what happened today before making plans.",
|
package/dist/types.d.ts
CHANGED
|
@@ -15,6 +15,7 @@ export interface SearchResult {
|
|
|
15
15
|
agentSlug?: string | null;
|
|
16
16
|
category?: string | null;
|
|
17
17
|
topic?: string | null;
|
|
18
|
+
pinned?: boolean;
|
|
18
19
|
}
|
|
19
20
|
export type ChunkCategory = 'facts' | 'events' | 'discoveries' | 'preferences' | 'advice';
|
|
20
21
|
export interface Chunk {
|
|
@@ -508,7 +509,7 @@ export interface SelfImproveExperiment {
|
|
|
508
509
|
baselineScore: number;
|
|
509
510
|
score: number;
|
|
510
511
|
accepted: boolean;
|
|
511
|
-
approvalStatus: 'pending' | 'approved' | 'denied' | 'expired';
|
|
512
|
+
approvalStatus: 'pending' | 'approved' | 'denied' | 'expired' | 'unsurfaced';
|
|
512
513
|
reason: string;
|
|
513
514
|
error?: string;
|
|
514
515
|
}
|
|
@@ -548,6 +549,13 @@ export interface SelfImproveConfig {
|
|
|
548
549
|
iterationBudgetMs: number;
|
|
549
550
|
maxDurationMs: number;
|
|
550
551
|
acceptThreshold: number;
|
|
552
|
+
/**
|
|
553
|
+
* Default: 0.85. Stricter floor for what reaches the user's pending-changes
|
|
554
|
+
* inbox. Proposals scoring >= acceptThreshold but < surfaceThreshold are
|
|
555
|
+
* marked 'unsurfaced' — kept in the experiment log for trend analysis but
|
|
556
|
+
* NOT written to PENDING_DIR. Cuts noise without losing signal data.
|
|
557
|
+
*/
|
|
558
|
+
surfaceThreshold?: number;
|
|
551
559
|
plateauLimit: number;
|
|
552
560
|
areas: ('soul' | 'cron' | 'workflow' | 'memory' | 'agent' | 'source' | 'communication' | 'goal' | 'advisor-rule' | 'prompt-override')[];
|
|
553
561
|
/** Enable tiered auto-apply: low-risk changes apply without approval. Default: false. */
|