claude-self-reflect 2.3.0 ā 2.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -9
- package/installer/cli.js +12 -1
- package/installer/setup-wizard.js +159 -106
- package/mcp-server/pyproject.toml +6 -5
- package/mcp-server/src/server.py +112 -25
- package/package.json +1 -1
- package/scripts/import-conversations-unified.py +269 -0
- package/scripts/import-current-conversation.py +3 -2
- package/scripts/import-live-conversation.py +5 -3
- package/scripts/import-recent-only.py +5 -1
- package/scripts/import-watcher.py +1 -1
package/README.md
CHANGED
|
@@ -2,6 +2,22 @@
|
|
|
2
2
|
|
|
3
3
|
Claude forgets everything. This fixes that.
|
|
4
4
|
|
|
5
|
+
## š Security & Privacy Notice
|
|
6
|
+
|
|
7
|
+
**v2.3.3 Security Update**: This version addresses critical security vulnerabilities. Please update immediately.
|
|
8
|
+
|
|
9
|
+
### Privacy Modes
|
|
10
|
+
- **Local Mode (Default)**: Your conversations stay on your machine. No external API calls.
|
|
11
|
+
- **Cloud Mode**: Uses Voyage AI for better search accuracy. Conversations are sent to Voyage for embedding generation.
|
|
12
|
+
|
|
13
|
+
### Security Improvements in v2.3.3
|
|
14
|
+
- ā
Removed all hardcoded API keys
|
|
15
|
+
- ā
Fixed command injection vulnerabilities
|
|
16
|
+
- ā
Patched vulnerable dependencies
|
|
17
|
+
- ā
Local embeddings by default for privacy
|
|
18
|
+
|
|
19
|
+
**Important**: If using cloud mode, review [Voyage AI's privacy policy](https://www.voyageai.com/privacy) to understand how your data is handled.
|
|
20
|
+
|
|
5
21
|
## What You Get
|
|
6
22
|
|
|
7
23
|
Ask Claude about past conversations. Get actual answers.
|
|
@@ -13,28 +29,30 @@ Your conversations become searchable. Your decisions stay remembered. Your conte
|
|
|
13
29
|
|
|
14
30
|
## Install
|
|
15
31
|
|
|
16
|
-
### Quick Start (
|
|
32
|
+
### Quick Start (Local Mode - Default)
|
|
17
33
|
```bash
|
|
18
|
-
#
|
|
19
|
-
# Sign up at https://www.voyageai.com/ - it takes 30 seconds
|
|
20
|
-
|
|
21
|
-
# Step 2: Install and run automatic setup
|
|
34
|
+
# Install and run automatic setup
|
|
22
35
|
npm install -g claude-self-reflect
|
|
23
|
-
claude-self-reflect setup
|
|
36
|
+
claude-self-reflect setup
|
|
24
37
|
|
|
25
38
|
# That's it! The setup will:
|
|
26
39
|
# ā
Configure everything automatically
|
|
27
40
|
# ā
Install the MCP in Claude Code
|
|
28
41
|
# ā
Start monitoring for new conversations
|
|
29
42
|
# ā
Verify the reflection tools work
|
|
43
|
+
# š Keep all data local - no API keys needed
|
|
30
44
|
```
|
|
31
45
|
|
|
32
|
-
###
|
|
46
|
+
### Cloud Mode (Better Search Accuracy)
|
|
33
47
|
```bash
|
|
48
|
+
# Step 1: Get your free Voyage AI key
|
|
49
|
+
# Sign up at https://www.voyageai.com/ - it takes 30 seconds
|
|
50
|
+
|
|
51
|
+
# Step 2: Install with Voyage key
|
|
34
52
|
npm install -g claude-self-reflect
|
|
35
|
-
claude-self-reflect setup --
|
|
53
|
+
claude-self-reflect setup --voyage-key=YOUR_ACTUAL_KEY_HERE
|
|
36
54
|
```
|
|
37
|
-
*Note:
|
|
55
|
+
*Note: Cloud mode provides more accurate semantic search but sends conversation data to Voyage AI for processing.*
|
|
38
56
|
|
|
39
57
|
5 minutes. Everything automatic. Just works.
|
|
40
58
|
|
package/installer/cli.js
CHANGED
|
@@ -95,8 +95,19 @@ async function doctor() {
|
|
|
95
95
|
console.log('\nš” Run "claude-self-reflect setup" to fix any issues');
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
+
const ASCII_ART = `
|
|
99
|
+
____ _____ _____ _ _____ ____ _____
|
|
100
|
+
| _ \\| ____| ___| | | ____/ ___|_ _|
|
|
101
|
+
| |_) | _| | |_ | | | _|| | | |
|
|
102
|
+
| _ <| |___| _| | |___| |__| |___ | |
|
|
103
|
+
|_| \\_\\_____|_| |_____|_____\\____| |_|
|
|
104
|
+
|
|
105
|
+
Memory that learns and forgets
|
|
106
|
+
`;
|
|
107
|
+
|
|
98
108
|
function help() {
|
|
99
|
-
console.log(
|
|
109
|
+
console.log(ASCII_ART);
|
|
110
|
+
console.log('\nClaude Self-Reflect - Perfect memory for Claude\n');
|
|
100
111
|
console.log('Usage: claude-self-reflect <command> [options]\n');
|
|
101
112
|
console.log('Commands:');
|
|
102
113
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
-
import { execSync, spawn } from 'child_process';
|
|
3
|
+
import { execSync, spawn, spawnSync } from 'child_process';
|
|
4
4
|
import { fileURLToPath } from 'url';
|
|
5
5
|
import { dirname, join } from 'path';
|
|
6
6
|
import fs from 'fs/promises';
|
|
@@ -11,20 +11,42 @@ const __filename = fileURLToPath(import.meta.url);
|
|
|
11
11
|
const __dirname = dirname(__filename);
|
|
12
12
|
const projectRoot = join(__dirname, '..');
|
|
13
13
|
|
|
14
|
+
// Safe command execution helper
|
|
15
|
+
function safeExec(command, args = [], options = {}) {
|
|
16
|
+
const result = spawnSync(command, args, {
|
|
17
|
+
...options,
|
|
18
|
+
shell: false // Never use shell to prevent injection
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
if (result.error) {
|
|
22
|
+
throw result.error;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
if (result.status !== 0) {
|
|
26
|
+
const error = new Error(`Command failed: ${command} ${args.join(' ')}`);
|
|
27
|
+
error.stdout = result.stdout;
|
|
28
|
+
error.stderr = result.stderr;
|
|
29
|
+
error.status = result.status;
|
|
30
|
+
throw error;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
return result.stdout?.toString() || '';
|
|
34
|
+
}
|
|
35
|
+
|
|
14
36
|
// Parse command line arguments
|
|
15
37
|
const args = process.argv.slice(2);
|
|
16
38
|
let voyageKey = null;
|
|
17
|
-
let localMode = false;
|
|
18
39
|
let mcpConfigured = false;
|
|
19
40
|
|
|
20
41
|
for (const arg of args) {
|
|
21
42
|
if (arg.startsWith('--voyage-key=')) {
|
|
22
43
|
voyageKey = arg.split('=')[1];
|
|
23
|
-
} else if (arg === '--local') {
|
|
24
|
-
localMode = true;
|
|
25
44
|
}
|
|
26
45
|
}
|
|
27
46
|
|
|
47
|
+
// Default to local mode unless Voyage key is provided
|
|
48
|
+
let localMode = !voyageKey;
|
|
49
|
+
|
|
28
50
|
const isInteractive = process.stdin.isTTY && process.stdout.isTTY;
|
|
29
51
|
|
|
30
52
|
const rl = isInteractive ? readline.createInterface({
|
|
@@ -43,31 +65,39 @@ const question = (query) => {
|
|
|
43
65
|
async function checkPython() {
|
|
44
66
|
console.log('\nš¦ Checking Python installation...');
|
|
45
67
|
try {
|
|
46
|
-
const version =
|
|
68
|
+
const version = safeExec('python3', ['--version']).trim();
|
|
47
69
|
console.log(`ā
Found ${version}`);
|
|
48
70
|
|
|
49
71
|
// Check if SSL module works
|
|
50
72
|
try {
|
|
51
|
-
|
|
73
|
+
safeExec('python3', ['-c', 'import ssl'], { stdio: 'pipe' });
|
|
52
74
|
return true;
|
|
53
75
|
} catch (sslError) {
|
|
54
76
|
console.log('ā ļø Python SSL module not working');
|
|
55
77
|
|
|
56
78
|
// Check if we're using pyenv
|
|
57
|
-
const whichPython =
|
|
79
|
+
const whichPython = safeExec('which', ['python3']).trim();
|
|
58
80
|
if (whichPython.includes('pyenv')) {
|
|
59
81
|
console.log('š Detected pyenv Python with broken SSL');
|
|
60
82
|
|
|
61
83
|
// Check if brew Python is available
|
|
62
84
|
try {
|
|
63
|
-
|
|
85
|
+
let brewPrefix = '';
|
|
86
|
+
for (const pythonVersion of ['python@3.11', 'python@3.10', 'python@3.12']) {
|
|
87
|
+
try {
|
|
88
|
+
brewPrefix = safeExec('brew', ['--prefix', pythonVersion]).trim();
|
|
89
|
+
if (brewPrefix) break;
|
|
90
|
+
} catch {}
|
|
91
|
+
}
|
|
64
92
|
if (brewPrefix) {
|
|
65
93
|
// Find the actual python executable
|
|
66
94
|
let pythonPath = null;
|
|
67
95
|
for (const exe of ['python3.11', 'python3.10', 'python3.12', 'python3']) {
|
|
68
96
|
try {
|
|
69
97
|
const fullPath = `${brewPrefix}/bin/${exe}`;
|
|
70
|
-
|
|
98
|
+
// Use fs.existsSync instead of shell test command
|
|
99
|
+
const { existsSync } = await import('fs');
|
|
100
|
+
if (!existsSync(fullPath)) throw new Error('File not found');
|
|
71
101
|
pythonPath = fullPath;
|
|
72
102
|
break;
|
|
73
103
|
} catch {}
|
|
@@ -77,7 +107,7 @@ async function checkPython() {
|
|
|
77
107
|
console.log(`ā
Found brew Python at ${pythonPath}`);
|
|
78
108
|
// Test if SSL works with brew Python
|
|
79
109
|
try {
|
|
80
|
-
|
|
110
|
+
safeExec(pythonPath, ['-c', 'import ssl'], { stdio: 'pipe' });
|
|
81
111
|
process.env.PYTHON_PATH = pythonPath;
|
|
82
112
|
return true;
|
|
83
113
|
} catch {
|
|
@@ -89,8 +119,8 @@ async function checkPython() {
|
|
|
89
119
|
|
|
90
120
|
console.log('\nš§ Attempting to install Python with brew...');
|
|
91
121
|
try {
|
|
92
|
-
|
|
93
|
-
const brewPython =
|
|
122
|
+
safeExec('brew', ['install', 'python@3.11'], { stdio: 'inherit' });
|
|
123
|
+
const brewPython = safeExec('brew', ['--prefix', 'python@3.11']).trim();
|
|
94
124
|
process.env.PYTHON_PATH = `${brewPython}/bin/python3`;
|
|
95
125
|
console.log('ā
Installed Python 3.11 with brew');
|
|
96
126
|
return true;
|
|
@@ -110,7 +140,7 @@ async function checkPython() {
|
|
|
110
140
|
|
|
111
141
|
async function checkDocker() {
|
|
112
142
|
try {
|
|
113
|
-
|
|
143
|
+
safeExec('docker', ['info'], { stdio: 'ignore' });
|
|
114
144
|
return true;
|
|
115
145
|
} catch {
|
|
116
146
|
return false;
|
|
@@ -152,15 +182,15 @@ async function checkQdrant() {
|
|
|
152
182
|
try {
|
|
153
183
|
// Check if a container named 'qdrant' already exists
|
|
154
184
|
try {
|
|
155
|
-
|
|
185
|
+
safeExec('docker', ['container', 'inspect', 'qdrant'], { stdio: 'ignore' });
|
|
156
186
|
console.log('Removing existing Qdrant container...');
|
|
157
|
-
|
|
187
|
+
safeExec('docker', ['rm', '-f', 'qdrant'], { stdio: 'ignore' });
|
|
158
188
|
} catch {
|
|
159
189
|
// Container doesn't exist, which is fine
|
|
160
190
|
}
|
|
161
191
|
|
|
162
192
|
console.log('Starting Qdrant...');
|
|
163
|
-
|
|
193
|
+
safeExec('docker', ['run', '-d', '--name', 'qdrant', '-p', '6333:6333', '-v', 'qdrant_storage:/qdrant/storage', 'qdrant/qdrant:latest'], { stdio: 'inherit' });
|
|
164
194
|
|
|
165
195
|
// Wait for Qdrant to be ready
|
|
166
196
|
console.log('Waiting for Qdrant to start...');
|
|
@@ -181,7 +211,9 @@ async function checkQdrant() {
|
|
|
181
211
|
console.log(` Still waiting... (${retries} seconds left)`);
|
|
182
212
|
// Check if container is still running
|
|
183
213
|
try {
|
|
184
|
-
|
|
214
|
+
// Check if container is running without using shell pipes
|
|
215
|
+
const psOutput = safeExec('docker', ['ps', '--filter', 'name=qdrant', '--format', '{{.Names}}'], { stdio: 'pipe' });
|
|
216
|
+
if (!psOutput.includes('qdrant')) throw new Error('Container not running');
|
|
185
217
|
} catch {
|
|
186
218
|
console.log('ā Qdrant container stopped unexpectedly');
|
|
187
219
|
return false;
|
|
@@ -226,11 +258,22 @@ async function setupPythonEnvironment() {
|
|
|
226
258
|
console.log('Creating virtual environment...');
|
|
227
259
|
const pythonCmd = process.env.PYTHON_PATH || 'python3';
|
|
228
260
|
try {
|
|
229
|
-
|
|
261
|
+
// Use spawn with proper path handling instead of shell execution
|
|
262
|
+
const { spawnSync } = require('child_process');
|
|
263
|
+
const result = spawnSync(pythonCmd, ['-m', 'venv', 'venv'], {
|
|
264
|
+
cwd: mcpPath,
|
|
265
|
+
stdio: 'inherit'
|
|
266
|
+
});
|
|
267
|
+
if (result.error) throw result.error;
|
|
230
268
|
} catch (venvError) {
|
|
231
269
|
console.log('ā ļø Failed to create venv with python3, trying python...');
|
|
232
270
|
try {
|
|
233
|
-
|
|
271
|
+
const { spawnSync } = require('child_process');
|
|
272
|
+
const result = spawnSync('python', ['-m', 'venv', 'venv'], {
|
|
273
|
+
cwd: mcpPath,
|
|
274
|
+
stdio: 'inherit'
|
|
275
|
+
});
|
|
276
|
+
if (result.error) throw result.error;
|
|
234
277
|
} catch {
|
|
235
278
|
console.log('ā Failed to create virtual environment');
|
|
236
279
|
console.log('š Fix: Install python3-venv package');
|
|
@@ -241,19 +284,21 @@ async function setupPythonEnvironment() {
|
|
|
241
284
|
}
|
|
242
285
|
}
|
|
243
286
|
|
|
244
|
-
//
|
|
287
|
+
// Setup paths for virtual environment
|
|
245
288
|
console.log('Setting up pip in virtual environment...');
|
|
246
|
-
const
|
|
247
|
-
? 'venv
|
|
248
|
-
: '
|
|
289
|
+
const venvPython = process.platform === 'win32'
|
|
290
|
+
? join(mcpPath, 'venv', 'Scripts', 'python.exe')
|
|
291
|
+
: join(mcpPath, 'venv', 'bin', 'python');
|
|
292
|
+
const venvPip = process.platform === 'win32'
|
|
293
|
+
? join(mcpPath, 'venv', 'Scripts', 'pip.exe')
|
|
294
|
+
: join(mcpPath, 'venv', 'bin', 'pip');
|
|
249
295
|
|
|
250
296
|
// First, try to install certifi to help with SSL issues
|
|
251
297
|
console.log('Installing certificate handler...');
|
|
252
298
|
try {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
});
|
|
299
|
+
safeExec(venvPip, [
|
|
300
|
+
'install', '--trusted-host', 'pypi.org', '--trusted-host', 'files.pythonhosted.org', 'certifi'
|
|
301
|
+
], { cwd: mcpPath, stdio: 'pipe' });
|
|
257
302
|
} catch {
|
|
258
303
|
// Continue even if certifi fails
|
|
259
304
|
}
|
|
@@ -261,10 +306,9 @@ async function setupPythonEnvironment() {
|
|
|
261
306
|
// Upgrade pip and install wheel first
|
|
262
307
|
try {
|
|
263
308
|
// Use --no-cache-dir and --timeout to fail faster
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
});
|
|
309
|
+
safeExec(venvPython, [
|
|
310
|
+
'-m', 'pip', 'install', '--no-cache-dir', '--timeout', '5', '--retries', '1', '--upgrade', 'pip', 'wheel', 'setuptools'
|
|
311
|
+
], { cwd: mcpPath, stdio: 'pipe' });
|
|
268
312
|
console.log('ā
Pip upgraded successfully');
|
|
269
313
|
} catch {
|
|
270
314
|
// If upgrade fails due to SSL, skip it and continue
|
|
@@ -274,10 +318,9 @@ async function setupPythonEnvironment() {
|
|
|
274
318
|
// Now install dependencies
|
|
275
319
|
console.log('Installing MCP server dependencies...');
|
|
276
320
|
try {
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
});
|
|
321
|
+
safeExec(venvPip, [
|
|
322
|
+
'install', '--no-cache-dir', '--timeout', '10', '--retries', '1', '-e', '.'
|
|
323
|
+
], { cwd: mcpPath, stdio: 'pipe' });
|
|
281
324
|
console.log('ā
MCP server dependencies installed');
|
|
282
325
|
} catch (error) {
|
|
283
326
|
// Check for SSL errors
|
|
@@ -286,29 +329,31 @@ async function setupPythonEnvironment() {
|
|
|
286
329
|
console.log('ā ļø SSL error detected. Attempting automatic fix...');
|
|
287
330
|
|
|
288
331
|
// Try different approaches to fix SSL
|
|
289
|
-
const
|
|
332
|
+
const sslFixes = [
|
|
290
333
|
{
|
|
291
334
|
name: 'Using trusted host flags',
|
|
292
|
-
|
|
335
|
+
install: () => safeExec(venvPip, [
|
|
336
|
+
'install', '--trusted-host', 'pypi.org', '--trusted-host', 'files.pythonhosted.org',
|
|
337
|
+
'--no-cache-dir', '-e', '.'
|
|
338
|
+
], { cwd: mcpPath, stdio: 'pipe' })
|
|
293
339
|
},
|
|
294
340
|
{
|
|
295
341
|
name: 'Using index-url without SSL',
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
342
|
+
install: () => {
|
|
343
|
+
safeExec(venvPip, ['config', 'set', 'global.index-url', 'https://pypi.org/simple/'],
|
|
344
|
+
{ cwd: mcpPath, stdio: 'pipe' });
|
|
345
|
+
safeExec(venvPip, ['config', 'set', 'global.trusted-host', 'pypi.org files.pythonhosted.org'],
|
|
346
|
+
{ cwd: mcpPath, stdio: 'pipe' });
|
|
347
|
+
return safeExec(venvPip, ['install', '--no-cache-dir', '-e', '.'],
|
|
348
|
+
{ cwd: mcpPath, stdio: 'pipe' });
|
|
349
|
+
}
|
|
301
350
|
}
|
|
302
351
|
];
|
|
303
352
|
|
|
304
|
-
for (const fix of
|
|
353
|
+
for (const fix of sslFixes) {
|
|
305
354
|
console.log(`\n Trying: ${fix.name}...`);
|
|
306
355
|
try {
|
|
307
|
-
|
|
308
|
-
stdio: 'pipe',
|
|
309
|
-
shell: true,
|
|
310
|
-
env: { ...process.env, PYTHONWARNINGS: 'ignore:Unverified HTTPS request' }
|
|
311
|
-
});
|
|
356
|
+
fix.install();
|
|
312
357
|
console.log(' ā
Success! Dependencies installed using workaround');
|
|
313
358
|
return true;
|
|
314
359
|
} catch (e) {
|
|
@@ -327,17 +372,16 @@ async function setupPythonEnvironment() {
|
|
|
327
372
|
// Install script dependencies
|
|
328
373
|
console.log('Installing import script dependencies...');
|
|
329
374
|
try {
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
});
|
|
375
|
+
safeExec(venvPip, [
|
|
376
|
+
'install', '-r', join(scriptsPath, 'requirements.txt')
|
|
377
|
+
], { cwd: mcpPath, stdio: 'inherit' });
|
|
334
378
|
} catch (error) {
|
|
335
379
|
// Try with trusted host if SSL error
|
|
336
380
|
try {
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
});
|
|
381
|
+
safeExec(venvPip, [
|
|
382
|
+
'install', '--trusted-host', 'pypi.org', '--trusted-host', 'files.pythonhosted.org',
|
|
383
|
+
'-r', join(scriptsPath, 'requirements.txt')
|
|
384
|
+
], { cwd: mcpPath, stdio: 'inherit' });
|
|
341
385
|
} catch {
|
|
342
386
|
console.log('ā ļø Could not install script dependencies automatically');
|
|
343
387
|
console.log(' You may need to install them manually later');
|
|
@@ -446,6 +490,9 @@ async function configureEnvironment() {
|
|
|
446
490
|
if (!envContent.includes('DECAY_SCALE_DAYS=')) {
|
|
447
491
|
envContent += 'DECAY_SCALE_DAYS=90\n';
|
|
448
492
|
}
|
|
493
|
+
if (!envContent.includes('PREFER_LOCAL_EMBEDDINGS=')) {
|
|
494
|
+
envContent += `PREFER_LOCAL_EMBEDDINGS=${localMode ? 'true' : 'false'}\n`;
|
|
495
|
+
}
|
|
449
496
|
|
|
450
497
|
await fs.writeFile(envPath, envContent.trim() + '\n');
|
|
451
498
|
console.log('ā
Environment file created/updated');
|
|
@@ -460,7 +507,7 @@ async function setupClaude() {
|
|
|
460
507
|
|
|
461
508
|
// Check if Claude CLI is available
|
|
462
509
|
try {
|
|
463
|
-
|
|
510
|
+
safeExec('which', ['claude'], { stdio: 'ignore' });
|
|
464
511
|
|
|
465
512
|
// Try to add the MCP automatically
|
|
466
513
|
try {
|
|
@@ -479,7 +526,16 @@ async function setupClaude() {
|
|
|
479
526
|
? `claude mcp add claude-self-reflect "${runScript}" -e QDRANT_URL="http://localhost:6333"`
|
|
480
527
|
: `claude mcp add claude-self-reflect "${runScript}" -e VOYAGE_KEY="${voyageKeyValue}" -e QDRANT_URL="http://localhost:6333"`;
|
|
481
528
|
|
|
482
|
-
|
|
529
|
+
// Parse the MCP command properly
|
|
530
|
+
const mcpArgs = ['mcp', 'add', 'claude-self-reflect', runScript];
|
|
531
|
+
if (voyageKeyValue) {
|
|
532
|
+
mcpArgs.push('-e', `VOYAGE_KEY=${voyageKeyValue}`);
|
|
533
|
+
}
|
|
534
|
+
mcpArgs.push('-e', 'QDRANT_URL=http://localhost:6333');
|
|
535
|
+
if (localMode) {
|
|
536
|
+
mcpArgs.push('-e', 'PREFER_LOCAL_EMBEDDINGS=true');
|
|
537
|
+
}
|
|
538
|
+
safeExec('claude', mcpArgs, { stdio: 'inherit' });
|
|
483
539
|
console.log('ā
MCP added successfully!');
|
|
484
540
|
console.log('\nā ļø You may need to restart Claude Code for the changes to take effect.');
|
|
485
541
|
|
|
@@ -540,7 +596,8 @@ async function showPreSetupInstructions() {
|
|
|
540
596
|
console.log('š Before we begin, you\'ll need:');
|
|
541
597
|
console.log(' 1. Docker Desktop installed and running');
|
|
542
598
|
console.log(' 2. Python 3.10 or higher');
|
|
543
|
-
console.log('
|
|
599
|
+
console.log('\nš By default, we use local embeddings (private but less accurate)');
|
|
600
|
+
console.log(' For better accuracy, run with: --voyage-key=<your-key>\n');
|
|
544
601
|
|
|
545
602
|
if (isInteractive) {
|
|
546
603
|
await question('Press Enter to continue...');
|
|
@@ -549,12 +606,7 @@ async function showPreSetupInstructions() {
|
|
|
549
606
|
|
|
550
607
|
async function importConversations() {
|
|
551
608
|
console.log('\nš Import Claude Conversations...');
|
|
552
|
-
|
|
553
|
-
// Skip import in local mode
|
|
554
|
-
if (localMode) {
|
|
555
|
-
console.log('š Skipping import in local mode (no API key for embeddings)');
|
|
556
|
-
return;
|
|
557
|
-
}
|
|
609
|
+
console.log(localMode ? 'š Using local embeddings for import' : 'š Using Voyage AI embeddings');
|
|
558
610
|
|
|
559
611
|
// Check if Claude logs directory exists
|
|
560
612
|
const logsDir = join(process.env.HOME || process.env.USERPROFILE, '.claude', 'projects');
|
|
@@ -694,7 +746,7 @@ async function showSystemDashboard() {
|
|
|
694
746
|
|
|
695
747
|
// Docker status
|
|
696
748
|
try {
|
|
697
|
-
|
|
749
|
+
safeExec('docker', ['info'], { stdio: 'ignore' });
|
|
698
750
|
status.docker = true;
|
|
699
751
|
} catch {}
|
|
700
752
|
|
|
@@ -710,7 +762,7 @@ async function showSystemDashboard() {
|
|
|
710
762
|
// Python status
|
|
711
763
|
try {
|
|
712
764
|
const pythonCmd = process.env.PYTHON_PATH || 'python3';
|
|
713
|
-
|
|
765
|
+
safeExec(pythonCmd, ['--version'], { stdio: 'ignore' });
|
|
714
766
|
status.python = true;
|
|
715
767
|
} catch {}
|
|
716
768
|
|
|
@@ -773,9 +825,9 @@ async function showSystemDashboard() {
|
|
|
773
825
|
status.watcherErrors = [];
|
|
774
826
|
status.lastImportTime = null;
|
|
775
827
|
try {
|
|
776
|
-
const watcherLogs =
|
|
777
|
-
|
|
778
|
-
})
|
|
828
|
+
const watcherLogs = safeExec('docker', [
|
|
829
|
+
'logs', 'claude-reflection-watcher', '--tail', '50'
|
|
830
|
+
], { encoding: 'utf-8' });
|
|
779
831
|
|
|
780
832
|
// Check for recent errors
|
|
781
833
|
const errorMatches = watcherLogs.match(/ERROR.*Import failed.*/g);
|
|
@@ -800,10 +852,9 @@ async function showSystemDashboard() {
|
|
|
800
852
|
|
|
801
853
|
// Check if watcher is running via Docker
|
|
802
854
|
try {
|
|
803
|
-
const dockerStatus =
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
}).toString().trim();
|
|
855
|
+
const dockerStatus = safeExec('docker', [
|
|
856
|
+
'ps', '--filter', 'name=claude-reflection-watcher', '--format', '{{.Names}}'
|
|
857
|
+
], { cwd: projectRoot, encoding: 'utf-8' }).trim();
|
|
807
858
|
|
|
808
859
|
if (dockerStatus.includes('watcher')) {
|
|
809
860
|
status.watcherRunning = true;
|
|
@@ -901,11 +952,8 @@ async function setupWatcher() {
|
|
|
901
952
|
await fs.access(watcherScript);
|
|
902
953
|
console.log('ā
Watcher script found');
|
|
903
954
|
|
|
904
|
-
//
|
|
905
|
-
|
|
906
|
-
console.log('š Skipping watcher in local mode');
|
|
907
|
-
return;
|
|
908
|
-
}
|
|
955
|
+
// Watcher works with both local and cloud embeddings
|
|
956
|
+
console.log(localMode ? 'š Watcher will use local embeddings' : 'š Watcher will use Voyage AI embeddings');
|
|
909
957
|
|
|
910
958
|
// Ask if user wants to enable watcher
|
|
911
959
|
let enableWatcher = 'y';
|
|
@@ -933,20 +981,26 @@ async function setupWatcher() {
|
|
|
933
981
|
console.log('š§¹ Cleaning up existing containers...');
|
|
934
982
|
try {
|
|
935
983
|
// Stop all claude-reflection containers
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
984
|
+
try {
|
|
985
|
+
safeExec('docker', ['compose', 'down'], {
|
|
986
|
+
cwd: projectRoot,
|
|
987
|
+
stdio: 'pipe'
|
|
988
|
+
});
|
|
989
|
+
} catch {}
|
|
940
990
|
|
|
941
991
|
// Also stop any standalone containers
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
992
|
+
try {
|
|
993
|
+
safeExec('docker', ['stop', 'claude-reflection-watcher', 'claude-reflection-qdrant', 'qdrant'], {
|
|
994
|
+
stdio: 'pipe'
|
|
995
|
+
});
|
|
996
|
+
} catch {}
|
|
945
997
|
|
|
946
998
|
// Remove them
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
999
|
+
try {
|
|
1000
|
+
safeExec('docker', ['rm', 'claude-reflection-watcher', 'claude-reflection-qdrant', 'qdrant'], {
|
|
1001
|
+
stdio: 'pipe'
|
|
1002
|
+
});
|
|
1003
|
+
} catch {}
|
|
950
1004
|
|
|
951
1005
|
// Wait a moment for cleanup
|
|
952
1006
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
@@ -954,7 +1008,7 @@ async function setupWatcher() {
|
|
|
954
1008
|
|
|
955
1009
|
// Start both services with compose
|
|
956
1010
|
console.log('š Starting Qdrant and Watcher services...');
|
|
957
|
-
|
|
1011
|
+
safeExec('docker', ['compose', '--profile', 'watch', 'up', '-d'], {
|
|
958
1012
|
cwd: projectRoot,
|
|
959
1013
|
stdio: 'pipe' // Use pipe to capture output
|
|
960
1014
|
});
|
|
@@ -964,10 +1018,9 @@ async function setupWatcher() {
|
|
|
964
1018
|
|
|
965
1019
|
// Check container status
|
|
966
1020
|
try {
|
|
967
|
-
const psOutput =
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
});
|
|
1021
|
+
const psOutput = safeExec('docker', [
|
|
1022
|
+
'ps', '--filter', 'name=claude-reflection', '--format', 'table {{.Names}}\t{{.Status}}'
|
|
1023
|
+
], { cwd: projectRoot, encoding: 'utf8' });
|
|
971
1024
|
|
|
972
1025
|
const qdrantReady = psOutput.includes('claude-reflection-qdrant') && psOutput.includes('Up');
|
|
973
1026
|
const watcherReady = psOutput.includes('claude-reflection-watcher') && psOutput.includes('Up');
|
|
@@ -1033,7 +1086,7 @@ async function verifyMCP() {
|
|
|
1033
1086
|
|
|
1034
1087
|
try {
|
|
1035
1088
|
// Check if MCP is listed
|
|
1036
|
-
const mcpList =
|
|
1089
|
+
const mcpList = safeExec('claude', ['mcp', 'list'], { encoding: 'utf8' });
|
|
1037
1090
|
if (!mcpList.includes('claude-self-reflect')) {
|
|
1038
1091
|
console.log('ā MCP not found in Claude Code');
|
|
1039
1092
|
return;
|
|
@@ -1103,10 +1156,15 @@ asyncio.run(test_mcp())
|
|
|
1103
1156
|
// Run the test
|
|
1104
1157
|
console.log('\nš§Ŗ Testing MCP functionality...');
|
|
1105
1158
|
try {
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1159
|
+
// Create test script that activates venv and runs test
|
|
1160
|
+
const testScriptPath = join(projectRoot, 'run-test.sh');
|
|
1161
|
+
const testScript = `#!/bin/bash\nsource mcp-server/venv/bin/activate\npython test-mcp.py`;
|
|
1162
|
+
await fs.writeFile(testScriptPath, testScript, { mode: 0o755 });
|
|
1163
|
+
const testResult = safeExec('bash', [testScriptPath], {
|
|
1164
|
+
cwd: projectRoot,
|
|
1165
|
+
encoding: 'utf8'
|
|
1109
1166
|
});
|
|
1167
|
+
await fs.unlink(testScriptPath);
|
|
1110
1168
|
console.log(testResult);
|
|
1111
1169
|
|
|
1112
1170
|
// Clean up test script
|
|
@@ -1151,14 +1209,9 @@ async function main() {
|
|
|
1151
1209
|
}
|
|
1152
1210
|
}
|
|
1153
1211
|
|
|
1154
|
-
//
|
|
1155
|
-
if (!isInteractive
|
|
1156
|
-
console.log('
|
|
1157
|
-
console.log('Usage:');
|
|
1158
|
-
console.log(' claude-self-reflect setup --voyage-key=<your-key>');
|
|
1159
|
-
console.log(' claude-self-reflect setup --local\n');
|
|
1160
|
-
console.log('Get your free API key at: https://www.voyageai.com/');
|
|
1161
|
-
process.exit(1);
|
|
1212
|
+
// In non-interactive mode, just use defaults (local mode unless key provided)
|
|
1213
|
+
if (!isInteractive) {
|
|
1214
|
+
console.log(voyageKey ? 'š Using Voyage AI embeddings' : 'š Using local embeddings for privacy');
|
|
1162
1215
|
}
|
|
1163
1216
|
|
|
1164
1217
|
await showPreSetupInstructions();
|
|
@@ -9,11 +9,12 @@ authors = [
|
|
|
9
9
|
]
|
|
10
10
|
dependencies = [
|
|
11
11
|
"fastmcp>=0.0.7",
|
|
12
|
-
"qdrant-client>=1.7.0",
|
|
13
|
-
"voyageai>=0.1.0",
|
|
14
|
-
"python-dotenv>=1.0.0",
|
|
15
|
-
"pydantic>=2.0.0",
|
|
16
|
-
"pydantic-settings>=2.0.0",
|
|
12
|
+
"qdrant-client>=1.7.0,<2.0.0",
|
|
13
|
+
"voyageai>=0.1.0,<1.0.0",
|
|
14
|
+
"python-dotenv>=1.0.0,<2.0.0",
|
|
15
|
+
"pydantic>=2.9.2,<3.0.0", # Pin to avoid CVE-2024-3772
|
|
16
|
+
"pydantic-settings>=2.0.0,<3.0.0",
|
|
17
|
+
"fastembed>=0.4.0,<1.0.0",
|
|
17
18
|
]
|
|
18
19
|
|
|
19
20
|
[project.scripts]
|
package/mcp-server/src/server.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"""Claude Reflect MCP Server with Memory Decay."""
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
|
+
import asyncio
|
|
4
5
|
from pathlib import Path
|
|
5
6
|
from typing import Any, Optional, List, Dict, Union
|
|
6
7
|
from datetime import datetime
|
|
@@ -30,8 +31,25 @@ DECAY_WEIGHT = float(os.getenv('DECAY_WEIGHT', '0.3'))
|
|
|
30
31
|
DECAY_SCALE_DAYS = float(os.getenv('DECAY_SCALE_DAYS', '90'))
|
|
31
32
|
USE_NATIVE_DECAY = os.getenv('USE_NATIVE_DECAY', 'false').lower() == 'true'
|
|
32
33
|
|
|
33
|
-
#
|
|
34
|
-
|
|
34
|
+
# Embedding configuration
|
|
35
|
+
PREFER_LOCAL_EMBEDDINGS = os.getenv('PREFER_LOCAL_EMBEDDINGS', 'false').lower() == 'true'
|
|
36
|
+
EMBEDDING_MODEL = os.getenv('EMBEDDING_MODEL', 'sentence-transformers/all-MiniLM-L6-v2')
|
|
37
|
+
|
|
38
|
+
# Initialize Voyage AI client (only if not using local embeddings)
|
|
39
|
+
voyage_client = None
|
|
40
|
+
if not PREFER_LOCAL_EMBEDDINGS and VOYAGE_API_KEY:
|
|
41
|
+
voyage_client = voyageai.Client(api_key=VOYAGE_API_KEY)
|
|
42
|
+
|
|
43
|
+
# Initialize local embedding model if needed
|
|
44
|
+
local_embedding_model = None
|
|
45
|
+
if PREFER_LOCAL_EMBEDDINGS or not VOYAGE_API_KEY:
|
|
46
|
+
try:
|
|
47
|
+
from fastembed import TextEmbedding
|
|
48
|
+
local_embedding_model = TextEmbedding(model_name=EMBEDDING_MODEL)
|
|
49
|
+
print(f"[DEBUG] Initialized local embedding model: {EMBEDDING_MODEL}")
|
|
50
|
+
except ImportError:
|
|
51
|
+
print("[ERROR] FastEmbed not available. Install with: pip install fastembed")
|
|
52
|
+
raise
|
|
35
53
|
|
|
36
54
|
# Debug environment loading
|
|
37
55
|
print(f"[DEBUG] Environment variables loaded:")
|
|
@@ -39,6 +57,8 @@ print(f"[DEBUG] ENABLE_MEMORY_DECAY: {ENABLE_MEMORY_DECAY}")
|
|
|
39
57
|
print(f"[DEBUG] USE_NATIVE_DECAY: {USE_NATIVE_DECAY}")
|
|
40
58
|
print(f"[DEBUG] DECAY_WEIGHT: {DECAY_WEIGHT}")
|
|
41
59
|
print(f"[DEBUG] DECAY_SCALE_DAYS: {DECAY_SCALE_DAYS}")
|
|
60
|
+
print(f"[DEBUG] PREFER_LOCAL_EMBEDDINGS: {PREFER_LOCAL_EMBEDDINGS}")
|
|
61
|
+
print(f"[DEBUG] EMBEDDING_MODEL: {EMBEDDING_MODEL}")
|
|
42
62
|
print(f"[DEBUG] env_path: {env_path}")
|
|
43
63
|
|
|
44
64
|
|
|
@@ -63,22 +83,50 @@ mcp = FastMCP(
|
|
|
63
83
|
# Create Qdrant client
|
|
64
84
|
qdrant_client = AsyncQdrantClient(url=QDRANT_URL)
|
|
65
85
|
|
|
66
|
-
async def
|
|
67
|
-
"""Get all Voyage
|
|
86
|
+
async def get_all_collections() -> List[str]:
|
|
87
|
+
"""Get all collections (both Voyage and local)."""
|
|
68
88
|
collections = await qdrant_client.get_collections()
|
|
69
|
-
|
|
89
|
+
# Support both _voyage and _local collections, plus reflections
|
|
90
|
+
return [c.name for c in collections.collections
|
|
91
|
+
if c.name.endswith('_voyage') or c.name.endswith('_local') or c.name.startswith('reflections')]
|
|
70
92
|
|
|
71
93
|
async def generate_embedding(text: str) -> List[float]:
|
|
72
|
-
"""Generate embedding using
|
|
73
|
-
if not voyage_client:
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
94
|
+
"""Generate embedding using configured provider."""
|
|
95
|
+
if PREFER_LOCAL_EMBEDDINGS or not voyage_client:
|
|
96
|
+
# Use local embeddings
|
|
97
|
+
if not local_embedding_model:
|
|
98
|
+
raise ValueError("Local embedding model not initialized")
|
|
99
|
+
|
|
100
|
+
# Run in executor since fastembed is synchronous
|
|
101
|
+
loop = asyncio.get_event_loop()
|
|
102
|
+
embeddings = await loop.run_in_executor(
|
|
103
|
+
None, lambda: list(local_embedding_model.embed([text]))
|
|
104
|
+
)
|
|
105
|
+
return embeddings[0].tolist()
|
|
106
|
+
else:
|
|
107
|
+
# Use Voyage AI
|
|
108
|
+
result = voyage_client.embed(
|
|
109
|
+
texts=[text],
|
|
110
|
+
model="voyage-3-large",
|
|
111
|
+
input_type="query"
|
|
112
|
+
)
|
|
113
|
+
return result.embeddings[0]
|
|
114
|
+
|
|
115
|
+
def get_embedding_dimension() -> int:
|
|
116
|
+
"""Get the dimension of embeddings based on the provider."""
|
|
117
|
+
if PREFER_LOCAL_EMBEDDINGS or not voyage_client:
|
|
118
|
+
# all-MiniLM-L6-v2 produces 384-dimensional embeddings
|
|
119
|
+
return 384
|
|
120
|
+
else:
|
|
121
|
+
# voyage-3-large produces 1024-dimensional embeddings
|
|
122
|
+
return 1024
|
|
123
|
+
|
|
124
|
+
def get_collection_suffix() -> str:
|
|
125
|
+
"""Get the collection suffix based on embedding provider."""
|
|
126
|
+
if PREFER_LOCAL_EMBEDDINGS or not voyage_client:
|
|
127
|
+
return "_local"
|
|
128
|
+
else:
|
|
129
|
+
return "_voyage"
|
|
82
130
|
|
|
83
131
|
# Register tools
|
|
84
132
|
@mcp.tool()
|
|
@@ -115,17 +163,18 @@ async def reflect_on_past(
|
|
|
115
163
|
# Generate embedding
|
|
116
164
|
query_embedding = await generate_embedding(query)
|
|
117
165
|
|
|
118
|
-
# Get all
|
|
119
|
-
|
|
120
|
-
if not
|
|
166
|
+
# Get all collections
|
|
167
|
+
all_collections = await get_all_collections()
|
|
168
|
+
if not all_collections:
|
|
121
169
|
return "No conversation collections found. Please import conversations first."
|
|
122
170
|
|
|
123
|
-
await ctx.debug(f"Searching across {len(
|
|
171
|
+
await ctx.debug(f"Searching across {len(all_collections)} collections")
|
|
172
|
+
await ctx.debug(f"Using {'local' if PREFER_LOCAL_EMBEDDINGS or not voyage_client else 'Voyage AI'} embeddings")
|
|
124
173
|
|
|
125
174
|
all_results = []
|
|
126
175
|
|
|
127
176
|
# Search each collection
|
|
128
|
-
for collection_name in
|
|
177
|
+
for collection_name in all_collections:
|
|
129
178
|
try:
|
|
130
179
|
if should_use_decay and USE_NATIVE_DECAY:
|
|
131
180
|
# Use native Qdrant decay
|
|
@@ -179,7 +228,7 @@ async def reflect_on_past(
|
|
|
179
228
|
timestamp=point.payload.get('timestamp', datetime.now().isoformat()),
|
|
180
229
|
role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
|
|
181
230
|
excerpt=(point.payload.get('text', '')[:500] + '...'),
|
|
182
|
-
project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '')),
|
|
231
|
+
project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', '')),
|
|
183
232
|
conversation_id=point.payload.get('conversation_id'),
|
|
184
233
|
collection_name=collection_name
|
|
185
234
|
))
|
|
@@ -240,7 +289,7 @@ async def reflect_on_past(
|
|
|
240
289
|
timestamp=point.payload.get('timestamp', datetime.now().isoformat()),
|
|
241
290
|
role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
|
|
242
291
|
excerpt=(point.payload.get('text', '')[:500] + '...'),
|
|
243
|
-
project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '')),
|
|
292
|
+
project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', '')),
|
|
244
293
|
conversation_id=point.payload.get('conversation_id'),
|
|
245
294
|
collection_name=collection_name
|
|
246
295
|
))
|
|
@@ -261,7 +310,7 @@ async def reflect_on_past(
|
|
|
261
310
|
timestamp=point.payload.get('timestamp', datetime.now().isoformat()),
|
|
262
311
|
role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
|
|
263
312
|
excerpt=(point.payload.get('text', '')[:500] + '...'),
|
|
264
|
-
project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '')),
|
|
313
|
+
project_name=point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', '')),
|
|
265
314
|
conversation_id=point.payload.get('conversation_id'),
|
|
266
315
|
collection_name=collection_name
|
|
267
316
|
))
|
|
@@ -302,8 +351,46 @@ async def store_reflection(
|
|
|
302
351
|
"""Store an important insight or reflection for future reference."""
|
|
303
352
|
|
|
304
353
|
try:
|
|
305
|
-
#
|
|
306
|
-
|
|
354
|
+
# Create reflections collection name
|
|
355
|
+
collection_name = f"reflections{get_collection_suffix()}"
|
|
356
|
+
|
|
357
|
+
# Ensure collection exists
|
|
358
|
+
try:
|
|
359
|
+
collection_info = await qdrant_client.get_collection(collection_name)
|
|
360
|
+
except:
|
|
361
|
+
# Create collection if it doesn't exist
|
|
362
|
+
await qdrant_client.create_collection(
|
|
363
|
+
collection_name=collection_name,
|
|
364
|
+
vectors_config=VectorParams(
|
|
365
|
+
size=get_embedding_dimension(),
|
|
366
|
+
distance=Distance.COSINE
|
|
367
|
+
)
|
|
368
|
+
)
|
|
369
|
+
await ctx.debug(f"Created reflections collection: {collection_name}")
|
|
370
|
+
|
|
371
|
+
# Generate embedding for the reflection
|
|
372
|
+
embedding = await generate_embedding(content)
|
|
373
|
+
|
|
374
|
+
# Create point with metadata
|
|
375
|
+
point_id = datetime.now().timestamp()
|
|
376
|
+
point = PointStruct(
|
|
377
|
+
id=int(point_id),
|
|
378
|
+
vector=embedding,
|
|
379
|
+
payload={
|
|
380
|
+
"text": content,
|
|
381
|
+
"tags": tags,
|
|
382
|
+
"timestamp": datetime.now().isoformat(),
|
|
383
|
+
"type": "reflection",
|
|
384
|
+
"role": "user_reflection"
|
|
385
|
+
}
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
# Store in Qdrant
|
|
389
|
+
await qdrant_client.upsert(
|
|
390
|
+
collection_name=collection_name,
|
|
391
|
+
points=[point]
|
|
392
|
+
)
|
|
393
|
+
|
|
307
394
|
tags_str = ', '.join(tags) if tags else 'none'
|
|
308
395
|
return f"Reflection stored successfully with tags: {tags_str}"
|
|
309
396
|
|
package/package.json
CHANGED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Unified import script that supports both local and Voyage AI embeddings.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
import json
|
|
9
|
+
import glob
|
|
10
|
+
import hashlib
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from typing import List, Dict, Any
|
|
13
|
+
import logging
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from qdrant_client import QdrantClient
|
|
17
|
+
from qdrant_client.models import (
|
|
18
|
+
VectorParams, Distance, PointStruct,
|
|
19
|
+
Filter, FieldCondition, MatchValue
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
# Configuration
|
|
23
|
+
QDRANT_URL = os.getenv("QDRANT_URL", "http://localhost:6333")
|
|
24
|
+
LOGS_DIR = os.getenv("LOGS_DIR", "/logs")
|
|
25
|
+
STATE_FILE = os.getenv("STATE_FILE", "/config/imported-files.json")
|
|
26
|
+
BATCH_SIZE = int(os.getenv("BATCH_SIZE", "100"))
|
|
27
|
+
PREFER_LOCAL_EMBEDDINGS = os.getenv("PREFER_LOCAL_EMBEDDINGS", "false").lower() == "true"
|
|
28
|
+
VOYAGE_API_KEY = os.getenv("VOYAGE_KEY")
|
|
29
|
+
|
|
30
|
+
# Set up logging
|
|
31
|
+
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
32
|
+
logger = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
# Initialize embedding provider
|
|
35
|
+
embedding_provider = None
|
|
36
|
+
embedding_dimension = None
|
|
37
|
+
collection_suffix = None
|
|
38
|
+
|
|
39
|
+
if PREFER_LOCAL_EMBEDDINGS or not VOYAGE_API_KEY:
|
|
40
|
+
# Use local embeddings
|
|
41
|
+
logger.info("Using local embeddings (fastembed)")
|
|
42
|
+
from fastembed import TextEmbedding
|
|
43
|
+
embedding_provider = TextEmbedding("sentence-transformers/all-MiniLM-L6-v2")
|
|
44
|
+
embedding_dimension = 384
|
|
45
|
+
collection_suffix = "_local"
|
|
46
|
+
else:
|
|
47
|
+
# Use Voyage AI
|
|
48
|
+
logger.info("Using Voyage AI embeddings")
|
|
49
|
+
import voyageai
|
|
50
|
+
voyage_client = voyageai.Client(api_key=VOYAGE_API_KEY)
|
|
51
|
+
embedding_dimension = 1024
|
|
52
|
+
collection_suffix = "_voyage"
|
|
53
|
+
|
|
54
|
+
# Initialize Qdrant client
|
|
55
|
+
client = QdrantClient(url=QDRANT_URL)
|
|
56
|
+
|
|
57
|
+
def generate_embeddings(texts: List[str]) -> List[List[float]]:
|
|
58
|
+
"""Generate embeddings for a list of texts."""
|
|
59
|
+
if PREFER_LOCAL_EMBEDDINGS or not VOYAGE_API_KEY:
|
|
60
|
+
# Local embeddings using FastEmbed
|
|
61
|
+
embeddings = list(embedding_provider.passage_embed(texts))
|
|
62
|
+
return [embedding.tolist() for embedding in embeddings]
|
|
63
|
+
else:
|
|
64
|
+
# Voyage AI embeddings
|
|
65
|
+
result = voyage_client.embed(
|
|
66
|
+
texts=texts,
|
|
67
|
+
model="voyage-3-large",
|
|
68
|
+
input_type="document"
|
|
69
|
+
)
|
|
70
|
+
return result.embeddings
|
|
71
|
+
|
|
72
|
+
def chunk_conversation(messages: List[Dict[str, Any]], chunk_size: int = 10) -> List[Dict[str, Any]]:
|
|
73
|
+
"""Chunk conversation into smaller segments."""
|
|
74
|
+
chunks = []
|
|
75
|
+
|
|
76
|
+
for i in range(0, len(messages), chunk_size):
|
|
77
|
+
chunk_messages = messages[i:i + chunk_size]
|
|
78
|
+
|
|
79
|
+
# Extract text content
|
|
80
|
+
texts = []
|
|
81
|
+
for msg in chunk_messages:
|
|
82
|
+
role = msg.get("role", "unknown")
|
|
83
|
+
content = msg.get("content", "")
|
|
84
|
+
|
|
85
|
+
if isinstance(content, list):
|
|
86
|
+
# Handle structured content
|
|
87
|
+
text_parts = []
|
|
88
|
+
for item in content:
|
|
89
|
+
if isinstance(item, dict) and item.get("type") == "text":
|
|
90
|
+
text_parts.append(item.get("text", ""))
|
|
91
|
+
elif isinstance(item, str):
|
|
92
|
+
text_parts.append(item)
|
|
93
|
+
content = " ".join(text_parts)
|
|
94
|
+
|
|
95
|
+
if content:
|
|
96
|
+
texts.append(f"{role.upper()}: {content}")
|
|
97
|
+
|
|
98
|
+
if texts:
|
|
99
|
+
chunks.append({
|
|
100
|
+
"text": "\n".join(texts),
|
|
101
|
+
"messages": chunk_messages,
|
|
102
|
+
"chunk_index": i // chunk_size,
|
|
103
|
+
"start_role": chunk_messages[0].get("role", "unknown") if chunk_messages else "unknown"
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
return chunks
|
|
107
|
+
|
|
108
|
+
def import_project(project_path: Path, collection_name: str) -> int:
|
|
109
|
+
"""Import all conversations from a project."""
|
|
110
|
+
jsonl_files = list(project_path.glob("*.jsonl"))
|
|
111
|
+
|
|
112
|
+
if not jsonl_files:
|
|
113
|
+
logger.warning(f"No JSONL files found in {project_path}")
|
|
114
|
+
return 0
|
|
115
|
+
|
|
116
|
+
# Check if collection exists
|
|
117
|
+
collections = client.get_collections().collections
|
|
118
|
+
if collection_name not in [c.name for c in collections]:
|
|
119
|
+
logger.info(f"Creating collection: {collection_name}")
|
|
120
|
+
client.create_collection(
|
|
121
|
+
collection_name=collection_name,
|
|
122
|
+
vectors_config=VectorParams(
|
|
123
|
+
size=embedding_dimension,
|
|
124
|
+
distance=Distance.COSINE
|
|
125
|
+
)
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
total_chunks = 0
|
|
129
|
+
|
|
130
|
+
for jsonl_file in jsonl_files:
|
|
131
|
+
logger.info(f"Processing file: {jsonl_file.name}")
|
|
132
|
+
try:
|
|
133
|
+
# Read JSONL file and extract messages
|
|
134
|
+
messages = []
|
|
135
|
+
created_at = None
|
|
136
|
+
|
|
137
|
+
with open(jsonl_file, 'r', encoding='utf-8') as f:
|
|
138
|
+
for line_num, line in enumerate(f, 1):
|
|
139
|
+
line = line.strip()
|
|
140
|
+
if not line:
|
|
141
|
+
continue
|
|
142
|
+
|
|
143
|
+
try:
|
|
144
|
+
data = json.loads(line)
|
|
145
|
+
|
|
146
|
+
# Extract timestamp from first message
|
|
147
|
+
if created_at is None and 'timestamp' in data:
|
|
148
|
+
created_at = data.get('timestamp')
|
|
149
|
+
|
|
150
|
+
# Skip non-message lines (summaries, etc.)
|
|
151
|
+
if data.get('type') == 'summary':
|
|
152
|
+
continue
|
|
153
|
+
|
|
154
|
+
# Extract message if present
|
|
155
|
+
if 'message' in data and data['message']:
|
|
156
|
+
msg = data['message']
|
|
157
|
+
if msg.get('role') and msg.get('content'):
|
|
158
|
+
# Handle content that's an array of objects
|
|
159
|
+
content = msg['content']
|
|
160
|
+
if isinstance(content, list):
|
|
161
|
+
text_parts = []
|
|
162
|
+
for item in content:
|
|
163
|
+
if isinstance(item, dict) and item.get('type') == 'text':
|
|
164
|
+
text_parts.append(item.get('text', ''))
|
|
165
|
+
elif isinstance(item, str):
|
|
166
|
+
text_parts.append(item)
|
|
167
|
+
content = '\n'.join(text_parts)
|
|
168
|
+
|
|
169
|
+
if content:
|
|
170
|
+
messages.append({
|
|
171
|
+
'role': msg['role'],
|
|
172
|
+
'content': content
|
|
173
|
+
})
|
|
174
|
+
except json.JSONDecodeError:
|
|
175
|
+
logger.debug(f"Skipping invalid JSON at line {line_num}")
|
|
176
|
+
except Exception as e:
|
|
177
|
+
logger.error(f"Error processing line {line_num}: {e}")
|
|
178
|
+
|
|
179
|
+
if not messages:
|
|
180
|
+
continue
|
|
181
|
+
|
|
182
|
+
# Extract metadata
|
|
183
|
+
if created_at is None:
|
|
184
|
+
created_at = datetime.now().isoformat()
|
|
185
|
+
conversation_id = jsonl_file.stem
|
|
186
|
+
|
|
187
|
+
# Chunk the conversation
|
|
188
|
+
chunks = chunk_conversation(messages)
|
|
189
|
+
|
|
190
|
+
if not chunks:
|
|
191
|
+
continue
|
|
192
|
+
|
|
193
|
+
# Process in batches
|
|
194
|
+
for batch_start in range(0, len(chunks), BATCH_SIZE):
|
|
195
|
+
batch = chunks[batch_start:batch_start + BATCH_SIZE]
|
|
196
|
+
texts = [chunk["text"] for chunk in batch]
|
|
197
|
+
|
|
198
|
+
# Generate embeddings
|
|
199
|
+
embeddings = generate_embeddings(texts)
|
|
200
|
+
|
|
201
|
+
# Create points
|
|
202
|
+
points = []
|
|
203
|
+
for chunk, embedding in zip(batch, embeddings):
|
|
204
|
+
point_id = hashlib.md5(
|
|
205
|
+
f"{conversation_id}_{chunk['chunk_index']}".encode()
|
|
206
|
+
).hexdigest()[:16]
|
|
207
|
+
|
|
208
|
+
points.append(PointStruct(
|
|
209
|
+
id=int(point_id, 16) % (2**63), # Convert to valid integer ID
|
|
210
|
+
vector=embedding,
|
|
211
|
+
payload={
|
|
212
|
+
"text": chunk["text"],
|
|
213
|
+
"conversation_id": conversation_id,
|
|
214
|
+
"chunk_index": chunk["chunk_index"],
|
|
215
|
+
"timestamp": created_at,
|
|
216
|
+
"project": project_path.name,
|
|
217
|
+
"start_role": chunk["start_role"]
|
|
218
|
+
}
|
|
219
|
+
))
|
|
220
|
+
|
|
221
|
+
# Upload to Qdrant
|
|
222
|
+
client.upsert(
|
|
223
|
+
collection_name=collection_name,
|
|
224
|
+
points=points
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
total_chunks += len(points)
|
|
228
|
+
|
|
229
|
+
logger.info(f"Imported {len(chunks)} chunks from {jsonl_file.name}")
|
|
230
|
+
|
|
231
|
+
except Exception as e:
|
|
232
|
+
logger.error(f"Failed to import {jsonl_file}: {e}")
|
|
233
|
+
import traceback
|
|
234
|
+
logger.error(traceback.format_exc())
|
|
235
|
+
|
|
236
|
+
return total_chunks
|
|
237
|
+
|
|
238
|
+
def main():
|
|
239
|
+
"""Main import function."""
|
|
240
|
+
logs_path = Path(LOGS_DIR)
|
|
241
|
+
|
|
242
|
+
if not logs_path.exists():
|
|
243
|
+
logger.error(f"Logs directory not found: {LOGS_DIR}")
|
|
244
|
+
return
|
|
245
|
+
|
|
246
|
+
# Find all project directories
|
|
247
|
+
project_dirs = [d for d in logs_path.iterdir() if d.is_dir()]
|
|
248
|
+
|
|
249
|
+
if not project_dirs:
|
|
250
|
+
logger.warning("No project directories found")
|
|
251
|
+
return
|
|
252
|
+
|
|
253
|
+
logger.info(f"Found {len(project_dirs)} projects to import")
|
|
254
|
+
|
|
255
|
+
# Import each project
|
|
256
|
+
total_imported = 0
|
|
257
|
+
for project_dir in project_dirs:
|
|
258
|
+
# Create collection name from project path
|
|
259
|
+
collection_name = f"conv_{hashlib.md5(project_dir.name.encode()).hexdigest()[:8]}{collection_suffix}"
|
|
260
|
+
|
|
261
|
+
logger.info(f"Importing project: {project_dir.name} -> {collection_name}")
|
|
262
|
+
chunks = import_project(project_dir, collection_name)
|
|
263
|
+
total_imported += chunks
|
|
264
|
+
logger.info(f"Imported {chunks} chunks from {project_dir.name}")
|
|
265
|
+
|
|
266
|
+
logger.info(f"Import complete! Total chunks imported: {total_imported}")
|
|
267
|
+
|
|
268
|
+
if __name__ == "__main__":
|
|
269
|
+
main()
|
|
@@ -16,8 +16,9 @@ def main():
|
|
|
16
16
|
# Import just the current conversation
|
|
17
17
|
importer = VoyageConversationImporter()
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
# Example: Update these to your project path and file
|
|
20
|
+
project_path = os.path.expanduser("~/.claude/projects/your-project-name")
|
|
21
|
+
target_file = "your-conversation-id.jsonl"
|
|
21
22
|
|
|
22
23
|
print(f"Importing current conversation: {target_file}")
|
|
23
24
|
|
|
@@ -13,7 +13,7 @@ import requests
|
|
|
13
13
|
|
|
14
14
|
# Configuration
|
|
15
15
|
QDRANT_URL = "http://localhost:6333"
|
|
16
|
-
VOYAGE_API_KEY = os.getenv("VOYAGE_KEY"
|
|
16
|
+
VOYAGE_API_KEY = os.getenv("VOYAGE_KEY")
|
|
17
17
|
VOYAGE_API_URL = "https://api.voyageai.com/v1/embeddings"
|
|
18
18
|
|
|
19
19
|
@backoff.on_exception(backoff.expo, Exception, max_tries=3)
|
|
@@ -43,7 +43,8 @@ def import_conversation(file_path):
|
|
|
43
43
|
client = QdrantClient(url=QDRANT_URL)
|
|
44
44
|
|
|
45
45
|
# Determine collection name
|
|
46
|
-
|
|
46
|
+
# Example: Update to your project path
|
|
47
|
+
project_path = os.path.expanduser("~/your-project-path")
|
|
47
48
|
project_hash = hashlib.md5(project_path.encode()).hexdigest()[:8]
|
|
48
49
|
collection_name = f"conv_{project_hash}_voyage"
|
|
49
50
|
|
|
@@ -138,7 +139,8 @@ def import_conversation(file_path):
|
|
|
138
139
|
print(f"\nCollection {collection_name} now has {info.points_count} points")
|
|
139
140
|
|
|
140
141
|
def main():
|
|
141
|
-
|
|
142
|
+
# Example: Update to your conversation file path
|
|
143
|
+
file_path = os.path.expanduser("~/.claude/projects/your-project/your-conversation-id.jsonl")
|
|
142
144
|
|
|
143
145
|
if not os.path.exists(file_path):
|
|
144
146
|
print(f"File not found: {file_path}")
|
|
@@ -23,7 +23,11 @@ for file in os.listdir(project_path):
|
|
|
23
23
|
print(f"Found {len(recent_files)} recent files to import")
|
|
24
24
|
|
|
25
25
|
# Set environment variable
|
|
26
|
-
|
|
26
|
+
# VOYAGE_KEY must be set as environment variable
|
|
27
|
+
if not os.getenv("VOYAGE_KEY"):
|
|
28
|
+
print("Error: VOYAGE_KEY environment variable not set")
|
|
29
|
+
print("Please set: export VOYAGE_KEY='your-voyage-api-key'")
|
|
30
|
+
sys.exit(1)
|
|
27
31
|
|
|
28
32
|
# Import the whole project (the script will handle individual files)
|
|
29
33
|
os.system(f"python {import_script} {project_path}")
|
|
@@ -19,7 +19,7 @@ WATCH_DIR = os.getenv("WATCH_DIR", "/logs")
|
|
|
19
19
|
STATE_FILE = os.getenv("STATE_FILE", "/config/imported-files.json")
|
|
20
20
|
WATCH_INTERVAL = int(os.getenv("WATCH_INTERVAL", "60")) # seconds
|
|
21
21
|
IMPORT_DELAY = int(os.getenv("IMPORT_DELAY", "30")) # Wait before importing new files
|
|
22
|
-
IMPORTER_SCRIPT = "/scripts/import-conversations-
|
|
22
|
+
IMPORTER_SCRIPT = "/scripts/import-conversations-unified.py"
|
|
23
23
|
|
|
24
24
|
# Set up logging
|
|
25
25
|
logging.basicConfig(
|