@geoql/mdr 0.0.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/bin/detect-user.sh +0 -0
- package/bin/index-conversations.ts +2 -2
- package/bin/macrodata-daemon.ts +1 -1
- package/bin/macrodata-hook.sh +1 -1
- package/dist/bin/index-conversations.js +0 -0
- package/dist/bin/macrodata-daemon.js +0 -0
- package/dist/opencode/skills/macrodata-distill/SKILL.md +1 -1
- package/opencode/context.ts +44 -44
- package/opencode/conversations.ts +20 -20
- package/opencode/index.ts +26 -26
- package/opencode/journal.ts +24 -24
- package/opencode/logger.ts +8 -8
- package/opencode/search.ts +31 -31
- package/opencode/skills/macrodata-distill/SKILL.md +1 -1
- package/opencode/tools.ts +62 -62
- package/package.json +17 -18
- package/src/config.ts +13 -13
- package/src/conversations.ts +61 -61
- package/src/daemon.ts +71 -71
- package/src/detect-user.ts +24 -24
- package/src/embeddings.ts +25 -25
- package/src/index.ts +129 -129
- package/src/indexer.ts +36 -36
package/src/conversations.ts
CHANGED
|
@@ -10,12 +10,12 @@
|
|
|
10
10
|
* - Metadata: project, branch, timestamp, session
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { readdirSync, readFileSync, existsSync, statSync, writeFileSync, mkdirSync } from
|
|
14
|
-
import { join, basename } from
|
|
15
|
-
import { homedir } from
|
|
16
|
-
import { embedBatch, embedQuery } from
|
|
17
|
-
import { LocalIndex } from
|
|
18
|
-
import { getIndexDir } from
|
|
13
|
+
import { readdirSync, readFileSync, existsSync, statSync, writeFileSync, mkdirSync } from 'fs';
|
|
14
|
+
import { join, basename } from 'path';
|
|
15
|
+
import { homedir } from 'os';
|
|
16
|
+
import { embedBatch, embedQuery } from './embeddings.js';
|
|
17
|
+
import { LocalIndex } from 'vectra';
|
|
18
|
+
import { getIndexDir } from './config.js';
|
|
19
19
|
|
|
20
20
|
// Index state tracking for incremental updates
|
|
21
21
|
interface IndexState {
|
|
@@ -24,19 +24,19 @@ interface IndexState {
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
function getIndexStatePath(): string {
|
|
27
|
-
return join(getIndexDir(),
|
|
27
|
+
return join(getIndexDir(), 'conversations-state.json');
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
function loadIndexState(): IndexState {
|
|
31
31
|
const statePath = getIndexStatePath();
|
|
32
32
|
if (existsSync(statePath)) {
|
|
33
33
|
try {
|
|
34
|
-
return JSON.parse(readFileSync(statePath,
|
|
34
|
+
return JSON.parse(readFileSync(statePath, 'utf-8'));
|
|
35
35
|
} catch {
|
|
36
|
-
console.warn(
|
|
36
|
+
console.warn('[Conversations] Index state corrupted, starting fresh');
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
|
-
return { files: {}, lastUpdate:
|
|
39
|
+
return { files: {}, lastUpdate: '' };
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
function saveIndexState(state: IndexState): void {
|
|
@@ -49,12 +49,12 @@ function saveIndexState(state: IndexState): void {
|
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
// Configuration
|
|
52
|
-
const CLAUDE_DIR = join(homedir(),
|
|
53
|
-
const PROJECTS_DIR = join(CLAUDE_DIR,
|
|
52
|
+
const CLAUDE_DIR = join(homedir(), '.claude');
|
|
53
|
+
const PROJECTS_DIR = join(CLAUDE_DIR, 'projects');
|
|
54
54
|
|
|
55
55
|
// Types
|
|
56
56
|
interface ConversationMessage {
|
|
57
|
-
type:
|
|
57
|
+
type: 'user' | 'assistant' | 'file-history-snapshot';
|
|
58
58
|
message?: {
|
|
59
59
|
role: string;
|
|
60
60
|
content: string | Array<{ type: string; text?: string; thinking?: string }>;
|
|
@@ -92,7 +92,7 @@ let convIndexPath: string | null = null;
|
|
|
92
92
|
|
|
93
93
|
async function getConversationIndex(): Promise<LocalIndex> {
|
|
94
94
|
const currentIndexDir = getIndexDir();
|
|
95
|
-
const currentIndexPath = join(currentIndexDir,
|
|
95
|
+
const currentIndexPath = join(currentIndexDir, 'conversations');
|
|
96
96
|
|
|
97
97
|
// Invalidate cache if path changed
|
|
98
98
|
if (convIndex && convIndexPath !== currentIndexPath) {
|
|
@@ -103,7 +103,7 @@ async function getConversationIndex(): Promise<LocalIndex> {
|
|
|
103
103
|
if (convIndex) return convIndex;
|
|
104
104
|
|
|
105
105
|
if (!existsSync(currentIndexDir)) {
|
|
106
|
-
const { mkdirSync } = await import(
|
|
106
|
+
const { mkdirSync } = await import('fs');
|
|
107
107
|
mkdirSync(currentIndexDir, { recursive: true });
|
|
108
108
|
}
|
|
109
109
|
|
|
@@ -111,7 +111,7 @@ async function getConversationIndex(): Promise<LocalIndex> {
|
|
|
111
111
|
convIndexPath = currentIndexPath;
|
|
112
112
|
|
|
113
113
|
if (!(await convIndex.isIndexCreated())) {
|
|
114
|
-
console.log(
|
|
114
|
+
console.log('[Conversations] Creating new conversation index...');
|
|
115
115
|
await convIndex.createIndex();
|
|
116
116
|
}
|
|
117
117
|
|
|
@@ -123,7 +123,7 @@ async function getConversationIndex(): Promise<LocalIndex> {
|
|
|
123
123
|
* e.g., "-Users-alex-Repos-my-project" -> "/Users/alex/Repos/my-project"
|
|
124
124
|
*/
|
|
125
125
|
function decodeProjectPath(encoded: string): string {
|
|
126
|
-
return encoded.replace(/^-/,
|
|
126
|
+
return encoded.replace(/^-/, '/').replace(/-/g, '/');
|
|
127
127
|
}
|
|
128
128
|
|
|
129
129
|
/**
|
|
@@ -137,17 +137,17 @@ function getProjectName(projectPath: string): string {
|
|
|
137
137
|
* Extract first text content from assistant message
|
|
138
138
|
*/
|
|
139
139
|
function extractAssistantText(content: string | Array<{ type: string; text?: string }>): string {
|
|
140
|
-
if (typeof content ===
|
|
140
|
+
if (typeof content === 'string') {
|
|
141
141
|
return content.slice(0, 500);
|
|
142
142
|
}
|
|
143
143
|
|
|
144
144
|
for (const block of content) {
|
|
145
|
-
if (block.type ===
|
|
145
|
+
if (block.type === 'text' && block.text) {
|
|
146
146
|
return block.text.slice(0, 500);
|
|
147
147
|
}
|
|
148
148
|
}
|
|
149
149
|
|
|
150
|
-
return
|
|
150
|
+
return '';
|
|
151
151
|
}
|
|
152
152
|
|
|
153
153
|
/**
|
|
@@ -156,7 +156,7 @@ function extractAssistantText(content: string | Array<{ type: string; text?: str
|
|
|
156
156
|
function isToolResult(content: unknown): boolean {
|
|
157
157
|
if (Array.isArray(content)) {
|
|
158
158
|
// Array content with tool_result or tool_use_id = tool result, not user prompt
|
|
159
|
-
return content.some((item) => item.type ===
|
|
159
|
+
return content.some((item) => item.type === 'tool_result' || item.tool_use_id !== undefined);
|
|
160
160
|
}
|
|
161
161
|
return false;
|
|
162
162
|
}
|
|
@@ -166,27 +166,27 @@ function isToolResult(content: unknown): boolean {
|
|
|
166
166
|
*/
|
|
167
167
|
function isNoiseContent(content: string): boolean {
|
|
168
168
|
// Skip compacted session summaries
|
|
169
|
-
if (content.startsWith(
|
|
169
|
+
if (content.startsWith('This session is being continued from a previous conversation')) {
|
|
170
170
|
return true;
|
|
171
171
|
}
|
|
172
172
|
|
|
173
173
|
// Skip local command outputs
|
|
174
174
|
if (
|
|
175
|
-
content.includes(
|
|
176
|
-
content.includes(
|
|
177
|
-
content.includes(
|
|
175
|
+
content.includes('<local-command-stdout>') ||
|
|
176
|
+
content.includes('<local-command-caveat>') ||
|
|
177
|
+
content.includes('<command-name>')
|
|
178
178
|
) {
|
|
179
179
|
return true;
|
|
180
180
|
}
|
|
181
181
|
|
|
182
182
|
// Skip hook-injected context (standalone, not part of agent context)
|
|
183
183
|
if (
|
|
184
|
-
content.startsWith(
|
|
185
|
-
content.startsWith(
|
|
186
|
-
content.startsWith(
|
|
187
|
-
content.startsWith(
|
|
188
|
-
content.startsWith(
|
|
189
|
-
content.startsWith(
|
|
184
|
+
content.startsWith('<current_time>') ||
|
|
185
|
+
content.startsWith('<context_status>') ||
|
|
186
|
+
content.startsWith('<state_files>') ||
|
|
187
|
+
content.startsWith('<system-reminder>') ||
|
|
188
|
+
content.startsWith('## Current State Files') ||
|
|
189
|
+
content.startsWith('Base directory for this skill:')
|
|
190
190
|
) {
|
|
191
191
|
return true;
|
|
192
192
|
}
|
|
@@ -203,9 +203,9 @@ function isNoiseContent(content: string): boolean {
|
|
|
203
203
|
* Extract actual user text from message content, filtering out tool results
|
|
204
204
|
*/
|
|
205
205
|
function extractUserText(content: string | unknown[]): string {
|
|
206
|
-
let text =
|
|
206
|
+
let text = '';
|
|
207
207
|
|
|
208
|
-
if (typeof content ===
|
|
208
|
+
if (typeof content === 'string') {
|
|
209
209
|
text = content;
|
|
210
210
|
} else {
|
|
211
211
|
// Array content - try to find actual text blocks
|
|
@@ -213,28 +213,28 @@ function extractUserText(content: string | unknown[]): string {
|
|
|
213
213
|
const b = block as Record<string, unknown>;
|
|
214
214
|
/* v8 ignore next 3 -- defensive: callers pre-filter tool-result arrays via
|
|
215
215
|
isToolResult(), so this redundant guard is never the deciding skip. */
|
|
216
|
-
if (b.type ===
|
|
216
|
+
if (b.type === 'tool_result' || b.tool_use_id !== undefined) {
|
|
217
217
|
continue;
|
|
218
218
|
}
|
|
219
219
|
// Look for text content
|
|
220
|
-
if (b.type ===
|
|
220
|
+
if (b.type === 'text' && typeof b.text === 'string') {
|
|
221
221
|
text = b.text;
|
|
222
222
|
break;
|
|
223
223
|
}
|
|
224
224
|
}
|
|
225
225
|
}
|
|
226
226
|
|
|
227
|
-
if (!text) return
|
|
227
|
+
if (!text) return '';
|
|
228
228
|
|
|
229
229
|
// Extract actual user message from agent context blocks
|
|
230
230
|
// These have format: "# Agent Context\n...\nUser message: <actual message>"
|
|
231
|
-
if (text.startsWith(
|
|
231
|
+
if (text.startsWith('# Agent Context') || text.includes('\nUser message: ')) {
|
|
232
232
|
const userMsgMatch = text.match(/\nUser message: (.+)$/s);
|
|
233
233
|
if (userMsgMatch) {
|
|
234
234
|
return userMsgMatch[1].trim();
|
|
235
235
|
}
|
|
236
236
|
// No user message found in context block - skip it
|
|
237
|
-
return
|
|
237
|
+
return '';
|
|
238
238
|
}
|
|
239
239
|
|
|
240
240
|
return text;
|
|
@@ -248,8 +248,8 @@ function parseConversationFile(filePath: string, projectPath: string): Conversat
|
|
|
248
248
|
const projectName = getProjectName(projectPath);
|
|
249
249
|
|
|
250
250
|
try {
|
|
251
|
-
const content = readFileSync(filePath,
|
|
252
|
-
const lines = content.trim().split(
|
|
251
|
+
const content = readFileSync(filePath, 'utf-8');
|
|
252
|
+
const lines = content.trim().split('\n').filter(Boolean);
|
|
253
253
|
|
|
254
254
|
let currentUser: { msg: ConversationMessage; text: string } | null = null;
|
|
255
255
|
let malformedLines = 0;
|
|
@@ -258,7 +258,7 @@ function parseConversationFile(filePath: string, projectPath: string): Conversat
|
|
|
258
258
|
try {
|
|
259
259
|
const msg: ConversationMessage = JSON.parse(line);
|
|
260
260
|
|
|
261
|
-
if (msg.type ===
|
|
261
|
+
if (msg.type === 'user' && msg.message?.content) {
|
|
262
262
|
// Skip tool results - these aren't actual user prompts
|
|
263
263
|
if (isToolResult(msg.message.content)) {
|
|
264
264
|
continue;
|
|
@@ -273,7 +273,7 @@ function parseConversationFile(filePath: string, projectPath: string): Conversat
|
|
|
273
273
|
}
|
|
274
274
|
|
|
275
275
|
currentUser = { msg, text: userText };
|
|
276
|
-
} else if (msg.type ===
|
|
276
|
+
} else if (msg.type === 'assistant' && currentUser && msg.message?.content) {
|
|
277
277
|
// Found a user-assistant pair. currentUser.text is always non-empty
|
|
278
278
|
// here (only truthy user text is stored above), so it needs no guard.
|
|
279
279
|
const assistantText = extractAssistantText(msg.message.content);
|
|
@@ -285,9 +285,9 @@ function parseConversationFile(filePath: string, projectPath: string): Conversat
|
|
|
285
285
|
projectPath: projectPath,
|
|
286
286
|
branch: currentUser.msg.gitBranch,
|
|
287
287
|
timestamp: currentUser.msg.timestamp || new Date().toISOString(),
|
|
288
|
-
sessionId: currentUser.msg.sessionId || basename(filePath,
|
|
288
|
+
sessionId: currentUser.msg.sessionId || basename(filePath, '.jsonl'),
|
|
289
289
|
sessionPath: filePath,
|
|
290
|
-
messageUuid: currentUser.msg.uuid ||
|
|
290
|
+
messageUuid: currentUser.msg.uuid || '',
|
|
291
291
|
});
|
|
292
292
|
|
|
293
293
|
currentUser = null; // Reset for next exchange
|
|
@@ -322,7 +322,7 @@ function* scanConversationFiles(): Generator<{
|
|
|
322
322
|
const projectDirs = readdirSync(PROJECTS_DIR);
|
|
323
323
|
|
|
324
324
|
for (const projectDir of projectDirs) {
|
|
325
|
-
if (projectDir.startsWith(
|
|
325
|
+
if (projectDir.startsWith('.')) continue;
|
|
326
326
|
|
|
327
327
|
const projectPath = decodeProjectPath(projectDir);
|
|
328
328
|
const projectFullPath = join(PROJECTS_DIR, projectDir);
|
|
@@ -333,7 +333,7 @@ function* scanConversationFiles(): Generator<{
|
|
|
333
333
|
|
|
334
334
|
for (const file of files) {
|
|
335
335
|
// Skip agent files, only process main conversation files
|
|
336
|
-
if (!file.endsWith(
|
|
336
|
+
if (!file.endsWith('.jsonl') || file.startsWith('agent-')) continue;
|
|
337
337
|
|
|
338
338
|
const filePath = join(projectFullPath, file);
|
|
339
339
|
const mtime = statSync(filePath).mtimeMs;
|
|
@@ -347,7 +347,7 @@ function* scanConversationFiles(): Generator<{
|
|
|
347
347
|
* Rebuild the conversation index from scratch
|
|
348
348
|
*/
|
|
349
349
|
export async function rebuildConversationIndex(): Promise<{ exchangeCount: number }> {
|
|
350
|
-
console.log(
|
|
350
|
+
console.log('[Conversations] Starting full index rebuild...');
|
|
351
351
|
const startTime = Date.now();
|
|
352
352
|
|
|
353
353
|
const allExchanges: ConversationExchange[] = [];
|
|
@@ -371,7 +371,7 @@ export async function rebuildConversationIndex(): Promise<{ exchangeCount: numbe
|
|
|
371
371
|
|
|
372
372
|
// Create embeddings for all exchanges
|
|
373
373
|
const texts = allExchanges.map(
|
|
374
|
-
(e) => `${e.project}${e.branch ? ` (${e.branch})` :
|
|
374
|
+
(e) => `${e.project}${e.branch ? ` (${e.branch})` : ''}: ${e.userPrompt}`,
|
|
375
375
|
);
|
|
376
376
|
|
|
377
377
|
console.log(`[Conversations] Generating embeddings...`);
|
|
@@ -392,7 +392,7 @@ export async function rebuildConversationIndex(): Promise<{ exchangeCount: numbe
|
|
|
392
392
|
assistantSummary: exchange.assistantSummary,
|
|
393
393
|
project: exchange.project,
|
|
394
394
|
projectPath: exchange.projectPath,
|
|
395
|
-
branch: exchange.branch ||
|
|
395
|
+
branch: exchange.branch || '',
|
|
396
396
|
timestamp: exchange.timestamp,
|
|
397
397
|
sessionId: exchange.sessionId,
|
|
398
398
|
sessionPath: exchange.sessionPath,
|
|
@@ -422,7 +422,7 @@ export async function updateConversationIndex(): Promise<{
|
|
|
422
422
|
filesUpdated: number;
|
|
423
423
|
skipped: number;
|
|
424
424
|
}> {
|
|
425
|
-
console.log(
|
|
425
|
+
console.log('[Conversations] Starting incremental update...');
|
|
426
426
|
const startTime = Date.now();
|
|
427
427
|
|
|
428
428
|
const state = loadIndexState();
|
|
@@ -432,7 +432,7 @@ export async function updateConversationIndex(): Promise<{
|
|
|
432
432
|
the index, so isIndexCreated() is never false here; this full-rebuild
|
|
433
433
|
fallback only guards a future change to that contract. */
|
|
434
434
|
if (!(await idx.isIndexCreated())) {
|
|
435
|
-
console.log(
|
|
435
|
+
console.log('[Conversations] No existing index, doing full rebuild');
|
|
436
436
|
const result = await rebuildConversationIndex();
|
|
437
437
|
return { exchangeCount: result.exchangeCount, filesUpdated: 0, skipped: 0 };
|
|
438
438
|
}
|
|
@@ -458,7 +458,7 @@ export async function updateConversationIndex(): Promise<{
|
|
|
458
458
|
|
|
459
459
|
if (exchanges.length > 0) {
|
|
460
460
|
const texts = exchanges.map(
|
|
461
|
-
(e) => `${e.project}${e.branch ? ` (${e.branch})` :
|
|
461
|
+
(e) => `${e.project}${e.branch ? ` (${e.branch})` : ''}: ${e.userPrompt}`,
|
|
462
462
|
);
|
|
463
463
|
const vectors = await embedBatch(texts);
|
|
464
464
|
|
|
@@ -474,7 +474,7 @@ export async function updateConversationIndex(): Promise<{
|
|
|
474
474
|
assistantSummary: exchange.assistantSummary,
|
|
475
475
|
project: exchange.project,
|
|
476
476
|
projectPath: exchange.projectPath,
|
|
477
|
-
branch: exchange.branch ||
|
|
477
|
+
branch: exchange.branch || '',
|
|
478
478
|
timestamp: exchange.timestamp,
|
|
479
479
|
sessionId: exchange.sessionId,
|
|
480
480
|
sessionPath: exchange.sessionPath,
|
|
@@ -547,7 +547,7 @@ export async function searchConversations(
|
|
|
547
547
|
const stats = await idx.listItems();
|
|
548
548
|
|
|
549
549
|
if (stats.length === 0) {
|
|
550
|
-
console.log(
|
|
550
|
+
console.log('[Conversations] Index is empty');
|
|
551
551
|
return [];
|
|
552
552
|
}
|
|
553
553
|
|
|
@@ -617,11 +617,11 @@ export async function expandConversation(
|
|
|
617
617
|
throw new Error(`Session file not found: ${sessionPath}`);
|
|
618
618
|
}
|
|
619
619
|
|
|
620
|
-
const content = readFileSync(sessionPath,
|
|
621
|
-
const lines = content.trim().split(
|
|
620
|
+
const content = readFileSync(sessionPath, 'utf-8');
|
|
621
|
+
const lines = content.trim().split('\n').filter(Boolean);
|
|
622
622
|
|
|
623
623
|
const messages: Array<{ role: string; content: string; timestamp?: string; uuid?: string }> = [];
|
|
624
|
-
let project =
|
|
624
|
+
let project = '';
|
|
625
625
|
let branch: string | undefined;
|
|
626
626
|
let malformedLines = 0;
|
|
627
627
|
|
|
@@ -630,7 +630,7 @@ export async function expandConversation(
|
|
|
630
630
|
try {
|
|
631
631
|
const msg: ConversationMessage = JSON.parse(line);
|
|
632
632
|
|
|
633
|
-
if (msg.type ===
|
|
633
|
+
if (msg.type === 'user' && msg.message?.content) {
|
|
634
634
|
// Skip tool results
|
|
635
635
|
if (isToolResult(msg.message.content)) {
|
|
636
636
|
continue;
|
|
@@ -644,7 +644,7 @@ export async function expandConversation(
|
|
|
644
644
|
}
|
|
645
645
|
|
|
646
646
|
messages.push({
|
|
647
|
-
role:
|
|
647
|
+
role: 'user',
|
|
648
648
|
content: text,
|
|
649
649
|
timestamp: msg.timestamp,
|
|
650
650
|
uuid: msg.uuid,
|
|
@@ -656,11 +656,11 @@ export async function expandConversation(
|
|
|
656
656
|
if (!branch && msg.gitBranch) {
|
|
657
657
|
branch = msg.gitBranch;
|
|
658
658
|
}
|
|
659
|
-
} else if (msg.type ===
|
|
659
|
+
} else if (msg.type === 'assistant' && msg.message?.content) {
|
|
660
660
|
const text = extractAssistantText(msg.message.content);
|
|
661
661
|
if (text) {
|
|
662
662
|
messages.push({
|
|
663
|
-
role:
|
|
663
|
+
role: 'assistant',
|
|
664
664
|
content: text,
|
|
665
665
|
timestamp: msg.timestamp,
|
|
666
666
|
uuid: msg.uuid,
|