@geoql/mdr 0.0.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/bin/detect-user.sh +0 -0
- package/bin/index-conversations.ts +2 -2
- package/bin/macrodata-daemon.ts +1 -1
- package/bin/macrodata-hook.sh +1 -1
- package/dist/bin/index-conversations.js +0 -0
- package/dist/bin/macrodata-daemon.js +0 -0
- package/dist/opencode/skills/macrodata-distill/SKILL.md +1 -1
- package/opencode/context.ts +44 -44
- package/opencode/conversations.ts +20 -20
- package/opencode/index.ts +26 -26
- package/opencode/journal.ts +24 -24
- package/opencode/logger.ts +8 -8
- package/opencode/search.ts +31 -31
- package/opencode/skills/macrodata-distill/SKILL.md +1 -1
- package/opencode/tools.ts +62 -62
- package/package.json +17 -18
- package/src/config.ts +13 -13
- package/src/conversations.ts +61 -61
- package/src/daemon.ts +71 -71
- package/src/detect-user.ts +24 -24
- package/src/embeddings.ts +25 -25
- package/src/index.ts +129 -129
- package/src/indexer.ts +36 -36
package/opencode/journal.ts
CHANGED
|
@@ -4,11 +4,11 @@
|
|
|
4
4
|
* Write journal entries and search memory
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import { existsSync, appendFileSync, mkdirSync, readFileSync, readdirSync } from
|
|
8
|
-
import { join } from
|
|
9
|
-
import { getStateRoot } from
|
|
10
|
-
import { indexJournalEntry } from
|
|
11
|
-
import { logger } from
|
|
7
|
+
import { existsSync, appendFileSync, mkdirSync, readFileSync, readdirSync } from 'fs';
|
|
8
|
+
import { join } from 'path';
|
|
9
|
+
import { getStateRoot } from './context.js';
|
|
10
|
+
import { indexJournalEntry } from './search.js';
|
|
11
|
+
import { logger } from './logger.js';
|
|
12
12
|
|
|
13
13
|
interface JournalEntry {
|
|
14
14
|
timestamp: string;
|
|
@@ -24,11 +24,11 @@ function ensureDirectories(): void {
|
|
|
24
24
|
const stateRoot = getStateRoot();
|
|
25
25
|
const dirs = [
|
|
26
26
|
stateRoot,
|
|
27
|
-
join(stateRoot,
|
|
28
|
-
join(stateRoot,
|
|
29
|
-
join(stateRoot,
|
|
30
|
-
join(stateRoot,
|
|
31
|
-
join(stateRoot,
|
|
27
|
+
join(stateRoot, 'state'),
|
|
28
|
+
join(stateRoot, 'entities'),
|
|
29
|
+
join(stateRoot, 'entities', 'people'),
|
|
30
|
+
join(stateRoot, 'entities', 'projects'),
|
|
31
|
+
join(stateRoot, 'journal'),
|
|
32
32
|
];
|
|
33
33
|
|
|
34
34
|
for (const dir of dirs) {
|
|
@@ -40,8 +40,8 @@ function ensureDirectories(): void {
|
|
|
40
40
|
|
|
41
41
|
function getTodayJournalPath(): string {
|
|
42
42
|
const stateRoot = getStateRoot();
|
|
43
|
-
const today = new Date().toISOString().split(
|
|
44
|
-
return join(stateRoot,
|
|
43
|
+
const today = new Date().toISOString().split('T')[0];
|
|
44
|
+
return join(stateRoot, 'journal', `${today}.jsonl`);
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
/**
|
|
@@ -58,11 +58,11 @@ export async function logJournal(
|
|
|
58
58
|
timestamp: new Date().toISOString(),
|
|
59
59
|
topic,
|
|
60
60
|
content,
|
|
61
|
-
metadata: metadata || { source:
|
|
61
|
+
metadata: metadata || { source: 'opencode-plugin' },
|
|
62
62
|
};
|
|
63
63
|
|
|
64
64
|
const journalPath = getTodayJournalPath();
|
|
65
|
-
appendFileSync(journalPath, JSON.stringify(entry) +
|
|
65
|
+
appendFileSync(journalPath, JSON.stringify(entry) + '\n');
|
|
66
66
|
|
|
67
67
|
// Index the entry for semantic search
|
|
68
68
|
try {
|
|
@@ -77,22 +77,22 @@ export async function logJournal(
|
|
|
77
77
|
*/
|
|
78
78
|
export function getRecentJournal(count: number, topic?: string): JournalEntry[] {
|
|
79
79
|
const stateRoot = getStateRoot();
|
|
80
|
-
const journalDir = join(stateRoot,
|
|
80
|
+
const journalDir = join(stateRoot, 'journal');
|
|
81
81
|
let entries: JournalEntry[] = [];
|
|
82
82
|
|
|
83
83
|
if (!existsSync(journalDir)) return entries;
|
|
84
84
|
|
|
85
85
|
try {
|
|
86
86
|
const files = readdirSync(journalDir)
|
|
87
|
-
.filter((f) => f.endsWith(
|
|
87
|
+
.filter((f) => f.endsWith('.jsonl'))
|
|
88
88
|
.sort()
|
|
89
89
|
.reverse();
|
|
90
90
|
|
|
91
91
|
for (const file of files) {
|
|
92
92
|
if (entries.length >= count * 2) break; // Get more for filtering
|
|
93
93
|
|
|
94
|
-
const content = readFileSync(join(journalDir, file),
|
|
95
|
-
const lines = content.trim().split(
|
|
94
|
+
const content = readFileSync(join(journalDir, file), 'utf-8');
|
|
95
|
+
const lines = content.trim().split('\n').filter(Boolean);
|
|
96
96
|
|
|
97
97
|
for (const line of lines.reverse()) {
|
|
98
98
|
try {
|
|
@@ -119,7 +119,7 @@ export function getRecentJournal(count: number, topic?: string): JournalEntry[]
|
|
|
119
119
|
* Get recent conversation summaries
|
|
120
120
|
*/
|
|
121
121
|
export function getRecentSummaries(count: number): JournalEntry[] {
|
|
122
|
-
return getRecentJournal(count,
|
|
122
|
+
return getRecentJournal(count, 'conversation-summary');
|
|
123
123
|
}
|
|
124
124
|
|
|
125
125
|
/**
|
|
@@ -135,19 +135,19 @@ export async function saveConversationSummary(options: {
|
|
|
135
135
|
const parts = [options.summary];
|
|
136
136
|
|
|
137
137
|
if (options.keyDecisions?.length) {
|
|
138
|
-
parts.push(`Decisions: ${options.keyDecisions.join(
|
|
138
|
+
parts.push(`Decisions: ${options.keyDecisions.join(', ')}`);
|
|
139
139
|
}
|
|
140
140
|
if (options.openThreads?.length) {
|
|
141
|
-
parts.push(`Open threads: ${options.openThreads.join(
|
|
141
|
+
parts.push(`Open threads: ${options.openThreads.join(', ')}`);
|
|
142
142
|
}
|
|
143
143
|
if (options.learnedPatterns?.length) {
|
|
144
|
-
parts.push(`Learned: ${options.learnedPatterns.join(
|
|
144
|
+
parts.push(`Learned: ${options.learnedPatterns.join(', ')}`);
|
|
145
145
|
}
|
|
146
146
|
if (options.notes) {
|
|
147
147
|
parts.push(`Notes: ${options.notes}`);
|
|
148
148
|
}
|
|
149
149
|
|
|
150
|
-
await logJournal(
|
|
151
|
-
source:
|
|
150
|
+
await logJournal('conversation-summary', parts.join('\n'), {
|
|
151
|
+
source: 'opencode-plugin',
|
|
152
152
|
});
|
|
153
153
|
}
|
package/opencode/logger.ts
CHANGED
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
* Writes to .macrodata.log in state root instead of console
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import { appendFileSync, mkdirSync } from
|
|
7
|
-
import { join } from
|
|
8
|
-
import { homedir } from
|
|
6
|
+
import { appendFileSync, mkdirSync } from 'fs';
|
|
7
|
+
import { join } from 'path';
|
|
8
|
+
import { homedir } from 'os';
|
|
9
9
|
|
|
10
|
-
const LOG_FILE = join(homedir(),
|
|
10
|
+
const LOG_FILE = join(homedir(), '.config', 'macrodata', '.macrodata.log');
|
|
11
11
|
|
|
12
12
|
// Ensure directory exists
|
|
13
|
-
mkdirSync(join(homedir(),
|
|
13
|
+
mkdirSync(join(homedir(), '.config', 'macrodata'), { recursive: true });
|
|
14
14
|
|
|
15
15
|
function formatMessage(level: string, message: string): string {
|
|
16
16
|
const timestamp = new Date().toISOString();
|
|
@@ -19,14 +19,14 @@ function formatMessage(level: string, message: string): string {
|
|
|
19
19
|
|
|
20
20
|
export const logger = {
|
|
21
21
|
log(message: string): void {
|
|
22
|
-
appendFileSync(LOG_FILE, formatMessage(
|
|
22
|
+
appendFileSync(LOG_FILE, formatMessage('INFO', message));
|
|
23
23
|
},
|
|
24
24
|
|
|
25
25
|
error(message: string): void {
|
|
26
|
-
appendFileSync(LOG_FILE, formatMessage(
|
|
26
|
+
appendFileSync(LOG_FILE, formatMessage('ERROR', message));
|
|
27
27
|
},
|
|
28
28
|
|
|
29
29
|
warn(message: string): void {
|
|
30
|
-
appendFileSync(LOG_FILE, formatMessage(
|
|
30
|
+
appendFileSync(LOG_FILE, formatMessage('WARN', message));
|
|
31
31
|
},
|
|
32
32
|
};
|
package/opencode/search.ts
CHANGED
|
@@ -5,12 +5,12 @@
|
|
|
5
5
|
* (local Transformers.js by default, remote provider when configured).
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
import { LocalIndex } from
|
|
9
|
-
import { existsSync, readFileSync, readdirSync, mkdirSync } from
|
|
10
|
-
import { join } from
|
|
11
|
-
import { embed, embedBatch, embedQuery } from
|
|
12
|
-
import { getStateRoot } from
|
|
13
|
-
import { logger } from
|
|
8
|
+
import { LocalIndex } from 'vectra';
|
|
9
|
+
import { existsSync, readFileSync, readdirSync, mkdirSync } from 'fs';
|
|
10
|
+
import { join } from 'path';
|
|
11
|
+
import { embed, embedBatch, embedQuery } from '../src/embeddings.js';
|
|
12
|
+
import { getStateRoot } from './context.js';
|
|
13
|
+
import { logger } from './logger.js';
|
|
14
14
|
|
|
15
15
|
// Memory index singleton
|
|
16
16
|
let memoryIndex: LocalIndex | null = null;
|
|
@@ -24,9 +24,9 @@ async function getMemoryIndex(): Promise<LocalIndex> {
|
|
|
24
24
|
if (memoryIndex) return memoryIndex;
|
|
25
25
|
|
|
26
26
|
const stateRoot = getStateRoot();
|
|
27
|
-
const indexPath = join(stateRoot,
|
|
27
|
+
const indexPath = join(stateRoot, '.index', 'vectors');
|
|
28
28
|
|
|
29
|
-
const indexDir = join(stateRoot,
|
|
29
|
+
const indexDir = join(stateRoot, '.index');
|
|
30
30
|
if (!existsSync(indexDir)) {
|
|
31
31
|
mkdirSync(indexDir, { recursive: true });
|
|
32
32
|
}
|
|
@@ -34,14 +34,14 @@ async function getMemoryIndex(): Promise<LocalIndex> {
|
|
|
34
34
|
memoryIndex = new LocalIndex(indexPath);
|
|
35
35
|
|
|
36
36
|
if (!(await memoryIndex.isIndexCreated())) {
|
|
37
|
-
logger.log(
|
|
37
|
+
logger.log('Creating new memory index...');
|
|
38
38
|
await memoryIndex.createIndex();
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
return memoryIndex;
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
export type MemoryItemType =
|
|
44
|
+
export type MemoryItemType = 'journal' | 'person' | 'project' | 'topic';
|
|
45
45
|
|
|
46
46
|
export interface SearchResult {
|
|
47
47
|
content: string;
|
|
@@ -102,7 +102,7 @@ export async function searchMemory(
|
|
|
102
102
|
* Rebuild memory index from journal and entity files
|
|
103
103
|
*/
|
|
104
104
|
export async function rebuildMemoryIndex(): Promise<{ itemCount: number }> {
|
|
105
|
-
logger.log(
|
|
105
|
+
logger.log('Rebuilding memory index...');
|
|
106
106
|
const startTime = Date.now();
|
|
107
107
|
const stateRoot = getStateRoot();
|
|
108
108
|
|
|
@@ -118,19 +118,19 @@ export async function rebuildMemoryIndex(): Promise<{ itemCount: number }> {
|
|
|
118
118
|
const allItems: MemoryItem[] = [];
|
|
119
119
|
|
|
120
120
|
// Index journal entries
|
|
121
|
-
const journalDir = join(stateRoot,
|
|
121
|
+
const journalDir = join(stateRoot, 'journal');
|
|
122
122
|
if (existsSync(journalDir)) {
|
|
123
|
-
const files = readdirSync(journalDir).filter((f) => f.endsWith(
|
|
123
|
+
const files = readdirSync(journalDir).filter((f) => f.endsWith('.jsonl'));
|
|
124
124
|
for (const file of files) {
|
|
125
125
|
try {
|
|
126
|
-
const content = readFileSync(join(journalDir, file),
|
|
127
|
-
const lines = content.trim().split(
|
|
126
|
+
const content = readFileSync(join(journalDir, file), 'utf-8');
|
|
127
|
+
const lines = content.trim().split('\n').filter(Boolean);
|
|
128
128
|
for (let i = 0; i < lines.length; i++) {
|
|
129
129
|
try {
|
|
130
130
|
const entry = JSON.parse(lines[i]);
|
|
131
131
|
allItems.push({
|
|
132
132
|
id: `journal-${file}-${i}`,
|
|
133
|
-
type:
|
|
133
|
+
type: 'journal',
|
|
134
134
|
content: `[${entry.topic}] ${entry.content}`,
|
|
135
135
|
source: file,
|
|
136
136
|
timestamp: entry.timestamp,
|
|
@@ -146,19 +146,19 @@ export async function rebuildMemoryIndex(): Promise<{ itemCount: number }> {
|
|
|
146
146
|
}
|
|
147
147
|
|
|
148
148
|
// Index entity files (people, projects)
|
|
149
|
-
const entitiesDir = join(stateRoot,
|
|
149
|
+
const entitiesDir = join(stateRoot, 'entities');
|
|
150
150
|
for (const [subdir, type] of [
|
|
151
|
-
[
|
|
152
|
-
[
|
|
151
|
+
['people', 'person'],
|
|
152
|
+
['projects', 'project'],
|
|
153
153
|
] as const) {
|
|
154
154
|
const dir = join(entitiesDir, subdir);
|
|
155
155
|
if (!existsSync(dir)) continue;
|
|
156
156
|
|
|
157
|
-
const files = readdirSync(dir).filter((f) => f.endsWith(
|
|
157
|
+
const files = readdirSync(dir).filter((f) => f.endsWith('.md'));
|
|
158
158
|
for (const file of files) {
|
|
159
159
|
try {
|
|
160
|
-
const content = readFileSync(join(dir, file),
|
|
161
|
-
const filename = file.replace(
|
|
160
|
+
const content = readFileSync(join(dir, file), 'utf-8');
|
|
161
|
+
const filename = file.replace('.md', '');
|
|
162
162
|
|
|
163
163
|
// Split by ## headers
|
|
164
164
|
const sections = content.split(/^## /m);
|
|
@@ -169,13 +169,13 @@ export async function rebuildMemoryIndex(): Promise<{ itemCount: number }> {
|
|
|
169
169
|
type,
|
|
170
170
|
content: sections[0].trim(),
|
|
171
171
|
source: `${subdir}/${file}`,
|
|
172
|
-
section:
|
|
172
|
+
section: 'preamble',
|
|
173
173
|
});
|
|
174
174
|
}
|
|
175
175
|
|
|
176
176
|
for (let i = 1; i < sections.length; i++) {
|
|
177
177
|
const section = sections[i];
|
|
178
|
-
const firstLine = section.split(
|
|
178
|
+
const firstLine = section.split('\n')[0];
|
|
179
179
|
const sectionTitle = firstLine.trim();
|
|
180
180
|
const sectionContent = section.slice(firstLine.length).trim();
|
|
181
181
|
|
|
@@ -196,16 +196,16 @@ export async function rebuildMemoryIndex(): Promise<{ itemCount: number }> {
|
|
|
196
196
|
}
|
|
197
197
|
|
|
198
198
|
// Index topics
|
|
199
|
-
const topicsDir = join(stateRoot,
|
|
199
|
+
const topicsDir = join(stateRoot, 'topics');
|
|
200
200
|
if (existsSync(topicsDir)) {
|
|
201
|
-
const files = readdirSync(topicsDir).filter((f) => f.endsWith(
|
|
201
|
+
const files = readdirSync(topicsDir).filter((f) => f.endsWith('.md'));
|
|
202
202
|
for (const file of files) {
|
|
203
203
|
try {
|
|
204
|
-
const content = readFileSync(join(topicsDir, file),
|
|
205
|
-
const filename = file.replace(
|
|
204
|
+
const content = readFileSync(join(topicsDir, file), 'utf-8');
|
|
205
|
+
const filename = file.replace('.md', '');
|
|
206
206
|
allItems.push({
|
|
207
207
|
id: `topic-${filename}`,
|
|
208
|
-
type:
|
|
208
|
+
type: 'topic',
|
|
209
209
|
content: content.trim(),
|
|
210
210
|
source: `topics/${file}`,
|
|
211
211
|
});
|
|
@@ -279,9 +279,9 @@ export async function indexJournalEntry(entry: {
|
|
|
279
279
|
id: `journal-${entry.timestamp}`,
|
|
280
280
|
vector,
|
|
281
281
|
metadata: {
|
|
282
|
-
type:
|
|
282
|
+
type: 'journal',
|
|
283
283
|
content: `[${entry.topic}] ${entry.content}`,
|
|
284
|
-
source:
|
|
284
|
+
source: 'journal',
|
|
285
285
|
timestamp: entry.timestamp,
|
|
286
286
|
},
|
|
287
287
|
});
|
|
@@ -148,7 +148,7 @@ macrodata_log_journal(topic="distill-summary", content="Processed N sessions. Ex
|
|
|
148
148
|
"distilled_actions": [
|
|
149
149
|
{
|
|
150
150
|
"summary": "Added /distill skill to macrodata plugin",
|
|
151
|
-
"files": ["
|
|
151
|
+
"files": ["packages/macrodata/skills/distill/SKILL.md"],
|
|
152
152
|
"outcome": "Skill extracts facts from conversations via sub-agents"
|
|
153
153
|
}
|
|
154
154
|
],
|