@sashabogi/foundation 2.1.0 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +236 -2
- package/dist/cli.js.map +1 -1
- package/dist/tools/demerzel/engine.d.ts +3 -2
- package/dist/tools/demerzel/engine.d.ts.map +1 -1
- package/dist/tools/demerzel/engine.js +7 -7
- package/dist/tools/demerzel/engine.js.map +1 -1
- package/dist/tools/demerzel/enhanced-snapshot.d.ts +10 -2
- package/dist/tools/demerzel/enhanced-snapshot.d.ts.map +1 -1
- package/dist/tools/demerzel/enhanced-snapshot.js +156 -77
- package/dist/tools/demerzel/enhanced-snapshot.js.map +1 -1
- package/dist/tools/demerzel/index.d.ts +1 -1
- package/dist/tools/demerzel/index.d.ts.map +1 -1
- package/dist/tools/demerzel/index.js +53 -36
- package/dist/tools/demerzel/index.js.map +1 -1
- package/dist/tools/demerzel/semantic-search.d.ts +2 -2
- package/dist/tools/demerzel/semantic-search.d.ts.map +1 -1
- package/dist/tools/demerzel/semantic-search.js +4 -3
- package/dist/tools/demerzel/semantic-search.js.map +1 -1
- package/dist/tools/demerzel/snapshot.d.ts +34 -1
- package/dist/tools/demerzel/snapshot.d.ts.map +1 -1
- package/dist/tools/demerzel/snapshot.js +70 -34
- package/dist/tools/demerzel/snapshot.js.map +1 -1
- package/dist/tools/demerzel/zstd-io.d.ts +40 -0
- package/dist/tools/demerzel/zstd-io.d.ts.map +1 -0
- package/dist/tools/demerzel/zstd-io.js +69 -0
- package/dist/tools/demerzel/zstd-io.js.map +1 -0
- package/dist/tools/gaia/index.d.ts +5 -4
- package/dist/tools/gaia/index.d.ts.map +1 -1
- package/dist/tools/gaia/index.js +155 -8
- package/dist/tools/gaia/index.js.map +1 -1
- package/dist/tools/gaia/storage.d.ts +31 -2
- package/dist/tools/gaia/storage.d.ts.map +1 -1
- package/dist/tools/gaia/storage.js +110 -6
- package/dist/tools/gaia/storage.js.map +1 -1
- package/dist/tools/gaia/transcript-ingester.d.ts +81 -0
- package/dist/tools/gaia/transcript-ingester.d.ts.map +1 -0
- package/dist/tools/gaia/transcript-ingester.js +418 -0
- package/dist/tools/gaia/transcript-ingester.js.map +1 -0
- package/dist/vault/dashboard.d.ts +10 -0
- package/dist/vault/dashboard.d.ts.map +1 -0
- package/dist/vault/dashboard.js +145 -0
- package/dist/vault/dashboard.js.map +1 -0
- package/dist/vault/sync.d.ts +64 -0
- package/dist/vault/sync.d.ts.map +1 -0
- package/dist/vault/sync.js +372 -0
- package/dist/vault/sync.js.map +1 -0
- package/dist/vault/transform.d.ts +25 -0
- package/dist/vault/transform.d.ts.map +1 -0
- package/dist/vault/transform.js +155 -0
- package/dist/vault/transform.js.map +1 -0
- package/package.json +2 -1
- package/packages/ui/dist/assets/index-BCBAHIpG.css +1 -0
- package/packages/ui/dist/assets/index-CxRK0aqp.js +387 -0
- package/packages/ui/dist/index.html +2 -2
- package/docs/foundation +0 -0
- package/packages/ui/dist/assets/index-DVS_pYGH.css +0 -1
- package/packages/ui/dist/assets/index-WNO_oIqP.js +0 -312
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Transcript Ingester
|
|
3
|
+
*
|
|
4
|
+
* Walks ~/.claude/projects/**/*.jsonl, parses Claude Code conversation
|
|
5
|
+
* transcripts, and indexes human + assistant prose into Gaia's FTS5 store.
|
|
6
|
+
*
|
|
7
|
+
* Design:
|
|
8
|
+
* - Streams JSONL line-by-line (never loads full file into memory)
|
|
9
|
+
* - Incremental: tracks last-ingested byte offset per file in SQLite
|
|
10
|
+
* - Dedupes on (file_path, message_index) to survive re-runs
|
|
11
|
+
* - Skips noise: tool_result, system messages, sidechain, pure tool_use-only turns
|
|
12
|
+
*/
|
|
13
|
+
import { createReadStream, existsSync, readdirSync, statSync } from 'fs';
|
|
14
|
+
import { join, basename } from 'path';
|
|
15
|
+
import { homedir } from 'os';
|
|
16
|
+
import { createInterface } from 'readline';
|
|
17
|
+
import { nanoid } from 'nanoid';
|
|
18
|
+
// =============================================================================
|
|
19
|
+
// Schema helpers — call against the MemoriaStorage db instance (accessed via
|
|
20
|
+
// the exposed `getDb` accessor we add to storage, or by passing db directly)
|
|
21
|
+
// =============================================================================
|
|
22
|
+
export const INGEST_STATE_DDL = `
|
|
23
|
+
CREATE TABLE IF NOT EXISTS transcript_ingest_state (
|
|
24
|
+
file_path TEXT NOT NULL PRIMARY KEY,
|
|
25
|
+
last_byte_offset INTEGER NOT NULL DEFAULT 0,
|
|
26
|
+
last_ingested_at INTEGER NOT NULL,
|
|
27
|
+
messages_ingested INTEGER NOT NULL DEFAULT 0
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
CREATE TABLE IF NOT EXISTS transcript_dedup (
|
|
31
|
+
file_path TEXT NOT NULL,
|
|
32
|
+
message_index INTEGER NOT NULL,
|
|
33
|
+
memory_id TEXT NOT NULL,
|
|
34
|
+
PRIMARY KEY (file_path, message_index)
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
CREATE INDEX IF NOT EXISTS idx_dedup_file ON transcript_dedup(file_path);
|
|
38
|
+
`;
|
|
39
|
+
// =============================================================================
|
|
40
|
+
// JSONL parsing helpers
|
|
41
|
+
// =============================================================================
|
|
42
|
+
/**
|
|
43
|
+
* Extract plain text from Claude Code message content.
|
|
44
|
+
* Content may be a plain string or an array of content blocks.
|
|
45
|
+
* We only keep `text` blocks — tool_use and tool_result blocks are dropped.
|
|
46
|
+
*/
|
|
47
|
+
function extractText(content) {
|
|
48
|
+
if (typeof content === 'string') {
|
|
49
|
+
const trimmed = content.trim();
|
|
50
|
+
return trimmed.length > 0 ? trimmed : null;
|
|
51
|
+
}
|
|
52
|
+
if (!Array.isArray(content))
|
|
53
|
+
return null;
|
|
54
|
+
const parts = [];
|
|
55
|
+
for (const block of content) {
|
|
56
|
+
if (typeof block !== 'object' || block === null)
|
|
57
|
+
continue;
|
|
58
|
+
const b = block;
|
|
59
|
+
if (b['type'] === 'text' && typeof b['text'] === 'string') {
|
|
60
|
+
const t = b['text'].trim();
|
|
61
|
+
if (t.length > 0)
|
|
62
|
+
parts.push(t);
|
|
63
|
+
}
|
|
64
|
+
// Explicitly skip: tool_use, tool_result, image, document, etc.
|
|
65
|
+
}
|
|
66
|
+
return parts.length > 0 ? parts.join('\n\n') : null;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Parse a single JSONL line into a ParsedMessage or null if it should be
|
|
70
|
+
* skipped (noise, metadata, sidechain, system).
|
|
71
|
+
*/
|
|
72
|
+
function parseLine(rawLine, lineIndex, filePath, projectSlug, sinceMs) {
|
|
73
|
+
let obj;
|
|
74
|
+
try {
|
|
75
|
+
obj = JSON.parse(rawLine);
|
|
76
|
+
}
|
|
77
|
+
catch {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
const type = obj['type'];
|
|
81
|
+
// Only process user + assistant prose turns
|
|
82
|
+
if (type !== 'user' && type !== 'assistant')
|
|
83
|
+
return null;
|
|
84
|
+
// Skip sidechain messages (sub-agent internal turns)
|
|
85
|
+
if (obj['isSidechain'] === true)
|
|
86
|
+
return null;
|
|
87
|
+
// Require a message wrapper
|
|
88
|
+
const msg = obj['message'];
|
|
89
|
+
if (typeof msg !== 'object' || msg === null)
|
|
90
|
+
return null;
|
|
91
|
+
const msgObj = msg;
|
|
92
|
+
const role = msgObj['role'];
|
|
93
|
+
if (role !== 'user' && role !== 'assistant')
|
|
94
|
+
return null;
|
|
95
|
+
const text = extractText(msgObj['content']);
|
|
96
|
+
if (!text)
|
|
97
|
+
return null;
|
|
98
|
+
// Parse timestamp — field is "timestamp" on both user and assistant rows
|
|
99
|
+
let timestamp = 0;
|
|
100
|
+
const ts = obj['timestamp'];
|
|
101
|
+
if (typeof ts === 'string') {
|
|
102
|
+
const parsed = Date.parse(ts);
|
|
103
|
+
if (!isNaN(parsed))
|
|
104
|
+
timestamp = parsed;
|
|
105
|
+
}
|
|
106
|
+
else if (typeof ts === 'number') {
|
|
107
|
+
timestamp = ts;
|
|
108
|
+
}
|
|
109
|
+
// Since filter
|
|
110
|
+
if (sinceMs !== null && timestamp > 0 && timestamp < sinceMs)
|
|
111
|
+
return null;
|
|
112
|
+
const sessionId = typeof obj['sessionId'] === 'string' ? obj['sessionId'] : 'unknown';
|
|
113
|
+
const cwd = typeof obj['cwd'] === 'string' ? obj['cwd'] : null;
|
|
114
|
+
return {
|
|
115
|
+
sessionId,
|
|
116
|
+
messageIndex: lineIndex,
|
|
117
|
+
timestamp,
|
|
118
|
+
role: role,
|
|
119
|
+
text,
|
|
120
|
+
projectSlug,
|
|
121
|
+
cwd,
|
|
122
|
+
filePath,
|
|
123
|
+
lineOffset: lineIndex,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
// =============================================================================
|
|
127
|
+
// Project slug extraction
|
|
128
|
+
// =============================================================================
|
|
129
|
+
/**
|
|
130
|
+
* Derive human-readable project name from the JSONL path.
|
|
131
|
+
* ~/.claude/projects/<slug>/<uuid>.jsonl → slug
|
|
132
|
+
*/
|
|
133
|
+
function slugFromPath(filePath, rootDir) {
|
|
134
|
+
// Strip rootDir prefix
|
|
135
|
+
const relative = filePath.startsWith(rootDir)
|
|
136
|
+
? filePath.slice(rootDir.length + 1)
|
|
137
|
+
: filePath;
|
|
138
|
+
// First path component is the project slug
|
|
139
|
+
const slug = relative.split('/')[0] ?? basename(filePath);
|
|
140
|
+
return slug;
|
|
141
|
+
}
|
|
142
|
+
// =============================================================================
|
|
143
|
+
// Core ingester
|
|
144
|
+
// =============================================================================
|
|
145
|
+
export class TranscriptIngester {
|
|
146
|
+
db;
|
|
147
|
+
memoria;
|
|
148
|
+
constructor(memoriaDb, memoria) {
|
|
149
|
+
this.db = memoriaDb;
|
|
150
|
+
this.memoria = memoria;
|
|
151
|
+
this.ensureSchema();
|
|
152
|
+
}
|
|
153
|
+
ensureSchema() {
|
|
154
|
+
this.db.exec(INGEST_STATE_DDL);
|
|
155
|
+
}
|
|
156
|
+
// ---------------------------------------------------------------------------
|
|
157
|
+
// Public API
|
|
158
|
+
// ---------------------------------------------------------------------------
|
|
159
|
+
async ingest(options = {}) {
|
|
160
|
+
const { rootDir = join(homedir(), '.claude', 'projects'), projectFilter, since, dryRun = false, } = options;
|
|
161
|
+
const sinceMs = since ? Date.parse(since) : null;
|
|
162
|
+
if (since && (sinceMs === null || isNaN(sinceMs))) {
|
|
163
|
+
throw new Error(`Invalid since timestamp: ${since}`);
|
|
164
|
+
}
|
|
165
|
+
const stats = {
|
|
166
|
+
filesFound: 0,
|
|
167
|
+
filesSkipped: 0,
|
|
168
|
+
messagesIngested: 0,
|
|
169
|
+
messagesSkipped: 0,
|
|
170
|
+
bytesProcessed: 0,
|
|
171
|
+
projectSlugs: [],
|
|
172
|
+
dateRange: { earliest: null, latest: null },
|
|
173
|
+
parseErrors: 0,
|
|
174
|
+
parseErrorSamples: [],
|
|
175
|
+
dryRun,
|
|
176
|
+
};
|
|
177
|
+
if (!existsSync(rootDir)) {
|
|
178
|
+
return stats;
|
|
179
|
+
}
|
|
180
|
+
// Walk rootDir/slug/*.jsonl
|
|
181
|
+
const slugSet = new Set();
|
|
182
|
+
const jsonlFiles = this.collectJsonlFiles(rootDir, projectFilter);
|
|
183
|
+
for (const filePath of jsonlFiles) {
|
|
184
|
+
const slug = slugFromPath(filePath, rootDir);
|
|
185
|
+
slugSet.add(slug);
|
|
186
|
+
stats.filesFound++;
|
|
187
|
+
try {
|
|
188
|
+
await this.processFile(filePath, slug, sinceMs, dryRun, stats);
|
|
189
|
+
}
|
|
190
|
+
catch (err) {
|
|
191
|
+
stats.parseErrors++;
|
|
192
|
+
if (stats.parseErrorSamples.length < 3) {
|
|
193
|
+
stats.parseErrorSamples.push(`${filePath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
stats.projectSlugs = Array.from(slugSet).sort();
|
|
198
|
+
return stats;
|
|
199
|
+
}
|
|
200
|
+
// ---------------------------------------------------------------------------
|
|
201
|
+
// File collection
|
|
202
|
+
// ---------------------------------------------------------------------------
|
|
203
|
+
collectJsonlFiles(rootDir, projectFilter) {
|
|
204
|
+
const results = [];
|
|
205
|
+
let slugDirs;
|
|
206
|
+
try {
|
|
207
|
+
slugDirs = readdirSync(rootDir);
|
|
208
|
+
}
|
|
209
|
+
catch {
|
|
210
|
+
return results;
|
|
211
|
+
}
|
|
212
|
+
for (const slug of slugDirs) {
|
|
213
|
+
if (projectFilter && !slug.includes(projectFilter))
|
|
214
|
+
continue;
|
|
215
|
+
const slugPath = join(rootDir, slug);
|
|
216
|
+
let isDir = false;
|
|
217
|
+
try {
|
|
218
|
+
isDir = statSync(slugPath).isDirectory();
|
|
219
|
+
}
|
|
220
|
+
catch {
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
if (!isDir)
|
|
224
|
+
continue;
|
|
225
|
+
let entries;
|
|
226
|
+
try {
|
|
227
|
+
entries = readdirSync(slugPath);
|
|
228
|
+
}
|
|
229
|
+
catch {
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
for (const entry of entries) {
|
|
233
|
+
if (entry.endsWith('.jsonl')) {
|
|
234
|
+
results.push(join(slugPath, entry));
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return results;
|
|
239
|
+
}
|
|
240
|
+
// ---------------------------------------------------------------------------
|
|
241
|
+
// Per-file processing
|
|
242
|
+
// ---------------------------------------------------------------------------
|
|
243
|
+
async processFile(filePath, projectSlug, sinceMs, dryRun, stats) {
|
|
244
|
+
// Get last known offset for incremental processing
|
|
245
|
+
const stateRow = this.db.prepare(`SELECT last_byte_offset FROM transcript_ingest_state WHERE file_path = ?`).get(filePath);
|
|
246
|
+
const startOffset = stateRow?.last_byte_offset ?? 0;
|
|
247
|
+
// Get current file size to determine if there's new content
|
|
248
|
+
let fileSize = 0;
|
|
249
|
+
try {
|
|
250
|
+
fileSize = statSync(filePath).size;
|
|
251
|
+
}
|
|
252
|
+
catch {
|
|
253
|
+
stats.filesSkipped++;
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
if (fileSize <= startOffset) {
|
|
257
|
+
stats.filesSkipped++;
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
// Stream the file from start (we need to track line indices from 0 for dedup)
|
|
261
|
+
const messages = await this.streamFile(filePath, projectSlug, sinceMs, startOffset, stats);
|
|
262
|
+
if (messages.length === 0) {
|
|
263
|
+
// Update state even if no messages were parseable (to advance offset)
|
|
264
|
+
if (!dryRun) {
|
|
265
|
+
this.updateIngestState(filePath, fileSize, 0);
|
|
266
|
+
}
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
if (dryRun) {
|
|
270
|
+
stats.messagesIngested += messages.length;
|
|
271
|
+
this.updateDateRange(stats, messages);
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
// Insert in a transaction for atomicity
|
|
275
|
+
const insertCount = this.db.transaction(() => {
|
|
276
|
+
let inserted = 0;
|
|
277
|
+
for (const msg of messages) {
|
|
278
|
+
// Check dedup
|
|
279
|
+
const exists = this.db.prepare(`SELECT 1 FROM transcript_dedup WHERE file_path = ? AND message_index = ?`).get(msg.filePath, msg.messageIndex);
|
|
280
|
+
if (exists) {
|
|
281
|
+
stats.messagesSkipped++;
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
// Build memory content — prefix with role for search quality
|
|
285
|
+
const roleLabel = msg.role === 'user' ? '[User]' : '[Assistant]';
|
|
286
|
+
const content = `${roleLabel} ${msg.text}`;
|
|
287
|
+
// Save into Gaia memory store
|
|
288
|
+
const memory = this.memoria.saveMemory({
|
|
289
|
+
tier: 'project',
|
|
290
|
+
content,
|
|
291
|
+
tags: ['transcript', msg.role, msg.projectSlug],
|
|
292
|
+
related_files: [],
|
|
293
|
+
session_id: msg.sessionId,
|
|
294
|
+
project_path: msg.cwd ?? msg.projectSlug,
|
|
295
|
+
metadata: {
|
|
296
|
+
source: 'transcript',
|
|
297
|
+
session_id: msg.sessionId,
|
|
298
|
+
message_index: msg.messageIndex,
|
|
299
|
+
role: msg.role,
|
|
300
|
+
project_slug: msg.projectSlug,
|
|
301
|
+
file_path: msg.filePath,
|
|
302
|
+
line_offset: msg.lineOffset,
|
|
303
|
+
original_timestamp: msg.timestamp > 0 ? new Date(msg.timestamp).toISOString() : null,
|
|
304
|
+
},
|
|
305
|
+
});
|
|
306
|
+
// Record dedup entry
|
|
307
|
+
this.db.prepare(`INSERT OR IGNORE INTO transcript_dedup (file_path, message_index, memory_id) VALUES (?, ?, ?)`).run(msg.filePath, msg.messageIndex, memory.id);
|
|
308
|
+
inserted++;
|
|
309
|
+
stats.messagesIngested++;
|
|
310
|
+
}
|
|
311
|
+
return inserted;
|
|
312
|
+
})();
|
|
313
|
+
// Update incremental state to the current file size
|
|
314
|
+
this.updateIngestState(filePath, fileSize, insertCount);
|
|
315
|
+
this.updateDateRange(stats, messages);
|
|
316
|
+
stats.bytesProcessed += fileSize - startOffset;
|
|
317
|
+
}
|
|
318
|
+
// ---------------------------------------------------------------------------
|
|
319
|
+
// Streaming JSONL parser
|
|
320
|
+
// ---------------------------------------------------------------------------
|
|
321
|
+
streamFile(filePath, projectSlug, sinceMs, startOffset, stats) {
|
|
322
|
+
return new Promise((resolve, reject) => {
|
|
323
|
+
const messages = [];
|
|
324
|
+
// We always read from byte 0 to maintain correct lineIndex for dedup.
|
|
325
|
+
// Lines before startOffset are checked against dedup table and skipped
|
|
326
|
+
// in processFile, but we need their indices. For very large files already
|
|
327
|
+
// fully ingested, startOffset == fileSize so we return early above.
|
|
328
|
+
// The cost here is line counting from the start — acceptable since we
|
|
329
|
+
// only reach this if there's new content beyond the last offset.
|
|
330
|
+
const stream = createReadStream(filePath, { encoding: 'utf8' });
|
|
331
|
+
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
|
332
|
+
let lineIndex = 0;
|
|
333
|
+
let currentByteOffset = 0;
|
|
334
|
+
rl.on('line', (rawLine) => {
|
|
335
|
+
const lineByteLength = Buffer.byteLength(rawLine + '\n', 'utf8');
|
|
336
|
+
const lineStartOffset = currentByteOffset;
|
|
337
|
+
currentByteOffset += lineByteLength;
|
|
338
|
+
// Lines within already-ingested region: skip parsing but increment index
|
|
339
|
+
if (currentByteOffset <= startOffset) {
|
|
340
|
+
lineIndex++;
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
// Parse line
|
|
344
|
+
let parsed = null;
|
|
345
|
+
try {
|
|
346
|
+
parsed = parseLine(rawLine, lineIndex, filePath, projectSlug, sinceMs);
|
|
347
|
+
}
|
|
348
|
+
catch (err) {
|
|
349
|
+
stats.parseErrors++;
|
|
350
|
+
if (stats.parseErrorSamples.length < 3) {
|
|
351
|
+
stats.parseErrorSamples.push(`${filePath}:${lineIndex}: ${err instanceof Error ? err.message : String(err)}`);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
if (parsed) {
|
|
355
|
+
messages.push(parsed);
|
|
356
|
+
}
|
|
357
|
+
lineIndex++;
|
|
358
|
+
});
|
|
359
|
+
rl.on('close', () => resolve(messages));
|
|
360
|
+
rl.on('error', reject);
|
|
361
|
+
stream.on('error', reject);
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
// ---------------------------------------------------------------------------
|
|
365
|
+
// State tracking
|
|
366
|
+
// ---------------------------------------------------------------------------
|
|
367
|
+
updateIngestState(filePath, byteOffset, inserted) {
|
|
368
|
+
this.db.prepare(`
|
|
369
|
+
INSERT INTO transcript_ingest_state (file_path, last_byte_offset, last_ingested_at, messages_ingested)
|
|
370
|
+
VALUES (?, ?, ?, ?)
|
|
371
|
+
ON CONFLICT(file_path) DO UPDATE SET
|
|
372
|
+
last_byte_offset = excluded.last_byte_offset,
|
|
373
|
+
last_ingested_at = excluded.last_ingested_at,
|
|
374
|
+
messages_ingested = messages_ingested + excluded.messages_ingested
|
|
375
|
+
`).run(filePath, byteOffset, Date.now(), inserted);
|
|
376
|
+
}
|
|
377
|
+
updateDateRange(stats, messages) {
|
|
378
|
+
for (const msg of messages) {
|
|
379
|
+
if (msg.timestamp <= 0)
|
|
380
|
+
continue;
|
|
381
|
+
const iso = new Date(msg.timestamp).toISOString();
|
|
382
|
+
if (!stats.dateRange.earliest || iso < stats.dateRange.earliest) {
|
|
383
|
+
stats.dateRange.earliest = iso;
|
|
384
|
+
}
|
|
385
|
+
if (!stats.dateRange.latest || iso > stats.dateRange.latest) {
|
|
386
|
+
stats.dateRange.latest = iso;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
// ---------------------------------------------------------------------------
|
|
391
|
+
// Utility: exposed for tests
|
|
392
|
+
// ---------------------------------------------------------------------------
|
|
393
|
+
/** Returns ingest state rows for testing incremental behavior */
|
|
394
|
+
getIngestState() {
|
|
395
|
+
return this.db.prepare(`SELECT file_path, last_byte_offset, messages_ingested FROM transcript_ingest_state`).all();
|
|
396
|
+
}
|
|
397
|
+
/** Returns dedup row count for a given file */
|
|
398
|
+
getDedupCount(filePath) {
|
|
399
|
+
const row = this.db.prepare(`SELECT COUNT(*) as count FROM transcript_dedup WHERE file_path = ?`).get(filePath);
|
|
400
|
+
return row.count;
|
|
401
|
+
}
|
|
402
|
+
/** Nano-id helper (re-exported so tests can use it) */
|
|
403
|
+
static generateId() {
|
|
404
|
+
return `mem_${nanoid()}`;
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
// =============================================================================
|
|
408
|
+
// Standalone helper: build a TranscriptIngester from an existing MemoriaStorage
|
|
409
|
+
// =============================================================================
|
|
410
|
+
/**
|
|
411
|
+
* MemoriaStorage keeps its `db` private. We expose it through this module
|
|
412
|
+
* by accepting the db instance directly (tests pass an in-memory DB;
|
|
413
|
+
* the MCP tool passes the real db through getInternalDb).
|
|
414
|
+
*/
|
|
415
|
+
export function createIngester(db, memoria) {
|
|
416
|
+
return new TranscriptIngester(db, memoria);
|
|
417
|
+
}
|
|
418
|
+
//# sourceMappingURL=transcript-ingester.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transcript-ingester.js","sourceRoot":"","sources":["../../../src/tools/gaia/transcript-ingester.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AACzE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA4ChC,gFAAgF;AAChF,6EAA6E;AAC7E,6EAA6E;AAC7E,gFAAgF;AAEhF,MAAM,CAAC,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;CAgB/B,CAAC;AAEF,gFAAgF;AAChF,wBAAwB;AACxB,gFAAgF;AAEhF;;;;GAIG;AACH,SAAS,WAAW,CAAC,OAAgB;IACnC,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC/B,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;YAAE,SAAS;QAC1D,MAAM,CAAC,GAAG,KAAgC,CAAC;QAE3C,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,MAAM,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC1D,MAAM,CAAC,GAAI,CAAC,CAAC,MAAM,CAAY,CAAC,IAAI,EAAE,CAAC;YACvC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,gEAAgE;IAClE,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAChB,OAAe,EACf,SAAiB,EACjB,QAAgB,EAChB,WAAmB,EACnB,OAAsB;IAEtB,IAAI,GAA4B,CAAC;IACjC,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAA4B,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;IAEzB,4CAA4C;IAC5C,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAEzD,qDAAqD;IACrD,IAAI,GAAG,CAAC,aAAa,CAAC,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAE7C,4BAA4B;IAC5B,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACzD,MAAM,MAAM,GAAG,GAA8B,CAAC;IAE9C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5B,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAEzD,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAC5C,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,yEAAyE;IACzE,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,EAAE,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;IAC5B,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YAAE,SAAS,GAAG,MAAM,CAAC;IACzC,CAAC;SAAM,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;QAClC,SAAS,GAAG,EAAE,CAAC;IACjB,CAAC;IAED,eAAe;IACf,IAAI,OAAO,KAAK,IAAI,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1E,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,WAAW,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtF,MAAM,GAAG,GAAG,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE/D,OAAO;QACL,SAAS;QACT,YAAY,EAAE,SAAS;QACvB,SAAS;QACT,IAAI,EAAE,IAA4B;QAClC,IAAI;QACJ,WAAW;QACX,GAAG;QACH,QAAQ;QACR,UAAU,EAAE,SAAS;KACtB,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,0BAA0B;AAC1B,gFAAgF;AAEhF;;;GAGG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,OAAe;IACrD,uBAAuB;IACvB,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC;QAC3C,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,CAAC,CAAC,QAAQ,CAAC;IAEb,2CAA2C;IAC3C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC1D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,gFAAgF;AAChF,gBAAgB;AAChB,gFAAgF;AAEhF,MAAM,OAAO,kBAAkB;IACrB,EAAE,CAAoB;IACtB,OAAO,CAAiB;IAEhC,YAAY,SAA4B,EAAE,OAAuB;QAC/D,IAAI,CAAC,EAAE,GAAG,SAAS,CAAC;QACpB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC;IAEO,YAAY;QAClB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACjC,CAAC;IAED,8EAA8E;IAC9E,aAAa;IACb,8EAA8E;IAE9E,KAAK,CAAC,MAAM,CAAC,UAAyB,EAAE;QACtC,MAAM,EACJ,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,EAChD,aAAa,EACb,KAAK,EACL,MAAM,GAAG,KAAK,GACf,GAAG,OAAO,CAAC;QAEZ,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACjD,IAAI,KAAK,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,KAAK,GAAgB;YACzB,UAAU,EAAE,CAAC;YACb,YAAY,EAAE,CAAC;YACf,gBAAgB,EAAE,CAAC;YACnB,eAAe,EAAE,CAAC;YAClB,cAAc,EAAE,CAAC;YACjB,YAAY,EAAE,EAAE;YAChB,SAAS,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;YAC3C,WAAW,EAAE,CAAC;YACd,iBAAiB,EAAE,EAAE;YACrB,MAAM;SACP,CAAC;QAEF,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,4BAA4B;QAC5B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAElE,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClB,KAAK,CAAC,UAAU,EAAE,CAAC;YAEnB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACjE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,KAAK,CAAC,WAAW,EAAE,CAAC;gBACpB,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvC,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAC1B,GAAG,QAAQ,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACnE,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,8EAA8E;IAC9E,kBAAkB;IAClB,8EAA8E;IAEtE,iBAAiB,CAAC,OAAe,EAAE,aAAsB;QAC/D,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;gBAAE,SAAS;YAE7D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YACrC,IAAI,KAAK,GAAG,KAAK,CAAC;YAClB,IAAI,CAAC;gBACH,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;YAC3C,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB,IAAI,OAAiB,CAAC;YACtB,IAAI,CAAC;gBACH,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;YAClC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,8EAA8E;IAC9E,sBAAsB;IACtB,8EAA8E;IAEtE,KAAK,CAAC,WAAW,CACvB,QAAgB,EAChB,WAAmB,EACnB,OAAsB,EACtB,MAAe,EACf,KAAkB;QAElB,mDAAmD;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAC9B,0EAA0E,CAC3E,CAAC,GAAG,CAAC,QAAQ,CAA6C,CAAC;QAE5D,MAAM,WAAW,GAAG,QAAQ,EAAE,gBAAgB,IAAI,CAAC,CAAC;QAEpD,4DAA4D;QAC5D,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,CAAC,YAAY,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QAED,IAAI,QAAQ,IAAI,WAAW,EAAE,CAAC;YAC5B,KAAK,CAAC,YAAY,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QAED,8EAA8E;QAC9E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;QAE3F,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,sEAAsE;YACtE,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;YAChD,CAAC;YACD,OAAO;QACT,CAAC;QAED,IAAI,MAAM,EAAE,CAAC;YACX,KAAK,CAAC,gBAAgB,IAAI,QAAQ,CAAC,MAAM,CAAC;YAC1C,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YACtC,OAAO;QACT,CAAC;QAED,wCAAwC;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAC3C,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,cAAc;gBACd,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAC5B,0EAA0E,CAC3E,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;gBAEtC,IAAI,MAAM,EAAE,CAAC;oBACX,KAAK,CAAC,eAAe,EAAE,CAAC;oBACxB,SAAS;gBACX,CAAC;gBAED,6DAA6D;gBAC7D,MAAM,SAAS,GAAG,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC;gBACjE,MAAM,OAAO,GAAG,GAAG,SAAS,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBAE3C,8BAA8B;gBAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;oBACrC,IAAI,EAAE,SAAS;oBACf,OAAO;oBACP,IAAI,EAAE,CAAC,YAAY,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,WAAW,CAAC;oBAC/C,aAAa,EAAE,EAAE;oBACjB,UAAU,EAAE,GAAG,CAAC,SAAS;oBACzB,YAAY,EAAE,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW;oBACxC,QAAQ,EAAE;wBACR,MAAM,EAAE,YAAY;wBACpB,UAAU,EAAE,GAAG,CAAC,SAAS;wBACzB,aAAa,EAAE,GAAG,CAAC,YAAY;wBAC/B,IAAI,EAAE,GAAG,CAAC,IAAI;wBACd,YAAY,EAAE,GAAG,CAAC,WAAW;wBAC7B,SAAS,EAAE,GAAG,CAAC,QAAQ;wBACvB,WAAW,EAAE,GAAG,CAAC,UAAU;wBAC3B,kBAAkB,EAAE,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;qBACrF;iBACF,CAAC,CAAC;gBAEH,qBAAqB;gBACrB,IAAI,CAAC,EAAE,CAAC,OAAO,CACb,+FAA+F,CAChG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;gBAEjD,QAAQ,EAAE,CAAC;gBACX,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC3B,CAAC;YACD,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,EAAE,CAAC;QAEL,oDAAoD;QACpD,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QACxD,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACtC,KAAK,CAAC,cAAc,IAAI,QAAQ,GAAG,WAAW,CAAC;IACjD,CAAC;IAED,8EAA8E;IAC9E,yBAAyB;IACzB,8EAA8E;IAEtE,UAAU,CAChB,QAAgB,EAChB,WAAmB,EACnB,OAAsB,EACtB,WAAmB,EACnB,KAAkB;QAElB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,QAAQ,GAAoB,EAAE,CAAC;YAErC,sEAAsE;YACtE,uEAAuE;YACvE,0EAA0E;YAC1E,oEAAoE;YACpE,sEAAsE;YACtE,iEAAiE;YACjE,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;YAChE,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;YAEnE,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,iBAAiB,GAAG,CAAC,CAAC;YAE1B,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAe,EAAE,EAAE;gBAChC,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;gBACjE,MAAM,eAAe,GAAG,iBAAiB,CAAC;gBAC1C,iBAAiB,IAAI,cAAc,CAAC;gBAEpC,yEAAyE;gBACzE,IAAI,iBAAiB,IAAI,WAAW,EAAE,CAAC;oBACrC,SAAS,EAAE,CAAC;oBACZ,OAAO;gBACT,CAAC;gBAED,aAAa;gBACb,IAAI,MAAM,GAAyB,IAAI,CAAC;gBACxC,IAAI,CAAC;oBACH,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;gBACzE,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,KAAK,CAAC,WAAW,EAAE,CAAC;oBACpB,IAAI,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACvC,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAC1B,GAAG,QAAQ,IAAI,SAAS,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAChF,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAED,IAAI,MAAM,EAAE,CAAC;oBACX,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACxB,CAAC;gBAED,SAAS,EAAE,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YACxC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACvB,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;IACL,CAAC;IAED,8EAA8E;IAC9E,iBAAiB;IACjB,8EAA8E;IAEtE,iBAAiB,CAAC,QAAgB,EAAE,UAAkB,EAAE,QAAgB;QAC9E,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;KAOf,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAC;IACrD,CAAC;IAEO,eAAe,CAAC,KAAkB,EAAE,QAAyB;QACnE,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,GAAG,CAAC,SAAS,IAAI,CAAC;gBAAE,SAAS;YACjC,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;YAClD,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;gBAChE,KAAK,CAAC,SAAS,CAAC,QAAQ,GAAG,GAAG,CAAC;YACjC,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;gBAC5D,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,GAAG,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,6BAA6B;IAC7B,8EAA8E;IAE9E,iEAAiE;IACjE,cAAc;QACZ,OAAO,IAAI,CAAC,EAAE,CAAC,OAAO,CACpB,oFAAoF,CACrF,CAAC,GAAG,EAAuF,CAAC;IAC/F,CAAC;IAED,+CAA+C;IAC/C,aAAa,CAAC,QAAgB;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACzB,oEAAoE,CACrE,CAAC,GAAG,CAAC,QAAQ,CAAsB,CAAC;QACrC,OAAO,GAAG,CAAC,KAAK,CAAC;IACnB,CAAC;IAED,uDAAuD;IACvD,MAAM,CAAC,UAAU;QACf,OAAO,OAAO,MAAM,EAAE,EAAE,CAAC;IAC3B,CAAC;CACF;AAED,gFAAgF;AAChF,gFAAgF;AAChF,gFAAgF;AAEhF;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,EAAqB,EAAE,OAAuB;IAC3E,OAAO,IAAI,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vault Dashboard Generator
|
|
3
|
+
*
|
|
4
|
+
* Generates an Obsidian-compatible markdown file with Dataview plugin queries.
|
|
5
|
+
* The resulting file is written to Foundation/_dashboard.md inside the vault.
|
|
6
|
+
* Dataview handles all dynamic querying at render time — this function only
|
|
7
|
+
* produces the static template with embedded query blocks.
|
|
8
|
+
*/
|
|
9
|
+
export declare function generateDashboard(): string;
|
|
10
|
+
//# sourceMappingURL=dashboard.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dashboard.d.ts","sourceRoot":"","sources":["../../src/vault/dashboard.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,wBAAgB,iBAAiB,IAAI,MAAM,CAqI1C"}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vault Dashboard Generator
|
|
3
|
+
*
|
|
4
|
+
* Generates an Obsidian-compatible markdown file with Dataview plugin queries.
|
|
5
|
+
* The resulting file is written to Foundation/_dashboard.md inside the vault.
|
|
6
|
+
* Dataview handles all dynamic querying at render time — this function only
|
|
7
|
+
* produces the static template with embedded query blocks.
|
|
8
|
+
*/
|
|
9
|
+
// =============================================================================
|
|
10
|
+
// Dashboard Generator
|
|
11
|
+
// =============================================================================
|
|
12
|
+
export function generateDashboard() {
|
|
13
|
+
const timestamp = new Date().toISOString();
|
|
14
|
+
return `# Foundation Memory Dashboard
|
|
15
|
+
> Auto-generated by \`foundation vault sync\`. Do not edit manually.
|
|
16
|
+
> Last sync: ${timestamp}
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## Overview Stats
|
|
21
|
+
|
|
22
|
+
\`\`\`dataviewjs
|
|
23
|
+
const pages = dv.pages('"Foundation"').where(p => p.tier);
|
|
24
|
+
const tiers = ["session", "project", "global", "note", "observation"];
|
|
25
|
+
|
|
26
|
+
const rows = tiers.map(tier => {
|
|
27
|
+
const matches = pages.where(p => p.tier === tier);
|
|
28
|
+
const latest = matches.sort(p => p.created, "desc").first();
|
|
29
|
+
return [
|
|
30
|
+
tier,
|
|
31
|
+
matches.length,
|
|
32
|
+
latest ? latest.created : "—"
|
|
33
|
+
];
|
|
34
|
+
}).filter(r => r[1] > 0);
|
|
35
|
+
|
|
36
|
+
dv.table(["Tier", "Count", "Latest Memory"], rows);
|
|
37
|
+
\`\`\`
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## Recently Accessed
|
|
42
|
+
|
|
43
|
+
\`\`\`dataview
|
|
44
|
+
TABLE tier, tags, accessed, access_count
|
|
45
|
+
FROM "Foundation"
|
|
46
|
+
WHERE accessed != null
|
|
47
|
+
SORT accessed DESC
|
|
48
|
+
LIMIT 15
|
|
49
|
+
\`\`\`
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Most Connected
|
|
54
|
+
|
|
55
|
+
\`\`\`dataviewjs
|
|
56
|
+
const pages = dv.pages('"Foundation"');
|
|
57
|
+
|
|
58
|
+
const rows = pages
|
|
59
|
+
.map(p => {
|
|
60
|
+
const content = p.file.content ?? "";
|
|
61
|
+
const linkCount = (content.match(/\[\[/g) ?? []).length;
|
|
62
|
+
return [p.file.link, p.tier ?? "—", linkCount];
|
|
63
|
+
})
|
|
64
|
+
.sort(r => r[2], "desc")
|
|
65
|
+
.slice(0, 20);
|
|
66
|
+
|
|
67
|
+
dv.table(["Memory", "Tier", "Link Count"], rows);
|
|
68
|
+
\`\`\`
|
|
69
|
+
|
|
70
|
+
---
|
|
71
|
+
|
|
72
|
+
## Contradictions & Reverts
|
|
73
|
+
|
|
74
|
+
> Memories flagged with **contradicts** or **reverts** links — require human review.
|
|
75
|
+
|
|
76
|
+
\`\`\`dataviewjs
|
|
77
|
+
const pages = dv.pages('"Foundation"');
|
|
78
|
+
|
|
79
|
+
const flagged = pages.filter(p => {
|
|
80
|
+
const content = p.file.content ?? "";
|
|
81
|
+
return content.includes("**contradicts**") || content.includes("**reverts**");
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
if (flagged.length === 0) {
|
|
85
|
+
dv.paragraph("No contradictions or reverts detected.");
|
|
86
|
+
} else {
|
|
87
|
+
dv.list(flagged.map(p => p.file.link));
|
|
88
|
+
}
|
|
89
|
+
\`\`\`
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## By Project
|
|
94
|
+
|
|
95
|
+
\`\`\`dataview
|
|
96
|
+
TABLE rows.file.link AS "Memories", length(rows) AS "Count"
|
|
97
|
+
FROM "Foundation"
|
|
98
|
+
WHERE project != null
|
|
99
|
+
GROUP BY project
|
|
100
|
+
SORT length(rows) DESC
|
|
101
|
+
\`\`\`
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## Tag Cloud
|
|
106
|
+
|
|
107
|
+
\`\`\`dataviewjs
|
|
108
|
+
const pages = dv.pages('"Foundation"');
|
|
109
|
+
|
|
110
|
+
const tagCounts = {};
|
|
111
|
+
for (const page of pages) {
|
|
112
|
+
for (const tag of (page.tags ?? [])) {
|
|
113
|
+
tagCounts[tag] = (tagCounts[tag] ?? 0) + 1;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const sorted = Object.entries(tagCounts)
|
|
118
|
+
.sort((a, b) => b[1] - a[1]);
|
|
119
|
+
|
|
120
|
+
dv.list(sorted.map(([tag, count]) => \`\${tag} — \${count}\`));
|
|
121
|
+
\`\`\`
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## Recent Sessions
|
|
126
|
+
|
|
127
|
+
\`\`\`dataview
|
|
128
|
+
TABLE tier, tags, created, access_count
|
|
129
|
+
FROM "Foundation/Sessions"
|
|
130
|
+
SORT created DESC
|
|
131
|
+
LIMIT 20
|
|
132
|
+
\`\`\`
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
## Knowledge Base
|
|
137
|
+
|
|
138
|
+
\`\`\`dataview
|
|
139
|
+
TABLE type, description
|
|
140
|
+
FROM "Knowledge"
|
|
141
|
+
SORT file.name ASC
|
|
142
|
+
\`\`\`
|
|
143
|
+
`;
|
|
144
|
+
}
|
|
145
|
+
//# sourceMappingURL=dashboard.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dashboard.js","sourceRoot":"","sources":["../../src/vault/dashboard.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,gFAAgF;AAChF,sBAAsB;AACtB,gFAAgF;AAEhF,MAAM,UAAU,iBAAiB;IAC/B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAE3C,OAAO;;eAEM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+HvB,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vault Sync Engine
|
|
3
|
+
*
|
|
4
|
+
* Syncs Gaia memories from SQLite to an Obsidian vault as interlinked
|
|
5
|
+
* markdown files. Supports full and incremental sync, Claude Code memory
|
|
6
|
+
* file ingestion, and orphan cleanup.
|
|
7
|
+
*/
|
|
8
|
+
export interface VaultConfig {
|
|
9
|
+
/** Absolute path to the Obsidian vault root. */
|
|
10
|
+
vaultPath: string;
|
|
11
|
+
/** Absolute path to the Gaia SQLite database. */
|
|
12
|
+
gaiaDbPath: string;
|
|
13
|
+
/** Absolute path to the sync state JSON file. */
|
|
14
|
+
syncStatePath: string;
|
|
15
|
+
/** Optional path to Claude Code memory directory (e.g. ~/.claude/projects/.../memory/). */
|
|
16
|
+
claudeMemoryPath?: string;
|
|
17
|
+
}
|
|
18
|
+
export interface SyncState {
|
|
19
|
+
vault_path: string;
|
|
20
|
+
last_full_sync: string;
|
|
21
|
+
synced_memories: Record<string, string>;
|
|
22
|
+
}
|
|
23
|
+
export interface SyncResult {
|
|
24
|
+
created: number;
|
|
25
|
+
updated: number;
|
|
26
|
+
deleted: number;
|
|
27
|
+
skipped: number;
|
|
28
|
+
errors: string[];
|
|
29
|
+
}
|
|
30
|
+
export declare class VaultSync {
|
|
31
|
+
private config;
|
|
32
|
+
constructor(config: VaultConfig);
|
|
33
|
+
/**
|
|
34
|
+
* Create the required folder structure in the vault and write an initial
|
|
35
|
+
* dashboard index file.
|
|
36
|
+
*/
|
|
37
|
+
init(): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Full sync: reads ALL memories from Gaia DB, transforms and writes every
|
|
40
|
+
* file, updates sync state. Also syncs Claude Code memory files if
|
|
41
|
+
* claudeMemoryPath is configured.
|
|
42
|
+
*/
|
|
43
|
+
sync(): Promise<SyncResult>;
|
|
44
|
+
/**
|
|
45
|
+
* Incremental sync: only writes memories whose rendered content has changed
|
|
46
|
+
* since the last sync (detected via MD5 hash comparison).
|
|
47
|
+
*/
|
|
48
|
+
syncIncremental(): Promise<SyncResult>;
|
|
49
|
+
/**
|
|
50
|
+
* Returns the current sync state, or null if no sync has run yet.
|
|
51
|
+
*/
|
|
52
|
+
getStatus(): SyncState | null;
|
|
53
|
+
private loadSyncState;
|
|
54
|
+
private saveSyncState;
|
|
55
|
+
private hashContent;
|
|
56
|
+
private syncGaiaMemories;
|
|
57
|
+
private syncClaudeMemories;
|
|
58
|
+
/**
|
|
59
|
+
* Walk Foundation/ in the vault and remove .md files whose memory id is no
|
|
60
|
+
* longer in currentIds. Returns the number of files deleted.
|
|
61
|
+
*/
|
|
62
|
+
private cleanOrphans;
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=sync.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../../src/vault/sync.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AA0BH,MAAM,WAAW,WAAW;IAC1B,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,iDAAiD;IACjD,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,aAAa,EAAE,MAAM,CAAC;IACtB,2FAA2F;IAC3F,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,SAAS;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAoBD,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAc;gBAEhB,MAAM,EAAE,WAAW;IAQ/B;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAc3B;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,UAAU,CAAC;IAajC;;;OAGG;IACG,eAAe,IAAI,OAAO,CAAC,UAAU,CAAC;IAa5C;;OAEG;IACH,SAAS,IAAI,SAAS,GAAG,IAAI;IAa7B,OAAO,CAAC,aAAa;IAWrB,OAAO,CAAC,aAAa;IAarB,OAAO,CAAC,WAAW;YAQL,gBAAgB;YAwFhB,kBAAkB;IAoEhC;;;OAGG;IACH,OAAO,CAAC,YAAY;CA4DrB"}
|