@rlabs-inc/memory 0.3.7 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +809 -548
- package/dist/index.mjs +809 -548
- package/dist/server/index.js +961 -590
- package/dist/server/index.mjs +961 -590
- package/package.json +1 -1
- package/skills/memory-management.md +16 -3
- package/src/cli/commands/ingest.ts +214 -0
- package/src/cli/index.ts +20 -1
- package/src/core/curator.ts +28 -12
- package/src/core/engine.ts +3 -2
- package/src/core/session-parser.ts +955 -0
- package/src/core/store.ts +38 -29
- package/src/types/schema.ts +13 -0
|
@@ -0,0 +1,955 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// SESSION PARSER - Parse Claude Code JSONL transcripts into API-ready format
|
|
3
|
+
// Preserves ALL content blocks: text, thinking, tool_use, tool_result, images
|
|
4
|
+
// Each JSONL file = one session = one complete conversation
|
|
5
|
+
// ============================================================================
|
|
6
|
+
|
|
7
|
+
import { readdir, stat } from 'fs/promises'
|
|
8
|
+
import { join, basename } from 'path'
|
|
9
|
+
import { homedir } from 'os'
|
|
10
|
+
|
|
11
|
+
// ============================================================================
|
|
12
|
+
// TYPES - All content block types from Claude Code sessions
|
|
13
|
+
// ============================================================================
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Text content block
|
|
17
|
+
*/
|
|
18
|
+
export interface TextBlock {
|
|
19
|
+
type: 'text'
|
|
20
|
+
text: string
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Thinking/reasoning block (Claude's internal reasoning)
|
|
25
|
+
*/
|
|
26
|
+
export interface ThinkingBlock {
|
|
27
|
+
type: 'thinking'
|
|
28
|
+
thinking: string
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Image content block
|
|
33
|
+
*/
|
|
34
|
+
export interface ImageBlock {
|
|
35
|
+
type: 'image'
|
|
36
|
+
source: {
|
|
37
|
+
type: 'base64'
|
|
38
|
+
media_type: string // e.g., 'image/png', 'image/jpeg'
|
|
39
|
+
data: string // base64 encoded image data
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Tool use block - Claude calling a tool
|
|
45
|
+
*/
|
|
46
|
+
export interface ToolUseBlock {
|
|
47
|
+
type: 'tool_use'
|
|
48
|
+
id: string // Unique tool call ID
|
|
49
|
+
name: string // Tool name: Bash, Read, Write, Edit, Glob, Grep, etc.
|
|
50
|
+
input: Record<string, any> // Tool-specific input parameters
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Tool result block - Result from a tool call
|
|
55
|
+
*/
|
|
56
|
+
export interface ToolResultBlock {
|
|
57
|
+
type: 'tool_result'
|
|
58
|
+
tool_use_id: string // References the tool_use id
|
|
59
|
+
content: string | ContentBlock[] // Can be string or nested blocks
|
|
60
|
+
is_error?: boolean // True if the tool call failed
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* All possible content block types
|
|
65
|
+
*/
|
|
66
|
+
export type ContentBlock =
|
|
67
|
+
| TextBlock
|
|
68
|
+
| ThinkingBlock
|
|
69
|
+
| ImageBlock
|
|
70
|
+
| ToolUseBlock
|
|
71
|
+
| ToolResultBlock
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* A single message in the conversation
|
|
75
|
+
*/
|
|
76
|
+
export interface ParsedMessage {
|
|
77
|
+
role: 'user' | 'assistant'
|
|
78
|
+
content: string | ContentBlock[]
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Raw entry from JSONL file - includes ALL possible fields
|
|
83
|
+
*/
|
|
84
|
+
interface RawLogEntry {
|
|
85
|
+
// Entry type
|
|
86
|
+
type: 'user' | 'assistant' | 'summary' | 'meta' | string
|
|
87
|
+
|
|
88
|
+
// Message content
|
|
89
|
+
message?: {
|
|
90
|
+
role: 'user' | 'assistant'
|
|
91
|
+
content: string | ContentBlock[]
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Timestamps and IDs
|
|
95
|
+
timestamp?: string
|
|
96
|
+
uuid?: string
|
|
97
|
+
sessionId?: string
|
|
98
|
+
leafUuid?: string
|
|
99
|
+
|
|
100
|
+
// Context metadata
|
|
101
|
+
cwd?: string
|
|
102
|
+
gitBranch?: string
|
|
103
|
+
|
|
104
|
+
// Flags
|
|
105
|
+
isCompactSummary?: boolean
|
|
106
|
+
isMeta?: boolean
|
|
107
|
+
|
|
108
|
+
// Summary entry specific
|
|
109
|
+
summary?: string
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* A complete parsed session
|
|
114
|
+
*/
|
|
115
|
+
export interface ParsedSession {
|
|
116
|
+
/** Session ID (filename without extension) */
|
|
117
|
+
id: string
|
|
118
|
+
/** Session ID from within the JSONL (if different from filename) */
|
|
119
|
+
internalSessionId?: string
|
|
120
|
+
/** Project ID (folder name) */
|
|
121
|
+
projectId: string
|
|
122
|
+
/** Human-readable project name */
|
|
123
|
+
projectName: string
|
|
124
|
+
/** Full path to the JSONL file */
|
|
125
|
+
filepath: string
|
|
126
|
+
/** All messages in API-ready format */
|
|
127
|
+
messages: ParsedMessage[]
|
|
128
|
+
/** Session timestamps */
|
|
129
|
+
timestamps: {
|
|
130
|
+
first?: string
|
|
131
|
+
last?: string
|
|
132
|
+
}
|
|
133
|
+
/** Session context (from first user message) */
|
|
134
|
+
context?: {
|
|
135
|
+
cwd?: string
|
|
136
|
+
gitBranch?: string
|
|
137
|
+
}
|
|
138
|
+
/** Session metadata */
|
|
139
|
+
metadata: {
|
|
140
|
+
messageCount: number
|
|
141
|
+
userMessageCount: number
|
|
142
|
+
assistantMessageCount: number
|
|
143
|
+
toolUseCount: number
|
|
144
|
+
toolResultCount: number
|
|
145
|
+
hasThinkingBlocks: boolean
|
|
146
|
+
hasImages: boolean
|
|
147
|
+
isCompactSummary: boolean
|
|
148
|
+
hasMetaMessages: boolean
|
|
149
|
+
/** Estimated token count (rough: ~4 chars per token) */
|
|
150
|
+
estimatedTokens: number
|
|
151
|
+
/** File size in bytes */
|
|
152
|
+
fileSize: number
|
|
153
|
+
}
|
|
154
|
+
/** Optional session summary if present in JSONL */
|
|
155
|
+
summary?: string
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* A conversation within a session (user prompt + all responses)
|
|
160
|
+
* This is the natural unit - user asks something, Claude responds
|
|
161
|
+
*/
|
|
162
|
+
export interface Conversation {
|
|
163
|
+
/** User's prompt text (extracted from content) */
|
|
164
|
+
userText: string
|
|
165
|
+
/** Timestamp of the user prompt */
|
|
166
|
+
timestamp?: string
|
|
167
|
+
/** All messages in this conversation (user prompt + assistant responses + tool results) */
|
|
168
|
+
messages: ParsedMessage[]
|
|
169
|
+
/** Whether this is a continuation after context compaction */
|
|
170
|
+
isContinuation: boolean
|
|
171
|
+
/** Estimated tokens for this conversation */
|
|
172
|
+
estimatedTokens: number
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* A segment of a session (one or more conversations batched together)
|
|
177
|
+
* Used when sessions are too large for a single API call
|
|
178
|
+
*/
|
|
179
|
+
export interface SessionSegment {
|
|
180
|
+
/** Parent session ID */
|
|
181
|
+
sessionId: string
|
|
182
|
+
/** Segment index (0-based) */
|
|
183
|
+
segmentIndex: number
|
|
184
|
+
/** Total segments in this session */
|
|
185
|
+
totalSegments: number
|
|
186
|
+
/** Project ID */
|
|
187
|
+
projectId: string
|
|
188
|
+
/** Human-readable project name */
|
|
189
|
+
projectName: string
|
|
190
|
+
/** Messages in this segment (flattened from conversations) */
|
|
191
|
+
messages: ParsedMessage[]
|
|
192
|
+
/** Conversations in this segment (original grouping) */
|
|
193
|
+
conversations: Conversation[]
|
|
194
|
+
/** Timestamps for this segment */
|
|
195
|
+
timestamps: {
|
|
196
|
+
first?: string
|
|
197
|
+
last?: string
|
|
198
|
+
}
|
|
199
|
+
/** Whether this segment starts with a continuation (compacted context) */
|
|
200
|
+
startsWithContinuation: boolean
|
|
201
|
+
/** Estimated tokens in this segment */
|
|
202
|
+
estimatedTokens: number
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Project with its sessions
|
|
207
|
+
*/
|
|
208
|
+
export interface ParsedProject {
|
|
209
|
+
/** Raw folder name (e.g., -home-user-projects-foo) */
|
|
210
|
+
folderId: string
|
|
211
|
+
/** Human-readable name (e.g., foo) */
|
|
212
|
+
name: string
|
|
213
|
+
/** Full path to project folder */
|
|
214
|
+
path: string
|
|
215
|
+
/** All sessions in this project */
|
|
216
|
+
sessions: ParsedSession[]
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
// ============================================================================
|
|
220
|
+
// PARSING FUNCTIONS
|
|
221
|
+
// ============================================================================
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Estimate token count from text (rough: ~4 chars per token)
|
|
225
|
+
*/
|
|
226
|
+
function estimateTokens(text: string): number {
|
|
227
|
+
return Math.ceil(text.length / 4)
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Estimate tokens for a content block or message
|
|
232
|
+
*/
|
|
233
|
+
function estimateContentTokens(content: string | ContentBlock[]): number {
|
|
234
|
+
if (typeof content === 'string') {
|
|
235
|
+
return estimateTokens(content)
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
let tokens = 0
|
|
239
|
+
for (const block of content) {
|
|
240
|
+
if (typeof block === 'object' && block !== null) {
|
|
241
|
+
if (block.type === 'text' && 'text' in block) {
|
|
242
|
+
tokens += estimateTokens(block.text)
|
|
243
|
+
} else if (block.type === 'thinking' && 'thinking' in block) {
|
|
244
|
+
tokens += estimateTokens(block.thinking)
|
|
245
|
+
} else if (block.type === 'tool_use' && 'input' in block) {
|
|
246
|
+
tokens += estimateTokens(JSON.stringify(block.input))
|
|
247
|
+
} else if (block.type === 'tool_result' && 'content' in block) {
|
|
248
|
+
if (typeof block.content === 'string') {
|
|
249
|
+
tokens += estimateTokens(block.content)
|
|
250
|
+
} else {
|
|
251
|
+
tokens += estimateContentTokens(block.content)
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
// Images: roughly 1000 tokens per image (varies by size)
|
|
255
|
+
if (block.type === 'image') {
|
|
256
|
+
tokens += 1000
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
return tokens
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Internal message type with metadata for segmentation
|
|
265
|
+
*/
|
|
266
|
+
interface ParsedMessageWithMeta extends ParsedMessage {
|
|
267
|
+
timestamp?: string
|
|
268
|
+
isCompactSummary?: boolean
|
|
269
|
+
estimatedTokens: number
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Parse a single JSONL file into a complete session
|
|
274
|
+
* Preserves ALL content blocks - nothing is lost
|
|
275
|
+
*/
|
|
276
|
+
export async function parseSessionFile(filepath: string): Promise<ParsedSession> {
|
|
277
|
+
const file = Bun.file(filepath)
|
|
278
|
+
const content = await file.text()
|
|
279
|
+
const fileSize = file.size
|
|
280
|
+
const lines = content.split('\n').filter(line => line.trim())
|
|
281
|
+
|
|
282
|
+
const messages: ParsedMessage[] = []
|
|
283
|
+
const timestamps: string[] = []
|
|
284
|
+
let summary: string | undefined
|
|
285
|
+
let isCompactSummary = false
|
|
286
|
+
let totalEstimatedTokens = 0
|
|
287
|
+
let internalSessionId: string | undefined
|
|
288
|
+
let context: { cwd?: string; gitBranch?: string } | undefined
|
|
289
|
+
|
|
290
|
+
// Stats
|
|
291
|
+
let toolUseCount = 0
|
|
292
|
+
let toolResultCount = 0
|
|
293
|
+
let hasThinkingBlocks = false
|
|
294
|
+
let hasImages = false
|
|
295
|
+
let hasMetaMessages = false
|
|
296
|
+
|
|
297
|
+
for (const line of lines) {
|
|
298
|
+
try {
|
|
299
|
+
const entry: RawLogEntry = JSON.parse(line)
|
|
300
|
+
|
|
301
|
+
// Capture summary if present
|
|
302
|
+
if (entry.type === 'summary' && entry.summary) {
|
|
303
|
+
summary = entry.summary
|
|
304
|
+
continue
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
// Skip non-message entries (meta, etc.)
|
|
308
|
+
if (entry.type !== 'user' && entry.type !== 'assistant') {
|
|
309
|
+
continue
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Track if we see meta messages (but still skip them like Python does)
|
|
313
|
+
if (entry.isMeta) {
|
|
314
|
+
hasMetaMessages = true
|
|
315
|
+
continue // Skip meta messages - they're system messages
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
// Skip entries without message data
|
|
319
|
+
if (!entry.message) {
|
|
320
|
+
continue
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// Capture session ID from entry (first one we see)
|
|
324
|
+
if (!internalSessionId && entry.sessionId) {
|
|
325
|
+
internalSessionId = entry.sessionId
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// Capture context from first user message
|
|
329
|
+
if (!context && entry.type === 'user') {
|
|
330
|
+
if (entry.cwd || entry.gitBranch) {
|
|
331
|
+
context = {
|
|
332
|
+
cwd: entry.cwd,
|
|
333
|
+
gitBranch: entry.gitBranch
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// Track compact summary flag
|
|
339
|
+
if (entry.isCompactSummary) {
|
|
340
|
+
isCompactSummary = true
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// Capture timestamp
|
|
344
|
+
if (entry.timestamp) {
|
|
345
|
+
timestamps.push(entry.timestamp)
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// Extract the message - preserve ALL content
|
|
349
|
+
const message: ParsedMessage = {
|
|
350
|
+
role: entry.message.role,
|
|
351
|
+
content: entry.message.content
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// Estimate tokens for this message
|
|
355
|
+
const msgTokens = estimateContentTokens(message.content)
|
|
356
|
+
totalEstimatedTokens += msgTokens
|
|
357
|
+
|
|
358
|
+
// Analyze content for stats
|
|
359
|
+
if (Array.isArray(message.content)) {
|
|
360
|
+
for (const block of message.content) {
|
|
361
|
+
if (typeof block === 'object' && block !== null) {
|
|
362
|
+
if (block.type === 'tool_use') toolUseCount++
|
|
363
|
+
if (block.type === 'tool_result') toolResultCount++
|
|
364
|
+
if (block.type === 'thinking') hasThinkingBlocks = true
|
|
365
|
+
if (block.type === 'image') hasImages = true
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
messages.push(message)
|
|
371
|
+
} catch {
|
|
372
|
+
// Skip malformed lines
|
|
373
|
+
continue
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
// Extract IDs from filepath
|
|
378
|
+
const filename = basename(filepath)
|
|
379
|
+
const sessionId = filename.replace(/\.jsonl$/, '')
|
|
380
|
+
const projectFolder = basename(join(filepath, '..'))
|
|
381
|
+
|
|
382
|
+
return {
|
|
383
|
+
id: sessionId,
|
|
384
|
+
internalSessionId,
|
|
385
|
+
projectId: projectFolder,
|
|
386
|
+
projectName: getProjectDisplayName(projectFolder),
|
|
387
|
+
filepath,
|
|
388
|
+
messages,
|
|
389
|
+
timestamps: {
|
|
390
|
+
first: timestamps[0],
|
|
391
|
+
last: timestamps[timestamps.length - 1]
|
|
392
|
+
},
|
|
393
|
+
context,
|
|
394
|
+
metadata: {
|
|
395
|
+
messageCount: messages.length,
|
|
396
|
+
userMessageCount: messages.filter(m => m.role === 'user').length,
|
|
397
|
+
assistantMessageCount: messages.filter(m => m.role === 'assistant').length,
|
|
398
|
+
toolUseCount,
|
|
399
|
+
toolResultCount,
|
|
400
|
+
hasThinkingBlocks,
|
|
401
|
+
hasImages,
|
|
402
|
+
isCompactSummary,
|
|
403
|
+
hasMetaMessages,
|
|
404
|
+
estimatedTokens: totalEstimatedTokens,
|
|
405
|
+
fileSize
|
|
406
|
+
},
|
|
407
|
+
summary
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/**
|
|
412
|
+
* Extract plain text from content (matches Python's extract_text_from_content)
|
|
413
|
+
*/
|
|
414
|
+
function extractTextFromContent(content: string | ContentBlock[]): string {
|
|
415
|
+
if (typeof content === 'string') {
|
|
416
|
+
return content.trim()
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
const texts: string[] = []
|
|
420
|
+
for (const block of content) {
|
|
421
|
+
if (typeof block === 'object' && block !== null && block.type === 'text' && 'text' in block) {
|
|
422
|
+
const text = block.text
|
|
423
|
+
if (text) texts.push(text)
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
return texts.join(' ').trim()
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* Parse a session file and group messages into conversations
|
|
431
|
+
* A conversation starts with a user prompt (text, not just tool results)
|
|
432
|
+
* and includes all subsequent messages until the next user prompt
|
|
433
|
+
*/
|
|
434
|
+
export async function parseSessionConversations(filepath: string): Promise<Conversation[]> {
|
|
435
|
+
const file = Bun.file(filepath)
|
|
436
|
+
const content = await file.text()
|
|
437
|
+
const lines = content.split('\n').filter(line => line.trim())
|
|
438
|
+
|
|
439
|
+
const conversations: Conversation[] = []
|
|
440
|
+
let currentConv: {
|
|
441
|
+
userText: string
|
|
442
|
+
timestamp?: string
|
|
443
|
+
messages: ParsedMessageWithMeta[]
|
|
444
|
+
isContinuation: boolean
|
|
445
|
+
} | null = null
|
|
446
|
+
|
|
447
|
+
for (const line of lines) {
|
|
448
|
+
try {
|
|
449
|
+
const entry: RawLogEntry = JSON.parse(line)
|
|
450
|
+
|
|
451
|
+
// Skip non-message entries
|
|
452
|
+
if (entry.type !== 'user' && entry.type !== 'assistant') continue
|
|
453
|
+
if (entry.isMeta) continue
|
|
454
|
+
if (!entry.message) continue
|
|
455
|
+
|
|
456
|
+
const message: ParsedMessageWithMeta = {
|
|
457
|
+
role: entry.message.role,
|
|
458
|
+
content: entry.message.content,
|
|
459
|
+
timestamp: entry.timestamp,
|
|
460
|
+
isCompactSummary: entry.isCompactSummary,
|
|
461
|
+
estimatedTokens: estimateContentTokens(entry.message.content)
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// Check if this is a user prompt (has text content, not just tool results)
|
|
465
|
+
let isUserPrompt = false
|
|
466
|
+
let userText = ''
|
|
467
|
+
|
|
468
|
+
if (entry.type === 'user') {
|
|
469
|
+
userText = extractTextFromContent(entry.message.content)
|
|
470
|
+
if (userText) {
|
|
471
|
+
isUserPrompt = true
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
if (isUserPrompt) {
|
|
476
|
+
// Flush previous conversation
|
|
477
|
+
if (currentConv) {
|
|
478
|
+
const tokens = currentConv.messages.reduce((sum, m) => sum + m.estimatedTokens, 0)
|
|
479
|
+
conversations.push({
|
|
480
|
+
userText: currentConv.userText,
|
|
481
|
+
timestamp: currentConv.timestamp,
|
|
482
|
+
messages: currentConv.messages.map(m => ({ role: m.role, content: m.content })),
|
|
483
|
+
isContinuation: currentConv.isContinuation,
|
|
484
|
+
estimatedTokens: tokens
|
|
485
|
+
})
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
// Start new conversation
|
|
489
|
+
currentConv = {
|
|
490
|
+
userText,
|
|
491
|
+
timestamp: entry.timestamp,
|
|
492
|
+
messages: [message],
|
|
493
|
+
isContinuation: Boolean(entry.isCompactSummary)
|
|
494
|
+
}
|
|
495
|
+
} else if (currentConv) {
|
|
496
|
+
// Add to current conversation
|
|
497
|
+
currentConv.messages.push(message)
|
|
498
|
+
}
|
|
499
|
+
} catch {
|
|
500
|
+
continue
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// Flush final conversation
|
|
505
|
+
if (currentConv) {
|
|
506
|
+
const tokens = currentConv.messages.reduce((sum, m) => sum + m.estimatedTokens, 0)
|
|
507
|
+
conversations.push({
|
|
508
|
+
userText: currentConv.userText,
|
|
509
|
+
timestamp: currentConv.timestamp,
|
|
510
|
+
messages: currentConv.messages.map(m => ({ role: m.role, content: m.content })),
|
|
511
|
+
isContinuation: currentConv.isContinuation,
|
|
512
|
+
estimatedTokens: tokens
|
|
513
|
+
})
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
return conversations
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* Parse a session file and return segments (batches of conversations)
|
|
521
|
+
* Splits at conversation boundaries to respect token limits
|
|
522
|
+
*
|
|
523
|
+
* @param filepath Path to the JSONL file
|
|
524
|
+
* @param maxTokensPerSegment Maximum tokens per segment (default: 150000 for Claude's context)
|
|
525
|
+
*/
|
|
526
|
+
export async function parseSessionFileWithSegments(
|
|
527
|
+
filepath: string,
|
|
528
|
+
maxTokensPerSegment = 150000
|
|
529
|
+
): Promise<SessionSegment[]> {
|
|
530
|
+
// First, get all conversations
|
|
531
|
+
const conversations = await parseSessionConversations(filepath)
|
|
532
|
+
|
|
533
|
+
if (conversations.length === 0) {
|
|
534
|
+
return []
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
// Extract IDs from filepath
|
|
538
|
+
const filename = basename(filepath)
|
|
539
|
+
const sessionId = filename.replace(/\.jsonl$/, '')
|
|
540
|
+
const projectFolder = basename(join(filepath, '..'))
|
|
541
|
+
const projectName = getProjectDisplayName(projectFolder)
|
|
542
|
+
|
|
543
|
+
// Batch conversations into segments based on token limits
|
|
544
|
+
const segments: SessionSegment[] = []
|
|
545
|
+
let currentConversations: Conversation[] = []
|
|
546
|
+
let currentTokens = 0
|
|
547
|
+
let segmentIndex = 0
|
|
548
|
+
|
|
549
|
+
const flushSegment = () => {
|
|
550
|
+
if (currentConversations.length === 0) return
|
|
551
|
+
|
|
552
|
+
// Flatten messages from all conversations
|
|
553
|
+
const allMessages = currentConversations.flatMap(c => c.messages)
|
|
554
|
+
const allTimestamps = currentConversations
|
|
555
|
+
.map(c => c.timestamp)
|
|
556
|
+
.filter((t): t is string => !!t)
|
|
557
|
+
|
|
558
|
+
segments.push({
|
|
559
|
+
sessionId,
|
|
560
|
+
segmentIndex,
|
|
561
|
+
totalSegments: 0, // Will update after all segments are created
|
|
562
|
+
projectId: projectFolder,
|
|
563
|
+
projectName,
|
|
564
|
+
messages: allMessages,
|
|
565
|
+
conversations: currentConversations,
|
|
566
|
+
timestamps: {
|
|
567
|
+
first: allTimestamps[0],
|
|
568
|
+
last: allTimestamps[allTimestamps.length - 1]
|
|
569
|
+
},
|
|
570
|
+
startsWithContinuation: currentConversations[0]?.isContinuation ?? false,
|
|
571
|
+
estimatedTokens: currentTokens
|
|
572
|
+
})
|
|
573
|
+
|
|
574
|
+
segmentIndex++
|
|
575
|
+
currentConversations = []
|
|
576
|
+
currentTokens = 0
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
for (const conv of conversations) {
|
|
580
|
+
// If this single conversation exceeds limit, it becomes its own segment
|
|
581
|
+
if (conv.estimatedTokens > maxTokensPerSegment) {
|
|
582
|
+
// Flush current batch first
|
|
583
|
+
flushSegment()
|
|
584
|
+
// Add oversized conversation as its own segment
|
|
585
|
+
currentConversations = [conv]
|
|
586
|
+
currentTokens = conv.estimatedTokens
|
|
587
|
+
flushSegment()
|
|
588
|
+
continue
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
// Check if adding this conversation would exceed limit
|
|
592
|
+
if (currentTokens + conv.estimatedTokens > maxTokensPerSegment && currentConversations.length > 0) {
|
|
593
|
+
flushSegment()
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
currentConversations.push(conv)
|
|
597
|
+
currentTokens += conv.estimatedTokens
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
// Flush final segment
|
|
601
|
+
flushSegment()
|
|
602
|
+
|
|
603
|
+
// Update totalSegments in all segments
|
|
604
|
+
const totalSegments = segments.length
|
|
605
|
+
for (const segment of segments) {
|
|
606
|
+
segment.totalSegments = totalSegments
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
return segments
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
/**
|
|
613
|
+
* Convert encoded folder name to readable project name
|
|
614
|
+
* e.g., -home-user-projects-myproject -> myproject
|
|
615
|
+
*/
|
|
616
|
+
export function getProjectDisplayName(folderName: string): string {
|
|
617
|
+
// Common path prefixes to strip
|
|
618
|
+
const prefixesToStrip = [
|
|
619
|
+
'-home-',
|
|
620
|
+
'-mnt-c-Users-',
|
|
621
|
+
'-mnt-c-users-',
|
|
622
|
+
'-Users-',
|
|
623
|
+
]
|
|
624
|
+
|
|
625
|
+
let name = folderName
|
|
626
|
+
for (const prefix of prefixesToStrip) {
|
|
627
|
+
if (name.toLowerCase().startsWith(prefix.toLowerCase())) {
|
|
628
|
+
name = name.slice(prefix.length)
|
|
629
|
+
break
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
// Split on dashes and find meaningful parts
|
|
634
|
+
const parts = name.split('-')
|
|
635
|
+
|
|
636
|
+
// Common intermediate directories to skip
|
|
637
|
+
const skipDirs = new Set(['projects', 'code', 'repos', 'src', 'dev', 'work', 'documents'])
|
|
638
|
+
|
|
639
|
+
// Find meaningful parts
|
|
640
|
+
const meaningfulParts: string[] = []
|
|
641
|
+
let foundProject = false
|
|
642
|
+
|
|
643
|
+
for (let i = 0; i < parts.length; i++) {
|
|
644
|
+
const part = parts[i]
|
|
645
|
+
if (!part) continue
|
|
646
|
+
|
|
647
|
+
// Skip first part if it looks like a username
|
|
648
|
+
if (i === 0 && !foundProject) {
|
|
649
|
+
const remaining = parts.slice(i + 1).map(p => p.toLowerCase())
|
|
650
|
+
if (remaining.some(d => skipDirs.has(d))) {
|
|
651
|
+
continue
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
if (skipDirs.has(part.toLowerCase())) {
|
|
656
|
+
foundProject = true
|
|
657
|
+
continue
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
meaningfulParts.push(part)
|
|
661
|
+
foundProject = true
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
if (meaningfulParts.length) {
|
|
665
|
+
return meaningfulParts.join('-')
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// Fallback: return last non-empty part
|
|
669
|
+
for (let i = parts.length - 1; i >= 0; i--) {
|
|
670
|
+
if (parts[i]) return parts[i]
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
return folderName
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
// ============================================================================
|
|
677
|
+
// SESSION DISCOVERY
|
|
678
|
+
// ============================================================================
|
|
679
|
+
|
|
680
|
+
/**
|
|
681
|
+
* Find all session files in a Claude projects folder
|
|
682
|
+
*/
|
|
683
|
+
export async function findAllSessions(
|
|
684
|
+
projectsFolder?: string,
|
|
685
|
+
options: {
|
|
686
|
+
includeAgents?: boolean
|
|
687
|
+
limit?: number
|
|
688
|
+
} = {}
|
|
689
|
+
): Promise<ParsedProject[]> {
|
|
690
|
+
const folder = projectsFolder ?? join(homedir(), '.claude', 'projects')
|
|
691
|
+
|
|
692
|
+
try {
|
|
693
|
+
await stat(folder)
|
|
694
|
+
} catch {
|
|
695
|
+
return []
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
const projects: Map<string, ParsedProject> = new Map()
|
|
699
|
+
|
|
700
|
+
// Read all project folders
|
|
701
|
+
const projectFolders = await readdir(folder)
|
|
702
|
+
|
|
703
|
+
for (const projectFolder of projectFolders) {
|
|
704
|
+
const projectPath = join(folder, projectFolder)
|
|
705
|
+
const projectStat = await stat(projectPath)
|
|
706
|
+
|
|
707
|
+
if (!projectStat.isDirectory()) continue
|
|
708
|
+
|
|
709
|
+
// Find all JSONL files in this project
|
|
710
|
+
const files = await readdir(projectPath)
|
|
711
|
+
const jsonlFiles = files.filter(f => f.endsWith('.jsonl'))
|
|
712
|
+
|
|
713
|
+
// Skip agent files unless requested
|
|
714
|
+
const sessionFiles = options.includeAgents
|
|
715
|
+
? jsonlFiles
|
|
716
|
+
: jsonlFiles.filter(f => !f.startsWith('agent-'))
|
|
717
|
+
|
|
718
|
+
if (sessionFiles.length === 0) continue
|
|
719
|
+
|
|
720
|
+
// Parse each session
|
|
721
|
+
const sessions: ParsedSession[] = []
|
|
722
|
+
|
|
723
|
+
for (const sessionFile of sessionFiles) {
|
|
724
|
+
const filepath = join(projectPath, sessionFile)
|
|
725
|
+
|
|
726
|
+
try {
|
|
727
|
+
const session = await parseSessionFile(filepath)
|
|
728
|
+
|
|
729
|
+
// Skip empty or warmup sessions
|
|
730
|
+
if (session.messages.length === 0) continue
|
|
731
|
+
if (session.summary?.toLowerCase() === 'warmup') continue
|
|
732
|
+
|
|
733
|
+
sessions.push(session)
|
|
734
|
+
} catch {
|
|
735
|
+
// Skip files that can't be parsed
|
|
736
|
+
continue
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
// Apply limit per project if specified
|
|
740
|
+
if (options.limit && sessions.length >= options.limit) break
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
if (sessions.length === 0) continue
|
|
744
|
+
|
|
745
|
+
// Sort sessions by timestamp (most recent first)
|
|
746
|
+
sessions.sort((a, b) => {
|
|
747
|
+
const aTime = a.timestamps.last ?? a.timestamps.first ?? ''
|
|
748
|
+
const bTime = b.timestamps.last ?? b.timestamps.first ?? ''
|
|
749
|
+
return bTime.localeCompare(aTime)
|
|
750
|
+
})
|
|
751
|
+
|
|
752
|
+
projects.set(projectFolder, {
|
|
753
|
+
folderId: projectFolder,
|
|
754
|
+
name: getProjectDisplayName(projectFolder),
|
|
755
|
+
path: projectPath,
|
|
756
|
+
sessions
|
|
757
|
+
})
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
// Convert to array and sort by most recent session
|
|
761
|
+
const result = Array.from(projects.values())
|
|
762
|
+
result.sort((a, b) => {
|
|
763
|
+
const aTime = a.sessions[0]?.timestamps.last ?? ''
|
|
764
|
+
const bTime = b.sessions[0]?.timestamps.last ?? ''
|
|
765
|
+
return bTime.localeCompare(aTime)
|
|
766
|
+
})
|
|
767
|
+
|
|
768
|
+
return result
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
/**
|
|
772
|
+
* Find sessions for a specific project
|
|
773
|
+
*/
|
|
774
|
+
export async function findProjectSessions(
|
|
775
|
+
projectPath: string,
|
|
776
|
+
options: {
|
|
777
|
+
includeAgents?: boolean
|
|
778
|
+
limit?: number
|
|
779
|
+
} = {}
|
|
780
|
+
): Promise<ParsedSession[]> {
|
|
781
|
+
try {
|
|
782
|
+
await stat(projectPath)
|
|
783
|
+
} catch {
|
|
784
|
+
return []
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
const files = await readdir(projectPath)
|
|
788
|
+
const jsonlFiles = files.filter(f => f.endsWith('.jsonl'))
|
|
789
|
+
|
|
790
|
+
// Skip agent files unless requested
|
|
791
|
+
const sessionFiles = options.includeAgents
|
|
792
|
+
? jsonlFiles
|
|
793
|
+
: jsonlFiles.filter(f => !f.startsWith('agent-'))
|
|
794
|
+
|
|
795
|
+
const sessions: ParsedSession[] = []
|
|
796
|
+
|
|
797
|
+
for (const sessionFile of sessionFiles) {
|
|
798
|
+
const filepath = join(projectPath, sessionFile)
|
|
799
|
+
|
|
800
|
+
try {
|
|
801
|
+
const session = await parseSessionFile(filepath)
|
|
802
|
+
|
|
803
|
+
// Skip empty sessions
|
|
804
|
+
if (session.messages.length === 0) continue
|
|
805
|
+
|
|
806
|
+
sessions.push(session)
|
|
807
|
+
} catch {
|
|
808
|
+
continue
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
if (options.limit && sessions.length >= options.limit) break
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
// Sort by timestamp (most recent first)
|
|
815
|
+
sessions.sort((a, b) => {
|
|
816
|
+
const aTime = a.timestamps.last ?? a.timestamps.first ?? ''
|
|
817
|
+
const bTime = b.timestamps.last ?? b.timestamps.first ?? ''
|
|
818
|
+
return bTime.localeCompare(aTime)
|
|
819
|
+
})
|
|
820
|
+
|
|
821
|
+
return sessions
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
// ============================================================================
|
|
825
|
+
// API CONVERSION
|
|
826
|
+
// ============================================================================
|
|
827
|
+
|
|
828
|
+
/**
|
|
829
|
+
* Convert a parsed session to API-ready messages format
|
|
830
|
+
* This is what you send to the Anthropic Messages API
|
|
831
|
+
*/
|
|
832
|
+
export function toApiMessages(session: ParsedSession): Array<{
|
|
833
|
+
role: 'user' | 'assistant'
|
|
834
|
+
content: string | ContentBlock[]
|
|
835
|
+
}> {
|
|
836
|
+
return session.messages.map(msg => ({
|
|
837
|
+
role: msg.role,
|
|
838
|
+
content: msg.content
|
|
839
|
+
}))
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
/**
|
|
843
|
+
* Extract plain text from a session (for summaries, search, etc.)
|
|
844
|
+
*/
|
|
845
|
+
export function extractSessionText(session: ParsedSession): string {
|
|
846
|
+
const texts: string[] = []
|
|
847
|
+
|
|
848
|
+
for (const message of session.messages) {
|
|
849
|
+
if (typeof message.content === 'string') {
|
|
850
|
+
texts.push(message.content)
|
|
851
|
+
} else if (Array.isArray(message.content)) {
|
|
852
|
+
for (const block of message.content) {
|
|
853
|
+
if (typeof block === 'object' && block !== null) {
|
|
854
|
+
if (block.type === 'text' && 'text' in block) {
|
|
855
|
+
texts.push(block.text)
|
|
856
|
+
} else if (block.type === 'thinking' && 'thinking' in block) {
|
|
857
|
+
texts.push(block.thinking)
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
return texts.join('\n\n')
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
/**
|
|
868
|
+
* Get a brief summary of the session (first user message or stored summary)
|
|
869
|
+
*/
|
|
870
|
+
export function getSessionSummary(session: ParsedSession, maxLength = 200): string {
|
|
871
|
+
// Use stored summary if available
|
|
872
|
+
if (session.summary) {
|
|
873
|
+
return session.summary.length > maxLength
|
|
874
|
+
? session.summary.slice(0, maxLength - 3) + '...'
|
|
875
|
+
: session.summary
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
// Find first user message
|
|
879
|
+
for (const message of session.messages) {
|
|
880
|
+
if (message.role === 'user') {
|
|
881
|
+
let text = ''
|
|
882
|
+
|
|
883
|
+
if (typeof message.content === 'string') {
|
|
884
|
+
text = message.content
|
|
885
|
+
} else if (Array.isArray(message.content)) {
|
|
886
|
+
for (const block of message.content) {
|
|
887
|
+
if (typeof block === 'object' && block.type === 'text' && 'text' in block) {
|
|
888
|
+
text = block.text
|
|
889
|
+
break
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
if (text && !text.startsWith('<')) { // Skip XML-like system messages
|
|
895
|
+
return text.length > maxLength
|
|
896
|
+
? text.slice(0, maxLength - 3) + '...'
|
|
897
|
+
: text
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
return '(no summary)'
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
// ============================================================================
|
|
906
|
+
// STATISTICS
|
|
907
|
+
// ============================================================================
|
|
908
|
+
|
|
909
|
+
/**
|
|
910
|
+
* Get statistics about all discovered sessions
|
|
911
|
+
*/
|
|
912
|
+
export interface SessionStats {
|
|
913
|
+
totalProjects: number
|
|
914
|
+
totalSessions: number
|
|
915
|
+
totalMessages: number
|
|
916
|
+
totalToolUses: number
|
|
917
|
+
sessionsWithThinking: number
|
|
918
|
+
sessionsWithImages: number
|
|
919
|
+
oldestSession?: string
|
|
920
|
+
newestSession?: string
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
export function calculateStats(projects: ParsedProject[]): SessionStats {
|
|
924
|
+
let totalSessions = 0
|
|
925
|
+
let totalMessages = 0
|
|
926
|
+
let totalToolUses = 0
|
|
927
|
+
let sessionsWithThinking = 0
|
|
928
|
+
let sessionsWithImages = 0
|
|
929
|
+
const timestamps: string[] = []
|
|
930
|
+
|
|
931
|
+
for (const project of projects) {
|
|
932
|
+
for (const session of project.sessions) {
|
|
933
|
+
totalSessions++
|
|
934
|
+
totalMessages += session.metadata.messageCount
|
|
935
|
+
totalToolUses += session.metadata.toolUseCount
|
|
936
|
+
if (session.metadata.hasThinkingBlocks) sessionsWithThinking++
|
|
937
|
+
if (session.metadata.hasImages) sessionsWithImages++
|
|
938
|
+
if (session.timestamps.first) timestamps.push(session.timestamps.first)
|
|
939
|
+
if (session.timestamps.last) timestamps.push(session.timestamps.last)
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
timestamps.sort()
|
|
944
|
+
|
|
945
|
+
return {
|
|
946
|
+
totalProjects: projects.length,
|
|
947
|
+
totalSessions,
|
|
948
|
+
totalMessages,
|
|
949
|
+
totalToolUses,
|
|
950
|
+
sessionsWithThinking,
|
|
951
|
+
sessionsWithImages,
|
|
952
|
+
oldestSession: timestamps[0],
|
|
953
|
+
newestSession: timestamps[timestamps.length - 1]
|
|
954
|
+
}
|
|
955
|
+
}
|