@ebowwa/claudecodehistory 1.6.0 → 1.7.0

package/rust/src/parser.rs

@@ -1,261 +0,0 @@
-//! High-performance JSONL parser for Claude Code history files
-//!
-//! Uses memory-mapped I/O and parallel processing for maximum speed.
-
-use memmap2::Mmap;
-use rayon::prelude::*;
-use std::fs::File;
-use std::path::Path;
-use walkdir::WalkDir;
-
-use crate::types::*;
-use crate::utils::*;
-
-/// Fast directory parser - parses all JSONL files in a directory
-///
-/// Uses memory-mapped I/O and parallel processing for maximum speed.
-/// This is the equivalent of `parseDirFast` from @ebowwa/jsonl-hft
-pub fn parse_dir_fast(dir_path: &str) -> Vec<ParsedEntry> {
-    let path = Path::new(dir_path);
-
-    if !path.exists() || !path.is_dir() {
-        return Vec::new();
-    }
-
-    // Collect all JSONL files
-    let jsonl_files: Vec<_> = WalkDir::new(path)
-        .max_depth(1)
-        .into_iter()
-        .filter_map(|e| e.ok())
-        .filter(|e| e.path().extension().map_or(false, |ext| ext == "jsonl"))
-        .map(|e| e.path().to_path_buf())
-        .collect();
-
-    // Parse files in parallel
-    jsonl_files
-        .par_iter()
-        .flat_map(|file_path| parse_file_fast(file_path))
-        .collect()
-}
-
-/// Fast file parser - parses a single JSONL file using memory-mapped I/O
-pub fn parse_file_fast(file_path: &Path) -> Vec<ParsedEntry> {
-    let file = match File::open(file_path) {
-        Ok(f) => f,
-        Err(_) => return Vec::new(),
-    };
-
-    let mmap = match unsafe { Mmap::map(&file) } {
-        Ok(m) => m,
-        Err(_) => return Vec::new(),
-    };
-
-    let file_path_str = file_path.to_string_lossy().to_string();
-
-    // Parse lines in parallel using rayon
-    mmap.split(|&b| b == b'\n')
-        .par_bridge()
-        .filter_map(|line| parse_line_fast(line, &file_path_str))
-        .collect()
-}
-
-/// Parse a single line into a ParsedEntry
-fn parse_line_fast(line: &[u8], file_path: &str) -> Option<ParsedEntry> {
-    // Skip empty lines
-    let trimmed = trim_whitespace(line);
-    if trimmed.is_empty() {
-        return None;
-    }
-
-    // Quick validation: must start with '{' and end with '}'
-    if trimmed.first()? != &b'{' || trimmed.last()? != &b'}' {
-        return None;
-    }
-
-    // Parse JSON
-    let value: serde_json::Value = serde_json::from_slice(trimmed).ok()?;
-
-    // Extract fields with minimal allocations
-    let obj = value.as_object()?;
-
-    let uuid = obj.get("uuid")?.as_str()?.to_string();
-    let session_id = obj.get("sessionId")?.as_str()?.to_string();
-    let timestamp = obj.get("timestamp")?.as_str()?.to_string();
-
-    // Extract type/role
-    let message_type = obj.get("type").and_then(|v| v.as_str()).unwrap_or("user");
-    let role = message_type.to_string();
-
-    // Extract content from message.content
-    let content = extract_content(obj).unwrap_or_default();
-
-    Some(ParsedEntry {
-        uuid,
-        session_id,
-        timestamp,
-        role,
-        content,
-        file_path: file_path.to_string(),
-    })
-}
-
-/// Extract content from the message field
-fn extract_content(obj: &serde_json::Map<String, serde_json::Value>) -> Option<String> {
-    let message = obj.get("message")?.as_object()?;
-    let content = message.get("content")?;
-
-    match content {
-        serde_json::Value::String(s) => Some(s.clone()),
-        serde_json::Value::Array(arr) => {
-            let texts: Vec<_> = arr
-                .iter()
-                .filter_map(|item| {
-                    if let serde_json::Value::String(s) = item {
-                        Some(s.clone())
-                    } else if let Some(text) = item.get("text").and_then(|t| t.as_str()) {
-                        Some(text.to_string())
-                    } else {
-                        None
-                    }
-                })
-                .collect();
-            Some(texts.join(" "))
-        }
-        _ => None,
-    }
-}
-
-/// Trim whitespace from bytes
-fn trim_whitespace(bytes: &[u8]) -> &[u8] {
-    let start = bytes.iter().position(|&b| !b.is_ascii_whitespace()).unwrap_or(0);
-    let end = bytes
-        .iter()
-        .rposition(|&b| !b.is_ascii_whitespace())
-        .map(|i| i + 1)
-        .unwrap_or(0);
-    &bytes[start..end]
-}
-
-/// Parse a JSONL file into ConversationEntry objects
-pub fn parse_jsonl_file(
-    file_path: &Path,
-    project_dir: &str,
-    start_date: Option<&str>,
-    end_date: Option<&str>,
-) -> Vec<ConversationEntry> {
-    let file = match File::open(file_path) {
-        Ok(f) => f,
-        Err(_) => return Vec::new(),
-    };
-
-    let mmap = match unsafe { Mmap::map(&file) } {
-        Ok(m) => m,
-        Err(_) => return Vec::new(),
-    };
-
-    let project_path = decode_project_path(project_dir);
-
-    mmap.split(|&b| b == b'\n')
-        .filter_map(|line| {
-            let trimmed = trim_whitespace(line);
-            if trimmed.is_empty() {
-                return None;
-            }
-
-            let msg: ClaudeCodeMessage = serde_json::from_slice(trimmed).ok()?;
-
-            // Date filtering
-            if let Some(start) = start_date {
-                if msg.timestamp.as_str() < start {
-                    return None;
-                }
-            }
-            if let Some(end) = end_date {
-                if msg.timestamp.as_str() > end {
-                    return None;
-                }
-            }
-
-            convert_message_to_entry(msg, &project_path)
-        })
-        .collect()
-}
-
-/// Convert a ClaudeCodeMessage to a ConversationEntry
-fn convert_message_to_entry(
-    msg: ClaudeCodeMessage,
-    project_path: &str,
-) -> Option<ConversationEntry> {
-    // Extract content
-    let content = msg.message.as_ref().map(|m| {
-        match &m.content {
-            MessageContentValue::String(s) => s.clone(),
-            MessageContentValue::Array(blocks) => blocks
-                .iter()
-                .filter_map(|b| {
-                    b.text.clone().or_else(|| {
-                        b.block_type
-                            .as_ref()
-                            .map(|t| format!("[{}]", t))
-                    })
-                })
-                .collect::<Vec<_>>()
-                .join(" "),
-        }
-    }).unwrap_or_default();
-
-    // Calculate time-related fields
-    let timestamp = msg.timestamp.clone();
-    let (formatted_time, time_ago, local_date) = format_timestamp(&timestamp);
-
-    Some(ConversationEntry {
-        session_id: msg.session_id,
-        timestamp,
-        entry_type: msg.message_type,
-        content,
-        project_path: project_path.to_string(),
-        uuid: msg.uuid,
-        formatted_time: Some(formatted_time),
-        time_ago: Some(time_ago),
-        local_date: Some(local_date),
-        metadata: msg.message.map(|m| EntryMetadata {
-            usage: m.usage,
-            model: m.model,
-            request_id: msg.request_id,
-            total_cost_usd: None,
-            num_turns: None,
-            duration_ms: None,
-            is_error: None,
-            error_type: None,
-        }),
-        project_name: None,
-    })
-}
-
-/// Decode project path from directory name (e.g., "-Users-ebowwa-Desktop-codespaces" -> "/Users/ebowwa/Desktop/codespaces")
-fn decode_project_path(project_dir: &str) -> String {
-    project_dir
-        .replace('-', "/")
-        .trim_start_matches('/')
-        .to_string()
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_decode_project_path() {
-        assert_eq!(
-            decode_project_path("-Users-ebowwa-Desktop-codespaces"),
-            "Users/ebowwa/Desktop/codespaces"
-        );
-    }
-
-    #[test]
-    fn test_trim_whitespace() {
-        assert_eq!(trim_whitespace(b"  hello  "), b"hello");
-        assert_eq!(trim_whitespace(b"\n\ttest\n"), b"test");
-        assert_eq!(trim_whitespace(b""), b"");
-    }
-}

package/rust/src/search/index.rs

@@ -1,206 +0,0 @@
-//! Search index operations
-//!
-//! Provides index management for conversation history search
-
-use tantivy::{Index, IndexWriter, TantivyDocument};
-use tantivy::schema::{Schema, Value};
-
-use super::schema::HistoryFields;
-use super::SearchError;
-use crate::types::*;
-
-/// Add a conversation entry to the search index
-pub fn add_entry_to_index(
-    writer: &mut IndexWriter<TantivyDocument>,
-    schema: &Schema,
-    entry: &ConversationEntry,
-) -> Result<(), SearchError> {
-    let fields = HistoryFields::new(schema);
-    let mut doc = TantivyDocument::new();
-
-    // Add primary identifiers
-    doc.add_text(fields.uuid, &entry.uuid);
-    doc.add_text(fields.session_id, &entry.session_id);
-    doc.add_text(fields.project_path, &entry.project_path);
-
-    // Add message type
-    let message_type = match entry.entry_type {
-        MessageType::User => "user",
-        MessageType::Assistant => "assistant",
-        MessageType::System => "system",
-        MessageType::Result => "result",
-    };
-    doc.add_text(fields.message_type, message_type);
-
-    // Add role (same as message_type for now)
-    doc.add_text(fields.role, message_type);
-
-    // Add timestamp
-    let timestamp_ms = super::schema::timestamp_to_ms(&entry.timestamp);
-    doc.add_i64(fields.timestamp_ms, timestamp_ms);
-
-    // Add content
-    doc.add_text(fields.content, &entry.content);
-
-    // Build combined searchable text
-    let project_name = super::schema::extract_project_name(&entry.project_path);
-    let all_text = format!(
-        "{} {} {} {}",
-        entry.content,
-        entry.session_id,
-        project_name,
-        entry.uuid
-    );
-    doc.add_text(fields.all_text, &all_text);
-
-    // Add metadata if present
-    if let Some(ref meta) = entry.metadata {
-        if let Some(ref model) = meta.model {
-            doc.add_text(fields.model, model);
-        }
-        if let Some(ref request_id) = meta.request_id {
-            doc.add_text(fields.request_id, request_id);
-        }
-        if let Some(cost_usd) = meta.total_cost_usd {
-            doc.add_f64(fields.cost_usd, cost_usd);
-        }
-        if let Some(duration_ms) = meta.duration_ms {
-            doc.add_i64(fields.duration_ms, duration_ms as i64);
-        }
-        if let Some(is_error) = meta.is_error {
-            doc.add_bool(fields.is_error, is_error);
-        }
-    }
-
-    // Add project name for faceted search
-    doc.add_text(fields.project_name, &project_name);
-
-    writer.add_document(doc).map_err(|e| SearchError::Commit(e.to_string()))?;
-
-    Ok(())
-}
-
-/// Delete all entries for a session from the index
-pub fn delete_session_from_index(
-    writer: &mut IndexWriter<TantivyDocument>,
-    schema: &Schema,
-    session_id: &str,
-) -> Result<(), SearchError> {
-    let field = schema.get_field("session_id")
-        .map_err(|e| SearchError::Schema(e.to_string()))?;
-
-    let term = tantivy::Term::from_field_text(field, session_id);
-    writer.delete_term(term);
-
-    Ok(())
-}
-
-/// Get index statistics
-pub fn get_index_stats(index: &Index, schema: &Schema) -> Result<super::IndexStats, SearchError> {
-    let reader = index.reader()
-        .map_err(|e| SearchError::Query(e.to_string()))?;
-
-    let searcher = reader.searcher();
-    let fields = HistoryFields::new(schema);
-
-    let total_docs = searcher.num_docs() as usize;
-
-    // Count unique sessions and projects
-    let mut session_ids: std::collections::HashSet<String> = std::collections::HashSet::new();
-    let mut project_paths: std::collections::HashSet<String> = std::collections::HashSet::new();
-    let mut user_messages = 0;
-    let mut assistant_messages = 0;
-
-    for segment_reader in searcher.segment_readers() {
-        let store_reader = segment_reader.get_store_reader(0)
-            .map_err(|e| SearchError::Query(e.to_string()))?;
-
-        for doc_id in segment_reader.doc_ids_alive() {
-            if let Ok(doc) = store_reader.get::<TantivyDocument>(doc_id) {
-                // Extract session_id
-                if let Some(value) = doc.get_first(fields.session_id) {
-                    if let Some(text) = value.as_str() {
-                        session_ids.insert(text.to_string());
-                    }
-                }
-
-                // Extract project_path
-                if let Some(value) = doc.get_first(fields.project_path) {
-                    if let Some(text) = value.as_str() {
-                        project_paths.insert(text.to_string());
-                    }
-                }
-
-                // Count message types
-                if let Some(value) = doc.get_first(fields.message_type) {
-                    if let Some(text) = value.as_str() {
-                        match text {
-                            "user" => user_messages += 1,
-                            "assistant" => assistant_messages += 1,
-                            _ => {}
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    // Calculate index size
-    let index_size_bytes = get_index_size(index);
-
-    Ok(super::IndexStats {
-        total_docs,
-        user_messages,
-        assistant_messages,
-        unique_sessions: session_ids.len(),
-        unique_projects: project_paths.len(),
-        index_size_bytes,
-    })
-}
-
-/// Get the size of the index directory in bytes
-/// Note: In Tantivy 0.22, ManagedDirectory doesn't expose path()
-/// This function returns 0 as a placeholder - index size can be tracked separately
-fn get_index_size(_index: &Index) -> u64 {
-    // Tantivy 0.22 doesn't expose directory path through ManagedDirectory
-    // Index size tracking would need to be done at a higher level
-    0
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use tempfile::tempdir;
-    use super::super::schema::create_history_schema;
-
-    #[test]
-    fn test_add_entry_to_index() {
-        let dir = tempdir().unwrap();
-        let schema = create_history_schema();
-
-        let index = Index::create_in_dir(dir.path(), schema.clone())
-            .expect("Failed to create index");
-
-        let mut writer = index.writer(50_000_000)
-            .expect("Failed to create writer");
-
-        let entry = ConversationEntry {
-            uuid: "test-uuid".to_string(),
-            session_id: "test-session".to_string(),
-            timestamp: "2024-01-01T00:00:00Z".to_string(),
-            project_path: "/test/project".to_string(),
-            entry_type: MessageType::User,
-            content: "Hello world".to_string(),
-            formatted_time: None,
-            time_ago: None,
-            local_date: None,
-            metadata: None,
-            project_name: None,
-        };
-
-        let result = add_entry_to_index(&mut writer, &schema, &entry);
-        assert!(result.is_ok());
-
-        writer.commit().expect("Failed to commit");
-    }
-}