solidstats 3.0.0.beta.1 → 3.0.0

data/app/services/solidstats/load_lens_service.rb

@@ -4,9 +4,9 @@ module Solidstats
   # LoadLens - Development Performance Monitoring Service
   # Parses Rails development logs to extract performance metrics
   class LoadLensService
-    DATA_DIR = Rails.root.join('solidstats')
-    LOG_FILE = Rails.root.join('log', 'development.log')
-    POSITION_FILE = DATA_DIR.join('last_position.txt')
+    DATA_DIR = Rails.root.join("solidstats")
+    LOG_FILE = Rails.root.join("log", "development.log")
+    POSITION_FILE = DATA_DIR.join("last_position.txt")
     CACHE_FILE = "loadlens.json"
     SUMMARY_FILE = "summary.json"
     RETENTION_DAYS = 7
@@ -22,7 +22,7 @@ module Solidstats
 
     def self.get_performance_data
       cache_file_path = solidstats_cache_path(CACHE_FILE)
-      
+
       if File.exist?(cache_file_path) && cache_fresh?(cache_file_path, 15.minutes)
         raw_data = JSON.parse(File.read(cache_file_path))
         deep_indifferent_access(raw_data)
@@ -36,13 +36,13 @@ module Solidstats
 
     def self.scan_and_cache
       performance_data = scan_development_log
-      
+
       # Cache the performance data
       cache_performance_data(performance_data)
-      
+
       # Update summary.json with performance monitoring card
       update_summary_json(performance_data)
-      
+
       deep_indifferent_access(performance_data)
     end
 
@@ -64,10 +64,10 @@ module Solidstats
         last_position = read_last_position
         processed_count = 0
         current_requests = []
-        
-        File.open(LOG_FILE, 'r') do |file|
+
+        File.open(LOG_FILE, "r") do |file|
           file.seek(last_position)
-          
+
           file.each_line do |line|
             process_line(line.strip, current_requests)
             update_last_position(file.pos)
@@ -81,7 +81,7 @@ module Solidstats
             processed_count += 1
           end
         end
-        
+
         Rails.logger.info("DevLogParser: Processed #{processed_count} requests")
         { success: true, processed: processed_count }
       rescue => e
@@ -105,23 +105,28 @@ module Solidstats
     end
 
     def self.scan_development_log
-      # Load recent performance data for dashboard
+      # First, ensure the log parser runs to create perf files if they don't exist
+      parse_recent_log_entries_if_needed
+
+      # Now, load the performance data from the perf files
       data_files = []
-      
+      found_perf_files = []
+
       # Get last 7 days of data files
       7.times do |i|
         date = i.days.ago.to_date
         file_path = DATA_DIR.join("perf_#{date.strftime('%Y-%m-%d')}.json")
-        
+
         if File.exist?(file_path)
           file_data = JSON.parse(File.read(file_path))
           data_files.concat(file_data)
+          found_perf_files << file_path
         end
       end
 
       # Fallback: If no data files exist, parse recent entries from development.log
       if data_files.empty? && File.exist?(LOG_FILE) && Rails.env.development?
-        Rails.logger.info("DevLogParser: No data files found, parsing recent development log entries")
+        Rails.logger.info("LoadLensService: No perf data files found, parsing recent log entries as a fallback.")
         data_files = parse_recent_log_entries
       end
 
@@ -129,58 +134,73 @@ module Solidstats
       calculate_performance_metrics(data_files)
     end
 
+    def self.parse_recent_log_entries_if_needed
+      # Check if any perf files exist from the last 7 days
+      has_recent_perf_files = 7.times.any? do |i|
+        date = i.days.ago.to_date
+        file_path = DATA_DIR.join("perf_#{date.strftime('%Y-%m-%d')}.json")
+        File.exist?(file_path)
+      end
+
+      # If no recent perf files, run the parser
+      unless has_recent_perf_files
+        Rails.logger.info("LoadLensService: No recent perf files found. Parsing development log to generate initial data.")
+        parse_recent_log_entries
+      end
+    end
+
     def self.parse_recent_log_entries
       # Parse recent entries from development.log as fallback when no data files exist
       recent_requests = []
       current_requests = []
       lines_processed = 0
       max_lines = 2000  # Process last 2000 lines for initial bootstrap
-      
+
       begin
         # Get the last N lines from the log file efficiently
         log_lines = []
-        File.open(LOG_FILE, 'r') do |file|
+        File.open(LOG_FILE, "r") do |file|
           file.each_line { |line| log_lines << line.strip }
         end
-        
+
         # Take the last max_lines entries
         log_lines = log_lines.last(max_lines) if log_lines.size > max_lines
-        
+
         # Process each line using existing parsing logic
         log_lines.each do |line|
           process_line(line, current_requests)
           lines_processed += 1
-          
+
           # Collect completed requests
           current_requests.each do |req|
             if req[:completed]
               # Convert to the same format as saved requests
               clean_request = {
-                'controller' => req[:controller],
-                'action' => req[:action],
-                'http_method' => req[:http_method],
-                'path' => req[:path],
-                'status' => req[:status],
-                'total_time_ms' => req[:total_time_ms] || 0.0,
-                'view_time_ms' => req[:view_time_ms] || 0.0,
-                'activerecord_time_ms' => req[:activerecord_time_ms] || 0.0,
-                'timestamp' => req[:timestamp]
+                "controller" => req[:controller],
+                "action" => req[:action],
+                "http_method" => req[:http_method],
+                "path" => req[:path],
+                "status" => req[:status],
+                "total_time_ms" => req[:total_time_ms] || 0.0,
+                "view_time_ms" => req[:view_time_ms] || 0.0,
+                "activerecord_time_ms" => req[:activerecord_time_ms] || 0.0,
+                "timestamp" => req[:timestamp]
               }
-              
+
               # Only include complete requests with controller/action
-              if clean_request['controller'] && clean_request['action']
+              if clean_request["controller"] && clean_request["action"]
                 recent_requests << clean_request
               end
             end
           end
-          
+
           # Remove completed requests from processing queue
           current_requests.reject! { |req| req[:completed] }
         end
-        
+
         Rails.logger.info("DevLogParser: Parsed #{recent_requests.size} recent requests from #{lines_processed} log lines")
         recent_requests
-        
+
       rescue => e
         Rails.logger.error("DevLogParser: Failed to parse recent log entries: #{e.message}")
         []
@@ -191,11 +211,11 @@ module Solidstats
       return default_metrics if requests.empty?
 
       total_requests = requests.size
-      avg_response_time = (requests.sum { |req| req['total_time_ms'] || 0 } / total_requests).round(2)
-      avg_view_time = (requests.sum { |req| req['view_time_ms'] || 0 } / total_requests).round(2)
-      avg_db_time = (requests.sum { |req| req['activerecord_time_ms'] || 0 } / total_requests).round(2)
-      slow_requests = requests.count { |req| (req['total_time_ms'] || 0) > 1000 }
-      error_rate = ((requests.count { |req| (req['status'] || 200) >= 400 }.to_f / total_requests) * 100).round(2)
+      avg_response_time = (requests.sum { |req| req["total_time_ms"] || 0 } / total_requests).round(2)
+      avg_view_time = (requests.sum { |req| req["view_time_ms"] || 0 } / total_requests).round(2)
+      avg_db_time = (requests.sum { |req| req["activerecord_time_ms"] || 0 } / total_requests).round(2)
+      slow_requests = requests.count { |req| (req["total_time_ms"] || 0) > 1000 }
+      error_rate = ((requests.count { |req| (req["status"] || 200) >= 400 }.to_f / total_requests) * 100).round(2)
 
       status = determine_status(avg_response_time, error_rate, slow_requests, total_requests)
 
@@ -210,7 +230,7 @@ module Solidstats
           status: status,
           last_updated: Time.current.iso8601
         },
-        recent_requests: requests.sort_by { |req| req['timestamp'] }.reverse.first(20),
+        recent_requests: requests.sort_by { |req| req["timestamp"] }.reverse.first(20),
         last_updated: Time.current.iso8601
       }
     end
@@ -224,7 +244,7 @@ module Solidstats
           avg_db_time: 0,
           slow_requests: 0,
           error_rate: 0,
-          status: 'info',
+          status: "info",
           last_updated: Time.current.iso8601
         },
         recent_requests: [],
@@ -233,9 +253,9 @@ module Solidstats
     end
 
     def self.determine_status(avg_response_time, error_rate, slow_requests, total_requests)
-      return 'error' if error_rate > 10
-      return 'warning' if avg_response_time > 1000 || slow_requests > (total_requests * 0.1)
-      return 'success'
+      return "error" if error_rate > 10
+      return "warning" if avg_response_time > 1000 || slow_requests > (total_requests * 0.1)
+      "success"
     end
 
     def self.process_line(line, current_requests)
@@ -248,7 +268,7 @@ module Solidstats
           timestamp: parse_timestamp(timestamp),
           started_at: Time.current
         }
-      
+
       # Controller and action info
       elsif match = line.match(CONTROLLER_ACTION_REGEX)
         controller, action = match.captures
@@ -256,7 +276,7 @@ module Solidstats
           current_request[:controller] = controller
           current_request[:action] = action
         end
-      
+
       # Request completion with timing
       elsif match = line.match(COMPLETED_REGEX)
         status, total_time, timing_details = match.captures
@@ -264,26 +284,26 @@ module Solidstats
           current_request[:status] = status.to_i
           current_request[:total_time_ms] = total_time.to_f
           current_request[:completed] = true
-          
+
           # Extract view and ActiveRecord times from timing details if present
           if timing_details
             if view_match = timing_details.match(VIEW_RENDERING_REGEX)
               current_request[:view_time_ms] = view_match[1].to_f
             end
-            
+
             if ar_match = timing_details.match(ACTIVERECORD_REGEX)
               current_request[:activerecord_time_ms] = ar_match[1].to_f
             end
           end
         end
-      
+
       # View rendering time
       elsif match = line.match(VIEW_RENDERING_REGEX)
         view_time = match.captures.first
         if current_request = current_requests.last
           current_request[:view_time_ms] = view_time.to_f
         end
-      
+
       # ActiveRecord time
       elsif match = line.match(ACTIVERECORD_REGEX)
         ar_time = match.captures.first
@@ -319,7 +339,7 @@ module Solidstats
       return unless clean_request[:controller] && clean_request[:action]
 
       perf_file = current_perf_file
-      
+
       # Read existing data or create new array
       existing_data = if File.exist?(perf_file)
         JSON.parse(File.read(perf_file))
@@ -335,7 +355,7 @@ module Solidstats
     end
 
     def self.current_perf_file
-      date_suffix = Date.current.strftime('%Y-%m-%d')
+      date_suffix = Date.current.strftime("%Y-%m-%d")
       DATA_DIR.join("perf_#{date_suffix}.json")
     end
 
@@ -345,8 +365,8 @@ module Solidstats
 
     def self.cleanup_old_files
       cutoff_date = RETENTION_DAYS.days.ago.to_date
-      
-      Dir.glob(DATA_DIR.join('perf_*.json')).each do |file|
+
+      Dir.glob(DATA_DIR.join("perf_*.json")).each do |file|
         if match = File.basename(file).match(/perf_(\d{4}-\d{2}-\d{2})\.json/)
           file_date = Date.parse(match[1])
           File.delete(file) if file_date < cutoff_date
@@ -373,7 +393,7 @@ module Solidstats
     def self.cache_performance_data(data)
       cache_file_path = solidstats_cache_path(CACHE_FILE)
       ensure_solidstats_directory
-      
+
       File.write(cache_file_path, JSON.pretty_generate(data))
     rescue => e
       Rails.logger.error("Failed to cache performance data: #{e.message}")
@@ -381,33 +401,33 @@ module Solidstats
 
     def self.update_summary_json(performance_data)
       summary_file_path = solidstats_cache_path(SUMMARY_FILE)
-      
+
       # Read existing summary or create new one
       begin
         existing_summary = File.exist?(summary_file_path) ? JSON.parse(File.read(summary_file_path)) : {}
       rescue JSON::ParserError
         existing_summary = {}
       end
-      
+
       summary = performance_data[:summary]
-      
+
       # Create badges based on performance metrics
       badges = []
       badges << { "text" => "#{summary[:total_requests]} Requests", "color" => "info" }
-      
+
       case summary[:status]
-      when 'error'
+      when "error"
         badges << { "text" => "High Errors", "color" => "error" }
-      when 'warning'
+      when "warning"
         badges << { "text" => "Slow Responses", "color" => "warning" }
       else
         badges << { "text" => "Healthy", "color" => "success" }
       end
-      
+
       if summary[:avg_response_time] > 0
         badges << { "text" => "#{summary[:avg_response_time]}ms avg", "color" => "neutral" }
       end
-      
+
       # Update the LoadLens monitoring entry
       existing_summary["LoadLens"] = {
         "icon" => "activity",
@@ -417,7 +437,7 @@ module Solidstats
         "url" => "/solidstats/performance/load_lens",
         "badges" => badges
       }
-      
+
       # Write updated summary
       File.write(summary_file_path, JSON.pretty_generate(existing_summary))
     rescue => e
@@ -427,9 +447,9 @@ module Solidstats
     def self.generate_performance_message(summary)
       if summary[:total_requests] == 0
         "No requests tracked"
-      elsif summary[:status] == 'error'
+      elsif summary[:status] == "error"
         "#{summary[:error_rate]}% error rate"
-      elsif summary[:status] == 'warning'
+      elsif summary[:status] == "warning"
         "#{summary[:avg_response_time]}ms avg"
       else
         "#{summary[:avg_response_time]}ms avg"
@@ -441,13 +461,13 @@ module Solidstats
     rescue
       false
     end
-    
+
     def self.solidstats_cache_path(filename)
-      Rails.root.join('solidstats', filename)
+      Rails.root.join("solidstats", filename)
     end
-    
+
     def self.ensure_solidstats_directory
-      dir_path = File.dirname(solidstats_cache_path('dummy'))
+      dir_path = File.dirname(solidstats_cache_path("dummy"))
       FileUtils.mkdir_p(dir_path) unless Dir.exist?(dir_path)
     end
   end

data/app/services/solidstats/log_size_monitor_service.rb

@@ -2,14 +2,14 @@ module Solidstats
   class LogSizeMonitorService
    CACHE_FILE = "logs.json"
     SUMMARY_FILE = "summary.json"
-    
+
     # Size thresholds in bytes
     WARNING_SIZE = 500.megabytes
     ERROR_SIZE = 1.gigabyte
-    
+
     def self.get_logs_data
       cache_file_path = solidstats_cache_path(CACHE_FILE)
-      
+
       if File.exist?(cache_file_path) && cache_fresh?(cache_file_path)
         JSON.parse(File.read(cache_file_path))
       else
@@ -19,40 +19,40 @@ module Solidstats
       Rails.logger.error("Error reading logs cache, regenerating...")
       scan_and_cache
     end
-    
+
     def self.scan_and_cache
       logs_data = scan_log_files
-      
+
       # Cache the logs data
       cache_logs_data(logs_data)
-      
+
       # Update summary.json with log monitoring card
       update_summary_json(logs_data)
-      
+
       logs_data
     end
-    
+
     def self.truncate_log(filename)
       return { status: "error", message: "Filename required" } if filename.blank?
-      
+
       log_file_path = find_log_file(filename)
       return { status: "error", message: "Log file not found" } unless log_file_path
-      
+
       begin
         # Check if file is writable
         unless File.writable?(log_file_path)
           return { status: "error", message: "File is not writable" }
         end
-        
+
         # Get original size for reporting
         original_size = File.size(log_file_path)
-        
+
         # Truncate the file
         File.truncate(log_file_path, 0)
-        
+
         # Update cache after truncation
         scan_and_cache
-        
+
         {
           status: "success",
           message: "Log file truncated successfully",
@@ -64,21 +64,21 @@ module Solidstats
         { status: "error", message: "Failed to truncate: #{e.message}" }
       end
     end
-    
+
     private
-    
+
     def self.scan_log_files
       log_files = discover_log_files
       total_size = 0
       files_data = []
-      
+
       log_files.each do |file_path|
         next unless File.exist?(file_path) && File.readable?(file_path)
-        
+
         file_stat = File.stat(file_path)
         file_size = file_stat.size
         total_size += file_size
-        
+
         files_data << {
           name: File.basename(file_path),
           path: file_path,
@@ -89,10 +89,10 @@ module Solidstats
           can_truncate: File.writable?(file_path)
         }
       end
-      
+
       # Sort by size descending
       files_data.sort_by! { |f| -f[:size_bytes] }
-      
+
       {
         summary: {
           total_files: files_data.length,
@@ -106,91 +106,91 @@ module Solidstats
         files: files_data
       }
     end
-    
+
     def self.discover_log_files
       log_paths = []
-      
+
       # Rails log directory
       if defined?(Rails) && Rails.root
-        rails_log_dir = Rails.root.join('log')
-        log_paths += Dir.glob(rails_log_dir.join('*.log')) if Dir.exist?(rails_log_dir)
+        rails_log_dir = Rails.root.join("log")
+        log_paths += Dir.glob(rails_log_dir.join("*.log")) if Dir.exist?(rails_log_dir)
       end
-      
+
       # Remove duplicates and filter readable files
       log_paths.uniq.select { |path| File.readable?(path) }
     end
-    
+
     def self.find_log_file(filename)
       discovered_files = discover_log_files
       discovered_files.find { |path| File.basename(path) == filename }
     end
-    
+
     def self.determine_file_status(size_bytes)
       if size_bytes >= ERROR_SIZE
-        'error'
+        "error"
       elsif size_bytes >= WARNING_SIZE
-        'warning'
+        "warning"
       else
-        'success'
+        "success"
       end
     end
-    
+
     def self.determine_overall_status(total_size, files_data)
-      error_files = files_data.count { |f| f[:status] == 'error' }
-      warning_files = files_data.count { |f| f[:status] == 'warning' }
-      
+      error_files = files_data.count { |f| f[:status] == "error" }
+      warning_files = files_data.count { |f| f[:status] == "warning" }
+
       if error_files > 0 || total_size >= ERROR_SIZE * 2
-        'error'
+        "error"
       elsif warning_files > 0 || total_size >= WARNING_SIZE * 3
-        'warning'
+        "warning"
       else
-        'success'
+        "success"
       end
     end
-    
+
     def self.format_file_size(size_bytes)
       return "0 B" if size_bytes.zero?
-      
-      units = ['B', 'KB', 'MB', 'GB', 'TB']
+
+      units = [ "B", "KB", "MB", "GB", "TB" ]
       base = 1024.0
       exp = (Math.log(size_bytes) / Math.log(base)).to_i
-      exp = [exp, units.length - 1].min
-      
-      "%.1f %s" % [size_bytes / (base ** exp), units[exp]]
+      exp = [ exp, units.length - 1 ].min
+
+      "%.1f %s" % [ size_bytes / (base ** exp), units[exp] ]
     end
-    
+
     def self.cache_logs_data(logs_data)
       cache_file_path = solidstats_cache_path(CACHE_FILE)
       ensure_solidstats_directory
-      
+
       File.write(cache_file_path, JSON.pretty_generate(logs_data))
     rescue => e
       Rails.logger.error("Failed to cache logs data: #{e.message}")
     end
-    
+
     def self.update_summary_json(logs_data)
       summary_file_path = solidstats_cache_path(SUMMARY_FILE)
-      
+
       # Read existing summary or create new one
       begin
         existing_summary = File.exist?(summary_file_path) ? JSON.parse(File.read(summary_file_path)) : {}
       rescue JSON::ParserError
         existing_summary = {}
       end
-      
+
       # Create badges based on status
       badges = []
       badges << { "text" => "#{logs_data[:summary][:total_files]} Files", "color" => "info" }
-      
+
       case logs_data[:summary][:status]
-      when 'error'
+      when "error"
         badges << { "text" => "Large Size", "color" => "error" }
-      when 'warning'
+      when "warning"
         badges << { "text" => "Growing", "color" => "warning" }
       else
         badges << { "text" => "Healthy", "color" => "success" }
       end
-      
+
       # Update the log monitoring entry
       existing_summary["Log Files"] = {
         "icon" => "file-text",
@@ -200,25 +200,25 @@ module Solidstats
         "url" => "/solidstats/logs/size",
         "badges" => badges
       }
-      
+
       # Write updated summary
       File.write(summary_file_path, JSON.pretty_generate(existing_summary))
     rescue => e
       Rails.logger.error("Failed to update summary.json: #{e.message}")
     end
-    
+
     def self.cache_fresh?(cache_file_path, max_age = 30.minutes)
       File.mtime(cache_file_path) > max_age.ago
     rescue
       false
     end
-    
+
     def self.solidstats_cache_path(filename)
-      Rails.root.join( 'solidstats', filename)
+      Rails.root.join("solidstats", filename)
     end
-    
+
     def self.ensure_solidstats_directory
-      dir_path = File.dirname(solidstats_cache_path('dummy'))
+      dir_path = File.dirname(solidstats_cache_path("dummy"))
       FileUtils.mkdir_p(dir_path) unless Dir.exist?(dir_path)
     end
   end