solid_log-core 0.1.0

data/lib/solid_log/core/configuration.rb

@@ -0,0 +1,60 @@
+module SolidLog
+  module Core
+    class Configuration
+      attr_accessor :database_url,
+                    :retention_days,
+                    :error_retention_days,
+                    :max_batch_size,
+                    :parser_batch_size,
+                    :parser_concurrency,
+                    :auto_promote_fields,
+                    :field_promotion_threshold,
+                    :facet_cache_ttl,
+                    :live_tail_mode
+
+      def initialize
+        # Load from ENV vars with defaults
+        @database_url = ENV["SOLIDLOG_DATABASE_URL"] || ENV["DATABASE_URL"]
+        @retention_days = env_to_int("SOLIDLOG_RETENTION_DAYS", 30)
+        @error_retention_days = env_to_int("SOLIDLOG_ERROR_RETENTION_DAYS", 90)
+        @max_batch_size = env_to_int("SOLIDLOG_MAX_BATCH_SIZE", 1000) # For API ingestion
+        @parser_batch_size = env_to_int("SOLIDLOG_PARSER_BATCH_SIZE", 200) # Number of raw entries to parse per job
+        @parser_concurrency = env_to_int("SOLIDLOG_PARSER_CONCURRENCY", 5)
+        @auto_promote_fields = env_to_bool("SOLIDLOG_AUTO_PROMOTE_FIELDS", false)
+        @field_promotion_threshold = env_to_int("SOLIDLOG_FIELD_PROMOTION_THRESHOLD", 1000)
+        @facet_cache_ttl = env_to_int("SOLIDLOG_FACET_CACHE_TTL", 300) # seconds (5 minutes)
+        @live_tail_mode = env_to_symbol("SOLIDLOG_LIVE_TAIL_MODE", :disabled) # :websocket, :polling, or :disabled
+      end
+
+      # Check if database is configured
+      def database_configured?
+        database_url.present?
+      end
+
+      # Get cache TTL in seconds
+      def cache_ttl_seconds
+        facet_cache_ttl.to_i
+      end
+
+      private
+
+      def env_to_int(key, default = nil)
+        value = ENV[key]
+        return default if value.nil? || value.empty?
+        value.to_i
+      end
+
+      def env_to_bool(key, default = false)
+        value = ENV[key]
+        return default if value.nil? || value.empty?
+        ["true", "1", "yes", "on"].include?(value.downcase)
+      end
+
+      def env_to_symbol(key, default = nil)
+        value = ENV[key]
+        return default if value.nil? || value.empty?
+        value.to_sym
+      end
+    end
+  end
+end

data/lib/solid_log/core/services/correlation_service.rb

@@ -0,0 +1,74 @@
+module SolidLog
+  module Core
+    class CorrelationService
+      # Get all entries for a specific request ID in timeline order
+      def self.request_timeline(request_id)
+        return Entry.none if request_id.blank?
+
+        Entry.by_request_id(request_id).recent
+      end
+
+      # Get all entries for a specific job ID in timeline order
+      def self.job_timeline(job_id)
+        return Entry.none if job_id.blank?
+
+        Entry.by_job_id(job_id).recent
+      end
+
+      # Get correlation stats for a request
+      def self.request_stats(request_id)
+        entries = request_timeline(request_id)
+
+        {
+          total_entries: entries.count,
+          duration: calculate_duration(entries),
+          levels: entries.group(:level).count,
+          first_timestamp: entries.first&.timestamp,
+          last_timestamp: entries.last&.timestamp
+        }
+      end
+
+      # Get correlation stats for a job
+      def self.job_stats(job_id)
+        entries = job_timeline(job_id)
+
+        {
+          total_entries: entries.count,
+          duration: calculate_duration(entries),
+          levels: entries.group(:level).count,
+          first_timestamp: entries.first&.timestamp,
+          last_timestamp: entries.last&.timestamp
+        }
+      end
+
+      # Find related entries (same request_id or job_id)
+      def self.find_related(entry)
+        related = []
+
+        if entry.request_id.present?
+          related += request_timeline(entry.request_id).where.not(id: entry.id).to_a
+        end
+
+        if entry.job_id.present?
+          related += job_timeline(entry.job_id).where.not(id: entry.id).to_a
+        end
+
+        related.uniq.sort_by(&:timestamp)
+      end
+
+      private
+
+      # Calculate total duration from first to last entry
+      def self.calculate_duration(entries)
+        return nil if entries.count < 2
+
+        first = entries.first
+        last = entries.last
+
+        return nil unless first&.timestamp && last&.timestamp
+
+        ((last.timestamp - first.timestamp) * 1000).round # milliseconds
+      end
+    end
+  end
+end

data/lib/solid_log/core/services/field_analyzer.rb

@@ -0,0 +1,108 @@
+module SolidLog
+  module Core
+    class FieldAnalyzer
+      # Analyze fields and return promotion recommendations
+      def self.analyze(threshold: 1000)
+        recommendations = []
+
+        # Get hot fields that are not yet promoted
+        hot_fields = Field
+          .hot_fields(threshold)
+          .unpromoted
+          .recently_seen(30) # Only consider fields seen in last 30 days
+
+        hot_fields.each do |field|
+          # Calculate priority score
+          priority = calculate_priority(field, threshold)
+
+          recommendations << {
+            field: field,
+            priority: priority,
+            reason: promotion_reason(field, threshold)
+          }
+        end
+
+        # Sort by priority (highest first)
+        recommendations.sort_by { |rec| -rec[:priority] }
+      end
+
+      # Auto-promote fields that meet the threshold
+      def self.auto_promote_candidates(threshold: 1000)
+        candidates = analyze(threshold: threshold)
+        promoted_count = 0
+
+        candidates.each do |candidate|
+          field = candidate[:field]
+
+          # Only auto-promote fields with high priority
+          if candidate[:priority] >= 80
+            Rails.logger.info "FieldAnalyzer: Auto-promoting field '#{field.name}' (usage: #{field.usage_count}, priority: #{candidate[:priority]})"
+
+            field.promote!
+            promoted_count += 1
+
+            # TODO: In real implementation, we would need to:
+            # 1. Generate migration to add column
+            # 2. Backfill existing data
+            # 3. Update queries to use promoted field
+            # For now, just mark as promoted
+          end
+        end
+
+        promoted_count
+      end
+
+      private
+
+      # Calculate promotion priority (0-100 scale)
+      def self.calculate_priority(field, threshold)
+        score = 0
+
+        # Usage score (0-50 points)
+        usage_score = [50, (field.usage_count.to_f / (threshold * 10) * 50).to_i].min
+        score += usage_score
+
+        # Recency score (0-25 points)
+        days_since_seen = (Time.current - field.last_seen_at) / 1.day
+        recency_score = [0, [25, (25 - days_since_seen).to_i].min].max
+        score += recency_score
+
+        # Type score (0-25 points)
+        # Favor simple types that are easier to index
+        type_score = case field.field_type
+        when "string", "number", "boolean"
+                      25
+        when "datetime"
+                      20
+        else
+                      10
+        end
+        score += type_score
+
+        [100, score].min
+      end
+
+      # Generate human-readable reason for promotion
+      def self.promotion_reason(field, threshold)
+        reasons = []
+
+        if field.usage_count >= threshold * 10
+          reasons << "extremely high usage (#{field.usage_count})"
+        elsif field.usage_count >= threshold * 5
+          reasons << "very high usage (#{field.usage_count})"
+        else
+          reasons << "high usage (#{field.usage_count})"
+        end
+
+        days_since_seen = (Time.current - field.last_seen_at) / 1.day
+        if days_since_seen < 1
+          reasons << "actively used today"
+        elsif days_since_seen < 7
+          reasons << "recently active"
+        end
+
+        reasons.join(", ")
+      end
+    end
+  end
+end

data/lib/solid_log/core/services/health_service.rb

@@ -0,0 +1,151 @@
+module SolidLog
+  module Core
+    class HealthService
+      # Get comprehensive health metrics
+      def self.metrics
+        {
+          ingestion: ingestion_metrics,
+          parsing: parsing_metrics,
+          storage: storage_metrics,
+          performance: performance_metrics
+        }
+      end
+
+      # Ingestion metrics
+      def self.ingestion_metrics
+        today_start = Time.current.beginning_of_day
+        hour_ago = 1.hour.ago
+
+        {
+          total_raw: RawEntry.count,
+          today_raw: RawEntry.where("received_at >= ?", today_start).count,
+          last_hour_raw: RawEntry.where("received_at >= ?", hour_ago).count,
+          last_ingestion: RawEntry.order(received_at: :desc).first&.received_at
+        }
+      end
+
+      # Parsing metrics
+      def self.parsing_metrics
+        unparsed_count = RawEntry.unparsed.count
+        total_raw = RawEntry.count
+        stale_threshold = 1.hour.ago
+
+        backlog_percentage = total_raw > 0 ? (unparsed_count.to_f / total_raw * 100).round(2) : 0
+        stale_unparsed = RawEntry.unparsed.where("received_at < ?", stale_threshold).count
+
+        health_status = case
+        when backlog_percentage > 50
+                         "critical"
+        when backlog_percentage > 20
+                         "warning"
+        when stale_unparsed > 100
+                         "degraded"
+        else
+                         "healthy"
+        end
+
+        {
+          unparsed_count: unparsed_count,
+          parse_backlog_percentage: backlog_percentage,
+          stale_unparsed: stale_unparsed,
+          health_status: health_status
+        }
+      end
+
+      # Storage metrics
+      def self.storage_metrics
+        promoted_fields = Field.promoted.count
+        hot_fields = Field.hot_fields(1000).count
+
+        {
+          total_entries: Entry.count,
+          total_fields: Field.count,
+          promoted_fields: promoted_fields,
+          hot_fields_count: hot_fields,
+          database_size: database_size
+        }
+      end
+
+      # Performance metrics
+      def self.performance_metrics
+        hour_ago = 1.hour.ago
+        hour_entries = Entry.where("timestamp >= ?", hour_ago)
+
+        error_count = hour_entries.errors.count
+        total_count = hour_entries.count
+        error_rate = total_count > 0 ? (error_count.to_f / total_count * 100).round(2) : 0
+
+        avg_duration = hour_entries
+          .where.not(duration: nil)
+          .average(:duration)
+          &.round(2) || 0
+
+        {
+          cache_entries: FacetCache.count,
+          expired_cache: FacetCache.where("expires_at < ?", Time.current).count,
+          error_rate: error_rate,
+          avg_duration: avg_duration
+        }
+      end
+
+      # Get database size (platform-specific)
+      def self.database_size
+        adapter_name = ActiveRecord::Base.connection.adapter_name.downcase
+
+        case adapter_name
+        when "sqlite"
+          sqlite_database_size
+        when "postgresql"
+          postgresql_database_size
+        when "mysql"
+          mysql_database_size
+        else
+          "Unknown"
+        end
+      rescue => e
+        Rails.logger.error "HealthService: Error getting database size: #{e.message}"
+        "Error"
+      end
+
+      private
+
+      def self.sqlite_database_size
+        db_path = ActiveRecord::Base.connection_db_config.database
+        return "Unknown" unless File.exist?(db_path)
+
+        size_bytes = File.size(db_path)
+        format_bytes(size_bytes)
+      end
+
+      def self.postgresql_database_size
+        db_name = ActiveRecord::Base.connection_db_config.database
+        result = ActiveRecord::Base.connection.execute(
+          "SELECT pg_database_size('#{db_name}')"
+        )
+        size_bytes = result.first["pg_database_size"].to_i
+        format_bytes(size_bytes)
+      end
+
+      def self.mysql_database_size
+        db_name = ActiveRecord::Base.connection_db_config.database
+        result = ActiveRecord::Base.connection.execute(
+          "SELECT SUM(data_length + index_length) as size
+           FROM information_schema.TABLES
+           WHERE table_schema = '#{db_name}'"
+        )
+        size_bytes = result.first[0].to_i
+        format_bytes(size_bytes)
+      end
+
+      def self.format_bytes(bytes)
+        return "0 B" if bytes == 0
+
+        units = %w[B KB MB GB TB]
+        exp = (Math.log(bytes) / Math.log(1024)).to_i
+        exp = [exp, units.size - 1].min
+
+        "%.2f %s" % [bytes.to_f / (1024 ** exp), units[exp]]
+      end
+    end
+  end
+end

data/lib/solid_log/core/services/retention_service.rb

@@ -0,0 +1,72 @@
+module SolidLog
+  module Core
+    class RetentionService
+      # Cleanup old entries based on retention policies
+      def self.cleanup(retention_days:, error_retention_days:)
+        stats = {
+          entries_deleted: 0,
+          raw_deleted: 0,
+          cache_cleared: 0
+        }
+
+        # Calculate retention thresholds
+        regular_threshold = retention_days.days.ago
+        error_threshold = error_retention_days.days.ago
+
+        # Delete old regular logs (not errors)
+        stats[:entries_deleted] = Entry
+          .where("timestamp < ?", regular_threshold)
+          .where.not(level: %w[error fatal])
+          .delete_all
+
+        # Delete old error logs
+        stats[:entries_deleted] += Entry
+          .where("timestamp < ?", error_threshold)
+          .where(level: %w[error fatal])
+          .delete_all
+
+        # Delete corresponding raw entries (keep unparsed for investigation)
+        raw_ids = Entry.pluck(:raw_id).compact
+        stats[:raw_deleted] = RawEntry
+          .parsed
+          .where.not(id: raw_ids)
+          .delete_all
+
+        # Clear old cache entries
+        stats[:cache_cleared] = FacetCache
+          .where("expires_at < ?", Time.current)
+          .delete_all
+
+        stats
+      end
+
+      # Vacuum database (SQLite only)
+      def self.vacuum_database
+        return false unless sqlite_database?
+
+        ActiveRecord::Base.connection.execute("VACUUM")
+        true
+      rescue => e
+        Rails.logger.error "RetentionService: VACUUM failed: #{e.message}"
+        false
+      end
+
+      # Optimize database (SQLite PRAGMA optimize)
+      def self.optimize_database
+        return false unless sqlite_database?
+
+        ActiveRecord::Base.connection.execute("PRAGMA optimize")
+        true
+      rescue => e
+        Rails.logger.error "RetentionService: PRAGMA optimize failed: #{e.message}"
+        false
+      end
+
+      private
+
+      def self.sqlite_database?
+        ActiveRecord::Base.connection.adapter_name.downcase == "sqlite"
+      end
+    end
+  end
+end