solid_log-core 0.1.0

data/lib/solid_log/core/services/search_service.rb

@@ -0,0 +1,269 @@
+module SolidLog
+  module Core
+    class SearchService
+      def initialize(params = {}, **options)
+        @params = params
+        @facet_cache_ttl = options[:facet_cache_ttl]
+        @cache_enabled = @facet_cache_ttl.present?
+      end
+
+      def search
+        scope = Entry.all
+
+        # Apply search query
+        scope = apply_search(scope)
+
+        # Apply filters
+        scope = apply_filters(scope)
+
+        # Apply pagination
+        scope = apply_pagination(scope)
+
+        # Return scoped results
+        scope.recent
+      end
+
+      def available_facets
+        return cached_facets if cache_enabled?
+
+        facets = {
+          levels: Entry.facets_for("level"),
+          apps: Entry.facets_for("app"),
+          envs: Entry.facets_for("env"),
+          controllers: Entry.facets_for("controller"),
+          actions: Entry.facets_for("action"),
+          paths: Entry.facets_for("path"),
+          methods: Entry.facets_for("method"),
+          status_codes: Entry.facets_for("status_code")
+        }
+
+        # Add promoted fields dynamically
+        facets.merge!(promoted_field_facets)
+
+        cache_facets(facets) if cache_enabled?
+        facets
+      end
+
+      private
+
+      def apply_search(scope)
+        return scope if @params[:query].blank?
+
+        # Full-text search starts from Entry.all (doesn't chain with existing scope)
+        # This is a limitation of FTS implementation
+        Entry.search_fts(@params[:query])
+      end
+
+      def apply_filters(scope)
+        scope = apply_level_filter(scope)
+        scope = apply_app_filter(scope)
+        scope = apply_env_filter(scope)
+        # scope = apply_controller_filter(scope)
+        scope = apply_action_filter(scope)
+        scope = apply_path_filter(scope)
+        scope = apply_method_filter(scope)
+        scope = apply_status_code_filter(scope)
+        scope = apply_duration_filter(scope)
+        scope = apply_time_range_filter(scope)
+        scope = apply_correlation_filters(scope)
+        scope = apply_promoted_field_filters(scope)
+        scope
+      end
+
+      def apply_level_filter(scope)
+        levels = Array(@params[:levels]).reject(&:blank?)
+        return scope if levels.empty?
+
+        scope.where(level: levels)
+      end
+
+      def apply_app_filter(scope)
+        return scope if @params[:app].blank?
+
+        scope.by_app(@params[:app])
+      end
+
+      def apply_env_filter(scope)
+        return scope if @params[:env].blank?
+
+        scope.by_env(@params[:env])
+      end
+
+      def apply_time_range_filter(scope)
+        start_time = parse_datetime(@params[:start_time])
+        end_time = parse_datetime(@params[:end_time])
+
+        scope.by_time_range(start_time, end_time)
+      end
+
+      def apply_correlation_filters(scope)
+        scope = scope.by_request_id(@params[:request_id]) if @params[:request_id].present?
+        scope = scope.by_job_id(@params[:job_id]) if @params[:job_id].present?
+        scope
+      end
+
+      def apply_controller_filter(scope)
+        return scope if @params[:controller].blank?
+        scope.by_controller(@params[:controller])
+      end
+
+      def apply_action_filter(scope)
+        return scope if @params[:action].blank?
+        scope.by_action(@params[:action])
+      end
+
+      def apply_path_filter(scope)
+        return scope if @params[:path].blank?
+        scope.by_path(@params[:path])
+      end
+
+      def apply_method_filter(scope)
+        return scope if @params[:method].blank?
+        scope.by_method(@params[:method])
+      end
+
+      def apply_status_code_filter(scope)
+        return scope if @params[:status_code].blank?
+        scope.by_status_code(@params[:status_code])
+      end
+
+      def apply_duration_filter(scope)
+        min_duration = @params[:min_duration].presence
+        max_duration = @params[:max_duration].presence
+        return scope if min_duration.blank? && max_duration.blank?
+
+        scope.by_duration_range(min_duration, max_duration)
+      end
+
+      def apply_promoted_field_filters(scope)
+        # Apply filters for any promoted fields that have columns
+        Field.promoted.each do |field|
+          next unless Entry.column_names.include?(field.name)
+
+          # Handle different filter types
+          case field.filter_type
+          when "multiselect"
+            values = Array(@params[field.name.to_sym]).reject(&:blank?)
+            next if values.empty?
+            scope = scope.where(field.name => values)
+          when "tokens"
+            values = parse_token_values(@params[field.name.to_sym])
+            next if values.empty?
+            scope = scope.where(field.name => values)
+          when "range"
+            min_value = @params["min_#{field.name}".to_sym].presence
+            max_value = @params["max_#{field.name}".to_sym].presence
+            next if min_value.blank? && max_value.blank?
+            scope = scope.where("#{field.name} >= ?", min_value) if min_value.present?
+            scope = scope.where("#{field.name} <= ?", max_value) if max_value.present?
+          when "exact", "contains"
+            param_value = @params[field.name.to_sym]
+            next if param_value.blank?
+            if field.filter_type == "contains"
+              scope = scope.where("#{field.name} LIKE ?", "%#{param_value}%")
+            else
+              scope = scope.where(field.name => param_value)
+            end
+          end
+        end
+        scope
+      end
+
+      def parse_datetime(datetime_str)
+        return nil if datetime_str.blank?
+
+        Time.zone.parse(datetime_str)
+      rescue ArgumentError
+        nil
+      end
+
+      def parse_token_values(input)
+        return [] if input.blank?
+
+        # Split by comma, semicolon, or newline and clean up
+        input.to_s.split(/[,;\n]/).map(&:strip).reject(&:blank?)
+      end
+
+      def limit
+        limit = @params[:limit].to_i
+        limit > 0 ? [limit, 1000].min : 200
+      end
+
+      def cache_enabled?
+        @cache_enabled
+      end
+
+      def promoted_field_facets
+        promoted_facets = {}
+
+        # Get all promoted fields that have actual database columns
+        Field.promoted.each do |field|
+          # Skip token fields - we don't fetch facets for high-cardinality fields
+          next if field.filter_type == "tokens"
+
+          # Check if the column exists on the entries table
+          if Entry.column_names.include?(field.name)
+            # Get distinct values for this field
+            values = Entry.facets_for(field.name)
+            promoted_facets[field.name.to_sym] = values if values.any?
+          end
+        end
+
+        promoted_facets
+      end
+
+      def cached_facets
+        FacetCache.fetch("facets:all", ttl: cache_ttl) do
+          facets = {
+            levels: Entry.facets_for("level"),
+            apps: Entry.facets_for("app"),
+            envs: Entry.facets_for("env"),
+            controllers: Entry.facets_for("controller"),
+            actions: Entry.facets_for("action"),
+            paths: Entry.facets_for("path"),
+            methods: Entry.facets_for("method"),
+            status_codes: Entry.facets_for("status_code")
+          }
+          facets.merge!(promoted_field_facets)
+          facets
+        end
+      end
+
+      def cache_facets(facets)
+        FacetCache.store("facets:all", facets, ttl: cache_ttl)
+      end
+
+      def cache_ttl
+        @facet_cache_ttl
+      end
+
+      def apply_pagination(scope)
+        # For infinite scroll: load logs before/after a specific ID
+        # - before_id: Load older logs (scroll up)
+        # - after_id: Load newer logs (scroll down / live tail)
+
+        if @params[:before_id].present?
+          # Loading older logs (scrolling up)
+          # Get logs with ID < before_id, ordered descending
+          # Partial will reverse to ASC for prepending
+          scope = scope.where("id < ?", @params[:before_id])
+                       .order(id: :desc)
+                       .limit(limit)
+        elsif @params[:after_id].present?
+          # Loading newer logs (scrolling down / live tail)
+          # Get logs with ID > after_id, ordered descending
+          # Partial will reverse to ASC for appending (oldest first, newest last)
+          scope = scope.where("id > ?", @params[:after_id])
+                       .order(id: :desc)
+                       .limit(limit)
+        else
+          # Initial load: get most recent logs
+          scope = scope.order(id: :desc)
+                       .limit(limit)
+        end
+
+        scope
+      end
+    end
+  end
+end

data/lib/solid_log/core/version.rb

@@ -0,0 +1,5 @@
+module SolidLog
+  module Core
+    VERSION = "0.1.0"
+  end
+end

data/lib/solid_log/core.rb

@@ -0,0 +1,106 @@
+require "solid_log/core/version"
+require "solid_log/core/configuration"
+require "solid_log/core/client"
+require "solid_log/silence_middleware"
+
+# Database adapters
+require "solid_log/adapters/base_adapter"
+require "solid_log/adapters/sqlite_adapter"
+require "solid_log/adapters/postgresql_adapter"
+require "solid_log/adapters/mysql_adapter"
+require "solid_log/adapters/adapter_factory"
+
+# Parser
+require "solid_log/parser"
+
+# Loggers
+require "solid_log/direct_logger"
+
+# Service objects
+require "solid_log/core/services/retention_service"
+require "solid_log/core/services/field_analyzer"
+require "solid_log/core/services/search_service"
+require "solid_log/core/services/correlation_service"
+require "solid_log/core/services/health_service"
+
+# Models (explicit requires - no engine, no app/ directory)
+require "solid_log/models/record"
+require "solid_log/models/raw_entry"
+require "solid_log/models/entry"
+require "solid_log/models/token"
+require "solid_log/models/field"
+require "solid_log/models/facet_cache"
+
+module SolidLog
+  module Core
+    class << self
+      attr_writer :configuration
+
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure
+        yield(configuration)
+      end
+
+      def configure_client(&block)
+        Client.configure(&block)
+      end
+
+      def reset_configuration!
+        @configuration = Configuration.new
+      end
+
+      # Get database adapter
+      def adapter
+        SolidLog::Adapters::AdapterFactory.adapter
+      end
+
+      # Execute block without logging (prevent recursion)
+      def without_logging
+        Thread.current[:solid_log_silenced] = true
+        yield
+      ensure
+        Thread.current[:solid_log_silenced] = nil
+      end
+
+      # Check if logging is silenced
+      def silenced?
+        Thread.current[:solid_log_silenced] == true
+      end
+    end
+  end
+end
+
+# Legacy SolidLog module for backward compatibility with models
+module SolidLog
+  class << self
+    def configuration
+      Core.configuration
+    end
+
+    def configure(&block)
+      Core.configure(&block)
+    end
+
+    def adapter
+      Core.adapter
+    end
+
+    def without_logging(&block)
+      Core.without_logging(&block)
+    end
+
+    def silenced?
+      Core.silenced?
+    end
+  end
+
+  # Alias service classes for easier access
+  RetentionService = Core::RetentionService
+  FieldAnalyzer = Core::FieldAnalyzer
+  SearchService = Core::SearchService
+  CorrelationService = Core::CorrelationService
+  HealthService = Core::HealthService
+end

data/lib/solid_log/direct_logger.rb

@@ -0,0 +1,197 @@
+module SolidLog
+  # DirectLogger writes logs directly to the database, bypassing HTTP overhead.
+  # This is optimized for the parent Rails application that has direct database access.
+  #
+  # Features:
+  # - Batches logs in memory for performance
+  # - Flushes on size threshold (default: 100 logs)
+  # - Flushes on time threshold (default: 5 seconds)
+  # - Thread-safe
+  # - Flushes remaining logs on process exit
+  #
+  # Usage:
+  #   config.lograge.logger = ActiveSupport::Logger.new(SolidLog::DirectLogger.new)
+  class DirectLogger
+    attr_reader :buffer_size, :flush_interval, :last_flush_time
+
+    def initialize(batch_size: nil, flush_interval: 5, token_id: nil, eager_flush_levels: [:error, :fatal])
+      @buffer = []
+      @mutex = Mutex.new
+      @batch_size = batch_size || SolidLog.configuration&.max_batch_size || 100
+      @flush_interval = flush_interval # seconds
+      @last_flush_time = Time.current
+      @closed = false
+      @eager_flush_levels = Array(eager_flush_levels).map(&:to_s)
+
+      # Token ID priority: explicit param > ENV var > nil (for DirectLogger)
+      # token_id is only needed for audit trail (tracking which source ingested the log)
+      # For DirectLogger, nil is fine since we're logging internally
+      @token_id = token_id || token_id_from_env
+
+      # Start background flusher thread
+      start_flush_thread
+
+      # Ensure we flush on exit
+      at_exit { close }
+    end
+
+    # Write a log message (called by Rails logger)
+    # This is non-blocking - logs are buffered and flushed asynchronously
+    # EXCEPT for error/fatal logs which flush immediately to prevent data loss on crash
+    def write(message)
+      return if @closed
+
+      log_entry = parse_message(message)
+      return unless log_entry # Skip if parsing failed
+
+      # Check if this is a critical log that should flush immediately
+      should_eager_flush = false
+      if @eager_flush_levels.any?
+        parsed_data = JSON.parse(log_entry[:payload]) rescue {}
+        log_level = parsed_data["level"]&.to_s&.downcase
+        should_eager_flush = @eager_flush_levels.include?(log_level)
+      end
+
+      @mutex.synchronize do
+        @buffer << log_entry
+
+        # Flush immediately if:
+        # 1. Batch size reached, OR
+        # 2. This is a critical log (error/fatal) to prevent loss on crash
+        flush_internal if @buffer.size >= @batch_size || should_eager_flush
+      end
+    end
+
+    # Explicitly flush all buffered logs
+    # Useful for testing or before shutdown
+    def flush
+      @mutex.synchronize { flush_internal }
+    end
+
+    # Close the logger and flush remaining logs
+    def close
+      return if @closed
+      @closed = true
+
+      # Stop the flush thread
+      @flush_thread&.kill
+
+      # Flush remaining logs
+      flush
+    end
+
+    # Get current buffer size (for monitoring/debugging)
+    def buffer_size
+      @mutex.synchronize { @buffer.size }
+    end
+
+    private
+
+    # Parse a log message into the format expected by RawEntry
+    def parse_message(message)
+      # Handle different message formats
+      if message.is_a?(String)
+        # Try to parse as JSON (from Lograge)
+        begin
+          log_data = JSON.parse(message)
+        rescue JSON::ParserError
+          # Plain text log - wrap in JSON
+          log_data = {
+            message: message.strip,
+            timestamp: Time.current.utc.iso8601,
+            level: "info"
+          }
+        end
+      elsif message.is_a?(Hash)
+        log_data = message
+      else
+        # Unsupported format
+        return nil
+      end
+
+      # Return in RawEntry format
+      {
+        payload: log_data.to_json,
+        token_id: @token_id,
+        received_at: Time.current
+      }
+    rescue => e
+      # If parsing fails, log to stderr but don't crash
+      $stderr.puts "SolidLog::DirectLogger parse error: #{e.message}"
+      nil
+    end
+
+    # Internal flush (must be called within mutex.synchronize)
+    def flush_internal
+      return if @buffer.empty?
+
+      batch = @buffer.dup
+      @buffer.clear
+      @last_flush_time = Time.current
+
+      # Release mutex before database write
+      @mutex.unlock
+
+      begin
+        # Write batch to database
+        write_batch(batch)
+      ensure
+        # Re-acquire mutex
+        @mutex.lock
+      end
+    rescue => e
+      # On error, log to stderr
+      $stderr.puts "SolidLog::DirectLogger flush error: #{e.message}"
+      $stderr.puts e.backtrace.first(5).join("\n")
+    end
+
+    # Write a batch of logs to the database
+    def write_batch(batch)
+      return if batch.empty?
+
+      # Prevent recursive logging
+      SolidLog.without_logging do
+        # Ensure we have an ActiveRecord connection in this thread
+        ActiveRecord::Base.connection_pool.with_connection do
+          # Use insert_all for performance (single SQL statement)
+          RawEntry.insert_all(batch)
+        end
+      end
+    end
+
+    # Try to get token_id from environment variable
+    # This is useful for audit trail (tracking which source ingested logs)
+    def token_id_from_env
+      # Check for SOLIDLOG_TOKEN_ID env var
+      token_id = ENV["SOLIDLOG_TOKEN_ID"]
+      return nil unless token_id
+
+      # Validate it's a number
+      token_id.to_i if token_id.match?(/^\d+$/)
+    end
+
+    # Start background thread that flushes periodically
+    def start_flush_thread
+      @flush_thread = Thread.new do
+        loop do
+          sleep @flush_interval
+
+          # Check if we need to flush based on time
+          @mutex.synchronize do
+            time_since_flush = Time.current - @last_flush_time
+            flush_internal if time_since_flush >= @flush_interval && @buffer.any?
+          end
+        end
+      rescue => e
+        # Thread died - log but don't crash
+        $stderr.puts "SolidLog::DirectLogger flush thread error: #{e.message}"
+      end
+
+      # Make it a daemon thread so it doesn't prevent process exit
+      @flush_thread.abort_on_exception = false
+
+      # Set thread priority lower so it doesn't interfere with app
+      @flush_thread.priority = -1
+    end
+  end
+end