solid_log-core 0.1.0

data/lib/solid_log/adapters/sqlite_adapter.rb

@@ -0,0 +1,149 @@
+module SolidLog
+  module Adapters
+    class SqliteAdapter < BaseAdapter
+      def search(query, base_scope = Entry)
+        sanitized_query = Entry.sanitize_sql_like(query)
+        table_name = Entry.table_name
+
+        # Build LIKE query for partial matching
+        like_condition = case_insensitive_like("#{table_name}.message", "%#{sanitized_query}%")
+        like_sql = "SELECT #{table_name}.* FROM #{table_name} WHERE #{like_condition}"
+
+        # Build FTS query for full-text matching
+        fts_fragment = fts_search(query)
+        fts_sql = "SELECT #{table_name}.* FROM #{table_name} #{fts_fragment}"
+
+        # UNION both queries (DISTINCT removes duplicates)
+        union_sql = "#{fts_sql} UNION #{like_sql}"
+        Entry.from("(#{union_sql}) AS #{table_name}")
+      end
+
+      def fts_search(query)
+        # SQLite FTS5 search
+        sanitized_query = connection.quote(query)
+        <<~SQL.squish
+          JOIN solid_log_entries_fts
+          ON solid_log_entries.id = solid_log_entries_fts.rowid
+          WHERE solid_log_entries_fts MATCH #{sanitized_query}
+        SQL
+      end
+
+      def case_insensitive_like(column, pattern)
+        # SQLite uses LIKE with COLLATE NOCASE for case-insensitive matching
+        "#{column} LIKE #{connection.quote(pattern)} COLLATE NOCASE"
+      end
+
+      def claim_batch(batch_size)
+        # SQLite doesn't support SKIP LOCKED, so we use a different approach
+        # Get IDs first, then update them
+        RawEntry.transaction do
+          ids = RawEntry.where(parsed: false)
+            .order(received_at: :asc)
+            .limit(batch_size)
+            .lock
+            .pluck(:id)
+
+          return [] if ids.empty?
+
+          # Mark as parsed immediately
+          RawEntry.where(id: ids).update_all(parsed: true, parsed_at: Time.current)
+
+          # Return the entries
+          RawEntry.where(id: ids).to_a
+        end
+      end
+
+      def extract_json_field(column, field_name)
+        "json_extract(#{column}, '$.#{field_name}')"
+      end
+
+      def facet_values(column)
+        "SELECT DISTINCT #{column} FROM solid_log_entries WHERE #{column} IS NOT NULL ORDER BY #{column}"
+      end
+
+      def optimize!
+        execute("PRAGMA optimize")
+        execute("PRAGMA wal_checkpoint(TRUNCATE)")
+      end
+
+      def database_size
+        db_path = connection.instance_variable_get(:@config)[:database]
+        File.size(db_path) rescue 0
+      end
+
+      def supports_skip_locked?
+        false
+      end
+
+      def supports_native_json?
+        false
+      end
+
+      def supports_full_text_search?
+        true  # FTS5
+      end
+
+      def configure!
+        execute("PRAGMA journal_mode=WAL")
+        execute("PRAGMA synchronous=NORMAL")
+        execute("PRAGMA busy_timeout=5000")
+        execute("PRAGMA cache_size=-64000")  # 64MB
+        execute("PRAGMA temp_store=MEMORY")
+        execute("PRAGMA mmap_size=268435456")  # 256MB
+      rescue ActiveRecord::StatementInvalid => e
+        # Some pragmas may not be supported in all SQLite versions
+        Rails.logger.warn("SQLite configuration warning: #{e.message}")
+      end
+
+      def timestamp_to_epoch_sql(column)
+        "strftime('%s', #{column})"
+      end
+
+      # FTS5-specific methods
+      def create_fts_table_sql
+        <<~SQL
+          CREATE VIRTUAL TABLE IF NOT EXISTS solid_log_entries_fts USING fts5(
+            message,
+            extra_text,
+            content='solid_log_entries',
+            content_rowid='id'
+          )
+        SQL
+      end
+
+      def create_fts_triggers_sql
+        [
+          # Insert trigger
+          <<~SQL,
+            CREATE TRIGGER IF NOT EXISTS solid_log_entries_fts_insert
+            AFTER INSERT ON solid_log_entries
+            BEGIN
+              INSERT INTO solid_log_entries_fts(rowid, message, extra_text)
+              VALUES (new.id, new.message, new.extra_fields);
+            END
+          SQL
+
+          # Update trigger
+          <<~SQL,
+            CREATE TRIGGER IF NOT EXISTS solid_log_entries_fts_update
+            AFTER UPDATE ON solid_log_entries
+            BEGIN
+              UPDATE solid_log_entries_fts
+              SET message = new.message, extra_text = new.extra_fields
+              WHERE rowid = new.id;
+            END
+          SQL
+
+          # Delete trigger
+          <<~SQL
+            CREATE TRIGGER IF NOT EXISTS solid_log_entries_fts_delete
+            AFTER DELETE ON solid_log_entries
+            BEGIN
+              DELETE FROM solid_log_entries_fts WHERE rowid = old.id;
+            END
+          SQL
+        ]
+      end
+    end
+  end
+end

data/lib/solid_log/core/client/buffer.rb

@@ -0,0 +1,112 @@
+require "thread"
+
+module SolidLog
+  module Core
+    class Buffer
+      attr_reader :queue, :mutex, :flush_thread
+
+      def initialize(http_sender:, batch_size: 100, flush_interval: 5, max_queue_size: 10_000)
+        @http_sender = http_sender
+        @batch_size = batch_size
+        @flush_interval = flush_interval
+        @max_queue_size = max_queue_size
+
+        @queue = []
+        @mutex = Mutex.new
+        @flush_thread = nil
+        @running = false
+      end
+
+      # Add entry to buffer
+      def add(entry)
+        @mutex.synchronize do
+          # Drop oldest entries if queue is full
+          @queue.shift if @queue.size >= @max_queue_size
+
+          @queue << normalize_entry(entry)
+
+          # Auto-flush if batch size reached
+          flush_if_needed
+        end
+      end
+
+      # Flush pending entries
+      def flush
+        entries_to_send = []
+
+        @mutex.synchronize do
+          return if @queue.empty?
+
+          # Take up to batch_size entries
+          entries_to_send = @queue.shift(@batch_size)
+        end
+
+        # Send outside of mutex lock
+        send_entries(entries_to_send) if entries_to_send.any?
+      end
+
+      # Start automatic flushing in background thread
+      def start_auto_flush
+        return if @running
+
+        @running = true
+        @flush_thread = Thread.new do
+          loop do
+            sleep @flush_interval
+            break unless @running
+
+            begin
+              flush
+            rescue => e
+              Rails.logger.error "SolidLog::Client: Auto-flush error: #{e.message}" if defined?(Rails)
+            end
+          end
+        end
+      end
+
+      # Stop automatic flushing
+      def stop_auto_flush
+        @running = false
+        @flush_thread&.join
+        @flush_thread = nil
+      end
+
+      # Get current queue size
+      def size
+        @mutex.synchronize { @queue.size }
+      end
+
+      private
+
+      def flush_if_needed
+        flush if @queue.size >= @batch_size
+      end
+
+      def send_entries(entries)
+        @http_sender.send_batch(entries)
+      rescue => e
+        Rails.logger.error "SolidLog::Client: Failed to send batch: #{e.message}" if defined?(Rails)
+
+        # Put entries back in queue for retry
+        @mutex.synchronize do
+          @queue.unshift(*entries)
+
+          # Trim queue if it exceeds max size
+          while @queue.size > @max_queue_size
+            @queue.shift
+          end
+        end
+      end
+
+      def normalize_entry(entry)
+        # Ensure entry is a hash
+        entry = entry.to_h if entry.respond_to?(:to_h)
+
+        # Add timestamp if missing
+        entry[:timestamp] ||= Time.current.iso8601
+
+        entry
+      end
+    end
+  end
+end

data/lib/solid_log/core/client/configuration.rb

@@ -0,0 +1,31 @@
+module SolidLog
+  module Core
+    class ClientConfiguration
+      attr_accessor :service_url,
+                    :token,
+                    :app_name,
+                    :environment,
+                    :batch_size,
+                    :flush_interval,
+                    :max_queue_size,
+                    :retry_max_attempts,
+                    :enabled
+
+      def initialize
+        @service_url = nil
+        @token = nil
+        @app_name = "app"
+        @environment = "production"
+        @batch_size = 100
+        @flush_interval = 5 # seconds
+        @max_queue_size = 10_000
+        @retry_max_attempts = 3
+        @enabled = true
+      end
+
+      def valid?
+        service_url.present? && token.present?
+      end
+    end
+  end
+end

data/lib/solid_log/core/client/http.rb

@@ -0,0 +1,89 @@
+require "net/http"
+require "json"
+require "uri"
+
+module SolidLog
+  module Core
+    class HttpSender
+      def initialize(url:, token:, retry_handler:)
+        @url = url
+        @token = token
+        @retry_handler = retry_handler
+      end
+
+      # Send a batch of entries to the ingestion endpoint
+      def send_batch(entries)
+        return if entries.empty?
+
+        @retry_handler.with_retry do
+          perform_request(entries)
+        end
+      end
+
+      private
+
+      def perform_request(entries)
+        uri = URI.parse(@url)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == "https")
+        http.open_timeout = 5
+        http.read_timeout = 10
+
+        request = build_request(uri.path, entries)
+        response = http.request(request)
+
+        handle_response(response, entries)
+      end
+
+      def build_request(path, entries)
+        request = Net::HTTP::Post.new(path)
+        request["Authorization"] = "Bearer #{@token}"
+        request["Content-Type"] = if entries.size == 1
+                                    "application/json"
+        else
+                                    "application/x-ndjson"
+        end
+
+        request.body = if entries.size == 1
+                        JSON.generate(entries.first)
+        else
+                        entries.map { |e| JSON.generate(e) }.join("\n")
+        end
+
+        request
+      end
+
+      def handle_response(response, entries)
+        case response.code.to_i
+        when 200..299
+          # Success
+          log_debug "Successfully sent #{entries.size} entries"
+        when 400..499
+          # Client error - don't retry
+          log_error "Client error (#{response.code}): #{response.body}"
+          raise ClientError, "HTTP #{response.code}: #{response.body}"
+        when 500..599
+          # Server error - retry
+          log_error "Server error (#{response.code}): #{response.body}"
+          raise ServerError, "HTTP #{response.code}: #{response.body}"
+        else
+          log_error "Unexpected response (#{response.code}): #{response.body}"
+          raise UnexpectedError, "HTTP #{response.code}: #{response.body}"
+        end
+      end
+
+      def log_debug(message)
+        Rails.logger.debug "SolidLog::Client: #{message}" if defined?(Rails)
+      end
+
+      def log_error(message)
+        Rails.logger.error "SolidLog::Client: #{message}" if defined?(Rails)
+      end
+
+      # Error classes
+      class ClientError < StandardError; end
+      class ServerError < StandardError; end
+      class UnexpectedError < StandardError; end
+    end
+  end
+end

data/lib/solid_log/core/client/lograge_formatter.rb

@@ -0,0 +1,99 @@
+module SolidLog
+  module Core
+    class LogrageFormatter
+      # Format Lograge output for SolidLog
+      def call(data)
+        # Lograge data is already a hash with structured fields
+        # Just ensure we have the required fields
+
+        formatted = {
+          timestamp: Time.current.iso8601,
+          level: infer_level(data),
+          message: build_message(data),
+          app: Client.configuration.app_name,
+          env: Client.configuration.environment
+        }
+
+        # Add Lograge fields
+        formatted.merge!(extract_lograge_fields(data))
+
+        # Send to client
+        Client.log(formatted)
+
+        # Return JSON for Rails logger (if also logging to file)
+        JSON.generate(formatted)
+      end
+
+      private
+
+      def infer_level(data)
+        # Determine log level based on status code or error
+        if data[:status].to_i >= 500
+          "error"
+        elsif data[:status].to_i >= 400
+          "warn"
+        elsif data[:exception].present?
+          "error"
+        else
+          "info"
+        end
+      end
+
+      def build_message(data)
+        parts = []
+
+        # HTTP method and path
+        parts << "#{data[:method]} #{data[:path]}" if data[:method] && data[:path]
+
+        # Status code
+        parts << "(#{data[:status]})" if data[:status]
+
+        # Duration
+        parts << "#{data[:duration]}ms" if data[:duration]
+
+        # Controller and action
+        if data[:controller] && data[:action]
+          parts << "#{data[:controller]}##{data[:action]}"
+        end
+
+        parts.join(" ")
+      end
+
+      def extract_lograge_fields(data)
+        fields = {}
+
+        # Standard Lograge fields
+        fields[:method] = data[:method] if data[:method]
+        fields[:path] = data[:path] if data[:path]
+        fields[:controller] = data[:controller] if data[:controller]
+        fields[:action] = data[:action] if data[:action]
+        fields[:status_code] = data[:status] if data[:status]
+        fields[:duration] = data[:duration] if data[:duration]
+
+        # Request ID for correlation
+        fields[:request_id] = data[:request_id] if data[:request_id]
+
+        # View rendering time
+        fields[:view_runtime] = data[:view] if data[:view]
+        fields[:db_runtime] = data[:db] if data[:db]
+
+        # Params (be careful with sensitive data)
+        fields[:params] = data[:params] if data[:params]
+
+        # Remote IP
+        fields[:ip] = data[:remote_ip] if data[:remote_ip]
+
+        # User agent
+        fields[:user_agent] = data[:user_agent] if data[:user_agent]
+
+        # Any other custom fields
+        data.except(:method, :path, :controller, :action, :status, :duration,
+                   :request_id, :view, :db, :params, :remote_ip, :user_agent).each do |key, value|
+          fields[key] = value unless value.nil?
+        end
+
+        fields
+      end
+    end
+  end
+end

data/lib/solid_log/core/client/retry_handler.rb

@@ -0,0 +1,48 @@
+module SolidLog
+  module Core
+    class RetryHandler
+      def initialize(max_attempts: 3)
+        @max_attempts = max_attempts
+      end
+
+      # Execute block with exponential backoff retry
+      def with_retry(&block)
+        attempts = 0
+
+        begin
+          attempts += 1
+          yield
+        rescue HttpSender::ServerError, HttpSender::UnexpectedError, StandardError => e
+          if attempts < @max_attempts
+            sleep_time = exponential_backoff(attempts)
+            log_retry(attempts, sleep_time, e)
+            sleep(sleep_time)
+            retry
+          else
+            log_error "Max retry attempts (#{@max_attempts}) exceeded: #{e.message}"
+            raise
+          end
+        rescue HttpSender::ClientError => e
+          # Don't retry client errors (4xx)
+          log_error "Client error, not retrying: #{e.message}"
+          raise
+        end
+      end
+
+      private
+
+      def exponential_backoff(attempt)
+        # 1s, 2s, 4s, 8s, etc.
+        [2 ** (attempt - 1), 30].min # Cap at 30 seconds
+      end
+
+      def log_retry(attempt, sleep_time, error)
+        Rails.logger.warn "SolidLog::Client: Retry attempt #{attempt}/#{@max_attempts} after #{sleep_time}s (#{error.class}: #{error.message})" if defined?(Rails)
+      end
+
+      def log_error(message)
+        Rails.logger.error "SolidLog::Client: #{message}" if defined?(Rails)
+      end
+    end
+  end
+end

data/lib/solid_log/core/client.rb

@@ -0,0 +1,138 @@
+require_relative "client/configuration"
+require_relative "client/buffer"
+require_relative "client/http"
+require_relative "client/retry_handler"
+require_relative "client/lograge_formatter"
+
+module SolidLog
+  module Core
+    class Client
+      class << self
+        attr_writer :configuration
+
+        def configuration
+          @configuration ||= ClientConfiguration.new
+        end
+
+        def configure
+          yield(configuration)
+          initialize_client
+        end
+
+        # Get logger instance for Lograge integration
+        def logger
+          @logger ||= BufferedLogger.new
+        end
+
+        # Log a single entry
+        def log(entry)
+          return unless configuration.enabled
+
+          buffer.add(entry)
+        end
+
+        # Flush pending logs immediately
+        def flush
+          buffer.flush
+        end
+
+        # Start background flushing (if not already started)
+        def start
+          buffer.start_auto_flush
+        end
+
+        # Stop background flushing and flush pending logs
+        def stop
+          buffer.stop_auto_flush
+          flush
+        end
+
+        private
+
+        def buffer
+          @buffer ||= Buffer.new(
+            http_sender: http_sender,
+            batch_size: configuration.batch_size,
+            flush_interval: configuration.flush_interval,
+            max_queue_size: configuration.max_queue_size
+          )
+        end
+
+        def http_sender
+          @http_sender ||= HttpSender.new(
+            url: configuration.service_url,
+            token: configuration.token,
+            retry_handler: retry_handler
+          )
+        end
+
+        def retry_handler
+          @retry_handler ||= RetryHandler.new(
+            max_attempts: configuration.retry_max_attempts
+          )
+        end
+
+        def initialize_client
+          # Reset instances when configuration changes
+          @buffer = nil
+          @http_sender = nil
+          @retry_handler = nil
+          @logger = nil
+        end
+      end
+
+      # BufferedLogger for Lograge integration
+      class BufferedLogger
+        def initialize
+          @client = Client
+        end
+
+        def info(message)
+          log_message(message, level: "info")
+        end
+
+        def debug(message)
+          log_message(message, level: "debug")
+        end
+
+        def warn(message)
+          log_message(message, level: "warn")
+        end
+
+        def error(message)
+          log_message(message, level: "error")
+        end
+
+        def fatal(message)
+          log_message(message, level: "fatal")
+        end
+
+        def <<(message)
+          log_message(message, level: "info")
+        end
+
+        private
+
+        def log_message(message, level:)
+          # Parse JSON if message is a string
+          entry = if message.is_a?(String)
+                   begin
+                     JSON.parse(message).merge(level: level)
+                   rescue JSON::ParserError
+                     { message: message, level: level }
+                   end
+          else
+                   message.merge(level: level)
+          end
+
+          # Add default fields
+          entry[:timestamp] ||= Time.current.iso8601
+          entry[:app] ||= Client.configuration.app_name
+          entry[:env] ||= Client.configuration.environment
+
+          @client.log(entry)
+        end
+      end
+    end
+  end
+end