solid_log-core 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '09b479494d3ac70c18b6d0611a5ee0879ff01b90c17108dcffe1eaa951a47f00'
-  data.tar.gz: ca5f502ca1f8c7b722f1c26b31c898bf3c0e04b512fbde82d0b1e1e888006e32
+  metadata.gz: 29584710fa24106c9fd34cf056db068a4cc728790d891b036f58efca98f3f622
+  data.tar.gz: ef9627f82cf2eda9f91be114bf583440a58b76e3722e4619ac6caff143ca128b
 SHA512:
-  metadata.gz: 2da7bc4a35999dc410ef3951325e1c61c7e67ef4a7f5028d386bb397e676edc50528e2b46209a4999dc94f95cf6ff6b7025db7d7010683e2dd73b207dcac9333
-  data.tar.gz: dff9a33b078a75d0019bfba2d9371b9d4804ce6bcfb52ed6be75bc2ce9a3892ba57d679885fd64ac56c4904945d101078eedfbbb709114d662b0978b57218431
+  metadata.gz: 3078d64ae373365594bffebd66914e29569b76142a0a92dc52ed73592507def8a6abdee08505afc55643cc7c74e9732dc798419f95d89cad2fa2d7f87d36fe1c
+  data.tar.gz: 2a7e1c672513f17cb133a737624d7b9c48b166aa507d38286b33a5e824a7697ae0d98e4e3d85e296d539986b42fce5bc6c4043b04f1a5d88a3ba90a8501c14a3

data/lib/puma/plugin/solid_log.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require "puma/plugin"
+
+# Puma plugin for inline SolidLog parsing
+#
+# This plugin spawns a background thread that continuously polls for unparsed
+# raw log entries and processes them using BatchParsingService.
+#
+# Usage in config/puma.rb:
+#   plugin :solid_log if ENV["SOLIDLOG_PUMA_PLUGIN_ENABLED"]
+#
+# Configuration:
+#   SOLIDLOG_PUMA_PLUGIN_ENABLED=true
+#   SOLIDLOG_INLINE_PARSING_ENABLED=true
+#   SOLIDLOG_PARSE_INTERVAL=10
+#   SOLIDLOG_PARSER_BATCH_SIZE=200
+Puma::Plugin.create do
+  def start(launcher)
+    @launcher = launcher
+
+    # Only start if we're actually in Puma server mode
+    # Don't start in console, rails runner, rake tasks, etc.
+    return unless puma_server_mode?
+
+    # Only start if explicitly enabled
+    return unless enabled?
+
+    # Startup logs to STDERR to avoid recursion
+    $stderr.puts "[SolidLog Puma Plugin] Starting inline parsing"
+    $stderr.puts "[SolidLog Puma Plugin]   Parse interval: #{config.parse_interval}s"
+    $stderr.puts "[SolidLog Puma Plugin]   Batch size: #{config.parser_batch_size}"
+
+    # Start background thread using Puma's in_background helper
+    in_background do
+      parser_loop
+    end
+  end
+
+  private
+
+  # Check if we're actually running as a Puma server
+  # Returns false for console, rails runner, rake tasks, etc.
+  def puma_server_mode?
+    # Check if launcher is present and valid
+    return false unless @launcher
+    return false unless @launcher.respond_to?(:binder)
+
+    # Check if we're not in a Rails console or runner
+    # Console and runner set DISABLE_SPRING or have IRB/Runner in ARGV
+    return false if defined?(Rails::Console)
+    return false if $PROGRAM_NAME.include?('rake')
+    return false if $PROGRAM_NAME.include?('runner')
+    return false if ARGV.include?('console')
+    return false if ARGV.include?('runner')
+    return false if ARGV.include?('c')
+
+    # If we made it here, we're likely in server mode
+    true
+  end
+
+  # Main parser loop that runs continuously until Puma shutdown
+  def parser_loop
+    loop do
+      # Thread will be killed by Puma on shutdown, no need to check running state
+
+      begin
+        # Ensure database connection in this thread
+        # Pattern from DirectLogger (lines 155-158)
+        ActiveRecord::Base.connection_pool.with_connection do
+          # Process batch using shared service
+          stats = SolidLog::Core::Services::BatchParsingService.process_batch(
+            batch_size: config.parser_batch_size,
+            logger: logger,
+            broadcast_callback: method(:broadcast_callback)
+          )
+
+          # Log stats if entries were processed (to STDERR to avoid recursion)
+          log_stats(stats) if stats[:processed] > 0
+        end
+
+        # Normal sleep between cycles
+        sleep config.parse_interval
+      rescue => e
+        # Log error to STDERR to avoid recursion
+        $stderr.puts "[SolidLog Puma Plugin] ERROR: #{e.class} - #{e.message}"
+        $stderr.puts e.backtrace.first(5).join("\n") if e.backtrace
+
+        # Exponential backoff on error (max 60 seconds)
+        # Prevents tight error loops
+        backoff_time = [config.parse_interval * 2, 60].min
+        $stderr.puts "[SolidLog Puma Plugin] Backing off for #{backoff_time}s" if ENV['SOLIDLOG_DEBUG']
+        sleep backoff_time
+      end
+    end
+
+  rescue => e
+    # Thread-level error - log to STDERR but don't crash Puma
+    $stderr.puts "[SolidLog Puma Plugin] FATAL: #{e.class} - #{e.message}"
+    $stderr.puts e.backtrace.join("\n") if e.backtrace
+  ensure
+    $stderr.puts "[SolidLog Puma Plugin] Parser loop exited"
+  end
+
+  # Check if plugin should be enabled
+  def enabled?
+    # Check ENV flag (explicit opt-in)
+    return false if ENV["SOLIDLOG_PUMA_PLUGIN_ENABLED"] == "false"
+    return false unless ENV["SOLIDLOG_PUMA_PLUGIN_ENABLED"]
+
+    # Check configuration flag
+    return false unless config.inline_parsing_enabled
+
+    true
+  rescue => e
+    # If SolidLog isn't loaded yet, disable
+    logger.error "SolidLog Puma Plugin: Configuration error: #{e.message}"
+    false
+  end
+
+  # Get SolidLog configuration
+  def config
+    SolidLog.configuration
+  end
+
+  # Get logger instance
+  def logger
+    SolidLog.logger
+  end
+
+  # Broadcast callback for ActionCable
+  def broadcast_callback(entry_ids)
+    return if entry_ids.empty?
+
+    if defined?(ActionCable) && ActionCable.server
+      ActionCable.server.broadcast(
+        "solid_log_new_entries",
+        { entry_ids: entry_ids }
+      )
+      $stderr.puts "[SolidLog Puma Plugin] Broadcasted #{entry_ids.size} entries" if ENV['SOLIDLOG_DEBUG']
+    end
+  rescue => e
+    # Silent failure - broadcasting is optional
+    $stderr.puts "[SolidLog Puma Plugin] Broadcast failed: #{e.message}" if ENV['SOLIDLOG_DEBUG']
+  end
+
+  # Log processing stats (directly to STDERR to avoid recursion)
+  def log_stats(stats)
+    $stderr.puts "[SolidLog Puma Plugin] Processed #{stats[:processed]}, inserted #{stats[:inserted]}, errors #{stats[:errors]}" if ENV['SOLIDLOG_DEBUG']
+  end
+end

data/lib/solid_log/core/configuration.rb

@@ -4,26 +4,36 @@ module SolidLog
       attr_accessor :database_url,
                     :retention_days,
                     :error_retention_days,
+                    :max_entries,
                     :max_batch_size,
                     :parser_batch_size,
                     :parser_concurrency,
                     :auto_promote_fields,
                     :field_promotion_threshold,
                     :facet_cache_ttl,
-                    :live_tail_mode
+                    :live_tail_mode,
+                    :inline_parsing_enabled,
+                    :parse_interval,
+                    :after_entries_inserted
 
       def initialize
         # Load from ENV vars with defaults
         @database_url = ENV["SOLIDLOG_DATABASE_URL"] || ENV["DATABASE_URL"]
         @retention_days = env_to_int("SOLIDLOG_RETENTION_DAYS", 30)
         @error_retention_days = env_to_int("SOLIDLOG_ERROR_RETENTION_DAYS", 90)
+        @max_entries = env_to_int("SOLIDLOG_MAX_ENTRIES", nil) # Optional row limit (nil = unlimited)
         @max_batch_size = env_to_int("SOLIDLOG_MAX_BATCH_SIZE", 1000) # For API ingestion
-        @parser_batch_size = env_to_int("SOLIDLOG_PARSER_BATCH_SIZE", 200) # Number of raw entries to parse per job
+        @parser_batch_size = env_to_int("SOLIDLOG_PARSER_BATCH_SIZE", 1000) # Number of raw entries to parse per job
         @parser_concurrency = env_to_int("SOLIDLOG_PARSER_CONCURRENCY", 5)
         @auto_promote_fields = env_to_bool("SOLIDLOG_AUTO_PROMOTE_FIELDS", false)
         @field_promotion_threshold = env_to_int("SOLIDLOG_FIELD_PROMOTION_THRESHOLD", 1000)
         @facet_cache_ttl = env_to_int("SOLIDLOG_FACET_CACHE_TTL", 300) # seconds (5 minutes)
         @live_tail_mode = env_to_symbol("SOLIDLOG_LIVE_TAIL_MODE", :disabled) # :websocket, :polling, or :disabled
+
+        # Inline parsing configuration
+        @inline_parsing_enabled = env_to_bool("SOLIDLOG_INLINE_PARSING_ENABLED", false)
+        @parse_interval = env_to_int("SOLIDLOG_PARSE_INTERVAL", 10) # seconds
+        @after_entries_inserted = nil # Optional callback for custom broadcasting
       end
 
       # Check if database is configured

data/lib/solid_log/core/jobs/cache_cleanup_job.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module SolidLog
+  module Core
+    module Jobs
+      # Conditionally inherit from ActiveJob if available
+      if defined?(ActiveJob::Base)
+        BaseJob = ActiveJob::Base
+      else
+        # Plain Ruby class when ActiveJob not available
+        BaseJob = Class.new do
+          def self.queue_as(*); end  # No-op when not using ActiveJob
+          def self.perform_later(*args, **kwargs)
+            new.perform(*args, **kwargs)
+          end
+          def self.perform_now(*args, **kwargs)
+            new.perform(*args, **kwargs)
+          end
+        end
+      end
+
+      class CacheCleanupJob < BaseJob
+        queue_as :default
+
+        def perform
+          SolidLog.without_logging do
+            expired_count = SolidLog::FacetCache.expired.count
+
+            if expired_count > 0
+              SolidLog::FacetCache.cleanup_expired!
+              logger.info "SolidLog::CacheCleanupJob: Cleaned up #{expired_count} expired cache entries"
+            end
+          end
+        rescue => e
+          logger.error "SolidLog::CacheCleanupJob failed: #{e.message}"
+          # Re-raise for ActiveJob retry if available, otherwise just log
+          raise if defined?(ActiveJob::Base)
+        end
+
+        private
+
+        def logger
+          defined?(SolidLog) && SolidLog.respond_to?(:logger) ? SolidLog.logger : Logger.new(STDOUT)
+        end
+      end
+    end
+  end
+end

data/lib/solid_log/core/jobs/field_analysis_job.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module SolidLog
+  module Core
+    module Jobs
+      # Conditionally inherit from ActiveJob if available
+      if defined?(ActiveJob::Base)
+        BaseJob = ActiveJob::Base
+      else
+        # Plain Ruby class when ActiveJob not available
+        BaseJob = Class.new do
+          def self.queue_as(*); end  # No-op when not using ActiveJob
+          def self.perform_later(*args, **kwargs)
+            new.perform(*args, **kwargs)
+          end
+          def self.perform_now(*args, **kwargs)
+            new.perform(*args, **kwargs)
+          end
+        end
+      end
+
+      class FieldAnalysisJob < BaseJob
+        queue_as :default
+
+        def perform(auto_promote: false)
+          SolidLog.without_logging do
+            recommendations = SolidLog::Core::FieldAnalyzer.analyze
+
+            if recommendations.any?
+              logger.info "SolidLog::FieldAnalysisJob: Found #{recommendations.size} fields for potential promotion"
+
+              recommendations.take(10).each do |rec|
+                logger.info "  - #{rec[:field].name} (#{rec[:field].usage_count} uses, priority: #{rec[:priority]})"
+              end
+
+              if auto_promote
+                promoted_count = SolidLog::Core::FieldAnalyzer.auto_promote_candidates
+                logger.info "SolidLog::FieldAnalysisJob: Auto-promoted #{promoted_count} fields"
+              end
+            else
+              logger.info "SolidLog::FieldAnalysisJob: No fields meet promotion threshold"
+            end
+          end
+        rescue => e
+          logger.error "SolidLog::FieldAnalysisJob failed: #{e.message}"
+          # Re-raise for ActiveJob retry if available, otherwise just log
+          raise if defined?(ActiveJob::Base)
+        end
+
+        private
+
+        def logger
+          defined?(SolidLog) && SolidLog.respond_to?(:logger) ? SolidLog.logger : Logger.new(STDOUT)
+        end
+      end
+    end
+  end
+end

data/lib/solid_log/core/jobs/parse_job.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module SolidLog
+  module Core
+    module Jobs
+      # Conditionally inherit from ActiveJob if available
+      # This allows the same job class to work in Rails apps AND solid_log-service
+      if defined?(ActiveJob::Base)
+        BaseJob = ActiveJob::Base
+      else
+        # Plain Ruby class when ActiveJob not available
+        BaseJob = Class.new do
+          def self.queue_as(*_args)
+            # No-op when not using ActiveJob
+          end
+
+          def self.perform_later(*args, **kwargs)
+            # Synchronous execution when ActiveJob not available
+            new.perform(*args, **kwargs)
+          end
+
+          def self.perform_now(*args, **kwargs)
+            new.perform(*args, **kwargs)
+          end
+        end
+      end
+
+      # ParseJob processes batches of unparsed raw log entries.
+      #
+      # This job can be used in three contexts:
+      # 1. Rails app with Solid Queue (scheduled in config/recurring.yml)
+      # 2. Rails app with other ActiveJob backends (Sidekiq, Resque, etc.)
+      # 3. solid_log-service without Rails (called directly)
+      #
+      # @example Solid Queue (config/recurring.yml)
+      #   solidlog_parse:
+      #     class: SolidLog::Core::Jobs::ParseJob
+      #     schedule: every 10 seconds
+      #     args: [{ batch_size: 200 }]
+      #
+      # @example Direct call (service)
+      #   SolidLog::Core::Jobs::ParseJob.perform_now(batch_size: 200)
+      class ParseJob < BaseJob
+        queue_as :default
+
+        # Process a batch of unparsed raw entries
+        #
+        # @param batch_size [Integer] Number of entries to process (default: config value)
+        # @return [Hash] Statistics hash with :processed, :inserted, :errors keys
+        def perform(batch_size: nil)
+          batch_size ||= SolidLog.configuration.parser_batch_size
+
+          # Delegate to BatchParsingService
+          stats = SolidLog::Core::Services::BatchParsingService.process_batch(
+            batch_size: batch_size,
+            logger: logger,
+            broadcast_callback: method(:broadcast_callback)
+          )
+
+          # Log results if entries were processed
+          if stats[:processed] > 0
+            logger.info "SolidLog::ParseJob: Processed #{stats[:processed]}, inserted #{stats[:inserted]}, errors #{stats[:errors]}"
+          end
+
+          stats
+        rescue => e
+          logger.error "SolidLog::ParseJob failed: #{e.class} - #{e.message}"
+          logger.error e.backtrace.first(10).join("\n") if e.backtrace
+
+          # Re-raise for ActiveJob retry if available, otherwise just log
+          raise if defined?(ActiveJob::Base) && self.class.ancestors.include?(ActiveJob::Base)
+
+          stats || { processed: 0, inserted: 0, errors: 1 }
+        end
+
+        private
+
+        # Get logger instance
+        def logger
+          if defined?(SolidLog) && SolidLog.respond_to?(:logger)
+            SolidLog.logger
+          else
+            Logger.new($stdout).tap { |log| log.level = Logger::INFO }
+          end
+        end
+
+        # Broadcast callback for ActionCable
+        def broadcast_callback(entry_ids)
+          return if entry_ids.empty?
+
+          if defined?(ActionCable) && ActionCable.server
+            ActionCable.server.broadcast(
+              "solid_log_new_entries",
+              { entry_ids: entry_ids }
+            )
+            logger.debug "SolidLog::ParseJob: Broadcasted #{entry_ids.size} entries via ActionCable"
+          end
+        rescue => e
+          # Silent failure - broadcasting is optional
+          logger.debug "SolidLog::ParseJob: Broadcast failed: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/solid_log/core/jobs/retention_job.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module SolidLog
+  module Core
+    module Jobs
+      # Conditionally inherit from ActiveJob if available
+      if defined?(ActiveJob::Base)
+        BaseJob = ActiveJob::Base
+      else
+        # Plain Ruby class when ActiveJob not available
+        BaseJob = Class.new do
+          def self.queue_as(*); end  # No-op when not using ActiveJob
+          def self.perform_later(*args, **kwargs)
+            new.perform(*args, **kwargs)
+          end
+          def self.perform_now(*args, **kwargs)
+            new.perform(*args, **kwargs)
+          end
+        end
+      end
+
+      class RetentionJob < BaseJob
+        queue_as :default
+
+        def perform(retention_days: nil, error_retention_days: nil, max_entries: nil, vacuum: false)
+          retention_days ||= SolidLog.configuration.retention_days
+          error_retention_days ||= SolidLog.configuration.error_retention_days
+          max_entries ||= SolidLog.configuration.max_entries
+
+          SolidLog.without_logging do
+            logger.info "SolidLog::RetentionJob: Starting cleanup (retention: #{retention_days} days, errors: #{error_retention_days} days, max_entries: #{max_entries || 'unlimited'})"
+
+            stats = SolidLog::Core::RetentionService.cleanup(
+              retention_days: retention_days,
+              error_retention_days: error_retention_days,
+              max_entries: max_entries
+            )
+
+            logger.info "SolidLog::RetentionJob: Deleted #{stats[:entries_deleted]} entries, #{stats[:raw_deleted]} raw entries, cleared #{stats[:cache_cleared]} cache entries"
+
+            if vacuum
+              logger.info "SolidLog::RetentionJob: Running VACUUM..."
+              SolidLog::Core::RetentionService.vacuum_database
+              logger.info "SolidLog::RetentionJob: VACUUM complete"
+            end
+          end
+
+          stats
+        rescue => e
+          logger.error "SolidLog::RetentionJob failed: #{e.message}"
+          # Re-raise for ActiveJob retry if available, otherwise just log
+          raise if defined?(ActiveJob::Base)
+        end
+
+        private
+
+        def logger
+          defined?(SolidLog) && SolidLog.respond_to?(:logger) ? SolidLog.logger : Logger.new(STDOUT)
+        end
+      end
+    end
+  end
+end