solid_log-core 0.1.0 → 0.2.1

data/lib/solid_log/core/services/batch_parsing_service.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+module SolidLog
+  module Core
+    module Services
+      # BatchParsingService processes batches of unparsed raw log entries.
+      #
+      # This service is the single source of truth for parsing logic, used by:
+      # - Puma plugin (inline processing)
+      # - ParseJob (Solid Queue / ActiveJob)
+      # - solid_log-service (dedicated service process)
+      #
+      # @example Basic usage
+      #   stats = BatchParsingService.process_batch(batch_size: 200)
+      #   # => { processed: 150, inserted: 145, errors: 5 }
+      #
+      # @example With custom callback
+      #   callback = ->(entry_ids) { puts "Processed #{entry_ids.size} entries" }
+      #   BatchParsingService.process_batch(broadcast_callback: callback)
+      class BatchParsingService
+        # Process a batch of unparsed raw entries
+        #
+        # @param batch_size [Integer] Number of entries to process (default: config value)
+        # @param logger [Logger] Logger instance (default: SolidLog.logger)
+        # @param broadcast_callback [Proc] Callback for broadcasting new entries
+        # @return [Hash] Statistics hash with :processed, :inserted, :errors keys
+        def self.process_batch(batch_size: nil, logger: nil, broadcast_callback: nil)
+          batch_size ||= SolidLog.configuration.parser_batch_size
+          logger ||= SolidLog.logger
+
+          stats = { processed: 0, inserted: 0, errors: 0 }
+
+          SolidLog.without_logging do
+            # Claim a batch of unparsed entries
+            raw_entries = RawEntry.claim_batch(batch_size: batch_size)
+
+            return stats if raw_entries.empty?
+
+            # Log to STDERR to avoid recursion (only in debug mode)
+            $stderr.puts "[SolidLog::BatchParsingService] Processing #{raw_entries.size} raw entries" if ENV["SOLIDLOG_DEBUG"]
+
+            # Get promoted fields cache (for performance)
+            promoted_fields = get_promoted_fields_cache
+
+            # Process each entry
+            entries_to_insert = []
+            fields_to_track = {}
+
+            raw_entries.each do |raw_entry|
+              begin
+                # Parse the raw payload
+                parsed = Parser.parse(raw_entry.payload)
+
+                # Extract dynamic fields for field registry
+                extra_fields = parsed.delete(:extra_fields) || {}
+                track_fields(fields_to_track, extra_fields)
+
+                # Prepare entry for insertion
+                entry_data = {
+                  raw_id: raw_entry.id,
+                  timestamp: parsed[:timestamp],
+                  created_at: Time.current, # When entry was parsed/created
+                  level: parsed[:level],
+                  app: parsed[:app],
+                  env: parsed[:env],
+                  message: parsed[:message],
+                  request_id: parsed[:request_id],
+                  job_id: parsed[:job_id],
+                  duration: parsed[:duration],
+                  status_code: parsed[:status_code],
+                  controller: parsed[:controller],
+                  action: parsed[:action],
+                  path: parsed[:path],
+                  method: parsed[:method],
+                  extra_fields: extra_fields.to_json
+                }
+
+                # Populate promoted field columns
+                populate_promoted_fields(entry_data, extra_fields, promoted_fields)
+
+                entries_to_insert << entry_data
+                stats[:processed] += 1
+              rescue StandardError => e
+                logger.error "SolidLog::BatchParsingService: Failed to parse entry #{raw_entry.id}: #{e.message}"
+                logger.error e.backtrace.first(5).join("\n") if e.backtrace
+                stats[:errors] += 1
+                # Entry remains unparsed so it can be retried or investigated
+              end
+            end
+
+            # Bulk insert parsed entries
+            if entries_to_insert.any?
+              raw_ids = entries_to_insert.map { |e| e[:raw_id] }
+
+              Entry.insert_all(entries_to_insert)
+              stats[:inserted] = entries_to_insert.size
+              # Log to STDERR to avoid recursion (only in debug mode)
+              $stderr.puts "[SolidLog::BatchParsingService] Inserted #{entries_to_insert.size} entries" if ENV["SOLIDLOG_DEBUG"]
+
+              # Broadcast new entry IDs for live tail
+              new_entry_ids = Entry.where(raw_id: raw_ids).pluck(:id)
+              broadcast_entries(new_entry_ids, broadcast_callback, logger)
+            end
+
+            # Update field registry
+            update_field_registry(fields_to_track, logger)
+          end
+
+          stats
+        end
+
+        private
+
+        # Track field occurrences for the registry
+        def self.track_fields(fields_hash, extra_fields)
+          extra_fields.each do |key, value|
+            fields_hash[key] ||= { values: [], count: 0 }
+            fields_hash[key][:count] += 1
+            fields_hash[key][:type] ||= infer_field_type(value)
+          end
+        end
+
+        # Update the field registry with tracked fields
+        def self.update_field_registry(fields_hash, logger)
+          fields_hash.each do |name, data|
+            field = Field.find_or_initialize_by(name: name)
+            field.field_type ||= data[:type]
+            field.usage_count += data[:count]
+            field.last_seen_at = Time.current
+            field.save!
+          end
+        rescue StandardError => e
+          logger.error "SolidLog::BatchParsingService: Failed to update field registry: #{e.message}"
+        end
+
+        # Infer field type from value
+        def self.infer_field_type(value)
+          case value
+          when String
+            "string"
+          when Numeric
+            "number"
+          when TrueClass, FalseClass
+            "boolean"
+          when Time, DateTime, Date
+            "datetime"
+          when Array
+            "array"
+          when Hash
+            "object"
+          else
+            "string"
+          end
+        end
+
+        # Get cache of promoted fields and their types
+        # Returns hash: { "field_name" => "field_type", ... }
+        def self.get_promoted_fields_cache
+          @promoted_fields_cache ||= {}
+          @promoted_fields_cache_time ||= Time.at(0)
+
+          # Refresh cache every 60 seconds
+          if Time.current - @promoted_fields_cache_time > 60
+            promoted = Field.promoted.pluck(:name, :field_type).to_h
+            available_columns = Entry.column_names
+
+            # Only include fields that have actual columns in the table
+            @promoted_fields_cache = promoted.select { |name, _type| available_columns.include?(name) }
+            @promoted_fields_cache_time = Time.current
+          end
+
+          @promoted_fields_cache
+        rescue StandardError
+          # If Field table doesn't exist yet or there's an error, return empty hash
+          {}
+        end
+
+        # Populate promoted field columns in entry_data
+        # Extracts values from extra_fields and adds them to entry_data
+        # Note: values remain in extra_fields JSON for backward compatibility
+        def self.populate_promoted_fields(entry_data, extra_fields, promoted_fields)
+          promoted_fields.each do |field_name, field_type|
+            next unless extra_fields.key?(field_name)
+
+            value = extra_fields[field_name]
+            next if value.nil?
+
+            # Type coercion for promoted field
+            begin
+              typed_value = coerce_field_value(value, field_type)
+              entry_data[field_name.to_sym] = typed_value
+            rescue StandardError
+              # If type coercion fails, skip this field
+              # Field remains in extra_fields JSON
+            end
+          end
+        end
+
+        # Coerce a value to the specified field type
+        def self.coerce_field_value(value, field_type)
+          case field_type
+          when "number"
+            value.is_a?(Numeric) ? value : value.to_f
+          when "boolean"
+            [true, "true", "1", 1].include?(value)
+          when "datetime"
+            value.is_a?(Time) ? value : Time.parse(value.to_s)
+          when "string"
+            value.to_s
+          when "array", "object"
+            # Arrays and objects stored as JSON text
+            value.is_a?(String) ? value : value.to_json
+          else
+            value
+          end
+        end
+
+        # Broadcast new entries via callback or ActionCable
+        def self.broadcast_entries(entry_ids, callback, logger)
+          return if entry_ids.empty?
+
+          # Try user-configured callback first
+          if callback&.respond_to?(:call)
+            callback.call(entry_ids)
+            return
+          end
+
+          # Try configuration callback
+          if SolidLog.configuration.after_entries_inserted&.respond_to?(:call)
+            SolidLog.configuration.after_entries_inserted.call(entry_ids)
+            return
+          end
+
+          # Fallback to ActionCable if available
+          if defined?(ActionCable) && ActionCable.server
+            ActionCable.server.broadcast(
+              "solid_log_new_entries",
+              { entry_ids: entry_ids }
+            )
+          end
+        rescue StandardError => e
+          # Silent failure - broadcasting is optional (log to STDERR to avoid recursion)
+          $stderr.puts "[SolidLog::BatchParsingService] Broadcast failed: #{e.message}" if ENV["SOLIDLOG_DEBUG"]
+        end
+      end
+    end
+  end
+end

data/lib/solid_log/core/services/field_analyzer.rb

@@ -36,7 +36,7 @@ module SolidLog
 
           # Only auto-promote fields with high priority
           if candidate[:priority] >= 80
-            Rails.logger.info "FieldAnalyzer: Auto-promoting field '#{field.name}' (usage: #{field.usage_count}, priority: #{candidate[:priority]})"
+            SolidLog.logger.info "FieldAnalyzer: Auto-promoting field '#{field.name}' (usage: #{field.usage_count}, priority: #{candidate[:priority]})"
 
             field.promote!
             promoted_count += 1

data/lib/solid_log/core/services/migration_generator.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module SolidLog
+  module Core
+    # Generates ActiveRecord migrations for field promotions
+    # When a field is promoted, this creates a timestamped migration file that:
+    # 1. Adds a new column to solid_log_entries
+    # 2. Creates an index on the new column
+    # 3. Backfills existing data from the extra_fields JSON
+    class MigrationGenerator
+    class << self
+      # Generate a migration file for promoting a field
+      # @param field [Field] The field to promote
+      # @return [String, nil] Path to generated migration file, or nil if error
+      def generate_promotion_migration(field)
+        return nil unless field.is_a?(Field)
+
+        timestamp = Time.current.utc.strftime("%Y%m%d%H%M%S")
+        class_name = "AddPromoted#{field.name.camelize}ToEntries"
+        filename = "#{timestamp}_#{class_name.underscore}.rb"
+
+        # Determine where to write the migration
+        migration_path = determine_migration_path
+        FileUtils.mkdir_p(migration_path) unless Dir.exist?(migration_path)
+
+        file_path = File.join(migration_path, filename)
+
+        # Generate migration content
+        content = generate_migration_content(field, class_name)
+
+        # Write file
+        File.write(file_path, content)
+
+        SolidLog.logger&.info("Generated promotion migration: #{file_path}")
+        file_path
+      rescue => e
+        SolidLog.logger&.error("Failed to generate migration for field #{field.name}: #{e.message}")
+        nil
+      end
+
+      private
+
+      # Determine where to write the migration file
+      # Tries multiple locations in order of preference
+      def determine_migration_path
+        candidates = []
+
+        # Rails app log_migrate directory
+        candidates << Rails.root.join("db", "log_migrate") if defined?(Rails)
+
+        # Standalone service directory
+        candidates << File.expand_path("../../../../../solid_log-service/db/log_migrate", __FILE__)
+
+        # Core gem directory (fallback)
+        candidates << File.expand_path("../../../../db/log_migrate", __FILE__)
+
+        # Use first existing directory, or first candidate if none exist
+        candidates.find { |path| Dir.exist?(path) } || candidates.first
+      end
+
+      # Generate the Ruby migration code
+      def generate_migration_content(field, class_name)
+        <<~RUBY
+          # frozen_string_literal: true
+
+          # Migration auto-generated by SolidLog field promotion
+          # Promotes field '#{field.name}' from JSON to dedicated column
+          class #{class_name} < ActiveRecord::Migration[8.0]
+            def up
+              # Add column for promoted field
+              add_column :solid_log_entries, :#{field.name}, :#{sql_type(field.field_type)}
+
+              # Add index for efficient querying
+              add_index :solid_log_entries, :#{field.name}, name: "idx_entries_#{field.name}"
+
+              # Backfill existing data from extra_fields JSON
+              # Process in batches to avoid memory issues
+              say_with_time "Backfilling #{field.name} from extra_fields JSON" do
+                batch_size = 1000
+                offset = 0
+
+                loop do
+                  entries = SolidLog::Entry
+                    .where.not(extra_fields: nil)
+                    .limit(batch_size)
+                    .offset(offset)
+
+                  break if entries.empty?
+
+                  entries.each do |entry|
+                    begin
+                      extra = JSON.parse(entry.extra_fields)
+                      value = extra['#{field.name}']
+
+                      if value.present?
+                        # Type coercion
+                        typed_value = coerce_value(value, '#{field.field_type}')
+                        entry.update_column(:#{field.name}, typed_value)
+                      end
+                    rescue JSON::ParserError, TypeError => e
+                      # Skip entries with invalid JSON or type conversion errors
+                      next
+                    end
+                  end
+
+                  offset += batch_size
+                end
+              end
+            end
+
+            def down
+              remove_index :solid_log_entries, name: "idx_entries_#{field.name}"
+              remove_column :solid_log_entries, :#{field.name}
+            end
+
+            private
+
+            def coerce_value(value, field_type)
+              case field_type
+              when 'number'
+                value.is_a?(Numeric) ? value : value.to_f
+              when 'boolean'
+                [true, 'true', '1', 1].include?(value)
+              when 'datetime'
+                value.is_a?(Time) ? value : Time.parse(value.to_s)
+              when 'string'
+                value.to_s
+              else
+                value
+              end
+            rescue => e
+              nil # Return nil for unconvertible values
+            end
+          end
+        RUBY
+      end
+
+      # Map field types to SQL column types
+      def sql_type(field_type)
+        case field_type
+        when "number"
+          :float
+        when "boolean"
+          :boolean
+        when "datetime"
+          :datetime
+        when "string"
+          :string
+        when "array"
+          :text  # Store as JSON text
+        when "object"
+          :text  # Store as JSON text
+        else
+          :string  # Default to string
+        end
+      end
+    end
+    end
+  end
+end

data/lib/solid_log/core/services/migration_runner.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module SolidLog
+  module Core
+    # Runs pending ActiveRecord migrations for the log database
+    # Used by standalone service to auto-migrate on startup
+    class MigrationRunner
+    class << self
+      # Run all pending migrations
+      # @param migration_paths [Array<String>] Paths to search for migrations (optional)
+      # @return [Boolean] true if migrations ran successfully, false otherwise
+      def run_pending_migrations(migration_paths: nil)
+        paths = migration_paths || default_migration_paths
+        paths = Array(paths).select { |p| Dir.exist?(p) }
+
+        if paths.empty?
+          SolidLog.logger&.warn("No migration directories found, skipping migrations")
+          return true
+        end
+
+        SolidLog.logger&.info("Checking for pending migrations in: #{paths.join(', ')}")
+
+        begin
+          # Create schema_migrations table if it doesn't exist
+          ensure_schema_migrations_table!
+
+          # Get currently applied migrations
+          applied_versions = get_applied_versions
+
+          # Find all migration files
+          migration_files = find_migration_files(paths)
+
+          # Filter to pending migrations
+          pending = migration_files.reject { |file| applied_versions.include?(version_from_file(file)) }
+
+          if pending.empty?
+            SolidLog.logger&.info("No pending migrations found")
+            return true
+          end
+
+          SolidLog.logger&.info("Found #{pending.size} pending migration(s)")
+
+          # Run each pending migration
+          pending.sort.each do |migration_file|
+            run_migration_file(migration_file)
+          end
+
+          SolidLog.logger&.info("Successfully ran #{pending.size} migration(s)")
+          true
+        rescue => e
+          SolidLog.logger&.error("Migration failed: #{e.message}")
+          SolidLog.logger&.error(e.backtrace.join("\n"))
+          false
+        end
+      end
+
+      # Check if there are pending migrations
+      # @param migration_paths [Array<String>] Paths to search for migrations (optional)
+      # @return [Boolean] true if pending migrations exist
+      def pending_migrations?(migration_paths: nil)
+        paths = migration_paths || default_migration_paths
+        paths = Array(paths).select { |p| Dir.exist?(p) }
+
+        return false if paths.empty?
+
+        applied_versions = get_applied_versions
+        migration_files = find_migration_files(paths)
+
+        migration_files.any? { |file| !applied_versions.include?(version_from_file(file)) }
+      end
+
+      private
+
+      # Default paths to search for migrations
+      def default_migration_paths
+        paths = []
+
+        # Rails app log_migrate directory
+        paths << Rails.root.join("db", "log_migrate") if defined?(Rails)
+
+        # Service directory
+        paths << File.expand_path("../../../../../solid_log-service/db/log_migrate", __FILE__)
+
+        # Core gem directory
+        paths << File.expand_path("../../../../db/log_migrate", __FILE__)
+
+        paths
+      end
+
+      # Ensure schema_migrations table exists
+      def ensure_schema_migrations_table!
+        ActiveRecord::Base.connection.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS schema_migrations (
+            version VARCHAR(255) NOT NULL PRIMARY KEY
+          )
+        SQL
+      end
+
+      # Get list of applied migration versions
+      def get_applied_versions
+        ActiveRecord::Base.connection
+          .execute("SELECT version FROM schema_migrations")
+          .map { |row| row.is_a?(Hash) ? row["version"] : row[0] }
+          .to_set
+      end
+
+      # Find all migration files in given paths
+      def find_migration_files(paths)
+        paths.flat_map do |path|
+          Dir[File.join(path, "*.rb")]
+        end.uniq
+      end
+
+      # Extract version (timestamp) from migration filename
+      def version_from_file(filepath)
+        File.basename(filepath).match(/^(\d+)_/)[1]
+      end
+
+      # Run a single migration file
+      def run_migration_file(filepath)
+        version = version_from_file(filepath)
+        filename = File.basename(filepath, ".rb")
+
+        SolidLog.logger&.info("Running migration: #{filename}")
+
+        # Load and instantiate migration
+        load filepath
+        migration_name = filename.split("_", 2).last.camelize
+        migration_class = migration_name.constantize
+
+        # Run migration
+        migration = migration_class.new
+        ActiveRecord::Base.connection.transaction do
+          migration.migrate(:up)
+
+          # Record migration as applied
+          ActiveRecord::Base.connection.execute(
+            "INSERT INTO schema_migrations (version) VALUES ('#{version}')"
+          )
+        end
+
+        SolidLog.logger&.info("Completed migration: #{filename}")
+      end
+    end
+    end
+  end
+end

data/lib/solid_log/core/services/retention_service.rb

@@ -2,7 +2,7 @@ module SolidLog
   module Core
     class RetentionService
       # Cleanup old entries based on retention policies
-      def self.cleanup(retention_days:, error_retention_days:)
+      def self.cleanup(retention_days:, error_retention_days:, max_entries: nil)
         stats = {
           entries_deleted: 0,
           raw_deleted: 0,
@@ -25,6 +25,17 @@ module SolidLog
           .where(level: %w[error fatal])
           .delete_all
 
+        # Enforce max entries limit (delete oldest entries beyond limit)
+        if max_entries && max_entries > 0
+          current_count = Entry.count
+          if current_count > max_entries
+            delete_count = current_count - max_entries
+            oldest_ids = Entry.order(:timestamp).limit(delete_count).pluck(:id)
+            deleted = Entry.where(id: oldest_ids).delete_all
+            stats[:entries_deleted] += deleted
+          end
+        end
+
         # Delete corresponding raw entries (keep unparsed for investigation)
         raw_ids = Entry.pluck(:raw_id).compact
         stats[:raw_deleted] = RawEntry

data/lib/solid_log/core/version.rb

@@ -1,5 +1,5 @@
 module SolidLog
   module Core
-    VERSION = "0.1.0"
+    VERSION = "0.2.1"
   end
 end

data/lib/solid_log/core.rb

@@ -2,6 +2,10 @@ require "solid_log/core/version"
 require "solid_log/core/configuration"
 require "solid_log/core/client"
 require "solid_log/silence_middleware"
+require "logger"
+
+# Load Railtie if Rails is available
+require "solid_log/railtie" if defined?(Rails::Railtie)
 
 # Database adapters
 require "solid_log/adapters/base_adapter"
@@ -22,6 +26,15 @@ require "solid_log/core/services/field_analyzer"
 require "solid_log/core/services/search_service"
 require "solid_log/core/services/correlation_service"
 require "solid_log/core/services/health_service"
+require "solid_log/core/services/migration_generator"
+require "solid_log/core/services/migration_runner"
+require "solid_log/core/services/batch_parsing_service"
+
+# Jobs
+require "solid_log/core/jobs/parse_job"
+require "solid_log/core/jobs/retention_job"
+require "solid_log/core/jobs/cache_cleanup_job"
+require "solid_log/core/jobs/field_analysis_job"
 
 # Models (explicit requires - no engine, no app/ directory)
 require "solid_log/models/record"
@@ -69,6 +82,19 @@ module SolidLog
       def silenced?
         Thread.current[:solid_log_silenced] == true
       end
+
+      # Logger singleton - works in Rails and non-Rails contexts
+      def logger
+        @logger ||= if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+          Rails.logger
+        else
+          Logger.new(STDOUT).tap { |log| log.level = Logger::INFO }
+        end
+      end
+
+      def logger=(logger)
+        @logger = logger
+      end
     end
   end
 end
@@ -95,6 +121,14 @@ module SolidLog
     def silenced?
       Core.silenced?
     end
+
+    def logger
+      Core.logger
+    end
+
+    def logger=(logger)
+      Core.logger = logger
+    end
   end
 
   # Alias service classes for easier access
@@ -103,4 +137,6 @@ module SolidLog
   SearchService = Core::SearchService
   CorrelationService = Core::CorrelationService
   HealthService = Core::HealthService
+  MigrationGenerator = Core::MigrationGenerator
+  MigrationRunner = Core::MigrationRunner
 end

data/lib/solid_log/direct_logger.rb

@@ -40,6 +40,14 @@ module SolidLog
     # EXCEPT for error/fatal logs which flush immediately to prevent data loss on crash
     def write(message)
       return if @closed
+      return if SolidLog.silenced? # Skip if logging is silenced (anti-recursion)
+
+      # Skip SolidLog's own view rendering logs
+      if message.is_a?(String)
+        # Strip whitespace before checking pattern
+        trimmed = message.strip
+        return if trimmed.match?(/^Rendering (layout |partial )?.*solid_log/i)
+      end
 
       log_entry = parse_message(message)
       return unless log_entry # Skip if parsing failed