solid_log-core 0.1.0

data/lib/generators/solid_log/install/install_generator.rb

@@ -0,0 +1,134 @@
+require "rails/generators/base"
+
+module SolidLog
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      class_option :database, type: :string, default: nil, desc: "Database adapter (sqlite, postgresql, mysql)"
+
+      desc "Install SolidLog configuration, migrations, and database structure"
+
+      def copy_migrations
+        say "Copying SolidLog migrations...", :green
+
+        # Create log_migrate directory
+        empty_directory "db/log_migrate"
+
+        # Copy all migrations from solid_log-core
+        migrations_path = File.expand_path("../../../../db/log_migrate", __dir__)
+        Dir[File.join(migrations_path, "*.rb")].each do |file|
+          copy_file file, "db/log_migrate/#{File.basename(file)}"
+        end
+      end
+
+      def copy_structure_file
+        database = options[:database] || detect_database_adapter
+
+        say "Copying #{database} database structure file...", :green
+
+        structure_file = case database
+        when "sqlite", "sqlite3"
+          "log_structure_sqlite.sql"
+        when "postgresql", "postgres", "pg"
+          "log_structure_postgresql.sql"
+        when "mysql", "mysql2", "trilogy"
+          "log_structure_mysql.sql"
+        else
+          say "Unknown database: #{database}, skipping structure file", :yellow
+          return
+        end
+
+        source_path = File.expand_path("../../../../db/#{structure_file}", __dir__)
+        if File.exist?(source_path)
+          copy_file source_path, "db/log_structure.sql"
+        else
+          say "Structure file not found: #{structure_file}", :red
+        end
+      end
+
+      def create_initializer
+        say "Creating initializer...", :green
+        template "solid_log.rb.tt", "config/initializers/solid_log.rb"
+      end
+
+      def show_instructions
+        say "\n" + "="*80, :green
+        say "SolidLog Installation Complete! 🎉", :green
+        say "="*80 + "\n", :green
+
+        say "Next steps:", :yellow
+        say ""
+        say "1. Configure your database in config/database.yml:", :cyan
+        say "   Add a :log database connection. Example:"
+        say ""
+        say "   production:"
+        say "     primary:"
+        say "       adapter: sqlite3"
+        say "       database: storage/production.sqlite3"
+        say "     log:"
+        say "       adapter: sqlite3"
+        say "       database: storage/production_log.sqlite3"
+        say "       migrations_paths: db/log_migrate"
+        say ""
+        say "2. Create databases and run migrations:", :cyan
+        say "   rails db:create"
+        say "   rails db:migrate"
+        say ""
+        say "3. (Alternative) Load structure file:", :cyan
+        say "   rails db:create"
+        say "   rails db:schema:load SCHEMA=db/log_structure.sql DATABASE=log"
+        say ""
+        say "4. Enable WAL mode for SQLite (HIGHLY RECOMMENDED):", :cyan
+        say "   Already configured in config/initializers/solid_log.rb"
+        say "   Provides 3.4x faster performance for crash-safe logging!"
+        say ""
+        say "5. Configure DirectLogger for your app (recommended):", :cyan
+        say "   See config/initializers/solid_log.rb for examples"
+        say "   DirectLogger is 9x faster than individual inserts"
+        say ""
+        say "6. Create an API token:", :cyan
+        say "   rails solid_log:create_token[\"Production API\"]"
+        say ""
+        say "7. Mount the UI (if using solid_log-ui):", :cyan
+        say "   Add to config/routes.rb:"
+        say "   mount SolidLog::UI::Engine => '/admin/logs'"
+        say ""
+
+        say "Useful commands:", :yellow
+        say "  rails solid_log:stats          # View log statistics"
+        say "  rails solid_log:health         # Check system health"
+        say "  rails solid_log:list_tokens    # List API tokens"
+        say "  rails solid_log:parse_logs     # Parse raw logs"
+        say "  rails solid_log:retention[30]  # Clean up old logs"
+        say ""
+        say "Documentation:", :green
+        say "  README:      README.md"
+        say "  Quickstart:  QUICKSTART.md"
+        say "  Benchmarks:  solid_log-core/BENCHMARK_RESULTS.md"
+        say "="*80 + "\n", :green
+      end
+
+      private
+
+      def detect_database_adapter
+        if defined?(ActiveRecord::Base)
+          adapter = ActiveRecord::Base.connection_db_config.adapter rescue nil
+          return adapter if adapter
+        end
+
+        # Try to read from database.yml
+        if File.exist?("config/database.yml")
+          require "yaml"
+          db_config = YAML.load_file("config/database.yml")
+          env = ENV["RAILS_ENV"] || "development"
+          adapter = db_config.dig(env, "adapter") || db_config.dig(env, "primary", "adapter")
+          return adapter if adapter
+        end
+
+        # Default to sqlite
+        "sqlite3"
+      end
+    end
+  end
+end

data/lib/generators/solid_log/install/templates/solid_log.rb.tt

@@ -0,0 +1,133 @@
+# SolidLog Configuration
+SolidLog::Core.configure do |config|
+  # Retention policies
+  config.retention_days = 30              # Keep regular logs for 30 days
+  config.error_retention_days = 90        # Keep errors for 90 days
+
+  # Parsing & ingestion
+  config.max_batch_size = 100             # Logs per batch insert
+  config.parser_batch_size = 200          # Parser job batch size
+
+  # Performance
+  config.facet_cache_ttl = 5.minutes      # Cache filter options for 5 min
+end
+
+# Enable WAL mode for SQLite (HIGHLY RECOMMENDED)
+# Provides 3.4x faster eager flush performance (16,882 vs 4,923 logs/sec)
+# Skip this if using PostgreSQL or MySQL
+if defined?(ActiveRecord::Base)
+  begin
+    ActiveRecord::Base.connected_to(role: :log) do
+      adapter = ActiveRecord::Base.connection_db_config.adapter
+      if adapter == "sqlite3"
+        ActiveRecord::Base.connection.execute("PRAGMA journal_mode=WAL")
+        ActiveRecord::Base.connection.execute("PRAGMA synchronous=NORMAL")
+        Rails.logger.info "SolidLog: WAL mode enabled for SQLite"
+      end
+    end
+  rescue => e
+    Rails.logger.warn "SolidLog: Could not enable WAL mode: #{e.message}"
+  end
+end
+
+# ============================================================================
+# DirectLogger Configuration (RECOMMENDED for parent app logging)
+# ============================================================================
+#
+# DirectLogger writes logs directly to the database for maximum performance.
+# It's 9x faster than individual inserts and 67x faster than HTTP logging.
+#
+# Performance with WAL mode:
+#   - With crash safety: 16,882 logs/sec
+#   - Without crash safety: 56,660 logs/sec
+#
+# See: solid_log-core/BENCHMARK_RESULTS.md for detailed benchmarks
+
+Rails.application.configure do
+  # Option 1: Use DirectLogger with Lograge (recommended)
+  #
+  # config.lograge.enabled = true
+  # config.lograge.formatter = Lograge::Formatters::Json.new
+  #
+  # config.lograge.logger = ActiveSupport::Logger.new(
+  #   SolidLog::DirectLogger.new(
+  #     batch_size: 100,                      # Flush after 100 logs
+  #     flush_interval: 5,                    # Or after 5 seconds
+  #     eager_flush_levels: [:error, :fatal]  # Flush errors immediately (crash safety)
+  #   )
+  # )
+
+  # Option 2: Use DirectLogger with standard Rails logger
+  #
+  # config.logger = ActiveSupport::Logger.new(
+  #   SolidLog::DirectLogger.new
+  # )
+
+  # Option 3: Use HTTP Logger (for external services)
+  #
+  # For services without direct database access, use HTTP:
+  #
+  # config.lograge.enabled = true
+  # config.lograge.formatter = Lograge::Formatters::Json.new
+  # config.lograge.logger = ActiveSupport::Logger.new(
+  #   SolidLog::HttpLogger.new(
+  #     url: ENV['SOLIDLOG_URL'] || "http://localhost:3000/admin/logs/api/v1/ingest",
+  #     token: ENV['SOLIDLOG_TOKEN']
+  #   )
+  # )
+end
+
+# ============================================================================
+# DirectLogger Crash Safety
+# ============================================================================
+#
+# By default, DirectLogger flushes error/fatal logs IMMEDIATELY to prevent
+# losing the logs that explain WHY your app crashed.
+#
+# How it works:
+#   - info/debug/warn logs: Buffered (fast)
+#   - error/fatal logs: Flushed immediately (safe)
+#   - When error occurs: All buffered logs + error log flushed together
+#
+# Example:
+#   10:00:00 - info: Request started (buffered)
+#   10:00:01 - info: Processing params (buffered)
+#   10:00:02 - ERROR: Database connection lost (FLUSHES ALL 3 LOGS IMMEDIATELY)
+#   [App crashes - but logs are safe!]
+#
+# Configuration options:
+#
+# # Maximum safety (flush everything)
+# SolidLog::DirectLogger.new(
+#   batch_size: 10,
+#   flush_interval: 1,
+#   eager_flush_levels: [:debug, :info, :warn, :error, :fatal]
+# )
+#
+# # Maximum performance (risky - may lose crash logs!)
+# SolidLog::DirectLogger.new(
+#   batch_size: 500,
+#   flush_interval: 30,
+#   eager_flush_levels: []  # Disable eager flush
+# )
+#
+# # Recommended balance (default)
+# SolidLog::DirectLogger.new(
+#   batch_size: 100,
+#   flush_interval: 5,
+#   eager_flush_levels: [:error, :fatal]  # Default
+# )
+
+# ============================================================================
+# Token Configuration (Optional)
+# ============================================================================
+#
+# DirectLogger's token_id is optional. Use it only for audit trail tracking.
+#
+# Priority order:
+#   1. Explicit token_id parameter
+#   2. ENV["SOLIDLOG_TOKEN_ID"]
+#   3. nil (default - no token required)
+#
+# Example:
+#   ENV["SOLIDLOG_TOKEN_ID"] = "123"  # Optional

data/lib/solid_log/adapters/adapter_factory.rb

@@ -0,0 +1,34 @@
+module SolidLog
+  module Adapters
+    class AdapterFactory
+      class << self
+        def adapter
+          @adapter ||= build_adapter
+        end
+
+        def reset!
+          @adapter = nil
+        end
+
+        private
+
+        def build_adapter
+          connection = ActiveRecord::Base.connection
+          adapter_name = connection.adapter_name.downcase
+
+          case adapter_name
+          when "sqlite"
+            SqliteAdapter.new(connection)
+          when "postgresql"
+            PostgresqlAdapter.new(connection)
+          when "mysql2", "trilogy"
+            MysqlAdapter.new(connection)
+          else
+            raise "Unsupported database adapter: #{adapter_name}. " \
+                  "SolidLog supports SQLite, PostgreSQL, and MySQL."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/solid_log/adapters/base_adapter.rb

@@ -0,0 +1,88 @@
+module SolidLog
+  module Adapters
+    class BaseAdapter
+      def initialize(connection)
+        @connection = connection
+      end
+
+      # Search entries - combines FTS with LIKE for comprehensive results
+      def search(query, base_scope = Entry)
+        raise NotImplementedError, "#{self.class} must implement #search"
+      end
+
+      # Full-text search (legacy - used internally by search)
+      def fts_search(query)
+        raise NotImplementedError, "#{self.class} must implement #fts_search"
+      end
+
+      # Case-insensitive LIKE (database-agnostic)
+      def case_insensitive_like(column, pattern)
+        raise NotImplementedError, "#{self.class} must implement #case_insensitive_like"
+      end
+
+      # Claim unparsed entries with lock
+      def claim_batch(batch_size)
+        raise NotImplementedError, "#{self.class} must implement #claim_batch"
+      end
+
+      # JSON field extraction
+      def extract_json_field(column, field_name)
+        raise NotImplementedError, "#{self.class} must implement #extract_json_field"
+      end
+
+      # Facet query (distinct values)
+      def facet_values(column)
+        raise NotImplementedError, "#{self.class} must implement #facet_values"
+      end
+
+      # Optimize database
+      def optimize!
+        # Default: no-op
+      end
+
+      # Database size in bytes
+      def database_size
+        raise NotImplementedError, "#{self.class} must implement #database_size"
+      end
+
+      # Supports feature?
+      def supports_skip_locked?
+        false
+      end
+
+      def supports_native_json?
+        false
+      end
+
+      def supports_full_text_search?
+        false
+      end
+
+      # Database-specific configurations
+      def configure!
+        # Default: no-op
+      end
+
+      # Database-specific epoch conversion (must be implemented by subclasses)
+      def timestamp_to_epoch_sql(column)
+        raise NotImplementedError, "#{self.class} must implement #timestamp_to_epoch_sql"
+      end
+
+      protected
+
+      attr_reader :connection
+
+      def execute(sql)
+        connection.execute(sql)
+      end
+
+      def select_all(sql)
+        connection.select_all(sql)
+      end
+
+      def select_value(sql)
+        connection.select_value(sql)
+      end
+    end
+  end
+end

data/lib/solid_log/adapters/mysql_adapter.rb

@@ -0,0 +1,163 @@
+module SolidLog
+  module Adapters
+    class MysqlAdapter < BaseAdapter
+      def search(query, base_scope = Entry)
+        sanitized_query = Entry.sanitize_sql_like(query)
+        table_name = Entry.table_name
+
+        # Build LIKE query for partial matching
+        like_condition = case_insensitive_like("#{table_name}.message", "%#{sanitized_query}%")
+        like_sql = "SELECT #{table_name}.* FROM #{table_name} WHERE #{like_condition}"
+
+        # Build FTS query for full-text matching
+        fts_fragment = fts_search(query)
+        fts_sql = "SELECT #{table_name}.* FROM #{table_name} #{fts_fragment}"
+
+        # UNION both queries (DISTINCT removes duplicates)
+        union_sql = "#{fts_sql} UNION #{like_sql}"
+        Entry.from("(#{union_sql}) AS #{table_name}")
+      end
+
+      def fts_search(query)
+        # MySQL FULLTEXT search
+        sanitized_query = connection.quote(query)
+        <<~SQL.squish
+          WHERE MATCH(message, extra_fields) AGAINST(#{sanitized_query} IN NATURAL LANGUAGE MODE)
+        SQL
+      end
+
+      def case_insensitive_like(column, pattern)
+        # MySQL LIKE is case-insensitive by default with default collation
+        "#{column} LIKE #{connection.quote(pattern)}"
+      end
+
+      def claim_batch(batch_size)
+        # MySQL 8.0+ supports SKIP LOCKED
+        RawEntry.transaction do
+          entries = RawEntry.where(parsed: false)
+            .order(received_at: :asc)
+            .limit(batch_size)
+            .lock("FOR UPDATE SKIP LOCKED")
+            .to_a
+
+          return [] if entries.empty?
+
+          # Mark as parsed
+          entry_ids = entries.map(&:id)
+          RawEntry.where(id: entry_ids).update_all(parsed: true, parsed_at: Time.current)
+
+          entries
+        end
+      rescue ActiveRecord::StatementInvalid => e
+        # Fallback for MySQL < 8.0 without SKIP LOCKED
+        if e.message.include?("syntax error")
+          claim_batch_legacy(batch_size)
+        else
+          raise
+        end
+      end
+
+      def extract_json_field(column, field_name)
+        # MySQL JSON_EXTRACT
+        "JSON_UNQUOTE(JSON_EXTRACT(#{column}, '$.#{field_name}'))"
+      end
+
+      def facet_values(column)
+        "SELECT DISTINCT #{column} FROM solid_log_entries WHERE #{column} IS NOT NULL ORDER BY #{column}"
+      end
+
+      def optimize!
+        execute("OPTIMIZE TABLE solid_log_entries")
+        execute("OPTIMIZE TABLE solid_log_raw")
+        execute("ANALYZE TABLE solid_log_entries")
+        execute("ANALYZE TABLE solid_log_raw")
+      rescue ActiveRecord::StatementInvalid => e
+        Rails.logger.warn("MySQL optimize warning: #{e.message}")
+      end
+
+      def database_size
+        result = select_value(<<~SQL)
+          SELECT SUM(data_length + index_length)
+          FROM information_schema.TABLES
+          WHERE table_schema = DATABASE()
+          AND table_name LIKE 'solid_log_%'
+        SQL
+        result.to_i
+      end
+
+      def supports_skip_locked?
+        # MySQL 8.0+
+        version = connection.get_database_version
+        version >= "8.0"
+      end
+
+      def supports_native_json?
+        true  # MySQL 5.7+
+      end
+
+      def supports_full_text_search?
+        true  # FULLTEXT indexes
+      end
+
+      def configure!
+        # Set optimal MySQL settings
+        execute("SET SESSION sql_mode = 'TRADITIONAL'") rescue nil
+        execute("SET SESSION innodb_lock_wait_timeout = 50") rescue nil
+      end
+
+      def timestamp_to_epoch_sql(column)
+        "UNIX_TIMESTAMP(#{column})"
+      end
+
+      # MySQL-specific methods
+      def create_fts_index_sql
+        <<~SQL
+          CREATE FULLTEXT INDEX idx_entries_fts
+          ON solid_log_entries(message, extra_fields)
+        SQL
+      end
+
+      def create_json_indexes_sql
+        # MySQL doesn't have native JSON indexes, use generated columns
+        [
+          # Example for common fields - these would be created via field promotion
+          # "ALTER TABLE solid_log_entries ADD COLUMN user_id_virtual INT GENERATED ALWAYS AS (JSON_UNQUOTE(JSON_EXTRACT(extra_fields, '$.user_id'))) STORED",
+          # "CREATE INDEX idx_entries_user_id ON solid_log_entries(user_id_virtual)"
+        ]
+      end
+
+      # Bulk insert optimization for MySQL
+      def bulk_insert(table_name, records)
+        return if records.empty?
+
+        columns = records.first.keys
+        values_sql = records.map do |record|
+          "(#{columns.map { |col| connection.quote(record[col]) }.join(', ')})"
+        end.join(", ")
+
+        execute(<<~SQL)
+          INSERT INTO #{table_name} (#{columns.join(', ')})
+          VALUES #{values_sql}
+        SQL
+      end
+
+      private
+
+      def claim_batch_legacy(batch_size)
+        # Fallback for MySQL < 8.0 without SKIP LOCKED
+        RawEntry.transaction do
+          ids = RawEntry.where(parsed: false)
+            .order(received_at: :asc)
+            .limit(batch_size)
+            .lock
+            .pluck(:id)
+
+          return [] if ids.empty?
+
+          RawEntry.where(id: ids).update_all(parsed: true, parsed_at: Time.current)
+          RawEntry.where(id: ids).to_a
+        end
+      end
+    end
+  end
+end

data/lib/solid_log/adapters/postgresql_adapter.rb

@@ -0,0 +1,141 @@
+module SolidLog
+  module Adapters
+    class PostgresqlAdapter < BaseAdapter
+      def search(query, base_scope = Entry)
+        sanitized_query = Entry.sanitize_sql_like(query)
+        table_name = Entry.table_name
+
+        # Build LIKE query for partial matching
+        like_condition = case_insensitive_like("#{table_name}.message", "%#{sanitized_query}%")
+        like_sql = "SELECT #{table_name}.* FROM #{table_name} WHERE #{like_condition}"
+
+        # Build FTS query for full-text matching
+        fts_fragment = fts_search(query)
+        fts_sql = "SELECT #{table_name}.* FROM #{table_name} #{fts_fragment}"
+
+        # UNION both queries (DISTINCT removes duplicates)
+        union_sql = "#{fts_sql} UNION #{like_sql}"
+        Entry.from("(#{union_sql}) AS #{table_name}")
+      end
+
+      def fts_search(query)
+        # PostgreSQL full-text search using tsvector
+        sanitized_query = connection.quote(query.gsub(/[^\w\s]/, " "))
+        <<~SQL.squish
+          WHERE to_tsvector('english', COALESCE(message, '') || ' ' || COALESCE(extra_fields::text, ''))
+          @@ plainto_tsquery('english', #{sanitized_query})
+        SQL
+      end
+
+      def case_insensitive_like(column, pattern)
+        # PostgreSQL uses ILIKE for case-insensitive matching
+        "#{column} ILIKE #{connection.quote(pattern)}"
+      end
+
+      def claim_batch(batch_size)
+        # PostgreSQL supports SKIP LOCKED
+        RawEntry.transaction do
+          entries = RawEntry.where(parsed: false)
+            .order(received_at: :asc)
+            .limit(batch_size)
+            .lock("FOR UPDATE SKIP LOCKED")
+            .to_a
+
+          return [] if entries.empty?
+
+          # Mark as parsed
+          entry_ids = entries.map(&:id)
+          RawEntry.where(id: entry_ids).update_all(parsed: true, parsed_at: Time.current)
+
+          entries
+        end
+      end
+
+      def extract_json_field(column, field_name)
+        # PostgreSQL JSONB operator
+        "#{column}->>'#{field_name}'"
+      end
+
+      def facet_values(column)
+        "SELECT DISTINCT #{column} FROM solid_log_entries WHERE #{column} IS NOT NULL ORDER BY #{column}"
+      end
+
+      def optimize!
+        # Analyze tables for query planning
+        execute("ANALYZE solid_log_entries")
+        execute("ANALYZE solid_log_raw")
+        execute("ANALYZE solid_log_fields")
+
+        # Vacuum if needed (non-blocking)
+        execute("VACUUM ANALYZE solid_log_entries")
+      rescue ActiveRecord::StatementInvalid => e
+        # VACUUM might fail if already running
+        Rails.logger.warn("PostgreSQL vacuum warning: #{e.message}")
+      end
+
+      def database_size
+        result = select_value(<<~SQL)
+          SELECT pg_database_size(current_database())
+        SQL
+        result.to_i
+      end
+
+      def supports_skip_locked?
+        true
+      end
+
+      def supports_native_json?
+        true  # JSONB
+      end
+
+      def supports_full_text_search?
+        true  # tsvector/tsquery
+      end
+
+      def configure!
+        # Set optimal PostgreSQL settings for logging workload
+        execute("SET work_mem = '64MB'") rescue nil
+        execute("SET maintenance_work_mem = '256MB'") rescue nil
+      end
+
+      def timestamp_to_epoch_sql(column)
+        "EXTRACT(EPOCH FROM #{column})::bigint"
+      end
+
+      # PostgreSQL-specific methods
+      def create_fts_index_sql
+        <<~SQL
+          CREATE INDEX IF NOT EXISTS idx_entries_fts
+          ON solid_log_entries
+          USING GIN (to_tsvector('english', COALESCE(message, '') || ' ' || COALESCE(extra_fields::text, '')))
+        SQL
+      end
+
+      def create_json_indexes_sql
+        # GIN index for JSONB queries
+        [
+          <<~SQL
+            CREATE INDEX IF NOT EXISTS idx_entries_extra_fields
+            ON solid_log_entries
+            USING GIN (extra_fields jsonb_path_ops)
+          SQL
+        ]
+      end
+
+      # Bulk insert optimization for PostgreSQL
+      def bulk_insert(table_name, records)
+        return if records.empty?
+
+        columns = records.first.keys
+        values_sql = records.map do |record|
+          "(#{columns.map { |col| connection.quote(record[col]) }.join(', ')})"
+        end.join(", ")
+
+        execute(<<~SQL)
+          INSERT INTO #{table_name} (#{columns.join(', ')})
+          VALUES #{values_sql}
+        SQL
+      end
+    end
+  end
+end