solid_log-service 0.1.0

data/app/jobs/solid_log/parser_job.rb

@@ -0,0 +1,130 @@
+require "action_cable" if defined?(Rails)
+
+module SolidLog
+  class ParserJob < ApplicationJob
+    queue_as :default
+
+    # Process a batch of unparsed raw entries
+    def perform(batch_size: nil)
+      batch_size ||= SolidLog.configuration.parser_batch_size
+
+      SolidLog.without_logging do
+        # Claim a batch of unparsed entries
+        raw_entries = RawEntry.claim_batch(batch_size: batch_size)
+
+        return if raw_entries.empty?
+
+        Rails.logger.info "SolidLog::ParserJob: Processing #{raw_entries.size} raw entries"
+
+        # Process each entry
+        entries_to_insert = []
+        fields_to_track = {}
+
+        raw_entries.each do |raw_entry|
+          begin
+            # Parse the raw payload
+            parsed = Parser.parse(raw_entry.payload)
+
+            # Extract dynamic fields for field registry
+            extra_fields = parsed.delete(:extra_fields) || {}
+            track_fields(fields_to_track, extra_fields)
+
+            # Prepare entry for insertion
+            entry_data = {
+              raw_id: raw_entry.id,
+              timestamp: parsed[:timestamp],
+              created_at: Time.current, # When entry was parsed/created
+              level: parsed[:level],
+              app: parsed[:app],
+              env: parsed[:env],
+              message: parsed[:message],
+              request_id: parsed[:request_id],
+              job_id: parsed[:job_id],
+              duration: parsed[:duration],
+              status_code: parsed[:status_code],
+              controller: parsed[:controller],
+              action: parsed[:action],
+              path: parsed[:path],
+              method: parsed[:method],
+              extra_fields: extra_fields.to_json
+            }
+
+            entries_to_insert << entry_data
+          rescue StandardError => e
+            Rails.logger.error "SolidLog::ParserJob: Failed to parse entry #{raw_entry.id}: #{e.message}"
+            Rails.logger.error e.backtrace.join("\n")
+            # Leave entry unparsed so it can be retried or investigated
+          end
+        end
+
+        # Bulk insert parsed entries
+        if entries_to_insert.any?
+          raw_ids = entries_to_insert.map { |e| e[:raw_id] }
+
+          Entry.insert_all(entries_to_insert)
+          Rails.logger.info "SolidLog::ParserJob: Inserted #{entries_to_insert.size} entries"
+
+          # Broadcast new entry IDs for live tail
+          begin
+            new_entry_ids = Entry.where(raw_id: raw_ids).pluck(:id)
+            if new_entry_ids.any?
+              ActionCable.server.broadcast(
+                "solid_log_new_entries",
+                { entry_ids: new_entry_ids }
+              )
+            end
+          rescue NameError => e
+            Rails.logger.error "SolidLog::ParserJob: ActionCable not available: #{e.message}"
+          rescue => e
+            Rails.logger.error "SolidLog::ParserJob: Failed to broadcast: #{e.class} - #{e.message}"
+          end
+        end
+
+        # Update field registry
+        update_field_registry(fields_to_track)
+      end
+    end
+
+    private
+
+    # Track field occurrences for the registry
+    def track_fields(fields_hash, extra_fields)
+      extra_fields.each do |key, value|
+        fields_hash[key] ||= { values: [], count: 0 }
+        fields_hash[key][:count] += 1
+        fields_hash[key][:type] ||= infer_field_type(value)
+      end
+    end
+
+    # Update the field registry with tracked fields
+    def update_field_registry(fields_hash)
+      fields_hash.each do |name, data|
+        field = Field.find_or_initialize_by(name: name)
+        field.field_type ||= data[:type]
+        field.usage_count += data[:count]
+        field.last_seen_at = Time.current
+        field.save!
+      end
+    end
+
+    # Infer field type from value
+    def infer_field_type(value)
+      case value
+      when String
+        "string"
+      when Numeric
+        "number"
+      when TrueClass, FalseClass
+        "boolean"
+      when Time, DateTime, Date
+        "datetime"
+      when Array
+        "array"
+      when Hash
+        "object"
+      else
+        "string"
+      end
+    end
+  end
+end

data/app/jobs/solid_log/retention_job.rb

@@ -0,0 +1,24 @@
+module SolidLog
+  class RetentionJob < ApplicationJob
+    queue_as :default
+
+    def perform(retention_days: 30, error_retention_days: 90, vacuum: false)
+      SolidLog.without_logging do
+        Rails.logger.info "SolidLog::RetentionJob: Starting cleanup (retention: #{retention_days} days, errors: #{error_retention_days} days)"
+
+        stats = RetentionService.cleanup(
+          retention_days: retention_days,
+          error_retention_days: error_retention_days
+        )
+
+        Rails.logger.info "SolidLog::RetentionJob: Deleted #{stats[:entries_deleted]} entries, #{stats[:raw_deleted]} raw entries, cleared #{stats[:cache_cleared]} cache entries"
+
+        if vacuum
+          Rails.logger.info "SolidLog::RetentionJob: Running VACUUM..."
+          RetentionService.vacuum_database
+          Rails.logger.info "SolidLog::RetentionJob: VACUUM complete"
+        end
+      end
+    end
+  end
+end

data/bin/solid_log_service

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'rack'
+require_relative '../lib/solid_log/service'
+
+# Parse command line options
+options = {
+  port: ENV['PORT'] || SolidLog::Service.configuration.port || 3001,
+  bind: ENV['BIND'] || SolidLog::Service.configuration.bind || '0.0.0.0',
+  environment: ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'production'
+}
+
+ARGV.each do |arg|
+  case arg
+  when /^--port=(\d+)$/
+    options[:port] = $1.to_i
+  when /^--bind=(.+)$/
+    options[:bind] = $1
+  when /^--environment=(.+)$/
+    options[:environment] = $1
+  when '--help', '-h'
+    puts <<~HELP
+      SolidLog Service - Standalone log ingestion and processing service
+
+      Usage: solid_log_service [options]
+
+      Options:
+        --port=PORT              Port to bind to (default: 3001)
+        --bind=ADDRESS           Address to bind to (default: 0.0.0.0)
+        --environment=ENV        Environment (default: production)
+        --help, -h               Show this help message
+
+      Environment Variables:
+        PORT                     Port to bind to
+        BIND                     Address to bind to
+        RAILS_ENV                Rails environment
+        DATABASE_URL             Database connection string
+        LOG_LEVEL                Log level (debug, info, warn, error)
+
+      Examples:
+        solid_log_service
+        solid_log_service --port=8080 --bind=127.0.0.1
+        PORT=8080 solid_log_service
+
+      Configuration:
+        Create config/solid_log_service.rb to configure the service.
+        See README.md for configuration options.
+    HELP
+    exit 0
+  end
+end
+
+# Set environment
+ENV['RAILS_ENV'] = options[:environment]
+ENV['RACK_ENV'] = options[:environment]
+
+puts "Starting SolidLog Service..."
+puts "  Environment: #{options[:environment]}"
+puts "  Binding: #{options[:bind]}:#{options[:port]}"
+
+# Load the application
+require_relative '../config.ru'
+
+# Run with Puma
+require 'puma/cli'
+
+puma_args = [
+  '--bind', "tcp://#{options[:bind]}:#{options[:port]}",
+  '--environment', options[:environment],
+  '--workers', ENV.fetch('WEB_CONCURRENCY', '2'),
+  '--threads', "#{ENV.fetch('RAILS_MIN_THREADS', '5')}:#{ENV.fetch('RAILS_MAX_THREADS', '5')}"
+]
+
+Puma::CLI.new(puma_args).run

data/config/cable.yml

@@ -0,0 +1,11 @@
+development:
+  adapter: async
+
+test:
+  adapter: test
+
+production:
+  adapter: solid_cable
+  connects_to:
+    database:
+      writing: cable

data/config/routes.rb

@@ -0,0 +1,26 @@
+SolidLog::Service::Engine.routes.draw do
+  namespace :api do
+    namespace :v1 do
+      # Ingestion
+      post "ingest", to: "ingest#create"
+
+      # Queries
+      resources :entries, only: [:index, :show]
+      post "search", to: "search#create"
+
+      # Facets
+      get "facets", to: "facets#index"
+      get "facets/all", to: "facets#all", as: :all_facets
+
+      # Timelines
+      get "timelines/request/:request_id", to: "timelines#show_request", as: :timeline_request
+      get "timelines/job/:job_id", to: "timelines#show_job", as: :timeline_job
+
+      # Health
+      get "health", to: "health#show"
+    end
+  end
+
+  # Root health check
+  get "/health", to: "api/v1/health#show"
+end

data/config.ru

@@ -0,0 +1,7 @@
+require_relative 'lib/solid_log/service'
+require_relative 'lib/solid_log/service/application'
+
+# Load routes
+require_relative 'config/routes'
+
+run SolidLog::Service::Application

data/lib/solid_log/service/application.rb

@@ -0,0 +1,72 @@
+require "rails"
+require "action_controller/railtie"
+require "active_record/railtie"
+require "active_job/railtie"
+require "action_cable/engine"
+
+module SolidLog
+  module Service
+    class Application < Rails::Application
+      config.load_defaults 8.0
+      config.api_only = true
+
+      # Enable caching with memory store
+      config.cache_store = :memory_store
+
+      # Load service configuration
+      config.before_initialize do
+        # Load configuration file if it exists
+        config_file = Rails.root.join("config", "solid_log_service.rb")
+        require config_file if File.exist?(config_file)
+
+        # Load Action Cable configuration
+        cable_config_path = Rails.root.join("config", "cable.yml")
+        if File.exist?(cable_config_path)
+          config.action_cable.cable = YAML.load_file(cable_config_path, aliases: true)[Rails.env]
+        end
+      end
+
+      # Set up database connection
+      config.before_initialize do
+        db_config = {
+          adapter: ENV["DB_ADAPTER"] || "sqlite3",
+          database: ENV["DATABASE_URL"] || Rails.root.join("storage", "production_log.sqlite3").to_s,
+          pool: ENV.fetch("RAILS_MAX_THREADS", 5)
+        }
+
+        ActiveRecord::Base.establish_connection(db_config)
+      end
+
+      # Start job processor after initialization (but not in console mode)
+      config.after_initialize do
+        unless defined?(Rails::Console)
+          SolidLog::Service.start!
+        end
+      end
+
+      # Stop job processor on shutdown
+      at_exit do
+        SolidLog::Service.stop!
+      end
+
+      # Eager load controllers and jobs
+      config.eager_load_paths << Rails.root.join("app", "controllers")
+      config.eager_load_paths << Rails.root.join("app", "jobs")
+
+      # CORS configuration
+      config.middleware.insert_before 0, Rack::Cors do
+        allow do
+          origins { |source, env| SolidLog::Service.configuration.cors_origins.include?(source) || SolidLog::Service.configuration.cors_origins.include?("*") }
+          resource "*",
+            headers: :any,
+            methods: [:get, :post, :put, :patch, :delete, :options, :head],
+            credentials: false
+        end
+      end if defined?(Rack::Cors)
+
+      # Logging
+      config.logger = ActiveSupport::Logger.new(STDOUT)
+      config.log_level = ENV.fetch("LOG_LEVEL", "info").to_sym
+    end
+  end
+end

data/lib/solid_log/service/configuration.rb

@@ -0,0 +1,84 @@
+require "solid_log/core"
+
+module SolidLog
+  module Service
+    class Configuration < SolidLog::Core::Configuration
+      attr_accessor :job_mode,
+                    :parser_interval,
+                    :cache_cleanup_interval,
+                    :retention_hour,
+                    :field_analysis_hour,
+                    :websocket_enabled,
+                    :cors_origins,
+                    :bind,
+                    :port
+
+      def initialize
+        super
+
+        # Load from ENV vars with defaults
+        # Job processing mode: :scheduler (default), :active_job, or :manual
+        @job_mode = env_to_symbol("SOLIDLOG_JOB_MODE", :scheduler)
+
+        # Scheduler intervals (only used when job_mode = :scheduler)
+        @parser_interval = env_to_int("SOLIDLOG_PARSER_INTERVAL", 10)  # seconds
+        @cache_cleanup_interval = env_to_int("SOLIDLOG_CACHE_CLEANUP_INTERVAL", 3600)  # seconds (1 hour)
+        @retention_hour = env_to_int("SOLIDLOG_RETENTION_HOUR", 2)  # Hour of day (0-23)
+        @field_analysis_hour = env_to_int("SOLIDLOG_FIELD_ANALYSIS_HOUR", 3)  # Hour of day (0-23)
+
+        # WebSocket support for live tail
+        @websocket_enabled = env_to_bool("SOLIDLOG_WEBSOCKET_ENABLED", false)
+
+        # CORS configuration for API
+        @cors_origins = env_to_array("SOLIDLOG_CORS_ORIGINS", [])
+
+        # Server configuration
+        @bind = ENV["SOLIDLOG_BIND"] || ENV["BIND"] || "0.0.0.0"
+        @port = env_to_int("SOLIDLOG_PORT") || env_to_int("PORT", 3001)
+      end
+
+      # Validate configuration
+      def valid?
+        errors = []
+
+        errors << "job_mode must be :scheduler, :active_job, or :manual" unless [:scheduler, :active_job, :manual].include?(job_mode)
+        errors << "parser_interval must be positive" unless parser_interval&.positive?
+        errors << "cache_cleanup_interval must be positive" unless cache_cleanup_interval&.positive?
+        errors << "retention_hour must be between 0 and 23" unless retention_hour&.between?(0, 23)
+        errors << "field_analysis_hour must be between 0 and 23" unless field_analysis_hour&.between?(0, 23)
+
+        if errors.any?
+          raise ArgumentError, "Invalid configuration:\n  #{errors.join("\n  ")}"
+        end
+
+        true
+      end
+
+      private
+
+      def env_to_int(key, default = nil)
+        value = ENV[key]
+        return default if value.nil? || value.empty?
+        value.to_i
+      end
+
+      def env_to_bool(key, default = false)
+        value = ENV[key]
+        return default if value.nil? || value.empty?
+        ["true", "1", "yes", "on"].include?(value.downcase)
+      end
+
+      def env_to_symbol(key, default = nil)
+        value = ENV[key]
+        return default if value.nil? || value.empty?
+        value.to_sym
+      end
+
+      def env_to_array(key, default = [])
+        value = ENV[key]
+        return default if value.nil? || value.empty?
+        value.split(",").map(&:strip)
+      end
+    end
+  end
+end

data/lib/solid_log/service/engine.rb

@@ -0,0 +1,25 @@
+require "rails/engine"
+
+module SolidLog
+  module Service
+    class Engine < ::Rails::Engine
+      isolate_namespace SolidLog
+
+      config.generators do |g|
+        g.test_framework :minitest
+        g.fixture_replacement :factory_bot
+      end
+
+      # Ensure controllers and jobs are autoloaded
+      config.autoload_paths << root.join("app/controllers")
+      config.autoload_paths << root.join("app/jobs")
+
+      # Add SilenceMiddleware to prevent recursive logging
+      # This intercepts all requests to the service and sets Thread.current[:solid_log_silenced]
+      # so the service doesn't log its own API requests, parser jobs, etc.
+      initializer "solid_log_service.add_middleware" do |app|
+        app.middleware.use SolidLog::SilenceMiddleware
+      end
+    end
+  end
+end

data/lib/solid_log/service/job_processor.rb

@@ -0,0 +1,82 @@
+module SolidLog
+  module Service
+    class JobProcessor
+      class << self
+        attr_reader :scheduler
+
+        def setup
+          case configuration.job_mode
+          when :scheduler
+            setup_scheduler
+          when :active_job
+            setup_active_job
+          when :manual
+            setup_manual
+          else
+            raise ArgumentError, "Invalid job_mode: #{configuration.job_mode}. Must be :scheduler, :active_job, or :manual"
+          end
+        end
+
+        def stop
+          case configuration.job_mode
+          when :scheduler
+            stop_scheduler
+          when :active_job, :manual
+            # Nothing to stop
+          end
+        end
+
+        private
+
+        def configuration
+          SolidLog::Service.configuration
+        end
+
+        def setup_scheduler
+          @scheduler = Scheduler.new(configuration)
+          @scheduler.start
+
+          Rails.logger.info "SolidLog::Service: Started built-in Scheduler"
+          Rails.logger.info "  Parser interval: #{configuration.parser_interval}s"
+          Rails.logger.info "  Cache cleanup interval: #{configuration.cache_cleanup_interval}s"
+          Rails.logger.info "  Retention hour: #{configuration.retention_hour}:00"
+          Rails.logger.info "  Field analysis hour: #{configuration.field_analysis_hour}:00"
+        end
+
+        def stop_scheduler
+          if @scheduler
+            @scheduler.stop
+            @scheduler = nil
+          end
+        end
+
+        def setup_active_job
+          # Jobs are enqueued via host app's ActiveJob backend
+          # Host app should configure recurring jobs using their job backend
+          # Example with Solid Queue:
+          #
+          # SolidQueue::RecurringTask.create!(
+          #   key: 'solidlog_parser',
+          #   schedule: 'every 10 seconds',
+          #   class_name: 'SolidLog::ParserJob'
+          # )
+
+          Rails.logger.info "SolidLog::Service: Using ActiveJob for background processing"
+          Rails.logger.info "  Make sure to configure recurring jobs in your host application"
+        end
+
+        def setup_manual
+          # User manages scheduling via cron or other external scheduler
+          # No setup needed
+
+          Rails.logger.info "SolidLog::Service: Manual job mode (no auto-scheduling)"
+          Rails.logger.info "  Set up cron jobs to run:"
+          Rails.logger.info "    - rails solid_log:parse_logs (every 10 seconds recommended)"
+          Rails.logger.info "    - rails solid_log:cache_cleanup (hourly recommended)"
+          Rails.logger.info "    - rails solid_log:retention (daily recommended)"
+          Rails.logger.info "    - rails solid_log:field_analysis (daily recommended)"
+        end
+      end
+    end
+  end
+end

data/lib/solid_log/service/scheduler.rb

@@ -0,0 +1,146 @@
+require "thread"
+
+module SolidLog
+  module Service
+    class Scheduler
+      attr_reader :threads, :running
+
+      def initialize(config = SolidLog::Service.configuration)
+        @config = config
+        @threads = []
+        @running = false
+        @mutex = Mutex.new
+      end
+
+      def start
+        return if @running
+
+        @mutex.synchronize do
+          return if @running
+          @running = true
+        end
+
+        Rails.logger.info "SolidLog::Service::Scheduler starting..."
+
+        # Parser job - frequent (configurable, default 10s)
+        thread = Thread.new { parser_loop }
+        thread.abort_on_exception = true
+        @threads << thread
+
+        # Cache cleanup - configurable (default hourly)
+        thread = Thread.new { cache_cleanup_loop }
+        thread.abort_on_exception = true
+        @threads << thread
+
+        # Daily jobs - retention and field analysis
+        thread = Thread.new { daily_jobs_loop }
+        thread.abort_on_exception = true
+        @threads << thread
+
+        Rails.logger.info "SolidLog::Service::Scheduler started with #{@threads.size} threads"
+      end
+
+      def stop
+        return unless @running
+
+        Rails.logger.info "SolidLog::Service::Scheduler stopping..."
+
+        @mutex.synchronize do
+          @running = false
+        end
+
+        # Give threads 5 seconds to finish gracefully
+        @threads.each { |t| t.join(5) }
+
+        # Force kill any remaining threads
+        @threads.each { |t| t.kill if t.alive? }
+        @threads.clear
+
+        Rails.logger.info "SolidLog::Service::Scheduler stopped"
+      end
+
+      def running?
+        @running
+      end
+
+      private
+
+      def parser_loop
+        loop do
+          break unless @running
+
+          begin
+            SolidLog::ParserJob.perform_now
+          rescue => e
+            Rails.logger.error "SolidLog::Scheduler: Parser job failed: #{e.message}"
+            Rails.logger.error e.backtrace.join("\n")
+          end
+
+          sleep @config.parser_interval
+        end
+      end
+
+      def cache_cleanup_loop
+        loop do
+          break unless @running
+
+          begin
+            SolidLog::CacheCleanupJob.perform_now
+          rescue => e
+            Rails.logger.error "SolidLog::Scheduler: Cache cleanup failed: #{e.message}"
+            Rails.logger.error e.backtrace.join("\n")
+          end
+
+          sleep @config.cache_cleanup_interval
+        end
+      end
+
+      def daily_jobs_loop
+        loop do
+          break unless @running
+
+          current_hour = Time.current.hour
+
+          # Run retention job at configured hour (default 2 AM)
+          if current_hour == @config.retention_hour
+            run_daily_job(:retention)
+            sleep 1.hour  # Wait an hour to avoid running multiple times
+          end
+
+          # Run field analysis at configured hour (default 3 AM)
+          if current_hour == @config.field_analysis_hour
+            run_daily_job(:field_analysis)
+            sleep 1.hour  # Wait an hour to avoid running multiple times
+          end
+
+          # Check every 10 minutes
+          sleep 10.minutes
+        end
+      end
+
+      def run_daily_job(job_name)
+        case job_name
+        when :retention
+          begin
+            SolidLog::RetentionJob.perform_now(
+              retention_days: @config.retention_days,
+              error_retention_days: @config.error_retention_days
+            )
+          rescue => e
+            Rails.logger.error "SolidLog::Scheduler: Retention job failed: #{e.message}"
+            Rails.logger.error e.backtrace.join("\n")
+          end
+        when :field_analysis
+          begin
+            SolidLog::FieldAnalysisJob.perform_now(
+              auto_promote: @config.auto_promote_fields
+            )
+          rescue => e
+            Rails.logger.error "SolidLog::Scheduler: Field analysis failed: #{e.message}"
+            Rails.logger.error e.backtrace.join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/solid_log/service/version.rb

@@ -0,0 +1,5 @@
+module SolidLog
+  module Service
+    VERSION = "0.1.0"
+  end
+end