solid_log-service 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c94d0fde51404dd97f27e8d437a9c9edab5366ba360bbd2719b0a658a28c5fe
-  data.tar.gz: 1addb2c0e3b2471fdadccbd3741905747679b1e6475e3f6a12f6a5f3a44f5193
+  metadata.gz: 6c115e88f98776a378b9c5938a31e7c4d09e26d255079da4847ce6b8a406671e
+  data.tar.gz: 5f77c6da8f89be28d3e08854013b4a980107b0aeba18342c6dbf91e55629bbfa
 SHA512:
-  metadata.gz: 8e3ab660ee56208d357bc6bf5baf5e81e89137df79b9fe5849fd8d3567934529335a77c264ae6de75432d07c3b1e1611827d2aa4c61fbba4c7b98edeebaccc72
-  data.tar.gz: 7edb569df3716558642f22c9a55d680eb2f022ff6832a8364ead35824959b1354873dab3caa2b6039c5aa5430545c005ca480dbc1f26bafc5ad4c343c79bc7f6
+  metadata.gz: e5521a9483b8a616e7df54cfc30849b9cde8aca8be35bba06ad5905eb3724822d068e478e8bdad880b4587d529bc3a48551a30b2770e7c6b9abf3fdd428971aa
+  data.tar.gz: baa483b28aaf4e0e009355ace039ee6ab59811acc5a51b5f52b7467e71a3978e46fe3bc9de8391e48af6f84be306168f0c5de7e707b9b5b5e92aa28abadda823

data/config/cable.yml

@@ -5,7 +5,4 @@ test:
   adapter: test
 
 production:
-  adapter: solid_cable
-  connects_to:
-    database:
-      writing: cable
+  adapter: async

data/config.ru

@@ -1,7 +1,56 @@
+# This file is used by Rack-based servers to start the application.
+require 'rubygems'
+
+# Only require bundler/setup if not using global gems (development/test)
+# In production Docker, gems are installed globally and SKIP_BUNDLER is set
+require 'bundler/setup' unless ENV['SKIP_BUNDLER'] == 'true'
+
+require 'active_support'
+require 'active_support/core_ext'
+require 'active_record'
+require 'action_cable'
+
+# Set up ActiveRecord database connection
+url = ENV["SOLIDLOG_DATABASE_URL"] || ENV["DATABASE_URL"]
+adapter = ENV["SOLIDLOG_DB_ADAPTER"] || ENV["DB_ADAPTER"] || "sqlite3"
+pool = ENV.fetch("RAILS_MAX_THREADS", 5).to_i
+
+db_config = if url&.include?("://")
+  { url: url, pool: pool }
+else
+  { adapter: adapter, database: url || "storage/production_log.sqlite3", pool: pool }
+end
+
+ActiveRecord::Base.establish_connection(db_config)
+
+# Load solid_log gems
+require 'solid_log/core'
 require_relative 'lib/solid_log/service'
-require_relative 'lib/solid_log/service/application'
 
-# Load routes
-require_relative 'config/routes'
+# Configure ActionCable for live-tailing
+cable_config_path = File.join(__dir__, 'config', 'cable.yml')
+if File.exist?(cable_config_path)
+  require 'yaml'
+  env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'production'
+  cable_config = YAML.load_file(cable_config_path)[env]
+  ActionCable.server.config.cable = cable_config if cable_config
+end
+ActionCable.server.config.logger = SolidLog::Service.logger
+
+# Load configuration file if it exists
+config_file = File.join(__dir__, 'config', 'solid_log_service.rb')
+require config_file if File.exist?(config_file)
+
+# Run pending migrations (auto-migrate unless disabled)
+unless ENV['SKIP_AUTO_MIGRATE'] == 'true'
+  SolidLog::MigrationRunner.run_pending_migrations
+end
+
+# Start job processor
+SolidLog::Service.start!
+
+# Shutdown hook
+at_exit { SolidLog::Service.stop! }
 
-run SolidLog::Service::Application
+# Run Rack app
+run SolidLog::Service::RackApp.new

data/lib/solid_log/service/job_processor.rb

@@ -36,11 +36,11 @@ module SolidLog
           @scheduler = Scheduler.new(configuration)
           @scheduler.start
 
-          Rails.logger.info "SolidLog::Service: Started built-in Scheduler"
-          Rails.logger.info "  Parser interval: #{configuration.parser_interval}s"
-          Rails.logger.info "  Cache cleanup interval: #{configuration.cache_cleanup_interval}s"
-          Rails.logger.info "  Retention hour: #{configuration.retention_hour}:00"
-          Rails.logger.info "  Field analysis hour: #{configuration.field_analysis_hour}:00"
+          SolidLog::Service.logger.info "SolidLog::Service: Started built-in Scheduler"
+          SolidLog::Service.logger.info "  Parser interval: #{configuration.parser_interval}s"
+          SolidLog::Service.logger.info "  Cache cleanup interval: #{configuration.cache_cleanup_interval}s"
+          SolidLog::Service.logger.info "  Retention hour: #{configuration.retention_hour}:00"
+          SolidLog::Service.logger.info "  Field analysis hour: #{configuration.field_analysis_hour}:00"
         end
 
         def stop_scheduler
@@ -58,23 +58,23 @@ module SolidLog
           # SolidQueue::RecurringTask.create!(
           #   key: 'solidlog_parser',
           #   schedule: 'every 10 seconds',
-          #   class_name: 'SolidLog::ParserJob'
+          #   class_name: 'SolidLog::Core::Jobs::ParseJob'
           # )
 
-          Rails.logger.info "SolidLog::Service: Using ActiveJob for background processing"
-          Rails.logger.info "  Make sure to configure recurring jobs in your host application"
+          SolidLog::Service.logger.info "SolidLog::Service: Using ActiveJob for background processing"
+          SolidLog::Service.logger.info "  Make sure to configure recurring jobs in your host application"
         end
 
         def setup_manual
           # User manages scheduling via cron or other external scheduler
           # No setup needed
 
-          Rails.logger.info "SolidLog::Service: Manual job mode (no auto-scheduling)"
-          Rails.logger.info "  Set up cron jobs to run:"
-          Rails.logger.info "    - rails solid_log:parse_logs (every 10 seconds recommended)"
-          Rails.logger.info "    - rails solid_log:cache_cleanup (hourly recommended)"
-          Rails.logger.info "    - rails solid_log:retention (daily recommended)"
-          Rails.logger.info "    - rails solid_log:field_analysis (daily recommended)"
+          SolidLog::Service.logger.info "SolidLog::Service: Manual job mode (no auto-scheduling)"
+          SolidLog::Service.logger.info "  Set up cron jobs to run:"
+          SolidLog::Service.logger.info "    - rails solid_log:parse_logs (every 10 seconds recommended)"
+          SolidLog::Service.logger.info "    - rails solid_log:cache_cleanup (hourly recommended)"
+          SolidLog::Service.logger.info "    - rails solid_log:retention (daily recommended)"
+          SolidLog::Service.logger.info "    - rails solid_log:field_analysis (daily recommended)"
         end
       end
     end

data/lib/solid_log/service/rack_app.rb

@@ -0,0 +1,382 @@
+require "rack"
+require "json"
+
+module SolidLog
+  module Service
+    class RackApp
+      def call(env)
+        request = Rack::Request.new(env)
+        method = request.request_method
+        path = request.path_info
+
+        # Route matching
+        route(method, path, request)
+      rescue JSON::ParserError => e
+        bad_request("Invalid JSON: #{e.message}")
+      rescue ActiveRecord::RecordInvalid => e
+        unprocessable_entity("Validation error", e.record.errors.full_messages)
+      rescue => e
+        internal_error(e)
+      end
+
+      private
+
+      # Router - matches HTTP method and path to handler using pattern matching
+      def route(method, path, request)
+        # Split path into segments for easier matching
+        segments = path.split("/").reject(&:empty?)
+
+        # Pattern match on [method, segments]
+        case [method, segments]
+        # POST routes
+        in ["POST", ["api", "v1", "ingest"]]
+          handle_ingest(request)
+        in ["POST", ["api", "v1", "search"]]
+          handle_search(request)
+
+        # GET routes - static
+        in ["GET", ["api", "v1", "entries"]]
+          handle_entries_index(request)
+        in ["GET", ["api", "v1", "facets"]]
+          handle_facets(request)
+        in ["GET", ["api", "v1", "facets", "all"]]
+          handle_facets_all(request)
+        in ["GET", ["health"]] | ["GET", ["api", "v1", "health"]]
+          handle_health(request)
+        in ["GET", ["cable"]]
+          ActionCable.server.call(request.env)
+
+        # GET routes - with parameters
+        in ["GET", ["api", "v1", "entries", id]]
+          handle_entries_show(request, id)
+        in ["GET", ["api", "v1", "timeline", "request", request_id]]
+          handle_timeline_request(request, request_id)
+        in ["GET", ["api", "v1", "timeline", "job", job_id]]
+          handle_timeline_job(request, job_id)
+
+        else
+          not_found
+        end
+      end
+
+      # POST /api/v1/ingest
+      def handle_ingest(request)
+        token = authenticate!(request)
+        return token unless token.is_a?(SolidLog::Token)
+
+        payload = parse_ingest_payload(request)
+
+        if payload.blank?
+          return bad_request("Empty payload")
+        end
+
+        entries = Array.wrap(payload)
+
+        max = SolidLog.configuration.max_batch_size
+        if entries.size > max
+          return response(413, {
+            error: "Batch too large",
+            max_size: max,
+            received: entries.size
+          })
+        end
+
+        # Create raw entries
+        raw_entries = entries.map do |entry|
+          {
+            token_id: token.id,
+            payload: entry.to_json,
+            received_at: Time.current,
+            parsed: false
+          }
+        end
+
+        # Bulk insert
+        SolidLog.without_logging do
+          SolidLog::RawEntry.insert_all(raw_entries)
+        end
+
+        token.touch_last_used!
+
+        response(202, {
+          status: "accepted",
+          count: entries.size,
+          message: "Log entries queued for processing"
+        })
+      end
+
+      # GET /api/v1/entries
+      def handle_entries_index(request)
+        token = authenticate!(request)
+        return token unless token.is_a?(SolidLog::Token)
+
+        search_params = build_filter_params(request)
+        search_service = SolidLog::SearchService.new(search_params)
+        entries = search_service.search
+
+        token.touch_last_used!
+
+        response(200, {
+          entries: entries.as_json(methods: [:extra_fields_hash]),
+          total: entries.count,
+          limit: request.params["limit"]&.to_i || 100
+        })
+      end
+
+      # GET /api/v1/entries/:id
+      def handle_entries_show(request, id)
+        token = authenticate!(request)
+        return token unless token.is_a?(SolidLog::Token)
+
+        entry = SolidLog::Entry.find(id)
+        token.touch_last_used!
+
+        response(200, {
+          entry: entry.as_json(methods: [:extra_fields_hash])
+        })
+      rescue ActiveRecord::RecordNotFound
+        not_found("Entry not found")
+      end
+
+      # GET /api/v1/facets?field=level or GET /api/v1/facets (returns all)
+      def handle_facets(request)
+        token = authenticate!(request)
+        return token unless token.is_a?(SolidLog::Token)
+
+        field = request.params["field"]
+
+        # If no field parameter, return all facets (same as /facets/all)
+        if field.blank?
+          return handle_facets_all(request)
+        end
+
+        limit = request.params["limit"]&.to_i || 100
+        facets = SolidLog::Entry.facets_for(field, limit: limit)
+
+        token.touch_last_used!
+
+        response(200, {
+          field: field,
+          values: facets,
+          total: facets.size
+        })
+      end
+
+      # GET /api/v1/facets/all
+      def handle_facets_all(request)
+        token = authenticate!(request)
+        return token unless token.is_a?(SolidLog::Token)
+
+        facets = {
+          level: SolidLog::Entry.facets_for("level"),
+          app: SolidLog::Entry.facets_for("app"),
+          env: SolidLog::Entry.facets_for("env"),
+          controller: SolidLog::Entry.facets_for("controller", limit: 50),
+          action: SolidLog::Entry.facets_for("action", limit: 50),
+          method: SolidLog::Entry.facets_for("method"),
+          status_code: SolidLog::Entry.facets_for("status_code")
+        }
+
+        token.touch_last_used!
+
+        response(200, { facets: facets })
+      end
+
+      # POST /api/v1/search
+      def handle_search(request)
+        token = authenticate!(request)
+        return token unless token.is_a?(SolidLog::Token)
+
+        # Parse JSON body
+        body = request.body.read
+        params = body.present? ? JSON.parse(body) : {}
+
+        query = params["q"] || params["query"] || request.params["q"] || request.params["query"]
+        if query.blank?
+          return bad_request("Query parameter required")
+        end
+
+        search_params = {
+          query: query,
+          limit: params["limit"] || request.params["limit"]
+        }.compact
+
+        search_service = SolidLog::SearchService.new(search_params)
+        entries = search_service.search
+
+        token.touch_last_used!
+
+        response(200, {
+          query: query,
+          entries: entries.as_json(methods: [:extra_fields_hash]),
+          total: entries.count,
+          limit: (params["limit"] || request.params["limit"])&.to_i || 100
+        })
+      end
+
+      # GET /api/v1/timelines/request/:request_id
+      def handle_timeline_request(request, request_id)
+        token = authenticate!(request)
+        return token unless token.is_a?(SolidLog::Token)
+
+        if request_id.blank?
+          return bad_request("Request ID required")
+        end
+
+        entries = SolidLog::CorrelationService.request_timeline(request_id)
+        stats = SolidLog::CorrelationService.request_stats(request_id)
+
+        token.touch_last_used!
+
+        response(200, {
+          request_id: request_id,
+          entries: entries.as_json(methods: [:extra_fields_hash]),
+          stats: stats
+        })
+      end
+
+      # GET /api/v1/timelines/job/:job_id
+      def handle_timeline_job(request, job_id)
+        token = authenticate!(request)
+        return token unless token.is_a?(SolidLog::Token)
+
+        if job_id.blank?
+          return bad_request("Job ID required")
+        end
+
+        entries = SolidLog::CorrelationService.job_timeline(job_id)
+        stats = SolidLog::CorrelationService.job_stats(job_id)
+
+        token.touch_last_used!
+
+        response(200, {
+          job_id: job_id,
+          entries: entries.as_json(methods: [:extra_fields_hash]),
+          stats: stats
+        })
+      end
+
+      # GET /health or /api/v1/health (no authentication)
+      def handle_health(request)
+        metrics = SolidLog::HealthService.metrics
+
+        status_code = case metrics[:parsing][:health_status]
+        when "critical"
+          503
+        when "warning", "degraded"
+          200
+        else
+          200
+        end
+
+        response(status_code, {
+          status: metrics[:parsing][:health_status],
+          timestamp: Time.current.iso8601,
+          metrics: metrics
+        })
+      end
+
+      # Authentication
+      def authenticate!(request)
+        header = request.get_header("HTTP_AUTHORIZATION")
+        unless header&.match?(/\A(Bearer|bearer) /)
+          return unauthorized("Missing or invalid Authorization header")
+        end
+
+        token_value = header.sub(/\A(Bearer|bearer) /, "")
+        token = SolidLog::Token.authenticate(token_value)
+
+        unless token
+          return unauthorized("Invalid token")
+        end
+
+        token
+      end
+
+      # Parse ingest payload (supports JSON, JSON array, and NDJSON)
+      def parse_ingest_payload(request)
+        # Check for _json param (Rails-style JSON array parsing)
+        return request.params["_json"] if request.params["_json"]
+
+        body = request.body.read
+        return [] if body.blank?
+
+        # Check if it's NDJSON (multiple lines) or regular JSON
+        if body.include?("\n")
+          # NDJSON format
+          body.lines.map do |line|
+            JSON.parse(line.strip) unless line.strip.empty?
+          end.compact
+        else
+          # Regular JSON (single entry or array)
+          JSON.parse(body)
+        end
+      end
+
+      # Build filter params from request
+      def build_filter_params(request)
+        params = request.params
+        search_params = {}
+
+        # Handle filters hash if present
+        if params["filters"].is_a?(Hash)
+          filters = params["filters"]
+          search_params[:levels] = [filters["level"]].compact if filters["level"].to_s.present?
+          search_params[:app] = filters["app"] if filters["app"].to_s.present?
+          search_params[:env] = filters["env"] if filters["env"].to_s.present?
+          search_params[:controller] = filters["controller"] if filters["controller"].to_s.present?
+          search_params[:action] = filters["action"] if filters["action"].to_s.present?
+          search_params[:path] = filters["path"] if filters["path"].to_s.present?
+          search_params[:method] = filters["method"] if filters["method"].to_s.present?
+          search_params[:status_code] = filters["status_code"] if filters["status_code"].to_s.present?
+          search_params[:start_time] = filters["start_time"] if filters["start_time"].to_s.present?
+          search_params[:end_time] = filters["end_time"] if filters["end_time"].to_s.present?
+          search_params[:min_duration] = filters["min_duration"] if filters["min_duration"].to_s.present?
+          search_params[:max_duration] = filters["max_duration"] if filters["max_duration"].to_s.present?
+        end
+
+        search_params[:query] = params["q"] if params["q"].to_s.present?
+        search_params[:limit] = params["limit"] if params["limit"].to_s.present?
+
+        search_params
+      end
+
+      # Response helpers
+      def response(status, data)
+        [status, json_headers, [JSON.generate(data)]]
+      end
+
+      def unauthorized(message = "Unauthorized")
+        [401, json_headers, [JSON.generate({ error: message })]]
+      end
+
+      def bad_request(message)
+        [400, json_headers, [JSON.generate({ error: message })]]
+      end
+
+      def not_found(message = "Not found")
+        [404, json_headers, [JSON.generate({ error: message })]]
+      end
+
+      def unprocessable_entity(error, details = nil)
+        data = { error: error }
+        data[:details] = details if details
+        [422, json_headers, [JSON.generate(data)]]
+      end
+
+      def internal_error(exception)
+        SolidLog::Service.logger.error "SolidLog API Error: #{exception.message}"
+        SolidLog::Service.logger.error exception.backtrace.join("\n")
+
+        [500, json_headers, [JSON.generate({
+          error: "Internal server error",
+          message: exception.message
+        })]]
+      end
+
+      def json_headers
+        { "Content-Type" => "application/json" }
+      end
+    end
+  end
+end

data/lib/solid_log/service/scheduler.rb

@@ -20,7 +20,7 @@ module SolidLog
           @running = true
         end
 
-        Rails.logger.info "SolidLog::Service::Scheduler starting..."
+        SolidLog::Service.logger.info "SolidLog::Service::Scheduler starting..."
 
         # Parser job - frequent (configurable, default 10s)
         thread = Thread.new { parser_loop }
@@ -37,13 +37,13 @@ module SolidLog
         thread.abort_on_exception = true
         @threads << thread
 
-        Rails.logger.info "SolidLog::Service::Scheduler started with #{@threads.size} threads"
+        SolidLog::Service.logger.info "SolidLog::Service::Scheduler started with #{@threads.size} threads"
       end
 
       def stop
         return unless @running
 
-        Rails.logger.info "SolidLog::Service::Scheduler stopping..."
+        SolidLog::Service.logger.info "SolidLog::Service::Scheduler stopping..."
 
         @mutex.synchronize do
           @running = false
@@ -56,7 +56,7 @@ module SolidLog
         @threads.each { |t| t.kill if t.alive? }
         @threads.clear
 
-        Rails.logger.info "SolidLog::Service::Scheduler stopped"
+        SolidLog::Service.logger.info "SolidLog::Service::Scheduler stopped"
       end
 
       def running?
@@ -70,10 +70,11 @@ module SolidLog
           break unless @running
 
           begin
-            SolidLog::ParserJob.perform_now
+            # Use core's ParseJob (works with or without ActiveJob)
+            SolidLog::Core::Jobs::ParseJob.perform_now(batch_size: @config.parser_batch_size)
           rescue => e
-            Rails.logger.error "SolidLog::Scheduler: Parser job failed: #{e.message}"
-            Rails.logger.error e.backtrace.join("\n")
+            SolidLog::Service.logger.error "SolidLog::Scheduler: Parser job failed: #{e.message}"
+            SolidLog::Service.logger.error e.backtrace.join("\n")
           end
 
           sleep @config.parser_interval
@@ -85,10 +86,11 @@ module SolidLog
           break unless @running
 
           begin
-            SolidLog::CacheCleanupJob.perform_now
+            # Use core's CacheCleanupJob (works with or without ActiveJob)
+            SolidLog::Core::Jobs::CacheCleanupJob.perform_now
           rescue => e
-            Rails.logger.error "SolidLog::Scheduler: Cache cleanup failed: #{e.message}"
-            Rails.logger.error e.backtrace.join("\n")
+            SolidLog::Service.logger.error "SolidLog::Scheduler: Cache cleanup failed: #{e.message}"
+            SolidLog::Service.logger.error e.backtrace.join("\n")
           end
 
           sleep @config.cache_cleanup_interval
@@ -122,22 +124,25 @@ module SolidLog
         case job_name
         when :retention
           begin
-            SolidLog::RetentionJob.perform_now(
+            # Use core's RetentionJob (works with or without ActiveJob)
+            SolidLog::Core::Jobs::RetentionJob.perform_now(
               retention_days: @config.retention_days,
-              error_retention_days: @config.error_retention_days
+              error_retention_days: @config.error_retention_days,
+              max_entries: SolidLog.configuration.max_entries
             )
           rescue => e
-            Rails.logger.error "SolidLog::Scheduler: Retention job failed: #{e.message}"
-            Rails.logger.error e.backtrace.join("\n")
+            SolidLog::Service.logger.error "SolidLog::Scheduler: Retention job failed: #{e.message}"
+            SolidLog::Service.logger.error e.backtrace.join("\n")
           end
         when :field_analysis
           begin
-            SolidLog::FieldAnalysisJob.perform_now(
+            # Use core's FieldAnalysisJob (works with or without ActiveJob)
+            SolidLog::Core::Jobs::FieldAnalysisJob.perform_now(
               auto_promote: @config.auto_promote_fields
             )
           rescue => e
-            Rails.logger.error "SolidLog::Scheduler: Field analysis failed: #{e.message}"
-            Rails.logger.error e.backtrace.join("\n")
+            SolidLog::Service.logger.error "SolidLog::Scheduler: Field analysis failed: #{e.message}"
+            SolidLog::Service.logger.error e.backtrace.join("\n")
           end
         end
       end

data/lib/solid_log/service/version.rb

@@ -1,5 +1,5 @@
 module SolidLog
   module Service
-    VERSION = "0.1.0"
+    VERSION = "0.2.1"
   end
 end

data/lib/solid_log/service.rb

@@ -3,7 +3,7 @@ require_relative "service/version"
 require_relative "service/configuration"
 require_relative "service/scheduler"
 require_relative "service/job_processor"
-require_relative "service/engine"
+require_relative "service/rack_app"
 
 module SolidLog
   module Service
@@ -23,8 +23,25 @@ module SolidLog
         @configuration = Configuration.new
       end
 
+      # Logger - delegates to SolidLog.logger (from core)
+      def logger
+        SolidLog.logger
+      end
+
+      def logger=(logger)
+        SolidLog.logger = logger
+      end
+
       # Start the service (job processor)
       def start!
+        # Configure core logger if not already set
+        unless SolidLog.logger
+          require "logger"
+          SolidLog.logger = Logger.new(STDOUT).tap do |log|
+            log.level = ENV.fetch("LOG_LEVEL", "info").to_sym
+          end
+        end
+
         JobProcessor.setup
       end