solid_log-service 0.1.0 → 0.2.1

data/app/jobs/solid_log/parser_job.rb

@@ -1,130 +0,0 @@
-require "action_cable" if defined?(Rails)
-
-module SolidLog
-  class ParserJob < ApplicationJob
-    queue_as :default
-
-    # Process a batch of unparsed raw entries
-    def perform(batch_size: nil)
-      batch_size ||= SolidLog.configuration.parser_batch_size
-
-      SolidLog.without_logging do
-        # Claim a batch of unparsed entries
-        raw_entries = RawEntry.claim_batch(batch_size: batch_size)
-
-        return if raw_entries.empty?
-
-        Rails.logger.info "SolidLog::ParserJob: Processing #{raw_entries.size} raw entries"
-
-        # Process each entry
-        entries_to_insert = []
-        fields_to_track = {}
-
-        raw_entries.each do |raw_entry|
-          begin
-            # Parse the raw payload
-            parsed = Parser.parse(raw_entry.payload)
-
-            # Extract dynamic fields for field registry
-            extra_fields = parsed.delete(:extra_fields) || {}
-            track_fields(fields_to_track, extra_fields)
-
-            # Prepare entry for insertion
-            entry_data = {
-              raw_id: raw_entry.id,
-              timestamp: parsed[:timestamp],
-              created_at: Time.current, # When entry was parsed/created
-              level: parsed[:level],
-              app: parsed[:app],
-              env: parsed[:env],
-              message: parsed[:message],
-              request_id: parsed[:request_id],
-              job_id: parsed[:job_id],
-              duration: parsed[:duration],
-              status_code: parsed[:status_code],
-              controller: parsed[:controller],
-              action: parsed[:action],
-              path: parsed[:path],
-              method: parsed[:method],
-              extra_fields: extra_fields.to_json
-            }
-
-            entries_to_insert << entry_data
-          rescue StandardError => e
-            Rails.logger.error "SolidLog::ParserJob: Failed to parse entry #{raw_entry.id}: #{e.message}"
-            Rails.logger.error e.backtrace.join("\n")
-            # Leave entry unparsed so it can be retried or investigated
-          end
-        end
-
-        # Bulk insert parsed entries
-        if entries_to_insert.any?
-          raw_ids = entries_to_insert.map { |e| e[:raw_id] }
-
-          Entry.insert_all(entries_to_insert)
-          Rails.logger.info "SolidLog::ParserJob: Inserted #{entries_to_insert.size} entries"
-
-          # Broadcast new entry IDs for live tail
-          begin
-            new_entry_ids = Entry.where(raw_id: raw_ids).pluck(:id)
-            if new_entry_ids.any?
-              ActionCable.server.broadcast(
-                "solid_log_new_entries",
-                { entry_ids: new_entry_ids }
-              )
-            end
-          rescue NameError => e
-            Rails.logger.error "SolidLog::ParserJob: ActionCable not available: #{e.message}"
-          rescue => e
-            Rails.logger.error "SolidLog::ParserJob: Failed to broadcast: #{e.class} - #{e.message}"
-          end
-        end
-
-        # Update field registry
-        update_field_registry(fields_to_track)
-      end
-    end
-
-    private
-
-    # Track field occurrences for the registry
-    def track_fields(fields_hash, extra_fields)
-      extra_fields.each do |key, value|
-        fields_hash[key] ||= { values: [], count: 0 }
-        fields_hash[key][:count] += 1
-        fields_hash[key][:type] ||= infer_field_type(value)
-      end
-    end
-
-    # Update the field registry with tracked fields
-    def update_field_registry(fields_hash)
-      fields_hash.each do |name, data|
-        field = Field.find_or_initialize_by(name: name)
-        field.field_type ||= data[:type]
-        field.usage_count += data[:count]
-        field.last_seen_at = Time.current
-        field.save!
-      end
-    end
-
-    # Infer field type from value
-    def infer_field_type(value)
-      case value
-      when String
-        "string"
-      when Numeric
-        "number"
-      when TrueClass, FalseClass
-        "boolean"
-      when Time, DateTime, Date
-        "datetime"
-      when Array
-        "array"
-      when Hash
-        "object"
-      else
-        "string"
-      end
-    end
-  end
-end

data/app/jobs/solid_log/retention_job.rb

@@ -1,24 +0,0 @@
-module SolidLog
-  class RetentionJob < ApplicationJob
-    queue_as :default
-
-    def perform(retention_days: 30, error_retention_days: 90, vacuum: false)
-      SolidLog.without_logging do
-        Rails.logger.info "SolidLog::RetentionJob: Starting cleanup (retention: #{retention_days} days, errors: #{error_retention_days} days)"
-
-        stats = RetentionService.cleanup(
-          retention_days: retention_days,
-          error_retention_days: error_retention_days
-        )
-
-        Rails.logger.info "SolidLog::RetentionJob: Deleted #{stats[:entries_deleted]} entries, #{stats[:raw_deleted]} raw entries, cleared #{stats[:cache_cleared]} cache entries"
-
-        if vacuum
-          Rails.logger.info "SolidLog::RetentionJob: Running VACUUM..."
-          RetentionService.vacuum_database
-          Rails.logger.info "SolidLog::RetentionJob: VACUUM complete"
-        end
-      end
-    end
-  end
-end

data/bin/solid_log_service

@@ -1,75 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'bundler/setup'
-require 'rack'
-require_relative '../lib/solid_log/service'
-
-# Parse command line options
-options = {
-  port: ENV['PORT'] || SolidLog::Service.configuration.port || 3001,
-  bind: ENV['BIND'] || SolidLog::Service.configuration.bind || '0.0.0.0',
-  environment: ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'production'
-}
-
-ARGV.each do |arg|
-  case arg
-  when /^--port=(\d+)$/
-    options[:port] = $1.to_i
-  when /^--bind=(.+)$/
-    options[:bind] = $1
-  when /^--environment=(.+)$/
-    options[:environment] = $1
-  when '--help', '-h'
-    puts <<~HELP
-      SolidLog Service - Standalone log ingestion and processing service
-
-      Usage: solid_log_service [options]
-
-      Options:
-        --port=PORT              Port to bind to (default: 3001)
-        --bind=ADDRESS           Address to bind to (default: 0.0.0.0)
-        --environment=ENV        Environment (default: production)
-        --help, -h               Show this help message
-
-      Environment Variables:
-        PORT                     Port to bind to
-        BIND                     Address to bind to
-        RAILS_ENV                Rails environment
-        DATABASE_URL             Database connection string
-        LOG_LEVEL                Log level (debug, info, warn, error)
-
-      Examples:
-        solid_log_service
-        solid_log_service --port=8080 --bind=127.0.0.1
-        PORT=8080 solid_log_service
-
-      Configuration:
-        Create config/solid_log_service.rb to configure the service.
-        See README.md for configuration options.
-    HELP
-    exit 0
-  end
-end
-
-# Set environment
-ENV['RAILS_ENV'] = options[:environment]
-ENV['RACK_ENV'] = options[:environment]
-
-puts "Starting SolidLog Service..."
-puts "  Environment: #{options[:environment]}"
-puts "  Binding: #{options[:bind]}:#{options[:port]}"
-
-# Load the application
-require_relative '../config.ru'
-
-# Run with Puma
-require 'puma/cli'
-
-puma_args = [
-  '--bind', "tcp://#{options[:bind]}:#{options[:port]}",
-  '--environment', options[:environment],
-  '--workers', ENV.fetch('WEB_CONCURRENCY', '2'),
-  '--threads', "#{ENV.fetch('RAILS_MIN_THREADS', '5')}:#{ENV.fetch('RAILS_MAX_THREADS', '5')}"
-]
-
-Puma::CLI.new(puma_args).run

data/config/routes.rb

@@ -1,26 +0,0 @@
-SolidLog::Service::Engine.routes.draw do
-  namespace :api do
-    namespace :v1 do
-      # Ingestion
-      post "ingest", to: "ingest#create"
-
-      # Queries
-      resources :entries, only: [:index, :show]
-      post "search", to: "search#create"
-
-      # Facets
-      get "facets", to: "facets#index"
-      get "facets/all", to: "facets#all", as: :all_facets
-
-      # Timelines
-      get "timelines/request/:request_id", to: "timelines#show_request", as: :timeline_request
-      get "timelines/job/:job_id", to: "timelines#show_job", as: :timeline_job
-
-      # Health
-      get "health", to: "health#show"
-    end
-  end
-
-  # Root health check
-  get "/health", to: "api/v1/health#show"
-end

data/lib/solid_log/service/application.rb

@@ -1,72 +0,0 @@
-require "rails"
-require "action_controller/railtie"
-require "active_record/railtie"
-require "active_job/railtie"
-require "action_cable/engine"
-
-module SolidLog
-  module Service
-    class Application < Rails::Application
-      config.load_defaults 8.0
-      config.api_only = true
-
-      # Enable caching with memory store
-      config.cache_store = :memory_store
-
-      # Load service configuration
-      config.before_initialize do
-        # Load configuration file if it exists
-        config_file = Rails.root.join("config", "solid_log_service.rb")
-        require config_file if File.exist?(config_file)
-
-        # Load Action Cable configuration
-        cable_config_path = Rails.root.join("config", "cable.yml")
-        if File.exist?(cable_config_path)
-          config.action_cable.cable = YAML.load_file(cable_config_path, aliases: true)[Rails.env]
-        end
-      end
-
-      # Set up database connection
-      config.before_initialize do
-        db_config = {
-          adapter: ENV["DB_ADAPTER"] || "sqlite3",
-          database: ENV["DATABASE_URL"] || Rails.root.join("storage", "production_log.sqlite3").to_s,
-          pool: ENV.fetch("RAILS_MAX_THREADS", 5)
-        }
-
-        ActiveRecord::Base.establish_connection(db_config)
-      end
-
-      # Start job processor after initialization (but not in console mode)
-      config.after_initialize do
-        unless defined?(Rails::Console)
-          SolidLog::Service.start!
-        end
-      end
-
-      # Stop job processor on shutdown
-      at_exit do
-        SolidLog::Service.stop!
-      end
-
-      # Eager load controllers and jobs
-      config.eager_load_paths << Rails.root.join("app", "controllers")
-      config.eager_load_paths << Rails.root.join("app", "jobs")
-
-      # CORS configuration
-      config.middleware.insert_before 0, Rack::Cors do
-        allow do
-          origins { |source, env| SolidLog::Service.configuration.cors_origins.include?(source) || SolidLog::Service.configuration.cors_origins.include?("*") }
-          resource "*",
-            headers: :any,
-            methods: [:get, :post, :put, :patch, :delete, :options, :head],
-            credentials: false
-        end
-      end if defined?(Rack::Cors)
-
-      # Logging
-      config.logger = ActiveSupport::Logger.new(STDOUT)
-      config.log_level = ENV.fetch("LOG_LEVEL", "info").to_sym
-    end
-  end
-end

data/lib/solid_log/service/engine.rb

@@ -1,25 +0,0 @@
-require "rails/engine"
-
-module SolidLog
-  module Service
-    class Engine < ::Rails::Engine
-      isolate_namespace SolidLog
-
-      config.generators do |g|
-        g.test_framework :minitest
-        g.fixture_replacement :factory_bot
-      end
-
-      # Ensure controllers and jobs are autoloaded
-      config.autoload_paths << root.join("app/controllers")
-      config.autoload_paths << root.join("app/jobs")
-
-      # Add SilenceMiddleware to prevent recursive logging
-      # This intercepts all requests to the service and sets Thread.current[:solid_log_silenced]
-      # so the service doesn't log its own API requests, parser jobs, etc.
-      initializer "solid_log_service.add_middleware" do |app|
-        app.middleware.use SolidLog::SilenceMiddleware
-      end
-    end
-  end
-end