solid_log-service 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4c94d0fde51404dd97f27e8d437a9c9edab5366ba360bbd2719b0a658a28c5fe
+  data.tar.gz: 1addb2c0e3b2471fdadccbd3741905747679b1e6475e3f6a12f6a5f3a44f5193
+SHA512:
+  metadata.gz: 8e3ab660ee56208d357bc6bf5baf5e81e89137df79b9fe5849fd8d3567934529335a77c264ae6de75432d07c3b1e1611827d2aa4c61fbba4c7b98edeebaccc72
+  data.tar.gz: 7edb569df3716558642f22c9a55d680eb2f022ff6832a8364ead35824959b1354873dab3caa2b6039c5aa5430545c005ca480dbc1f26bafc5ad4c343c79bc7f6

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright Dan Loman
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,112 @@
+# SolidLog::Service
+
+Standalone log ingestion and processing service with HTTP API and built-in job scheduler.
+
+## Overview
+
+`solid_log-service` provides:
+
+- **HTTP Ingestion API**: Accept logs via POST with bearer token auth
+- **Query APIs**: REST endpoints for searching, filtering, and retrieving logs
+- **Background Processing**: Parse raw logs, retention cleanup, field analysis
+- **Built-in Scheduler**: No external dependencies (or use ActiveJob/cron)
+- **Health Monitoring**: Metrics endpoint for observability
+
+## Installation
+
+```ruby
+gem 'solid_log-service'
+
+# Database adapter (choose one)
+gem 'sqlite3', '>= 2.1'   # For SQLite (recommended for most deployments)
+# OR
+gem 'pg', '>= 1.1'        # For PostgreSQL
+# OR
+gem 'mysql2', '>= 0.5'    # For MySQL
+```
+
+## Standalone Deployment
+
+**1. Create configuration file:**
+
+```ruby
+# config/solid_log_service.rb
+SolidLog::Service.configure do |config|
+  config.database_url = ENV['DATABASE_URL'] || 'sqlite3:///data/production_log.sqlite'
+
+  # Job processing mode (default: :scheduler)
+  config.job_mode = :scheduler  # or :active_job, :manual
+
+  # Scheduler intervals (only used when job_mode = :scheduler)
+  config.parser_interval = 10.seconds
+  config.cache_cleanup_interval = 1.hour
+  config.retention_hour = 2  # Run at 2 AM
+  config.field_analysis_hour = 3  # Run at 3 AM
+
+  # Retention policies
+  config.retention_days = 30
+  config.error_retention_days = 90
+end
+```
+
+**2. Run the service:**
+
+```bash
+bundle exec solid_log_service
+```
+
+Or with a Procfile:
+```
+service: bundle exec solid_log_service
+```
+
+## Kamal Deployment
+
+See main SolidLog documentation for Kamal deployment examples.
+
+## API Endpoints
+
+### Ingestion
+- `POST /api/v1/ingest` - Ingest logs (single or batch)
+
+### Queries
+- `GET /api/v1/entries` - List/filter entries
+- `GET /api/v1/entries/:id` - Get single entry
+- `GET /api/v1/search` - Full-text search
+- `GET /api/v1/facets` - Get filter options
+- `GET /api/v1/timelines/request/:id` - Request timeline
+- `GET /api/v1/timelines/job/:id` - Job timeline
+- `GET /api/v1/health` - Health metrics
+
+## Job Processing Modes
+
+### Built-in Scheduler (Default)
+No external dependencies. Runs jobs in background threads.
+
+```ruby
+config.job_mode = :scheduler
+config.parser_interval = 10.seconds
+```
+
+### ActiveJob Integration
+Leverages host app's job backend (Solid Queue, Sidekiq, etc.)
+
+```ruby
+config.job_mode = :active_job
+```
+
+### Manual (Cron)
+You manage scheduling via cron.
+
+```ruby
+config.job_mode = :manual
+```
+
+Then in crontab:
+```
+*/1 * * * * cd /app && rails solid_log:parse_logs
+```
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+end
+
+task default: :test

data/app/controllers/solid_log/api/base_controller.rb

@@ -0,0 +1,80 @@
+module SolidLog
+  module Api
+    class BaseController < ActionController::API
+      before_action :authenticate_token!
+
+      rescue_from ActionDispatch::Http::Parameters::ParseError do |exception|
+        render json: {
+          error: "Invalid JSON",
+          message: exception.message
+        }, status: :unprocessable_entity
+      end
+
+      rescue_from ActionController::BadRequest do |exception|
+        render json: {
+          error: "Invalid JSON",
+          message: exception.message
+        }, status: :unprocessable_entity
+      end
+
+      rescue_from StandardError do |exception|
+        # Check if it's a parameter parsing error based on message
+        if exception.message.include?("parsing request parameters")
+          render json: {
+            error: "Invalid JSON",
+            message: exception.message
+          }, status: :unprocessable_entity
+        else
+          Rails.logger.error "SolidLog API Error: #{exception.message}"
+          Rails.logger.error exception.backtrace.join("\n")
+
+          render json: {
+            error: "Internal server error",
+            message: exception.message
+          }, status: :internal_server_error
+        end
+      end
+
+      rescue_from ActiveRecord::RecordInvalid do |exception|
+        render json: {
+          error: "Validation error",
+          details: exception.record.errors.full_messages
+        }, status: :unprocessable_entity
+      end
+
+      private
+
+      def authenticate_token!
+        token_value = extract_bearer_token
+
+        unless token_value
+          render json: { error: "Missing or invalid Authorization header" }, status: :unauthorized
+          return
+        end
+
+        @current_token = SolidLog::Token.authenticate(token_value)
+
+        unless @current_token
+          render json: { error: "Invalid token" }, status: :unauthorized
+          return
+        end
+
+        # Touch last_used_at timestamp
+        @current_token.touch_last_used!
+      end
+
+      def current_token
+        @current_token
+      end
+
+      def extract_bearer_token
+        header = request.headers["Authorization"]
+        return nil unless header
+
+        # Expected format: "Bearer <token>"
+        matches = header.match(/^Bearer (.+)$/i)
+        matches[1] if matches
+      end
+    end
+  end
+end

data/app/controllers/solid_log/api/v1/entries_controller.rb

@@ -0,0 +1,55 @@
+module SolidLog
+  module Api
+    module V1
+      class EntriesController < Api::BaseController
+        # GET /api/v1/entries
+        def index
+          search_service = SolidLog::SearchService.new(filter_params)
+          entries = search_service.search
+
+          render json: {
+            entries: entries.as_json(methods: [:extra_fields_hash]),
+            total: entries.count,
+            limit: params[:limit]&.to_i || 100
+          }
+        end
+
+        # GET /api/v1/entries/:id
+        def show
+          entry = Entry.find(params[:id])
+
+          render json: {
+            entry: entry.as_json(methods: [:extra_fields_hash])
+          }
+        rescue ActiveRecord::RecordNotFound
+          render json: { error: "Entry not found" }, status: :not_found
+        end
+
+        private
+
+        def filter_params
+          # Extract filters from params[:filters] to avoid Rails routing params collision
+          search_params = {}
+          filters = params[:filters] || {}
+
+          search_params[:levels] = [filters[:level]].compact if filters[:level].present?
+          search_params[:app] = filters[:app] if filters[:app].present?
+          search_params[:env] = filters[:env] if filters[:env].present?
+          search_params[:controller] = filters[:controller] if filters[:controller].present?
+          search_params[:action] = filters[:action] if filters[:action].present?
+          search_params[:path] = filters[:path] if filters[:path].present?
+          search_params[:method] = filters[:method] if filters[:method].present?
+          search_params[:status_code] = filters[:status_code] if filters[:status_code].present?
+          search_params[:start_time] = filters[:start_time] if filters[:start_time].present?
+          search_params[:end_time] = filters[:end_time] if filters[:end_time].present?
+          search_params[:min_duration] = filters[:min_duration] if filters[:min_duration].present?
+          search_params[:max_duration] = filters[:max_duration] if filters[:max_duration].present?
+          search_params[:query] = params[:q] if params[:q].present?
+          search_params[:limit] = params[:limit] if params[:limit].present?
+
+          search_params
+        end
+      end
+    end
+  end
+end

data/app/controllers/solid_log/api/v1/facets_controller.rb

@@ -0,0 +1,41 @@
+module SolidLog
+  module Api
+    module V1
+      class FacetsController < Api::BaseController
+        # GET /api/v1/facets
+        def index
+          field = params[:field]
+
+          if field.blank?
+            return render json: { error: "Field parameter required" }, status: :bad_request
+          end
+
+          # Use Entry model directly for facets
+          limit = params[:limit]&.to_i || 100
+          facets = SolidLog::Entry.facets_for(field, limit: limit)
+
+          render json: {
+            field: field,
+            values: facets,
+            total: facets.size
+          }
+        end
+
+        # GET /api/v1/facets/all
+        def all
+          facets = {
+            level: SolidLog::Entry.facets_for("level"),
+            app: SolidLog::Entry.facets_for("app"),
+            env: SolidLog::Entry.facets_for("env"),
+            controller: SolidLog::Entry.facets_for("controller", limit: 50),
+            action: SolidLog::Entry.facets_for("action", limit: 50),
+            method: SolidLog::Entry.facets_for("method"),
+            status_code: SolidLog::Entry.facets_for("status_code")
+          }
+
+          render json: { facets: facets }
+        end
+      end
+    end
+  end
+end

data/app/controllers/solid_log/api/v1/health_controller.rb

@@ -0,0 +1,29 @@
+module SolidLog
+  module Api
+    module V1
+      class HealthController < Api::BaseController
+        skip_before_action :authenticate_token!, only: [:show]
+
+        # GET /api/v1/health
+        def show
+          metrics = SolidLog::HealthService.metrics
+
+          status = case metrics[:parsing][:health_status]
+          when "critical"
+                    :service_unavailable
+          when "warning", "degraded"
+                    :ok  # Still functional
+          else
+                    :ok
+          end
+
+          render json: {
+            status: metrics[:parsing][:health_status],
+            timestamp: Time.current.iso8601,
+            metrics: metrics
+          }, status: status
+        end
+      end
+    end
+  end
+end

data/app/controllers/solid_log/api/v1/ingest_controller.rb

@@ -0,0 +1,80 @@
+module SolidLog
+  module Api
+    module V1
+      class IngestController < Api::BaseController
+        # POST /api/v1/ingest
+        # Accepts single log entry (hash) or batch (array of hashes)
+        def create
+          payload = params[:_json] || parse_ndjson_body
+
+          if payload.blank?
+            render json: { error: "Empty payload" }, status: :bad_request
+            return
+          end
+
+          entries = Array.wrap(payload)
+
+          if entries.size > max_batch_size
+            render json: {
+              error: "Batch too large",
+              max_size: max_batch_size,
+              received: entries.size
+            }, status: :payload_too_large
+            return
+          end
+
+          # Create raw entries
+          raw_entries = entries.map do |entry|
+            {
+              token_id: current_token.id,
+              payload: entry.to_json,
+              received_at: Time.current,
+              parsed: false
+            }
+          end
+
+          # Bulk insert
+          SolidLog.without_logging do
+            SolidLog::RawEntry.insert_all(raw_entries)
+          end
+
+          render json: {
+            status: "accepted",
+            count: entries.size,
+            message: "Log entries queued for processing"
+          }, status: :accepted
+        rescue JSON::ParserError => e
+          render json: {
+            error: "Invalid JSON",
+            message: e.message
+          }, status: :bad_request
+        end
+
+        private
+
+        def max_batch_size
+          SolidLog.configuration.max_batch_size
+        end
+
+        # Parse NDJSON (newline-delimited JSON) from request body
+        def parse_ndjson_body
+          return [] unless request.body
+
+          body = request.body.read
+          return [] if body.blank?
+
+          # Check if it's NDJSON (multiple lines) or regular JSON
+          if body.include?("\n")
+            # NDJSON format
+            body.lines.map do |line|
+              JSON.parse(line.strip) unless line.strip.empty?
+            end.compact
+          else
+            # Regular JSON (single entry or array)
+            JSON.parse(body)
+          end
+        end
+      end
+    end
+  end
+end

data/app/controllers/solid_log/api/v1/search_controller.rb

@@ -0,0 +1,31 @@
+module SolidLog
+  module Api
+    module V1
+      class SearchController < Api::BaseController
+        # POST /api/v1/search
+        def create
+          query = params[:q] || params[:query]
+
+          if query.blank?
+            return render json: { error: "Query parameter required" }, status: :bad_request
+          end
+
+          search_params = {
+            query: query,
+            limit: params[:limit]
+          }.compact
+
+          search_service = SolidLog::SearchService.new(search_params)
+          entries = search_service.search
+
+          render json: {
+            query: query,
+            entries: entries.as_json(methods: [:extra_fields_hash]),
+            total: entries.count,
+            limit: params[:limit]&.to_i || 100
+          }
+        end
+      end
+    end
+  end
+end

data/app/controllers/solid_log/api/v1/timelines_controller.rb

@@ -0,0 +1,43 @@
+module SolidLog
+  module Api
+    module V1
+      class TimelinesController < Api::BaseController
+        # GET /api/v1/timelines/request/:request_id
+        def show_request
+          request_id = params[:request_id]
+
+          if request_id.blank?
+            return render json: { error: "Request ID required" }, status: :bad_request
+          end
+
+          entries = SolidLog::CorrelationService.request_timeline(request_id)
+          stats = SolidLog::CorrelationService.request_stats(request_id)
+
+          render json: {
+            request_id: request_id,
+            entries: entries.as_json(methods: [:extra_fields_hash]),
+            stats: stats
+          }
+        end
+
+        # GET /api/v1/timelines/job/:job_id
+        def show_job
+          job_id = params[:job_id]
+
+          if job_id.blank?
+            return render json: { error: "Job ID required" }, status: :bad_request
+          end
+
+          entries = SolidLog::CorrelationService.job_timeline(job_id)
+          stats = SolidLog::CorrelationService.job_stats(job_id)
+
+          render json: {
+            job_id: job_id,
+            entries: entries.as_json(methods: [:extra_fields_hash]),
+            stats: stats
+          }
+        end
+      end
+    end
+  end
+end

data/app/jobs/solid_log/application_job.rb

@@ -0,0 +1,4 @@
+module SolidLog
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/solid_log/cache_cleanup_job.rb

@@ -0,0 +1,16 @@
+module SolidLog
+  class CacheCleanupJob < ApplicationJob
+    queue_as :default
+
+    def perform
+      SolidLog.without_logging do
+        expired_count = FacetCache.expired.count
+
+        if expired_count > 0
+          FacetCache.cleanup_expired!
+          Rails.logger.info "SolidLog::CacheCleanupJob: Cleaned up #{expired_count} expired cache entries"
+        end
+      end
+    end
+  end
+end

data/app/jobs/solid_log/field_analysis_job.rb

@@ -0,0 +1,26 @@
+module SolidLog
+  class FieldAnalysisJob < ApplicationJob
+    queue_as :default
+
+    def perform(auto_promote: false)
+      SolidLog.without_logging do
+        recommendations = FieldAnalyzer.analyze
+
+        if recommendations.any?
+          Rails.logger.info "SolidLog::FieldAnalysisJob: Found #{recommendations.size} fields for potential promotion"
+
+          recommendations.take(10).each do |rec|
+            Rails.logger.info "  - #{rec[:field].name} (#{rec[:field].usage_count} uses, priority: #{rec[:priority]})"
+          end
+
+          if auto_promote
+            promoted_count = FieldAnalyzer.auto_promote_candidates
+            Rails.logger.info "SolidLog::FieldAnalysisJob: Auto-promoted #{promoted_count} fields"
+          end
+        else
+          Rails.logger.info "SolidLog::FieldAnalysisJob: No fields meet promotion threshold"
+        end
+      end
+    end
+  end
+end