solid_log-core 0.1.0

data/lib/solid_log/models/entry.rb

@@ -0,0 +1,185 @@
+module SolidLog
+  class Entry < Record
+    self.table_name = "solid_log_entries"
+
+    belongs_to :raw_entry, foreign_key: :raw_id, optional: true
+
+    # Override timestamp getter to ensure datetime casting for SQLite
+    # (SQLite stores as string, column name doesn't end in _at)
+    def timestamp
+      value = super
+      value.is_a?(String) ? Time.zone.parse(value) : value
+    end
+
+    validates :level, presence: true
+    validates :timestamp, presence: true
+    validates :created_at, presence: true
+
+    LOG_LEVELS = %w[debug info warn error fatal unknown].freeze
+
+    # Scopes for filtering (support both single values and arrays for multi-select)
+    scope :by_level, ->(level) { where(level: level) if level.present? }
+    scope :by_app, ->(app) {
+      return all if app.blank?
+      app.is_a?(Array) ? where(app: Array(app).flatten.reject(&:blank?)) : where(app: app)
+    }
+    scope :by_env, ->(env) {
+      return all if env.blank?
+      env.is_a?(Array) ? where(env: Array(env).flatten.reject(&:blank?)) : where(env: env)
+    }
+    scope :by_controller, ->(controller) {
+      return all if controller.blank?
+      controller.is_a?(Array) ? where(controller: Array(controller).flatten.reject(&:blank?)) : where(controller: controller)
+    }
+    scope :by_action, ->(action) {
+      return all if action.blank?
+      action.is_a?(Array) ? where(action: Array(action).flatten.reject(&:blank?)) : where(action: action)
+    }
+    scope :by_path, ->(path) {
+      return all if path.blank?
+      path.is_a?(Array) ? where(path: Array(path).flatten.reject(&:blank?)) : where(path: path)
+    }
+    scope :by_method, ->(method) {
+      return all if method.blank?
+      method.is_a?(Array) ? where(method: Array(method).flatten.reject(&:blank?)) : where(method: method)
+    }
+    scope :by_status_code, ->(status_code) {
+      return all if status_code.blank?
+      status_code.is_a?(Array) ? where(status_code: Array(status_code).flatten.reject(&:blank?)) : where(status_code: status_code)
+    }
+    scope :by_request_id, ->(request_id) { where(request_id: request_id) if request_id.present? }
+    scope :by_job_id, ->(job_id) { where(job_id: job_id) if job_id.present? }
+    scope :by_time_range, ->(start_time, end_time) {
+      scope = all
+      scope = scope.where("timestamp >= ?", start_time) if start_time.present?
+      scope = scope.where("timestamp <= ?", end_time) if end_time.present?
+      scope
+    }
+    scope :by_duration_range, ->(min_duration, max_duration) {
+      scope = all
+      scope = scope.where("duration >= ?", min_duration) if min_duration.present?
+      scope = scope.where("duration <= ?", max_duration) if max_duration.present?
+      scope
+    }
+    scope :recent, -> { order(timestamp: :asc) } # Ascending for terminal-style view (newest at bottom)
+    scope :errors, -> { where(level: %w[error fatal]) }
+
+    # Full-text search combined with LIKE for partial matching
+    # Delegates to the database adapter to build the appropriate query
+    def self.search_fts(query)
+      return all if query.blank?
+
+      SolidLog.adapter.search(query, all)
+    rescue => e
+      Rails.logger.error("Full-text search error: #{e.message}")
+      # Fall back to simple LIKE
+      adapter = SolidLog.adapter
+      sanitized_query = sanitize_sql_like(query)
+      like_condition = adapter.case_insensitive_like("message", "%#{sanitized_query}%")
+      where(like_condition)
+    end
+
+    # Filter by a dynamic field in extra_fields JSON (database-agnostic)
+    def self.filter_by_field(field_name, field_value)
+      return all if field_name.blank?
+
+      adapter = SolidLog.adapter
+      json_extract = adapter.extract_json_field("extra_fields", field_name)
+
+      # SQLite json_extract returns values with their JSON types
+      # For numeric values, we need to handle both string and number comparisons
+      where("#{json_extract} = ? OR #{json_extract} = ?", field_value.to_s, field_value)
+    rescue => e
+      Rails.logger.error("Field filter error: #{e.message}")
+      all
+    end
+
+    # Get correlation timeline for a request
+    def self.correlation_timeline_for_request(request_id)
+      by_request_id(request_id).recent
+    end
+
+    # Get correlation timeline for a job
+    def self.correlation_timeline_for_job(job_id)
+      by_job_id(job_id).recent
+    end
+
+    # Get available facets for a field
+    def self.facets_for(field, limit: 100)
+      return [] unless column_names.include?(field.to_s)
+
+      # Get distinct values
+      values = distinct.pluck(field).compact
+
+      # Sort and limit
+      sorted_values = case field.to_s
+      when "level"
+        # Sort by severity
+        values.sort_by { |l| LOG_LEVELS.index(l) || 999 }
+      when "status_code"
+        # Sort numerically
+        values.sort
+      when "controller", "action", "path"
+        # Sort these potentially large lists
+        values.sort
+      else
+        # Default: sort alphabetically
+        values.sort
+      end
+
+      # Apply limit to all fields
+      sorted_values.take(limit)
+    end
+
+    # Parse extra_fields JSON
+    def extra_fields_hash
+      return {} if extra_fields.blank?
+      @extra_fields_hash ||= JSON.parse(extra_fields)
+    rescue JSON::ParserError
+      {}
+    end
+
+    # Format log level with color class
+    def level_badge_class
+      case level
+      when "debug"
+        "badge-gray"
+      when "info"
+        "badge-blue"
+      when "warn"
+        "badge-yellow"
+      when "error"
+        "badge-red"
+      when "fatal"
+        "badge-dark-red"
+      else
+        "badge-gray"
+      end
+    end
+
+    # Check if this entry has correlation data
+    def correlated?
+      request_id.present? || job_id.present?
+    end
+
+    # Prevent recursive logging
+    around_save :without_logging_wrapper
+    around_create :without_logging_wrapper
+    around_update :without_logging_wrapper
+    around_destroy :without_logging_wrapper
+
+    def self.destroy_all
+      SolidLog.without_logging { super }
+    end
+
+    def self.delete_all
+      SolidLog.without_logging { super }
+    end
+
+    private
+
+    def without_logging_wrapper
+      SolidLog.without_logging { yield }
+    end
+  end
+end

data/lib/solid_log/models/facet_cache.rb

@@ -0,0 +1,58 @@
+module SolidLog
+  class FacetCache < Record
+    self.table_name = "solid_log_facet_cache"
+
+    validates :key_name, presence: true, uniqueness: true
+    validates :cache_value, presence: true
+
+    scope :expired, -> { where("expires_at < ?", Time.current) }
+    scope :valid, -> { where("expires_at IS NULL OR expires_at >= ?", Time.current) }
+
+    # Fetch from cache or compute and store (thread-safe with database locking)
+    def self.fetch(key, ttl: 5.minutes, &block)
+      # First attempt: check for existing valid cache (no lock for read-heavy workloads)
+      cached = valid.find_by(key_name: key)
+      return JSON.parse(cached.cache_value) if cached
+
+      # Use database-level locking to prevent race condition
+      transaction do
+        # Double-check after acquiring lock (ensures only one thread computes)
+        cached = valid.lock.find_by(key_name: key)
+        return JSON.parse(cached.cache_value) if cached
+
+        # No valid cache exists, compute value
+        value = block.call
+        store(key, value, ttl: ttl)
+        value
+      end
+    end
+
+    # Store a value in the cache
+    def self.store(key, value, ttl: 5.minutes)
+      expires_at = ttl ? Time.current + ttl : nil
+      cache_value = value.to_json
+
+      upsert(
+        { key_name: key, cache_value: cache_value, expires_at: expires_at, updated_at: Time.current },
+        unique_by: :key_name
+      )
+
+      value
+    end
+
+    # Invalidate a specific cache key
+    def self.invalidate(key)
+      find_by(key_name: key)&.destroy
+    end
+
+    # Clear all expired cache entries
+    def self.cleanup_expired!
+      expired.delete_all
+    end
+
+    # Clear all cache entries
+    def self.clear_all!
+      delete_all
+    end
+  end
+end

data/lib/solid_log/models/field.rb

@@ -0,0 +1,100 @@
+module SolidLog
+  class Field < Record
+    self.table_name = "solid_log_fields"
+
+    FIELD_TYPES = %w[string number boolean datetime array object].freeze
+    FILTER_TYPES = %w[multiselect range exact contains tokens].freeze
+
+    # High-cardinality fields that should default to tokens
+    HIGH_CARDINALITY_PATTERNS = %w[user_id session_id ip_address uuid transaction_id].freeze
+
+    validates :name, presence: true, uniqueness: true
+    validates :field_type, presence: true, inclusion: { in: FIELD_TYPES }
+    validates :filter_type, presence: true, inclusion: { in: FILTER_TYPES }
+    validates :usage_count, numericality: { greater_than_or_equal_to: 0 }
+
+    scope :hot_fields, ->(threshold = 1000) { where("usage_count >= ?", threshold).order(usage_count: :desc) }
+    scope :promoted, -> { where(promoted: true) }
+    scope :unpromoted, -> { where(promoted: false) }
+    scope :recently_seen, ->(days = 7) { where("last_seen_at >= ?", days.days.ago) }
+
+    # Increment usage count and update last_seen_at
+    def increment_usage!
+      increment!(:usage_count)
+      touch(:last_seen_at)
+    end
+
+    # Mark field as promoted (has its own column)
+    def promote!
+      update!(promoted: true)
+    end
+
+    # Mark field as unpromoted (stored in JSON)
+    def demote!
+      update!(promoted: false)
+    end
+
+    # Check if field is promotable (high usage and not already promoted)
+    def promotable?(threshold: 1000)
+      !promoted? && usage_count >= threshold
+    end
+
+    # Track a field occurrence
+    def self.track(name, value)
+      field = find_or_initialize_by(name: name)
+      field.field_type ||= infer_type(value)
+      field.filter_type ||= infer_filter_type(field.field_type, name)
+
+      # Save if new record before calling increment_usage!
+      field.save! if field.new_record?
+
+      field.increment_usage!
+      field
+    end
+
+    private
+
+    # Infer field type from value
+    def self.infer_type(value)
+      case value
+      when Time, DateTime, Date
+        "datetime"
+      when TrueClass, FalseClass
+        "boolean"
+      when Numeric
+        "number"
+      when Array
+        "array"
+      when Hash
+        "object"
+      when String
+        # Try to parse as datetime
+        begin
+          Time.parse(value)
+          "datetime"
+        rescue ArgumentError
+          "string"
+        end
+      else
+        "string"
+      end
+    end
+
+    # Infer filter type from field type and name
+    def self.infer_filter_type(field_type, field_name = nil)
+      # Check if field name suggests high cardinality
+      if field_name && HIGH_CARDINALITY_PATTERNS.any? { |pattern| field_name.to_s.include?(pattern) }
+        return "tokens"
+      end
+
+      case field_type
+      when "number", "datetime"
+        "range"
+      when "boolean"
+        "exact"
+      else
+        "multiselect"
+      end
+    end
+  end
+end

data/lib/solid_log/models/raw_entry.rb

@@ -0,0 +1,33 @@
+module SolidLog
+  class RawEntry < Record
+    self.table_name = "solid_log_raw"
+
+    belongs_to :token, foreign_key: :token_id, optional: true
+    has_one :entry, foreign_key: :raw_id, dependent: :destroy
+
+    validates :payload, presence: true
+
+    scope :unparsed, -> { where(parsed: false) }
+    scope :parsed, -> { where(parsed: true) }
+    scope :stale_unparsed, ->(threshold = 1.hour.ago) { unparsed.where("received_at < ?", threshold) }
+    scope :recent, -> { order(received_at: :desc) }
+
+    # Get the parsed payload as a hash
+    def payload_hash
+      @payload_hash ||= JSON.parse(payload)
+    rescue JSON::ParserError => e
+      Rails.logger.error "SolidLog: Failed to parse payload for RawEntry #{id}: #{e.message}"
+      {}
+    end
+
+    # Class method to claim unparsed entries for processing
+    # Returns an array of RawEntry records
+    # Uses database-specific locking strategy
+    def self.claim_batch(batch_size: 100)
+      SolidLog.adapter.claim_batch(batch_size)
+    rescue => e
+      Rails.logger.error "SolidLog: Failed to claim batch: #{e.message}"
+      []
+    end
+  end
+end

data/lib/solid_log/models/record.rb

@@ -0,0 +1,5 @@
+module SolidLog
+  class Record < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/lib/solid_log/models/token.rb

@@ -0,0 +1,61 @@
+require "openssl"
+require "active_support/security_utils"
+
+module SolidLog
+  class Token < Record
+    self.table_name = "solid_log_tokens"
+
+    has_many :raw_entries, foreign_key: :token_id, dependent: :nullify
+
+    validates :name, presence: true
+    validates :token_hash, presence: true, uniqueness: true
+
+    # Generate a new token and return it (only time it's visible)
+    def self.generate!(name)
+      plaintext = "slk_" + SecureRandom.hex(32)
+      token = new(name: name)
+      token.token_hash = hash_token(plaintext)
+      token.save!
+
+      {
+        id: token.id,
+        name: token.name,
+        token: plaintext,
+        created_at: token.created_at
+      }
+    end
+
+    # Authenticate a plaintext token - O(1) database lookup
+    def self.authenticate(plaintext)
+      return nil if plaintext.blank?
+
+      hashed = hash_token(plaintext)
+      find_by(token_hash: hashed)
+    end
+
+    # Authenticate a plaintext token against this token's hash
+    def authenticate(plaintext)
+      return false if plaintext.blank? || token_hash.blank?
+
+      # Use constant-time comparison to prevent timing attacks
+      ActiveSupport::SecurityUtils.secure_compare(
+        self.class.hash_token(plaintext),
+        token_hash
+      )
+    end
+
+    # Touch last_used_at timestamp
+    def touch_last_used!
+      update_column(:last_used_at, Time.current)
+    end
+
+    private
+
+    # Generate deterministic hash using HMAC-SHA256
+    # This allows O(1) database lookups while maintaining security
+    def self.hash_token(plaintext)
+      secret_key = Rails.application.secret_key_base || raise("secret_key_base not configured")
+      OpenSSL::HMAC.hexdigest("SHA256", secret_key, plaintext)
+    end
+  end
+end

data/lib/solid_log/parser.rb

@@ -0,0 +1,179 @@
+module SolidLog
+  class Parser
+    STANDARD_FIELDS = %w[
+      timestamp time occurred_at created_at
+      level severity
+      message msg text
+      request_id
+      job_id
+      duration duration_ms
+      status status_code
+      controller
+      action
+      path
+      method http_method
+      app application
+      env environment
+    ].freeze
+
+    VALID_LEVELS = %w[debug info warn error fatal unknown].freeze
+
+    # Instance method for parsing (used by tests and jobs)
+    def parse(raw_json)
+      return nil if raw_json.blank?
+
+      payload = JSON.parse(raw_json)
+      return nil if payload.blank?
+
+      extract_fields(payload)
+    rescue JSON::ParserError
+      nil
+    end
+
+    # Class method for parsing (for convenience)
+    def self.parse(raw_json)
+      new.parse(raw_json)
+    end
+
+    private
+
+    # Extract standard and dynamic fields from payload
+    def extract_fields(payload)
+      standard = extract_standard_fields(payload)
+      dynamic = extract_dynamic_fields(payload, standard.keys)
+
+      # Return extra_fields as hash (ParserJob will convert to JSON)
+      result = standard.merge(extra_fields: dynamic.empty? ? {} : dynamic)
+
+      result
+    end
+
+    # Extract known/standard fields
+    def extract_standard_fields(payload)
+      fields = {}
+
+      # Timestamp (required) - when the log event occurred
+      # created_at will be set automatically by Rails when the entry is saved
+      fields[:timestamp] = extract_timestamp(payload)
+
+      # Level (required, default to info)
+      fields[:level] = normalize_level(payload["level"] || payload["severity"] || "info")
+
+      # Message
+      fields[:message] = payload["message"] || payload["msg"] || payload["text"]
+
+      # App and env
+      fields[:app] = payload["app"] || payload["application"]
+      fields[:env] = payload["env"] || payload["environment"]
+
+      # Correlation IDs
+      fields[:request_id] = payload["request_id"]
+      fields[:job_id] = payload["job_id"]
+
+      # HTTP/Controller fields
+      fields[:controller] = payload["controller"]
+      fields[:action] = payload["action"]
+      fields[:path] = payload["path"]
+      fields[:method] = payload["method"] || payload["http_method"]
+
+      # Performance fields
+      fields[:duration] = extract_duration(payload)
+      fields[:status_code] = extract_status_code(payload)
+
+      # Remove nil values
+      fields.compact
+    end
+
+    # Extract dynamic fields (anything not in standard fields)
+    def extract_dynamic_fields(payload, standard_keys)
+      dynamic = payload.dup
+
+      # Remove standard fields
+      STANDARD_FIELDS.each { |f| dynamic.delete(f) }
+      standard_keys.each { |k| dynamic.delete(k.to_s) }
+
+      dynamic
+    end
+
+    # Normalize log level to standard values
+    def normalize_level(level)
+      return "info" if level.blank?
+
+      level_str = level.to_s.downcase
+      VALID_LEVELS.include?(level_str) ? level_str : "info"
+    end
+
+    # Extract timestamp from various field names (when the log event occurred)
+    def extract_timestamp(payload)
+      timestamp_fields = %w[timestamp time occurred_at created_at]
+
+      timestamp_fields.each do |field|
+        value = payload[field]
+        next if value.blank?
+
+        begin
+          return Time.parse(value) if value.is_a?(String)
+          # Handle both seconds and milliseconds
+          if value.is_a?(Numeric)
+            return value > 10_000_000_000 ? Time.at(value / 1000.0) : Time.at(value)
+          end
+          return value if value.is_a?(Time) || value.is_a?(DateTime)
+        rescue ArgumentError
+          next
+        end
+      end
+
+      # Fallback to current time
+      Time.current
+    end
+
+    # Extract duration in milliseconds
+    def extract_duration(payload)
+      duration = payload["duration"] || payload["duration_ms"]
+      return nil if duration.blank?
+
+      duration.to_f
+    end
+
+    # Extract HTTP status code
+    def extract_status_code(payload)
+      status = payload["status"] || payload["status_code"]
+      return nil if status.blank?
+
+      status.to_i
+    end
+
+    # Track fields in the registry
+    def track_fields(payload)
+      payload.each do |key, value|
+        next if STANDARD_FIELDS.include?(key)
+
+        SolidLog.without_logging do
+          field = Field.find_or_initialize_by(name: key)
+          field.field_type ||= infer_type(value)
+          field.usage_count ||= 0
+          field.usage_count += 1
+          field.last_seen_at = Time.current
+          field.save if field.changed?
+        end
+      end
+    rescue => e
+      # Silently fail if field tracking has issues
+      Rails.logger.debug("SolidLog: Failed to track fields: #{e.message}") if defined?(Rails)
+    end
+
+    # Infer the type of a value
+    def infer_type(value)
+      case value
+      when TrueClass, FalseClass
+        "boolean"
+      when Integer, Float
+        "number"
+      when Time, DateTime, Date
+        "datetime"
+      else
+        "string"
+      end
+    end
+  end
+end

data/lib/solid_log/silence_middleware.rb

@@ -0,0 +1,34 @@
+module SolidLog
+  class SilenceMiddleware
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      request = Rack::Request.new(env)
+
+      # Check if this is a SolidLog request (UI or API)
+      if solid_log_request?(request)
+        # Set thread-local flag to prevent SolidLog from logging its own requests
+        Thread.current[:solid_log_silenced] = true
+
+        begin
+          @app.call(env)
+        ensure
+          Thread.current[:solid_log_silenced] = nil
+        end
+      else
+        @app.call(env)
+      end
+    end
+
+    private
+
+    def solid_log_request?(request)
+      # Match both UI routes (/admin/logs, /solid_log) and API routes (/api/v1/ingest)
+      request.path.start_with?("/admin/logs") ||
+        request.path.include?("solid_log") ||
+        request.path.start_with?("/api/v1/ingest")
+    end
+  end
+end

data/lib/solid_log-core.rb

@@ -0,0 +1,2 @@
+# Top-level require for solid_log-core gem
+require "solid_log/core"