flare 0.1.0

data/lib/flare/metric_submitter.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "zlib"
+require "stringio"
+require "securerandom"
+require "socket"
+
+module Flare
+  # Submits metrics to the Flare metrics service via HTTP.
+  # Handles gzip compression, retries with exponential backoff, and error handling.
+  class MetricSubmitter
+    SCHEMA_VERSION = "V1"
+    GZIP_ENCODING = "gzip"
+    USER_AGENT = "Flare Ruby/#{Flare::VERSION}"
+
+    # Default timeouts (in seconds)
+    DEFAULT_OPEN_TIMEOUT = 2
+    DEFAULT_READ_TIMEOUT = 5
+    DEFAULT_WRITE_TIMEOUT = 5
+
+    # Max retries before giving up
+    MAX_RETRIES = 3
+
+    class SubmissionError < StandardError
+      attr_reader :request_id, :response_code, :response_body
+
+      def initialize(message, request_id:, response_code: nil, response_body: nil)
+        @request_id = request_id
+        @response_code = response_code
+        @response_body = response_body
+        super(message)
+      end
+    end
+
+    class ClientError < StandardError
+      attr_reader :request_id, :response_code, :response_body
+
+      def initialize(message, request_id:, response_code: nil, response_body: nil)
+        @request_id = request_id
+        @response_code = response_code
+        @response_body = response_body
+        super(message)
+      end
+    end
+
+    attr_reader :endpoint, :api_key, :backoff_policy
+
+    def initialize(endpoint:, api_key:, project: nil, environment: nil, backoff_policy: nil, open_timeout: nil, read_timeout: nil, write_timeout: nil)
+      @endpoint = URI("#{endpoint.to_s.chomp('/')}/api/metrics")
+      @api_key = api_key
+      @project = project || default_project
+      @environment = environment || default_environment
+      @backoff_policy = backoff_policy || BackoffPolicy.new
+      @open_timeout = open_timeout || DEFAULT_OPEN_TIMEOUT
+      @read_timeout = read_timeout || DEFAULT_READ_TIMEOUT
+      @write_timeout = write_timeout || DEFAULT_WRITE_TIMEOUT
+    end
+
+    # Submit drained metrics to the server.
+    # Returns [success_count, error] where error may be nil on success.
+    def submit(drained)
+      return [0, nil] if drained.empty?
+
+      request_id = SecureRandom.uuid
+      Flare.log "Submitting #{drained.size} metrics to #{@endpoint} (request_id=#{request_id})"
+
+      body = build_body(drained, request_id)
+      return [0, nil] if body.nil?
+
+      @backoff_policy.reset
+      response, error = retry_with_backoff(MAX_RETRIES) { post(body, request_id) }
+
+      if error
+        Flare.log "Submission failed: #{error.message} (request_id=#{request_id})"
+        [0, error]
+      else
+        Flare.log "Submission succeeded: #{response.code} (request_id=#{request_id})"
+        [drained.size, nil]
+      end
+    end
+
+    private
+
+    def build_body(drained, request_id)
+      metrics = drained.map do |key, values|
+        {
+          bucket: format_time(key.bucket),
+          namespace: key.namespace,
+          service: key.service,
+          target: key.target || "",
+          operation: key.operation,
+          count: values[:count],
+          sum_ms: values[:sum_ms],
+          error_count: values[:error_count]
+        }
+      end
+
+      payload = {
+        request_id: request_id,
+        schema_version: SCHEMA_VERSION,
+        project: @project,
+        environment: @environment,
+        metrics: metrics
+      }
+
+      gzip(JSON.generate(payload))
+    rescue => e
+      warn "[Flare] Failed to build submission body: #{e.message}"
+      nil
+    end
+
+    def post(body, request_id)
+      http = Net::HTTP.new(@endpoint.host, @endpoint.port)
+      http.use_ssl = @endpoint.scheme == "https"
+      http.open_timeout = @open_timeout
+      http.read_timeout = @read_timeout
+      http.write_timeout = @write_timeout if http.respond_to?(:write_timeout=)
+
+      request_uri = @endpoint.request_uri
+      request = Net::HTTP::Post.new(request_uri == "" ? "/" : request_uri)
+      request["Content-Type"] = "application/json"
+      request["Content-Encoding"] = GZIP_ENCODING
+      request["Authorization"] = "Bearer #{@api_key}"
+      request["User-Agent"] = USER_AGENT
+      request["X-Request-Id"] = request_id
+      request["X-Schema-Version"] = SCHEMA_VERSION
+
+      # Client metadata headers (like Flipper)
+      request["X-Client-Language"] = "ruby"
+      request["X-Client-Language-Version"] = RUBY_VERSION
+      request["X-Client-Platform"] = RUBY_PLATFORM
+      request["X-Client-Pid"] = Process.pid.to_s
+      request["X-Client-Hostname"] = Socket.gethostname rescue "unknown"
+
+      request.body = body
+      response = http.request(request)
+
+      code = response.code.to_i
+
+      # Success
+      if code >= 200 && code < 300
+        return [response, false] # [result, should_retry]
+      end
+
+      # Retriable errors: rate limiting, server errors
+      if code == 429 || code >= 500
+        raise SubmissionError.new(
+          "Retriable error: #{code}",
+          request_id: request_id,
+          response_code: code,
+          response_body: response.body
+        )
+      end
+
+      # Non-retriable client errors (4xx except 429)
+      raise ClientError.new(
+        "Client error: #{code}",
+        request_id: request_id,
+        response_code: code,
+        response_body: response.body
+      )
+    end
+
+    def retry_with_backoff(max_attempts)
+      attempts_remaining = max_attempts
+      last_error = nil
+
+      while attempts_remaining > 0
+        begin
+          result, should_retry = yield
+          return [result, nil] unless should_retry
+        rescue SubmissionError, Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNREFUSED, Errno::ECONNRESET => e
+          last_error = e
+          attempts_remaining -= 1
+
+          if attempts_remaining > 0
+            sleep_time = @backoff_policy.next_interval / 1000.0
+            sleep(sleep_time)
+          end
+          next
+        rescue => e
+          # Unexpected errors - don't retry
+          return [nil, e]
+        end
+      end
+
+      [nil, last_error]
+    end
+
+    def gzip(string)
+      io = StringIO.new
+      io.set_encoding("BINARY")
+      gz = Zlib::GzipWriter.new(io)
+      gz.write(string)
+      gz.close
+      io.string
+    end
+
+    def format_time(time)
+      time.utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+    end
+
+    def default_project
+      if defined?(Rails) && Rails.application
+        Rails.application.class.module_parent_name.underscore rescue "rails_app"
+      else
+        "app"
+      end
+    end
+
+    def default_environment
+      if defined?(Rails)
+        Rails.env.to_s
+      else
+        ENV.fetch("RACK_ENV", "development")
+      end
+    end
+  end
+end

data/lib/flare/source_location.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Flare
+  # Utility for finding the source location (file, line, method) of app code
+  # that triggered a database query or other instrumented operation.
+  module SourceLocation
+    # How many app code lines to capture for the backtrace
+    MAX_TRACE_LINES = 8
+
+    # Patterns to filter out from backtraces (gems, framework code)
+    IGNORE_PATTERNS = [
+      /\/gems\//,
+      /\/ruby\//,
+      /\/rubygems\//,
+      /lib\/active_record/,
+      /lib\/active_support/,
+      /lib\/action_/,
+      /opentelemetry/,
+      /flare/,
+      /<internal:/,
+      /\/bin\//,
+    ].freeze
+
+    module_function
+
+    # Find the first app source location from the current backtrace
+    # Returns a hash with :filepath, :lineno, :function or nil
+    def find
+      backtrace = caller(2, 50)
+      return nil unless backtrace
+
+      # Find the first line that's app code (not gems/framework)
+      app_line = backtrace.find { |line| app_code?(line) }
+      return nil unless app_line
+
+      parse_backtrace_line(app_line)
+    end
+
+    # Find multiple app source locations from the current backtrace
+    # Returns an array of cleaned backtrace lines (up to MAX_TRACE_LINES)
+    def find_trace
+      backtrace = caller(2, 100)
+      return [] unless backtrace
+
+      # Filter to only app code lines
+      app_lines = backtrace.select { |line| app_code?(line) }
+      return [] if app_lines.empty?
+
+      # Clean and format each line, limit to MAX_TRACE_LINES
+      app_lines.first(MAX_TRACE_LINES).map { |line| clean_backtrace_line(line) }
+    end
+
+    # Add source location attributes to a hash (for span attributes)
+    def add_to_attributes(attrs)
+      location = find
+      return attrs unless location
+
+      attrs["code.filepath"] = location[:filepath]
+      attrs["code.lineno"] = location[:lineno]
+      attrs["code.function"] = location[:function] if location[:function]
+
+      # Add full trace as a single string attribute
+      trace = find_trace
+      attrs["code.stacktrace"] = trace.join("\n") if trace.any?
+
+      attrs
+    end
+
+    def app_code?(line)
+      # Must contain /app/ (Rails convention) and not match ignore patterns
+      return false unless line.include?("/app/")
+
+      IGNORE_PATTERNS.none? { |pattern| line.match?(pattern) }
+    end
+
+    def parse_backtrace_line(line)
+      # Parse: /path/to/file.rb:123:in `method_name'
+      if line =~ /\A(.+):(\d+):in [`'](.+)'\z/
+        {
+          filepath: clean_path($1),
+          lineno: $2.to_i,
+          function: $3
+        }
+      elsif line =~ /\A(.+):(\d+)\z/
+        {
+          filepath: clean_path($1),
+          lineno: $2.to_i,
+          function: nil
+        }
+      end
+    end
+
+    def clean_backtrace_line(line)
+      # Parse and reformat: "app/models/user.rb:42:in `find_by_email'"
+      if line =~ /\A(.+):(\d+):in [`'](.+)'\z/
+        "#{clean_path($1)}:#{$2} in `#{$3}'"
+      elsif line =~ /\A(.+):(\d+)\z/
+        "#{clean_path($1)}:#{$2}"
+      else
+        clean_path(line)
+      end
+    end
+
+    def clean_path(path)
+      # Remove Rails.root prefix if present
+      if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+        path.sub(/\A#{Regexp.escape(Rails.root.to_s)}\//, "")
+      else
+        path
+      end
+    end
+  end
+end

data/lib/flare/sqlite_exporter.rb

@@ -0,0 +1,279 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require "json"
+
+module Flare
+  class SQLiteExporter
+    SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
+    FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
+    TIMEOUT = OpenTelemetry::SDK::Trace::Export::TIMEOUT
+
+    # Prune roughly every 100 exports (1% chance per export)
+    PRUNE_PROBABILITY = 0.01
+
+    def initialize(database_path)
+      @database_path = database_path
+      @mutex = Mutex.new
+      @setup = false
+    end
+
+    # Maximum number of retry attempts when the database is busy.
+    # Mirrors ActiveRecord's retry strategy for SQLite.
+    MAX_RETRIES = 3
+
+    def export(span_datas, timeout: nil)
+      setup_database unless @setup
+
+      retries = 0
+      exported = 0
+
+      begin
+        @mutex.synchronize do
+          connection.transaction do
+            span_datas.each do |span_data|
+              next if should_ignore_span?(span_data)
+
+              create_span(span_data)
+              exported += 1
+            end
+          end
+        end
+      rescue ::SQLite3::BusyException
+        retries += 1
+        if retries <= MAX_RETRIES
+          sleep 0.1 * retries
+          retry
+        end
+        warn "[Flare] SQLite export error: database is busy after #{MAX_RETRIES} retries"
+        return FAILURE
+      end
+
+      Flare.log "Exported #{exported} spans to SQLite" if exported > 0
+
+      # Periodically prune old data
+      maybe_prune
+
+      SUCCESS
+    rescue => e
+      warn "[Flare] SQLite export error: #{e.message}"
+      FAILURE
+    end
+
+    def force_flush(timeout: nil)
+      SUCCESS
+    end
+
+    def shutdown(timeout: nil)
+      SUCCESS
+    end
+
+    private
+
+    def maybe_prune
+      return unless rand < PRUNE_PROBABILITY
+
+      Flare.storage.prune(
+        retention_hours: Flare.configuration.retention_hours,
+        max_spans: Flare.configuration.max_spans
+      )
+    rescue => e
+      warn "[Flare] Prune error: #{e.message}"
+    end
+
+    def should_ignore_span?(span_data)
+      span_data.name&.start_with?("Flare::")
+    end
+
+    def create_span(span_data)
+      now = Time.now.iso8601(6)
+
+      sql = <<~SQL
+        INSERT INTO flare_spans (name, kind, span_id, trace_id, parent_span_id, start_timestamp, end_timestamp, total_recorded_links, total_recorded_events, total_recorded_properties, created_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      SQL
+
+      values = [
+        span_data.name,
+        span_data.kind.to_s,
+        span_data.hex_span_id,
+        span_data.hex_trace_id,
+        span_data.hex_parent_span_id,
+        span_data.start_timestamp,
+        span_data.end_timestamp,
+        span_data.total_recorded_links,
+        span_data.total_recorded_events,
+        span_data.total_recorded_attributes,
+        now,
+        now
+      ]
+
+      connection.execute(sql, values)
+      span_record_id = connection.last_insert_row_id
+
+      span_data.events&.each do |span_event|
+        create_event(span_record_id, span_event)
+      end
+
+      create_properties("Flare::Span", span_record_id, span_data.attributes)
+    end
+
+    def create_event(span_record_id, span_event)
+      now = Time.now.iso8601(6)
+      timestamp = span_event.timestamp ? Time.at(span_event.timestamp / 1_000_000_000.0).iso8601(6) : now
+
+      sql = <<~SQL
+        INSERT INTO flare_events (span_id, name, created_at, updated_at)
+        VALUES (?, ?, ?, ?)
+      SQL
+
+      connection.execute(sql, [span_record_id, span_event.name, timestamp, now])
+      event_record_id = connection.last_insert_row_id
+      create_properties("Flare::Event", event_record_id, span_event.attributes)
+    end
+
+    def create_properties(owner_type, owner_id, attributes)
+      return unless attributes
+
+      now = Time.now.iso8601(6)
+
+      sql = <<~SQL
+        INSERT INTO flare_properties (key, value, value_type, owner_type, owner_id, created_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+      SQL
+
+      attributes.each do |key, value|
+        next if value.nil?
+
+        value_type = determine_value_type(value)
+        serialized_value = JSON.generate(value)
+        connection.execute(sql, [key, serialized_value, value_type, owner_type, owner_id, now, now])
+      end
+    end
+
+    def determine_value_type(value)
+      case value
+      when String then 0  # string
+      when Integer then 1 # integer
+      when Float then 2   # float
+      when TrueClass, FalseClass then 3 # boolean
+      when Array then 4   # array
+      else 0              # default to string
+      end
+    end
+
+    def setup_database
+      @mutex.synchronize do
+        return if @setup
+
+        db = connection
+        configure_pragmas(db)
+
+        db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS flare_spans (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL,
+            kind TEXT NOT NULL,
+            span_id TEXT NOT NULL,
+            trace_id TEXT NOT NULL,
+            parent_span_id TEXT,
+            start_timestamp INTEGER NOT NULL,
+            end_timestamp INTEGER NOT NULL,
+            total_recorded_properties INTEGER NOT NULL DEFAULT 0,
+            total_recorded_events INTEGER NOT NULL DEFAULT 0,
+            total_recorded_links INTEGER NOT NULL DEFAULT 0,
+            created_at TEXT NOT NULL,
+            updated_at TEXT NOT NULL
+          )
+        SQL
+
+        db.execute(<<~SQL)
+          CREATE INDEX IF NOT EXISTS idx_spans_span_id ON flare_spans(span_id)
+        SQL
+
+        db.execute(<<~SQL)
+          CREATE INDEX IF NOT EXISTS idx_spans_trace_id ON flare_spans(trace_id)
+        SQL
+
+        db.execute(<<~SQL)
+          CREATE INDEX IF NOT EXISTS idx_spans_parent_span_id ON flare_spans(parent_span_id)
+        SQL
+
+        db.execute(<<~SQL)
+          CREATE INDEX IF NOT EXISTS idx_spans_created_at ON flare_spans(created_at)
+        SQL
+
+        db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS flare_events (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            span_id INTEGER NOT NULL,
+            name TEXT NOT NULL,
+            created_at TEXT NOT NULL,
+            updated_at TEXT NOT NULL,
+            FOREIGN KEY (span_id) REFERENCES flare_spans(id)
+          )
+        SQL
+
+        db.execute(<<~SQL)
+          CREATE INDEX IF NOT EXISTS idx_events_span_id ON flare_events(span_id)
+        SQL
+
+        db.execute(<<~SQL)
+          CREATE TABLE IF NOT EXISTS flare_properties (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            key TEXT NOT NULL,
+            value TEXT,
+            value_type INTEGER NOT NULL DEFAULT 0,
+            owner_type TEXT NOT NULL,
+            owner_id INTEGER NOT NULL,
+            created_at TEXT NOT NULL,
+            updated_at TEXT NOT NULL
+          )
+        SQL
+
+        db.execute(<<~SQL)
+          CREATE INDEX IF NOT EXISTS idx_properties_owner ON flare_properties(owner_type, owner_id)
+        SQL
+
+        db.execute(<<~SQL)
+          CREATE INDEX IF NOT EXISTS idx_properties_key ON flare_properties(key)
+        SQL
+
+        close_connection # avoid inheriting connection across fork
+        @setup = true
+      end
+    end
+
+    # Applies the same SQLite pragmas that ActiveRecord uses for good
+    # concurrency and performance with threaded/multi-process access.
+    def configure_pragmas(db)
+      db.execute("PRAGMA journal_mode=WAL")
+      db.execute("PRAGMA synchronous=NORMAL")
+      db.execute("PRAGMA mmap_size=134217728")        # 128MB
+      db.execute("PRAGMA journal_size_limit=67108864") # 64MB
+      db.execute("PRAGMA cache_size=2000")
+    end
+
+    def connection_key
+      :"flare_sqlite_db_#{@database_path.hash}"
+    end
+
+    def connection
+      Thread.current[connection_key] ||= begin
+        dir = File.dirname(@database_path)
+        FileUtils.mkdir_p(dir) unless File.directory?(dir)
+        db = ::SQLite3::Database.new(@database_path, results_as_hash: true)
+        db.busy_timeout = 5000
+        db
+      end
+    end
+
+    def close_connection
+      key = connection_key
+      if db = Thread.current[key]
+        db.close rescue nil
+        Thread.current[key] = nil
+      end
+    end
+  end
+end