rails_loki_exporter 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bf716f8af36e10dc3e446c2dc7e044835927161c00cca433f666ef4c34afcd50
+  data.tar.gz: fd8d98a489d8123d88e5e28b048d33fedd5d069e3d88e884ab0e1ef5a4e8ff5c
+SHA512:
+  metadata.gz: b0e41dfe931cefead14aa113b5c35268757a43d517ba296ba48b410612b3f03721808d67896f64ca187cb5d36f9797c16152b2e9f50bdeef814913b4f6163cf0
+  data.tar.gz: 2a949bbe2ef8a94c40c5096992dea844598e043a20dba564f2aa43e86030cd2a0ede0ce34cbcfe1e2def25701a81c74bdedc0e6fee1f9a7df9f3c9a4e7c02ad8

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,61 @@
+# Rails Loki Exporter
+
+:gem: **Rails Loki Exporter** :gem: is a simple log epxporter for Rails.
+
+Export logs for your Rails application to Loki instance and access them through Grafana dashboard. 
+## Prerequisites
+:exclamation: Before you start make sure you set up the following:
+- Grafana Dashboard
+- Loki Server
+
+
+## Installation
+
+**Rails Loki Exporter**'s installation is pretty straightforward.
+
+Using Bundler:
+- Add a line for **Rails Loki Exporter** gem in your Rails application `Gemfile`:
+```rb
+...
+gem 'rails_loki_exporter', '~> <version>'
+...
+```
+- Install dependencies using `bundler`:
+```sh
+$ bundle install
+```
+- In your Rails application create `config/config.yml` file:
+```
+auth_enabled: true
+base_url: 'Your grafana loki url' 
+user_name: 'Your User number'
+password: 'Your Grafana.com API Token'
+log_file_path: 'log/#{Rails.env}.log'
+logs_type: '%w(ERROR WARN FATAL INFO DEBUG)'
+intercept_logs: true
+``` 
+- Add block for **Rails Loki Exporter** in your `application.rb` file:
+```
+require 'ruby_for_grafana_loki'
+   ...
+   ...
+   ...
+
+   config.after_initialize do
+      config_file_path = File.join(Rails.root, 'config', 'config.yml')
+      logger = RailsLokiExporters.create_logger(config_file_path)
+      Rails.logger = logger
+   end
+```
+- Start your Rails application:
+```sh
+$ rails s
+```
+
+## Deployment
+
+Add additional notes about how to deploy this on a production system.
+
+## Resources
+
+Add links to external resources for this project, such as CI server, bug tracker, etc.

data/lib/rails_loki_exporter/client.rb

@@ -0,0 +1,92 @@
+require 'socket'
+module RailsLokiExporter
+  LOGS_TYPE = %w(ERROR WARN FATAL INFO DEBUG).freeze
+
+  class Client
+    include RailsLokiExporter::Connection
+
+    attr_accessor :job_name
+    attr_accessor :host_name
+    attr_accessor :max_buffer_size
+    attr_accessor :interaction_interval
+    attr_accessor :connection
+
+    def initialize(config)
+      @base_url = config['base_url']
+      @log_file_path = config['log_file_path']
+      @logs_type = config['logs_type']
+      @intercept_logs = config['intercept_logs']
+      @job_name = config['job_name'] || "#{$0}_#{Process.pid}"
+      @host_name = config['host_name'] || Socket.gethostname
+      @log_buffer = []
+      @last_interaction_time = nil
+      @interaction_interval = 1 # in seconds, adjust as needed
+      @max_buffer_size = 10 # set the maximum number of logs to buffer
+      @connection = connection
+    end
+
+    def send_all_logs
+      File.open(@log_file_path, 'r') do |file|
+        file.each_line do |line|
+          send_log(line) 
+        end
+      end
+    end
+
+    def send_log(log_message)
+        @log_buffer << log_message
+        if @log_buffer.size >= @max_buffer_size || can_send_log?
+          send_buffered_logs
+          @last_interaction_time = Time.now
+        else
+          # @logger.info('Log buffered. Waiting for more logs or interaction interval.')
+        end
+    end
+
+    private
+    def send_buffered_logs
+      return if @log_buffer.empty?
+
+      curr_datetime = Time.now.to_i * 1_000_000_000
+      msg = "On server #{@host_name} detected error"
+      payload = {
+        'streams' => [
+          {
+            'stream' => {
+              'job' => @job_name,
+              'host' => @host_name
+            },
+            'values' => @log_buffer.map { |log| [curr_datetime.to_s, log] },
+            'entries' => @log_buffer.map do |_|
+              {
+                'ts' => curr_datetime,
+                'line' => "[WARN] " + msg
+              }
+            end
+          }
+        ]
+      }
+
+      json_payload = JSON.generate(payload)
+      uri = '/loki/api/v1/push'
+      @connection.post(uri, json_payload)
+
+      @log_buffer.clear
+    end
+
+    def can_send_log?
+      return true if @last_interaction_time.nil?
+
+      elapsed_time = Time.now - @last_interaction_time
+      elapsed_time >= @interaction_interval
+    end
+
+    def match_logs_type?(log_line)
+      return false if log_line.nil?
+
+      type_match = log_line.match(/(ERROR|WARN|FATAL|INFO|DEBUG)/)
+      type = type_match ? type_match.to_s : 'UNMATCHED'
+      type == 'UNMATCHED' || @logs_type.include?(type)
+    end
+  end
+end

data/lib/rails_loki_exporter/connection.rb

@@ -0,0 +1,60 @@
+require 'json'
+require 'net/http'
+require 'uri'
+
+module RailsLokiExporter
+  module Connection
+    def initialize(base_url, user_name, password, auth_enabled, host_name, job_name)
+      uri = URI.parse(base_url)
+      @base_url = uri
+      @user_name = user_name
+      @password = password
+      @auth_enabled = auth_enabled
+    end
+    def connection
+      http = Net::HTTP.new(@base_url.to_s, @base_url.port)
+      http.use_ssl = @base_url.scheme == 'https'
+      http.read_timeout = 30 # Adjust as needed
+      http.open_timeout = 30 # Adjust as needed
+      http
+    end
+    def post(url_loki, body)
+      url = @base_url.to_s + url_loki
+      username = @user_name
+      password = @password
+      send_authenticated_post(url, body, username, password)
+    end
+    def send_authenticated_post(url, body, username, password)
+      uri = URI.parse(url)
+      request = Net::HTTP::Post.new(uri.path)
+      request['Content-Type'] = 'application/json'
+      request.body = body
+
+      if username && password && @auth_enabled
+        request.basic_auth(username, password)
+      else
+        raise "Username or password is nil."
+      end
+
+      response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+        http.request(request)
+      end
+
+      case response
+      when Net::HTTPSuccess
+        response.body ? JSON.parse(response.body) : nil
+      when Net::HTTPNoContent
+        puts "Request successful, but no content was returned."
+        nil
+      else
+        raise "Failed to make POST request. Response code: #{response.code}, Response body: #{response.body}"
+      end
+    rescue StandardError => e
+      puts "Error: #{e.message}"
+    end
+    def base64_encode_credentials(user_name, password)
+      credentials = "#{user_name}:#{password}"
+      Base64.strict_encode64(credentials)
+    end
+  end
+end

data/lib/rails_loki_exporter/custom_log_subscriber.rb

@@ -0,0 +1,35 @@
+
+require 'action_controller/log_subscriber'
+
+module RailsLokiExporter
+  class CustomLogSubscriber < ActiveSupport::LogSubscriber
+    INTERNAL_PARAMS = %w(controller action format _method only_path)
+    class << self
+      attr_accessor :client
+    end
+
+    def process_action(event)
+      if self.class.client
+        
+        payload = event.payload
+        controller = payload[:controller]
+        action = payload[:action]
+        status = payload[:status]
+        duration = event.duration.round(2)
+        path = payload[:path]
+        params = payload[:params].except(*INTERNAL_PARAMS)
+        query_string = params.to_query
+
+        request_url = query_string.blank? ? path : "#{path}?#{query_string}"
+        log_message = "LogSubscriber - [#{controller}##{action}] URL:#{request_url}, Params: #{params}, Status: #{status}, Duration: #{duration}ms"
+        self.class.client.send_log(log_message)
+      end
+    end
+  end
+end
+
+%w(process_action start_processing).each do |evt|
+    ActiveSupport::Notifications.unsubscribe "#{evt}.action_controller"
+end
+  
+RailsLokiExporter::CustomLogSubscriber .attach_to :action_controller

data/lib/rails_loki_exporter/intercepting_logger.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'active_support/logger'
+
+module RailsLokiExporter
+  class InterceptingLogger < ActiveSupport::Logger
+    attr_accessor :client
+
+    SEVERITY_NAMES = %w(DEBUG INFO WARN ERROR FATAL).freeze
+
+    def initialize(intercept_logs: false)
+      @intercept_logs = intercept_logs
+      @log = ""
+      super(STDOUT)
+      self.level = Logger::DEBUG
+    end
+
+    def add(severity, message = nil, progname = nil, &block)
+      severity_name = severity_name(severity)
+      log_message = message
+      if log_message.nil?
+        if block_given?
+          log_message = yield
+        end
+      end
+
+      if @intercept_logs
+        if log_message.nil?
+          puts caller
+        else
+          client.send_log(@log) if client
+        end
+      end
+      super(severity, message, progname, &block)
+    end
+
+    def debug(log_message = "")
+      client.send_log("#{log_message}") if client
+    end
+
+    def info(log_message = "")
+      client.send_log("#{log_message}") if client
+    end
+
+    def fatal(log_message = "")
+      client.send_log("#{log_message}") if client
+    end
+
+    def warn(log_message = "")
+      client.send_log("#{log_message}") if client
+    end
+
+    def error(log_message = "")
+      client.send_log("#{log_message}") if client
+    end
+
+    def broadcast_to(console)
+      client.send_log(@log) if client
+    end
+
+    private
+
+    def format_message(severity, datetime, progname, msg)
+          puts "severity: #{severity}"
+          puts "datetime: #{datetime}"
+          puts "progname: #{progname}"
+          puts "msg: #{msg}"
+      "#{severity} #{progname}: #{msg}\n"
+    end
+
+    def severity_name(severity)
+      SEVERITY_NAMES[severity] || "UNKNOWN"
+    end
+  end
+end

data/lib/rails_loki_exporter/my_connection.rb

@@ -0,0 +1,8 @@
+module RailsLokiExporter
+    class MyConnection
+      include Connection
+      def self.create(base_url, user_name, password, auth_enabled, host_name, job_name)
+        new(base_url, user_name, password, auth_enabled, host_name, job_name).connection
+      end
+    end
+end

data/lib/rails_loki_exporter/query.rb

@@ -0,0 +1,8 @@
+module RailsLokiExporter
+    class Query
+        attr_reader :streams_data
+        def initialize(response) 
+            @streams_data = response['streams']
+        end
+    end     
+end 

data/lib/rails_loki_exporter/version.rb

@@ -0,0 +1,3 @@
+module RailsLokiExporter
+    VERSION = '1.0.2'
+end 

data/lib/rails_loki_exporter.rb

@@ -0,0 +1,46 @@
+require 'zeitwerk'
+
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+
+module RailsLokiExporter
+  class << self
+    def create_logger(config_file_path)
+      config = load_config(config_file_path)
+
+      connection_instance =  MyConnection.new(
+        config['base_url'],
+        config['user_name'],
+        config['password'],
+        config['auth_enabled'],
+        config['host_name'],
+        config['job_name']
+      )
+
+      client = Client.new(config)
+      logger = InterceptingLogger.new(intercept_logs: config['intercept_logs'])
+      if config['enable_log_subscriber'] 
+        CustomLogSubscriber.client = client
+      end
+      logger.client = client
+      client.connection = connection_instance
+      logger
+    end
+
+    private
+
+    def load_config(config_file_path)
+      expanded_path = File.expand_path(config_file_path, __dir__)
+
+      if File.exist?(expanded_path)
+        config_erb = ERB.new(File.read(expanded_path)).result
+        config = YAML.safe_load(config_erb, aliases: true)
+        puts config.to_json
+        return config
+      else
+        puts "Config file not found: #{expanded_path}"
+        return {}
+      end
+    end
+  end
+end

data/rails_loki_exporter.gemspec

@@ -0,0 +1,21 @@
+require File.expand_path('lib/rails_loki_exporter/version', __dir__)
+
+Gem::Specification.new do |spec|
+    spec.name                  = 'rails_loki_exporter'
+    spec.version               = RailsLokiExporter::VERSION
+    spec.authors               = ['Oleg Ten', 'Assiya Kalykova']
+    spec.email                 = ['tennet0505@gmail.com']
+    spec.summary               = 'Ruby for Grafana Loki'
+    spec.description           = 'Attempt to make gem'
+    spec.homepage              = 'https://rubygems.org/gems/hello_app_ak_gem'
+    spec.license               = 'MIT'
+    spec.platform              = Gem::Platform::RUBY
+    spec.required_ruby_version = '>= 2.7.0'
+    spec.files = Dir['README.md', 'lib/**/*.rb',
+                     'rails_loki_exporter.gemspec',
+                     'Gemfile']
+    spec.extra_rdoc_files      = ['README.md']
+    spec.require_paths         = ['lib']
+    spec.add_dependency        'zeitwerk', '~> 2.4'
+    spec.add_dependency        'rspec'
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: rails_loki_exporter
+version: !ruby/object:Gem::Version
+  version: 1.0.2
+platform: ruby
+authors:
+- Oleg Ten
+- Assiya Kalykova
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2024-02-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Attempt to make gem
+email:
+- tennet0505@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- Gemfile
+- README.md
+- lib/rails_loki_exporter.rb
+- lib/rails_loki_exporter/client.rb
+- lib/rails_loki_exporter/connection.rb
+- lib/rails_loki_exporter/custom_log_subscriber.rb
+- lib/rails_loki_exporter/intercepting_logger.rb
+- lib/rails_loki_exporter/my_connection.rb
+- lib/rails_loki_exporter/query.rb
+- lib/rails_loki_exporter/version.rb
+- rails_loki_exporter.gemspec
+homepage: https://rubygems.org/gems/hello_app_ak_gem
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key: 
+specification_version: 4
+summary: Ruby for Grafana Loki
+test_files: []