injest-client 0.1.2 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a527ee5b2ec9e083f3d841a8a01bbfd62ac72d5dd5772455b10d8cde4ce1e015
-  data.tar.gz: e0ac5772bbac8b9e9bafa24ff78327fd20fa1f904df65a2bc7129f66f5f16fa2
+  metadata.gz: 5b3054fcd879d5b947d3babca7db973a464f569b32b6de529f41dcdba731e4e3
+  data.tar.gz: 72abf9a2575b870373be452eb8f52bfb05d57979badafed40b726843a806489c
 SHA512:
-  metadata.gz: 4002ab02de4c80a77c9359d030ae93ca73dd90a89dd138966b0d847b1fc12e1931f48b8b1274ed70b28addb0fa222fa69c789ec7e5c77421688b6bfcbdba85bf
-  data.tar.gz: 463a06c6a30c442323149bb33e5cfaf2cc0d164c63939a33fd4509cb844c90d51128d5128b444b4531add4ef47970fcf7b6feb9927f86b3042779116151ce049
+  metadata.gz: 329b5be30aa1f679e790e1406a46ef3c0337278e5b2e3e5ff6bb428154350e77ee067d4825283e45ed24df226ba4f90935d6e7e2f3630b79ba75563be4c1a54f
+  data.tar.gz: f01211a252f24a637df6641f693830410864253655d51395b601b850249168190e4550e22bdff976705f1d2738e061aa52b839eedc01add61276ba8e90acbce7

data/lib/README.md

@@ -0,0 +1,24 @@
+# Injest Client
+
+## Changelog
+
+### 0.1.4
+
+- add raw_search to Injest::HttpClient
+
+ENV vars
+
+- `INJEST_STRATEGY`: stdout by default
+  
+  Allowed values: Values: stdout | null | http | push
+  
+  `http` and `push` have the same behavior.
+
+- `INJEST_ROOT`: a URL required with strategy `http` or `push`
+- `INJEST_JWT`: a JWT required with strategy `http` or `push`
+- `INJEST_CLIENT`: (optional) a string to bind to a specific client
+
+## TODO:
+
+- Customization with procs
+- Tests: https://guides.rubygems.org/make-your-own-gem/#writing-tests

data/lib/injest/configuration.rb

@@ -0,0 +1,18 @@
+class Injest::Configuration
+  include Singleton
+
+  attr_reader :strategy, 
+    :injest_root, :injest_token, :injest_client
+
+  private
+  def initialize
+    @strategy = ENV.fetch('INJEST_STRATEGY', 'stdout')
+
+    if strategy == 'http' || strategy == 'push'
+      @injest_root = ENV.fetch('INJEST_ROOT')
+      @injest_token = ENV.fetch('INJEST_JWT')
+    end
+    
+    @injest_client = ENV.fetch('INJEST_CLIENT', nil)
+  end
+end

data/lib/injest/http_client.rb

@@ -0,0 +1,198 @@
+require 'net/http'
+require 'json'
+
+class Injest::HttpClient
+  attr_reader :root_url,
+              :push_url,
+              :search_url,
+              :raw_search_url,
+              :jwt,
+              :client_code
+  def initialize(configuration)
+    @root_url = configuration.injest_root
+    @client_code = configuration.injest_client
+    if client_code.nil? || client_code == ''
+      @push_url = [root_url, 'api/v1/logs'].join('/')
+    else
+      @push_url = [root_url, 'api/v1/logs', client_code].join('/')
+    end
+    @search_url = [root_url, 'api/v1/logs/search'].join('/')
+    @raw_search_url = [root_url, 'api/v1/logs/raw_search'].join('/')
+    @jwt = configuration.injest_token
+  end
+
+  # Push an audit log
+  def audit(data)
+    push data.merge(log_type: 'audit')
+  end
+
+  def search(page: 1, per: 20, **other)
+    payload = {
+      page: page,
+      per: per,
+      logs_search_form: {}
+    }
+    other.each do |k, v|
+      payload[:logs_search_form][k] = v
+    end
+
+    post_http_request search_url, payload
+  end
+
+  def raw_search(body:, index_postfix: nil)
+    payload = { body: body }
+    payload[:index_postfix] = index_postfix unless index_postfix.nil?
+
+    post_http_request raw_search_url, payload
+  end
+
+  # Push a log
+  def push(data)
+    data = data_as_hash(data)
+    create_log(data)
+  end
+
+  def json(time:, severity:, data:,
+           correlation_id: nil,
+           request_id: nil,
+           partition_key: nil,
+           owner: nil,
+           delta: nil)
+    request_id ||= SecureRandom.uuid
+    data = {
+      request_id: request_id,
+      correlation_id: correlation_id,
+      time: time,
+      human_time: Time.at(time).to_s,
+      log_type: 'json',
+      severity: severity,
+      partition_key: partition_key,
+      owner: owner,
+      delta: delta,
+      data: data_as_hash(data)
+    }
+    create_log data
+  end
+
+  def text(time:, severity:, message:,
+           correlation_id: nil,
+           request_id: nil,
+           partition_key: nil,
+           owner: nil,
+           delta: nil)
+    request_id ||= SecureRandom.uuid
+    data = {
+      request_id: request_id,
+      correlation_id: correlation_id,
+      time: time,
+      human_time: Time.at(time).to_s,
+      log_type: 'text',
+      severity: severity,
+      partition_key: partition_key,
+      owner: owner,
+      delta: delta,
+      text: message
+    }
+    create_log data
+  end
+
+  def process_response!(response)
+    case response.code
+    when '200', '201'
+      unless response.body.empty?
+        JSON.parse(response.body)
+      else
+        {}
+      end
+    else
+      data = JSON.parse(response.body) unless response.body.blank?
+      puts "Endpoint replied with #{response.code} - #{data['error_code']}", data: data
+      raise "Endpoint replied with #{response.code} - #{data['error_code']}", data: data
+    end
+  end
+
+  private
+  def create_log(data)
+    return if ENV['INJEST_STRATEGY'] == 'null'
+
+    if ENV['INJEST_STRATEGY'] == 'stdout'
+      puts "== Injest STDOUT data =="
+      # puts data.inspect
+      print_log(data)
+      return
+    end
+
+    post_http_request push_url, data
+    # uri = URI.parse push_url
+    # request = Net::HTTP::Post.new(uri)
+    # request["Content-Type"] = "application/json"
+    # request["Accept"] = "application/json"
+    # request['Authorization'] = "Bearer #{jwt}"
+    # case data
+    # when String
+    #   request.body = data
+    # when Hash
+    #   request.body = data.to_json
+    # end
+    # req_options = { use_ssl: uri.scheme == "https" }
+    # response = Net::HTTP.start(uri.hostname, uri.port, req_options) do |http|
+    #   http.request(request)
+    # end
+    #
+    # process_response!(response)
+  end
+
+  def post_http_request(url, data)
+    uri = URI.parse url
+    request = Net::HTTP::Post.new(uri)
+    request["Content-Type"] = "application/json"
+    request["Accept"] = "application/json"
+    request['Authorization'] = "Bearer #{jwt}"
+    case data
+    when String
+      request.body = data
+    when Hash
+      request.body = data.to_json
+    end
+    req_options = { use_ssl: uri.scheme == "https" }
+    response = Net::HTTP.start(uri.hostname, uri.port, req_options) do |http|
+      http.request(request)
+    end
+
+    process_response!(response)
+  end
+
+  def data_as_hash(data)
+    case data
+    when String
+      JSON.parse(data)
+    when Hash
+      data
+    end
+  end
+
+  def print_log(data)
+    puts "LOGTYPE '#{data[:log_type]}'"
+    if data[:log_type] != 'audit'
+      puts data.inspect
+      return
+    end
+
+    puts "#{data[:human_time]} - Request #{data[:request_id]}"
+    puts "#{data[:request][:method]} #{data[:request][:path]}"
+    puts ''
+    puts "Subject"
+    puts data[:request][:subject].inspect
+    puts ''
+    puts 'Context'
+    puts data[:request][:context].inspect
+    puts ''
+    puts "Headers"
+    data[:request][:headers].each do |k, v|
+      puts "#{k}: #{v}"
+    end
+    puts ''
+    puts "Response: #{data[:response][:status]}"
+    puts data[:response][:body].inspect
+  end
+end

data/lib/injest/middleware.rb

@@ -0,0 +1,173 @@
+class Injest::Middleware
+  HEADERS_BLACKLIST = %w[HTTP_COOKIE].freeze
+
+  attr_reader :configuration
+  attr_reader :request_type, :env, :consumer, :subject
+
+  def initialize(app, processor: nil)
+    @app = app
+    init_configuration
+
+    @processor = processor
+
+    @consumer = nil
+    @subject = nil
+  end
+
+  def call(env)
+    dup._call(env)
+  end
+
+  def _call(env)
+    @env = env
+    set_request_type
+    return @app.call(env) unless trackable?
+
+    @request_started_on = Time.now
+    @status, @headers, @response = @app.call(env)
+    @request_ended_on = Time.now
+
+    set_subject
+    set_consumer
+
+    data = build_data
+    data = @processor.call(env, data) unless @processor.nil?
+
+    append(data)
+
+    [@status, @headers, @response]
+  end
+
+  def init_configuration
+    @configuration = Injest::Configuration.instance
+  end
+
+  private
+
+  def append(data)
+    Injest::Writer.instance.append(data)
+  end
+
+  def build_data
+    {
+      request_id: env["action_dispatch.request_id"],
+      time: @request_started_on.to_i,
+      human_time: @request_started_on,
+      type: request_type,
+      request: build_request,
+      response: {
+        status: @status,
+        body: response_body(@response),
+        headers: @headers
+      },
+      delta: @request_ended_on - @request_started_on,
+      log_type: 'audit'
+    }
+  end
+
+  def build_request
+    if env.key?('rack.session')
+      auth_scheme = env["rack.session"].fetch('auth_scheme', [])
+    else
+      auth_scheme = []
+    end
+    
+    {
+      method: env['REQUEST_METHOD'],
+      server_name: env['SERVER_NAME'],
+      port: env['SERVER_PORT'],
+      query_string: env['QUERY_STRING'],
+      path: env['PATH_INFO'],
+      scheme: env["rack.url_scheme"],
+      remote_addr: env['REMOTE_ADDR'],
+      remote_ip: env['action_dispatch.remote_ip'].to_s,
+      formats: (env["action_dispatch.request.formats"] || []).map(&:to_s),
+      headers: get_request_headers,
+      subject: subject,
+      context: get_context(env['action_dispatch.request.parameters'], env["action_dispatch.request.request_parameters"]),
+      auth_scheme: auth_scheme
+    }
+  end
+
+  def get_context(data, params)
+    data ||= {}
+    {
+      controller: data['controller'],
+      action: data['action'],
+      params: params,
+      version: Gem.loaded_specs["injest-client"].version.to_s,
+      consumer: consumer
+    }
+  end
+
+  def get_request_headers
+    result = {}
+    env.each do |k, v|
+      if (k.to_s.start_with?('HTTP_') || k.to_s.start_with?('X-')) && !HEADERS_BLACKLIST.include?(k) && !v.blank?
+        result[k] = v
+      end
+    end
+    result
+  end
+
+  def response_body(response)
+    if @headers['Content-Type'] == 'text/csv' ||
+      @headers['Content-Type'] == 'application/pdf'
+      return { raw: "* * * RESPONSE NOT SHOWN FOR #{@headers['Content-Type']} * * *" }
+    end
+
+    if request_type == 'api'
+      if response.respond_to?(:body)
+        JSON.parse(response.body) rescue { raw: "* * * Unable to parse JSON response for #{@headers['Content-Type']} * * *" }
+      else
+        response
+      end
+    else
+      if @headers['Content-Type']&.downcase&.include?('application/json')
+        if response.respond_to?(:body)
+          JSON.parse(response.body) rescue { raw: "* * * Unable to parse JSON response for #{@headers['Content-Type']} * * *" }
+        else
+          { raw: "* * * RESPONSE NOT SHOWN * * *" }
+        end
+      else
+        { raw: "* * * RESPONSE NOT SHOWN FOR #{@headers['Content-Type']} * * *" }
+      end
+    end
+  end
+
+  def set_consumer
+    if !subject.nil?
+      @consumer = [subject[:type].downcase, subject[:id]].join(':')
+    elsif env.key?("action_dispatch.request.parameters")
+      @consumer = env["action_dispatch.request.parameters"].fetch('_consumer', 'unknown')
+    end
+  end
+
+  def set_request_type
+    # TODO: maybe we should check some header instead
+    if env['PATH_INFO'].match?(/.*\.(?:png|css|jpeg|ico|gif)/)
+      @request_type = 'static_asset'
+    elsif env['PATH_INFO'].match?(/^\/api\/v[0-9]+/)
+      @request_type = 'api'
+    elsif env['PATH_INFO'].include?('/sidekiq')
+      @request_type = 'sidekiq'
+    elsif env['PATH_INFO'].include?('/rails')
+      @request_type = 'rails'
+    else
+      @request_type = 'browser'
+    end
+  end
+
+  def set_subject
+    return if env['rack.session'].nil?
+
+    @subject = env["rack.session"].fetch('current_subject', {}).with_indifferent_access
+    if @subject.nil?
+      @subject = env["rack.session"].fetch(:current_subject, {}).with_indifferent_access
+    end
+  end
+
+  def trackable?
+    !%w[static_asset sidekiq rails].include?(request_type)
+  end
+end

data/lib/injest/worker.rb

@@ -0,0 +1,6 @@
+class Injest::Worker
+  include Sidekiq::Worker
+  def perform(data)
+    Injest::Writer.instance.append(data, sync: true)
+  end
+end

data/lib/injest/writer.rb

@@ -0,0 +1,63 @@
+class Injest::Writer
+  include Singleton
+
+  attr_reader :configuration, :http_client
+
+  def append(data, sync: false)
+    case configuration.strategy
+    when 'push', 'http'
+      if sync
+        http_client.push(data)
+      else
+        Injest::Worker.perform_async(data)
+      end
+    
+    when 'stdout'
+      print(data)
+
+    when 'null'
+      # Do nothing
+    else
+      # Do nothing
+    end
+  end
+
+  private
+  def initialize
+    @configuration = Injest::Configuration.instance
+    @http_client = Injest::HttpClient.new(configuration)
+  end
+
+  def print(data)
+    puts "=== Injest output ==="
+    if data[:log_type] != 'audit'
+      data.each do |k, v|
+        if v.is_a?(Hash)
+          puts "  - #{k}:"
+          v.each do |k1, v1|
+            puts "    - #{k1}: #{v1}"
+          end
+        else
+          puts "  - #{k}: #{v}"
+        end
+      end
+      return
+    end
+
+    puts "#{data[:request][:method]} #{data[:request][:path]} @ #{data[:human_time]} -- Request #{data[:request_id]}"
+    puts ''
+    puts "Subject"
+    puts "  " + data[:request][:subject].inspect
+    puts ''
+    puts 'Context'
+    puts "  " + data[:request][:context].inspect
+    puts ''
+    puts "Headers"
+    data[:request][:headers].each do |k, v|
+      puts "  - #{k}: #{v}"
+    end
+    puts ''
+    puts "Response: #{data[:response][:status]}"
+    puts data[:response][:body].inspect
+  end
+end

data/lib/injest.rb

@@ -5,9 +5,9 @@ end
 require 'singleton'
 require 'sidekiq'
 
-require_relative 'injest/configuration'
-require_relative 'injest/writer'
-require_relative 'injest/http_client'
-require_relative 'injest/worker'
+require 'injest/configuration'
+require 'injest/writer'
+require 'injest/http_client'
+require 'injest/worker'
 
-require_relative 'injest/middleware'
+require 'injest/middleware'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: injest-client
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - Stefano Lampis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-09 00:00:00.000000000 Z
+date: 2023-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -58,7 +58,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/README.md
 - lib/injest.rb
+- lib/injest/configuration.rb
+- lib/injest/http_client.rb
+- lib/injest/middleware.rb
+- lib/injest/worker.rb
+- lib/injest/writer.rb
 homepage: https://rubygems.org/gems/slampis-injest-client
 licenses:
 - MIT