proxes 0.9.13 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95d76307001a88f8c450617ba32a844f2b572d221590971d3d2e78648d437c86
-  data.tar.gz: 58d82f430b8c22f1999bc488db50422e0f57045579c4951488c32bb4be02ec18
+  metadata.gz: 3d4050f2df15c55cf749b63f4bb462a1e7bd5eda06dd03dff38ff96fad0f2cde
+  data.tar.gz: e84bcff209deac8cb463f1099637d3df5502ff0bcf31811f9da3c8ae6579f9f9
 SHA512:
-  metadata.gz: 5cfc7eb05f037be6502e5f15dc8e2add30c14579489bdcf382b6fda300859b910585ba0c94df82221f8e611d87050f8ea1693fa4812b4c15307b5cd1b9f93178
-  data.tar.gz: 918a2bcc7730276a8e9cd27c532bdef62da8e5f1509b330da0cc4f5ffe6afa912a4a230dff464fa3eb7dc184c502a3cd61ed4b9ceefb34c4496e8245285d4662
+  metadata.gz: 91355b745d0af5560ad753f052bcb8729b431eb4ed4618f8574c052242f5589c1689b60078cd812daff1f295541da2d308a41ab8cc99240c2919459fde82247c
+  data.tar.gz: 851dc6564798817a56e7e504dfce4594fc01f1f448ba9ead76a2e088070a8c6b90d29d68183604a092377ae3e6e0c9f71093a6dc128f22d4326d1b9c7d1ae0f3

data/lib/ditty/components/proxes.rb

@@ -7,6 +7,7 @@ module Ditty
     def self.load
       controllers = File.expand_path('../../proxes/controllers', __dir__)
       Dir.glob("#{controllers}/*.rb").each { |f| require f }
+      ENV['ELASTICSEARCH_URL'] ||= 'http://localhost:9200'
       require 'proxes/models/permission'
       require 'proxes/services/listener'
     end
@@ -28,6 +29,7 @@ module Ditty
       {
         '/search' => ::ProxES::Search,
         '/status' => ::ProxES::Status,
+        '/status-checks' => ::ProxES::StatusChecks,
         '/permissions' => ::ProxES::Permissions
       }
     end
@@ -36,7 +38,8 @@ module Ditty
       load
       [
         { order: 0, link: '/status/check', text: 'Status Check', target: ::ProxES::Status, icon: 'dashboard' },
-        { order: 1, link: '/search', text: 'Search', target: ::ProxES::Status, icon: 'search' },
+        { order: 1, link: '/status-checks', text: 'Mange Status Checks', target: ::ProxES::StatusCheck, icon: 'dashboard' },
+        { order: 1, link: '/search', text: 'Search', target: ::ProxES::Search, icon: 'search' },
         { order: 15, link: '/permissions', text: 'Permissions', target: ::ProxES::Permission, icon: 'check-square' }
       ]
     end
@@ -46,10 +49,11 @@ module Ditty
         require 'ditty/models/user'
         require 'ditty/models/role'
         require 'proxes/models/permission'
+        require 'proxes/models/status_check'
 
         sa = ::Ditty::Role.find_or_create(name: 'super_admin')
-        %w[GET POST PUT DELETE HEAD OPTIONS INDEX].each do |verb|
-          ::ProxES::Permission.find_or_create(role: sa, verb: verb, pattern: '.*')
+        %w[GET POST PUT DELETE HEAD OPTIONS].each do |verb|
+          ::ProxES::Permission.find_or_create(role: sa, verb: verb, pattern: '*', index: '*')
         end
 
         # Admin Role
@@ -61,25 +65,66 @@ module Ditty
         ::ProxES::Permission.find_or_create(role: user_role, verb: 'GET', pattern: '/_nodes')
         ::ProxES::Permission.find_or_create(role: user_role, verb: 'GET', pattern: '/_nodes/stats')
         ::ProxES::Permission.find_or_create(role: user_role, verb: 'GET', pattern: '/_stats')
-        ::ProxES::Permission.find_or_create(role: user_role, verb: 'INDEX', pattern: 'user-{user.id}.*')
+        # TODO
+        # ::ProxES::Permission.find_or_create(role: user_role, verb: 'INDEX', pattern: 'user-{user.id}*')
 
         # Kibana Specific
-        # actions: ["indices:data/read/field_stats", "indices:admin/mappings/fields/get", "indices:admin/get", "indices:data/read/msearch"]
         anon_role = ::Ditty::Role.find_or_create(name: 'anonymous')
         ::Ditty::User.create_anonymous_user('anonymous@proxes.io')
-        ::ProxES::Permission.find_or_create(role: anon_role, verb: 'GET', pattern: '/.kibana/config/.*')
-        ::ProxES::Permission.find_or_create(role: anon_role, verb: 'INDEX', pattern: '.kibana')
+        ::ProxES::Permission.find_or_create(role: anon_role, verb: 'GET', pattern: '/.kibana/config/*', index: '.kibana')
+        ::ProxES::Permission.find_or_create(role: anon_role, verb: 'GET', pattern: '/.kibana/doc/config/*', index: '.kibana')
 
         kibana = ::Ditty::Role.find_or_create(name: 'kibana')
-        ::ProxES::Permission.find_or_create(role: kibana, verb: 'INDEX', pattern: '.kibana')
-        ::ProxES::Permission.find_or_create(role: kibana, verb: 'HEAD', pattern: '/')
-        ::ProxES::Permission.find_or_create(role: kibana, verb: 'GET', pattern: '/_nodes*')
-        ::ProxES::Permission.find_or_create(role: kibana, verb: 'GET', pattern: '/_cluster/health.*')
-        ::ProxES::Permission.find_or_create(role: kibana, verb: 'GET', pattern: '/_cluster/settings.*')
-        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_mget')
-        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_search')
-        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_msearch')
-        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_refresh')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'HEAD', pattern: '/', index: '.kibana')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'GET', pattern: '/_nodes*', index: '.kibana')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'GET', pattern: '/_cluster/health*', index: '.kibana')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'GET', pattern: '/_cluster/settings*', index: '.kibana')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_mget', index: '.kibana')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_search', index: '.kibana')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_msearch', index: '.kibana')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_refresh', index: '.kibana')
+
+        # Status Check
+        ::ProxES::StatusCheck.find_or_create(
+          type: 'ProxES::ClusterHealthStatusCheck',
+          name: 'Cluster Health',
+          source: 'health'
+        ) { |r| r.set(required_value: 'green', order: 20) }
+        ::ProxES::StatusCheck.find_or_create(
+          type: 'ProxES::MasterNodesStatusCheck',
+          name: 'Master Nodes',
+          source: 'node_stats'
+        ) { |r| r.set(required_value: 1, order: 30) }
+        ::ProxES::StatusCheck.find_or_create(
+          type: 'ProxES::DataNodesStatusCheck',
+          name: 'Data Nodes',
+          source: 'node_stats'
+        ) { |r| r.set(required_value: 1, order: 40) }
+        ::ProxES::StatusCheck.find_or_create(
+          type: 'ProxES::IngestNodesStatusCheck',
+          name: 'Ingest Nodes',
+          source: 'node_stats'
+        ) { |r| r.set(required_value: 1, order: 50) }
+        ::ProxES::StatusCheck.find_or_create(
+          type: 'ProxES::FileSystemStatusCheck',
+          name: 'Node File Systems',
+          source: 'node_stats'
+        ) { |r| r.set(required_value: 10, order: 60) }
+        ::ProxES::StatusCheck.find_or_create(
+          type: 'ProxES::JVMHeapStatusCheck',
+          name: 'Node JVM Heap',
+          source: 'node_stats'
+        ) { |r| r.set(required_value: 85, order: 70) }
+        ::ProxES::StatusCheck.find_or_create(
+          type: 'ProxES::CPUStatusCheck',
+          name: 'Node CPU Usage',
+          source: 'node_stats'
+        ) { |r| r.set(required_value: 70, order: 80) }
+        ::ProxES::StatusCheck.find_or_create(
+          type: 'ProxES::MemoryStatusCheck',
+          name: 'Node Memory Usage',
+          source: 'node_stats'
+        ) { |r| r.set(required_value: 99, order: 90) }
       end
     end
   end

data/lib/proxes/controllers/permissions.rb

@@ -9,6 +9,7 @@ require 'proxes/policies/permission_policy'
 module ProxES
   class Permissions < Ditty::Component
     set model_class: Permission
+    set view_folder: ::Ditty::ProxES.view_folder
 
     FILTERS = [
       { name: :user, field: 'user.email' },
@@ -31,11 +32,5 @@ module ProxES
         ProxES::Permission.verbs
       end
     end
-
-    def find_template(views, name, engine, &block)
-      super(views, name, engine, &block) # Root
-      super(::Ditty::ProxES.view_folder, name, engine, &block) # This Component
-      super(::Ditty::App.view_folder, name, engine, &block) # Ditty
-    end
   end
 end

data/lib/proxes/controllers/search.rb

@@ -8,9 +8,17 @@ require 'proxes/policies/search_policy'
 module ProxES
   class Search < Ditty::Application
     set base_path: "#{settings.map_path}/search"
+    set view_folder: ::Ditty::ProxES.view_folder
+
+    helpers do
+      def fields(indices, names_only)
+        standard = { '_index' => 'text', '_type' => 'text', '_id' => 'text' }
+        standard.merge ProxES::Services::Search.fields(index: indices, names_only: names_only)
+      end
+    end
 
     get '/' do
-      authorize self, :search
+      authorize self, :list
 
       param :page, Integer, min: 0, default: 1
       param :count, Integer, min: 0, default: 25
@@ -21,16 +29,16 @@ module ProxES
            locals: {
              title: 'Search',
              indices: ProxES::Services::Search.indices,
-             fields: ProxES::Services::Search.fields(index: params[:indices], names_only: true),
+             fields: fields(params[:indices], true),
              result: result
            }
     end
 
     get '/fields/?:indices?/?' do
+      param :names_only, Boolean, default: false
       authorize self, :fields
 
-      param :names_only, Boolean, default: false
-      json ProxES::Services::Search.fields index: params[:indices], names_only: params[:names_only]
+      json fields(params[:indices], params[:names_only])
     end
 
     get '/indices/?' do
@@ -45,11 +53,5 @@ module ProxES
       param :size, Integer, min: 0, default: 25
       json ProxES::Services::Search.values(field, size: params[:size], index: params[:indices])
     end
-
-    def find_template(views, name, engine, &block)
-      super(views, name, engine, &block) # Root
-      super(::Ditty::ProxES.view_folder, name, engine, &block) # This Component
-      super(::Ditty::App.view_folder, name, engine, &block) # Ditty
-    end
   end
 end

data/lib/proxes/controllers/status.rb

@@ -2,112 +2,49 @@
 
 require 'ditty/controllers/application'
 require 'proxes/policies/status_policy'
-require 'proxes/services/es'
+require 'proxes/models/status_check'
 
 module ProxES
   class Status < Ditty::Application
-    helpers ProxES::Services::ES
-
-    def find_template(views, name, engine, &block)
-      super(views, name, engine, &block) # Root
-      super(::Ditty::ProxES.view_folder, name, engine, &block) # This Component
-      super(::Ditty::App.view_folder, name, engine, &block) # Ditty
-    end
+    set view_folder: ::Ditty::ProxES.view_folder
 
     # This provides a URL that can be polled by a monitoring system. It will return
     # 200 OK if all the checks pass, or 500 if any of the checks fail.
     get '/check' do
       checks = []
       begin
-        health = client.cluster.health level: 'cluster'
-        checks << { text: 'Cluster Reachable', passed: true, value: health['cluster_name'] }
-        checks << { text: 'Cluster Health', passed: health['status'] == 'green', value: health['status'] }
-
-        node_stats = client.nodes.stats
-
-        master_nodes = []
-        data_nodes = []
-        ingestion_nodes = []
-        node_stats['nodes'].each_value do |node|
-          if node['roles']
-            master_nodes << node['name'] if node['roles'].include? 'master'
-            data_nodes << node['name'] if node['roles'].include? 'data'
-            ingestion_nodes << node['name'] if node['roles'].include? 'ingest'
-          elsif node['attributes']
-            master_nodes << node['name'] unless node['attributes']['master'] == 'false'
-            data_nodes << node['name'] unless node['attributes']['data'] == 'false'
-            ingestion_nodes << node['name'] unless node['attributes']['ingest'] == 'false'
-          elsif node['settings']
-            master_nodes << node['name'] unless node['settings']['node']['master'] == 'false'
-            data_nodes << node['name'] unless node['settings']['node']['data'] == 'false'
-            ingestion_nodes << node['name'] unless node['settings']['node']['ingest'] == 'false'
-          end
-        end
-        checks << {
-          text: 'Master Nodes',
-          passed: master_nodes.count > 0,
-          value: master_nodes.count > 0 ? master_nodes.sort : 'None'
-        }
-        checks << {
-          text: 'Data Nodes',
-          passed: data_nodes.count > 0,
-          value: data_nodes.count > 0 ? data_nodes.sort : 'None'
-        }
-        checks << {
-          text: 'Ingestion Nodes',
-          passed: true,
-          value: ingestion_nodes.count > 0 ? ingestion_nodes.sort : 'None'
-        }
-
-        jvm_values = []
-        jvm_passed = true
-        node_stats['nodes'].each_value do |node|
-          jvm_values << "#{node['name']}: #{node['jvm']['mem']['heap_used_percent']}%"
-          jvm_passed = false if node['jvm']['mem']['heap_used_percent'] > 85
+        # Programmed checks
+        ProxES::StatusCheck.order(:order).all.each do |sc|
+          checks << {
+            text: sc.name,
+            passed: sc.passed?,
+            check: sc.required_value,
+            value: sc.value,
+            children: sc.children,
+            entity: sc
+          }
         end
-        checks << { text: 'Node JVM Heap', passed: jvm_passed, value: jvm_values.sort }
 
-        fs_values = []
-        fs_passed = true
-        node_stats['nodes'].each_value do |node|
-          next if node['attributes'] && node['attributes']['data'] == 'false'
-          next if node['roles'] && node['roles'].include?('data') == false
-          stats = node['fs']['total']
-          left = stats['available_in_bytes'] / stats['total_in_bytes'].to_f * 100
-          fs_values << "#{node['name']}: #{format('%.02f', left)}% Free"
-          fs_passed = false if left < 10
-        end
-        checks << { text: 'Node File Systems', passed: fs_passed, value: fs_values.sort }
-
-        cpu_values = []
-        cpu_passed = true
-        node_stats['nodes'].each_value do |node|
-          value = (node['os']['cpu_percent'] || node['os']['cpu']['percent'])
-          cpu_values << "#{node['name']}: #{value}"
-          cpu_passed = false if value.to_i > 70
-        end
-        checks << { text: 'Node CPU Usage', passed: cpu_passed, value: cpu_values.sort }
-
-        memory_values = []
-        memory_sum = 0
-        node_stats['nodes'].each_value do |node|
-          memory_sum += node['os']['mem']['used_percent']
-          memory_values << "#{node['name']}: #{node['os']['mem']['used_percent']}"
-        end
-        memory_passed = (memory_sum / memory_values.size).to_i < 100
-        checks << { text: 'Node Memory Usage', passed: memory_passed, value: memory_values.sort }
+        # Default Check
+        checks.unshift(
+          text: 'Cluster Reachable',
+          passed: true,
+          value: ProxES::StatusCheck.source_result('health')['cluster_name']['cluster_name']
+        )
       rescue Faraday::Error => e
         checks << { text: 'Cluster Reachable', passed: false, value: e.message }
       end
 
-      status checks.find { |c| c[:passed] == false } ? 500 : 200
+      passed = checks.find { |c| c[:passed] == false }.nil?
+      code = passed ? 200 : 500
 
+      status code
       respond_to do |format|
         format.html do
-          haml :'status/check', locals: { title: 'Status Check', checks: checks }
+          haml :'status/check', locals: { title: 'Status Check', checks: checks, passed: passed }
         end
         format.json do
-          json checks
+          json checks: checks, passed: passed, code: code
         end
       end
     end

data/lib/proxes/controllers/status_checks.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'ditty/controllers/component'
+require 'proxes/policies/status_check_policy'
+require 'proxes/models/status_check'
+
+module ProxES
+  class StatusChecks < Ditty::Component
+    set model_class: StatusCheck
+    set view_folder: ::Ditty::ProxES.view_folder
+
+    def ordering
+      return Sequel.asc(:order) if params[:sort].blank?
+
+      Sequel.send(params[:order].to_sym, params[:sort].to_sym)
+    end
+  end
+end

data/lib/proxes/forwarder.rb

@@ -1,5 +1,8 @@
+# frozen_string_literal: true
+
 require 'proxes/services/es'
-require 'net/http/persistent'
+require 'typhoeus'
+require 'typhoeus/adapters/faraday'
 require 'singleton'
 require 'rack'
 
@@ -10,40 +13,61 @@ module ProxES
     include ProxES::Services::ES
 
     def call(env)
-      source = Rack::Request.new(env)
-      response = conn.send(source.request_method.downcase) do |req|
-        source_body = body_from(source)
-        req.body = source_body if source_body
-        req.url source.fullpath == '' ? URI.parse(env['REQUEST_URI']).request_uri : source.fullpath
+      rewrite_response(
+        perform_request(
+          Rack::Request.new(
+            rewrite_env(env)
+          )
+        )
+      )
+    end
+
+    def rewrite_env(env)
+      env
+    end
+
+    # TODO: Consider moving these methods to the ProxES ES Service to enable reuse
+    def perform_request(request)
+      conn.send(request.request_method.downcase) do |req|
+        body = body_from(request)
+        req.body = body if body
+        req.url request.fullpath == '' ? URI.parse(env['REQUEST_URI']).request_uri : request.fullpath
       end
-      mangle response
     end
 
-    def mangle(response)
-      headers = (response.respond_to?(:headers) && response.headers) || self.class.normalize_headers(response.to_hash)
+    def rewrite_response(response)
+      headers = (response.respond_to?(:headers) && response.headers) || normalize_headers(response.to_hash)
       body    = response.body || ['']
       body    = [body] unless body.respond_to?(:each)
 
-      # Not sure where this is coming from, but it causes timeouts on the client
-      headers.delete('transfer-encoding')
-      # Ensure that the content length rack middleware kicks in
-      headers.delete('content-length')
+      # Only keep certain headers.
+      # See point 1 on https://www.mnot.net/blog/2011/07/11/what_proxies_must_do
+      # TODO: Extend on the above
+      headers.delete_if { |k, _v| !header_list.include? k.downcase }
 
       [response.status, headers, body]
     end
 
+    def header_list
+      [
+        'date',
+        'content-type',
+        'cache-control',
+      ]
+    end
+
     def body_from(request)
-      return nil if request.body.nil? || (Kernel.const_defined?('::Puma::NullIO') && request.body.is_a?(Puma::NullIO))
+      return if request.body.nil? || request.body.respond_to?(:read) == false
+
       request.body.read.tap { |_r| request.body.rewind }
     end
 
-    class << self
-      def normalize_headers(headers)
-        mapped = headers.map do |k, v|
+    def normalize_headers(headers)
+      Rack::Utils::HeaderHash.new(
+        headers.map do |k, v|
           [k, v.is_a?(Array) ? v.join("\n") : v]
-        end
-        Rack::Utils::HeaderHash.new Hash[mapped]
-      end
+        end.to_h
+      )
     end
   end
 end

data/lib/proxes/middleware/error_handling.rb

@@ -13,7 +13,7 @@ module ProxES
 
       def initialize(app, logger = nil)
         @app = app
-        @logger = logger || ::Ditty::Services::Logger.instance
+        @logger = logger || ::Ditty::Services::Logger
       end
 
       def call(env)
@@ -30,24 +30,26 @@ module ProxES
         log_not_authorized request
         raise e if ENV['APP_ENV'] == 'development'
         return [401, {}, []] if request.head?
+
         request.html? && request.user.nil? ? login_and_redirect(request) : error('Not Authorized', 401)
       rescue StandardError => e
         broadcast(:es_request_denied, request, e)
         log_not_authorized request
         raise e if ENV['APP_ENV'] == 'development'
         return [403, {}. []] if request.head?
+
         error 'Forbidden', 403
       end
 
       def log_not_authorized(request)
         user = request.user ? request.user.email : 'unauthenticated request'
-        logger.error "Access denied for #{user} by security layer: #{request.detail}"
+        logger.error "Access denied for #{user} by security layer: #{request.detail} #{request.indices.join(',')}"
       end
 
       # Response Helpers
       def error(message, code = 500)
         headers = { 'Content-Type' => 'application/json' }
-        headers['WWW-Authenticate'] = 'Basic realm="security"' if code == 401
+        headers['WWW-Authenticate'] = 'Basic realm="Authorization Required"' if code == 401
         [code, headers, ['{"error":"' + message + '"}']]
       end