proxes 0.9.8 → 0.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19cf2985a0f5c1cc1e76a9e0acd7319ed6e600a8f43b0fe34ddb99ddad7a47f7
-  data.tar.gz: ae63df70c4d596af1fd8ec87656ffbc05e35b433c0719ac1ae7497978f30bc17
+  metadata.gz: 9f06c68171f233ccce5d7cb06ab76f6503254ae8159c8906a53170af8e6e4b78
+  data.tar.gz: c4022844adcbf0d5740fe3b78f841a2be8c0db6b6d182a9d383f79ceb6d28299
 SHA512:
-  metadata.gz: 57f5d54d14b06f89f2067a0f6ca18e0dc08307a18d156d7748569a1b9ef1b1feef54e4650218957555b7c028491899fdc1be2874d8317b2850c39824a3313634
-  data.tar.gz: 7089ca97c7dd821183d82248c6820bb59d5c76b9d61f547855a1930548bb6ba13407690b74f799f13311ea7d53c1088b92b26832f2b7b33ddcf92feb78514912
+  metadata.gz: 9b7d4ad313c188b7dd2e9db490870828bf6b6c2590c15349ccc58d167bc29c6f1e455e3706a6202cb78d072a1b105308c0c9004cb239d06cefb4fe0ae951e7cf
+  data.tar.gz: 3ed39b26565420d629984d4e932bce2b239369554b84d0daf2744cca469a65ec634ee17f514790b11d8a5a4d7833d23b15e55519051fdec046371d9b0e49524d

data/lib/ditty/components/proxes.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'ditty'
+
+module Ditty
+  class ProxES
+    def self.load
+      controllers = File.expand_path('../../proxes/controllers', __dir__)
+      Dir.glob("#{controllers}/*.rb").each { |f| require f }
+      require 'proxes/models/permission'
+      require 'proxes/services/listener'
+    end
+
+    def self.migrations
+      File.expand_path('../../../migrate', __dir__)
+    end
+
+    def self.view_folder
+      File.expand_path('../../../views', __dir__)
+    end
+
+    def self.public_folder
+      File.expand_path('../../../public', __dir__)
+    end
+
+    def self.routes
+      load
+      {
+        '/search' => ::ProxES::Search,
+        '/status' => ::ProxES::Status,
+        '/permissions' => ::ProxES::Permissions
+      }
+    end
+
+    def self.navigation
+      load
+      [
+        { order: 0, link: '/status/check', text: 'Status Check', target: ::ProxES::Status, icon: 'dashboard' },
+        { order: 1, link: '/search', text: 'Search', target: ::ProxES::Status, icon: 'search' },
+        { order: 15, link: '/permissions', text: 'Permissions', target: ::ProxES::Permission, icon: 'check-square' }
+      ]
+    end
+
+    def self.seeder
+      proc do
+        require 'ditty/models/user'
+        require 'ditty/models/role'
+        require 'proxes/models/permission'
+
+        sa = ::Ditty::Role.find_or_create(name: 'super_admin')
+        %w[GET POST PUT DELETE HEAD OPTIONS INDEX].each do |verb|
+          ::ProxES::Permission.find_or_create(role: sa, verb: verb, pattern: '.*')
+        end
+
+        # Admin Role
+        ::Ditty::Role.find_or_create(name: 'admin')
+
+        # User Role
+        user_role = ::Ditty::Role.find_or_create(name: 'user')
+        ::ProxES::Permission.find_or_create(role: user_role, verb: 'GET', pattern: '/_cluster/stats')
+        ::ProxES::Permission.find_or_create(role: user_role, verb: 'GET', pattern: '/_nodes')
+        ::ProxES::Permission.find_or_create(role: user_role, verb: 'GET', pattern: '/_nodes/stats')
+        ::ProxES::Permission.find_or_create(role: user_role, verb: 'GET', pattern: '/_stats')
+        ::ProxES::Permission.find_or_create(role: user_role, verb: 'INDEX', pattern: 'user-{user.id}')
+
+        # Kibana Specific
+        anon = ::Ditty::User.find_or_create(email: 'anonymous@proxes.io')
+        anon.remove_role user_role
+        anon_role = ::Ditty::Role.find_or_create(name: 'anonymous')
+        anon.add_role anon_role unless anon.role?('anonymous')
+        ::ProxES::Permission.find_or_create(role: anon_role, verb: 'GET', pattern: '/.kibana/config/*')
+        ::ProxES::Permission.find_or_create(role: anon_role, verb: 'INDEX', pattern: '.kibana')
+
+        kibana = ::Ditty::Role.find_or_create(name: 'kibana')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'INDEX', pattern: '.kibana')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'HEAD', pattern: '/')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'GET', pattern: '/_nodes*')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'GET', pattern: '/_cluster/health*')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'GET', pattern: '/_cluster/settings*')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_mget')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_search')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_msearch')
+        ::ProxES::Permission.find_or_create(role: kibana, verb: 'POST', pattern: '/_refresh')
+      end
+    end
+  end
+end
+
+Ditty::Components.register_component(:proxes, Ditty::ProxES)

data/lib/proxes/controllers/permissions.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'ditty/controllers/component'
+require 'proxes/models/permission'
+require 'ditty/policies/user_policy'
+require 'ditty/policies/role_policy'
+require 'proxes/policies/permission_policy'
+
+module ProxES
+  class Permissions < Ditty::Component
+    set model_class: Permission
+
+    FILTERS = [
+      { name: :user, field: 'user.email' },
+      { name: :role, field: 'role.name' },
+      { name: :verb }
+    ].freeze
+
+    SEARCHABLE = %i[pattern].freeze
+
+    helpers do
+      def user_options
+        policy_scope(::Ditty::User).as_hash(:email, :email)
+      end
+
+      def role_options
+        policy_scope(::Ditty::Role).as_hash(:name, :name)
+      end
+
+      def verb_options
+        ProxES::Permission.verbs
+      end
+    end
+
+    def find_template(views, name, engine, &block)
+      super(views, name, engine, &block) # Root
+      super(::Ditty::ProxES.view_folder, name, engine, &block) # This Component
+      super(::Ditty::App.view_folder, name, engine, &block) # Ditty
+    end
+  end
+end

data/lib/proxes/controllers/search.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object/blank'
+require 'ditty/controllers/application'
+require 'proxes/services/search'
+require 'proxes/policies/search_policy'
+
+module ProxES
+  class Search < Ditty::Application
+    set base_path: "#{settings.map_path}/search"
+
+    get '/' do
+      authorize self, :search
+
+      param :page, Integer, min: 0, default: 1
+      param :count, Integer, min: 0, default: 25
+      from = ((params[:page] - 1) * params[:count])
+      params[:q] = '*' if params[:q].blank?
+      result = ProxES::Services::Search.search(params[:q], index: params[:indices], from: from, size: params[:count])
+      haml :"#{view_location}/index",
+           locals: {
+             title: 'Search',
+             indices: ProxES::Services::Search.indices,
+             fields: ProxES::Services::Search.fields(index: params[:indices], names_only: true),
+             result: result
+           }
+    end
+
+    get '/fields/?:indices?/?' do
+      authorize self, :fields
+
+      param :names_only, Boolean, default: false
+      json ProxES::Services::Search.fields index: params[:indices], names_only: params[:names_only]
+    end
+
+    get '/indices/?' do
+      authorize self, :indices
+
+      json ProxES::Services::Search.indices
+    end
+
+    get '/values/:field/?:indices?/?' do |field|
+      authorize self, :values
+
+      param :size, Integer, min: 0, default: 25
+      json ProxES::Services::Search.values(field, size: params[:size], index: params[:indices])
+    end
+
+    def find_template(views, name, engine, &block)
+      super(views, name, engine, &block) # Root
+      super(::Ditty::ProxES.view_folder, name, engine, &block) # This Component
+      super(::Ditty::App.view_folder, name, engine, &block) # Ditty
+    end
+  end
+end

data/lib/proxes/controllers/status.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require 'ditty/controllers/application'
+require 'proxes/policies/status_policy'
+require 'proxes/services/es'
+
+module ProxES
+  class Status < Ditty::Application
+    helpers ProxES::Services::ES
+
+    def find_template(views, name, engine, &block)
+      super(views, name, engine, &block) # Root
+      super(::Ditty::ProxES.view_folder, name, engine, &block) # This Component
+      super(::Ditty::App.view_folder, name, engine, &block) # Ditty
+    end
+
+    # This provides a URL that can be polled by a monitoring system. It will return
+    # 200 OK if all the checks pass, or 500 if any of the checks fail.
+    get '/check' do
+      checks = []
+      begin
+        health = client.cluster.health level: 'cluster'
+        checks << { text: 'Cluster Reachable', passed: true, value: health['cluster_name'] }
+        checks << { text: 'Cluster Health', passed: health['status'] == 'green', value: health['status'] }
+
+        node_stats = client.nodes.stats
+
+        master_nodes = []
+        data_nodes = []
+        ingestion_nodes = []
+        node_stats['nodes'].each_value do |node|
+          if node['roles']
+            master_nodes << node['name'] if node['roles'].include? 'master'
+            data_nodes << node['name'] if node['roles'].include? 'data'
+            ingestion_nodes << node['name'] if node['roles'].include? 'ingest'
+          elsif node['attributes']
+            master_nodes << node['name'] unless node['attributes']['master'] == 'false'
+            data_nodes << node['name'] unless node['attributes']['data'] == 'false'
+            ingestion_nodes << node['name'] unless node['attributes']['ingest'] == 'false'
+          elsif node['settings']
+            master_nodes << node['name'] unless node['settings']['node']['master'] == 'false'
+            data_nodes << node['name'] unless node['settings']['node']['data'] == 'false'
+            ingestion_nodes << node['name'] unless node['settings']['node']['ingest'] == 'false'
+          end
+        end
+        checks << {
+          text: 'Master Nodes',
+          passed: master_nodes.count > 0,
+          value: master_nodes.count > 0 ? master_nodes.sort : 'None'
+        }
+        checks << {
+          text: 'Data Nodes',
+          passed: data_nodes.count > 0,
+          value: data_nodes.count > 0 ? data_nodes.sort : 'None'
+        }
+        checks << {
+          text: 'Ingestion Nodes',
+          passed: true,
+          value: ingestion_nodes.count > 0 ? ingestion_nodes.sort : 'None'
+        }
+
+        jvm_values = []
+        jvm_passed = true
+        node_stats['nodes'].each_value do |node|
+          jvm_values << "#{node['name']}: #{node['jvm']['mem']['heap_used_percent']}%"
+          jvm_passed = false if node['jvm']['mem']['heap_used_percent'] > 85
+        end
+        checks << { text: 'Node JVM Heap', passed: jvm_passed, value: jvm_values.sort }
+
+        fs_values = []
+        fs_passed = true
+        node_stats['nodes'].each_value do |node|
+          next if node['attributes'] && node['attributes']['data'] == 'false'
+          next if node['roles'] && node['roles'].include?('data') == false
+          stats = node['fs']['total']
+          left = stats['available_in_bytes'] / stats['total_in_bytes'].to_f * 100
+          fs_values << "#{node['name']}: #{format('%.02f', left)}% Free"
+          fs_passed = false if left < 10
+        end
+        checks << { text: 'Node File Systems', passed: fs_passed, value: fs_values.sort }
+
+        cpu_values = []
+        cpu_passed = true
+        node_stats['nodes'].each_value do |node|
+          value = (node['os']['cpu_percent'] || node['os']['cpu']['percent'])
+          cpu_values << "#{node['name']}: #{value}"
+          cpu_passed = false if value.to_i > 70
+        end
+        checks << { text: 'Node CPU Usage', passed: cpu_passed, value: cpu_values.sort }
+
+        memory_values = []
+        memory_sum = 0
+        node_stats['nodes'].each_value do |node|
+          memory_sum += node['os']['mem']['used_percent']
+          memory_values << "#{node['name']}: #{node['os']['mem']['used_percent']}"
+        end
+        memory_passed = (memory_sum / memory_values.size).to_i < 100
+        checks << { text: 'Node Memory Usage', passed: memory_passed, value: memory_values.sort }
+      rescue Faraday::Error => e
+        checks << { text: 'Cluster Reachable', passed: false, value: e.message }
+      end
+
+      status checks.find { |c| c[:passed] == false } ? 500 : 200
+
+      respond_to do |format|
+        format.html do
+          haml :'status/check', locals: { title: 'Status Check', checks: checks }
+        end
+        format.json do
+          json checks
+        end
+      end
+    end
+  end
+end

data/lib/proxes/forwarder.rb

@@ -0,0 +1,49 @@
+require 'proxes/services/es'
+require 'net/http/persistent'
+require 'singleton'
+require 'rack'
+
+module ProxES
+  # A lot of code in this comes from Rack::Proxy
+  class Forwarder
+    include Singleton
+    include ProxES::Services::ES
+
+    def call(env)
+      source = Rack::Request.new(env)
+      response = conn.send(source.request_method.downcase) do |req|
+        source_body = body_from(source)
+        req.body = source_body if source_body
+        req.url source.fullpath == '' ? URI.parse(env['REQUEST_URI']).request_uri : source.fullpath
+      end
+      mangle response
+    end
+
+    def mangle(response)
+      headers = (response.respond_to?(:headers) && response.headers) || self.class.normalize_headers(response.to_hash)
+      body    = response.body || ['']
+      body    = [body] unless body.respond_to?(:each)
+
+      # Not sure where this is coming from, but it causes timeouts on the client
+      headers.delete('transfer-encoding')
+      # Ensure that the content length rack middleware kicks in
+      headers.delete('content-length')
+
+      [response.status, headers, body]
+    end
+
+    def body_from(request)
+      return nil if request.body.nil? || (Kernel.const_defined?('::Puma::NullIO') && request.body.is_a?(Puma::NullIO))
+      request.body.read.tap { |_r| request.body.rewind }
+    end
+
+    class << self
+      def normalize_headers(headers)
+        mapped = headers.map do |k, v|
+          [k, v.is_a?(Array) ? v.join("\n") : v]
+        end
+        Rack::Utils::HeaderHash.new Hash[mapped]
+      end
+    end
+  end
+end

data/lib/proxes/helpers/indices.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/object/blank'
+
+module ProxES
+  module Helpers
+    module Indices
+      def filter(asked, against)
+        return against.map { |a| a.gsub(/\.\*/, '*') } if asked == ['*'] || asked.blank?
+
+        answer = []
+        against.each do |pattern|
+          answer.concat(asked.select { |idx| idx =~ /#{pattern}/ })
+        end
+        answer
+      end
+
+      def patterns
+        return [] if user.nil?
+        patterns_for('INDEX').map do |permission|
+          return nil if permission.pattern.blank?
+          permission.pattern.gsub(/\{user.(.*)\}/) { |_match| user.send(Regexp.last_match[1].to_sym) }
+        end.compact
+      end
+
+      def patterns_for(action)
+        return [] if user.nil?
+        Permission.for_user(user, action)
+      end
+    end
+  end
+end

data/lib/proxes/loggers/elasticsearch.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module ProxES
+  module Loggers
+    class Elasticsearch < ::Logger
+    end
+  end
+end

data/lib/proxes/middleware/error_handling.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'wisper'
+require 'proxes/request'
+require 'ditty/services/logger'
+
+module ProxES
+  module Middleware
+    class ErrorHandling
+      attr_reader :logger
+
+      include Wisper::Publisher
+
+      def initialize(app, logger = nil)
+        @app = app
+        @logger = logger || ::Ditty::Services::Logger.instance
+      end
+
+      def call(env)
+        request = ProxES::Request.from_env(env)
+        response = @app.call env
+        broadcast(:es_request_failed, request, response) unless (200..299).cover?(response[0])
+        response
+      rescue Errno::EHOSTUNREACH
+        error 'Could not reach Elasticsearch at ' + ENV['ELASTICSEARCH_URL']
+      rescue Errno::ECONNREFUSED, Faraday::ConnectionFailed, SocketError
+        error 'Elasticsearch not listening at ' + ENV['ELASTICSEARCH_URL']
+      rescue Pundit::NotAuthorizedError, Ditty::Helpers::NotAuthenticated => e
+        broadcast(:es_request_denied, request, e)
+        log_not_authorized request
+        raise e if env['APP_ENV'] == 'development'
+        return [401, {}, []] if request.head?
+        request.html? && request.user.nil? ? login_and_redirect(request) : error('Not Authorized', 401)
+      rescue StandardError => e
+        broadcast(:es_request_denied, request, e)
+        log_not_authorized request
+        raise e if env['APP_ENV'] == 'development'
+        return [403, {}. []] if request.head?
+        error 'Forbidden', 403
+      end
+
+      def log_not_authorized(request)
+        user = request.user ? request.user.email : 'unauthenticated request'
+        logger.error "Access denied for #{user} by security layer: #{request.detail}"
+      end
+
+      # Response Helpers
+      def error(message, code = 500)
+        headers = { 'Content-Type' => 'application/json' }
+        headers['WWW-Authenticate'] = 'Basic realm="security"' if code == 401
+        [code, headers, ['{"error":"' + message + '"}']]
+      end
+
+      def login_and_redirect(request)
+        request.session['omniauth.origin'] = request.url unless request.url == '/_proxes/auth/login'
+        redirect '/_proxes/auth/login'
+      end
+
+      def redirect(destination, code = 302)
+        [code, { 'Location' => destination }, []]
+      end
+    end
+  end
+end