proxes 0.9.2 → 0.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23a733cd75da2400f2490d5586a3a8039e395bdede3ecdb8ac380d986a23c74e
-  data.tar.gz: 8c316a8c6f51a14ec99c381594b1755b54064426d3885669fa0c77f40b1b2ad4
+  metadata.gz: a89e0871c6d36ce3b7e6ff19c4b7fa7556cedeedd0d364bd78d0aa741b06c6e9
+  data.tar.gz: b126f70689158d474c5f096386c13a46ffdb8e58c90c7c77181a11cef3de14d9
 SHA512:
-  metadata.gz: 45bdf16dc25e971b2b7b0ff43f7e6cf1f49fe8d9d62a8b3ba7b28a846734a775b7d416dfbfbc07057ac79768308abb7ea114d3a35bc4e0dbe91961e044c463e3
-  data.tar.gz: c104db7338a96b33a5a44b46d2b92d90ad531b8335fe2864669014524f0cbfc151a51a7fe4ec00efa723eba1e8925432ff036d115fceb85d9424801ebb7fba73
+  metadata.gz: 83df2e988cb4f9d64fae3693603dddb03912483b29e04b3c65e0ee691b341b2fe68d029e2253559534c23ee15fad129d83b7954261a297f2ab004e97dc81f2ba
+  data.tar.gz: 591c032478769ea14170e7dfb8ae92c7e34f0a5ad23dc4569bfcb5745ba47bcaa1e1585c1b4e2a5ec669c88fbb2417b9253f6427fbf671b181e8d8b21db3cbbe

data/.travis.yml

@@ -1,7 +1,7 @@
 sudo: true
 language: ruby
 rvm:
-  - 2.5.0
+  - 2.5.1
   - 2.4.3
   - 2.3.6
   - 2.2.9
@@ -9,6 +9,7 @@ gemfile: Gemfile.ci
 env:
   global:
     - CC_TEST_REPORTER_ID=1f562305f75e169f5f5eca3b738fee879550c98e50099c2b9cd6ae71478007a0
+    - DATABASE_CLEANER_ALLOW_REMOTE_DATABASE_URL=true
     - secure: wTIBP3ZtltfLyIFS6p4XgzPff70EZLxIOpzL4tWCsB2XmEMtCBPdfm9CXLw88yxH4mCJbCVljJHRzrav9N5RimVGb3h5AX5ODpqJYO4PWRZzNYGglXM3f/3QWFXExEdJ97y0m2XytOzlVWtYjShqpCq1V9z88XwmJ2YQtiFMma4n2fjW8+WK30eGwNw60GAf7a2IAA2qMSIymEdM0OcRRVjmyWRtdAslnjANQ/m9lRYeNTfwfG5Xw26BRzv46Urmlw9MZsFGV5bG5bAKRFbp3h00UsPjKlAUL22bqEQ5XmRCuXCMzffoPhfJHDao+3iaVZlViwbCmoHRuuwHnTmUBFvvef/MRSUdLLwBsKBeDVofpGhLEInPX+D/kicNEnidYTtz+O8XZ0Hgur+VN+ayItIVZqFyaw96c9n+2H/KR7TG2lsccdFHNgATTX/fVOCc2AuBI84z3qM1LFYpJibwoTktQS1dnlmX6hwwEg+lJFhY0Spr3+aKOHf/zgqK+Q0jrEDQMo0khDTuPaOy2kznzK6ktrZVizw1tZyH80Eop/Yxpx1bIab8CF49pWlMZAkwmTP3e24qewUtgExyxUK1yBZ23L7QDnYLtfwCFfUbiEScjghJ9Aez1yerMCxNUrSOltyk1XxZw9xMrptA8VgN/0sFFmfICTXTZSqJdIGkfGY=
     - secure: z4AUz0bssd0C7rfPsh71aH6ki8Zp49WZrYNRhj50MytJympFsEOYRVdVKuuAoq7jaD0cahIioDHrTf1J9bIOqMPc3l2lk9RFWA27Qk46C5zNxspnNBin0disEeO/AENlb5Zwe/Cqamg7Sc/NVpELPqUl8gHfX3FVv4Po3dZZWQb1d/pLXBpl84HGLWgFwz6ku+74XW6aqX1Qzd8G0s9gsqsAgUszwEX3ppltBrai1VdtZyFbv0XCOds3F687Ecyht3FJBQCOaTopPQztdmA8rBTPW+ffhfoZuQWfo8aILqmn9xCGr4vhMjwaOP7cVQdXcPs1ecsFBKFS0XRm5kJKBtBPLDfEc4zl6UIIe5xZBh5m0dysxu78ZoNFVui3QszyUP+dARf3AHsy1m3d8iRXMNEqKTTglKdPIdFKZLnWGoVSf+fidR7NIyJH7MGhr4VC8esGFGazzOXBnqVAX+z1tEse6n7llFcy0dREW0FNMpKMSP6TqN4LRiKXtRdSbeSvHXP3UiH6vPjAE/lHhf7qXSNOLkZDtCWrD0us4kT1ACeL2AV2SArJf7pmGdEL9pUyp4jSuStwxtRPhYJj2kuS6op9MYoU6Q8eyXhIw8Uju4SitGtkTnb8t5L50Xw7vUQT9faEah9ewJwB2kxa2RZxCdIzylwuYssCON4xkdXI1fY=
     - secure: lUfjVwbIxUBQTGkDTOSMNvSYONBwI5yObn8fCByzGkfleddWPjLEeMPjGBZboOu2CVZ/q/FG3xbn63Hmss7CU83FmWRoV5dMP9Yc7X7cHaGmi31mZcaY2Yah+jgx7PcNIdK8LOAB3/zhxDNOcr8U4O9gqxGZWnV2NLR8SB+KiNTg5Hou3nOeVw7l1Iz+h1hXvZfxkMv7aX/3TLMT8j2JMkfgCN8moMlNRAYnvm/B68FPcVs6UFcaZIwg2mBnq3FrARo127jG1Ozms2SOPTqkPY9y8yoGyo3TYH2iQOKdRxCCECBnkSO8HYRYTL9CUoMk/D4ZTp7TGNRIKxjHtlt+9O1WtfmZB2aevZX5c7ChCZ0wNhQpzlsyPDz0GoIOkms5Bfv7nsvbloejr+vsRxe8YsNaVAVG+9RVZziKxGCWsxTopfnFMfFBK7KIoJ4N60ijhtBr3qcYQbSeoZWviwSd/GCjl9I2d2mTPVI0DsiFVt8lP92ZZitPJEhPRcxHdHu47Wu7oftndcVqzhhgdGnOxMchYKgzccR17VWlactGxffJP+ZIUkHaWCkxVR4etcrE753uaUX9x+M1fMoKKrwvycstX/YWLncgsYM1YensRUN7oW87wk216anlKfHubO2aVeDNnXn+wszK87M12o9gFu1khTBowqZpXsVDaUaa+oc=

data/Dockerfile

@@ -18,13 +18,12 @@ RUN apk add --update \
   postgresql-dev \
   && rm -rf /var/cache/apk/* \
   && mkdir /root/.ssh \
-  && mkdir /usr/src/app/tmp \
-  && mkdir /usr/src/app/logs \
   && mkdir /usr/src/app/config \
   && touch /var/log/cron.log \
   && ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts \
   && gem install bundler
 
+ADD views /usr/src/app/views
 COPY config.ru /usr/src/app/
 COPY config/logger.yml /usr/src/app/config/
 COPY config/puma.rb /usr/src/app/config/

data/Gemfile.deploy.lock

@@ -44,7 +44,7 @@ GEM
       tilt
     hashie (3.5.7)
     highline (1.7.10)
-    i18n (0.9.3)
+    i18n (0.9.4)
       concurrent-ruby (~> 1.0)
     logger (1.2.8)
     minitest (5.11.3)
@@ -62,7 +62,7 @@ GEM
       bcrypt-ruby (~> 3.0)
       omniauth (~> 1.0)
     pg (1.0.0)
-    proxes (0.9.1)
+    proxes (0.9.2)
       activesupport (>= 3)
       bcrypt (~> 3.1)
       ditty (>= 0.2)

data/Gemfile.dev

@@ -3,9 +3,9 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in proxes.gemspec
 gemspec
 
+gem 'dotenv'
+gem 'pry-byebug'
+gem 'puma'
 gem 'rerun', git: 'https://github.com/alexch/rerun.git', branch: 'master'
-gem 'sqlite3'
 gem 'simplecov'
-gem 'pry-byebug'
-gem 'dotenv'
-
+gem 'sqlite3'

data/config.ru

@@ -5,6 +5,8 @@ $LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)
 
 require 'dotenv/load'
 
+require 'ditty/services/logger'
+use Rack::CommonLogger, Ditty::Services::Logger.instance
 # Session
 use Rack::Session::Cookie,
     key: '_ProxES_session',
@@ -53,11 +55,15 @@ end
 
 map '/' do
   # Proxy all Elasticsearch requests
-  require 'proxes/security'
+  require 'proxes/middleware/metrics'
+  require 'proxes/middleware/error_handling'
+  require 'proxes/middleware/security'
   require 'proxes/forwarder'
 
   # Security
-  use ProxES::Security, Ditty::Services::Logger.instance
+  use ProxES::Middleware::Metrics
+  use ProxES::Middleware::ErrorHandling
+  use ProxES::Middleware::Security, Ditty::Services::Logger.instance unless ENV['PROXES_PASSTHROUGH']
   use Rack::ContentLength
 
   # Forward requests to ES

data/config/logger.yml

@@ -1,3 +1,5 @@
 ---
 - name: default
   class: Logger
+  level: WARN
+  options: $stdout

data/docker-compose.yml

@@ -1,7 +1,11 @@
 version: '3'
 services:
+  db:
+    image: postgres
+  es:
+    image: elasticsearch
   web:
-    build: .
+    image: eagerelk/proxes:latest
     command: web-proxes
     ports:
       - '9292:9292'
@@ -11,7 +15,3 @@ services:
     depends_on:
       - db
       - es
-  db:
-    image: postgres
-  es:
-    image: elasticsearch

data/lib/proxes/controllers/permissions.rb

@@ -16,7 +16,7 @@ module ProxES
       { name: :verb }
     ].freeze
 
-    SEARCHABLE = %i[pattern]
+    SEARCHABLE = %i[pattern].freeze
 
     helpers do
       def user_options

data/lib/proxes/controllers/status.rb

@@ -28,7 +28,7 @@ module ProxES
         master_nodes = []
         data_nodes = []
         ingestion_nodes = []
-        node_stats['nodes'].values.each do |node|
+        node_stats['nodes'].each_value do |node|
           if node['roles']
             master_nodes << node['name'] if node['roles'].include? 'master'
             data_nodes << node['name'] if node['roles'].include? 'data'
@@ -61,7 +61,7 @@ module ProxES
 
         jvm_values = []
         jvm_passed = true
-        node_stats['nodes'].values.each do |node|
+        node_stats['nodes'].each_value do |node|
           jvm_values << "#{node['name']}: #{node['jvm']['mem']['heap_used_percent']}%"
           jvm_passed = false if node['jvm']['mem']['heap_used_percent'] > 85
         end
@@ -69,19 +69,19 @@ module ProxES
 
         fs_values = []
         fs_passed = true
-        node_stats['nodes'].values.each do |node|
+        node_stats['nodes'].each_value do |node|
           next if node['attributes'] && node['attributes']['data'] == 'false'
           next if node['roles'] && node['roles'].include?('data') == false
           stats = node['fs']['total']
           left = stats['available_in_bytes'] / stats['total_in_bytes'].to_f * 100
-          fs_values << "#{node['name']}: #{'%.02f' % left}% Free"
+          fs_values << "#{node['name']}: #{format('%.02f', left)}% Free"
           fs_passed = false if left < 10
         end
         checks << { text: 'Node File Systems', passed: fs_passed, value: fs_values.sort }
 
         cpu_values = []
         cpu_passed = true
-        node_stats['nodes'].values.each do |node|
+        node_stats['nodes'].each_value do |node|
           value = (node['os']['cpu_percent'] || node['os']['cpu']['percent'])
           cpu_values << "#{node['name']}: #{value}"
           cpu_passed = false if value.to_i > 70
@@ -89,18 +89,27 @@ module ProxES
         checks << { text: 'Node CPU Usage', passed: cpu_passed, value: cpu_values.sort }
 
         memory_values = []
-        memory_passed = true
-        node_stats['nodes'].values.each do |node|
+        memory_sum = 0
+        node_stats['nodes'].each_value do |node|
+          memory_sum += node['os']['mem']['used_percent']
           memory_values << "#{node['name']}: #{node['os']['mem']['used_percent']}"
-          memory_passed = false if node['os']['mem']['used_percent'].to_i >= 100
         end
+        memory_passed = (memory_sum / memory_values.size).to_i < 100
         checks << { text: 'Node Memory Usage', passed: memory_passed, value: memory_values.sort }
       rescue Faraday::Error => e
         checks << { text: 'Cluster Reachable', passed: false, value: e.message }
       end
 
       status checks.find { |c| c[:passed] == false } ? 500 : 200
-      haml :'status/check', locals: { title: 'Status Check', checks: checks }
+
+      respond_to do |format|
+        format.html do
+          haml :'status/check', locals: { title: 'Status Check', checks: checks }
+        end
+        format.json do
+          json checks
+        end
+      end
     end
   end
 end

data/lib/proxes/forwarder.rb

@@ -9,16 +9,6 @@ module ProxES
     include Singleton
     include ProxES::Services::ES
 
-    attr_reader :streaming
-
-    def backend
-      @backend ||= URI(ENV['ELASTICSEARCH_URL'])
-    end
-
-    def backend=(var)
-      @backend = URI(var)
-    end
-
     def call(env)
       forward(env)
     rescue SocketError
@@ -28,7 +18,6 @@ module ProxES
 
     def forward(env)
       source = Rack::Request.new(env)
-      conn.basic_auth backend.user, backend.password
       response = conn.send(source.request_method.downcase) do |req|
         source_body = body_from(source)
         req.body = source_body if source_body

data/lib/proxes/helpers/indices.rb

@@ -1,10 +1,13 @@
 # frozen_string_literal: true
 
+require 'active_support'
+require 'active_support/core_ext/object/blank'
+
 module ProxES
   module Helpers
     module Indices
       def filter(asked, against)
-        return against.map { |a| a.gsub(/\.\*/, '*') } if asked == ['*'] || asked == [] || asked.nil?
+        return against.map { |a| a.gsub(/\.\*/, '*') } if asked == ['*'] || asked.blank?
 
         answer = []
         against.each do |pattern|
@@ -12,6 +15,19 @@ module ProxES
         end
         answer
       end
+
+      def patterns
+        return [] if user.nil?
+        patterns_for('INDEX').map do |permission|
+          return nil if permission.pattern.blank?
+          permission.pattern.gsub(/\{user.(.*)\}/) { |_match| user.send(Regexp.last_match[1].to_sym) }
+        end.compact
+      end
+
+      def patterns_for(action)
+        return [] if user.nil?
+        Permission.for_user(user, action)
+      end
     end
   end
 end

data/lib/proxes/middleware/error_handling.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'ditty/helpers/wisper'
+
+module ProxES
+  module Middleware
+    class ErrorHandling
+      attr_reader :logger
+
+      include Wisper::Publisher
+      include Ditty::Helpers::Wisper
+
+      def initialize(app, logger = nil)
+        @app = app
+        @logger = logger || ::Ditty::Services::Logger.instance
+      end
+
+      def call(env)
+        request = Request.from_env(env)
+        code, headers, body = @app.call env
+        unless (200..299).cover? code
+          log_action(
+            :es_request_failed,
+            user: request.user,
+            details: "#{request.request_method.upcase} #{request.fullpath} (#{request.class.name})"
+          )
+        end
+        [code, headers, body]
+      rescue Errno::EHOSTUNREACH
+        error 'Could not reach Elasticsearch at ' + ENV['ELASTICSEARCH_URL']
+      rescue Errno::ECONNREFUSED, Faraday::ConnectionFailed
+        error 'Elasticsearch not listening at ' + ENV['ELASTICSEARCH_URL']
+      rescue Pundit::NotAuthorizedError, Ditty::Helpers::NotAuthenticated
+        if request.html? && request.user.nil?
+          env['rack.session']['omniauth.origin'] = request.url
+          return redirect '/_proxes/auth/identity'
+        end
+
+        user = request.user ? request.user.email : 'unauthenticated request'
+        logger.error "Access denied for #{user} by security layer: #{request.detail}"
+
+        failed request, 'Not Authorized', :es_request_denied, 401
+      rescue StandardError => e
+        raise e if env['RACK_ENV'] != 'production'
+
+        user = request.user ? request.user.email : 'unauthenticated request'
+        logger.error "Access denied for #{user} by security exception: #{request.detail}"
+        logger.error e
+
+        failed request, 'Forbidden', :es_request_denied, 403
+      end
+
+      def failed(request, message, action, code)
+        log_action(
+          action,
+          user: request.user,
+          details: "#{request.request_method.upcase} #{request.fullpath} (#{request.class.name})"
+        )
+        error message, code
+      end
+
+      # Response Helpers
+      def error(message, code = 500)
+        headers = { 'Content-Type' => 'application/json' }
+        headers['WWW-Authenticate'] = 'Basic realm="security"' if code == 401
+        [code, headers, ['{"error":"' + message + '"}']]
+      end
+
+      def redirect(destination, code = 302)
+        [code, { 'Location' => destination }, []]
+      end
+    end
+  end
+end

data/lib/proxes/middleware/metrics.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'wisper'
+
+module ProxES
+  module Middleware
+    class Metrics
+      include Wisper::Publisher
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        request = Request.from_env(env)
+        broadcast(:call_started, request)
+
+        result = @app.call request.env
+
+        broadcast(:call_completed, request) if result[0].to_i >= 200 && result[0].to_i < 300
+        result
+      end
+    end
+  end
+end

data/lib/proxes/middleware/security.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'proxes/request'
+require 'proxes/policies/request_policy'
+require 'ditty/services/logger'
+require 'ditty/helpers/pundit'
+require 'ditty/helpers/authentication'
+
+module ProxES
+  module Middleware
+    class Security
+      attr_reader :env, :logger
+
+      include Ditty::Helpers::Authentication
+      include Ditty::Helpers::Pundit
+      include Ditty::Helpers::Wisper
+
+      def initialize(app, logger = nil)
+        @app = app
+        @logger = logger || ::Ditty::Services::Logger.instance
+      end
+
+      def call(env)
+        @env = env
+        request = ProxES::Request.from_env(env)
+        log(request, 'BEFORE')
+
+        check_basic request
+        authorize request
+
+        request.index = policy_scope(request) if request.indices?
+        log(request, 'AFTER')
+
+        @app.call env
+      end
+
+      def check_basic(request)
+        auth = Rack::Auth::Basic::Request.new(request.env)
+        return false unless auth.provided? && auth.basic?
+
+        identity = ::Ditty::Identity.find(username: auth.credentials[0])
+        identity ||= ::Ditty::Identity.find(username: CGI.unescape(auth.credentials[0]))
+        return false unless identity && identity.authenticate(auth.credentials[1])
+        request.env['rack.session'] ||= {}
+        request.env['rack.session']['user_id'] = identity.user_id
+      end
+
+      def authorize(request)
+        Pundit.authorize(request.user, request, request.request_method.downcase + '?')
+      end
+
+      def log(request, stage)
+        logger.debug '============' + stage.ljust(56) + '============'
+        logger.debug '= ' + "Request: #{request.detail}".ljust(76) + ' ='
+        logger.debug '= ' + "Endpoint: #{request.endpoint}".ljust(76) + ' ='
+        logger.debug '================================================================================'
+      end
+    end
+  end
+end

data/lib/proxes/models/permission.rb

@@ -29,3 +29,9 @@ module ProxES
     end
   end
 end
+
+module Ditty
+  class User < ::Sequel::Model
+    one_to_many :permissions, class: ::ProxES::Permission
+  end
+end