proxes 0.9.4 → 0.9.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a89e0871c6d36ce3b7e6ff19c4b7fa7556cedeedd0d364bd78d0aa741b06c6e9
-  data.tar.gz: b126f70689158d474c5f096386c13a46ffdb8e58c90c7c77181a11cef3de14d9
+  metadata.gz: 0ef233fd8fd3cf4118e820483e703279eac3921ae2c92dee336f802c0c6f5df2
+  data.tar.gz: 8e7fb52b8b0e152de449d490545d96944502cce8db7491878d90e9042b572c1b
 SHA512:
-  metadata.gz: 83df2e988cb4f9d64fae3693603dddb03912483b29e04b3c65e0ee691b341b2fe68d029e2253559534c23ee15fad129d83b7954261a297f2ab004e97dc81f2ba
-  data.tar.gz: 591c032478769ea14170e7dfb8ae92c7e34f0a5ad23dc4569bfcb5745ba47bcaa1e1585c1b4e2a5ec669c88fbb2417b9253f6427fbf671b181e8d8b21db3cbbe
+  metadata.gz: 4301310adb06293c72b22595985959883c46a316a702995074d5f74b30535c54541ab185725f20edc2b8ba54dcd19d31076bac1e3427b78ae93f0f701b8bd371
+  data.tar.gz: a40f54478277272699c56a5067dd4cf13c8f59feebed6e6cc8d98ec06678e030b6ef1a683c4f3c676f74e7613afb04b8d1e3a8b4bf9d604015d096c58f0014e6

data/.gitignore

@@ -13,6 +13,8 @@
 .vagrant
 *.db
 /Gemfile.dev.lock
+/Gemfile.ci.lock
 proxes-*.gem
 migrations
 .env
+pids

data/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:2.4-alpine
+FROM ruby:2.4-alpine3.4
 MAINTAINER Jurgens du Toit <jurgens@datatools.io>
 
 EXPOSE 9292
@@ -25,7 +25,7 @@ RUN apk add --update \
 
 ADD views /usr/src/app/views
 COPY config.ru /usr/src/app/
-COPY config/logger.yml /usr/src/app/config/
+COPY config/settings.yml /usr/src/app/config/
 COPY config/puma.rb /usr/src/app/config/
 COPY Gemfile.deploy /usr/src/app/Gemfile
 COPY Gemfile.deploy.lock /usr/src/app/Gemfile.lock

data/Gemfile.deploy

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 source 'https://rubygems.org'
 
 gem 'dotenv'

data/Gemfile.deploy.lock

@@ -1,58 +1,70 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.4)
+    activesupport (5.2.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    backports (3.11.1)
-    bcrypt (3.1.11)
+    ansi (1.5.0)
+    ast (2.4.0)
+    backports (3.11.3)
+    bcrypt (3.1.12)
     bcrypt-ruby (3.1.5)
       bcrypt (>= 3.1.3)
     concurrent-ruby (1.0.5)
-    connection_pool (2.2.1)
-    ditty (0.4.0)
+    connection_pool (2.2.2)
+    ditty (0.6.0)
       activesupport (>= 3)
       bcrypt (~> 3.1)
       haml (~> 5.0)
       logger (~> 1.0)
+      mail (>= 1.7)
+      oga (>= 2.14)
       omniauth (~> 1.0)
       omniauth-identity (~> 1.0)
       pundit (~> 1.0)
       rack-contrib (~> 1.0)
       rake (~> 12.0)
-      sequel (~> 4.0)
-      sinatra (~> 2.0)
+      sequel (>= 4.0)
+      sinatra (>= 2.0)
       sinatra-contrib (~> 2.0)
       sinatra-flash (~> 0.3)
+      thor (>= 0.20)
       tilt (>= 2)
+      will_paginate (>= 3.1)
       wisper (~> 2.0)
-    dotenv (2.2.1)
-    elasticsearch (6.0.1)
-      elasticsearch-api (= 6.0.1)
-      elasticsearch-transport (= 6.0.1)
-    elasticsearch-api (6.0.1)
+    dotenv (2.5.0)
+    elasticsearch (6.1.0)
+      elasticsearch-api (= 6.1.0)
+      elasticsearch-transport (= 6.1.0)
+    elasticsearch-api (6.1.0)
       multi_json
-    elasticsearch-transport (6.0.1)
+    elasticsearch-transport (6.1.0)
       faraday
       multi_json
-    faraday (0.14.0)
+    faraday (0.15.2)
       multipart-post (>= 1.2, < 3)
     haml (5.0.4)
       temple (>= 0.8.0)
       tilt
     hashie (3.5.7)
     highline (1.7.10)
-    i18n (0.9.4)
+    i18n (1.0.1)
       concurrent-ruby (~> 1.0)
     logger (1.2.8)
+    mail (2.7.0)
+      mini_mime (>= 0.1.1)
+    mini_mime (1.0.0)
     minitest (5.11.3)
     multi_json (1.13.1)
     multipart-post (2.0.0)
-    mustermann (1.0.1)
+    mustermann (1.0.2)
     net-http-persistent (3.0.0)
       connection_pool (~> 2.2)
+    oga (2.15)
+      ast
+      ruby-ll (~> 2.1)
     omniauth (1.8.1)
       hashie (>= 3.4.6, < 3.6.0)
       rack (>= 1.6.2, < 3)
@@ -62,7 +74,7 @@ GEM
       bcrypt-ruby (~> 3.0)
       omniauth (~> 1.0)
     pg (1.0.0)
-    proxes (0.9.2)
+    proxes (0.9.4)
       activesupport (>= 3)
       bcrypt (~> 3.1)
       ditty (>= 0.2)
@@ -84,35 +96,41 @@ GEM
       sinatra-flash (~> 0.3)
       tilt (>= 2)
       wisper (~> 2.0)
-    puma (3.11.2)
+    puma (3.12.0)
     pundit (1.1.0)
       activesupport (>= 3.0.0)
-    rack (2.0.4)
+    rack (2.0.5)
     rack-contrib (1.2.0)
       rack (>= 0.9.1)
-    rack-protection (2.0.0)
+    rack-protection (2.0.3)
       rack
-    rake (12.3.0)
+    rake (12.3.1)
+    ruby-ll (2.1.2)
+      ansi
+      ast
     sequel (4.49.0)
-    sinatra (2.0.0)
+    sinatra (2.0.3)
       mustermann (~> 1.0)
       rack (~> 2.0)
-      rack-protection (= 2.0.0)
+      rack-protection (= 2.0.3)
       tilt (~> 2.0)
-    sinatra-contrib (2.0.0)
-      backports (>= 2.0)
+    sinatra-contrib (2.0.3)
+      activesupport (>= 4.0.0)
+      backports (>= 2.8.2)
       multi_json
       mustermann (~> 1.0)
-      rack-protection (= 2.0.0)
-      sinatra (= 2.0.0)
+      rack-protection (= 2.0.3)
+      sinatra (= 2.0.3)
       tilt (>= 1.3, < 3)
     sinatra-flash (0.3.0)
       sinatra (>= 1.0.0)
     temple (0.8.0)
+    thor (0.20.0)
     thread_safe (0.3.6)
     tilt (2.0.8)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
+    will_paginate (3.1.6)
     wisper (2.0.0)
 
 PLATFORMS

data/Gemfile.dev

@@ -1,11 +1,13 @@
+# frozen_string_literal: true
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in proxes.gemspec
 gemspec
 
+gem 'ditty', path: '../ditty'
 gem 'dotenv'
 gem 'pry-byebug'
 gem 'puma'
 gem 'rerun', git: 'https://github.com/alexch/rerun.git', branch: 'master'
 gem 'simplecov'
-gem 'sqlite3'
+gem 'pg'

data/config/settings.yml

@@ -0,0 +1,7 @@
+---
+logger:
+  loggers:
+  - name: default
+    class: Logger
+    level: WARN
+    options: $stdout

data/docker-compose.yml

@@ -2,16 +2,19 @@ version: '3'
 services:
   db:
     image: postgres
-  es:
+    container_name: postgres
+  elasticsearch:
     image: elasticsearch
+    container_name: elasticsearch
   web:
     image: eagerelk/proxes:latest
+    container_name: web-proxes
     command: web-proxes
     ports:
       - '9292:9292'
     environment:
       - DATABASE_URL=postgres://postgres:@db/postgres
-      - ELASTICSEARCH_URL=http://es:9200
+      - ELASTICSEARCH_URL=http://elasticsearch:9200
     depends_on:
       - db
-      - es
+      - elasticsearch

data/lib/ditty/components/proxes.rb

@@ -8,6 +8,7 @@ module Ditty
       controllers = File.expand_path('../../../proxes/controllers', __FILE__)
       Dir.glob("#{controllers}/*.rb").each { |f| require f }
       require 'proxes/models/permission'
+      require 'proxes/services/listener'
     end
 
     def self.migrations
@@ -25,6 +26,7 @@ module Ditty
     def self.routes
       load
       {
+        '/search' => ::ProxES::Search,
         '/status' => ::ProxES::Status,
         '/permissions' => ::ProxES::Permissions
       }
@@ -34,6 +36,7 @@ module Ditty
       load
       [
         { order: 0, link: '/status/check', text: 'Status Check', target: ::ProxES::Status, icon: 'dashboard' },
+        { order: 1, link: '/search', text: 'Search', target: ::ProxES::Status, icon: 'search' },
         { order: 15, link: '/permissions/', text: 'Permissions', target: ::ProxES::Permission, icon: 'check-square' }
       ]
     end

data/lib/proxes/controllers/search.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object/blank'
+require 'ditty/controllers/application'
+require 'proxes/services/search'
+
+module ProxES
+  class Search < Ditty::Application
+    set base_path: "#{settings.map_path}/search"
+
+    get '/' do
+      page = (params['page'] || 1).to_i
+      size = (params['count'] || 25).to_i
+      from = ((page - 1) * size)
+      params['q'] = '*' if params['q'].blank?
+      result = ProxES::Services::Search.search(params['q'], index: params['indices'], from: from, size: size)
+      haml :"#{view_location}/index",
+           locals: {
+             title: 'Search',
+             indices: ProxES::Services::Search.indices,
+             fields: ProxES::Services::Search.fields(params['indices']),
+             result: result
+           }
+    end
+
+    get '/fields/?:indices?/?' do
+      json ProxES::Services::Search.fields params['indices']
+    end
+
+    get '/indices/?' do
+      json ProxES::Services::Search.indices
+    end
+
+    get '/values/:field/?:indices?/?' do |field|
+      size = params['size'] || 25
+      json ProxES::Services::Search.values(field, size: size, index: params['indices'])
+    end
+
+    def find_template(views, name, engine, &block)
+      super(views, name, engine, &block) # Root
+      super(::Ditty::ProxES.view_folder, name, engine, &block) # This Component
+      super(::Ditty::App.view_folder, name, engine, &block) # Ditty
+    end
+  end
+end

data/lib/proxes/controllers/status.rb

@@ -19,7 +19,7 @@ module ProxES
     get '/check' do
       checks = []
       begin
-        health = cluster_health
+        health = client.cluster.health level: 'cluster'
         checks << { text: 'Cluster Reachable', passed: true, value: health['cluster_name'] }
         checks << { text: 'Cluster Health', passed: health['status'] == 'green', value: health['status'] }
 

data/lib/proxes/forwarder.rb

@@ -10,13 +10,6 @@ module ProxES
     include ProxES::Services::ES
 
     def call(env)
-      forward(env)
-    rescue SocketError
-      headers = { 'Content-Type' => 'application/json' }
-      [500, headers, ['{"error":"Could not connect to Elasticsearch"}']]
-    end
-
-    def forward(env)
       source = Rack::Request.new(env)
       response = conn.send(source.request_method.downcase) do |req|
         source_body = body_from(source)

data/lib/proxes/middleware/error_handling.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-require 'ditty/helpers/wisper'
+require 'wisper'
+require 'proxes/request'
+require 'ditty/services/logger'
 
 module ProxES
   module Middleware
@@ -8,7 +10,6 @@ module ProxES
       attr_reader :logger
 
       include Wisper::Publisher
-      include Ditty::Helpers::Wisper
 
       def initialize(app, logger = nil)
         @app = app
@@ -16,47 +17,29 @@ module ProxES
       end
 
       def call(env)
-        request = Request.from_env(env)
-        code, headers, body = @app.call env
-        unless (200..299).cover? code
-          log_action(
-            :es_request_failed,
-            user: request.user,
-            details: "#{request.request_method.upcase} #{request.fullpath} (#{request.class.name})"
-          )
-        end
-        [code, headers, body]
+        request = ProxES::Request.from_env(env)
+        response = @app.call env
+        broadcast(:es_request_failed, request, response) unless (200..299).cover?(response[0])
+        response
       rescue Errno::EHOSTUNREACH
         error 'Could not reach Elasticsearch at ' + ENV['ELASTICSEARCH_URL']
-      rescue Errno::ECONNREFUSED, Faraday::ConnectionFailed
+      rescue Errno::ECONNREFUSED, Faraday::ConnectionFailed, SocketError
         error 'Elasticsearch not listening at ' + ENV['ELASTICSEARCH_URL']
-      rescue Pundit::NotAuthorizedError, Ditty::Helpers::NotAuthenticated
-        if request.html? && request.user.nil?
-          env['rack.session']['omniauth.origin'] = request.url
-          return redirect '/_proxes/auth/identity'
-        end
-
-        user = request.user ? request.user.email : 'unauthenticated request'
-        logger.error "Access denied for #{user} by security layer: #{request.detail}"
-
-        failed request, 'Not Authorized', :es_request_denied, 401
+      rescue Pundit::NotAuthorizedError, Ditty::Helpers::NotAuthenticated => e
+        broadcast(:es_request_denied, request, e)
+        log_not_authorized request
+        raise e if env['APP_ENV'] == 'development'
+        request.html? && request.user.nil? ? login_and_redirect(request) : error('Not Authorized', 401)
       rescue StandardError => e
-        raise e if env['RACK_ENV'] != 'production'
-
-        user = request.user ? request.user.email : 'unauthenticated request'
-        logger.error "Access denied for #{user} by security exception: #{request.detail}"
-        logger.error e
-
-        failed request, 'Forbidden', :es_request_denied, 403
+        broadcast(:es_request_denied, request, e)
+        log_not_authorized request
+        raise e if env['APP_ENV'] == 'development'
+        error 'Forbidden', 403
       end
 
-      def failed(request, message, action, code)
-        log_action(
-          action,
-          user: request.user,
-          details: "#{request.request_method.upcase} #{request.fullpath} (#{request.class.name})"
-        )
-        error message, code
+      def log_not_authorized(request)
+        user = request.user ? request.user.email : 'unauthenticated request'
+        logger.error "Access denied for #{user} by security layer: #{request.detail}"
       end
 
       # Response Helpers
@@ -66,6 +49,11 @@ module ProxES
         [code, headers, ['{"error":"' + message + '"}']]
       end
 
+      def login_and_redirect(request)
+        request.session['omniauth.origin'] = request.url unless request.url == '/_proxes/auth/identity'
+        redirect '/_proxes/auth/identity'
+      end
+
       def redirect(destination, code = 302)
         [code, { 'Location' => destination }, []]
       end

data/lib/proxes/middleware/security.rb

@@ -13,7 +13,6 @@ module ProxES
 
       include Ditty::Helpers::Authentication
       include Ditty::Helpers::Pundit
-      include Ditty::Helpers::Wisper
 
       def initialize(app, logger = nil)
         @app = app

data/lib/proxes/models/permission.rb

@@ -24,8 +24,18 @@ module ProxES
       validates_includes self.class.verbs, :verb
     end
 
-    def self.verbs
-      %w[GET POST PUT DELETE HEAD OPTIONS TRACE INDEX]
+    class << self
+      def verbs
+        %w[GET POST PUT DELETE HEAD OPTIONS TRACE INDEX]
+      end
+
+      def from_audit_log(audit_log)
+        match = audit_log.details.match(/^(\w)+ (\S+)/)
+        {
+          verb: match[1],
+          path: match[2]
+        }
+      end
     end
   end
 end
@@ -35,3 +45,9 @@ module Ditty
     one_to_many :permissions, class: ::ProxES::Permission
   end
 end
+
+module Ditty
+  class Role < ::Sequel::Model
+    one_to_many :permissions, class: ::ProxES::Permission
+  end
+end