ksconnect 0.1.13 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ce87324cd6e4c21f489c752a6eb33944bd778e5
-  data.tar.gz: b41e6e41d0d139d45d1cc7f2180fd171ba4358b1
+  metadata.gz: 06207d64b5e16054f9879283f9b3635d43486380
+  data.tar.gz: e716c4150e97ebb0f600a99dacbd27502b4d23b3
 SHA512:
-  metadata.gz: 549d3295ac65cf384297a8077faf3610a2f9bbaeaa41d982945849d3f94a69a28f1ed7536696b0084f51dbf21d8b8f58dc33834f0b589ea1205cc5ed58fd3204
-  data.tar.gz: 54bfee7aefbf5e248271a9c04b7d253d659e3126c20244835f5c457274e1c76e0632b1b2b2e0a9426fe0f4daae1eec037994126c982c2bde1c87030a4d3e3e56
+  metadata.gz: c1cd24055c673e2585f2dc362ffe2a456561f1934a72725ca342f26f5687c4423ae450cc50c3dbed8fde16ecd6e7ca08a8f916d5e774485c9afe7ca59720d2ac
+  data.tar.gz: 51c868da513607c93a184ae2e52ef8fe3da918a211400c3cefa9daffe3796dccb635dd1b0d5219df3348c54f3c9de6c449bd1cea16f65cc97856fa17ff9fac2a

data/lib/ksconnect/api/plugin/data.rb

@@ -19,46 +19,61 @@ class KSConnect
 
           @cache = ActiveSupport::HashWithIndifferentAccess.new if @use_cache
           @cache_uuid = nil
+
+          Redis.current ||= Redis.new(driver: :hiredis)
+          $redis ||= ConnectionPool.new(size: MAX_THREADS, timeout: 8) { Redis.current }
         end
 
         def []=(field, value)
           @cache[field] = value if @use_cache
-          redis.hset(key, field, value)
-          redis.publish("core:push", { plugin_name: @plugin_name, domain_name: @domain_name, request_type: 'update' }.to_json) if @auto_notify
+          $redis.with { |redis|
+            redis.hset(key, field, value)
+            redis.publish("core:push", { plugin_name: @plugin_name, domain_name: @domain_name, request_type: 'update' }.to_json) if @auto_notify
+          }
         end
 
         def setall(hash)
           @cache.merge!(hash) if @use_cache
-          redis.mapped_hmset(key, hash)
-          redis.publish("core:push", { plugin_name: @plugin_name, domain_name: @domain_name, request_type: 'update' }.to_json) if @auto_notify
+          $redis.with { |redis|
+            redis.mapped_hmset(key, hash)
+            redis.publish("core:push", { plugin_name: @plugin_name, domain_name: @domain_name, request_type: 'update' }.to_json) if @auto_notify
+          }
         end
 
         def [](field)
-          if @use_cache
-            @cache = redis.hgetall(key) if @cache.empty?
-            @cache[field]
-          else
-            redis.hget(key, field)
-          end
+          $redis.with { |redis|
+            if @use_cache
+              @cache = redis.hgetall(key) if @cache.empty?
+              @cache[field]
+            else
+              redis.hget(key, field)
+            end
+          }
         end
 
         def getall
-          if @use_cache
-            @cache = redis.hgetall(key) if @cache.empty?; @cache
-          else
-            redis.hgetall(key)
-          end
+          $redis.with { |redis|
+            if @use_cache
+              @cache = redis.hgetall(key) if @cache.empty?; @cache
+            else
+              redis.hgetall(key)
+            end
+          }
         end
 
         def reload
-          @cache = redis.hgetall(key) if @use_cache
-          @cache
+          $redis.with { |redis|
+            @cache = redis.hgetall(key) if @use_cache
+            @cache
+          }
         end
 
         def delete(field)
-          @cache.delete(field) if @use_cache
-          redis.hdel(key, field)
-          redis.publish("core:push", { plugin_name: @plugin_name, domain_name: @domain_name, request_type: 'update' }.to_json) if @auto_notify
+          $redis.with { |redis|
+            @cache.delete(field) if @use_cache
+            redis.hdel(key, field)
+            redis.publish("core:push", { plugin_name: @plugin_name, domain_name: @domain_name, request_type: 'update' }.to_json) if @auto_notify
+          }
         end
 
         def key
@@ -69,23 +84,21 @@ class KSConnect
         private
 
         def set_data_uuid
-          begin
-            tries ||= 3
-            id = redis.hget("#{@plugin_name}:#{@type}", @domain_name)
-            if id
-              @cache_uuid = id
-            else
-              @cache_uuid = SecureRandom.uuid
-              raise "Race on setting data key failed." unless redis.hsetnx("#{@plugin_name}:#{@type}", @domain_name, @cache_uuid)
+          $redis.with { |redis|
+            begin
+              tries ||= 3
+              id = redis.hget("#{@plugin_name}:#{@type}", @domain_name)
+              if id
+                @cache_uuid = id
+              else
+                @cache_uuid = SecureRandom.uuid
+                raise "Race on setting data key failed." unless redis.hsetnx("#{@plugin_name}:#{@type}", @domain_name, @cache_uuid)
+              end
+            rescue Exception => e
+              logger.error e.message
+              retry unless (tries -= 1).zero?
             end
-          rescue Exception => e
-            logger.error e.message
-            retry unless (tries -= 1).zero?
-          end
-        end
-
-        def redis
-          Redis.current
+          }
         end
       end
     end

data/lib/ksconnect/api/plugin/domain.rb

@@ -17,38 +17,47 @@ class KSConnect
           @plugin_name = plugin_name
           @data = Data.new(plugin_name, name, type: :data)
           @private_data = Data.new(plugin_name, name, type: :private_data, auto_notify: true)
+
+          Redis.current ||= Redis.new(driver: :hiredis)
+          $redis ||= ConnectionPool.new(size: MAX_THREADS, timeout: 8) { Redis.current }
         end
 
         def update_ip_address
-          @ip_address = redis.hget(domains_key, name)
+          $redis.with { |redis|
+            @ip_address = redis.hget(domains_key, name)
+          }
         end
 
         def ip_address=(new_ip)
           @ip_address = new_ip
-          redis.hset(domains_key, name, new_ip)
-          redis.publish("core:push", { domain_name: name, plugin_name: "core", request_type: "update"}.to_json) if plugin_name == 'core'
+          $redis.with { |redis|
+            redis.hset(domains_key, name, new_ip)
+            redis.publish("core:push", { domain_name: name, plugin_name: "core", request_type: "update"}.to_json) if plugin_name == 'core'
+          }
         end
 
         def notify(data)
-          redis.lpush("ks:notifications_push", data.merge({ domain_name: @name, plugin_name: @plugin_name }).to_json)
+          $redis.with { |redis|
+            redis.lpush("ks:notifications_push", data.merge({ domain_name: @name, plugin_name: @plugin_name }).to_json)
+          }
         end
 
         def add_issue(issue_type, data)
-          redis.lpush("ks:issues_push", data.merge({ domain_name: @name, plugin_name: @plugin_name, issue_type: issue_type, request_type: 'add' }).to_json)
+          $redis.with { |redis|
+            redis.lpush("ks:issues_push", data.merge({ domain_name: @name, plugin_name: @plugin_name, issue_type: issue_type, request_type: 'add' }).to_json)
+          }
         end
 
         def clear_issue(issue_type)
-          redis.lpush("ks:issues_push", { domain_name: @name, plugin_name: @plugin_name, issue_type: issue_type, request_type: 'remove' }.to_json)
+          $redis.with { |redis|
+            redis.lpush("ks:issues_push", { domain_name: @name, plugin_name: @plugin_name, issue_type: issue_type, request_type: 'remove' }.to_json)
+          }
         end
 
         private
 
-        def redis
-          Redis.current
-        end
-
         def domains_key
-          @domain_list_uuid ||= redis.hget("#{plugin_name}:data", "domain_names")
+          @domain_list_uuid ||= $redis.with { |redis| redis.hget("#{plugin_name}:data", "domain_names") }
           "kloudsec_data:#{@domain_list_uuid}"
         end
       end

data/lib/ksconnect/api/plugin.rb

@@ -15,6 +15,9 @@ class KSConnect
         @main_plugin = main
         @domains = {}
 
+        Redis.current ||= Redis.new(driver: :hiredis)
+        $redis ||= ConnectionPool.new(size: MAX_THREADS, timeout: 8) { Redis.current }
+
         load_domains
         subscribe_to_events
       end
@@ -31,7 +34,7 @@ class KSConnect
       # Note that it does not update the ip address of existing domains.
       def load_domains
         # load domain list
-        domain_to_ip = redis.hgetall(domains_key)
+        domain_to_ip = $redis.with { |redis| redis.hgetall(domains_key) }
 
         # add new domains
         new_domains = domain_to_ip.keys - @domains.values.map(&:name)
@@ -67,14 +70,16 @@ class KSConnect
 
       def update_action(action_id, state, body="")
         raise "Updating action with empty action_id" if action_id.nil? or action_id.empty?
-        redis.multi do |redis_transaction|
-          redis_transaction.mapped_hmset "kloudsec_data:#{action_id}", {
-            plugin_name: @name,
-            uuid: action_id,
-            state: state,
-            response: body,
-          }
-          redis_transaction.expire "kloudsec_data:#{action_id}", 300
+        $redis.with do |redis|
+          redis.multi do |redis_transaction|
+            redis_transaction.mapped_hmset "kloudsec_data:#{action_id}", {
+                plugin_name: @name,
+                uuid: action_id,
+                state: state,
+                response: body,
+            }
+            redis_transaction.expire "kloudsec_data:#{action_id}", 300
+          end
         end
       end
 
@@ -103,11 +108,13 @@ class KSConnect
             raise "Invalid request type"
         end
 
-        redis.publish("core:push", { domain_name: domain_name, plugin_name: @name, request_type: request_type }.to_json) if should_repush?
+        $redis.with { |redis|
+          redis.publish("core:push", { domain_name: domain_name, plugin_name: @name, request_type: request_type }.to_json) if should_repush?
+        }
       end
 
       def domains_key
-        @domain_list_uuid ||= redis.hget("#{@name}:data", "domain_names")
+        @domain_list_uuid ||= $redis.with { |redis| redis.hget("#{@name}:data", "domain_names") }
         "kloudsec_data:#{@domain_list_uuid}"
       end
 
@@ -118,11 +125,7 @@ class KSConnect
       end
 
       def get_ip_for(domain_name)
-        redis.hget(domains_key, domain_name)
-      end
-
-      def redis
-        Redis.current
+        $redis.with { |redis| redis.hget(domains_key, domain_name) }
       end
     end
   end

data/lib/ksconnect/helpers.rb

@@ -1,8 +1,10 @@
+require 'active_support/core_ext/object/try'
+
 class KSConnect
   module Helpers
     def ip_address_for(domain)
-      if all_domains[domain].present?
-        all_domains[domain].ip_address
+      if all_domains[domain.name]
+        all_domains[domain.name].ip_address
       else
         "kloudsec.com" # go to something safe if ip is invalid
       end
@@ -14,32 +16,40 @@ class KSConnect
     end
 
     def https_redirect_enabled?(domain)
-      https_enabled?(domain) && plugins[:ssl].domains[domain].data['redirect'] == "true"
+      https_enabled?(domain) && plugins[:ssl].domains[domain.name].data['redirect'] == "true"
     end
 
     def https_rewriting_enabled?(domain)
-      https_enabled?(domain) && plugins[:ssl].domains[domain].data['rewriteHTTPS'] == "true"
+      https_enabled?(domain) && plugins[:ssl].domains[domain.name].data['rewriteHTTPS'] == "true"
     end
 
     def waf_enabled?(domain)
-      plugins[:web_shield].domains.include?(domain)
+      plugins[:web_shield].domains.has_key?(domain.name)
     end
 
     def waf_learning?(domain)
-      waf_enabled?(domain) && plugins[:web_shield].domains[domain].data['learning']
+      waf_enabled?(domain) && plugins[:web_shield].domains[domain.name].data['learning']
+    end
+
+    def waf_location_config(domain)
+      waf_enabled?(domain) ? plugins[:web_shield].domains[domain.name].private_data['location.conf'] : ""
+    end
+
+    def waf_server_config(domain)
+      waf_enabled?(domain) ? plugins[:web_shield].domains[domain.name].private_data['server.conf'] : ""
     end
 
     def pagespeed_enabled?(domain)
-      plugins[:mod_cache].domains.include?(domain)
+      plugins[:mod_cache].domains.has_key?(domain.name)
     end
 
     def pending_autossl?(domain)
       p, k = autossl_verification_path_and_key_for(domain)
-      plugins[:autossl].domains.include?(domain) && p && k
+      plugins[:autossl].domains.has_key?(domain.name) && p && k
     end
 
     def autossl_verification_path_and_key_for(domain)
-      d = plugins[:autossl].domains[domain].private_data
+      d = plugins[:autossl].domains[domain.name].try(:private_data)
       if d
         return d['verify_endpoint'], d['verify_content']
       else
@@ -48,8 +58,8 @@ class KSConnect
     end
 
     def ssl_key_and_cert_for(domain)
-      ssl = plugins[:ssl].domains[domain].private_data
-      autossl = plugins[:autossl].domains[domain].private_data
+      ssl = plugins[:ssl].domains[domain.name].try(:private_data)
+      autossl = plugins[:autossl].domains[domain.name].try(:private_data)
 
       if ssl && ssl['key'] && ssl['cert']
         return ssl['key'], ssl['cert']

data/lib/ksconnect.rb

@@ -1,4 +1,6 @@
 require 'redis'
+require 'thread/pool'
+require 'connection_pool'
 require 'ksconnect/api'
 require 'ksconnect/api/plugin'
 require 'ksconnect/api/plugin/config'
@@ -6,15 +8,19 @@ require 'ksconnect/api/plugin/data'
 require 'ksconnect/api/plugin/domain'
 require 'ksconnect/helpers'
 require 'ksconnect/logs'
-require 'thread/pool'
 
 class KSConnect
   include Logs
   attr_reader :api
   attr_reader :plugin
 
+  MIN_THREADS = ENV['MIN_THREADS'] || 5
+  MAX_THREADS = ENV['MAX_THREADS'] || 25
+
   def initialize(*args)
     Redis.current ||= Redis.new(driver: :hiredis)
+    $redis ||= ConnectionPool.new(size: MAX_THREADS, timeout: 8) { Redis.current }
+
     plugins = args
 
     additional_options = args.last.is_a?(Hash) ? args.last : nil
@@ -28,7 +34,7 @@ class KSConnect
   end
 
   def self.thread_pool
-    @@thread_pool ||= Thread.pool(3, 50)
+    @@thread_pool ||= Thread.pool(MIN_THREADS, MAX_THREADS)
   end
 
   def self.channel(name)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ksconnect
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.2.0
 platform: ruby
 authors:
 - Ivan Poon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-01 00:00:00.000000000 Z
+date: 2016-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -66,6 +66,90 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |-
   KSConnect provides a connection interface for Kloudsec plugins by exposing a simple to use
   API for managing and synchronizing plugin data.