synapse 0.13.5 → 0.13.7

data/README.md

@@ -152,16 +152,17 @@ The base watcher is useful in situations where you only want to use the servers
 It has the following options:
 
 * `method`: base
-* `label_filter`: optional filter to be applied to discovered service nodes
+* `label_filters`: optional list of filters to be applied to discovered service nodes
 
 ###### Filtering service nodes ######
-Synapse can be configured to only return service nodes that match a `label_filter` predicate. If provided, the `label_filter` hash should contain the following:
 
-* `label`: The label for which the filter is applied
+Synapse can be configured to only return service nodes that match a `label_filters` predicate. If provided, `label_filters` should be an array of hashes which contain the following:
+
+* `label`: The name of the label for which the filter is applied
 * `value`: The comparison value
-* `condition` (one of ['`equals`']): The type of filter condition to be applied. Only `equals` is supported at present
+* `condition` (one of ['`equals`', '`not-equals`']): The type of filter condition to be applied.
 
-Given a `label_filter`: `{ "label": "cluster", "value": "dev", "condition": "equals" }`, this will return only service nodes that contain the label value `{ "cluster": "dev" }`.
+Given a `label_filters`: `[{ "label": "cluster", "value": "dev", "condition": "equals" }]`, this will return only service nodes that contain the label value `{ "cluster": "dev" }`.
 
 ##### Zookeeper #####
 
@@ -284,6 +285,7 @@ The top level `haproxy` section of the config file has the following options:
 * `do_writes`: whether or not the config file will be written (default to `true`)
 * `do_reloads`: whether or not Synapse will reload HAProxy (default to `true`)
 * `do_socket`: whether or not Synapse will use the HAProxy socket commands to prevent reloads (default to `true`)
+* `socket_file_path`: where to find the haproxy stats socket. can be a list (if using `nbproc`)
 * `global`: options listed here will be written into the `global` section of the HAProxy config
 * `defaults`: options listed here will be written into the `defaults` section of the HAProxy config
 * `extra_sections`: additional, manually-configured `frontend`, `backend`, or `listen` stanzas
@@ -332,7 +334,9 @@ For example:
    - "bind 127.0.0.1:8081"
   reload_command: "service haproxy reload"
   config_file_path: "/etc/haproxy/haproxy.cfg"
-  socket_file_path: "/var/run/haproxy.sock"
+  socket_file_path:
+    - /var/run/haproxy.sock
+    - /var/run/haproxy2.sock
   global:
    - "daemon"
    - "user    haproxy"

data/lib/synapse/haproxy.rb

@@ -815,6 +815,10 @@ module Synapse
       @opts['do_socket'] = true unless @opts.key?('do_socket')
       @opts['do_reloads'] = true unless @opts.key?('do_reloads')
 
+      # socket_file_path can be a string or a list
+      # lets make a new option which is always a list (plural)
+      @opts['socket_file_paths'] = [@opts['socket_file_path']].flatten
+
       # how to restart haproxy
       @restart_interval = @opts.fetch('restart_interval', 2).to_i
       @restart_jitter = @opts.fetch('restart_jitter', 0).to_f
@@ -850,7 +854,9 @@ module Synapse
     def update_config(watchers)
       # if we support updating backends, try that whenever possible
       if @opts['do_socket']
-        update_backends(watchers)
+        @opts['socket_file_paths'].each do |socket_path|
+          update_backends_at(socket_path, watchers)
+        end
       else
         @restart_required = true
       end
@@ -1030,8 +1036,8 @@ module Synapse
       ]
     end
 
-    def haproxy_exec(command)
-      s = UNIXSocket.new(@opts['socket_file_path'])
+    def talk_to_socket(socket_file_path, command)
+      s = UNIXSocket.new(socket_file_path)
       s.write(command)
       s.read
     ensure
@@ -1040,11 +1046,11 @@ module Synapse
 
     # tries to set active backends via haproxy's stats socket
     # because we can't add backends via the socket, we might still need to restart haproxy
-    def update_backends(watchers)
+    def update_backends_at(socket_file_path, watchers)
       # first, get a list of existing servers for various backends
       begin
         stat_command = "show stat\n"
-        info = haproxy_exec(stat_command)
+        info = talk_to_socket(socket_file_path, stat_command)
       rescue StandardError => e
         log.warn "synapse: restart required because socket command #{stat_command} failed "\
                  "with error #{e.inspect}"
@@ -1098,7 +1104,7 @@ module Synapse
 
           # actually write the command to the socket
           begin
-            output = haproxy_exec(command)
+            output = talk_to_socket(socket_file_path, command)
           rescue StandardError => e
             log.warn "synapse: restart required because socket command #{command} failed with "\
                      "error #{e.inspect}"
@@ -1113,7 +1119,7 @@ module Synapse
         end
       end
 
-      log.info "synapse: reconfigured haproxy"
+      log.info "synapse: reconfigured haproxy via #{socket_file_path}"
     end
 
     # writes the config

data/lib/synapse/service_watcher/base.rb

@@ -21,7 +21,14 @@ class Synapse::ServiceWatcher
 
       @name = opts['name']
       @discovery = opts['discovery']
-      @label_filter = @discovery['label_filter'] || false
+
+      # deprecated singular filter
+      @singular_label_filter = @discovery['label_filter']
+      unless @singular_label_filter.nil?
+        log.warn "synapse: `label_filter` parameter is deprecated; use `label_filters` -- an array"
+      end
+
+      @label_filters = [@singular_label_filter, @discovery['label_filters']].flatten.compact
 
       @leader_election = opts['leader_election'] || false
       @leader_last_warn = Time.now - LEADER_WARN_INTERVAL
@@ -73,25 +80,36 @@ class Synapse::ServiceWatcher
     end
 
     def backends
+      filtered = backends_filtered_by_labels
+
       if @leader_election
-        if @backends.all?{|b| b.key?('id') && b['id']}
-          smallest = @backends.sort_by{ |b| b['id']}.first
-          log.debug "synapse: leader election chose one of #{@backends.count} backends " \
+        failure_warning = nil
+        if filtered.empty?
+          failure_warning = "synapse: service #{@name}: leader election failed: no backends to choose from"
+        end
+
+        all_backends_have_ids = filtered.all?{|b| b.key?('id') && b['id']}
+        unless all_backends_have_ids
+          failure_warning = "synapse: service #{@name}: leader election failed; not all backends include an id"
+        end
+
+        # no problems encountered, lets do the leader election
+        if failure_warning.nil?
+          smallest = filtered.sort_by{ |b| b['id']}.first
+          log.debug "synapse: leader election chose one of #{filtered.count} backends " \
             "(#{smallest['host']}:#{smallest['port']} with id #{smallest['id']})"
 
           return [smallest]
+
+        # we had some sort of problem, lets log about it
         elsif (Time.now - @leader_last_warn) > LEADER_WARN_INTERVAL
-          log.warn "synapse: service #{@name}: leader election failed; not all backends include an id"
           @leader_last_warn = Time.now
+          log.warn failure_warning
+          return []
         end
-
-        # if leader election fails, return no backends
-        return []
-      elsif @label_filter
-        return filter_backends_by_label(@backends, @label_filter)
       end
 
-      return @backends
+      return filtered
     end
 
     private
@@ -102,15 +120,16 @@ class Synapse::ServiceWatcher
       log.warn "synapse: warning: a stub watcher with no default servers is pretty useless" if @default_servers.empty?
     end
 
-    def filter_backends_by_label(backends, label_filter)
-      filtered_backends = []
-      backends.each do |backend|
+    def backends_filtered_by_labels
+      filtered_backends = @backends.select do |backend|
         backend_labels = backend['labels'] || {}
-        if label_filter['condition'] == 'equals' and backend_labels[label_filter['label']] == label_filter['value']
-          filtered_backends << backend
+        @label_filters.all? do |label_filter|
+          (label_filter['condition'] == 'equals' &&
+            backend_labels[label_filter['label']] == label_filter['value']) ||
+          (label_filter['condition'] == 'not-equals' &&
+            backend_labels[label_filter['label']] != label_filter['value'])
         end
       end
-      return filtered_backends
     end
 
     def set_backends(new_backends)

data/lib/synapse/version.rb

@@ -1,3 +1,3 @@
 module Synapse
-  VERSION = "0.13.5"
+  VERSION = "0.13.7"
 end

data/spec/factories/backend.rb

@@ -0,0 +1,20 @@
+FactoryGirl.define do
+  factory :backend, :class => Hash do
+    sequence(:name) { |n| "server#{n}" }
+    sequence(:host) { |n| "hostname#{n}" }
+    sequence(:port)
+
+    labels  {}
+
+    # needed to build hashes instead of classes
+    initialize_with { attributes }
+
+    # convert keys to strings, since backends always have string keys
+    after(:build) do |backend|
+      backend.keys.each do |k|
+        backend[k.to_s] = backend[k]
+        backend.delete(k)
+      end
+    end
+  end
+end

data/spec/lib/synapse/haproxy_spec.rb

@@ -66,16 +66,27 @@ describe Synapse::Haproxy do
     end
 
     context 'when we support socket updates' do
-      include_context 'generate_config is stubbed out'
+      let(:socket_file_path) { 'socket_file_path' }
       before do
         config['haproxy']['do_socket'] = true
-        config['haproxy']['socket_file_path'] = 'socket_file_path'
+        config['haproxy']['socket_file_path'] = socket_file_path
       end
 
+      include_context 'generate_config is stubbed out'
+
       it 'updates backends via the socket' do
-        expect(subject).to receive(:update_backends).with(watchers)
+        expect(subject).to receive(:update_backends_at).with(socket_file_path, watchers)
         subject.update_config(watchers)
       end
+
+      context 'when we specify multiple stats sockets' do
+        let(:socket_file_path) { ['socket_file_path1', 'socket_file_path2'] }
+
+        it 'updates all of them' do
+          expect(subject).to receive(:update_backends_at).exactly(socket_file_path.count).times
+          subject.update_config(watchers)
+        end
+      end
     end
 
     context 'when we do not support socket updates' do
@@ -83,7 +94,7 @@ describe Synapse::Haproxy do
       before { config['haproxy']['do_socket'] = false }
 
       it 'does not update the backends' do
-        expect(subject).to_not receive(:update_backends)
+        expect(subject).to_not receive(:update_backends_at)
         subject.update_config(watchers)
       end
     end

data/spec/lib/synapse/service_watcher_base_spec.rb

@@ -111,28 +111,70 @@ describe Synapse::ServiceWatcher::BaseWatcher do
     end
 
     context 'with label_filter set' do
-      let(:matching_labeled_backends) { [
-        { 'name' => 'server1', 'host' => 'server1', 'port' => 1111, 'labels' => { 'az' => 'us-east-1a' } },
-        { 'name' => 'server2', 'host' => 'server2', 'port' => 2222, 'labels' => { 'az' => 'us-east-1a' } },
-      ] }
-      let(:non_matching_labeled_backends) { [
-        { 'name' => 'server3', 'host' => 'server3', 'port' => 3333, 'labels' => { 'az' => 'us-west-1c' } },
-        { 'name' => 'server4', 'host' => 'server4', 'port' => 4444, 'labels' => { 'az' => 'us-west-2a' } },
-      ] }
-      let(:non_labeled_backends) { [
-        { 'name' => 'server5', 'host' => 'server5', 'port' => 5555 },
-      ] }
-      let(:args) {
-        testargs.merge({ 'discovery' => {
-          'method' => 'base',
-          'label_filter' => { 'condition' => 'equals', 'label' => 'az', 'value' => 'us-east-1a' } }
+      let(:matching_az) { 'us-east-1a' }
+      let(:matching_labels) { [{'az' => matching_az}] * 2 }
+      let(:non_matching_labels) { [{'az' => 'us-east-1b'}, {'az' => 'us-west-1a'}] }
+
+      let(:matching_labeled_backends) do
+        matching_labels.map{ |l| FactoryGirl.build(:backend, :labels => l) }
+      end
+      let(:non_matching_labeled_backends) do
+        non_matching_labels.map{ |l| FactoryGirl.build(:backend, :labels => l) }
+      end
+      let(:non_labeled_backends) do
+        [FactoryGirl.build(:backend, :labels => {})]
+      end
+
+      before do
+        expect(subject).to receive(:'reconfigure!').exactly(:once)
+        subject.send(:set_backends,
+          matching_labeled_backends + non_matching_labeled_backends + non_labeled_backends)
+      end
+
+      let(:condition) { 'equals' }
+      let(:label_filters) { [{ 'condition' => condition, 'label' => 'az', 'value' => 'us-east-1a' }] }
+      let(:args) do
+        testargs.merge({
+          'discovery' => {
+            'method' => 'base',
+            'label_filters' => label_filters,
+          }
         })
-      }
+      end
+
       it 'removes all backends that do not match the label_filter' do
-        expect(subject).to receive(:'reconfigure!').exactly(:once)
-        subject.send(:set_backends, matching_labeled_backends + non_matching_labeled_backends +
-                     non_labeled_backends)
-        expect(subject.backends).to eq(matching_labeled_backends)
+        expect(subject.backends).to contain_exactly(*matching_labeled_backends)
+      end
+
+      context 'when the condition is not-equals' do
+        let(:condition) { 'not-equals' }
+
+        it 'removes all backends that DO match the label_filter' do
+          expect(subject.backends).to contain_exactly(*(non_labeled_backends + non_matching_labeled_backends))
+        end
+      end
+
+      context 'with multiple labels and conditions conditions' do
+        let(:matching_region) { 'region1' }
+        let(:matching_labels) { [{'az' => matching_az, 'region' => matching_region}] * 2 }
+        let(:non_matching_labels) do
+          [
+            {'az' => matching_az, 'region' => 'non-matching'},
+            {'az' => 'non-matching', 'region' => matching_region},
+            {'az' => 'non-matching', 'region' => 'non-matching'},
+          ]
+        end
+
+        let(:label_filters) do
+          [
+            { 'condition' => 'equals', 'label' => 'az', 'value' => matching_az },
+            { 'condition' => 'equals', 'label' => 'region', 'value' => matching_region },
+          ]
+        end
+
+        it 'returns only backends that match all labels' do
+          expect(subject.backends).to contain_exactly(*matching_labeled_backends)
+        end
       end
     end
   end

data/spec/spec_helper.rb

@@ -13,6 +13,16 @@ require 'webmock/rspec'
 require 'timecop'
 Timecop.safe_mode = true
 
+# configure factory girl
+require 'factory_girl'
+RSpec.configure do |config|
+  config.include FactoryGirl::Syntax::Methods
+  config.before(:suite) do
+    FactoryGirl.find_definitions
+  end
+end
+
+# general RSpec config
 RSpec.configure do |config|
   config.run_all_when_everything_filtered = true
   config.filter_run :focus

data/synapse.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |gem|
 
   gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec", "~> 3.1.0"
+  gem.add_development_dependency "factory_girl"
   gem.add_development_dependency "pry"
   gem.add_development_dependency "pry-nav"
   gem.add_development_dependency "webmock"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: synapse
 version: !ruby/object:Gem::Version
-  version: 0.13.5
+  version: 0.13.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-08 00:00:00.000000000 Z
+date: 2016-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -107,6 +107,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: factory_girl
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -210,6 +226,7 @@ files:
 - lib/synapse/service_watcher/zookeeper.rb
 - lib/synapse/service_watcher/zookeeper_dns.rb
 - lib/synapse/version.rb
+- spec/factories/backend.rb
 - spec/lib/synapse/file_output_spec.rb
 - spec/lib/synapse/haproxy_spec.rb
 - spec/lib/synapse/service_watcher_base_spec.rb
@@ -247,6 +264,7 @@ signing_key:
 specification_version: 3
 summary: ': Write a gem summary'
 test_files:
+- spec/factories/backend.rb
 - spec/lib/synapse/file_output_spec.rb
 - spec/lib/synapse/haproxy_spec.rb
 - spec/lib/synapse/service_watcher_base_spec.rb