synapse 0.11.1 → 0.12.1

data/lib/synapse/service_watcher/zookeeper.rb

@@ -1,13 +1,18 @@
 require "synapse/service_watcher/base"
 
+require 'thread'
 require 'zk'
 
 module Synapse
   class ZookeeperWatcher < BaseWatcher
     NUMBERS_RE = /^\d+$/
 
+    @@zk_pool = {}
+    @@zk_pool_count = {}
+    @@zk_pool_lock = Mutex.new
+
     def start
-      @zk_hosts = @discovery['hosts'].shuffle.join(',')
+      @zk_hosts = @discovery['hosts'].sort.join(',')
 
       @watcher = nil
       @zk = nil
@@ -22,6 +27,8 @@ module Synapse
     end
 
     def ping?
+      # @zk being nil implies no session *or* a lost session, do not remove
+      # the check on @zk being truthy
       @zk && @zk.connected?
     end
 
@@ -45,7 +52,7 @@ module Synapse
       @zk.create(path, ignore: :node_exists)
     end
 
-    # find the current backends at the discovery path; sets @backends
+    # find the current backends at the discovery path
     def discover
       log.info "synapse: discovering backends for service #{@name}"
 
@@ -54,7 +61,7 @@ module Synapse
         node = @zk.get("#{@discovery['path']}/#{id}")
 
         begin
-          host, port, name = deserialize_service_instance(node.first)
+          host, port, name, weight = deserialize_service_instance(node.first)
         rescue StandardError => e
           log.error "synapse: invalid data in ZK node #{id} at #{@discovery['path']}: #{e}"
         else
@@ -65,26 +72,17 @@ module Synapse
           numeric_id = NUMBERS_RE =~ numeric_id ? numeric_id.to_i : nil
 
           log.debug "synapse: discovered backend #{name} at #{host}:#{server_port} for service #{@name}"
-          new_backends << { 'name' => name, 'host' => host, 'port' => server_port, 'id' => numeric_id}
+          new_backends << { 'name' => name, 'host' => host, 'port' => server_port, 'id' => numeric_id, 'weight' => weight }
         end
       end
 
-      if new_backends.empty?
-        if @default_servers.empty?
-          log.warn "synapse: no backends and no default servers for service #{@name}; using previous backends: #{@backends.inspect}"
-        else
-          log.warn "synapse: no backends for service #{@name}; using default servers: #{@default_servers.inspect}"
-          @backends = @default_servers
-        end
-      else
-        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
-        set_backends(new_backends)
-      end
+      set_backends(new_backends)
     end
 
     # sets up zookeeper callbacks if the data at the discovery path changes
     def watch
       return if @zk.nil?
+      log.debug "synapse: setting watch at #{@discovery['path']}"
 
       @watcher.unsubscribe unless @watcher.nil?
       @watcher = @zk.register(@discovery['path'], &watcher_callback)
@@ -94,6 +92,7 @@ module Synapse
         log.error "synapse: zookeeper watcher path #{@discovery['path']} does not exist!"
         raise RuntimeError.new('could not set a ZK watch on a node that should exist')
       end
+      log.debug "synapse: set watch at #{@discovery['path']}"
     end
 
     # handles the event that a watched path has changed in zookeeper
@@ -103,26 +102,55 @@ module Synapse
         watch
         # Rediscover
         discover
-        # send a message to calling class to reconfigure
-        reconfigure!
       end
     end
 
     def zk_cleanup
       log.info "synapse: zookeeper watcher cleaning up"
 
-      @watcher.unsubscribe unless @watcher.nil?
-      @watcher = nil
-
-      @zk.close! unless @zk.nil?
-      @zk = nil
+      begin
+        @watcher.unsubscribe unless @watcher.nil?
+        @watcher = nil
+      ensure
+        @@zk_pool_lock.synchronize {
+          if @@zk_pool.has_key?(@zk_hosts)
+            @@zk_pool_count[@zk_hosts] -= 1
+            # Last thread to use the connection closes it
+            if @@zk_pool_count[@zk_hosts] == 0
+              log.info "synapse: closing zk connection to #{@zk_hosts}"
+              begin
+                @zk.close! unless @zk.nil?
+              ensure
+                @@zk_pool.delete(@zk_hosts)
+              end
+            end
+          end
+          @zk = nil
+        }
+      end
 
       log.info "synapse: zookeeper watcher cleaned up successfully"
     end
 
     def zk_connect
       log.info "synapse: zookeeper watcher connecting to ZK at #{@zk_hosts}"
-      @zk = ZK.new(@zk_hosts)
+
+      # Ensure that all Zookeeper watcher re-use a single zookeeper
+      # connection to any given set of zk hosts.
+      @@zk_pool_lock.synchronize {
+        unless @@zk_pool.has_key?(@zk_hosts)
+          log.info "synapse: creating pooled connection to #{@zk_hosts}"
+          @@zk_pool[@zk_hosts] = ZK.new(@zk_hosts, :timeout => 5, :thread => :per_callback)
+          @@zk_pool_count[@zk_hosts] = 1
+          log.info "synapse: successfully created zk connection to #{@zk_hosts}"
+        else
+          @@zk_pool_count[@zk_hosts] += 1
+          log.info "synapse: re-using existing zookeeper connection to #{@zk_hosts}"
+        end
+      }
+
+      @zk = @@zk_pool[@zk_hosts]
+      log.info "synapse: retrieved zk connection to #{@zk_hosts}"
 
       # handle session expiry -- by cleaning up zk, this will make `ping?`
       # fail and so synapse will exit
@@ -146,8 +174,9 @@ module Synapse
       host = decoded['host'] || (raise ValueError, 'instance json data does not have host key')
       port = decoded['port'] || (raise ValueError, 'instance json data does not have port key')
       name = decoded['name'] || nil
+      weight = decoded['weight'] || nil
 
-      return host, port, name
+      return host, port, name, weight
     end
   end
 end

data/lib/synapse/version.rb

@@ -1,3 +1,3 @@
 module Synapse
-  VERSION = "0.11.1"
+  VERSION = "0.12.1"
 end

data/spec/lib/synapse/haproxy_spec.rb

@@ -5,9 +5,28 @@ class MockWatcher; end;
 describe Synapse::Haproxy do
   subject { Synapse::Haproxy.new(config['haproxy']) }
 
-  it 'updating the config' do
+  let(:mockwatcher) do
     mockWatcher = double(Synapse::ServiceWatcher)
-    subject.should_receive(:generate_config)
-    subject.update_config([mockWatcher])
+    allow(mockWatcher).to receive(:name).and_return('example_service')
+    backends = [{ 'host' => 'somehost', 'port' => '5555'}]
+    allow(mockWatcher).to receive(:backends).and_return(backends)
+    allow(mockWatcher).to receive(:haproxy).and_return({'server_options' => "check inter 2000 rise 3 fall 2"})
+    mockWatcher
   end
+
+  it 'updating the config' do
+    expect(subject).to receive(:generate_config)
+    subject.update_config([mockwatcher])
+  end
+
+  it 'generates backend stanza' do
+    mockConfig = []
+    expect(subject.generate_backend_stanza(mockwatcher, mockConfig)).to eql(["\nbackend example_service", [], ["\tserver somehost:5555 somehost:5555 cookie somehost:5555 check inter 2000 rise 3 fall 2"]])
+  end
+
+  it 'generates backend stanza without cookies for tcp mode' do
+    mockConfig = ['mode tcp']
+    expect(subject.generate_backend_stanza(mockwatcher, mockConfig)).to eql(["\nbackend example_service", ["\tmode tcp"], ["\tserver somehost:5555 somehost:5555 check inter 2000 rise 3 fall 2"]])
+  end
+
 end

data/spec/lib/synapse/service_watcher_base_spec.rb

@@ -6,7 +6,7 @@ end
 
 describe Synapse::BaseWatcher do
   let(:mocksynapse) { double() }
-  subject { Synapse::BaseWatcher.new(args, mocksynapse) } 
+  subject { Synapse::BaseWatcher.new(args, mocksynapse) }
   let(:testargs) { { 'name' => 'foo', 'discovery' => { 'method' => 'base' }, 'haproxy' => {} }}
 
   def remove_arg(name)
@@ -37,18 +37,76 @@ describe Synapse::BaseWatcher do
     end
   end
 
-  context "with default_servers" do
-    default_servers = ['server1', 'server2']
+  context 'set_backends test' do
+    default_servers = [
+      {'name' => 'default_server1', 'host' => 'default_server1', 'port' => 123},
+      {'name' => 'default_server2', 'host' => 'default_server2', 'port' => 123}
+    ]
+    backends = [
+      {'name' => 'server1', 'host' => 'server1', 'port' => 123},
+      {'name' => 'server2', 'host' => 'server2', 'port' => 123}
+    ]
     let(:args) { testargs.merge({'default_servers' => default_servers}) }
-    it('sets default backends to default_servers') { expect(subject.backends).to equal(default_servers) }
 
-    context "with keep_default_servers set" do
-      let(:args) { testargs.merge({'default_servers' => default_servers, 'keep_default_servers' => true}) }
-      let(:new_backends) { ['discovered1', 'discovered2'] }
+    it 'sets backends' do
+      expect(subject).to receive(:'reconfigure!').exactly(:once)
+      expect(subject.send(:set_backends, backends)).to equal(true)
+      expect(subject.backends).to eq(backends)
+    end
+
+    it 'removes duplicate backends' do
+      expect(subject).to receive(:'reconfigure!').exactly(:once)
+      duplicate_backends = backends + backends
+      expect(subject.send(:set_backends, duplicate_backends)).to equal(true)
+      expect(subject.backends).to eq(backends)
+    end
+
+    it 'sets backends to default_servers if no backends discovered' do
+      expect(subject).to receive(:'reconfigure!').exactly(:once)
+      expect(subject.send(:set_backends, [])).to equal(true)
+      expect(subject.backends).to eq(default_servers)
+    end
+
+    context 'with no default_servers' do
+      let(:args) { remove_arg 'default_servers' }
+      it 'uses previous backends if no default_servers set' do
+        expect(subject).to receive(:'reconfigure!').exactly(:once)
+        expect(subject.send(:set_backends, backends)).to equal(true)
+        expect(subject.send(:set_backends, [])).to equal(false)
+        expect(subject.backends).to eq(backends)
+      end
+    end
+
+    context 'with no default_servers set and use_previous_backends disabled' do
+      let(:args) {
+        remove_arg 'default_servers'
+        testargs.merge({'use_previous_backends' => false})
+      }
+      it 'removes all backends if no default_servers set and use_previous_backends disabled' do
+        expect(subject).to receive(:'reconfigure!').exactly(:twice)
+        expect(subject.send(:set_backends, backends)).to equal(true)
+        expect(subject.backends).to eq(backends)
+        expect(subject.send(:set_backends, [])).to equal(true)
+        expect(subject.backends).to eq([])
+      end
+    end
+
+    it 'calls reconfigure only once for duplicate backends' do
+      expect(subject).to receive(:'reconfigure!').exactly(:once)
+      expect(subject.send(:set_backends, backends)).to equal(true)
+      expect(subject.backends).to eq(backends)
+      expect(subject.send(:set_backends, backends)).to equal(false)
+      expect(subject.backends).to eq(backends)
+    end
 
+    context 'with keep_default_servers set' do
+      let(:args) {
+        testargs.merge({'default_servers' => default_servers, 'keep_default_servers' => true})
+      }
       it('keeps default_servers when setting backends') do
-        subject.send(:set_backends, new_backends)
-        expect(subject.backends).to eq(default_servers + new_backends)
+        expect(subject).to receive(:'reconfigure!').exactly(:once)
+        expect(subject.send(:set_backends, backends)).to equal(true)
+        expect(subject.backends).to eq(backends + default_servers)
       end
     end
   end

data/spec/lib/synapse/service_watcher_docker_spec.rb

@@ -46,12 +46,7 @@ describe Synapse::DockerWatcher do
     end
     it('has a happy first run path, configuring backends') do
       expect(subject).to receive(:containers).and_return(['container1'])
-      expect(subject).to receive(:configure_backends).with(['container1'])
-      subject.send(:watch)
-    end
-    it('does not call configure_backends if there is no change') do
-      expect(subject).to receive(:containers).and_return([])
-      expect(subject).to_not receive(:configure_backends)
+      expect(subject).to receive(:set_backends).with(['container1'])
       subject.send(:watch)
     end
   end
@@ -65,33 +60,6 @@ describe Synapse::DockerWatcher do
     end
   end
 
-  context "configure_backends tests" do
-    before(:each) do
-      expect(subject.synapse).to receive(:'reconfigure!').at_least(:once)
-    end
-    it 'runs' do
-      expect { subject.send(:configure_backends, []) }.not_to raise_error
-    end
-    it 'sets backends right' do
-      subject.send(:configure_backends, ['foo'])
-      expect(subject.backends).to eq(['foo'])
-    end
-    it 'resets to default backends if no container found' do
-      subject.default_servers = ['fallback1']
-      subject.send(:configure_backends, ['foo'])
-      expect(subject.backends).to eq(['foo'])
-      subject.send(:configure_backends, [])
-      expect(subject.backends).to eq(['fallback1'])
-    end
-    it 'does not reset to default backends if there are no default backends' do
-      subject.default_servers = []
-      subject.send(:configure_backends, ['foo'])
-      expect(subject.backends).to eq(['foo'])
-      subject.send(:configure_backends, [])
-      expect(subject.backends).to eq(['foo'])
-    end
-  end
-
   context "rewrite_container_ports tests" do
     it 'doesnt break if Ports => nil' do
         subject.send(:rewrite_container_ports, nil)

data/spec/lib/synapse/service_watcher_ec2tags_spec.rb

@@ -0,0 +1,187 @@
+require 'spec_helper'
+require 'logging'
+
+class Synapse::EC2Watcher
+  attr_reader   :synapse
+  attr_accessor :default_servers, :ec2
+end
+
+class FakeAWSInstance
+  def ip_address
+    @ip_address ||= fake_address
+  end
+
+  def private_ip_address
+    @private_ip_address ||= fake_address
+  end
+
+  def dns_name
+    @dns_name ||= "ec2-#{ip_address.gsub('.', '-')}.eu-test-1.compute.amazonaws.com"
+  end
+
+  def private_dns_name
+    @private_dns_name ||= "ip-#{private_ip_address.gsub('.', '-')}.eu-test-1.compute.internal"
+  end
+
+  def fake_address
+    4.times.map { (0...254).to_a.shuffle.pop.to_s }.join('.')
+  end
+end
+
+describe Synapse::EC2Watcher do
+  let(:mock_synapse) { double }
+  subject { Synapse::EC2Watcher.new(basic_config, mock_synapse) }
+
+  let(:basic_config) do
+    { 'name' => 'ec2tagtest',
+      'haproxy' => {
+        'port' => '8080',
+        'server_port_override' => '8081'
+      },
+      "discovery" => {
+        "method" => "ec2tag",
+        "tag_name"   => "fuNNy_tag_name",
+        "tag_value"  => "funkyTagValue",
+        "aws_region" => 'eu-test-1',
+        "aws_access_key_id" => 'ABCDEFGHIJKLMNOPQRSTU',
+        "aws_secret_access_key" => 'verylongfakekeythatireallyneedtogenerate'
+      }
+    }
+  end
+
+  before(:all) do
+    # Clean up ENV so we don't inherit any actual AWS config.
+    %w[AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION].each { |k| ENV.delete(k) }
+  end
+
+  before(:each) do
+    # https://ruby.awsblog.com/post/Tx2SU6TYJWQQLC3/Stubbing-AWS-Responses
+    # always returns empty results, so data may have to be faked.
+    AWS.stub!
+  end
+
+  def remove_discovery_arg(name)
+    args = basic_config.clone
+    args['discovery'].delete name
+    args
+  end
+
+  def remove_haproxy_arg(name)
+    args = basic_config.clone
+    args['haproxy'].delete name
+    args
+  end
+
+  def munge_haproxy_arg(name, new_value)
+    args = basic_config.clone
+    args['haproxy'][name] = new_value
+    args
+  end
+
+  describe '#new' do
+    let(:args) { basic_config }
+
+    it 'instantiates cleanly with basic config' do
+      expect { subject }.not_to raise_error
+    end
+
+    context 'when missing arguments' do
+      it 'complains if aws_region is missing' do
+        expect {
+          Synapse::EC2Watcher.new(remove_discovery_arg('aws_region'), mock_synapse)
+        }.to raise_error(ArgumentError, /Missing aws_region/)
+      end
+      it 'complains if aws_access_key_id is missing' do
+        expect {
+          Synapse::EC2Watcher.new(remove_discovery_arg('aws_access_key_id'), mock_synapse)
+        }.to raise_error(ArgumentError, /Missing aws_access_key_id/)
+      end
+      it 'complains if aws_secret_access_key is missing' do
+        expect {
+          Synapse::EC2Watcher.new(remove_discovery_arg('aws_secret_access_key'), mock_synapse)
+        }.to raise_error(ArgumentError, /Missing aws_secret_access_key/)
+      end
+      it 'complains if server_port_override is missing' do
+        expect {
+          Synapse::EC2Watcher.new(remove_haproxy_arg('server_port_override'), mock_synapse)
+        }.to raise_error(ArgumentError, /Missing server_port_override/)
+      end
+    end
+
+    context 'invalid data' do
+      it 'complains if the haproxy server_port_override is not a number' do
+          expect {
+            Synapse::EC2Watcher.new(munge_haproxy_arg('server_port_override', '80deadbeef'), mock_synapse)
+          }.to raise_error(ArgumentError, /Invalid server_port_override/)
+      end
+    end
+  end
+
+  context "instance discovery" do
+    let(:instance1) { FakeAWSInstance.new }
+    let(:instance2) { FakeAWSInstance.new }
+
+    context 'using the AWS API' do
+      let(:ec2_client) { double('AWS::EC2') }
+      let(:instance_collection) { double('AWS::EC2::InstanceCollection') }
+
+      before do
+        subject.ec2 = ec2_client
+      end
+
+      it 'fetches instances and filter instances' do
+        # Unfortunately there's quite a bit going on here, but this is
+        # a chained call to get then filter EC2 instances, which is
+        # done remotely; breaking into separate calls would result in
+        # unnecessary data being retrieved.
+
+        expect(subject.ec2).to receive(:instances).and_return(instance_collection)
+
+        expect(instance_collection).to receive(:tagged).with('foo').and_return(instance_collection)
+        expect(instance_collection).to receive(:tagged_values).with('bar').and_return(instance_collection)
+        expect(instance_collection).to receive(:select).and_return(instance_collection)
+
+        subject.send(:instances_with_tags, 'foo', 'bar')
+      end
+    end
+
+    context 'returned backend data structure' do
+      before do
+        allow(subject).to receive(:instances_with_tags).and_return([instance1, instance2])
+      end
+
+      let(:backends) { subject.send(:discover_instances) }
+
+      it 'returns an Array of backend name/host/port Hashes' do
+        required_keys = %w[name host port]
+        expect(
+          backends.all?{|b| required_keys.each{|k| b.has_key?(k)}}
+        ).to be_truthy
+      end
+
+      it 'sets the backend port to server_port_override for all backends' do
+        backends = subject.send(:discover_instances)
+        expect(
+          backends.all? { |b| b['port'] == basic_config['haproxy']['server_port_override'] }
+        ).to be_truthy
+      end
+    end
+
+    context 'returned instance fields' do
+      before do
+        allow(subject).to receive(:instances_with_tags).and_return([instance1])
+      end
+
+      let(:backend) { subject.send(:discover_instances).pop }
+
+      it "returns an instance's private IP as the hostname" do
+        expect( backend['host'] ).to eq instance1.private_ip_address
+      end
+
+      it "returns an instance's private hostname as the server name" do
+        expect( backend['name'] ).to eq instance1.private_dns_name
+      end
+    end
+  end
+end
+