synapse 0.8.0 → 0.9.1

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+cache: bundler
+rvm:
+  - 1.9.3
+

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    synapse (0.8.0)
+    synapse (0.9.1)
       docker-api (~> 1.7.2)
       zk (~> 1.9.2)
 
@@ -27,6 +27,7 @@ GEM
       slop (~> 3.4)
     pry-nav (0.2.3)
       pry (~> 0.9.10)
+    rake (10.1.1)
     rspec (2.14.1)
       rspec-core (~> 2.14.0)
       rspec-expectations (~> 2.14.0)
@@ -36,10 +37,10 @@ GEM
       diff-lcs (>= 1.1.3, < 2.0)
     rspec-mocks (2.14.3)
     slop (3.4.6)
-    zk (1.9.2)
+    zk (1.9.3)
       logging (~> 1.7.2)
       zookeeper (~> 1.4.0)
-    zookeeper (1.4.7)
+    zookeeper (1.4.8)
 
 PLATFORMS
   ruby
@@ -47,5 +48,6 @@ PLATFORMS
 DEPENDENCIES
   pry
   pry-nav
+  rake
   rspec
   synapse!

data/README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/airbnb/synapse.png?branch=master)](https://travis-ci.org/airbnb/synapse)
+
 # Synapse #
 
 Synapse is Airbnb's new system for service discovery.

data/Rakefile

@@ -1 +1,8 @@
 require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :test => :spec
+task :default => :spec
+

data/bin/synapse

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'json'
+require 'yaml'
 require 'optparse'
 
 require 'synapse'
@@ -32,15 +32,15 @@ optparse.parse!
 def parseconfig(filename)
   # parse synapse config file
   begin
-    c = JSON::parse(File.read(filename))
+    c = YAML::parse(File.read(filename))
   rescue Errno::ENOENT => e
     raise ArgumentError, "config file does not exist:\n#{e.inspect}"
   rescue Errno::EACCES => e
     raise ArgumentError, "could not open config file:\n#{e.inspect}"
-  rescue JSON::ParserError => e
-    raise "config file #{filename} is not json:\n#{e.inspect}"
+  rescue YAML::ParseError => e
+    raise "config file #{filename} is not yaml:\n#{e.inspect}"
   end
-  return c
+  return c.to_ruby
 end
 
 config = parseconfig(options[:config])
@@ -51,8 +51,8 @@ if config.has_key?('service_conf_dir')
   if ! Dir.exists?(cdir)
     raise "service conf dir does not exist:#{cdir}"
   end
-  cfiles = Dir.glob(File.join(cdir, '*.json'))
-  cfiles.each { |x| config['services'][File.basename(x[/(.*)\.json$/, 1])] = parseconfig(x) }
+  cfiles = Dir.glob(File.join(cdir, '*.{json,yaml}'))
+  cfiles.each { |x| config['services'][File.basename(x[/(.*)\.(json|yaml)$/, 1])] = parseconfig(x) }
 end
 
 # run synapse

data/lib/synapse/haproxy.rb

@@ -1,7 +1,5 @@
 require 'synapse/log'
-
 require 'socket'
-require 'digest'
 
 module Synapse
   class Haproxy
@@ -750,8 +748,12 @@ module Synapse
 
     # used to build unique, consistent haproxy names for backends
     def construct_name(backend)
-      address_digest = Digest::SHA256.hexdigest(backend['host'])[0..7]
-      return "#{backend['name']}:#{backend['port']}_#{address_digest}"
+      name = "#{backend['host']}:#{backend['port']}"
+      if backend['name'] && !backend['name'].empty?
+        name = "#{name}_#{backend['name']}"
+      end
+
+      return name
     end
   end
 end

data/lib/synapse/service_watcher/base.rb

@@ -1,6 +1,12 @@
+require 'synapse/log'
+
 module Synapse
   class BaseWatcher
-    attr_reader :name, :backends, :haproxy
+    include Logging
+
+    LEADER_WARN_INTERVAL = 30
+
+    attr_reader :name, :haproxy
 
     def initialize(opts={}, synapse)
       super()
@@ -15,6 +21,9 @@ module Synapse
       @name = opts['name']
       @discovery = opts['discovery']
 
+      @leader_election = opts['leader_election'] || false
+      @leader_last_warn = Time.now - LEADER_WARN_INTERVAL
+
       # the haproxy config
       @haproxy = opts['haproxy']
       @haproxy['server_options'] ||= ""
@@ -55,6 +64,26 @@ module Synapse
       true
     end
 
+    def backends
+      if @leader_election
+        if @backends.all?{|b| b.key?('id') && b['id']}
+          smallest = @backends.sort_by{ |b| b['id']}.first
+          log.debug "synapse: leader election chose one of #{@backends.count} backends " \
+            "(#{smallest['host']}:#{smallest['port']} with id #{smallest['id']})"
+
+          return [smallest]
+        elsif (Time.now - @leader_last_warn) > LEADER_WARN_INTERVAL
+          log.warn "synapse: service #{@name}: leader election failed; not all backends include an id"
+          @leader_last_warn = Time.now
+        end
+
+        # if leader election fails, return no backends
+        return []
+      end
+
+      return @backends
+    end
+
     private
     def validate_discovery_opts
       raise ArgumentError, "invalid discovery method '#{@discovery['method']}' for base watcher" \

data/lib/synapse/service_watcher/dns.rb

@@ -1,12 +1,10 @@
 require "synapse/service_watcher/base"
-require "synapse/log"
 
 require 'thread'
 require 'resolv'
 
 module Synapse
   class DnsWatcher < BaseWatcher
-    include Logging
     def start
       @check_interval = @discovery['check_interval'] || 30.0
       @nameserver = @discovery['nameserver']
@@ -80,7 +78,6 @@ module Synapse
       new_backends = servers.flat_map do |(server, addresses)|
         addresses.map do |address|
           {
-            'name' => server['name'],
             'host' => address,
             'port' => server['port']
           }

data/lib/synapse/service_watcher/docker.rb

@@ -34,7 +34,7 @@ module Synapse
           end
 
           sleep_until_next_check(start)
-        rescue => e
+        rescue Exception => e
           log.warn "synapse: error in watcher thread: #{e.inspect}"
           log.warn e.backtrace
         end
@@ -50,6 +50,24 @@ module Synapse
       end
     end
 
+    def rewrite_container_ports(ports)
+      pairs = []
+      if ports.is_a?(String)
+        # "Ports" comes through (as of 0.6.5) as a string like "0.0.0.0:49153->6379/tcp, 0.0.0.0:49153->6379/tcp"
+        # Convert string to a map of container port to host port: {"7000"->"49158", "6379": "49159"}
+        pairs = ports.split(", ").collect do |v|
+          pair = v.split('->')
+          [ pair[1].rpartition("/").first, pair[0].rpartition(":").last ]
+        end
+      elsif ports.is_a?(Array)
+        # New style API, ports is an array of hashes, with numeric values (or nil if no ports forwarded)
+        pairs = ports.collect do |v|
+          [v['PrivatePort'].to_s, v['PublicPort'].to_s]
+        end
+      end
+      Hash[pairs]
+    end
+
     def containers
       backends = @discovery['servers'].map do |server|
         Docker.url = "http://#{server['host']}:#{server['port'] || 4243}"
@@ -59,14 +77,8 @@ module Synapse
           log.warn "synapse: error polling docker host #{Docker.url}: #{e.inspect}"
           next []
         end
-        # "Ports" comes through (as of 0.6.5) as a string like "0.0.0.0:49153->6379/tcp, 0.0.0.0:49153->6379/tcp"
-        # Convert string to a map of container port to host port: {"7000"->"49158", "6379": "49159"}
         cnts.each do |cnt|
-          pairs = cnt["Ports"].split(", ").collect do |v|
-            pair = v.split('->')
-            [ pair[1].rpartition("/").first, pair[0].rpartition(":").last ]
-          end
-          cnt["Ports"] = Hash[pairs]
+          cnt['Ports'] = rewrite_container_ports cnt['Ports']
         end
         # Discover containers that match the image/port we're interested in
         cnts = cnts.find_all do |cnt|

data/lib/synapse/service_watcher/zookeeper.rb

@@ -1,11 +1,11 @@
 require "synapse/service_watcher/base"
-require "synapse/log"
 
 require 'zk'
 
 module Synapse
   class ZookeeperWatcher < BaseWatcher
-    include Logging
+    NUMBERS_RE = /^\d+$/
+
     def start
       zk_hosts = @discovery['hosts'].shuffle.join(',')
 
@@ -55,18 +55,22 @@ module Synapse
 
       new_backends = []
       begin
-        @zk.children(@discovery['path'], :watch => true).map do |name|
-          node = @zk.get("#{@discovery['path']}/#{name}")
+        @zk.children(@discovery['path'], :watch => true).each do |id|
+          node = @zk.get("#{@discovery['path']}/#{id}")
 
           begin
-            host, port = deserialize_service_instance(node.first)
-          rescue
-            log.error "synapse: invalid data in ZK node #{name} at #{@discovery['path']}"
+            host, port, name = deserialize_service_instance(node.first)
+          rescue StandardError => e
+            log.error "synapse: invalid data in ZK node #{id} at #{@discovery['path']}: #{e}"
           else
             server_port = @server_port_override ? @server_port_override : port
 
+            # find the numberic id in the node name; used for leader elections if enabled
+            numeric_id = id.split('_').last
+            numeric_id = NUMBERS_RE =~ numeric_id ? numeric_id.to_i : nil
+
             log.debug "synapse: discovered backend #{name} at #{host}:#{server_port} for service #{@name}"
-            new_backends << { 'name' => name, 'host' => host, 'port' => server_port}
+            new_backends << { 'name' => name, 'host' => host, 'port' => server_port, 'id' => numeric_id}
           end
         end
       rescue ZK::Exceptions::NoNode
@@ -108,28 +112,16 @@ module Synapse
       end
     end
 
-    # tries to extract host/port from a json hash
-    def parse_json(data)
-      begin
-        json = JSON.parse data
-      rescue Object => o
-        return false
-      end
-      raise 'instance json data does not have host key' unless json.has_key?('host')
-      raise 'instance json data does not have port key' unless json.has_key?('port')
-      return json['host'], json['port']
-    end
-
     # decode the data at a zookeeper endpoint
     def deserialize_service_instance(data)
       log.debug "synapse: deserializing process data"
+      decoded = JSON.parse(data)
 
-      # if that does not work, try json
-      host, port = parse_json(data)
-      return host, port if host
+      host = decoded['host'] || (raise ValueError, 'instance json data does not have host key')
+      port = decoded['port'] || (raise ValueError, 'instance json data does not have port key')
+      name = decoded['name'] || nil
 
-      # if we got this far, then we have a problem
-      raise "could not decode this data:\n#{data}"
+      return host, port, name
     end
   end
 end

data/lib/synapse/version.rb

@@ -1,3 +1,3 @@
 module Synapse
-  VERSION = "0.8.0"
+  VERSION = "0.9.1"
 end

data/spec/lib/synapse/haproxy_spec.rb

@@ -6,8 +6,7 @@ describe Synapse::Haproxy do
   subject { Synapse::Haproxy.new(config['haproxy']) }
 
   it 'updating the config' do
-    mockWatcher = mock(Synapse::ServiceWatcher)
-    binding.pry
+    mockWatcher = double(Synapse::ServiceWatcher)
     subject.should_receive(:generate_config)
     subject.update_config([mockWatcher])
   end

data/spec/lib/synapse/service_watcher_base_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+class Synapse::BaseWatcher
+  attr_reader :should_exit, :default_servers
+end
+
+describe Synapse::BaseWatcher do
+  let(:mocksynapse) { double() }
+  subject { Synapse::BaseWatcher.new(args, mocksynapse) } 
+  let(:testargs) { { 'name' => 'foo', 'discovery' => { 'method' => 'base' }, 'haproxy' => {} }}
+
+  def remove_arg(name)
+    args = testargs.clone
+    args.delete name
+    args
+  end
+
+  context "can construct normally" do
+    let(:args) { testargs }
+    it('can at least construct') { expect { subject }.not_to raise_error }
+  end
+
+  ['name', 'discovery', 'haproxy'].each do |to_remove|
+    context "without #{to_remove} argument" do
+      let(:args) { remove_arg to_remove }
+      it('gots bang') { expect { subject }.to raise_error(ArgumentError, "missing required option #{to_remove}") }
+    end
+  end
+
+  context "normal tests" do
+    let(:args) { testargs }
+    it('is running') { expect(subject.should_exit).to equal(false) }
+    it('can ping') { expect(subject.ping?).to equal(true) }
+    it('can be stopped') do
+      subject.stop
+      expect(subject.should_exit).to equal(true)
+    end
+  end
+
+  context "with default_servers" do
+    default_servers = ['server1', 'server2']
+    let(:args) { testargs.merge({'default_servers' => default_servers}) }
+    it('sets default backends to default_servers') { expect(subject.backends).to equal(default_servers) }
+  end
+end
+

data/spec/lib/synapse/service_watcher_docker_spec.rb

@@ -0,0 +1,152 @@
+require 'spec_helper'
+
+class Synapse::DockerWatcher
+  attr_reader :check_interval, :watcher, :synapse
+  attr_accessor :default_servers
+end
+
+describe Synapse::DockerWatcher do
+  let(:mocksynapse) { double() }
+  subject { Synapse::DockerWatcher.new(testargs, mocksynapse) }
+  let(:testargs) { { 'name' => 'foo', 'discovery' => { 'method' => 'docker', 'servers' => [{'host' => 'server1.local', 'name' => 'mainserver'}], 'image_name' => 'mycool/image', 'container_port' => 6379 }, 'haproxy' => {} }}
+  before(:each) do
+    allow(subject.log).to receive(:warn)
+    allow(subject.log).to receive(:info)
+  end
+
+  def add_arg(name, value)
+    args = testargs.clone
+    args['discovery'][name] = value
+    args
+  end
+
+  context "can construct normally" do
+    it('can at least construct') { expect { subject }.not_to raise_error }
+  end
+
+  context "normal tests" do
+    it('starts a watcher thread') do
+      watcher_mock = double()
+      expect(Thread).to receive(:new).and_return(watcher_mock)
+      subject.start
+      expect(subject.watcher).to equal(watcher_mock)
+    end
+    it('sets default check interval') do
+      expect(Thread).to receive(:new).and_return(double)
+      subject.start
+      expect(subject.check_interval).to eq(15.0)
+    end
+  end
+
+  context "watch tests" do
+    before(:each) do
+      expect(subject).to receive(:sleep_until_next_check) do |arg|
+        subject.instance_variable_set('@should_exit', true)
+      end
+    end
+    it('has a happy first run path, configuring backends') do
+      expect(subject).to receive(:containers).and_return(['container1'])
+      expect(subject).to receive(:configure_backends).with(['container1'])
+      subject.send(:watch)
+    end
+    it('does not call configure_backends if there is no change') do
+      expect(subject).to receive(:containers).and_return([])
+      expect(subject).to_not receive(:configure_backends)
+      subject.send(:watch)
+    end
+  end
+  context "watch eats exceptions" do
+    it "blows up when finding containers" do
+      expect(subject).to receive(:containers) do |arg|
+        subject.instance_variable_set('@should_exit', true)
+        raise('throw exception inside watch')
+      end
+      expect { subject.send(:watch) }.not_to raise_error
+    end
+  end
+
+  context "configure_backends tests" do
+    before(:each) do
+      expect(subject.synapse).to receive(:'reconfigure!').at_least(:once)
+    end
+    it 'runs' do
+      expect { subject.send(:configure_backends, []) }.not_to raise_error
+    end
+    it 'sets backends right' do
+      subject.send(:configure_backends, ['foo'])
+      expect(subject.backends).to eq(['foo'])
+    end
+    it 'resets to default backends if no container found' do
+      subject.default_servers = ['fallback1']
+      subject.send(:configure_backends, ['foo'])
+      expect(subject.backends).to eq(['foo'])
+      subject.send(:configure_backends, [])
+      expect(subject.backends).to eq(['fallback1'])
+    end
+    it 'does not reset to default backends if there are no default backends' do
+      subject.default_servers = []
+      subject.send(:configure_backends, ['foo'])
+      expect(subject.backends).to eq(['foo'])
+      subject.send(:configure_backends, [])
+      expect(subject.backends).to eq(['foo'])
+    end
+  end
+
+  context "rewrite_container_ports tests" do
+    it 'doesnt break if Ports => nil' do
+        subject.send(:rewrite_container_ports, nil)
+    end
+    it 'works for old style port mappings' do
+      expect(subject.send(:rewrite_container_ports, "0.0.0.0:49153->6379/tcp, 0.0.0.0:49154->6390/tcp")).to \
+        eql({'6379' => '49153', '6390' => '49154'})
+    end
+    it 'works for new style port mappings' do
+      expect(subject.send(:rewrite_container_ports, [{'PrivatePort' => 6379, 'PublicPort' => 49153}, {'PublicPort' => 49154, 'PrivatePort' => 6390}])).to \
+        eql({'6379' => '49153', '6390' => '49154'})
+    end
+  end
+
+  context "container discovery tests" do
+    before(:each) do
+      getter = double()
+      expect(getter).to receive(:get)
+      expect(Docker).to receive(:connection).and_return(getter)
+    end
+
+    it('has a sane uri') { subject.send(:containers); expect(Docker.url).to eql('http://server1.local:4243') }
+
+    context 'old style port mappings' do
+      context 'works for one container' do
+        let(:docker_data) { [{"Ports" => "0.0.0.0:49153->6379/tcp, 0.0.0.0:49154->6390/tcp", "Image" => "mycool/image:tagname"}] }
+        it do 
+          expect(Docker::Util).to receive(:parse_json).and_return(docker_data)
+          expect(subject.send(:containers)).to eql([{"name"=>"mainserver", "host"=>"server1.local", "port"=>"49153"}])
+         end
+      end
+      context 'works for multiple containers' do
+        let(:docker_data) { [{"Ports" => "0.0.0.0:49153->6379/tcp, 0.0.0.0:49154->6390/tcp", "Image" => "mycool/image:tagname"}, {"Ports" => "0.0.0.0:49155->6379/tcp", "Image" => "mycool/image:tagname"}] }
+        it do
+          expect(Docker::Util).to receive(:parse_json).and_return(docker_data)
+          expect(subject.send(:containers)).to eql([{"name"=>"mainserver", "host"=>"server1.local", "port"=>"49153"},{"name"=>"mainserver", "host"=>"server1.local", "port"=>"49155"}])
+        end
+      end
+    end
+
+    context 'new style port mappings' do
+      let(:docker_data) { [{"Ports" => [{'PrivatePort' => 6379, 'PublicPort' => 49153}, {'PublicPort' => 49154, 'PrivatePort' => 6390}], "Image" => "mycool/image:tagname"}] }
+      it do
+        expect(Docker::Util).to receive(:parse_json).and_return(docker_data)
+        expect(subject.send(:containers)).to eql([{"name"=>"mainserver", "host"=>"server1.local", "port"=>"49153"}])
+      end
+    end
+
+    context 'filters out wrong images' do
+      let(:docker_data) { [{"Ports" => "0.0.0.0:49153->6379/tcp, 0.0.0.0:49154->6390/tcp", "Image" => "mycool/image:tagname"}, {"Ports" => "0.0.0.0:49155->6379/tcp", "Image" => "wrong/image:tagname"}] }
+      it do
+        expect(Docker::Util).to receive(:parse_json).and_return(docker_data)
+        expect(subject.send(:containers)).to eql([{"name"=>"mainserver", "host"=>"server1.local", "port"=>"49153"}])
+      end
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -4,7 +4,7 @@
 # loaded once.
 #
 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
-require '../lib/synapse'
+require "#{File.dirname(__FILE__)}/../lib/synapse"
 require 'pry'
 require 'support/config'
 

data/synapse.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "zk", "~> 1.9.2"
   gem.add_runtime_dependency "docker-api", "~> 1.7.2"
 
+  gem.add_development_dependency "rake"
   gem.add_development_dependency "rspec"
   gem.add_development_dependency "pry"
   gem.add_development_dependency "pry-nav"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: synapse
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-27 00:00:00.000000000 Z
+date: 2014-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zk
@@ -43,6 +43,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -102,6 +118,7 @@ files:
 - .gitignore
 - .mailmap
 - .rspec
+- .travis.yml
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -126,6 +143,8 @@ files:
 - lib/synapse/service_watcher/zookeeper.rb
 - lib/synapse/version.rb
 - spec/lib/synapse/haproxy_spec.rb
+- spec/lib/synapse/service_watcher_base_spec.rb
+- spec/lib/synapse/service_watcher_docker_spec.rb
 - spec/spec_helper.rb
 - spec/support/config.rb
 - spec/support/minimum.conf.yaml
@@ -142,12 +161,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 575568743231626432
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 575568743231626432
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -156,6 +181,8 @@ specification_version: 3
 summary: ': Write a gem summary'
 test_files:
 - spec/lib/synapse/haproxy_spec.rb
+- spec/lib/synapse/service_watcher_base_spec.rb
+- spec/lib/synapse/service_watcher_docker_spec.rb
 - spec/spec_helper.rb
 - spec/support/config.rb
 - spec/support/minimum.conf.yaml