synapse 0.0.1 → 0.2.1

data/lib/synapse/service_watcher/dns.rb

@@ -0,0 +1,98 @@
+require_relative "./base"
+
+require 'thread'
+require 'resolv'
+
+module Synapse
+  class DnsWatcher < BaseWatcher
+    def start
+      @check_interval = @discovery['check_interval'] || 30.0
+      @nameserver = @discovery['nameserver']
+
+      watch
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'dns'
+      raise ArgumentError, "a non-empty list of servers is required" \
+        if @discovery['servers'].empty?
+    end
+
+    def watch
+      @watcher = Thread.new do
+        last_resolution = resolve_servers
+        configure_backends(last_resolution)
+        while true
+          begin
+            start = Time.now
+            current_resolution = resolve_servers
+            unless last_resolution == current_resolution
+              last_resolution = current_resolution
+              configure_backends(last_resolution)
+            end
+
+            sleep_until_next_check(start)
+          rescue => e
+            log.warn "Error in watcher thread: #{e.inspect}"
+            log.warn e.backtrace
+          end
+        end
+      end
+    end
+
+    def sleep_until_next_check(start_time)
+      sleep_time = @check_interval - (Time.now - start_time)
+      if sleep_time > 0.0
+        sleep(sleep_time)
+      end
+    end
+
+    def resolve_servers
+      resolver.tap do |dns|
+        resolution = @discovery['servers'].map do |server|
+          addresses = dns.getaddresses(server['host']).map(&:to_s)
+          [server, addresses.sort]
+        end
+
+        return resolution
+      end
+    rescue => e
+      log.warn "Error while resolving host names: #{e.inspect}"
+      []
+    end
+
+    def resolver
+      args = [{:nameserver => @nameserver}] if @nameserver
+      Resolv::DNS.open(*args)
+    end
+
+    def configure_backends(servers)
+      new_backends = servers.flat_map do |(server, addresses)|
+        addresses.map do |address|
+          {
+            'name' => "#{server['name']}-#{[address, server['port']].hash}",
+            'host' => address,
+            'port' => server['port']
+          }
+        end
+      end
+
+      if new_backends.empty?
+        if @default_servers.empty?
+          log.warn "synapse: no backends and no default servers for service #{@name};" \
+            " using previous backends: #{@backends.inspect}"
+        else
+          log.warn "synapse: no backends for service #{@name};" \
+            " using default servers: #{@default_servers.inspect}"
+          @backends = @default_servers
+        end
+      else
+        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
+        @backends = new_backends
+      end
+      @synapse.configure
+    end
+  end
+end

data/lib/synapse/service_watcher/ec2tag.rb

@@ -0,0 +1,26 @@
+require_relative "./base"
+
+module Synapse
+  class EC2Watcher < BaseWatcher
+    def start
+      # connect to ec2
+      # find all servers whose @discovery['tag_name'] matches @discovery['tag_value']
+      # call @synapse.configure
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'ec2tag' 
+      raise ArgumentError, "a `server_port_override` option is required for ec2tag watchers" \
+        unless @server_port_override
+      raise ArgumentError, "missing aws credentials for service #{@name}" \
+        unless (@discovery['aws_key'] && @discovery['aws_secret'])
+      raise ArgumentError, "aws tag name is required for service #{@name}" \
+        unless @discovery['tag_name']
+      raise ArgumentError, "aws tag value required for service #{@name}" \
+        unless @discovery['tag_value']
+    end
+  end
+end
+

data/lib/synapse/service_watcher/zookeeper.rb

@@ -0,0 +1,138 @@
+require_relative "./base"
+
+require_relative "../../gen-rb/endpoint_types"
+require_relative "../../gen-rb/thrift"
+require 'zk'
+
+module Synapse
+  class ZookeeperWatcher < BaseWatcher
+    def start
+      zk_hosts = @discovery['hosts'].shuffle.join(',')
+
+      log.info "synapse: starting ZK watcher #{@name} @ hosts: #{zk_hosts}, path: #{@discovery['path']}"
+      @zk = ZK.new(zk_hosts)
+
+      @deserializer = Thrift::Deserializer.new
+
+      watch
+      discover
+    end
+
+    private
+    def validate_discovery_opts
+      raise ArgumentError, "invalid discovery method #{@discovery['method']}" \
+        unless @discovery['method'] == 'zookeeper'
+      raise ArgumentError, "missing or invalid zookeeper host for service #{@name}" \
+        unless @discovery['hosts']
+      raise ArgumentError, "invalid zookeeper path for service #{@name}" \
+        unless @discovery['path']
+    end
+
+    # helper method that ensures that the discovery path exists
+    def create(path)
+      log.debug "synapse: creating ZK path: #{path}"
+      # recurse if the parent node does not exist
+      create File.dirname(path) unless @zk.exists? File.dirname(path)
+      @zk.create(path, ignore: :node_exists)
+    end
+
+    # find the current backends at the discovery path; sets @backends
+    def discover
+      log.info "synapse: discovering backends for service #{@name}"
+
+      new_backends = []
+      begin
+        @zk.children(@discovery['path'], :watch => true).map do |name|
+          node = @zk.get("#{@discovery['path']}/#{name}")
+
+          begin
+            host, port = deserialize_service_instance(node.first)
+          rescue
+            log.error "synapse: invalid data in ZK node #{name} at #{@discovery['path']}"
+          else
+            server_port = @server_port_override ? @server_port_override : port
+            backend_name = "#{name}-#{[host, server_port].hash}"
+
+            log.debug "synapse: discovered backend #{backend_name} at #{host}:#{server_port} for service #{@name}"
+            new_backends << { 'name' => backend_name, 'host' => host, 'port' => server_port}
+          end
+        end
+      rescue ZK::Exceptions::NoNode
+        # the path must exist, otherwise watch callbacks will not work
+        create(@discovery['path'])
+        retry
+      end
+
+      if new_backends.empty?
+        if @default_servers.empty?
+          log.warn "synapse: no backends and no default servers for service #{@name}; using previous backends: #{@backends.inspect}"
+        else
+          log.warn "synapse: no backends for service #{@name}; using default servers: #{@default_servers.inspect}"
+          @backends = @default_servers
+        end
+      else
+        log.info "synapse: discovered #{new_backends.length} backends for service #{@name}"
+        @backends = new_backends
+      end
+    end
+
+    # sets up zookeeper callbacks if the data at the discovery path changes
+    def watch
+      @watcher.unsubscribe if defined? @watcher
+      @watcher = @zk.register(@discovery['path'], &watcher_callback)
+    end
+
+    # handles the event that a watched path has changed in zookeeper
+    def watcher_callback
+      Proc.new do |event|
+        # Set new watcher
+        watch
+        # Rediscover
+        discover
+        # send a message to calling class to reconfigure
+        @synapse.configure
+      end
+    end
+
+    # tries to extract host/port from a json hash
+    def parse_json(data)
+      begin
+        json = JSON.parse data
+      rescue Object => o
+        return false
+      end
+      raise 'instance json data does not have host key' unless json.has_key?('host')
+      raise 'instance json data does not have port key' unless json.has_key?('port')
+      return json['host'], json['port']
+    end
+
+    # tries to extract a host/port from twitter thrift data
+    def parse_thrift(data)
+      begin
+        service = Twitter::Thrift::ServiceInstance.new
+        @deserializer.deserialize(service, data)
+      rescue Object => o
+        return false
+      end
+      raise "instance thrift data does not have host" if service.serviceEndpoint.host.nil?
+      raise "instance thrift data does not have port" if service.serviceEndpoint.port.nil?
+      return service.serviceEndpoint.host, service.serviceEndpoint.port
+    end
+
+    # decode the data at a zookeeper endpoint
+    def deserialize_service_instance(data)
+      log.debug "synapse: deserializing process data"
+
+      # first, lets try parsing this as thrift
+      host, port = parse_thrift(data)
+      return host, port if host
+
+      # if that does not work, try json
+      host, port = parse_json(data)
+      return host, port if host
+
+      # if we got this far, then we have a problem
+      raise "could not decode this data:\n#{data}"
+    end
+  end
+end

data/lib/synapse/service_watcher.rb

@@ -0,0 +1,28 @@
+require_relative "./service_watcher/base"
+require_relative "./service_watcher/zookeeper"
+require_relative "./service_watcher/ec2tag"
+require_relative "./service_watcher/dns"
+
+module Synapse
+  class ServiceWatcher
+
+    @watchers = {
+      'base'=>BaseWatcher,
+      'zookeeper'=>ZookeeperWatcher,
+      'ec2tag'=>EC2Watcher,
+      'dns' => DnsWatcher
+    }
+
+    # the method which actually dispatches watcher creation requests
+    def self.create(opts, synapse)
+      raise ArgumentError, "Missing discovery method when trying to create watcher" \
+        unless opts.has_key?('discovery') && opts['discovery'].has_key?('method')
+
+      discovery_method = opts['discovery']['method']
+      raise ArgumentError, "Invalid discovery method #{discovery_method}" \
+        unless @watchers.has_key?(discovery_method)
+
+      return @watchers[discovery_method].new(opts, synapse)
+    end
+  end
+end

data/lib/synapse/version.rb

@@ -1,3 +1,3 @@
 module Synapse
-  VERSION = "0.0.1"
+  VERSION = "0.2.1"
 end

data/lib/synapse.rb

@@ -1,5 +1,64 @@
-require "synapse/version"
+require_relative "synapse/version"
+require_relative "synapse/base"
+require_relative "synapse/haproxy"
+require_relative "synapse/service_watcher"
+
+require 'logger'
+require 'json'
+
+include Synapse
 
 module Synapse
-  # Your code goes here...
+  class Synapse
+    def initialize(opts={})
+      # disable configuration until this is started
+      @configure_enabled = false
+
+      # create the service watchers for all our services
+      raise "specify a list of services to connect in the config" unless opts.has_key?('services')
+      @service_watchers = create_service_watchers(opts['services'])
+
+      # create the haproxy object
+      raise "haproxy config section is missing" unless opts.has_key?('haproxy')
+      @haproxy = Haproxy.new(opts['haproxy'])
+    end
+
+    # start all the watchers and enable haproxy configuration
+    def run
+      log.info "synapse: starting..."
+
+      @service_watchers.map { |watcher| watcher.start }
+      @configure_enabled = true
+      configure
+
+      # loop forever
+      loops = 0
+      loop do 
+        sleep 1
+        loops += 1
+        log.debug "synapse: still running at #{Time.now}" if (loops % 60) == 0
+      end
+    end
+
+    # reconfigure haproxy based on our watchers
+    def configure
+      if @configure_enabled
+        log.info "synapse: regenerating haproxy config"
+        @haproxy.update_config(@service_watchers)
+      else
+        log.info "synapse: reconfigure requested, but it's not yet enabled"
+      end
+    end
+
+    private
+    def create_service_watchers(services={})
+      service_watchers =[]
+      services.each do |service_config|
+        service_watchers << ServiceWatcher.create(service_config, self)
+      end
+      
+      return service_watchers
+    end
+    
+  end
 end

data/spec/lib/synapse/haproxy_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+class MockWatcher; end;
+
+describe Synapse::Haproxy do
+  subject { Synapse::Haproxy.new(config['haproxy']) }
+
+  it 'updating the config' do
+    mockWatcher = mock(Synapse::ServiceWatcher)
+    binding.pry
+    subject.should_receive(:generate_config)
+    subject.update_config([mockWatcher])
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+require_relative '../lib/synapse'
+require 'pry'
+require_relative 'support/config'
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+  config.include Config
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

data/spec/support/config.rb

@@ -0,0 +1,9 @@
+require "yaml"
+
+module Config
+
+  def config
+    @config ||= YAML::load_file(File.join(File.dirname(File.expand_path(__FILE__)), 'minimum.conf.yaml'))
+  end
+
+end

data/spec/support/minimum.conf.yaml

@@ -0,0 +1,27 @@
+# list the services to connect
+services:
+  - name: test
+    local_port: 3210
+    server_options: test_option
+    default_servers:
+      - { name: default1, host: localhost, port: 8080}
+    discovery:
+      method: zookeeper
+      path: /airbnb/service/logging/event_collector
+      hosts:
+        - localhost:2181
+    listen:
+      - test_option
+
+
+# settings for haproxy including the global config
+haproxy:
+  reload_command: "sudo service haproxy reload"
+  config_file_path: "/etc/haproxy/haproxy.cfg"
+  do_writes: false
+  do_reloads: false
+  global:
+    - global_test_option
+    
+  defaults:
+    - default_test_option

data/synapse.gemspec

@@ -16,4 +16,11 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "zk", "~> 1.7.4"
+  gem.add_runtime_dependency "thrift", "~> 0.9.0"
+
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "pry-nav"
 end

data/test.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -ex
+
+echo running synapse script

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: synapse
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,23 +9,137 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-09 00:00:00.000000000 Z
-dependencies: []
+date: 2013-10-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: zk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.4
+- !ruby/object:Gem::Dependency
+  name: thrift
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ': Write a gem description'
 email:
 - martin.rhoads@airbnb.com
-executables: []
+executables:
+- synapse
 extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .rspec
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
+- Vagrantfile
+- bin/synapse
+- chef/converge
+- chef/cookbooks/lxc/recipes/default.rb
+- chef/cookbooks/synapse/attributes/default.rb
+- chef/cookbooks/synapse/recipes/default.rb
+- chef/run.json
+- chef/run.rb
+- client/.RData
+- client/.Rhistory
+- client/bench_rewrite_config.dat
+- client/benchmark-client.iml
+- client/pom.xml
+- client/src/main/java/ClientArsch.java
+- client/src/main/java/META-INF/MANIFEST.MF
+- config/synapse.conf.json
+- haproxy.pid
+- lib/gen-rb/endpoint_types.rb
+- lib/gen-rb/thrift.rb
 - lib/synapse.rb
+- lib/synapse/base.rb
+- lib/synapse/haproxy.rb
+- lib/synapse/service_watcher.rb
+- lib/synapse/service_watcher/base.rb
+- lib/synapse/service_watcher/dns.rb
+- lib/synapse/service_watcher/ec2tag.rb
+- lib/synapse/service_watcher/zookeeper.rb
 - lib/synapse/version.rb
+- spec/lib/synapse/haproxy_spec.rb
+- spec/spec_helper.rb
+- spec/support/config.rb
+- spec/support/minimum.conf.yaml
 - synapse.gemspec
+- test.sh
 homepage: ''
 licenses: []
 post_install_message: 
@@ -50,4 +164,8 @@ rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: ': Write a gem summary'
-test_files: []
+test_files:
+- spec/lib/synapse/haproxy_spec.rb
+- spec/spec_helper.rb
+- spec/support/config.rb
+- spec/support/minimum.conf.yaml