synapse-aurora 0.11.2

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :test => :spec
+task :default => :spec
+

data/bin/synapse

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'optparse'
+
+require 'synapse'
+
+options={}
+
+# set command line options
+optparse = OptionParser.new do |opts|
+  opts.banner =<<-EOB
+Welcome to synapse
+
+Usage: synapse --config /path/to/synapse/config
+  EOB
+
+  options[:config] = ENV['SYNAPSE_CONFIG']
+  opts.on('-c config','--config config', String, 'path to synapse config') do |key,value|
+    options[:config] = key
+  end
+
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+end
+
+# parse command line arguments
+optparse.parse!
+
+def parseconfig(filename)
+  # parse synapse config file
+  begin
+    c = YAML::parse(File.read(filename))
+  rescue Errno::ENOENT => e
+    raise ArgumentError, "config file does not exist:\n#{e.inspect}"
+  rescue Errno::EACCES => e
+    raise ArgumentError, "could not open config file:\n#{e.inspect}"
+  rescue YAML::SyntaxError => e
+    raise "config file #{filename} is not yaml:\n#{e.inspect}"
+  end
+  return c.to_ruby
+end
+
+config = parseconfig(options[:config])
+config['services'] ||= {}
+
+if config.has_key?('service_conf_dir')
+  cdir = File.expand_path(config['service_conf_dir'])
+  if ! Dir.exists?(cdir)
+    raise "service conf dir does not exist:#{cdir}"
+  end
+  cfiles = Dir.glob(File.join(cdir, '*.{json,yaml}'))
+  cfiles.each { |x| config['services'][File.basename(x[/(.*)\.(json|yaml)$/, 1])] = parseconfig(x) }
+end
+
+# run synapse
+s = Synapse::Synapse.new(config)
+s.run
+
+puts "synapse has exited"

data/config/hostheader_test.json

@@ -0,0 +1,71 @@
+{
+  "services": {
+    "service1": {
+      "default_servers": [
+        { "name": "default1", "host": "localhost", "port": 8080 }
+      ],
+      "discovery": {
+        "method": "dns",
+        "nameserver": "127.0.0.1",
+        "servers": [
+          "0.www.example.com",
+          "1.www.example.com"
+        ]
+      },
+      "haproxy": {
+        "server_options": "check inter 2s rise 3 fall 2",
+        "listen": [
+          "mode http",
+          "option httplog"
+        ],
+        "backend": [
+          "mode http",
+          "option httpchk GET /health HTTP/1.0"
+        ]
+      }
+    }
+  },
+  "haproxy": {
+    "reload_command": "sudo service haproxy reload",
+    "config_file_path": "/etc/haproxy/haproxy.cfg",
+    "socket_file_path": "/var/haproxy/stats.sock",
+    "do_writes": true,
+    "do_reloads": true,
+    "do_socket": true,
+    "global": [
+      "daemon",
+      "user haproxy",
+      "group haproxy",
+      "maxconn 4096",
+      "log     127.0.0.1 local0",
+      "log     127.0.0.1 local1 notice",
+      "stats   socket /var/haproxy/stats.sock mode 666 level admin"
+    ],
+    "defaults": [
+      "log      global",
+      "option   dontlognull",
+      "maxconn  2000",
+      "retries  3",
+      "timeout  connect 5s",
+      "timeout  client  1m",
+      "timeout  server  1m",
+      "option   redispatch",
+      "balance  roundrobin"
+    ],
+    "extra_sections": {
+      "listen stats :3212": [
+        "mode http",
+        "stats enable",
+        "stats uri /",
+        "stats refresh 5s"
+      ],
+      "frontend http-generic-in": [
+        "bind 127.0.0.1:80",
+        "acl is_service1 hdr_dom(host) -i service1.lb",
+        "acl is_cache hdr_dom(host) -i cache.lb",
+        "use_backend service1 if is_service1",
+        "use_backend cache if is_cache"
+      ]
+    }
+  }
+}

data/config/svcdir_test.json

@@ -0,0 +1,46 @@
+{
+  "service_conf_dir": "config/synapse_services",
+  "haproxy": {
+    "reload_command": "sudo service haproxy reload",
+    "config_file_path": "/etc/haproxy/haproxy.cfg",
+    "socket_file_path": "/var/haproxy/stats.sock",
+    "do_writes": true,
+    "do_reloads": true,
+    "do_socket": true,
+    "global": [
+      "daemon",
+      "user haproxy",
+      "group haproxy",
+      "maxconn 4096",
+      "log     127.0.0.1 local0",
+      "log     127.0.0.1 local1 notice",
+      "stats   socket /var/haproxy/stats.sock mode 666 level admin"
+    ],
+    "defaults": [
+      "log      global",
+      "option   dontlognull",
+      "maxconn  2000",
+      "retries  3",
+      "timeout  connect 5s",
+      "timeout  client  1m",
+      "timeout  server  1m",
+      "option   redispatch",
+      "balance  roundrobin"
+    ],
+    "extra_sections": {
+      "listen stats :3212": [
+        "mode http",
+        "stats enable",
+        "stats uri /",
+        "stats refresh 5s"
+      ],
+      "frontend http-generic-in": [
+        "bind 127.0.0.1:80",
+        "acl is_service1 hdr_dom(host) -i service1.lb",
+        "acl is_cache hdr_dom(host) -i cache.lb",
+        "use_backend service1 if is_service1",
+        "use_backend cache if is_cache"
+      ]
+    }
+  }
+}

data/config/synapse.conf.json

@@ -0,0 +1,90 @@
+{
+  "services": {
+    "service1": {
+      "default_servers": [
+        {
+          "name": "default1",
+          "host": "localhost",
+          "port": 8423
+        }
+      ],
+      "discovery": {
+        "method": "zookeeper",
+        "path": "/services/service1",
+        "hosts": [
+          "localhost:2181"
+        ]
+      },
+      "haproxy": {
+        "port": 3213,
+        "server_options": "check inter 2s rise 3 fall 2",
+        "listen": [
+          "mode http",
+          "option httpchk /health",
+          "http-check expect string OK"
+        ]
+      }
+      
+    },
+    "service2": {
+      "default_servers": [
+        {
+          "name": "default1",
+          "host": "localhost",
+          "port": 8422
+        }
+      ],
+      "discovery": {
+        "method": "zookeeper",
+        "path": "/services/service2",
+        "hosts": [
+          "localhost:2181"
+        ]
+      },
+      "haproxy": {
+        "port": 3214,
+        "server_options": "check inter 2s rise 3 fall 2",
+        "listen": [
+          "mode http",
+          "option httpchk /health"
+        ]
+      }
+    }
+  },
+  "haproxy": {
+    "reload_command": "sudo service haproxy reload",
+    "config_file_path": "/etc/haproxy/haproxy.cfg",
+    "socket_file_path": "/var/haproxy/stats.sock",
+    "do_writes": false,
+    "do_reloads": false,
+    "do_socket": false,
+    "global": [
+      "daemon",
+      "user haproxy",
+      "group haproxy",
+      "maxconn 4096",
+      "log     127.0.0.1 local0",
+      "log     127.0.0.1 local1 notice",
+      "stats   socket /var/haproxy/stats.sock mode 666 level admin"
+    ],
+    "defaults": [
+      "log      global",
+      "option   dontlognull",
+      "maxconn  2000",
+      "retries  3",
+      "timeout  connect 5s",
+      "timeout  client  1m",
+      "timeout  server  1m",
+      "option   redispatch",
+      "balance  roundrobin"
+    ],
+    "extra_sections": {
+      "listen stats :3212": [
+        "mode http",
+        "stats enable",
+        "stats uri /",
+        "stats refresh 5s"
+      ]
+    }
+  }
+}

data/config/synapse_services/service1.json

@@ -0,0 +1,24 @@
+{
+  "default_servers": [
+    { "name": "default1", "host": "localhost", "port": 8080 }
+  ],
+  "discovery": {
+    "method": "dns",
+    "nameserver": "10.10.1.13",
+    "servers": [
+      "0.www.example.com",
+      "1.www.example.com"
+    ]
+  },
+  "haproxy": {
+    "server_options": "check inter 2s rise 3 fall 2",
+    "listen": [
+      "mode http",
+      "option httplog"
+    ],
+    "backend": [
+      "mode http",
+      "option httpchk GET /health HTTP/1.0"
+    ]
+  }
+}

data/config/synapse_services/service2.json

@@ -0,0 +1,24 @@
+{
+  "default_servers": [
+    { "name": "default1", "host": "localhost", "port": 8080 }
+  ],
+  "discovery": {
+    "method": "dns",
+    "nameserver": "10.10.1.13",
+    "servers": [
+      "0.www.example.com",
+      "1.www.example.com"
+    ]
+  },
+  "haproxy": {
+    "server_options": "check inter 2s rise 3 fall 2",
+    "listen": [
+      "mode http",
+      "option httplog"
+    ],
+    "backend": [
+      "mode http",
+      "option httpchk GET /health HTTP/1.0"
+    ]
+  }
+}

data/default.nix

@@ -0,0 +1,66 @@
+# Build a snapshot of synapse from the current git working copy.
+
+with import <nixpkgs> { };
+
+let
+  myGems = recurseIntoAttrs (callPackage ./.nix/rubylibs.nix { });
+  synapse_requirements = [
+    rubyLibs.aws_sdk
+    myGems.zk
+    myGems.docker_api
+  ];
+in
+  stdenv.mkDerivation rec {
+    name = "synapse-git";
+    version = "0.11.1";
+
+    src = ./.;
+
+    meta = {
+      description = "Development build of synapse from git";
+      homepage = "https://github.com/benley/synapse";
+    };
+
+    buildInputs = [ ruby rubygems git makeWrapper ];
+    propagatedBuildInputs = synapse_requirements;
+
+    buildPhase = "gem build synapse.gemspec";
+
+    installPhase = ''
+      export HOME=$TMP/home; mkdir -pv "$HOME"
+
+      gem install \
+        --ignore-dependencies \
+        --install-dir "$out/${ruby.gemPath}" \
+        --bindir "$out/bin" \
+        "synapse-${version}.gem" $gemFlags -- $buildFlags
+
+      # Don't keep the .gem file in the finished package
+      rm -frv "$out/${ruby.gemPath}/cache"
+
+      addToSearchPath GEM_PATH "$out/${ruby.gemPath}"
+
+      for prog in $out/bin/*; do
+        wrapProgram "$prog" \
+          --prefix GEM_PATH : "$GEM_PATH" \
+          --prefix RUBYLIB : "${rubygems}/lib" \
+          --set RUBYOPT rubygems \
+          $extraWrapperFlags ''${extraWrapperFlagsArray[@]}
+      done
+
+      # Make sure all the executables are actually in the package
+      for prog in $out/gems/*/bin/*; do
+        [[ -e "$out/bin/$(basename $prog)" ]]
+      done
+
+      # looks like useless files which break build repeatability and consume space
+      rm $out/${ruby.gemPath}/doc/*/*/created.rid \
+         $out/${ruby.gemPath}/gems/*/ext/*/mkmf.log || true
+
+      runHook postInstall
+    '';
+
+    propagatedUserEnvPkgs = synapse_requirements;
+
+    passthru.isRubyGem = true;
+  }

data/lib/synapse.rb

@@ -0,0 +1,85 @@
+require "synapse/version"
+require "synapse/service_watcher/base"
+require "synapse/haproxy"
+require "synapse/service_watcher"
+require "synapse/log"
+
+require 'logger'
+require 'json'
+
+include Synapse
+
+module Synapse
+  class Synapse
+    include Logging
+    def initialize(opts={})
+      # create the service watchers for all our services
+      raise "specify a list of services to connect in the config" unless opts.has_key?('services')
+      @service_watchers = create_service_watchers(opts['services'])
+
+      # create the haproxy object
+      raise "haproxy config section is missing" unless opts.has_key?('haproxy')
+      @haproxy = Haproxy.new(opts['haproxy'])
+
+      # configuration is initially enabled to configure on first loop
+      @config_updated = true
+
+      # Any exceptions in the watcher threads should wake the main thread so
+      # that we can fail fast.
+      Thread.abort_on_exception = true
+
+      log.debug "synapse: completed init"
+    end
+
+    # start all the watchers and enable haproxy configuration
+    def run
+      log.info "synapse: starting..."
+
+      # start all the watchers
+      @service_watchers.map { |watcher| watcher.start }
+
+      # main loop
+      loops = 0
+      loop do
+        @service_watchers.each do |w|
+          raise "synapse: service watcher #{w.name} failed ping!" unless w.ping?
+        end
+
+        if @config_updated
+          @config_updated = false
+          log.info "synapse: regenerating haproxy config"
+          @haproxy.update_config(@service_watchers)
+        else
+          sleep 1
+        end
+
+        loops += 1
+        log.debug "synapse: still running at #{Time.now}" if (loops % 60) == 0
+      end
+
+    rescue StandardError => e
+      log.error "synapse: encountered unexpected exception #{e.inspect} in main thread"
+      raise e
+    ensure
+      log.warn "synapse: exiting; sending stop signal to all watchers"
+
+      # stop all the watchers
+      @service_watchers.map(&:stop)
+    end
+
+    def reconfigure!
+      @config_updated = true
+    end
+
+    private
+    def create_service_watchers(services={})
+      service_watchers =[]
+      services.each do |service_name, service_config|
+        service_watchers << ServiceWatcher.create(service_name, service_config, self)
+      end
+
+      return service_watchers
+    end
+
+  end
+end