democracyworks-synapse 0.9.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 035f5185668b79655c47279251a52255efa293a0
+  data.tar.gz: 578f659d158fde0be97f2c852ecbe44df03692a8
+SHA512:
+  metadata.gz: 32eda48b033735638ca19e21c2022ca2db80886829ee303a0c03a368398a4207e01e5d39211cdd23ccf8fc65540f95b0ae32ca9d3b7ab846ec13772bbd71e6a2
+  data.tar.gz: 6ebf7d1cfd7f504793bf1d09191c7e49f67f564fc11e9196065d3d458d499e53457d190a8000e4f7d33abcee303a703a581346c670dc0efb0698482a4e94c7a7

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*~
+.vagrant
+.*sw?
+vendor/
+
+synapse.jar

data/.mailmap

@@ -0,0 +1,3 @@
+<igor.serebryany@airbedandbreakfast.com> <igor47@moomers.org>
+<martin.rhoads@airbnb.com> <ermal14@gmail.com>
+Pierre Carrier <pierre@gcarrier.fr>

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in synapse.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,23 @@
+Original work Copyright (c) 2013 Airbnb, Inc.
+Modified work Copyright (c) 2014 Democracy Works, Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Makefile

@@ -0,0 +1,6 @@
+build: synapse.jar
+
+synapse.jar:
+	jruby -S warble jar
+
+.PHONY: build push

data/README.md

@@ -0,0 +1,241 @@
+# Synapse #
+
+Synapse is Airbnb's new system for service discovery.
+Synapse solves the problem of automated fail-over in the cloud, where failover via network re-configuration is impossible.
+The end result is the ability to connect internal services together in a scalable, fault-tolerant way.
+
+## Motivation ##
+
+Synapse emerged from the need to maintain high-availability applications in the cloud.
+Traditional high-availability techniques, which involve using a CRM like [pacemaker](http://linux-ha.org/wiki/Pacemaker), do not work in environments where the end-user has no control over the networking.
+In an environment like Amazon's EC2, all of the available workarounds are suboptimal:
+
+* Round-robin DNS: Slow to converge, and doesn't work when applications cache DNS lookups (which is frequent)
+* Elastic IPs: slow to converge, limited in number, public-facing-only, which makes them less useful for internal services
+* ELB: Again, public-facing only, and only useful for HTTP
+
+One solution to this problem is a discovery service, like [Apache Zookeeper](http://zookeeper.apache.org/).
+However, Zookeeper and similar services have their own problems:
+
+* Service discovery is embedded in all of your apps; often, integration is not simple
+* The discovery layer itself it subject to failure
+* Requires additional servers/instances
+
+Synapse solves these difficulties in a simple and fault-tolerant way.
+
+## How Synapse Works ##
+
+Synapse runs on your application servers; here at Airbnb, we just run it on every box we deploy.
+The heart of synapse is actually [HAProxy](http://haproxy.1wt.eu/), a stable and proven routing component.
+For every external service that your application talks to, we assign a synapse local port on localhost.
+Synapse creates a proxy from the local port to the service, and you reconfigure your application to talk to the proxy.
+
+Synapse comes with a number of `watchers`, which are responsible for service discovery.
+The synapse watchers take care of re-configuring the proxy so that it always points at available servers.
+We've included a number of default watchers, including ones that query zookeeper and ones using the AWS API.
+It is easy to write your own watchers for your use case, and we encourage submitting them back to the project.
+
+## Example Migration ##
+
+Lets suppose your rails application depends on a Postgre database instance.
+The database.yaml file has the DB host and port hardcoded:
+
+```yaml
+production:
+  database: mydb
+  host: mydb.example.com
+  port: 5432
+```
+
+You would like to be able to fail over to a different database in case the original dies.
+Let's suppose your instance is running in AWS and you're using the tag 'proddb' set to 'true' to indicate the prod DB.
+You set up synapse to proxy the DB connection on `localhost:3219` in the `synapse.conf.json` file.
+Add a hash under `services` that looks like this:
+
+```json
+{"services":
+    "proddb": {
+      "default_servers": [
+        {
+          "name": "default-db",
+          "host": "mydb.example.com",
+          "port": 5432
+        }
+      ],
+      "discovery": {
+        "method": "awstag",
+        "tag": "proddb",
+        "value": "true"
+      },
+      "haproxy": {
+        "port": 3219,
+        "server_options": "check inter 2000 rise 3 fall 2",
+        "frontend": [
+          "mode tcp",
+        ],
+        "backend": [
+          "mode tcp",
+        ],
+      },
+    },
+...
+```
+
+And then change your database.yaml file to look like this:
+
+```yaml
+production:
+  database: mydb
+  host: localhost
+  port: 3219
+```
+
+Start up synapse.
+It will configure HAProxy with a proxy from `localhost:3219` to your DB.
+It will attempt to find the DB using the AWS API; if that does not work, it will default to the DB given in `default_servers`.
+In the worst case, if AWS API is down and you need to change which DB your application talks to, simply edit the `synapse.conf.json` file, update the `default_servers` and restart synapse.
+HAProxy will be transparently reloaded, and your application will keep running without a hiccup.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'synapse'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install synapse
+
+## Configuration ##
+
+Synapse depends on a single config file in JSON format; it's usually called `synapse.conf.json`.
+The file has two main sections.
+The first is the `services` section, which lists the services you'd like to connect.
+The second is the `haproxy` section, which specifies how to configure and interact with HAProxy.
+
+### Configuring a Service ###
+
+The services are a hash, where the keys are the `name` of the service to be configured.
+The name is just a human-readable string; it will be used in logs and notifications.
+Each value in the services hash is also a hash, and should contain the following keys:
+
+* `discovery`: how synapse will discover hosts providing this service (see below)
+* `default_servers`: the list of default servers providing this service; synapse uses these if none others can be discovered
+* `haproxy`: how will the haproxy section for this service be configured
+
+#### Service Discovery ####
+
+We've included a number of `watchers` which provide service discovery.
+Put these into the `discovery` section of the service hash, with these options:
+
+##### Stub #####
+
+The stub watcher, this is useful in situations where you only want to use the servers in the `default_servers` list.
+It has only one option:
+
+* `method`: stub
+
+##### Zookeeper #####
+
+This watcher retrieves a list of servers from zookeeper.
+It takes the following options:
+
+* `method`: zookeeper
+* `path`: the zookeeper path where ephemeral nodes will be created for each available service server
+* `hosts`: the list of zookeeper servers to query
+
+The watcher assumes that each node under `path` represents a service server.
+Synapse attempts to decode the data in each of these nodes using JSON and also using Thrift under the standard Twitter service encoding.
+We assume that the data contains a hostname and a port for service servers.
+
+##### Docker #####
+
+This watcher retrieves a list of [docker](http://www.docker.io/) containers via docker's [HTTP API](http://docs.docker.io/en/latest/api/docker_remote_api/).
+It takes the following options:
+
+* `method`: docker
+* `servers`: a list of servers running docker as a daemon. Format is `{"name":"...", "host": "..."[, port: 4243]}`
+* `image_name`: find containers running this image
+* `container_port`: find containers forwarding this port
+* `check_interval`: how often to poll the docker API on each server. Default is 15s.
+
+#### Listing Default Servers ####
+
+You may list a number of default servers providing a service.
+Each hash in that section has the following options:
+
+* `name`: a human-readable name for the default server; must be unique
+* `host`: the host or IP address of the server
+* `port`: the port where the service runs on the `host`
+
+The `default_servers` list is used only when service discovery returns no servers.
+In that case, the service proxy will be created with the servers listed here.
+If you do not list any default servers, no proxy will be created.  The
+`default_servers` will also be used in addition to discovered servers if the
+`keep_default_servers` option is set.
+
+#### The `haproxy` Section ####
+
+This section is it's own hash, which should contain the following keys:
+
+* `port`: the port (on localhost) where HAProxy will listen for connections to the service.
+* `server_port_override`: the port that discovered servers listen on; you should specify this if your discovery mechanism only discovers names or addresses (like the DNS watcher). If the discovery method discovers a port along with hostnames (like the zookeeper watcher) this option may be left out, but will be used in preference if given.
+* `server_options`: the haproxy options for each `server` line of the service in HAProxy config; it may be left out.
+* `frontend`: additional lines passed to the HAProxy config in the `frontend` stanza of this service
+* `backend`: additional lines passed to the HAProxy config in the `backend` stanza of this service
+* `listen`: these lines will be parsed and placed in the correct `frontend`/`backend` section as applicable; you can put lines which are the same for the frontend and backend here.
+
+### Configuring HAProxy ###
+
+The `haproxy` section of the config file has the following options:
+
+* `reload_command`: the command Synapse will run to reload HAProxy
+* `config_file_path`: where Synapse will write the HAProxy config file
+* `do_writes`: whether or not the config file will be written (default to `true`)
+* `do_reloads`: whether or not Synapse will reload HAProxy (default to `true`)
+* `global`: options listed here will be written into the `global` section of the HAProxy config
+* `defaults`: options listed here will be written into the `defaults` section of the HAProxy config
+* `bind_address`: force HAProxy to listen  on this address (default is localhost)
+
+Note that a non-default `bind_address` can be dangerous: it is up to you to ensure that HAProxy will not attempt to bind an address:port combination that is not already in use by one of your services.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+### Creating a Service Watcher ###
+
+If you'd like to create a new service watcher:
+
+1. Create a file for your watcher in `service_watcher` dir
+2. Use the following template:
+```ruby
+require 'synapse/service_watcher/base'
+
+module Synapse
+  class NewWatcher < BaseWatcher
+    def start
+      # write code which begins running service discovery
+    end
+
+    private
+    def validate_discovery_opts
+      # here, validate any required options in @discovery
+    end
+  end
+end
+```
+
+3. Implement the `start` and `validate_discovery_opts` methods
+4. Implement whatever additional methods your discovery requires
+
+When your watcher detects a list of new backends, they should be written to `@backends`.
+You should then call `@synapse.configure` to force synapse to update the HAProxy config.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/synapse

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'optparse'
+
+require 'synapse'
+
+options={}
+
+# set command line options
+optparse = OptionParser.new do |opts|
+  opts.banner =<<-EOB
+Welcome to synapse
+
+Usage: synapse --config /path/to/synapse/config
+  EOB
+
+  options[:config] = ENV['SYNAPSE_CONFIG']
+  opts.on('-c config','--config config', String, 'path to synapse config') do |key,value|
+    options[:config] = key
+  end
+
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts opts
+    exit
+  end
+end
+
+# parse command line arguments
+optparse.parse!
+
+def parseconfig(filename)
+  # parse synapse config file
+  begin
+    c = YAML::parse(File.read(filename))
+  rescue Errno::ENOENT => e
+    raise ArgumentError, "config file does not exist:\n#{e.inspect}"
+  rescue Errno::EACCES => e
+    raise ArgumentError, "could not open config file:\n#{e.inspect}"
+  rescue YAML::ParseError => e
+    raise "config file #{filename} is not yaml:\n#{e.inspect}"
+  end
+  return c.to_ruby
+end
+
+config = parseconfig(options[:config])
+config['services'] ||= {}
+
+if config.has_key?('service_conf_dir')
+  cdir = File.expand_path(config['service_conf_dir'])
+  if ! Dir.exists?(cdir)
+    raise "service conf dir does not exist:#{cdir}"
+  end
+  cfiles = Dir.glob(File.join(cdir, '*.{json,yaml}'))
+  cfiles.each { |x| config['services'][File.basename(x[/(.*)\.(json|yaml)$/, 1])] = parseconfig(x) }
+end
+
+# run synapse
+s = Synapse::Synapse.new(config)
+s.run
+
+puts "synapse has exited"

data/config/hostheader_test.json

@@ -0,0 +1,71 @@
+{
+  "services": {
+    "service1": {
+      "default_servers": [
+        { "name": "default1", "host": "localhost", "port": 8080 }
+      ],
+      "discovery": {
+        "method": "dns",
+        "nameserver": "127.0.0.1",
+        "servers": [
+          "0.www.example.com",
+          "1.www.example.com"
+        ]
+      },
+      "haproxy": {
+        "server_options": "check inter 2s rise 3 fall 2",
+        "listen": [
+          "mode http",
+          "option httplog"
+        ],
+        "backend": [
+          "mode http",
+          "option httpchk GET /health HTTP/1.0"
+        ]
+      }
+    }
+  },
+  "haproxy": {
+    "reload_command": "sudo service haproxy reload",
+    "config_file_path": "/etc/haproxy/haproxy.cfg",
+    "socket_file_path": "/var/haproxy/stats.sock",
+    "do_writes": true,
+    "do_reloads": true,
+    "do_socket": true,
+    "global": [
+      "daemon",
+      "user haproxy",
+      "group haproxy",
+      "maxconn 4096",
+      "log     127.0.0.1 local0",
+      "log     127.0.0.1 local1 notice",
+      "stats   socket /var/haproxy/stats.sock mode 666 level admin"
+    ],
+    "defaults": [
+      "log      global",
+      "option   dontlognull",
+      "maxconn  2000",
+      "retries  3",
+      "timeout  connect 5s",
+      "timeout  client  1m",
+      "timeout  server  1m",
+      "option   redispatch",
+      "balance  roundrobin"
+    ],
+    "extra_sections": {
+      "listen stats :3212": [
+        "mode http",
+        "stats enable",
+        "stats uri /",
+        "stats refresh 5s"
+      ],
+      "frontend http-generic-in": [
+        "bind 127.0.0.1:80",
+        "acl is_service1 hdr_dom(host) -i service1.lb",
+        "acl is_cache hdr_dom(host) -i cache.lb",
+        "use_backend service1 if is_service1",
+        "use_backend cache if is_cache"
+      ]
+    }
+  }
+}