synapse 0.11.1 → 0.12.1

data/.gitignore

@@ -21,3 +21,4 @@ tmp
 vendor/
 
 synapse.jar
+.ruby-version

data/.travis.yml

@@ -2,4 +2,6 @@ language: ruby
 cache: bundler
 rvm:
   - 1.9.3
-
+  - 2.0.0
+  - 2.1.6
+  - 2.2.2

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    synapse (0.11.1)
+    synapse (0.12.1)
+      aws-sdk (~> 1.39)
       docker-api (~> 1.7.2)
       zk (~> 1.9.4)
 
@@ -9,21 +10,29 @@ GEM
   remote: https://rubygems.org/
   specs:
     archive-tar-minitar (0.5.2)
+    aws-sdk (1.64.0)
+      aws-sdk-v1 (= 1.64.0)
+    aws-sdk-v1 (1.64.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
     coderay (1.0.9)
-    diff-lcs (1.2.4)
+    diff-lcs (1.2.5)
     docker-api (1.7.6)
       archive-tar-minitar
       excon (>= 0.28)
       json
-    excon (0.32.1)
+    excon (0.45.4)
     ffi (1.9.3-java)
-    json (1.8.1)
+    json (1.8.3)
     little-plugger (1.1.3)
     logging (1.8.2)
       little-plugger (>= 1.1.3)
       multi_json (>= 1.8.4)
     method_source (0.8.2)
-    multi_json (1.9.2)
+    mini_portile (0.6.2)
+    multi_json (1.11.2)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
     pry (0.9.12.2)
       coderay (~> 1.0.5)
       method_source (~> 0.8)
@@ -36,21 +45,25 @@ GEM
     pry-nav (0.2.3)
       pry (~> 0.9.10)
     rake (10.1.1)
-    rspec (2.14.1)
-      rspec-core (~> 2.14.0)
-      rspec-expectations (~> 2.14.0)
-      rspec-mocks (~> 2.14.0)
-    rspec-core (2.14.5)
-    rspec-expectations (2.14.2)
-      diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.14.3)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.7)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.2)
     slop (3.4.6)
     spoon (0.0.4)
       ffi
-    zk (1.9.4)
+    zk (1.9.5)
       logging (~> 1.8.2)
       zookeeper (~> 1.4.0)
-    zookeeper (1.4.8)
+    zookeeper (1.4.10)
 
 PLATFORMS
   java
@@ -60,5 +73,8 @@ DEPENDENCIES
   pry
   pry-nav
   rake
-  rspec
+  rspec (~> 3.1.0)
   synapse!
+
+BUNDLED WITH
+   1.10.5

data/README.md

@@ -52,36 +52,27 @@ production:
 
 You would like to be able to fail over to a different database in case the original dies.
 Let's suppose your instance is running in AWS and you're using the tag 'proddb' set to 'true' to indicate the prod DB.
-You set up synapse to proxy the DB connection on `localhost:3219` in the `synapse.conf.json` file.
+You set up synapse to proxy the DB connection on `localhost:3219` in the `synapse.conf.yaml` file.
 Add a hash under `services` that looks like this:
 
-```json
-{"services":
-    "proddb": {
-      "default_servers": [
-        {
-          "name": "default-db",
-          "host": "mydb.example.com",
-          "port": 5432
-        }
-      ],
-      "discovery": {
-        "method": "awstag",
-        "tag": "proddb",
-        "value": "true"
-      },
-      "haproxy": {
-        "port": 3219,
-        "server_options": "check inter 2000 rise 3 fall 2",
-        "frontend": [
-          "mode tcp",
-        ],
-        "backend": [
-          "mode tcp",
-        ],
-      },
-    },
-...
+```yaml
+---
+ services:
+  proddb:
+   default_servers:
+    -
+     name: "default-db"
+     host: "mydb.example.com"
+     port: 5432
+   discovery:
+    method: "awstag"
+    tag_name: "proddb"
+    tag_value: "true"
+   haproxy:
+    port: 3219
+    server_options: "check inter 2000 rise 3 fall 2"
+    frontend: mode tcp
+    backend: mode tcp
 ```
 
 And then change your database.yaml file to look like this:
@@ -112,6 +103,9 @@ And then execute:
 Or install it yourself as:
 
     $ gem install synapse
+    
+
+Don't forget to install HAProxy prior to installing Synapse.
 
 ## Configuration ##
 
@@ -129,7 +123,6 @@ Each value in the services hash is also a hash, and should contain the following
 * `discovery`: how synapse will discover hosts providing this service (see below)
 * `default_servers`: the list of default servers providing this service; synapse uses these if no others can be discovered
 * `haproxy`: how will the haproxy section for this service be configured
-* `shared_frontend`: optional: haproxy configuration directives for a shared http frontend (see below)
 
 #### Service Discovery ####
 
@@ -167,6 +160,29 @@ It takes the following options:
 * `container_port`: find containers forwarding this port
 * `check_interval`: how often to poll the docker API on each server. Default is 15s.
 
+##### AWS EC2 tags #####
+
+This watcher retrieves a list of Amazon EC2 instances that have a tag
+with particular value using the AWS API.
+It takes the following options:
+
+* `method`: ec2tag
+* `tag_name`: the name of the tag to inspect. As per the AWS docs,
+  this is case-sensitive.
+* `tag_value`: the value to match on. Case-sensitive.
+
+Additionally, you MUST supply `server_port_override` in the `haproxy`
+section of the configuration as this watcher does not know which port
+the backend service is listening on.
+
+The following options are optional, provided the well-known `AWS_`
+environment variables shown are set. If supplied, these options will
+be used in preference to the `AWS_` environment variables.
+
+* `aws_access_key_id`: AWS key or set `AWS_ACCESS_KEY_ID` in the environment.
+* `aws_secret_access_key`: AWS secret key or set `AWS_SECRET_ACCESS_KEY` in the environment.
+* `aws_region`: AWS region (i.e. `us-east-1`) or set `AWS_REGION` in the environment.
+
 #### Listing Default Servers ####
 
 You may list a number of default servers providing a service.
@@ -182,16 +198,30 @@ If you do not list any default servers, no proxy will be created.  The
 `default_servers` will also be used in addition to discovered servers if the
 `keep_default_servers` option is set.
 
+If you do not list any `default_servers`, and all backends for a service
+disappear then the previous known backends will be used.  Disable this behavior
+by unsetting `use_previous_backends`.
+
+#### The `file_output` Section ####
+
+This section controls whether or not synapse will write out service state
+to the filesystem in json format. This can be used for services that want to
+use discovery information but not go through HAProxy.
+
+* `output_directory`: the path to a directory on disk that service registrations
+should be written to.
+
 #### The `haproxy` Section ####
 
 This section is its own hash, which should contain the following keys:
 
-* `port`: the port (on localhost) where HAProxy will listen for connections to the service.
+* `port`: the port (on localhost) where HAProxy will listen for connections to the service. If this is omitted, only a backend stanza (and no frontend stanza) will be generated for this service; you'll need to get traffic to your service yourself via the `shared_frontend` or manual frontends in `extra_sections`
 * `server_port_override`: the port that discovered servers listen on; you should specify this if your discovery mechanism only discovers names or addresses (like the DNS watcher). If the discovery method discovers a port along with hostnames (like the zookeeper watcher) this option may be left out, but will be used in preference if given.
 * `server_options`: the haproxy options for each `server` line of the service in HAProxy config; it may be left out.
 * `frontend`: additional lines passed to the HAProxy config in the `frontend` stanza of this service
 * `backend`: additional lines passed to the HAProxy config in the `backend` stanza of this service
 * `listen`: these lines will be parsed and placed in the correct `frontend`/`backend` section as applicable; you can put lines which are the same for the frontend and backend here.
+* `shared_frontend`: optional: haproxy configuration directives for a shared http frontend (see below)
 
 ### Configuring HAProxy ###
 
@@ -201,80 +231,75 @@ The `haproxy` section of the config file has the following options:
 * `config_file_path`: where Synapse will write the HAProxy config file
 * `do_writes`: whether or not the config file will be written (default to `true`)
 * `do_reloads`: whether or not Synapse will reload HAProxy (default to `true`)
+* `do_socket`: whether or not Synapse will use the HAProxy socket commands to prevent reloads (default to `true`)
 * `global`: options listed here will be written into the `global` section of the HAProxy config
 * `defaults`: options listed here will be written into the `defaults` section of the HAProxy config
+* `extra_sections`: additional, manually-configured `frontend`, `backend`, or `listen` stanzas
 * `bind_address`: force HAProxy to listen on this address (default is localhost)
-* `shared_fronted`: (OPTIONAL) additional lines passed to the HAProxy config used to configure a shared HTTP frontend (see below)
-
-Note that a non-default `bind_address` can be dangerous: it is up to you to ensure that HAProxy will not attempt to bind an address:port combination that is not already in use by one of your services.
+* `shared_frontend`: (OPTIONAL) additional lines passed to the HAProxy config used to configure a shared HTTP frontend (see below)
+* `restart_interval`: number of seconds to wait between restarts of haproxy (default: 2)
+* `restart_jitter`: percentage, expressed as a float, of jitter to multiply the `restart_interval` by when determining the next
+  restart time. Use this to help prevent healthcheck storms when HAProxy restarts. (default: 0.0)
+* `state_file_path`: full path on disk (e.g. /tmp/synapse/state.json) for caching haproxy state between reloads.
+  If provided, synapse will store recently seen backends at this location and can "remember" backends across both synapse and
+  HAProxy restarts. Any backends that are "down" in the reporter but listed in the cache will be put into HAProxy disabled (default: nil)
+* `state_file_ttl`: the number of seconds that backends should be kept in the state file cache.
+  This only applies if `state_file_path` is provided (default: 86400)
+
+Note that a non-default `bind_address` can be dangerous.
+If you configure an `address:port` combination that is already in use on the system, haproxy will fail to start.
 
 ### HAProxy shared HTTP Frontend ###
 
 For HTTP-only services, it is not always necessary or desirable to dedicate a TCP port per service, since HAProxy can route traffic based on host headers.
-To support this, the optional `shared_fronted` section can be added to both the `haproxy` section and each indvidual service definition: synapse will concatenate them all into a single frontend section in the generated haproxy.cfg file.
-Note that synapse does not assemble the routing ACLs for you: you have to do that yourself based on your needs.
+To support this, the optional `shared_frontend` section can be added to both the `haproxy` section and each indvidual service definition.
+Synapse will concatenate them all into a single frontend section in the generated haproxy.cfg file.
+Note that synapse does not assemble the routing ACLs for you; you have to do that yourself based on your needs.
 This is probably most useful in combination with the `service_conf_dir` directive in a case where the individual service config files are being distributed by a configuration manager such as puppet or chef, or bundled into service packages.
 For example:
 
 ```yaml
-{
-  "haproxy": {
-    "shared_frontend": [
-      "bind 127.0.0.1:8081"
-    ],
-    "reload_command": "service haproxy reload",
-    "config_file_path": "/etc/haproxy/haproxy.cfg",
-    "socket_file_path": "/var/run/haproxy.sock",
-    "global": [
-      "daemon",
-      "user    haproxy",
-      "group   haproxy",
-      "maxconn 4096",
-      "log     127.0.0.1 local2 notice",
-      "stats   socket /var/run/haproxy.sock"
-    ],
-    "defaults": [
-      "log      global",
-      "balance  roundrobin"
-    ]
-  },
-  "services": {
-    "service1": {
-      "discovery": {
-        "method": "zookeeper",
-        "path":  "/nerve/services/service1",
-        "hosts": [ "0.zookeeper.example.com:2181" ]
-      },
-      "haproxy": {
-        "server_options": "check inter 2s rise 3 fall 2",
-        "shared_frontend": [
-          "acl is_service1 hdr_dom(host) -i service1.lb.example.com",
-          "use_backend service1 if is_service1"
-        ],
-        "backend": [
-          "mode http"
-        ]
-      }
-    },
-    "service2": {
-      "discovery": {
-        "method": "zookeeper",
-        "path":  "/nerve/services/service2",
-        "hosts": [ "0.zookeeper.example.com:2181" ]
-      },
-      "haproxy": {
-        "server_options": "check inter 2s rise 3 fall 2",
-        "shared_frontend": [
-          "acl is_service1 hdr_dom(host) -i service2.lb.example.com",
-          "use_backend service2 if is_service2"
-        ],
-        "backend": [
-          "mode http"
-        ]
-      }
-    }
-  }
-}
+ haproxy:
+  shared_frontend: "bind 127.0.0.1:8081"
+  reload_command: "service haproxy reload"
+  config_file_path: "/etc/haproxy/haproxy.cfg"
+  socket_file_path: "/var/run/haproxy.sock"
+  global:
+   - "daemon"
+   - "user    haproxy"
+   - "group   haproxy"
+   - "maxconn 4096"
+   - "log     127.0.0.1 local2 notice"
+   - "stats   socket /var/run/haproxy.sock"
+  defaults:
+   - "log      global"
+   - "balance  roundrobin"
+ services:
+  service1:
+   discovery: 
+    method: "zookeeper"
+    path:  "/nerve/services/service1"
+    hosts: "0.zookeeper.example.com:2181"
+   haproxy:
+    server_options: "check inter 2s rise 3 fall 2"
+    shared_frontend:
+     - "acl is_service1 hdr_dom(host) -i service1.lb.example.com"
+     - "use_backend service1 if is_service1"
+    backend: "mode http"
+
+  service2:
+   discovery:
+    method: "zookeeper"
+    path:  "/nerve/services/service2"
+    hosts: "0.zookeeper.example.com:2181"
+
+   haproxy:
+    server_options: "check inter 2s rise 3 fall 2"
+    shared_frontend:
+     - "acl is_service1 hdr_dom(host) -i service2.lb.example.com"
+     - "use_backend service2 if is_service2
+    backend: "mode http"
+
 ```
 
 This would produce an haproxy.cfg much like the following:
@@ -332,5 +357,5 @@ end
 3. Implement the `start` and `validate_discovery_opts` methods
 4. Implement whatever additional methods your discovery requires
 
-When your watcher detects a list of new backends, they should be written to `@backends`.
-You should then call `@synapse.configure` to force synapse to update the HAProxy config.
+When your watcher detects a list of new backends, you should call `set_backends` to
+store the new backends and update the HAProxy config.

data/lib/synapse.rb

@@ -1,6 +1,7 @@
 require "synapse/version"
 require "synapse/service_watcher/base"
 require "synapse/haproxy"
+require "synapse/file_output"
 require "synapse/service_watcher"
 require "synapse/log"
 
@@ -17,9 +18,17 @@ module Synapse
       raise "specify a list of services to connect in the config" unless opts.has_key?('services')
       @service_watchers = create_service_watchers(opts['services'])
 
-      # create the haproxy object
+      # create objects that need to be notified of service changes
+      @config_generators = []
+      # create the haproxy config generator, this is mandatory
       raise "haproxy config section is missing" unless opts.has_key?('haproxy')
-      @haproxy = Haproxy.new(opts['haproxy'])
+      @config_generators << Haproxy.new(opts['haproxy'])
+
+      # possibly create a file manifestation for services that do not
+      # want to communicate via haproxy, e.g. cassandra
+      if opts.has_key?('file_output')
+        @config_generators << FileOutput.new(opts['file_output'])
+      end
 
       # configuration is initially enabled to configure on first loop
       @config_updated = true
@@ -47,10 +56,15 @@ module Synapse
 
         if @config_updated
           @config_updated = false
-          log.info "synapse: regenerating haproxy config"
-          @haproxy.update_config(@service_watchers)
-        else
-          sleep 1
+          @config_generators.each do |config_generator|
+            log.info "synapse: configuring #{config_generator.name}"
+            config_generator.update_config(@service_watchers)
+          end
+        end
+
+        sleep 1
+        @config_generators.each do |config_generator|
+          config_generator.tick(@service_watchers)
         end
 
         loops += 1

data/lib/synapse/file_output.rb

@@ -0,0 +1,57 @@
+require 'synapse/log'
+require 'fileutils'
+require 'tempfile'
+
+module Synapse
+  class FileOutput
+    include Logging
+    attr_reader :opts, :name
+
+    def initialize(opts)
+      unless opts.has_key?("output_directory")
+        raise ArgumentError, "flat file generation requires an output_directory key"
+      end
+
+      begin
+        FileUtils.mkdir_p(opts['output_directory'])
+      rescue SystemCallError => err
+        raise ArgumentError, "provided output directory #{opts['output_directory']} is not present or creatable"
+      end
+
+      @opts = opts
+      @name = 'file_output'
+    end
+
+    def tick(watchers)
+    end
+
+    def update_config(watchers)
+      watchers.each do |watcher|
+        write_backends_to_file(watcher.name, watcher.backends)
+      end
+    end
+
+    def write_backends_to_file(service_name, new_backends)
+      data_path = File.join(@opts['output_directory'], "#{service_name}.json")
+      begin
+        old_backends = JSON.load(File.read(data_path))
+      rescue Errno::ENOENT
+        old_backends = nil
+      end
+
+      if old_backends == new_backends
+        # Prevent modifying the file unless something has actually changed
+        # This way clients can set watches on this file and update their
+        # internal state only when the smartstack state has actually changed
+        return false
+      else
+        # Atomically write new sevice configuration file
+        temp_path = File.join(@opts['output_directory'],
+                              ".#{service_name}.json.tmp")
+        File.open(temp_path, 'w', 0644) {|f| f.write(new_backends.to_json)}
+        FileUtils.mv(temp_path, data_path)
+        return true
+      end
+    end
+  end
+end