etcd-rb 1.0.0.pre1 → 1.0.0

data/lib/etcd/observer.rb

@@ -0,0 +1,76 @@
+module Etcd
+  class Observer
+    include Etcd::Loggable
+
+    def initialize(client, prefix, handler)
+      @client  = client
+      @prefix  = prefix
+      @handler = handler
+      @index   = nil
+      reset_logger!(Logger::DEBUG)
+    end
+
+    def run
+      @running = true
+      @thread = Thread.start do
+        while @running
+          logger.debug "********* watching #{@prefix} with index #{@index}"
+          @client.watch(@prefix, index: @index) do |value, key, info|
+            if @running
+              logger.debug "watch fired for #{@prefix} with #{info.inspect} "
+              call_handler_in_needed(value, key, info)
+            end
+          end
+        end
+      end
+      self
+    end
+
+    ## etcd has a bug: after restart watches with index fire __sometimes__ with all the previous values
+    ## workaround:
+    ##  - execute @handler only if info[:index] had higher value than the last index
+    def call_handler_in_needed(value, key, info)
+      if info[:index] && @index.to_i <= info[:index]
+        # next time start watching from next index
+        @index = info[:index] + 1
+        logger.debug "index for #{@prefix} ----  #{@index} "
+        @handler.call(value, key, info)
+      # first-time-fire
+      elsif @index == nil
+        @handler.call(value, key, info)
+      end
+    end
+
+    def cancel
+      @running = false
+      self
+    end
+
+    def rerun
+      logger.debug "rerun for #{@prefix}"
+      @thread.terminate if @thread.alive?
+      logger.debug "after termination for #{@prefix}"
+      run
+    end
+
+    def join
+      @thread.join
+      self
+    end
+
+    def status
+      @thread.status
+    end
+
+    def pp_status
+      "#{@prefix}: #{pp_thread_status}"
+    end
+
+    def pp_thread_status
+      st = @thread.status
+      st = 'dead by exception'   if st == nil
+      st = 'dead by termination' if st == false
+      st
+    end
+  end
+end

data/lib/etcd/requestable.rb

@@ -0,0 +1,24 @@
+module Etcd
+  module Requestable
+    include Etcd::Loggable
+    def http_client
+      @http_client ||= reset_http_client!
+    end
+
+    def reset_http_client!
+      @http_client = HTTPClient.new(agent_name: "etcd-rb/#{VERSION}")
+    end
+
+    def request(method, uri, args={})
+      logger.debug("request - #{method} #{uri} #{args.inspect}")
+      http_client.request(method, uri, args.merge(follow_redirect: true))
+    end
+
+    def request_data(method, uri, args={})
+      response = request(method, uri, args)
+      if response.status_code == 200
+        MultiJson.load(response.body)
+      end
+    end
+  end
+end

data/lib/etcd/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module Etcd
-  VERSION = '1.0.0.pre1'.freeze
+  VERSION = '1.0.0'.freeze
 end

data/lib/etcd.rb

@@ -1,10 +1,23 @@
 # encoding: utf-8
 
 module Etcd
-  EtcdError = Class.new(StandardError)
-  ConnectionError = Class.new(EtcdError)
+  EtcdError         = Class.new(StandardError)
+  ConnectionError   = Class.new(EtcdError)
   AllNodesDownError = Class.new(EtcdError)
 end
 
+require 'time'
+require 'thread'
+require 'httpclient'
+require 'multi_json'
+require 'logger'
+
 require 'etcd/version'
-require 'etcd/client'
+require 'etcd/constants'
+require 'etcd/loggable'
+require 'etcd/requestable'
+require 'etcd/heartbeat'
+require 'etcd/node'
+require 'etcd/cluster'
+require 'etcd/observer'
+require 'etcd/client'

data/spec/etcd/client_spec.rb

@@ -1,61 +1,17 @@
 # encoding: utf-8
-
 require 'spec_helper'
 
-
 module Etcd
   describe Client do
-    let :client do
-      described_class.new(uri: "http://#{host}:#{port}").connect
-    end
-
-    let :host do
-      'example.com'
-    end
-
-    let :port do
-      50_000
-    end
-
-    let :base_uri do
-      "http://#{host}:#{port}/v1"
-    end
-
-    let :machines_uri do
-      "http://#{host}:#{port}/v1/machines"
-    end
-
-    let :leader_uri do
-      "http://#{host}:#{port}/v1/leader"
-    end
+    include ClusterHelper
+    include ClientHelper
 
-    let :machines do
-      %W[http://#{host}:#{port} http://#{host}:#{port + 2} http://#{host}:#{port + 1}]
+    def base_uri
+      "http://127.0.0.1:4001/v1"
     end
 
-    before do
-      stub_request(:get, leader_uri).to_return(body: "http://#{host}:#{port}")
-      stub_request(:get, machines_uri).to_return(body: machines.join(', '))
-    end
-
-    describe '#connect' do
-      it 'gets the list of machines' do
-        client
-        WebMock.should have_requested(:get, machines_uri)
-      end
-
-      it 'raises an error when the seed node cannot be contacted' do
-        stub_request(:get, machines_uri).to_timeout
-        expect { client }.to raise_error(ConnectionError)
-      end
-
-      it 'discards the seed node and instead talks to the leader (the first in the list of machines)' do
-        machines.push(machines.shift)
-        stub_request(:get, machines_uri).to_return(body: machines.join(','))
-        stub_request(:get, "#{base_uri}/keys/foo").to_return(body: MultiJson.dump({'value' => 'wrong value'}))
-        stub_request(:get, "#{machines[0]}/v1/keys/foo").to_return(body: MultiJson.dump({'value' => 'right value'}))
-        client.get('/foo').should == 'right value'
-      end
+    let :client do
+      default_client
     end
 
     describe '#get' do
@@ -92,68 +48,9 @@ module Etcd
           client.get('/foo').should eql({'/foo/bar' => 'bar', '/foo/baz' => 'baz'})
         end
       end
-
-      context 'when not talking to the master' do
-        before do
-          stub_request(:get, "#{base_uri}/keys/foo").to_return(status: 307, headers: {'Location' => 'http://example.com:7654/v1/keys/foo'})
-          stub_request(:get, 'http://example.com:7654/v1/machines').to_return(body: %w[http://example.com:7654 http://example.com:7655].join(', '))
-          stub_request(:get, 'http://example.com:7654/v1/keys/foo').to_return(body: MultiJson.dump({'value' => 'bar'}))
-        end
-
-        it 'follows redirects' do
-          client.get('/foo').should == 'bar'
-        end
-
-        it 'sets the new host as the new leader when it is redirected' do
-          client.get('/foo')
-          client.get('/foo').should == 'bar'
-          WebMock.should have_requested(:get, "#{base_uri}/keys/foo").once
-        end
-
-        it 'asks for a new list of cluster machines' do
-          client.get('/foo')
-          WebMock.should have_requested(:get, 'http://example.com:7654/v1/machines').once
-        end
-      end
-
-      context 'when nodes go down' do
-        let :uris do
-          machines.map { |m| "#{m}/v1/keys/thing" }
-        end
-
-        it 'tries with another node when the leader appears to be down' do
-          stub_request(:get, uris[0]).to_timeout
-          stub_request(:get, uris[1]).to_return(body: MultiJson.dump({'value' => 'bar'}))
-          stub_request(:get, uris[2]).to_return(body: MultiJson.dump({'value' => 'baz'}))
-          client.get('/thing').should == 'bar'
-        end
-
-        it 'tries other nodes until it finds one that responds' do
-          stub_request(:get, uris[0]).to_timeout
-          stub_request(:get, uris[1]).to_timeout
-          stub_request(:get, uris[2]).to_return(body: MultiJson.dump({'value' => 'bar'}))
-          client.get('/thing').should == 'bar'
-        end
-
-        it 'follows redirects when trying a node after a leader failure, and saves the new leader' do
-          stub_request(:get, uris[0]).to_timeout
-          stub_request(:get, uris[1]).to_return(status: 307, headers: {'Location' => uris[2]})
-          stub_request(:get, uris[2]).to_return(body: MultiJson.dump({'value' => 'bar'}))
-          stub_request(:get, uris[2].sub('/keys/thing', '/machines')).to_return(body: [uris[1], uris[2]].join(', '))
-          stub_request(:get, uris[2].sub('/thing', '/bar')).to_return(body: MultiJson.dump({'value' => 'baz'}))
-          client.get('/thing').should == 'bar'
-          client.get('/bar').should == 'baz'
-        end
-
-        it 'raises an error when all leader candidates have been exhausted' do
-          stub_request(:get, uris[0]).to_timeout
-          stub_request(:get, uris[1]).to_timeout
-          stub_request(:get, uris[2]).to_timeout
-          expect { client.get('/thing') }.to raise_error(AllNodesDownError)
-        end
-      end
     end
 
+
     describe '#set' do
       before do
         stub_request(:post, "#{base_uri}/keys/foo").to_return(body: MultiJson.dump({}))
@@ -197,12 +94,12 @@ module Etcd
 
       it 'returns true when the key is successfully changed' do
         stub_request(:post, "#{base_uri}/keys/foo").to_return(body: MultiJson.dump({}))
-        client.update('/foo', 'bar', 'baz').should be_true
+        client.update('/foo', 'bar', 'baz').should eq(true)
       end
 
       it 'returns false when an error is returned' do
         stub_request(:post, "#{base_uri}/keys/foo").to_return(status: 400, body: MultiJson.dump({}))
-        client.update('/foo', 'bar', 'baz').should be_false
+        client.update('/foo', 'bar', 'baz').should eq(false)
       end
 
       it 'sets a TTL when the :ttl option is given' do
@@ -211,6 +108,7 @@ module Etcd
       end
     end
 
+
     describe '#delete' do
       before do
         stub_request(:delete, "#{base_uri}/keys/foo").to_return(body: MultiJson.dump({}))
@@ -235,12 +133,12 @@ module Etcd
     describe '#exists?' do
       it 'returns true if the key has a value' do
         stub_request(:get, "#{base_uri}/keys/foo").to_return(body: MultiJson.dump({'value' => 'bar'}))
-        client.exists?('/foo').should be_true
+        client.exists?('/foo').should eq(true)
       end
 
       it 'returns false if the key does not exist' do
         stub_request(:get, "#{base_uri}/keys/foo").to_return(status: 404, body: 'Not found')
-        client.exists?('/foo').should be_false
+        client.exists?('/foo').should eq(false)
       end
     end
 
@@ -252,7 +150,8 @@ module Etcd
         info[:key].should == '/foo'
         info[:value].should == 'bar'
         info[:index].should == 31
-        info[:expiration].to_f.should == (Time.utc(2013, 12, 11, 10, 9, 8) + 0.123).to_f
+        # rounding because of ruby 2.0 time parsing bug @see https://gist.github.com/mindreframer/6746829
+        info[:expiration].to_f.round.should == (Time.utc(2013, 12, 11, 10, 9, 8) + 0.123).to_f.round
         info[:ttl].should == 7
       end
 
@@ -261,7 +160,7 @@ module Etcd
         stub_request(:get, "#{base_uri}/keys/foo").to_return(body: body)
         info = client.info('/foo')
         info[:key].should == '/foo'
-        info[:dir].should be_true
+        info[:dir].should eq(true)
       end
 
       it 'returns only the pieces of information that are returned' do
@@ -296,156 +195,34 @@ module Etcd
       end
     end
 
-    describe '#watch' do
-      it 'sends a GET request for a watch of a key prefix' do
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {}).to_return(body: MultiJson.dump({}))
-        client.watch('/foo') { }
-        WebMock.should have_requested(:get, "#{base_uri}/watch/foo").with(query: {})
-      end
-
-      it 'sends a GET request for a watch of a key prefix from a specified index' do
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {'index' => 3}).to_return(body: MultiJson.dump({}))
-        client.watch('/foo', index: 3) { }
-        WebMock.should have_requested(:get, "#{base_uri}/watch/foo").with(query: {'index' => 3})
-      end
-
-      it 'yields the value' do
-        body = MultiJson.dump({'value' => 'bar'})
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {}).to_return(body: body)
-        value = nil
-        client.watch('/foo') do |v|
-          value = v
-        end
-        value.should == 'bar'
-      end
-
-      it 'yields the changed key' do
-        body = MultiJson.dump({'key' => '/foo/bar', 'value' => 'bar'})
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {}).to_return(body: body)
-        key = nil
-        client.watch('/foo') do |_, k|
-          key = k
-        end
-        key.should == '/foo/bar'
-      end
-
-      it 'yields info about the key, when it is a new key' do
-        body = MultiJson.dump({'action' => 'SET', 'key' => '/foo/bar', 'value' => 'bar', 'index' => 3, 'newKey' => true})
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {}).to_return(body: body)
-        info = nil
-        client.watch('/foo') do |_, _, i|
-          info = i
-        end
-        info[:action].should == :set
-        info[:key].should == '/foo/bar'
-        info[:value].should == 'bar'
-        info[:index].should == 3
-        info[:new_key].should be_true
-      end
+    describe "when cluster leader changes" do
+      include ClusterHelper
 
-      it 'yields info about the key, when the key was changed' do
-        body = MultiJson.dump({'action' => 'SET', 'key' => '/foo/bar', 'value' => 'bar', 'prevValue' => 'baz', 'index' => 3})
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {}).to_return(body: body)
-        info = nil
-        client.watch('/foo') do |_, _, i|
-          info = i
-        end
-        info[:action].should == :set
-        info[:key].should == '/foo/bar'
-        info[:value].should == 'bar'
-        info[:index].should == 3
-        info[:previous_value].should == 'baz'
+      let :etcd1_uri do
+        "http://127.0.0.1:4001"
       end
 
-      it 'yields info about the key, when the key has a TTL' do
-        body = MultiJson.dump({'action' => 'SET', 'key' => '/foo/bar', 'value' => 'bar', 'index' => 3, 'expiration' => '2013-12-11T12:09:08.123+02:00', 'ttl' => 7})
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {}).to_return(body: body)
-        info = nil
-        client.watch('/foo') do |_, _, i|
-          info = i
-        end
-        info[:action].should == :set
-        info[:key].should == '/foo/bar'
-        info[:value].should == 'bar'
-        info[:index].should == 3
-        info[:expiration].to_f.should == (Time.utc(2013, 12, 11, 10, 9, 8) + 0.123).to_f
-        info[:ttl].should == 7
+      let :etcd2_uri do
+        "http://127.0.0.1:4002"
       end
 
-      it 'returns the return value of the block' do
-        body = MultiJson.dump({'action' => 'SET', 'key' => '/foo/bar', 'value' => 'bar', 'index' => 3, 'expiration' => '2013-12-11T12:09:08.123+02:00', 'ttl' => 7})
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {}).to_return(body: body)
-        return_value = client.watch('/foo') do |_, k, _|
-          k
-        end
-        return_value.should == '/foo/bar'
-      end
-    end
+      it "#post - follows redirects and updates the cluster status transparently" do
+        with_stubbed_status(etcd1_uri)
+        with_stubbed_leaders(healthy_cluster_config)
 
-    describe '#observe' do
-      it 'watches the specified key prefix' do
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {}).to_return(body: MultiJson.dump({}))
-        barrier = Queue.new
-        observer = client.observe('/foo') do
-          barrier << :ping
-          observer.cancel
-          observer.join
-        end
-        barrier.pop
-        WebMock.should have_requested(:get, "#{base_uri}/watch/foo").with(query: {})
-      end
+        client = Etcd::Client.connect(:uris => etcd1_uri)
+        client.leader.etcd.should == etcd1_uri
+        client.leader.name.should == "node1"
 
-      it 're-watches the prefix with the last seen index immediately' do
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {}).to_return(body: MultiJson.dump({'index' => 3}))
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {'index' => 3}).to_return(body: MultiJson.dump({'index' => 4}))
-        barrier = Queue.new
-        observer = client.observe('/foo') do |_, _, info|
-          if info[:index] == 4
-            barrier << :ping
-            observer.cancel
-            observer.join
-          end
-        end
-        barrier.pop
-        WebMock.should have_requested(:get, "#{base_uri}/watch/foo").with(query: {})
-        WebMock.should have_requested(:get, "#{base_uri}/watch/foo").with(query: {'index' => 3})
-      end
+        with_stubbed_leaders(healthy_cluster_changed_leader_config)
 
-      it 'yields the value, key and info to the block given' do
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {}).to_return(body: MultiJson.dump({'action' => 'SET', 'key' => '/foo/bar', 'value' => 'bar', 'index' => 3, 'newKey' => true}))
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {'index' => 3}).to_return(body: MultiJson.dump({'action' => 'DELETE', 'key' => '/foo/baz', 'value' => 'foo', 'index' => 4}))
-        stub_request(:get, "#{base_uri}/watch/foo").with(query: {'index' => 4}).to_return(body: MultiJson.dump({'action' => 'SET', 'key' => '/foo/bar', 'value' => 'hello', 'index' => 5}))
-        barrier = Queue.new
-        values = []
-        keys = []
-        actions = []
-        new_keys = []
-        observer = client.observe('/foo') do |value, key, info|
-          values << value
-          keys << key
-          actions << info[:action]
-          new_keys << info[:new_key]
-          if info[:index] == 5
-            barrier << :ping
-            observer.cancel
-            observer.join
-          end
-        end
-        barrier.pop
-        values.should == %w[bar foo hello]
-        keys.should == %w[/foo/bar /foo/baz /foo/bar]
-        actions.should == [:set, :delete, :set]
-        new_keys.should == [true, nil, nil]
+        stub_request(:post, "#{etcd1_uri}/v1/keys/foo").to_return(status: 307, headers: {'Location' => "#{etcd2_uri}/v1/keys/foo"})
+        stub_request(:post, "#{etcd2_uri}/v1/keys/foo").to_return(body: MultiJson.dump({'value' => 'bar'}))
+        client.set("foo", "bar")
+        client.leader.etcd.should == etcd2_uri
+        client.leader.name.should == "node2"
       end
     end
 
-    describe '#machines' do
-      it 'returns a list of host and ports of the machines in the etcd cluster' do
-        body = 'http://host01:4001, http://host02:4001, http://host03:4001'
-        stub_request(:get, machines_uri).to_return(body: body)
-        stub_request(:get, machines_uri.sub(base_uri, 'http://host01:4001/v1')).to_return(body: body)
-        client.machines.should == %w[http://host01:4001 http://host02:4001 http://host03:4001]
-      end
-    end
   end
-end
+end

data/spec/etcd/cluster_spec.rb

@@ -0,0 +1,176 @@
+require 'spec_helper'
+
+
+module Etcd
+  describe Cluster do
+
+    include ClusterHelper
+
+    let :cluster_uri do
+      "http://127.0.0.1:4001"
+    end
+
+    describe :class_methods do
+      describe '#cluster_status' do
+        it "returns parsed data" do
+          with_stubbed_status(cluster_uri) do
+            res = Etcd::Cluster.cluster_status(cluster_uri)
+            res[0][:name].should == "node1"
+          end
+        end
+      end
+
+      describe '#parse_cluster_status' do
+        it "works with correct parsed json" do
+          res = Etcd::Cluster.parse_cluster_status(status_data)
+          res[0].should == {
+            :name => "node1",
+            :raft => "http://127.0.0.1:7001",
+            :etcd => "http://127.0.0.1:4001",
+          }
+        end
+      end
+
+      describe '#nodes_from_uri' do
+        it "returns node instances created from uri" do
+          with_stubbed_status(cluster_uri) do
+            nodes = Etcd::Cluster.nodes_from_uri(cluster_uri)
+            nodes.size.should == 3
+            nodes.first.class.should == Etcd::Node
+          end
+        end
+
+        it "but those instances have no real status yet" do
+          with_stubbed_status(cluster_uri) do
+            nodes = Etcd::Cluster.nodes_from_uri(cluster_uri)
+            nodes.size.should == 3
+            nodes.first.status == :unknown
+          end
+        end
+      end
+
+      describe '#init_from_uris', "- preferred way to initialize cluster" do
+        describe "in healthy cluster" do
+          it "has all nodes at status :running" do
+            with_stubbed_status(cluster_uri) do
+              with_stubbed_leaders(healthy_cluster_config) do
+                cluster = Etcd::Cluster.init_from_uris(cluster_uri)
+                nodes = cluster.nodes
+                nodes.size.should == 3
+                nodes.map{|x| x.status}.uniq.should == [:running]
+              end
+            end
+          end
+
+          it "has one leader node" do
+            with_stubbed_status(cluster_uri) do
+              with_stubbed_leaders(healthy_cluster_config) do
+                cluster = Etcd::Cluster.init_from_uris(cluster_uri)
+                leader  = cluster.leader
+                leader.etcd.should == cluster_uri
+              end
+            end
+          end
+        end
+
+
+        describe "in un-healthy cluster" do
+          it "has some  nodes at status :down" do
+            with_stubbed_status(cluster_uri) do
+              with_stubbed_leaders(one_down_cluster_config) do
+                cluster = Etcd::Cluster.init_from_uris(cluster_uri)
+                nodes = cluster.nodes
+                nodes.size.should == 3
+                nodes.map{|x| x.status}.uniq.should == [:running, :down]
+              end
+            end
+          end
+        end
+      end
+    end
+
+    describe :instance_methods do
+      describe '#new' do
+        it "will not request any info on initialization" do
+          cluster = Etcd::Cluster.new(cluster_uri)
+          WebMock.should_not have_requested(:get, "http://127.0.0.1:4001/v1/keys/_etcd/machines/")
+        end
+      end
+
+      describe '#nodes' do
+        it "will update nodes info on first nodes access" do
+          with_stubbed_status(cluster_uri) do
+            with_stubbed_leaders(healthy_cluster_config) do
+              cluster = Etcd::Cluster.new(cluster_uri)
+              nodes   = cluster.nodes
+              nodes.size.should == 3
+            end
+          end
+        end
+
+        it "caches result on further queries" do
+          with_stubbed_status(cluster_uri) do
+            with_stubbed_leaders(healthy_cluster_config) do
+              cluster = Etcd::Cluster.new(cluster_uri)
+              nodes   = cluster.nodes
+              nodes.map{|x| x.status}.uniq.should == [:running]
+              with_stubbed_leaders(one_down_cluster_config) do
+                nodes   = cluster.nodes
+                nodes.map{|x| x.status}.uniq.should_not == [:running, :down]
+                # now update for real
+                nodes   = cluster.update_status
+                nodes.map{|x| x.status}.uniq.should == [:running, :down]
+              end
+            end
+          end
+        end
+
+      end
+
+      describe '#update_status' do
+        it "will re-update node stati from etcd" do
+          with_stubbed_status(cluster_uri) do
+            with_stubbed_leaders(healthy_cluster_config) do
+              cluster = Etcd::Cluster.new(cluster_uri)
+              nodes   = cluster.update_status
+              nodes.size.should == 3
+              nodes.map{|x| x.status}.uniq.should == [:running]
+              with_stubbed_leaders(one_down_cluster_config) do
+                nodes   = cluster.update_status
+                nodes.map{|x| x.status}.uniq.should == [:running, :down]
+              end
+            end
+          end
+        end
+      end
+
+      describe '#leader' do
+
+        it "returns the leader node from nodes" do
+          with_stubbed_status(cluster_uri) do
+            with_stubbed_leaders(healthy_cluster_config) do
+              cluster = Etcd::Cluster.new(cluster_uri)
+              leader = cluster.leader
+              leader.etcd.should == cluster_uri
+              leader.is_leader.should eq(true)
+            end
+          end
+        end
+
+        it "re-sets leader after every update_status" do
+          with_stubbed_status(cluster_uri) do
+            with_stubbed_leaders(healthy_cluster_config) do
+              cluster = Etcd::Cluster.new(cluster_uri)
+              cluster.leader.etcd.should == cluster_uri
+              with_stubbed_leaders(healthy_cluster_changed_leader_config) do
+                nodes   = cluster.update_status
+                cluster.leader.etcd.should == "http://127.0.0.1:4002"
+              end
+            end
+          end
+        end
+
+      end
+    end
+  end
+end