chef-provisioning 0.15

data/lib/chef/provisioning/convergence_strategy/install_msi.rb

@@ -0,0 +1,58 @@
+require 'chef/provisioning/convergence_strategy/precreate_chef_objects'
+require 'pathname'
+
+class Chef
+module Provisioning
+  class ConvergenceStrategy
+    class InstallMsi < PrecreateChefObjects
+      @@install_msi_cache = {}
+
+      def initialize(convergence_options, config)
+        super
+        @install_msi_url = convergence_options[:install_msi_url] || 'http://www.opscode.com/chef/install.msi'
+        @install_msi_path = convergence_options[:install_msi_path] || "$env:TEMP\\#{File.basename(@install_msi_url)}"
+        @chef_client_timeout = convergence_options.has_key?(:chef_client_timeout) ? convergence_options[:chef_client_timeout] : 120*60 # Default: 2 hours
+      end
+
+      attr_reader :install_msi_url
+      attr_reader :install_msi_path
+
+      def setup_convergence(action_handler, machine)
+        if !convergence_options.has_key?(:client_rb_path) || !convergence_options.has_key?(:client_pem_path)
+          system_drive = machine.execute_always('$env:SystemDrive').stdout.strip
+          @convergence_options = Cheffish::MergedConfig.new(convergence_options, {
+            :client_rb_path => "#{system_drive}\\chef\\client.rb",
+            :client_pem_path => "#{system_drive}\\chef\\client.pem"
+          })
+        end
+
+        super
+
+        # Install chef-client.  TODO check and update version if not latest / not desired
+        if machine.execute_always('chef-client -v').exitstatus != 0
+          # TODO ssh verification of install.msi before running arbtrary code would be nice?
+          # TODO find a way to cache this on the host like with the Unix stuff.
+          # Limiter is we don't know how to efficiently upload large files to
+          # the remote machine with WMI.
+          machine.execute(action_handler, "(New-Object System.Net.WebClient).DownloadFile(#{machine.escape(install_msi_url)}, #{machine.escape(install_msi_path)})")
+          machine.execute(action_handler, "msiexec /qn /i #{machine.escape(install_msi_path)}")
+        end
+      end
+
+      def converge(action_handler, machine)
+        super
+
+        # TODO For some reason I get a 500 back if I don't do -l debug
+        action_handler.open_stream(machine.node['name']) do |stdout|
+          action_handler.open_stream(machine.node['name']) do |stderr|
+            machine.execute(action_handler, "chef-client -l debug",
+              :stream_stdout => stdout,
+              :stream_stderr => stderr,
+              :timeout => @chef_client_timeout)
+          end
+        end
+      end
+    end
+  end
+end
+end

data/lib/chef/provisioning/convergence_strategy/install_sh.rb

@@ -0,0 +1,55 @@
+require 'chef/provisioning/convergence_strategy/precreate_chef_objects'
+require 'pathname'
+
+class Chef
+module Provisioning
+  class ConvergenceStrategy
+    class InstallSh < PrecreateChefObjects
+      @@install_sh_cache = {}
+
+      def initialize(convergence_options, config)
+        convergence_options = Cheffish::MergedConfig.new(convergence_options, {
+          :client_rb_path => '/etc/chef/client.rb',
+          :client_pem_path => '/etc/chef/client.pem'
+        })
+        super(convergence_options, config)
+        @install_sh_url = convergence_options[:install_sh_url] || 'http://www.opscode.com/chef/install.sh'
+        @install_sh_path = convergence_options[:install_sh_path] || '/tmp/chef-install.sh'
+        @bootstrap_env = convergence_options[:bootstrap_proxy] ? "http_proxy=#{convergence_options[:bootstrap_proxy]}" : ""
+        @chef_client_timeout = convergence_options.has_key?(:chef_client_timeout) ? convergence_options[:chef_client_timeout] : 120*60 # Default: 2 hours
+      end
+
+      attr_reader :install_sh_url
+      attr_reader :install_sh_path
+      attr_reader :bootstrap_env
+
+      def setup_convergence(action_handler, machine)
+        super
+
+        # Install chef-client.  TODO check and update version if not latest / not desired
+        if machine.execute_always('chef-client -v').exitstatus != 0
+          # TODO ssh verification of install.sh before running arbtrary code would be nice?
+          @@install_sh_cache[install_sh_url] ||= Net::HTTP.get(URI(install_sh_url))
+          machine.write_file(action_handler, install_sh_path, @@install_sh_cache[install_sh_url], :ensure_dir => true)
+          machine.execute(action_handler, "bash -c '#{bootstrap_env} bash #{install_sh_path}'")
+        end
+      end
+
+      def converge(action_handler, machine)
+        super
+
+        action_handler.open_stream(machine.node['name']) do |stdout|
+          action_handler.open_stream(machine.node['name']) do |stderr|
+            command_line = "chef-client"
+            command_line << " -l #{config[:log_level].to_s}" if config[:log_level]
+            machine.execute(action_handler, command_line,
+              :stream_stdout => stdout,
+              :stream_stderr => stderr,
+              :timeout => @chef_client_timeout)
+          end
+        end
+      end
+    end
+  end
+end
+end

data/lib/chef/provisioning/convergence_strategy/no_converge.rb

@@ -0,0 +1,39 @@
+require 'chef/provisioning/convergence_strategy'
+require 'pathname'
+require 'cheffish'
+
+class Chef
+module Provisioning
+  class ConvergenceStrategy
+    class NoConverge < ConvergenceStrategy
+      def initialize(convergence_options, config)
+        super
+      end
+
+      def chef_server
+        @chef_server ||= convergence_options[:chef_server] || Cheffish.default_chef_server(config)
+      end
+
+      def setup_convergence(action_handler, machine)
+      end
+
+      def converge(action_handler, machine)
+      end
+
+      def cleanup_convergence(action_handler, machine_spec)
+        _self = self
+        Chef::Provisioning.inline_resource(action_handler) do
+          chef_node machine_spec.name do
+            chef_server _self.chef_server
+            action :delete
+          end
+          chef_client machine_spec.name do
+            chef_server _self.chef_server
+            action :delete
+          end
+        end
+      end
+    end
+  end
+end
+end

data/lib/chef/provisioning/convergence_strategy/precreate_chef_objects.rb

@@ -0,0 +1,183 @@
+require 'chef/provisioning/convergence_strategy'
+require 'pathname'
+require 'cheffish'
+
+class Chef
+module Provisioning
+  class ConvergenceStrategy
+    class PrecreateChefObjects < ConvergenceStrategy
+      def initialize(convergence_options, config)
+        super
+      end
+
+      def chef_server
+        @chef_server ||= convergence_options[:chef_server] || Cheffish.default_chef_server(config)
+      end
+
+      def setup_convergence(action_handler, machine)
+        # Create keys on machine
+        public_key = create_keys(action_handler, machine)
+        # Create node and client on chef server
+        create_chef_objects(action_handler, machine, public_key)
+
+        # If the chef server lives on localhost, tunnel the port through to the guest
+        # (we need to know what got tunneled!)
+        chef_server_url = chef_server[:chef_server_url]
+        chef_server_url = machine.make_url_available_to_remote(chef_server_url)
+
+        # Support for multiple ohai hints, required on some platforms
+        create_ohai_files(action_handler, machine)
+
+        # Create client.rb and client.pem on machine
+        content = client_rb_content(chef_server_url, machine.node['name'])
+        machine.write_file(action_handler, convergence_options[:client_rb_path], content, :ensure_dir => true)
+      end
+
+      def converge(action_handler, machine)
+        machine.make_url_available_to_remote(chef_server[:chef_server_url])
+      end
+
+      def cleanup_convergence(action_handler, machine_spec)
+        _self = self
+        Chef::Provisioning.inline_resource(action_handler) do
+          chef_node machine_spec.name do
+            chef_server _self.chef_server
+            action :delete
+          end
+          chef_client machine_spec.name do
+            chef_server _self.chef_server
+            action :delete
+          end
+        end
+      end
+
+      protected
+
+      def create_keys(action_handler, machine)
+        server_private_key = machine.read_file(convergence_options[:client_pem_path])
+        if server_private_key
+          begin
+            server_private_key, format = Cheffish::KeyFormatter.decode(server_private_key)
+          rescue
+            server_private_key = nil
+          end
+        end
+
+        if server_private_key
+          if source_key && server_private_key.to_pem != source_key.to_pem
+            # If the server private key does not match our source key, overwrite it
+            server_private_key = source_key
+            if convergence_options[:allow_overwrite_keys]
+              machine.write_file(action_handler, convergence_options[:client_pem_path], server_private_key.to_pem, :ensure_dir => true)
+            else
+              raise "Private key on machine #{machine.name} does not match desired input key."
+            end
+          end
+
+        else
+
+          # If the server does not already have keys, create them and upload
+          _convergence_options = convergence_options
+          Chef::Provisioning.inline_resource(action_handler) do
+            private_key 'in_memory' do
+              path :none
+              if _convergence_options[:private_key_options]
+                _convergence_options[:private_key_options].each_pair do |key,value|
+                  send(key, value)
+                end
+              end
+              after { |resource, private_key| server_private_key = private_key }
+            end
+          end
+
+          machine.write_file(action_handler, convergence_options[:client_pem_path], server_private_key.to_pem, :ensure_dir => true)
+        end
+
+        server_private_key.public_key
+      end
+
+      def is_localhost(host)
+        host == '127.0.0.1' || host == 'localhost' || host == '[::1]'
+      end
+
+      def source_key
+        if convergence_options[:source_key].is_a?(String)
+          key, format = Cheffish::KeyFormatter.decode(convergence_options[:source_key], convergence_options[:source_key_pass_phrase])
+          key
+        elsif convergence_options[:source_key]
+          convergence_options[:source_key]
+        elsif convergence_options[:source_key_path]
+          key, format = Cheffish::KeyFormatter.decode(IO.read(convergence_options[:source_key_path]), convergence_options[:source_key_pass_phrase], convergence_options[:source_key_path])
+          key
+        else
+          nil
+        end
+      end
+
+      # Create the ohai file(s)
+      def create_ohai_files(action_handler, machine)
+        if convergence_options[:ohai_hints]
+          convergence_options[:ohai_hints].each_pair do |hint, data|
+            # The location of the ohai hint
+            ohai_hint = "/etc/chef/ohai/hints/#{hint}.json"
+            machine.write_file(action_handler, ohai_hint, data.to_json, :ensure_dir => true)
+          end
+        end
+      end
+
+      def create_chef_objects(action_handler, machine, public_key)
+        _convergence_options = convergence_options
+        _chef_server = chef_server
+        # Save the node and create the client keys and client.
+        Chef::Provisioning.inline_resource(action_handler) do
+          # Create client
+          chef_client machine.name do
+            chef_server _chef_server
+            source_key public_key
+            output_key_path _convergence_options[:public_key_path]
+            output_key_format _convergence_options[:public_key_format]
+            admin _convergence_options[:admin]
+            validator _convergence_options[:validator]
+          end
+
+          # Create node
+          # TODO strip automatic attributes first so we don't race with "current state"
+          chef_node machine.name do
+            chef_server _chef_server
+            raw_json machine.node
+          end
+        end
+
+        # If using enterprise/hosted chef, fix acls
+        if chef_server[:chef_server_url] =~ /\/+organizations\/.+/
+          grant_client_node_permissions(action_handler, chef_server, machine.name, ["read", "update"])
+        end
+      end
+
+      # Grant the client permissions to the node
+      # This procedure assumes that the client name and node name are the same
+      def grant_client_node_permissions(action_handler, chef_server, node_name, perms)
+        api = Cheffish.chef_server_api(chef_server)
+        node_perms = api.get("/nodes/#{node_name}/_acl")
+        perms.each do |p|
+          if !node_perms[p]['actors'].include?(node_name)
+            action_handler.perform_action "Add #{node_name} to client #{p} ACLs" do
+              node_perms[p]['actors'] << node_name
+              api.put("/nodes/#{node_name}/_acl/#{p}", p => node_perms[p])
+            end
+          end
+        end
+      end
+
+      def client_rb_content(chef_server_url, node_name)
+        <<EOM
+chef_server_url #{chef_server_url.inspect}
+node_name #{node_name.inspect}
+client_key #{convergence_options[:client_pem_path].inspect}
+ssl_verify_mode :verify_peer
+EOM
+      end
+    end
+  end
+end
+end

data/lib/chef/provisioning/driver.rb

@@ -0,0 +1,304 @@
+class Chef
+module Provisioning
+  #
+  # A Driver instance represents a place where machines can be created and found,
+  # and contains methods to create, delete, start, stop, and find them.
+  #
+  # For AWS, a Driver instance corresponds to a single account.
+  # For Vagrant, it is a directory where VM files are found.
+  #
+  # = How to Make a Driver
+  #
+  # To implement a Driver, you must implement the following methods:
+  #
+  # * initialize(driver_url) - create a new driver with the given URL
+  # * driver_url - a URL representing everything unique about your driver. (NOT credentials)
+  # * allocate_machine - ask the driver to allocate a machine to you.
+  # * ready_machine - get the machine "ready" - wait for it to be booted and accessible (for example, accessible via SSH transport).
+  # * stop_machine - stop the machine.
+  # * destroy_machine - delete the machine.
+  # * connect_to_machine - connect to the given machine.
+  #
+  # Optionally, you can also implement:
+  # * allocate_machines - allocate an entire group of machines.
+  # * ready_machines - get a group of machines warm and booted.
+  # * stop_machines - stop a group of machines.
+  # * destroy_machines - delete a group of machines.
+  #
+  # Additionally, you must create a file named `chef/provisioning/driver_init/<scheme>.rb`,
+  # where <scheme> is the name of the scheme you chose for your driver_url. This
+  # file, when required, must call Chef::Provisioning.add_registered_driver(<scheme>, <class>).
+  # The given <class>.from_url(url, config) will be called with a driver_url and
+  # configuration.
+  #
+  # All of these methods must be idempotent - if the work is already done, they
+  # just don't do anything.
+  #
+  class Driver
+    #
+    # Inflate a driver from a driver URL.
+    #
+    #
+    # @param [String] driver_url  the URL to inflate the driver
+    # config - a configuration hash.  See "config" for a list of known keys.
+    #
+    # == Returns
+    # A Driver representing the given driver_url.
+    #
+    def initialize(driver_url, config)
+      @driver_url = driver_url
+      @config = config
+    end
+
+    #
+    # Override this on specific driver classes
+    #
+    def self.from_url(driver_url, config)
+      Chef::Provisioning.from_url(driver_url, config)
+    end
+
+    #
+    # A URL representing the driver and the place where machines come from.
+    # This will be stuffed in machine_spec.location['driver_url'] so that the
+    # machine can be re-inflated.  URLs must have a unique scheme identifying the
+    # driver class, and enough information to identify the place where created
+    # machines can be found.  For AWS, this is the account number; for lxc and
+    # vagrant, it is the directory in which VMs and containers are.
+    #
+    # For example:
+    # - fog:AWS:123456789012
+    # - vagrant:/var/vms
+    # - lxc:
+    # - docker:
+    #
+    attr_reader :driver_url
+
+    # A configuration hash.  These keys may be present:
+    #   - :driver_options: a driver-defined object containing driver config.
+    #   - :private_keys: a hash of private keys, with a "name" and a "value".  Values are either strings (paths) or PrivateKey objects.
+    #   - :private_key_paths: a list of paths to directories containing private keys.
+    #   - :write_private_key_path: the path to which we write new keys by default.
+    #   - :log_level: :debug/:info/:warn/:error/:fatal
+    #   - :chef_server_url: url to chef server
+    #   - :node_name: username to talk to chef server
+    #   - :client_key: path to key used to talk to chef server
+    attr_reader :config
+
+    #
+    # Driver configuration. Equivalent to config[:driver_options] || {}
+    #
+    def driver_options
+      config[:driver_options] || {}
+    end
+
+
+    # Allocate a machine from the underlying service.  This method
+    # does not need to wait for the machine to boot or have an IP, but it must
+    # store enough information in machine_spec.location to find the machine
+    # later in ready_machine.
+    #
+    # If a machine is powered off or otherwise unusable, this method may start
+    # it, but does not need to wait until it is started.  The idea is to get the
+    # gears moving, but the job doesn't need to be done :)
+    #
+    # @param [Chef::Provisioning::ActionHandler] action_handler The action_handler object that is calling this method
+    # @param [Chef::Provisioning::MachineSpec] machine_spec A machine specification representing this machine.
+    # @param [Hash] machine_options A set of options representing the desired options when
+    # constructing the machine
+    #
+    # @return [Chef::Provisioning::MachineSpec] Modifies the passed-in machine_spec.  Anything in here will be saved
+    # back after allocate_machine completes.
+    #
+    def allocate_machine(action_handler, machine_spec, machine_options)
+      raise "#{self.class} does not implement allocate_machine"
+    end
+
+    # Ready a machine, to the point where it is running and accessible via a
+    # transport. This will NOT allocate a machine, but may kick it if it is down.
+    # This method waits for the machine to be usable, returning a Machine object
+    # pointing at the machine, allowing useful actions like setup, converge,
+    # execute, file and directory.
+    #
+    #
+    # @param [Chef::Provisioning::ActionHandler] action_handler The action_handler object that is calling this method
+    # @param [Chef::Provisioning::MachineSpec] machine_spec A machine specification representing this machine.
+    # @param [Hash] machine_options A set of options representing the desired state of the machine
+    #
+    # @return [Machine] A machine object pointing at the machine, allowing useful actions like setup,
+    # converge, execute, file and directory.
+    #
+    def ready_machine(action_handler, machine_spec, machine_options)
+      raise "#{self.class} does not implement ready_machine"
+    end
+
+    # Connect to a machine without allocating or readying it.  This method will
+    # NOT make any changes to anything, or attempt to wait.
+    #
+    # @param [Chef::Provisioning::MachineSpec] machine_spec MachineSpec representing this machine.
+    # @param [Hash] machine_options
+    # @return [Machine] A machine object pointing at the machine, allowing useful actions like setup,
+    # converge, execute, file and directory.
+    #
+    def connect_to_machine(machine_spec, machine_options)
+      raise "#{self.class} does not implement connect_to_machine"
+    end
+
+
+    # Delete the given machine --  destroy the machine,
+    # returning things to the state before allocate_machine was called.
+    #
+    # @param [Chef::Provisioning::ActionHandler] action_handler The action_handler object that is calling this method
+    # @param [Chef::Provisioning::MachineSpec] machine_spec A machine specification representing this machine.
+    # @param [Hash] machine_options A set of options representing the desired state of the machine
+    def destroy_machine(action_handler, machine_spec, machine_options)
+      raise "#{self.class} does not implement destroy_machine"
+    end
+
+    # Stop the given machine.
+    #
+    # @param [Chef::Provisioning::ActionHandler] action_handler The action_handler object that is calling this method
+    # @param [Chef::Provisioning::MachineSpec] machine_spec A machine specification representing this machine.
+    # @param [Hash] machine_options A set of options representing the desired state of the machine
+    def stop_machine(action_handler, machine_spec, machine_options)
+      raise "#{self.class} does not implement stop_machine"
+    end
+
+    # Allocate an image. Returns quickly with an ID that tracks the image.
+    #
+    # @param [Chef::Provisioning::ActionHandler] action_handler The action_handler object that is calling this method
+    # @param [Chef::Provisioning::ImageSpec] image_spec A machine specification representing this machine.
+    # @param [Hash] image_options A set of options representing the desired state of the machine
+    def allocate_image(action_handler, image_spec, image_options, machine_spec)
+      raise "#{self.class} does not implement create_image"
+    end
+
+    # Ready an image, waiting till the point where it is ready to be used.
+    #
+    # @param [Chef::Provisioning::ActionHandler] action_handler The action_handler object that is calling this method
+    # @param [Chef::Provisioning::ImageSpec] image_spec A machine specification representing this machine.
+    # @param [Hash] image_options A set of options representing the desired state of the machine
+    def ready_image(action_handler, image_spec, image_options)
+      raise "#{self.class} does not implement ready_image"
+    end
+
+    # Destroy an image using this service.
+    #
+    # @param [Chef::Provisioning::ActionHandler] action_handler The action_handler object that is calling this method
+    # @param [Chef::Provisioning::ImageSpec] image_spec A machine specification representing this machine.
+    # @param [Hash] image_options A set of options representing the desired state of the machine
+    def destroy_image(action_handler, image_spec, image_options)
+      raise "#{self.class} does not implement destroy_image"
+    end
+
+    #
+    # Optional interface methods
+    #
+
+    #
+    # Allocate a set of machines.  This should have the same effect as running
+    # allocate_machine on all machine_specs.
+    #
+    # Drivers do not need to implement this; the default implementation
+    # calls acquire_machine in parallel.
+    #
+    # == Parallelizing
+    #
+    # The parallelizer must implement #parallelize
+    # @example Example parallelizer
+    #   parallelizer.parallelize(specs_and_options) do |machine_spec|
+    #     allocate_machine(action_handler, machine_spec)
+    #   end.to_a
+    #   # The to_a at the end causes you to wait until the parallelization is done
+    #
+    # This object is shared among other chef-provisioning actions, ensuring that you do
+    # not go over parallelization limits set by the user.  Use of the parallelizer
+    # to parallelizer machines is not required.
+    #
+    # == Passing a block
+    #
+    # If you pass a block to this function, each machine will be yielded to you
+    # as it completes, and then the function will return when all machines are
+    # yielded.
+    #
+    # @example Passing a block
+    #   allocate_machines(action_handler, specs_and_options, parallelizer) do |machine_spec|
+    #     ...
+    #   end
+    #
+    # @param [Chef::Provisioning::ActionHandler] action_handler The action_handler object that is calling this method; this
+    #        is generally a driver, but could be anything that can support the
+    #        interface (i.e., in the case of the test kitchen provisioning driver for
+    #        acquiring and destroying VMs).
+    # @param [Hash] specs_and_options A hash of machine_spec -> machine_options representing the
+    #                 machines to allocate.
+    # @param [Parallelizer] parallelizer an object with a parallelize() method that works like this:
+    # @return [Array<Machine>] An array of machine objects created
+    def allocate_machines(action_handler, specs_and_options, parallelizer)
+      parallelizer.parallelize(specs_and_options) do |machine_spec, machine_options|
+        allocate_machine(add_prefix(machine_spec, action_handler), machine_spec, machine_options)
+        yield machine_spec if block_given?
+        machine_spec
+      end.to_a
+    end
+
+    # Ready machines in batch, in parallel if possible.
+    def ready_machines(action_handler, specs_and_options, parallelizer)
+      parallelizer.parallelize(specs_and_options) do |machine_spec, machine_options|
+        machine = ready_machine(add_prefix(machine_spec, action_handler), machine_spec, machine_options)
+        yield machine if block_given?
+        machine
+      end.to_a
+    end
+
+    # Stop machines in batch, in parallel if possible.
+    def stop_machines(action_handler, specs_and_options, parallelizer)
+      parallelizer.parallelize(specs_and_options) do |machine_spec, machine_options|
+        stop_machine(add_prefix(machine_spec, action_handler), machine_spec, machine_options)
+        yield machine_spec if block_given?
+      end.to_a
+    end
+
+    # Delete machines in batch, in parallel if possible.
+    def destroy_machines(action_handler, specs_and_options, parallelizer)
+      parallelizer.parallelize(specs_and_options) do |machine_spec, machine_options|
+        destroy_machine(add_prefix(machine_spec, action_handler), machine_spec, machine_options)
+        yield machine_spec if block_given?
+      end.to_a
+    end
+
+    # Allocate a load balancer
+    # @param [ChefMetal::ActionHandler] action_handler The action handler
+    # @param [ChefMetal::LoadBalancerSpec] lb_spec Frozen LB specification
+    # @param [Hash] lb_options A hash of options to pass the LB
+    def allocate_load_balancer(action_handler, lb_spec, lb_options)
+    end
+
+    # Make the load balancer ready
+    # @param [ChefMetal::ActionHandler] action_handler The action handler
+    # @param [ChefMetal::LoadBalancerSpec] lb_spec Frozen LB specification
+    # @param [Hash] lb_options A hash of options to pass the LB
+    def ready_load_balancer(action_handler, lb_spec, lb_options)
+    end
+
+    # Destroy the load balancer
+    # @param [ChefMetal::ActionHandler] action_handler The action handler
+    # @param [ChefMetal::LoadBalancerSpec] lb_spec Frozen LB specification
+    # @param [Hash] lb_options A hash of options to pass the LB
+    def destroy_load_balancer(action_handler, lb_spec, lb_options)
+    end
+
+    def update_load_balancer(action_handler, lb_spec, lb_options, opts = {})
+    end
+
+    protected
+
+    def add_prefix(machine_spec, action_handler)
+      AddPrefixActionHandler.new(action_handler, "[#{machine_spec.name}] ")
+    end
+
+    def get_private_key(name)
+      Cheffish.get_private_key(name, config)
+    end
+  end
+end
+end