souffle 0.0.3 → 0.0.4

data/CHANGELOG.md

@@ -1,19 +1,30 @@
 # Changelog
 
+### v0.0.4
 
-###  v0.0.1
+* Added chef-client support
+* Added warning for cli one-off runs
+* Added provider plugins support
+* Added provider helper methods
+* Updated documentation
+* System timeout set to 10 minutes
+* Updated handler for node provisioning
+* Updated handler for system provisioning
+* Node provisioning updates are now evented
+* Updated eventmachine to 1.0.0
+* Fixed timing issue with mdadm install
 
-* Application framework laid out
-* Initial node state machine
-* Initial system state machine
-* Added AWS provider
-* Initial command line interface
-* Added daemonization
-* Added singleton configuration
-* Initial runlist parser
-* Initial dependency traversal
-* System rebalancing (pre-provisioning)
-* Added REST API
+### v0.0.3
+
+* Multiple documentation updates
+* AWS provider
+  * Tags now use `hex(4)` instead of `hex(6)`
+  * Deprecated default tag prefix
+* Added chef roles support to providers
+* Fixed /version url link
+* Added `tag`, `domain`, and `fqdn` node helpers
+* Bugfix for repo_path
+* Hostname is now set at provisioning
 
 ### v0.0.2
 
@@ -29,14 +40,16 @@
 * Optimized AWS polling mechanism
 * Optimized AWS creation ordering
 
-### v0.0.3
+###  v0.0.1
 
-* Multiple documentation updates
-* AWS provider
-  * Tags now use `hex(4)` instead of `hex(6)`
-  * Deprecated default tag prefix
-* Added chef roles support to providers
-* Fixed /version url link
-* Added `tag`, `domain`, and `fqdn` node helpers
-* Bugfix for repo_path
-* Hostname is now set at provisioning
+* Application framework laid out
+* Initial node state machine
+* Initial system state machine
+* Added AWS provider
+* Initial command line interface
+* Added daemonization
+* Added singleton configuration
+* Initial runlist parser
+* Initial dependency traversal
+* System rebalancing (pre-provisioning)
+* Added REST API

data/Gemfile

@@ -4,7 +4,7 @@ source "http://rubygems.org"
 #   gem "activesupport", ">= 2.3.5"
 
 gem "yajl-ruby", "~> 1.1.0"
-gem "eventmachine", "~> 0.12.10"
+gem "eventmachine", "~> 1.0.0"
 gem "amqp", "~> 0.9.7"
 gem "state_machine", "~> 1.1.2"
 gem "em-ssh", "~> 0.4.0"

data/README.md

@@ -48,9 +48,9 @@ As an example system we'll generate two nodes that both are provisioned with `so
 
 ```json
 {
-  "provider": "aws",
   "user": "josh",
   "options": {
+    "provider": "aws",
     "domain": "mydomain.com",
     "type": "solo",
     "aws_ebs_size": 10,

data/lib/souffle/exceptions.rb

@@ -30,4 +30,7 @@ class Souffle::Exceptions
 
   # The AWS Keys are invalid.
   class InvalidAwsKeys < RuntimeError; end
+
+  # Plugin does not exist.
+  class PluginDoesNotExist < RuntimeError; end
 end

data/lib/souffle/http.rb

@@ -35,9 +35,8 @@ class Souffle::Http < Sinatra::Base
     Souffle::Log.debug msg
     Souffle::Log.debug data.to_s
 
-    provider = Souffle::Provider::AWS.new
-
     system = Souffle::System.from_hash(data)
+    provider = Souffle::Provider.plugin(system.try_opt(:provider)).new
     system_tag = provider.create_system(system)
 
     begin

data/lib/souffle/node.rb

@@ -99,15 +99,22 @@ class Souffle::Node
     nil
   end
 
+  # Returns the current system provider.
+  #
+  # @return [ Souffle::Provider::Base ] The current system provider.
+  def provider
+    system.provider
+  end
+
   # The logging prefix for the given node.
-  # 
+  #
   # @return [ String ] The logging prefix for the given node.
   def log_prefix
     "[#{tag}: #{name}]"
   end
 
   # The tag for the given node.
-  # 
+  #
   # @return [ String ] The tag for the given node.
   def tag
     try_opt(:tag)

data/lib/souffle/provider.rb

@@ -3,12 +3,33 @@ require 'tmpdir'
 
 # A metal provider module (Describes AWS, Softlayer, etc).
 module Souffle::Provider
+  class << self
+    # Returns the list of available provider plugins.
+    #
+    # @return [ Array ] The list of available provider plugins.
+    def plugins
+      constants.map { |k| k.to_s.downcase }
+    end
+
+    # Returns the plugin with the given name.
+    #
+    # @param [ String ] name The name of the plugin to select.
+    #
+    # @return [ Souffle::Provider::Base ] The plugin with the given name.
+    def plugin(name)
+      plug = constants.select { |k| k.to_s.downcase == name.downcase }.first
+      Souffle::Provider.const_get(plug)
+    rescue Souffle::Exceptions::PluginDoesNotExist => e
+      Souffle::Log.error "#{e.message}:\n#{e.backtrace.join("\n")}"
+    end
+  end
+
   # The souffle cloud provider class.
   class Base
     attr_accessor :system
 
     # Initialize a new provider for a given system.
-    # 
+    #
     # @param [ Souffle::System ] system The system to provision.
     def initialize(system=Souffle::System.new)
       @system ||= system
@@ -16,7 +37,7 @@ module Souffle::Provider
     end
 
     # The name of the given provider.
-    # 
+    #
     # @return [ String ] The name of the given provider.
     def name
       self.class.name.split('::').last
@@ -30,7 +51,7 @@ module Souffle::Provider
     #
     # @raise [Souffle::Exceptions::Provider] This definition must be
     # overrridden.
-    # 
+    #
     # @param [ Souffle::System ] system The system to instantiate.
     # @param [ String ] tag The tag to use for the system.
     def create_system(system, tag=nil)
@@ -39,7 +60,7 @@ module Souffle::Provider
     end
 
     # Takes a node definition and begins the provisioning process.
-    # 
+    #
     # @param [ Souffle::Node ] node The node to instantiate.
     # @param [ String ] tag The tag to use for the node.
     def create_node(node, tag=nil)
@@ -48,7 +69,7 @@ module Souffle::Provider
     end
 
     # Creates a raid array for a given provider. Intended to be overridden.
-    # 
+    #
     # @raise [Souffle::Exceptions::Provider] This definition must be
     # overridden.
     def create_raid
@@ -57,9 +78,9 @@ module Souffle::Provider
     end
 
     # Generates the json required for chef-solo to run on a node.
-    # 
+    #
     # @param [ Souffle::Node ] node The node to generate chef-solo json for.
-    # 
+    #
     # @return [ String ] The chef-solo json for the particular node.
     def generate_chef_json(node)
       json_info = Hash.new
@@ -71,7 +92,7 @@ module Souffle::Provider
 
     # Waits for ssh to be accessible for a node for the initial connection and
     # yields an ssh object to manage the commands naturally from there.
-    # 
+    #
     # @param [ String ] address The address of the machine to connect to.
     # @param [ String ] user The user to connect as.
     # @param [ String, NilClass ] pass By default publickey and password auth
@@ -83,7 +104,7 @@ module Souffle::Provider
     # @option opts [ Hash ] :timeout (TIMEOUT) default timeout for all
     # #wait_for and #send_wait calls.
     # @option opts [ Boolean ] :reconnect When disconnected reconnect.
-    # 
+    #
     # @yield [ Eventmachine::Ssh:Session ] The ssh session.
     def wait_for_boot(address, user="root", pass=nil, opts={},
                       timeout=200)
@@ -111,7 +132,7 @@ module Souffle::Provider
     end
 
     # Yields an ssh object to manage the commands naturally from there.
-    # 
+    #
     # @param [ String ] address The address of the machine to connect to.
     # @param [ String ] user The user to connect as.
     # @param [ String, NilClass ] pass By default publickey and password auth
@@ -122,7 +143,7 @@ module Souffle::Provider
     # @option opts [ Hash ] :timeout (TIMEOUT) default timeout for all
     # #wait_for and #send_wait calls.
     # @option opts [ Boolean ] :reconnect When disconnected reconnect.
-    # 
+    #
     # @yield [ EventMachine::Ssh::Session ] The ssh session.
     def ssh_block(address, user="root", pass=nil, opts={})
       opts[:password] = pass unless pass.nil?
@@ -136,18 +157,18 @@ module Souffle::Provider
     end
 
     # The path to the ssh key with the given name.
-    # 
+    #
     # @param [ String ] key_name The name fo the ssh key to lookup.
-    # 
+    #
     # @return [ String ] The path to the ssh key with the given name.
     def ssh_key(key_name)
       "#{ssh_key_path}/#{key_name}"
     end
 
     # Grabs an ssh key for a given aws node.
-    # 
+    #
     # @param [ String ] key_name The name fo the ssh key to lookup.
-    # 
+    #
     # @return [ Boolean ] Whether or not the ssh_key exists
     # for the node.
     def ssh_key_exists?(key_name)
@@ -160,11 +181,11 @@ module Souffle::Provider
     rescue
       error_msg =  "The ssh key directory does not have write permissions: "
       error_msg << ssh_key_path
-      raise PermissionErrorSshKeys, error_msg
+      raise Souffle::Exceptions::PermissionErrorSshKeys, error_msg
     end
 
     # The path to the ssh keys for the provider.
-    # 
+    #
     # @return [ String ] The path to the ssh keys for the provider.
     def ssh_key_path
       File.join(File.dirname(
@@ -172,7 +193,7 @@ module Souffle::Provider
     end
 
     # Rsync's a file to a remote node.
-    # 
+    #
     # @param [ String ] ipaddress The ipaddress of the node to connect to.
     # @param [ String ] file The file to rsync.
     # @param [ String ] path The remote path to rsync.
@@ -189,7 +210,7 @@ module Souffle::Provider
     end
 
     # The list of cookbooks and their full paths.
-    # 
+    #
     # @return [ Array ] The list of cookbooks and their full paths.
     def cookbook_paths
       Array(Souffle::Config[:chef_cookbook_path]).inject([]) do |_paths, path|
@@ -201,7 +222,7 @@ module Souffle::Provider
     end
 
     # The list of roles and their full paths.
-    # 
+    #
     # @return [ Array ] The list of roles and their full paths.
     def role_paths
       Array(Souffle::Config[:chef_role_path]).inject([]) do |_paths, path|
@@ -213,7 +234,7 @@ module Souffle::Provider
     end
 
     # Creates a new cookbook tarball for the deployment.
-    # 
+    #
     # @return [ String ] The path to the created tarball.
     def create_cookbooks_tarball
       tarball_name = "cookbooks-latest.tar.gz"

data/lib/souffle/provider/aws.rb

@@ -210,7 +210,7 @@ class Souffle::Provider::AWS < Souffle::Provider::Base
 
       pre_event do
         @partitions = 0
-        @provider = node.provisioner.provider
+        @provider = node.provider
         node.options[:volumes].each_with_index do |volume, index|
           @provider.partition_device(
             node, @provider.volume_id_to_device(index)) do |count|
@@ -284,10 +284,10 @@ class Souffle::Provider::AWS < Souffle::Provider::Base
   # 
   # @param [ Souffle::Node ] node The node to install mdadm on.
   def setup_mdadm(node)
-    ssh_block(node) do |ssh|
+    n = node; ssh_block(node) do |ssh|
       ssh.exec!("/usr/bin/yum install -y mdadm")
+      n.provisioner.mdadm_installed
     end
-    node.provisioner.mdadm_installed
   end
 
   # Sets up software raid for the given node.
@@ -330,7 +330,7 @@ class Souffle::Provider::AWS < Souffle::Provider::Base
 
       pre_event do
         Souffle::Log.info "#{node.log_prefix} Waiting for node running..."
-        @provider = node.provisioner.provider
+        @provider = node.provider
         @blk = blk
       end
 
@@ -364,7 +364,7 @@ class Souffle::Provider::AWS < Souffle::Provider::Base
 
       pre_event do
         Souffle::Log.info "#{node.log_prefix} Waiting for EBS to be ready..."
-        @provider = node.provisioner.provider
+        @provider = node.provider
         @volume_ids = node.options[:volumes].map { |v| v[:aws_id] }
       end
 
@@ -479,12 +479,11 @@ class Souffle::Provider::AWS < Souffle::Provider::Base
   # @todo Setup the chef/chef-solo tar gzip and ssh connections.
   def provision(node)
     set_hostname(node)
-    if node.try_opt(:chef_provisioner) == :solo
+    if node.try_opt(:chef_provisioner).to_s.downcase == "solo"
       provision_chef_solo(node, generate_chef_json(node))
     else
       provision_chef_client(node)
     end
-    node.provisioner.provisioned
   end
 
   # Waits for ssh to be accessible for a node for the initial connection and
@@ -515,7 +514,7 @@ class Souffle::Provider::AWS < Souffle::Provider::Base
 
       pre_event do
         Souffle::Log.info "#{node.log_prefix} Waiting for ssh..."
-        @provider = node.provisioner.provider
+        @provider = node.provider
         @blk = blk
       end
 
@@ -572,7 +571,7 @@ class Souffle::Provider::AWS < Souffle::Provider::Base
     solo_config =  "node_name \"#{node.fqdn}\"\n"
     solo_config << "cookbook_path \"/tmp/cookbooks\"\n"
     solo_config << 'role_path "/tmp/roles"'
-    ssh_block(node) do |ssh|
+    n = node; ssh_block(node) do |ssh|
       ssh.exec!("sleep 2; tar -zxf /tmp/cookbooks-latest.tar.gz -C /tmp")
       ssh.exec!("echo '#{solo_config}' >/tmp/solo.rb")
       ssh.exec!("echo '#{solo_json}' >/tmp/solo.json")
@@ -580,6 +579,7 @@ class Souffle::Provider::AWS < Souffle::Provider::Base
       rm_files = %w{ /tmp/cookbooks /tmp/cookbooks-latest.tar.gz
         /tmp/roles /tmp/solo.rb /tmp/solo.json /tmp/chef_bootstrap }
       ssh.exec!("rm -rf #{rm_files}")
+      n.provisioner.provisioned
     end
   end
 
@@ -587,8 +587,14 @@ class Souffle::Provider::AWS < Souffle::Provider::Base
   # 
   # @todo Chef client provisioner needs to be completed.
   def provision_chef_client(node)
-    ssh_block(node) do |ssh|
-      ssh.exec!("chef-client")
+    client_cmds =  "chef-client -N #{node.fqdn} "
+    client_cmds << "-j /tmp/client.json "
+    client_cmds << "-S #{node.try_opt(:chef_server)} "
+    n = node; ssh_block(node) do |ssh|
+      write_temp_chef_json(ssh, n)
+      ssh.exec!(client_cmds)
+      cleanup_temp_chef_files(ssh, n)
+      node.provisioner.provisioned
     end
   end
 
@@ -725,4 +731,23 @@ class Souffle::Provider::AWS < Souffle::Provider::Base
     end
   end
 
+  private
+
+  # Writes a temporary chef-client json file.
+  #
+  # @param [ EventMachine::Ssh::Connection ] ssh The em-ssh connection.
+  # @param [ Souffle::Node ] node The given node to work with.
+  def write_temp_chef_json(ssh, node)
+    ssh.exec!("echo '''#{generate_chef_json(node)}''' > /tmp/client.json")
+  end
+
+  # Removes the temporary chef-client files.
+  #
+  # @param [ EventMachine::Ssh::Connection ] ssh The em-ssh connection.
+  # @param [ Souffle::Node ] node The given node to work with.
+  def cleanup_temp_chef_files(ssh, node)
+    ssh.exec!("rm -f /tmp/client.json")
+    ssh.exec!("rm -f /etc/chef/validation.pem")
+  end
+
 end

data/lib/souffle/provisioner/node.rb

@@ -19,6 +19,7 @@ class Souffle::Provisioner::Node
     after_transition :formatting_device => :ready_to_provision,
                           :do => :ready
     after_transition any => :provisioning, :do => :provision
+    after_transition any => :complete, :do => :node_provisioned
 
     event :reclaimed do
       transition any => :creating
@@ -133,6 +134,12 @@ class Souffle::Provisioner::Node
     provider.provision(@node)
   end
 
+  # Notifies the system that the current node has completed provisioning.
+  def node_provisioned
+    Souffle::Log.info "#{@node.log_prefix} Node provisioned."
+    system_provisioner.node_provisioned
+  end
+
   # Kills the node entirely.
   def kill
     Souffle::Log.info "#{@node.log_prefix} Killing node..."
@@ -152,6 +159,11 @@ class Souffle::Provisioner::Node
 
   # Helper function for the node's system provider.
   def provider
-    @node.system.provisioner.provider
+    @node.provider
+  end
+
+  # Helper function for the system provisioner.
+  def system_provisioner
+    @node.system.provisioner
   end
 end

data/lib/souffle/provisioner/system.rb

@@ -13,6 +13,7 @@ class Souffle::Provisioner::System
     after_transition :initializing => :creating, :do => :create
     after_transition :creating => :provisioning, :do => :provision
     after_transition any => :initializing, :do => :create
+    after_transition :provisioning => :complete, :do => :system_provisioned
 
     event :initialized do
       transition :initializing => :creating
@@ -51,13 +52,14 @@ class Souffle::Provisioner::System
   # @param [ Souffle::Provider::Base ] provider The provider to use.
   # @param [ Fixnum ] max_failures the maximum number of failures.
   # @param [ Fixnum ] timeout The maximum time to wait for node creation.
-  def initialize(system, provider, max_failures=3, timeout=500)
+  def initialize(system, provider, max_failures=3, timeout=600)
     @failures = 0
     @system = system
     @provider = provider
     @time_used = 0
     @timeout = timeout
     @max_failures = max_failures
+    @nodes_completed = 0
     super() # NOTE: This is here to initialize state_machine.
   end
 
@@ -113,12 +115,23 @@ class Souffle::Provisioner::System
     end
   end
 
+  # System has completed provisioning.
+  def system_provisioned
+    Souffle::Log.info "[#{system_tag}] System provisioned."
+  end
+
+  # Updates the number of nodes provisioned for the system provisioner.
+  def node_provisioned
+    @nodes_completed += 1
+    provisioned if @nodes_completed == @system.nodes.size
+  end
+
   # Kills the system.
   def kill_system
     # @provider.kill(@system.nodes)
   end
 
-  # Handles the error state and recreates the system
+  # Handles the error state and recreates the system.
   def error_handler
     @failures += 1
     if @failures >= @max_failures
@@ -169,24 +182,9 @@ class Souffle::Provisioner::System
 
   # Wait until all of the nodes are provisioned and then continue.
   def wait_until_complete
-    total_nodes = @system.nodes.size
-    all_complete = false
-    timer = EM::PeriodicTimer.new(2) do
-      nodes_ready = @system.nodes.select do |n|
-        n.provisioner.state == "complete"
-      end.size
-
-      if nodes_ready == total_nodes
-        all_complete = true
-        timer.cancel
-        created
-      end
-    end
-
     EM::Timer.new(@timeout) do
-      unless all_complete
+      unless @nodes_completed == @system.nodes.size
         Souffle::Log.error "[#{system_tag}] System provision timeout reached."
-        timer.cancel
         error_occurred
       end
     end

data/lib/souffle/server.rb

@@ -13,15 +13,9 @@ class Souffle::Server
   # Runs the server.
   def run
     if Souffle::Config[:server]
-      EM.synchrony do
-        @app = Rack::Builder.new do
-          use Rack::Lint
-          use Rack::ShowExceptions
-          run Rack::Cascade.new([Souffle::Http])
-        end.to_app
-
-        Rack::Handler.get(:thin).run(@app, rack_options)
-      end
+      run_server
+    else
+      run_instance
     end
   end
 
@@ -44,5 +38,26 @@ class Souffle::Server
     end
     opts
   end
+
+  # Starts a long-running webserver that continuously handles requests.
+  def run_server
+    EM.synchrony do
+      @app = Rack::Builder.new do
+        use Rack::Lint
+        use Rack::ShowExceptions
+        run Rack::Cascade.new([Souffle::Http])
+      end.to_app
+
+      Rack::Handler.get(:thin).run(@app, rack_options)
+    end
+  end
+
+  # Starts a single instance of the provisioner.
+  def run_instance
+    Souffle::Log.info "Single instance runs are not currently implemented..."
+    # system = Souffle::System.from_hash(data)
+    # provider = Souffle::Provider.plugin(system.try_opt(:provider)).new
+    # system_tag = provider.create_system(system)
+  end
   
 end

data/lib/souffle/system.rb

@@ -103,6 +103,13 @@ class Souffle::System
     @nodes.select { |n| n.dependencies.any? }
   end
 
+  # Returns the current system provider.
+  #
+  # @return [ Souffle::Provider::Base ] The current system provider.
+  def provider
+    @provisioner.provider
+  end
+
   # Tries to fetch an option parameter otherwise it grabs it from config.
   # 
   # @param [ Symbol ] opt The option to try and fetch.

data/lib/souffle/version.rb

@@ -1,6 +1,5 @@
 # An orchestrator for setting up isolated chef-managed systems.
 module Souffle
   # The current souffle version.
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end
-

data/spec/provider_spec.rb

@@ -53,4 +53,14 @@ describe "Souffle::Provider::Base" do
 
     JSON.parse(@provider.generate_chef_json(node)).should eql(runlist)
   end
+
+  it "should be able to generate a list of provider plugins" do
+    ["base", "vagrant", "aws"].each do |plugin|
+      Souffle::Provider.plugins.include?(plugin).should eql(true)
+    end
+  end
+
+  it "should be able to select a particular plugin" do
+    Souffle::Provider.plugin("aws").should eql(Souffle::Provider::AWS)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: souffle
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-19 00:00:00.000000000 Z
+date: 2012-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.12.10
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.12.10
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: amqp
   requirement: !ruby/object:Gem::Requirement