souffle 0.0.1 → 0.0.2

data/Gemfile

@@ -7,21 +7,27 @@ gem "yajl-ruby", "~> 1.1.0"
 gem "eventmachine", "~> 0.12.10"
 gem "amqp", "~> 0.9.7"
 gem "state_machine", "~> 1.1.2"
+gem "em-ssh", "~> 0.4.0"
+gem "em-synchrony", "~> 0.2.0"
 
 gem "right_aws", "~> 3.0.4"
+gem "tilt", "~> 1.3.3"
+gem "redis", "~> 3.0.1"
 
-gem "mixlib-cli"
+gem "mixlib-cli", ">= 1.2.2"
 gem "mixlib-config", ">= 1.1.0"
 gem "mixlib-log", ">= 1.3.0"
 
+gem "thin", "~> 1.4.1"
+gem "rack", "~> 1.4.1"
+gem "sinatra", "~> 1.3.2"
+
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
   gem "rspec", "~> 2.10.0"
-  gem "fakefs", "~> 0.4.0"
   gem "yard", "~> 0.8"
   gem "redcarpet", "~> 2.1.1"
-  gem "rdoc", "~> 3.12"
   gem "cucumber", ">= 0"
   gem "bundler", "~> 1.1.0"
   gem "jeweler", "~> 1.8.3"

data/README.md

@@ -2,6 +2,12 @@
 
 An orchestrator for setting up isolated chef-managed systems.
 
+## A note on tests
+
+In order to avoid painfully silly charges and costs, all of the AWS tests
+that require you to pay (spinning up machines, etc), will only run if you
+have the environment variable `AWS_LIVE` set to `true`.
+
 ## Contributing to souffle
 
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.

data/lib/souffle.rb

@@ -1,17 +1,17 @@
 $:.unshift File.dirname(__FILE__)
 require 'yajl'
+require 'yajl/json_gem'
 require 'eventmachine'
-require 'state_machine'
-require 'right_aws'
-
-# An orchestrator for setting up isolated chef-managed systems.
-module Souffle
-  VERSION = "0.0.1"
-end
+require 'em-ssh'
+require 'em-synchrony'
 
+require 'souffle/version'
+require 'souffle/ssh_monkey'
 require 'souffle/log'
 require 'souffle/exceptions'
 require 'souffle/config'
-require 'souffle/providers'
+require 'souffle/daemon'
 require 'souffle/node'
 require 'souffle/system'
+require 'souffle/provider'
+require 'souffle/provisioner'

data/lib/souffle/application.rb

@@ -4,11 +4,15 @@ require 'mixlib/cli'
 class Souffle::Application
   include Mixlib::CLI
 
+  # The commands that were left unparsed from parse_options.
+  attr_accessor :commands
+
   # Added a Wakeup exception.
   class Wakeup < Exception; end
 
   # Initialize the application, setting up default handlers.
   def initialize
+    @commands = []
     super
 
     trap("TERM") do
@@ -42,10 +46,11 @@ class Souffle::Application
     begin
       ::File.open(config[:config_file]) { |f| apply_config(f.path) }
     rescue Errno::ENOENT => error
-      noconfig = "Did not find config file: #{config[:config_file]}"
-      Souffle::Log.warn("*****************************************")
-      Souffle::Log.warn("#{noconfig}, using command line options.")
-      Souffle::Log.warn("*****************************************")
+      msg =  "Did not find the config file: #{config[:config_file]}"
+      msg << ", Using command line options."
+      Souffle::Log.warn "*****************************************"
+      Souffle::Log.warn msg
+      Souffle::Log.warn "*****************************************"
     end
   end
 
@@ -56,9 +61,9 @@ class Souffle::Application
     Souffle::Log.init(Souffle::Config[:log_location])
     if ( Souffle::Config[:log_location] != STDOUT ) && STDOUT.tty? &&
       ( !Souffle::Config[:daemonize] )
-      stdout_loger = Logger.new(STDOUT)
+      stdout_logger = Logger.new(STDOUT)
       STDOUT.sync = true
-      stdout_logger = Souffle::Log.logger.formatter
+      stdout_logger.formatter = Souffle::Log.logger.formatter
       Souffle::Log.loggers << stdout_logger
     end
     Souffle::Log.level = Souffle::Config[:log_level]
@@ -113,8 +118,8 @@ class Souffle::Application
     # Log a fatal error message to both STDERR and the Logger,
     # exit the application with a fatal message.
     #
-    # @param [ msg ] String The message to log.
-    # @param [ err ] Integer The exit level.
+    # @param [ String ] msg The message to log.
+    # @param [ Fixnum ] err The exit level.
     def fatal!(msg, err = -1)
       Souffle::Log.fatal(msg)
       Process.exit err
@@ -123,8 +128,8 @@ class Souffle::Application
     # Log a fatal error message to both STDERR and the Logger,
     # exit the application with a debug message.
     #
-    # @param [ msg ] String The message to log.
-    # @param [ err ] Integer The exit level.
+    # @param [ String ] msg The message to log.
+    # @param [ Fixnum ] err The exit level.
     def exit!(msg, err = -1)
       Souffle::Log.debug(msg)
       Process.exit err

data/lib/souffle/application/souffle-server.rb

@@ -7,7 +7,7 @@ class Souffle::Application::Server < Souffle::Application
   option :config_file,
     :short => "-c CONFIG",
     :long  => "--config CONFIG",
-    :default => "/etc/souffle/worker.rb",
+    :default => "/etc/souffle/souffle.rb",
     :description => "The configuration file to use"
 
   option :log_level,
@@ -17,10 +17,78 @@ class Souffle::Application::Server < Souffle::Application
     :proc => lambda { |l| l.to_sym }
 
   option :log_location,
-    :short        => "-L LOGLOCATION",
-    :long         => "--logfile LOGLOCATION",
-    :description  => "Set the log file location, defaults to STDOUT",
-    :proc         => nil
+    :short => "-L LOG_LOCATION",
+    :long =>  "--logfile LOG_LOCATION",
+    :description => "Set the log file location, defaults to STDOUT",
+    :proc => nil
+
+  option :user,
+    :short => "-u USER",
+    :long => "--user USER",
+    :description => "User to set privilege to",
+    :proc => nil
+
+  option :group,
+    :short => "-g GROUP",
+    :long => "--group GROUP",
+    :description => "Group to set privilege to",
+    :proc => nil
+
+  option :daemonize,
+    :short => "-d",
+    :long  => "--daemonize",
+    :default => false,
+    :description => "Run the application as a daemon (forces `-s`)",
+    :proc => lambda { |p| true }
+
+  option :environment,
+    :short => "-E",
+    :long  => "--environment",
+    :description => "The environment profile to use",
+    :proc => nil
+
+  option :rack_host,
+    :short => "-H HOSTNAME",
+    :long  => "--hostname HOSTNAME",
+    :description => "Hostname to listen on (default: 0.0.0.0)",
+    :proc => nil
+
+  option :rack_port,
+    :short => "-P PORT",
+    :long  => "--port PORT",
+    :description => "Port to listen on (default: 8080)",
+    :proc => lambda { |p| p.to_i }
+
+  option :vagrant_dir,
+    :short => "-V VAGRANT_DIR",
+    :long  => "--vagrant_dir VAGRANT_DIR",
+    :description => "The path to the base vagrant vm directory",
+    :proc => nil
+
+  option :pid_file,
+    :short => "-f PID_FILE",
+    :long  => "--pid PID_FILE",
+    :description => "Set the PID file location, defaults to /tmp/souffle.pid",
+    :proc => nil
+
+  option :provider,
+    :short => "-p PROVIDER",
+    :long  => "--provider PROVIDER",
+    :description => "The provider to use (overrides config)",
+    :proc => nil
+
+  option :json,
+    :short => "-j JSON",
+    :long  => "--json JSON",
+    :description => "The json for a single provision (negates `-s`)",
+    :proc => nil
+
+  option :server,
+    :short => "-s",
+    :long  => "--server",
+    :default => false,
+    :description => "Start the application as a server",
+    :proc => nil
 
   option :help,
     :short => "-h",
@@ -39,8 +107,25 @@ class Souffle::Application::Server < Souffle::Application
     :proc => lambda { |v| puts "Souffle: #{::Souffle::VERSION}"},
     :exit => 0
 
+  # Grabs all of the cli parameters and generates the mixlib config object.
   def initialize
     super
+    Souffle::Config.merge!(config)
+  end
+
+  # Configures the souffle server based on the cli parameters.
+  def setup_application
+    Souffle::Daemon.change_privilege
+    Souffle::Config[:server] = true if Souffle::Config[:daemonize]
+    @app = Souffle::Server.new
+  end
 
+  # Runs the Souffle Server.
+  def run_application
+    if Souffle::Config[:daemonize]
+      Souffle::Config[:server] = true
+      Souffle::Daemon.daemonize("souffle")
+    end
+    @app.run
   end
 end

data/lib/souffle/config.rb

@@ -2,74 +2,103 @@ require 'souffle/log'
 require 'mixlib/config'
 require 'yajl'
 
-module Souffle
-  # The configuration object for the souffle server.
-  class Config
+# The configuration object for the souffle server.
+class Souffle::Config
+  extend Mixlib::Config
 
-    extend Mixlib::Config
+  # Return the configuration itself upon inspection.
+  def self.inspect
+    configuration.inspect
+  end
 
-    # Return the configuration itself upon inspection.
-    def self.inspect
-      configuration.inspect
-    end
+  # Loads a given file and passes it to the appropriate parser.
+  #
+  # @raise [ IOError ] Any IO Exceptions that occur.
+  #
+  # @param [ String ] filename The filename to read.
+  def self.from_file(filename, parser="ruby")
+    send("from_file_#{parser}".to_sym, filename)
+  end
 
-    # Loads a given file and passes it to the appropriate parser.
-    #
-    # @raise [ IOError ] Any IO Exceptions that occur.
-    #
-    # @param [ String ] filename The filename to read.
-    def self.from_file(filename, parser="ruby")
-      send("from_file_#{parser}".to_sym, filename)
-    end
+  # Loads a given ruby file and runs instance_eval against it
+  # in the context of the current object.
+  #
+  # @raise [ IOError ] Any IO Exceptions that occur.
+  #
+  # @param [ String ] filename The file to read.
+  def self.from_file_ruby(filename)
+    self.instance_eval(IO.read(filename), filename, 1)
+  end
 
-    # Loads a given ruby file and runs instance_eval against it
-    # in the context of the current object.
-    #
-    # @raise [ IOError ] Any IO Exceptions that occur.
-    #
-    # @param [ String ] filename The file to read.
-    def self.from_file_ruby(filename)
-      self.instance_eval(IO.read(filename), filename, 1)
-    end
+  # Loads a given json file and merges the current context
+  # configuration with the updated hash.
+  #
+  # @raise [ IOError ] Any IO Exceptions that occur.
+  # @raise [ Yajl::ParseError ] Raises Yajl Parsing error on improper json.
+  #
+  # @param [ String ] filename The file to read.
+  def self.from_file_json(filename)
+    self.from_stream_json(IO.read(filename))
+  end
 
-    # Loads a given json file and merges the current context
-    # configuration with the updated hash.
-    #
-    # @raise [ IOError ] Any IO Exceptions that occur.
-    # @raise [ Yajl::ParseError ] Raises Yajl Parsing error on improper json.
-    #
-    # @param [ String ] filename The file to read.
-    def self.from_file_json(filename)
-      self.from_input_json(IO.read(filename))
-    end
+  # Loads a given json input and merges the current context
+  # configuration with the updated hash.
+  #
+  # @raise [ IOError ] Any IO Exceptions that occur.
+  # @raise [ Yajl::ParseError ] Raises Yajl Parsing error on improper json.
+  #
+  # @param [ String ] input The json configuration input.
+  def self.from_stream_json(input)
+    parser = Yajl::Parser.new(:symbolize_keys => true)
+    configuration.merge!(parser.parse(input))
+  end
 
-    # Loads a given json input and merges the current context
-    # configuration with the updated hash.
-    #
-    # @raise [ IOError ] Any IO Exceptions that occur.
-    # @raise [ Yajl::ParseError ] Raises Yajl Parsing error on improper json.
-    #
-    # @param [ String ] input The json configuration input.
-    def self.from_stream_json(input)
-      parser = Yajl::Parser.new
-      configuration.merge!(parser.parse(input))
+  # When you are using ActiveSupport, they monkey-patch 'daemonize' into
+  # Kernel. So while this is basically identical to what method_missing
+  # would do, we pull it up here and get a real method written so that
+  # things get dispatched properly.
+  config_attr_writer :daemonize do |v|
+    configure do |c|
+      c[:daemonize] = v
     end
+  end
 
-    # When you are using ActiveSupport, they monkey-patch 'daemonize' into
-    # Kernel. So while this is basically identical to what method_missing
-    # would do, we pull it up here and get a real method written so that
-    # things get dispatched properly.
-    config_attr_writer :daemonize do |v|
-      configure do |c|
-        c[:daemonize] = v
-      end
-    end
+  # Configuration Settings
+  config_file "/etc/souffle/souffle.rb"
 
-    log_level :info
-    log_location STDOUT
+  # Enable debug
+  debug false
 
-    aws_access_key = ""
-    aws_access_secret = ""
+  # Logging Settings
+  log_level :info
+  log_location STDOUT
 
-  end
+  # Chef Settings
+  chef_cookbook_path []
+  chef_provisioner :solo
+  chef_domain "souffle"
+
+  # Provider Settings
+  provider "Vagrant"
+
+  # Daemonization Settings
+  user nil
+  group nil
+  umask 0022
+
+  pid_file nil
+
+  # AWS Settings
+  aws_access_key ""
+  aws_access_secret ""
+  aws_region "us-west-1"
+  delete_on_termination true
+
+  # Rack Settings
+  rack_host "0.0.0.0"
+  rack_port 8080
+  rack_environment "development"
+
+  # Vagrant Settings
+  vagrant_dir "#{ENV['HOME']}/vagrant/vms"
 end

data/lib/souffle/daemon.rb

@@ -0,0 +1,156 @@
+require 'etc'
+
+# Daemon helper routines.
+class Souffle::Daemon
+  class << self
+    attr_accessor :name
+
+    # Daemonize the current process, managing pidfiles and process uid/gid.
+    #
+    # @param [ String ] name The name to be used for the pid file
+    def daemonize(name)
+      @name = name
+      pid = pid_from_file
+      unless running?
+        remove_pid_file()
+        Souffle::Log.info("Daemonizing...")
+        begin
+          exit if fork; Process.setsid; exit if fork
+          msg =  "Forked, in #{Process.pid}. "
+          msg << "Privileges: #{Process.euid} #{Process.egid}"
+          Souffle::Log.info(msg)
+          File.umask Souffle::Config[:umask]
+          $stdin.reopen("/dev/null")
+          $stdout.reopen("/dev/null", "a")
+          $stderr.reopen($stdout)
+          save_pid_file;
+          at_exit { remove_pid_file }
+        rescue NotImplementedError => e
+          Souffle::Application.fatal!("There is no fork: #{e.message}")
+        end
+      else
+        Souffle::Application.fatal!("Souffle is already running pid #{pid}")
+      end
+    end
+
+    # Checks if Souffle is running based on the pid_file.
+    # 
+    # @return [ Boolean ] Whether or not Souffle is running.
+    def running?
+      if pid_from_file.nil?
+        false
+      else
+        Process.kill(0, pid_from_file)
+        true
+      end
+    rescue Errno::ESRCH, Errno::ENOENT
+      false
+    rescue Errno::EACCES => e
+      msg =  "You don't have access to the PID "
+      msg << "file at #{pid_file}: #{e.message}"
+      Souffle::Application.fatal!(msg)
+    end
+    
+    # Gets the pid file for @name.
+    # 
+    # @return [ String ] Location of the pid file for @name.
+    def pid_file
+       Souffle::Config[:pid_file] or "/tmp/#{@name}.pid"
+    end
+    
+    # Sucks the pid out of pid_file.
+    # 
+    # @return [ Fixnum,NilClass ] The PID from pid_file or nil if it doesn't
+    # exist.
+    def pid_from_file
+      File.read(pid_file).chomp.to_i
+    rescue Errno::ENOENT, Errno::EACCES
+      nil
+    end
+  
+    # Store the PID on the filesystem.
+    # 
+    # @note
+    #   This uses the Souffle::Config[:pid_file] option or "/tmp/name.pid"
+    #   by default.
+    def save_pid_file
+      file = pid_file
+      begin
+        FileUtils.mkdir_p(File.dirname(file))
+      rescue Errno::EACCES => e
+        msg =  "Failed store pid in #{File.dirname(file)}, "
+        msg << "permission denied: #{e.message}"
+        Souffle::Application.fatal!(msg)
+      end
+    
+      begin
+        File.open(file, "w") { |f| f.write(Process.pid.to_s) }
+      rescue Errno::EACCES => e
+        msg =  "Couldn't write to pidfile #{file}, "
+        msg << "permission denied: #{e.message}"
+        Souffle::Application.fatal!(msg)
+      end
+    end
+  
+    # Delete the PID from the filesystem
+    def remove_pid_file
+      FileUtils.rm(pid_file) if File.exists?(pid_file)
+    end
+         
+    # Change process user/group to those specified in Souffle::Config
+    def change_privilege
+      Dir.chdir("/")
+
+      msg =  "About to change privilege to "
+      if Souffle::Config[:user] and Souffle::Config[:group]
+        msg << "#{Souffle::Config[:user]}:#{Souffle::Config[:group]}"
+        Souffle::Log.info(msg)
+        _change_privilege(Souffle::Config[:user], Souffle::Config[:group])
+      elsif Souffle::Config[:user]
+        msg << "#{Souffle::Config[:user]}"
+        Souffle::Log.info(msg)
+        _change_privilege(Souffle::Config[:user])
+      end
+    end
+  
+    # Change privileges of the process to be the specified user and group
+    # 
+    # @param [ String ] user The user to change the process to.
+    # @param [ String ] group The group to change the process to.
+    # 
+    # @note
+    #   The group parameter defaults to user unless specified.
+    def _change_privilege(user, group=user)
+      uid, gid = Process.euid, Process.egid
+
+      begin
+        target_uid = Etc.getpwnam(user).uid
+      rescue ArgumentError => e
+        msg =  "Failed to get UID for user #{user}, does it exist? "
+        msg << e.message
+        Souffle::Application.fatal!(msg)
+        return false
+      end
+ 
+      begin
+        target_gid = Etc.getgrnam(group).gid
+      rescue ArgumentError => e
+        msg =  "Failed to get GID for group #{group}, does it exist? "
+        msg << e.message
+        Souffle::Application.fatal!(msg)
+        return false
+      end
+    
+      if (uid != target_uid) or (gid != target_gid)
+        Process.initgroups(user, target_gid)
+        Process::GID.change_privilege(target_gid)
+        Process::UID.change_privilege(target_uid)
+      end
+      true
+    rescue Errno::EPERM => e
+      msg =  "Permission denied when trying to change #{uid}:#{gid} "
+      msg << "to #{target_uid}:#{target_gid}. #{e.message}"
+      Souffle::Application.fatal!(msg)
+    end
+  end
+end