circus-deployment 0.0.1 → 0.2

data/lib/bundler/circus_util.rb

@@ -1,8 +1,7 @@
-require 'bundler'
-
 module Bundler
   class CircusUtil
     def self.fix_external_paths(dir)
+      require 'bundler'
       ENV['BUNDLE_GEMFILE'] = File.join(dir, 'Gemfile')
       
       # Correct any path based components in the Gemfile

data/lib/bundler/gem_cacher.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'bundler/cli'
+require 'bundler'
+
+Bundler.ui = Bundler::UI::Shell.new(Thor::Shell::Basic.new)
+
+# TODO: Make RUBY_FRAMEWORK_VERSION resolvable instead of using 1.8
+gem_install_root = "vendor/bundle/ruby/1.8/gems"
+spec_install_root = "vendor/bundle/ruby/1.8/specifications"
+FileUtils.mkdir_p(gem_install_root)
+FileUtils.mkdir_p(spec_install_root)
+
+Bundler.definition.specs.each do |spec|
+  install_path = "#{gem_install_root}/#{spec.name}-#{spec.version}"
+  
+  unless install_path == spec.full_gem_path
+    FileUtils.mkdir_p(File.dirname(install_path))
+    FileUtils.rm_rf install_path
+    FileUtils.cp_r spec.full_gem_path, install_path
+  end
+  
+  File.open(File.join(spec_install_root, "#{spec.name}-#{spec.version}.spec"), 'w') do |sf|
+    sf.write(spec.to_ruby)
+  end
+  in_dir_spec = "#{install_path}/#{spec.name}.gemspec"
+  unless File.exists? in_dir_spec
+    File.open(in_dir_spec, 'w') do |sf|
+      sf.write(spec.to_ruby)
+    end
+  end
+end
+
+File.open('Gemfile', 'w') do |f|
+  Bundler.definition.specs.each do |s|
+    f << "gem #{s.name.inspect}, #{s.version.to_s.inspect}, :path => '#{gem_install_root}/#{s.name}-#{s.version}'\n"
+  end
+end
+# FileUtils.mkdir_p('.bundle')
+# File.open('.bundle/config', 'w') do |f|
+#   f.puts '---'
+#   f.puts 'BUNDLE_FROZEN: "1"'
+#   f.puts 'BUNDLE_DISABLE_SHARED_GEMS: "1"'
+#   f.puts 'BUNDLE_PATH: vendor/bundle'
+# end

data/lib/circus.rb

@@ -6,4 +6,5 @@ require File.expand_path('../circus/repos', __FILE__)
 require File.expand_path('../circus/booth_client', __FILE__)
 require File.expand_path('../circus/actstore_client', __FILE__)
 require File.expand_path('../circus/local_config', __FILE__)
-require File.expand_path('../circus/connection_builder', __FILE__)
+require File.expand_path('../circus/connection_builder', __FILE__)
+require File.expand_path('../circus/processes/daemontools', __FILE__)

data/lib/circus/act.rb

@@ -3,7 +3,7 @@ require 'yaml'
 
 module Circus
   class Act
-    attr_reader :name, :dir, :props, :profile
+    attr_reader :name, :dir, :props
     
     def initialize(name, dir, props = {})
       @name = name
@@ -14,9 +14,18 @@ module Circus
       if File.exists? act_file
         act_cfg = YAML.load(File.read(act_file))
         @props.merge! act_cfg
+        
+        # Allow act name to be overriden
+        @name = @props['name'] if @props['name']
       end
     end
     
+    def profile
+      detect! unless @profile
+      
+      @profile
+    end
+    
     def should_package?
       return false if @props['no-package']
       true
@@ -47,14 +56,14 @@ module Circus
       return false unless profile.package_for_deploy(logger, overlay_dir)
       
       begin
-        # Squash the output file
-        include_dirs = ["#{overlay_dir}/*"]
-        include_dirs << "#{@dir}/*" if profile.package_base_dir?
-        include_dirs.concat(profile.extra_dirs)
+        # Package up the output file
+        include_dirs = ["-C #{overlay_dir} ."]
+        include_dirs << "-C #{@dir} ." if profile.package_base_dir?
+        include_dirs.concat(profile.extra_dirs.map { |d| "-C #{File.dirname(d)} #{File.basename(d)}"})
         output_name = File.join(output_root_dir, "#{name}.act")
       
         ExternalUtil.run_external(logger, 'Output packaging', 
-          "mksquashfs #{include_dirs.join(' ')} #{output_name} -noappend -noI -noD -noF 2>&1")
+          "tar -czf #{output_name} --exclude .circus #{include_dirs.join(' ')} 2>&1")
       ensure
         profile.cleanup_after_deploy(logger, overlay_dir)
       end
@@ -70,5 +79,19 @@ module Circus
     def act_file
       File.join(@dir, 'act.yaml')
     end
+    
+    def pause(logger, run_root)
+      working_dir = File.join(run_root, name)
+      process_mgmt = Circus::Processes::Daemontools.new
+      
+      process_mgmt.pause_service(name, working_dir, logger)
+    end
+    
+    def resume(logger, run_root)
+      working_dir = File.join(run_root, name)
+      process_mgmt = Circus::Processes::Daemontools.new
+      
+      process_mgmt.resume_service(name, working_dir, logger)
+    end
   end
 end

data/lib/circus/application.rb

@@ -29,20 +29,61 @@ module Circus
     # Instructs the application to startup in place and activate all of its associated
     # acts.
     def go!(logger)
-      load! unless @acts
-      
-      FileUtils.rm_rf(private_run_root)
-      acts.each do |act|
-        act.package_for_dev(logger, private_run_root)
+      # Ensure that we have svscan available. If we don't, die nice and early
+      `which svscan`
+      if $? != 0
+        logger.error 'The svscan utility (usually provided by daemontools) is not available. ' +
+          ' This utility must be installed before the circus go command can function.'
+        # TODO: Detect OS and suggest installation mechanism?
+          
+        return
       end
-      acts.each do |act|
+      
+      load! unless @acts
+
+      run_acts = acts.select { |a| a.profile.supported_for_development? }
+      run_acts.each do |act|
         logger.info "Starting act #{act.name} at #{act.dir} using profile #{act.profile.name}"
       end
       logger.info "---------------------"
+      FileUtils.rm_rf(private_run_root)
+      run_acts.each do |act|
+        return unless act.package_for_dev(logger, private_run_root)
+      end
       
+      # If we've loaded bundler ourselves, then we need to remove its environment variables; otherwise,
+      # it will screw up child applications!
+      ENV['BUNDLE_GEMFILE'] = nil
+      ENV['BUNDLE_BIN_PATH'] = nil
       system("svscan #{private_run_root}")
     end
     
+    # Instructs the application to stop the given act that is running under development via go.
+    def pause(logger, act_name)
+      load! unless @acts
+      
+      target_act = acts.find { |a| a.name == act_name }
+      unless target_act
+        logger.error "Act #{act_name} could not be found"
+        return
+      end
+      
+      target_act.pause(logger, private_run_root)
+    end
+    
+    # Instructs the application to resume the given act that is running under development via go.
+    def resume(logger, act_name)
+      load! unless @acts
+      
+      target_act = acts.find { |a| a.name == act_name }
+      unless target_act
+        logger.error "Act #{act_name} could not be found"
+        return
+      end
+      
+      target_act.resume(logger, private_run_root)
+    end
+    
     # Instructs the application to assemble it's components for deployment and generate
     # act output files.
     def assemble!(output_dir, logger, dev = false, only_acts = nil)
@@ -58,6 +99,8 @@ module Circus
       
       assembly_acts = acts.select {|a| only_acts.nil? or only_acts.include? a.name }
       
+      FileUtils.rm_rf(output_dir)
+      FileUtils.rm_rf(private_overlay_root)
       FileUtils.mkdir_p(output_dir)
       assembly_acts.each do |act|
         act.detect!

data/lib/circus/booth_tool.rb

@@ -25,6 +25,11 @@ module Circus
           repo_helper.repo_url
         end
       
+      unless repo_url
+        @logger.error("Could not detect repository source url")
+        return false
+      end
+
       app_name = options[:app_name] || File.basename(File.expand_path('.'))
 
       @logger.info("Creating booth connection #{name} on #{booth} for #{repo_url} as #{app_name}")
@@ -42,10 +47,15 @@ module Circus
     
     def admit(name, apps, options)
       booth = get_booth_or_default(name)
-      return unless booth
+      return false unless booth
 
       repo_helper = Repos.find_repo_by_id(booth[:repo_type]).new(File.expand_path('.'))
       current_rev = repo_helper.current_revision
+
+      unless current_rev
+        @logger.error("Could not detect current repository version")
+        return false
+      end
       
       @logger.info("Starting admission into #{name} of version #{current_rev}")
       connection = ConnectionBuilder.new(booth).build(booth[:booth])
@@ -61,12 +71,15 @@ module Circus
       end
       admitted = Circus::Agents::Encoding.decode(client.admit(booth[:booth], booth[:booth_id], current_rev, apply_patch_fn, apps).result)
       
+      return false if booth[:target] == 'none'
       clown_connection = ConnectionBuilder.new(options).build(booth[:target])
       clown_client = ClownClient.new(clown_connection, @logger)
       admitted.each do |name, url|
         @logger.info("Executing deployment of #{name} from #{url} to #{booth[:target]}")
         clown_client.deploy(booth[:target], name, url).result
       end
+      
+      true
     end
       
     private

data/lib/circus/cli.rb

@@ -27,6 +27,18 @@ module Circus
       @app.go!(LOGGER)
     end
     
+    desc "pause ACT_NAME", "Temporarily halts an act that is running in development (via go)"
+    def pause(act_name)
+      load!
+      @app.pause(LOGGER, act_name)
+    end
+    
+    desc "resume ACT_NAME", "Resumes a paused act that is running in development (via go)"
+    def resume(act_name)
+      load!
+      @app.resume(LOGGER, act_name)
+    end
+    
     desc "assemble", "Assemble the application's acts for deployment"
     method_option :output, :default => '.circus/acts/', :desc => 'Destination directory for generated acts'
     method_option :actstore, :desc => 'URL of the act store to upload the built acts to'
@@ -82,7 +94,7 @@ module Circus
     method_option :uncommitted, :desc => 'Indicates that the current uncommitted changes should be sent to the booth and included in the app'
     def admit(name = nil, *apps)
       tool = BoothTool.new(LOGGER, LocalConfig.new)
-      tool.admit(name, apps, options)
+      exit 1 unless tool.admit(name, apps, options)
     end
 
     desc "exec TARGET [ACT] [CMD]", "Executes the given command in the deployed context of the given act"
@@ -95,7 +107,14 @@ module Circus
       
       connection = ConnectionBuilder.new(options).build(target)
       client = ClownClient.new(connection, LOGGER)
-      client.exec(target, act_name, cmd).result
+      exit 1 unless client.exec(target, act_name, cmd).result
+    end
+    
+    desc "reset TARGET ACT", "Restarts the given act on the target host"
+    def reset(target, act_name)
+      connection = ConnectionBuilder.new(options).build(target)
+      client = ClownClient.new(connection, LOGGER)
+      exit 1 unless client.reset(target, act_name).result
     end
     
     desc "deploy TARGET NAME ACT", "Deploy the named object using the given act onto the given target server"

data/lib/circus/clown_client.rb

@@ -23,5 +23,9 @@ module Circus
     def exec(node, name, cmd)
       @connection.call(node, 'Clown', 'Clown', 'exec', {'name' => name, 'command' => cmd}, @logger)
     end
+    
+    def reset(node, name)
+      @connection.call(node, 'Clown', 'Clown', 'reset', {'name' => name}, @logger)
+    end
   end
 end

data/lib/circus/external_util.rb

@@ -10,5 +10,15 @@ module Circus
         true
       end
     end
+    
+    def self.run_and_show_external(logger, desc, cmd)
+      IO.popen("#{cmd} 2>&1", 'r') do |pipe|
+        while (line = pipe.gets)
+          logger.info(line)
+        end
+      end
+      
+      true
+    end
   end
 end

data/lib/circus/processes/daemontools.rb

@@ -0,0 +1,65 @@
+module Circus
+  module Processes
+    # Support for working with Daemontools
+    class Daemontools
+      def pause_service(name, working_dir, logger)
+        res = `svc -d #{working_dir}`
+        if $? != 0
+          logger.error("Failed to initiate service shutdown for #{name}")
+          return
+        end
+        
+        unless await_service_pause(name, working_dir, logger)
+          logger.error("Service did not shutdown")
+        end
+      end
+      
+      def resume_service(name, working_dir, logger)
+        res = `svc -u #{working_dir}`
+        if $? != 0
+          logger.error("Failed to initiate service startup for #{name}")
+          return
+        end
+        
+        unless await_service_startup(name, working_dir, logger)
+          logger.error("Service did not resume")
+        end
+      end
+      
+      def await_service_startup(name, working_dir, logger)
+        logger.info("Waiting for startup of #{name}")
+        
+        (1..100).each do |i|
+          return true if is_service_running?(working_dir)
+          sleep 1
+        end
+        
+        return false
+      end
+      
+      def await_service_pause(name, working_dir, logger)
+        logger.info("Waiting for shutdown of #{name}")
+
+        (1..5).each do |i|
+          return true unless is_service_running?(working_dir)
+          sleep 1
+        end
+
+        (1..100).each do |i|
+          # Attempt a more brutal shutdown
+          `svc -k #{working_dir}`
+          
+          return true unless is_service_running?(working_dir)
+          sleep 1
+        end
+        
+        return false
+      end
+      
+      def is_service_running?(working_dir)
+        res = `svstat #{working_dir}`
+        $? == 0 && res.include?('up (pid')
+      end
+    end
+  end
+end

data/lib/circus/profiles.rb

@@ -8,3 +8,5 @@ require File.expand_path('../profiles/make_base', __FILE__)
 require File.expand_path('../profiles/shell', __FILE__)
 require File.expand_path('../profiles/django', __FILE__)
 require File.expand_path('../profiles/jekyll', __FILE__)
+require File.expand_path('../profiles/maven_webapp', __FILE__)
+require File.expand_path('../profiles/chef_stack', __FILE__)

data/lib/circus/profiles/base.rb

@@ -14,6 +14,16 @@ module Circus
         @props = props
       end
       
+      def mark_for_persistent_run?
+        true
+      end
+      
+      # Whether acts using the given profile can be run during development. If this returns false,
+      # then local only commands (such as go) will ignore these items.
+      def supported_for_development?
+        true
+      end
+      
       def package_base_dir?
         true
       end
@@ -34,8 +44,10 @@ module Circus
         write_run_script(run_dir) do |f|
           f.write(deploy_run_script_content.strip)
         end
+        
+        reqs = build_default_requirements.merge(requirements)
         File.open(File.join(run_dir, 'requirements.yaml'), 'w') do |f|
-          f.write(requirements.to_yaml)
+          f.write(reqs.to_yaml)
         end
       end
       
@@ -110,6 +122,13 @@ module Circus
         def run_external(logger, desc, cmd)
           ExternalUtil.run_external(logger, desc, cmd)
         end
+        
+      private
+        def build_default_requirements
+          {
+            'persistent-run' => mark_for_persistent_run?
+          }
+        end
     end
   end
 end

data/lib/circus/profiles/chef_stack.rb

@@ -0,0 +1,75 @@
+require 'json'
+
+module Circus
+  module Profiles
+    # A Chef Stack provides the ability to trigger deployment of system packages and configuration.
+    class ChefStack < Base
+      CHEF_STACK_PROPERTY='chef-stack'
+      
+      # Checks if this is a chef stack applcation. Will accept the application if it 
+      # has a file named stack.yaml, or has a 'chef-stack' property describing the entry point.
+      def self.accepts?(name, dir, props)
+        return true if props.include? CHEF_STACK_PROPERTY
+        return File.exists?(File.join(dir, "stack.yaml"))
+      end
+      
+      def initialize(name, dir, props)
+        super(name, dir, props)
+        
+        @stack_script = props[CHEF_STACK_PROPERTY] || "stack.yaml"
+      end
+      
+      # The name of this profile
+      def name
+        "chef-stack"
+      end
+      
+      def requirements
+        reqs = super
+        reqs['execution-user'] = 'root'
+        reqs
+      end
+      
+      def mark_for_persistent_run?
+        false
+      end
+      
+      # Stacks are not useful in development
+      def supported_for_development?
+        false
+      end
+      
+      def dev_run_script_content
+        shell_run_script do
+          <<-EOT
+          cd #{@dir}
+          exec echo "Stacks cannot be run in development"
+          EOT
+        end
+      end
+      
+      def prepare_for_deploy(logger, overlay_dir)
+        # Generate a chef configuration file
+        stack_config = YAML::load(File.read(File.join(@dir, @stack_script)))
+        File.open(File.join(overlay_dir, 'stack.json'), 'w') do |f|
+          f.write(stack_config.to_json)
+        end
+        File.open(File.join(overlay_dir, 'solo-stack.rb'), 'w') do |f|
+          f << "cookbook_path File.expand_path('../cookbooks', __FILE__)"
+        end
+        
+        true
+      end
+
+      def deploy_run_script_content
+        shell_run_script do
+          <<-EOT
+          exec chef-solo -c solo-stack.rb -j stack.json
+          EOT
+        end
+      end
+    end
+    
+    PROFILES << ChefStack
+  end
+end