circus-deployment 0.0.1

data/lib/circus/agents/params.rb

@@ -0,0 +1,47 @@
+require 'circus/agents/encoding'
+
+module Circus
+  module Agents
+    class CommandParams
+      def initialize(str)
+        @params = if str
+          Encoding.decode(str)
+        else
+          {}
+        end
+      end
+    
+      def required(*names)
+        res = names.map do |n|
+          raise MissingParameterException.new(n) unless @params.has_key? n
+        
+          @params[n]
+        end
+        format_result(res)
+      end
+    
+      def optional(*names)
+        res = names.map do |n|
+          if  @params.has_key? n
+            @params[n]
+          else
+            nil
+          end
+        end
+        format_result(res)
+      end
+      
+      private
+        def format_result(result)
+          return result.first if result.length == 1
+          result
+        end
+    end
+  
+    class MissingParameterException < ArgumentError
+      def initialize(name)
+        super("missing parameter #{name}")
+      end
+    end
+  end
+end

data/lib/circus/agents/ssh_connection.rb

@@ -0,0 +1,120 @@
+require 'dbus'
+require 'uuid'
+require 'net/ssh'
+require 'net/scp'
+require 'uri'
+require 'circus/agents/dbus_connection'
+
+module Circus
+  module Agents
+    class SSHConnection < DBusConnection
+      def initialize(target)
+        addr = "/tmp/circus-#{UUID.generate}"
+        @uri = URI.parse(target)
+        @ssh = open_ssh
+        # Determine our remote uid so we can use that in authentication requests
+        remote_uid = @ssh.exec!('id -u').strip
+
+        @acceptor = Socket.new(Socket::Constants::PF_UNIX, Socket::Constants::SOCK_STREAM, 0)
+        @acceptor.bind(Socket.pack_sockaddr_un(addr))
+        @acceptor.listen(5)
+        
+        Thread.new do
+          begin
+            socket, info = @acceptor.accept
+            @ssh.open_channel do |channel|
+              channel.exec('nc -U /var/run/dbus/system_bus_socket') do |ch, success|
+                abort "Couldn't open DBus connection" unless success
+
+                socket.extend(Net::SSH::BufferedIo)
+                @ssh.listen_to(socket)
+
+                ch.on_process do
+                  if socket.available > 0
+                    ch.send_data(socket.read_available)
+                  end
+                end
+
+                ch.on_data do |ch, data|
+                  socket.write(data)
+                end
+
+                ch.on_close do
+                  @ssh.stop_listening_to(socket)
+                  socket.close
+                end
+              end
+            end
+            @ssh.loop
+          rescue
+            puts "SSH connector thread died:"
+            puts $!, $@
+          end
+        end
+        
+        bus = RemoteDBusConnection.new("unix:path=#{addr}", remote_uid)
+        
+        super(bus)
+      end
+      
+      def send_file(fn)
+        # Perform an SCP upload of the file into the /tmp of the target
+        target_fn = File.join('/tmp', File.basename(fn))
+        
+        scp_ssh = open_ssh
+        scp = Net::SCP.new(scp_ssh)
+        scp.upload!(fn, target_fn)
+        scp_ssh.exec!("chmod ugo+r #{target_fn}")
+        scp_ssh.close
+        target_fn
+      end
+      
+      private
+        def open_ssh
+          Net::SSH.start(@uri.host, @uri.user || ENV['USER'], :port => @uri.port || 22)
+        end
+    end
+    
+    class RemoteDBusConnection < DBus::Connection
+      def initialize(path, remote_uid)
+        super(path)
+        
+        @remote_uid = remote_uid
+        
+        connect
+        send_hello
+      end
+      
+      def init_connection
+        @client = DBus::Client.new(@socket)
+        @client.auth_list = [RemoteExternal.new(@remote_uid)]
+        @client.authenticate
+      end
+    end
+    
+    class RemoteExternal
+      def initialize(uid)
+        @uid = uid
+      end
+      
+      # Make this look like a class that can be instantiated
+      def new
+        self
+      end
+      
+      def name
+        'EXTERNAL'
+      end
+      
+      def authenticate
+        @uid.to_s.split(//).collect { |a| "%x" % a[0].ord }.join
+      end
+    end
+  end
+end
+
+module DBus
+  class Client
+    attr_accessor :auth_list
+  end
+end

data/lib/circus/application.rb

@@ -0,0 +1,99 @@
+require 'yaml'
+require 'fileutils'
+
+module Circus
+  class Application
+    attr_reader :acts
+    
+    def initialize(dir, name = nil)
+      @dir = dir
+      @name = name || File.basename(dir)
+      @acts = nil
+    end
+    
+    def load!
+      @acts = if File.exists?(acts_file)
+        acts_cfg = YAML.load(File.read(acts_file))
+        acts_cfg.map do |name, props|
+            # If no props are specified in the YAML, this could be nil. Default it.
+          props ||= {}
+          
+          act_dir = File.join(@dir, props['dir'] || name)
+          Act.new(name, act_dir, props)
+        end
+      else
+        [Act.new(@name, @dir)]
+      end
+    end
+    
+    # Instructs the application to startup in place and activate all of its associated
+    # acts.
+    def go!(logger)
+      load! unless @acts
+      
+      FileUtils.rm_rf(private_run_root)
+      acts.each do |act|
+        act.package_for_dev(logger, private_run_root)
+      end
+      acts.each do |act|
+        logger.info "Starting act #{act.name} at #{act.dir} using profile #{act.profile.name}"
+      end
+      logger.info "---------------------"
+      
+      system("svscan #{private_run_root}")
+    end
+    
+    # Instructs the application to assemble it's components for deployment and generate
+    # act output files.
+    def assemble!(output_dir, logger, dev = false, only_acts = nil)
+      load! unless @acts
+
+      if dev
+        acts.each do |act|
+          logger.info "Assembling development act #{act.name} at #{act.dir}"
+          act.package_for_dev(logger, private_run_root)
+        end
+        return true
+      end
+      
+      assembly_acts = acts.select {|a| only_acts.nil? or only_acts.include? a.name }
+      
+      FileUtils.mkdir_p(output_dir)
+      assembly_acts.each do |act|
+        act.detect!
+        if act.should_package?
+          logger.info "Assembling act #{act.name} at #{act.dir} using profile #{act.profile.name}"
+        end
+      end
+      logger.info "---------------------"
+      assembly_acts.each do |act|
+        if act.should_package?
+          return false unless act.assemble(logger, output_dir, private_overlay_root)
+        end
+      end
+      
+      true
+    end
+    
+    def upload(output_dir, act_store, only_acts = nil)
+      load! unless @acts
+      upload_acts = acts.select {|a| only_acts.nil? or only_acts.include? a.name }
+      
+      upload_acts.each do |act|
+        act.upload(output_dir, act_store)
+      end
+    end
+        
+    def private_run_root
+      File.join(@dir, '.circus', 'run-dev')
+    end
+    
+    def private_overlay_root
+      File.join(@dir, '.circus', 'overlays')
+    end
+    
+    def acts_file
+      File.join(@dir, 'acts.yaml')
+    end
+  end
+end

data/lib/circus/booth_client.rb

@@ -0,0 +1,25 @@
+require 'circus/agents/client'
+require 'circus/agents/encoding'
+
+module Circus
+  class BoothClient < Circus::Agents::Client
+    def initialize(connection, logger)
+      super(connection)
+      
+      @logger = logger
+    end
+    
+    def create_booth(booth, name, scm_type, scm_url)
+      @connection.call(booth, 'Booth', 'Booth', 'create', {'name' => name, 'scm' => scm_type, 'scm_url' => scm_url}, @logger)
+    end
+    
+    def admit(booth, booth_id, commit_id, patch_fn, acts)
+      @connection.call(booth, 'Booth', 'Booth', 'admit', 
+        {'app_id' => booth_id, 'commit_id' => commit_id, 'patch_fn' => patch_fn || '', 'acts' => acts.join(',')}, @logger)
+    end
+    
+    def get_ssh_key(booth)
+      @connection.call(booth, 'Booth', 'Booth', 'get_ssh_key', {}, @logger)
+    end
+  end
+end

data/lib/circus/booth_tool.rb

@@ -0,0 +1,98 @@
+require 'circus/booth_client'
+require 'circus/clown_client'
+require 'circus/agents/encoding'
+require 'tempfile'
+
+module Circus
+  class BoothTool
+    def initialize(logger, config)
+      @logger = logger
+      @config = config
+    end
+    
+    def connect(name, booth, options = {})
+      # Detect the facets of the repository
+      repo_type_clazz = if options[:repo_type] 
+          Repos.find_repo_by_id(options[:repo_type])
+        else
+          Repos.find_repo_from_dir(File.expand_path('.'))
+        end
+      repo_type = repo_type_clazz.type_id
+      repo_url = if options[:repo_url]
+          options[:repo_url]
+        else
+          repo_helper = repo_type_clazz.new(File.expand_path('.'))
+          repo_helper.repo_url
+        end
+      
+      app_name = options[:app_name] || File.basename(File.expand_path('.'))
+
+      @logger.info("Creating booth connection #{name} on #{booth} for #{repo_url} as #{app_name}")
+      connection = ConnectionBuilder.new(options).build(booth)
+      
+      client = BoothClient.new(connection, @logger)
+      booth_id = client.create_booth(booth, app_name, repo_type, repo_url).result
+      
+      @config.booths[name] = {
+        :booth => booth, :booth_id => booth_id,
+        :repo_type => repo_type,
+        :target => options[:deploy_target] || booth
+      }
+    end
+    
+    def admit(name, apps, options)
+      booth = get_booth_or_default(name)
+      return unless booth
+
+      repo_helper = Repos.find_repo_by_id(booth[:repo_type]).new(File.expand_path('.'))
+      current_rev = repo_helper.current_revision
+      
+      @logger.info("Starting admission into #{name} of version #{current_rev}")
+      connection = ConnectionBuilder.new(booth).build(booth[:booth])
+      # connection.configure_bg!(OpenStruct.new(booth))
+      client = BoothClient.new(connection, @logger)
+      apply_patch_fn = if options[:uncommitted]
+        patch_fn = Tempfile.new('booth').path
+        repo_helper.write_patch(patch_fn)
+        
+        connection.send_file(patch_fn)
+      else
+        ''
+      end
+      admitted = Circus::Agents::Encoding.decode(client.admit(booth[:booth], booth[:booth_id], current_rev, apply_patch_fn, apps).result)
+      
+      clown_connection = ConnectionBuilder.new(options).build(booth[:target])
+      clown_client = ClownClient.new(clown_connection, @logger)
+      admitted.each do |name, url|
+        @logger.info("Executing deployment of #{name} from #{url} to #{booth[:target]}")
+        clown_client.deploy(booth[:target], name, url).result
+      end
+    end
+      
+    private
+      def get_booth_or_default(name)
+        unless name
+          if @config.booths.count == 0
+            @logger.error("No booths configured. Please configure a booth with 'connect' first.")
+            return nil
+          end
+          
+          if @config.booths.count > 1
+            @logger.error("A booth name needs to be provided when multiple booths have been configured. " + 
+                          "One of #{local_config.booths.keys.join(', ')} should be specified")
+            return nil
+          end
+
+          name = @config.booths.keys.first
+        end
+        
+        booth = @config.booths[name]
+        unless booth
+          @logger.error("No booth #{name} is configured. Configure it first with 'connect'.")
+          return nil
+        end
+        
+        booth
+      end
+  end
+end

data/lib/circus/cli.rb

@@ -0,0 +1,147 @@
+require 'ostruct'
+require 'thor'
+#require 'circus/agents/connection'
+require 'circus/clown_client'
+require 'circus/booth_client'
+require 'circus/booth_tool'
+require 'circus/version'
+
+module Circus
+  class CLI < Thor
+    LOGGER = StdoutLogger.new
+    
+    #
+    # Global Options
+    #
+    
+    class_option :source, :default => '.', :desc => 'Source directory containing the application'
+    
+    
+    #
+    # Actions
+    #
+    
+    desc "go", "Run the current application in place for development"
+    def go
+      load!
+      @app.go!(LOGGER)
+    end
+    
+    desc "assemble", "Assemble the application's acts for deployment"
+    method_option :output, :default => '.circus/acts/', :desc => 'Destination directory for generated acts'
+    method_option :actstore, :desc => 'URL of the act store to upload the built acts to'
+    method_option :dev, :type => :boolean, :default => false, :desc => 'Make assembly for local dev use only'
+    def assemble
+      load!
+      
+      output_path = File.expand_path(options[:output])
+      dev = options[:dev]
+
+      @app.assemble!(output_path, LOGGER, dev)
+      if options[:actstore]
+        store = ActStoreClient.new(options[:actstore], LOGGER)
+        @app.upload(output_path, store)
+      end
+    end
+
+    desc "upload FILENAME", "Uploads the given set of act files to act store"
+    method_option :actstore, :required => true, :desc => 'URL of the act store to upload builds to'
+    def upload(fn)
+      store = ActStoreClient.new(options[:actstore], LOGGER)
+      store.upload_act(fn)
+    end
+    
+    desc "get-booth-key BOOTH", "Retrieves the public key that a booth will use when connecting to SSH resources"
+    def get_booth_key(booth)
+      connection = ConnectionBuilder.new(options).build(booth)
+      
+      client = BoothClient.new(connection, LOGGER)
+      key = client.get_ssh_key(booth).result
+      puts "SSH key for: #{booth}"
+      puts "  #{key}"
+    end
+    
+    desc "connect NAME BOOTH", "Associates the local application with the provided booth to allow for deployment"
+    method_option :deploy_target, :desc => 'The target for application deployment. Uses same host as booth if not specified.'
+    method_option :app_name, :desc => 'The name of the application being configured. Defaults to application directory name.'
+    method_option :repo_type, :desc => 'The type of the repository being deployed. Defaults to type of current working directory.'
+    method_option :repo_url, :desc => 'The URL of the repository being deployed. Defaults to the primary url of current working directory.'
+    
+    # method_option :jid, :required => true, :desc => 'The deployer jid to connect as'
+    # method_option :host, :required => true, :desc => 'The XMPP host to connect to'
+    # method_option :port, :type => :numeric, :default => 5222, :desc => 'The XMPP port to connect to'
+    # method_option :password, :required => true, :desc => 'The XMPP user password'
+    def connect(name, booth)
+      config = LocalConfig.new
+      tool = BoothTool.new(LOGGER, config)
+      tool.connect(name, booth, options)
+      config.save!
+    end
+    
+    desc "admit [NAME] [ACT1 ACT2]", "Admits and deploys the application with the given booth reference"
+    method_option :uncommitted, :desc => 'Indicates that the current uncommitted changes should be sent to the booth and included in the app'
+    def admit(name = nil, *apps)
+      tool = BoothTool.new(LOGGER, LocalConfig.new)
+      tool.admit(name, apps, options)
+    end
+
+    desc "exec TARGET [ACT] [CMD]", "Executes the given command in the deployed context of the given act"
+    # method_option :jid, :required => true, :desc => 'The deployer jid to connect as'
+    # method_option :host, :required => true, :desc => 'The XMPP host to connect to'
+    # method_option :port, :type => :numeric, :default => 5222, :desc => 'The XMPP port to connect to'
+    # method_option :password, :required => true, :desc => 'The XMPP user password'
+    def exec(target, act_name, *cmd_parts)
+      cmd = cmd_parts.join(' ')
+      
+      connection = ConnectionBuilder.new(options).build(target)
+      client = ClownClient.new(connection, LOGGER)
+      client.exec(target, act_name, cmd).result
+    end
+    
+    desc "deploy TARGET NAME ACT", "Deploy the named object using the given act onto the given target server"
+    method_option :config, :desc => 'URL of configuration object to use at deployment'
+    # method_option :actstore, :required => true, :desc => 'The store to retrieve the act from'
+    # method_option :jid, :required => true, :desc => 'The deployer jid to connect as'
+    # method_option :host, :required => true, :desc => 'The XMPP host to connect to'
+    # method_option :port, :type => :numeric, :default => 5222, :desc => 'The XMPP port to connect to'
+    # method_option :password, :required => true, :desc => 'The XMPP user password'
+    def deploy(target, name, act)
+      connection = ConnectionBuilder.new(options).build(target)
+      
+      client = ClownClient.new(connection, LOGGER)
+      client.deploy(target, name, act, options[:config]).result
+    end
+    
+    desc "undeploy TARGET NAME", "Undeploys the named act from the given target server"
+    # method_option :jid, :required => true, :desc => 'The deployer jid to connect as'
+    # method_option :host, :required => true, :desc => 'The XMPP host to connect to'
+    # method_option :port, :type => :numeric, :default => 5222, :desc => 'The XMPP port to connect to'
+    # method_option :password, :required => true, :desc => 'The XMPP user password'
+    def undeploy(target, name)
+      connection = ConnectionBuilder.new(options).build(target)
+      
+      client = ClownClient.new(connection, LOGGER)
+      client.undeploy(target, name).result
+    end
+    
+    desc "configure TARGET NAME CONFIG", "(Re)configures the given act on the given target using the provided config details"
+    def configure(target, name, config)
+      connection = ConnectionBuilder.new(options).build(target)
+      
+      client = ClownClient.new(connection, LOGGER)
+      client.configure(target, name, config).result
+    end
+    
+    desc "alias NAME TARGET", "Adds an alias so that NAME can be used when target TARGET is desired"
+    def alias(name, target)
+      local_config = LocalConfig.new
+      local_config.aliases[name] = target
+      local_config.save!
+    end
+    
+    private
+      def load!
+        @app = Circus::Application.new(File.expand_path(options[:source]))
+      end
+  end
+end

data/lib/circus/clown_client.rb

@@ -0,0 +1,27 @@
+require 'circus/agents/client'
+
+module Circus
+  class ClownClient < Circus::Agents::Client
+    def initialize(connection, logger)
+      super(connection)
+      
+      @logger = logger
+    end
+    
+    def deploy(node, name, act_url, config_url = nil)
+      @connection.call(node, 'Clown', 'Clown', 'deploy', {'name' => name, 'url' => act_url, 'config_url' => config_url || ''}, @logger)
+    end
+    
+    def configure(node, name, config_url)
+      @connection.call(node, 'Clown', 'Clown', 'configure', {'name' => name, 'config_url' => config_url}, @logger)
+    end
+    
+    def undeploy(node, name)
+      @connection.call(node, 'Clown', 'Clown', 'undeploy', {'name' => name}, @logger)
+    end
+    
+    def exec(node, name, cmd)
+      @connection.call(node, 'Clown', 'Clown', 'exec', {'name' => name, 'command' => cmd}, @logger)
+    end
+  end
+end

data/lib/circus/connection_builder.rb

@@ -0,0 +1,32 @@
+require 'circus/agents/dbus_connection'
+require 'circus/agents/ssh_connection'
+
+module Circus
+  class ConnectionBuilder
+    def initialize(options)
+      @options = options
+      @local_config = LocalConfig.new
+    end
+    
+    def build(target)
+      if not target.include? ':'
+        if @local_config.aliases.include? target
+          build(@local_config.aliases[target])
+        else
+          raise ArgumentError.new("Invalid target #{target} - looks like an alias, but none configured to match")
+        end
+      elsif target.start_with? 'local:'
+        # Local connection
+        client = Agents::DBusConnection.new
+        # client.configure_bg!
+        client
+      elsif target.start_with? 'ssh://'
+        # SSH connection
+        client = Agents::SSHConnection.new(target)
+        client
+      elsif
+        raise ArgumentError.new("Ivalid target #{target} - unknown protocol")
+      end
+    end
+  end
+end

data/lib/circus/external_util.rb

@@ -0,0 +1,14 @@
+module Circus
+  class ExternalUtil
+    def self.run_external(logger, desc, cmd)
+      res = `#{cmd} 2>&1`
+      if $? != 0
+        logger.error "#{desc} failed:"
+        logger.error res
+        false
+      else
+        true
+      end
+    end
+  end
+end

data/lib/circus/local_config.rb

@@ -0,0 +1,65 @@
+require 'fileutils'
+
+module Circus
+  class LocalConfig
+    def initialize(dir = '.')
+      @dir = File.expand_path(dir)
+      @is_new = true
+      
+      @store_fn = find_store_fn
+      load!
+    end
+    
+    def booths
+      @state[:booths] ||= {}
+    end
+    
+    def aliases
+      @state[:aliases] ||= {}
+    end
+    
+    def new?
+      @is_new
+    end
+    
+    def load!
+      if File.exists? @store_fn
+        @is_new = false
+        @state = YAML::load(File.read(@store_fn))
+      else
+        @state = {}
+      end
+    end
+    
+    def save!
+      FileUtils.mkdir_p(File.dirname(@store_fn))
+      File.open(@store_fn, 'w') do |f|
+        YAML::dump(@state, f)
+      end
+      @is_new = false
+    end
+    
+    private
+      def find_store_fn
+        path = @dir
+        fn = store_fn_for_dir(path)
+        
+        until File.exists? fn
+          next_path = File.dirname(path)
+            
+          # If we've hit the root, then the filename will keep repeating. 
+          return store_fn_for_dir(@dir) if path == next_path
+          
+          # Build for the next loop
+          path = next_path
+          fn = store_fn_for_dir(path)
+        end
+        
+        fn
+      end
+      
+      def store_fn_for_dir(dir)
+        File.join(dir, '.circus/config')
+      end
+  end
+end