buildbox 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17c1b2a9d1b7bc186af0aa5a3d93d7dea50e9864
-  data.tar.gz: dd7a3713cfed6a4f8f04eeb77fece760132833cd
+  metadata.gz: 0652374e7d5bf3b28c6d9a2af33a53c3882d6143
+  data.tar.gz: d76455f156b6fda572ea2d1276784f57005f9654
 SHA512:
-  metadata.gz: b15ed4fb9913f0e48dae8b18d05695b556f0714cf9a042b7f08ffec7773945e2cee92554a0f8f7a4af8870ccdbc92b0159c16df970dcd94f80a3bd16b1260a44
-  data.tar.gz: b266bba36f25497145ff582886caae1aa0da899119092deda3d4a304a24d4e35abc0c4caaca0a518859e19999038901c527f6bad9443ef83d6cb3c6d38af4356
+  metadata.gz: 45597206e22ef07fcd6948c9cd9071e66aabac2b44760fb9724ba5367785b626d3116b6c8c19e47e0c074916fb206b56c52d5ebf3ccf5580f25ebafcfe24e5f3
+  data.tar.gz: 9097f5b48ebbd24f4a90c70d9e6409bd518ec9884058cdbde153814c3a3ca196f2c7121d9d5cba6e43ce21109dbc2a147feded052077c5c7884611f2116a7408

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    buildbox (0.3.3)
+    buildbox (0.3.4)
       celluloid (~> 0.14)
       childprocess (~> 0.3)
       faraday (~> 0.8)

data/README.md

@@ -5,14 +5,18 @@
 Install the gem
 
     $ gem install buildbox
+    
+Authenticate
+
+    $ buildbox auth:login [api_key]
 
 Add your worker tokens
 
-    $ buildbox worker:add [token]
+    $ buildbox agent:setup [token]
 
 Then you can start monitoring for builds like so:
 
-    $ buildbox worker:start
+    $ buildbox agent:start
 
 For more help with the command line interface
 

data/bin/buildbox

@@ -3,8 +3,18 @@
 # Disable stdout,stderr buffering
 STDERR.sync = STDOUT.sync = true
 
-dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
+# Set the process name before we go to far in
+$PROGRAM_NAME = 'buildbox'
+
+# Add the buildbox lib directory to the load path
+root_dir = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+lib_dir  = File.join(root_dir, 'lib')
+$LOAD_PATH.unshift(lib_dir) unless $LOAD_PATH.include?(lib_dir)
+
+# Setup bundler correctly
+ENV['BUNDLE_GEMFILE'] ||= File.join(root_dir, "Gemfile")
+require 'bundler/setup'
 
 require 'buildbox'
+Buildbox.logger.level = Logger::DEBUG if ENV['DEBUG'] == 'true'
 Buildbox::CLI.new(ARGV).parse

data/lib/buildbox.rb

@@ -1,3 +1,4 @@
+require 'rubygems'
 require 'pathname'
 require 'logger'
 

data/lib/buildbox/agent.rb

@@ -1,38 +1,39 @@
-require 'rubygems'
 require 'celluloid'
 
 module Buildbox
   class Agent
+    include Celluloid
     include Celluloid::Logger
 
-    def initialize(access_token, api)
+    def initialize(access_token, api = Buildbox::API.new)
       @api          = api
       @access_token = access_token
     end
 
     def work
-      running_builds = scheduled_builds.map do |build|
-        Monitor.new(build, @api).async.monitor
-        Runner.new(build).future(:start)
+      builds = scheduled_builds
+
+      # Start off by letting each build know that it's been picked up
+      # by an agent.
+      builds.each do |build|
+        @api.update_build(build, :agent_accepted => @access_token)
       end
 
-      # wait for all the running builds to finish
-      running_builds.map(&:value)
+      # Run the builds one at a time
+      builds.each do |build|
+        Monitor.new(build, @api).async.monitor
+        Runner.new(build).start
+      end
     end
 
     private
 
-    def projects
-      @api.agent(@access_token, hostname).projects
-    rescue Faraday::Error::ClientError
-      warn "Agent #{@access_token} doesn't exist"
-      [] # return empty array to avoid breakage
-    end
-
     def scheduled_builds
-      projects.map do |project|
-        @api.scheduled_builds(project)
-      end.flatten
+      agent = @api.agent(@access_token, hostname)
+      @api.scheduled_builds agent
+    rescue Buildbox::API::AgentNotFoundError
+      warn "Agent `#{@access_token}` doesn't exist"
+      [] # return empty array to avoid breakage
     end
 
     def hostname

data/lib/buildbox/api.rb

@@ -1,32 +1,56 @@
-require 'rubygems'
 require 'faraday'
 require 'faraday_middleware'
 require 'hashie/mash'
+require 'delegate'
 
 module Buildbox
   class API
-    def initialize(config = Buildbox.config)
-      @config  = config
+    # Faraday uses debug to show response information, but when the agent is in
+    # DEBUG mode, it's kinda useless noise. So we use a ProxyLogger to only push
+    # the information we care about to the logger.
+    class ProxyLogger
+      def initialize(logger)
+        @logger = logger
+      end
+      def info(*args)
+        @logger.debug(*args)
+      end
+
+      def debug(*args)
+        # no-op
+      end
+    end
+
+    class AgentNotFoundError < Faraday::Error::ClientError; end
+    class ServerError < Faraday::Error::ClientError; end
+
+    def initialize(config = Buildbox.config, logger = Buildbox.logger)
+      @config = config
+      @logger = logger
     end
 
     def authenticate(api_key)
       @api_key = api_key
+
       get("user")
     end
 
     def agent(access_token, hostname)
       put("agents/#{access_token}", :hostname => hostname)
+    rescue Faraday::Error::ClientError => e
+      if e.response[:status] == 404
+        raise AgentNotFoundError.new(e, e.response)
+      else
+        raise ServerError.new(e, e.response)
+      end
     end
 
-    def scheduled_builds(project)
-      get(project.scheduled_builds_url).map { |build| Buildbox::Build.new(build) }
+    def scheduled_builds(agent)
+      get(agent.scheduled_builds_url).map { |build| Buildbox::Build.new(build) }
     end
 
-    def update_build(build)
-      put(build.url, :started_at  => build.started_at,
-                     :finished_at => build.finished_at,
-                     :output      => build.output,
-                     :exit_status => build.exit_status)
+    def update_build(build, options)
+      put(build.url, options)
     end
 
     private
@@ -34,24 +58,29 @@ module Buildbox
     def connection
       @connection ||= Faraday.new(:url => @config.api_endpoint) do |faraday|
         faraday.basic_auth @api_key || @config.api_key, ''
+        faraday.request :retry
+        faraday.request :json
 
-        faraday.request  :json
-
-        faraday.response :logger, Buildbox.logger
+        faraday.response :logger, ProxyLogger.new(@logger)
         faraday.response :mashify
 
-        # json needs to come after mashify as it needs to run before the mashify
+        # JSON needs to come after mashify as it needs to run before the mashify
         # middleware.
         faraday.response :json
         faraday.response :raise_error
 
         faraday.adapter Faraday.default_adapter
+
+        # Set some sensible defaults on the adapter.
+        faraday.options[:timeout]      = 60
+        faraday.options[:open_timeout] = 60
       end
     end
 
     def post(path, body = {})
       connection.post(path) do |request|
-        request.body = body
+        request.body                    = body
+        request.headers['Content-Type'] = 'application/json'
       end.body
     end
 

data/lib/buildbox/canceler.rb

@@ -1,8 +1,9 @@
+require 'celluloid'
+require 'timeout'
+
 module Buildbox
   class Canceler
-    def self.cancel(build)
-      new(build).cancel
-    end
+    include Celluloid
 
     def initialize(build)
       @build = build
@@ -11,8 +12,69 @@ module Buildbox
     def cancel
       @build.cancel_started = true
 
-      # Kill that damn process, yo!
-      Process.kill 'INT', @build.pid
+      # Store all the child processes before we stop so we can reap them after.
+      # A new process may start up between this and the process stopping,
+      # but that should be OK for now.
+      child_processes = process_map[@build.process.pid]
+
+      # Stop the process
+      Buildbox.logger.info "Cancelling build #{@build.namespace}/#{@build.id} with PID #{@build.process.pid}"
+      @build.process.stop
+
+      begin
+        @build.process.wait
+      rescue Errno::ECHILD
+        # Wow! That finished quickly...
+      end
+
+      kill_processes(child_processes)
+    end
+
+    private
+
+    def kill_processes(processes)
+      processes.each do |pid|
+        Buildbox.logger.debug "Sending a TERM signal to child process with PID #{pid}"
+
+        begin
+          Process.kill("TERM", pid)
+
+          # If the child process doesn't die within 5 seconds, try a more
+          # forceful kill command
+          begin
+            Timeout.timeout(5) do
+              Process.wait
+            end
+          rescue Timeout::Error
+            Buildbox.logger.debug "Sending a KILL signal to child process with PID #{pid}"
+
+            Process.kill("KILL", pid)
+          rescue Errno::ECHILD
+            # Killed already
+          end
+        rescue Errno::ESRCH
+          # No such process
+        end
+      end
+    end
+
+    # Generates a map of parent process and child processes. This method
+    # will currently only work on unix.
+    def process_map
+      output    = `ps -eo ppid,pid`
+      processes = {}
+
+      output.split("\n").each do |line|
+        if result = line.match(/(\d+)\s(\d+)/)
+          parent = result[1].to_i
+          child  = result[2].to_i
+
+          processes[parent] ||= []
+          processes[parent] << child
+        end
+      end
+
+      processes
     end
   end
 end

data/lib/buildbox/cli.rb

@@ -72,7 +72,8 @@ module Buildbox
           Buildbox.config.update(:agent_access_tokens => agent_access_tokens << access_token)
 
           puts "Successfully added agent access token"
-          puts "You can now start the agent with: buildbox agent:start"
+          puts "You can now start the agent with: buildbox agent:start."
+          puts "If the agent is already running, you'll have to restart it for the new changes to take effect"
         elsif command == "auth:login"
           if @argv.length == 0
             puts "No api key provided"

data/lib/buildbox/command.rb

@@ -13,35 +13,33 @@ module Buildbox
     # the given timeout.
     class TimeoutExceeded < StandardError; end
 
-    attr_reader :pid, :output, :exit_status
+    attr_reader :output, :exit_status
 
     def self.run(*args, &block)
-      options   = args.last.is_a?(Hash) ? args.pop : {}
-      arguments = args.dup
-
-      # Run the command
-      command = new(arguments, options, &block)
+      command = new(*args, &block)
       command.start(&block)
       command
     end
 
-    def initialize(arguments, options = {})
-      @arguments = arguments
-      @options   = options
+    def initialize(*args)
+      @options   = args.last.is_a?(Hash) ? args.pop : {}
+      @arguments = args.dup
       @logger    = Buildbox.logger
     end
 
+    def arguments
+      [ *@arguments ].compact.map(&:to_s) # all arguments must be a string
+    end
+
+    def process
+      @process ||= ChildProcess.build(*arguments)
+    end
+
     def start(&block)
       # Get the timeout, if we have one
       timeout = @options[:timeout]
 
-      # Build the command we're going to run
-      arguments = [ *@arguments ].compact.map(&:to_s) # all arguments must be a string
-
-      # Build the ChildProcess
-      @logger.info("Starting process: #{arguments}")
-
-      process     = ChildProcess.build(*arguments)
+      # Set the directory for the process
       process.cwd = File.expand_path(@options[:directory] || Dir.pwd)
 
       # Create the pipes so we can read the output in real time. PTY
@@ -70,6 +68,8 @@ module Buildbox
       # Make sure the stdin does not buffer
       process.io.stdin.sync = true
 
+      @logger.debug("Process #{arguments} started with PID: #{process.pid} and Group ID: #{Process.getpgid(process.pid)}")
+
       if RUBY_PLATFORM != "java"
         # On Java, we have to close after. See down the method...
         # Otherwise, we close the writer right here, since we're
@@ -77,9 +77,6 @@ module Buildbox
         write_pipe.close
       end
 
-      # Store the process id for later cancelling!
-      @pid = process.pid
-
       # Record the start time for timeout purposes
       start_time = Time.now.to_i
 
@@ -107,7 +104,7 @@ module Buildbox
             next if data.empty?
 
             output << cleaned_data = UTF8.clean(data)
-            yield self, cleaned_data if block_given?
+            yield cleaned_data if block_given?
           end
         end
 
@@ -140,7 +137,7 @@ module Buildbox
       # If there's some that we missed
       if extra_data != ""
         output << cleaned_data = UTF8.clean(extra_data)
-        yield self, cleaned_data if block_given?
+        yield cleaned_data if block_given?
       end
 
       if RUBY_PLATFORM == "java"

data/lib/buildbox/monitor.rb

@@ -1,4 +1,3 @@
-require 'rubygems'
 require 'celluloid'
 
 module Buildbox
@@ -12,25 +11,25 @@ module Buildbox
 
     def monitor
       loop do
-        # There is an edge case where the build finishes between making the
-        # update_build http call, and breaking. So to make sure we're using the
-        # same build object throughout this call, we can just deep dup it.
-        build = Marshal.load(Marshal.dump(@build))
+        if @build.started?
+          # As the build can finish in between doing the update_build api_call
+          # and checking to see if the build has finished, we make sure we use the
+          # same finished_at timestamp throughout the entire method.
+          finished_at = @build.finished_at
 
-        if build.started? || build.finished?
-          new_build = @api.update_build(build)
+          updated_build = @api.update_build(@build, :started_at  => @build.started_at,
+                                                    :finished_at => finished_at,
+                                                    :output      => @build.output,
+                                                    :exit_status => @build.exit_status)
 
-          # Try and cancel the build if we haven't tried already
-          if new_build.state == 'canceled' && !@build.cancelling?
-            Buildbox::Canceler.cancel(@build)
+          if updated_build.state == 'canceled' && !@build.cancelling?
+            Buildbox::Canceler.new(@build).async.cancel
           end
-        end
 
-        if build.finished?
-          break
-        else
-          sleep 2 # 2 seconds seems reasonable for now
+          break if finished_at
         end
+
+        sleep 1
       end
     end
   end

data/lib/buildbox/runner.rb

@@ -1,4 +1,3 @@
-require 'rubygems'
 require 'celluloid'
 require 'fileutils'
 
@@ -7,8 +6,6 @@ module Buildbox
     include Celluloid
     include Celluloid::Logger
 
-    attr_reader :build
-
     def initialize(build)
       @build = build
     end
@@ -20,24 +17,22 @@ module Buildbox
       File.open(script_path, 'w+') { |file| file.write(@build.script) }
       File.chmod(0777, script_path)
 
-      info "Running script: #{script_path}"
+      command = Command.new(script_path, :environment => @build.env, :directory => directory_path)
 
+      @build.output     = ""
+      @build.process    = command.process
       @build.started_at = Time.now.utc
 
-      build.output = ""
-      result = Command.run(script_path, :environment => @build.env, :directory => directory_path) do |command, chunk|
-        build.pid = command.pid
-        build.output << chunk
-      end
+      command.start { |chunk| @build.output << chunk }
 
-      build.output      = result.output
-      build.exit_status = result.exit_status
+      @build.output      = command.output
+      @build.exit_status = command.exit_status
 
       File.delete(script_path)
 
       @build.finished_at = Time.now.utc
 
-      info "#{@build.namespace} ##{@build.id} finished with exit status #{result.exit_status}"
+      info "#{@build.namespace} ##{@build.id} finished with exit status #{command.exit_status}"
     end
 
     private

data/lib/buildbox/server.rb

@@ -1,3 +1,5 @@
+require 'celluloid'
+
 module Buildbox
   class Server
     INTERVAL = 5
@@ -5,26 +7,33 @@ module Buildbox
     def initialize(config = Buildbox.config, logger = Buildbox.logger)
       @config = config
       @logger = logger
+      @supervisors = []
     end
 
     def start
-      loop do
-        @config.check
-        @config.reload
+      Celluloid.logger = @logger
 
-        agent_access_tokens.each do |access_token|
-          Buildbox::Agent.new(access_token, api).work
+      agent_access_tokens.each do |access_token|
+        @supervisors << Buildbox::Agent.supervise(access_token)
+
+        @logger.info "Agent with access token `#{access_token}` has started."
+      end
+
+      loop do
+        @supervisors.each do |supervisor|
+          supervisor.actors.first.async.work
         end
 
-        @logger.info "Sleeping for #{INTERVAL} seconds"
-        sleep INTERVAL
+        wait INTERVAL
       end
     end
 
     private
 
-    def api
-      @api ||= Buildbox::API.new
+    def wait(interval)
+      @logger.debug "Sleeping for #{interval} seconds"
+
+      sleep interval
     end
 
     def agent_access_tokens

data/lib/buildbox/version.rb

@@ -1,3 +1,3 @@
 module Buildbox
-  VERSION = "0.3.3"
+  VERSION = "0.3.4"
 end

data/spec/buildbox/buildbox/command_spec.rb

@@ -45,7 +45,7 @@ describe Buildbox::Command do
 
     it "can collect output in chunks" do
       chunked_output = ''
-      result = Buildbox::Command.run('echo', 'hello world') do |command, chunk|
+      result = Buildbox::Command.run('echo', 'hello world') do |chunk|
         unless chunk.nil?
           chunked_output += chunk
         end
@@ -76,7 +76,7 @@ describe Buildbox::Command do
 
     it "captures color'd output from a command" do
       chunked_output = ''
-      result = Buildbox::Command.run('rspec', FIXTURES_PATH.join('rspec', 'test_spec.rb')) do |command, chunk|
+      result = Buildbox::Command.run('rspec', FIXTURES_PATH.join('rspec', 'test_spec.rb')) do |chunk|
         chunked_output += chunk unless chunk.nil?
       end
 
@@ -87,7 +87,7 @@ describe Buildbox::Command do
 
     it "runs scripts in a tty" do
       chunked_output = ''
-      result = Buildbox::Command.run(FIXTURES_PATH.join('tty_script')) do |command, chunk|
+      result = Buildbox::Command.run(FIXTURES_PATH.join('tty_script')) do |chunk|
         chunked_output += chunk unless chunk.nil?
       end
 
@@ -115,11 +115,11 @@ describe Buildbox::Command do
     it "can collect chunks from within a thread" do
       chunks = []
 
-      result = Buildbox::Command.run(FIXTURES_PATH.join('sleep_script')) do |command, chunk|
+      result = Buildbox::Command.run(FIXTURES_PATH.join('sleep_script')) do |chunk|
         chunks << chunk
       end
 
-      chunks.should == ["test 0\r\n", "test 1\r\n", "test 2\r\n"]
+      chunks.map(&:chomp).reject(&:empty?).should == ["test 0", "test 1", "test 2"]
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: buildbox
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Keith Pitt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-16 00:00:00.000000000 Z
+date: 2013-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday